1 //===- ScalarEvolution.cpp - Scalar Evolution Analysis --------------------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // This file contains the implementation of the scalar evolution analysis 11 // engine, which is used primarily to analyze expressions involving induction 12 // variables in loops. 13 // 14 // There are several aspects to this library. First is the representation of 15 // scalar expressions, which are represented as subclasses of the SCEV class. 16 // These classes are used to represent certain types of subexpressions that we 17 // can handle. We only create one SCEV of a particular shape, so 18 // pointer-comparisons for equality are legal. 19 // 20 // One important aspect of the SCEV objects is that they are never cyclic, even 21 // if there is a cycle in the dataflow for an expression (ie, a PHI node). If 22 // the PHI node is one of the idioms that we can represent (e.g., a polynomial 23 // recurrence) then we represent it directly as a recurrence node, otherwise we 24 // represent it as a SCEVUnknown node. 25 // 26 // In addition to being able to represent expressions of various types, we also 27 // have folders that are used to build the *canonical* representation for a 28 // particular expression. These folders are capable of using a variety of 29 // rewrite rules to simplify the expressions. 30 // 31 // Once the folders are defined, we can implement the more interesting 32 // higher-level code, such as the code that recognizes PHI nodes of various 33 // types, computes the execution count of a loop, etc. 34 // 35 // TODO: We should use these routines and value representations to implement 36 // dependence analysis! 37 // 38 //===----------------------------------------------------------------------===// 39 // 40 // There are several good references for the techniques used in this analysis. 41 // 42 // Chains of recurrences -- a method to expedite the evaluation 43 // of closed-form functions 44 // Olaf Bachmann, Paul S. Wang, Eugene V. Zima 45 // 46 // On computational properties of chains of recurrences 47 // Eugene V. Zima 48 // 49 // Symbolic Evaluation of Chains of Recurrences for Loop Optimization 50 // Robert A. van Engelen 51 // 52 // Efficient Symbolic Analysis for Optimizing Compilers 53 // Robert A. van Engelen 54 // 55 // Using the chains of recurrences algebra for data dependence testing and 56 // induction variable substitution 57 // MS Thesis, Johnie Birch 58 // 59 //===----------------------------------------------------------------------===// 60 61 #include "llvm/Analysis/ScalarEvolution.h" 62 #include "llvm/ADT/Optional.h" 63 #include "llvm/ADT/STLExtras.h" 64 #include "llvm/ADT/SmallPtrSet.h" 65 #include "llvm/ADT/Statistic.h" 66 #include "llvm/Analysis/AssumptionCache.h" 67 #include "llvm/Analysis/ConstantFolding.h" 68 #include "llvm/Analysis/InstructionSimplify.h" 69 #include "llvm/Analysis/LoopInfo.h" 70 #include "llvm/Analysis/ScalarEvolutionExpressions.h" 71 #include "llvm/Analysis/TargetLibraryInfo.h" 72 #include "llvm/Analysis/ValueTracking.h" 73 #include "llvm/IR/ConstantRange.h" 74 #include "llvm/IR/Constants.h" 75 #include "llvm/IR/DataLayout.h" 76 #include "llvm/IR/DerivedTypes.h" 77 #include "llvm/IR/Dominators.h" 78 #include "llvm/IR/GetElementPtrTypeIterator.h" 79 #include "llvm/IR/GlobalAlias.h" 80 #include "llvm/IR/GlobalVariable.h" 81 #include "llvm/IR/InstIterator.h" 82 #include "llvm/IR/Instructions.h" 83 #include "llvm/IR/LLVMContext.h" 84 #include "llvm/IR/Metadata.h" 85 #include "llvm/IR/Operator.h" 86 #include "llvm/IR/PatternMatch.h" 87 #include "llvm/Support/CommandLine.h" 88 #include "llvm/Support/Debug.h" 89 #include "llvm/Support/ErrorHandling.h" 90 #include "llvm/Support/MathExtras.h" 91 #include "llvm/Support/raw_ostream.h" 92 #include "llvm/Support/SaveAndRestore.h" 93 #include <algorithm> 94 using namespace llvm; 95 96 #define DEBUG_TYPE "scalar-evolution" 97 98 STATISTIC(NumArrayLenItCounts, 99 "Number of trip counts computed with array length"); 100 STATISTIC(NumTripCountsComputed, 101 "Number of loops with predictable loop counts"); 102 STATISTIC(NumTripCountsNotComputed, 103 "Number of loops without predictable loop counts"); 104 STATISTIC(NumBruteForceTripCountsComputed, 105 "Number of loops with trip counts computed by force"); 106 107 static cl::opt<unsigned> 108 MaxBruteForceIterations("scalar-evolution-max-iterations", cl::ReallyHidden, 109 cl::desc("Maximum number of iterations SCEV will " 110 "symbolically execute a constant " 111 "derived loop"), 112 cl::init(100)); 113 114 // FIXME: Enable this with EXPENSIVE_CHECKS when the test suite is clean. 115 static cl::opt<bool> 116 VerifySCEV("verify-scev", 117 cl::desc("Verify ScalarEvolution's backedge taken counts (slow)")); 118 static cl::opt<bool> 119 VerifySCEVMap("verify-scev-maps", 120 cl::desc("Verify no dangling value in ScalarEvolution's " 121 "ExprValueMap (slow)")); 122 123 //===----------------------------------------------------------------------===// 124 // SCEV class definitions 125 //===----------------------------------------------------------------------===// 126 127 //===----------------------------------------------------------------------===// 128 // Implementation of the SCEV class. 129 // 130 131 LLVM_DUMP_METHOD 132 void SCEV::dump() const { 133 print(dbgs()); 134 dbgs() << '\n'; 135 } 136 137 void SCEV::print(raw_ostream &OS) const { 138 switch (static_cast<SCEVTypes>(getSCEVType())) { 139 case scConstant: 140 cast<SCEVConstant>(this)->getValue()->printAsOperand(OS, false); 141 return; 142 case scTruncate: { 143 const SCEVTruncateExpr *Trunc = cast<SCEVTruncateExpr>(this); 144 const SCEV *Op = Trunc->getOperand(); 145 OS << "(trunc " << *Op->getType() << " " << *Op << " to " 146 << *Trunc->getType() << ")"; 147 return; 148 } 149 case scZeroExtend: { 150 const SCEVZeroExtendExpr *ZExt = cast<SCEVZeroExtendExpr>(this); 151 const SCEV *Op = ZExt->getOperand(); 152 OS << "(zext " << *Op->getType() << " " << *Op << " to " 153 << *ZExt->getType() << ")"; 154 return; 155 } 156 case scSignExtend: { 157 const SCEVSignExtendExpr *SExt = cast<SCEVSignExtendExpr>(this); 158 const SCEV *Op = SExt->getOperand(); 159 OS << "(sext " << *Op->getType() << " " << *Op << " to " 160 << *SExt->getType() << ")"; 161 return; 162 } 163 case scAddRecExpr: { 164 const SCEVAddRecExpr *AR = cast<SCEVAddRecExpr>(this); 165 OS << "{" << *AR->getOperand(0); 166 for (unsigned i = 1, e = AR->getNumOperands(); i != e; ++i) 167 OS << ",+," << *AR->getOperand(i); 168 OS << "}<"; 169 if (AR->hasNoUnsignedWrap()) 170 OS << "nuw><"; 171 if (AR->hasNoSignedWrap()) 172 OS << "nsw><"; 173 if (AR->hasNoSelfWrap() && 174 !AR->getNoWrapFlags((NoWrapFlags)(FlagNUW | FlagNSW))) 175 OS << "nw><"; 176 AR->getLoop()->getHeader()->printAsOperand(OS, /*PrintType=*/false); 177 OS << ">"; 178 return; 179 } 180 case scAddExpr: 181 case scMulExpr: 182 case scUMaxExpr: 183 case scSMaxExpr: { 184 const SCEVNAryExpr *NAry = cast<SCEVNAryExpr>(this); 185 const char *OpStr = nullptr; 186 switch (NAry->getSCEVType()) { 187 case scAddExpr: OpStr = " + "; break; 188 case scMulExpr: OpStr = " * "; break; 189 case scUMaxExpr: OpStr = " umax "; break; 190 case scSMaxExpr: OpStr = " smax "; break; 191 } 192 OS << "("; 193 for (SCEVNAryExpr::op_iterator I = NAry->op_begin(), E = NAry->op_end(); 194 I != E; ++I) { 195 OS << **I; 196 if (std::next(I) != E) 197 OS << OpStr; 198 } 199 OS << ")"; 200 switch (NAry->getSCEVType()) { 201 case scAddExpr: 202 case scMulExpr: 203 if (NAry->hasNoUnsignedWrap()) 204 OS << "<nuw>"; 205 if (NAry->hasNoSignedWrap()) 206 OS << "<nsw>"; 207 } 208 return; 209 } 210 case scUDivExpr: { 211 const SCEVUDivExpr *UDiv = cast<SCEVUDivExpr>(this); 212 OS << "(" << *UDiv->getLHS() << " /u " << *UDiv->getRHS() << ")"; 213 return; 214 } 215 case scUnknown: { 216 const SCEVUnknown *U = cast<SCEVUnknown>(this); 217 Type *AllocTy; 218 if (U->isSizeOf(AllocTy)) { 219 OS << "sizeof(" << *AllocTy << ")"; 220 return; 221 } 222 if (U->isAlignOf(AllocTy)) { 223 OS << "alignof(" << *AllocTy << ")"; 224 return; 225 } 226 227 Type *CTy; 228 Constant *FieldNo; 229 if (U->isOffsetOf(CTy, FieldNo)) { 230 OS << "offsetof(" << *CTy << ", "; 231 FieldNo->printAsOperand(OS, false); 232 OS << ")"; 233 return; 234 } 235 236 // Otherwise just print it normally. 237 U->getValue()->printAsOperand(OS, false); 238 return; 239 } 240 case scCouldNotCompute: 241 OS << "***COULDNOTCOMPUTE***"; 242 return; 243 } 244 llvm_unreachable("Unknown SCEV kind!"); 245 } 246 247 Type *SCEV::getType() const { 248 switch (static_cast<SCEVTypes>(getSCEVType())) { 249 case scConstant: 250 return cast<SCEVConstant>(this)->getType(); 251 case scTruncate: 252 case scZeroExtend: 253 case scSignExtend: 254 return cast<SCEVCastExpr>(this)->getType(); 255 case scAddRecExpr: 256 case scMulExpr: 257 case scUMaxExpr: 258 case scSMaxExpr: 259 return cast<SCEVNAryExpr>(this)->getType(); 260 case scAddExpr: 261 return cast<SCEVAddExpr>(this)->getType(); 262 case scUDivExpr: 263 return cast<SCEVUDivExpr>(this)->getType(); 264 case scUnknown: 265 return cast<SCEVUnknown>(this)->getType(); 266 case scCouldNotCompute: 267 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!"); 268 } 269 llvm_unreachable("Unknown SCEV kind!"); 270 } 271 272 bool SCEV::isZero() const { 273 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(this)) 274 return SC->getValue()->isZero(); 275 return false; 276 } 277 278 bool SCEV::isOne() const { 279 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(this)) 280 return SC->getValue()->isOne(); 281 return false; 282 } 283 284 bool SCEV::isAllOnesValue() const { 285 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(this)) 286 return SC->getValue()->isAllOnesValue(); 287 return false; 288 } 289 290 /// isNonConstantNegative - Return true if the specified scev is negated, but 291 /// not a constant. 292 bool SCEV::isNonConstantNegative() const { 293 const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(this); 294 if (!Mul) return false; 295 296 // If there is a constant factor, it will be first. 297 const SCEVConstant *SC = dyn_cast<SCEVConstant>(Mul->getOperand(0)); 298 if (!SC) return false; 299 300 // Return true if the value is negative, this matches things like (-42 * V). 301 return SC->getAPInt().isNegative(); 302 } 303 304 SCEVCouldNotCompute::SCEVCouldNotCompute() : 305 SCEV(FoldingSetNodeIDRef(), scCouldNotCompute) {} 306 307 bool SCEVCouldNotCompute::classof(const SCEV *S) { 308 return S->getSCEVType() == scCouldNotCompute; 309 } 310 311 const SCEV *ScalarEvolution::getConstant(ConstantInt *V) { 312 FoldingSetNodeID ID; 313 ID.AddInteger(scConstant); 314 ID.AddPointer(V); 315 void *IP = nullptr; 316 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S; 317 SCEV *S = new (SCEVAllocator) SCEVConstant(ID.Intern(SCEVAllocator), V); 318 UniqueSCEVs.InsertNode(S, IP); 319 return S; 320 } 321 322 const SCEV *ScalarEvolution::getConstant(const APInt &Val) { 323 return getConstant(ConstantInt::get(getContext(), Val)); 324 } 325 326 const SCEV * 327 ScalarEvolution::getConstant(Type *Ty, uint64_t V, bool isSigned) { 328 IntegerType *ITy = cast<IntegerType>(getEffectiveSCEVType(Ty)); 329 return getConstant(ConstantInt::get(ITy, V, isSigned)); 330 } 331 332 SCEVCastExpr::SCEVCastExpr(const FoldingSetNodeIDRef ID, 333 unsigned SCEVTy, const SCEV *op, Type *ty) 334 : SCEV(ID, SCEVTy), Op(op), Ty(ty) {} 335 336 SCEVTruncateExpr::SCEVTruncateExpr(const FoldingSetNodeIDRef ID, 337 const SCEV *op, Type *ty) 338 : SCEVCastExpr(ID, scTruncate, op, ty) { 339 assert((Op->getType()->isIntegerTy() || Op->getType()->isPointerTy()) && 340 (Ty->isIntegerTy() || Ty->isPointerTy()) && 341 "Cannot truncate non-integer value!"); 342 } 343 344 SCEVZeroExtendExpr::SCEVZeroExtendExpr(const FoldingSetNodeIDRef ID, 345 const SCEV *op, Type *ty) 346 : SCEVCastExpr(ID, scZeroExtend, op, ty) { 347 assert((Op->getType()->isIntegerTy() || Op->getType()->isPointerTy()) && 348 (Ty->isIntegerTy() || Ty->isPointerTy()) && 349 "Cannot zero extend non-integer value!"); 350 } 351 352 SCEVSignExtendExpr::SCEVSignExtendExpr(const FoldingSetNodeIDRef ID, 353 const SCEV *op, Type *ty) 354 : SCEVCastExpr(ID, scSignExtend, op, ty) { 355 assert((Op->getType()->isIntegerTy() || Op->getType()->isPointerTy()) && 356 (Ty->isIntegerTy() || Ty->isPointerTy()) && 357 "Cannot sign extend non-integer value!"); 358 } 359 360 void SCEVUnknown::deleted() { 361 // Clear this SCEVUnknown from various maps. 362 SE->forgetMemoizedResults(this); 363 364 // Remove this SCEVUnknown from the uniquing map. 365 SE->UniqueSCEVs.RemoveNode(this); 366 367 // Release the value. 368 setValPtr(nullptr); 369 } 370 371 void SCEVUnknown::allUsesReplacedWith(Value *New) { 372 // Clear this SCEVUnknown from various maps. 373 SE->forgetMemoizedResults(this); 374 375 // Remove this SCEVUnknown from the uniquing map. 376 SE->UniqueSCEVs.RemoveNode(this); 377 378 // Update this SCEVUnknown to point to the new value. This is needed 379 // because there may still be outstanding SCEVs which still point to 380 // this SCEVUnknown. 381 setValPtr(New); 382 } 383 384 bool SCEVUnknown::isSizeOf(Type *&AllocTy) const { 385 if (ConstantExpr *VCE = dyn_cast<ConstantExpr>(getValue())) 386 if (VCE->getOpcode() == Instruction::PtrToInt) 387 if (ConstantExpr *CE = dyn_cast<ConstantExpr>(VCE->getOperand(0))) 388 if (CE->getOpcode() == Instruction::GetElementPtr && 389 CE->getOperand(0)->isNullValue() && 390 CE->getNumOperands() == 2) 391 if (ConstantInt *CI = dyn_cast<ConstantInt>(CE->getOperand(1))) 392 if (CI->isOne()) { 393 AllocTy = cast<PointerType>(CE->getOperand(0)->getType()) 394 ->getElementType(); 395 return true; 396 } 397 398 return false; 399 } 400 401 bool SCEVUnknown::isAlignOf(Type *&AllocTy) const { 402 if (ConstantExpr *VCE = dyn_cast<ConstantExpr>(getValue())) 403 if (VCE->getOpcode() == Instruction::PtrToInt) 404 if (ConstantExpr *CE = dyn_cast<ConstantExpr>(VCE->getOperand(0))) 405 if (CE->getOpcode() == Instruction::GetElementPtr && 406 CE->getOperand(0)->isNullValue()) { 407 Type *Ty = 408 cast<PointerType>(CE->getOperand(0)->getType())->getElementType(); 409 if (StructType *STy = dyn_cast<StructType>(Ty)) 410 if (!STy->isPacked() && 411 CE->getNumOperands() == 3 && 412 CE->getOperand(1)->isNullValue()) { 413 if (ConstantInt *CI = dyn_cast<ConstantInt>(CE->getOperand(2))) 414 if (CI->isOne() && 415 STy->getNumElements() == 2 && 416 STy->getElementType(0)->isIntegerTy(1)) { 417 AllocTy = STy->getElementType(1); 418 return true; 419 } 420 } 421 } 422 423 return false; 424 } 425 426 bool SCEVUnknown::isOffsetOf(Type *&CTy, Constant *&FieldNo) const { 427 if (ConstantExpr *VCE = dyn_cast<ConstantExpr>(getValue())) 428 if (VCE->getOpcode() == Instruction::PtrToInt) 429 if (ConstantExpr *CE = dyn_cast<ConstantExpr>(VCE->getOperand(0))) 430 if (CE->getOpcode() == Instruction::GetElementPtr && 431 CE->getNumOperands() == 3 && 432 CE->getOperand(0)->isNullValue() && 433 CE->getOperand(1)->isNullValue()) { 434 Type *Ty = 435 cast<PointerType>(CE->getOperand(0)->getType())->getElementType(); 436 // Ignore vector types here so that ScalarEvolutionExpander doesn't 437 // emit getelementptrs that index into vectors. 438 if (Ty->isStructTy() || Ty->isArrayTy()) { 439 CTy = Ty; 440 FieldNo = CE->getOperand(2); 441 return true; 442 } 443 } 444 445 return false; 446 } 447 448 //===----------------------------------------------------------------------===// 449 // SCEV Utilities 450 //===----------------------------------------------------------------------===// 451 452 namespace { 453 /// SCEVComplexityCompare - Return true if the complexity of the LHS is less 454 /// than the complexity of the RHS. This comparator is used to canonicalize 455 /// expressions. 456 class SCEVComplexityCompare { 457 const LoopInfo *const LI; 458 public: 459 explicit SCEVComplexityCompare(const LoopInfo *li) : LI(li) {} 460 461 // Return true or false if LHS is less than, or at least RHS, respectively. 462 bool operator()(const SCEV *LHS, const SCEV *RHS) const { 463 return compare(LHS, RHS) < 0; 464 } 465 466 // Return negative, zero, or positive, if LHS is less than, equal to, or 467 // greater than RHS, respectively. A three-way result allows recursive 468 // comparisons to be more efficient. 469 int compare(const SCEV *LHS, const SCEV *RHS) const { 470 // Fast-path: SCEVs are uniqued so we can do a quick equality check. 471 if (LHS == RHS) 472 return 0; 473 474 // Primarily, sort the SCEVs by their getSCEVType(). 475 unsigned LType = LHS->getSCEVType(), RType = RHS->getSCEVType(); 476 if (LType != RType) 477 return (int)LType - (int)RType; 478 479 // Aside from the getSCEVType() ordering, the particular ordering 480 // isn't very important except that it's beneficial to be consistent, 481 // so that (a + b) and (b + a) don't end up as different expressions. 482 switch (static_cast<SCEVTypes>(LType)) { 483 case scUnknown: { 484 const SCEVUnknown *LU = cast<SCEVUnknown>(LHS); 485 const SCEVUnknown *RU = cast<SCEVUnknown>(RHS); 486 487 // Sort SCEVUnknown values with some loose heuristics. TODO: This is 488 // not as complete as it could be. 489 const Value *LV = LU->getValue(), *RV = RU->getValue(); 490 491 // Order pointer values after integer values. This helps SCEVExpander 492 // form GEPs. 493 bool LIsPointer = LV->getType()->isPointerTy(), 494 RIsPointer = RV->getType()->isPointerTy(); 495 if (LIsPointer != RIsPointer) 496 return (int)LIsPointer - (int)RIsPointer; 497 498 // Compare getValueID values. 499 unsigned LID = LV->getValueID(), 500 RID = RV->getValueID(); 501 if (LID != RID) 502 return (int)LID - (int)RID; 503 504 // Sort arguments by their position. 505 if (const Argument *LA = dyn_cast<Argument>(LV)) { 506 const Argument *RA = cast<Argument>(RV); 507 unsigned LArgNo = LA->getArgNo(), RArgNo = RA->getArgNo(); 508 return (int)LArgNo - (int)RArgNo; 509 } 510 511 // For instructions, compare their loop depth, and their operand 512 // count. This is pretty loose. 513 if (const Instruction *LInst = dyn_cast<Instruction>(LV)) { 514 const Instruction *RInst = cast<Instruction>(RV); 515 516 // Compare loop depths. 517 const BasicBlock *LParent = LInst->getParent(), 518 *RParent = RInst->getParent(); 519 if (LParent != RParent) { 520 unsigned LDepth = LI->getLoopDepth(LParent), 521 RDepth = LI->getLoopDepth(RParent); 522 if (LDepth != RDepth) 523 return (int)LDepth - (int)RDepth; 524 } 525 526 // Compare the number of operands. 527 unsigned LNumOps = LInst->getNumOperands(), 528 RNumOps = RInst->getNumOperands(); 529 return (int)LNumOps - (int)RNumOps; 530 } 531 532 return 0; 533 } 534 535 case scConstant: { 536 const SCEVConstant *LC = cast<SCEVConstant>(LHS); 537 const SCEVConstant *RC = cast<SCEVConstant>(RHS); 538 539 // Compare constant values. 540 const APInt &LA = LC->getAPInt(); 541 const APInt &RA = RC->getAPInt(); 542 unsigned LBitWidth = LA.getBitWidth(), RBitWidth = RA.getBitWidth(); 543 if (LBitWidth != RBitWidth) 544 return (int)LBitWidth - (int)RBitWidth; 545 return LA.ult(RA) ? -1 : 1; 546 } 547 548 case scAddRecExpr: { 549 const SCEVAddRecExpr *LA = cast<SCEVAddRecExpr>(LHS); 550 const SCEVAddRecExpr *RA = cast<SCEVAddRecExpr>(RHS); 551 552 // Compare addrec loop depths. 553 const Loop *LLoop = LA->getLoop(), *RLoop = RA->getLoop(); 554 if (LLoop != RLoop) { 555 unsigned LDepth = LLoop->getLoopDepth(), 556 RDepth = RLoop->getLoopDepth(); 557 if (LDepth != RDepth) 558 return (int)LDepth - (int)RDepth; 559 } 560 561 // Addrec complexity grows with operand count. 562 unsigned LNumOps = LA->getNumOperands(), RNumOps = RA->getNumOperands(); 563 if (LNumOps != RNumOps) 564 return (int)LNumOps - (int)RNumOps; 565 566 // Lexicographically compare. 567 for (unsigned i = 0; i != LNumOps; ++i) { 568 long X = compare(LA->getOperand(i), RA->getOperand(i)); 569 if (X != 0) 570 return X; 571 } 572 573 return 0; 574 } 575 576 case scAddExpr: 577 case scMulExpr: 578 case scSMaxExpr: 579 case scUMaxExpr: { 580 const SCEVNAryExpr *LC = cast<SCEVNAryExpr>(LHS); 581 const SCEVNAryExpr *RC = cast<SCEVNAryExpr>(RHS); 582 583 // Lexicographically compare n-ary expressions. 584 unsigned LNumOps = LC->getNumOperands(), RNumOps = RC->getNumOperands(); 585 if (LNumOps != RNumOps) 586 return (int)LNumOps - (int)RNumOps; 587 588 for (unsigned i = 0; i != LNumOps; ++i) { 589 if (i >= RNumOps) 590 return 1; 591 long X = compare(LC->getOperand(i), RC->getOperand(i)); 592 if (X != 0) 593 return X; 594 } 595 return (int)LNumOps - (int)RNumOps; 596 } 597 598 case scUDivExpr: { 599 const SCEVUDivExpr *LC = cast<SCEVUDivExpr>(LHS); 600 const SCEVUDivExpr *RC = cast<SCEVUDivExpr>(RHS); 601 602 // Lexicographically compare udiv expressions. 603 long X = compare(LC->getLHS(), RC->getLHS()); 604 if (X != 0) 605 return X; 606 return compare(LC->getRHS(), RC->getRHS()); 607 } 608 609 case scTruncate: 610 case scZeroExtend: 611 case scSignExtend: { 612 const SCEVCastExpr *LC = cast<SCEVCastExpr>(LHS); 613 const SCEVCastExpr *RC = cast<SCEVCastExpr>(RHS); 614 615 // Compare cast expressions by operand. 616 return compare(LC->getOperand(), RC->getOperand()); 617 } 618 619 case scCouldNotCompute: 620 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!"); 621 } 622 llvm_unreachable("Unknown SCEV kind!"); 623 } 624 }; 625 } // end anonymous namespace 626 627 /// GroupByComplexity - Given a list of SCEV objects, order them by their 628 /// complexity, and group objects of the same complexity together by value. 629 /// When this routine is finished, we know that any duplicates in the vector are 630 /// consecutive and that complexity is monotonically increasing. 631 /// 632 /// Note that we go take special precautions to ensure that we get deterministic 633 /// results from this routine. In other words, we don't want the results of 634 /// this to depend on where the addresses of various SCEV objects happened to 635 /// land in memory. 636 /// 637 static void GroupByComplexity(SmallVectorImpl<const SCEV *> &Ops, 638 LoopInfo *LI) { 639 if (Ops.size() < 2) return; // Noop 640 if (Ops.size() == 2) { 641 // This is the common case, which also happens to be trivially simple. 642 // Special case it. 643 const SCEV *&LHS = Ops[0], *&RHS = Ops[1]; 644 if (SCEVComplexityCompare(LI)(RHS, LHS)) 645 std::swap(LHS, RHS); 646 return; 647 } 648 649 // Do the rough sort by complexity. 650 std::stable_sort(Ops.begin(), Ops.end(), SCEVComplexityCompare(LI)); 651 652 // Now that we are sorted by complexity, group elements of the same 653 // complexity. Note that this is, at worst, N^2, but the vector is likely to 654 // be extremely short in practice. Note that we take this approach because we 655 // do not want to depend on the addresses of the objects we are grouping. 656 for (unsigned i = 0, e = Ops.size(); i != e-2; ++i) { 657 const SCEV *S = Ops[i]; 658 unsigned Complexity = S->getSCEVType(); 659 660 // If there are any objects of the same complexity and same value as this 661 // one, group them. 662 for (unsigned j = i+1; j != e && Ops[j]->getSCEVType() == Complexity; ++j) { 663 if (Ops[j] == S) { // Found a duplicate. 664 // Move it to immediately after i'th element. 665 std::swap(Ops[i+1], Ops[j]); 666 ++i; // no need to rescan it. 667 if (i == e-2) return; // Done! 668 } 669 } 670 } 671 } 672 673 // Returns the size of the SCEV S. 674 static inline int sizeOfSCEV(const SCEV *S) { 675 struct FindSCEVSize { 676 int Size; 677 FindSCEVSize() : Size(0) {} 678 679 bool follow(const SCEV *S) { 680 ++Size; 681 // Keep looking at all operands of S. 682 return true; 683 } 684 bool isDone() const { 685 return false; 686 } 687 }; 688 689 FindSCEVSize F; 690 SCEVTraversal<FindSCEVSize> ST(F); 691 ST.visitAll(S); 692 return F.Size; 693 } 694 695 namespace { 696 697 struct SCEVDivision : public SCEVVisitor<SCEVDivision, void> { 698 public: 699 // Computes the Quotient and Remainder of the division of Numerator by 700 // Denominator. 701 static void divide(ScalarEvolution &SE, const SCEV *Numerator, 702 const SCEV *Denominator, const SCEV **Quotient, 703 const SCEV **Remainder) { 704 assert(Numerator && Denominator && "Uninitialized SCEV"); 705 706 SCEVDivision D(SE, Numerator, Denominator); 707 708 // Check for the trivial case here to avoid having to check for it in the 709 // rest of the code. 710 if (Numerator == Denominator) { 711 *Quotient = D.One; 712 *Remainder = D.Zero; 713 return; 714 } 715 716 if (Numerator->isZero()) { 717 *Quotient = D.Zero; 718 *Remainder = D.Zero; 719 return; 720 } 721 722 // A simple case when N/1. The quotient is N. 723 if (Denominator->isOne()) { 724 *Quotient = Numerator; 725 *Remainder = D.Zero; 726 return; 727 } 728 729 // Split the Denominator when it is a product. 730 if (const SCEVMulExpr *T = dyn_cast<const SCEVMulExpr>(Denominator)) { 731 const SCEV *Q, *R; 732 *Quotient = Numerator; 733 for (const SCEV *Op : T->operands()) { 734 divide(SE, *Quotient, Op, &Q, &R); 735 *Quotient = Q; 736 737 // Bail out when the Numerator is not divisible by one of the terms of 738 // the Denominator. 739 if (!R->isZero()) { 740 *Quotient = D.Zero; 741 *Remainder = Numerator; 742 return; 743 } 744 } 745 *Remainder = D.Zero; 746 return; 747 } 748 749 D.visit(Numerator); 750 *Quotient = D.Quotient; 751 *Remainder = D.Remainder; 752 } 753 754 // Except in the trivial case described above, we do not know how to divide 755 // Expr by Denominator for the following functions with empty implementation. 756 void visitTruncateExpr(const SCEVTruncateExpr *Numerator) {} 757 void visitZeroExtendExpr(const SCEVZeroExtendExpr *Numerator) {} 758 void visitSignExtendExpr(const SCEVSignExtendExpr *Numerator) {} 759 void visitUDivExpr(const SCEVUDivExpr *Numerator) {} 760 void visitSMaxExpr(const SCEVSMaxExpr *Numerator) {} 761 void visitUMaxExpr(const SCEVUMaxExpr *Numerator) {} 762 void visitUnknown(const SCEVUnknown *Numerator) {} 763 void visitCouldNotCompute(const SCEVCouldNotCompute *Numerator) {} 764 765 void visitConstant(const SCEVConstant *Numerator) { 766 if (const SCEVConstant *D = dyn_cast<SCEVConstant>(Denominator)) { 767 APInt NumeratorVal = Numerator->getAPInt(); 768 APInt DenominatorVal = D->getAPInt(); 769 uint32_t NumeratorBW = NumeratorVal.getBitWidth(); 770 uint32_t DenominatorBW = DenominatorVal.getBitWidth(); 771 772 if (NumeratorBW > DenominatorBW) 773 DenominatorVal = DenominatorVal.sext(NumeratorBW); 774 else if (NumeratorBW < DenominatorBW) 775 NumeratorVal = NumeratorVal.sext(DenominatorBW); 776 777 APInt QuotientVal(NumeratorVal.getBitWidth(), 0); 778 APInt RemainderVal(NumeratorVal.getBitWidth(), 0); 779 APInt::sdivrem(NumeratorVal, DenominatorVal, QuotientVal, RemainderVal); 780 Quotient = SE.getConstant(QuotientVal); 781 Remainder = SE.getConstant(RemainderVal); 782 return; 783 } 784 } 785 786 void visitAddRecExpr(const SCEVAddRecExpr *Numerator) { 787 const SCEV *StartQ, *StartR, *StepQ, *StepR; 788 if (!Numerator->isAffine()) 789 return cannotDivide(Numerator); 790 divide(SE, Numerator->getStart(), Denominator, &StartQ, &StartR); 791 divide(SE, Numerator->getStepRecurrence(SE), Denominator, &StepQ, &StepR); 792 // Bail out if the types do not match. 793 Type *Ty = Denominator->getType(); 794 if (Ty != StartQ->getType() || Ty != StartR->getType() || 795 Ty != StepQ->getType() || Ty != StepR->getType()) 796 return cannotDivide(Numerator); 797 Quotient = SE.getAddRecExpr(StartQ, StepQ, Numerator->getLoop(), 798 Numerator->getNoWrapFlags()); 799 Remainder = SE.getAddRecExpr(StartR, StepR, Numerator->getLoop(), 800 Numerator->getNoWrapFlags()); 801 } 802 803 void visitAddExpr(const SCEVAddExpr *Numerator) { 804 SmallVector<const SCEV *, 2> Qs, Rs; 805 Type *Ty = Denominator->getType(); 806 807 for (const SCEV *Op : Numerator->operands()) { 808 const SCEV *Q, *R; 809 divide(SE, Op, Denominator, &Q, &R); 810 811 // Bail out if types do not match. 812 if (Ty != Q->getType() || Ty != R->getType()) 813 return cannotDivide(Numerator); 814 815 Qs.push_back(Q); 816 Rs.push_back(R); 817 } 818 819 if (Qs.size() == 1) { 820 Quotient = Qs[0]; 821 Remainder = Rs[0]; 822 return; 823 } 824 825 Quotient = SE.getAddExpr(Qs); 826 Remainder = SE.getAddExpr(Rs); 827 } 828 829 void visitMulExpr(const SCEVMulExpr *Numerator) { 830 SmallVector<const SCEV *, 2> Qs; 831 Type *Ty = Denominator->getType(); 832 833 bool FoundDenominatorTerm = false; 834 for (const SCEV *Op : Numerator->operands()) { 835 // Bail out if types do not match. 836 if (Ty != Op->getType()) 837 return cannotDivide(Numerator); 838 839 if (FoundDenominatorTerm) { 840 Qs.push_back(Op); 841 continue; 842 } 843 844 // Check whether Denominator divides one of the product operands. 845 const SCEV *Q, *R; 846 divide(SE, Op, Denominator, &Q, &R); 847 if (!R->isZero()) { 848 Qs.push_back(Op); 849 continue; 850 } 851 852 // Bail out if types do not match. 853 if (Ty != Q->getType()) 854 return cannotDivide(Numerator); 855 856 FoundDenominatorTerm = true; 857 Qs.push_back(Q); 858 } 859 860 if (FoundDenominatorTerm) { 861 Remainder = Zero; 862 if (Qs.size() == 1) 863 Quotient = Qs[0]; 864 else 865 Quotient = SE.getMulExpr(Qs); 866 return; 867 } 868 869 if (!isa<SCEVUnknown>(Denominator)) 870 return cannotDivide(Numerator); 871 872 // The Remainder is obtained by replacing Denominator by 0 in Numerator. 873 ValueToValueMap RewriteMap; 874 RewriteMap[cast<SCEVUnknown>(Denominator)->getValue()] = 875 cast<SCEVConstant>(Zero)->getValue(); 876 Remainder = SCEVParameterRewriter::rewrite(Numerator, SE, RewriteMap, true); 877 878 if (Remainder->isZero()) { 879 // The Quotient is obtained by replacing Denominator by 1 in Numerator. 880 RewriteMap[cast<SCEVUnknown>(Denominator)->getValue()] = 881 cast<SCEVConstant>(One)->getValue(); 882 Quotient = 883 SCEVParameterRewriter::rewrite(Numerator, SE, RewriteMap, true); 884 return; 885 } 886 887 // Quotient is (Numerator - Remainder) divided by Denominator. 888 const SCEV *Q, *R; 889 const SCEV *Diff = SE.getMinusSCEV(Numerator, Remainder); 890 // This SCEV does not seem to simplify: fail the division here. 891 if (sizeOfSCEV(Diff) > sizeOfSCEV(Numerator)) 892 return cannotDivide(Numerator); 893 divide(SE, Diff, Denominator, &Q, &R); 894 if (R != Zero) 895 return cannotDivide(Numerator); 896 Quotient = Q; 897 } 898 899 private: 900 SCEVDivision(ScalarEvolution &S, const SCEV *Numerator, 901 const SCEV *Denominator) 902 : SE(S), Denominator(Denominator) { 903 Zero = SE.getZero(Denominator->getType()); 904 One = SE.getOne(Denominator->getType()); 905 906 // We generally do not know how to divide Expr by Denominator. We 907 // initialize the division to a "cannot divide" state to simplify the rest 908 // of the code. 909 cannotDivide(Numerator); 910 } 911 912 // Convenience function for giving up on the division. We set the quotient to 913 // be equal to zero and the remainder to be equal to the numerator. 914 void cannotDivide(const SCEV *Numerator) { 915 Quotient = Zero; 916 Remainder = Numerator; 917 } 918 919 ScalarEvolution &SE; 920 const SCEV *Denominator, *Quotient, *Remainder, *Zero, *One; 921 }; 922 923 } 924 925 //===----------------------------------------------------------------------===// 926 // Simple SCEV method implementations 927 //===----------------------------------------------------------------------===// 928 929 /// BinomialCoefficient - Compute BC(It, K). The result has width W. 930 /// Assume, K > 0. 931 static const SCEV *BinomialCoefficient(const SCEV *It, unsigned K, 932 ScalarEvolution &SE, 933 Type *ResultTy) { 934 // Handle the simplest case efficiently. 935 if (K == 1) 936 return SE.getTruncateOrZeroExtend(It, ResultTy); 937 938 // We are using the following formula for BC(It, K): 939 // 940 // BC(It, K) = (It * (It - 1) * ... * (It - K + 1)) / K! 941 // 942 // Suppose, W is the bitwidth of the return value. We must be prepared for 943 // overflow. Hence, we must assure that the result of our computation is 944 // equal to the accurate one modulo 2^W. Unfortunately, division isn't 945 // safe in modular arithmetic. 946 // 947 // However, this code doesn't use exactly that formula; the formula it uses 948 // is something like the following, where T is the number of factors of 2 in 949 // K! (i.e. trailing zeros in the binary representation of K!), and ^ is 950 // exponentiation: 951 // 952 // BC(It, K) = (It * (It - 1) * ... * (It - K + 1)) / 2^T / (K! / 2^T) 953 // 954 // This formula is trivially equivalent to the previous formula. However, 955 // this formula can be implemented much more efficiently. The trick is that 956 // K! / 2^T is odd, and exact division by an odd number *is* safe in modular 957 // arithmetic. To do exact division in modular arithmetic, all we have 958 // to do is multiply by the inverse. Therefore, this step can be done at 959 // width W. 960 // 961 // The next issue is how to safely do the division by 2^T. The way this 962 // is done is by doing the multiplication step at a width of at least W + T 963 // bits. This way, the bottom W+T bits of the product are accurate. Then, 964 // when we perform the division by 2^T (which is equivalent to a right shift 965 // by T), the bottom W bits are accurate. Extra bits are okay; they'll get 966 // truncated out after the division by 2^T. 967 // 968 // In comparison to just directly using the first formula, this technique 969 // is much more efficient; using the first formula requires W * K bits, 970 // but this formula less than W + K bits. Also, the first formula requires 971 // a division step, whereas this formula only requires multiplies and shifts. 972 // 973 // It doesn't matter whether the subtraction step is done in the calculation 974 // width or the input iteration count's width; if the subtraction overflows, 975 // the result must be zero anyway. We prefer here to do it in the width of 976 // the induction variable because it helps a lot for certain cases; CodeGen 977 // isn't smart enough to ignore the overflow, which leads to much less 978 // efficient code if the width of the subtraction is wider than the native 979 // register width. 980 // 981 // (It's possible to not widen at all by pulling out factors of 2 before 982 // the multiplication; for example, K=2 can be calculated as 983 // It/2*(It+(It*INT_MIN/INT_MIN)+-1). However, it requires 984 // extra arithmetic, so it's not an obvious win, and it gets 985 // much more complicated for K > 3.) 986 987 // Protection from insane SCEVs; this bound is conservative, 988 // but it probably doesn't matter. 989 if (K > 1000) 990 return SE.getCouldNotCompute(); 991 992 unsigned W = SE.getTypeSizeInBits(ResultTy); 993 994 // Calculate K! / 2^T and T; we divide out the factors of two before 995 // multiplying for calculating K! / 2^T to avoid overflow. 996 // Other overflow doesn't matter because we only care about the bottom 997 // W bits of the result. 998 APInt OddFactorial(W, 1); 999 unsigned T = 1; 1000 for (unsigned i = 3; i <= K; ++i) { 1001 APInt Mult(W, i); 1002 unsigned TwoFactors = Mult.countTrailingZeros(); 1003 T += TwoFactors; 1004 Mult = Mult.lshr(TwoFactors); 1005 OddFactorial *= Mult; 1006 } 1007 1008 // We need at least W + T bits for the multiplication step 1009 unsigned CalculationBits = W + T; 1010 1011 // Calculate 2^T, at width T+W. 1012 APInt DivFactor = APInt::getOneBitSet(CalculationBits, T); 1013 1014 // Calculate the multiplicative inverse of K! / 2^T; 1015 // this multiplication factor will perform the exact division by 1016 // K! / 2^T. 1017 APInt Mod = APInt::getSignedMinValue(W+1); 1018 APInt MultiplyFactor = OddFactorial.zext(W+1); 1019 MultiplyFactor = MultiplyFactor.multiplicativeInverse(Mod); 1020 MultiplyFactor = MultiplyFactor.trunc(W); 1021 1022 // Calculate the product, at width T+W 1023 IntegerType *CalculationTy = IntegerType::get(SE.getContext(), 1024 CalculationBits); 1025 const SCEV *Dividend = SE.getTruncateOrZeroExtend(It, CalculationTy); 1026 for (unsigned i = 1; i != K; ++i) { 1027 const SCEV *S = SE.getMinusSCEV(It, SE.getConstant(It->getType(), i)); 1028 Dividend = SE.getMulExpr(Dividend, 1029 SE.getTruncateOrZeroExtend(S, CalculationTy)); 1030 } 1031 1032 // Divide by 2^T 1033 const SCEV *DivResult = SE.getUDivExpr(Dividend, SE.getConstant(DivFactor)); 1034 1035 // Truncate the result, and divide by K! / 2^T. 1036 1037 return SE.getMulExpr(SE.getConstant(MultiplyFactor), 1038 SE.getTruncateOrZeroExtend(DivResult, ResultTy)); 1039 } 1040 1041 /// evaluateAtIteration - Return the value of this chain of recurrences at 1042 /// the specified iteration number. We can evaluate this recurrence by 1043 /// multiplying each element in the chain by the binomial coefficient 1044 /// corresponding to it. In other words, we can evaluate {A,+,B,+,C,+,D} as: 1045 /// 1046 /// A*BC(It, 0) + B*BC(It, 1) + C*BC(It, 2) + D*BC(It, 3) 1047 /// 1048 /// where BC(It, k) stands for binomial coefficient. 1049 /// 1050 const SCEV *SCEVAddRecExpr::evaluateAtIteration(const SCEV *It, 1051 ScalarEvolution &SE) const { 1052 const SCEV *Result = getStart(); 1053 for (unsigned i = 1, e = getNumOperands(); i != e; ++i) { 1054 // The computation is correct in the face of overflow provided that the 1055 // multiplication is performed _after_ the evaluation of the binomial 1056 // coefficient. 1057 const SCEV *Coeff = BinomialCoefficient(It, i, SE, getType()); 1058 if (isa<SCEVCouldNotCompute>(Coeff)) 1059 return Coeff; 1060 1061 Result = SE.getAddExpr(Result, SE.getMulExpr(getOperand(i), Coeff)); 1062 } 1063 return Result; 1064 } 1065 1066 //===----------------------------------------------------------------------===// 1067 // SCEV Expression folder implementations 1068 //===----------------------------------------------------------------------===// 1069 1070 const SCEV *ScalarEvolution::getTruncateExpr(const SCEV *Op, 1071 Type *Ty) { 1072 assert(getTypeSizeInBits(Op->getType()) > getTypeSizeInBits(Ty) && 1073 "This is not a truncating conversion!"); 1074 assert(isSCEVable(Ty) && 1075 "This is not a conversion to a SCEVable type!"); 1076 Ty = getEffectiveSCEVType(Ty); 1077 1078 FoldingSetNodeID ID; 1079 ID.AddInteger(scTruncate); 1080 ID.AddPointer(Op); 1081 ID.AddPointer(Ty); 1082 void *IP = nullptr; 1083 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S; 1084 1085 // Fold if the operand is constant. 1086 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Op)) 1087 return getConstant( 1088 cast<ConstantInt>(ConstantExpr::getTrunc(SC->getValue(), Ty))); 1089 1090 // trunc(trunc(x)) --> trunc(x) 1091 if (const SCEVTruncateExpr *ST = dyn_cast<SCEVTruncateExpr>(Op)) 1092 return getTruncateExpr(ST->getOperand(), Ty); 1093 1094 // trunc(sext(x)) --> sext(x) if widening or trunc(x) if narrowing 1095 if (const SCEVSignExtendExpr *SS = dyn_cast<SCEVSignExtendExpr>(Op)) 1096 return getTruncateOrSignExtend(SS->getOperand(), Ty); 1097 1098 // trunc(zext(x)) --> zext(x) if widening or trunc(x) if narrowing 1099 if (const SCEVZeroExtendExpr *SZ = dyn_cast<SCEVZeroExtendExpr>(Op)) 1100 return getTruncateOrZeroExtend(SZ->getOperand(), Ty); 1101 1102 // trunc(x1+x2+...+xN) --> trunc(x1)+trunc(x2)+...+trunc(xN) if we can 1103 // eliminate all the truncates, or we replace other casts with truncates. 1104 if (const SCEVAddExpr *SA = dyn_cast<SCEVAddExpr>(Op)) { 1105 SmallVector<const SCEV *, 4> Operands; 1106 bool hasTrunc = false; 1107 for (unsigned i = 0, e = SA->getNumOperands(); i != e && !hasTrunc; ++i) { 1108 const SCEV *S = getTruncateExpr(SA->getOperand(i), Ty); 1109 if (!isa<SCEVCastExpr>(SA->getOperand(i))) 1110 hasTrunc = isa<SCEVTruncateExpr>(S); 1111 Operands.push_back(S); 1112 } 1113 if (!hasTrunc) 1114 return getAddExpr(Operands); 1115 UniqueSCEVs.FindNodeOrInsertPos(ID, IP); // Mutates IP, returns NULL. 1116 } 1117 1118 // trunc(x1*x2*...*xN) --> trunc(x1)*trunc(x2)*...*trunc(xN) if we can 1119 // eliminate all the truncates, or we replace other casts with truncates. 1120 if (const SCEVMulExpr *SM = dyn_cast<SCEVMulExpr>(Op)) { 1121 SmallVector<const SCEV *, 4> Operands; 1122 bool hasTrunc = false; 1123 for (unsigned i = 0, e = SM->getNumOperands(); i != e && !hasTrunc; ++i) { 1124 const SCEV *S = getTruncateExpr(SM->getOperand(i), Ty); 1125 if (!isa<SCEVCastExpr>(SM->getOperand(i))) 1126 hasTrunc = isa<SCEVTruncateExpr>(S); 1127 Operands.push_back(S); 1128 } 1129 if (!hasTrunc) 1130 return getMulExpr(Operands); 1131 UniqueSCEVs.FindNodeOrInsertPos(ID, IP); // Mutates IP, returns NULL. 1132 } 1133 1134 // If the input value is a chrec scev, truncate the chrec's operands. 1135 if (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(Op)) { 1136 SmallVector<const SCEV *, 4> Operands; 1137 for (const SCEV *Op : AddRec->operands()) 1138 Operands.push_back(getTruncateExpr(Op, Ty)); 1139 return getAddRecExpr(Operands, AddRec->getLoop(), SCEV::FlagAnyWrap); 1140 } 1141 1142 // The cast wasn't folded; create an explicit cast node. We can reuse 1143 // the existing insert position since if we get here, we won't have 1144 // made any changes which would invalidate it. 1145 SCEV *S = new (SCEVAllocator) SCEVTruncateExpr(ID.Intern(SCEVAllocator), 1146 Op, Ty); 1147 UniqueSCEVs.InsertNode(S, IP); 1148 return S; 1149 } 1150 1151 // Get the limit of a recurrence such that incrementing by Step cannot cause 1152 // signed overflow as long as the value of the recurrence within the 1153 // loop does not exceed this limit before incrementing. 1154 static const SCEV *getSignedOverflowLimitForStep(const SCEV *Step, 1155 ICmpInst::Predicate *Pred, 1156 ScalarEvolution *SE) { 1157 unsigned BitWidth = SE->getTypeSizeInBits(Step->getType()); 1158 if (SE->isKnownPositive(Step)) { 1159 *Pred = ICmpInst::ICMP_SLT; 1160 return SE->getConstant(APInt::getSignedMinValue(BitWidth) - 1161 SE->getSignedRange(Step).getSignedMax()); 1162 } 1163 if (SE->isKnownNegative(Step)) { 1164 *Pred = ICmpInst::ICMP_SGT; 1165 return SE->getConstant(APInt::getSignedMaxValue(BitWidth) - 1166 SE->getSignedRange(Step).getSignedMin()); 1167 } 1168 return nullptr; 1169 } 1170 1171 // Get the limit of a recurrence such that incrementing by Step cannot cause 1172 // unsigned overflow as long as the value of the recurrence within the loop does 1173 // not exceed this limit before incrementing. 1174 static const SCEV *getUnsignedOverflowLimitForStep(const SCEV *Step, 1175 ICmpInst::Predicate *Pred, 1176 ScalarEvolution *SE) { 1177 unsigned BitWidth = SE->getTypeSizeInBits(Step->getType()); 1178 *Pred = ICmpInst::ICMP_ULT; 1179 1180 return SE->getConstant(APInt::getMinValue(BitWidth) - 1181 SE->getUnsignedRange(Step).getUnsignedMax()); 1182 } 1183 1184 namespace { 1185 1186 struct ExtendOpTraitsBase { 1187 typedef const SCEV *(ScalarEvolution::*GetExtendExprTy)(const SCEV *, Type *); 1188 }; 1189 1190 // Used to make code generic over signed and unsigned overflow. 1191 template <typename ExtendOp> struct ExtendOpTraits { 1192 // Members present: 1193 // 1194 // static const SCEV::NoWrapFlags WrapType; 1195 // 1196 // static const ExtendOpTraitsBase::GetExtendExprTy GetExtendExpr; 1197 // 1198 // static const SCEV *getOverflowLimitForStep(const SCEV *Step, 1199 // ICmpInst::Predicate *Pred, 1200 // ScalarEvolution *SE); 1201 }; 1202 1203 template <> 1204 struct ExtendOpTraits<SCEVSignExtendExpr> : public ExtendOpTraitsBase { 1205 static const SCEV::NoWrapFlags WrapType = SCEV::FlagNSW; 1206 1207 static const GetExtendExprTy GetExtendExpr; 1208 1209 static const SCEV *getOverflowLimitForStep(const SCEV *Step, 1210 ICmpInst::Predicate *Pred, 1211 ScalarEvolution *SE) { 1212 return getSignedOverflowLimitForStep(Step, Pred, SE); 1213 } 1214 }; 1215 1216 const ExtendOpTraitsBase::GetExtendExprTy ExtendOpTraits< 1217 SCEVSignExtendExpr>::GetExtendExpr = &ScalarEvolution::getSignExtendExpr; 1218 1219 template <> 1220 struct ExtendOpTraits<SCEVZeroExtendExpr> : public ExtendOpTraitsBase { 1221 static const SCEV::NoWrapFlags WrapType = SCEV::FlagNUW; 1222 1223 static const GetExtendExprTy GetExtendExpr; 1224 1225 static const SCEV *getOverflowLimitForStep(const SCEV *Step, 1226 ICmpInst::Predicate *Pred, 1227 ScalarEvolution *SE) { 1228 return getUnsignedOverflowLimitForStep(Step, Pred, SE); 1229 } 1230 }; 1231 1232 const ExtendOpTraitsBase::GetExtendExprTy ExtendOpTraits< 1233 SCEVZeroExtendExpr>::GetExtendExpr = &ScalarEvolution::getZeroExtendExpr; 1234 } 1235 1236 // The recurrence AR has been shown to have no signed/unsigned wrap or something 1237 // close to it. Typically, if we can prove NSW/NUW for AR, then we can just as 1238 // easily prove NSW/NUW for its preincrement or postincrement sibling. This 1239 // allows normalizing a sign/zero extended AddRec as such: {sext/zext(Step + 1240 // Start),+,Step} => {(Step + sext/zext(Start),+,Step} As a result, the 1241 // expression "Step + sext/zext(PreIncAR)" is congruent with 1242 // "sext/zext(PostIncAR)" 1243 template <typename ExtendOpTy> 1244 static const SCEV *getPreStartForExtend(const SCEVAddRecExpr *AR, Type *Ty, 1245 ScalarEvolution *SE) { 1246 auto WrapType = ExtendOpTraits<ExtendOpTy>::WrapType; 1247 auto GetExtendExpr = ExtendOpTraits<ExtendOpTy>::GetExtendExpr; 1248 1249 const Loop *L = AR->getLoop(); 1250 const SCEV *Start = AR->getStart(); 1251 const SCEV *Step = AR->getStepRecurrence(*SE); 1252 1253 // Check for a simple looking step prior to loop entry. 1254 const SCEVAddExpr *SA = dyn_cast<SCEVAddExpr>(Start); 1255 if (!SA) 1256 return nullptr; 1257 1258 // Create an AddExpr for "PreStart" after subtracting Step. Full SCEV 1259 // subtraction is expensive. For this purpose, perform a quick and dirty 1260 // difference, by checking for Step in the operand list. 1261 SmallVector<const SCEV *, 4> DiffOps; 1262 for (const SCEV *Op : SA->operands()) 1263 if (Op != Step) 1264 DiffOps.push_back(Op); 1265 1266 if (DiffOps.size() == SA->getNumOperands()) 1267 return nullptr; 1268 1269 // Try to prove `WrapType` (SCEV::FlagNSW or SCEV::FlagNUW) on `PreStart` + 1270 // `Step`: 1271 1272 // 1. NSW/NUW flags on the step increment. 1273 auto PreStartFlags = 1274 ScalarEvolution::maskFlags(SA->getNoWrapFlags(), SCEV::FlagNUW); 1275 const SCEV *PreStart = SE->getAddExpr(DiffOps, PreStartFlags); 1276 const SCEVAddRecExpr *PreAR = dyn_cast<SCEVAddRecExpr>( 1277 SE->getAddRecExpr(PreStart, Step, L, SCEV::FlagAnyWrap)); 1278 1279 // "{S,+,X} is <nsw>/<nuw>" and "the backedge is taken at least once" implies 1280 // "S+X does not sign/unsign-overflow". 1281 // 1282 1283 const SCEV *BECount = SE->getBackedgeTakenCount(L); 1284 if (PreAR && PreAR->getNoWrapFlags(WrapType) && 1285 !isa<SCEVCouldNotCompute>(BECount) && SE->isKnownPositive(BECount)) 1286 return PreStart; 1287 1288 // 2. Direct overflow check on the step operation's expression. 1289 unsigned BitWidth = SE->getTypeSizeInBits(AR->getType()); 1290 Type *WideTy = IntegerType::get(SE->getContext(), BitWidth * 2); 1291 const SCEV *OperandExtendedStart = 1292 SE->getAddExpr((SE->*GetExtendExpr)(PreStart, WideTy), 1293 (SE->*GetExtendExpr)(Step, WideTy)); 1294 if ((SE->*GetExtendExpr)(Start, WideTy) == OperandExtendedStart) { 1295 if (PreAR && AR->getNoWrapFlags(WrapType)) { 1296 // If we know `AR` == {`PreStart`+`Step`,+,`Step`} is `WrapType` (FlagNSW 1297 // or FlagNUW) and that `PreStart` + `Step` is `WrapType` too, then 1298 // `PreAR` == {`PreStart`,+,`Step`} is also `WrapType`. Cache this fact. 1299 const_cast<SCEVAddRecExpr *>(PreAR)->setNoWrapFlags(WrapType); 1300 } 1301 return PreStart; 1302 } 1303 1304 // 3. Loop precondition. 1305 ICmpInst::Predicate Pred; 1306 const SCEV *OverflowLimit = 1307 ExtendOpTraits<ExtendOpTy>::getOverflowLimitForStep(Step, &Pred, SE); 1308 1309 if (OverflowLimit && 1310 SE->isLoopEntryGuardedByCond(L, Pred, PreStart, OverflowLimit)) 1311 return PreStart; 1312 1313 return nullptr; 1314 } 1315 1316 // Get the normalized zero or sign extended expression for this AddRec's Start. 1317 template <typename ExtendOpTy> 1318 static const SCEV *getExtendAddRecStart(const SCEVAddRecExpr *AR, Type *Ty, 1319 ScalarEvolution *SE) { 1320 auto GetExtendExpr = ExtendOpTraits<ExtendOpTy>::GetExtendExpr; 1321 1322 const SCEV *PreStart = getPreStartForExtend<ExtendOpTy>(AR, Ty, SE); 1323 if (!PreStart) 1324 return (SE->*GetExtendExpr)(AR->getStart(), Ty); 1325 1326 return SE->getAddExpr((SE->*GetExtendExpr)(AR->getStepRecurrence(*SE), Ty), 1327 (SE->*GetExtendExpr)(PreStart, Ty)); 1328 } 1329 1330 // Try to prove away overflow by looking at "nearby" add recurrences. A 1331 // motivating example for this rule: if we know `{0,+,4}` is `ult` `-1` and it 1332 // does not itself wrap then we can conclude that `{1,+,4}` is `nuw`. 1333 // 1334 // Formally: 1335 // 1336 // {S,+,X} == {S-T,+,X} + T 1337 // => Ext({S,+,X}) == Ext({S-T,+,X} + T) 1338 // 1339 // If ({S-T,+,X} + T) does not overflow ... (1) 1340 // 1341 // RHS == Ext({S-T,+,X} + T) == Ext({S-T,+,X}) + Ext(T) 1342 // 1343 // If {S-T,+,X} does not overflow ... (2) 1344 // 1345 // RHS == Ext({S-T,+,X}) + Ext(T) == {Ext(S-T),+,Ext(X)} + Ext(T) 1346 // == {Ext(S-T)+Ext(T),+,Ext(X)} 1347 // 1348 // If (S-T)+T does not overflow ... (3) 1349 // 1350 // RHS == {Ext(S-T)+Ext(T),+,Ext(X)} == {Ext(S-T+T),+,Ext(X)} 1351 // == {Ext(S),+,Ext(X)} == LHS 1352 // 1353 // Thus, if (1), (2) and (3) are true for some T, then 1354 // Ext({S,+,X}) == {Ext(S),+,Ext(X)} 1355 // 1356 // (3) is implied by (1) -- "(S-T)+T does not overflow" is simply "({S-T,+,X}+T) 1357 // does not overflow" restricted to the 0th iteration. Therefore we only need 1358 // to check for (1) and (2). 1359 // 1360 // In the current context, S is `Start`, X is `Step`, Ext is `ExtendOpTy` and T 1361 // is `Delta` (defined below). 1362 // 1363 template <typename ExtendOpTy> 1364 bool ScalarEvolution::proveNoWrapByVaryingStart(const SCEV *Start, 1365 const SCEV *Step, 1366 const Loop *L) { 1367 auto WrapType = ExtendOpTraits<ExtendOpTy>::WrapType; 1368 1369 // We restrict `Start` to a constant to prevent SCEV from spending too much 1370 // time here. It is correct (but more expensive) to continue with a 1371 // non-constant `Start` and do a general SCEV subtraction to compute 1372 // `PreStart` below. 1373 // 1374 const SCEVConstant *StartC = dyn_cast<SCEVConstant>(Start); 1375 if (!StartC) 1376 return false; 1377 1378 APInt StartAI = StartC->getAPInt(); 1379 1380 for (unsigned Delta : {-2, -1, 1, 2}) { 1381 const SCEV *PreStart = getConstant(StartAI - Delta); 1382 1383 FoldingSetNodeID ID; 1384 ID.AddInteger(scAddRecExpr); 1385 ID.AddPointer(PreStart); 1386 ID.AddPointer(Step); 1387 ID.AddPointer(L); 1388 void *IP = nullptr; 1389 const auto *PreAR = 1390 static_cast<SCEVAddRecExpr *>(UniqueSCEVs.FindNodeOrInsertPos(ID, IP)); 1391 1392 // Give up if we don't already have the add recurrence we need because 1393 // actually constructing an add recurrence is relatively expensive. 1394 if (PreAR && PreAR->getNoWrapFlags(WrapType)) { // proves (2) 1395 const SCEV *DeltaS = getConstant(StartC->getType(), Delta); 1396 ICmpInst::Predicate Pred = ICmpInst::BAD_ICMP_PREDICATE; 1397 const SCEV *Limit = ExtendOpTraits<ExtendOpTy>::getOverflowLimitForStep( 1398 DeltaS, &Pred, this); 1399 if (Limit && isKnownPredicate(Pred, PreAR, Limit)) // proves (1) 1400 return true; 1401 } 1402 } 1403 1404 return false; 1405 } 1406 1407 const SCEV *ScalarEvolution::getZeroExtendExpr(const SCEV *Op, 1408 Type *Ty) { 1409 assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) && 1410 "This is not an extending conversion!"); 1411 assert(isSCEVable(Ty) && 1412 "This is not a conversion to a SCEVable type!"); 1413 Ty = getEffectiveSCEVType(Ty); 1414 1415 // Fold if the operand is constant. 1416 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Op)) 1417 return getConstant( 1418 cast<ConstantInt>(ConstantExpr::getZExt(SC->getValue(), Ty))); 1419 1420 // zext(zext(x)) --> zext(x) 1421 if (const SCEVZeroExtendExpr *SZ = dyn_cast<SCEVZeroExtendExpr>(Op)) 1422 return getZeroExtendExpr(SZ->getOperand(), Ty); 1423 1424 // Before doing any expensive analysis, check to see if we've already 1425 // computed a SCEV for this Op and Ty. 1426 FoldingSetNodeID ID; 1427 ID.AddInteger(scZeroExtend); 1428 ID.AddPointer(Op); 1429 ID.AddPointer(Ty); 1430 void *IP = nullptr; 1431 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S; 1432 1433 // zext(trunc(x)) --> zext(x) or x or trunc(x) 1434 if (const SCEVTruncateExpr *ST = dyn_cast<SCEVTruncateExpr>(Op)) { 1435 // It's possible the bits taken off by the truncate were all zero bits. If 1436 // so, we should be able to simplify this further. 1437 const SCEV *X = ST->getOperand(); 1438 ConstantRange CR = getUnsignedRange(X); 1439 unsigned TruncBits = getTypeSizeInBits(ST->getType()); 1440 unsigned NewBits = getTypeSizeInBits(Ty); 1441 if (CR.truncate(TruncBits).zeroExtend(NewBits).contains( 1442 CR.zextOrTrunc(NewBits))) 1443 return getTruncateOrZeroExtend(X, Ty); 1444 } 1445 1446 // If the input value is a chrec scev, and we can prove that the value 1447 // did not overflow the old, smaller, value, we can zero extend all of the 1448 // operands (often constants). This allows analysis of something like 1449 // this: for (unsigned char X = 0; X < 100; ++X) { int Y = X; } 1450 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Op)) 1451 if (AR->isAffine()) { 1452 const SCEV *Start = AR->getStart(); 1453 const SCEV *Step = AR->getStepRecurrence(*this); 1454 unsigned BitWidth = getTypeSizeInBits(AR->getType()); 1455 const Loop *L = AR->getLoop(); 1456 1457 if (!AR->hasNoUnsignedWrap()) { 1458 auto NewFlags = proveNoWrapViaConstantRanges(AR); 1459 const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(NewFlags); 1460 } 1461 1462 // If we have special knowledge that this addrec won't overflow, 1463 // we don't need to do any further analysis. 1464 if (AR->hasNoUnsignedWrap()) 1465 return getAddRecExpr( 1466 getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, this), 1467 getZeroExtendExpr(Step, Ty), L, AR->getNoWrapFlags()); 1468 1469 // Check whether the backedge-taken count is SCEVCouldNotCompute. 1470 // Note that this serves two purposes: It filters out loops that are 1471 // simply not analyzable, and it covers the case where this code is 1472 // being called from within backedge-taken count analysis, such that 1473 // attempting to ask for the backedge-taken count would likely result 1474 // in infinite recursion. In the later case, the analysis code will 1475 // cope with a conservative value, and it will take care to purge 1476 // that value once it has finished. 1477 const SCEV *MaxBECount = getMaxBackedgeTakenCount(L); 1478 if (!isa<SCEVCouldNotCompute>(MaxBECount)) { 1479 // Manually compute the final value for AR, checking for 1480 // overflow. 1481 1482 // Check whether the backedge-taken count can be losslessly casted to 1483 // the addrec's type. The count is always unsigned. 1484 const SCEV *CastedMaxBECount = 1485 getTruncateOrZeroExtend(MaxBECount, Start->getType()); 1486 const SCEV *RecastedMaxBECount = 1487 getTruncateOrZeroExtend(CastedMaxBECount, MaxBECount->getType()); 1488 if (MaxBECount == RecastedMaxBECount) { 1489 Type *WideTy = IntegerType::get(getContext(), BitWidth * 2); 1490 // Check whether Start+Step*MaxBECount has no unsigned overflow. 1491 const SCEV *ZMul = getMulExpr(CastedMaxBECount, Step); 1492 const SCEV *ZAdd = getZeroExtendExpr(getAddExpr(Start, ZMul), WideTy); 1493 const SCEV *WideStart = getZeroExtendExpr(Start, WideTy); 1494 const SCEV *WideMaxBECount = 1495 getZeroExtendExpr(CastedMaxBECount, WideTy); 1496 const SCEV *OperandExtendedAdd = 1497 getAddExpr(WideStart, 1498 getMulExpr(WideMaxBECount, 1499 getZeroExtendExpr(Step, WideTy))); 1500 if (ZAdd == OperandExtendedAdd) { 1501 // Cache knowledge of AR NUW, which is propagated to this AddRec. 1502 const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNUW); 1503 // Return the expression with the addrec on the outside. 1504 return getAddRecExpr( 1505 getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, this), 1506 getZeroExtendExpr(Step, Ty), L, AR->getNoWrapFlags()); 1507 } 1508 // Similar to above, only this time treat the step value as signed. 1509 // This covers loops that count down. 1510 OperandExtendedAdd = 1511 getAddExpr(WideStart, 1512 getMulExpr(WideMaxBECount, 1513 getSignExtendExpr(Step, WideTy))); 1514 if (ZAdd == OperandExtendedAdd) { 1515 // Cache knowledge of AR NW, which is propagated to this AddRec. 1516 // Negative step causes unsigned wrap, but it still can't self-wrap. 1517 const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNW); 1518 // Return the expression with the addrec on the outside. 1519 return getAddRecExpr( 1520 getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, this), 1521 getSignExtendExpr(Step, Ty), L, AR->getNoWrapFlags()); 1522 } 1523 } 1524 1525 // If the backedge is guarded by a comparison with the pre-inc value 1526 // the addrec is safe. Also, if the entry is guarded by a comparison 1527 // with the start value and the backedge is guarded by a comparison 1528 // with the post-inc value, the addrec is safe. 1529 if (isKnownPositive(Step)) { 1530 const SCEV *N = getConstant(APInt::getMinValue(BitWidth) - 1531 getUnsignedRange(Step).getUnsignedMax()); 1532 if (isLoopBackedgeGuardedByCond(L, ICmpInst::ICMP_ULT, AR, N) || 1533 (isLoopEntryGuardedByCond(L, ICmpInst::ICMP_ULT, Start, N) && 1534 isLoopBackedgeGuardedByCond(L, ICmpInst::ICMP_ULT, 1535 AR->getPostIncExpr(*this), N))) { 1536 // Cache knowledge of AR NUW, which is propagated to this AddRec. 1537 const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNUW); 1538 // Return the expression with the addrec on the outside. 1539 return getAddRecExpr( 1540 getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, this), 1541 getZeroExtendExpr(Step, Ty), L, AR->getNoWrapFlags()); 1542 } 1543 } else if (isKnownNegative(Step)) { 1544 const SCEV *N = getConstant(APInt::getMaxValue(BitWidth) - 1545 getSignedRange(Step).getSignedMin()); 1546 if (isLoopBackedgeGuardedByCond(L, ICmpInst::ICMP_UGT, AR, N) || 1547 (isLoopEntryGuardedByCond(L, ICmpInst::ICMP_UGT, Start, N) && 1548 isLoopBackedgeGuardedByCond(L, ICmpInst::ICMP_UGT, 1549 AR->getPostIncExpr(*this), N))) { 1550 // Cache knowledge of AR NW, which is propagated to this AddRec. 1551 // Negative step causes unsigned wrap, but it still can't self-wrap. 1552 const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNW); 1553 // Return the expression with the addrec on the outside. 1554 return getAddRecExpr( 1555 getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, this), 1556 getSignExtendExpr(Step, Ty), L, AR->getNoWrapFlags()); 1557 } 1558 } 1559 } 1560 1561 if (proveNoWrapByVaryingStart<SCEVZeroExtendExpr>(Start, Step, L)) { 1562 const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNUW); 1563 return getAddRecExpr( 1564 getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, this), 1565 getZeroExtendExpr(Step, Ty), L, AR->getNoWrapFlags()); 1566 } 1567 } 1568 1569 if (auto *SA = dyn_cast<SCEVAddExpr>(Op)) { 1570 // zext((A + B + ...)<nuw>) --> (zext(A) + zext(B) + ...)<nuw> 1571 if (SA->hasNoUnsignedWrap()) { 1572 // If the addition does not unsign overflow then we can, by definition, 1573 // commute the zero extension with the addition operation. 1574 SmallVector<const SCEV *, 4> Ops; 1575 for (const auto *Op : SA->operands()) 1576 Ops.push_back(getZeroExtendExpr(Op, Ty)); 1577 return getAddExpr(Ops, SCEV::FlagNUW); 1578 } 1579 } 1580 1581 // The cast wasn't folded; create an explicit cast node. 1582 // Recompute the insert position, as it may have been invalidated. 1583 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S; 1584 SCEV *S = new (SCEVAllocator) SCEVZeroExtendExpr(ID.Intern(SCEVAllocator), 1585 Op, Ty); 1586 UniqueSCEVs.InsertNode(S, IP); 1587 return S; 1588 } 1589 1590 const SCEV *ScalarEvolution::getSignExtendExpr(const SCEV *Op, 1591 Type *Ty) { 1592 assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) && 1593 "This is not an extending conversion!"); 1594 assert(isSCEVable(Ty) && 1595 "This is not a conversion to a SCEVable type!"); 1596 Ty = getEffectiveSCEVType(Ty); 1597 1598 // Fold if the operand is constant. 1599 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Op)) 1600 return getConstant( 1601 cast<ConstantInt>(ConstantExpr::getSExt(SC->getValue(), Ty))); 1602 1603 // sext(sext(x)) --> sext(x) 1604 if (const SCEVSignExtendExpr *SS = dyn_cast<SCEVSignExtendExpr>(Op)) 1605 return getSignExtendExpr(SS->getOperand(), Ty); 1606 1607 // sext(zext(x)) --> zext(x) 1608 if (const SCEVZeroExtendExpr *SZ = dyn_cast<SCEVZeroExtendExpr>(Op)) 1609 return getZeroExtendExpr(SZ->getOperand(), Ty); 1610 1611 // Before doing any expensive analysis, check to see if we've already 1612 // computed a SCEV for this Op and Ty. 1613 FoldingSetNodeID ID; 1614 ID.AddInteger(scSignExtend); 1615 ID.AddPointer(Op); 1616 ID.AddPointer(Ty); 1617 void *IP = nullptr; 1618 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S; 1619 1620 // sext(trunc(x)) --> sext(x) or x or trunc(x) 1621 if (const SCEVTruncateExpr *ST = dyn_cast<SCEVTruncateExpr>(Op)) { 1622 // It's possible the bits taken off by the truncate were all sign bits. If 1623 // so, we should be able to simplify this further. 1624 const SCEV *X = ST->getOperand(); 1625 ConstantRange CR = getSignedRange(X); 1626 unsigned TruncBits = getTypeSizeInBits(ST->getType()); 1627 unsigned NewBits = getTypeSizeInBits(Ty); 1628 if (CR.truncate(TruncBits).signExtend(NewBits).contains( 1629 CR.sextOrTrunc(NewBits))) 1630 return getTruncateOrSignExtend(X, Ty); 1631 } 1632 1633 // sext(C1 + (C2 * x)) --> C1 + sext(C2 * x) if C1 < C2 1634 if (auto *SA = dyn_cast<SCEVAddExpr>(Op)) { 1635 if (SA->getNumOperands() == 2) { 1636 auto *SC1 = dyn_cast<SCEVConstant>(SA->getOperand(0)); 1637 auto *SMul = dyn_cast<SCEVMulExpr>(SA->getOperand(1)); 1638 if (SMul && SC1) { 1639 if (auto *SC2 = dyn_cast<SCEVConstant>(SMul->getOperand(0))) { 1640 const APInt &C1 = SC1->getAPInt(); 1641 const APInt &C2 = SC2->getAPInt(); 1642 if (C1.isStrictlyPositive() && C2.isStrictlyPositive() && 1643 C2.ugt(C1) && C2.isPowerOf2()) 1644 return getAddExpr(getSignExtendExpr(SC1, Ty), 1645 getSignExtendExpr(SMul, Ty)); 1646 } 1647 } 1648 } 1649 1650 // sext((A + B + ...)<nsw>) --> (sext(A) + sext(B) + ...)<nsw> 1651 if (SA->hasNoSignedWrap()) { 1652 // If the addition does not sign overflow then we can, by definition, 1653 // commute the sign extension with the addition operation. 1654 SmallVector<const SCEV *, 4> Ops; 1655 for (const auto *Op : SA->operands()) 1656 Ops.push_back(getSignExtendExpr(Op, Ty)); 1657 return getAddExpr(Ops, SCEV::FlagNSW); 1658 } 1659 } 1660 // If the input value is a chrec scev, and we can prove that the value 1661 // did not overflow the old, smaller, value, we can sign extend all of the 1662 // operands (often constants). This allows analysis of something like 1663 // this: for (signed char X = 0; X < 100; ++X) { int Y = X; } 1664 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Op)) 1665 if (AR->isAffine()) { 1666 const SCEV *Start = AR->getStart(); 1667 const SCEV *Step = AR->getStepRecurrence(*this); 1668 unsigned BitWidth = getTypeSizeInBits(AR->getType()); 1669 const Loop *L = AR->getLoop(); 1670 1671 if (!AR->hasNoSignedWrap()) { 1672 auto NewFlags = proveNoWrapViaConstantRanges(AR); 1673 const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(NewFlags); 1674 } 1675 1676 // If we have special knowledge that this addrec won't overflow, 1677 // we don't need to do any further analysis. 1678 if (AR->hasNoSignedWrap()) 1679 return getAddRecExpr( 1680 getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, this), 1681 getSignExtendExpr(Step, Ty), L, SCEV::FlagNSW); 1682 1683 // Check whether the backedge-taken count is SCEVCouldNotCompute. 1684 // Note that this serves two purposes: It filters out loops that are 1685 // simply not analyzable, and it covers the case where this code is 1686 // being called from within backedge-taken count analysis, such that 1687 // attempting to ask for the backedge-taken count would likely result 1688 // in infinite recursion. In the later case, the analysis code will 1689 // cope with a conservative value, and it will take care to purge 1690 // that value once it has finished. 1691 const SCEV *MaxBECount = getMaxBackedgeTakenCount(L); 1692 if (!isa<SCEVCouldNotCompute>(MaxBECount)) { 1693 // Manually compute the final value for AR, checking for 1694 // overflow. 1695 1696 // Check whether the backedge-taken count can be losslessly casted to 1697 // the addrec's type. The count is always unsigned. 1698 const SCEV *CastedMaxBECount = 1699 getTruncateOrZeroExtend(MaxBECount, Start->getType()); 1700 const SCEV *RecastedMaxBECount = 1701 getTruncateOrZeroExtend(CastedMaxBECount, MaxBECount->getType()); 1702 if (MaxBECount == RecastedMaxBECount) { 1703 Type *WideTy = IntegerType::get(getContext(), BitWidth * 2); 1704 // Check whether Start+Step*MaxBECount has no signed overflow. 1705 const SCEV *SMul = getMulExpr(CastedMaxBECount, Step); 1706 const SCEV *SAdd = getSignExtendExpr(getAddExpr(Start, SMul), WideTy); 1707 const SCEV *WideStart = getSignExtendExpr(Start, WideTy); 1708 const SCEV *WideMaxBECount = 1709 getZeroExtendExpr(CastedMaxBECount, WideTy); 1710 const SCEV *OperandExtendedAdd = 1711 getAddExpr(WideStart, 1712 getMulExpr(WideMaxBECount, 1713 getSignExtendExpr(Step, WideTy))); 1714 if (SAdd == OperandExtendedAdd) { 1715 // Cache knowledge of AR NSW, which is propagated to this AddRec. 1716 const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNSW); 1717 // Return the expression with the addrec on the outside. 1718 return getAddRecExpr( 1719 getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, this), 1720 getSignExtendExpr(Step, Ty), L, AR->getNoWrapFlags()); 1721 } 1722 // Similar to above, only this time treat the step value as unsigned. 1723 // This covers loops that count up with an unsigned step. 1724 OperandExtendedAdd = 1725 getAddExpr(WideStart, 1726 getMulExpr(WideMaxBECount, 1727 getZeroExtendExpr(Step, WideTy))); 1728 if (SAdd == OperandExtendedAdd) { 1729 // If AR wraps around then 1730 // 1731 // abs(Step) * MaxBECount > unsigned-max(AR->getType()) 1732 // => SAdd != OperandExtendedAdd 1733 // 1734 // Thus (AR is not NW => SAdd != OperandExtendedAdd) <=> 1735 // (SAdd == OperandExtendedAdd => AR is NW) 1736 1737 const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNW); 1738 1739 // Return the expression with the addrec on the outside. 1740 return getAddRecExpr( 1741 getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, this), 1742 getZeroExtendExpr(Step, Ty), L, AR->getNoWrapFlags()); 1743 } 1744 } 1745 1746 // If the backedge is guarded by a comparison with the pre-inc value 1747 // the addrec is safe. Also, if the entry is guarded by a comparison 1748 // with the start value and the backedge is guarded by a comparison 1749 // with the post-inc value, the addrec is safe. 1750 ICmpInst::Predicate Pred; 1751 const SCEV *OverflowLimit = 1752 getSignedOverflowLimitForStep(Step, &Pred, this); 1753 if (OverflowLimit && 1754 (isLoopBackedgeGuardedByCond(L, Pred, AR, OverflowLimit) || 1755 (isLoopEntryGuardedByCond(L, Pred, Start, OverflowLimit) && 1756 isLoopBackedgeGuardedByCond(L, Pred, AR->getPostIncExpr(*this), 1757 OverflowLimit)))) { 1758 // Cache knowledge of AR NSW, then propagate NSW to the wide AddRec. 1759 const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNSW); 1760 return getAddRecExpr( 1761 getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, this), 1762 getSignExtendExpr(Step, Ty), L, AR->getNoWrapFlags()); 1763 } 1764 } 1765 // If Start and Step are constants, check if we can apply this 1766 // transformation: 1767 // sext{C1,+,C2} --> C1 + sext{0,+,C2} if C1 < C2 1768 auto *SC1 = dyn_cast<SCEVConstant>(Start); 1769 auto *SC2 = dyn_cast<SCEVConstant>(Step); 1770 if (SC1 && SC2) { 1771 const APInt &C1 = SC1->getAPInt(); 1772 const APInt &C2 = SC2->getAPInt(); 1773 if (C1.isStrictlyPositive() && C2.isStrictlyPositive() && C2.ugt(C1) && 1774 C2.isPowerOf2()) { 1775 Start = getSignExtendExpr(Start, Ty); 1776 const SCEV *NewAR = getAddRecExpr(getZero(AR->getType()), Step, L, 1777 AR->getNoWrapFlags()); 1778 return getAddExpr(Start, getSignExtendExpr(NewAR, Ty)); 1779 } 1780 } 1781 1782 if (proveNoWrapByVaryingStart<SCEVSignExtendExpr>(Start, Step, L)) { 1783 const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNSW); 1784 return getAddRecExpr( 1785 getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, this), 1786 getSignExtendExpr(Step, Ty), L, AR->getNoWrapFlags()); 1787 } 1788 } 1789 1790 // If the input value is provably positive and we could not simplify 1791 // away the sext build a zext instead. 1792 if (isKnownNonNegative(Op)) 1793 return getZeroExtendExpr(Op, Ty); 1794 1795 // The cast wasn't folded; create an explicit cast node. 1796 // Recompute the insert position, as it may have been invalidated. 1797 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S; 1798 SCEV *S = new (SCEVAllocator) SCEVSignExtendExpr(ID.Intern(SCEVAllocator), 1799 Op, Ty); 1800 UniqueSCEVs.InsertNode(S, IP); 1801 return S; 1802 } 1803 1804 /// getAnyExtendExpr - Return a SCEV for the given operand extended with 1805 /// unspecified bits out to the given type. 1806 /// 1807 const SCEV *ScalarEvolution::getAnyExtendExpr(const SCEV *Op, 1808 Type *Ty) { 1809 assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) && 1810 "This is not an extending conversion!"); 1811 assert(isSCEVable(Ty) && 1812 "This is not a conversion to a SCEVable type!"); 1813 Ty = getEffectiveSCEVType(Ty); 1814 1815 // Sign-extend negative constants. 1816 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Op)) 1817 if (SC->getAPInt().isNegative()) 1818 return getSignExtendExpr(Op, Ty); 1819 1820 // Peel off a truncate cast. 1821 if (const SCEVTruncateExpr *T = dyn_cast<SCEVTruncateExpr>(Op)) { 1822 const SCEV *NewOp = T->getOperand(); 1823 if (getTypeSizeInBits(NewOp->getType()) < getTypeSizeInBits(Ty)) 1824 return getAnyExtendExpr(NewOp, Ty); 1825 return getTruncateOrNoop(NewOp, Ty); 1826 } 1827 1828 // Next try a zext cast. If the cast is folded, use it. 1829 const SCEV *ZExt = getZeroExtendExpr(Op, Ty); 1830 if (!isa<SCEVZeroExtendExpr>(ZExt)) 1831 return ZExt; 1832 1833 // Next try a sext cast. If the cast is folded, use it. 1834 const SCEV *SExt = getSignExtendExpr(Op, Ty); 1835 if (!isa<SCEVSignExtendExpr>(SExt)) 1836 return SExt; 1837 1838 // Force the cast to be folded into the operands of an addrec. 1839 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Op)) { 1840 SmallVector<const SCEV *, 4> Ops; 1841 for (const SCEV *Op : AR->operands()) 1842 Ops.push_back(getAnyExtendExpr(Op, Ty)); 1843 return getAddRecExpr(Ops, AR->getLoop(), SCEV::FlagNW); 1844 } 1845 1846 // If the expression is obviously signed, use the sext cast value. 1847 if (isa<SCEVSMaxExpr>(Op)) 1848 return SExt; 1849 1850 // Absent any other information, use the zext cast value. 1851 return ZExt; 1852 } 1853 1854 /// CollectAddOperandsWithScales - Process the given Ops list, which is 1855 /// a list of operands to be added under the given scale, update the given 1856 /// map. This is a helper function for getAddRecExpr. As an example of 1857 /// what it does, given a sequence of operands that would form an add 1858 /// expression like this: 1859 /// 1860 /// m + n + 13 + (A * (o + p + (B * (q + m + 29)))) + r + (-1 * r) 1861 /// 1862 /// where A and B are constants, update the map with these values: 1863 /// 1864 /// (m, 1+A*B), (n, 1), (o, A), (p, A), (q, A*B), (r, 0) 1865 /// 1866 /// and add 13 + A*B*29 to AccumulatedConstant. 1867 /// This will allow getAddRecExpr to produce this: 1868 /// 1869 /// 13+A*B*29 + n + (m * (1+A*B)) + ((o + p) * A) + (q * A*B) 1870 /// 1871 /// This form often exposes folding opportunities that are hidden in 1872 /// the original operand list. 1873 /// 1874 /// Return true iff it appears that any interesting folding opportunities 1875 /// may be exposed. This helps getAddRecExpr short-circuit extra work in 1876 /// the common case where no interesting opportunities are present, and 1877 /// is also used as a check to avoid infinite recursion. 1878 /// 1879 static bool 1880 CollectAddOperandsWithScales(DenseMap<const SCEV *, APInt> &M, 1881 SmallVectorImpl<const SCEV *> &NewOps, 1882 APInt &AccumulatedConstant, 1883 const SCEV *const *Ops, size_t NumOperands, 1884 const APInt &Scale, 1885 ScalarEvolution &SE) { 1886 bool Interesting = false; 1887 1888 // Iterate over the add operands. They are sorted, with constants first. 1889 unsigned i = 0; 1890 while (const SCEVConstant *C = dyn_cast<SCEVConstant>(Ops[i])) { 1891 ++i; 1892 // Pull a buried constant out to the outside. 1893 if (Scale != 1 || AccumulatedConstant != 0 || C->getValue()->isZero()) 1894 Interesting = true; 1895 AccumulatedConstant += Scale * C->getAPInt(); 1896 } 1897 1898 // Next comes everything else. We're especially interested in multiplies 1899 // here, but they're in the middle, so just visit the rest with one loop. 1900 for (; i != NumOperands; ++i) { 1901 const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(Ops[i]); 1902 if (Mul && isa<SCEVConstant>(Mul->getOperand(0))) { 1903 APInt NewScale = 1904 Scale * cast<SCEVConstant>(Mul->getOperand(0))->getAPInt(); 1905 if (Mul->getNumOperands() == 2 && isa<SCEVAddExpr>(Mul->getOperand(1))) { 1906 // A multiplication of a constant with another add; recurse. 1907 const SCEVAddExpr *Add = cast<SCEVAddExpr>(Mul->getOperand(1)); 1908 Interesting |= 1909 CollectAddOperandsWithScales(M, NewOps, AccumulatedConstant, 1910 Add->op_begin(), Add->getNumOperands(), 1911 NewScale, SE); 1912 } else { 1913 // A multiplication of a constant with some other value. Update 1914 // the map. 1915 SmallVector<const SCEV *, 4> MulOps(Mul->op_begin()+1, Mul->op_end()); 1916 const SCEV *Key = SE.getMulExpr(MulOps); 1917 auto Pair = M.insert({Key, NewScale}); 1918 if (Pair.second) { 1919 NewOps.push_back(Pair.first->first); 1920 } else { 1921 Pair.first->second += NewScale; 1922 // The map already had an entry for this value, which may indicate 1923 // a folding opportunity. 1924 Interesting = true; 1925 } 1926 } 1927 } else { 1928 // An ordinary operand. Update the map. 1929 std::pair<DenseMap<const SCEV *, APInt>::iterator, bool> Pair = 1930 M.insert({Ops[i], Scale}); 1931 if (Pair.second) { 1932 NewOps.push_back(Pair.first->first); 1933 } else { 1934 Pair.first->second += Scale; 1935 // The map already had an entry for this value, which may indicate 1936 // a folding opportunity. 1937 Interesting = true; 1938 } 1939 } 1940 } 1941 1942 return Interesting; 1943 } 1944 1945 // We're trying to construct a SCEV of type `Type' with `Ops' as operands and 1946 // `OldFlags' as can't-wrap behavior. Infer a more aggressive set of 1947 // can't-overflow flags for the operation if possible. 1948 static SCEV::NoWrapFlags 1949 StrengthenNoWrapFlags(ScalarEvolution *SE, SCEVTypes Type, 1950 const SmallVectorImpl<const SCEV *> &Ops, 1951 SCEV::NoWrapFlags Flags) { 1952 using namespace std::placeholders; 1953 typedef OverflowingBinaryOperator OBO; 1954 1955 bool CanAnalyze = 1956 Type == scAddExpr || Type == scAddRecExpr || Type == scMulExpr; 1957 (void)CanAnalyze; 1958 assert(CanAnalyze && "don't call from other places!"); 1959 1960 int SignOrUnsignMask = SCEV::FlagNUW | SCEV::FlagNSW; 1961 SCEV::NoWrapFlags SignOrUnsignWrap = 1962 ScalarEvolution::maskFlags(Flags, SignOrUnsignMask); 1963 1964 // If FlagNSW is true and all the operands are non-negative, infer FlagNUW. 1965 auto IsKnownNonNegative = [&](const SCEV *S) { 1966 return SE->isKnownNonNegative(S); 1967 }; 1968 1969 if (SignOrUnsignWrap == SCEV::FlagNSW && all_of(Ops, IsKnownNonNegative)) 1970 Flags = 1971 ScalarEvolution::setFlags(Flags, (SCEV::NoWrapFlags)SignOrUnsignMask); 1972 1973 SignOrUnsignWrap = ScalarEvolution::maskFlags(Flags, SignOrUnsignMask); 1974 1975 if (SignOrUnsignWrap != SignOrUnsignMask && Type == scAddExpr && 1976 Ops.size() == 2 && isa<SCEVConstant>(Ops[0])) { 1977 1978 // (A + C) --> (A + C)<nsw> if the addition does not sign overflow 1979 // (A + C) --> (A + C)<nuw> if the addition does not unsign overflow 1980 1981 const APInt &C = cast<SCEVConstant>(Ops[0])->getAPInt(); 1982 if (!(SignOrUnsignWrap & SCEV::FlagNSW)) { 1983 auto NSWRegion = ConstantRange::makeGuaranteedNoWrapRegion( 1984 Instruction::Add, C, OBO::NoSignedWrap); 1985 if (NSWRegion.contains(SE->getSignedRange(Ops[1]))) 1986 Flags = ScalarEvolution::setFlags(Flags, SCEV::FlagNSW); 1987 } 1988 if (!(SignOrUnsignWrap & SCEV::FlagNUW)) { 1989 auto NUWRegion = ConstantRange::makeGuaranteedNoWrapRegion( 1990 Instruction::Add, C, OBO::NoUnsignedWrap); 1991 if (NUWRegion.contains(SE->getUnsignedRange(Ops[1]))) 1992 Flags = ScalarEvolution::setFlags(Flags, SCEV::FlagNUW); 1993 } 1994 } 1995 1996 return Flags; 1997 } 1998 1999 /// getAddExpr - Get a canonical add expression, or something simpler if 2000 /// possible. 2001 const SCEV *ScalarEvolution::getAddExpr(SmallVectorImpl<const SCEV *> &Ops, 2002 SCEV::NoWrapFlags Flags) { 2003 assert(!(Flags & ~(SCEV::FlagNUW | SCEV::FlagNSW)) && 2004 "only nuw or nsw allowed"); 2005 assert(!Ops.empty() && "Cannot get empty add!"); 2006 if (Ops.size() == 1) return Ops[0]; 2007 #ifndef NDEBUG 2008 Type *ETy = getEffectiveSCEVType(Ops[0]->getType()); 2009 for (unsigned i = 1, e = Ops.size(); i != e; ++i) 2010 assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy && 2011 "SCEVAddExpr operand types don't match!"); 2012 #endif 2013 2014 // Sort by complexity, this groups all similar expression types together. 2015 GroupByComplexity(Ops, &LI); 2016 2017 Flags = StrengthenNoWrapFlags(this, scAddExpr, Ops, Flags); 2018 2019 // If there are any constants, fold them together. 2020 unsigned Idx = 0; 2021 if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(Ops[0])) { 2022 ++Idx; 2023 assert(Idx < Ops.size()); 2024 while (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Ops[Idx])) { 2025 // We found two constants, fold them together! 2026 Ops[0] = getConstant(LHSC->getAPInt() + RHSC->getAPInt()); 2027 if (Ops.size() == 2) return Ops[0]; 2028 Ops.erase(Ops.begin()+1); // Erase the folded element 2029 LHSC = cast<SCEVConstant>(Ops[0]); 2030 } 2031 2032 // If we are left with a constant zero being added, strip it off. 2033 if (LHSC->getValue()->isZero()) { 2034 Ops.erase(Ops.begin()); 2035 --Idx; 2036 } 2037 2038 if (Ops.size() == 1) return Ops[0]; 2039 } 2040 2041 // Okay, check to see if the same value occurs in the operand list more than 2042 // once. If so, merge them together into an multiply expression. Since we 2043 // sorted the list, these values are required to be adjacent. 2044 Type *Ty = Ops[0]->getType(); 2045 bool FoundMatch = false; 2046 for (unsigned i = 0, e = Ops.size(); i != e-1; ++i) 2047 if (Ops[i] == Ops[i+1]) { // X + Y + Y --> X + Y*2 2048 // Scan ahead to count how many equal operands there are. 2049 unsigned Count = 2; 2050 while (i+Count != e && Ops[i+Count] == Ops[i]) 2051 ++Count; 2052 // Merge the values into a multiply. 2053 const SCEV *Scale = getConstant(Ty, Count); 2054 const SCEV *Mul = getMulExpr(Scale, Ops[i]); 2055 if (Ops.size() == Count) 2056 return Mul; 2057 Ops[i] = Mul; 2058 Ops.erase(Ops.begin()+i+1, Ops.begin()+i+Count); 2059 --i; e -= Count - 1; 2060 FoundMatch = true; 2061 } 2062 if (FoundMatch) 2063 return getAddExpr(Ops, Flags); 2064 2065 // Check for truncates. If all the operands are truncated from the same 2066 // type, see if factoring out the truncate would permit the result to be 2067 // folded. eg., trunc(x) + m*trunc(n) --> trunc(x + trunc(m)*n) 2068 // if the contents of the resulting outer trunc fold to something simple. 2069 for (; Idx < Ops.size() && isa<SCEVTruncateExpr>(Ops[Idx]); ++Idx) { 2070 const SCEVTruncateExpr *Trunc = cast<SCEVTruncateExpr>(Ops[Idx]); 2071 Type *DstType = Trunc->getType(); 2072 Type *SrcType = Trunc->getOperand()->getType(); 2073 SmallVector<const SCEV *, 8> LargeOps; 2074 bool Ok = true; 2075 // Check all the operands to see if they can be represented in the 2076 // source type of the truncate. 2077 for (unsigned i = 0, e = Ops.size(); i != e; ++i) { 2078 if (const SCEVTruncateExpr *T = dyn_cast<SCEVTruncateExpr>(Ops[i])) { 2079 if (T->getOperand()->getType() != SrcType) { 2080 Ok = false; 2081 break; 2082 } 2083 LargeOps.push_back(T->getOperand()); 2084 } else if (const SCEVConstant *C = dyn_cast<SCEVConstant>(Ops[i])) { 2085 LargeOps.push_back(getAnyExtendExpr(C, SrcType)); 2086 } else if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(Ops[i])) { 2087 SmallVector<const SCEV *, 8> LargeMulOps; 2088 for (unsigned j = 0, f = M->getNumOperands(); j != f && Ok; ++j) { 2089 if (const SCEVTruncateExpr *T = 2090 dyn_cast<SCEVTruncateExpr>(M->getOperand(j))) { 2091 if (T->getOperand()->getType() != SrcType) { 2092 Ok = false; 2093 break; 2094 } 2095 LargeMulOps.push_back(T->getOperand()); 2096 } else if (const auto *C = dyn_cast<SCEVConstant>(M->getOperand(j))) { 2097 LargeMulOps.push_back(getAnyExtendExpr(C, SrcType)); 2098 } else { 2099 Ok = false; 2100 break; 2101 } 2102 } 2103 if (Ok) 2104 LargeOps.push_back(getMulExpr(LargeMulOps)); 2105 } else { 2106 Ok = false; 2107 break; 2108 } 2109 } 2110 if (Ok) { 2111 // Evaluate the expression in the larger type. 2112 const SCEV *Fold = getAddExpr(LargeOps, Flags); 2113 // If it folds to something simple, use it. Otherwise, don't. 2114 if (isa<SCEVConstant>(Fold) || isa<SCEVUnknown>(Fold)) 2115 return getTruncateExpr(Fold, DstType); 2116 } 2117 } 2118 2119 // Skip past any other cast SCEVs. 2120 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scAddExpr) 2121 ++Idx; 2122 2123 // If there are add operands they would be next. 2124 if (Idx < Ops.size()) { 2125 bool DeletedAdd = false; 2126 while (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Ops[Idx])) { 2127 // If we have an add, expand the add operands onto the end of the operands 2128 // list. 2129 Ops.erase(Ops.begin()+Idx); 2130 Ops.append(Add->op_begin(), Add->op_end()); 2131 DeletedAdd = true; 2132 } 2133 2134 // If we deleted at least one add, we added operands to the end of the list, 2135 // and they are not necessarily sorted. Recurse to resort and resimplify 2136 // any operands we just acquired. 2137 if (DeletedAdd) 2138 return getAddExpr(Ops); 2139 } 2140 2141 // Skip over the add expression until we get to a multiply. 2142 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scMulExpr) 2143 ++Idx; 2144 2145 // Check to see if there are any folding opportunities present with 2146 // operands multiplied by constant values. 2147 if (Idx < Ops.size() && isa<SCEVMulExpr>(Ops[Idx])) { 2148 uint64_t BitWidth = getTypeSizeInBits(Ty); 2149 DenseMap<const SCEV *, APInt> M; 2150 SmallVector<const SCEV *, 8> NewOps; 2151 APInt AccumulatedConstant(BitWidth, 0); 2152 if (CollectAddOperandsWithScales(M, NewOps, AccumulatedConstant, 2153 Ops.data(), Ops.size(), 2154 APInt(BitWidth, 1), *this)) { 2155 struct APIntCompare { 2156 bool operator()(const APInt &LHS, const APInt &RHS) const { 2157 return LHS.ult(RHS); 2158 } 2159 }; 2160 2161 // Some interesting folding opportunity is present, so its worthwhile to 2162 // re-generate the operands list. Group the operands by constant scale, 2163 // to avoid multiplying by the same constant scale multiple times. 2164 std::map<APInt, SmallVector<const SCEV *, 4>, APIntCompare> MulOpLists; 2165 for (const SCEV *NewOp : NewOps) 2166 MulOpLists[M.find(NewOp)->second].push_back(NewOp); 2167 // Re-generate the operands list. 2168 Ops.clear(); 2169 if (AccumulatedConstant != 0) 2170 Ops.push_back(getConstant(AccumulatedConstant)); 2171 for (auto &MulOp : MulOpLists) 2172 if (MulOp.first != 0) 2173 Ops.push_back(getMulExpr(getConstant(MulOp.first), 2174 getAddExpr(MulOp.second))); 2175 if (Ops.empty()) 2176 return getZero(Ty); 2177 if (Ops.size() == 1) 2178 return Ops[0]; 2179 return getAddExpr(Ops); 2180 } 2181 } 2182 2183 // If we are adding something to a multiply expression, make sure the 2184 // something is not already an operand of the multiply. If so, merge it into 2185 // the multiply. 2186 for (; Idx < Ops.size() && isa<SCEVMulExpr>(Ops[Idx]); ++Idx) { 2187 const SCEVMulExpr *Mul = cast<SCEVMulExpr>(Ops[Idx]); 2188 for (unsigned MulOp = 0, e = Mul->getNumOperands(); MulOp != e; ++MulOp) { 2189 const SCEV *MulOpSCEV = Mul->getOperand(MulOp); 2190 if (isa<SCEVConstant>(MulOpSCEV)) 2191 continue; 2192 for (unsigned AddOp = 0, e = Ops.size(); AddOp != e; ++AddOp) 2193 if (MulOpSCEV == Ops[AddOp]) { 2194 // Fold W + X + (X * Y * Z) --> W + (X * ((Y*Z)+1)) 2195 const SCEV *InnerMul = Mul->getOperand(MulOp == 0); 2196 if (Mul->getNumOperands() != 2) { 2197 // If the multiply has more than two operands, we must get the 2198 // Y*Z term. 2199 SmallVector<const SCEV *, 4> MulOps(Mul->op_begin(), 2200 Mul->op_begin()+MulOp); 2201 MulOps.append(Mul->op_begin()+MulOp+1, Mul->op_end()); 2202 InnerMul = getMulExpr(MulOps); 2203 } 2204 const SCEV *One = getOne(Ty); 2205 const SCEV *AddOne = getAddExpr(One, InnerMul); 2206 const SCEV *OuterMul = getMulExpr(AddOne, MulOpSCEV); 2207 if (Ops.size() == 2) return OuterMul; 2208 if (AddOp < Idx) { 2209 Ops.erase(Ops.begin()+AddOp); 2210 Ops.erase(Ops.begin()+Idx-1); 2211 } else { 2212 Ops.erase(Ops.begin()+Idx); 2213 Ops.erase(Ops.begin()+AddOp-1); 2214 } 2215 Ops.push_back(OuterMul); 2216 return getAddExpr(Ops); 2217 } 2218 2219 // Check this multiply against other multiplies being added together. 2220 for (unsigned OtherMulIdx = Idx+1; 2221 OtherMulIdx < Ops.size() && isa<SCEVMulExpr>(Ops[OtherMulIdx]); 2222 ++OtherMulIdx) { 2223 const SCEVMulExpr *OtherMul = cast<SCEVMulExpr>(Ops[OtherMulIdx]); 2224 // If MulOp occurs in OtherMul, we can fold the two multiplies 2225 // together. 2226 for (unsigned OMulOp = 0, e = OtherMul->getNumOperands(); 2227 OMulOp != e; ++OMulOp) 2228 if (OtherMul->getOperand(OMulOp) == MulOpSCEV) { 2229 // Fold X + (A*B*C) + (A*D*E) --> X + (A*(B*C+D*E)) 2230 const SCEV *InnerMul1 = Mul->getOperand(MulOp == 0); 2231 if (Mul->getNumOperands() != 2) { 2232 SmallVector<const SCEV *, 4> MulOps(Mul->op_begin(), 2233 Mul->op_begin()+MulOp); 2234 MulOps.append(Mul->op_begin()+MulOp+1, Mul->op_end()); 2235 InnerMul1 = getMulExpr(MulOps); 2236 } 2237 const SCEV *InnerMul2 = OtherMul->getOperand(OMulOp == 0); 2238 if (OtherMul->getNumOperands() != 2) { 2239 SmallVector<const SCEV *, 4> MulOps(OtherMul->op_begin(), 2240 OtherMul->op_begin()+OMulOp); 2241 MulOps.append(OtherMul->op_begin()+OMulOp+1, OtherMul->op_end()); 2242 InnerMul2 = getMulExpr(MulOps); 2243 } 2244 const SCEV *InnerMulSum = getAddExpr(InnerMul1,InnerMul2); 2245 const SCEV *OuterMul = getMulExpr(MulOpSCEV, InnerMulSum); 2246 if (Ops.size() == 2) return OuterMul; 2247 Ops.erase(Ops.begin()+Idx); 2248 Ops.erase(Ops.begin()+OtherMulIdx-1); 2249 Ops.push_back(OuterMul); 2250 return getAddExpr(Ops); 2251 } 2252 } 2253 } 2254 } 2255 2256 // If there are any add recurrences in the operands list, see if any other 2257 // added values are loop invariant. If so, we can fold them into the 2258 // recurrence. 2259 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scAddRecExpr) 2260 ++Idx; 2261 2262 // Scan over all recurrences, trying to fold loop invariants into them. 2263 for (; Idx < Ops.size() && isa<SCEVAddRecExpr>(Ops[Idx]); ++Idx) { 2264 // Scan all of the other operands to this add and add them to the vector if 2265 // they are loop invariant w.r.t. the recurrence. 2266 SmallVector<const SCEV *, 8> LIOps; 2267 const SCEVAddRecExpr *AddRec = cast<SCEVAddRecExpr>(Ops[Idx]); 2268 const Loop *AddRecLoop = AddRec->getLoop(); 2269 for (unsigned i = 0, e = Ops.size(); i != e; ++i) 2270 if (isLoopInvariant(Ops[i], AddRecLoop)) { 2271 LIOps.push_back(Ops[i]); 2272 Ops.erase(Ops.begin()+i); 2273 --i; --e; 2274 } 2275 2276 // If we found some loop invariants, fold them into the recurrence. 2277 if (!LIOps.empty()) { 2278 // NLI + LI + {Start,+,Step} --> NLI + {LI+Start,+,Step} 2279 LIOps.push_back(AddRec->getStart()); 2280 2281 SmallVector<const SCEV *, 4> AddRecOps(AddRec->op_begin(), 2282 AddRec->op_end()); 2283 AddRecOps[0] = getAddExpr(LIOps); 2284 2285 // Build the new addrec. Propagate the NUW and NSW flags if both the 2286 // outer add and the inner addrec are guaranteed to have no overflow. 2287 // Always propagate NW. 2288 Flags = AddRec->getNoWrapFlags(setFlags(Flags, SCEV::FlagNW)); 2289 const SCEV *NewRec = getAddRecExpr(AddRecOps, AddRecLoop, Flags); 2290 2291 // If all of the other operands were loop invariant, we are done. 2292 if (Ops.size() == 1) return NewRec; 2293 2294 // Otherwise, add the folded AddRec by the non-invariant parts. 2295 for (unsigned i = 0;; ++i) 2296 if (Ops[i] == AddRec) { 2297 Ops[i] = NewRec; 2298 break; 2299 } 2300 return getAddExpr(Ops); 2301 } 2302 2303 // Okay, if there weren't any loop invariants to be folded, check to see if 2304 // there are multiple AddRec's with the same loop induction variable being 2305 // added together. If so, we can fold them. 2306 for (unsigned OtherIdx = Idx+1; 2307 OtherIdx < Ops.size() && isa<SCEVAddRecExpr>(Ops[OtherIdx]); 2308 ++OtherIdx) 2309 if (AddRecLoop == cast<SCEVAddRecExpr>(Ops[OtherIdx])->getLoop()) { 2310 // Other + {A,+,B}<L> + {C,+,D}<L> --> Other + {A+C,+,B+D}<L> 2311 SmallVector<const SCEV *, 4> AddRecOps(AddRec->op_begin(), 2312 AddRec->op_end()); 2313 for (; OtherIdx != Ops.size() && isa<SCEVAddRecExpr>(Ops[OtherIdx]); 2314 ++OtherIdx) 2315 if (const auto *OtherAddRec = dyn_cast<SCEVAddRecExpr>(Ops[OtherIdx])) 2316 if (OtherAddRec->getLoop() == AddRecLoop) { 2317 for (unsigned i = 0, e = OtherAddRec->getNumOperands(); 2318 i != e; ++i) { 2319 if (i >= AddRecOps.size()) { 2320 AddRecOps.append(OtherAddRec->op_begin()+i, 2321 OtherAddRec->op_end()); 2322 break; 2323 } 2324 AddRecOps[i] = getAddExpr(AddRecOps[i], 2325 OtherAddRec->getOperand(i)); 2326 } 2327 Ops.erase(Ops.begin() + OtherIdx); --OtherIdx; 2328 } 2329 // Step size has changed, so we cannot guarantee no self-wraparound. 2330 Ops[Idx] = getAddRecExpr(AddRecOps, AddRecLoop, SCEV::FlagAnyWrap); 2331 return getAddExpr(Ops); 2332 } 2333 2334 // Otherwise couldn't fold anything into this recurrence. Move onto the 2335 // next one. 2336 } 2337 2338 // Okay, it looks like we really DO need an add expr. Check to see if we 2339 // already have one, otherwise create a new one. 2340 FoldingSetNodeID ID; 2341 ID.AddInteger(scAddExpr); 2342 for (unsigned i = 0, e = Ops.size(); i != e; ++i) 2343 ID.AddPointer(Ops[i]); 2344 void *IP = nullptr; 2345 SCEVAddExpr *S = 2346 static_cast<SCEVAddExpr *>(UniqueSCEVs.FindNodeOrInsertPos(ID, IP)); 2347 if (!S) { 2348 const SCEV **O = SCEVAllocator.Allocate<const SCEV *>(Ops.size()); 2349 std::uninitialized_copy(Ops.begin(), Ops.end(), O); 2350 S = new (SCEVAllocator) SCEVAddExpr(ID.Intern(SCEVAllocator), 2351 O, Ops.size()); 2352 UniqueSCEVs.InsertNode(S, IP); 2353 } 2354 S->setNoWrapFlags(Flags); 2355 return S; 2356 } 2357 2358 static uint64_t umul_ov(uint64_t i, uint64_t j, bool &Overflow) { 2359 uint64_t k = i*j; 2360 if (j > 1 && k / j != i) Overflow = true; 2361 return k; 2362 } 2363 2364 /// Compute the result of "n choose k", the binomial coefficient. If an 2365 /// intermediate computation overflows, Overflow will be set and the return will 2366 /// be garbage. Overflow is not cleared on absence of overflow. 2367 static uint64_t Choose(uint64_t n, uint64_t k, bool &Overflow) { 2368 // We use the multiplicative formula: 2369 // n(n-1)(n-2)...(n-(k-1)) / k(k-1)(k-2)...1 . 2370 // At each iteration, we take the n-th term of the numeral and divide by the 2371 // (k-n)th term of the denominator. This division will always produce an 2372 // integral result, and helps reduce the chance of overflow in the 2373 // intermediate computations. However, we can still overflow even when the 2374 // final result would fit. 2375 2376 if (n == 0 || n == k) return 1; 2377 if (k > n) return 0; 2378 2379 if (k > n/2) 2380 k = n-k; 2381 2382 uint64_t r = 1; 2383 for (uint64_t i = 1; i <= k; ++i) { 2384 r = umul_ov(r, n-(i-1), Overflow); 2385 r /= i; 2386 } 2387 return r; 2388 } 2389 2390 /// Determine if any of the operands in this SCEV are a constant or if 2391 /// any of the add or multiply expressions in this SCEV contain a constant. 2392 static bool containsConstantSomewhere(const SCEV *StartExpr) { 2393 SmallVector<const SCEV *, 4> Ops; 2394 Ops.push_back(StartExpr); 2395 while (!Ops.empty()) { 2396 const SCEV *CurrentExpr = Ops.pop_back_val(); 2397 if (isa<SCEVConstant>(*CurrentExpr)) 2398 return true; 2399 2400 if (isa<SCEVAddExpr>(*CurrentExpr) || isa<SCEVMulExpr>(*CurrentExpr)) { 2401 const auto *CurrentNAry = cast<SCEVNAryExpr>(CurrentExpr); 2402 Ops.append(CurrentNAry->op_begin(), CurrentNAry->op_end()); 2403 } 2404 } 2405 return false; 2406 } 2407 2408 /// getMulExpr - Get a canonical multiply expression, or something simpler if 2409 /// possible. 2410 const SCEV *ScalarEvolution::getMulExpr(SmallVectorImpl<const SCEV *> &Ops, 2411 SCEV::NoWrapFlags Flags) { 2412 assert(Flags == maskFlags(Flags, SCEV::FlagNUW | SCEV::FlagNSW) && 2413 "only nuw or nsw allowed"); 2414 assert(!Ops.empty() && "Cannot get empty mul!"); 2415 if (Ops.size() == 1) return Ops[0]; 2416 #ifndef NDEBUG 2417 Type *ETy = getEffectiveSCEVType(Ops[0]->getType()); 2418 for (unsigned i = 1, e = Ops.size(); i != e; ++i) 2419 assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy && 2420 "SCEVMulExpr operand types don't match!"); 2421 #endif 2422 2423 // Sort by complexity, this groups all similar expression types together. 2424 GroupByComplexity(Ops, &LI); 2425 2426 Flags = StrengthenNoWrapFlags(this, scMulExpr, Ops, Flags); 2427 2428 // If there are any constants, fold them together. 2429 unsigned Idx = 0; 2430 if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(Ops[0])) { 2431 2432 // C1*(C2+V) -> C1*C2 + C1*V 2433 if (Ops.size() == 2) 2434 if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Ops[1])) 2435 // If any of Add's ops are Adds or Muls with a constant, 2436 // apply this transformation as well. 2437 if (Add->getNumOperands() == 2) 2438 if (containsConstantSomewhere(Add)) 2439 return getAddExpr(getMulExpr(LHSC, Add->getOperand(0)), 2440 getMulExpr(LHSC, Add->getOperand(1))); 2441 2442 ++Idx; 2443 while (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Ops[Idx])) { 2444 // We found two constants, fold them together! 2445 ConstantInt *Fold = 2446 ConstantInt::get(getContext(), LHSC->getAPInt() * RHSC->getAPInt()); 2447 Ops[0] = getConstant(Fold); 2448 Ops.erase(Ops.begin()+1); // Erase the folded element 2449 if (Ops.size() == 1) return Ops[0]; 2450 LHSC = cast<SCEVConstant>(Ops[0]); 2451 } 2452 2453 // If we are left with a constant one being multiplied, strip it off. 2454 if (cast<SCEVConstant>(Ops[0])->getValue()->equalsInt(1)) { 2455 Ops.erase(Ops.begin()); 2456 --Idx; 2457 } else if (cast<SCEVConstant>(Ops[0])->getValue()->isZero()) { 2458 // If we have a multiply of zero, it will always be zero. 2459 return Ops[0]; 2460 } else if (Ops[0]->isAllOnesValue()) { 2461 // If we have a mul by -1 of an add, try distributing the -1 among the 2462 // add operands. 2463 if (Ops.size() == 2) { 2464 if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Ops[1])) { 2465 SmallVector<const SCEV *, 4> NewOps; 2466 bool AnyFolded = false; 2467 for (const SCEV *AddOp : Add->operands()) { 2468 const SCEV *Mul = getMulExpr(Ops[0], AddOp); 2469 if (!isa<SCEVMulExpr>(Mul)) AnyFolded = true; 2470 NewOps.push_back(Mul); 2471 } 2472 if (AnyFolded) 2473 return getAddExpr(NewOps); 2474 } else if (const auto *AddRec = dyn_cast<SCEVAddRecExpr>(Ops[1])) { 2475 // Negation preserves a recurrence's no self-wrap property. 2476 SmallVector<const SCEV *, 4> Operands; 2477 for (const SCEV *AddRecOp : AddRec->operands()) 2478 Operands.push_back(getMulExpr(Ops[0], AddRecOp)); 2479 2480 return getAddRecExpr(Operands, AddRec->getLoop(), 2481 AddRec->getNoWrapFlags(SCEV::FlagNW)); 2482 } 2483 } 2484 } 2485 2486 if (Ops.size() == 1) 2487 return Ops[0]; 2488 } 2489 2490 // Skip over the add expression until we get to a multiply. 2491 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scMulExpr) 2492 ++Idx; 2493 2494 // If there are mul operands inline them all into this expression. 2495 if (Idx < Ops.size()) { 2496 bool DeletedMul = false; 2497 while (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(Ops[Idx])) { 2498 // If we have an mul, expand the mul operands onto the end of the operands 2499 // list. 2500 Ops.erase(Ops.begin()+Idx); 2501 Ops.append(Mul->op_begin(), Mul->op_end()); 2502 DeletedMul = true; 2503 } 2504 2505 // If we deleted at least one mul, we added operands to the end of the list, 2506 // and they are not necessarily sorted. Recurse to resort and resimplify 2507 // any operands we just acquired. 2508 if (DeletedMul) 2509 return getMulExpr(Ops); 2510 } 2511 2512 // If there are any add recurrences in the operands list, see if any other 2513 // added values are loop invariant. If so, we can fold them into the 2514 // recurrence. 2515 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scAddRecExpr) 2516 ++Idx; 2517 2518 // Scan over all recurrences, trying to fold loop invariants into them. 2519 for (; Idx < Ops.size() && isa<SCEVAddRecExpr>(Ops[Idx]); ++Idx) { 2520 // Scan all of the other operands to this mul and add them to the vector if 2521 // they are loop invariant w.r.t. the recurrence. 2522 SmallVector<const SCEV *, 8> LIOps; 2523 const SCEVAddRecExpr *AddRec = cast<SCEVAddRecExpr>(Ops[Idx]); 2524 const Loop *AddRecLoop = AddRec->getLoop(); 2525 for (unsigned i = 0, e = Ops.size(); i != e; ++i) 2526 if (isLoopInvariant(Ops[i], AddRecLoop)) { 2527 LIOps.push_back(Ops[i]); 2528 Ops.erase(Ops.begin()+i); 2529 --i; --e; 2530 } 2531 2532 // If we found some loop invariants, fold them into the recurrence. 2533 if (!LIOps.empty()) { 2534 // NLI * LI * {Start,+,Step} --> NLI * {LI*Start,+,LI*Step} 2535 SmallVector<const SCEV *, 4> NewOps; 2536 NewOps.reserve(AddRec->getNumOperands()); 2537 const SCEV *Scale = getMulExpr(LIOps); 2538 for (unsigned i = 0, e = AddRec->getNumOperands(); i != e; ++i) 2539 NewOps.push_back(getMulExpr(Scale, AddRec->getOperand(i))); 2540 2541 // Build the new addrec. Propagate the NUW and NSW flags if both the 2542 // outer mul and the inner addrec are guaranteed to have no overflow. 2543 // 2544 // No self-wrap cannot be guaranteed after changing the step size, but 2545 // will be inferred if either NUW or NSW is true. 2546 Flags = AddRec->getNoWrapFlags(clearFlags(Flags, SCEV::FlagNW)); 2547 const SCEV *NewRec = getAddRecExpr(NewOps, AddRecLoop, Flags); 2548 2549 // If all of the other operands were loop invariant, we are done. 2550 if (Ops.size() == 1) return NewRec; 2551 2552 // Otherwise, multiply the folded AddRec by the non-invariant parts. 2553 for (unsigned i = 0;; ++i) 2554 if (Ops[i] == AddRec) { 2555 Ops[i] = NewRec; 2556 break; 2557 } 2558 return getMulExpr(Ops); 2559 } 2560 2561 // Okay, if there weren't any loop invariants to be folded, check to see if 2562 // there are multiple AddRec's with the same loop induction variable being 2563 // multiplied together. If so, we can fold them. 2564 2565 // {A1,+,A2,+,...,+,An}<L> * {B1,+,B2,+,...,+,Bn}<L> 2566 // = {x=1 in [ sum y=x..2x [ sum z=max(y-x, y-n)..min(x,n) [ 2567 // choose(x, 2x)*choose(2x-y, x-z)*A_{y-z}*B_z 2568 // ]]],+,...up to x=2n}. 2569 // Note that the arguments to choose() are always integers with values 2570 // known at compile time, never SCEV objects. 2571 // 2572 // The implementation avoids pointless extra computations when the two 2573 // addrec's are of different length (mathematically, it's equivalent to 2574 // an infinite stream of zeros on the right). 2575 bool OpsModified = false; 2576 for (unsigned OtherIdx = Idx+1; 2577 OtherIdx != Ops.size() && isa<SCEVAddRecExpr>(Ops[OtherIdx]); 2578 ++OtherIdx) { 2579 const SCEVAddRecExpr *OtherAddRec = 2580 dyn_cast<SCEVAddRecExpr>(Ops[OtherIdx]); 2581 if (!OtherAddRec || OtherAddRec->getLoop() != AddRecLoop) 2582 continue; 2583 2584 bool Overflow = false; 2585 Type *Ty = AddRec->getType(); 2586 bool LargerThan64Bits = getTypeSizeInBits(Ty) > 64; 2587 SmallVector<const SCEV*, 7> AddRecOps; 2588 for (int x = 0, xe = AddRec->getNumOperands() + 2589 OtherAddRec->getNumOperands() - 1; x != xe && !Overflow; ++x) { 2590 const SCEV *Term = getZero(Ty); 2591 for (int y = x, ye = 2*x+1; y != ye && !Overflow; ++y) { 2592 uint64_t Coeff1 = Choose(x, 2*x - y, Overflow); 2593 for (int z = std::max(y-x, y-(int)AddRec->getNumOperands()+1), 2594 ze = std::min(x+1, (int)OtherAddRec->getNumOperands()); 2595 z < ze && !Overflow; ++z) { 2596 uint64_t Coeff2 = Choose(2*x - y, x-z, Overflow); 2597 uint64_t Coeff; 2598 if (LargerThan64Bits) 2599 Coeff = umul_ov(Coeff1, Coeff2, Overflow); 2600 else 2601 Coeff = Coeff1*Coeff2; 2602 const SCEV *CoeffTerm = getConstant(Ty, Coeff); 2603 const SCEV *Term1 = AddRec->getOperand(y-z); 2604 const SCEV *Term2 = OtherAddRec->getOperand(z); 2605 Term = getAddExpr(Term, getMulExpr(CoeffTerm, Term1,Term2)); 2606 } 2607 } 2608 AddRecOps.push_back(Term); 2609 } 2610 if (!Overflow) { 2611 const SCEV *NewAddRec = getAddRecExpr(AddRecOps, AddRec->getLoop(), 2612 SCEV::FlagAnyWrap); 2613 if (Ops.size() == 2) return NewAddRec; 2614 Ops[Idx] = NewAddRec; 2615 Ops.erase(Ops.begin() + OtherIdx); --OtherIdx; 2616 OpsModified = true; 2617 AddRec = dyn_cast<SCEVAddRecExpr>(NewAddRec); 2618 if (!AddRec) 2619 break; 2620 } 2621 } 2622 if (OpsModified) 2623 return getMulExpr(Ops); 2624 2625 // Otherwise couldn't fold anything into this recurrence. Move onto the 2626 // next one. 2627 } 2628 2629 // Okay, it looks like we really DO need an mul expr. Check to see if we 2630 // already have one, otherwise create a new one. 2631 FoldingSetNodeID ID; 2632 ID.AddInteger(scMulExpr); 2633 for (unsigned i = 0, e = Ops.size(); i != e; ++i) 2634 ID.AddPointer(Ops[i]); 2635 void *IP = nullptr; 2636 SCEVMulExpr *S = 2637 static_cast<SCEVMulExpr *>(UniqueSCEVs.FindNodeOrInsertPos(ID, IP)); 2638 if (!S) { 2639 const SCEV **O = SCEVAllocator.Allocate<const SCEV *>(Ops.size()); 2640 std::uninitialized_copy(Ops.begin(), Ops.end(), O); 2641 S = new (SCEVAllocator) SCEVMulExpr(ID.Intern(SCEVAllocator), 2642 O, Ops.size()); 2643 UniqueSCEVs.InsertNode(S, IP); 2644 } 2645 S->setNoWrapFlags(Flags); 2646 return S; 2647 } 2648 2649 /// getUDivExpr - Get a canonical unsigned division expression, or something 2650 /// simpler if possible. 2651 const SCEV *ScalarEvolution::getUDivExpr(const SCEV *LHS, 2652 const SCEV *RHS) { 2653 assert(getEffectiveSCEVType(LHS->getType()) == 2654 getEffectiveSCEVType(RHS->getType()) && 2655 "SCEVUDivExpr operand types don't match!"); 2656 2657 if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(RHS)) { 2658 if (RHSC->getValue()->equalsInt(1)) 2659 return LHS; // X udiv 1 --> x 2660 // If the denominator is zero, the result of the udiv is undefined. Don't 2661 // try to analyze it, because the resolution chosen here may differ from 2662 // the resolution chosen in other parts of the compiler. 2663 if (!RHSC->getValue()->isZero()) { 2664 // Determine if the division can be folded into the operands of 2665 // its operands. 2666 // TODO: Generalize this to non-constants by using known-bits information. 2667 Type *Ty = LHS->getType(); 2668 unsigned LZ = RHSC->getAPInt().countLeadingZeros(); 2669 unsigned MaxShiftAmt = getTypeSizeInBits(Ty) - LZ - 1; 2670 // For non-power-of-two values, effectively round the value up to the 2671 // nearest power of two. 2672 if (!RHSC->getAPInt().isPowerOf2()) 2673 ++MaxShiftAmt; 2674 IntegerType *ExtTy = 2675 IntegerType::get(getContext(), getTypeSizeInBits(Ty) + MaxShiftAmt); 2676 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(LHS)) 2677 if (const SCEVConstant *Step = 2678 dyn_cast<SCEVConstant>(AR->getStepRecurrence(*this))) { 2679 // {X,+,N}/C --> {X/C,+,N/C} if safe and N/C can be folded. 2680 const APInt &StepInt = Step->getAPInt(); 2681 const APInt &DivInt = RHSC->getAPInt(); 2682 if (!StepInt.urem(DivInt) && 2683 getZeroExtendExpr(AR, ExtTy) == 2684 getAddRecExpr(getZeroExtendExpr(AR->getStart(), ExtTy), 2685 getZeroExtendExpr(Step, ExtTy), 2686 AR->getLoop(), SCEV::FlagAnyWrap)) { 2687 SmallVector<const SCEV *, 4> Operands; 2688 for (const SCEV *Op : AR->operands()) 2689 Operands.push_back(getUDivExpr(Op, RHS)); 2690 return getAddRecExpr(Operands, AR->getLoop(), SCEV::FlagNW); 2691 } 2692 /// Get a canonical UDivExpr for a recurrence. 2693 /// {X,+,N}/C => {Y,+,N}/C where Y=X-(X%N). Safe when C%N=0. 2694 // We can currently only fold X%N if X is constant. 2695 const SCEVConstant *StartC = dyn_cast<SCEVConstant>(AR->getStart()); 2696 if (StartC && !DivInt.urem(StepInt) && 2697 getZeroExtendExpr(AR, ExtTy) == 2698 getAddRecExpr(getZeroExtendExpr(AR->getStart(), ExtTy), 2699 getZeroExtendExpr(Step, ExtTy), 2700 AR->getLoop(), SCEV::FlagAnyWrap)) { 2701 const APInt &StartInt = StartC->getAPInt(); 2702 const APInt &StartRem = StartInt.urem(StepInt); 2703 if (StartRem != 0) 2704 LHS = getAddRecExpr(getConstant(StartInt - StartRem), Step, 2705 AR->getLoop(), SCEV::FlagNW); 2706 } 2707 } 2708 // (A*B)/C --> A*(B/C) if safe and B/C can be folded. 2709 if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(LHS)) { 2710 SmallVector<const SCEV *, 4> Operands; 2711 for (const SCEV *Op : M->operands()) 2712 Operands.push_back(getZeroExtendExpr(Op, ExtTy)); 2713 if (getZeroExtendExpr(M, ExtTy) == getMulExpr(Operands)) 2714 // Find an operand that's safely divisible. 2715 for (unsigned i = 0, e = M->getNumOperands(); i != e; ++i) { 2716 const SCEV *Op = M->getOperand(i); 2717 const SCEV *Div = getUDivExpr(Op, RHSC); 2718 if (!isa<SCEVUDivExpr>(Div) && getMulExpr(Div, RHSC) == Op) { 2719 Operands = SmallVector<const SCEV *, 4>(M->op_begin(), 2720 M->op_end()); 2721 Operands[i] = Div; 2722 return getMulExpr(Operands); 2723 } 2724 } 2725 } 2726 // (A+B)/C --> (A/C + B/C) if safe and A/C and B/C can be folded. 2727 if (const SCEVAddExpr *A = dyn_cast<SCEVAddExpr>(LHS)) { 2728 SmallVector<const SCEV *, 4> Operands; 2729 for (const SCEV *Op : A->operands()) 2730 Operands.push_back(getZeroExtendExpr(Op, ExtTy)); 2731 if (getZeroExtendExpr(A, ExtTy) == getAddExpr(Operands)) { 2732 Operands.clear(); 2733 for (unsigned i = 0, e = A->getNumOperands(); i != e; ++i) { 2734 const SCEV *Op = getUDivExpr(A->getOperand(i), RHS); 2735 if (isa<SCEVUDivExpr>(Op) || 2736 getMulExpr(Op, RHS) != A->getOperand(i)) 2737 break; 2738 Operands.push_back(Op); 2739 } 2740 if (Operands.size() == A->getNumOperands()) 2741 return getAddExpr(Operands); 2742 } 2743 } 2744 2745 // Fold if both operands are constant. 2746 if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(LHS)) { 2747 Constant *LHSCV = LHSC->getValue(); 2748 Constant *RHSCV = RHSC->getValue(); 2749 return getConstant(cast<ConstantInt>(ConstantExpr::getUDiv(LHSCV, 2750 RHSCV))); 2751 } 2752 } 2753 } 2754 2755 FoldingSetNodeID ID; 2756 ID.AddInteger(scUDivExpr); 2757 ID.AddPointer(LHS); 2758 ID.AddPointer(RHS); 2759 void *IP = nullptr; 2760 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S; 2761 SCEV *S = new (SCEVAllocator) SCEVUDivExpr(ID.Intern(SCEVAllocator), 2762 LHS, RHS); 2763 UniqueSCEVs.InsertNode(S, IP); 2764 return S; 2765 } 2766 2767 static const APInt gcd(const SCEVConstant *C1, const SCEVConstant *C2) { 2768 APInt A = C1->getAPInt().abs(); 2769 APInt B = C2->getAPInt().abs(); 2770 uint32_t ABW = A.getBitWidth(); 2771 uint32_t BBW = B.getBitWidth(); 2772 2773 if (ABW > BBW) 2774 B = B.zext(ABW); 2775 else if (ABW < BBW) 2776 A = A.zext(BBW); 2777 2778 return APIntOps::GreatestCommonDivisor(A, B); 2779 } 2780 2781 /// getUDivExactExpr - Get a canonical unsigned division expression, or 2782 /// something simpler if possible. There is no representation for an exact udiv 2783 /// in SCEV IR, but we can attempt to remove factors from the LHS and RHS. 2784 /// We can't do this when it's not exact because the udiv may be clearing bits. 2785 const SCEV *ScalarEvolution::getUDivExactExpr(const SCEV *LHS, 2786 const SCEV *RHS) { 2787 // TODO: we could try to find factors in all sorts of things, but for now we 2788 // just deal with u/exact (multiply, constant). See SCEVDivision towards the 2789 // end of this file for inspiration. 2790 2791 const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(LHS); 2792 if (!Mul) 2793 return getUDivExpr(LHS, RHS); 2794 2795 if (const SCEVConstant *RHSCst = dyn_cast<SCEVConstant>(RHS)) { 2796 // If the mulexpr multiplies by a constant, then that constant must be the 2797 // first element of the mulexpr. 2798 if (const auto *LHSCst = dyn_cast<SCEVConstant>(Mul->getOperand(0))) { 2799 if (LHSCst == RHSCst) { 2800 SmallVector<const SCEV *, 2> Operands; 2801 Operands.append(Mul->op_begin() + 1, Mul->op_end()); 2802 return getMulExpr(Operands); 2803 } 2804 2805 // We can't just assume that LHSCst divides RHSCst cleanly, it could be 2806 // that there's a factor provided by one of the other terms. We need to 2807 // check. 2808 APInt Factor = gcd(LHSCst, RHSCst); 2809 if (!Factor.isIntN(1)) { 2810 LHSCst = 2811 cast<SCEVConstant>(getConstant(LHSCst->getAPInt().udiv(Factor))); 2812 RHSCst = 2813 cast<SCEVConstant>(getConstant(RHSCst->getAPInt().udiv(Factor))); 2814 SmallVector<const SCEV *, 2> Operands; 2815 Operands.push_back(LHSCst); 2816 Operands.append(Mul->op_begin() + 1, Mul->op_end()); 2817 LHS = getMulExpr(Operands); 2818 RHS = RHSCst; 2819 Mul = dyn_cast<SCEVMulExpr>(LHS); 2820 if (!Mul) 2821 return getUDivExactExpr(LHS, RHS); 2822 } 2823 } 2824 } 2825 2826 for (int i = 0, e = Mul->getNumOperands(); i != e; ++i) { 2827 if (Mul->getOperand(i) == RHS) { 2828 SmallVector<const SCEV *, 2> Operands; 2829 Operands.append(Mul->op_begin(), Mul->op_begin() + i); 2830 Operands.append(Mul->op_begin() + i + 1, Mul->op_end()); 2831 return getMulExpr(Operands); 2832 } 2833 } 2834 2835 return getUDivExpr(LHS, RHS); 2836 } 2837 2838 /// getAddRecExpr - Get an add recurrence expression for the specified loop. 2839 /// Simplify the expression as much as possible. 2840 const SCEV *ScalarEvolution::getAddRecExpr(const SCEV *Start, const SCEV *Step, 2841 const Loop *L, 2842 SCEV::NoWrapFlags Flags) { 2843 SmallVector<const SCEV *, 4> Operands; 2844 Operands.push_back(Start); 2845 if (const SCEVAddRecExpr *StepChrec = dyn_cast<SCEVAddRecExpr>(Step)) 2846 if (StepChrec->getLoop() == L) { 2847 Operands.append(StepChrec->op_begin(), StepChrec->op_end()); 2848 return getAddRecExpr(Operands, L, maskFlags(Flags, SCEV::FlagNW)); 2849 } 2850 2851 Operands.push_back(Step); 2852 return getAddRecExpr(Operands, L, Flags); 2853 } 2854 2855 /// getAddRecExpr - Get an add recurrence expression for the specified loop. 2856 /// Simplify the expression as much as possible. 2857 const SCEV * 2858 ScalarEvolution::getAddRecExpr(SmallVectorImpl<const SCEV *> &Operands, 2859 const Loop *L, SCEV::NoWrapFlags Flags) { 2860 if (Operands.size() == 1) return Operands[0]; 2861 #ifndef NDEBUG 2862 Type *ETy = getEffectiveSCEVType(Operands[0]->getType()); 2863 for (unsigned i = 1, e = Operands.size(); i != e; ++i) 2864 assert(getEffectiveSCEVType(Operands[i]->getType()) == ETy && 2865 "SCEVAddRecExpr operand types don't match!"); 2866 for (unsigned i = 0, e = Operands.size(); i != e; ++i) 2867 assert(isLoopInvariant(Operands[i], L) && 2868 "SCEVAddRecExpr operand is not loop-invariant!"); 2869 #endif 2870 2871 if (Operands.back()->isZero()) { 2872 Operands.pop_back(); 2873 return getAddRecExpr(Operands, L, SCEV::FlagAnyWrap); // {X,+,0} --> X 2874 } 2875 2876 // It's tempting to want to call getMaxBackedgeTakenCount count here and 2877 // use that information to infer NUW and NSW flags. However, computing a 2878 // BE count requires calling getAddRecExpr, so we may not yet have a 2879 // meaningful BE count at this point (and if we don't, we'd be stuck 2880 // with a SCEVCouldNotCompute as the cached BE count). 2881 2882 Flags = StrengthenNoWrapFlags(this, scAddRecExpr, Operands, Flags); 2883 2884 // Canonicalize nested AddRecs in by nesting them in order of loop depth. 2885 if (const SCEVAddRecExpr *NestedAR = dyn_cast<SCEVAddRecExpr>(Operands[0])) { 2886 const Loop *NestedLoop = NestedAR->getLoop(); 2887 if (L->contains(NestedLoop) 2888 ? (L->getLoopDepth() < NestedLoop->getLoopDepth()) 2889 : (!NestedLoop->contains(L) && 2890 DT.dominates(L->getHeader(), NestedLoop->getHeader()))) { 2891 SmallVector<const SCEV *, 4> NestedOperands(NestedAR->op_begin(), 2892 NestedAR->op_end()); 2893 Operands[0] = NestedAR->getStart(); 2894 // AddRecs require their operands be loop-invariant with respect to their 2895 // loops. Don't perform this transformation if it would break this 2896 // requirement. 2897 bool AllInvariant = all_of( 2898 Operands, [&](const SCEV *Op) { return isLoopInvariant(Op, L); }); 2899 2900 if (AllInvariant) { 2901 // Create a recurrence for the outer loop with the same step size. 2902 // 2903 // The outer recurrence keeps its NW flag but only keeps NUW/NSW if the 2904 // inner recurrence has the same property. 2905 SCEV::NoWrapFlags OuterFlags = 2906 maskFlags(Flags, SCEV::FlagNW | NestedAR->getNoWrapFlags()); 2907 2908 NestedOperands[0] = getAddRecExpr(Operands, L, OuterFlags); 2909 AllInvariant = all_of(NestedOperands, [&](const SCEV *Op) { 2910 return isLoopInvariant(Op, NestedLoop); 2911 }); 2912 2913 if (AllInvariant) { 2914 // Ok, both add recurrences are valid after the transformation. 2915 // 2916 // The inner recurrence keeps its NW flag but only keeps NUW/NSW if 2917 // the outer recurrence has the same property. 2918 SCEV::NoWrapFlags InnerFlags = 2919 maskFlags(NestedAR->getNoWrapFlags(), SCEV::FlagNW | Flags); 2920 return getAddRecExpr(NestedOperands, NestedLoop, InnerFlags); 2921 } 2922 } 2923 // Reset Operands to its original state. 2924 Operands[0] = NestedAR; 2925 } 2926 } 2927 2928 // Okay, it looks like we really DO need an addrec expr. Check to see if we 2929 // already have one, otherwise create a new one. 2930 FoldingSetNodeID ID; 2931 ID.AddInteger(scAddRecExpr); 2932 for (unsigned i = 0, e = Operands.size(); i != e; ++i) 2933 ID.AddPointer(Operands[i]); 2934 ID.AddPointer(L); 2935 void *IP = nullptr; 2936 SCEVAddRecExpr *S = 2937 static_cast<SCEVAddRecExpr *>(UniqueSCEVs.FindNodeOrInsertPos(ID, IP)); 2938 if (!S) { 2939 const SCEV **O = SCEVAllocator.Allocate<const SCEV *>(Operands.size()); 2940 std::uninitialized_copy(Operands.begin(), Operands.end(), O); 2941 S = new (SCEVAllocator) SCEVAddRecExpr(ID.Intern(SCEVAllocator), 2942 O, Operands.size(), L); 2943 UniqueSCEVs.InsertNode(S, IP); 2944 } 2945 S->setNoWrapFlags(Flags); 2946 return S; 2947 } 2948 2949 const SCEV * 2950 ScalarEvolution::getGEPExpr(Type *PointeeType, const SCEV *BaseExpr, 2951 const SmallVectorImpl<const SCEV *> &IndexExprs, 2952 bool InBounds) { 2953 // getSCEV(Base)->getType() has the same address space as Base->getType() 2954 // because SCEV::getType() preserves the address space. 2955 Type *IntPtrTy = getEffectiveSCEVType(BaseExpr->getType()); 2956 // FIXME(PR23527): Don't blindly transfer the inbounds flag from the GEP 2957 // instruction to its SCEV, because the Instruction may be guarded by control 2958 // flow and the no-overflow bits may not be valid for the expression in any 2959 // context. This can be fixed similarly to how these flags are handled for 2960 // adds. 2961 SCEV::NoWrapFlags Wrap = InBounds ? SCEV::FlagNSW : SCEV::FlagAnyWrap; 2962 2963 const SCEV *TotalOffset = getZero(IntPtrTy); 2964 // The address space is unimportant. The first thing we do on CurTy is getting 2965 // its element type. 2966 Type *CurTy = PointerType::getUnqual(PointeeType); 2967 for (const SCEV *IndexExpr : IndexExprs) { 2968 // Compute the (potentially symbolic) offset in bytes for this index. 2969 if (StructType *STy = dyn_cast<StructType>(CurTy)) { 2970 // For a struct, add the member offset. 2971 ConstantInt *Index = cast<SCEVConstant>(IndexExpr)->getValue(); 2972 unsigned FieldNo = Index->getZExtValue(); 2973 const SCEV *FieldOffset = getOffsetOfExpr(IntPtrTy, STy, FieldNo); 2974 2975 // Add the field offset to the running total offset. 2976 TotalOffset = getAddExpr(TotalOffset, FieldOffset); 2977 2978 // Update CurTy to the type of the field at Index. 2979 CurTy = STy->getTypeAtIndex(Index); 2980 } else { 2981 // Update CurTy to its element type. 2982 CurTy = cast<SequentialType>(CurTy)->getElementType(); 2983 // For an array, add the element offset, explicitly scaled. 2984 const SCEV *ElementSize = getSizeOfExpr(IntPtrTy, CurTy); 2985 // Getelementptr indices are signed. 2986 IndexExpr = getTruncateOrSignExtend(IndexExpr, IntPtrTy); 2987 2988 // Multiply the index by the element size to compute the element offset. 2989 const SCEV *LocalOffset = getMulExpr(IndexExpr, ElementSize, Wrap); 2990 2991 // Add the element offset to the running total offset. 2992 TotalOffset = getAddExpr(TotalOffset, LocalOffset); 2993 } 2994 } 2995 2996 // Add the total offset from all the GEP indices to the base. 2997 return getAddExpr(BaseExpr, TotalOffset, Wrap); 2998 } 2999 3000 const SCEV *ScalarEvolution::getSMaxExpr(const SCEV *LHS, 3001 const SCEV *RHS) { 3002 SmallVector<const SCEV *, 2> Ops; 3003 Ops.push_back(LHS); 3004 Ops.push_back(RHS); 3005 return getSMaxExpr(Ops); 3006 } 3007 3008 const SCEV * 3009 ScalarEvolution::getSMaxExpr(SmallVectorImpl<const SCEV *> &Ops) { 3010 assert(!Ops.empty() && "Cannot get empty smax!"); 3011 if (Ops.size() == 1) return Ops[0]; 3012 #ifndef NDEBUG 3013 Type *ETy = getEffectiveSCEVType(Ops[0]->getType()); 3014 for (unsigned i = 1, e = Ops.size(); i != e; ++i) 3015 assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy && 3016 "SCEVSMaxExpr operand types don't match!"); 3017 #endif 3018 3019 // Sort by complexity, this groups all similar expression types together. 3020 GroupByComplexity(Ops, &LI); 3021 3022 // If there are any constants, fold them together. 3023 unsigned Idx = 0; 3024 if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(Ops[0])) { 3025 ++Idx; 3026 assert(Idx < Ops.size()); 3027 while (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Ops[Idx])) { 3028 // We found two constants, fold them together! 3029 ConstantInt *Fold = ConstantInt::get( 3030 getContext(), APIntOps::smax(LHSC->getAPInt(), RHSC->getAPInt())); 3031 Ops[0] = getConstant(Fold); 3032 Ops.erase(Ops.begin()+1); // Erase the folded element 3033 if (Ops.size() == 1) return Ops[0]; 3034 LHSC = cast<SCEVConstant>(Ops[0]); 3035 } 3036 3037 // If we are left with a constant minimum-int, strip it off. 3038 if (cast<SCEVConstant>(Ops[0])->getValue()->isMinValue(true)) { 3039 Ops.erase(Ops.begin()); 3040 --Idx; 3041 } else if (cast<SCEVConstant>(Ops[0])->getValue()->isMaxValue(true)) { 3042 // If we have an smax with a constant maximum-int, it will always be 3043 // maximum-int. 3044 return Ops[0]; 3045 } 3046 3047 if (Ops.size() == 1) return Ops[0]; 3048 } 3049 3050 // Find the first SMax 3051 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scSMaxExpr) 3052 ++Idx; 3053 3054 // Check to see if one of the operands is an SMax. If so, expand its operands 3055 // onto our operand list, and recurse to simplify. 3056 if (Idx < Ops.size()) { 3057 bool DeletedSMax = false; 3058 while (const SCEVSMaxExpr *SMax = dyn_cast<SCEVSMaxExpr>(Ops[Idx])) { 3059 Ops.erase(Ops.begin()+Idx); 3060 Ops.append(SMax->op_begin(), SMax->op_end()); 3061 DeletedSMax = true; 3062 } 3063 3064 if (DeletedSMax) 3065 return getSMaxExpr(Ops); 3066 } 3067 3068 // Okay, check to see if the same value occurs in the operand list twice. If 3069 // so, delete one. Since we sorted the list, these values are required to 3070 // be adjacent. 3071 for (unsigned i = 0, e = Ops.size()-1; i != e; ++i) 3072 // X smax Y smax Y --> X smax Y 3073 // X smax Y --> X, if X is always greater than Y 3074 if (Ops[i] == Ops[i+1] || 3075 isKnownPredicate(ICmpInst::ICMP_SGE, Ops[i], Ops[i+1])) { 3076 Ops.erase(Ops.begin()+i+1, Ops.begin()+i+2); 3077 --i; --e; 3078 } else if (isKnownPredicate(ICmpInst::ICMP_SLE, Ops[i], Ops[i+1])) { 3079 Ops.erase(Ops.begin()+i, Ops.begin()+i+1); 3080 --i; --e; 3081 } 3082 3083 if (Ops.size() == 1) return Ops[0]; 3084 3085 assert(!Ops.empty() && "Reduced smax down to nothing!"); 3086 3087 // Okay, it looks like we really DO need an smax expr. Check to see if we 3088 // already have one, otherwise create a new one. 3089 FoldingSetNodeID ID; 3090 ID.AddInteger(scSMaxExpr); 3091 for (unsigned i = 0, e = Ops.size(); i != e; ++i) 3092 ID.AddPointer(Ops[i]); 3093 void *IP = nullptr; 3094 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S; 3095 const SCEV **O = SCEVAllocator.Allocate<const SCEV *>(Ops.size()); 3096 std::uninitialized_copy(Ops.begin(), Ops.end(), O); 3097 SCEV *S = new (SCEVAllocator) SCEVSMaxExpr(ID.Intern(SCEVAllocator), 3098 O, Ops.size()); 3099 UniqueSCEVs.InsertNode(S, IP); 3100 return S; 3101 } 3102 3103 const SCEV *ScalarEvolution::getUMaxExpr(const SCEV *LHS, 3104 const SCEV *RHS) { 3105 SmallVector<const SCEV *, 2> Ops; 3106 Ops.push_back(LHS); 3107 Ops.push_back(RHS); 3108 return getUMaxExpr(Ops); 3109 } 3110 3111 const SCEV * 3112 ScalarEvolution::getUMaxExpr(SmallVectorImpl<const SCEV *> &Ops) { 3113 assert(!Ops.empty() && "Cannot get empty umax!"); 3114 if (Ops.size() == 1) return Ops[0]; 3115 #ifndef NDEBUG 3116 Type *ETy = getEffectiveSCEVType(Ops[0]->getType()); 3117 for (unsigned i = 1, e = Ops.size(); i != e; ++i) 3118 assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy && 3119 "SCEVUMaxExpr operand types don't match!"); 3120 #endif 3121 3122 // Sort by complexity, this groups all similar expression types together. 3123 GroupByComplexity(Ops, &LI); 3124 3125 // If there are any constants, fold them together. 3126 unsigned Idx = 0; 3127 if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(Ops[0])) { 3128 ++Idx; 3129 assert(Idx < Ops.size()); 3130 while (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Ops[Idx])) { 3131 // We found two constants, fold them together! 3132 ConstantInt *Fold = ConstantInt::get( 3133 getContext(), APIntOps::umax(LHSC->getAPInt(), RHSC->getAPInt())); 3134 Ops[0] = getConstant(Fold); 3135 Ops.erase(Ops.begin()+1); // Erase the folded element 3136 if (Ops.size() == 1) return Ops[0]; 3137 LHSC = cast<SCEVConstant>(Ops[0]); 3138 } 3139 3140 // If we are left with a constant minimum-int, strip it off. 3141 if (cast<SCEVConstant>(Ops[0])->getValue()->isMinValue(false)) { 3142 Ops.erase(Ops.begin()); 3143 --Idx; 3144 } else if (cast<SCEVConstant>(Ops[0])->getValue()->isMaxValue(false)) { 3145 // If we have an umax with a constant maximum-int, it will always be 3146 // maximum-int. 3147 return Ops[0]; 3148 } 3149 3150 if (Ops.size() == 1) return Ops[0]; 3151 } 3152 3153 // Find the first UMax 3154 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scUMaxExpr) 3155 ++Idx; 3156 3157 // Check to see if one of the operands is a UMax. If so, expand its operands 3158 // onto our operand list, and recurse to simplify. 3159 if (Idx < Ops.size()) { 3160 bool DeletedUMax = false; 3161 while (const SCEVUMaxExpr *UMax = dyn_cast<SCEVUMaxExpr>(Ops[Idx])) { 3162 Ops.erase(Ops.begin()+Idx); 3163 Ops.append(UMax->op_begin(), UMax->op_end()); 3164 DeletedUMax = true; 3165 } 3166 3167 if (DeletedUMax) 3168 return getUMaxExpr(Ops); 3169 } 3170 3171 // Okay, check to see if the same value occurs in the operand list twice. If 3172 // so, delete one. Since we sorted the list, these values are required to 3173 // be adjacent. 3174 for (unsigned i = 0, e = Ops.size()-1; i != e; ++i) 3175 // X umax Y umax Y --> X umax Y 3176 // X umax Y --> X, if X is always greater than Y 3177 if (Ops[i] == Ops[i+1] || 3178 isKnownPredicate(ICmpInst::ICMP_UGE, Ops[i], Ops[i+1])) { 3179 Ops.erase(Ops.begin()+i+1, Ops.begin()+i+2); 3180 --i; --e; 3181 } else if (isKnownPredicate(ICmpInst::ICMP_ULE, Ops[i], Ops[i+1])) { 3182 Ops.erase(Ops.begin()+i, Ops.begin()+i+1); 3183 --i; --e; 3184 } 3185 3186 if (Ops.size() == 1) return Ops[0]; 3187 3188 assert(!Ops.empty() && "Reduced umax down to nothing!"); 3189 3190 // Okay, it looks like we really DO need a umax expr. Check to see if we 3191 // already have one, otherwise create a new one. 3192 FoldingSetNodeID ID; 3193 ID.AddInteger(scUMaxExpr); 3194 for (unsigned i = 0, e = Ops.size(); i != e; ++i) 3195 ID.AddPointer(Ops[i]); 3196 void *IP = nullptr; 3197 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S; 3198 const SCEV **O = SCEVAllocator.Allocate<const SCEV *>(Ops.size()); 3199 std::uninitialized_copy(Ops.begin(), Ops.end(), O); 3200 SCEV *S = new (SCEVAllocator) SCEVUMaxExpr(ID.Intern(SCEVAllocator), 3201 O, Ops.size()); 3202 UniqueSCEVs.InsertNode(S, IP); 3203 return S; 3204 } 3205 3206 const SCEV *ScalarEvolution::getSMinExpr(const SCEV *LHS, 3207 const SCEV *RHS) { 3208 // ~smax(~x, ~y) == smin(x, y). 3209 return getNotSCEV(getSMaxExpr(getNotSCEV(LHS), getNotSCEV(RHS))); 3210 } 3211 3212 const SCEV *ScalarEvolution::getUMinExpr(const SCEV *LHS, 3213 const SCEV *RHS) { 3214 // ~umax(~x, ~y) == umin(x, y) 3215 return getNotSCEV(getUMaxExpr(getNotSCEV(LHS), getNotSCEV(RHS))); 3216 } 3217 3218 const SCEV *ScalarEvolution::getSizeOfExpr(Type *IntTy, Type *AllocTy) { 3219 // We can bypass creating a target-independent 3220 // constant expression and then folding it back into a ConstantInt. 3221 // This is just a compile-time optimization. 3222 return getConstant(IntTy, getDataLayout().getTypeAllocSize(AllocTy)); 3223 } 3224 3225 const SCEV *ScalarEvolution::getOffsetOfExpr(Type *IntTy, 3226 StructType *STy, 3227 unsigned FieldNo) { 3228 // We can bypass creating a target-independent 3229 // constant expression and then folding it back into a ConstantInt. 3230 // This is just a compile-time optimization. 3231 return getConstant( 3232 IntTy, getDataLayout().getStructLayout(STy)->getElementOffset(FieldNo)); 3233 } 3234 3235 const SCEV *ScalarEvolution::getUnknown(Value *V) { 3236 // Don't attempt to do anything other than create a SCEVUnknown object 3237 // here. createSCEV only calls getUnknown after checking for all other 3238 // interesting possibilities, and any other code that calls getUnknown 3239 // is doing so in order to hide a value from SCEV canonicalization. 3240 3241 FoldingSetNodeID ID; 3242 ID.AddInteger(scUnknown); 3243 ID.AddPointer(V); 3244 void *IP = nullptr; 3245 if (SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) { 3246 assert(cast<SCEVUnknown>(S)->getValue() == V && 3247 "Stale SCEVUnknown in uniquing map!"); 3248 return S; 3249 } 3250 SCEV *S = new (SCEVAllocator) SCEVUnknown(ID.Intern(SCEVAllocator), V, this, 3251 FirstUnknown); 3252 FirstUnknown = cast<SCEVUnknown>(S); 3253 UniqueSCEVs.InsertNode(S, IP); 3254 return S; 3255 } 3256 3257 //===----------------------------------------------------------------------===// 3258 // Basic SCEV Analysis and PHI Idiom Recognition Code 3259 // 3260 3261 /// isSCEVable - Test if values of the given type are analyzable within 3262 /// the SCEV framework. This primarily includes integer types, and it 3263 /// can optionally include pointer types if the ScalarEvolution class 3264 /// has access to target-specific information. 3265 bool ScalarEvolution::isSCEVable(Type *Ty) const { 3266 // Integers and pointers are always SCEVable. 3267 return Ty->isIntegerTy() || Ty->isPointerTy(); 3268 } 3269 3270 /// getTypeSizeInBits - Return the size in bits of the specified type, 3271 /// for which isSCEVable must return true. 3272 uint64_t ScalarEvolution::getTypeSizeInBits(Type *Ty) const { 3273 assert(isSCEVable(Ty) && "Type is not SCEVable!"); 3274 return getDataLayout().getTypeSizeInBits(Ty); 3275 } 3276 3277 /// getEffectiveSCEVType - Return a type with the same bitwidth as 3278 /// the given type and which represents how SCEV will treat the given 3279 /// type, for which isSCEVable must return true. For pointer types, 3280 /// this is the pointer-sized integer type. 3281 Type *ScalarEvolution::getEffectiveSCEVType(Type *Ty) const { 3282 assert(isSCEVable(Ty) && "Type is not SCEVable!"); 3283 3284 if (Ty->isIntegerTy()) 3285 return Ty; 3286 3287 // The only other support type is pointer. 3288 assert(Ty->isPointerTy() && "Unexpected non-pointer non-integer type!"); 3289 return getDataLayout().getIntPtrType(Ty); 3290 } 3291 3292 const SCEV *ScalarEvolution::getCouldNotCompute() { 3293 return CouldNotCompute.get(); 3294 } 3295 3296 3297 bool ScalarEvolution::checkValidity(const SCEV *S) const { 3298 // Helper class working with SCEVTraversal to figure out if a SCEV contains 3299 // a SCEVUnknown with null value-pointer. FindInvalidSCEVUnknown::FindOne 3300 // is set iff if find such SCEVUnknown. 3301 // 3302 struct FindInvalidSCEVUnknown { 3303 bool FindOne; 3304 FindInvalidSCEVUnknown() { FindOne = false; } 3305 bool follow(const SCEV *S) { 3306 switch (static_cast<SCEVTypes>(S->getSCEVType())) { 3307 case scConstant: 3308 return false; 3309 case scUnknown: 3310 if (!cast<SCEVUnknown>(S)->getValue()) 3311 FindOne = true; 3312 return false; 3313 default: 3314 return true; 3315 } 3316 } 3317 bool isDone() const { return FindOne; } 3318 }; 3319 3320 FindInvalidSCEVUnknown F; 3321 SCEVTraversal<FindInvalidSCEVUnknown> ST(F); 3322 ST.visitAll(S); 3323 3324 return !F.FindOne; 3325 } 3326 3327 namespace { 3328 // Helper class working with SCEVTraversal to figure out if a SCEV contains 3329 // a sub SCEV of scAddRecExpr type. FindInvalidSCEVUnknown::FoundOne is set 3330 // iff if such sub scAddRecExpr type SCEV is found. 3331 struct FindAddRecurrence { 3332 bool FoundOne; 3333 FindAddRecurrence() : FoundOne(false) {} 3334 3335 bool follow(const SCEV *S) { 3336 switch (static_cast<SCEVTypes>(S->getSCEVType())) { 3337 case scAddRecExpr: 3338 FoundOne = true; 3339 case scConstant: 3340 case scUnknown: 3341 case scCouldNotCompute: 3342 return false; 3343 default: 3344 return true; 3345 } 3346 } 3347 bool isDone() const { return FoundOne; } 3348 }; 3349 } 3350 3351 bool ScalarEvolution::containsAddRecurrence(const SCEV *S) { 3352 HasRecMapType::iterator I = HasRecMap.find_as(S); 3353 if (I != HasRecMap.end()) 3354 return I->second; 3355 3356 FindAddRecurrence F; 3357 SCEVTraversal<FindAddRecurrence> ST(F); 3358 ST.visitAll(S); 3359 HasRecMap.insert({S, F.FoundOne}); 3360 return F.FoundOne; 3361 } 3362 3363 /// getSCEVValues - Return the Value set from S. 3364 SetVector<Value *> *ScalarEvolution::getSCEVValues(const SCEV *S) { 3365 ExprValueMapType::iterator SI = ExprValueMap.find_as(S); 3366 if (SI == ExprValueMap.end()) 3367 return nullptr; 3368 #ifndef NDEBUG 3369 if (VerifySCEVMap) { 3370 // Check there is no dangling Value in the set returned. 3371 for (const auto &VE : SI->second) 3372 assert(ValueExprMap.count(VE)); 3373 } 3374 #endif 3375 return &SI->second; 3376 } 3377 3378 /// eraseValueFromMap - Erase Value from ValueExprMap and ExprValueMap. 3379 /// If ValueExprMap.erase(V) is not used together with forgetMemoizedResults(S), 3380 /// eraseValueFromMap should be used instead to ensure whenever V->S is removed 3381 /// from ValueExprMap, V is also removed from the set of ExprValueMap[S]. 3382 void ScalarEvolution::eraseValueFromMap(Value *V) { 3383 ValueExprMapType::iterator I = ValueExprMap.find_as(V); 3384 if (I != ValueExprMap.end()) { 3385 const SCEV *S = I->second; 3386 SetVector<Value *> *SV = getSCEVValues(S); 3387 // Remove V from the set of ExprValueMap[S] 3388 if (SV) 3389 SV->remove(V); 3390 ValueExprMap.erase(V); 3391 } 3392 } 3393 3394 /// getSCEV - Return an existing SCEV if it exists, otherwise analyze the 3395 /// expression and create a new one. 3396 const SCEV *ScalarEvolution::getSCEV(Value *V) { 3397 assert(isSCEVable(V->getType()) && "Value is not SCEVable!"); 3398 3399 const SCEV *S = getExistingSCEV(V); 3400 if (S == nullptr) { 3401 S = createSCEV(V); 3402 // During PHI resolution, it is possible to create two SCEVs for the same 3403 // V, so it is needed to double check whether V->S is inserted into 3404 // ValueExprMap before insert S->V into ExprValueMap. 3405 std::pair<ValueExprMapType::iterator, bool> Pair = 3406 ValueExprMap.insert({SCEVCallbackVH(V, this), S}); 3407 if (Pair.second) 3408 ExprValueMap[S].insert(V); 3409 } 3410 return S; 3411 } 3412 3413 const SCEV *ScalarEvolution::getExistingSCEV(Value *V) { 3414 assert(isSCEVable(V->getType()) && "Value is not SCEVable!"); 3415 3416 ValueExprMapType::iterator I = ValueExprMap.find_as(V); 3417 if (I != ValueExprMap.end()) { 3418 const SCEV *S = I->second; 3419 if (checkValidity(S)) 3420 return S; 3421 forgetMemoizedResults(S); 3422 ValueExprMap.erase(I); 3423 } 3424 return nullptr; 3425 } 3426 3427 /// getNegativeSCEV - Return a SCEV corresponding to -V = -1*V 3428 /// 3429 const SCEV *ScalarEvolution::getNegativeSCEV(const SCEV *V, 3430 SCEV::NoWrapFlags Flags) { 3431 if (const SCEVConstant *VC = dyn_cast<SCEVConstant>(V)) 3432 return getConstant( 3433 cast<ConstantInt>(ConstantExpr::getNeg(VC->getValue()))); 3434 3435 Type *Ty = V->getType(); 3436 Ty = getEffectiveSCEVType(Ty); 3437 return getMulExpr( 3438 V, getConstant(cast<ConstantInt>(Constant::getAllOnesValue(Ty))), Flags); 3439 } 3440 3441 /// getNotSCEV - Return a SCEV corresponding to ~V = -1-V 3442 const SCEV *ScalarEvolution::getNotSCEV(const SCEV *V) { 3443 if (const SCEVConstant *VC = dyn_cast<SCEVConstant>(V)) 3444 return getConstant( 3445 cast<ConstantInt>(ConstantExpr::getNot(VC->getValue()))); 3446 3447 Type *Ty = V->getType(); 3448 Ty = getEffectiveSCEVType(Ty); 3449 const SCEV *AllOnes = 3450 getConstant(cast<ConstantInt>(Constant::getAllOnesValue(Ty))); 3451 return getMinusSCEV(AllOnes, V); 3452 } 3453 3454 /// getMinusSCEV - Return LHS-RHS. Minus is represented in SCEV as A+B*-1. 3455 const SCEV *ScalarEvolution::getMinusSCEV(const SCEV *LHS, const SCEV *RHS, 3456 SCEV::NoWrapFlags Flags) { 3457 // Fast path: X - X --> 0. 3458 if (LHS == RHS) 3459 return getZero(LHS->getType()); 3460 3461 // We represent LHS - RHS as LHS + (-1)*RHS. This transformation 3462 // makes it so that we cannot make much use of NUW. 3463 auto AddFlags = SCEV::FlagAnyWrap; 3464 const bool RHSIsNotMinSigned = 3465 !getSignedRange(RHS).getSignedMin().isMinSignedValue(); 3466 if (maskFlags(Flags, SCEV::FlagNSW) == SCEV::FlagNSW) { 3467 // Let M be the minimum representable signed value. Then (-1)*RHS 3468 // signed-wraps if and only if RHS is M. That can happen even for 3469 // a NSW subtraction because e.g. (-1)*M signed-wraps even though 3470 // -1 - M does not. So to transfer NSW from LHS - RHS to LHS + 3471 // (-1)*RHS, we need to prove that RHS != M. 3472 // 3473 // If LHS is non-negative and we know that LHS - RHS does not 3474 // signed-wrap, then RHS cannot be M. So we can rule out signed-wrap 3475 // either by proving that RHS > M or that LHS >= 0. 3476 if (RHSIsNotMinSigned || isKnownNonNegative(LHS)) { 3477 AddFlags = SCEV::FlagNSW; 3478 } 3479 } 3480 3481 // FIXME: Find a correct way to transfer NSW to (-1)*M when LHS - 3482 // RHS is NSW and LHS >= 0. 3483 // 3484 // The difficulty here is that the NSW flag may have been proven 3485 // relative to a loop that is to be found in a recurrence in LHS and 3486 // not in RHS. Applying NSW to (-1)*M may then let the NSW have a 3487 // larger scope than intended. 3488 auto NegFlags = RHSIsNotMinSigned ? SCEV::FlagNSW : SCEV::FlagAnyWrap; 3489 3490 return getAddExpr(LHS, getNegativeSCEV(RHS, NegFlags), AddFlags); 3491 } 3492 3493 /// getTruncateOrZeroExtend - Return a SCEV corresponding to a conversion of the 3494 /// input value to the specified type. If the type must be extended, it is zero 3495 /// extended. 3496 const SCEV * 3497 ScalarEvolution::getTruncateOrZeroExtend(const SCEV *V, Type *Ty) { 3498 Type *SrcTy = V->getType(); 3499 assert((SrcTy->isIntegerTy() || SrcTy->isPointerTy()) && 3500 (Ty->isIntegerTy() || Ty->isPointerTy()) && 3501 "Cannot truncate or zero extend with non-integer arguments!"); 3502 if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty)) 3503 return V; // No conversion 3504 if (getTypeSizeInBits(SrcTy) > getTypeSizeInBits(Ty)) 3505 return getTruncateExpr(V, Ty); 3506 return getZeroExtendExpr(V, Ty); 3507 } 3508 3509 /// getTruncateOrSignExtend - Return a SCEV corresponding to a conversion of the 3510 /// input value to the specified type. If the type must be extended, it is sign 3511 /// extended. 3512 const SCEV * 3513 ScalarEvolution::getTruncateOrSignExtend(const SCEV *V, 3514 Type *Ty) { 3515 Type *SrcTy = V->getType(); 3516 assert((SrcTy->isIntegerTy() || SrcTy->isPointerTy()) && 3517 (Ty->isIntegerTy() || Ty->isPointerTy()) && 3518 "Cannot truncate or zero extend with non-integer arguments!"); 3519 if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty)) 3520 return V; // No conversion 3521 if (getTypeSizeInBits(SrcTy) > getTypeSizeInBits(Ty)) 3522 return getTruncateExpr(V, Ty); 3523 return getSignExtendExpr(V, Ty); 3524 } 3525 3526 /// getNoopOrZeroExtend - Return a SCEV corresponding to a conversion of the 3527 /// input value to the specified type. If the type must be extended, it is zero 3528 /// extended. The conversion must not be narrowing. 3529 const SCEV * 3530 ScalarEvolution::getNoopOrZeroExtend(const SCEV *V, Type *Ty) { 3531 Type *SrcTy = V->getType(); 3532 assert((SrcTy->isIntegerTy() || SrcTy->isPointerTy()) && 3533 (Ty->isIntegerTy() || Ty->isPointerTy()) && 3534 "Cannot noop or zero extend with non-integer arguments!"); 3535 assert(getTypeSizeInBits(SrcTy) <= getTypeSizeInBits(Ty) && 3536 "getNoopOrZeroExtend cannot truncate!"); 3537 if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty)) 3538 return V; // No conversion 3539 return getZeroExtendExpr(V, Ty); 3540 } 3541 3542 /// getNoopOrSignExtend - Return a SCEV corresponding to a conversion of the 3543 /// input value to the specified type. If the type must be extended, it is sign 3544 /// extended. The conversion must not be narrowing. 3545 const SCEV * 3546 ScalarEvolution::getNoopOrSignExtend(const SCEV *V, Type *Ty) { 3547 Type *SrcTy = V->getType(); 3548 assert((SrcTy->isIntegerTy() || SrcTy->isPointerTy()) && 3549 (Ty->isIntegerTy() || Ty->isPointerTy()) && 3550 "Cannot noop or sign extend with non-integer arguments!"); 3551 assert(getTypeSizeInBits(SrcTy) <= getTypeSizeInBits(Ty) && 3552 "getNoopOrSignExtend cannot truncate!"); 3553 if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty)) 3554 return V; // No conversion 3555 return getSignExtendExpr(V, Ty); 3556 } 3557 3558 /// getNoopOrAnyExtend - Return a SCEV corresponding to a conversion of 3559 /// the input value to the specified type. If the type must be extended, 3560 /// it is extended with unspecified bits. The conversion must not be 3561 /// narrowing. 3562 const SCEV * 3563 ScalarEvolution::getNoopOrAnyExtend(const SCEV *V, Type *Ty) { 3564 Type *SrcTy = V->getType(); 3565 assert((SrcTy->isIntegerTy() || SrcTy->isPointerTy()) && 3566 (Ty->isIntegerTy() || Ty->isPointerTy()) && 3567 "Cannot noop or any extend with non-integer arguments!"); 3568 assert(getTypeSizeInBits(SrcTy) <= getTypeSizeInBits(Ty) && 3569 "getNoopOrAnyExtend cannot truncate!"); 3570 if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty)) 3571 return V; // No conversion 3572 return getAnyExtendExpr(V, Ty); 3573 } 3574 3575 /// getTruncateOrNoop - Return a SCEV corresponding to a conversion of the 3576 /// input value to the specified type. The conversion must not be widening. 3577 const SCEV * 3578 ScalarEvolution::getTruncateOrNoop(const SCEV *V, Type *Ty) { 3579 Type *SrcTy = V->getType(); 3580 assert((SrcTy->isIntegerTy() || SrcTy->isPointerTy()) && 3581 (Ty->isIntegerTy() || Ty->isPointerTy()) && 3582 "Cannot truncate or noop with non-integer arguments!"); 3583 assert(getTypeSizeInBits(SrcTy) >= getTypeSizeInBits(Ty) && 3584 "getTruncateOrNoop cannot extend!"); 3585 if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty)) 3586 return V; // No conversion 3587 return getTruncateExpr(V, Ty); 3588 } 3589 3590 /// getUMaxFromMismatchedTypes - Promote the operands to the wider of 3591 /// the types using zero-extension, and then perform a umax operation 3592 /// with them. 3593 const SCEV *ScalarEvolution::getUMaxFromMismatchedTypes(const SCEV *LHS, 3594 const SCEV *RHS) { 3595 const SCEV *PromotedLHS = LHS; 3596 const SCEV *PromotedRHS = RHS; 3597 3598 if (getTypeSizeInBits(LHS->getType()) > getTypeSizeInBits(RHS->getType())) 3599 PromotedRHS = getZeroExtendExpr(RHS, LHS->getType()); 3600 else 3601 PromotedLHS = getNoopOrZeroExtend(LHS, RHS->getType()); 3602 3603 return getUMaxExpr(PromotedLHS, PromotedRHS); 3604 } 3605 3606 /// getUMinFromMismatchedTypes - Promote the operands to the wider of 3607 /// the types using zero-extension, and then perform a umin operation 3608 /// with them. 3609 const SCEV *ScalarEvolution::getUMinFromMismatchedTypes(const SCEV *LHS, 3610 const SCEV *RHS) { 3611 const SCEV *PromotedLHS = LHS; 3612 const SCEV *PromotedRHS = RHS; 3613 3614 if (getTypeSizeInBits(LHS->getType()) > getTypeSizeInBits(RHS->getType())) 3615 PromotedRHS = getZeroExtendExpr(RHS, LHS->getType()); 3616 else 3617 PromotedLHS = getNoopOrZeroExtend(LHS, RHS->getType()); 3618 3619 return getUMinExpr(PromotedLHS, PromotedRHS); 3620 } 3621 3622 /// getPointerBase - Transitively follow the chain of pointer-type operands 3623 /// until reaching a SCEV that does not have a single pointer operand. This 3624 /// returns a SCEVUnknown pointer for well-formed pointer-type expressions, 3625 /// but corner cases do exist. 3626 const SCEV *ScalarEvolution::getPointerBase(const SCEV *V) { 3627 // A pointer operand may evaluate to a nonpointer expression, such as null. 3628 if (!V->getType()->isPointerTy()) 3629 return V; 3630 3631 if (const SCEVCastExpr *Cast = dyn_cast<SCEVCastExpr>(V)) { 3632 return getPointerBase(Cast->getOperand()); 3633 } else if (const SCEVNAryExpr *NAry = dyn_cast<SCEVNAryExpr>(V)) { 3634 const SCEV *PtrOp = nullptr; 3635 for (const SCEV *NAryOp : NAry->operands()) { 3636 if (NAryOp->getType()->isPointerTy()) { 3637 // Cannot find the base of an expression with multiple pointer operands. 3638 if (PtrOp) 3639 return V; 3640 PtrOp = NAryOp; 3641 } 3642 } 3643 if (!PtrOp) 3644 return V; 3645 return getPointerBase(PtrOp); 3646 } 3647 return V; 3648 } 3649 3650 /// PushDefUseChildren - Push users of the given Instruction 3651 /// onto the given Worklist. 3652 static void 3653 PushDefUseChildren(Instruction *I, 3654 SmallVectorImpl<Instruction *> &Worklist) { 3655 // Push the def-use children onto the Worklist stack. 3656 for (User *U : I->users()) 3657 Worklist.push_back(cast<Instruction>(U)); 3658 } 3659 3660 /// ForgetSymbolicValue - This looks up computed SCEV values for all 3661 /// instructions that depend on the given instruction and removes them from 3662 /// the ValueExprMapType map if they reference SymName. This is used during PHI 3663 /// resolution. 3664 void ScalarEvolution::forgetSymbolicName(Instruction *PN, const SCEV *SymName) { 3665 SmallVector<Instruction *, 16> Worklist; 3666 PushDefUseChildren(PN, Worklist); 3667 3668 SmallPtrSet<Instruction *, 8> Visited; 3669 Visited.insert(PN); 3670 while (!Worklist.empty()) { 3671 Instruction *I = Worklist.pop_back_val(); 3672 if (!Visited.insert(I).second) 3673 continue; 3674 3675 auto It = ValueExprMap.find_as(static_cast<Value *>(I)); 3676 if (It != ValueExprMap.end()) { 3677 const SCEV *Old = It->second; 3678 3679 // Short-circuit the def-use traversal if the symbolic name 3680 // ceases to appear in expressions. 3681 if (Old != SymName && !hasOperand(Old, SymName)) 3682 continue; 3683 3684 // SCEVUnknown for a PHI either means that it has an unrecognized 3685 // structure, it's a PHI that's in the progress of being computed 3686 // by createNodeForPHI, or it's a single-value PHI. In the first case, 3687 // additional loop trip count information isn't going to change anything. 3688 // In the second case, createNodeForPHI will perform the necessary 3689 // updates on its own when it gets to that point. In the third, we do 3690 // want to forget the SCEVUnknown. 3691 if (!isa<PHINode>(I) || 3692 !isa<SCEVUnknown>(Old) || 3693 (I != PN && Old == SymName)) { 3694 forgetMemoizedResults(Old); 3695 ValueExprMap.erase(It); 3696 } 3697 } 3698 3699 PushDefUseChildren(I, Worklist); 3700 } 3701 } 3702 3703 namespace { 3704 class SCEVInitRewriter : public SCEVRewriteVisitor<SCEVInitRewriter> { 3705 public: 3706 static const SCEV *rewrite(const SCEV *S, const Loop *L, 3707 ScalarEvolution &SE) { 3708 SCEVInitRewriter Rewriter(L, SE); 3709 const SCEV *Result = Rewriter.visit(S); 3710 return Rewriter.isValid() ? Result : SE.getCouldNotCompute(); 3711 } 3712 3713 SCEVInitRewriter(const Loop *L, ScalarEvolution &SE) 3714 : SCEVRewriteVisitor(SE), L(L), Valid(true) {} 3715 3716 const SCEV *visitUnknown(const SCEVUnknown *Expr) { 3717 if (!(SE.getLoopDisposition(Expr, L) == ScalarEvolution::LoopInvariant)) 3718 Valid = false; 3719 return Expr; 3720 } 3721 3722 const SCEV *visitAddRecExpr(const SCEVAddRecExpr *Expr) { 3723 // Only allow AddRecExprs for this loop. 3724 if (Expr->getLoop() == L) 3725 return Expr->getStart(); 3726 Valid = false; 3727 return Expr; 3728 } 3729 3730 bool isValid() { return Valid; } 3731 3732 private: 3733 const Loop *L; 3734 bool Valid; 3735 }; 3736 3737 class SCEVShiftRewriter : public SCEVRewriteVisitor<SCEVShiftRewriter> { 3738 public: 3739 static const SCEV *rewrite(const SCEV *S, const Loop *L, 3740 ScalarEvolution &SE) { 3741 SCEVShiftRewriter Rewriter(L, SE); 3742 const SCEV *Result = Rewriter.visit(S); 3743 return Rewriter.isValid() ? Result : SE.getCouldNotCompute(); 3744 } 3745 3746 SCEVShiftRewriter(const Loop *L, ScalarEvolution &SE) 3747 : SCEVRewriteVisitor(SE), L(L), Valid(true) {} 3748 3749 const SCEV *visitUnknown(const SCEVUnknown *Expr) { 3750 // Only allow AddRecExprs for this loop. 3751 if (!(SE.getLoopDisposition(Expr, L) == ScalarEvolution::LoopInvariant)) 3752 Valid = false; 3753 return Expr; 3754 } 3755 3756 const SCEV *visitAddRecExpr(const SCEVAddRecExpr *Expr) { 3757 if (Expr->getLoop() == L && Expr->isAffine()) 3758 return SE.getMinusSCEV(Expr, Expr->getStepRecurrence(SE)); 3759 Valid = false; 3760 return Expr; 3761 } 3762 bool isValid() { return Valid; } 3763 3764 private: 3765 const Loop *L; 3766 bool Valid; 3767 }; 3768 } // end anonymous namespace 3769 3770 SCEV::NoWrapFlags 3771 ScalarEvolution::proveNoWrapViaConstantRanges(const SCEVAddRecExpr *AR) { 3772 if (!AR->isAffine()) 3773 return SCEV::FlagAnyWrap; 3774 3775 typedef OverflowingBinaryOperator OBO; 3776 SCEV::NoWrapFlags Result = SCEV::FlagAnyWrap; 3777 3778 if (!AR->hasNoSignedWrap()) { 3779 ConstantRange AddRecRange = getSignedRange(AR); 3780 ConstantRange IncRange = getSignedRange(AR->getStepRecurrence(*this)); 3781 3782 auto NSWRegion = ConstantRange::makeGuaranteedNoWrapRegion( 3783 Instruction::Add, IncRange, OBO::NoSignedWrap); 3784 if (NSWRegion.contains(AddRecRange)) 3785 Result = ScalarEvolution::setFlags(Result, SCEV::FlagNSW); 3786 } 3787 3788 if (!AR->hasNoUnsignedWrap()) { 3789 ConstantRange AddRecRange = getUnsignedRange(AR); 3790 ConstantRange IncRange = getUnsignedRange(AR->getStepRecurrence(*this)); 3791 3792 auto NUWRegion = ConstantRange::makeGuaranteedNoWrapRegion( 3793 Instruction::Add, IncRange, OBO::NoUnsignedWrap); 3794 if (NUWRegion.contains(AddRecRange)) 3795 Result = ScalarEvolution::setFlags(Result, SCEV::FlagNUW); 3796 } 3797 3798 return Result; 3799 } 3800 3801 namespace { 3802 /// Represents an abstract binary operation. This may exist as a 3803 /// normal instruction or constant expression, or may have been 3804 /// derived from an expression tree. 3805 struct BinaryOp { 3806 unsigned Opcode; 3807 Value *LHS; 3808 Value *RHS; 3809 bool IsNSW; 3810 bool IsNUW; 3811 3812 /// Op is set if this BinaryOp corresponds to a concrete LLVM instruction or 3813 /// constant expression. 3814 Operator *Op; 3815 3816 explicit BinaryOp(Operator *Op) 3817 : Opcode(Op->getOpcode()), LHS(Op->getOperand(0)), RHS(Op->getOperand(1)), 3818 IsNSW(false), IsNUW(false), Op(Op) { 3819 if (auto *OBO = dyn_cast<OverflowingBinaryOperator>(Op)) { 3820 IsNSW = OBO->hasNoSignedWrap(); 3821 IsNUW = OBO->hasNoUnsignedWrap(); 3822 } 3823 } 3824 3825 explicit BinaryOp(unsigned Opcode, Value *LHS, Value *RHS, bool IsNSW = false, 3826 bool IsNUW = false) 3827 : Opcode(Opcode), LHS(LHS), RHS(RHS), IsNSW(IsNSW), IsNUW(IsNUW), 3828 Op(nullptr) {} 3829 }; 3830 } 3831 3832 3833 /// Try to map \p V into a BinaryOp, and return \c None on failure. 3834 static Optional<BinaryOp> MatchBinaryOp(Value *V) { 3835 auto *Op = dyn_cast<Operator>(V); 3836 if (!Op) 3837 return None; 3838 3839 // Implementation detail: all the cleverness here should happen without 3840 // creating new SCEV expressions -- our caller knowns tricks to avoid creating 3841 // SCEV expressions when possible, and we should not break that. 3842 3843 switch (Op->getOpcode()) { 3844 case Instruction::Add: 3845 case Instruction::Sub: 3846 case Instruction::Mul: 3847 case Instruction::UDiv: 3848 case Instruction::And: 3849 case Instruction::Or: 3850 case Instruction::AShr: 3851 case Instruction::Shl: 3852 return BinaryOp(Op); 3853 3854 case Instruction::Xor: 3855 if (auto *RHSC = dyn_cast<ConstantInt>(Op->getOperand(1))) 3856 // If the RHS of the xor is a signbit, then this is just an add. 3857 // Instcombine turns add of signbit into xor as a strength reduction step. 3858 if (RHSC->getValue().isSignBit()) 3859 return BinaryOp(Instruction::Add, Op->getOperand(0), Op->getOperand(1)); 3860 return BinaryOp(Op); 3861 3862 case Instruction::LShr: 3863 // Turn logical shift right of a constant into a unsigned divide. 3864 if (ConstantInt *SA = dyn_cast<ConstantInt>(Op->getOperand(1))) { 3865 uint32_t BitWidth = cast<IntegerType>(Op->getType())->getBitWidth(); 3866 3867 // If the shift count is not less than the bitwidth, the result of 3868 // the shift is undefined. Don't try to analyze it, because the 3869 // resolution chosen here may differ from the resolution chosen in 3870 // other parts of the compiler. 3871 if (SA->getValue().ult(BitWidth)) { 3872 Constant *X = 3873 ConstantInt::get(SA->getContext(), 3874 APInt::getOneBitSet(BitWidth, SA->getZExtValue())); 3875 return BinaryOp(Instruction::UDiv, Op->getOperand(0), X); 3876 } 3877 } 3878 return BinaryOp(Op); 3879 3880 default: 3881 break; 3882 } 3883 3884 return None; 3885 } 3886 3887 const SCEV *ScalarEvolution::createAddRecFromPHI(PHINode *PN) { 3888 const Loop *L = LI.getLoopFor(PN->getParent()); 3889 if (!L || L->getHeader() != PN->getParent()) 3890 return nullptr; 3891 3892 // The loop may have multiple entrances or multiple exits; we can analyze 3893 // this phi as an addrec if it has a unique entry value and a unique 3894 // backedge value. 3895 Value *BEValueV = nullptr, *StartValueV = nullptr; 3896 for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) { 3897 Value *V = PN->getIncomingValue(i); 3898 if (L->contains(PN->getIncomingBlock(i))) { 3899 if (!BEValueV) { 3900 BEValueV = V; 3901 } else if (BEValueV != V) { 3902 BEValueV = nullptr; 3903 break; 3904 } 3905 } else if (!StartValueV) { 3906 StartValueV = V; 3907 } else if (StartValueV != V) { 3908 StartValueV = nullptr; 3909 break; 3910 } 3911 } 3912 if (BEValueV && StartValueV) { 3913 // While we are analyzing this PHI node, handle its value symbolically. 3914 const SCEV *SymbolicName = getUnknown(PN); 3915 assert(ValueExprMap.find_as(PN) == ValueExprMap.end() && 3916 "PHI node already processed?"); 3917 ValueExprMap.insert({SCEVCallbackVH(PN, this), SymbolicName}); 3918 3919 // Using this symbolic name for the PHI, analyze the value coming around 3920 // the back-edge. 3921 const SCEV *BEValue = getSCEV(BEValueV); 3922 3923 // NOTE: If BEValue is loop invariant, we know that the PHI node just 3924 // has a special value for the first iteration of the loop. 3925 3926 // If the value coming around the backedge is an add with the symbolic 3927 // value we just inserted, then we found a simple induction variable! 3928 if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(BEValue)) { 3929 // If there is a single occurrence of the symbolic value, replace it 3930 // with a recurrence. 3931 unsigned FoundIndex = Add->getNumOperands(); 3932 for (unsigned i = 0, e = Add->getNumOperands(); i != e; ++i) 3933 if (Add->getOperand(i) == SymbolicName) 3934 if (FoundIndex == e) { 3935 FoundIndex = i; 3936 break; 3937 } 3938 3939 if (FoundIndex != Add->getNumOperands()) { 3940 // Create an add with everything but the specified operand. 3941 SmallVector<const SCEV *, 8> Ops; 3942 for (unsigned i = 0, e = Add->getNumOperands(); i != e; ++i) 3943 if (i != FoundIndex) 3944 Ops.push_back(Add->getOperand(i)); 3945 const SCEV *Accum = getAddExpr(Ops); 3946 3947 // This is not a valid addrec if the step amount is varying each 3948 // loop iteration, but is not itself an addrec in this loop. 3949 if (isLoopInvariant(Accum, L) || 3950 (isa<SCEVAddRecExpr>(Accum) && 3951 cast<SCEVAddRecExpr>(Accum)->getLoop() == L)) { 3952 SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap; 3953 3954 // If the increment doesn't overflow, then neither the addrec nor 3955 // the post-increment will overflow. 3956 if (auto BO = MatchBinaryOp(BEValueV)) { 3957 if (BO->Opcode == Instruction::Add && BO->LHS == PN) { 3958 if (BO->IsNUW) 3959 Flags = setFlags(Flags, SCEV::FlagNUW); 3960 if (BO->IsNSW) 3961 Flags = setFlags(Flags, SCEV::FlagNSW); 3962 } 3963 } else if (GEPOperator *GEP = dyn_cast<GEPOperator>(BEValueV)) { 3964 // If the increment is an inbounds GEP, then we know the address 3965 // space cannot be wrapped around. We cannot make any guarantee 3966 // about signed or unsigned overflow because pointers are 3967 // unsigned but we may have a negative index from the base 3968 // pointer. We can guarantee that no unsigned wrap occurs if the 3969 // indices form a positive value. 3970 if (GEP->isInBounds() && GEP->getOperand(0) == PN) { 3971 Flags = setFlags(Flags, SCEV::FlagNW); 3972 3973 const SCEV *Ptr = getSCEV(GEP->getPointerOperand()); 3974 if (isKnownPositive(getMinusSCEV(getSCEV(GEP), Ptr))) 3975 Flags = setFlags(Flags, SCEV::FlagNUW); 3976 } 3977 3978 // We cannot transfer nuw and nsw flags from subtraction 3979 // operations -- sub nuw X, Y is not the same as add nuw X, -Y 3980 // for instance. 3981 } 3982 3983 const SCEV *StartVal = getSCEV(StartValueV); 3984 const SCEV *PHISCEV = getAddRecExpr(StartVal, Accum, L, Flags); 3985 3986 // Since the no-wrap flags are on the increment, they apply to the 3987 // post-incremented value as well. 3988 if (isLoopInvariant(Accum, L)) 3989 (void)getAddRecExpr(getAddExpr(StartVal, Accum), Accum, L, Flags); 3990 3991 // Okay, for the entire analysis of this edge we assumed the PHI 3992 // to be symbolic. We now need to go back and purge all of the 3993 // entries for the scalars that use the symbolic expression. 3994 forgetSymbolicName(PN, SymbolicName); 3995 ValueExprMap[SCEVCallbackVH(PN, this)] = PHISCEV; 3996 return PHISCEV; 3997 } 3998 } 3999 } else { 4000 // Otherwise, this could be a loop like this: 4001 // i = 0; for (j = 1; ..; ++j) { .... i = j; } 4002 // In this case, j = {1,+,1} and BEValue is j. 4003 // Because the other in-value of i (0) fits the evolution of BEValue 4004 // i really is an addrec evolution. 4005 // 4006 // We can generalize this saying that i is the shifted value of BEValue 4007 // by one iteration: 4008 // PHI(f(0), f({1,+,1})) --> f({0,+,1}) 4009 const SCEV *Shifted = SCEVShiftRewriter::rewrite(BEValue, L, *this); 4010 const SCEV *Start = SCEVInitRewriter::rewrite(Shifted, L, *this); 4011 if (Shifted != getCouldNotCompute() && 4012 Start != getCouldNotCompute()) { 4013 const SCEV *StartVal = getSCEV(StartValueV); 4014 if (Start == StartVal) { 4015 // Okay, for the entire analysis of this edge we assumed the PHI 4016 // to be symbolic. We now need to go back and purge all of the 4017 // entries for the scalars that use the symbolic expression. 4018 forgetSymbolicName(PN, SymbolicName); 4019 ValueExprMap[SCEVCallbackVH(PN, this)] = Shifted; 4020 return Shifted; 4021 } 4022 } 4023 } 4024 4025 // Remove the temporary PHI node SCEV that has been inserted while intending 4026 // to create an AddRecExpr for this PHI node. We can not keep this temporary 4027 // as it will prevent later (possibly simpler) SCEV expressions to be added 4028 // to the ValueExprMap. 4029 ValueExprMap.erase(PN); 4030 } 4031 4032 return nullptr; 4033 } 4034 4035 // Checks if the SCEV S is available at BB. S is considered available at BB 4036 // if S can be materialized at BB without introducing a fault. 4037 static bool IsAvailableOnEntry(const Loop *L, DominatorTree &DT, const SCEV *S, 4038 BasicBlock *BB) { 4039 struct CheckAvailable { 4040 bool TraversalDone = false; 4041 bool Available = true; 4042 4043 const Loop *L = nullptr; // The loop BB is in (can be nullptr) 4044 BasicBlock *BB = nullptr; 4045 DominatorTree &DT; 4046 4047 CheckAvailable(const Loop *L, BasicBlock *BB, DominatorTree &DT) 4048 : L(L), BB(BB), DT(DT) {} 4049 4050 bool setUnavailable() { 4051 TraversalDone = true; 4052 Available = false; 4053 return false; 4054 } 4055 4056 bool follow(const SCEV *S) { 4057 switch (S->getSCEVType()) { 4058 case scConstant: case scTruncate: case scZeroExtend: case scSignExtend: 4059 case scAddExpr: case scMulExpr: case scUMaxExpr: case scSMaxExpr: 4060 // These expressions are available if their operand(s) is/are. 4061 return true; 4062 4063 case scAddRecExpr: { 4064 // We allow add recurrences that are on the loop BB is in, or some 4065 // outer loop. This guarantees availability because the value of the 4066 // add recurrence at BB is simply the "current" value of the induction 4067 // variable. We can relax this in the future; for instance an add 4068 // recurrence on a sibling dominating loop is also available at BB. 4069 const auto *ARLoop = cast<SCEVAddRecExpr>(S)->getLoop(); 4070 if (L && (ARLoop == L || ARLoop->contains(L))) 4071 return true; 4072 4073 return setUnavailable(); 4074 } 4075 4076 case scUnknown: { 4077 // For SCEVUnknown, we check for simple dominance. 4078 const auto *SU = cast<SCEVUnknown>(S); 4079 Value *V = SU->getValue(); 4080 4081 if (isa<Argument>(V)) 4082 return false; 4083 4084 if (isa<Instruction>(V) && DT.dominates(cast<Instruction>(V), BB)) 4085 return false; 4086 4087 return setUnavailable(); 4088 } 4089 4090 case scUDivExpr: 4091 case scCouldNotCompute: 4092 // We do not try to smart about these at all. 4093 return setUnavailable(); 4094 } 4095 llvm_unreachable("switch should be fully covered!"); 4096 } 4097 4098 bool isDone() { return TraversalDone; } 4099 }; 4100 4101 CheckAvailable CA(L, BB, DT); 4102 SCEVTraversal<CheckAvailable> ST(CA); 4103 4104 ST.visitAll(S); 4105 return CA.Available; 4106 } 4107 4108 // Try to match a control flow sequence that branches out at BI and merges back 4109 // at Merge into a "C ? LHS : RHS" select pattern. Return true on a successful 4110 // match. 4111 static bool BrPHIToSelect(DominatorTree &DT, BranchInst *BI, PHINode *Merge, 4112 Value *&C, Value *&LHS, Value *&RHS) { 4113 C = BI->getCondition(); 4114 4115 BasicBlockEdge LeftEdge(BI->getParent(), BI->getSuccessor(0)); 4116 BasicBlockEdge RightEdge(BI->getParent(), BI->getSuccessor(1)); 4117 4118 if (!LeftEdge.isSingleEdge()) 4119 return false; 4120 4121 assert(RightEdge.isSingleEdge() && "Follows from LeftEdge.isSingleEdge()"); 4122 4123 Use &LeftUse = Merge->getOperandUse(0); 4124 Use &RightUse = Merge->getOperandUse(1); 4125 4126 if (DT.dominates(LeftEdge, LeftUse) && DT.dominates(RightEdge, RightUse)) { 4127 LHS = LeftUse; 4128 RHS = RightUse; 4129 return true; 4130 } 4131 4132 if (DT.dominates(LeftEdge, RightUse) && DT.dominates(RightEdge, LeftUse)) { 4133 LHS = RightUse; 4134 RHS = LeftUse; 4135 return true; 4136 } 4137 4138 return false; 4139 } 4140 4141 const SCEV *ScalarEvolution::createNodeFromSelectLikePHI(PHINode *PN) { 4142 if (PN->getNumIncomingValues() == 2) { 4143 const Loop *L = LI.getLoopFor(PN->getParent()); 4144 4145 // We don't want to break LCSSA, even in a SCEV expression tree. 4146 for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) 4147 if (LI.getLoopFor(PN->getIncomingBlock(i)) != L) 4148 return nullptr; 4149 4150 // Try to match 4151 // 4152 // br %cond, label %left, label %right 4153 // left: 4154 // br label %merge 4155 // right: 4156 // br label %merge 4157 // merge: 4158 // V = phi [ %x, %left ], [ %y, %right ] 4159 // 4160 // as "select %cond, %x, %y" 4161 4162 BasicBlock *IDom = DT[PN->getParent()]->getIDom()->getBlock(); 4163 assert(IDom && "At least the entry block should dominate PN"); 4164 4165 auto *BI = dyn_cast<BranchInst>(IDom->getTerminator()); 4166 Value *Cond = nullptr, *LHS = nullptr, *RHS = nullptr; 4167 4168 if (BI && BI->isConditional() && 4169 BrPHIToSelect(DT, BI, PN, Cond, LHS, RHS) && 4170 IsAvailableOnEntry(L, DT, getSCEV(LHS), PN->getParent()) && 4171 IsAvailableOnEntry(L, DT, getSCEV(RHS), PN->getParent())) 4172 return createNodeForSelectOrPHI(PN, Cond, LHS, RHS); 4173 } 4174 4175 return nullptr; 4176 } 4177 4178 const SCEV *ScalarEvolution::createNodeForPHI(PHINode *PN) { 4179 if (const SCEV *S = createAddRecFromPHI(PN)) 4180 return S; 4181 4182 if (const SCEV *S = createNodeFromSelectLikePHI(PN)) 4183 return S; 4184 4185 // If the PHI has a single incoming value, follow that value, unless the 4186 // PHI's incoming blocks are in a different loop, in which case doing so 4187 // risks breaking LCSSA form. Instcombine would normally zap these, but 4188 // it doesn't have DominatorTree information, so it may miss cases. 4189 if (Value *V = SimplifyInstruction(PN, getDataLayout(), &TLI, &DT, &AC)) 4190 if (LI.replacementPreservesLCSSAForm(PN, V)) 4191 return getSCEV(V); 4192 4193 // If it's not a loop phi, we can't handle it yet. 4194 return getUnknown(PN); 4195 } 4196 4197 const SCEV *ScalarEvolution::createNodeForSelectOrPHI(Instruction *I, 4198 Value *Cond, 4199 Value *TrueVal, 4200 Value *FalseVal) { 4201 // Handle "constant" branch or select. This can occur for instance when a 4202 // loop pass transforms an inner loop and moves on to process the outer loop. 4203 if (auto *CI = dyn_cast<ConstantInt>(Cond)) 4204 return getSCEV(CI->isOne() ? TrueVal : FalseVal); 4205 4206 // Try to match some simple smax or umax patterns. 4207 auto *ICI = dyn_cast<ICmpInst>(Cond); 4208 if (!ICI) 4209 return getUnknown(I); 4210 4211 Value *LHS = ICI->getOperand(0); 4212 Value *RHS = ICI->getOperand(1); 4213 4214 switch (ICI->getPredicate()) { 4215 case ICmpInst::ICMP_SLT: 4216 case ICmpInst::ICMP_SLE: 4217 std::swap(LHS, RHS); 4218 // fall through 4219 case ICmpInst::ICMP_SGT: 4220 case ICmpInst::ICMP_SGE: 4221 // a >s b ? a+x : b+x -> smax(a, b)+x 4222 // a >s b ? b+x : a+x -> smin(a, b)+x 4223 if (getTypeSizeInBits(LHS->getType()) <= getTypeSizeInBits(I->getType())) { 4224 const SCEV *LS = getNoopOrSignExtend(getSCEV(LHS), I->getType()); 4225 const SCEV *RS = getNoopOrSignExtend(getSCEV(RHS), I->getType()); 4226 const SCEV *LA = getSCEV(TrueVal); 4227 const SCEV *RA = getSCEV(FalseVal); 4228 const SCEV *LDiff = getMinusSCEV(LA, LS); 4229 const SCEV *RDiff = getMinusSCEV(RA, RS); 4230 if (LDiff == RDiff) 4231 return getAddExpr(getSMaxExpr(LS, RS), LDiff); 4232 LDiff = getMinusSCEV(LA, RS); 4233 RDiff = getMinusSCEV(RA, LS); 4234 if (LDiff == RDiff) 4235 return getAddExpr(getSMinExpr(LS, RS), LDiff); 4236 } 4237 break; 4238 case ICmpInst::ICMP_ULT: 4239 case ICmpInst::ICMP_ULE: 4240 std::swap(LHS, RHS); 4241 // fall through 4242 case ICmpInst::ICMP_UGT: 4243 case ICmpInst::ICMP_UGE: 4244 // a >u b ? a+x : b+x -> umax(a, b)+x 4245 // a >u b ? b+x : a+x -> umin(a, b)+x 4246 if (getTypeSizeInBits(LHS->getType()) <= getTypeSizeInBits(I->getType())) { 4247 const SCEV *LS = getNoopOrZeroExtend(getSCEV(LHS), I->getType()); 4248 const SCEV *RS = getNoopOrZeroExtend(getSCEV(RHS), I->getType()); 4249 const SCEV *LA = getSCEV(TrueVal); 4250 const SCEV *RA = getSCEV(FalseVal); 4251 const SCEV *LDiff = getMinusSCEV(LA, LS); 4252 const SCEV *RDiff = getMinusSCEV(RA, RS); 4253 if (LDiff == RDiff) 4254 return getAddExpr(getUMaxExpr(LS, RS), LDiff); 4255 LDiff = getMinusSCEV(LA, RS); 4256 RDiff = getMinusSCEV(RA, LS); 4257 if (LDiff == RDiff) 4258 return getAddExpr(getUMinExpr(LS, RS), LDiff); 4259 } 4260 break; 4261 case ICmpInst::ICMP_NE: 4262 // n != 0 ? n+x : 1+x -> umax(n, 1)+x 4263 if (getTypeSizeInBits(LHS->getType()) <= getTypeSizeInBits(I->getType()) && 4264 isa<ConstantInt>(RHS) && cast<ConstantInt>(RHS)->isZero()) { 4265 const SCEV *One = getOne(I->getType()); 4266 const SCEV *LS = getNoopOrZeroExtend(getSCEV(LHS), I->getType()); 4267 const SCEV *LA = getSCEV(TrueVal); 4268 const SCEV *RA = getSCEV(FalseVal); 4269 const SCEV *LDiff = getMinusSCEV(LA, LS); 4270 const SCEV *RDiff = getMinusSCEV(RA, One); 4271 if (LDiff == RDiff) 4272 return getAddExpr(getUMaxExpr(One, LS), LDiff); 4273 } 4274 break; 4275 case ICmpInst::ICMP_EQ: 4276 // n == 0 ? 1+x : n+x -> umax(n, 1)+x 4277 if (getTypeSizeInBits(LHS->getType()) <= getTypeSizeInBits(I->getType()) && 4278 isa<ConstantInt>(RHS) && cast<ConstantInt>(RHS)->isZero()) { 4279 const SCEV *One = getOne(I->getType()); 4280 const SCEV *LS = getNoopOrZeroExtend(getSCEV(LHS), I->getType()); 4281 const SCEV *LA = getSCEV(TrueVal); 4282 const SCEV *RA = getSCEV(FalseVal); 4283 const SCEV *LDiff = getMinusSCEV(LA, One); 4284 const SCEV *RDiff = getMinusSCEV(RA, LS); 4285 if (LDiff == RDiff) 4286 return getAddExpr(getUMaxExpr(One, LS), LDiff); 4287 } 4288 break; 4289 default: 4290 break; 4291 } 4292 4293 return getUnknown(I); 4294 } 4295 4296 /// createNodeForGEP - Expand GEP instructions into add and multiply 4297 /// operations. This allows them to be analyzed by regular SCEV code. 4298 /// 4299 const SCEV *ScalarEvolution::createNodeForGEP(GEPOperator *GEP) { 4300 // Don't attempt to analyze GEPs over unsized objects. 4301 if (!GEP->getSourceElementType()->isSized()) 4302 return getUnknown(GEP); 4303 4304 SmallVector<const SCEV *, 4> IndexExprs; 4305 for (auto Index = GEP->idx_begin(); Index != GEP->idx_end(); ++Index) 4306 IndexExprs.push_back(getSCEV(*Index)); 4307 return getGEPExpr(GEP->getSourceElementType(), 4308 getSCEV(GEP->getPointerOperand()), 4309 IndexExprs, GEP->isInBounds()); 4310 } 4311 4312 /// GetMinTrailingZeros - Determine the minimum number of zero bits that S is 4313 /// guaranteed to end in (at every loop iteration). It is, at the same time, 4314 /// the minimum number of times S is divisible by 2. For example, given {4,+,8} 4315 /// it returns 2. If S is guaranteed to be 0, it returns the bitwidth of S. 4316 uint32_t 4317 ScalarEvolution::GetMinTrailingZeros(const SCEV *S) { 4318 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(S)) 4319 return C->getAPInt().countTrailingZeros(); 4320 4321 if (const SCEVTruncateExpr *T = dyn_cast<SCEVTruncateExpr>(S)) 4322 return std::min(GetMinTrailingZeros(T->getOperand()), 4323 (uint32_t)getTypeSizeInBits(T->getType())); 4324 4325 if (const SCEVZeroExtendExpr *E = dyn_cast<SCEVZeroExtendExpr>(S)) { 4326 uint32_t OpRes = GetMinTrailingZeros(E->getOperand()); 4327 return OpRes == getTypeSizeInBits(E->getOperand()->getType()) ? 4328 getTypeSizeInBits(E->getType()) : OpRes; 4329 } 4330 4331 if (const SCEVSignExtendExpr *E = dyn_cast<SCEVSignExtendExpr>(S)) { 4332 uint32_t OpRes = GetMinTrailingZeros(E->getOperand()); 4333 return OpRes == getTypeSizeInBits(E->getOperand()->getType()) ? 4334 getTypeSizeInBits(E->getType()) : OpRes; 4335 } 4336 4337 if (const SCEVAddExpr *A = dyn_cast<SCEVAddExpr>(S)) { 4338 // The result is the min of all operands results. 4339 uint32_t MinOpRes = GetMinTrailingZeros(A->getOperand(0)); 4340 for (unsigned i = 1, e = A->getNumOperands(); MinOpRes && i != e; ++i) 4341 MinOpRes = std::min(MinOpRes, GetMinTrailingZeros(A->getOperand(i))); 4342 return MinOpRes; 4343 } 4344 4345 if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(S)) { 4346 // The result is the sum of all operands results. 4347 uint32_t SumOpRes = GetMinTrailingZeros(M->getOperand(0)); 4348 uint32_t BitWidth = getTypeSizeInBits(M->getType()); 4349 for (unsigned i = 1, e = M->getNumOperands(); 4350 SumOpRes != BitWidth && i != e; ++i) 4351 SumOpRes = std::min(SumOpRes + GetMinTrailingZeros(M->getOperand(i)), 4352 BitWidth); 4353 return SumOpRes; 4354 } 4355 4356 if (const SCEVAddRecExpr *A = dyn_cast<SCEVAddRecExpr>(S)) { 4357 // The result is the min of all operands results. 4358 uint32_t MinOpRes = GetMinTrailingZeros(A->getOperand(0)); 4359 for (unsigned i = 1, e = A->getNumOperands(); MinOpRes && i != e; ++i) 4360 MinOpRes = std::min(MinOpRes, GetMinTrailingZeros(A->getOperand(i))); 4361 return MinOpRes; 4362 } 4363 4364 if (const SCEVSMaxExpr *M = dyn_cast<SCEVSMaxExpr>(S)) { 4365 // The result is the min of all operands results. 4366 uint32_t MinOpRes = GetMinTrailingZeros(M->getOperand(0)); 4367 for (unsigned i = 1, e = M->getNumOperands(); MinOpRes && i != e; ++i) 4368 MinOpRes = std::min(MinOpRes, GetMinTrailingZeros(M->getOperand(i))); 4369 return MinOpRes; 4370 } 4371 4372 if (const SCEVUMaxExpr *M = dyn_cast<SCEVUMaxExpr>(S)) { 4373 // The result is the min of all operands results. 4374 uint32_t MinOpRes = GetMinTrailingZeros(M->getOperand(0)); 4375 for (unsigned i = 1, e = M->getNumOperands(); MinOpRes && i != e; ++i) 4376 MinOpRes = std::min(MinOpRes, GetMinTrailingZeros(M->getOperand(i))); 4377 return MinOpRes; 4378 } 4379 4380 if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(S)) { 4381 // For a SCEVUnknown, ask ValueTracking. 4382 unsigned BitWidth = getTypeSizeInBits(U->getType()); 4383 APInt Zeros(BitWidth, 0), Ones(BitWidth, 0); 4384 computeKnownBits(U->getValue(), Zeros, Ones, getDataLayout(), 0, &AC, 4385 nullptr, &DT); 4386 return Zeros.countTrailingOnes(); 4387 } 4388 4389 // SCEVUDivExpr 4390 return 0; 4391 } 4392 4393 /// GetRangeFromMetadata - Helper method to assign a range to V from 4394 /// metadata present in the IR. 4395 static Optional<ConstantRange> GetRangeFromMetadata(Value *V) { 4396 if (Instruction *I = dyn_cast<Instruction>(V)) 4397 if (MDNode *MD = I->getMetadata(LLVMContext::MD_range)) 4398 return getConstantRangeFromMetadata(*MD); 4399 4400 return None; 4401 } 4402 4403 /// getRange - Determine the range for a particular SCEV. If SignHint is 4404 /// HINT_RANGE_UNSIGNED (resp. HINT_RANGE_SIGNED) then getRange prefers ranges 4405 /// with a "cleaner" unsigned (resp. signed) representation. 4406 /// 4407 ConstantRange 4408 ScalarEvolution::getRange(const SCEV *S, 4409 ScalarEvolution::RangeSignHint SignHint) { 4410 DenseMap<const SCEV *, ConstantRange> &Cache = 4411 SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED ? UnsignedRanges 4412 : SignedRanges; 4413 4414 // See if we've computed this range already. 4415 DenseMap<const SCEV *, ConstantRange>::iterator I = Cache.find(S); 4416 if (I != Cache.end()) 4417 return I->second; 4418 4419 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(S)) 4420 return setRange(C, SignHint, ConstantRange(C->getAPInt())); 4421 4422 unsigned BitWidth = getTypeSizeInBits(S->getType()); 4423 ConstantRange ConservativeResult(BitWidth, /*isFullSet=*/true); 4424 4425 // If the value has known zeros, the maximum value will have those known zeros 4426 // as well. 4427 uint32_t TZ = GetMinTrailingZeros(S); 4428 if (TZ != 0) { 4429 if (SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED) 4430 ConservativeResult = 4431 ConstantRange(APInt::getMinValue(BitWidth), 4432 APInt::getMaxValue(BitWidth).lshr(TZ).shl(TZ) + 1); 4433 else 4434 ConservativeResult = ConstantRange( 4435 APInt::getSignedMinValue(BitWidth), 4436 APInt::getSignedMaxValue(BitWidth).ashr(TZ).shl(TZ) + 1); 4437 } 4438 4439 if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) { 4440 ConstantRange X = getRange(Add->getOperand(0), SignHint); 4441 for (unsigned i = 1, e = Add->getNumOperands(); i != e; ++i) 4442 X = X.add(getRange(Add->getOperand(i), SignHint)); 4443 return setRange(Add, SignHint, ConservativeResult.intersectWith(X)); 4444 } 4445 4446 if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(S)) { 4447 ConstantRange X = getRange(Mul->getOperand(0), SignHint); 4448 for (unsigned i = 1, e = Mul->getNumOperands(); i != e; ++i) 4449 X = X.multiply(getRange(Mul->getOperand(i), SignHint)); 4450 return setRange(Mul, SignHint, ConservativeResult.intersectWith(X)); 4451 } 4452 4453 if (const SCEVSMaxExpr *SMax = dyn_cast<SCEVSMaxExpr>(S)) { 4454 ConstantRange X = getRange(SMax->getOperand(0), SignHint); 4455 for (unsigned i = 1, e = SMax->getNumOperands(); i != e; ++i) 4456 X = X.smax(getRange(SMax->getOperand(i), SignHint)); 4457 return setRange(SMax, SignHint, ConservativeResult.intersectWith(X)); 4458 } 4459 4460 if (const SCEVUMaxExpr *UMax = dyn_cast<SCEVUMaxExpr>(S)) { 4461 ConstantRange X = getRange(UMax->getOperand(0), SignHint); 4462 for (unsigned i = 1, e = UMax->getNumOperands(); i != e; ++i) 4463 X = X.umax(getRange(UMax->getOperand(i), SignHint)); 4464 return setRange(UMax, SignHint, ConservativeResult.intersectWith(X)); 4465 } 4466 4467 if (const SCEVUDivExpr *UDiv = dyn_cast<SCEVUDivExpr>(S)) { 4468 ConstantRange X = getRange(UDiv->getLHS(), SignHint); 4469 ConstantRange Y = getRange(UDiv->getRHS(), SignHint); 4470 return setRange(UDiv, SignHint, 4471 ConservativeResult.intersectWith(X.udiv(Y))); 4472 } 4473 4474 if (const SCEVZeroExtendExpr *ZExt = dyn_cast<SCEVZeroExtendExpr>(S)) { 4475 ConstantRange X = getRange(ZExt->getOperand(), SignHint); 4476 return setRange(ZExt, SignHint, 4477 ConservativeResult.intersectWith(X.zeroExtend(BitWidth))); 4478 } 4479 4480 if (const SCEVSignExtendExpr *SExt = dyn_cast<SCEVSignExtendExpr>(S)) { 4481 ConstantRange X = getRange(SExt->getOperand(), SignHint); 4482 return setRange(SExt, SignHint, 4483 ConservativeResult.intersectWith(X.signExtend(BitWidth))); 4484 } 4485 4486 if (const SCEVTruncateExpr *Trunc = dyn_cast<SCEVTruncateExpr>(S)) { 4487 ConstantRange X = getRange(Trunc->getOperand(), SignHint); 4488 return setRange(Trunc, SignHint, 4489 ConservativeResult.intersectWith(X.truncate(BitWidth))); 4490 } 4491 4492 if (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(S)) { 4493 // If there's no unsigned wrap, the value will never be less than its 4494 // initial value. 4495 if (AddRec->hasNoUnsignedWrap()) 4496 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(AddRec->getStart())) 4497 if (!C->getValue()->isZero()) 4498 ConservativeResult = ConservativeResult.intersectWith( 4499 ConstantRange(C->getAPInt(), APInt(BitWidth, 0))); 4500 4501 // If there's no signed wrap, and all the operands have the same sign or 4502 // zero, the value won't ever change sign. 4503 if (AddRec->hasNoSignedWrap()) { 4504 bool AllNonNeg = true; 4505 bool AllNonPos = true; 4506 for (unsigned i = 0, e = AddRec->getNumOperands(); i != e; ++i) { 4507 if (!isKnownNonNegative(AddRec->getOperand(i))) AllNonNeg = false; 4508 if (!isKnownNonPositive(AddRec->getOperand(i))) AllNonPos = false; 4509 } 4510 if (AllNonNeg) 4511 ConservativeResult = ConservativeResult.intersectWith( 4512 ConstantRange(APInt(BitWidth, 0), 4513 APInt::getSignedMinValue(BitWidth))); 4514 else if (AllNonPos) 4515 ConservativeResult = ConservativeResult.intersectWith( 4516 ConstantRange(APInt::getSignedMinValue(BitWidth), 4517 APInt(BitWidth, 1))); 4518 } 4519 4520 // TODO: non-affine addrec 4521 if (AddRec->isAffine()) { 4522 const SCEV *MaxBECount = getMaxBackedgeTakenCount(AddRec->getLoop()); 4523 if (!isa<SCEVCouldNotCompute>(MaxBECount) && 4524 getTypeSizeInBits(MaxBECount->getType()) <= BitWidth) { 4525 auto RangeFromAffine = getRangeForAffineAR( 4526 AddRec->getStart(), AddRec->getStepRecurrence(*this), MaxBECount, 4527 BitWidth); 4528 if (!RangeFromAffine.isFullSet()) 4529 ConservativeResult = 4530 ConservativeResult.intersectWith(RangeFromAffine); 4531 4532 auto RangeFromFactoring = getRangeViaFactoring( 4533 AddRec->getStart(), AddRec->getStepRecurrence(*this), MaxBECount, 4534 BitWidth); 4535 if (!RangeFromFactoring.isFullSet()) 4536 ConservativeResult = 4537 ConservativeResult.intersectWith(RangeFromFactoring); 4538 } 4539 } 4540 4541 return setRange(AddRec, SignHint, ConservativeResult); 4542 } 4543 4544 if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(S)) { 4545 // Check if the IR explicitly contains !range metadata. 4546 Optional<ConstantRange> MDRange = GetRangeFromMetadata(U->getValue()); 4547 if (MDRange.hasValue()) 4548 ConservativeResult = ConservativeResult.intersectWith(MDRange.getValue()); 4549 4550 // Split here to avoid paying the compile-time cost of calling both 4551 // computeKnownBits and ComputeNumSignBits. This restriction can be lifted 4552 // if needed. 4553 const DataLayout &DL = getDataLayout(); 4554 if (SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED) { 4555 // For a SCEVUnknown, ask ValueTracking. 4556 APInt Zeros(BitWidth, 0), Ones(BitWidth, 0); 4557 computeKnownBits(U->getValue(), Zeros, Ones, DL, 0, &AC, nullptr, &DT); 4558 if (Ones != ~Zeros + 1) 4559 ConservativeResult = 4560 ConservativeResult.intersectWith(ConstantRange(Ones, ~Zeros + 1)); 4561 } else { 4562 assert(SignHint == ScalarEvolution::HINT_RANGE_SIGNED && 4563 "generalize as needed!"); 4564 unsigned NS = ComputeNumSignBits(U->getValue(), DL, 0, &AC, nullptr, &DT); 4565 if (NS > 1) 4566 ConservativeResult = ConservativeResult.intersectWith( 4567 ConstantRange(APInt::getSignedMinValue(BitWidth).ashr(NS - 1), 4568 APInt::getSignedMaxValue(BitWidth).ashr(NS - 1) + 1)); 4569 } 4570 4571 return setRange(U, SignHint, ConservativeResult); 4572 } 4573 4574 return setRange(S, SignHint, ConservativeResult); 4575 } 4576 4577 ConstantRange ScalarEvolution::getRangeForAffineAR(const SCEV *Start, 4578 const SCEV *Step, 4579 const SCEV *MaxBECount, 4580 unsigned BitWidth) { 4581 assert(!isa<SCEVCouldNotCompute>(MaxBECount) && 4582 getTypeSizeInBits(MaxBECount->getType()) <= BitWidth && 4583 "Precondition!"); 4584 4585 ConstantRange Result(BitWidth, /* isFullSet = */ true); 4586 4587 // Check for overflow. This must be done with ConstantRange arithmetic 4588 // because we could be called from within the ScalarEvolution overflow 4589 // checking code. 4590 4591 MaxBECount = getNoopOrZeroExtend(MaxBECount, Start->getType()); 4592 ConstantRange MaxBECountRange = getUnsignedRange(MaxBECount); 4593 ConstantRange ZExtMaxBECountRange = 4594 MaxBECountRange.zextOrTrunc(BitWidth * 2 + 1); 4595 4596 ConstantRange StepSRange = getSignedRange(Step); 4597 ConstantRange SExtStepSRange = StepSRange.sextOrTrunc(BitWidth * 2 + 1); 4598 4599 ConstantRange StartURange = getUnsignedRange(Start); 4600 ConstantRange EndURange = 4601 StartURange.add(MaxBECountRange.multiply(StepSRange)); 4602 4603 // Check for unsigned overflow. 4604 ConstantRange ZExtStartURange = StartURange.zextOrTrunc(BitWidth * 2 + 1); 4605 ConstantRange ZExtEndURange = EndURange.zextOrTrunc(BitWidth * 2 + 1); 4606 if (ZExtStartURange.add(ZExtMaxBECountRange.multiply(SExtStepSRange)) == 4607 ZExtEndURange) { 4608 APInt Min = APIntOps::umin(StartURange.getUnsignedMin(), 4609 EndURange.getUnsignedMin()); 4610 APInt Max = APIntOps::umax(StartURange.getUnsignedMax(), 4611 EndURange.getUnsignedMax()); 4612 bool IsFullRange = Min.isMinValue() && Max.isMaxValue(); 4613 if (!IsFullRange) 4614 Result = 4615 Result.intersectWith(ConstantRange(Min, Max + 1)); 4616 } 4617 4618 ConstantRange StartSRange = getSignedRange(Start); 4619 ConstantRange EndSRange = 4620 StartSRange.add(MaxBECountRange.multiply(StepSRange)); 4621 4622 // Check for signed overflow. This must be done with ConstantRange 4623 // arithmetic because we could be called from within the ScalarEvolution 4624 // overflow checking code. 4625 ConstantRange SExtStartSRange = StartSRange.sextOrTrunc(BitWidth * 2 + 1); 4626 ConstantRange SExtEndSRange = EndSRange.sextOrTrunc(BitWidth * 2 + 1); 4627 if (SExtStartSRange.add(ZExtMaxBECountRange.multiply(SExtStepSRange)) == 4628 SExtEndSRange) { 4629 APInt Min = 4630 APIntOps::smin(StartSRange.getSignedMin(), EndSRange.getSignedMin()); 4631 APInt Max = 4632 APIntOps::smax(StartSRange.getSignedMax(), EndSRange.getSignedMax()); 4633 bool IsFullRange = Min.isMinSignedValue() && Max.isMaxSignedValue(); 4634 if (!IsFullRange) 4635 Result = 4636 Result.intersectWith(ConstantRange(Min, Max + 1)); 4637 } 4638 4639 return Result; 4640 } 4641 4642 ConstantRange ScalarEvolution::getRangeViaFactoring(const SCEV *Start, 4643 const SCEV *Step, 4644 const SCEV *MaxBECount, 4645 unsigned BitWidth) { 4646 // RangeOf({C?A:B,+,C?P:Q}) == RangeOf(C?{A,+,P}:{B,+,Q}) 4647 // == RangeOf({A,+,P}) union RangeOf({B,+,Q}) 4648 4649 struct SelectPattern { 4650 Value *Condition = nullptr; 4651 APInt TrueValue; 4652 APInt FalseValue; 4653 4654 explicit SelectPattern(ScalarEvolution &SE, unsigned BitWidth, 4655 const SCEV *S) { 4656 Optional<unsigned> CastOp; 4657 APInt Offset(BitWidth, 0); 4658 4659 assert(SE.getTypeSizeInBits(S->getType()) == BitWidth && 4660 "Should be!"); 4661 4662 // Peel off a constant offset: 4663 if (auto *SA = dyn_cast<SCEVAddExpr>(S)) { 4664 // In the future we could consider being smarter here and handle 4665 // {Start+Step,+,Step} too. 4666 if (SA->getNumOperands() != 2 || !isa<SCEVConstant>(SA->getOperand(0))) 4667 return; 4668 4669 Offset = cast<SCEVConstant>(SA->getOperand(0))->getAPInt(); 4670 S = SA->getOperand(1); 4671 } 4672 4673 // Peel off a cast operation 4674 if (auto *SCast = dyn_cast<SCEVCastExpr>(S)) { 4675 CastOp = SCast->getSCEVType(); 4676 S = SCast->getOperand(); 4677 } 4678 4679 using namespace llvm::PatternMatch; 4680 4681 auto *SU = dyn_cast<SCEVUnknown>(S); 4682 const APInt *TrueVal, *FalseVal; 4683 if (!SU || 4684 !match(SU->getValue(), m_Select(m_Value(Condition), m_APInt(TrueVal), 4685 m_APInt(FalseVal)))) { 4686 Condition = nullptr; 4687 return; 4688 } 4689 4690 TrueValue = *TrueVal; 4691 FalseValue = *FalseVal; 4692 4693 // Re-apply the cast we peeled off earlier 4694 if (CastOp.hasValue()) 4695 switch (*CastOp) { 4696 default: 4697 llvm_unreachable("Unknown SCEV cast type!"); 4698 4699 case scTruncate: 4700 TrueValue = TrueValue.trunc(BitWidth); 4701 FalseValue = FalseValue.trunc(BitWidth); 4702 break; 4703 case scZeroExtend: 4704 TrueValue = TrueValue.zext(BitWidth); 4705 FalseValue = FalseValue.zext(BitWidth); 4706 break; 4707 case scSignExtend: 4708 TrueValue = TrueValue.sext(BitWidth); 4709 FalseValue = FalseValue.sext(BitWidth); 4710 break; 4711 } 4712 4713 // Re-apply the constant offset we peeled off earlier 4714 TrueValue += Offset; 4715 FalseValue += Offset; 4716 } 4717 4718 bool isRecognized() { return Condition != nullptr; } 4719 }; 4720 4721 SelectPattern StartPattern(*this, BitWidth, Start); 4722 if (!StartPattern.isRecognized()) 4723 return ConstantRange(BitWidth, /* isFullSet = */ true); 4724 4725 SelectPattern StepPattern(*this, BitWidth, Step); 4726 if (!StepPattern.isRecognized()) 4727 return ConstantRange(BitWidth, /* isFullSet = */ true); 4728 4729 if (StartPattern.Condition != StepPattern.Condition) { 4730 // We don't handle this case today; but we could, by considering four 4731 // possibilities below instead of two. I'm not sure if there are cases where 4732 // that will help over what getRange already does, though. 4733 return ConstantRange(BitWidth, /* isFullSet = */ true); 4734 } 4735 4736 // NB! Calling ScalarEvolution::getConstant is fine, but we should not try to 4737 // construct arbitrary general SCEV expressions here. This function is called 4738 // from deep in the call stack, and calling getSCEV (on a sext instruction, 4739 // say) can end up caching a suboptimal value. 4740 4741 // FIXME: without the explicit `this` receiver below, MSVC errors out with 4742 // C2352 and C2512 (otherwise it isn't needed). 4743 4744 const SCEV *TrueStart = this->getConstant(StartPattern.TrueValue); 4745 const SCEV *TrueStep = this->getConstant(StepPattern.TrueValue); 4746 const SCEV *FalseStart = this->getConstant(StartPattern.FalseValue); 4747 const SCEV *FalseStep = this->getConstant(StepPattern.FalseValue); 4748 4749 ConstantRange TrueRange = 4750 this->getRangeForAffineAR(TrueStart, TrueStep, MaxBECount, BitWidth); 4751 ConstantRange FalseRange = 4752 this->getRangeForAffineAR(FalseStart, FalseStep, MaxBECount, BitWidth); 4753 4754 return TrueRange.unionWith(FalseRange); 4755 } 4756 4757 SCEV::NoWrapFlags ScalarEvolution::getNoWrapFlagsFromUB(const Value *V) { 4758 if (isa<ConstantExpr>(V)) return SCEV::FlagAnyWrap; 4759 const BinaryOperator *BinOp = cast<BinaryOperator>(V); 4760 4761 // Return early if there are no flags to propagate to the SCEV. 4762 SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap; 4763 if (BinOp->hasNoUnsignedWrap()) 4764 Flags = ScalarEvolution::setFlags(Flags, SCEV::FlagNUW); 4765 if (BinOp->hasNoSignedWrap()) 4766 Flags = ScalarEvolution::setFlags(Flags, SCEV::FlagNSW); 4767 if (Flags == SCEV::FlagAnyWrap) 4768 return SCEV::FlagAnyWrap; 4769 4770 return isSCEVExprNeverPoison(BinOp) ? Flags : SCEV::FlagAnyWrap; 4771 } 4772 4773 bool ScalarEvolution::isSCEVExprNeverPoison(const Instruction *I) { 4774 // Here we check that I is in the header of the innermost loop containing I, 4775 // since we only deal with instructions in the loop header. The actual loop we 4776 // need to check later will come from an add recurrence, but getting that 4777 // requires computing the SCEV of the operands, which can be expensive. This 4778 // check we can do cheaply to rule out some cases early. 4779 Loop *InnermostContainingLoop = LI.getLoopFor(I->getParent()); 4780 if (InnermostContainingLoop == nullptr || 4781 InnermostContainingLoop->getHeader() != I->getParent()) 4782 return false; 4783 4784 // Only proceed if we can prove that I does not yield poison. 4785 if (!isKnownNotFullPoison(I)) return false; 4786 4787 // At this point we know that if I is executed, then it does not wrap 4788 // according to at least one of NSW or NUW. If I is not executed, then we do 4789 // not know if the calculation that I represents would wrap. Multiple 4790 // instructions can map to the same SCEV. If we apply NSW or NUW from I to 4791 // the SCEV, we must guarantee no wrapping for that SCEV also when it is 4792 // derived from other instructions that map to the same SCEV. We cannot make 4793 // that guarantee for cases where I is not executed. So we need to find the 4794 // loop that I is considered in relation to and prove that I is executed for 4795 // every iteration of that loop. That implies that the value that I 4796 // calculates does not wrap anywhere in the loop, so then we can apply the 4797 // flags to the SCEV. 4798 // 4799 // We check isLoopInvariant to disambiguate in case we are adding recurrences 4800 // from different loops, so that we know which loop to prove that I is 4801 // executed in. 4802 for (unsigned OpIndex = 0; OpIndex < I->getNumOperands(); ++OpIndex) { 4803 const SCEV *Op = getSCEV(I->getOperand(OpIndex)); 4804 if (auto *AddRec = dyn_cast<SCEVAddRecExpr>(Op)) { 4805 bool AllOtherOpsLoopInvariant = true; 4806 for (unsigned OtherOpIndex = 0; OtherOpIndex < I->getNumOperands(); 4807 ++OtherOpIndex) { 4808 if (OtherOpIndex != OpIndex) { 4809 const SCEV *OtherOp = getSCEV(I->getOperand(OtherOpIndex)); 4810 if (!isLoopInvariant(OtherOp, AddRec->getLoop())) { 4811 AllOtherOpsLoopInvariant = false; 4812 break; 4813 } 4814 } 4815 } 4816 if (AllOtherOpsLoopInvariant && 4817 isGuaranteedToExecuteForEveryIteration(I, AddRec->getLoop())) 4818 return true; 4819 } 4820 } 4821 return false; 4822 } 4823 4824 /// createSCEV - We know that there is no SCEV for the specified value. Analyze 4825 /// the expression. 4826 /// 4827 const SCEV *ScalarEvolution::createSCEV(Value *V) { 4828 if (!isSCEVable(V->getType())) 4829 return getUnknown(V); 4830 4831 if (Instruction *I = dyn_cast<Instruction>(V)) { 4832 // Don't attempt to analyze instructions in blocks that aren't 4833 // reachable. Such instructions don't matter, and they aren't required 4834 // to obey basic rules for definitions dominating uses which this 4835 // analysis depends on. 4836 if (!DT.isReachableFromEntry(I->getParent())) 4837 return getUnknown(V); 4838 } else if (ConstantInt *CI = dyn_cast<ConstantInt>(V)) 4839 return getConstant(CI); 4840 else if (isa<ConstantPointerNull>(V)) 4841 return getZero(V->getType()); 4842 else if (GlobalAlias *GA = dyn_cast<GlobalAlias>(V)) 4843 return GA->isInterposable() ? getUnknown(V) : getSCEV(GA->getAliasee()); 4844 else if (!isa<ConstantExpr>(V)) 4845 return getUnknown(V); 4846 4847 Operator *U = cast<Operator>(V); 4848 if (auto BO = MatchBinaryOp(U)) { 4849 switch (BO->Opcode) { 4850 case Instruction::Add: { 4851 // The simple thing to do would be to just call getSCEV on both operands 4852 // and call getAddExpr with the result. However if we're looking at a 4853 // bunch of things all added together, this can be quite inefficient, 4854 // because it leads to N-1 getAddExpr calls for N ultimate operands. 4855 // Instead, gather up all the operands and make a single getAddExpr call. 4856 // LLVM IR canonical form means we need only traverse the left operands. 4857 SmallVector<const SCEV *, 4> AddOps; 4858 do { 4859 if (BO->Op) { 4860 if (auto *OpSCEV = getExistingSCEV(BO->Op)) { 4861 AddOps.push_back(OpSCEV); 4862 break; 4863 } 4864 4865 // If a NUW or NSW flag can be applied to the SCEV for this 4866 // addition, then compute the SCEV for this addition by itself 4867 // with a separate call to getAddExpr. We need to do that 4868 // instead of pushing the operands of the addition onto AddOps, 4869 // since the flags are only known to apply to this particular 4870 // addition - they may not apply to other additions that can be 4871 // formed with operands from AddOps. 4872 const SCEV *RHS = getSCEV(BO->RHS); 4873 SCEV::NoWrapFlags Flags = getNoWrapFlagsFromUB(BO->Op); 4874 if (Flags != SCEV::FlagAnyWrap) { 4875 const SCEV *LHS = getSCEV(BO->LHS); 4876 if (BO->Opcode == Instruction::Sub) 4877 AddOps.push_back(getMinusSCEV(LHS, RHS, Flags)); 4878 else 4879 AddOps.push_back(getAddExpr(LHS, RHS, Flags)); 4880 break; 4881 } 4882 } 4883 4884 if (BO->Opcode == Instruction::Sub) 4885 AddOps.push_back(getNegativeSCEV(getSCEV(BO->RHS))); 4886 else 4887 AddOps.push_back(getSCEV(BO->RHS)); 4888 4889 auto NewBO = MatchBinaryOp(BO->LHS); 4890 if (!NewBO || (NewBO->Opcode != Instruction::Add && 4891 NewBO->Opcode != Instruction::Sub)) { 4892 AddOps.push_back(getSCEV(BO->LHS)); 4893 break; 4894 } 4895 BO = NewBO; 4896 } while (true); 4897 4898 return getAddExpr(AddOps); 4899 } 4900 4901 case Instruction::Mul: { 4902 SmallVector<const SCEV *, 4> MulOps; 4903 do { 4904 if (BO->Op) { 4905 if (auto *OpSCEV = getExistingSCEV(BO->Op)) { 4906 MulOps.push_back(OpSCEV); 4907 break; 4908 } 4909 4910 SCEV::NoWrapFlags Flags = getNoWrapFlagsFromUB(BO->Op); 4911 if (Flags != SCEV::FlagAnyWrap) { 4912 MulOps.push_back( 4913 getMulExpr(getSCEV(BO->LHS), getSCEV(BO->RHS), Flags)); 4914 break; 4915 } 4916 } 4917 4918 MulOps.push_back(getSCEV(BO->RHS)); 4919 auto NewBO = MatchBinaryOp(BO->LHS); 4920 if (!NewBO || NewBO->Opcode != Instruction::Mul) { 4921 MulOps.push_back(getSCEV(BO->LHS)); 4922 break; 4923 } 4924 BO = NewBO; 4925 } while (true); 4926 4927 return getMulExpr(MulOps); 4928 } 4929 case Instruction::UDiv: 4930 return getUDivExpr(getSCEV(BO->LHS), getSCEV(BO->RHS)); 4931 case Instruction::Sub: { 4932 SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap; 4933 if (BO->Op) 4934 Flags = getNoWrapFlagsFromUB(BO->Op); 4935 return getMinusSCEV(getSCEV(BO->LHS), getSCEV(BO->RHS), Flags); 4936 } 4937 case Instruction::And: 4938 // For an expression like x&255 that merely masks off the high bits, 4939 // use zext(trunc(x)) as the SCEV expression. 4940 if (ConstantInt *CI = dyn_cast<ConstantInt>(BO->RHS)) { 4941 if (CI->isNullValue()) 4942 return getSCEV(BO->RHS); 4943 if (CI->isAllOnesValue()) 4944 return getSCEV(BO->LHS); 4945 const APInt &A = CI->getValue(); 4946 4947 // Instcombine's ShrinkDemandedConstant may strip bits out of 4948 // constants, obscuring what would otherwise be a low-bits mask. 4949 // Use computeKnownBits to compute what ShrinkDemandedConstant 4950 // knew about to reconstruct a low-bits mask value. 4951 unsigned LZ = A.countLeadingZeros(); 4952 unsigned TZ = A.countTrailingZeros(); 4953 unsigned BitWidth = A.getBitWidth(); 4954 APInt KnownZero(BitWidth, 0), KnownOne(BitWidth, 0); 4955 computeKnownBits(BO->LHS, KnownZero, KnownOne, getDataLayout(), 4956 0, &AC, nullptr, &DT); 4957 4958 APInt EffectiveMask = 4959 APInt::getLowBitsSet(BitWidth, BitWidth - LZ - TZ).shl(TZ); 4960 if ((LZ != 0 || TZ != 0) && !((~A & ~KnownZero) & EffectiveMask)) { 4961 const SCEV *MulCount = getConstant(ConstantInt::get( 4962 getContext(), APInt::getOneBitSet(BitWidth, TZ))); 4963 return getMulExpr( 4964 getZeroExtendExpr( 4965 getTruncateExpr( 4966 getUDivExactExpr(getSCEV(BO->LHS), MulCount), 4967 IntegerType::get(getContext(), BitWidth - LZ - TZ)), 4968 BO->LHS->getType()), 4969 MulCount); 4970 } 4971 } 4972 break; 4973 4974 case Instruction::Or: 4975 // If the RHS of the Or is a constant, we may have something like: 4976 // X*4+1 which got turned into X*4|1. Handle this as an Add so loop 4977 // optimizations will transparently handle this case. 4978 // 4979 // In order for this transformation to be safe, the LHS must be of the 4980 // form X*(2^n) and the Or constant must be less than 2^n. 4981 if (ConstantInt *CI = dyn_cast<ConstantInt>(BO->RHS)) { 4982 const SCEV *LHS = getSCEV(BO->LHS); 4983 const APInt &CIVal = CI->getValue(); 4984 if (GetMinTrailingZeros(LHS) >= 4985 (CIVal.getBitWidth() - CIVal.countLeadingZeros())) { 4986 // Build a plain add SCEV. 4987 const SCEV *S = getAddExpr(LHS, getSCEV(CI)); 4988 // If the LHS of the add was an addrec and it has no-wrap flags, 4989 // transfer the no-wrap flags, since an or won't introduce a wrap. 4990 if (const SCEVAddRecExpr *NewAR = dyn_cast<SCEVAddRecExpr>(S)) { 4991 const SCEVAddRecExpr *OldAR = cast<SCEVAddRecExpr>(LHS); 4992 const_cast<SCEVAddRecExpr *>(NewAR)->setNoWrapFlags( 4993 OldAR->getNoWrapFlags()); 4994 } 4995 return S; 4996 } 4997 } 4998 break; 4999 5000 case Instruction::Xor: 5001 if (ConstantInt *CI = dyn_cast<ConstantInt>(BO->RHS)) { 5002 // If the RHS of xor is -1, then this is a not operation. 5003 if (CI->isAllOnesValue()) 5004 return getNotSCEV(getSCEV(BO->LHS)); 5005 5006 // Model xor(and(x, C), C) as and(~x, C), if C is a low-bits mask. 5007 // This is a variant of the check for xor with -1, and it handles 5008 // the case where instcombine has trimmed non-demanded bits out 5009 // of an xor with -1. 5010 if (auto *LBO = dyn_cast<BinaryOperator>(BO->LHS)) 5011 if (ConstantInt *LCI = dyn_cast<ConstantInt>(LBO->getOperand(1))) 5012 if (LBO->getOpcode() == Instruction::And && 5013 LCI->getValue() == CI->getValue()) 5014 if (const SCEVZeroExtendExpr *Z = 5015 dyn_cast<SCEVZeroExtendExpr>(getSCEV(BO->LHS))) { 5016 Type *UTy = BO->LHS->getType(); 5017 const SCEV *Z0 = Z->getOperand(); 5018 Type *Z0Ty = Z0->getType(); 5019 unsigned Z0TySize = getTypeSizeInBits(Z0Ty); 5020 5021 // If C is a low-bits mask, the zero extend is serving to 5022 // mask off the high bits. Complement the operand and 5023 // re-apply the zext. 5024 if (APIntOps::isMask(Z0TySize, CI->getValue())) 5025 return getZeroExtendExpr(getNotSCEV(Z0), UTy); 5026 5027 // If C is a single bit, it may be in the sign-bit position 5028 // before the zero-extend. In this case, represent the xor 5029 // using an add, which is equivalent, and re-apply the zext. 5030 APInt Trunc = CI->getValue().trunc(Z0TySize); 5031 if (Trunc.zext(getTypeSizeInBits(UTy)) == CI->getValue() && 5032 Trunc.isSignBit()) 5033 return getZeroExtendExpr(getAddExpr(Z0, getConstant(Trunc)), 5034 UTy); 5035 } 5036 } 5037 break; 5038 5039 case Instruction::Shl: 5040 // Turn shift left of a constant amount into a multiply. 5041 if (ConstantInt *SA = dyn_cast<ConstantInt>(BO->RHS)) { 5042 uint32_t BitWidth = cast<IntegerType>(SA->getType())->getBitWidth(); 5043 5044 // If the shift count is not less than the bitwidth, the result of 5045 // the shift is undefined. Don't try to analyze it, because the 5046 // resolution chosen here may differ from the resolution chosen in 5047 // other parts of the compiler. 5048 if (SA->getValue().uge(BitWidth)) 5049 break; 5050 5051 // It is currently not resolved how to interpret NSW for left 5052 // shift by BitWidth - 1, so we avoid applying flags in that 5053 // case. Remove this check (or this comment) once the situation 5054 // is resolved. See 5055 // http://lists.llvm.org/pipermail/llvm-dev/2015-April/084195.html 5056 // and http://reviews.llvm.org/D8890 . 5057 auto Flags = SCEV::FlagAnyWrap; 5058 if (BO->Op && SA->getValue().ult(BitWidth - 1)) 5059 Flags = getNoWrapFlagsFromUB(BO->Op); 5060 5061 Constant *X = ConstantInt::get(getContext(), 5062 APInt::getOneBitSet(BitWidth, SA->getZExtValue())); 5063 return getMulExpr(getSCEV(BO->LHS), getSCEV(X), Flags); 5064 } 5065 break; 5066 5067 case Instruction::AShr: 5068 // For a two-shift sext-inreg, use sext(trunc(x)) as the SCEV expression. 5069 if (ConstantInt *CI = dyn_cast<ConstantInt>(BO->RHS)) 5070 if (Operator *L = dyn_cast<Operator>(BO->LHS)) 5071 if (L->getOpcode() == Instruction::Shl && 5072 L->getOperand(1) == BO->RHS) { 5073 uint64_t BitWidth = getTypeSizeInBits(BO->LHS->getType()); 5074 5075 // If the shift count is not less than the bitwidth, the result of 5076 // the shift is undefined. Don't try to analyze it, because the 5077 // resolution chosen here may differ from the resolution chosen in 5078 // other parts of the compiler. 5079 if (CI->getValue().uge(BitWidth)) 5080 break; 5081 5082 uint64_t Amt = BitWidth - CI->getZExtValue(); 5083 if (Amt == BitWidth) 5084 return getSCEV(L->getOperand(0)); // shift by zero --> noop 5085 return getSignExtendExpr( 5086 getTruncateExpr(getSCEV(L->getOperand(0)), 5087 IntegerType::get(getContext(), Amt)), 5088 BO->LHS->getType()); 5089 } 5090 break; 5091 } 5092 } 5093 5094 switch (U->getOpcode()) { 5095 case Instruction::Trunc: 5096 return getTruncateExpr(getSCEV(U->getOperand(0)), U->getType()); 5097 5098 case Instruction::ZExt: 5099 return getZeroExtendExpr(getSCEV(U->getOperand(0)), U->getType()); 5100 5101 case Instruction::SExt: 5102 return getSignExtendExpr(getSCEV(U->getOperand(0)), U->getType()); 5103 5104 case Instruction::BitCast: 5105 // BitCasts are no-op casts so we just eliminate the cast. 5106 if (isSCEVable(U->getType()) && isSCEVable(U->getOperand(0)->getType())) 5107 return getSCEV(U->getOperand(0)); 5108 break; 5109 5110 // It's tempting to handle inttoptr and ptrtoint as no-ops, however this can 5111 // lead to pointer expressions which cannot safely be expanded to GEPs, 5112 // because ScalarEvolution doesn't respect the GEP aliasing rules when 5113 // simplifying integer expressions. 5114 5115 case Instruction::GetElementPtr: 5116 return createNodeForGEP(cast<GEPOperator>(U)); 5117 5118 case Instruction::PHI: 5119 return createNodeForPHI(cast<PHINode>(U)); 5120 5121 case Instruction::Select: 5122 // U can also be a select constant expr, which let fall through. Since 5123 // createNodeForSelect only works for a condition that is an `ICmpInst`, and 5124 // constant expressions cannot have instructions as operands, we'd have 5125 // returned getUnknown for a select constant expressions anyway. 5126 if (isa<Instruction>(U)) 5127 return createNodeForSelectOrPHI(cast<Instruction>(U), U->getOperand(0), 5128 U->getOperand(1), U->getOperand(2)); 5129 } 5130 5131 return getUnknown(V); 5132 } 5133 5134 5135 5136 //===----------------------------------------------------------------------===// 5137 // Iteration Count Computation Code 5138 // 5139 5140 unsigned ScalarEvolution::getSmallConstantTripCount(Loop *L) { 5141 if (BasicBlock *ExitingBB = L->getExitingBlock()) 5142 return getSmallConstantTripCount(L, ExitingBB); 5143 5144 // No trip count information for multiple exits. 5145 return 0; 5146 } 5147 5148 /// getSmallConstantTripCount - Returns the maximum trip count of this loop as a 5149 /// normal unsigned value. Returns 0 if the trip count is unknown or not 5150 /// constant. Will also return 0 if the maximum trip count is very large (>= 5151 /// 2^32). 5152 /// 5153 /// This "trip count" assumes that control exits via ExitingBlock. More 5154 /// precisely, it is the number of times that control may reach ExitingBlock 5155 /// before taking the branch. For loops with multiple exits, it may not be the 5156 /// number times that the loop header executes because the loop may exit 5157 /// prematurely via another branch. 5158 unsigned ScalarEvolution::getSmallConstantTripCount(Loop *L, 5159 BasicBlock *ExitingBlock) { 5160 assert(ExitingBlock && "Must pass a non-null exiting block!"); 5161 assert(L->isLoopExiting(ExitingBlock) && 5162 "Exiting block must actually branch out of the loop!"); 5163 const SCEVConstant *ExitCount = 5164 dyn_cast<SCEVConstant>(getExitCount(L, ExitingBlock)); 5165 if (!ExitCount) 5166 return 0; 5167 5168 ConstantInt *ExitConst = ExitCount->getValue(); 5169 5170 // Guard against huge trip counts. 5171 if (ExitConst->getValue().getActiveBits() > 32) 5172 return 0; 5173 5174 // In case of integer overflow, this returns 0, which is correct. 5175 return ((unsigned)ExitConst->getZExtValue()) + 1; 5176 } 5177 5178 unsigned ScalarEvolution::getSmallConstantTripMultiple(Loop *L) { 5179 if (BasicBlock *ExitingBB = L->getExitingBlock()) 5180 return getSmallConstantTripMultiple(L, ExitingBB); 5181 5182 // No trip multiple information for multiple exits. 5183 return 0; 5184 } 5185 5186 /// getSmallConstantTripMultiple - Returns the largest constant divisor of the 5187 /// trip count of this loop as a normal unsigned value, if possible. This 5188 /// means that the actual trip count is always a multiple of the returned 5189 /// value (don't forget the trip count could very well be zero as well!). 5190 /// 5191 /// Returns 1 if the trip count is unknown or not guaranteed to be the 5192 /// multiple of a constant (which is also the case if the trip count is simply 5193 /// constant, use getSmallConstantTripCount for that case), Will also return 1 5194 /// if the trip count is very large (>= 2^32). 5195 /// 5196 /// As explained in the comments for getSmallConstantTripCount, this assumes 5197 /// that control exits the loop via ExitingBlock. 5198 unsigned 5199 ScalarEvolution::getSmallConstantTripMultiple(Loop *L, 5200 BasicBlock *ExitingBlock) { 5201 assert(ExitingBlock && "Must pass a non-null exiting block!"); 5202 assert(L->isLoopExiting(ExitingBlock) && 5203 "Exiting block must actually branch out of the loop!"); 5204 const SCEV *ExitCount = getExitCount(L, ExitingBlock); 5205 if (ExitCount == getCouldNotCompute()) 5206 return 1; 5207 5208 // Get the trip count from the BE count by adding 1. 5209 const SCEV *TCMul = getAddExpr(ExitCount, getOne(ExitCount->getType())); 5210 // FIXME: SCEV distributes multiplication as V1*C1 + V2*C1. We could attempt 5211 // to factor simple cases. 5212 if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(TCMul)) 5213 TCMul = Mul->getOperand(0); 5214 5215 const SCEVConstant *MulC = dyn_cast<SCEVConstant>(TCMul); 5216 if (!MulC) 5217 return 1; 5218 5219 ConstantInt *Result = MulC->getValue(); 5220 5221 // Guard against huge trip counts (this requires checking 5222 // for zero to handle the case where the trip count == -1 and the 5223 // addition wraps). 5224 if (!Result || Result->getValue().getActiveBits() > 32 || 5225 Result->getValue().getActiveBits() == 0) 5226 return 1; 5227 5228 return (unsigned)Result->getZExtValue(); 5229 } 5230 5231 // getExitCount - Get the expression for the number of loop iterations for which 5232 // this loop is guaranteed not to exit via ExitingBlock. Otherwise return 5233 // SCEVCouldNotCompute. 5234 const SCEV *ScalarEvolution::getExitCount(Loop *L, BasicBlock *ExitingBlock) { 5235 return getBackedgeTakenInfo(L).getExact(ExitingBlock, this); 5236 } 5237 5238 const SCEV * 5239 ScalarEvolution::getPredicatedBackedgeTakenCount(const Loop *L, 5240 SCEVUnionPredicate &Preds) { 5241 return getPredicatedBackedgeTakenInfo(L).getExact(this, &Preds); 5242 } 5243 5244 /// getBackedgeTakenCount - If the specified loop has a predictable 5245 /// backedge-taken count, return it, otherwise return a SCEVCouldNotCompute 5246 /// object. The backedge-taken count is the number of times the loop header 5247 /// will be branched to from within the loop. This is one less than the 5248 /// trip count of the loop, since it doesn't count the first iteration, 5249 /// when the header is branched to from outside the loop. 5250 /// 5251 /// Note that it is not valid to call this method on a loop without a 5252 /// loop-invariant backedge-taken count (see 5253 /// hasLoopInvariantBackedgeTakenCount). 5254 /// 5255 const SCEV *ScalarEvolution::getBackedgeTakenCount(const Loop *L) { 5256 return getBackedgeTakenInfo(L).getExact(this); 5257 } 5258 5259 /// getMaxBackedgeTakenCount - Similar to getBackedgeTakenCount, except 5260 /// return the least SCEV value that is known never to be less than the 5261 /// actual backedge taken count. 5262 const SCEV *ScalarEvolution::getMaxBackedgeTakenCount(const Loop *L) { 5263 return getBackedgeTakenInfo(L).getMax(this); 5264 } 5265 5266 /// PushLoopPHIs - Push PHI nodes in the header of the given loop 5267 /// onto the given Worklist. 5268 static void 5269 PushLoopPHIs(const Loop *L, SmallVectorImpl<Instruction *> &Worklist) { 5270 BasicBlock *Header = L->getHeader(); 5271 5272 // Push all Loop-header PHIs onto the Worklist stack. 5273 for (BasicBlock::iterator I = Header->begin(); 5274 PHINode *PN = dyn_cast<PHINode>(I); ++I) 5275 Worklist.push_back(PN); 5276 } 5277 5278 const ScalarEvolution::BackedgeTakenInfo & 5279 ScalarEvolution::getPredicatedBackedgeTakenInfo(const Loop *L) { 5280 auto &BTI = getBackedgeTakenInfo(L); 5281 if (BTI.hasFullInfo()) 5282 return BTI; 5283 5284 auto Pair = PredicatedBackedgeTakenCounts.insert({L, BackedgeTakenInfo()}); 5285 5286 if (!Pair.second) 5287 return Pair.first->second; 5288 5289 BackedgeTakenInfo Result = 5290 computeBackedgeTakenCount(L, /*AllowPredicates=*/true); 5291 5292 return PredicatedBackedgeTakenCounts.find(L)->second = Result; 5293 } 5294 5295 const ScalarEvolution::BackedgeTakenInfo & 5296 ScalarEvolution::getBackedgeTakenInfo(const Loop *L) { 5297 // Initially insert an invalid entry for this loop. If the insertion 5298 // succeeds, proceed to actually compute a backedge-taken count and 5299 // update the value. The temporary CouldNotCompute value tells SCEV 5300 // code elsewhere that it shouldn't attempt to request a new 5301 // backedge-taken count, which could result in infinite recursion. 5302 std::pair<DenseMap<const Loop *, BackedgeTakenInfo>::iterator, bool> Pair = 5303 BackedgeTakenCounts.insert({L, BackedgeTakenInfo()}); 5304 if (!Pair.second) 5305 return Pair.first->second; 5306 5307 // computeBackedgeTakenCount may allocate memory for its result. Inserting it 5308 // into the BackedgeTakenCounts map transfers ownership. Otherwise, the result 5309 // must be cleared in this scope. 5310 BackedgeTakenInfo Result = computeBackedgeTakenCount(L); 5311 5312 if (Result.getExact(this) != getCouldNotCompute()) { 5313 assert(isLoopInvariant(Result.getExact(this), L) && 5314 isLoopInvariant(Result.getMax(this), L) && 5315 "Computed backedge-taken count isn't loop invariant for loop!"); 5316 ++NumTripCountsComputed; 5317 } 5318 else if (Result.getMax(this) == getCouldNotCompute() && 5319 isa<PHINode>(L->getHeader()->begin())) { 5320 // Only count loops that have phi nodes as not being computable. 5321 ++NumTripCountsNotComputed; 5322 } 5323 5324 // Now that we know more about the trip count for this loop, forget any 5325 // existing SCEV values for PHI nodes in this loop since they are only 5326 // conservative estimates made without the benefit of trip count 5327 // information. This is similar to the code in forgetLoop, except that 5328 // it handles SCEVUnknown PHI nodes specially. 5329 if (Result.hasAnyInfo()) { 5330 SmallVector<Instruction *, 16> Worklist; 5331 PushLoopPHIs(L, Worklist); 5332 5333 SmallPtrSet<Instruction *, 8> Visited; 5334 while (!Worklist.empty()) { 5335 Instruction *I = Worklist.pop_back_val(); 5336 if (!Visited.insert(I).second) 5337 continue; 5338 5339 ValueExprMapType::iterator It = 5340 ValueExprMap.find_as(static_cast<Value *>(I)); 5341 if (It != ValueExprMap.end()) { 5342 const SCEV *Old = It->second; 5343 5344 // SCEVUnknown for a PHI either means that it has an unrecognized 5345 // structure, or it's a PHI that's in the progress of being computed 5346 // by createNodeForPHI. In the former case, additional loop trip 5347 // count information isn't going to change anything. In the later 5348 // case, createNodeForPHI will perform the necessary updates on its 5349 // own when it gets to that point. 5350 if (!isa<PHINode>(I) || !isa<SCEVUnknown>(Old)) { 5351 forgetMemoizedResults(Old); 5352 ValueExprMap.erase(It); 5353 } 5354 if (PHINode *PN = dyn_cast<PHINode>(I)) 5355 ConstantEvolutionLoopExitValue.erase(PN); 5356 } 5357 5358 PushDefUseChildren(I, Worklist); 5359 } 5360 } 5361 5362 // Re-lookup the insert position, since the call to 5363 // computeBackedgeTakenCount above could result in a 5364 // recusive call to getBackedgeTakenInfo (on a different 5365 // loop), which would invalidate the iterator computed 5366 // earlier. 5367 return BackedgeTakenCounts.find(L)->second = Result; 5368 } 5369 5370 /// forgetLoop - This method should be called by the client when it has 5371 /// changed a loop in a way that may effect ScalarEvolution's ability to 5372 /// compute a trip count, or if the loop is deleted. 5373 void ScalarEvolution::forgetLoop(const Loop *L) { 5374 // Drop any stored trip count value. 5375 auto RemoveLoopFromBackedgeMap = 5376 [L](DenseMap<const Loop *, BackedgeTakenInfo> &Map) { 5377 auto BTCPos = Map.find(L); 5378 if (BTCPos != Map.end()) { 5379 BTCPos->second.clear(); 5380 Map.erase(BTCPos); 5381 } 5382 }; 5383 5384 RemoveLoopFromBackedgeMap(BackedgeTakenCounts); 5385 RemoveLoopFromBackedgeMap(PredicatedBackedgeTakenCounts); 5386 5387 // Drop information about expressions based on loop-header PHIs. 5388 SmallVector<Instruction *, 16> Worklist; 5389 PushLoopPHIs(L, Worklist); 5390 5391 SmallPtrSet<Instruction *, 8> Visited; 5392 while (!Worklist.empty()) { 5393 Instruction *I = Worklist.pop_back_val(); 5394 if (!Visited.insert(I).second) 5395 continue; 5396 5397 ValueExprMapType::iterator It = 5398 ValueExprMap.find_as(static_cast<Value *>(I)); 5399 if (It != ValueExprMap.end()) { 5400 forgetMemoizedResults(It->second); 5401 ValueExprMap.erase(It); 5402 if (PHINode *PN = dyn_cast<PHINode>(I)) 5403 ConstantEvolutionLoopExitValue.erase(PN); 5404 } 5405 5406 PushDefUseChildren(I, Worklist); 5407 } 5408 5409 // Forget all contained loops too, to avoid dangling entries in the 5410 // ValuesAtScopes map. 5411 for (Loop::iterator I = L->begin(), E = L->end(); I != E; ++I) 5412 forgetLoop(*I); 5413 } 5414 5415 /// forgetValue - This method should be called by the client when it has 5416 /// changed a value in a way that may effect its value, or which may 5417 /// disconnect it from a def-use chain linking it to a loop. 5418 void ScalarEvolution::forgetValue(Value *V) { 5419 Instruction *I = dyn_cast<Instruction>(V); 5420 if (!I) return; 5421 5422 // Drop information about expressions based on loop-header PHIs. 5423 SmallVector<Instruction *, 16> Worklist; 5424 Worklist.push_back(I); 5425 5426 SmallPtrSet<Instruction *, 8> Visited; 5427 while (!Worklist.empty()) { 5428 I = Worklist.pop_back_val(); 5429 if (!Visited.insert(I).second) 5430 continue; 5431 5432 ValueExprMapType::iterator It = 5433 ValueExprMap.find_as(static_cast<Value *>(I)); 5434 if (It != ValueExprMap.end()) { 5435 forgetMemoizedResults(It->second); 5436 ValueExprMap.erase(It); 5437 if (PHINode *PN = dyn_cast<PHINode>(I)) 5438 ConstantEvolutionLoopExitValue.erase(PN); 5439 } 5440 5441 PushDefUseChildren(I, Worklist); 5442 } 5443 } 5444 5445 /// getExact - Get the exact loop backedge taken count considering all loop 5446 /// exits. A computable result can only be returned for loops with a single 5447 /// exit. Returning the minimum taken count among all exits is incorrect 5448 /// because one of the loop's exit limit's may have been skipped. HowFarToZero 5449 /// assumes that the limit of each loop test is never skipped. This is a valid 5450 /// assumption as long as the loop exits via that test. For precise results, it 5451 /// is the caller's responsibility to specify the relevant loop exit using 5452 /// getExact(ExitingBlock, SE). 5453 const SCEV * 5454 ScalarEvolution::BackedgeTakenInfo::getExact( 5455 ScalarEvolution *SE, SCEVUnionPredicate *Preds) const { 5456 // If any exits were not computable, the loop is not computable. 5457 if (!ExitNotTaken.isCompleteList()) return SE->getCouldNotCompute(); 5458 5459 // We need exactly one computable exit. 5460 if (!ExitNotTaken.ExitingBlock) return SE->getCouldNotCompute(); 5461 assert(ExitNotTaken.ExactNotTaken && "uninitialized not-taken info"); 5462 5463 const SCEV *BECount = nullptr; 5464 for (auto &ENT : ExitNotTaken) { 5465 assert(ENT.ExactNotTaken != SE->getCouldNotCompute() && "bad exit SCEV"); 5466 5467 if (!BECount) 5468 BECount = ENT.ExactNotTaken; 5469 else if (BECount != ENT.ExactNotTaken) 5470 return SE->getCouldNotCompute(); 5471 if (Preds && ENT.getPred()) 5472 Preds->add(ENT.getPred()); 5473 5474 assert((Preds || ENT.hasAlwaysTruePred()) && 5475 "Predicate should be always true!"); 5476 } 5477 5478 assert(BECount && "Invalid not taken count for loop exit"); 5479 return BECount; 5480 } 5481 5482 /// getExact - Get the exact not taken count for this loop exit. 5483 const SCEV * 5484 ScalarEvolution::BackedgeTakenInfo::getExact(BasicBlock *ExitingBlock, 5485 ScalarEvolution *SE) const { 5486 for (auto &ENT : ExitNotTaken) 5487 if (ENT.ExitingBlock == ExitingBlock && ENT.hasAlwaysTruePred()) 5488 return ENT.ExactNotTaken; 5489 5490 return SE->getCouldNotCompute(); 5491 } 5492 5493 /// getMax - Get the max backedge taken count for the loop. 5494 const SCEV * 5495 ScalarEvolution::BackedgeTakenInfo::getMax(ScalarEvolution *SE) const { 5496 for (auto &ENT : ExitNotTaken) 5497 if (!ENT.hasAlwaysTruePred()) 5498 return SE->getCouldNotCompute(); 5499 5500 return Max ? Max : SE->getCouldNotCompute(); 5501 } 5502 5503 bool ScalarEvolution::BackedgeTakenInfo::hasOperand(const SCEV *S, 5504 ScalarEvolution *SE) const { 5505 if (Max && Max != SE->getCouldNotCompute() && SE->hasOperand(Max, S)) 5506 return true; 5507 5508 if (!ExitNotTaken.ExitingBlock) 5509 return false; 5510 5511 for (auto &ENT : ExitNotTaken) 5512 if (ENT.ExactNotTaken != SE->getCouldNotCompute() && 5513 SE->hasOperand(ENT.ExactNotTaken, S)) 5514 return true; 5515 5516 return false; 5517 } 5518 5519 /// Allocate memory for BackedgeTakenInfo and copy the not-taken count of each 5520 /// computable exit into a persistent ExitNotTakenInfo array. 5521 ScalarEvolution::BackedgeTakenInfo::BackedgeTakenInfo( 5522 SmallVectorImpl<EdgeInfo> &ExitCounts, bool Complete, const SCEV *MaxCount) 5523 : Max(MaxCount) { 5524 5525 if (!Complete) 5526 ExitNotTaken.setIncomplete(); 5527 5528 unsigned NumExits = ExitCounts.size(); 5529 if (NumExits == 0) return; 5530 5531 ExitNotTaken.ExitingBlock = ExitCounts[0].ExitBlock; 5532 ExitNotTaken.ExactNotTaken = ExitCounts[0].Taken; 5533 5534 // Determine the number of ExitNotTakenExtras structures that we need. 5535 unsigned ExtraInfoSize = 0; 5536 if (NumExits > 1) 5537 ExtraInfoSize = 1 + std::count_if(std::next(ExitCounts.begin()), 5538 ExitCounts.end(), [](EdgeInfo &Entry) { 5539 return !Entry.Pred.isAlwaysTrue(); 5540 }); 5541 else if (!ExitCounts[0].Pred.isAlwaysTrue()) 5542 ExtraInfoSize = 1; 5543 5544 ExitNotTakenExtras *ENT = nullptr; 5545 5546 // Allocate the ExitNotTakenExtras structures and initialize the first 5547 // element (ExitNotTaken). 5548 if (ExtraInfoSize > 0) { 5549 ENT = new ExitNotTakenExtras[ExtraInfoSize]; 5550 ExitNotTaken.ExtraInfo = &ENT[0]; 5551 *ExitNotTaken.getPred() = std::move(ExitCounts[0].Pred); 5552 } 5553 5554 if (NumExits == 1) 5555 return; 5556 5557 auto &Exits = ExitNotTaken.ExtraInfo->Exits; 5558 5559 // Handle the rare case of multiple computable exits. 5560 for (unsigned i = 1, PredPos = 1; i < NumExits; ++i) { 5561 ExitNotTakenExtras *Ptr = nullptr; 5562 if (!ExitCounts[i].Pred.isAlwaysTrue()) { 5563 Ptr = &ENT[PredPos++]; 5564 Ptr->Pred = std::move(ExitCounts[i].Pred); 5565 } 5566 5567 Exits.emplace_back(ExitCounts[i].ExitBlock, ExitCounts[i].Taken, Ptr); 5568 } 5569 } 5570 5571 /// clear - Invalidate this result and free the ExitNotTakenInfo array. 5572 void ScalarEvolution::BackedgeTakenInfo::clear() { 5573 ExitNotTaken.ExitingBlock = nullptr; 5574 ExitNotTaken.ExactNotTaken = nullptr; 5575 delete[] ExitNotTaken.ExtraInfo; 5576 } 5577 5578 /// computeBackedgeTakenCount - Compute the number of times the backedge 5579 /// of the specified loop will execute. 5580 ScalarEvolution::BackedgeTakenInfo 5581 ScalarEvolution::computeBackedgeTakenCount(const Loop *L, 5582 bool AllowPredicates) { 5583 SmallVector<BasicBlock *, 8> ExitingBlocks; 5584 L->getExitingBlocks(ExitingBlocks); 5585 5586 SmallVector<EdgeInfo, 4> ExitCounts; 5587 bool CouldComputeBECount = true; 5588 BasicBlock *Latch = L->getLoopLatch(); // may be NULL. 5589 const SCEV *MustExitMaxBECount = nullptr; 5590 const SCEV *MayExitMaxBECount = nullptr; 5591 5592 // Compute the ExitLimit for each loop exit. Use this to populate ExitCounts 5593 // and compute maxBECount. 5594 // Do a union of all the predicates here. 5595 for (unsigned i = 0, e = ExitingBlocks.size(); i != e; ++i) { 5596 BasicBlock *ExitBB = ExitingBlocks[i]; 5597 ExitLimit EL = computeExitLimit(L, ExitBB, AllowPredicates); 5598 5599 assert((AllowPredicates || EL.Pred.isAlwaysTrue()) && 5600 "Predicated exit limit when predicates are not allowed!"); 5601 5602 // 1. For each exit that can be computed, add an entry to ExitCounts. 5603 // CouldComputeBECount is true only if all exits can be computed. 5604 if (EL.Exact == getCouldNotCompute()) 5605 // We couldn't compute an exact value for this exit, so 5606 // we won't be able to compute an exact value for the loop. 5607 CouldComputeBECount = false; 5608 else 5609 ExitCounts.emplace_back(EdgeInfo(ExitBB, EL.Exact, EL.Pred)); 5610 5611 // 2. Derive the loop's MaxBECount from each exit's max number of 5612 // non-exiting iterations. Partition the loop exits into two kinds: 5613 // LoopMustExits and LoopMayExits. 5614 // 5615 // If the exit dominates the loop latch, it is a LoopMustExit otherwise it 5616 // is a LoopMayExit. If any computable LoopMustExit is found, then 5617 // MaxBECount is the minimum EL.Max of computable LoopMustExits. Otherwise, 5618 // MaxBECount is conservatively the maximum EL.Max, where CouldNotCompute is 5619 // considered greater than any computable EL.Max. 5620 if (EL.Max != getCouldNotCompute() && Latch && 5621 DT.dominates(ExitBB, Latch)) { 5622 if (!MustExitMaxBECount) 5623 MustExitMaxBECount = EL.Max; 5624 else { 5625 MustExitMaxBECount = 5626 getUMinFromMismatchedTypes(MustExitMaxBECount, EL.Max); 5627 } 5628 } else if (MayExitMaxBECount != getCouldNotCompute()) { 5629 if (!MayExitMaxBECount || EL.Max == getCouldNotCompute()) 5630 MayExitMaxBECount = EL.Max; 5631 else { 5632 MayExitMaxBECount = 5633 getUMaxFromMismatchedTypes(MayExitMaxBECount, EL.Max); 5634 } 5635 } 5636 } 5637 const SCEV *MaxBECount = MustExitMaxBECount ? MustExitMaxBECount : 5638 (MayExitMaxBECount ? MayExitMaxBECount : getCouldNotCompute()); 5639 return BackedgeTakenInfo(ExitCounts, CouldComputeBECount, MaxBECount); 5640 } 5641 5642 ScalarEvolution::ExitLimit 5643 ScalarEvolution::computeExitLimit(const Loop *L, BasicBlock *ExitingBlock, 5644 bool AllowPredicates) { 5645 5646 // Okay, we've chosen an exiting block. See what condition causes us to exit 5647 // at this block and remember the exit block and whether all other targets 5648 // lead to the loop header. 5649 bool MustExecuteLoopHeader = true; 5650 BasicBlock *Exit = nullptr; 5651 for (auto *SBB : successors(ExitingBlock)) 5652 if (!L->contains(SBB)) { 5653 if (Exit) // Multiple exit successors. 5654 return getCouldNotCompute(); 5655 Exit = SBB; 5656 } else if (SBB != L->getHeader()) { 5657 MustExecuteLoopHeader = false; 5658 } 5659 5660 // At this point, we know we have a conditional branch that determines whether 5661 // the loop is exited. However, we don't know if the branch is executed each 5662 // time through the loop. If not, then the execution count of the branch will 5663 // not be equal to the trip count of the loop. 5664 // 5665 // Currently we check for this by checking to see if the Exit branch goes to 5666 // the loop header. If so, we know it will always execute the same number of 5667 // times as the loop. We also handle the case where the exit block *is* the 5668 // loop header. This is common for un-rotated loops. 5669 // 5670 // If both of those tests fail, walk up the unique predecessor chain to the 5671 // header, stopping if there is an edge that doesn't exit the loop. If the 5672 // header is reached, the execution count of the branch will be equal to the 5673 // trip count of the loop. 5674 // 5675 // More extensive analysis could be done to handle more cases here. 5676 // 5677 if (!MustExecuteLoopHeader && ExitingBlock != L->getHeader()) { 5678 // The simple checks failed, try climbing the unique predecessor chain 5679 // up to the header. 5680 bool Ok = false; 5681 for (BasicBlock *BB = ExitingBlock; BB; ) { 5682 BasicBlock *Pred = BB->getUniquePredecessor(); 5683 if (!Pred) 5684 return getCouldNotCompute(); 5685 TerminatorInst *PredTerm = Pred->getTerminator(); 5686 for (const BasicBlock *PredSucc : PredTerm->successors()) { 5687 if (PredSucc == BB) 5688 continue; 5689 // If the predecessor has a successor that isn't BB and isn't 5690 // outside the loop, assume the worst. 5691 if (L->contains(PredSucc)) 5692 return getCouldNotCompute(); 5693 } 5694 if (Pred == L->getHeader()) { 5695 Ok = true; 5696 break; 5697 } 5698 BB = Pred; 5699 } 5700 if (!Ok) 5701 return getCouldNotCompute(); 5702 } 5703 5704 bool IsOnlyExit = (L->getExitingBlock() != nullptr); 5705 TerminatorInst *Term = ExitingBlock->getTerminator(); 5706 if (BranchInst *BI = dyn_cast<BranchInst>(Term)) { 5707 assert(BI->isConditional() && "If unconditional, it can't be in loop!"); 5708 // Proceed to the next level to examine the exit condition expression. 5709 return computeExitLimitFromCond( 5710 L, BI->getCondition(), BI->getSuccessor(0), BI->getSuccessor(1), 5711 /*ControlsExit=*/IsOnlyExit, AllowPredicates); 5712 } 5713 5714 if (SwitchInst *SI = dyn_cast<SwitchInst>(Term)) 5715 return computeExitLimitFromSingleExitSwitch(L, SI, Exit, 5716 /*ControlsExit=*/IsOnlyExit); 5717 5718 return getCouldNotCompute(); 5719 } 5720 5721 /// computeExitLimitFromCond - Compute the number of times the 5722 /// backedge of the specified loop will execute if its exit condition 5723 /// were a conditional branch of ExitCond, TBB, and FBB. 5724 /// 5725 /// @param ControlsExit is true if ExitCond directly controls the exit 5726 /// branch. In this case, we can assume that the loop exits only if the 5727 /// condition is true and can infer that failing to meet the condition prior to 5728 /// integer wraparound results in undefined behavior. 5729 ScalarEvolution::ExitLimit 5730 ScalarEvolution::computeExitLimitFromCond(const Loop *L, 5731 Value *ExitCond, 5732 BasicBlock *TBB, 5733 BasicBlock *FBB, 5734 bool ControlsExit, 5735 bool AllowPredicates) { 5736 // Check if the controlling expression for this loop is an And or Or. 5737 if (BinaryOperator *BO = dyn_cast<BinaryOperator>(ExitCond)) { 5738 if (BO->getOpcode() == Instruction::And) { 5739 // Recurse on the operands of the and. 5740 bool EitherMayExit = L->contains(TBB); 5741 ExitLimit EL0 = computeExitLimitFromCond(L, BO->getOperand(0), TBB, FBB, 5742 ControlsExit && !EitherMayExit, 5743 AllowPredicates); 5744 ExitLimit EL1 = computeExitLimitFromCond(L, BO->getOperand(1), TBB, FBB, 5745 ControlsExit && !EitherMayExit, 5746 AllowPredicates); 5747 const SCEV *BECount = getCouldNotCompute(); 5748 const SCEV *MaxBECount = getCouldNotCompute(); 5749 if (EitherMayExit) { 5750 // Both conditions must be true for the loop to continue executing. 5751 // Choose the less conservative count. 5752 if (EL0.Exact == getCouldNotCompute() || 5753 EL1.Exact == getCouldNotCompute()) 5754 BECount = getCouldNotCompute(); 5755 else 5756 BECount = getUMinFromMismatchedTypes(EL0.Exact, EL1.Exact); 5757 if (EL0.Max == getCouldNotCompute()) 5758 MaxBECount = EL1.Max; 5759 else if (EL1.Max == getCouldNotCompute()) 5760 MaxBECount = EL0.Max; 5761 else 5762 MaxBECount = getUMinFromMismatchedTypes(EL0.Max, EL1.Max); 5763 } else { 5764 // Both conditions must be true at the same time for the loop to exit. 5765 // For now, be conservative. 5766 assert(L->contains(FBB) && "Loop block has no successor in loop!"); 5767 if (EL0.Max == EL1.Max) 5768 MaxBECount = EL0.Max; 5769 if (EL0.Exact == EL1.Exact) 5770 BECount = EL0.Exact; 5771 } 5772 5773 SCEVUnionPredicate NP; 5774 NP.add(&EL0.Pred); 5775 NP.add(&EL1.Pred); 5776 // There are cases (e.g. PR26207) where computeExitLimitFromCond is able 5777 // to be more aggressive when computing BECount than when computing 5778 // MaxBECount. In these cases it is possible for EL0.Exact and EL1.Exact 5779 // to match, but for EL0.Max and EL1.Max to not. 5780 if (isa<SCEVCouldNotCompute>(MaxBECount) && 5781 !isa<SCEVCouldNotCompute>(BECount)) 5782 MaxBECount = BECount; 5783 5784 return ExitLimit(BECount, MaxBECount, NP); 5785 } 5786 if (BO->getOpcode() == Instruction::Or) { 5787 // Recurse on the operands of the or. 5788 bool EitherMayExit = L->contains(FBB); 5789 ExitLimit EL0 = computeExitLimitFromCond(L, BO->getOperand(0), TBB, FBB, 5790 ControlsExit && !EitherMayExit, 5791 AllowPredicates); 5792 ExitLimit EL1 = computeExitLimitFromCond(L, BO->getOperand(1), TBB, FBB, 5793 ControlsExit && !EitherMayExit, 5794 AllowPredicates); 5795 const SCEV *BECount = getCouldNotCompute(); 5796 const SCEV *MaxBECount = getCouldNotCompute(); 5797 if (EitherMayExit) { 5798 // Both conditions must be false for the loop to continue executing. 5799 // Choose the less conservative count. 5800 if (EL0.Exact == getCouldNotCompute() || 5801 EL1.Exact == getCouldNotCompute()) 5802 BECount = getCouldNotCompute(); 5803 else 5804 BECount = getUMinFromMismatchedTypes(EL0.Exact, EL1.Exact); 5805 if (EL0.Max == getCouldNotCompute()) 5806 MaxBECount = EL1.Max; 5807 else if (EL1.Max == getCouldNotCompute()) 5808 MaxBECount = EL0.Max; 5809 else 5810 MaxBECount = getUMinFromMismatchedTypes(EL0.Max, EL1.Max); 5811 } else { 5812 // Both conditions must be false at the same time for the loop to exit. 5813 // For now, be conservative. 5814 assert(L->contains(TBB) && "Loop block has no successor in loop!"); 5815 if (EL0.Max == EL1.Max) 5816 MaxBECount = EL0.Max; 5817 if (EL0.Exact == EL1.Exact) 5818 BECount = EL0.Exact; 5819 } 5820 5821 SCEVUnionPredicate NP; 5822 NP.add(&EL0.Pred); 5823 NP.add(&EL1.Pred); 5824 return ExitLimit(BECount, MaxBECount, NP); 5825 } 5826 } 5827 5828 // With an icmp, it may be feasible to compute an exact backedge-taken count. 5829 // Proceed to the next level to examine the icmp. 5830 if (ICmpInst *ExitCondICmp = dyn_cast<ICmpInst>(ExitCond)) { 5831 ExitLimit EL = 5832 computeExitLimitFromICmp(L, ExitCondICmp, TBB, FBB, ControlsExit); 5833 if (EL.hasFullInfo() || !AllowPredicates) 5834 return EL; 5835 5836 // Try again, but use SCEV predicates this time. 5837 return computeExitLimitFromICmp(L, ExitCondICmp, TBB, FBB, ControlsExit, 5838 /*AllowPredicates=*/true); 5839 } 5840 5841 // Check for a constant condition. These are normally stripped out by 5842 // SimplifyCFG, but ScalarEvolution may be used by a pass which wishes to 5843 // preserve the CFG and is temporarily leaving constant conditions 5844 // in place. 5845 if (ConstantInt *CI = dyn_cast<ConstantInt>(ExitCond)) { 5846 if (L->contains(FBB) == !CI->getZExtValue()) 5847 // The backedge is always taken. 5848 return getCouldNotCompute(); 5849 else 5850 // The backedge is never taken. 5851 return getZero(CI->getType()); 5852 } 5853 5854 // If it's not an integer or pointer comparison then compute it the hard way. 5855 return computeExitCountExhaustively(L, ExitCond, !L->contains(TBB)); 5856 } 5857 5858 ScalarEvolution::ExitLimit 5859 ScalarEvolution::computeExitLimitFromICmp(const Loop *L, 5860 ICmpInst *ExitCond, 5861 BasicBlock *TBB, 5862 BasicBlock *FBB, 5863 bool ControlsExit, 5864 bool AllowPredicates) { 5865 5866 // If the condition was exit on true, convert the condition to exit on false 5867 ICmpInst::Predicate Cond; 5868 if (!L->contains(FBB)) 5869 Cond = ExitCond->getPredicate(); 5870 else 5871 Cond = ExitCond->getInversePredicate(); 5872 5873 // Handle common loops like: for (X = "string"; *X; ++X) 5874 if (LoadInst *LI = dyn_cast<LoadInst>(ExitCond->getOperand(0))) 5875 if (Constant *RHS = dyn_cast<Constant>(ExitCond->getOperand(1))) { 5876 ExitLimit ItCnt = 5877 computeLoadConstantCompareExitLimit(LI, RHS, L, Cond); 5878 if (ItCnt.hasAnyInfo()) 5879 return ItCnt; 5880 } 5881 5882 ExitLimit ShiftEL = computeShiftCompareExitLimit( 5883 ExitCond->getOperand(0), ExitCond->getOperand(1), L, Cond); 5884 if (ShiftEL.hasAnyInfo()) 5885 return ShiftEL; 5886 5887 const SCEV *LHS = getSCEV(ExitCond->getOperand(0)); 5888 const SCEV *RHS = getSCEV(ExitCond->getOperand(1)); 5889 5890 // Try to evaluate any dependencies out of the loop. 5891 LHS = getSCEVAtScope(LHS, L); 5892 RHS = getSCEVAtScope(RHS, L); 5893 5894 // At this point, we would like to compute how many iterations of the 5895 // loop the predicate will return true for these inputs. 5896 if (isLoopInvariant(LHS, L) && !isLoopInvariant(RHS, L)) { 5897 // If there is a loop-invariant, force it into the RHS. 5898 std::swap(LHS, RHS); 5899 Cond = ICmpInst::getSwappedPredicate(Cond); 5900 } 5901 5902 // Simplify the operands before analyzing them. 5903 (void)SimplifyICmpOperands(Cond, LHS, RHS); 5904 5905 // If we have a comparison of a chrec against a constant, try to use value 5906 // ranges to answer this query. 5907 if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(RHS)) 5908 if (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(LHS)) 5909 if (AddRec->getLoop() == L) { 5910 // Form the constant range. 5911 ConstantRange CompRange( 5912 ICmpInst::makeConstantRange(Cond, RHSC->getAPInt())); 5913 5914 const SCEV *Ret = AddRec->getNumIterationsInRange(CompRange, *this); 5915 if (!isa<SCEVCouldNotCompute>(Ret)) return Ret; 5916 } 5917 5918 switch (Cond) { 5919 case ICmpInst::ICMP_NE: { // while (X != Y) 5920 // Convert to: while (X-Y != 0) 5921 ExitLimit EL = HowFarToZero(getMinusSCEV(LHS, RHS), L, ControlsExit, 5922 AllowPredicates); 5923 if (EL.hasAnyInfo()) return EL; 5924 break; 5925 } 5926 case ICmpInst::ICMP_EQ: { // while (X == Y) 5927 // Convert to: while (X-Y == 0) 5928 ExitLimit EL = HowFarToNonZero(getMinusSCEV(LHS, RHS), L); 5929 if (EL.hasAnyInfo()) return EL; 5930 break; 5931 } 5932 case ICmpInst::ICMP_SLT: 5933 case ICmpInst::ICMP_ULT: { // while (X < Y) 5934 bool IsSigned = Cond == ICmpInst::ICMP_SLT; 5935 ExitLimit EL = HowManyLessThans(LHS, RHS, L, IsSigned, ControlsExit, 5936 AllowPredicates); 5937 if (EL.hasAnyInfo()) return EL; 5938 break; 5939 } 5940 case ICmpInst::ICMP_SGT: 5941 case ICmpInst::ICMP_UGT: { // while (X > Y) 5942 bool IsSigned = Cond == ICmpInst::ICMP_SGT; 5943 ExitLimit EL = 5944 HowManyGreaterThans(LHS, RHS, L, IsSigned, ControlsExit, 5945 AllowPredicates); 5946 if (EL.hasAnyInfo()) return EL; 5947 break; 5948 } 5949 default: 5950 break; 5951 } 5952 return computeExitCountExhaustively(L, ExitCond, !L->contains(TBB)); 5953 } 5954 5955 ScalarEvolution::ExitLimit 5956 ScalarEvolution::computeExitLimitFromSingleExitSwitch(const Loop *L, 5957 SwitchInst *Switch, 5958 BasicBlock *ExitingBlock, 5959 bool ControlsExit) { 5960 assert(!L->contains(ExitingBlock) && "Not an exiting block!"); 5961 5962 // Give up if the exit is the default dest of a switch. 5963 if (Switch->getDefaultDest() == ExitingBlock) 5964 return getCouldNotCompute(); 5965 5966 assert(L->contains(Switch->getDefaultDest()) && 5967 "Default case must not exit the loop!"); 5968 const SCEV *LHS = getSCEVAtScope(Switch->getCondition(), L); 5969 const SCEV *RHS = getConstant(Switch->findCaseDest(ExitingBlock)); 5970 5971 // while (X != Y) --> while (X-Y != 0) 5972 ExitLimit EL = HowFarToZero(getMinusSCEV(LHS, RHS), L, ControlsExit); 5973 if (EL.hasAnyInfo()) 5974 return EL; 5975 5976 return getCouldNotCompute(); 5977 } 5978 5979 static ConstantInt * 5980 EvaluateConstantChrecAtConstant(const SCEVAddRecExpr *AddRec, ConstantInt *C, 5981 ScalarEvolution &SE) { 5982 const SCEV *InVal = SE.getConstant(C); 5983 const SCEV *Val = AddRec->evaluateAtIteration(InVal, SE); 5984 assert(isa<SCEVConstant>(Val) && 5985 "Evaluation of SCEV at constant didn't fold correctly?"); 5986 return cast<SCEVConstant>(Val)->getValue(); 5987 } 5988 5989 /// computeLoadConstantCompareExitLimit - Given an exit condition of 5990 /// 'icmp op load X, cst', try to see if we can compute the backedge 5991 /// execution count. 5992 ScalarEvolution::ExitLimit 5993 ScalarEvolution::computeLoadConstantCompareExitLimit( 5994 LoadInst *LI, 5995 Constant *RHS, 5996 const Loop *L, 5997 ICmpInst::Predicate predicate) { 5998 5999 if (LI->isVolatile()) return getCouldNotCompute(); 6000 6001 // Check to see if the loaded pointer is a getelementptr of a global. 6002 // TODO: Use SCEV instead of manually grubbing with GEPs. 6003 GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(LI->getOperand(0)); 6004 if (!GEP) return getCouldNotCompute(); 6005 6006 // Make sure that it is really a constant global we are gepping, with an 6007 // initializer, and make sure the first IDX is really 0. 6008 GlobalVariable *GV = dyn_cast<GlobalVariable>(GEP->getOperand(0)); 6009 if (!GV || !GV->isConstant() || !GV->hasDefinitiveInitializer() || 6010 GEP->getNumOperands() < 3 || !isa<Constant>(GEP->getOperand(1)) || 6011 !cast<Constant>(GEP->getOperand(1))->isNullValue()) 6012 return getCouldNotCompute(); 6013 6014 // Okay, we allow one non-constant index into the GEP instruction. 6015 Value *VarIdx = nullptr; 6016 std::vector<Constant*> Indexes; 6017 unsigned VarIdxNum = 0; 6018 for (unsigned i = 2, e = GEP->getNumOperands(); i != e; ++i) 6019 if (ConstantInt *CI = dyn_cast<ConstantInt>(GEP->getOperand(i))) { 6020 Indexes.push_back(CI); 6021 } else if (!isa<ConstantInt>(GEP->getOperand(i))) { 6022 if (VarIdx) return getCouldNotCompute(); // Multiple non-constant idx's. 6023 VarIdx = GEP->getOperand(i); 6024 VarIdxNum = i-2; 6025 Indexes.push_back(nullptr); 6026 } 6027 6028 // Loop-invariant loads may be a byproduct of loop optimization. Skip them. 6029 if (!VarIdx) 6030 return getCouldNotCompute(); 6031 6032 // Okay, we know we have a (load (gep GV, 0, X)) comparison with a constant. 6033 // Check to see if X is a loop variant variable value now. 6034 const SCEV *Idx = getSCEV(VarIdx); 6035 Idx = getSCEVAtScope(Idx, L); 6036 6037 // We can only recognize very limited forms of loop index expressions, in 6038 // particular, only affine AddRec's like {C1,+,C2}. 6039 const SCEVAddRecExpr *IdxExpr = dyn_cast<SCEVAddRecExpr>(Idx); 6040 if (!IdxExpr || !IdxExpr->isAffine() || isLoopInvariant(IdxExpr, L) || 6041 !isa<SCEVConstant>(IdxExpr->getOperand(0)) || 6042 !isa<SCEVConstant>(IdxExpr->getOperand(1))) 6043 return getCouldNotCompute(); 6044 6045 unsigned MaxSteps = MaxBruteForceIterations; 6046 for (unsigned IterationNum = 0; IterationNum != MaxSteps; ++IterationNum) { 6047 ConstantInt *ItCst = ConstantInt::get( 6048 cast<IntegerType>(IdxExpr->getType()), IterationNum); 6049 ConstantInt *Val = EvaluateConstantChrecAtConstant(IdxExpr, ItCst, *this); 6050 6051 // Form the GEP offset. 6052 Indexes[VarIdxNum] = Val; 6053 6054 Constant *Result = ConstantFoldLoadThroughGEPIndices(GV->getInitializer(), 6055 Indexes); 6056 if (!Result) break; // Cannot compute! 6057 6058 // Evaluate the condition for this iteration. 6059 Result = ConstantExpr::getICmp(predicate, Result, RHS); 6060 if (!isa<ConstantInt>(Result)) break; // Couldn't decide for sure 6061 if (cast<ConstantInt>(Result)->getValue().isMinValue()) { 6062 ++NumArrayLenItCounts; 6063 return getConstant(ItCst); // Found terminating iteration! 6064 } 6065 } 6066 return getCouldNotCompute(); 6067 } 6068 6069 ScalarEvolution::ExitLimit ScalarEvolution::computeShiftCompareExitLimit( 6070 Value *LHS, Value *RHSV, const Loop *L, ICmpInst::Predicate Pred) { 6071 ConstantInt *RHS = dyn_cast<ConstantInt>(RHSV); 6072 if (!RHS) 6073 return getCouldNotCompute(); 6074 6075 const BasicBlock *Latch = L->getLoopLatch(); 6076 if (!Latch) 6077 return getCouldNotCompute(); 6078 6079 const BasicBlock *Predecessor = L->getLoopPredecessor(); 6080 if (!Predecessor) 6081 return getCouldNotCompute(); 6082 6083 // Return true if V is of the form "LHS `shift_op` <positive constant>". 6084 // Return LHS in OutLHS and shift_opt in OutOpCode. 6085 auto MatchPositiveShift = 6086 [](Value *V, Value *&OutLHS, Instruction::BinaryOps &OutOpCode) { 6087 6088 using namespace PatternMatch; 6089 6090 ConstantInt *ShiftAmt; 6091 if (match(V, m_LShr(m_Value(OutLHS), m_ConstantInt(ShiftAmt)))) 6092 OutOpCode = Instruction::LShr; 6093 else if (match(V, m_AShr(m_Value(OutLHS), m_ConstantInt(ShiftAmt)))) 6094 OutOpCode = Instruction::AShr; 6095 else if (match(V, m_Shl(m_Value(OutLHS), m_ConstantInt(ShiftAmt)))) 6096 OutOpCode = Instruction::Shl; 6097 else 6098 return false; 6099 6100 return ShiftAmt->getValue().isStrictlyPositive(); 6101 }; 6102 6103 // Recognize a "shift recurrence" either of the form %iv or of %iv.shifted in 6104 // 6105 // loop: 6106 // %iv = phi i32 [ %iv.shifted, %loop ], [ %val, %preheader ] 6107 // %iv.shifted = lshr i32 %iv, <positive constant> 6108 // 6109 // Return true on a succesful match. Return the corresponding PHI node (%iv 6110 // above) in PNOut and the opcode of the shift operation in OpCodeOut. 6111 auto MatchShiftRecurrence = 6112 [&](Value *V, PHINode *&PNOut, Instruction::BinaryOps &OpCodeOut) { 6113 Optional<Instruction::BinaryOps> PostShiftOpCode; 6114 6115 { 6116 Instruction::BinaryOps OpC; 6117 Value *V; 6118 6119 // If we encounter a shift instruction, "peel off" the shift operation, 6120 // and remember that we did so. Later when we inspect %iv's backedge 6121 // value, we will make sure that the backedge value uses the same 6122 // operation. 6123 // 6124 // Note: the peeled shift operation does not have to be the same 6125 // instruction as the one feeding into the PHI's backedge value. We only 6126 // really care about it being the same *kind* of shift instruction -- 6127 // that's all that is required for our later inferences to hold. 6128 if (MatchPositiveShift(LHS, V, OpC)) { 6129 PostShiftOpCode = OpC; 6130 LHS = V; 6131 } 6132 } 6133 6134 PNOut = dyn_cast<PHINode>(LHS); 6135 if (!PNOut || PNOut->getParent() != L->getHeader()) 6136 return false; 6137 6138 Value *BEValue = PNOut->getIncomingValueForBlock(Latch); 6139 Value *OpLHS; 6140 6141 return 6142 // The backedge value for the PHI node must be a shift by a positive 6143 // amount 6144 MatchPositiveShift(BEValue, OpLHS, OpCodeOut) && 6145 6146 // of the PHI node itself 6147 OpLHS == PNOut && 6148 6149 // and the kind of shift should be match the kind of shift we peeled 6150 // off, if any. 6151 (!PostShiftOpCode.hasValue() || *PostShiftOpCode == OpCodeOut); 6152 }; 6153 6154 PHINode *PN; 6155 Instruction::BinaryOps OpCode; 6156 if (!MatchShiftRecurrence(LHS, PN, OpCode)) 6157 return getCouldNotCompute(); 6158 6159 const DataLayout &DL = getDataLayout(); 6160 6161 // The key rationale for this optimization is that for some kinds of shift 6162 // recurrences, the value of the recurrence "stabilizes" to either 0 or -1 6163 // within a finite number of iterations. If the condition guarding the 6164 // backedge (in the sense that the backedge is taken if the condition is true) 6165 // is false for the value the shift recurrence stabilizes to, then we know 6166 // that the backedge is taken only a finite number of times. 6167 6168 ConstantInt *StableValue = nullptr; 6169 switch (OpCode) { 6170 default: 6171 llvm_unreachable("Impossible case!"); 6172 6173 case Instruction::AShr: { 6174 // {K,ashr,<positive-constant>} stabilizes to signum(K) in at most 6175 // bitwidth(K) iterations. 6176 Value *FirstValue = PN->getIncomingValueForBlock(Predecessor); 6177 bool KnownZero, KnownOne; 6178 ComputeSignBit(FirstValue, KnownZero, KnownOne, DL, 0, nullptr, 6179 Predecessor->getTerminator(), &DT); 6180 auto *Ty = cast<IntegerType>(RHS->getType()); 6181 if (KnownZero) 6182 StableValue = ConstantInt::get(Ty, 0); 6183 else if (KnownOne) 6184 StableValue = ConstantInt::get(Ty, -1, true); 6185 else 6186 return getCouldNotCompute(); 6187 6188 break; 6189 } 6190 case Instruction::LShr: 6191 case Instruction::Shl: 6192 // Both {K,lshr,<positive-constant>} and {K,shl,<positive-constant>} 6193 // stabilize to 0 in at most bitwidth(K) iterations. 6194 StableValue = ConstantInt::get(cast<IntegerType>(RHS->getType()), 0); 6195 break; 6196 } 6197 6198 auto *Result = 6199 ConstantFoldCompareInstOperands(Pred, StableValue, RHS, DL, &TLI); 6200 assert(Result->getType()->isIntegerTy(1) && 6201 "Otherwise cannot be an operand to a branch instruction"); 6202 6203 if (Result->isZeroValue()) { 6204 unsigned BitWidth = getTypeSizeInBits(RHS->getType()); 6205 const SCEV *UpperBound = 6206 getConstant(getEffectiveSCEVType(RHS->getType()), BitWidth); 6207 SCEVUnionPredicate P; 6208 return ExitLimit(getCouldNotCompute(), UpperBound, P); 6209 } 6210 6211 return getCouldNotCompute(); 6212 } 6213 6214 /// CanConstantFold - Return true if we can constant fold an instruction of the 6215 /// specified type, assuming that all operands were constants. 6216 static bool CanConstantFold(const Instruction *I) { 6217 if (isa<BinaryOperator>(I) || isa<CmpInst>(I) || 6218 isa<SelectInst>(I) || isa<CastInst>(I) || isa<GetElementPtrInst>(I) || 6219 isa<LoadInst>(I)) 6220 return true; 6221 6222 if (const CallInst *CI = dyn_cast<CallInst>(I)) 6223 if (const Function *F = CI->getCalledFunction()) 6224 return canConstantFoldCallTo(F); 6225 return false; 6226 } 6227 6228 /// Determine whether this instruction can constant evolve within this loop 6229 /// assuming its operands can all constant evolve. 6230 static bool canConstantEvolve(Instruction *I, const Loop *L) { 6231 // An instruction outside of the loop can't be derived from a loop PHI. 6232 if (!L->contains(I)) return false; 6233 6234 if (isa<PHINode>(I)) { 6235 // We don't currently keep track of the control flow needed to evaluate 6236 // PHIs, so we cannot handle PHIs inside of loops. 6237 return L->getHeader() == I->getParent(); 6238 } 6239 6240 // If we won't be able to constant fold this expression even if the operands 6241 // are constants, bail early. 6242 return CanConstantFold(I); 6243 } 6244 6245 /// getConstantEvolvingPHIOperands - Implement getConstantEvolvingPHI by 6246 /// recursing through each instruction operand until reaching a loop header phi. 6247 static PHINode * 6248 getConstantEvolvingPHIOperands(Instruction *UseInst, const Loop *L, 6249 DenseMap<Instruction *, PHINode *> &PHIMap) { 6250 6251 // Otherwise, we can evaluate this instruction if all of its operands are 6252 // constant or derived from a PHI node themselves. 6253 PHINode *PHI = nullptr; 6254 for (Value *Op : UseInst->operands()) { 6255 if (isa<Constant>(Op)) continue; 6256 6257 Instruction *OpInst = dyn_cast<Instruction>(Op); 6258 if (!OpInst || !canConstantEvolve(OpInst, L)) return nullptr; 6259 6260 PHINode *P = dyn_cast<PHINode>(OpInst); 6261 if (!P) 6262 // If this operand is already visited, reuse the prior result. 6263 // We may have P != PHI if this is the deepest point at which the 6264 // inconsistent paths meet. 6265 P = PHIMap.lookup(OpInst); 6266 if (!P) { 6267 // Recurse and memoize the results, whether a phi is found or not. 6268 // This recursive call invalidates pointers into PHIMap. 6269 P = getConstantEvolvingPHIOperands(OpInst, L, PHIMap); 6270 PHIMap[OpInst] = P; 6271 } 6272 if (!P) 6273 return nullptr; // Not evolving from PHI 6274 if (PHI && PHI != P) 6275 return nullptr; // Evolving from multiple different PHIs. 6276 PHI = P; 6277 } 6278 // This is a expression evolving from a constant PHI! 6279 return PHI; 6280 } 6281 6282 /// getConstantEvolvingPHI - Given an LLVM value and a loop, return a PHI node 6283 /// in the loop that V is derived from. We allow arbitrary operations along the 6284 /// way, but the operands of an operation must either be constants or a value 6285 /// derived from a constant PHI. If this expression does not fit with these 6286 /// constraints, return null. 6287 static PHINode *getConstantEvolvingPHI(Value *V, const Loop *L) { 6288 Instruction *I = dyn_cast<Instruction>(V); 6289 if (!I || !canConstantEvolve(I, L)) return nullptr; 6290 6291 if (PHINode *PN = dyn_cast<PHINode>(I)) 6292 return PN; 6293 6294 // Record non-constant instructions contained by the loop. 6295 DenseMap<Instruction *, PHINode *> PHIMap; 6296 return getConstantEvolvingPHIOperands(I, L, PHIMap); 6297 } 6298 6299 /// EvaluateExpression - Given an expression that passes the 6300 /// getConstantEvolvingPHI predicate, evaluate its value assuming the PHI node 6301 /// in the loop has the value PHIVal. If we can't fold this expression for some 6302 /// reason, return null. 6303 static Constant *EvaluateExpression(Value *V, const Loop *L, 6304 DenseMap<Instruction *, Constant *> &Vals, 6305 const DataLayout &DL, 6306 const TargetLibraryInfo *TLI) { 6307 // Convenient constant check, but redundant for recursive calls. 6308 if (Constant *C = dyn_cast<Constant>(V)) return C; 6309 Instruction *I = dyn_cast<Instruction>(V); 6310 if (!I) return nullptr; 6311 6312 if (Constant *C = Vals.lookup(I)) return C; 6313 6314 // An instruction inside the loop depends on a value outside the loop that we 6315 // weren't given a mapping for, or a value such as a call inside the loop. 6316 if (!canConstantEvolve(I, L)) return nullptr; 6317 6318 // An unmapped PHI can be due to a branch or another loop inside this loop, 6319 // or due to this not being the initial iteration through a loop where we 6320 // couldn't compute the evolution of this particular PHI last time. 6321 if (isa<PHINode>(I)) return nullptr; 6322 6323 std::vector<Constant*> Operands(I->getNumOperands()); 6324 6325 for (unsigned i = 0, e = I->getNumOperands(); i != e; ++i) { 6326 Instruction *Operand = dyn_cast<Instruction>(I->getOperand(i)); 6327 if (!Operand) { 6328 Operands[i] = dyn_cast<Constant>(I->getOperand(i)); 6329 if (!Operands[i]) return nullptr; 6330 continue; 6331 } 6332 Constant *C = EvaluateExpression(Operand, L, Vals, DL, TLI); 6333 Vals[Operand] = C; 6334 if (!C) return nullptr; 6335 Operands[i] = C; 6336 } 6337 6338 if (CmpInst *CI = dyn_cast<CmpInst>(I)) 6339 return ConstantFoldCompareInstOperands(CI->getPredicate(), Operands[0], 6340 Operands[1], DL, TLI); 6341 if (LoadInst *LI = dyn_cast<LoadInst>(I)) { 6342 if (!LI->isVolatile()) 6343 return ConstantFoldLoadFromConstPtr(Operands[0], LI->getType(), DL); 6344 } 6345 return ConstantFoldInstOperands(I, Operands, DL, TLI); 6346 } 6347 6348 6349 // If every incoming value to PN except the one for BB is a specific Constant, 6350 // return that, else return nullptr. 6351 static Constant *getOtherIncomingValue(PHINode *PN, BasicBlock *BB) { 6352 Constant *IncomingVal = nullptr; 6353 6354 for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) { 6355 if (PN->getIncomingBlock(i) == BB) 6356 continue; 6357 6358 auto *CurrentVal = dyn_cast<Constant>(PN->getIncomingValue(i)); 6359 if (!CurrentVal) 6360 return nullptr; 6361 6362 if (IncomingVal != CurrentVal) { 6363 if (IncomingVal) 6364 return nullptr; 6365 IncomingVal = CurrentVal; 6366 } 6367 } 6368 6369 return IncomingVal; 6370 } 6371 6372 /// getConstantEvolutionLoopExitValue - If we know that the specified Phi is 6373 /// in the header of its containing loop, we know the loop executes a 6374 /// constant number of times, and the PHI node is just a recurrence 6375 /// involving constants, fold it. 6376 Constant * 6377 ScalarEvolution::getConstantEvolutionLoopExitValue(PHINode *PN, 6378 const APInt &BEs, 6379 const Loop *L) { 6380 auto I = ConstantEvolutionLoopExitValue.find(PN); 6381 if (I != ConstantEvolutionLoopExitValue.end()) 6382 return I->second; 6383 6384 if (BEs.ugt(MaxBruteForceIterations)) 6385 return ConstantEvolutionLoopExitValue[PN] = nullptr; // Not going to evaluate it. 6386 6387 Constant *&RetVal = ConstantEvolutionLoopExitValue[PN]; 6388 6389 DenseMap<Instruction *, Constant *> CurrentIterVals; 6390 BasicBlock *Header = L->getHeader(); 6391 assert(PN->getParent() == Header && "Can't evaluate PHI not in loop header!"); 6392 6393 BasicBlock *Latch = L->getLoopLatch(); 6394 if (!Latch) 6395 return nullptr; 6396 6397 for (auto &I : *Header) { 6398 PHINode *PHI = dyn_cast<PHINode>(&I); 6399 if (!PHI) break; 6400 auto *StartCST = getOtherIncomingValue(PHI, Latch); 6401 if (!StartCST) continue; 6402 CurrentIterVals[PHI] = StartCST; 6403 } 6404 if (!CurrentIterVals.count(PN)) 6405 return RetVal = nullptr; 6406 6407 Value *BEValue = PN->getIncomingValueForBlock(Latch); 6408 6409 // Execute the loop symbolically to determine the exit value. 6410 if (BEs.getActiveBits() >= 32) 6411 return RetVal = nullptr; // More than 2^32-1 iterations?? Not doing it! 6412 6413 unsigned NumIterations = BEs.getZExtValue(); // must be in range 6414 unsigned IterationNum = 0; 6415 const DataLayout &DL = getDataLayout(); 6416 for (; ; ++IterationNum) { 6417 if (IterationNum == NumIterations) 6418 return RetVal = CurrentIterVals[PN]; // Got exit value! 6419 6420 // Compute the value of the PHIs for the next iteration. 6421 // EvaluateExpression adds non-phi values to the CurrentIterVals map. 6422 DenseMap<Instruction *, Constant *> NextIterVals; 6423 Constant *NextPHI = 6424 EvaluateExpression(BEValue, L, CurrentIterVals, DL, &TLI); 6425 if (!NextPHI) 6426 return nullptr; // Couldn't evaluate! 6427 NextIterVals[PN] = NextPHI; 6428 6429 bool StoppedEvolving = NextPHI == CurrentIterVals[PN]; 6430 6431 // Also evaluate the other PHI nodes. However, we don't get to stop if we 6432 // cease to be able to evaluate one of them or if they stop evolving, 6433 // because that doesn't necessarily prevent us from computing PN. 6434 SmallVector<std::pair<PHINode *, Constant *>, 8> PHIsToCompute; 6435 for (const auto &I : CurrentIterVals) { 6436 PHINode *PHI = dyn_cast<PHINode>(I.first); 6437 if (!PHI || PHI == PN || PHI->getParent() != Header) continue; 6438 PHIsToCompute.emplace_back(PHI, I.second); 6439 } 6440 // We use two distinct loops because EvaluateExpression may invalidate any 6441 // iterators into CurrentIterVals. 6442 for (const auto &I : PHIsToCompute) { 6443 PHINode *PHI = I.first; 6444 Constant *&NextPHI = NextIterVals[PHI]; 6445 if (!NextPHI) { // Not already computed. 6446 Value *BEValue = PHI->getIncomingValueForBlock(Latch); 6447 NextPHI = EvaluateExpression(BEValue, L, CurrentIterVals, DL, &TLI); 6448 } 6449 if (NextPHI != I.second) 6450 StoppedEvolving = false; 6451 } 6452 6453 // If all entries in CurrentIterVals == NextIterVals then we can stop 6454 // iterating, the loop can't continue to change. 6455 if (StoppedEvolving) 6456 return RetVal = CurrentIterVals[PN]; 6457 6458 CurrentIterVals.swap(NextIterVals); 6459 } 6460 } 6461 6462 const SCEV *ScalarEvolution::computeExitCountExhaustively(const Loop *L, 6463 Value *Cond, 6464 bool ExitWhen) { 6465 PHINode *PN = getConstantEvolvingPHI(Cond, L); 6466 if (!PN) return getCouldNotCompute(); 6467 6468 // If the loop is canonicalized, the PHI will have exactly two entries. 6469 // That's the only form we support here. 6470 if (PN->getNumIncomingValues() != 2) return getCouldNotCompute(); 6471 6472 DenseMap<Instruction *, Constant *> CurrentIterVals; 6473 BasicBlock *Header = L->getHeader(); 6474 assert(PN->getParent() == Header && "Can't evaluate PHI not in loop header!"); 6475 6476 BasicBlock *Latch = L->getLoopLatch(); 6477 assert(Latch && "Should follow from NumIncomingValues == 2!"); 6478 6479 for (auto &I : *Header) { 6480 PHINode *PHI = dyn_cast<PHINode>(&I); 6481 if (!PHI) 6482 break; 6483 auto *StartCST = getOtherIncomingValue(PHI, Latch); 6484 if (!StartCST) continue; 6485 CurrentIterVals[PHI] = StartCST; 6486 } 6487 if (!CurrentIterVals.count(PN)) 6488 return getCouldNotCompute(); 6489 6490 // Okay, we find a PHI node that defines the trip count of this loop. Execute 6491 // the loop symbolically to determine when the condition gets a value of 6492 // "ExitWhen". 6493 unsigned MaxIterations = MaxBruteForceIterations; // Limit analysis. 6494 const DataLayout &DL = getDataLayout(); 6495 for (unsigned IterationNum = 0; IterationNum != MaxIterations;++IterationNum){ 6496 auto *CondVal = dyn_cast_or_null<ConstantInt>( 6497 EvaluateExpression(Cond, L, CurrentIterVals, DL, &TLI)); 6498 6499 // Couldn't symbolically evaluate. 6500 if (!CondVal) return getCouldNotCompute(); 6501 6502 if (CondVal->getValue() == uint64_t(ExitWhen)) { 6503 ++NumBruteForceTripCountsComputed; 6504 return getConstant(Type::getInt32Ty(getContext()), IterationNum); 6505 } 6506 6507 // Update all the PHI nodes for the next iteration. 6508 DenseMap<Instruction *, Constant *> NextIterVals; 6509 6510 // Create a list of which PHIs we need to compute. We want to do this before 6511 // calling EvaluateExpression on them because that may invalidate iterators 6512 // into CurrentIterVals. 6513 SmallVector<PHINode *, 8> PHIsToCompute; 6514 for (const auto &I : CurrentIterVals) { 6515 PHINode *PHI = dyn_cast<PHINode>(I.first); 6516 if (!PHI || PHI->getParent() != Header) continue; 6517 PHIsToCompute.push_back(PHI); 6518 } 6519 for (PHINode *PHI : PHIsToCompute) { 6520 Constant *&NextPHI = NextIterVals[PHI]; 6521 if (NextPHI) continue; // Already computed! 6522 6523 Value *BEValue = PHI->getIncomingValueForBlock(Latch); 6524 NextPHI = EvaluateExpression(BEValue, L, CurrentIterVals, DL, &TLI); 6525 } 6526 CurrentIterVals.swap(NextIterVals); 6527 } 6528 6529 // Too many iterations were needed to evaluate. 6530 return getCouldNotCompute(); 6531 } 6532 6533 /// getSCEVAtScope - Return a SCEV expression for the specified value 6534 /// at the specified scope in the program. The L value specifies a loop 6535 /// nest to evaluate the expression at, where null is the top-level or a 6536 /// specified loop is immediately inside of the loop. 6537 /// 6538 /// This method can be used to compute the exit value for a variable defined 6539 /// in a loop by querying what the value will hold in the parent loop. 6540 /// 6541 /// In the case that a relevant loop exit value cannot be computed, the 6542 /// original value V is returned. 6543 const SCEV *ScalarEvolution::getSCEVAtScope(const SCEV *V, const Loop *L) { 6544 SmallVector<std::pair<const Loop *, const SCEV *>, 2> &Values = 6545 ValuesAtScopes[V]; 6546 // Check to see if we've folded this expression at this loop before. 6547 for (auto &LS : Values) 6548 if (LS.first == L) 6549 return LS.second ? LS.second : V; 6550 6551 Values.emplace_back(L, nullptr); 6552 6553 // Otherwise compute it. 6554 const SCEV *C = computeSCEVAtScope(V, L); 6555 for (auto &LS : reverse(ValuesAtScopes[V])) 6556 if (LS.first == L) { 6557 LS.second = C; 6558 break; 6559 } 6560 return C; 6561 } 6562 6563 /// This builds up a Constant using the ConstantExpr interface. That way, we 6564 /// will return Constants for objects which aren't represented by a 6565 /// SCEVConstant, because SCEVConstant is restricted to ConstantInt. 6566 /// Returns NULL if the SCEV isn't representable as a Constant. 6567 static Constant *BuildConstantFromSCEV(const SCEV *V) { 6568 switch (static_cast<SCEVTypes>(V->getSCEVType())) { 6569 case scCouldNotCompute: 6570 case scAddRecExpr: 6571 break; 6572 case scConstant: 6573 return cast<SCEVConstant>(V)->getValue(); 6574 case scUnknown: 6575 return dyn_cast<Constant>(cast<SCEVUnknown>(V)->getValue()); 6576 case scSignExtend: { 6577 const SCEVSignExtendExpr *SS = cast<SCEVSignExtendExpr>(V); 6578 if (Constant *CastOp = BuildConstantFromSCEV(SS->getOperand())) 6579 return ConstantExpr::getSExt(CastOp, SS->getType()); 6580 break; 6581 } 6582 case scZeroExtend: { 6583 const SCEVZeroExtendExpr *SZ = cast<SCEVZeroExtendExpr>(V); 6584 if (Constant *CastOp = BuildConstantFromSCEV(SZ->getOperand())) 6585 return ConstantExpr::getZExt(CastOp, SZ->getType()); 6586 break; 6587 } 6588 case scTruncate: { 6589 const SCEVTruncateExpr *ST = cast<SCEVTruncateExpr>(V); 6590 if (Constant *CastOp = BuildConstantFromSCEV(ST->getOperand())) 6591 return ConstantExpr::getTrunc(CastOp, ST->getType()); 6592 break; 6593 } 6594 case scAddExpr: { 6595 const SCEVAddExpr *SA = cast<SCEVAddExpr>(V); 6596 if (Constant *C = BuildConstantFromSCEV(SA->getOperand(0))) { 6597 if (PointerType *PTy = dyn_cast<PointerType>(C->getType())) { 6598 unsigned AS = PTy->getAddressSpace(); 6599 Type *DestPtrTy = Type::getInt8PtrTy(C->getContext(), AS); 6600 C = ConstantExpr::getBitCast(C, DestPtrTy); 6601 } 6602 for (unsigned i = 1, e = SA->getNumOperands(); i != e; ++i) { 6603 Constant *C2 = BuildConstantFromSCEV(SA->getOperand(i)); 6604 if (!C2) return nullptr; 6605 6606 // First pointer! 6607 if (!C->getType()->isPointerTy() && C2->getType()->isPointerTy()) { 6608 unsigned AS = C2->getType()->getPointerAddressSpace(); 6609 std::swap(C, C2); 6610 Type *DestPtrTy = Type::getInt8PtrTy(C->getContext(), AS); 6611 // The offsets have been converted to bytes. We can add bytes to an 6612 // i8* by GEP with the byte count in the first index. 6613 C = ConstantExpr::getBitCast(C, DestPtrTy); 6614 } 6615 6616 // Don't bother trying to sum two pointers. We probably can't 6617 // statically compute a load that results from it anyway. 6618 if (C2->getType()->isPointerTy()) 6619 return nullptr; 6620 6621 if (PointerType *PTy = dyn_cast<PointerType>(C->getType())) { 6622 if (PTy->getElementType()->isStructTy()) 6623 C2 = ConstantExpr::getIntegerCast( 6624 C2, Type::getInt32Ty(C->getContext()), true); 6625 C = ConstantExpr::getGetElementPtr(PTy->getElementType(), C, C2); 6626 } else 6627 C = ConstantExpr::getAdd(C, C2); 6628 } 6629 return C; 6630 } 6631 break; 6632 } 6633 case scMulExpr: { 6634 const SCEVMulExpr *SM = cast<SCEVMulExpr>(V); 6635 if (Constant *C = BuildConstantFromSCEV(SM->getOperand(0))) { 6636 // Don't bother with pointers at all. 6637 if (C->getType()->isPointerTy()) return nullptr; 6638 for (unsigned i = 1, e = SM->getNumOperands(); i != e; ++i) { 6639 Constant *C2 = BuildConstantFromSCEV(SM->getOperand(i)); 6640 if (!C2 || C2->getType()->isPointerTy()) return nullptr; 6641 C = ConstantExpr::getMul(C, C2); 6642 } 6643 return C; 6644 } 6645 break; 6646 } 6647 case scUDivExpr: { 6648 const SCEVUDivExpr *SU = cast<SCEVUDivExpr>(V); 6649 if (Constant *LHS = BuildConstantFromSCEV(SU->getLHS())) 6650 if (Constant *RHS = BuildConstantFromSCEV(SU->getRHS())) 6651 if (LHS->getType() == RHS->getType()) 6652 return ConstantExpr::getUDiv(LHS, RHS); 6653 break; 6654 } 6655 case scSMaxExpr: 6656 case scUMaxExpr: 6657 break; // TODO: smax, umax. 6658 } 6659 return nullptr; 6660 } 6661 6662 const SCEV *ScalarEvolution::computeSCEVAtScope(const SCEV *V, const Loop *L) { 6663 if (isa<SCEVConstant>(V)) return V; 6664 6665 // If this instruction is evolved from a constant-evolving PHI, compute the 6666 // exit value from the loop without using SCEVs. 6667 if (const SCEVUnknown *SU = dyn_cast<SCEVUnknown>(V)) { 6668 if (Instruction *I = dyn_cast<Instruction>(SU->getValue())) { 6669 const Loop *LI = this->LI[I->getParent()]; 6670 if (LI && LI->getParentLoop() == L) // Looking for loop exit value. 6671 if (PHINode *PN = dyn_cast<PHINode>(I)) 6672 if (PN->getParent() == LI->getHeader()) { 6673 // Okay, there is no closed form solution for the PHI node. Check 6674 // to see if the loop that contains it has a known backedge-taken 6675 // count. If so, we may be able to force computation of the exit 6676 // value. 6677 const SCEV *BackedgeTakenCount = getBackedgeTakenCount(LI); 6678 if (const SCEVConstant *BTCC = 6679 dyn_cast<SCEVConstant>(BackedgeTakenCount)) { 6680 // Okay, we know how many times the containing loop executes. If 6681 // this is a constant evolving PHI node, get the final value at 6682 // the specified iteration number. 6683 Constant *RV = 6684 getConstantEvolutionLoopExitValue(PN, BTCC->getAPInt(), LI); 6685 if (RV) return getSCEV(RV); 6686 } 6687 } 6688 6689 // Okay, this is an expression that we cannot symbolically evaluate 6690 // into a SCEV. Check to see if it's possible to symbolically evaluate 6691 // the arguments into constants, and if so, try to constant propagate the 6692 // result. This is particularly useful for computing loop exit values. 6693 if (CanConstantFold(I)) { 6694 SmallVector<Constant *, 4> Operands; 6695 bool MadeImprovement = false; 6696 for (Value *Op : I->operands()) { 6697 if (Constant *C = dyn_cast<Constant>(Op)) { 6698 Operands.push_back(C); 6699 continue; 6700 } 6701 6702 // If any of the operands is non-constant and if they are 6703 // non-integer and non-pointer, don't even try to analyze them 6704 // with scev techniques. 6705 if (!isSCEVable(Op->getType())) 6706 return V; 6707 6708 const SCEV *OrigV = getSCEV(Op); 6709 const SCEV *OpV = getSCEVAtScope(OrigV, L); 6710 MadeImprovement |= OrigV != OpV; 6711 6712 Constant *C = BuildConstantFromSCEV(OpV); 6713 if (!C) return V; 6714 if (C->getType() != Op->getType()) 6715 C = ConstantExpr::getCast(CastInst::getCastOpcode(C, false, 6716 Op->getType(), 6717 false), 6718 C, Op->getType()); 6719 Operands.push_back(C); 6720 } 6721 6722 // Check to see if getSCEVAtScope actually made an improvement. 6723 if (MadeImprovement) { 6724 Constant *C = nullptr; 6725 const DataLayout &DL = getDataLayout(); 6726 if (const CmpInst *CI = dyn_cast<CmpInst>(I)) 6727 C = ConstantFoldCompareInstOperands(CI->getPredicate(), Operands[0], 6728 Operands[1], DL, &TLI); 6729 else if (const LoadInst *LI = dyn_cast<LoadInst>(I)) { 6730 if (!LI->isVolatile()) 6731 C = ConstantFoldLoadFromConstPtr(Operands[0], LI->getType(), DL); 6732 } else 6733 C = ConstantFoldInstOperands(I, Operands, DL, &TLI); 6734 if (!C) return V; 6735 return getSCEV(C); 6736 } 6737 } 6738 } 6739 6740 // This is some other type of SCEVUnknown, just return it. 6741 return V; 6742 } 6743 6744 if (const SCEVCommutativeExpr *Comm = dyn_cast<SCEVCommutativeExpr>(V)) { 6745 // Avoid performing the look-up in the common case where the specified 6746 // expression has no loop-variant portions. 6747 for (unsigned i = 0, e = Comm->getNumOperands(); i != e; ++i) { 6748 const SCEV *OpAtScope = getSCEVAtScope(Comm->getOperand(i), L); 6749 if (OpAtScope != Comm->getOperand(i)) { 6750 // Okay, at least one of these operands is loop variant but might be 6751 // foldable. Build a new instance of the folded commutative expression. 6752 SmallVector<const SCEV *, 8> NewOps(Comm->op_begin(), 6753 Comm->op_begin()+i); 6754 NewOps.push_back(OpAtScope); 6755 6756 for (++i; i != e; ++i) { 6757 OpAtScope = getSCEVAtScope(Comm->getOperand(i), L); 6758 NewOps.push_back(OpAtScope); 6759 } 6760 if (isa<SCEVAddExpr>(Comm)) 6761 return getAddExpr(NewOps); 6762 if (isa<SCEVMulExpr>(Comm)) 6763 return getMulExpr(NewOps); 6764 if (isa<SCEVSMaxExpr>(Comm)) 6765 return getSMaxExpr(NewOps); 6766 if (isa<SCEVUMaxExpr>(Comm)) 6767 return getUMaxExpr(NewOps); 6768 llvm_unreachable("Unknown commutative SCEV type!"); 6769 } 6770 } 6771 // If we got here, all operands are loop invariant. 6772 return Comm; 6773 } 6774 6775 if (const SCEVUDivExpr *Div = dyn_cast<SCEVUDivExpr>(V)) { 6776 const SCEV *LHS = getSCEVAtScope(Div->getLHS(), L); 6777 const SCEV *RHS = getSCEVAtScope(Div->getRHS(), L); 6778 if (LHS == Div->getLHS() && RHS == Div->getRHS()) 6779 return Div; // must be loop invariant 6780 return getUDivExpr(LHS, RHS); 6781 } 6782 6783 // If this is a loop recurrence for a loop that does not contain L, then we 6784 // are dealing with the final value computed by the loop. 6785 if (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(V)) { 6786 // First, attempt to evaluate each operand. 6787 // Avoid performing the look-up in the common case where the specified 6788 // expression has no loop-variant portions. 6789 for (unsigned i = 0, e = AddRec->getNumOperands(); i != e; ++i) { 6790 const SCEV *OpAtScope = getSCEVAtScope(AddRec->getOperand(i), L); 6791 if (OpAtScope == AddRec->getOperand(i)) 6792 continue; 6793 6794 // Okay, at least one of these operands is loop variant but might be 6795 // foldable. Build a new instance of the folded commutative expression. 6796 SmallVector<const SCEV *, 8> NewOps(AddRec->op_begin(), 6797 AddRec->op_begin()+i); 6798 NewOps.push_back(OpAtScope); 6799 for (++i; i != e; ++i) 6800 NewOps.push_back(getSCEVAtScope(AddRec->getOperand(i), L)); 6801 6802 const SCEV *FoldedRec = 6803 getAddRecExpr(NewOps, AddRec->getLoop(), 6804 AddRec->getNoWrapFlags(SCEV::FlagNW)); 6805 AddRec = dyn_cast<SCEVAddRecExpr>(FoldedRec); 6806 // The addrec may be folded to a nonrecurrence, for example, if the 6807 // induction variable is multiplied by zero after constant folding. Go 6808 // ahead and return the folded value. 6809 if (!AddRec) 6810 return FoldedRec; 6811 break; 6812 } 6813 6814 // If the scope is outside the addrec's loop, evaluate it by using the 6815 // loop exit value of the addrec. 6816 if (!AddRec->getLoop()->contains(L)) { 6817 // To evaluate this recurrence, we need to know how many times the AddRec 6818 // loop iterates. Compute this now. 6819 const SCEV *BackedgeTakenCount = getBackedgeTakenCount(AddRec->getLoop()); 6820 if (BackedgeTakenCount == getCouldNotCompute()) return AddRec; 6821 6822 // Then, evaluate the AddRec. 6823 return AddRec->evaluateAtIteration(BackedgeTakenCount, *this); 6824 } 6825 6826 return AddRec; 6827 } 6828 6829 if (const SCEVZeroExtendExpr *Cast = dyn_cast<SCEVZeroExtendExpr>(V)) { 6830 const SCEV *Op = getSCEVAtScope(Cast->getOperand(), L); 6831 if (Op == Cast->getOperand()) 6832 return Cast; // must be loop invariant 6833 return getZeroExtendExpr(Op, Cast->getType()); 6834 } 6835 6836 if (const SCEVSignExtendExpr *Cast = dyn_cast<SCEVSignExtendExpr>(V)) { 6837 const SCEV *Op = getSCEVAtScope(Cast->getOperand(), L); 6838 if (Op == Cast->getOperand()) 6839 return Cast; // must be loop invariant 6840 return getSignExtendExpr(Op, Cast->getType()); 6841 } 6842 6843 if (const SCEVTruncateExpr *Cast = dyn_cast<SCEVTruncateExpr>(V)) { 6844 const SCEV *Op = getSCEVAtScope(Cast->getOperand(), L); 6845 if (Op == Cast->getOperand()) 6846 return Cast; // must be loop invariant 6847 return getTruncateExpr(Op, Cast->getType()); 6848 } 6849 6850 llvm_unreachable("Unknown SCEV type!"); 6851 } 6852 6853 /// getSCEVAtScope - This is a convenience function which does 6854 /// getSCEVAtScope(getSCEV(V), L). 6855 const SCEV *ScalarEvolution::getSCEVAtScope(Value *V, const Loop *L) { 6856 return getSCEVAtScope(getSCEV(V), L); 6857 } 6858 6859 /// SolveLinEquationWithOverflow - Finds the minimum unsigned root of the 6860 /// following equation: 6861 /// 6862 /// A * X = B (mod N) 6863 /// 6864 /// where N = 2^BW and BW is the common bit width of A and B. The signedness of 6865 /// A and B isn't important. 6866 /// 6867 /// If the equation does not have a solution, SCEVCouldNotCompute is returned. 6868 static const SCEV *SolveLinEquationWithOverflow(const APInt &A, const APInt &B, 6869 ScalarEvolution &SE) { 6870 uint32_t BW = A.getBitWidth(); 6871 assert(BW == B.getBitWidth() && "Bit widths must be the same."); 6872 assert(A != 0 && "A must be non-zero."); 6873 6874 // 1. D = gcd(A, N) 6875 // 6876 // The gcd of A and N may have only one prime factor: 2. The number of 6877 // trailing zeros in A is its multiplicity 6878 uint32_t Mult2 = A.countTrailingZeros(); 6879 // D = 2^Mult2 6880 6881 // 2. Check if B is divisible by D. 6882 // 6883 // B is divisible by D if and only if the multiplicity of prime factor 2 for B 6884 // is not less than multiplicity of this prime factor for D. 6885 if (B.countTrailingZeros() < Mult2) 6886 return SE.getCouldNotCompute(); 6887 6888 // 3. Compute I: the multiplicative inverse of (A / D) in arithmetic 6889 // modulo (N / D). 6890 // 6891 // (N / D) may need BW+1 bits in its representation. Hence, we'll use this 6892 // bit width during computations. 6893 APInt AD = A.lshr(Mult2).zext(BW + 1); // AD = A / D 6894 APInt Mod(BW + 1, 0); 6895 Mod.setBit(BW - Mult2); // Mod = N / D 6896 APInt I = AD.multiplicativeInverse(Mod); 6897 6898 // 4. Compute the minimum unsigned root of the equation: 6899 // I * (B / D) mod (N / D) 6900 APInt Result = (I * B.lshr(Mult2).zext(BW + 1)).urem(Mod); 6901 6902 // The result is guaranteed to be less than 2^BW so we may truncate it to BW 6903 // bits. 6904 return SE.getConstant(Result.trunc(BW)); 6905 } 6906 6907 /// SolveQuadraticEquation - Find the roots of the quadratic equation for the 6908 /// given quadratic chrec {L,+,M,+,N}. This returns either the two roots (which 6909 /// might be the same) or two SCEVCouldNotCompute objects. 6910 /// 6911 static std::pair<const SCEV *,const SCEV *> 6912 SolveQuadraticEquation(const SCEVAddRecExpr *AddRec, ScalarEvolution &SE) { 6913 assert(AddRec->getNumOperands() == 3 && "This is not a quadratic chrec!"); 6914 const SCEVConstant *LC = dyn_cast<SCEVConstant>(AddRec->getOperand(0)); 6915 const SCEVConstant *MC = dyn_cast<SCEVConstant>(AddRec->getOperand(1)); 6916 const SCEVConstant *NC = dyn_cast<SCEVConstant>(AddRec->getOperand(2)); 6917 6918 // We currently can only solve this if the coefficients are constants. 6919 if (!LC || !MC || !NC) { 6920 const SCEV *CNC = SE.getCouldNotCompute(); 6921 return {CNC, CNC}; 6922 } 6923 6924 uint32_t BitWidth = LC->getAPInt().getBitWidth(); 6925 const APInt &L = LC->getAPInt(); 6926 const APInt &M = MC->getAPInt(); 6927 const APInt &N = NC->getAPInt(); 6928 APInt Two(BitWidth, 2); 6929 APInt Four(BitWidth, 4); 6930 6931 { 6932 using namespace APIntOps; 6933 const APInt& C = L; 6934 // Convert from chrec coefficients to polynomial coefficients AX^2+BX+C 6935 // The B coefficient is M-N/2 6936 APInt B(M); 6937 B -= sdiv(N,Two); 6938 6939 // The A coefficient is N/2 6940 APInt A(N.sdiv(Two)); 6941 6942 // Compute the B^2-4ac term. 6943 APInt SqrtTerm(B); 6944 SqrtTerm *= B; 6945 SqrtTerm -= Four * (A * C); 6946 6947 if (SqrtTerm.isNegative()) { 6948 // The loop is provably infinite. 6949 const SCEV *CNC = SE.getCouldNotCompute(); 6950 return {CNC, CNC}; 6951 } 6952 6953 // Compute sqrt(B^2-4ac). This is guaranteed to be the nearest 6954 // integer value or else APInt::sqrt() will assert. 6955 APInt SqrtVal(SqrtTerm.sqrt()); 6956 6957 // Compute the two solutions for the quadratic formula. 6958 // The divisions must be performed as signed divisions. 6959 APInt NegB(-B); 6960 APInt TwoA(A << 1); 6961 if (TwoA.isMinValue()) { 6962 const SCEV *CNC = SE.getCouldNotCompute(); 6963 return {CNC, CNC}; 6964 } 6965 6966 LLVMContext &Context = SE.getContext(); 6967 6968 ConstantInt *Solution1 = 6969 ConstantInt::get(Context, (NegB + SqrtVal).sdiv(TwoA)); 6970 ConstantInt *Solution2 = 6971 ConstantInt::get(Context, (NegB - SqrtVal).sdiv(TwoA)); 6972 6973 return {SE.getConstant(Solution1), SE.getConstant(Solution2)}; 6974 } // end APIntOps namespace 6975 } 6976 6977 /// HowFarToZero - Return the number of times a backedge comparing the specified 6978 /// value to zero will execute. If not computable, return CouldNotCompute. 6979 /// 6980 /// This is only used for loops with a "x != y" exit test. The exit condition is 6981 /// now expressed as a single expression, V = x-y. So the exit test is 6982 /// effectively V != 0. We know and take advantage of the fact that this 6983 /// expression only being used in a comparison by zero context. 6984 ScalarEvolution::ExitLimit 6985 ScalarEvolution::HowFarToZero(const SCEV *V, const Loop *L, bool ControlsExit, 6986 bool AllowPredicates) { 6987 SCEVUnionPredicate P; 6988 // If the value is a constant 6989 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(V)) { 6990 // If the value is already zero, the branch will execute zero times. 6991 if (C->getValue()->isZero()) return C; 6992 return getCouldNotCompute(); // Otherwise it will loop infinitely. 6993 } 6994 6995 const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(V); 6996 if (!AddRec && AllowPredicates) 6997 // Try to make this an AddRec using runtime tests, in the first X 6998 // iterations of this loop, where X is the SCEV expression found by the 6999 // algorithm below. 7000 AddRec = convertSCEVToAddRecWithPredicates(V, L, P); 7001 7002 if (!AddRec || AddRec->getLoop() != L) 7003 return getCouldNotCompute(); 7004 7005 // If this is a quadratic (3-term) AddRec {L,+,M,+,N}, find the roots of 7006 // the quadratic equation to solve it. 7007 if (AddRec->isQuadratic() && AddRec->getType()->isIntegerTy()) { 7008 std::pair<const SCEV *,const SCEV *> Roots = 7009 SolveQuadraticEquation(AddRec, *this); 7010 const SCEVConstant *R1 = dyn_cast<SCEVConstant>(Roots.first); 7011 const SCEVConstant *R2 = dyn_cast<SCEVConstant>(Roots.second); 7012 if (R1 && R2) { 7013 // Pick the smallest positive root value. 7014 if (ConstantInt *CB = 7015 dyn_cast<ConstantInt>(ConstantExpr::getICmp(CmpInst::ICMP_ULT, 7016 R1->getValue(), 7017 R2->getValue()))) { 7018 if (!CB->getZExtValue()) 7019 std::swap(R1, R2); // R1 is the minimum root now. 7020 7021 // We can only use this value if the chrec ends up with an exact zero 7022 // value at this index. When solving for "X*X != 5", for example, we 7023 // should not accept a root of 2. 7024 const SCEV *Val = AddRec->evaluateAtIteration(R1, *this); 7025 if (Val->isZero()) 7026 return ExitLimit(R1, R1, P); // We found a quadratic root! 7027 } 7028 } 7029 return getCouldNotCompute(); 7030 } 7031 7032 // Otherwise we can only handle this if it is affine. 7033 if (!AddRec->isAffine()) 7034 return getCouldNotCompute(); 7035 7036 // If this is an affine expression, the execution count of this branch is 7037 // the minimum unsigned root of the following equation: 7038 // 7039 // Start + Step*N = 0 (mod 2^BW) 7040 // 7041 // equivalent to: 7042 // 7043 // Step*N = -Start (mod 2^BW) 7044 // 7045 // where BW is the common bit width of Start and Step. 7046 7047 // Get the initial value for the loop. 7048 const SCEV *Start = getSCEVAtScope(AddRec->getStart(), L->getParentLoop()); 7049 const SCEV *Step = getSCEVAtScope(AddRec->getOperand(1), L->getParentLoop()); 7050 7051 // For now we handle only constant steps. 7052 // 7053 // TODO: Handle a nonconstant Step given AddRec<NUW>. If the 7054 // AddRec is NUW, then (in an unsigned sense) it cannot be counting up to wrap 7055 // to 0, it must be counting down to equal 0. Consequently, N = Start / -Step. 7056 // We have not yet seen any such cases. 7057 const SCEVConstant *StepC = dyn_cast<SCEVConstant>(Step); 7058 if (!StepC || StepC->getValue()->equalsInt(0)) 7059 return getCouldNotCompute(); 7060 7061 // For positive steps (counting up until unsigned overflow): 7062 // N = -Start/Step (as unsigned) 7063 // For negative steps (counting down to zero): 7064 // N = Start/-Step 7065 // First compute the unsigned distance from zero in the direction of Step. 7066 bool CountDown = StepC->getAPInt().isNegative(); 7067 const SCEV *Distance = CountDown ? Start : getNegativeSCEV(Start); 7068 7069 // Handle unitary steps, which cannot wraparound. 7070 // 1*N = -Start; -1*N = Start (mod 2^BW), so: 7071 // N = Distance (as unsigned) 7072 if (StepC->getValue()->equalsInt(1) || StepC->getValue()->isAllOnesValue()) { 7073 ConstantRange CR = getUnsignedRange(Start); 7074 const SCEV *MaxBECount; 7075 if (!CountDown && CR.getUnsignedMin().isMinValue()) 7076 // When counting up, the worst starting value is 1, not 0. 7077 MaxBECount = CR.getUnsignedMax().isMinValue() 7078 ? getConstant(APInt::getMinValue(CR.getBitWidth())) 7079 : getConstant(APInt::getMaxValue(CR.getBitWidth())); 7080 else 7081 MaxBECount = getConstant(CountDown ? CR.getUnsignedMax() 7082 : -CR.getUnsignedMin()); 7083 return ExitLimit(Distance, MaxBECount, P); 7084 } 7085 7086 // As a special case, handle the instance where Step is a positive power of 7087 // two. In this case, determining whether Step divides Distance evenly can be 7088 // done by counting and comparing the number of trailing zeros of Step and 7089 // Distance. 7090 if (!CountDown) { 7091 const APInt &StepV = StepC->getAPInt(); 7092 // StepV.isPowerOf2() returns true if StepV is an positive power of two. It 7093 // also returns true if StepV is maximally negative (eg, INT_MIN), but that 7094 // case is not handled as this code is guarded by !CountDown. 7095 if (StepV.isPowerOf2() && 7096 GetMinTrailingZeros(Distance) >= StepV.countTrailingZeros()) { 7097 // Here we've constrained the equation to be of the form 7098 // 7099 // 2^(N + k) * Distance' = (StepV == 2^N) * X (mod 2^W) ... (0) 7100 // 7101 // where we're operating on a W bit wide integer domain and k is 7102 // non-negative. The smallest unsigned solution for X is the trip count. 7103 // 7104 // (0) is equivalent to: 7105 // 7106 // 2^(N + k) * Distance' - 2^N * X = L * 2^W 7107 // <=> 2^N(2^k * Distance' - X) = L * 2^(W - N) * 2^N 7108 // <=> 2^k * Distance' - X = L * 2^(W - N) 7109 // <=> 2^k * Distance' = L * 2^(W - N) + X ... (1) 7110 // 7111 // The smallest X satisfying (1) is unsigned remainder of dividing the LHS 7112 // by 2^(W - N). 7113 // 7114 // <=> X = 2^k * Distance' URem 2^(W - N) ... (2) 7115 // 7116 // E.g. say we're solving 7117 // 7118 // 2 * Val = 2 * X (in i8) ... (3) 7119 // 7120 // then from (2), we get X = Val URem i8 128 (k = 0 in this case). 7121 // 7122 // Note: It is tempting to solve (3) by setting X = Val, but Val is not 7123 // necessarily the smallest unsigned value of X that satisfies (3). 7124 // E.g. if Val is i8 -127 then the smallest value of X that satisfies (3) 7125 // is i8 1, not i8 -127 7126 7127 const auto *ModuloResult = getUDivExactExpr(Distance, Step); 7128 7129 // Since SCEV does not have a URem node, we construct one using a truncate 7130 // and a zero extend. 7131 7132 unsigned NarrowWidth = StepV.getBitWidth() - StepV.countTrailingZeros(); 7133 auto *NarrowTy = IntegerType::get(getContext(), NarrowWidth); 7134 auto *WideTy = Distance->getType(); 7135 7136 const SCEV *Limit = 7137 getZeroExtendExpr(getTruncateExpr(ModuloResult, NarrowTy), WideTy); 7138 return ExitLimit(Limit, Limit, P); 7139 } 7140 } 7141 7142 // If the condition controls loop exit (the loop exits only if the expression 7143 // is true) and the addition is no-wrap we can use unsigned divide to 7144 // compute the backedge count. In this case, the step may not divide the 7145 // distance, but we don't care because if the condition is "missed" the loop 7146 // will have undefined behavior due to wrapping. 7147 if (ControlsExit && AddRec->hasNoSelfWrap()) { 7148 const SCEV *Exact = 7149 getUDivExpr(Distance, CountDown ? getNegativeSCEV(Step) : Step); 7150 return ExitLimit(Exact, Exact, P); 7151 } 7152 7153 // Then, try to solve the above equation provided that Start is constant. 7154 if (const SCEVConstant *StartC = dyn_cast<SCEVConstant>(Start)) { 7155 const SCEV *E = SolveLinEquationWithOverflow( 7156 StepC->getValue()->getValue(), -StartC->getValue()->getValue(), *this); 7157 return ExitLimit(E, E, P); 7158 } 7159 return getCouldNotCompute(); 7160 } 7161 7162 /// HowFarToNonZero - Return the number of times a backedge checking the 7163 /// specified value for nonzero will execute. If not computable, return 7164 /// CouldNotCompute 7165 ScalarEvolution::ExitLimit 7166 ScalarEvolution::HowFarToNonZero(const SCEV *V, const Loop *L) { 7167 // Loops that look like: while (X == 0) are very strange indeed. We don't 7168 // handle them yet except for the trivial case. This could be expanded in the 7169 // future as needed. 7170 7171 // If the value is a constant, check to see if it is known to be non-zero 7172 // already. If so, the backedge will execute zero times. 7173 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(V)) { 7174 if (!C->getValue()->isNullValue()) 7175 return getZero(C->getType()); 7176 return getCouldNotCompute(); // Otherwise it will loop infinitely. 7177 } 7178 7179 // We could implement others, but I really doubt anyone writes loops like 7180 // this, and if they did, they would already be constant folded. 7181 return getCouldNotCompute(); 7182 } 7183 7184 /// getPredecessorWithUniqueSuccessorForBB - Return a predecessor of BB 7185 /// (which may not be an immediate predecessor) which has exactly one 7186 /// successor from which BB is reachable, or null if no such block is 7187 /// found. 7188 /// 7189 std::pair<BasicBlock *, BasicBlock *> 7190 ScalarEvolution::getPredecessorWithUniqueSuccessorForBB(BasicBlock *BB) { 7191 // If the block has a unique predecessor, then there is no path from the 7192 // predecessor to the block that does not go through the direct edge 7193 // from the predecessor to the block. 7194 if (BasicBlock *Pred = BB->getSinglePredecessor()) 7195 return {Pred, BB}; 7196 7197 // A loop's header is defined to be a block that dominates the loop. 7198 // If the header has a unique predecessor outside the loop, it must be 7199 // a block that has exactly one successor that can reach the loop. 7200 if (Loop *L = LI.getLoopFor(BB)) 7201 return {L->getLoopPredecessor(), L->getHeader()}; 7202 7203 return {nullptr, nullptr}; 7204 } 7205 7206 /// HasSameValue - SCEV structural equivalence is usually sufficient for 7207 /// testing whether two expressions are equal, however for the purposes of 7208 /// looking for a condition guarding a loop, it can be useful to be a little 7209 /// more general, since a front-end may have replicated the controlling 7210 /// expression. 7211 /// 7212 static bool HasSameValue(const SCEV *A, const SCEV *B) { 7213 // Quick check to see if they are the same SCEV. 7214 if (A == B) return true; 7215 7216 auto ComputesEqualValues = [](const Instruction *A, const Instruction *B) { 7217 // Not all instructions that are "identical" compute the same value. For 7218 // instance, two distinct alloca instructions allocating the same type are 7219 // identical and do not read memory; but compute distinct values. 7220 return A->isIdenticalTo(B) && (isa<BinaryOperator>(A) || isa<GetElementPtrInst>(A)); 7221 }; 7222 7223 // Otherwise, if they're both SCEVUnknown, it's possible that they hold 7224 // two different instructions with the same value. Check for this case. 7225 if (const SCEVUnknown *AU = dyn_cast<SCEVUnknown>(A)) 7226 if (const SCEVUnknown *BU = dyn_cast<SCEVUnknown>(B)) 7227 if (const Instruction *AI = dyn_cast<Instruction>(AU->getValue())) 7228 if (const Instruction *BI = dyn_cast<Instruction>(BU->getValue())) 7229 if (ComputesEqualValues(AI, BI)) 7230 return true; 7231 7232 // Otherwise assume they may have a different value. 7233 return false; 7234 } 7235 7236 /// SimplifyICmpOperands - Simplify LHS and RHS in a comparison with 7237 /// predicate Pred. Return true iff any changes were made. 7238 /// 7239 bool ScalarEvolution::SimplifyICmpOperands(ICmpInst::Predicate &Pred, 7240 const SCEV *&LHS, const SCEV *&RHS, 7241 unsigned Depth) { 7242 bool Changed = false; 7243 7244 // If we hit the max recursion limit bail out. 7245 if (Depth >= 3) 7246 return false; 7247 7248 // Canonicalize a constant to the right side. 7249 if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(LHS)) { 7250 // Check for both operands constant. 7251 if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(RHS)) { 7252 if (ConstantExpr::getICmp(Pred, 7253 LHSC->getValue(), 7254 RHSC->getValue())->isNullValue()) 7255 goto trivially_false; 7256 else 7257 goto trivially_true; 7258 } 7259 // Otherwise swap the operands to put the constant on the right. 7260 std::swap(LHS, RHS); 7261 Pred = ICmpInst::getSwappedPredicate(Pred); 7262 Changed = true; 7263 } 7264 7265 // If we're comparing an addrec with a value which is loop-invariant in the 7266 // addrec's loop, put the addrec on the left. Also make a dominance check, 7267 // as both operands could be addrecs loop-invariant in each other's loop. 7268 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(RHS)) { 7269 const Loop *L = AR->getLoop(); 7270 if (isLoopInvariant(LHS, L) && properlyDominates(LHS, L->getHeader())) { 7271 std::swap(LHS, RHS); 7272 Pred = ICmpInst::getSwappedPredicate(Pred); 7273 Changed = true; 7274 } 7275 } 7276 7277 // If there's a constant operand, canonicalize comparisons with boundary 7278 // cases, and canonicalize *-or-equal comparisons to regular comparisons. 7279 if (const SCEVConstant *RC = dyn_cast<SCEVConstant>(RHS)) { 7280 const APInt &RA = RC->getAPInt(); 7281 switch (Pred) { 7282 default: llvm_unreachable("Unexpected ICmpInst::Predicate value!"); 7283 case ICmpInst::ICMP_EQ: 7284 case ICmpInst::ICMP_NE: 7285 // Fold ((-1) * %a) + %b == 0 (equivalent to %b-%a == 0) into %a == %b. 7286 if (!RA) 7287 if (const SCEVAddExpr *AE = dyn_cast<SCEVAddExpr>(LHS)) 7288 if (const SCEVMulExpr *ME = dyn_cast<SCEVMulExpr>(AE->getOperand(0))) 7289 if (AE->getNumOperands() == 2 && ME->getNumOperands() == 2 && 7290 ME->getOperand(0)->isAllOnesValue()) { 7291 RHS = AE->getOperand(1); 7292 LHS = ME->getOperand(1); 7293 Changed = true; 7294 } 7295 break; 7296 case ICmpInst::ICMP_UGE: 7297 if ((RA - 1).isMinValue()) { 7298 Pred = ICmpInst::ICMP_NE; 7299 RHS = getConstant(RA - 1); 7300 Changed = true; 7301 break; 7302 } 7303 if (RA.isMaxValue()) { 7304 Pred = ICmpInst::ICMP_EQ; 7305 Changed = true; 7306 break; 7307 } 7308 if (RA.isMinValue()) goto trivially_true; 7309 7310 Pred = ICmpInst::ICMP_UGT; 7311 RHS = getConstant(RA - 1); 7312 Changed = true; 7313 break; 7314 case ICmpInst::ICMP_ULE: 7315 if ((RA + 1).isMaxValue()) { 7316 Pred = ICmpInst::ICMP_NE; 7317 RHS = getConstant(RA + 1); 7318 Changed = true; 7319 break; 7320 } 7321 if (RA.isMinValue()) { 7322 Pred = ICmpInst::ICMP_EQ; 7323 Changed = true; 7324 break; 7325 } 7326 if (RA.isMaxValue()) goto trivially_true; 7327 7328 Pred = ICmpInst::ICMP_ULT; 7329 RHS = getConstant(RA + 1); 7330 Changed = true; 7331 break; 7332 case ICmpInst::ICMP_SGE: 7333 if ((RA - 1).isMinSignedValue()) { 7334 Pred = ICmpInst::ICMP_NE; 7335 RHS = getConstant(RA - 1); 7336 Changed = true; 7337 break; 7338 } 7339 if (RA.isMaxSignedValue()) { 7340 Pred = ICmpInst::ICMP_EQ; 7341 Changed = true; 7342 break; 7343 } 7344 if (RA.isMinSignedValue()) goto trivially_true; 7345 7346 Pred = ICmpInst::ICMP_SGT; 7347 RHS = getConstant(RA - 1); 7348 Changed = true; 7349 break; 7350 case ICmpInst::ICMP_SLE: 7351 if ((RA + 1).isMaxSignedValue()) { 7352 Pred = ICmpInst::ICMP_NE; 7353 RHS = getConstant(RA + 1); 7354 Changed = true; 7355 break; 7356 } 7357 if (RA.isMinSignedValue()) { 7358 Pred = ICmpInst::ICMP_EQ; 7359 Changed = true; 7360 break; 7361 } 7362 if (RA.isMaxSignedValue()) goto trivially_true; 7363 7364 Pred = ICmpInst::ICMP_SLT; 7365 RHS = getConstant(RA + 1); 7366 Changed = true; 7367 break; 7368 case ICmpInst::ICMP_UGT: 7369 if (RA.isMinValue()) { 7370 Pred = ICmpInst::ICMP_NE; 7371 Changed = true; 7372 break; 7373 } 7374 if ((RA + 1).isMaxValue()) { 7375 Pred = ICmpInst::ICMP_EQ; 7376 RHS = getConstant(RA + 1); 7377 Changed = true; 7378 break; 7379 } 7380 if (RA.isMaxValue()) goto trivially_false; 7381 break; 7382 case ICmpInst::ICMP_ULT: 7383 if (RA.isMaxValue()) { 7384 Pred = ICmpInst::ICMP_NE; 7385 Changed = true; 7386 break; 7387 } 7388 if ((RA - 1).isMinValue()) { 7389 Pred = ICmpInst::ICMP_EQ; 7390 RHS = getConstant(RA - 1); 7391 Changed = true; 7392 break; 7393 } 7394 if (RA.isMinValue()) goto trivially_false; 7395 break; 7396 case ICmpInst::ICMP_SGT: 7397 if (RA.isMinSignedValue()) { 7398 Pred = ICmpInst::ICMP_NE; 7399 Changed = true; 7400 break; 7401 } 7402 if ((RA + 1).isMaxSignedValue()) { 7403 Pred = ICmpInst::ICMP_EQ; 7404 RHS = getConstant(RA + 1); 7405 Changed = true; 7406 break; 7407 } 7408 if (RA.isMaxSignedValue()) goto trivially_false; 7409 break; 7410 case ICmpInst::ICMP_SLT: 7411 if (RA.isMaxSignedValue()) { 7412 Pred = ICmpInst::ICMP_NE; 7413 Changed = true; 7414 break; 7415 } 7416 if ((RA - 1).isMinSignedValue()) { 7417 Pred = ICmpInst::ICMP_EQ; 7418 RHS = getConstant(RA - 1); 7419 Changed = true; 7420 break; 7421 } 7422 if (RA.isMinSignedValue()) goto trivially_false; 7423 break; 7424 } 7425 } 7426 7427 // Check for obvious equality. 7428 if (HasSameValue(LHS, RHS)) { 7429 if (ICmpInst::isTrueWhenEqual(Pred)) 7430 goto trivially_true; 7431 if (ICmpInst::isFalseWhenEqual(Pred)) 7432 goto trivially_false; 7433 } 7434 7435 // If possible, canonicalize GE/LE comparisons to GT/LT comparisons, by 7436 // adding or subtracting 1 from one of the operands. 7437 switch (Pred) { 7438 case ICmpInst::ICMP_SLE: 7439 if (!getSignedRange(RHS).getSignedMax().isMaxSignedValue()) { 7440 RHS = getAddExpr(getConstant(RHS->getType(), 1, true), RHS, 7441 SCEV::FlagNSW); 7442 Pred = ICmpInst::ICMP_SLT; 7443 Changed = true; 7444 } else if (!getSignedRange(LHS).getSignedMin().isMinSignedValue()) { 7445 LHS = getAddExpr(getConstant(RHS->getType(), (uint64_t)-1, true), LHS, 7446 SCEV::FlagNSW); 7447 Pred = ICmpInst::ICMP_SLT; 7448 Changed = true; 7449 } 7450 break; 7451 case ICmpInst::ICMP_SGE: 7452 if (!getSignedRange(RHS).getSignedMin().isMinSignedValue()) { 7453 RHS = getAddExpr(getConstant(RHS->getType(), (uint64_t)-1, true), RHS, 7454 SCEV::FlagNSW); 7455 Pred = ICmpInst::ICMP_SGT; 7456 Changed = true; 7457 } else if (!getSignedRange(LHS).getSignedMax().isMaxSignedValue()) { 7458 LHS = getAddExpr(getConstant(RHS->getType(), 1, true), LHS, 7459 SCEV::FlagNSW); 7460 Pred = ICmpInst::ICMP_SGT; 7461 Changed = true; 7462 } 7463 break; 7464 case ICmpInst::ICMP_ULE: 7465 if (!getUnsignedRange(RHS).getUnsignedMax().isMaxValue()) { 7466 RHS = getAddExpr(getConstant(RHS->getType(), 1, true), RHS, 7467 SCEV::FlagNUW); 7468 Pred = ICmpInst::ICMP_ULT; 7469 Changed = true; 7470 } else if (!getUnsignedRange(LHS).getUnsignedMin().isMinValue()) { 7471 LHS = getAddExpr(getConstant(RHS->getType(), (uint64_t)-1, true), LHS); 7472 Pred = ICmpInst::ICMP_ULT; 7473 Changed = true; 7474 } 7475 break; 7476 case ICmpInst::ICMP_UGE: 7477 if (!getUnsignedRange(RHS).getUnsignedMin().isMinValue()) { 7478 RHS = getAddExpr(getConstant(RHS->getType(), (uint64_t)-1, true), RHS); 7479 Pred = ICmpInst::ICMP_UGT; 7480 Changed = true; 7481 } else if (!getUnsignedRange(LHS).getUnsignedMax().isMaxValue()) { 7482 LHS = getAddExpr(getConstant(RHS->getType(), 1, true), LHS, 7483 SCEV::FlagNUW); 7484 Pred = ICmpInst::ICMP_UGT; 7485 Changed = true; 7486 } 7487 break; 7488 default: 7489 break; 7490 } 7491 7492 // TODO: More simplifications are possible here. 7493 7494 // Recursively simplify until we either hit a recursion limit or nothing 7495 // changes. 7496 if (Changed) 7497 return SimplifyICmpOperands(Pred, LHS, RHS, Depth+1); 7498 7499 return Changed; 7500 7501 trivially_true: 7502 // Return 0 == 0. 7503 LHS = RHS = getConstant(ConstantInt::getFalse(getContext())); 7504 Pred = ICmpInst::ICMP_EQ; 7505 return true; 7506 7507 trivially_false: 7508 // Return 0 != 0. 7509 LHS = RHS = getConstant(ConstantInt::getFalse(getContext())); 7510 Pred = ICmpInst::ICMP_NE; 7511 return true; 7512 } 7513 7514 bool ScalarEvolution::isKnownNegative(const SCEV *S) { 7515 return getSignedRange(S).getSignedMax().isNegative(); 7516 } 7517 7518 bool ScalarEvolution::isKnownPositive(const SCEV *S) { 7519 return getSignedRange(S).getSignedMin().isStrictlyPositive(); 7520 } 7521 7522 bool ScalarEvolution::isKnownNonNegative(const SCEV *S) { 7523 return !getSignedRange(S).getSignedMin().isNegative(); 7524 } 7525 7526 bool ScalarEvolution::isKnownNonPositive(const SCEV *S) { 7527 return !getSignedRange(S).getSignedMax().isStrictlyPositive(); 7528 } 7529 7530 bool ScalarEvolution::isKnownNonZero(const SCEV *S) { 7531 return isKnownNegative(S) || isKnownPositive(S); 7532 } 7533 7534 bool ScalarEvolution::isKnownPredicate(ICmpInst::Predicate Pred, 7535 const SCEV *LHS, const SCEV *RHS) { 7536 // Canonicalize the inputs first. 7537 (void)SimplifyICmpOperands(Pred, LHS, RHS); 7538 7539 // If LHS or RHS is an addrec, check to see if the condition is true in 7540 // every iteration of the loop. 7541 // If LHS and RHS are both addrec, both conditions must be true in 7542 // every iteration of the loop. 7543 const SCEVAddRecExpr *LAR = dyn_cast<SCEVAddRecExpr>(LHS); 7544 const SCEVAddRecExpr *RAR = dyn_cast<SCEVAddRecExpr>(RHS); 7545 bool LeftGuarded = false; 7546 bool RightGuarded = false; 7547 if (LAR) { 7548 const Loop *L = LAR->getLoop(); 7549 if (isLoopEntryGuardedByCond(L, Pred, LAR->getStart(), RHS) && 7550 isLoopBackedgeGuardedByCond(L, Pred, LAR->getPostIncExpr(*this), RHS)) { 7551 if (!RAR) return true; 7552 LeftGuarded = true; 7553 } 7554 } 7555 if (RAR) { 7556 const Loop *L = RAR->getLoop(); 7557 if (isLoopEntryGuardedByCond(L, Pred, LHS, RAR->getStart()) && 7558 isLoopBackedgeGuardedByCond(L, Pred, LHS, RAR->getPostIncExpr(*this))) { 7559 if (!LAR) return true; 7560 RightGuarded = true; 7561 } 7562 } 7563 if (LeftGuarded && RightGuarded) 7564 return true; 7565 7566 if (isKnownPredicateViaSplitting(Pred, LHS, RHS)) 7567 return true; 7568 7569 // Otherwise see what can be done with known constant ranges. 7570 return isKnownPredicateViaConstantRanges(Pred, LHS, RHS); 7571 } 7572 7573 bool ScalarEvolution::isMonotonicPredicate(const SCEVAddRecExpr *LHS, 7574 ICmpInst::Predicate Pred, 7575 bool &Increasing) { 7576 bool Result = isMonotonicPredicateImpl(LHS, Pred, Increasing); 7577 7578 #ifndef NDEBUG 7579 // Verify an invariant: inverting the predicate should turn a monotonically 7580 // increasing change to a monotonically decreasing one, and vice versa. 7581 bool IncreasingSwapped; 7582 bool ResultSwapped = isMonotonicPredicateImpl( 7583 LHS, ICmpInst::getSwappedPredicate(Pred), IncreasingSwapped); 7584 7585 assert(Result == ResultSwapped && "should be able to analyze both!"); 7586 if (ResultSwapped) 7587 assert(Increasing == !IncreasingSwapped && 7588 "monotonicity should flip as we flip the predicate"); 7589 #endif 7590 7591 return Result; 7592 } 7593 7594 bool ScalarEvolution::isMonotonicPredicateImpl(const SCEVAddRecExpr *LHS, 7595 ICmpInst::Predicate Pred, 7596 bool &Increasing) { 7597 7598 // A zero step value for LHS means the induction variable is essentially a 7599 // loop invariant value. We don't really depend on the predicate actually 7600 // flipping from false to true (for increasing predicates, and the other way 7601 // around for decreasing predicates), all we care about is that *if* the 7602 // predicate changes then it only changes from false to true. 7603 // 7604 // A zero step value in itself is not very useful, but there may be places 7605 // where SCEV can prove X >= 0 but not prove X > 0, so it is helpful to be 7606 // as general as possible. 7607 7608 switch (Pred) { 7609 default: 7610 return false; // Conservative answer 7611 7612 case ICmpInst::ICMP_UGT: 7613 case ICmpInst::ICMP_UGE: 7614 case ICmpInst::ICMP_ULT: 7615 case ICmpInst::ICMP_ULE: 7616 if (!LHS->hasNoUnsignedWrap()) 7617 return false; 7618 7619 Increasing = Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_UGE; 7620 return true; 7621 7622 case ICmpInst::ICMP_SGT: 7623 case ICmpInst::ICMP_SGE: 7624 case ICmpInst::ICMP_SLT: 7625 case ICmpInst::ICMP_SLE: { 7626 if (!LHS->hasNoSignedWrap()) 7627 return false; 7628 7629 const SCEV *Step = LHS->getStepRecurrence(*this); 7630 7631 if (isKnownNonNegative(Step)) { 7632 Increasing = Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SGE; 7633 return true; 7634 } 7635 7636 if (isKnownNonPositive(Step)) { 7637 Increasing = Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SLE; 7638 return true; 7639 } 7640 7641 return false; 7642 } 7643 7644 } 7645 7646 llvm_unreachable("switch has default clause!"); 7647 } 7648 7649 bool ScalarEvolution::isLoopInvariantPredicate( 7650 ICmpInst::Predicate Pred, const SCEV *LHS, const SCEV *RHS, const Loop *L, 7651 ICmpInst::Predicate &InvariantPred, const SCEV *&InvariantLHS, 7652 const SCEV *&InvariantRHS) { 7653 7654 // If there is a loop-invariant, force it into the RHS, otherwise bail out. 7655 if (!isLoopInvariant(RHS, L)) { 7656 if (!isLoopInvariant(LHS, L)) 7657 return false; 7658 7659 std::swap(LHS, RHS); 7660 Pred = ICmpInst::getSwappedPredicate(Pred); 7661 } 7662 7663 const SCEVAddRecExpr *ArLHS = dyn_cast<SCEVAddRecExpr>(LHS); 7664 if (!ArLHS || ArLHS->getLoop() != L) 7665 return false; 7666 7667 bool Increasing; 7668 if (!isMonotonicPredicate(ArLHS, Pred, Increasing)) 7669 return false; 7670 7671 // If the predicate "ArLHS `Pred` RHS" monotonically increases from false to 7672 // true as the loop iterates, and the backedge is control dependent on 7673 // "ArLHS `Pred` RHS" == true then we can reason as follows: 7674 // 7675 // * if the predicate was false in the first iteration then the predicate 7676 // is never evaluated again, since the loop exits without taking the 7677 // backedge. 7678 // * if the predicate was true in the first iteration then it will 7679 // continue to be true for all future iterations since it is 7680 // monotonically increasing. 7681 // 7682 // For both the above possibilities, we can replace the loop varying 7683 // predicate with its value on the first iteration of the loop (which is 7684 // loop invariant). 7685 // 7686 // A similar reasoning applies for a monotonically decreasing predicate, by 7687 // replacing true with false and false with true in the above two bullets. 7688 7689 auto P = Increasing ? Pred : ICmpInst::getInversePredicate(Pred); 7690 7691 if (!isLoopBackedgeGuardedByCond(L, P, LHS, RHS)) 7692 return false; 7693 7694 InvariantPred = Pred; 7695 InvariantLHS = ArLHS->getStart(); 7696 InvariantRHS = RHS; 7697 return true; 7698 } 7699 7700 bool ScalarEvolution::isKnownPredicateViaConstantRanges( 7701 ICmpInst::Predicate Pred, const SCEV *LHS, const SCEV *RHS) { 7702 if (HasSameValue(LHS, RHS)) 7703 return ICmpInst::isTrueWhenEqual(Pred); 7704 7705 // This code is split out from isKnownPredicate because it is called from 7706 // within isLoopEntryGuardedByCond. 7707 7708 auto CheckRanges = 7709 [&](const ConstantRange &RangeLHS, const ConstantRange &RangeRHS) { 7710 return ConstantRange::makeSatisfyingICmpRegion(Pred, RangeRHS) 7711 .contains(RangeLHS); 7712 }; 7713 7714 // The check at the top of the function catches the case where the values are 7715 // known to be equal. 7716 if (Pred == CmpInst::ICMP_EQ) 7717 return false; 7718 7719 if (Pred == CmpInst::ICMP_NE) 7720 return CheckRanges(getSignedRange(LHS), getSignedRange(RHS)) || 7721 CheckRanges(getUnsignedRange(LHS), getUnsignedRange(RHS)) || 7722 isKnownNonZero(getMinusSCEV(LHS, RHS)); 7723 7724 if (CmpInst::isSigned(Pred)) 7725 return CheckRanges(getSignedRange(LHS), getSignedRange(RHS)); 7726 7727 return CheckRanges(getUnsignedRange(LHS), getUnsignedRange(RHS)); 7728 } 7729 7730 bool ScalarEvolution::isKnownPredicateViaNoOverflow(ICmpInst::Predicate Pred, 7731 const SCEV *LHS, 7732 const SCEV *RHS) { 7733 7734 // Match Result to (X + Y)<ExpectedFlags> where Y is a constant integer. 7735 // Return Y via OutY. 7736 auto MatchBinaryAddToConst = 7737 [this](const SCEV *Result, const SCEV *X, APInt &OutY, 7738 SCEV::NoWrapFlags ExpectedFlags) { 7739 const SCEV *NonConstOp, *ConstOp; 7740 SCEV::NoWrapFlags FlagsPresent; 7741 7742 if (!splitBinaryAdd(Result, ConstOp, NonConstOp, FlagsPresent) || 7743 !isa<SCEVConstant>(ConstOp) || NonConstOp != X) 7744 return false; 7745 7746 OutY = cast<SCEVConstant>(ConstOp)->getAPInt(); 7747 return (FlagsPresent & ExpectedFlags) == ExpectedFlags; 7748 }; 7749 7750 APInt C; 7751 7752 switch (Pred) { 7753 default: 7754 break; 7755 7756 case ICmpInst::ICMP_SGE: 7757 std::swap(LHS, RHS); 7758 case ICmpInst::ICMP_SLE: 7759 // X s<= (X + C)<nsw> if C >= 0 7760 if (MatchBinaryAddToConst(RHS, LHS, C, SCEV::FlagNSW) && C.isNonNegative()) 7761 return true; 7762 7763 // (X + C)<nsw> s<= X if C <= 0 7764 if (MatchBinaryAddToConst(LHS, RHS, C, SCEV::FlagNSW) && 7765 !C.isStrictlyPositive()) 7766 return true; 7767 break; 7768 7769 case ICmpInst::ICMP_SGT: 7770 std::swap(LHS, RHS); 7771 case ICmpInst::ICMP_SLT: 7772 // X s< (X + C)<nsw> if C > 0 7773 if (MatchBinaryAddToConst(RHS, LHS, C, SCEV::FlagNSW) && 7774 C.isStrictlyPositive()) 7775 return true; 7776 7777 // (X + C)<nsw> s< X if C < 0 7778 if (MatchBinaryAddToConst(LHS, RHS, C, SCEV::FlagNSW) && C.isNegative()) 7779 return true; 7780 break; 7781 } 7782 7783 return false; 7784 } 7785 7786 bool ScalarEvolution::isKnownPredicateViaSplitting(ICmpInst::Predicate Pred, 7787 const SCEV *LHS, 7788 const SCEV *RHS) { 7789 if (Pred != ICmpInst::ICMP_ULT || ProvingSplitPredicate) 7790 return false; 7791 7792 // Allowing arbitrary number of activations of isKnownPredicateViaSplitting on 7793 // the stack can result in exponential time complexity. 7794 SaveAndRestore<bool> Restore(ProvingSplitPredicate, true); 7795 7796 // If L >= 0 then I `ult` L <=> I >= 0 && I `slt` L 7797 // 7798 // To prove L >= 0 we use isKnownNonNegative whereas to prove I >= 0 we use 7799 // isKnownPredicate. isKnownPredicate is more powerful, but also more 7800 // expensive; and using isKnownNonNegative(RHS) is sufficient for most of the 7801 // interesting cases seen in practice. We can consider "upgrading" L >= 0 to 7802 // use isKnownPredicate later if needed. 7803 return isKnownNonNegative(RHS) && 7804 isKnownPredicate(CmpInst::ICMP_SGE, LHS, getZero(LHS->getType())) && 7805 isKnownPredicate(CmpInst::ICMP_SLT, LHS, RHS); 7806 } 7807 7808 /// isLoopBackedgeGuardedByCond - Test whether the backedge of the loop is 7809 /// protected by a conditional between LHS and RHS. This is used to 7810 /// to eliminate casts. 7811 bool 7812 ScalarEvolution::isLoopBackedgeGuardedByCond(const Loop *L, 7813 ICmpInst::Predicate Pred, 7814 const SCEV *LHS, const SCEV *RHS) { 7815 // Interpret a null as meaning no loop, where there is obviously no guard 7816 // (interprocedural conditions notwithstanding). 7817 if (!L) return true; 7818 7819 if (isKnownPredicateViaConstantRanges(Pred, LHS, RHS)) 7820 return true; 7821 7822 BasicBlock *Latch = L->getLoopLatch(); 7823 if (!Latch) 7824 return false; 7825 7826 BranchInst *LoopContinuePredicate = 7827 dyn_cast<BranchInst>(Latch->getTerminator()); 7828 if (LoopContinuePredicate && LoopContinuePredicate->isConditional() && 7829 isImpliedCond(Pred, LHS, RHS, 7830 LoopContinuePredicate->getCondition(), 7831 LoopContinuePredicate->getSuccessor(0) != L->getHeader())) 7832 return true; 7833 7834 // We don't want more than one activation of the following loops on the stack 7835 // -- that can lead to O(n!) time complexity. 7836 if (WalkingBEDominatingConds) 7837 return false; 7838 7839 SaveAndRestore<bool> ClearOnExit(WalkingBEDominatingConds, true); 7840 7841 // See if we can exploit a trip count to prove the predicate. 7842 const auto &BETakenInfo = getBackedgeTakenInfo(L); 7843 const SCEV *LatchBECount = BETakenInfo.getExact(Latch, this); 7844 if (LatchBECount != getCouldNotCompute()) { 7845 // We know that Latch branches back to the loop header exactly 7846 // LatchBECount times. This means the backdege condition at Latch is 7847 // equivalent to "{0,+,1} u< LatchBECount". 7848 Type *Ty = LatchBECount->getType(); 7849 auto NoWrapFlags = SCEV::NoWrapFlags(SCEV::FlagNUW | SCEV::FlagNW); 7850 const SCEV *LoopCounter = 7851 getAddRecExpr(getZero(Ty), getOne(Ty), L, NoWrapFlags); 7852 if (isImpliedCond(Pred, LHS, RHS, ICmpInst::ICMP_ULT, LoopCounter, 7853 LatchBECount)) 7854 return true; 7855 } 7856 7857 // Check conditions due to any @llvm.assume intrinsics. 7858 for (auto &AssumeVH : AC.assumptions()) { 7859 if (!AssumeVH) 7860 continue; 7861 auto *CI = cast<CallInst>(AssumeVH); 7862 if (!DT.dominates(CI, Latch->getTerminator())) 7863 continue; 7864 7865 if (isImpliedCond(Pred, LHS, RHS, CI->getArgOperand(0), false)) 7866 return true; 7867 } 7868 7869 // If the loop is not reachable from the entry block, we risk running into an 7870 // infinite loop as we walk up into the dom tree. These loops do not matter 7871 // anyway, so we just return a conservative answer when we see them. 7872 if (!DT.isReachableFromEntry(L->getHeader())) 7873 return false; 7874 7875 for (DomTreeNode *DTN = DT[Latch], *HeaderDTN = DT[L->getHeader()]; 7876 DTN != HeaderDTN; DTN = DTN->getIDom()) { 7877 7878 assert(DTN && "should reach the loop header before reaching the root!"); 7879 7880 BasicBlock *BB = DTN->getBlock(); 7881 BasicBlock *PBB = BB->getSinglePredecessor(); 7882 if (!PBB) 7883 continue; 7884 7885 BranchInst *ContinuePredicate = dyn_cast<BranchInst>(PBB->getTerminator()); 7886 if (!ContinuePredicate || !ContinuePredicate->isConditional()) 7887 continue; 7888 7889 Value *Condition = ContinuePredicate->getCondition(); 7890 7891 // If we have an edge `E` within the loop body that dominates the only 7892 // latch, the condition guarding `E` also guards the backedge. This 7893 // reasoning works only for loops with a single latch. 7894 7895 BasicBlockEdge DominatingEdge(PBB, BB); 7896 if (DominatingEdge.isSingleEdge()) { 7897 // We're constructively (and conservatively) enumerating edges within the 7898 // loop body that dominate the latch. The dominator tree better agree 7899 // with us on this: 7900 assert(DT.dominates(DominatingEdge, Latch) && "should be!"); 7901 7902 if (isImpliedCond(Pred, LHS, RHS, Condition, 7903 BB != ContinuePredicate->getSuccessor(0))) 7904 return true; 7905 } 7906 } 7907 7908 return false; 7909 } 7910 7911 /// isLoopEntryGuardedByCond - Test whether entry to the loop is protected 7912 /// by a conditional between LHS and RHS. This is used to help avoid max 7913 /// expressions in loop trip counts, and to eliminate casts. 7914 bool 7915 ScalarEvolution::isLoopEntryGuardedByCond(const Loop *L, 7916 ICmpInst::Predicate Pred, 7917 const SCEV *LHS, const SCEV *RHS) { 7918 // Interpret a null as meaning no loop, where there is obviously no guard 7919 // (interprocedural conditions notwithstanding). 7920 if (!L) return false; 7921 7922 if (isKnownPredicateViaConstantRanges(Pred, LHS, RHS)) 7923 return true; 7924 7925 // Starting at the loop predecessor, climb up the predecessor chain, as long 7926 // as there are predecessors that can be found that have unique successors 7927 // leading to the original header. 7928 for (std::pair<BasicBlock *, BasicBlock *> 7929 Pair(L->getLoopPredecessor(), L->getHeader()); 7930 Pair.first; 7931 Pair = getPredecessorWithUniqueSuccessorForBB(Pair.first)) { 7932 7933 BranchInst *LoopEntryPredicate = 7934 dyn_cast<BranchInst>(Pair.first->getTerminator()); 7935 if (!LoopEntryPredicate || 7936 LoopEntryPredicate->isUnconditional()) 7937 continue; 7938 7939 if (isImpliedCond(Pred, LHS, RHS, 7940 LoopEntryPredicate->getCondition(), 7941 LoopEntryPredicate->getSuccessor(0) != Pair.second)) 7942 return true; 7943 } 7944 7945 // Check conditions due to any @llvm.assume intrinsics. 7946 for (auto &AssumeVH : AC.assumptions()) { 7947 if (!AssumeVH) 7948 continue; 7949 auto *CI = cast<CallInst>(AssumeVH); 7950 if (!DT.dominates(CI, L->getHeader())) 7951 continue; 7952 7953 if (isImpliedCond(Pred, LHS, RHS, CI->getArgOperand(0), false)) 7954 return true; 7955 } 7956 7957 return false; 7958 } 7959 7960 namespace { 7961 /// RAII wrapper to prevent recursive application of isImpliedCond. 7962 /// ScalarEvolution's PendingLoopPredicates set must be empty unless we are 7963 /// currently evaluating isImpliedCond. 7964 struct MarkPendingLoopPredicate { 7965 Value *Cond; 7966 DenseSet<Value*> &LoopPreds; 7967 bool Pending; 7968 7969 MarkPendingLoopPredicate(Value *C, DenseSet<Value*> &LP) 7970 : Cond(C), LoopPreds(LP) { 7971 Pending = !LoopPreds.insert(Cond).second; 7972 } 7973 ~MarkPendingLoopPredicate() { 7974 if (!Pending) 7975 LoopPreds.erase(Cond); 7976 } 7977 }; 7978 } // end anonymous namespace 7979 7980 /// isImpliedCond - Test whether the condition described by Pred, LHS, 7981 /// and RHS is true whenever the given Cond value evaluates to true. 7982 bool ScalarEvolution::isImpliedCond(ICmpInst::Predicate Pred, 7983 const SCEV *LHS, const SCEV *RHS, 7984 Value *FoundCondValue, 7985 bool Inverse) { 7986 MarkPendingLoopPredicate Mark(FoundCondValue, PendingLoopPredicates); 7987 if (Mark.Pending) 7988 return false; 7989 7990 // Recursively handle And and Or conditions. 7991 if (BinaryOperator *BO = dyn_cast<BinaryOperator>(FoundCondValue)) { 7992 if (BO->getOpcode() == Instruction::And) { 7993 if (!Inverse) 7994 return isImpliedCond(Pred, LHS, RHS, BO->getOperand(0), Inverse) || 7995 isImpliedCond(Pred, LHS, RHS, BO->getOperand(1), Inverse); 7996 } else if (BO->getOpcode() == Instruction::Or) { 7997 if (Inverse) 7998 return isImpliedCond(Pred, LHS, RHS, BO->getOperand(0), Inverse) || 7999 isImpliedCond(Pred, LHS, RHS, BO->getOperand(1), Inverse); 8000 } 8001 } 8002 8003 ICmpInst *ICI = dyn_cast<ICmpInst>(FoundCondValue); 8004 if (!ICI) return false; 8005 8006 // Now that we found a conditional branch that dominates the loop or controls 8007 // the loop latch. Check to see if it is the comparison we are looking for. 8008 ICmpInst::Predicate FoundPred; 8009 if (Inverse) 8010 FoundPred = ICI->getInversePredicate(); 8011 else 8012 FoundPred = ICI->getPredicate(); 8013 8014 const SCEV *FoundLHS = getSCEV(ICI->getOperand(0)); 8015 const SCEV *FoundRHS = getSCEV(ICI->getOperand(1)); 8016 8017 return isImpliedCond(Pred, LHS, RHS, FoundPred, FoundLHS, FoundRHS); 8018 } 8019 8020 bool ScalarEvolution::isImpliedCond(ICmpInst::Predicate Pred, const SCEV *LHS, 8021 const SCEV *RHS, 8022 ICmpInst::Predicate FoundPred, 8023 const SCEV *FoundLHS, 8024 const SCEV *FoundRHS) { 8025 // Balance the types. 8026 if (getTypeSizeInBits(LHS->getType()) < 8027 getTypeSizeInBits(FoundLHS->getType())) { 8028 if (CmpInst::isSigned(Pred)) { 8029 LHS = getSignExtendExpr(LHS, FoundLHS->getType()); 8030 RHS = getSignExtendExpr(RHS, FoundLHS->getType()); 8031 } else { 8032 LHS = getZeroExtendExpr(LHS, FoundLHS->getType()); 8033 RHS = getZeroExtendExpr(RHS, FoundLHS->getType()); 8034 } 8035 } else if (getTypeSizeInBits(LHS->getType()) > 8036 getTypeSizeInBits(FoundLHS->getType())) { 8037 if (CmpInst::isSigned(FoundPred)) { 8038 FoundLHS = getSignExtendExpr(FoundLHS, LHS->getType()); 8039 FoundRHS = getSignExtendExpr(FoundRHS, LHS->getType()); 8040 } else { 8041 FoundLHS = getZeroExtendExpr(FoundLHS, LHS->getType()); 8042 FoundRHS = getZeroExtendExpr(FoundRHS, LHS->getType()); 8043 } 8044 } 8045 8046 // Canonicalize the query to match the way instcombine will have 8047 // canonicalized the comparison. 8048 if (SimplifyICmpOperands(Pred, LHS, RHS)) 8049 if (LHS == RHS) 8050 return CmpInst::isTrueWhenEqual(Pred); 8051 if (SimplifyICmpOperands(FoundPred, FoundLHS, FoundRHS)) 8052 if (FoundLHS == FoundRHS) 8053 return CmpInst::isFalseWhenEqual(FoundPred); 8054 8055 // Check to see if we can make the LHS or RHS match. 8056 if (LHS == FoundRHS || RHS == FoundLHS) { 8057 if (isa<SCEVConstant>(RHS)) { 8058 std::swap(FoundLHS, FoundRHS); 8059 FoundPred = ICmpInst::getSwappedPredicate(FoundPred); 8060 } else { 8061 std::swap(LHS, RHS); 8062 Pred = ICmpInst::getSwappedPredicate(Pred); 8063 } 8064 } 8065 8066 // Check whether the found predicate is the same as the desired predicate. 8067 if (FoundPred == Pred) 8068 return isImpliedCondOperands(Pred, LHS, RHS, FoundLHS, FoundRHS); 8069 8070 // Check whether swapping the found predicate makes it the same as the 8071 // desired predicate. 8072 if (ICmpInst::getSwappedPredicate(FoundPred) == Pred) { 8073 if (isa<SCEVConstant>(RHS)) 8074 return isImpliedCondOperands(Pred, LHS, RHS, FoundRHS, FoundLHS); 8075 else 8076 return isImpliedCondOperands(ICmpInst::getSwappedPredicate(Pred), 8077 RHS, LHS, FoundLHS, FoundRHS); 8078 } 8079 8080 // Unsigned comparison is the same as signed comparison when both the operands 8081 // are non-negative. 8082 if (CmpInst::isUnsigned(FoundPred) && 8083 CmpInst::getSignedPredicate(FoundPred) == Pred && 8084 isKnownNonNegative(FoundLHS) && isKnownNonNegative(FoundRHS)) 8085 return isImpliedCondOperands(Pred, LHS, RHS, FoundLHS, FoundRHS); 8086 8087 // Check if we can make progress by sharpening ranges. 8088 if (FoundPred == ICmpInst::ICMP_NE && 8089 (isa<SCEVConstant>(FoundLHS) || isa<SCEVConstant>(FoundRHS))) { 8090 8091 const SCEVConstant *C = nullptr; 8092 const SCEV *V = nullptr; 8093 8094 if (isa<SCEVConstant>(FoundLHS)) { 8095 C = cast<SCEVConstant>(FoundLHS); 8096 V = FoundRHS; 8097 } else { 8098 C = cast<SCEVConstant>(FoundRHS); 8099 V = FoundLHS; 8100 } 8101 8102 // The guarding predicate tells us that C != V. If the known range 8103 // of V is [C, t), we can sharpen the range to [C + 1, t). The 8104 // range we consider has to correspond to same signedness as the 8105 // predicate we're interested in folding. 8106 8107 APInt Min = ICmpInst::isSigned(Pred) ? 8108 getSignedRange(V).getSignedMin() : getUnsignedRange(V).getUnsignedMin(); 8109 8110 if (Min == C->getAPInt()) { 8111 // Given (V >= Min && V != Min) we conclude V >= (Min + 1). 8112 // This is true even if (Min + 1) wraps around -- in case of 8113 // wraparound, (Min + 1) < Min, so (V >= Min => V >= (Min + 1)). 8114 8115 APInt SharperMin = Min + 1; 8116 8117 switch (Pred) { 8118 case ICmpInst::ICMP_SGE: 8119 case ICmpInst::ICMP_UGE: 8120 // We know V `Pred` SharperMin. If this implies LHS `Pred` 8121 // RHS, we're done. 8122 if (isImpliedCondOperands(Pred, LHS, RHS, V, 8123 getConstant(SharperMin))) 8124 return true; 8125 8126 case ICmpInst::ICMP_SGT: 8127 case ICmpInst::ICMP_UGT: 8128 // We know from the range information that (V `Pred` Min || 8129 // V == Min). We know from the guarding condition that !(V 8130 // == Min). This gives us 8131 // 8132 // V `Pred` Min || V == Min && !(V == Min) 8133 // => V `Pred` Min 8134 // 8135 // If V `Pred` Min implies LHS `Pred` RHS, we're done. 8136 8137 if (isImpliedCondOperands(Pred, LHS, RHS, V, getConstant(Min))) 8138 return true; 8139 8140 default: 8141 // No change 8142 break; 8143 } 8144 } 8145 } 8146 8147 // Check whether the actual condition is beyond sufficient. 8148 if (FoundPred == ICmpInst::ICMP_EQ) 8149 if (ICmpInst::isTrueWhenEqual(Pred)) 8150 if (isImpliedCondOperands(Pred, LHS, RHS, FoundLHS, FoundRHS)) 8151 return true; 8152 if (Pred == ICmpInst::ICMP_NE) 8153 if (!ICmpInst::isTrueWhenEqual(FoundPred)) 8154 if (isImpliedCondOperands(FoundPred, LHS, RHS, FoundLHS, FoundRHS)) 8155 return true; 8156 8157 // Otherwise assume the worst. 8158 return false; 8159 } 8160 8161 bool ScalarEvolution::splitBinaryAdd(const SCEV *Expr, 8162 const SCEV *&L, const SCEV *&R, 8163 SCEV::NoWrapFlags &Flags) { 8164 const auto *AE = dyn_cast<SCEVAddExpr>(Expr); 8165 if (!AE || AE->getNumOperands() != 2) 8166 return false; 8167 8168 L = AE->getOperand(0); 8169 R = AE->getOperand(1); 8170 Flags = AE->getNoWrapFlags(); 8171 return true; 8172 } 8173 8174 bool ScalarEvolution::computeConstantDifference(const SCEV *Less, 8175 const SCEV *More, 8176 APInt &C) { 8177 // We avoid subtracting expressions here because this function is usually 8178 // fairly deep in the call stack (i.e. is called many times). 8179 8180 if (isa<SCEVAddRecExpr>(Less) && isa<SCEVAddRecExpr>(More)) { 8181 const auto *LAR = cast<SCEVAddRecExpr>(Less); 8182 const auto *MAR = cast<SCEVAddRecExpr>(More); 8183 8184 if (LAR->getLoop() != MAR->getLoop()) 8185 return false; 8186 8187 // We look at affine expressions only; not for correctness but to keep 8188 // getStepRecurrence cheap. 8189 if (!LAR->isAffine() || !MAR->isAffine()) 8190 return false; 8191 8192 if (LAR->getStepRecurrence(*this) != MAR->getStepRecurrence(*this)) 8193 return false; 8194 8195 Less = LAR->getStart(); 8196 More = MAR->getStart(); 8197 8198 // fall through 8199 } 8200 8201 if (isa<SCEVConstant>(Less) && isa<SCEVConstant>(More)) { 8202 const auto &M = cast<SCEVConstant>(More)->getAPInt(); 8203 const auto &L = cast<SCEVConstant>(Less)->getAPInt(); 8204 C = M - L; 8205 return true; 8206 } 8207 8208 const SCEV *L, *R; 8209 SCEV::NoWrapFlags Flags; 8210 if (splitBinaryAdd(Less, L, R, Flags)) 8211 if (const auto *LC = dyn_cast<SCEVConstant>(L)) 8212 if (R == More) { 8213 C = -(LC->getAPInt()); 8214 return true; 8215 } 8216 8217 if (splitBinaryAdd(More, L, R, Flags)) 8218 if (const auto *LC = dyn_cast<SCEVConstant>(L)) 8219 if (R == Less) { 8220 C = LC->getAPInt(); 8221 return true; 8222 } 8223 8224 return false; 8225 } 8226 8227 bool ScalarEvolution::isImpliedCondOperandsViaNoOverflow( 8228 ICmpInst::Predicate Pred, const SCEV *LHS, const SCEV *RHS, 8229 const SCEV *FoundLHS, const SCEV *FoundRHS) { 8230 if (Pred != CmpInst::ICMP_SLT && Pred != CmpInst::ICMP_ULT) 8231 return false; 8232 8233 const auto *AddRecLHS = dyn_cast<SCEVAddRecExpr>(LHS); 8234 if (!AddRecLHS) 8235 return false; 8236 8237 const auto *AddRecFoundLHS = dyn_cast<SCEVAddRecExpr>(FoundLHS); 8238 if (!AddRecFoundLHS) 8239 return false; 8240 8241 // We'd like to let SCEV reason about control dependencies, so we constrain 8242 // both the inequalities to be about add recurrences on the same loop. This 8243 // way we can use isLoopEntryGuardedByCond later. 8244 8245 const Loop *L = AddRecFoundLHS->getLoop(); 8246 if (L != AddRecLHS->getLoop()) 8247 return false; 8248 8249 // FoundLHS u< FoundRHS u< -C => (FoundLHS + C) u< (FoundRHS + C) ... (1) 8250 // 8251 // FoundLHS s< FoundRHS s< INT_MIN - C => (FoundLHS + C) s< (FoundRHS + C) 8252 // ... (2) 8253 // 8254 // Informal proof for (2), assuming (1) [*]: 8255 // 8256 // We'll also assume (A s< B) <=> ((A + INT_MIN) u< (B + INT_MIN)) ... (3)[**] 8257 // 8258 // Then 8259 // 8260 // FoundLHS s< FoundRHS s< INT_MIN - C 8261 // <=> (FoundLHS + INT_MIN) u< (FoundRHS + INT_MIN) u< -C [ using (3) ] 8262 // <=> (FoundLHS + INT_MIN + C) u< (FoundRHS + INT_MIN + C) [ using (1) ] 8263 // <=> (FoundLHS + INT_MIN + C + INT_MIN) s< 8264 // (FoundRHS + INT_MIN + C + INT_MIN) [ using (3) ] 8265 // <=> FoundLHS + C s< FoundRHS + C 8266 // 8267 // [*]: (1) can be proved by ruling out overflow. 8268 // 8269 // [**]: This can be proved by analyzing all the four possibilities: 8270 // (A s< 0, B s< 0), (A s< 0, B s>= 0), (A s>= 0, B s< 0) and 8271 // (A s>= 0, B s>= 0). 8272 // 8273 // Note: 8274 // Despite (2), "FoundRHS s< INT_MIN - C" does not mean that "FoundRHS + C" 8275 // will not sign underflow. For instance, say FoundLHS = (i8 -128), FoundRHS 8276 // = (i8 -127) and C = (i8 -100). Then INT_MIN - C = (i8 -28), and FoundRHS 8277 // s< (INT_MIN - C). Lack of sign overflow / underflow in "FoundRHS + C" is 8278 // neither necessary nor sufficient to prove "(FoundLHS + C) s< (FoundRHS + 8279 // C)". 8280 8281 APInt LDiff, RDiff; 8282 if (!computeConstantDifference(FoundLHS, LHS, LDiff) || 8283 !computeConstantDifference(FoundRHS, RHS, RDiff) || 8284 LDiff != RDiff) 8285 return false; 8286 8287 if (LDiff == 0) 8288 return true; 8289 8290 APInt FoundRHSLimit; 8291 8292 if (Pred == CmpInst::ICMP_ULT) { 8293 FoundRHSLimit = -RDiff; 8294 } else { 8295 assert(Pred == CmpInst::ICMP_SLT && "Checked above!"); 8296 FoundRHSLimit = APInt::getSignedMinValue(getTypeSizeInBits(RHS->getType())) - RDiff; 8297 } 8298 8299 // Try to prove (1) or (2), as needed. 8300 return isLoopEntryGuardedByCond(L, Pred, FoundRHS, 8301 getConstant(FoundRHSLimit)); 8302 } 8303 8304 /// isImpliedCondOperands - Test whether the condition described by Pred, 8305 /// LHS, and RHS is true whenever the condition described by Pred, FoundLHS, 8306 /// and FoundRHS is true. 8307 bool ScalarEvolution::isImpliedCondOperands(ICmpInst::Predicate Pred, 8308 const SCEV *LHS, const SCEV *RHS, 8309 const SCEV *FoundLHS, 8310 const SCEV *FoundRHS) { 8311 if (isImpliedCondOperandsViaRanges(Pred, LHS, RHS, FoundLHS, FoundRHS)) 8312 return true; 8313 8314 if (isImpliedCondOperandsViaNoOverflow(Pred, LHS, RHS, FoundLHS, FoundRHS)) 8315 return true; 8316 8317 return isImpliedCondOperandsHelper(Pred, LHS, RHS, 8318 FoundLHS, FoundRHS) || 8319 // ~x < ~y --> x > y 8320 isImpliedCondOperandsHelper(Pred, LHS, RHS, 8321 getNotSCEV(FoundRHS), 8322 getNotSCEV(FoundLHS)); 8323 } 8324 8325 8326 /// If Expr computes ~A, return A else return nullptr 8327 static const SCEV *MatchNotExpr(const SCEV *Expr) { 8328 const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Expr); 8329 if (!Add || Add->getNumOperands() != 2 || 8330 !Add->getOperand(0)->isAllOnesValue()) 8331 return nullptr; 8332 8333 const SCEVMulExpr *AddRHS = dyn_cast<SCEVMulExpr>(Add->getOperand(1)); 8334 if (!AddRHS || AddRHS->getNumOperands() != 2 || 8335 !AddRHS->getOperand(0)->isAllOnesValue()) 8336 return nullptr; 8337 8338 return AddRHS->getOperand(1); 8339 } 8340 8341 8342 /// Is MaybeMaxExpr an SMax or UMax of Candidate and some other values? 8343 template<typename MaxExprType> 8344 static bool IsMaxConsistingOf(const SCEV *MaybeMaxExpr, 8345 const SCEV *Candidate) { 8346 const MaxExprType *MaxExpr = dyn_cast<MaxExprType>(MaybeMaxExpr); 8347 if (!MaxExpr) return false; 8348 8349 return find(MaxExpr->operands(), Candidate) != MaxExpr->op_end(); 8350 } 8351 8352 8353 /// Is MaybeMinExpr an SMin or UMin of Candidate and some other values? 8354 template<typename MaxExprType> 8355 static bool IsMinConsistingOf(ScalarEvolution &SE, 8356 const SCEV *MaybeMinExpr, 8357 const SCEV *Candidate) { 8358 const SCEV *MaybeMaxExpr = MatchNotExpr(MaybeMinExpr); 8359 if (!MaybeMaxExpr) 8360 return false; 8361 8362 return IsMaxConsistingOf<MaxExprType>(MaybeMaxExpr, SE.getNotSCEV(Candidate)); 8363 } 8364 8365 static bool IsKnownPredicateViaAddRecStart(ScalarEvolution &SE, 8366 ICmpInst::Predicate Pred, 8367 const SCEV *LHS, const SCEV *RHS) { 8368 8369 // If both sides are affine addrecs for the same loop, with equal 8370 // steps, and we know the recurrences don't wrap, then we only 8371 // need to check the predicate on the starting values. 8372 8373 if (!ICmpInst::isRelational(Pred)) 8374 return false; 8375 8376 const SCEVAddRecExpr *LAR = dyn_cast<SCEVAddRecExpr>(LHS); 8377 if (!LAR) 8378 return false; 8379 const SCEVAddRecExpr *RAR = dyn_cast<SCEVAddRecExpr>(RHS); 8380 if (!RAR) 8381 return false; 8382 if (LAR->getLoop() != RAR->getLoop()) 8383 return false; 8384 if (!LAR->isAffine() || !RAR->isAffine()) 8385 return false; 8386 8387 if (LAR->getStepRecurrence(SE) != RAR->getStepRecurrence(SE)) 8388 return false; 8389 8390 SCEV::NoWrapFlags NW = ICmpInst::isSigned(Pred) ? 8391 SCEV::FlagNSW : SCEV::FlagNUW; 8392 if (!LAR->getNoWrapFlags(NW) || !RAR->getNoWrapFlags(NW)) 8393 return false; 8394 8395 return SE.isKnownPredicate(Pred, LAR->getStart(), RAR->getStart()); 8396 } 8397 8398 /// Is LHS `Pred` RHS true on the virtue of LHS or RHS being a Min or Max 8399 /// expression? 8400 static bool IsKnownPredicateViaMinOrMax(ScalarEvolution &SE, 8401 ICmpInst::Predicate Pred, 8402 const SCEV *LHS, const SCEV *RHS) { 8403 switch (Pred) { 8404 default: 8405 return false; 8406 8407 case ICmpInst::ICMP_SGE: 8408 std::swap(LHS, RHS); 8409 // fall through 8410 case ICmpInst::ICMP_SLE: 8411 return 8412 // min(A, ...) <= A 8413 IsMinConsistingOf<SCEVSMaxExpr>(SE, LHS, RHS) || 8414 // A <= max(A, ...) 8415 IsMaxConsistingOf<SCEVSMaxExpr>(RHS, LHS); 8416 8417 case ICmpInst::ICMP_UGE: 8418 std::swap(LHS, RHS); 8419 // fall through 8420 case ICmpInst::ICMP_ULE: 8421 return 8422 // min(A, ...) <= A 8423 IsMinConsistingOf<SCEVUMaxExpr>(SE, LHS, RHS) || 8424 // A <= max(A, ...) 8425 IsMaxConsistingOf<SCEVUMaxExpr>(RHS, LHS); 8426 } 8427 8428 llvm_unreachable("covered switch fell through?!"); 8429 } 8430 8431 /// isImpliedCondOperandsHelper - Test whether the condition described by 8432 /// Pred, LHS, and RHS is true whenever the condition described by Pred, 8433 /// FoundLHS, and FoundRHS is true. 8434 bool 8435 ScalarEvolution::isImpliedCondOperandsHelper(ICmpInst::Predicate Pred, 8436 const SCEV *LHS, const SCEV *RHS, 8437 const SCEV *FoundLHS, 8438 const SCEV *FoundRHS) { 8439 auto IsKnownPredicateFull = 8440 [this](ICmpInst::Predicate Pred, const SCEV *LHS, const SCEV *RHS) { 8441 return isKnownPredicateViaConstantRanges(Pred, LHS, RHS) || 8442 IsKnownPredicateViaMinOrMax(*this, Pred, LHS, RHS) || 8443 IsKnownPredicateViaAddRecStart(*this, Pred, LHS, RHS) || 8444 isKnownPredicateViaNoOverflow(Pred, LHS, RHS); 8445 }; 8446 8447 switch (Pred) { 8448 default: llvm_unreachable("Unexpected ICmpInst::Predicate value!"); 8449 case ICmpInst::ICMP_EQ: 8450 case ICmpInst::ICMP_NE: 8451 if (HasSameValue(LHS, FoundLHS) && HasSameValue(RHS, FoundRHS)) 8452 return true; 8453 break; 8454 case ICmpInst::ICMP_SLT: 8455 case ICmpInst::ICMP_SLE: 8456 if (IsKnownPredicateFull(ICmpInst::ICMP_SLE, LHS, FoundLHS) && 8457 IsKnownPredicateFull(ICmpInst::ICMP_SGE, RHS, FoundRHS)) 8458 return true; 8459 break; 8460 case ICmpInst::ICMP_SGT: 8461 case ICmpInst::ICMP_SGE: 8462 if (IsKnownPredicateFull(ICmpInst::ICMP_SGE, LHS, FoundLHS) && 8463 IsKnownPredicateFull(ICmpInst::ICMP_SLE, RHS, FoundRHS)) 8464 return true; 8465 break; 8466 case ICmpInst::ICMP_ULT: 8467 case ICmpInst::ICMP_ULE: 8468 if (IsKnownPredicateFull(ICmpInst::ICMP_ULE, LHS, FoundLHS) && 8469 IsKnownPredicateFull(ICmpInst::ICMP_UGE, RHS, FoundRHS)) 8470 return true; 8471 break; 8472 case ICmpInst::ICMP_UGT: 8473 case ICmpInst::ICMP_UGE: 8474 if (IsKnownPredicateFull(ICmpInst::ICMP_UGE, LHS, FoundLHS) && 8475 IsKnownPredicateFull(ICmpInst::ICMP_ULE, RHS, FoundRHS)) 8476 return true; 8477 break; 8478 } 8479 8480 return false; 8481 } 8482 8483 /// isImpliedCondOperandsViaRanges - helper function for isImpliedCondOperands. 8484 /// Tries to get cases like "X `sgt` 0 => X - 1 `sgt` -1". 8485 bool ScalarEvolution::isImpliedCondOperandsViaRanges(ICmpInst::Predicate Pred, 8486 const SCEV *LHS, 8487 const SCEV *RHS, 8488 const SCEV *FoundLHS, 8489 const SCEV *FoundRHS) { 8490 if (!isa<SCEVConstant>(RHS) || !isa<SCEVConstant>(FoundRHS)) 8491 // The restriction on `FoundRHS` be lifted easily -- it exists only to 8492 // reduce the compile time impact of this optimization. 8493 return false; 8494 8495 const SCEVAddExpr *AddLHS = dyn_cast<SCEVAddExpr>(LHS); 8496 if (!AddLHS || AddLHS->getOperand(1) != FoundLHS || 8497 !isa<SCEVConstant>(AddLHS->getOperand(0))) 8498 return false; 8499 8500 APInt ConstFoundRHS = cast<SCEVConstant>(FoundRHS)->getAPInt(); 8501 8502 // `FoundLHSRange` is the range we know `FoundLHS` to be in by virtue of the 8503 // antecedent "`FoundLHS` `Pred` `FoundRHS`". 8504 ConstantRange FoundLHSRange = 8505 ConstantRange::makeAllowedICmpRegion(Pred, ConstFoundRHS); 8506 8507 // Since `LHS` is `FoundLHS` + `AddLHS->getOperand(0)`, we can compute a range 8508 // for `LHS`: 8509 APInt Addend = cast<SCEVConstant>(AddLHS->getOperand(0))->getAPInt(); 8510 ConstantRange LHSRange = FoundLHSRange.add(ConstantRange(Addend)); 8511 8512 // We can also compute the range of values for `LHS` that satisfy the 8513 // consequent, "`LHS` `Pred` `RHS`": 8514 APInt ConstRHS = cast<SCEVConstant>(RHS)->getAPInt(); 8515 ConstantRange SatisfyingLHSRange = 8516 ConstantRange::makeSatisfyingICmpRegion(Pred, ConstRHS); 8517 8518 // The antecedent implies the consequent if every value of `LHS` that 8519 // satisfies the antecedent also satisfies the consequent. 8520 return SatisfyingLHSRange.contains(LHSRange); 8521 } 8522 8523 // Verify if an linear IV with positive stride can overflow when in a 8524 // less-than comparison, knowing the invariant term of the comparison, the 8525 // stride and the knowledge of NSW/NUW flags on the recurrence. 8526 bool ScalarEvolution::doesIVOverflowOnLT(const SCEV *RHS, const SCEV *Stride, 8527 bool IsSigned, bool NoWrap) { 8528 if (NoWrap) return false; 8529 8530 unsigned BitWidth = getTypeSizeInBits(RHS->getType()); 8531 const SCEV *One = getOne(Stride->getType()); 8532 8533 if (IsSigned) { 8534 APInt MaxRHS = getSignedRange(RHS).getSignedMax(); 8535 APInt MaxValue = APInt::getSignedMaxValue(BitWidth); 8536 APInt MaxStrideMinusOne = getSignedRange(getMinusSCEV(Stride, One)) 8537 .getSignedMax(); 8538 8539 // SMaxRHS + SMaxStrideMinusOne > SMaxValue => overflow! 8540 return (MaxValue - MaxStrideMinusOne).slt(MaxRHS); 8541 } 8542 8543 APInt MaxRHS = getUnsignedRange(RHS).getUnsignedMax(); 8544 APInt MaxValue = APInt::getMaxValue(BitWidth); 8545 APInt MaxStrideMinusOne = getUnsignedRange(getMinusSCEV(Stride, One)) 8546 .getUnsignedMax(); 8547 8548 // UMaxRHS + UMaxStrideMinusOne > UMaxValue => overflow! 8549 return (MaxValue - MaxStrideMinusOne).ult(MaxRHS); 8550 } 8551 8552 // Verify if an linear IV with negative stride can overflow when in a 8553 // greater-than comparison, knowing the invariant term of the comparison, 8554 // the stride and the knowledge of NSW/NUW flags on the recurrence. 8555 bool ScalarEvolution::doesIVOverflowOnGT(const SCEV *RHS, const SCEV *Stride, 8556 bool IsSigned, bool NoWrap) { 8557 if (NoWrap) return false; 8558 8559 unsigned BitWidth = getTypeSizeInBits(RHS->getType()); 8560 const SCEV *One = getOne(Stride->getType()); 8561 8562 if (IsSigned) { 8563 APInt MinRHS = getSignedRange(RHS).getSignedMin(); 8564 APInt MinValue = APInt::getSignedMinValue(BitWidth); 8565 APInt MaxStrideMinusOne = getSignedRange(getMinusSCEV(Stride, One)) 8566 .getSignedMax(); 8567 8568 // SMinRHS - SMaxStrideMinusOne < SMinValue => overflow! 8569 return (MinValue + MaxStrideMinusOne).sgt(MinRHS); 8570 } 8571 8572 APInt MinRHS = getUnsignedRange(RHS).getUnsignedMin(); 8573 APInt MinValue = APInt::getMinValue(BitWidth); 8574 APInt MaxStrideMinusOne = getUnsignedRange(getMinusSCEV(Stride, One)) 8575 .getUnsignedMax(); 8576 8577 // UMinRHS - UMaxStrideMinusOne < UMinValue => overflow! 8578 return (MinValue + MaxStrideMinusOne).ugt(MinRHS); 8579 } 8580 8581 // Compute the backedge taken count knowing the interval difference, the 8582 // stride and presence of the equality in the comparison. 8583 const SCEV *ScalarEvolution::computeBECount(const SCEV *Delta, const SCEV *Step, 8584 bool Equality) { 8585 const SCEV *One = getOne(Step->getType()); 8586 Delta = Equality ? getAddExpr(Delta, Step) 8587 : getAddExpr(Delta, getMinusSCEV(Step, One)); 8588 return getUDivExpr(Delta, Step); 8589 } 8590 8591 /// HowManyLessThans - Return the number of times a backedge containing the 8592 /// specified less-than comparison will execute. If not computable, return 8593 /// CouldNotCompute. 8594 /// 8595 /// @param ControlsExit is true when the LHS < RHS condition directly controls 8596 /// the branch (loops exits only if condition is true). In this case, we can use 8597 /// NoWrapFlags to skip overflow checks. 8598 ScalarEvolution::ExitLimit 8599 ScalarEvolution::HowManyLessThans(const SCEV *LHS, const SCEV *RHS, 8600 const Loop *L, bool IsSigned, 8601 bool ControlsExit, bool AllowPredicates) { 8602 SCEVUnionPredicate P; 8603 // We handle only IV < Invariant 8604 if (!isLoopInvariant(RHS, L)) 8605 return getCouldNotCompute(); 8606 8607 const SCEVAddRecExpr *IV = dyn_cast<SCEVAddRecExpr>(LHS); 8608 if (!IV && AllowPredicates) 8609 // Try to make this an AddRec using runtime tests, in the first X 8610 // iterations of this loop, where X is the SCEV expression found by the 8611 // algorithm below. 8612 IV = convertSCEVToAddRecWithPredicates(LHS, L, P); 8613 8614 // Avoid weird loops 8615 if (!IV || IV->getLoop() != L || !IV->isAffine()) 8616 return getCouldNotCompute(); 8617 8618 bool NoWrap = ControlsExit && 8619 IV->getNoWrapFlags(IsSigned ? SCEV::FlagNSW : SCEV::FlagNUW); 8620 8621 const SCEV *Stride = IV->getStepRecurrence(*this); 8622 8623 // Avoid negative or zero stride values 8624 if (!isKnownPositive(Stride)) 8625 return getCouldNotCompute(); 8626 8627 // Avoid proven overflow cases: this will ensure that the backedge taken count 8628 // will not generate any unsigned overflow. Relaxed no-overflow conditions 8629 // exploit NoWrapFlags, allowing to optimize in presence of undefined 8630 // behaviors like the case of C language. 8631 if (!Stride->isOne() && doesIVOverflowOnLT(RHS, Stride, IsSigned, NoWrap)) 8632 return getCouldNotCompute(); 8633 8634 ICmpInst::Predicate Cond = IsSigned ? ICmpInst::ICMP_SLT 8635 : ICmpInst::ICMP_ULT; 8636 const SCEV *Start = IV->getStart(); 8637 const SCEV *End = RHS; 8638 if (!isLoopEntryGuardedByCond(L, Cond, getMinusSCEV(Start, Stride), RHS)) { 8639 const SCEV *Diff = getMinusSCEV(RHS, Start); 8640 // If we have NoWrap set, then we can assume that the increment won't 8641 // overflow, in which case if RHS - Start is a constant, we don't need to 8642 // do a max operation since we can just figure it out statically 8643 if (NoWrap && isa<SCEVConstant>(Diff)) { 8644 APInt D = dyn_cast<const SCEVConstant>(Diff)->getAPInt(); 8645 if (D.isNegative()) 8646 End = Start; 8647 } else 8648 End = IsSigned ? getSMaxExpr(RHS, Start) 8649 : getUMaxExpr(RHS, Start); 8650 } 8651 8652 const SCEV *BECount = computeBECount(getMinusSCEV(End, Start), Stride, false); 8653 8654 APInt MinStart = IsSigned ? getSignedRange(Start).getSignedMin() 8655 : getUnsignedRange(Start).getUnsignedMin(); 8656 8657 APInt MinStride = IsSigned ? getSignedRange(Stride).getSignedMin() 8658 : getUnsignedRange(Stride).getUnsignedMin(); 8659 8660 unsigned BitWidth = getTypeSizeInBits(LHS->getType()); 8661 APInt Limit = IsSigned ? APInt::getSignedMaxValue(BitWidth) - (MinStride - 1) 8662 : APInt::getMaxValue(BitWidth) - (MinStride - 1); 8663 8664 // Although End can be a MAX expression we estimate MaxEnd considering only 8665 // the case End = RHS. This is safe because in the other case (End - Start) 8666 // is zero, leading to a zero maximum backedge taken count. 8667 APInt MaxEnd = 8668 IsSigned ? APIntOps::smin(getSignedRange(RHS).getSignedMax(), Limit) 8669 : APIntOps::umin(getUnsignedRange(RHS).getUnsignedMax(), Limit); 8670 8671 const SCEV *MaxBECount; 8672 if (isa<SCEVConstant>(BECount)) 8673 MaxBECount = BECount; 8674 else 8675 MaxBECount = computeBECount(getConstant(MaxEnd - MinStart), 8676 getConstant(MinStride), false); 8677 8678 if (isa<SCEVCouldNotCompute>(MaxBECount)) 8679 MaxBECount = BECount; 8680 8681 return ExitLimit(BECount, MaxBECount, P); 8682 } 8683 8684 ScalarEvolution::ExitLimit 8685 ScalarEvolution::HowManyGreaterThans(const SCEV *LHS, const SCEV *RHS, 8686 const Loop *L, bool IsSigned, 8687 bool ControlsExit, bool AllowPredicates) { 8688 SCEVUnionPredicate P; 8689 // We handle only IV > Invariant 8690 if (!isLoopInvariant(RHS, L)) 8691 return getCouldNotCompute(); 8692 8693 const SCEVAddRecExpr *IV = dyn_cast<SCEVAddRecExpr>(LHS); 8694 if (!IV && AllowPredicates) 8695 // Try to make this an AddRec using runtime tests, in the first X 8696 // iterations of this loop, where X is the SCEV expression found by the 8697 // algorithm below. 8698 IV = convertSCEVToAddRecWithPredicates(LHS, L, P); 8699 8700 // Avoid weird loops 8701 if (!IV || IV->getLoop() != L || !IV->isAffine()) 8702 return getCouldNotCompute(); 8703 8704 bool NoWrap = ControlsExit && 8705 IV->getNoWrapFlags(IsSigned ? SCEV::FlagNSW : SCEV::FlagNUW); 8706 8707 const SCEV *Stride = getNegativeSCEV(IV->getStepRecurrence(*this)); 8708 8709 // Avoid negative or zero stride values 8710 if (!isKnownPositive(Stride)) 8711 return getCouldNotCompute(); 8712 8713 // Avoid proven overflow cases: this will ensure that the backedge taken count 8714 // will not generate any unsigned overflow. Relaxed no-overflow conditions 8715 // exploit NoWrapFlags, allowing to optimize in presence of undefined 8716 // behaviors like the case of C language. 8717 if (!Stride->isOne() && doesIVOverflowOnGT(RHS, Stride, IsSigned, NoWrap)) 8718 return getCouldNotCompute(); 8719 8720 ICmpInst::Predicate Cond = IsSigned ? ICmpInst::ICMP_SGT 8721 : ICmpInst::ICMP_UGT; 8722 8723 const SCEV *Start = IV->getStart(); 8724 const SCEV *End = RHS; 8725 if (!isLoopEntryGuardedByCond(L, Cond, getAddExpr(Start, Stride), RHS)) { 8726 const SCEV *Diff = getMinusSCEV(RHS, Start); 8727 // If we have NoWrap set, then we can assume that the increment won't 8728 // overflow, in which case if RHS - Start is a constant, we don't need to 8729 // do a max operation since we can just figure it out statically 8730 if (NoWrap && isa<SCEVConstant>(Diff)) { 8731 APInt D = dyn_cast<const SCEVConstant>(Diff)->getAPInt(); 8732 if (!D.isNegative()) 8733 End = Start; 8734 } else 8735 End = IsSigned ? getSMinExpr(RHS, Start) 8736 : getUMinExpr(RHS, Start); 8737 } 8738 8739 const SCEV *BECount = computeBECount(getMinusSCEV(Start, End), Stride, false); 8740 8741 APInt MaxStart = IsSigned ? getSignedRange(Start).getSignedMax() 8742 : getUnsignedRange(Start).getUnsignedMax(); 8743 8744 APInt MinStride = IsSigned ? getSignedRange(Stride).getSignedMin() 8745 : getUnsignedRange(Stride).getUnsignedMin(); 8746 8747 unsigned BitWidth = getTypeSizeInBits(LHS->getType()); 8748 APInt Limit = IsSigned ? APInt::getSignedMinValue(BitWidth) + (MinStride - 1) 8749 : APInt::getMinValue(BitWidth) + (MinStride - 1); 8750 8751 // Although End can be a MIN expression we estimate MinEnd considering only 8752 // the case End = RHS. This is safe because in the other case (Start - End) 8753 // is zero, leading to a zero maximum backedge taken count. 8754 APInt MinEnd = 8755 IsSigned ? APIntOps::smax(getSignedRange(RHS).getSignedMin(), Limit) 8756 : APIntOps::umax(getUnsignedRange(RHS).getUnsignedMin(), Limit); 8757 8758 8759 const SCEV *MaxBECount = getCouldNotCompute(); 8760 if (isa<SCEVConstant>(BECount)) 8761 MaxBECount = BECount; 8762 else 8763 MaxBECount = computeBECount(getConstant(MaxStart - MinEnd), 8764 getConstant(MinStride), false); 8765 8766 if (isa<SCEVCouldNotCompute>(MaxBECount)) 8767 MaxBECount = BECount; 8768 8769 return ExitLimit(BECount, MaxBECount, P); 8770 } 8771 8772 /// getNumIterationsInRange - Return the number of iterations of this loop that 8773 /// produce values in the specified constant range. Another way of looking at 8774 /// this is that it returns the first iteration number where the value is not in 8775 /// the condition, thus computing the exit count. If the iteration count can't 8776 /// be computed, an instance of SCEVCouldNotCompute is returned. 8777 const SCEV *SCEVAddRecExpr::getNumIterationsInRange(ConstantRange Range, 8778 ScalarEvolution &SE) const { 8779 if (Range.isFullSet()) // Infinite loop. 8780 return SE.getCouldNotCompute(); 8781 8782 // If the start is a non-zero constant, shift the range to simplify things. 8783 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(getStart())) 8784 if (!SC->getValue()->isZero()) { 8785 SmallVector<const SCEV *, 4> Operands(op_begin(), op_end()); 8786 Operands[0] = SE.getZero(SC->getType()); 8787 const SCEV *Shifted = SE.getAddRecExpr(Operands, getLoop(), 8788 getNoWrapFlags(FlagNW)); 8789 if (const auto *ShiftedAddRec = dyn_cast<SCEVAddRecExpr>(Shifted)) 8790 return ShiftedAddRec->getNumIterationsInRange( 8791 Range.subtract(SC->getAPInt()), SE); 8792 // This is strange and shouldn't happen. 8793 return SE.getCouldNotCompute(); 8794 } 8795 8796 // The only time we can solve this is when we have all constant indices. 8797 // Otherwise, we cannot determine the overflow conditions. 8798 if (any_of(operands(), [](const SCEV *Op) { return !isa<SCEVConstant>(Op); })) 8799 return SE.getCouldNotCompute(); 8800 8801 // Okay at this point we know that all elements of the chrec are constants and 8802 // that the start element is zero. 8803 8804 // First check to see if the range contains zero. If not, the first 8805 // iteration exits. 8806 unsigned BitWidth = SE.getTypeSizeInBits(getType()); 8807 if (!Range.contains(APInt(BitWidth, 0))) 8808 return SE.getZero(getType()); 8809 8810 if (isAffine()) { 8811 // If this is an affine expression then we have this situation: 8812 // Solve {0,+,A} in Range === Ax in Range 8813 8814 // We know that zero is in the range. If A is positive then we know that 8815 // the upper value of the range must be the first possible exit value. 8816 // If A is negative then the lower of the range is the last possible loop 8817 // value. Also note that we already checked for a full range. 8818 APInt One(BitWidth,1); 8819 APInt A = cast<SCEVConstant>(getOperand(1))->getAPInt(); 8820 APInt End = A.sge(One) ? (Range.getUpper() - One) : Range.getLower(); 8821 8822 // The exit value should be (End+A)/A. 8823 APInt ExitVal = (End + A).udiv(A); 8824 ConstantInt *ExitValue = ConstantInt::get(SE.getContext(), ExitVal); 8825 8826 // Evaluate at the exit value. If we really did fall out of the valid 8827 // range, then we computed our trip count, otherwise wrap around or other 8828 // things must have happened. 8829 ConstantInt *Val = EvaluateConstantChrecAtConstant(this, ExitValue, SE); 8830 if (Range.contains(Val->getValue())) 8831 return SE.getCouldNotCompute(); // Something strange happened 8832 8833 // Ensure that the previous value is in the range. This is a sanity check. 8834 assert(Range.contains( 8835 EvaluateConstantChrecAtConstant(this, 8836 ConstantInt::get(SE.getContext(), ExitVal - One), SE)->getValue()) && 8837 "Linear scev computation is off in a bad way!"); 8838 return SE.getConstant(ExitValue); 8839 } else if (isQuadratic()) { 8840 // If this is a quadratic (3-term) AddRec {L,+,M,+,N}, find the roots of the 8841 // quadratic equation to solve it. To do this, we must frame our problem in 8842 // terms of figuring out when zero is crossed, instead of when 8843 // Range.getUpper() is crossed. 8844 SmallVector<const SCEV *, 4> NewOps(op_begin(), op_end()); 8845 NewOps[0] = SE.getNegativeSCEV(SE.getConstant(Range.getUpper())); 8846 const SCEV *NewAddRec = SE.getAddRecExpr(NewOps, getLoop(), 8847 // getNoWrapFlags(FlagNW) 8848 FlagAnyWrap); 8849 8850 // Next, solve the constructed addrec 8851 auto Roots = SolveQuadraticEquation(cast<SCEVAddRecExpr>(NewAddRec), SE); 8852 const SCEVConstant *R1 = dyn_cast<SCEVConstant>(Roots.first); 8853 const SCEVConstant *R2 = dyn_cast<SCEVConstant>(Roots.second); 8854 if (R1) { 8855 // Pick the smallest positive root value. 8856 if (ConstantInt *CB = dyn_cast<ConstantInt>(ConstantExpr::getICmp( 8857 ICmpInst::ICMP_ULT, R1->getValue(), R2->getValue()))) { 8858 if (!CB->getZExtValue()) 8859 std::swap(R1, R2); // R1 is the minimum root now. 8860 8861 // Make sure the root is not off by one. The returned iteration should 8862 // not be in the range, but the previous one should be. When solving 8863 // for "X*X < 5", for example, we should not return a root of 2. 8864 ConstantInt *R1Val = EvaluateConstantChrecAtConstant(this, 8865 R1->getValue(), 8866 SE); 8867 if (Range.contains(R1Val->getValue())) { 8868 // The next iteration must be out of the range... 8869 ConstantInt *NextVal = 8870 ConstantInt::get(SE.getContext(), R1->getAPInt() + 1); 8871 8872 R1Val = EvaluateConstantChrecAtConstant(this, NextVal, SE); 8873 if (!Range.contains(R1Val->getValue())) 8874 return SE.getConstant(NextVal); 8875 return SE.getCouldNotCompute(); // Something strange happened 8876 } 8877 8878 // If R1 was not in the range, then it is a good return value. Make 8879 // sure that R1-1 WAS in the range though, just in case. 8880 ConstantInt *NextVal = 8881 ConstantInt::get(SE.getContext(), R1->getAPInt() - 1); 8882 R1Val = EvaluateConstantChrecAtConstant(this, NextVal, SE); 8883 if (Range.contains(R1Val->getValue())) 8884 return R1; 8885 return SE.getCouldNotCompute(); // Something strange happened 8886 } 8887 } 8888 } 8889 8890 return SE.getCouldNotCompute(); 8891 } 8892 8893 namespace { 8894 struct FindUndefs { 8895 bool Found; 8896 FindUndefs() : Found(false) {} 8897 8898 bool follow(const SCEV *S) { 8899 if (const SCEVUnknown *C = dyn_cast<SCEVUnknown>(S)) { 8900 if (isa<UndefValue>(C->getValue())) 8901 Found = true; 8902 } else if (const SCEVConstant *C = dyn_cast<SCEVConstant>(S)) { 8903 if (isa<UndefValue>(C->getValue())) 8904 Found = true; 8905 } 8906 8907 // Keep looking if we haven't found it yet. 8908 return !Found; 8909 } 8910 bool isDone() const { 8911 // Stop recursion if we have found an undef. 8912 return Found; 8913 } 8914 }; 8915 } 8916 8917 // Return true when S contains at least an undef value. 8918 static inline bool 8919 containsUndefs(const SCEV *S) { 8920 FindUndefs F; 8921 SCEVTraversal<FindUndefs> ST(F); 8922 ST.visitAll(S); 8923 8924 return F.Found; 8925 } 8926 8927 namespace { 8928 // Collect all steps of SCEV expressions. 8929 struct SCEVCollectStrides { 8930 ScalarEvolution &SE; 8931 SmallVectorImpl<const SCEV *> &Strides; 8932 8933 SCEVCollectStrides(ScalarEvolution &SE, SmallVectorImpl<const SCEV *> &S) 8934 : SE(SE), Strides(S) {} 8935 8936 bool follow(const SCEV *S) { 8937 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S)) 8938 Strides.push_back(AR->getStepRecurrence(SE)); 8939 return true; 8940 } 8941 bool isDone() const { return false; } 8942 }; 8943 8944 // Collect all SCEVUnknown and SCEVMulExpr expressions. 8945 struct SCEVCollectTerms { 8946 SmallVectorImpl<const SCEV *> &Terms; 8947 8948 SCEVCollectTerms(SmallVectorImpl<const SCEV *> &T) 8949 : Terms(T) {} 8950 8951 bool follow(const SCEV *S) { 8952 if (isa<SCEVUnknown>(S) || isa<SCEVMulExpr>(S)) { 8953 if (!containsUndefs(S)) 8954 Terms.push_back(S); 8955 8956 // Stop recursion: once we collected a term, do not walk its operands. 8957 return false; 8958 } 8959 8960 // Keep looking. 8961 return true; 8962 } 8963 bool isDone() const { return false; } 8964 }; 8965 8966 // Check if a SCEV contains an AddRecExpr. 8967 struct SCEVHasAddRec { 8968 bool &ContainsAddRec; 8969 8970 SCEVHasAddRec(bool &ContainsAddRec) : ContainsAddRec(ContainsAddRec) { 8971 ContainsAddRec = false; 8972 } 8973 8974 bool follow(const SCEV *S) { 8975 if (isa<SCEVAddRecExpr>(S)) { 8976 ContainsAddRec = true; 8977 8978 // Stop recursion: once we collected a term, do not walk its operands. 8979 return false; 8980 } 8981 8982 // Keep looking. 8983 return true; 8984 } 8985 bool isDone() const { return false; } 8986 }; 8987 8988 // Find factors that are multiplied with an expression that (possibly as a 8989 // subexpression) contains an AddRecExpr. In the expression: 8990 // 8991 // 8 * (100 + %p * %q * (%a + {0, +, 1}_loop)) 8992 // 8993 // "%p * %q" are factors multiplied by the expression "(%a + {0, +, 1}_loop)" 8994 // that contains the AddRec {0, +, 1}_loop. %p * %q are likely to be array size 8995 // parameters as they form a product with an induction variable. 8996 // 8997 // This collector expects all array size parameters to be in the same MulExpr. 8998 // It might be necessary to later add support for collecting parameters that are 8999 // spread over different nested MulExpr. 9000 struct SCEVCollectAddRecMultiplies { 9001 SmallVectorImpl<const SCEV *> &Terms; 9002 ScalarEvolution &SE; 9003 9004 SCEVCollectAddRecMultiplies(SmallVectorImpl<const SCEV *> &T, ScalarEvolution &SE) 9005 : Terms(T), SE(SE) {} 9006 9007 bool follow(const SCEV *S) { 9008 if (auto *Mul = dyn_cast<SCEVMulExpr>(S)) { 9009 bool HasAddRec = false; 9010 SmallVector<const SCEV *, 0> Operands; 9011 for (auto Op : Mul->operands()) { 9012 if (isa<SCEVUnknown>(Op)) { 9013 Operands.push_back(Op); 9014 } else { 9015 bool ContainsAddRec; 9016 SCEVHasAddRec ContiansAddRec(ContainsAddRec); 9017 visitAll(Op, ContiansAddRec); 9018 HasAddRec |= ContainsAddRec; 9019 } 9020 } 9021 if (Operands.size() == 0) 9022 return true; 9023 9024 if (!HasAddRec) 9025 return false; 9026 9027 Terms.push_back(SE.getMulExpr(Operands)); 9028 // Stop recursion: once we collected a term, do not walk its operands. 9029 return false; 9030 } 9031 9032 // Keep looking. 9033 return true; 9034 } 9035 bool isDone() const { return false; } 9036 }; 9037 } 9038 9039 /// Find parametric terms in this SCEVAddRecExpr. We first for parameters in 9040 /// two places: 9041 /// 1) The strides of AddRec expressions. 9042 /// 2) Unknowns that are multiplied with AddRec expressions. 9043 void ScalarEvolution::collectParametricTerms(const SCEV *Expr, 9044 SmallVectorImpl<const SCEV *> &Terms) { 9045 SmallVector<const SCEV *, 4> Strides; 9046 SCEVCollectStrides StrideCollector(*this, Strides); 9047 visitAll(Expr, StrideCollector); 9048 9049 DEBUG({ 9050 dbgs() << "Strides:\n"; 9051 for (const SCEV *S : Strides) 9052 dbgs() << *S << "\n"; 9053 }); 9054 9055 for (const SCEV *S : Strides) { 9056 SCEVCollectTerms TermCollector(Terms); 9057 visitAll(S, TermCollector); 9058 } 9059 9060 DEBUG({ 9061 dbgs() << "Terms:\n"; 9062 for (const SCEV *T : Terms) 9063 dbgs() << *T << "\n"; 9064 }); 9065 9066 SCEVCollectAddRecMultiplies MulCollector(Terms, *this); 9067 visitAll(Expr, MulCollector); 9068 } 9069 9070 static bool findArrayDimensionsRec(ScalarEvolution &SE, 9071 SmallVectorImpl<const SCEV *> &Terms, 9072 SmallVectorImpl<const SCEV *> &Sizes) { 9073 int Last = Terms.size() - 1; 9074 const SCEV *Step = Terms[Last]; 9075 9076 // End of recursion. 9077 if (Last == 0) { 9078 if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(Step)) { 9079 SmallVector<const SCEV *, 2> Qs; 9080 for (const SCEV *Op : M->operands()) 9081 if (!isa<SCEVConstant>(Op)) 9082 Qs.push_back(Op); 9083 9084 Step = SE.getMulExpr(Qs); 9085 } 9086 9087 Sizes.push_back(Step); 9088 return true; 9089 } 9090 9091 for (const SCEV *&Term : Terms) { 9092 // Normalize the terms before the next call to findArrayDimensionsRec. 9093 const SCEV *Q, *R; 9094 SCEVDivision::divide(SE, Term, Step, &Q, &R); 9095 9096 // Bail out when GCD does not evenly divide one of the terms. 9097 if (!R->isZero()) 9098 return false; 9099 9100 Term = Q; 9101 } 9102 9103 // Remove all SCEVConstants. 9104 Terms.erase(std::remove_if(Terms.begin(), Terms.end(), [](const SCEV *E) { 9105 return isa<SCEVConstant>(E); 9106 }), 9107 Terms.end()); 9108 9109 if (Terms.size() > 0) 9110 if (!findArrayDimensionsRec(SE, Terms, Sizes)) 9111 return false; 9112 9113 Sizes.push_back(Step); 9114 return true; 9115 } 9116 9117 // Returns true when S contains at least a SCEVUnknown parameter. 9118 static inline bool 9119 containsParameters(const SCEV *S) { 9120 struct FindParameter { 9121 bool FoundParameter; 9122 FindParameter() : FoundParameter(false) {} 9123 9124 bool follow(const SCEV *S) { 9125 if (isa<SCEVUnknown>(S)) { 9126 FoundParameter = true; 9127 // Stop recursion: we found a parameter. 9128 return false; 9129 } 9130 // Keep looking. 9131 return true; 9132 } 9133 bool isDone() const { 9134 // Stop recursion if we have found a parameter. 9135 return FoundParameter; 9136 } 9137 }; 9138 9139 FindParameter F; 9140 SCEVTraversal<FindParameter> ST(F); 9141 ST.visitAll(S); 9142 9143 return F.FoundParameter; 9144 } 9145 9146 // Returns true when one of the SCEVs of Terms contains a SCEVUnknown parameter. 9147 static inline bool 9148 containsParameters(SmallVectorImpl<const SCEV *> &Terms) { 9149 for (const SCEV *T : Terms) 9150 if (containsParameters(T)) 9151 return true; 9152 return false; 9153 } 9154 9155 // Return the number of product terms in S. 9156 static inline int numberOfTerms(const SCEV *S) { 9157 if (const SCEVMulExpr *Expr = dyn_cast<SCEVMulExpr>(S)) 9158 return Expr->getNumOperands(); 9159 return 1; 9160 } 9161 9162 static const SCEV *removeConstantFactors(ScalarEvolution &SE, const SCEV *T) { 9163 if (isa<SCEVConstant>(T)) 9164 return nullptr; 9165 9166 if (isa<SCEVUnknown>(T)) 9167 return T; 9168 9169 if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(T)) { 9170 SmallVector<const SCEV *, 2> Factors; 9171 for (const SCEV *Op : M->operands()) 9172 if (!isa<SCEVConstant>(Op)) 9173 Factors.push_back(Op); 9174 9175 return SE.getMulExpr(Factors); 9176 } 9177 9178 return T; 9179 } 9180 9181 /// Return the size of an element read or written by Inst. 9182 const SCEV *ScalarEvolution::getElementSize(Instruction *Inst) { 9183 Type *Ty; 9184 if (StoreInst *Store = dyn_cast<StoreInst>(Inst)) 9185 Ty = Store->getValueOperand()->getType(); 9186 else if (LoadInst *Load = dyn_cast<LoadInst>(Inst)) 9187 Ty = Load->getType(); 9188 else 9189 return nullptr; 9190 9191 Type *ETy = getEffectiveSCEVType(PointerType::getUnqual(Ty)); 9192 return getSizeOfExpr(ETy, Ty); 9193 } 9194 9195 /// Second step of delinearization: compute the array dimensions Sizes from the 9196 /// set of Terms extracted from the memory access function of this SCEVAddRec. 9197 void ScalarEvolution::findArrayDimensions(SmallVectorImpl<const SCEV *> &Terms, 9198 SmallVectorImpl<const SCEV *> &Sizes, 9199 const SCEV *ElementSize) const { 9200 9201 if (Terms.size() < 1 || !ElementSize) 9202 return; 9203 9204 // Early return when Terms do not contain parameters: we do not delinearize 9205 // non parametric SCEVs. 9206 if (!containsParameters(Terms)) 9207 return; 9208 9209 DEBUG({ 9210 dbgs() << "Terms:\n"; 9211 for (const SCEV *T : Terms) 9212 dbgs() << *T << "\n"; 9213 }); 9214 9215 // Remove duplicates. 9216 std::sort(Terms.begin(), Terms.end()); 9217 Terms.erase(std::unique(Terms.begin(), Terms.end()), Terms.end()); 9218 9219 // Put larger terms first. 9220 std::sort(Terms.begin(), Terms.end(), [](const SCEV *LHS, const SCEV *RHS) { 9221 return numberOfTerms(LHS) > numberOfTerms(RHS); 9222 }); 9223 9224 ScalarEvolution &SE = *const_cast<ScalarEvolution *>(this); 9225 9226 // Try to divide all terms by the element size. If term is not divisible by 9227 // element size, proceed with the original term. 9228 for (const SCEV *&Term : Terms) { 9229 const SCEV *Q, *R; 9230 SCEVDivision::divide(SE, Term, ElementSize, &Q, &R); 9231 if (!Q->isZero()) 9232 Term = Q; 9233 } 9234 9235 SmallVector<const SCEV *, 4> NewTerms; 9236 9237 // Remove constant factors. 9238 for (const SCEV *T : Terms) 9239 if (const SCEV *NewT = removeConstantFactors(SE, T)) 9240 NewTerms.push_back(NewT); 9241 9242 DEBUG({ 9243 dbgs() << "Terms after sorting:\n"; 9244 for (const SCEV *T : NewTerms) 9245 dbgs() << *T << "\n"; 9246 }); 9247 9248 if (NewTerms.empty() || 9249 !findArrayDimensionsRec(SE, NewTerms, Sizes)) { 9250 Sizes.clear(); 9251 return; 9252 } 9253 9254 // The last element to be pushed into Sizes is the size of an element. 9255 Sizes.push_back(ElementSize); 9256 9257 DEBUG({ 9258 dbgs() << "Sizes:\n"; 9259 for (const SCEV *S : Sizes) 9260 dbgs() << *S << "\n"; 9261 }); 9262 } 9263 9264 /// Third step of delinearization: compute the access functions for the 9265 /// Subscripts based on the dimensions in Sizes. 9266 void ScalarEvolution::computeAccessFunctions( 9267 const SCEV *Expr, SmallVectorImpl<const SCEV *> &Subscripts, 9268 SmallVectorImpl<const SCEV *> &Sizes) { 9269 9270 // Early exit in case this SCEV is not an affine multivariate function. 9271 if (Sizes.empty()) 9272 return; 9273 9274 if (auto *AR = dyn_cast<SCEVAddRecExpr>(Expr)) 9275 if (!AR->isAffine()) 9276 return; 9277 9278 const SCEV *Res = Expr; 9279 int Last = Sizes.size() - 1; 9280 for (int i = Last; i >= 0; i--) { 9281 const SCEV *Q, *R; 9282 SCEVDivision::divide(*this, Res, Sizes[i], &Q, &R); 9283 9284 DEBUG({ 9285 dbgs() << "Res: " << *Res << "\n"; 9286 dbgs() << "Sizes[i]: " << *Sizes[i] << "\n"; 9287 dbgs() << "Res divided by Sizes[i]:\n"; 9288 dbgs() << "Quotient: " << *Q << "\n"; 9289 dbgs() << "Remainder: " << *R << "\n"; 9290 }); 9291 9292 Res = Q; 9293 9294 // Do not record the last subscript corresponding to the size of elements in 9295 // the array. 9296 if (i == Last) { 9297 9298 // Bail out if the remainder is too complex. 9299 if (isa<SCEVAddRecExpr>(R)) { 9300 Subscripts.clear(); 9301 Sizes.clear(); 9302 return; 9303 } 9304 9305 continue; 9306 } 9307 9308 // Record the access function for the current subscript. 9309 Subscripts.push_back(R); 9310 } 9311 9312 // Also push in last position the remainder of the last division: it will be 9313 // the access function of the innermost dimension. 9314 Subscripts.push_back(Res); 9315 9316 std::reverse(Subscripts.begin(), Subscripts.end()); 9317 9318 DEBUG({ 9319 dbgs() << "Subscripts:\n"; 9320 for (const SCEV *S : Subscripts) 9321 dbgs() << *S << "\n"; 9322 }); 9323 } 9324 9325 /// Splits the SCEV into two vectors of SCEVs representing the subscripts and 9326 /// sizes of an array access. Returns the remainder of the delinearization that 9327 /// is the offset start of the array. The SCEV->delinearize algorithm computes 9328 /// the multiples of SCEV coefficients: that is a pattern matching of sub 9329 /// expressions in the stride and base of a SCEV corresponding to the 9330 /// computation of a GCD (greatest common divisor) of base and stride. When 9331 /// SCEV->delinearize fails, it returns the SCEV unchanged. 9332 /// 9333 /// For example: when analyzing the memory access A[i][j][k] in this loop nest 9334 /// 9335 /// void foo(long n, long m, long o, double A[n][m][o]) { 9336 /// 9337 /// for (long i = 0; i < n; i++) 9338 /// for (long j = 0; j < m; j++) 9339 /// for (long k = 0; k < o; k++) 9340 /// A[i][j][k] = 1.0; 9341 /// } 9342 /// 9343 /// the delinearization input is the following AddRec SCEV: 9344 /// 9345 /// AddRec: {{{%A,+,(8 * %m * %o)}<%for.i>,+,(8 * %o)}<%for.j>,+,8}<%for.k> 9346 /// 9347 /// From this SCEV, we are able to say that the base offset of the access is %A 9348 /// because it appears as an offset that does not divide any of the strides in 9349 /// the loops: 9350 /// 9351 /// CHECK: Base offset: %A 9352 /// 9353 /// and then SCEV->delinearize determines the size of some of the dimensions of 9354 /// the array as these are the multiples by which the strides are happening: 9355 /// 9356 /// CHECK: ArrayDecl[UnknownSize][%m][%o] with elements of sizeof(double) bytes. 9357 /// 9358 /// Note that the outermost dimension remains of UnknownSize because there are 9359 /// no strides that would help identifying the size of the last dimension: when 9360 /// the array has been statically allocated, one could compute the size of that 9361 /// dimension by dividing the overall size of the array by the size of the known 9362 /// dimensions: %m * %o * 8. 9363 /// 9364 /// Finally delinearize provides the access functions for the array reference 9365 /// that does correspond to A[i][j][k] of the above C testcase: 9366 /// 9367 /// CHECK: ArrayRef[{0,+,1}<%for.i>][{0,+,1}<%for.j>][{0,+,1}<%for.k>] 9368 /// 9369 /// The testcases are checking the output of a function pass: 9370 /// DelinearizationPass that walks through all loads and stores of a function 9371 /// asking for the SCEV of the memory access with respect to all enclosing 9372 /// loops, calling SCEV->delinearize on that and printing the results. 9373 9374 void ScalarEvolution::delinearize(const SCEV *Expr, 9375 SmallVectorImpl<const SCEV *> &Subscripts, 9376 SmallVectorImpl<const SCEV *> &Sizes, 9377 const SCEV *ElementSize) { 9378 // First step: collect parametric terms. 9379 SmallVector<const SCEV *, 4> Terms; 9380 collectParametricTerms(Expr, Terms); 9381 9382 if (Terms.empty()) 9383 return; 9384 9385 // Second step: find subscript sizes. 9386 findArrayDimensions(Terms, Sizes, ElementSize); 9387 9388 if (Sizes.empty()) 9389 return; 9390 9391 // Third step: compute the access functions for each subscript. 9392 computeAccessFunctions(Expr, Subscripts, Sizes); 9393 9394 if (Subscripts.empty()) 9395 return; 9396 9397 DEBUG({ 9398 dbgs() << "succeeded to delinearize " << *Expr << "\n"; 9399 dbgs() << "ArrayDecl[UnknownSize]"; 9400 for (const SCEV *S : Sizes) 9401 dbgs() << "[" << *S << "]"; 9402 9403 dbgs() << "\nArrayRef"; 9404 for (const SCEV *S : Subscripts) 9405 dbgs() << "[" << *S << "]"; 9406 dbgs() << "\n"; 9407 }); 9408 } 9409 9410 //===----------------------------------------------------------------------===// 9411 // SCEVCallbackVH Class Implementation 9412 //===----------------------------------------------------------------------===// 9413 9414 void ScalarEvolution::SCEVCallbackVH::deleted() { 9415 assert(SE && "SCEVCallbackVH called with a null ScalarEvolution!"); 9416 if (PHINode *PN = dyn_cast<PHINode>(getValPtr())) 9417 SE->ConstantEvolutionLoopExitValue.erase(PN); 9418 SE->eraseValueFromMap(getValPtr()); 9419 // this now dangles! 9420 } 9421 9422 void ScalarEvolution::SCEVCallbackVH::allUsesReplacedWith(Value *V) { 9423 assert(SE && "SCEVCallbackVH called with a null ScalarEvolution!"); 9424 9425 // Forget all the expressions associated with users of the old value, 9426 // so that future queries will recompute the expressions using the new 9427 // value. 9428 Value *Old = getValPtr(); 9429 SmallVector<User *, 16> Worklist(Old->user_begin(), Old->user_end()); 9430 SmallPtrSet<User *, 8> Visited; 9431 while (!Worklist.empty()) { 9432 User *U = Worklist.pop_back_val(); 9433 // Deleting the Old value will cause this to dangle. Postpone 9434 // that until everything else is done. 9435 if (U == Old) 9436 continue; 9437 if (!Visited.insert(U).second) 9438 continue; 9439 if (PHINode *PN = dyn_cast<PHINode>(U)) 9440 SE->ConstantEvolutionLoopExitValue.erase(PN); 9441 SE->eraseValueFromMap(U); 9442 Worklist.insert(Worklist.end(), U->user_begin(), U->user_end()); 9443 } 9444 // Delete the Old value. 9445 if (PHINode *PN = dyn_cast<PHINode>(Old)) 9446 SE->ConstantEvolutionLoopExitValue.erase(PN); 9447 SE->eraseValueFromMap(Old); 9448 // this now dangles! 9449 } 9450 9451 ScalarEvolution::SCEVCallbackVH::SCEVCallbackVH(Value *V, ScalarEvolution *se) 9452 : CallbackVH(V), SE(se) {} 9453 9454 //===----------------------------------------------------------------------===// 9455 // ScalarEvolution Class Implementation 9456 //===----------------------------------------------------------------------===// 9457 9458 ScalarEvolution::ScalarEvolution(Function &F, TargetLibraryInfo &TLI, 9459 AssumptionCache &AC, DominatorTree &DT, 9460 LoopInfo &LI) 9461 : F(F), TLI(TLI), AC(AC), DT(DT), LI(LI), 9462 CouldNotCompute(new SCEVCouldNotCompute()), 9463 WalkingBEDominatingConds(false), ProvingSplitPredicate(false), 9464 ValuesAtScopes(64), LoopDispositions(64), BlockDispositions(64), 9465 FirstUnknown(nullptr) {} 9466 9467 ScalarEvolution::ScalarEvolution(ScalarEvolution &&Arg) 9468 : F(Arg.F), TLI(Arg.TLI), AC(Arg.AC), DT(Arg.DT), LI(Arg.LI), 9469 CouldNotCompute(std::move(Arg.CouldNotCompute)), 9470 ValueExprMap(std::move(Arg.ValueExprMap)), 9471 WalkingBEDominatingConds(false), ProvingSplitPredicate(false), 9472 BackedgeTakenCounts(std::move(Arg.BackedgeTakenCounts)), 9473 PredicatedBackedgeTakenCounts( 9474 std::move(Arg.PredicatedBackedgeTakenCounts)), 9475 ConstantEvolutionLoopExitValue( 9476 std::move(Arg.ConstantEvolutionLoopExitValue)), 9477 ValuesAtScopes(std::move(Arg.ValuesAtScopes)), 9478 LoopDispositions(std::move(Arg.LoopDispositions)), 9479 BlockDispositions(std::move(Arg.BlockDispositions)), 9480 UnsignedRanges(std::move(Arg.UnsignedRanges)), 9481 SignedRanges(std::move(Arg.SignedRanges)), 9482 UniqueSCEVs(std::move(Arg.UniqueSCEVs)), 9483 UniquePreds(std::move(Arg.UniquePreds)), 9484 SCEVAllocator(std::move(Arg.SCEVAllocator)), 9485 FirstUnknown(Arg.FirstUnknown) { 9486 Arg.FirstUnknown = nullptr; 9487 } 9488 9489 ScalarEvolution::~ScalarEvolution() { 9490 // Iterate through all the SCEVUnknown instances and call their 9491 // destructors, so that they release their references to their values. 9492 for (SCEVUnknown *U = FirstUnknown; U;) { 9493 SCEVUnknown *Tmp = U; 9494 U = U->Next; 9495 Tmp->~SCEVUnknown(); 9496 } 9497 FirstUnknown = nullptr; 9498 9499 ExprValueMap.clear(); 9500 ValueExprMap.clear(); 9501 HasRecMap.clear(); 9502 9503 // Free any extra memory created for ExitNotTakenInfo in the unlikely event 9504 // that a loop had multiple computable exits. 9505 for (auto &BTCI : BackedgeTakenCounts) 9506 BTCI.second.clear(); 9507 for (auto &BTCI : PredicatedBackedgeTakenCounts) 9508 BTCI.second.clear(); 9509 9510 assert(PendingLoopPredicates.empty() && "isImpliedCond garbage"); 9511 assert(!WalkingBEDominatingConds && "isLoopBackedgeGuardedByCond garbage!"); 9512 assert(!ProvingSplitPredicate && "ProvingSplitPredicate garbage!"); 9513 } 9514 9515 bool ScalarEvolution::hasLoopInvariantBackedgeTakenCount(const Loop *L) { 9516 return !isa<SCEVCouldNotCompute>(getBackedgeTakenCount(L)); 9517 } 9518 9519 static void PrintLoopInfo(raw_ostream &OS, ScalarEvolution *SE, 9520 const Loop *L) { 9521 // Print all inner loops first 9522 for (Loop::iterator I = L->begin(), E = L->end(); I != E; ++I) 9523 PrintLoopInfo(OS, SE, *I); 9524 9525 OS << "Loop "; 9526 L->getHeader()->printAsOperand(OS, /*PrintType=*/false); 9527 OS << ": "; 9528 9529 SmallVector<BasicBlock *, 8> ExitBlocks; 9530 L->getExitBlocks(ExitBlocks); 9531 if (ExitBlocks.size() != 1) 9532 OS << "<multiple exits> "; 9533 9534 if (SE->hasLoopInvariantBackedgeTakenCount(L)) { 9535 OS << "backedge-taken count is " << *SE->getBackedgeTakenCount(L); 9536 } else { 9537 OS << "Unpredictable backedge-taken count. "; 9538 } 9539 9540 OS << "\n" 9541 "Loop "; 9542 L->getHeader()->printAsOperand(OS, /*PrintType=*/false); 9543 OS << ": "; 9544 9545 if (!isa<SCEVCouldNotCompute>(SE->getMaxBackedgeTakenCount(L))) { 9546 OS << "max backedge-taken count is " << *SE->getMaxBackedgeTakenCount(L); 9547 } else { 9548 OS << "Unpredictable max backedge-taken count. "; 9549 } 9550 9551 OS << "\n" 9552 "Loop "; 9553 L->getHeader()->printAsOperand(OS, /*PrintType=*/false); 9554 OS << ": "; 9555 9556 SCEVUnionPredicate Pred; 9557 auto PBT = SE->getPredicatedBackedgeTakenCount(L, Pred); 9558 if (!isa<SCEVCouldNotCompute>(PBT)) { 9559 OS << "Predicated backedge-taken count is " << *PBT << "\n"; 9560 OS << " Predicates:\n"; 9561 Pred.print(OS, 4); 9562 } else { 9563 OS << "Unpredictable predicated backedge-taken count. "; 9564 } 9565 OS << "\n"; 9566 } 9567 9568 static StringRef loopDispositionToStr(ScalarEvolution::LoopDisposition LD) { 9569 switch (LD) { 9570 case ScalarEvolution::LoopVariant: 9571 return "Variant"; 9572 case ScalarEvolution::LoopInvariant: 9573 return "Invariant"; 9574 case ScalarEvolution::LoopComputable: 9575 return "Computable"; 9576 } 9577 llvm_unreachable("Unknown ScalarEvolution::LoopDisposition kind!"); 9578 } 9579 9580 void ScalarEvolution::print(raw_ostream &OS) const { 9581 // ScalarEvolution's implementation of the print method is to print 9582 // out SCEV values of all instructions that are interesting. Doing 9583 // this potentially causes it to create new SCEV objects though, 9584 // which technically conflicts with the const qualifier. This isn't 9585 // observable from outside the class though, so casting away the 9586 // const isn't dangerous. 9587 ScalarEvolution &SE = *const_cast<ScalarEvolution *>(this); 9588 9589 OS << "Classifying expressions for: "; 9590 F.printAsOperand(OS, /*PrintType=*/false); 9591 OS << "\n"; 9592 for (Instruction &I : instructions(F)) 9593 if (isSCEVable(I.getType()) && !isa<CmpInst>(I)) { 9594 OS << I << '\n'; 9595 OS << " --> "; 9596 const SCEV *SV = SE.getSCEV(&I); 9597 SV->print(OS); 9598 if (!isa<SCEVCouldNotCompute>(SV)) { 9599 OS << " U: "; 9600 SE.getUnsignedRange(SV).print(OS); 9601 OS << " S: "; 9602 SE.getSignedRange(SV).print(OS); 9603 } 9604 9605 const Loop *L = LI.getLoopFor(I.getParent()); 9606 9607 const SCEV *AtUse = SE.getSCEVAtScope(SV, L); 9608 if (AtUse != SV) { 9609 OS << " --> "; 9610 AtUse->print(OS); 9611 if (!isa<SCEVCouldNotCompute>(AtUse)) { 9612 OS << " U: "; 9613 SE.getUnsignedRange(AtUse).print(OS); 9614 OS << " S: "; 9615 SE.getSignedRange(AtUse).print(OS); 9616 } 9617 } 9618 9619 if (L) { 9620 OS << "\t\t" "Exits: "; 9621 const SCEV *ExitValue = SE.getSCEVAtScope(SV, L->getParentLoop()); 9622 if (!SE.isLoopInvariant(ExitValue, L)) { 9623 OS << "<<Unknown>>"; 9624 } else { 9625 OS << *ExitValue; 9626 } 9627 9628 bool First = true; 9629 for (auto *Iter = L; Iter; Iter = Iter->getParentLoop()) { 9630 if (First) { 9631 OS << "\t\t" "LoopDispositions: { "; 9632 First = false; 9633 } else { 9634 OS << ", "; 9635 } 9636 9637 Iter->getHeader()->printAsOperand(OS, /*PrintType=*/false); 9638 OS << ": " << loopDispositionToStr(SE.getLoopDisposition(SV, Iter)); 9639 } 9640 9641 for (auto *InnerL : depth_first(L)) { 9642 if (InnerL == L) 9643 continue; 9644 if (First) { 9645 OS << "\t\t" "LoopDispositions: { "; 9646 First = false; 9647 } else { 9648 OS << ", "; 9649 } 9650 9651 InnerL->getHeader()->printAsOperand(OS, /*PrintType=*/false); 9652 OS << ": " << loopDispositionToStr(SE.getLoopDisposition(SV, InnerL)); 9653 } 9654 9655 OS << " }"; 9656 } 9657 9658 OS << "\n"; 9659 } 9660 9661 OS << "Determining loop execution counts for: "; 9662 F.printAsOperand(OS, /*PrintType=*/false); 9663 OS << "\n"; 9664 for (LoopInfo::iterator I = LI.begin(), E = LI.end(); I != E; ++I) 9665 PrintLoopInfo(OS, &SE, *I); 9666 } 9667 9668 ScalarEvolution::LoopDisposition 9669 ScalarEvolution::getLoopDisposition(const SCEV *S, const Loop *L) { 9670 auto &Values = LoopDispositions[S]; 9671 for (auto &V : Values) { 9672 if (V.getPointer() == L) 9673 return V.getInt(); 9674 } 9675 Values.emplace_back(L, LoopVariant); 9676 LoopDisposition D = computeLoopDisposition(S, L); 9677 auto &Values2 = LoopDispositions[S]; 9678 for (auto &V : make_range(Values2.rbegin(), Values2.rend())) { 9679 if (V.getPointer() == L) { 9680 V.setInt(D); 9681 break; 9682 } 9683 } 9684 return D; 9685 } 9686 9687 ScalarEvolution::LoopDisposition 9688 ScalarEvolution::computeLoopDisposition(const SCEV *S, const Loop *L) { 9689 switch (static_cast<SCEVTypes>(S->getSCEVType())) { 9690 case scConstant: 9691 return LoopInvariant; 9692 case scTruncate: 9693 case scZeroExtend: 9694 case scSignExtend: 9695 return getLoopDisposition(cast<SCEVCastExpr>(S)->getOperand(), L); 9696 case scAddRecExpr: { 9697 const SCEVAddRecExpr *AR = cast<SCEVAddRecExpr>(S); 9698 9699 // If L is the addrec's loop, it's computable. 9700 if (AR->getLoop() == L) 9701 return LoopComputable; 9702 9703 // Add recurrences are never invariant in the function-body (null loop). 9704 if (!L) 9705 return LoopVariant; 9706 9707 // This recurrence is variant w.r.t. L if L contains AR's loop. 9708 if (L->contains(AR->getLoop())) 9709 return LoopVariant; 9710 9711 // This recurrence is invariant w.r.t. L if AR's loop contains L. 9712 if (AR->getLoop()->contains(L)) 9713 return LoopInvariant; 9714 9715 // This recurrence is variant w.r.t. L if any of its operands 9716 // are variant. 9717 for (auto *Op : AR->operands()) 9718 if (!isLoopInvariant(Op, L)) 9719 return LoopVariant; 9720 9721 // Otherwise it's loop-invariant. 9722 return LoopInvariant; 9723 } 9724 case scAddExpr: 9725 case scMulExpr: 9726 case scUMaxExpr: 9727 case scSMaxExpr: { 9728 bool HasVarying = false; 9729 for (auto *Op : cast<SCEVNAryExpr>(S)->operands()) { 9730 LoopDisposition D = getLoopDisposition(Op, L); 9731 if (D == LoopVariant) 9732 return LoopVariant; 9733 if (D == LoopComputable) 9734 HasVarying = true; 9735 } 9736 return HasVarying ? LoopComputable : LoopInvariant; 9737 } 9738 case scUDivExpr: { 9739 const SCEVUDivExpr *UDiv = cast<SCEVUDivExpr>(S); 9740 LoopDisposition LD = getLoopDisposition(UDiv->getLHS(), L); 9741 if (LD == LoopVariant) 9742 return LoopVariant; 9743 LoopDisposition RD = getLoopDisposition(UDiv->getRHS(), L); 9744 if (RD == LoopVariant) 9745 return LoopVariant; 9746 return (LD == LoopInvariant && RD == LoopInvariant) ? 9747 LoopInvariant : LoopComputable; 9748 } 9749 case scUnknown: 9750 // All non-instruction values are loop invariant. All instructions are loop 9751 // invariant if they are not contained in the specified loop. 9752 // Instructions are never considered invariant in the function body 9753 // (null loop) because they are defined within the "loop". 9754 if (auto *I = dyn_cast<Instruction>(cast<SCEVUnknown>(S)->getValue())) 9755 return (L && !L->contains(I)) ? LoopInvariant : LoopVariant; 9756 return LoopInvariant; 9757 case scCouldNotCompute: 9758 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!"); 9759 } 9760 llvm_unreachable("Unknown SCEV kind!"); 9761 } 9762 9763 bool ScalarEvolution::isLoopInvariant(const SCEV *S, const Loop *L) { 9764 return getLoopDisposition(S, L) == LoopInvariant; 9765 } 9766 9767 bool ScalarEvolution::hasComputableLoopEvolution(const SCEV *S, const Loop *L) { 9768 return getLoopDisposition(S, L) == LoopComputable; 9769 } 9770 9771 ScalarEvolution::BlockDisposition 9772 ScalarEvolution::getBlockDisposition(const SCEV *S, const BasicBlock *BB) { 9773 auto &Values = BlockDispositions[S]; 9774 for (auto &V : Values) { 9775 if (V.getPointer() == BB) 9776 return V.getInt(); 9777 } 9778 Values.emplace_back(BB, DoesNotDominateBlock); 9779 BlockDisposition D = computeBlockDisposition(S, BB); 9780 auto &Values2 = BlockDispositions[S]; 9781 for (auto &V : make_range(Values2.rbegin(), Values2.rend())) { 9782 if (V.getPointer() == BB) { 9783 V.setInt(D); 9784 break; 9785 } 9786 } 9787 return D; 9788 } 9789 9790 ScalarEvolution::BlockDisposition 9791 ScalarEvolution::computeBlockDisposition(const SCEV *S, const BasicBlock *BB) { 9792 switch (static_cast<SCEVTypes>(S->getSCEVType())) { 9793 case scConstant: 9794 return ProperlyDominatesBlock; 9795 case scTruncate: 9796 case scZeroExtend: 9797 case scSignExtend: 9798 return getBlockDisposition(cast<SCEVCastExpr>(S)->getOperand(), BB); 9799 case scAddRecExpr: { 9800 // This uses a "dominates" query instead of "properly dominates" query 9801 // to test for proper dominance too, because the instruction which 9802 // produces the addrec's value is a PHI, and a PHI effectively properly 9803 // dominates its entire containing block. 9804 const SCEVAddRecExpr *AR = cast<SCEVAddRecExpr>(S); 9805 if (!DT.dominates(AR->getLoop()->getHeader(), BB)) 9806 return DoesNotDominateBlock; 9807 } 9808 // FALL THROUGH into SCEVNAryExpr handling. 9809 case scAddExpr: 9810 case scMulExpr: 9811 case scUMaxExpr: 9812 case scSMaxExpr: { 9813 const SCEVNAryExpr *NAry = cast<SCEVNAryExpr>(S); 9814 bool Proper = true; 9815 for (const SCEV *NAryOp : NAry->operands()) { 9816 BlockDisposition D = getBlockDisposition(NAryOp, BB); 9817 if (D == DoesNotDominateBlock) 9818 return DoesNotDominateBlock; 9819 if (D == DominatesBlock) 9820 Proper = false; 9821 } 9822 return Proper ? ProperlyDominatesBlock : DominatesBlock; 9823 } 9824 case scUDivExpr: { 9825 const SCEVUDivExpr *UDiv = cast<SCEVUDivExpr>(S); 9826 const SCEV *LHS = UDiv->getLHS(), *RHS = UDiv->getRHS(); 9827 BlockDisposition LD = getBlockDisposition(LHS, BB); 9828 if (LD == DoesNotDominateBlock) 9829 return DoesNotDominateBlock; 9830 BlockDisposition RD = getBlockDisposition(RHS, BB); 9831 if (RD == DoesNotDominateBlock) 9832 return DoesNotDominateBlock; 9833 return (LD == ProperlyDominatesBlock && RD == ProperlyDominatesBlock) ? 9834 ProperlyDominatesBlock : DominatesBlock; 9835 } 9836 case scUnknown: 9837 if (Instruction *I = 9838 dyn_cast<Instruction>(cast<SCEVUnknown>(S)->getValue())) { 9839 if (I->getParent() == BB) 9840 return DominatesBlock; 9841 if (DT.properlyDominates(I->getParent(), BB)) 9842 return ProperlyDominatesBlock; 9843 return DoesNotDominateBlock; 9844 } 9845 return ProperlyDominatesBlock; 9846 case scCouldNotCompute: 9847 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!"); 9848 } 9849 llvm_unreachable("Unknown SCEV kind!"); 9850 } 9851 9852 bool ScalarEvolution::dominates(const SCEV *S, const BasicBlock *BB) { 9853 return getBlockDisposition(S, BB) >= DominatesBlock; 9854 } 9855 9856 bool ScalarEvolution::properlyDominates(const SCEV *S, const BasicBlock *BB) { 9857 return getBlockDisposition(S, BB) == ProperlyDominatesBlock; 9858 } 9859 9860 bool ScalarEvolution::hasOperand(const SCEV *S, const SCEV *Op) const { 9861 // Search for a SCEV expression node within an expression tree. 9862 // Implements SCEVTraversal::Visitor. 9863 struct SCEVSearch { 9864 const SCEV *Node; 9865 bool IsFound; 9866 9867 SCEVSearch(const SCEV *N): Node(N), IsFound(false) {} 9868 9869 bool follow(const SCEV *S) { 9870 IsFound |= (S == Node); 9871 return !IsFound; 9872 } 9873 bool isDone() const { return IsFound; } 9874 }; 9875 9876 SCEVSearch Search(Op); 9877 visitAll(S, Search); 9878 return Search.IsFound; 9879 } 9880 9881 void ScalarEvolution::forgetMemoizedResults(const SCEV *S) { 9882 ValuesAtScopes.erase(S); 9883 LoopDispositions.erase(S); 9884 BlockDispositions.erase(S); 9885 UnsignedRanges.erase(S); 9886 SignedRanges.erase(S); 9887 ExprValueMap.erase(S); 9888 HasRecMap.erase(S); 9889 9890 auto RemoveSCEVFromBackedgeMap = 9891 [S, this](DenseMap<const Loop *, BackedgeTakenInfo> &Map) { 9892 for (auto I = Map.begin(), E = Map.end(); I != E;) { 9893 BackedgeTakenInfo &BEInfo = I->second; 9894 if (BEInfo.hasOperand(S, this)) { 9895 BEInfo.clear(); 9896 Map.erase(I++); 9897 } else 9898 ++I; 9899 } 9900 }; 9901 9902 RemoveSCEVFromBackedgeMap(BackedgeTakenCounts); 9903 RemoveSCEVFromBackedgeMap(PredicatedBackedgeTakenCounts); 9904 } 9905 9906 typedef DenseMap<const Loop *, std::string> VerifyMap; 9907 9908 /// replaceSubString - Replaces all occurrences of From in Str with To. 9909 static void replaceSubString(std::string &Str, StringRef From, StringRef To) { 9910 size_t Pos = 0; 9911 while ((Pos = Str.find(From, Pos)) != std::string::npos) { 9912 Str.replace(Pos, From.size(), To.data(), To.size()); 9913 Pos += To.size(); 9914 } 9915 } 9916 9917 /// getLoopBackedgeTakenCounts - Helper method for verifyAnalysis. 9918 static void 9919 getLoopBackedgeTakenCounts(Loop *L, VerifyMap &Map, ScalarEvolution &SE) { 9920 std::string &S = Map[L]; 9921 if (S.empty()) { 9922 raw_string_ostream OS(S); 9923 SE.getBackedgeTakenCount(L)->print(OS); 9924 9925 // false and 0 are semantically equivalent. This can happen in dead loops. 9926 replaceSubString(OS.str(), "false", "0"); 9927 // Remove wrap flags, their use in SCEV is highly fragile. 9928 // FIXME: Remove this when SCEV gets smarter about them. 9929 replaceSubString(OS.str(), "<nw>", ""); 9930 replaceSubString(OS.str(), "<nsw>", ""); 9931 replaceSubString(OS.str(), "<nuw>", ""); 9932 } 9933 9934 for (auto *R : reverse(*L)) 9935 getLoopBackedgeTakenCounts(R, Map, SE); // recurse. 9936 } 9937 9938 void ScalarEvolution::verify() const { 9939 ScalarEvolution &SE = *const_cast<ScalarEvolution *>(this); 9940 9941 // Gather stringified backedge taken counts for all loops using SCEV's caches. 9942 // FIXME: It would be much better to store actual values instead of strings, 9943 // but SCEV pointers will change if we drop the caches. 9944 VerifyMap BackedgeDumpsOld, BackedgeDumpsNew; 9945 for (LoopInfo::reverse_iterator I = LI.rbegin(), E = LI.rend(); I != E; ++I) 9946 getLoopBackedgeTakenCounts(*I, BackedgeDumpsOld, SE); 9947 9948 // Gather stringified backedge taken counts for all loops using a fresh 9949 // ScalarEvolution object. 9950 ScalarEvolution SE2(F, TLI, AC, DT, LI); 9951 for (LoopInfo::reverse_iterator I = LI.rbegin(), E = LI.rend(); I != E; ++I) 9952 getLoopBackedgeTakenCounts(*I, BackedgeDumpsNew, SE2); 9953 9954 // Now compare whether they're the same with and without caches. This allows 9955 // verifying that no pass changed the cache. 9956 assert(BackedgeDumpsOld.size() == BackedgeDumpsNew.size() && 9957 "New loops suddenly appeared!"); 9958 9959 for (VerifyMap::iterator OldI = BackedgeDumpsOld.begin(), 9960 OldE = BackedgeDumpsOld.end(), 9961 NewI = BackedgeDumpsNew.begin(); 9962 OldI != OldE; ++OldI, ++NewI) { 9963 assert(OldI->first == NewI->first && "Loop order changed!"); 9964 9965 // Compare the stringified SCEVs. We don't care if undef backedgetaken count 9966 // changes. 9967 // FIXME: We currently ignore SCEV changes from/to CouldNotCompute. This 9968 // means that a pass is buggy or SCEV has to learn a new pattern but is 9969 // usually not harmful. 9970 if (OldI->second != NewI->second && 9971 OldI->second.find("undef") == std::string::npos && 9972 NewI->second.find("undef") == std::string::npos && 9973 OldI->second != "***COULDNOTCOMPUTE***" && 9974 NewI->second != "***COULDNOTCOMPUTE***") { 9975 dbgs() << "SCEVValidator: SCEV for loop '" 9976 << OldI->first->getHeader()->getName() 9977 << "' changed from '" << OldI->second 9978 << "' to '" << NewI->second << "'!\n"; 9979 std::abort(); 9980 } 9981 } 9982 9983 // TODO: Verify more things. 9984 } 9985 9986 char ScalarEvolutionAnalysis::PassID; 9987 9988 ScalarEvolution ScalarEvolutionAnalysis::run(Function &F, 9989 AnalysisManager<Function> &AM) { 9990 return ScalarEvolution(F, AM.getResult<TargetLibraryAnalysis>(F), 9991 AM.getResult<AssumptionAnalysis>(F), 9992 AM.getResult<DominatorTreeAnalysis>(F), 9993 AM.getResult<LoopAnalysis>(F)); 9994 } 9995 9996 PreservedAnalyses 9997 ScalarEvolutionPrinterPass::run(Function &F, AnalysisManager<Function> &AM) { 9998 AM.getResult<ScalarEvolutionAnalysis>(F).print(OS); 9999 return PreservedAnalyses::all(); 10000 } 10001 10002 INITIALIZE_PASS_BEGIN(ScalarEvolutionWrapperPass, "scalar-evolution", 10003 "Scalar Evolution Analysis", false, true) 10004 INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker) 10005 INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass) 10006 INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass) 10007 INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass) 10008 INITIALIZE_PASS_END(ScalarEvolutionWrapperPass, "scalar-evolution", 10009 "Scalar Evolution Analysis", false, true) 10010 char ScalarEvolutionWrapperPass::ID = 0; 10011 10012 ScalarEvolutionWrapperPass::ScalarEvolutionWrapperPass() : FunctionPass(ID) { 10013 initializeScalarEvolutionWrapperPassPass(*PassRegistry::getPassRegistry()); 10014 } 10015 10016 bool ScalarEvolutionWrapperPass::runOnFunction(Function &F) { 10017 SE.reset(new ScalarEvolution( 10018 F, getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(), 10019 getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F), 10020 getAnalysis<DominatorTreeWrapperPass>().getDomTree(), 10021 getAnalysis<LoopInfoWrapperPass>().getLoopInfo())); 10022 return false; 10023 } 10024 10025 void ScalarEvolutionWrapperPass::releaseMemory() { SE.reset(); } 10026 10027 void ScalarEvolutionWrapperPass::print(raw_ostream &OS, const Module *) const { 10028 SE->print(OS); 10029 } 10030 10031 void ScalarEvolutionWrapperPass::verifyAnalysis() const { 10032 if (!VerifySCEV) 10033 return; 10034 10035 SE->verify(); 10036 } 10037 10038 void ScalarEvolutionWrapperPass::getAnalysisUsage(AnalysisUsage &AU) const { 10039 AU.setPreservesAll(); 10040 AU.addRequiredTransitive<AssumptionCacheTracker>(); 10041 AU.addRequiredTransitive<LoopInfoWrapperPass>(); 10042 AU.addRequiredTransitive<DominatorTreeWrapperPass>(); 10043 AU.addRequiredTransitive<TargetLibraryInfoWrapperPass>(); 10044 } 10045 10046 const SCEVPredicate * 10047 ScalarEvolution::getEqualPredicate(const SCEVUnknown *LHS, 10048 const SCEVConstant *RHS) { 10049 FoldingSetNodeID ID; 10050 // Unique this node based on the arguments 10051 ID.AddInteger(SCEVPredicate::P_Equal); 10052 ID.AddPointer(LHS); 10053 ID.AddPointer(RHS); 10054 void *IP = nullptr; 10055 if (const auto *S = UniquePreds.FindNodeOrInsertPos(ID, IP)) 10056 return S; 10057 SCEVEqualPredicate *Eq = new (SCEVAllocator) 10058 SCEVEqualPredicate(ID.Intern(SCEVAllocator), LHS, RHS); 10059 UniquePreds.InsertNode(Eq, IP); 10060 return Eq; 10061 } 10062 10063 const SCEVPredicate *ScalarEvolution::getWrapPredicate( 10064 const SCEVAddRecExpr *AR, 10065 SCEVWrapPredicate::IncrementWrapFlags AddedFlags) { 10066 FoldingSetNodeID ID; 10067 // Unique this node based on the arguments 10068 ID.AddInteger(SCEVPredicate::P_Wrap); 10069 ID.AddPointer(AR); 10070 ID.AddInteger(AddedFlags); 10071 void *IP = nullptr; 10072 if (const auto *S = UniquePreds.FindNodeOrInsertPos(ID, IP)) 10073 return S; 10074 auto *OF = new (SCEVAllocator) 10075 SCEVWrapPredicate(ID.Intern(SCEVAllocator), AR, AddedFlags); 10076 UniquePreds.InsertNode(OF, IP); 10077 return OF; 10078 } 10079 10080 namespace { 10081 10082 class SCEVPredicateRewriter : public SCEVRewriteVisitor<SCEVPredicateRewriter> { 10083 public: 10084 // Rewrites \p S in the context of a loop L and the predicate A. 10085 // If Assume is true, rewrite is free to add further predicates to A 10086 // such that the result will be an AddRecExpr. 10087 static const SCEV *rewrite(const SCEV *S, const Loop *L, ScalarEvolution &SE, 10088 SCEVUnionPredicate &A, bool Assume) { 10089 SCEVPredicateRewriter Rewriter(L, SE, A, Assume); 10090 return Rewriter.visit(S); 10091 } 10092 10093 SCEVPredicateRewriter(const Loop *L, ScalarEvolution &SE, 10094 SCEVUnionPredicate &P, bool Assume) 10095 : SCEVRewriteVisitor(SE), P(P), L(L), Assume(Assume) {} 10096 10097 const SCEV *visitUnknown(const SCEVUnknown *Expr) { 10098 auto ExprPreds = P.getPredicatesForExpr(Expr); 10099 for (auto *Pred : ExprPreds) 10100 if (const auto *IPred = dyn_cast<const SCEVEqualPredicate>(Pred)) 10101 if (IPred->getLHS() == Expr) 10102 return IPred->getRHS(); 10103 10104 return Expr; 10105 } 10106 10107 const SCEV *visitZeroExtendExpr(const SCEVZeroExtendExpr *Expr) { 10108 const SCEV *Operand = visit(Expr->getOperand()); 10109 const SCEVAddRecExpr *AR = dyn_cast<const SCEVAddRecExpr>(Operand); 10110 if (AR && AR->getLoop() == L && AR->isAffine()) { 10111 // This couldn't be folded because the operand didn't have the nuw 10112 // flag. Add the nusw flag as an assumption that we could make. 10113 const SCEV *Step = AR->getStepRecurrence(SE); 10114 Type *Ty = Expr->getType(); 10115 if (addOverflowAssumption(AR, SCEVWrapPredicate::IncrementNUSW)) 10116 return SE.getAddRecExpr(SE.getZeroExtendExpr(AR->getStart(), Ty), 10117 SE.getSignExtendExpr(Step, Ty), L, 10118 AR->getNoWrapFlags()); 10119 } 10120 return SE.getZeroExtendExpr(Operand, Expr->getType()); 10121 } 10122 10123 const SCEV *visitSignExtendExpr(const SCEVSignExtendExpr *Expr) { 10124 const SCEV *Operand = visit(Expr->getOperand()); 10125 const SCEVAddRecExpr *AR = dyn_cast<const SCEVAddRecExpr>(Operand); 10126 if (AR && AR->getLoop() == L && AR->isAffine()) { 10127 // This couldn't be folded because the operand didn't have the nsw 10128 // flag. Add the nssw flag as an assumption that we could make. 10129 const SCEV *Step = AR->getStepRecurrence(SE); 10130 Type *Ty = Expr->getType(); 10131 if (addOverflowAssumption(AR, SCEVWrapPredicate::IncrementNSSW)) 10132 return SE.getAddRecExpr(SE.getSignExtendExpr(AR->getStart(), Ty), 10133 SE.getSignExtendExpr(Step, Ty), L, 10134 AR->getNoWrapFlags()); 10135 } 10136 return SE.getSignExtendExpr(Operand, Expr->getType()); 10137 } 10138 10139 private: 10140 bool addOverflowAssumption(const SCEVAddRecExpr *AR, 10141 SCEVWrapPredicate::IncrementWrapFlags AddedFlags) { 10142 auto *A = SE.getWrapPredicate(AR, AddedFlags); 10143 if (!Assume) { 10144 // Check if we've already made this assumption. 10145 if (P.implies(A)) 10146 return true; 10147 return false; 10148 } 10149 P.add(A); 10150 return true; 10151 } 10152 10153 SCEVUnionPredicate &P; 10154 const Loop *L; 10155 bool Assume; 10156 }; 10157 } // end anonymous namespace 10158 10159 const SCEV *ScalarEvolution::rewriteUsingPredicate(const SCEV *S, const Loop *L, 10160 SCEVUnionPredicate &Preds) { 10161 return SCEVPredicateRewriter::rewrite(S, L, *this, Preds, false); 10162 } 10163 10164 const SCEVAddRecExpr * 10165 ScalarEvolution::convertSCEVToAddRecWithPredicates(const SCEV *S, const Loop *L, 10166 SCEVUnionPredicate &Preds) { 10167 SCEVUnionPredicate TransformPreds; 10168 S = SCEVPredicateRewriter::rewrite(S, L, *this, TransformPreds, true); 10169 auto *AddRec = dyn_cast<SCEVAddRecExpr>(S); 10170 10171 if (!AddRec) 10172 return nullptr; 10173 10174 // Since the transformation was successful, we can now transfer the SCEV 10175 // predicates. 10176 Preds.add(&TransformPreds); 10177 return AddRec; 10178 } 10179 10180 /// SCEV predicates 10181 SCEVPredicate::SCEVPredicate(const FoldingSetNodeIDRef ID, 10182 SCEVPredicateKind Kind) 10183 : FastID(ID), Kind(Kind) {} 10184 10185 SCEVEqualPredicate::SCEVEqualPredicate(const FoldingSetNodeIDRef ID, 10186 const SCEVUnknown *LHS, 10187 const SCEVConstant *RHS) 10188 : SCEVPredicate(ID, P_Equal), LHS(LHS), RHS(RHS) {} 10189 10190 bool SCEVEqualPredicate::implies(const SCEVPredicate *N) const { 10191 const auto *Op = dyn_cast<const SCEVEqualPredicate>(N); 10192 10193 if (!Op) 10194 return false; 10195 10196 return Op->LHS == LHS && Op->RHS == RHS; 10197 } 10198 10199 bool SCEVEqualPredicate::isAlwaysTrue() const { return false; } 10200 10201 const SCEV *SCEVEqualPredicate::getExpr() const { return LHS; } 10202 10203 void SCEVEqualPredicate::print(raw_ostream &OS, unsigned Depth) const { 10204 OS.indent(Depth) << "Equal predicate: " << *LHS << " == " << *RHS << "\n"; 10205 } 10206 10207 SCEVWrapPredicate::SCEVWrapPredicate(const FoldingSetNodeIDRef ID, 10208 const SCEVAddRecExpr *AR, 10209 IncrementWrapFlags Flags) 10210 : SCEVPredicate(ID, P_Wrap), AR(AR), Flags(Flags) {} 10211 10212 const SCEV *SCEVWrapPredicate::getExpr() const { return AR; } 10213 10214 bool SCEVWrapPredicate::implies(const SCEVPredicate *N) const { 10215 const auto *Op = dyn_cast<SCEVWrapPredicate>(N); 10216 10217 return Op && Op->AR == AR && setFlags(Flags, Op->Flags) == Flags; 10218 } 10219 10220 bool SCEVWrapPredicate::isAlwaysTrue() const { 10221 SCEV::NoWrapFlags ScevFlags = AR->getNoWrapFlags(); 10222 IncrementWrapFlags IFlags = Flags; 10223 10224 if (ScalarEvolution::setFlags(ScevFlags, SCEV::FlagNSW) == ScevFlags) 10225 IFlags = clearFlags(IFlags, IncrementNSSW); 10226 10227 return IFlags == IncrementAnyWrap; 10228 } 10229 10230 void SCEVWrapPredicate::print(raw_ostream &OS, unsigned Depth) const { 10231 OS.indent(Depth) << *getExpr() << " Added Flags: "; 10232 if (SCEVWrapPredicate::IncrementNUSW & getFlags()) 10233 OS << "<nusw>"; 10234 if (SCEVWrapPredicate::IncrementNSSW & getFlags()) 10235 OS << "<nssw>"; 10236 OS << "\n"; 10237 } 10238 10239 SCEVWrapPredicate::IncrementWrapFlags 10240 SCEVWrapPredicate::getImpliedFlags(const SCEVAddRecExpr *AR, 10241 ScalarEvolution &SE) { 10242 IncrementWrapFlags ImpliedFlags = IncrementAnyWrap; 10243 SCEV::NoWrapFlags StaticFlags = AR->getNoWrapFlags(); 10244 10245 // We can safely transfer the NSW flag as NSSW. 10246 if (ScalarEvolution::setFlags(StaticFlags, SCEV::FlagNSW) == StaticFlags) 10247 ImpliedFlags = IncrementNSSW; 10248 10249 if (ScalarEvolution::setFlags(StaticFlags, SCEV::FlagNUW) == StaticFlags) { 10250 // If the increment is positive, the SCEV NUW flag will also imply the 10251 // WrapPredicate NUSW flag. 10252 if (const auto *Step = dyn_cast<SCEVConstant>(AR->getStepRecurrence(SE))) 10253 if (Step->getValue()->getValue().isNonNegative()) 10254 ImpliedFlags = setFlags(ImpliedFlags, IncrementNUSW); 10255 } 10256 10257 return ImpliedFlags; 10258 } 10259 10260 /// Union predicates don't get cached so create a dummy set ID for it. 10261 SCEVUnionPredicate::SCEVUnionPredicate() 10262 : SCEVPredicate(FoldingSetNodeIDRef(nullptr, 0), P_Union) {} 10263 10264 bool SCEVUnionPredicate::isAlwaysTrue() const { 10265 return all_of(Preds, 10266 [](const SCEVPredicate *I) { return I->isAlwaysTrue(); }); 10267 } 10268 10269 ArrayRef<const SCEVPredicate *> 10270 SCEVUnionPredicate::getPredicatesForExpr(const SCEV *Expr) { 10271 auto I = SCEVToPreds.find(Expr); 10272 if (I == SCEVToPreds.end()) 10273 return ArrayRef<const SCEVPredicate *>(); 10274 return I->second; 10275 } 10276 10277 bool SCEVUnionPredicate::implies(const SCEVPredicate *N) const { 10278 if (const auto *Set = dyn_cast<const SCEVUnionPredicate>(N)) 10279 return all_of(Set->Preds, 10280 [this](const SCEVPredicate *I) { return this->implies(I); }); 10281 10282 auto ScevPredsIt = SCEVToPreds.find(N->getExpr()); 10283 if (ScevPredsIt == SCEVToPreds.end()) 10284 return false; 10285 auto &SCEVPreds = ScevPredsIt->second; 10286 10287 return any_of(SCEVPreds, 10288 [N](const SCEVPredicate *I) { return I->implies(N); }); 10289 } 10290 10291 const SCEV *SCEVUnionPredicate::getExpr() const { return nullptr; } 10292 10293 void SCEVUnionPredicate::print(raw_ostream &OS, unsigned Depth) const { 10294 for (auto Pred : Preds) 10295 Pred->print(OS, Depth); 10296 } 10297 10298 void SCEVUnionPredicate::add(const SCEVPredicate *N) { 10299 if (const auto *Set = dyn_cast<const SCEVUnionPredicate>(N)) { 10300 for (auto Pred : Set->Preds) 10301 add(Pred); 10302 return; 10303 } 10304 10305 if (implies(N)) 10306 return; 10307 10308 const SCEV *Key = N->getExpr(); 10309 assert(Key && "Only SCEVUnionPredicate doesn't have an " 10310 " associated expression!"); 10311 10312 SCEVToPreds[Key].push_back(N); 10313 Preds.push_back(N); 10314 } 10315 10316 PredicatedScalarEvolution::PredicatedScalarEvolution(ScalarEvolution &SE, 10317 Loop &L) 10318 : SE(SE), L(L), Generation(0), BackedgeCount(nullptr) {} 10319 10320 const SCEV *PredicatedScalarEvolution::getSCEV(Value *V) { 10321 const SCEV *Expr = SE.getSCEV(V); 10322 RewriteEntry &Entry = RewriteMap[Expr]; 10323 10324 // If we already have an entry and the version matches, return it. 10325 if (Entry.second && Generation == Entry.first) 10326 return Entry.second; 10327 10328 // We found an entry but it's stale. Rewrite the stale entry 10329 // acording to the current predicate. 10330 if (Entry.second) 10331 Expr = Entry.second; 10332 10333 const SCEV *NewSCEV = SE.rewriteUsingPredicate(Expr, &L, Preds); 10334 Entry = {Generation, NewSCEV}; 10335 10336 return NewSCEV; 10337 } 10338 10339 const SCEV *PredicatedScalarEvolution::getBackedgeTakenCount() { 10340 if (!BackedgeCount) { 10341 SCEVUnionPredicate BackedgePred; 10342 BackedgeCount = SE.getPredicatedBackedgeTakenCount(&L, BackedgePred); 10343 addPredicate(BackedgePred); 10344 } 10345 return BackedgeCount; 10346 } 10347 10348 void PredicatedScalarEvolution::addPredicate(const SCEVPredicate &Pred) { 10349 if (Preds.implies(&Pred)) 10350 return; 10351 Preds.add(&Pred); 10352 updateGeneration(); 10353 } 10354 10355 const SCEVUnionPredicate &PredicatedScalarEvolution::getUnionPredicate() const { 10356 return Preds; 10357 } 10358 10359 void PredicatedScalarEvolution::updateGeneration() { 10360 // If the generation number wrapped recompute everything. 10361 if (++Generation == 0) { 10362 for (auto &II : RewriteMap) { 10363 const SCEV *Rewritten = II.second.second; 10364 II.second = {Generation, SE.rewriteUsingPredicate(Rewritten, &L, Preds)}; 10365 } 10366 } 10367 } 10368 10369 void PredicatedScalarEvolution::setNoOverflow( 10370 Value *V, SCEVWrapPredicate::IncrementWrapFlags Flags) { 10371 const SCEV *Expr = getSCEV(V); 10372 const auto *AR = cast<SCEVAddRecExpr>(Expr); 10373 10374 auto ImpliedFlags = SCEVWrapPredicate::getImpliedFlags(AR, SE); 10375 10376 // Clear the statically implied flags. 10377 Flags = SCEVWrapPredicate::clearFlags(Flags, ImpliedFlags); 10378 addPredicate(*SE.getWrapPredicate(AR, Flags)); 10379 10380 auto II = FlagsMap.insert({V, Flags}); 10381 if (!II.second) 10382 II.first->second = SCEVWrapPredicate::setFlags(Flags, II.first->second); 10383 } 10384 10385 bool PredicatedScalarEvolution::hasNoOverflow( 10386 Value *V, SCEVWrapPredicate::IncrementWrapFlags Flags) { 10387 const SCEV *Expr = getSCEV(V); 10388 const auto *AR = cast<SCEVAddRecExpr>(Expr); 10389 10390 Flags = SCEVWrapPredicate::clearFlags( 10391 Flags, SCEVWrapPredicate::getImpliedFlags(AR, SE)); 10392 10393 auto II = FlagsMap.find(V); 10394 10395 if (II != FlagsMap.end()) 10396 Flags = SCEVWrapPredicate::clearFlags(Flags, II->second); 10397 10398 return Flags == SCEVWrapPredicate::IncrementAnyWrap; 10399 } 10400 10401 const SCEVAddRecExpr *PredicatedScalarEvolution::getAsAddRec(Value *V) { 10402 const SCEV *Expr = this->getSCEV(V); 10403 auto *New = SE.convertSCEVToAddRecWithPredicates(Expr, &L, Preds); 10404 10405 if (!New) 10406 return nullptr; 10407 10408 updateGeneration(); 10409 RewriteMap[SE.getSCEV(V)] = {Generation, New}; 10410 return New; 10411 } 10412 10413 PredicatedScalarEvolution::PredicatedScalarEvolution( 10414 const PredicatedScalarEvolution &Init) 10415 : RewriteMap(Init.RewriteMap), SE(Init.SE), L(Init.L), Preds(Init.Preds), 10416 Generation(Init.Generation), BackedgeCount(Init.BackedgeCount) { 10417 for (auto I = Init.FlagsMap.begin(), E = Init.FlagsMap.end(); I != E; ++I) 10418 FlagsMap.insert(*I); 10419 } 10420 10421 void PredicatedScalarEvolution::print(raw_ostream &OS, unsigned Depth) const { 10422 // For each block. 10423 for (auto *BB : L.getBlocks()) 10424 for (auto &I : *BB) { 10425 if (!SE.isSCEVable(I.getType())) 10426 continue; 10427 10428 auto *Expr = SE.getSCEV(&I); 10429 auto II = RewriteMap.find(Expr); 10430 10431 if (II == RewriteMap.end()) 10432 continue; 10433 10434 // Don't print things that are not interesting. 10435 if (II->second.second == Expr) 10436 continue; 10437 10438 OS.indent(Depth) << "[PSE]" << I << ":\n"; 10439 OS.indent(Depth + 2) << *Expr << "\n"; 10440 OS.indent(Depth + 2) << "--> " << *II->second.second << "\n"; 10441 } 10442 } 10443