1; REQUIRES: asserts 2; RUN: opt < %s -passes=loop-vectorize -force-vector-width=2 -force-vector-interleave=1 -force-widen-divrem-via-safe-divisor=0 -disable-output -debug-only=loop-vectorize 2>&1 | FileCheck %s 3 4target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128" 5 6; Test cases for PR50009, which require sinking a replicate-region due to a 7; first-order recurrence. 8 9define void @sink_replicate_region_1(i32 %x, ptr %ptr, ptr noalias %dst) optsize { 10; CHECK-LABEL: sink_replicate_region_1 11; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { 12; CHECK-NEXT: Live-in vp<[[VF:%.+]]> = VF 13; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF 14; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count 15; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count 16; CHECK-NEXT: Live-in ir<20001> = original trip-count 17; CHECK-EMPTY: 18; CHECK-NEXT: ir-bb<entry>: 19; CHECK-NEXT: Successor(s): vector.ph 20; CHECK-EMPTY: 21; CHECK-NEXT: vector.ph: 22; CHECK-NEXT: Successor(s): vector loop 23; CHECK-EMPTY: 24; CHECK-NEXT: <x1> vector loop: { 25; CHECK-NEXT: vector.body: 26; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION 27; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%0> = phi ir<0>, ir<%conv> 28; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VF]]> 29; CHECK-NEXT: vp<[[STEPS:%.]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> 30; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]> 31; CHECK-NEXT: Successor(s): pred.load 32; CHECK-EMPTY: 33; CHECK-NEXT: <xVFxUF> pred.load: { 34; CHECK-NEXT: pred.load.entry: 35; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]> 36; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue 37; CHECK-EMPTY: 38; CHECK-NEXT: pred.load.if: 39; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%ptr>, vp<[[STEPS]]> 40; CHECK-NEXT: REPLICATE ir<%lv> = load ir<%gep> (S->V) 41; CHECK-NEXT: Successor(s): pred.load.continue 42; CHECK-EMPTY: 43; CHECK-NEXT: pred.load.continue: 44; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED1:%.+]]> = ir<%lv> 45; CHECK-NEXT: No successors 46; CHECK-NEXT: } 47; CHECK-NEXT: Successor(s): loop.0 48; CHECK-EMPTY: 49; CHECK-NEXT: loop.0: 50; CHECK-NEXT: WIDEN-CAST ir<%conv> = sext vp<[[PRED1]]> to i32 51; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%0>, ir<%conv> 52; CHECK-NEXT: Successor(s): pred.store 53; CHECK-EMPTY: 54; CHECK-NEXT: <xVFxUF> pred.store: { 55; CHECK-NEXT: pred.store.entry: 56; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]> 57; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue 58; CHECK-EMPTY: 59; CHECK-NEXT: pred.store.if: 60; CHECK-NEXT: REPLICATE ir<%rem> = srem vp<[[SPLICE]]>, ir<%x> 61; CHECK-NEXT: REPLICATE ir<%gep.dst> = getelementptr ir<%dst>, vp<[[STEPS]]> 62; CHECK-NEXT: REPLICATE ir<%add> = add ir<%conv>, ir<%rem> 63; CHECK-NEXT: REPLICATE store ir<%add>, ir<%gep.dst> 64; CHECK-NEXT: Successor(s): pred.store.continue 65; CHECK-EMPTY: 66; CHECK-NEXT: pred.store.continue: 67; CHECK-NEXT: No successors 68; CHECK-NEXT: } 69; CHECK-NEXT: Successor(s): loop.1 70; CHECK-EMPTY: 71; CHECK-NEXT: loop.1: 72; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> 73; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> 74; CHECK-NEXT: No successors 75; CHECK-NEXT: } 76; CHECK-NEXT: Successor(s): middle.block 77; CHECK-EMPTY: 78; CHECK-NEXT: middle.block: 79; CHECK-NEXT: EMIT vp<[[RESUME_1:%.+]]> = extract-from-end ir<%conv>, ir<1> 80; CHECK-NEXT: EMIT branch-on-cond ir<true> 81; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph 82; CHECK-EMPTY: 83; CHECK-NEXT: scalar.ph 84; CHECK-NEXT: EMIT vp<[[RESUME_1_P:%.*]]> = resume-phi vp<[[RESUME_1]]>, ir<0> 85; CHECK-NEXT: EMIT vp<[[RESUME_IV:%.*]]> = resume-phi vp<[[VEC_TC]]>, ir<0> 86; CHECK-NEXT: Successor(s): ir-bb<loop> 87; CHECK-EMPTY: 88; CHECK-NEXT: ir-bb<loop>: 89; CHECK-NEXT: IR %0 = phi i32 [ 0, %entry ], [ %conv, %loop ] (extra operand: vp<[[RESUME_1_P]]> from scalar.ph) 90; CHECK-NEXT: IR %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ] (extra operand: vp<[[RESUME_IV]]> from scalar.ph) 91; CHECK: IR %ec = icmp eq i32 %iv.next, 20001 92; CHECK-NEXT: No successors 93; CHECK-EMPTY: 94; CHECK-NEXT: ir-bb<exit> 95; CHECK-NEXT: No successors 96; CHECK-NEXT: } 97; 98entry: 99 br label %loop 100 101loop: 102 %0 = phi i32 [ 0, %entry ], [ %conv, %loop ] 103 %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ] 104 %rem = srem i32 %0, %x 105 %gep = getelementptr i8, ptr %ptr, i32 %iv 106 %lv = load i8, ptr %gep 107 %conv = sext i8 %lv to i32 108 %add = add i32 %conv, %rem 109 %gep.dst = getelementptr i32, ptr %dst, i32 %iv 110 store i32 %add, ptr %gep.dst 111 %iv.next = add nsw i32 %iv, 1 112 %ec = icmp eq i32 %iv.next, 20001 113 br i1 %ec, label %exit, label %loop 114 115exit: 116 ret void 117} 118 119define void @sink_replicate_region_2(i32 %x, i8 %y, ptr %ptr) optsize { 120; CHECK-LABEL: sink_replicate_region_2 121; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { 122; CHECK-NEXT: Live-in vp<[[VF:%.+]]> = VF 123; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF 124; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count 125; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count 126; CHECK-NEXT: Live-in ir<20001> = original trip-count 127; CHECK-EMPTY: 128; CHECK-NEXT: ir-bb<entry>: 129; CHECK-NEXT: Successor(s): vector.ph 130; CHECK-EMPTY: 131; CHECK-NEXT: vector.ph: 132; CHECK-NEXT: WIDEN-CAST ir<%recur.next> = sext ir<%y> to i32 133; CHECK-NEXT: Successor(s): vector loop 134; CHECK-EMPTY: 135; CHECK-NEXT: <x1> vector loop: { 136; CHECK-NEXT: vector.body: 137; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION 138; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%recur> = phi ir<0>, ir<%recur.next> 139; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VF]]> 140; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]> 141; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%recur>, ir<%recur.next> 142; CHECK-NEXT: Successor(s): pred.store 143; CHECK-EMPTY: 144; CHECK-NEXT: <xVFxUF> pred.store: { 145; CHECK-NEXT: pred.store.entry: 146; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]> 147; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue 148; CHECK-EMPTY: 149; CHECK-NEXT: pred.store.if: 150; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> 151; CHECK-NEXT: REPLICATE ir<%rem> = srem vp<[[SPLICE]]>, ir<%x> 152; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%ptr>, vp<[[STEPS]]> 153; CHECK-NEXT: REPLICATE ir<%add> = add ir<%rem>, ir<%recur.next> 154; CHECK-NEXT: REPLICATE store ir<%add>, ir<%gep> 155; CHECK-NEXT: Successor(s): pred.store.continue 156; CHECK-EMPTY: 157; CHECK-NEXT: pred.store.continue: 158; CHECK-NEXT: No successors 159; CHECK-NEXT: } 160; CHECK-NEXT: Successor(s): loop.0 161; CHECK-EMPTY: 162; CHECK-NEXT: loop.0: 163; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> 164; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> 165; CHECK-NEXT: No successors 166; CHECK-NEXT: } 167; CHECK-NEXT: Successor(s): middle.block 168; CHECK-EMPTY: 169; CHECK-NEXT: middle.block: 170; CHECK-NEXT: EMIT vp<[[RESUME_1:%.+]]> = extract-from-end ir<%recur.next>, ir<1> 171; CHECK-NEXT: EMIT branch-on-cond ir<true> 172; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph 173; CHECK-EMPTY: 174; CHECK-NEXT: scalar.ph 175; CHECK-NEXT: EMIT vp<[[RESUME_1_P:%.*]]> = resume-phi vp<[[RESUME_1]]>, ir<0> 176; CHECK-NEXT: EMIT vp<[[RESUME_IV:%.*]]> = resume-phi vp<[[VEC_TC]]>, ir<0> 177; CHECK-NEXT: Successor(s): ir-bb<loop> 178; CHECK-EMPTY: 179; CHECK-NEXT: ir-bb<loop>: 180; CHECK-NEXT: IR %recur = phi i32 [ 0, %entry ], [ %recur.next, %loop ] (extra operand: vp<[[RESUME_1_P]]> from scalar.ph) 181; CHECK-NEXT: IR %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ] (extra operand: vp<[[RESUME_IV]]> from scalar.ph) 182; CHECK: IR %ec = icmp eq i32 %iv.next, 20001 183; CHECK-NEXT: No successors 184; CHECK-EMPTY: 185; CHECK-NEXT: ir-bb<exit> 186; CHECK-NEXT: No successors 187; CHECK-NEXT: } 188; 189entry: 190 br label %loop 191 192loop: 193 %recur = phi i32 [ 0, %entry ], [ %recur.next, %loop ] 194 %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ] 195 %rem = srem i32 %recur, %x 196 %recur.next = sext i8 %y to i32 197 %add = add i32 %rem, %recur.next 198 %gep = getelementptr i32, ptr %ptr, i32 %iv 199 store i32 %add, ptr %gep 200 %iv.next = add nsw i32 %iv, 1 201 %ec = icmp eq i32 %iv.next, 20001 202 br i1 %ec, label %exit, label %loop 203 204exit: 205 ret void 206} 207 208define i32 @sink_replicate_region_3_reduction(i32 %x, i8 %y, ptr %ptr) optsize { 209; CHECK-LABEL: sink_replicate_region_3_reduction 210; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { 211; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF 212; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count 213; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count 214; CHECK-NEXT: Live-in ir<20001> = original trip-count 215; CHECK-EMPTY: 216; CHECK-NEXT: ir-bb<entry>: 217; CHECK-NEXT: Successor(s): vector.ph 218; CHECK-EMPTY: 219; CHECK-NEXT: vector.ph: 220; CHECK-NEXT: WIDEN-CAST ir<%recur.next> = sext ir<%y> to i32 221; CHECK-NEXT: Successor(s): vector loop 222; CHECK-EMPTY: 223; CHECK-NEXT: <x1> vector loop: { 224; CHECK-NEXT: vector.body: 225; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION 226; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%recur> = phi ir<0>, ir<%recur.next> 227; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%and.red> = phi ir<1234>, ir<%and.red.next> 228; CHECK-NEXT: EMIT vp<[[WIDEN_CAN:%.+]]> = WIDEN-CANONICAL-INDUCTION vp<[[CAN_IV]]> 229; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<[[WIDEN_CAN]]>, vp<[[BTC]]> 230; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%recur>, ir<%recur.next> 231; CHECK-NEXT: WIDEN ir<%rem> = srem vp<[[SPLICE]]>, ir<%x> 232; CHECK-NEXT: WIDEN ir<%add> = add ir<%rem>, ir<%recur.next> 233; CHECK-NEXT: WIDEN ir<%and.red.next> = and ir<%and.red>, ir<%add> 234; CHECK-NEXT: EMIT vp<[[SEL:%.+]]> = select vp<[[MASK]]>, ir<%and.red.next>, ir<%and.red> 235; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> 236; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> 237; CHECK-NEXT: No successors 238; CHECK-NEXT: } 239; CHECK-NEXT: Successor(s): middle.block 240; CHECK-EMPTY: 241; CHECK-NEXT: middle.block: 242; CHECK-NEXT: EMIT vp<[[RED_RES:%.+]]> = compute-reduction-result ir<%and.red>, vp<[[SEL]]> 243; CHECK-NEXT: EMIT vp<[[RED_EX:%.+]]> = extract-from-end vp<[[RED_RES]]>, ir<1> 244; CHECK-NEXT: EMIT vp<[[RESUME_1:%.+]]> = extract-from-end ir<%recur.next>, ir<1> 245; CHECK-NEXT: EMIT branch-on-cond ir<true> 246; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph 247; CHECK-EMPTY: 248; CHECK-NEXT: scalar.ph 249; CHECK-NEXT: EMIT vp<[[RESUME_1_P:%.*]]> = resume-phi vp<[[RESUME_1]]>, ir<0> 250; CHECK-NEXT: EMIT vp<[[RESUME_IV:%.*]]> = resume-phi vp<[[VEC_TC]]>, ir<0> 251; CHECK-NEXT: EMIT vp<[[RESUME_RED:%.+]]> = resume-phi vp<[[RED_RES]]>, ir<1234> 252; CHECK-NEXT: Successor(s): ir-bb<loop> 253; CHECK-EMPTY: 254; CHECK-NEXT: ir-bb<loop>: 255; CHECK-NEXT: IR %recur = phi i32 [ 0, %entry ], [ %recur.next, %loop ] (extra operand: vp<[[RESUME_1_P]]> from scalar.ph) 256; CHECK-NEXT: IR %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ] (extra operand: vp<[[RESUME_IV]]> from scalar.ph) 257; CHECK-NEXT: IR %and.red = phi i32 [ 1234, %entry ], [ %and.red.next, %loop ] 258; CHECK: IR %ec = icmp eq i32 %iv.next, 20001 259; CHECK-NEXT: No successors 260; CHECK-EMPTY: 261; CHECK-NEXT: ir-bb<exit> 262; CHECK-NEXT: IR %res = phi i32 [ %and.red.next, %loop ] (extra operand: vp<[[RED_EX]]> from middle.block) 263; CHECK-NEXT: No successors 264; CHECK-NEXT: } 265; 266entry: 267 br label %loop 268 269loop: 270 %recur = phi i32 [ 0, %entry ], [ %recur.next, %loop ] 271 %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ] 272 %and.red = phi i32 [ 1234, %entry ], [ %and.red.next, %loop ] 273 %rem = srem i32 %recur, %x 274 %recur.next = sext i8 %y to i32 275 %add = add i32 %rem, %recur.next 276 %and.red.next = and i32 %and.red, %add 277 %iv.next = add nsw i32 %iv, 1 278 %ec = icmp eq i32 %iv.next, 20001 279 br i1 %ec, label %exit, label %loop 280 281exit: 282 %res = phi i32 [ %and.red.next, %loop ] 283 ret i32 %res 284} 285 286; To sink the replicate region containing %rem, we need to split the block 287; containing %conv at the end, because %conv is the last recipe in the block. 288define void @sink_replicate_region_4_requires_split_at_end_of_block(i32 %x, ptr %ptr, ptr noalias %dst) optsize { 289; CHECK-LABEL: sink_replicate_region_4_requires_split_at_end_of_block 290; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { 291; CHECK-NEXT: Live-in vp<[[VF:%.+]]> = VF 292; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF 293; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count 294; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count 295; CHECK-NEXT: Live-in ir<20001> = original trip-count 296; CHECK-EMPTY: 297; CHECK-NEXT: ir-bb<entry>: 298; CHECK-NEXT: Successor(s): vector.ph 299; CHECK-EMPTY: 300; CHECK-NEXT: vector.ph: 301; CHECK-NEXT: Successor(s): vector loop 302; CHECK-EMPTY: 303; CHECK-NEXT: <x1> vector loop: { 304; CHECK-NEXT: vector.body: 305; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION 306; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%0> = phi ir<0>, ir<%conv> 307; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VF]]> 308; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> 309; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]> 310; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%ptr>, vp<[[STEPS]]> 311; CHECK-NEXT: Successor(s): pred.load 312; CHECK-EMPTY: 313; CHECK-NEXT: <xVFxUF> pred.load: { 314; CHECK-NEXT: pred.load.entry: 315; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]> 316; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue 317; CHECK-EMPTY: 318; CHECK-NEXT: pred.load.if: 319; CHECK-NEXT: REPLICATE ir<%lv> = load ir<%gep> (S->V) 320; CHECK-NEXT: Successor(s): pred.load.continue 321; CHECK-EMPTY: 322; CHECK-NEXT: pred.load.continue: 323; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%lv> 324; CHECK-NEXT: No successors 325; CHECK-NEXT: } 326; CHECK-NEXT: Successor(s): loop.0 327; CHECK-EMPTY: 328; CHECK-NEXT: loop.0: 329; CHECK-NEXT: WIDEN-CAST ir<%conv> = sext vp<[[PRED]]> to i32 330; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%0>, ir<%conv> 331; CHECK-NEXT: Successor(s): pred.store 332; CHECK-EMPTY: 333; CHECK: <xVFxUF> pred.store: { 334; CHECK-NEXT: pred.store.entry: 335; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]> 336; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue 337; CHECK-EMPTY: 338; CHECK: pred.store.if: 339; CHECK-NEXT: REPLICATE ir<%lv.2> = load ir<%gep> 340; CHECK-NEXT: REPLICATE ir<%rem> = srem vp<[[SPLICE]]>, ir<%x> 341; CHECK-NEXT: REPLICATE ir<%conv.lv.2> = sext ir<%lv.2> 342; CHECK-NEXT: REPLICATE ir<%add.1> = add ir<%conv>, ir<%rem> 343; CHECK-NEXT: REPLICATE ir<%gep.dst> = getelementptr ir<%dst>, vp<[[STEPS]]> 344; CHECK-NEXT: REPLICATE ir<%add> = add ir<%add.1>, ir<%conv.lv.2> 345; CHECK-NEXT: REPLICATE store ir<%add>, ir<%gep.dst> 346; CHECK-NEXT: Successor(s): pred.store.continue 347; CHECK-EMPTY: 348; CHECK: pred.store.continue: 349; CHECK-NEXT: No successors 350; CHECK-NEXT: } 351; CHECK-NEXT: Successor(s): loop.2 352; CHECK-EMPTY: 353; CHECK: loop.2: 354; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> 355; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> 356; CHECK-NEXT: No successors 357; CHECK-NEXT: } 358; CHECK-NEXT: Successor(s): middle.block 359; CHECK-EMPTY: 360; CHECK-NEXT: middle.block: 361; CHECK-NEXT: EMIT vp<[[RESUME_1:%.+]]> = extract-from-end ir<%conv>, ir<1> 362; CHECK-NEXT: EMIT branch-on-cond ir<true> 363; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph 364; CHECK-EMPTY: 365; CHECK-NEXT: scalar.ph 366; CHECK-NEXT: EMIT vp<[[RESUME_1_P:%.*]]> = resume-phi vp<[[RESUME_1]]>, ir<0> 367; CHECK-NEXT: EMIT vp<[[RESUME_IV:%.*]]> = resume-phi vp<[[VEC_TC]]>, ir<0> 368; CHECK-NEXT: Successor(s): ir-bb<loop> 369; CHECK-EMPTY: 370; CHECK-NEXT: ir-bb<loop>: 371; CHECK-NEXT: IR %0 = phi i32 [ 0, %entry ], [ %conv, %loop ] (extra operand: vp<[[RESUME_1_P]]> from scalar.ph) 372; CHECK-NEXT: IR %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ] (extra operand: vp<[[RESUME_IV]]> from scalar.ph) 373; CHECK: IR %ec = icmp eq i32 %iv.next, 20001 374; CHECK-NEXT: No successors 375; CHECK-EMPTY: 376; CHECK-NEXT: ir-bb<exit> 377; CHECK-NEXT: No successors 378; CHECK-NEXT: } 379; 380entry: 381 br label %loop 382 383loop: 384 %0 = phi i32 [ 0, %entry ], [ %conv, %loop ] 385 %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ] 386 %gep = getelementptr i8, ptr %ptr, i32 %iv 387 %rem = srem i32 %0, %x 388 %lv = load i8, ptr %gep 389 %conv = sext i8 %lv to i32 390 %lv.2 = load i8, ptr %gep 391 %add.1 = add i32 %conv, %rem 392 %conv.lv.2 = sext i8 %lv.2 to i32 393 %add = add i32 %add.1, %conv.lv.2 394 %gep.dst = getelementptr i32, ptr %dst, i32 %iv 395 store i32 %add, ptr %gep.dst 396 %iv.next = add nsw i32 %iv, 1 397 %ec = icmp eq i32 %iv.next, 20001 398 br i1 %ec, label %exit, label %loop 399 400exit: 401 ret void 402} 403 404; Test case that requires sinking a recipe in a replicate region after another replicate region. 405define void @sink_replicate_region_after_replicate_region(ptr %ptr, ptr noalias %dst.2, i32 %x, i8 %y) optsize { 406; CHECK-LABEL: sink_replicate_region_after_replicate_region 407; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { 408; CHECK-NEXT: Live-in vp<[[VF:%.+]]> = VF 409; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF 410; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count 411; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count 412; CHECK-NEXT: vp<[[TC:%.+]]> = original trip-count 413; CHECK-EMPTY: 414; CHECK-NEXT: ir-bb<entry>: 415; CHECK-NEXT: EMIT vp<[[TC]]> = EXPAND SCEV (1 smax (1 + (sext i8 %y to i32))<nsw>) 416; CHECK-NEXT: Successor(s): vector.ph 417; CHECK-EMPTY: 418; CHECK-NEXT: vector.ph: 419; CHECK-NEXT: WIDEN-CAST ir<%recur.next> = sext ir<%y> to i32 420; CHECK-NEXT: Successor(s): vector loop 421; CHECK-EMPTY: 422; CHECK-NEXT: <x1> vector loop: { 423; CHECK-NEXT: vector.body: 424; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION 425; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%recur> = phi ir<0>, ir<%recur.next> 426; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VF]]> 427; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]> 428; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%recur>, ir<%recur.next> 429; CHECK-NEXT: Successor(s): pred.store 430; CHECK-EMPTY: 431; CHECK-NEXT: <xVFxUF> pred.store: { 432; CHECK-NEXT: pred.store.entry: 433; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]> 434; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue 435; CHECK-EMPTY: 436; CHECK-NEXT: pred.store.if: 437; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> 438; CHECK-NEXT: REPLICATE ir<%rem> = srem vp<[[SPLICE]]>, ir<%x> 439; CHECK-NEXT: REPLICATE ir<%rem.div> = sdiv ir<20>, ir<%rem> 440; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%ptr>, vp<[[STEPS]]> 441; CHECK-NEXT: REPLICATE store ir<%rem.div>, ir<%gep> 442; CHECK-NEXT: REPLICATE ir<%gep.2> = getelementptr ir<%dst.2>, vp<[[STEPS]]> 443; CHECK-NEXT: REPLICATE store ir<%rem.div>, ir<%gep.2> 444; CHECK-NEXT: Successor(s): pred.store.continue 445; CHECK-EMPTY: 446; CHECK-NEXT: pred.store.continue: 447; CHECK-NEXT: No successors 448; CHECK-NEXT: } 449; CHECK-NEXT: Successor(s): loop.2 450; CHECK-EMPTY: 451; CHECK-NEXT: loop.2: 452; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> 453; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> 454; CHECK-NEXT: No successors 455; CHECK-NEXT: } 456; CHECK-NEXT: Successor(s): middle.block 457; CHECK-EMPTY: 458; CHECK-NEXT: middle.block: 459; CHECK-NEXT: EMIT vp<[[RESUME_1:%.+]]> = extract-from-end ir<%recur.next>, ir<1> 460; CHECK-NEXT: EMIT branch-on-cond ir<true> 461; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph 462; CHECK-EMPTY: 463; CHECK-NEXT: scalar.ph 464; CHECK-NEXT: EMIT vp<[[RESUME_1_P:%.*]]> = resume-phi vp<[[RESUME_1]]>, ir<0> 465; CHECK-NEXT: EMIT vp<[[RESUME_IV:%.*]]> = resume-phi vp<[[VEC_TC]]>, ir<0> 466; CHECK-NEXT: Successor(s): ir-bb<loop> 467; CHECK-EMPTY: 468; CHECK-NEXT: ir-bb<loop>: 469; CHECK-NEXT: IR %recur = phi i32 [ 0, %entry ], [ %recur.next, %loop ] (extra operand: vp<[[RESUME_1_P]]> from scalar.ph) 470; CHECK-NEXT: IR %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ] (extra operand: vp<[[RESUME_IV]]> from scalar.ph) 471; CHECK: IR %C = icmp sgt i32 %iv.next, %recur.next 472; CHECK-NEXT: No successors 473; CHECK-EMPTY: 474; CHECK-NEXT: ir-bb<exit> 475; CHECK-NEXT: No successors 476; CHECK-NEXT: } 477; 478entry: 479 br label %loop 480 481loop: ; preds = %loop, %entry 482 %recur = phi i32 [ 0, %entry ], [ %recur.next, %loop ] 483 %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ] 484 %rem = srem i32 %recur, %x 485 %rem.div = sdiv i32 20, %rem 486 %recur.next = sext i8 %y to i32 487 %gep = getelementptr i32, ptr %ptr, i32 %iv 488 store i32 %rem.div, ptr %gep 489 %gep.2 = getelementptr i32, ptr %dst.2, i32 %iv 490 store i32 %rem.div, ptr %gep.2 491 %iv.next = add nsw i32 %iv, 1 492 %C = icmp sgt i32 %iv.next, %recur.next 493 br i1 %C, label %exit, label %loop 494 495exit: ; preds = %loop 496 ret void 497} 498 499define void @need_new_block_after_sinking_pr56146(i32 %x, ptr %src, ptr noalias %dst) { 500; CHECK-LABEL: need_new_block_after_sinking_pr56146 501; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { 502; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF 503; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count 504; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count 505; CHECK-NEXT: Live-in ir<3> = original trip-count 506; CHECK-EMPTY: 507; CHECK-NEXT: ir-bb<entry>: 508; CHECK-NEXT: Successor(s): vector.ph 509; CHECK-EMPTY: 510; CHECK-NEXT: vector.ph: 511; CHECK-NEXT: vp<[[END:%.+]]> = DERIVED-IV ir<2> + vp<[[VEC_TC]]> * ir<1> 512; CHECK-NEXT: Successor(s): vector loop 513; CHECK-EMPTY: 514; CHECK-NEXT: <x1> vector loop: { 515; CHECK-NEXT: vector.body: 516; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION 517; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%.pn> = phi ir<0>, ir<[[L:%.+]]> 518; CHECK-NEXT: vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<2> + vp<[[CAN_IV]]> * ir<1> 519; CHECK-NEXT: EMIT vp<[[WIDE_IV:%.+]]> = WIDEN-CANONICAL-INDUCTION vp<[[CAN_IV]]> 520; CHECK-NEXT: EMIT vp<[[CMP:%.+]]> = icmp ule vp<[[WIDE_IV]]>, vp<[[BTC]]> 521; CHECK-NEXT: CLONE ir<[[L]]> = load ir<%src> 522; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%.pn>, ir<[[L]]> 523; CHECK-NEXT: Successor(s): pred.store 524; CHECK-EMPTY: 525; CHECK-NEXT: <xVFxUF> pred.store: { 526; CHECK-NEXT: pred.store.entry: 527; CHECK-NEXT: BRANCH-ON-MASK vp<[[CMP]]> 528; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue 529; CHECK-EMPTY: 530; CHECK-NEXT: pred.store.if: 531; CHECK-NEXT: vp<[[SCALAR_STEPS:%.+]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir<1> 532; CHECK-NEXT: REPLICATE ir<%gep.dst> = getelementptr ir<%dst>, vp<[[SCALAR_STEPS]]> 533; CHECK-NEXT: REPLICATE ir<%val> = sdiv vp<[[SPLICE]]>, ir<%x> 534; CHECK-NEXT: REPLICATE store ir<%val>, ir<%gep.dst> 535; CHECK-NEXT: Successor(s): pred.store.continue 536; CHECK-EMPTY: 537; CHECK-NEXT: pred.store.continue: 538; CHECK-NEXT: No successors 539; CHECK-NEXT: } 540; CHECK-NEXT: Successor(s): loop.0 541; CHECK-EMPTY: 542; CHECK-NEXT: loop.0: 543; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> 544; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> 545; CHECK-NEXT: No successors 546; CHECK-NEXT: } 547; CHECK-NEXT: Successor(s): middle.block 548; CHECK-EMPTY: 549; CHECK-NEXT: middle.block: 550; CHECK-NEXT: EMIT vp<[[RESUME_1:%.+]]> = extract-from-end ir<%l>, ir<1> 551; CHECK-NEXT: EMIT branch-on-cond ir<true> 552; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph 553; CHECK-EMPTY: 554; CHECK-NEXT: scalar.ph 555; CHECK-NEXT: EMIT vp<[[RESUME_IV:%.*]]> = resume-phi vp<[[END]]>, ir<2> 556; CHECK-NEXT: EMIT vp<[[RESUME_1_P:%.*]]> = resume-phi vp<[[RESUME_1]]>, ir<0> 557; CHECK-NEXT: Successor(s): ir-bb<loop> 558; CHECK-EMPTY: 559; CHECK-NEXT: ir-bb<loop>: 560; CHECK-NEXT: IR %iv = phi i64 [ 2, %entry ], [ %iv.next, %loop ] (extra operand: vp<[[RESUME_IV]]> from scalar.ph) 561; CHECK-NEXT: IR %.pn = phi i32 [ 0, %entry ], [ %l, %loop ] (extra operand: vp<[[RESUME_1_P]]> from scalar.ph) 562; CHECK: IR %ec = icmp ugt i64 %iv, 3 563; CHECK-NEXT: No successors 564; CHECK-EMPTY: 565; CHECK-NEXT: ir-bb<exit> 566; CHECK-NEXT: No successors 567; CHECK-NEXT: } 568; 569entry: 570 br label %loop 571 572loop: 573 %iv = phi i64 [ 2, %entry ], [ %iv.next, %loop ] 574 %.pn = phi i32 [ 0, %entry ], [ %l, %loop ] 575 %val = sdiv i32 %.pn, %x 576 %l = load i32, ptr %src, align 4 577 %gep.dst = getelementptr i32, ptr %dst, i64 %iv 578 store i32 %val, ptr %gep.dst 579 %iv.next = add nuw nsw i64 %iv, 1 580 %ec = icmp ugt i64 %iv, 3 581 br i1 %ec, label %exit, label %loop 582 583exit: 584 ret void 585} 586