1; REQUIRES: asserts 2; RUN: opt -mattr=+neon,+dotprod -passes=loop-vectorize -debug-only=loop-vectorize -force-vector-interleave=1 -enable-epilogue-vectorization -epilogue-vectorization-force-VF=2 -disable-output %s 2>&1 | FileCheck %s 3 4target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128" 5target triple = "aarch64-none-unknown-elf" 6 7; Tests for printing VPlans that are enabled under AArch64 8 9define i32 @print_partial_reduction(ptr %a, ptr %b) { 10; CHECK: VPlan 'Initial VPlan for VF={8,16},UF>=1' { 11; CHECK-NEXT: Live-in vp<[[VFxUF:%.]]> = VF * UF 12; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count 13; CHECK-NEXT: Live-in ir<1024> = original trip-count 14; CHECK-EMPTY: 15; CHECK-NEXT: ir-bb<entry>: 16; CHECK-NEXT: Successor(s): vector.ph 17; CHECK-EMPTY: 18; CHECK-NEXT: vector.ph: 19; CHECK-NEXT: Successor(s): vector loop 20; CHECK-EMPTY: 21; CHECK-NEXT: <x1> vector loop: { 22; CHECK-NEXT: vector.body: 23; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]> 24; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<[[ACC:%.+]]> = phi ir<0>, ir<[[REDUCE:%.+]]> (VF scaled by 1/4) 25; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> 26; CHECK-NEXT: CLONE ir<%gep.a> = getelementptr ir<%a>, vp<[[STEPS]]> 27; CHECK-NEXT: vp<[[PTR_A:%.+]]> = vector-pointer ir<%gep.a> 28; CHECK-NEXT: WIDEN ir<%load.a> = load vp<[[PTR_A]]> 29; CHECK-NEXT: WIDEN-CAST ir<%ext.a> = zext ir<%load.a> to i32 30; CHECK-NEXT: CLONE ir<%gep.b> = getelementptr ir<%b>, vp<[[STEPS]]> 31; CHECK-NEXT: vp<[[PTR_B:%.+]]> = vector-pointer ir<%gep.b> 32; CHECK-NEXT: WIDEN ir<%load.b> = load vp<[[PTR_B]]> 33; CHECK-NEXT: WIDEN-CAST ir<%ext.b> = zext ir<%load.b> to i32 34; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%ext.b>, ir<%ext.a> 35; CHECK-NEXT: PARTIAL-REDUCE ir<[[REDUCE]]> = add ir<%mul>, ir<[[ACC]]> 36; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> 37; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> 38; CHECK-NEXT: No successors 39; CHECK-NEXT: } 40; CHECK-NEXT: Successor(s): middle.block 41; CHECK-EMPTY: 42; CHECK-NEXT: middle.block: 43; CHECK-NEXT: EMIT vp<[[RED_RESULT:%.+]]> = compute-reduction-result ir<[[ACC]]>, ir<[[REDUCE]]> 44; CHECK-NEXT: EMIT vp<[[EXTRACT:%.+]]> = extract-from-end vp<[[RED_RESULT]]>, ir<1> 45; CHECK-NEXT: EMIT vp<[[CMP:%.+]]> = icmp eq ir<1024>, vp<%1> 46; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> 47; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph 48; CHECK-EMPTY: 49; CHECK-NEXT: scalar.ph: 50; CHECK-NEXT: EMIT vp<%bc.resume.val> = resume-phi vp<[[VEC_TC]]>, ir<0> 51; CHECK-NEXT: EMIT vp<%bc.merge.rdx> = resume-phi vp<[[RED_RESULT]]>, ir<0> 52; CHECK-NEXT: Successor(s): ir-bb<for.body> 53; CHECK-EMPTY: 54; CHECK-NEXT: ir-bb<for.body>: 55; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ] 56; CHECK-NEXT: IR %accum = phi i32 [ 0, %entry ], [ %add, %for.body ] (extra operand: vp<%bc.merge.rdx> from scalar.ph) 57; CHECK-NEXT: IR %gep.a = getelementptr i8, ptr %a, i64 %iv 58; CHECK-NEXT: IR %load.a = load i8, ptr %gep.a, align 1 59; CHECK-NEXT: IR %ext.a = zext i8 %load.a to i32 60; CHECK-NEXT: IR %gep.b = getelementptr i8, ptr %b, i64 %iv 61; CHECK-NEXT: IR %load.b = load i8, ptr %gep.b, align 1 62; CHECK-NEXT: IR %ext.b = zext i8 %load.b to i32 63; CHECK-NEXT: IR %mul = mul i32 %ext.b, %ext.a 64; CHECK-NEXT: IR %add = add i32 %mul, %accum 65; CHECK-NEXT: IR %iv.next = add i64 %iv, 1 66; CHECK-NEXT: IR %exitcond.not = icmp eq i64 %iv.next, 1024 67; CHECK-NEXT: No successors 68; CHECK-EMPTY: 69; CHECK-NEXT: ir-bb<exit>: 70; CHECK-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[EXTRACT]]> from middle.block) 71; CHECK-NEXT: No successors 72; CHECK-NEXT: } 73; CHECK: VPlan 'Final VPlan for VF={8,16},UF={1}' { 74; CHECK-NEXT: Live-in ir<[[EP_VFxUF:.+]]> = VF * UF 75; CHECK-NEXT: Live-in ir<[[EP_VEC_TC:.+]]> = vector-trip-count 76; CHECK-NEXT: Live-in ir<1024> = original trip-count 77; CHECK-EMPTY: 78; CHECK-NEXT: ir-bb<entry>: 79; CHECK-NEXT: Successor(s): ir-bb<scalar.ph>, ir-bb<vector.main.loop.iter.check> 80; CHECK-EMPTY: 81; CHECK-NEXT: ir-bb<vector.main.loop.iter.check>: 82; CHECK-NEXT: Successor(s): ir-bb<scalar.ph>, ir-bb<vector.ph> 83; CHECK-EMPTY: 84; CHECK-NEXT: ir-bb<vector.ph>: 85; CHECK-NEXT: Successor(s): vector loop 86; CHECK-EMPTY: 87; CHECK-NEXT: <x1> vector loop: { 88; CHECK-NEXT: vector.body: 89; CHECK-NEXT: SCALAR-PHI vp<[[EP_IV:%.+]]> = phi ir<0>, vp<%index.next> 90; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%accum> = phi ir<0>, ir<%add> (VF scaled by 1/4) 91; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[EP_IV]]>, ir<1> 92; CHECK-NEXT: CLONE ir<%gep.a> = getelementptr ir<%a>, vp<[[STEPS]]> 93; CHECK-NEXT: vp<[[PTR_A:%.+]]> = vector-pointer ir<%gep.a> 94; CHECK-NEXT: WIDEN ir<%load.a> = load vp<[[PTR_A]]> 95; CHECK-NEXT: WIDEN-CAST ir<%ext.a> = zext ir<%load.a> to i32 96; CHECK-NEXT: CLONE ir<%gep.b> = getelementptr ir<%b>, vp<[[STEPS]]> 97; CHECK-NEXT: vp<[[PTR_B:%.+]]> = vector-pointer ir<%gep.b> 98; CHECK-NEXT: WIDEN ir<%load.b> = load vp<[[PTR_B]]> 99; CHECK-NEXT: WIDEN-CAST ir<%ext.b> = zext ir<%load.b> to i32 100; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%ext.b>, ir<%ext.a> 101; CHECK-NEXT: PARTIAL-REDUCE ir<%add> = add ir<%mul>, ir<%accum> 102; CHECK-NEXT: EMIT vp<[[EP_IV_NEXT:%.+]]> = add nuw vp<[[EP_IV]]>, ir<16> 103; CHECK-NEXT: EMIT branch-on-count vp<[[EP_IV_NEXT]]>, ir<1024> 104; CHECK-NEXT: No successors 105; CHECK-NEXT: } 106; CHECK-NEXT: Successor(s): ir-bb<middle.block> 107; CHECK-EMPTY: 108; CHECK-NEXT: ir-bb<middle.block>: 109; CHECK-NEXT: EMIT vp<[[RED_RESULT:%.+]]> = compute-reduction-result ir<%accum>, ir<%add> 110; CHECK-NEXT: EMIT vp<[[EXTRACT:%.+]]> = extract-from-end vp<[[RED_RESULT]]>, ir<1> 111; CHECK-NEXT: EMIT vp<[[CMP:%.+]]> = icmp eq ir<1024>, ir<1024> 112; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> 113; CHECK-NEXT: Successor(s): ir-bb<exit>, ir-bb<scalar.ph> 114; CHECK-EMPTY: 115; CHECK-NEXT: ir-bb<exit>: 116; CHECK-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[EXTRACT]]> from ir-bb<middle.block>) 117; CHECK-NEXT: No successors 118; CHECK-EMPTY: 119; CHECK-NEXT: ir-bb<scalar.ph>: 120; CHECK-NEXT: EMIT vp<[[EP_RESUME:%.+]]> = resume-phi ir<1024>, ir<0> 121; CHECK-NEXT: EMIT vp<[[EP_MERGE:%.+]]> = resume-phi vp<[[RED_RESULT]]>, ir<0> 122; CHECK-NEXT: Successor(s): ir-bb<for.body> 123; CHECK-EMPTY: 124; CHECK-NEXT: ir-bb<for.body>: 125; CHECK-NEXT: IR %accum = phi i32 [ 0, %scalar.ph ], [ %add, %for.body ] (extra operand: vp<[[EP_MERGE]]> from ir-bb<scalar.ph>) 126; CHECK-NEXT: IR %gep.a = getelementptr i8, ptr %a, i64 %iv 127; CHECK-NEXT: IR %load.a = load i8, ptr %gep.a, align 1 128; CHECK-NEXT: IR %ext.a = zext i8 %load.a to i32 129; CHECK-NEXT: IR %gep.b = getelementptr i8, ptr %b, i64 %iv 130; CHECK-NEXT: IR %load.b = load i8, ptr %gep.b, align 1 131; CHECK-NEXT: IR %ext.b = zext i8 %load.b to i32 132; CHECK-NEXT: IR %mul = mul i32 %ext.b, %ext.a 133; CHECK-NEXT: IR %add = add i32 %mul, %accum 134; CHECK-NEXT: IR %iv.next = add i64 %iv, 1 135; CHECK-NEXT: IR %exitcond.not = icmp eq i64 %iv.next, 1024 136; CHECK-NEXT: No successors 137; CHECK-NEXT: } 138entry: 139 br label %for.body 140 141for.body: ; preds = %for.body, %entry 142 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ] 143 %accum = phi i32 [ 0, %entry ], [ %add, %for.body ] 144 %gep.a = getelementptr i8, ptr %a, i64 %iv 145 %load.a = load i8, ptr %gep.a, align 1 146 %ext.a = zext i8 %load.a to i32 147 %gep.b = getelementptr i8, ptr %b, i64 %iv 148 %load.b = load i8, ptr %gep.b, align 1 149 %ext.b = zext i8 %load.b to i32 150 %mul = mul i32 %ext.b, %ext.a 151 %add = add i32 %mul, %accum 152 %iv.next = add i64 %iv, 1 153 %exitcond.not = icmp eq i64 %iv.next, 1024 154 br i1 %exitcond.not, label %exit, label %for.body 155 156exit: 157 ret i32 %add 158} 159