xref: /llvm-project/llvm/test/Transforms/LoopVectorize/X86/epilog-vectorization-inductions.ll (revision 7f3428d3ed71d87a2088b77b6cab9f3d86544234)
1d58f6707SFlorian Hahn; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
2d58f6707SFlorian Hahn; RUN: opt -passes=loop-vectorize -mcpu=znver2 -S %s | FileCheck %s
3d58f6707SFlorian Hahn
4d58f6707SFlorian Hahntarget triple = "x86_64-unknown-linux-gnu"
5d58f6707SFlorian Hahn
6d58f6707SFlorian Hahndefine void @test_pr59459(i64 %iv.start, ptr %arr) {
7d58f6707SFlorian Hahn; CHECK-LABEL: @test_pr59459(
8d58f6707SFlorian Hahn; CHECK-NEXT:  iter.check:
9d58f6707SFlorian Hahn; CHECK-NEXT:    [[TMP0:%.*]] = trunc i64 [[IV_START:%.*]] to i32
10d58f6707SFlorian Hahn; CHECK-NEXT:    [[SMAX1:%.*]] = call i32 @llvm.smax.i32(i32 [[TMP0]], i32 92)
11d58f6707SFlorian Hahn; CHECK-NEXT:    [[TMP1:%.*]] = sub i32 [[SMAX1]], [[TMP0]]
12d58f6707SFlorian Hahn; CHECK-NEXT:    [[TMP2:%.*]] = zext i32 [[TMP1]] to i64
13d58f6707SFlorian Hahn; CHECK-NEXT:    [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1
14a8538b91SJulian Nagele; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], 4
15d58f6707SFlorian Hahn; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]]
16d58f6707SFlorian Hahn; CHECK:       vector.scevcheck:
17d58f6707SFlorian Hahn; CHECK-NEXT:    [[TMP4:%.*]] = trunc i64 [[IV_START]] to i32
18d58f6707SFlorian Hahn; CHECK-NEXT:    [[SMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[TMP4]], i32 92)
19d58f6707SFlorian Hahn; CHECK-NEXT:    [[TMP5:%.*]] = sub i32 [[SMAX]], [[TMP4]]
20d58f6707SFlorian Hahn; CHECK-NEXT:    [[TMP6:%.*]] = add i32 [[TMP4]], -1
21d58f6707SFlorian Hahn; CHECK-NEXT:    [[TMP7:%.*]] = add i32 [[TMP6]], [[TMP5]]
22d58f6707SFlorian Hahn; CHECK-NEXT:    [[TMP8:%.*]] = icmp ult i32 [[TMP7]], [[TMP6]]
23d58f6707SFlorian Hahn; CHECK-NEXT:    br i1 [[TMP8]], label [[VEC_EPILOG_SCALAR_PH]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
24d58f6707SFlorian Hahn; CHECK:       vector.main.loop.iter.check:
25d58f6707SFlorian Hahn; CHECK-NEXT:    [[MIN_ITERS_CHECK2:%.*]] = icmp ult i64 [[TMP3]], 16
26d58f6707SFlorian Hahn; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK2]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]
27d58f6707SFlorian Hahn; CHECK:       vector.ph:
28d58f6707SFlorian Hahn; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], 16
29d58f6707SFlorian Hahn; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]]
307f7f540aSFlorian Hahn; CHECK-NEXT:    [[IND_END:%.*]] = add i64 [[IV_START]], [[N_VEC]]
31d58f6707SFlorian Hahn; CHECK-NEXT:    [[TMP9:%.*]] = trunc i64 [[IV_START]] to i32
32eae26b66SPaul Walker; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <16 x i32> poison, i32 [[TMP9]], i64 0
33d58f6707SFlorian Hahn; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <16 x i32> [[DOTSPLATINSERT]], <16 x i32> poison, <16 x i32> zeroinitializer
34d58f6707SFlorian Hahn; CHECK-NEXT:    [[INDUCTION:%.*]] = add <16 x i32> [[DOTSPLAT]], <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
35d58f6707SFlorian Hahn; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
36d58f6707SFlorian Hahn; CHECK:       vector.body:
37d58f6707SFlorian Hahn; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
38d58f6707SFlorian Hahn; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <16 x i32> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
39d58f6707SFlorian Hahn; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 [[IV_START]], [[INDEX]]
40d58f6707SFlorian Hahn; CHECK-NEXT:    [[TMP10:%.*]] = trunc i64 [[OFFSET_IDX]] to i32
41d58f6707SFlorian Hahn; CHECK-NEXT:    [[TMP11:%.*]] = add i32 [[TMP10]], 0
42d58f6707SFlorian Hahn; CHECK-NEXT:    [[TMP12:%.*]] = add i32 [[TMP11]], -1
4338fffa63SPaul Walker; CHECK-NEXT:    [[TMP13:%.*]] = mul <16 x i32> [[VEC_IND]], splat (i32 196608)
4438fffa63SPaul Walker; CHECK-NEXT:    [[TMP14:%.*]] = lshr exact <16 x i32> [[TMP13]], splat (i32 16)
45d58f6707SFlorian Hahn; CHECK-NEXT:    [[TMP15:%.*]] = trunc <16 x i32> [[TMP14]] to <16 x i16>
46d58f6707SFlorian Hahn; CHECK-NEXT:    [[TMP16:%.*]] = zext i32 [[TMP12]] to i64
47d58f6707SFlorian Hahn; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr i16, ptr [[ARR:%.*]], i64 [[TMP16]]
48d58f6707SFlorian Hahn; CHECK-NEXT:    [[TMP18:%.*]] = getelementptr i16, ptr [[TMP17]], i32 0
49d58f6707SFlorian Hahn; CHECK-NEXT:    store <16 x i16> [[TMP15]], ptr [[TMP18]], align 2
50d58f6707SFlorian Hahn; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
5138fffa63SPaul Walker; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <16 x i32> [[VEC_IND]], splat (i32 16)
52d58f6707SFlorian Hahn; CHECK-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
53d58f6707SFlorian Hahn; CHECK-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
54d58f6707SFlorian Hahn; CHECK:       middle.block:
55d58f6707SFlorian Hahn; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]]
56d58f6707SFlorian Hahn; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
57d58f6707SFlorian Hahn; CHECK:       vec.epilog.iter.check:
58f69ac9a2SFlorian Hahn; CHECK-NEXT:    [[IND_END6:%.*]] = add i64 [[IV_START]], [[N_VEC]]
59d58f6707SFlorian Hahn; CHECK-NEXT:    [[N_VEC_REMAINING:%.*]] = sub i64 [[TMP3]], [[N_VEC]]
60a8538b91SJulian Nagele; CHECK-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC_REMAINING]], 4
61d58f6707SFlorian Hahn; CHECK-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]]
62d58f6707SFlorian Hahn; CHECK:       vec.epilog.ph:
63d58f6707SFlorian Hahn; CHECK-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
64*7f3428d3SFlorian Hahn; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[IV_START]], [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
65a8538b91SJulian Nagele; CHECK-NEXT:    [[N_MOD_VF3:%.*]] = urem i64 [[TMP3]], 4
66d58f6707SFlorian Hahn; CHECK-NEXT:    [[N_VEC4:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF3]]
67f69ac9a2SFlorian Hahn; CHECK-NEXT:    [[IND_END5:%.*]] = add i64 [[IV_START]], [[N_VEC4]]
68f69ac9a2SFlorian Hahn; CHECK-NEXT:    [[TMP20:%.*]] = trunc i64 [[BC_RESUME_VAL]] to i32
69a8538b91SJulian Nagele; CHECK-NEXT:    [[DOTSPLATINSERT9:%.*]] = insertelement <4 x i32> poison, i32 [[TMP20]], i64 0
70a8538b91SJulian Nagele; CHECK-NEXT:    [[DOTSPLAT10:%.*]] = shufflevector <4 x i32> [[DOTSPLATINSERT9]], <4 x i32> poison, <4 x i32> zeroinitializer
71a8538b91SJulian Nagele; CHECK-NEXT:    [[INDUCTION11:%.*]] = add <4 x i32> [[DOTSPLAT10]], <i32 0, i32 1, i32 2, i32 3>
72d58f6707SFlorian Hahn; CHECK-NEXT:    br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
73d58f6707SFlorian Hahn; CHECK:       vec.epilog.vector.body:
74a8538b91SJulian Nagele; CHECK-NEXT:    [[INDEX8:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT15:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
75a8538b91SJulian Nagele; CHECK-NEXT:    [[VEC_IND12:%.*]] = phi <4 x i32> [ [[INDUCTION11]], [[VEC_EPILOG_PH]] ], [ [[VEC_IND_NEXT13:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
76a8538b91SJulian Nagele; CHECK-NEXT:    [[OFFSET_IDX14:%.*]] = add i64 [[IV_START]], [[INDEX8]]
77a8538b91SJulian Nagele; CHECK-NEXT:    [[TMP21:%.*]] = trunc i64 [[OFFSET_IDX14]] to i32
78d58f6707SFlorian Hahn; CHECK-NEXT:    [[TMP22:%.*]] = add i32 [[TMP21]], 0
79d58f6707SFlorian Hahn; CHECK-NEXT:    [[TMP23:%.*]] = add i32 [[TMP22]], -1
80a8538b91SJulian Nagele; CHECK-NEXT:    [[TMP24:%.*]] = mul <4 x i32> [[VEC_IND12]], splat (i32 196608)
81a8538b91SJulian Nagele; CHECK-NEXT:    [[TMP25:%.*]] = lshr exact <4 x i32> [[TMP24]], splat (i32 16)
82a8538b91SJulian Nagele; CHECK-NEXT:    [[TMP26:%.*]] = trunc <4 x i32> [[TMP25]] to <4 x i16>
83d58f6707SFlorian Hahn; CHECK-NEXT:    [[TMP27:%.*]] = zext i32 [[TMP23]] to i64
84d58f6707SFlorian Hahn; CHECK-NEXT:    [[TMP28:%.*]] = getelementptr i16, ptr [[ARR]], i64 [[TMP27]]
85d58f6707SFlorian Hahn; CHECK-NEXT:    [[TMP29:%.*]] = getelementptr i16, ptr [[TMP28]], i32 0
86a8538b91SJulian Nagele; CHECK-NEXT:    store <4 x i16> [[TMP26]], ptr [[TMP29]], align 2
87a8538b91SJulian Nagele; CHECK-NEXT:    [[INDEX_NEXT15]] = add nuw i64 [[INDEX8]], 4
88a8538b91SJulian Nagele; CHECK-NEXT:    [[VEC_IND_NEXT13]] = add <4 x i32> [[VEC_IND12]], splat (i32 4)
89a8538b91SJulian Nagele; CHECK-NEXT:    [[TMP30:%.*]] = icmp eq i64 [[INDEX_NEXT15]], [[N_VEC4]]
903d4eed01SFlorian Hahn; CHECK-NEXT:    br i1 [[TMP30]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
91d58f6707SFlorian Hahn; CHECK:       vec.epilog.middle.block:
92a8538b91SJulian Nagele; CHECK-NEXT:    [[CMP_N16:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC4]]
93a8538b91SJulian Nagele; CHECK-NEXT:    br i1 [[CMP_N16]], label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
94d58f6707SFlorian Hahn; CHECK:       vec.epilog.scalar.ph:
954ad0fdd1SFlorian Hahn; CHECK-NEXT:    [[BC_RESUME_VAL7:%.*]] = phi i64 [ [[IND_END5]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IV_START]], [[VECTOR_SCEVCHECK]] ], [ [[IV_START]], [[ITER_CHECK:%.*]] ], [ [[IND_END6]], [[VEC_EPILOG_ITER_CHECK]] ]
96d58f6707SFlorian Hahn; CHECK-NEXT:    br label [[LOOP:%.*]]
97d58f6707SFlorian Hahn; CHECK:       loop:
98f69ac9a2SFlorian Hahn; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL7]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
99d58f6707SFlorian Hahn; CHECK-NEXT:    [[IV_TRUNC:%.*]] = trunc i64 [[IV]] to i32
100d58f6707SFlorian Hahn; CHECK-NEXT:    [[STORE_IDX:%.*]] = add i32 [[IV_TRUNC]], -1
101d58f6707SFlorian Hahn; CHECK-NEXT:    [[X:%.*]] = mul i32 [[IV_TRUNC]], 196608
102d58f6707SFlorian Hahn; CHECK-NEXT:    [[Y:%.*]] = lshr exact i32 [[X]], 16
103d58f6707SFlorian Hahn; CHECK-NEXT:    [[STORE_VAL:%.*]] = trunc i32 [[Y]] to i16
104d58f6707SFlorian Hahn; CHECK-NEXT:    [[STORE_IDX_WIDE:%.*]] = zext i32 [[STORE_IDX]] to i64
105d58f6707SFlorian Hahn; CHECK-NEXT:    [[ADDR:%.*]] = getelementptr i16, ptr [[ARR]], i64 [[STORE_IDX_WIDE]]
106d58f6707SFlorian Hahn; CHECK-NEXT:    store i16 [[STORE_VAL]], ptr [[ADDR]], align 2
107d58f6707SFlorian Hahn; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
108d58f6707SFlorian Hahn; CHECK-NEXT:    [[LOOP_COND:%.*]] = icmp sgt i32 [[IV_TRUNC]], 91
109d58f6707SFlorian Hahn; CHECK-NEXT:    br i1 [[LOOP_COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP4:![0-9]+]]
110d58f6707SFlorian Hahn; CHECK:       exit:
111d58f6707SFlorian Hahn; CHECK-NEXT:    ret void
112d58f6707SFlorian Hahn;
113a8538b91SJulian Nagele
114a8538b91SJulian Nagele
115d58f6707SFlorian Hahnentry:
116d58f6707SFlorian Hahn  br label %loop
117d58f6707SFlorian Hahn
118d58f6707SFlorian Hahnloop:
119d58f6707SFlorian Hahn  %iv = phi i64 [ %iv.start, %entry ], [ %iv.next, %loop ]
120d58f6707SFlorian Hahn  %iv.trunc = trunc i64 %iv to i32
121d58f6707SFlorian Hahn  %store.idx = add i32 %iv.trunc, -1
122d58f6707SFlorian Hahn  %x = mul i32 %iv.trunc, 196608
123d58f6707SFlorian Hahn  %y = lshr exact i32 %x, 16
124d58f6707SFlorian Hahn  %store.val = trunc i32 %y to i16
125d58f6707SFlorian Hahn  %store.idx.wide = zext i32 %store.idx to i64
126d58f6707SFlorian Hahn  %addr = getelementptr i16, ptr %arr, i64 %store.idx.wide
127d58f6707SFlorian Hahn  store i16 %store.val, ptr %addr
128d58f6707SFlorian Hahn  %iv.next = add nuw nsw i64 %iv, 1
129d58f6707SFlorian Hahn  %loop.cond = icmp sgt i32 %iv.trunc, 91
130d58f6707SFlorian Hahn  br i1 %loop.cond, label %exit, label %loop
131d58f6707SFlorian Hahn
132d58f6707SFlorian Hahnexit:
133d58f6707SFlorian Hahn  ret void
134d58f6707SFlorian Hahn}
135d58f6707SFlorian Hahn
1363d4eed01SFlorian Hahndefine void @test_induction_step_needs_expansion(ptr noalias %j, ptr %k, i64 %l, i16 %off) {
1373d4eed01SFlorian Hahn; CHECK-LABEL: @test_induction_step_needs_expansion(
1383d4eed01SFlorian Hahn; CHECK-NEXT:  iter.check:
1393d4eed01SFlorian Hahn; CHECK-NEXT:    [[TMP0:%.*]] = sub i16 0, [[OFF:%.*]]
1403d4eed01SFlorian Hahn; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[L:%.*]], 8
1413d4eed01SFlorian Hahn; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
1423d4eed01SFlorian Hahn; CHECK:       vector.main.loop.iter.check:
1433d4eed01SFlorian Hahn; CHECK-NEXT:    [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[L]], 64
1443d4eed01SFlorian Hahn; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK1]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]
1453d4eed01SFlorian Hahn; CHECK:       vector.ph:
1463d4eed01SFlorian Hahn; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[L]], 64
1473d4eed01SFlorian Hahn; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[L]], [[N_MOD_VF]]
1487f7f540aSFlorian Hahn; CHECK-NEXT:    [[DOTCAST:%.*]] = trunc i64 [[N_VEC]] to i16
1497f7f540aSFlorian Hahn; CHECK-NEXT:    [[IND_END:%.*]] = mul i16 [[DOTCAST]], [[TMP0]]
1508ec40675SFlorian Hahn; CHECK-NEXT:    [[DOTSPLATINSERT2:%.*]] = insertelement <16 x i16> poison, i16 [[TMP0]], i64 0
1518ec40675SFlorian Hahn; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i16> [[DOTSPLATINSERT2]], <16 x i16> poison, <16 x i32> zeroinitializer
15238fffa63SPaul Walker; CHECK-NEXT:    [[TMP1:%.*]] = mul <16 x i16> splat (i16 16), [[TMP2]]
1533d4eed01SFlorian Hahn; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <16 x i16> poison, i16 [[TMP0]], i64 0
1543d4eed01SFlorian Hahn; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <16 x i16> [[DOTSPLATINSERT]], <16 x i16> poison, <16 x i32> zeroinitializer
1557f7f540aSFlorian Hahn; CHECK-NEXT:    [[TMP11:%.*]] = mul <16 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>, [[DOTSPLAT]]
1567f7f540aSFlorian Hahn; CHECK-NEXT:    [[INDUCTION:%.*]] = add <16 x i16> zeroinitializer, [[TMP11]]
1577f7f540aSFlorian Hahn; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x i16> poison, i16 [[OFF]], i64 0
1587f7f540aSFlorian Hahn; CHECK-NEXT:    [[BROADCAST_SPLAT3:%.*]] = shufflevector <16 x i16> [[BROADCAST_SPLATINSERT]], <16 x i16> poison, <16 x i32> zeroinitializer
1593d4eed01SFlorian Hahn; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
1603d4eed01SFlorian Hahn; CHECK:       vector.body:
1613d4eed01SFlorian Hahn; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
1623d4eed01SFlorian Hahn; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <16 x i16> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
16338fffa63SPaul Walker; CHECK-NEXT:    [[STEP_ADD:%.*]] = add <16 x i16> [[VEC_IND]], [[TMP1]]
16438fffa63SPaul Walker; CHECK-NEXT:    [[STEP_ADD_2:%.*]] = add <16 x i16> [[STEP_ADD]], [[TMP1]]
16538fffa63SPaul Walker; CHECK-NEXT:    [[STEP_ADD_3:%.*]] = add <16 x i16> [[STEP_ADD_2]], [[TMP1]]
1663d4eed01SFlorian Hahn; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[INDEX]], 0
16738fffa63SPaul Walker; CHECK-NEXT:    [[TMP4:%.*]] = sub <16 x i16> [[VEC_IND]], [[BROADCAST_SPLAT3]]
16838fffa63SPaul Walker; CHECK-NEXT:    [[TMP5:%.*]] = sub <16 x i16> [[STEP_ADD]], [[BROADCAST_SPLAT3]]
16938fffa63SPaul Walker; CHECK-NEXT:    [[TMP6:%.*]] = sub <16 x i16> [[STEP_ADD_2]], [[BROADCAST_SPLAT3]]
17038fffa63SPaul Walker; CHECK-NEXT:    [[TMP7:%.*]] = sub <16 x i16> [[STEP_ADD_3]], [[BROADCAST_SPLAT3]]
17138fffa63SPaul Walker; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i16, ptr [[K:%.*]], i64 [[TMP3]]
17238fffa63SPaul Walker; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i16, ptr [[TMP8]], i32 0
17338fffa63SPaul Walker; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i16, ptr [[TMP8]], i32 16
174*7f3428d3SFlorian Hahn; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i16, ptr [[TMP8]], i32 32
17538fffa63SPaul Walker; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i16, ptr [[TMP8]], i32 48
17638fffa63SPaul Walker; CHECK-NEXT:    store <16 x i16> [[TMP4]], ptr [[TMP9]], align 2
17738fffa63SPaul Walker; CHECK-NEXT:    store <16 x i16> [[TMP5]], ptr [[TMP10]], align 2
178*7f3428d3SFlorian Hahn; CHECK-NEXT:    store <16 x i16> [[TMP6]], ptr [[TMP21]], align 2
17938fffa63SPaul Walker; CHECK-NEXT:    store <16 x i16> [[TMP7]], ptr [[TMP12]], align 2
1803d4eed01SFlorian Hahn; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 64
18138fffa63SPaul Walker; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <16 x i16> [[STEP_ADD_3]], [[TMP1]]
18238fffa63SPaul Walker; CHECK-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
18338fffa63SPaul Walker; CHECK-NEXT:    br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
1843d4eed01SFlorian Hahn; CHECK:       middle.block:
1853d4eed01SFlorian Hahn; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[L]], [[N_VEC]]
1863d4eed01SFlorian Hahn; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
1873d4eed01SFlorian Hahn; CHECK:       vec.epilog.iter.check:
18838fffa63SPaul Walker; CHECK-NEXT:    [[DOTCAST9:%.*]] = trunc i64 [[N_VEC]] to i16
18938fffa63SPaul Walker; CHECK-NEXT:    [[IND_END10:%.*]] = mul i16 [[DOTCAST9]], [[TMP0]]
1903d4eed01SFlorian Hahn; CHECK-NEXT:    [[N_VEC_REMAINING:%.*]] = sub i64 [[L]], [[N_VEC]]
1913d4eed01SFlorian Hahn; CHECK-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC_REMAINING]], 8
1923d4eed01SFlorian Hahn; CHECK-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]]
1933d4eed01SFlorian Hahn; CHECK:       vec.epilog.ph:
1947f7f540aSFlorian Hahn; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
195*7f3428d3SFlorian Hahn; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ [[IND_END]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
19638fffa63SPaul Walker; CHECK-NEXT:    [[N_MOD_VF4:%.*]] = urem i64 [[L]], 8
19738fffa63SPaul Walker; CHECK-NEXT:    [[N_VEC5:%.*]] = sub i64 [[L]], [[N_MOD_VF4]]
19838fffa63SPaul Walker; CHECK-NEXT:    [[DOTCAST7:%.*]] = trunc i64 [[N_VEC5]] to i16
19938fffa63SPaul Walker; CHECK-NEXT:    [[IND_END8:%.*]] = mul i16 [[DOTCAST7]], [[TMP0]]
20038fffa63SPaul Walker; CHECK-NEXT:    [[DOTSPLATINSERT13:%.*]] = insertelement <8 x i16> poison, i16 [[BC_RESUME_VAL]], i64 0
20138fffa63SPaul Walker; CHECK-NEXT:    [[DOTSPLAT14:%.*]] = shufflevector <8 x i16> [[DOTSPLATINSERT13]], <8 x i16> poison, <8 x i32> zeroinitializer
20238fffa63SPaul Walker; CHECK-NEXT:    [[DOTSPLATINSERT15:%.*]] = insertelement <8 x i16> poison, i16 [[TMP0]], i64 0
20338fffa63SPaul Walker; CHECK-NEXT:    [[DOTSPLAT16:%.*]] = shufflevector <8 x i16> [[DOTSPLATINSERT15]], <8 x i16> poison, <8 x i32> zeroinitializer
20438fffa63SPaul Walker; CHECK-NEXT:    [[TMP14:%.*]] = mul <8 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>, [[DOTSPLAT16]]
20538fffa63SPaul Walker; CHECK-NEXT:    [[INDUCTION17:%.*]] = add <8 x i16> [[DOTSPLAT14]], [[TMP14]]
20638fffa63SPaul Walker; CHECK-NEXT:    [[TMP15:%.*]] = mul i16 [[TMP0]], 8
20738fffa63SPaul Walker; CHECK-NEXT:    [[DOTSPLATINSERT18:%.*]] = insertelement <8 x i16> poison, i16 [[TMP15]], i64 0
20838fffa63SPaul Walker; CHECK-NEXT:    [[DOTSPLAT19:%.*]] = shufflevector <8 x i16> [[DOTSPLATINSERT18]], <8 x i16> poison, <8 x i32> zeroinitializer
20938fffa63SPaul Walker; CHECK-NEXT:    [[BROADCAST_SPLATINSERT22:%.*]] = insertelement <8 x i16> poison, i16 [[OFF]], i64 0
21038fffa63SPaul Walker; CHECK-NEXT:    [[BROADCAST_SPLAT23:%.*]] = shufflevector <8 x i16> [[BROADCAST_SPLATINSERT22]], <8 x i16> poison, <8 x i32> zeroinitializer
2113d4eed01SFlorian Hahn; CHECK-NEXT:    br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
2123d4eed01SFlorian Hahn; CHECK:       vec.epilog.vector.body:
2137f7f540aSFlorian Hahn; CHECK-NEXT:    [[INDEX12:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT24:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
21438fffa63SPaul Walker; CHECK-NEXT:    [[VEC_IND20:%.*]] = phi <8 x i16> [ [[INDUCTION17]], [[VEC_EPILOG_PH]] ], [ [[VEC_IND_NEXT21:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
21538fffa63SPaul Walker; CHECK-NEXT:    [[TMP16:%.*]] = add i64 [[INDEX12]], 0
21638fffa63SPaul Walker; CHECK-NEXT:    [[TMP17:%.*]] = sub <8 x i16> [[VEC_IND20]], [[BROADCAST_SPLAT23]]
21738fffa63SPaul Walker; CHECK-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i16, ptr [[K]], i64 [[TMP16]]
21838fffa63SPaul Walker; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i16, ptr [[TMP18]], i32 0
21938fffa63SPaul Walker; CHECK-NEXT:    store <8 x i16> [[TMP17]], ptr [[TMP19]], align 2
22038fffa63SPaul Walker; CHECK-NEXT:    [[INDEX_NEXT24]] = add nuw i64 [[INDEX12]], 8
22138fffa63SPaul Walker; CHECK-NEXT:    [[VEC_IND_NEXT21]] = add <8 x i16> [[VEC_IND20]], [[DOTSPLAT19]]
22238fffa63SPaul Walker; CHECK-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT24]], [[N_VEC5]]
22338fffa63SPaul Walker; CHECK-NEXT:    br i1 [[TMP20]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
2243d4eed01SFlorian Hahn; CHECK:       vec.epilog.middle.block:
22538fffa63SPaul Walker; CHECK-NEXT:    [[CMP_N25:%.*]] = icmp eq i64 [[L]], [[N_VEC5]]
22638fffa63SPaul Walker; CHECK-NEXT:    br i1 [[CMP_N25]], label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
2273d4eed01SFlorian Hahn; CHECK:       vec.epilog.scalar.ph:
2284ad0fdd1SFlorian Hahn; CHECK-NEXT:    [[BC_RESUME_VAL6:%.*]] = phi i64 [ [[N_VEC5]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ]
2294ad0fdd1SFlorian Hahn; CHECK-NEXT:    [[BC_RESUME_VAL11:%.*]] = phi i16 [ [[IND_END8]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK]] ], [ [[IND_END10]], [[VEC_EPILOG_ITER_CHECK]] ]
2303d4eed01SFlorian Hahn; CHECK-NEXT:    br label [[LOOP:%.*]]
2313d4eed01SFlorian Hahn; CHECK:       loop:
23238fffa63SPaul Walker; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL6]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
23338fffa63SPaul Walker; CHECK-NEXT:    [[P_09:%.*]] = phi i16 [ [[BC_RESUME_VAL11]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[ADD:%.*]], [[LOOP]] ]
2343d4eed01SFlorian Hahn; CHECK-NEXT:    [[ADD]] = sub i16 [[P_09]], [[OFF]]
2353d4eed01SFlorian Hahn; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i16, ptr [[K]], i64 [[IV]]
2363d4eed01SFlorian Hahn; CHECK-NEXT:    store i16 [[ADD]], ptr [[ARRAYIDX3]], align 2
2373d4eed01SFlorian Hahn; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
2383d4eed01SFlorian Hahn; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[L]]
2393d4eed01SFlorian Hahn; CHECK-NEXT:    br i1 [[EC]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
2403d4eed01SFlorian Hahn; CHECK:       exit:
2413d4eed01SFlorian Hahn; CHECK-NEXT:    ret void
2423d4eed01SFlorian Hahn;
24338fffa63SPaul Walker
24438fffa63SPaul Walker
2453d4eed01SFlorian Hahnentry:
2463d4eed01SFlorian Hahn  br label %loop
2473d4eed01SFlorian Hahn
2483d4eed01SFlorian Hahnloop:
2493d4eed01SFlorian Hahn  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
2503d4eed01SFlorian Hahn  %p.09 = phi i16 [ 0, %entry ], [ %add, %loop ]
2513d4eed01SFlorian Hahn  %add = sub i16 %p.09, %off
2523d4eed01SFlorian Hahn  %arrayidx3 = getelementptr inbounds i16, ptr %k, i64 %iv
2533d4eed01SFlorian Hahn  store i16 %add, ptr %arrayidx3, align 2
2543d4eed01SFlorian Hahn  %iv.next = add nuw nsw i64 %iv, 1
2553d4eed01SFlorian Hahn  %ec = icmp eq i64 %iv.next, %l
2563d4eed01SFlorian Hahn  br i1 %ec , label %exit, label %loop
2573d4eed01SFlorian Hahn
2583d4eed01SFlorian Hahnexit:
2593d4eed01SFlorian Hahn  ret void
2603d4eed01SFlorian Hahn}
261