xref: /llvm-project/llvm/test/Transforms/LoopVectorize/X86/fixed-order-recurrence.ll (revision 7f3428d3ed71d87a2088b77b6cab9f3d86544234) 
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
2; RUN: opt -passes=loop-vectorize -S -o - %s  | FileCheck %s
3
4target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
5target triple = "x86_64-pc_linux"
6
7define void @firstorderrec(ptr nocapture noundef readonly %x, ptr noalias nocapture noundef writeonly %y, i32 noundef %n) {
8; CHECK-LABEL: @firstorderrec(
9; CHECK-NEXT:  entry:
10; CHECK-NEXT:    [[CMP18:%.*]] = icmp sgt i32 [[N:%.*]], 1
11; CHECK-NEXT:    br i1 [[CMP18]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
12; CHECK:       for.body.preheader:
13; CHECK-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[N]] to i64
14; CHECK-NEXT:    [[DOTPRE:%.*]] = load i8, ptr [[X:%.*]], align 1
15; CHECK-NEXT:    [[TMP0:%.*]] = add nsw i64 [[WIDE_TRIP_COUNT]], -1
16; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 32
17; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
18; CHECK:       vector.ph:
19; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 32
20; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
21; CHECK-NEXT:    [[IND_END:%.*]] = add i64 1, [[N_VEC]]
22; CHECK-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <16 x i8> poison, i8 [[DOTPRE]], i32 15
23; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
24; CHECK:       vector.body:
25; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
26; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <16 x i8> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[WIDE_LOAD1:%.*]], [[VECTOR_BODY]] ]
27; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 1, [[INDEX]]
28; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 0
29; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[X]], i64 [[TMP1]]
30; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP3]], i32 0
31; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[TMP3]], i32 16
32; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP5]], align 1
33; CHECK-NEXT:    [[WIDE_LOAD1]] = load <16 x i8>, ptr [[TMP6]], align 1
34; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <16 x i8> [[VECTOR_RECUR]], <16 x i8> [[WIDE_LOAD]], <16 x i32> <i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30>
35; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <16 x i8> [[WIDE_LOAD]], <16 x i8> [[WIDE_LOAD1]], <16 x i32> <i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30>
36; CHECK-NEXT:    [[TMP9:%.*]] = add <16 x i8> [[WIDE_LOAD]], [[TMP7]]
37; CHECK-NEXT:    [[TMP10:%.*]] = add <16 x i8> [[WIDE_LOAD1]], [[TMP8]]
38; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[Y:%.*]], i64 [[TMP1]]
39; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[TMP11]], i32 0
40; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[TMP11]], i32 16
41; CHECK-NEXT:    store <16 x i8> [[TMP9]], ptr [[TMP13]], align 1
42; CHECK-NEXT:    store <16 x i8> [[TMP10]], ptr [[TMP14]], align 1
43; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
44; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
45; CHECK-NEXT:    br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
46; CHECK:       middle.block:
47; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <16 x i8> [[WIDE_LOAD1]], i32 15
48; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
49; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]]
50; CHECK:       scalar.ph:
51; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i8 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ [[DOTPRE]], [[FOR_BODY_PREHEADER]] ]
52; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 1, [[FOR_BODY_PREHEADER]] ]
53; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
54; CHECK:       for.cond.cleanup.loopexit:
55; CHECK-NEXT:    br label [[FOR_COND_CLEANUP]]
56; CHECK:       for.cond.cleanup:
57; CHECK-NEXT:    ret void
58; CHECK:       for.body:
59; CHECK-NEXT:    [[TMP16:%.*]] = phi i8 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP17:%.*]], [[FOR_BODY]] ]
60; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
61; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds i8, ptr [[X]], i64 [[INDVARS_IV]]
62; CHECK-NEXT:    [[TMP17]] = load i8, ptr [[ARRAYIDX4]], align 1
63; CHECK-NEXT:    [[ADD7:%.*]] = add i8 [[TMP17]], [[TMP16]]
64; CHECK-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds i8, ptr [[Y]], i64 [[INDVARS_IV]]
65; CHECK-NEXT:    store i8 [[ADD7]], ptr [[ARRAYIDX10]], align 1
66; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
67; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
68; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
69;
70entry:
71  %cmp18 = icmp sgt i32 %n, 1
72  br i1 %cmp18, label %for.body.preheader, label %for.cond.cleanup
73
74for.body.preheader:                               ; preds = %entry
75  %wide.trip.count = zext i32 %n to i64
76  %.pre = load i8, ptr %x, align 1
77  br label %for.body
78
79for.cond.cleanup:                                 ; preds = %for.body, %entry
80  ret void
81
82for.body:                                         ; preds = %for.body.preheader, %for.body
83  %0 = phi i8 [ %.pre, %for.body.preheader ], [ %1, %for.body ]
84  %indvars.iv = phi i64 [ 1, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
85  %arrayidx4 = getelementptr inbounds i8, ptr %x, i64 %indvars.iv
86  %1 = load i8, ptr %arrayidx4, align 1
87  %add7 = add i8 %1, %0
88  %arrayidx10 = getelementptr inbounds i8, ptr %y, i64 %indvars.iv
89  store i8 %add7, ptr %arrayidx10, align 1
90  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
91  %exitcond.not = icmp eq i64 %indvars.iv.next, %wide.trip.count
92  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
93}
94
95define void @thirdorderrec(ptr nocapture noundef readonly %x, ptr noalias nocapture noundef writeonly %y, i32 noundef %n) #0 {
96; CHECK-LABEL: @thirdorderrec(
97; CHECK-NEXT:  entry:
98; CHECK-NEXT:    [[CMP38:%.*]] = icmp sgt i32 [[N:%.*]], 3
99; CHECK-NEXT:    br i1 [[CMP38]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
100; CHECK:       for.body.preheader:
101; CHECK-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[N]] to i64
102; CHECK-NEXT:    [[DOTPRE:%.*]] = load i8, ptr [[X:%.*]], align 1
103; CHECK-NEXT:    [[ARRAYIDX5_PHI_TRANS_INSERT:%.*]] = getelementptr inbounds i8, ptr [[X]], i64 1
104; CHECK-NEXT:    [[DOTPRE44:%.*]] = load i8, ptr [[ARRAYIDX5_PHI_TRANS_INSERT]], align 1
105; CHECK-NEXT:    [[ARRAYIDX12_PHI_TRANS_INSERT:%.*]] = getelementptr inbounds i8, ptr [[X]], i64 2
106; CHECK-NEXT:    [[DOTPRE45:%.*]] = load i8, ptr [[ARRAYIDX12_PHI_TRANS_INSERT]], align 1
107; CHECK-NEXT:    [[TMP0:%.*]] = add nsw i64 [[WIDE_TRIP_COUNT]], -3
108; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 32
109; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
110; CHECK:       vector.ph:
111; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 32
112; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
113; CHECK-NEXT:    [[IND_END:%.*]] = add i64 3, [[N_VEC]]
114; CHECK-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <16 x i8> poison, i8 [[DOTPRE45]], i32 15
115; CHECK-NEXT:    [[VECTOR_RECUR_INIT1:%.*]] = insertelement <16 x i8> poison, i8 [[DOTPRE44]], i32 15
116; CHECK-NEXT:    [[VECTOR_RECUR_INIT3:%.*]] = insertelement <16 x i8> poison, i8 [[DOTPRE]], i32 15
117; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
118; CHECK:       vector.body:
119; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
120; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <16 x i8> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[WIDE_LOAD5:%.*]], [[VECTOR_BODY]] ]
121; CHECK-NEXT:    [[VECTOR_RECUR2:%.*]] = phi <16 x i8> [ [[VECTOR_RECUR_INIT1]], [[VECTOR_PH]] ], [ [[TMP8:%.*]], [[VECTOR_BODY]] ]
122; CHECK-NEXT:    [[VECTOR_RECUR4:%.*]] = phi <16 x i8> [ [[VECTOR_RECUR_INIT3]], [[VECTOR_PH]] ], [ [[TMP10:%.*]], [[VECTOR_BODY]] ]
123; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 3, [[INDEX]]
124; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 0
125; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[X]], i64 [[TMP1]]
126; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP3]], i32 0
127; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[TMP3]], i32 16
128; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP5]], align 1
129; CHECK-NEXT:    [[WIDE_LOAD5]] = load <16 x i8>, ptr [[TMP6]], align 1
130; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <16 x i8> [[VECTOR_RECUR]], <16 x i8> [[WIDE_LOAD]], <16 x i32> <i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30>
131; CHECK-NEXT:    [[TMP8]] = shufflevector <16 x i8> [[WIDE_LOAD]], <16 x i8> [[WIDE_LOAD5]], <16 x i32> <i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30>
132; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <16 x i8> [[VECTOR_RECUR2]], <16 x i8> [[TMP7]], <16 x i32> <i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30>
133; CHECK-NEXT:    [[TMP10]] = shufflevector <16 x i8> [[TMP7]], <16 x i8> [[TMP8]], <16 x i32> <i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30>
134; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <16 x i8> [[VECTOR_RECUR4]], <16 x i8> [[TMP9]], <16 x i32> <i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30>
135; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <16 x i8> [[TMP9]], <16 x i8> [[TMP10]], <16 x i32> <i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30>
136; CHECK-NEXT:    [[TMP13:%.*]] = add <16 x i8> [[TMP9]], [[TMP11]]
137; CHECK-NEXT:    [[TMP14:%.*]] = add <16 x i8> [[TMP10]], [[TMP12]]
138; CHECK-NEXT:    [[TMP15:%.*]] = add <16 x i8> [[TMP13]], [[TMP7]]
139; CHECK-NEXT:    [[TMP16:%.*]] = add <16 x i8> [[TMP14]], [[TMP8]]
140; CHECK-NEXT:    [[TMP17:%.*]] = add <16 x i8> [[TMP15]], [[WIDE_LOAD]]
141; CHECK-NEXT:    [[TMP18:%.*]] = add <16 x i8> [[TMP16]], [[WIDE_LOAD5]]
142; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i8, ptr [[Y:%.*]], i64 [[TMP1]]
143; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i8, ptr [[TMP19]], i32 0
144; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[TMP19]], i32 16
145; CHECK-NEXT:    store <16 x i8> [[TMP17]], ptr [[TMP21]], align 1
146; CHECK-NEXT:    store <16 x i8> [[TMP18]], ptr [[TMP22]], align 1
147; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
148; CHECK-NEXT:    [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
149; CHECK-NEXT:    br i1 [[TMP23]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
150; CHECK:       middle.block:
151; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <16 x i8> [[WIDE_LOAD5]], i32 15
152; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT6:%.*]] = extractelement <16 x i8> [[TMP8]], i32 15
153; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT7:%.*]] = extractelement <16 x i8> [[TMP10]], i32 15
154; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
155; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]]
156; CHECK:       scalar.ph:
157; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i8 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ [[DOTPRE45]], [[FOR_BODY_PREHEADER]] ]
158; CHECK-NEXT:    [[SCALAR_RECUR_INIT8:%.*]] = phi i8 [ [[VECTOR_RECUR_EXTRACT6]], [[MIDDLE_BLOCK]] ], [ [[DOTPRE44]], [[FOR_BODY_PREHEADER]] ]
159; CHECK-NEXT:    [[SCALAR_RECUR_INIT9:%.*]] = phi i8 [ [[VECTOR_RECUR_EXTRACT7]], [[MIDDLE_BLOCK]] ], [ [[DOTPRE]], [[FOR_BODY_PREHEADER]] ]
160; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 3, [[FOR_BODY_PREHEADER]] ]
161; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
162; CHECK:       for.cond.cleanup.loopexit:
163; CHECK-NEXT:    br label [[FOR_COND_CLEANUP]]
164; CHECK:       for.cond.cleanup:
165; CHECK-NEXT:    ret void
166; CHECK:       for.body:
167; CHECK-NEXT:    [[TMP24:%.*]] = phi i8 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP27:%.*]], [[FOR_BODY]] ]
168; CHECK-NEXT:    [[TMP25:%.*]] = phi i8 [ [[SCALAR_RECUR_INIT8]], [[SCALAR_PH]] ], [ [[TMP24]], [[FOR_BODY]] ]
169; CHECK-NEXT:    [[TMP26:%.*]] = phi i8 [ [[SCALAR_RECUR_INIT9]], [[SCALAR_PH]] ], [ [[TMP25]], [[FOR_BODY]] ]
170; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
171; CHECK-NEXT:    [[ADD8:%.*]] = add i8 [[TMP25]], [[TMP26]]
172; CHECK-NEXT:    [[ADD15:%.*]] = add i8 [[ADD8]], [[TMP24]]
173; CHECK-NEXT:    [[ARRAYIDX18:%.*]] = getelementptr inbounds i8, ptr [[X]], i64 [[INDVARS_IV]]
174; CHECK-NEXT:    [[TMP27]] = load i8, ptr [[ARRAYIDX18]], align 1
175; CHECK-NEXT:    [[ADD21:%.*]] = add i8 [[ADD15]], [[TMP27]]
176; CHECK-NEXT:    [[ARRAYIDX24:%.*]] = getelementptr inbounds i8, ptr [[Y]], i64 [[INDVARS_IV]]
177; CHECK-NEXT:    store i8 [[ADD21]], ptr [[ARRAYIDX24]], align 1
178; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
179; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
180; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
181;
182entry:
183  %cmp38 = icmp sgt i32 %n, 3
184  br i1 %cmp38, label %for.body.preheader, label %for.cond.cleanup
185
186for.body.preheader:                               ; preds = %entry
187  %wide.trip.count = zext i32 %n to i64
188  %.pre = load i8, ptr %x, align 1
189  %arrayidx5.phi.trans.insert = getelementptr inbounds i8, ptr %x, i64 1
190  %.pre44 = load i8, ptr %arrayidx5.phi.trans.insert, align 1
191  %arrayidx12.phi.trans.insert = getelementptr inbounds i8, ptr %x, i64 2
192  %.pre45 = load i8, ptr %arrayidx12.phi.trans.insert, align 1
193  br label %for.body
194
195for.cond.cleanup:                                 ; preds = %for.body, %entry
196  ret void
197
198for.body:                                         ; preds = %for.body.preheader, %for.body
199  %0 = phi i8 [ %.pre45, %for.body.preheader ], [ %3, %for.body ]
200  %1 = phi i8 [ %.pre44, %for.body.preheader ], [ %0, %for.body ]
201  %2 = phi i8 [ %.pre, %for.body.preheader ], [ %1, %for.body ]
202  %indvars.iv = phi i64 [ 3, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
203  %add8 = add i8 %1, %2
204  %add15 = add i8 %add8, %0
205  %arrayidx18 = getelementptr inbounds i8, ptr %x, i64 %indvars.iv
206  %3 = load i8, ptr %arrayidx18, align 1
207  %add21 = add i8 %add15, %3
208  %arrayidx24 = getelementptr inbounds i8, ptr %y, i64 %indvars.iv
209  store i8 %add21, ptr %arrayidx24, align 1
210  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
211  %exitcond.not = icmp eq i64 %indvars.iv.next, %wide.trip.count
212  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
213}
214
215define i64 @test_pr62954_scalar_epilogue_required(ptr %A, ptr noalias %B, ptr %C)  {
216; CHECK-LABEL: @test_pr62954_scalar_epilogue_required(
217; CHECK-NEXT:  entry:
218; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 872
219; CHECK-NEXT:    [[REC_START:%.*]] = load i64, ptr [[GEP]], align 8
220; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
221; CHECK:       vector.ph:
222; CHECK-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <2 x i64> poison, i64 [[REC_START]], i32 1
223; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
224; CHECK:       vector.body:
225; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
226; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 1, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
227; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <2 x i64> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[TMP1:%.*]], [[VECTOR_BODY]] ]
228; CHECK-NEXT:    [[STEP_ADD:%.*]] = add <2 x i64> [[VEC_IND]], splat (i64 4)
229; CHECK-NEXT:    [[TMP1]] = sub nsw <2 x i64> zeroinitializer, [[STEP_ADD]]
230; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x i64> [[TMP1]], i32 1
231; CHECK-NEXT:    store i64 [[TMP2]], ptr [[GEP]], align 8
232; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
233; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[STEP_ADD]], splat (i64 4)
234; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 36
235; CHECK-NEXT:    br i1 [[TMP3]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
236; CHECK:       middle.block:
237; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <2 x i64> [[TMP1]], i32 1
238; CHECK-NEXT:    br label [[SCALAR_PH]]
239; CHECK:       scalar.ph:
240; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 73, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
241; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i64 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ [[REC_START]], [[ENTRY]] ]
242; CHECK-NEXT:    br label [[LOOP:%.*]]
243; CHECK:       loop:
244; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
245; CHECK-NEXT:    [[FOR:%.*]] = phi i64 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[NEG_IV:%.*]], [[LOOP]] ]
246; CHECK-NEXT:    [[GEP_B:%.*]] = getelementptr double, ptr [[B:%.*]], i64 [[IV]]
247; CHECK-NEXT:    [[L_B:%.*]] = load double, ptr [[GEP_B]], align 8
248; CHECK-NEXT:    [[NEG_IV]] = sub nsw i64 0, [[IV]]
249; CHECK-NEXT:    store i64 [[NEG_IV]], ptr [[GEP]], align 8
250; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 2
251; CHECK-NEXT:    [[EC:%.*]] = icmp ugt i64 [[IV]], 74
252; CHECK-NEXT:    br i1 [[EC]], label [[EXIT:%.*]], label [[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
253; CHECK:       exit:
254; CHECK-NEXT:    [[DOTIN_LCSSA:%.*]] = phi i64 [ [[FOR]], [[LOOP]] ]
255; CHECK-NEXT:    [[DOTLCSSA:%.*]] = phi double [ [[L_B]], [[LOOP]] ]
256; CHECK-NEXT:    store double [[DOTLCSSA]], ptr [[C:%.*]], align 8
257; CHECK-NEXT:    ret i64 [[DOTIN_LCSSA]]
258;
259entry:
260  %gep = getelementptr i8, ptr %A, i64 872
261  %rec.start = load i64, ptr  %gep, align 8
262  br label %loop
263
264loop:
265  %iv = phi i64 [ 1, %entry ], [ %iv.next, %loop ]
266  %for = phi i64 [ %rec.start, %entry ], [ %neg.iv, %loop ]
267  %gep.B = getelementptr double, ptr  %B, i64 %iv
268  %l.B = load double, ptr  %gep.B, align 8
269  %neg.iv = sub nsw i64 0, %iv
270  store i64 %neg.iv, ptr %gep, align 8
271  %iv.next = add nuw nsw i64 %iv, 2
272  %ec = icmp ugt i64 %iv, 74
273  br i1 %ec, label %exit, label %loop
274
275exit:
276  %.in.lcssa = phi i64 [ %for, %loop ]
277  %.lcssa = phi double [ %l.B, %loop ]
278  store double %.lcssa, ptr %C
279  ret i64 %.in.lcssa
280}
281
282; Test for https://github.com/llvm/llvm-project/issues/106523.
283; %for.2 requires no code motion, as its previous (%or) precedes its (first)
284; user (store). Furthermore, its user cannot sink, being a store.
285;
286; %for.1 requires code motion, as its previous (%trunc) follows its (first)
287; user (%or). Sinking %or past %trunc seems possible, as %or has no uses
288; (except for feeding %for.2; worth strengthening VPlan's dce?). However, %or
289; is both the user of %for.1 and the previous of %for.2, and we refrain from
290; sinking instructions that act as previous because they (may) serve points to
291; sink after.
292
293; Instead, %for.1 can be reconciled by hoisting its previous above its user
294; %or, as this user %trunc depends only on %iv.
295define void @for_iv_trunc_optimized(ptr %dst) {
296; CHECK-LABEL: @for_iv_trunc_optimized(
297; CHECK-NEXT:  bb:
298; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
299; CHECK:       vector.ph:
300; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
301; CHECK:       vector.body:
302; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
303; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 1>, [[VECTOR_PH]] ], [ [[STEP_ADD:%.*]], [[VECTOR_BODY]] ]
304; CHECK-NEXT:    [[VECTOR_RECUR1:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[TMP3:%.*]], [[VECTOR_BODY]] ]
305; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 1, i32 2, i32 3, i32 4>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
306; CHECK-NEXT:    [[STEP_ADD]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
307; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[VEC_IND]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
308; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[VEC_IND]], <4 x i32> [[STEP_ADD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
309; CHECK-NEXT:    [[TMP2:%.*]] = or <4 x i32> [[TMP0]], splat (i32 3)
310; CHECK-NEXT:    [[TMP3]] = or <4 x i32> [[TMP1]], splat (i32 3)
311; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> [[TMP3]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
312; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x i32> [[TMP5]], i32 3
313; CHECK-NEXT:    store i32 [[TMP6]], ptr [[DST:%.*]], align 4
314; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
315; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[STEP_ADD]], splat (i32 4)
316; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 336
317; CHECK-NEXT:    br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
318; CHECK:       middle.block:
319; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[STEP_ADD]], i32 3
320; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT3:%.*]] = extractelement <4 x i32> [[TMP3]], i32 3
321; CHECK-NEXT:    br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]]
322; CHECK:       scalar.ph:
323; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 337, [[MIDDLE_BLOCK]] ], [ 1, [[BB:%.*]] ]
324; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ 1, [[BB]] ]
325; CHECK-NEXT:    [[SCALAR_RECUR_INIT4:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT3]], [[MIDDLE_BLOCK]] ], [ 0, [[BB]] ]
326; CHECK-NEXT:    br label [[LOOP:%.*]]
327; CHECK:       loop:
328; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[LOOP]] ]
329; CHECK-NEXT:    [[FOR_1:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TRUNC:%.*]], [[LOOP]] ]
330; CHECK-NEXT:    [[FOR_2:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT4]], [[SCALAR_PH]] ], [ [[OR:%.*]], [[LOOP]] ]
331; CHECK-NEXT:    [[OR]] = or i32 [[FOR_1]], 3
332; CHECK-NEXT:    [[ADD]] = add i64 [[IV]], 1
333; CHECK-NEXT:    store i32 [[FOR_2]], ptr [[DST]], align 4
334; CHECK-NEXT:    [[ICMP:%.*]] = icmp ult i64 [[IV]], 337
335; CHECK-NEXT:    [[TRUNC]] = trunc i64 [[IV]] to i32
336; CHECK-NEXT:    br i1 [[ICMP]], label [[LOOP]], label [[EXIT]], !llvm.loop [[LOOP9:![0-9]+]]
337; CHECK:       exit:
338; CHECK-NEXT:    ret void
339;
340bb:
341  br label %loop
342
343loop:
344  %iv = phi i64 [ 1, %bb ], [ %add, %loop ]
345  %for.1 = phi i32 [ 1, %bb ], [ %trunc, %loop ]
346  %for.2 = phi i32 [ 0, %bb ], [ %or, %loop ]
347  %or = or i32 %for.1, 3
348  %add = add i64 %iv, 1
349  store i32 %for.2, ptr %dst, align 4
350  %icmp = icmp ult i64 %iv, 337
351  %trunc = trunc i64 %iv to i32
352  br i1 %icmp, label %loop, label %exit
353
354exit:
355  ret void
356}
357