xref: /llvm-project/llvm/unittests/Transforms/Vectorize/VPlanSlpTest.cpp (revision 09e516c54b0c5ec3fe6b3ef6c70d9e09e89abc95)
1 //===- llvm/unittest/Transforms/Vectorize/VPlanSlpTest.cpp ---------------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 
10 #include "../lib/Transforms/Vectorize/VPlan.h"
11 #include "../lib/Transforms/Vectorize/VPlanHCFGBuilder.h"
12 #include "../lib/Transforms/Vectorize/VPlanHCFGTransforms.h"
13 #include "VPlanTestBase.h"
14 #include "llvm/Analysis/VectorUtils.h"
15 #include "gtest/gtest.h"
16 
17 namespace llvm {
18 namespace {
19 
20 class VPlanSlpTest : public VPlanTestBase {
21 protected:
22   TargetLibraryInfoImpl TLII;
23   TargetLibraryInfo TLI;
24   DataLayout DL;
25 
26   std::unique_ptr<AssumptionCache> AC;
27   std::unique_ptr<ScalarEvolution> SE;
28   std::unique_ptr<AAResults> AARes;
29   std::unique_ptr<BasicAAResult> BasicAA;
30   std::unique_ptr<LoopAccessInfo> LAI;
31   std::unique_ptr<PredicatedScalarEvolution> PSE;
32   std::unique_ptr<InterleavedAccessInfo> IAI;
33 
34   VPlanSlpTest()
35       : TLII(), TLI(TLII),
36         DL("e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-"
37            "f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:"
38            "16:32:64-S128") {}
39 
40   VPInterleavedAccessInfo getInterleavedAccessInfo(Function &F, Loop *L,
41                                                    VPlan &Plan) {
42     AC.reset(new AssumptionCache(F));
43     SE.reset(new ScalarEvolution(F, TLI, *AC, *DT, *LI));
44     BasicAA.reset(new BasicAAResult(DL, F, TLI, *AC, &*DT, &*LI));
45     AARes.reset(new AAResults(TLI));
46     AARes->addAAResult(*BasicAA);
47     PSE.reset(new PredicatedScalarEvolution(*SE, *L));
48     LAI.reset(new LoopAccessInfo(L, &*SE, &TLI, &*AARes, &*DT, &*LI));
49     IAI.reset(new InterleavedAccessInfo(*PSE, L, &*DT, &*LI, &*LAI));
50     IAI->analyzeInterleaving(false);
51     return {Plan, *IAI};
52   }
53 };
54 
55 TEST_F(VPlanSlpTest, testSlpSimple_2) {
56   const char *ModuleString =
57       "%struct.Test = type { i32, i32 }\n"
58       "%struct.Test3 = type { i32, i32, i32 }\n"
59       "%struct.Test4xi8 = type { i8, i8, i8 }\n"
60       "define void @add_x2(%struct.Test* nocapture readonly %A, %struct.Test* "
61       "nocapture readonly %B, %struct.Test* nocapture %C)  {\n"
62       "entry:\n"
63       "  br label %for.body\n"
64       "for.body:                                         ; preds = %for.body, "
65       "%entry\n"
66       "  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n"
67       "  %A0 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 "
68       "%indvars.iv, i32 0\n"
69       "  %vA0 = load i32, i32* %A0, align 4\n"
70       "  %B0 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 "
71       "%indvars.iv, i32 0\n"
72       "  %vB0 = load i32, i32* %B0, align 4\n"
73       "  %add0 = add nsw i32 %vA0, %vB0\n"
74       "  %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 "
75       "%indvars.iv, i32 1\n"
76       "  %vA1 = load i32, i32* %A1, align 4\n"
77       "  %B1 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 "
78       "%indvars.iv, i32 1\n"
79       "  %vB1 = load i32, i32* %B1, align 4\n"
80       "  %add1 = add nsw i32 %vA1, %vB1\n"
81       "  %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
82       "%indvars.iv, i32 0\n"
83       "  store i32 %add0, i32* %C0, align 4\n"
84       "  %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
85       "%indvars.iv, i32 1\n"
86       "  store i32 %add1, i32* %C1, align 4\n"
87       "  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n"
88       "  %exitcond = icmp eq i64 %indvars.iv.next, 1024\n"
89       "  br i1 %exitcond, label %for.cond.cleanup, label %for.body\n"
90       "for.cond.cleanup:                                 ; preds = %for.body\n"
91       "  ret void\n"
92       "}\n";
93 
94   Module &M = parseModule(ModuleString);
95 
96   Function *F = M.getFunction("add_x2");
97   BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor();
98   auto Plan = buildHCFG(LoopHeader);
99   auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan);
100 
101   VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock();
102   EXPECT_NE(nullptr, Entry->getSingleSuccessor());
103   VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock();
104 
105   VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 12));
106   VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 14));
107 
108   VPlanSlp Slp(VPIAI, *Body);
109   SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2};
110   VPInstruction *CombinedStore = Slp.buildGraph(StoreRoot);
111   EXPECT_EQ(64u, Slp.getWidestBundleBits());
112   EXPECT_EQ(VPInstruction::SLPStore, CombinedStore->getOpcode());
113 
114   auto *CombinedAdd = cast<VPInstruction>(CombinedStore->getOperand(0));
115   EXPECT_EQ(Instruction::Add, CombinedAdd->getOpcode());
116 
117   auto *CombinedLoadA = cast<VPInstruction>(CombinedAdd->getOperand(0));
118   auto *CombinedLoadB = cast<VPInstruction>(CombinedAdd->getOperand(1));
119   EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadA->getOpcode());
120   EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadB->getOpcode());
121 }
122 
123 TEST_F(VPlanSlpTest, testSlpSimple_3) {
124   const char *ModuleString =
125       "%struct.Test = type { i32, i32 }\n"
126       "%struct.Test3 = type { i32, i32, i32 }\n"
127       "%struct.Test4xi8 = type { i8, i8, i8 }\n"
128       "define void @add_x2(%struct.Test* nocapture readonly %A, %struct.Test* "
129       "nocapture readonly %B, %struct.Test* nocapture %C)  {\n"
130       "entry:\n"
131       "  br label %for.body\n"
132       "for.body:                                         ; preds = %for.body, "
133       "%entry\n"
134       "  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n"
135       "  %A0 = getelementptr %struct.Test, %struct.Test* %A, i64 "
136       "                      %indvars.iv, i32 0\n"
137       "  %vA0 = load i32, i32* %A0, align 4\n"
138       "  %B0 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 "
139       "                      %indvars.iv, i32 0\n"
140       "  %vB0 = load i32, i32* %B0, align 4\n"
141       "  %add0 = add nsw i32 %vA0, %vB0\n"
142       "  %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 "
143       "                      %indvars.iv, i32 1\n"
144       "  %vA1 = load i32, i32* %A1, align 4\n"
145       "  %B1 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 "
146       "                      %indvars.iv, i32 1\n"
147       "  %vB1 = load i32, i32* %B1, align 4\n"
148       "  %add1 = add nsw i32 %vA1, %vB1\n"
149       "  %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
150       "                      %indvars.iv, i32 0\n"
151       "  store i32 %add0, i32* %C0, align 4\n"
152       "  %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
153       "                      %indvars.iv, i32 1\n"
154       "  store i32 %add1, i32* %C1, align 4\n"
155       "  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n"
156       "  %exitcond = icmp eq i64 %indvars.iv.next, 1024\n"
157       "  br i1 %exitcond, label %for.cond.cleanup, label %for.body\n"
158       "for.cond.cleanup:                                 ; preds = %for.body\n"
159       "  ret void\n"
160       "}\n";
161 
162   Module &M = parseModule(ModuleString);
163 
164   Function *F = M.getFunction("add_x2");
165   BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor();
166   auto Plan = buildHCFG(LoopHeader);
167 
168   VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock();
169   EXPECT_NE(nullptr, Entry->getSingleSuccessor());
170   VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock();
171 
172   VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 12));
173   VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 14));
174 
175   auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan);
176 
177   VPlanSlp Slp(VPIAI, *Body);
178   SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2};
179   VPInstruction *CombinedStore = Slp.buildGraph(StoreRoot);
180   EXPECT_EQ(64u, Slp.getWidestBundleBits());
181   EXPECT_EQ(VPInstruction::SLPStore, CombinedStore->getOpcode());
182 
183   auto *CombinedAdd = cast<VPInstruction>(CombinedStore->getOperand(0));
184   EXPECT_EQ(Instruction::Add, CombinedAdd->getOpcode());
185 
186   auto *CombinedLoadA = cast<VPInstruction>(CombinedAdd->getOperand(0));
187   auto *CombinedLoadB = cast<VPInstruction>(CombinedAdd->getOperand(1));
188   EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadA->getOpcode());
189   EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadB->getOpcode());
190 
191   VPInstruction *GetA = cast<VPInstruction>(&*std::next(Body->begin(), 1));
192   VPInstruction *GetB = cast<VPInstruction>(&*std::next(Body->begin(), 3));
193   EXPECT_EQ(GetA, CombinedLoadA->getOperand(0));
194   EXPECT_EQ(GetB, CombinedLoadB->getOperand(0));
195 }
196 
197 TEST_F(VPlanSlpTest, testSlpReuse_1) {
198   const char *ModuleString =
199       "%struct.Test = type { i32, i32 }\n"
200       "define void @add_x2(%struct.Test* nocapture readonly %A, %struct.Test* "
201       "nocapture readonly %B, %struct.Test* nocapture %C)  {\n"
202       "entry:\n"
203       "  br label %for.body\n"
204       "for.body:                                         ; preds = %for.body, "
205       "%entry\n"
206       "  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n"
207       "  %A0 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 "
208       "%indvars.iv, i32 0\n"
209       "  %vA0 = load i32, i32* %A0, align 4\n"
210       "  %add0 = add nsw i32 %vA0, %vA0\n"
211       "  %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 "
212       "%indvars.iv, i32 1\n"
213       "  %vA1 = load i32, i32* %A1, align 4\n"
214       "  %add1 = add nsw i32 %vA1, %vA1\n"
215       "  %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
216       "%indvars.iv, i32 0\n"
217       "  store i32 %add0, i32* %C0, align 4\n"
218       "  %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
219       "%indvars.iv, i32 1\n"
220       "  store i32 %add1, i32* %C1, align 4\n"
221       "  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n"
222       "  %exitcond = icmp eq i64 %indvars.iv.next, 1024\n"
223       "  br i1 %exitcond, label %for.cond.cleanup, label %for.body\n"
224       "for.cond.cleanup:                                 ; preds = %for.body\n"
225       "  ret void\n"
226       "}\n";
227 
228   Module &M = parseModule(ModuleString);
229 
230   Function *F = M.getFunction("add_x2");
231   BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor();
232   auto Plan = buildHCFG(LoopHeader);
233   auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan);
234 
235   VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock();
236   EXPECT_NE(nullptr, Entry->getSingleSuccessor());
237   VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock();
238 
239   VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 8));
240   VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 10));
241 
242   VPlanSlp Slp(VPIAI, *Body);
243   SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2};
244   VPInstruction *CombinedStore = Slp.buildGraph(StoreRoot);
245   EXPECT_EQ(64u, Slp.getWidestBundleBits());
246   EXPECT_EQ(VPInstruction::SLPStore, CombinedStore->getOpcode());
247 
248   auto *CombinedAdd = cast<VPInstruction>(CombinedStore->getOperand(0));
249   EXPECT_EQ(Instruction::Add, CombinedAdd->getOpcode());
250 
251   auto *CombinedLoadA = cast<VPInstruction>(CombinedAdd->getOperand(0));
252   EXPECT_EQ(CombinedLoadA, CombinedAdd->getOperand(1));
253   EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadA->getOpcode());
254 }
255 
256 TEST_F(VPlanSlpTest, testSlpReuse_2) {
257   const char *ModuleString =
258       "%struct.Test = type { i32, i32 }\n"
259       "define i32 @add_x2(%struct.Test* nocapture readonly %A, %struct.Test* "
260       "nocapture readonly %B, %struct.Test* nocapture %C)  {\n"
261       "entry:\n"
262       "  br label %for.body\n"
263       "for.body:                                         ; preds = %for.body, "
264       "%entry\n"
265       "  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n"
266       "  %A0 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 "
267       "%indvars.iv, i32 0\n"
268       "  %vA0 = load i32, i32* %A0, align 4\n"
269       "  %add0 = add nsw i32 %vA0, %vA0\n"
270       "  %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
271       "%indvars.iv, i32 0\n"
272       "  store i32 %add0, i32* %C0, align 4\n"
273       "  %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 "
274       "%indvars.iv, i32 1\n"
275       "  %vA1 = load i32, i32* %A1, align 4\n"
276       "  %add1 = add nsw i32 %vA1, %vA1\n"
277       "  %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
278       "%indvars.iv, i32 1\n"
279       "  store i32 %add1, i32* %C1, align 4\n"
280       "  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n"
281       "  %exitcond = icmp eq i64 %indvars.iv.next, 1024\n"
282       "  br i1 %exitcond, label %for.cond.cleanup, label %for.body\n"
283       "for.cond.cleanup:                                 ; preds = %for.body\n"
284       "  ret i32 %vA1\n"
285       "}\n";
286 
287   Module &M = parseModule(ModuleString);
288 
289   Function *F = M.getFunction("add_x2");
290   BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor();
291   auto Plan = buildHCFG(LoopHeader);
292   auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan);
293 
294   VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock();
295   EXPECT_NE(nullptr, Entry->getSingleSuccessor());
296   VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock();
297 
298   VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 5));
299   VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 10));
300 
301   VPlanSlp Slp(VPIAI, *Body);
302   SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2};
303   Slp.buildGraph(StoreRoot);
304   EXPECT_FALSE(Slp.isCompletelySLP());
305 }
306 
307 static void checkReorderExample(VPInstruction *Store1, VPInstruction *Store2,
308                                 VPBasicBlock *Body,
309                                 VPInterleavedAccessInfo &&IAI) {
310   VPlanSlp Slp(IAI, *Body);
311   SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2};
312   VPInstruction *CombinedStore = Slp.buildGraph(StoreRoot);
313 
314   EXPECT_TRUE(Slp.isCompletelySLP());
315   EXPECT_EQ(CombinedStore->getOpcode(), VPInstruction::SLPStore);
316 
317   VPInstruction *CombinedAdd =
318       cast<VPInstruction>(CombinedStore->getOperand(0));
319   EXPECT_EQ(CombinedAdd->getOpcode(), Instruction::Add);
320 
321   VPInstruction *CombinedMulAB =
322       cast<VPInstruction>(CombinedAdd->getOperand(0));
323   VPInstruction *CombinedMulCD =
324       cast<VPInstruction>(CombinedAdd->getOperand(1));
325   EXPECT_EQ(CombinedMulAB->getOpcode(), Instruction::Mul);
326 
327   VPInstruction *CombinedLoadA =
328       cast<VPInstruction>(CombinedMulAB->getOperand(0));
329   EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadA->getOpcode());
330   VPInstruction *LoadvA0 = cast<VPInstruction>(&*std::next(Body->begin(), 2));
331   VPInstruction *LoadvA1 = cast<VPInstruction>(&*std::next(Body->begin(), 12));
332   EXPECT_EQ(LoadvA0->getOperand(0), CombinedLoadA->getOperand(0));
333   EXPECT_EQ(LoadvA1->getOperand(0), CombinedLoadA->getOperand(1));
334 
335   VPInstruction *CombinedLoadB =
336       cast<VPInstruction>(CombinedMulAB->getOperand(1));
337   EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadB->getOpcode());
338   VPInstruction *LoadvB0 = cast<VPInstruction>(&*std::next(Body->begin(), 4));
339   VPInstruction *LoadvB1 = cast<VPInstruction>(&*std::next(Body->begin(), 14));
340   EXPECT_EQ(LoadvB0->getOperand(0), CombinedLoadB->getOperand(0));
341   EXPECT_EQ(LoadvB1->getOperand(0), CombinedLoadB->getOperand(1));
342 
343   EXPECT_EQ(CombinedMulCD->getOpcode(), Instruction::Mul);
344 
345   VPInstruction *CombinedLoadC =
346       cast<VPInstruction>(CombinedMulCD->getOperand(0));
347   EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadC->getOpcode());
348   VPInstruction *LoadvC0 = cast<VPInstruction>(&*std::next(Body->begin(), 7));
349   VPInstruction *LoadvC1 = cast<VPInstruction>(&*std::next(Body->begin(), 17));
350   EXPECT_EQ(LoadvC0->getOperand(0), CombinedLoadC->getOperand(0));
351   EXPECT_EQ(LoadvC1->getOperand(0), CombinedLoadC->getOperand(1));
352 
353   VPInstruction *CombinedLoadD =
354       cast<VPInstruction>(CombinedMulCD->getOperand(1));
355   EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadD->getOpcode());
356   VPInstruction *LoadvD0 = cast<VPInstruction>(&*std::next(Body->begin(), 9));
357   VPInstruction *LoadvD1 = cast<VPInstruction>(&*std::next(Body->begin(), 19));
358   EXPECT_EQ(LoadvD0->getOperand(0), CombinedLoadD->getOperand(0));
359   EXPECT_EQ(LoadvD1->getOperand(0), CombinedLoadD->getOperand(1));
360 }
361 
362 TEST_F(VPlanSlpTest, testSlpReorder_1) {
363   LLVMContext Ctx;
364   const char *ModuleString =
365       "%struct.Test = type { i32, i32 }\n"
366       "define void @add_x3(%struct.Test* %A, %struct.Test* %B, %struct.Test* "
367       "%C,  %struct.Test* %D,  %struct.Test* %E)  {\n"
368       "entry:\n"
369       "  br label %for.body\n"
370       "for.body:                                         ; preds = %for.body, "
371       "%entry\n"
372       "  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n"
373       "  %A0 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 "
374       "%indvars.iv, i32 0\n"
375       "  %vA0 = load i32, i32* %A0, align 4\n"
376       "  %B0 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 "
377       "%indvars.iv, i32 0\n"
378       "  %vB0 = load i32, i32* %B0, align 4\n"
379       "  %mul11 = mul nsw i32 %vA0, %vB0\n"
380       "  %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
381       "%indvars.iv, i32 0\n"
382       "  %vC0 = load i32, i32* %C0, align 4\n"
383       "  %D0 = getelementptr inbounds %struct.Test, %struct.Test* %D, i64 "
384       "%indvars.iv, i32 0\n"
385       "  %vD0 = load i32, i32* %D0, align 4\n"
386       "  %mul12 = mul nsw i32 %vC0, %vD0\n"
387       "  %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 "
388       "%indvars.iv, i32 1\n"
389       "  %vA1 = load i32, i32* %A1, align 4\n"
390       "  %B1 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 "
391       "%indvars.iv, i32 1\n"
392       "  %vB1 = load i32, i32* %B1, align 4\n"
393       "  %mul21 = mul nsw i32 %vA1, %vB1\n"
394       "  %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
395       "%indvars.iv, i32 1\n"
396       "  %vC1 = load i32, i32* %C1, align 4\n"
397       "  %D1 = getelementptr inbounds %struct.Test, %struct.Test* %D, i64 "
398       "%indvars.iv, i32 1\n"
399       "  %vD1 = load i32, i32* %D1, align 4\n"
400       "  %mul22 = mul nsw i32 %vC1, %vD1\n"
401       "  %add1 = add nsw i32 %mul11, %mul12\n"
402       "  %add2 = add nsw i32 %mul22, %mul21\n"
403       "  %E0 = getelementptr inbounds %struct.Test, %struct.Test* %E, i64 "
404       "%indvars.iv, i32 0\n"
405       "  store i32 %add1, i32* %E0, align 4\n"
406       "  %E1 = getelementptr inbounds %struct.Test, %struct.Test* %E, i64 "
407       "%indvars.iv, i32 1\n"
408       "  store i32 %add2, i32* %E1, align 4\n"
409       "  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n"
410       "  %exitcond = icmp eq i64 %indvars.iv.next, 1024\n"
411       "  br i1 %exitcond, label %for.cond.cleanup, label %for.body\n"
412       "for.cond.cleanup:                                 ; preds = %for.body\n"
413       "  ret void\n"
414       "}\n";
415 
416   Module &M = parseModule(ModuleString);
417 
418   Function *F = M.getFunction("add_x3");
419   BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor();
420   auto Plan = buildHCFG(LoopHeader);
421 
422   VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock();
423   EXPECT_NE(nullptr, Entry->getSingleSuccessor());
424   VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock();
425 
426   VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 24));
427   VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 26));
428 
429   checkReorderExample(
430       Store1, Store2, Body,
431       getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan));
432 }
433 
434 TEST_F(VPlanSlpTest, testSlpReorder_2) {
435   LLVMContext Ctx;
436   const char *ModuleString =
437       "%struct.Test = type { i32, i32 }\n"
438       "define void @add_x3(%struct.Test* %A, %struct.Test* %B, %struct.Test* "
439       "%C,  %struct.Test* %D,  %struct.Test* %E)  {\n"
440       "entry:\n"
441       "  br label %for.body\n"
442       "for.body:                                         ; preds = %for.body, "
443       "%entry\n"
444       "  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n"
445       "  %A0 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 "
446       "%indvars.iv, i32 0\n"
447       "  %vA0 = load i32, i32* %A0, align 4\n"
448       "  %B0 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 "
449       "%indvars.iv, i32 0\n"
450       "  %vB0 = load i32, i32* %B0, align 4\n"
451       "  %mul11 = mul nsw i32 %vA0, %vB0\n"
452       "  %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
453       "%indvars.iv, i32 0\n"
454       "  %vC0 = load i32, i32* %C0, align 4\n"
455       "  %D0 = getelementptr inbounds %struct.Test, %struct.Test* %D, i64 "
456       "%indvars.iv, i32 0\n"
457       "  %vD0 = load i32, i32* %D0, align 4\n"
458       "  %mul12 = mul nsw i32 %vC0, %vD0\n"
459       "  %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 "
460       "%indvars.iv, i32 1\n"
461       "  %vA1 = load i32, i32* %A1, align 4\n"
462       "  %B1 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 "
463       "%indvars.iv, i32 1\n"
464       "  %vB1 = load i32, i32* %B1, align 4\n"
465       "  %mul21 = mul nsw i32 %vB1, %vA1\n"
466       "  %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
467       "%indvars.iv, i32 1\n"
468       "  %vC1 = load i32, i32* %C1, align 4\n"
469       "  %D1 = getelementptr inbounds %struct.Test, %struct.Test* %D, i64 "
470       "%indvars.iv, i32 1\n"
471       "  %vD1 = load i32, i32* %D1, align 4\n"
472       "  %mul22 = mul nsw i32 %vD1, %vC1\n"
473       "  %add1 = add nsw i32 %mul11, %mul12\n"
474       "  %add2 = add nsw i32 %mul22, %mul21\n"
475       "  %E0 = getelementptr inbounds %struct.Test, %struct.Test* %E, i64 "
476       "%indvars.iv, i32 0\n"
477       "  store i32 %add1, i32* %E0, align 4\n"
478       "  %E1 = getelementptr inbounds %struct.Test, %struct.Test* %E, i64 "
479       "%indvars.iv, i32 1\n"
480       "  store i32 %add2, i32* %E1, align 4\n"
481       "  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n"
482       "  %exitcond = icmp eq i64 %indvars.iv.next, 1024\n"
483       "  br i1 %exitcond, label %for.cond.cleanup, label %for.body\n"
484       "for.cond.cleanup:                                 ; preds = %for.body\n"
485       "  ret void\n"
486       "}\n";
487 
488   Module &M = parseModule(ModuleString);
489 
490   Function *F = M.getFunction("add_x3");
491   BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor();
492   auto Plan = buildHCFG(LoopHeader);
493 
494   VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock();
495   EXPECT_NE(nullptr, Entry->getSingleSuccessor());
496   VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock();
497 
498   VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 24));
499   VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 26));
500 
501   checkReorderExample(
502       Store1, Store2, Body,
503       getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan));
504 }
505 
506 TEST_F(VPlanSlpTest, testSlpReorder_3) {
507   LLVMContext Ctx;
508   const char *ModuleString =
509       "%struct.Test = type { i32, i32 }\n"
510       "define void @add_x3(%struct.Test* %A, %struct.Test* %B, %struct.Test* "
511       "%C,  %struct.Test* %D,  %struct.Test* %E)  {\n"
512       "entry:\n"
513       "  br label %for.body\n"
514       "for.body:                                         ; preds = %for.body, "
515       "%entry\n"
516       "  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n"
517       "  %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 "
518       "%indvars.iv, i32 1\n"
519       "  %vA1 = load i32, i32* %A1, align 4\n"
520       "  %B0 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 "
521       "%indvars.iv, i32 0\n"
522       "  %vB0 = load i32, i32* %B0, align 4\n"
523       "  %mul11 = mul nsw i32 %vA1, %vB0\n"
524       "  %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
525       "%indvars.iv, i32 0\n"
526       "  %vC0 = load i32, i32* %C0, align 4\n"
527       "  %D0 = getelementptr inbounds %struct.Test, %struct.Test* %D, i64 "
528       "%indvars.iv, i32 0\n"
529       "  %vD0 = load i32, i32* %D0, align 4\n"
530       "  %mul12 = mul nsw i32 %vC0, %vD0\n"
531       "  %A0 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 "
532       "%indvars.iv, i32 0\n"
533       "  %vA0 = load i32, i32* %A0, align 4\n"
534       "  %B1 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 "
535       "%indvars.iv, i32 1\n"
536       "  %vB1 = load i32, i32* %B1, align 4\n"
537       "  %mul21 = mul nsw i32 %vB1, %vA0\n"
538       "  %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
539       "%indvars.iv, i32 1\n"
540       "  %vC1 = load i32, i32* %C1, align 4\n"
541       "  %D1 = getelementptr inbounds %struct.Test, %struct.Test* %D, i64 "
542       "%indvars.iv, i32 1\n"
543       "  %vD1 = load i32, i32* %D1, align 4\n"
544       "  %mul22 = mul nsw i32 %vD1, %vC1\n"
545       "  %add1 = add nsw i32 %mul11, %mul12\n"
546       "  %add2 = add nsw i32 %mul22, %mul21\n"
547       "  %E0 = getelementptr inbounds %struct.Test, %struct.Test* %E, i64 "
548       "%indvars.iv, i32 0\n"
549       "  store i32 %add1, i32* %E0, align 4\n"
550       "  %E1 = getelementptr inbounds %struct.Test, %struct.Test* %E, i64 "
551       "%indvars.iv, i32 1\n"
552       "  store i32 %add2, i32* %E1, align 4\n"
553       "  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n"
554       "  %exitcond = icmp eq i64 %indvars.iv.next, 1024\n"
555       "  br i1 %exitcond, label %for.cond.cleanup, label %for.body\n"
556       "for.cond.cleanup:                                 ; preds = %for.body\n"
557       "  ret void\n"
558       "}\n";
559 
560   Module &M = parseModule(ModuleString);
561 
562   Function *F = M.getFunction("add_x3");
563   BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor();
564   auto Plan = buildHCFG(LoopHeader);
565 
566   VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock();
567   EXPECT_NE(nullptr, Entry->getSingleSuccessor());
568   VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock();
569 
570   VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 24));
571   VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 26));
572 
573   auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan);
574   VPlanSlp Slp(VPIAI, *Body);
575   SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2};
576   EXPECT_EQ(nullptr, Slp.buildGraph(StoreRoot));
577 
578   // FIXME Need to select better first value for lane0.
579   EXPECT_FALSE(Slp.isCompletelySLP());
580 }
581 
582 TEST_F(VPlanSlpTest, testSlpReorder_4) {
583   LLVMContext Ctx;
584   const char *ModuleString =
585       "%struct.Test = type { i32, i32 }\n"
586       "define void @add_x3(%struct.Test* %A, %struct.Test* %B, %struct.Test* "
587       "%C,  %struct.Test* %D,  %struct.Test* %E)  {\n"
588       "entry:\n"
589       "  br label %for.body\n"
590       "for.body:                                         ; preds = %for.body, "
591       "%entry\n"
592       "  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n"
593       "  %A0 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 "
594       "%indvars.iv, i32 0\n"
595       "  %vA0 = load i32, i32* %A0, align 4\n"
596       "  %B0 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 "
597       "%indvars.iv, i32 0\n"
598       "  %vB0 = load i32, i32* %B0, align 4\n"
599       "  %mul11 = mul nsw i32 %vA0, %vB0\n"
600       "  %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
601       "%indvars.iv, i32 0\n"
602       "  %vC0 = load i32, i32* %C0, align 4\n"
603       "  %D0 = getelementptr inbounds %struct.Test, %struct.Test* %D, i64 "
604       "%indvars.iv, i32 0\n"
605       "  %vD0 = load i32, i32* %D0, align 4\n"
606       "  %mul12 = mul nsw i32 %vC0, %vD0\n"
607       "  %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 "
608       "%indvars.iv, i32 1\n"
609       "  %vA1 = load i32, i32* %A1, align 4\n"
610       "  %B1 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 "
611       "%indvars.iv, i32 1\n"
612       "  %vB1 = load i32, i32* %B1, align 4\n"
613       "  %mul21 = mul nsw i32 %vA1, %vB1\n"
614       "  %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
615       "%indvars.iv, i32 1\n"
616       "  %vC1 = load i32, i32* %C1, align 4\n"
617       "  %D1 = getelementptr inbounds %struct.Test, %struct.Test* %D, i64 "
618       "%indvars.iv, i32 1\n"
619       "  %vD1 = load i32, i32* %D1, align 4\n"
620       "  %mul22 = mul nsw i32 %vC1, %vD1\n"
621       "  %add1 = add nsw i32 %mul11, %mul12\n"
622       "  %add2 = add nsw i32 %mul22, %mul21\n"
623       "  %E0 = getelementptr inbounds %struct.Test, %struct.Test* %E, i64 "
624       "%indvars.iv, i32 0\n"
625       "  store i32 %add1, i32* %E0, align 4\n"
626       "  %E1 = getelementptr inbounds %struct.Test, %struct.Test* %E, i64 "
627       "%indvars.iv, i32 1\n"
628       "  store i32 %add2, i32* %E1, align 4\n"
629       "  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n"
630       "  %exitcond = icmp eq i64 %indvars.iv.next, 1024\n"
631       "  br i1 %exitcond, label %for.cond.cleanup, label %for.body\n"
632       "for.cond.cleanup:                                 ; preds = %for.body\n"
633       "  ret void\n"
634       "}\n";
635 
636   Module &M = parseModule(ModuleString);
637 
638   Function *F = M.getFunction("add_x3");
639   BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor();
640   auto Plan = buildHCFG(LoopHeader);
641 
642   VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock();
643   EXPECT_NE(nullptr, Entry->getSingleSuccessor());
644   VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock();
645 
646   VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 24));
647   VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 26));
648 
649   checkReorderExample(
650       Store1, Store2, Body,
651       getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan));
652 }
653 
654 // Make sure we do not combine instructions with operands in different BBs.
655 TEST_F(VPlanSlpTest, testInstrsInDifferentBBs) {
656   const char *ModuleString =
657       "%struct.Test = type { i32, i32 }\n"
658       "%struct.Test3 = type { i32, i32, i32 }\n"
659       "%struct.Test4xi8 = type { i8, i8, i8 }\n"
660       "define void @add_x2(%struct.Test* nocapture readonly %A, %struct.Test* "
661       "nocapture readonly %B, %struct.Test* nocapture %C)  {\n"
662       "entry:\n"
663       "  br label %for.body\n"
664       "for.body:                                         ; preds = %for.body, "
665       "%entry\n"
666       "  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n"
667       "  %A0 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 "
668       "%indvars.iv, i32 0\n"
669       "  %vA0 = load i32, i32* %A0, align 4\n"
670       "  %B0 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 "
671       "%indvars.iv, i32 0\n"
672       "  %vB0 = load i32, i32* %B0, align 4\n"
673       "  %add0 = add nsw i32 %vA0, %vB0\n"
674       "  %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 "
675       "%indvars.iv, i32 1\n"
676       "  %vA1 = load i32, i32* %A1, align 4\n"
677       "  %B1 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 "
678       "%indvars.iv, i32 1\n"
679       "  br label %bb2\n"
680       "bb2:\n"
681       "  %vB1 = load i32, i32* %B1, align 4\n"
682       "  %add1 = add nsw i32 %vA1, %vB1\n"
683       "  %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
684       "%indvars.iv, i32 0\n"
685       "  store i32 %add0, i32* %C0, align 4\n"
686       "  %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
687       "%indvars.iv, i32 1\n"
688       "  store i32 %add1, i32* %C1, align 4\n"
689       "  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n"
690       "  %exitcond = icmp eq i64 %indvars.iv.next, 1024\n"
691       "  br i1 %exitcond, label %for.cond.cleanup, label %for.body\n"
692       "for.cond.cleanup:                                 ; preds = %for.body\n"
693       "  ret void\n"
694       "}\n";
695 
696   Module &M = parseModule(ModuleString);
697 
698   Function *F = M.getFunction("add_x2");
699   BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor();
700   auto Plan = buildHCFG(LoopHeader);
701   auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan);
702 
703   VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock();
704   EXPECT_NE(nullptr, Entry->getSingleSuccessor());
705   VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock();
706   VPBasicBlock *BB2 = Body->getSingleSuccessor()->getEntryBasicBlock();
707 
708   VPInstruction *Store1 = cast<VPInstruction>(&*std::next(BB2->begin(), 3));
709   VPInstruction *Store2 = cast<VPInstruction>(&*std::next(BB2->begin(), 5));
710 
711   VPlanSlp Slp(VPIAI, *BB2);
712   SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2};
713   EXPECT_EQ(nullptr, Slp.buildGraph(StoreRoot));
714   EXPECT_EQ(0u, Slp.getWidestBundleBits());
715 }
716 
717 // Make sure we do not combine instructions with operands in different BBs.
718 TEST_F(VPlanSlpTest, testInstrsInDifferentBBs2) {
719   const char *ModuleString =
720       "%struct.Test = type { i32, i32 }\n"
721       "%struct.Test3 = type { i32, i32, i32 }\n"
722       "%struct.Test4xi8 = type { i8, i8, i8 }\n"
723       "define void @add_x2(%struct.Test* nocapture readonly %A, %struct.Test* "
724       "nocapture readonly %B, %struct.Test* nocapture %C)  {\n"
725       "entry:\n"
726       "  br label %for.body\n"
727       "for.body:                                         ; preds = %for.body, "
728       "%entry\n"
729       "  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n"
730       "  %A0 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 "
731       "%indvars.iv, i32 0\n"
732       "  %vA0 = load i32, i32* %A0, align 4\n"
733       "  %B0 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 "
734       "%indvars.iv, i32 0\n"
735       "  %vB0 = load i32, i32* %B0, align 4\n"
736       "  %add0 = add nsw i32 %vA0, %vB0\n"
737       "  %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 "
738       "%indvars.iv, i32 1\n"
739       "  %vA1 = load i32, i32* %A1, align 4\n"
740       "  %B1 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 "
741       "%indvars.iv, i32 1\n"
742       "  %vB1 = load i32, i32* %B1, align 4\n"
743       "  %add1 = add nsw i32 %vA1, %vB1\n"
744       "  br label %bb2\n"
745       "bb2:\n"
746       "  %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
747       "%indvars.iv, i32 0\n"
748       "  store i32 %add0, i32* %C0, align 4\n"
749       "  %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
750       "%indvars.iv, i32 1\n"
751       "  store i32 %add1, i32* %C1, align 4\n"
752       "  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n"
753       "  %exitcond = icmp eq i64 %indvars.iv.next, 1024\n"
754       "  br i1 %exitcond, label %for.cond.cleanup, label %for.body\n"
755       "for.cond.cleanup:                                 ; preds = %for.body\n"
756       "  ret void\n"
757       "}\n";
758 
759   Module &M = parseModule(ModuleString);
760 
761   Function *F = M.getFunction("add_x2");
762   BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor();
763   auto Plan = buildHCFG(LoopHeader);
764   auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan);
765 
766   VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock();
767   EXPECT_NE(nullptr, Entry->getSingleSuccessor());
768   VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock();
769   VPBasicBlock *BB2 = Body->getSingleSuccessor()->getEntryBasicBlock();
770 
771   VPInstruction *Store1 = cast<VPInstruction>(&*std::next(BB2->begin(), 1));
772   VPInstruction *Store2 = cast<VPInstruction>(&*std::next(BB2->begin(), 3));
773 
774   VPlanSlp Slp(VPIAI, *BB2);
775   SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2};
776   EXPECT_EQ(nullptr, Slp.buildGraph(StoreRoot));
777   EXPECT_EQ(0u, Slp.getWidestBundleBits());
778 }
779 
780 TEST_F(VPlanSlpTest, testSlpAtomicLoad) {
781   const char *ModuleString =
782       "%struct.Test = type { i32, i32 }\n"
783       "%struct.Test3 = type { i32, i32, i32 }\n"
784       "%struct.Test4xi8 = type { i8, i8, i8 }\n"
785       "define void @add_x2(%struct.Test* nocapture readonly %A, %struct.Test* "
786       "nocapture readonly %B, %struct.Test* nocapture %C)  {\n"
787       "entry:\n"
788       "  br label %for.body\n"
789       "for.body:                                         ; preds = %for.body, "
790       "%entry\n"
791       "  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n"
792       "  %A0 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 "
793       "%indvars.iv, i32 0\n"
794       "  %vA0 = load atomic i32, i32* %A0 monotonic, align 4\n"
795       "  %B0 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 "
796       "%indvars.iv, i32 0\n"
797       "  %vB0 = load i32, i32* %B0, align 4\n"
798       "  %add0 = add nsw i32 %vA0, %vB0\n"
799       "  %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 "
800       "%indvars.iv, i32 1\n"
801       "  %vA1 = load i32, i32* %A1, align 4\n"
802       "  %B1 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 "
803       "%indvars.iv, i32 1\n"
804       "  %vB1 = load i32, i32* %B1, align 4\n"
805       "  %add1 = add nsw i32 %vA1, %vB1\n"
806       "  %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
807       "%indvars.iv, i32 0\n"
808       "  store i32 %add0, i32* %C0, align 4\n"
809       "  %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
810       "%indvars.iv, i32 1\n"
811       "  store i32 %add1, i32* %C1, align 4\n"
812       "  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n"
813       "  %exitcond = icmp eq i64 %indvars.iv.next, 1024\n"
814       "  br i1 %exitcond, label %for.cond.cleanup, label %for.body\n"
815       "for.cond.cleanup:                                 ; preds = %for.body\n"
816       "  ret void\n"
817       "}\n";
818 
819   Module &M = parseModule(ModuleString);
820 
821   Function *F = M.getFunction("add_x2");
822   BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor();
823   auto Plan = buildHCFG(LoopHeader);
824   auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan);
825 
826   VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock();
827   EXPECT_NE(nullptr, Entry->getSingleSuccessor());
828   VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock();
829 
830   VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 12));
831   VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 14));
832 
833   VPlanSlp Slp(VPIAI, *Body);
834   SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2};
835   EXPECT_EQ(nullptr, Slp.buildGraph(StoreRoot));
836   EXPECT_FALSE(Slp.isCompletelySLP());
837 }
838 
839 TEST_F(VPlanSlpTest, testSlpAtomicStore) {
840   const char *ModuleString =
841       "%struct.Test = type { i32, i32 }\n"
842       "%struct.Test3 = type { i32, i32, i32 }\n"
843       "%struct.Test4xi8 = type { i8, i8, i8 }\n"
844       "define void @add_x2(%struct.Test* nocapture readonly %A, %struct.Test* "
845       "nocapture readonly %B, %struct.Test* nocapture %C)  {\n"
846       "entry:\n"
847       "  br label %for.body\n"
848       "for.body:                                         ; preds = %for.body, "
849       "%entry\n"
850       "  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n"
851       "  %A0 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 "
852       "%indvars.iv, i32 0\n"
853       "  %vA0 = load i32, i32* %A0, align 4\n"
854       "  %B0 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 "
855       "%indvars.iv, i32 0\n"
856       "  %vB0 = load i32, i32* %B0, align 4\n"
857       "  %add0 = add nsw i32 %vA0, %vB0\n"
858       "  %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 "
859       "%indvars.iv, i32 1\n"
860       "  %vA1 = load i32, i32* %A1, align 4\n"
861       "  %B1 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 "
862       "%indvars.iv, i32 1\n"
863       "  %vB1 = load i32, i32* %B1, align 4\n"
864       "  %add1 = add nsw i32 %vA1, %vB1\n"
865       "  %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
866       "%indvars.iv, i32 0\n"
867       "  store atomic i32 %add0, i32* %C0 monotonic, align 4\n"
868       "  %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
869       "%indvars.iv, i32 1\n"
870       "  store i32 %add1, i32* %C1, align 4\n"
871       "  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n"
872       "  %exitcond = icmp eq i64 %indvars.iv.next, 1024\n"
873       "  br i1 %exitcond, label %for.cond.cleanup, label %for.body\n"
874       "for.cond.cleanup:                                 ; preds = %for.body\n"
875       "  ret void\n"
876       "}\n";
877 
878   Module &M = parseModule(ModuleString);
879 
880   Function *F = M.getFunction("add_x2");
881   BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor();
882   auto Plan = buildHCFG(LoopHeader);
883   auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan);
884 
885   VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock();
886   EXPECT_NE(nullptr, Entry->getSingleSuccessor());
887   VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock();
888 
889   VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 12));
890   VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 14));
891 
892   VPlanSlp Slp(VPIAI, *Body);
893   SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2};
894   Slp.buildGraph(StoreRoot);
895   EXPECT_FALSE(Slp.isCompletelySLP());
896 }
897 
898 } // namespace
899 } // namespace llvm
900