1 //===- llvm/unittest/Transforms/Vectorize/VPlanSlpTest.cpp ---------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "../lib/Transforms/Vectorize/VPlan.h" 10 #include "../lib/Transforms/Vectorize/VPlanHCFGBuilder.h" 11 #include "../lib/Transforms/Vectorize/VPlanHCFGTransforms.h" 12 #include "VPlanTestBase.h" 13 #include "llvm/Analysis/VectorUtils.h" 14 #include "gtest/gtest.h" 15 16 namespace llvm { 17 namespace { 18 19 class VPlanSlpTest : public VPlanTestBase { 20 protected: 21 TargetLibraryInfoImpl TLII; 22 TargetLibraryInfo TLI; 23 DataLayout DL; 24 25 std::unique_ptr<AssumptionCache> AC; 26 std::unique_ptr<ScalarEvolution> SE; 27 std::unique_ptr<AAResults> AARes; 28 std::unique_ptr<BasicAAResult> BasicAA; 29 std::unique_ptr<LoopAccessInfo> LAI; 30 std::unique_ptr<PredicatedScalarEvolution> PSE; 31 std::unique_ptr<InterleavedAccessInfo> IAI; 32 33 VPlanSlpTest() 34 : TLII(), TLI(TLII), 35 DL("e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-" 36 "f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:" 37 "16:32:64-S128") {} 38 39 VPInterleavedAccessInfo getInterleavedAccessInfo(Function &F, Loop *L, 40 VPlan &Plan) { 41 AC.reset(new AssumptionCache(F)); 42 SE.reset(new ScalarEvolution(F, TLI, *AC, *DT, *LI)); 43 BasicAA.reset(new BasicAAResult(DL, F, TLI, *AC, &*DT, &*LI)); 44 AARes.reset(new AAResults(TLI)); 45 AARes->addAAResult(*BasicAA); 46 PSE.reset(new PredicatedScalarEvolution(*SE, *L)); 47 LAI.reset(new LoopAccessInfo(L, &*SE, &TLI, &*AARes, &*DT, &*LI)); 48 IAI.reset(new InterleavedAccessInfo(*PSE, L, &*DT, &*LI, &*LAI)); 49 IAI->analyzeInterleaving(false); 50 return {Plan, *IAI}; 51 } 52 }; 53 54 TEST_F(VPlanSlpTest, testSlpSimple_2) { 55 const char *ModuleString = 56 "%struct.Test = type { i32, i32 }\n" 57 "%struct.Test3 = type { i32, i32, i32 }\n" 58 "%struct.Test4xi8 = type { i8, i8, i8 }\n" 59 "define void @add_x2(%struct.Test* nocapture readonly %A, %struct.Test* " 60 "nocapture readonly %B, %struct.Test* nocapture %C) {\n" 61 "entry:\n" 62 " br label %for.body\n" 63 "for.body: ; preds = %for.body, " 64 "%entry\n" 65 " %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n" 66 " %A0 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 " 67 "%indvars.iv, i32 0\n" 68 " %vA0 = load i32, i32* %A0, align 4\n" 69 " %B0 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 " 70 "%indvars.iv, i32 0\n" 71 " %vB0 = load i32, i32* %B0, align 4\n" 72 " %add0 = add nsw i32 %vA0, %vB0\n" 73 " %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 " 74 "%indvars.iv, i32 1\n" 75 " %vA1 = load i32, i32* %A1, align 4\n" 76 " %B1 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 " 77 "%indvars.iv, i32 1\n" 78 " %vB1 = load i32, i32* %B1, align 4\n" 79 " %add1 = add nsw i32 %vA1, %vB1\n" 80 " %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 81 "%indvars.iv, i32 0\n" 82 " store i32 %add0, i32* %C0, align 4\n" 83 " %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 84 "%indvars.iv, i32 1\n" 85 " store i32 %add1, i32* %C1, align 4\n" 86 " %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n" 87 " %exitcond = icmp eq i64 %indvars.iv.next, 1024\n" 88 " br i1 %exitcond, label %for.cond.cleanup, label %for.body\n" 89 "for.cond.cleanup: ; preds = %for.body\n" 90 " ret void\n" 91 "}\n"; 92 93 Module &M = parseModule(ModuleString); 94 95 Function *F = M.getFunction("add_x2"); 96 BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor(); 97 auto Plan = buildHCFG(LoopHeader); 98 auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan); 99 100 VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock(); 101 EXPECT_NE(nullptr, Entry->getSingleSuccessor()); 102 VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock(); 103 104 VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 12)); 105 VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 14)); 106 107 VPlanSlp Slp(VPIAI, *Body); 108 SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2}; 109 VPInstruction *CombinedStore = Slp.buildGraph(StoreRoot); 110 EXPECT_EQ(64u, Slp.getWidestBundleBits()); 111 EXPECT_EQ(VPInstruction::SLPStore, CombinedStore->getOpcode()); 112 113 auto *CombinedAdd = cast<VPInstruction>(CombinedStore->getOperand(0)); 114 EXPECT_EQ(Instruction::Add, CombinedAdd->getOpcode()); 115 116 auto *CombinedLoadA = cast<VPInstruction>(CombinedAdd->getOperand(0)); 117 auto *CombinedLoadB = cast<VPInstruction>(CombinedAdd->getOperand(1)); 118 EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadA->getOpcode()); 119 EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadB->getOpcode()); 120 } 121 122 TEST_F(VPlanSlpTest, testSlpSimple_3) { 123 const char *ModuleString = 124 "%struct.Test = type { i32, i32 }\n" 125 "%struct.Test3 = type { i32, i32, i32 }\n" 126 "%struct.Test4xi8 = type { i8, i8, i8 }\n" 127 "define void @add_x2(%struct.Test* nocapture readonly %A, %struct.Test* " 128 "nocapture readonly %B, %struct.Test* nocapture %C) {\n" 129 "entry:\n" 130 " br label %for.body\n" 131 "for.body: ; preds = %for.body, " 132 "%entry\n" 133 " %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n" 134 " %A0 = getelementptr %struct.Test, %struct.Test* %A, i64 " 135 " %indvars.iv, i32 0\n" 136 " %vA0 = load i32, i32* %A0, align 4\n" 137 " %B0 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 " 138 " %indvars.iv, i32 0\n" 139 " %vB0 = load i32, i32* %B0, align 4\n" 140 " %add0 = add nsw i32 %vA0, %vB0\n" 141 " %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 " 142 " %indvars.iv, i32 1\n" 143 " %vA1 = load i32, i32* %A1, align 4\n" 144 " %B1 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 " 145 " %indvars.iv, i32 1\n" 146 " %vB1 = load i32, i32* %B1, align 4\n" 147 " %add1 = add nsw i32 %vA1, %vB1\n" 148 " %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 149 " %indvars.iv, i32 0\n" 150 " store i32 %add0, i32* %C0, align 4\n" 151 " %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 152 " %indvars.iv, i32 1\n" 153 " store i32 %add1, i32* %C1, align 4\n" 154 " %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n" 155 " %exitcond = icmp eq i64 %indvars.iv.next, 1024\n" 156 " br i1 %exitcond, label %for.cond.cleanup, label %for.body\n" 157 "for.cond.cleanup: ; preds = %for.body\n" 158 " ret void\n" 159 "}\n"; 160 161 Module &M = parseModule(ModuleString); 162 163 Function *F = M.getFunction("add_x2"); 164 BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor(); 165 auto Plan = buildHCFG(LoopHeader); 166 167 VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock(); 168 EXPECT_NE(nullptr, Entry->getSingleSuccessor()); 169 VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock(); 170 171 VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 12)); 172 VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 14)); 173 174 auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan); 175 176 VPlanSlp Slp(VPIAI, *Body); 177 SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2}; 178 VPInstruction *CombinedStore = Slp.buildGraph(StoreRoot); 179 EXPECT_EQ(64u, Slp.getWidestBundleBits()); 180 EXPECT_EQ(VPInstruction::SLPStore, CombinedStore->getOpcode()); 181 182 auto *CombinedAdd = cast<VPInstruction>(CombinedStore->getOperand(0)); 183 EXPECT_EQ(Instruction::Add, CombinedAdd->getOpcode()); 184 185 auto *CombinedLoadA = cast<VPInstruction>(CombinedAdd->getOperand(0)); 186 auto *CombinedLoadB = cast<VPInstruction>(CombinedAdd->getOperand(1)); 187 EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadA->getOpcode()); 188 EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadB->getOpcode()); 189 190 VPInstruction *GetA = cast<VPInstruction>(&*std::next(Body->begin(), 1)); 191 VPInstruction *GetB = cast<VPInstruction>(&*std::next(Body->begin(), 3)); 192 EXPECT_EQ(GetA, CombinedLoadA->getOperand(0)); 193 EXPECT_EQ(GetB, CombinedLoadB->getOperand(0)); 194 } 195 196 TEST_F(VPlanSlpTest, testSlpReuse_1) { 197 const char *ModuleString = 198 "%struct.Test = type { i32, i32 }\n" 199 "define void @add_x2(%struct.Test* nocapture readonly %A, %struct.Test* " 200 "nocapture readonly %B, %struct.Test* nocapture %C) {\n" 201 "entry:\n" 202 " br label %for.body\n" 203 "for.body: ; preds = %for.body, " 204 "%entry\n" 205 " %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n" 206 " %A0 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 " 207 "%indvars.iv, i32 0\n" 208 " %vA0 = load i32, i32* %A0, align 4\n" 209 " %add0 = add nsw i32 %vA0, %vA0\n" 210 " %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 " 211 "%indvars.iv, i32 1\n" 212 " %vA1 = load i32, i32* %A1, align 4\n" 213 " %add1 = add nsw i32 %vA1, %vA1\n" 214 " %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 215 "%indvars.iv, i32 0\n" 216 " store i32 %add0, i32* %C0, align 4\n" 217 " %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 218 "%indvars.iv, i32 1\n" 219 " store i32 %add1, i32* %C1, align 4\n" 220 " %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n" 221 " %exitcond = icmp eq i64 %indvars.iv.next, 1024\n" 222 " br i1 %exitcond, label %for.cond.cleanup, label %for.body\n" 223 "for.cond.cleanup: ; preds = %for.body\n" 224 " ret void\n" 225 "}\n"; 226 227 Module &M = parseModule(ModuleString); 228 229 Function *F = M.getFunction("add_x2"); 230 BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor(); 231 auto Plan = buildHCFG(LoopHeader); 232 auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan); 233 234 VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock(); 235 EXPECT_NE(nullptr, Entry->getSingleSuccessor()); 236 VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock(); 237 238 VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 8)); 239 VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 10)); 240 241 VPlanSlp Slp(VPIAI, *Body); 242 SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2}; 243 VPInstruction *CombinedStore = Slp.buildGraph(StoreRoot); 244 EXPECT_EQ(64u, Slp.getWidestBundleBits()); 245 EXPECT_EQ(VPInstruction::SLPStore, CombinedStore->getOpcode()); 246 247 auto *CombinedAdd = cast<VPInstruction>(CombinedStore->getOperand(0)); 248 EXPECT_EQ(Instruction::Add, CombinedAdd->getOpcode()); 249 250 auto *CombinedLoadA = cast<VPInstruction>(CombinedAdd->getOperand(0)); 251 EXPECT_EQ(CombinedLoadA, CombinedAdd->getOperand(1)); 252 EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadA->getOpcode()); 253 } 254 255 TEST_F(VPlanSlpTest, testSlpReuse_2) { 256 const char *ModuleString = 257 "%struct.Test = type { i32, i32 }\n" 258 "define i32 @add_x2(%struct.Test* nocapture readonly %A, %struct.Test* " 259 "nocapture readonly %B, %struct.Test* nocapture %C) {\n" 260 "entry:\n" 261 " br label %for.body\n" 262 "for.body: ; preds = %for.body, " 263 "%entry\n" 264 " %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n" 265 " %A0 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 " 266 "%indvars.iv, i32 0\n" 267 " %vA0 = load i32, i32* %A0, align 4\n" 268 " %add0 = add nsw i32 %vA0, %vA0\n" 269 " %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 270 "%indvars.iv, i32 0\n" 271 " store i32 %add0, i32* %C0, align 4\n" 272 " %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 " 273 "%indvars.iv, i32 1\n" 274 " %vA1 = load i32, i32* %A1, align 4\n" 275 " %add1 = add nsw i32 %vA1, %vA1\n" 276 " %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 277 "%indvars.iv, i32 1\n" 278 " store i32 %add1, i32* %C1, align 4\n" 279 " %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n" 280 " %exitcond = icmp eq i64 %indvars.iv.next, 1024\n" 281 " br i1 %exitcond, label %for.cond.cleanup, label %for.body\n" 282 "for.cond.cleanup: ; preds = %for.body\n" 283 " ret i32 %vA1\n" 284 "}\n"; 285 286 Module &M = parseModule(ModuleString); 287 288 Function *F = M.getFunction("add_x2"); 289 BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor(); 290 auto Plan = buildHCFG(LoopHeader); 291 auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan); 292 293 VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock(); 294 EXPECT_NE(nullptr, Entry->getSingleSuccessor()); 295 VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock(); 296 297 VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 5)); 298 VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 10)); 299 300 VPlanSlp Slp(VPIAI, *Body); 301 SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2}; 302 Slp.buildGraph(StoreRoot); 303 EXPECT_FALSE(Slp.isCompletelySLP()); 304 } 305 306 static void checkReorderExample(VPInstruction *Store1, VPInstruction *Store2, 307 VPBasicBlock *Body, 308 VPInterleavedAccessInfo &&IAI) { 309 VPlanSlp Slp(IAI, *Body); 310 SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2}; 311 VPInstruction *CombinedStore = Slp.buildGraph(StoreRoot); 312 313 EXPECT_TRUE(Slp.isCompletelySLP()); 314 EXPECT_EQ(CombinedStore->getOpcode(), VPInstruction::SLPStore); 315 316 VPInstruction *CombinedAdd = 317 cast<VPInstruction>(CombinedStore->getOperand(0)); 318 EXPECT_EQ(CombinedAdd->getOpcode(), Instruction::Add); 319 320 VPInstruction *CombinedMulAB = 321 cast<VPInstruction>(CombinedAdd->getOperand(0)); 322 VPInstruction *CombinedMulCD = 323 cast<VPInstruction>(CombinedAdd->getOperand(1)); 324 EXPECT_EQ(CombinedMulAB->getOpcode(), Instruction::Mul); 325 326 VPInstruction *CombinedLoadA = 327 cast<VPInstruction>(CombinedMulAB->getOperand(0)); 328 EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadA->getOpcode()); 329 VPInstruction *LoadvA0 = cast<VPInstruction>(&*std::next(Body->begin(), 2)); 330 VPInstruction *LoadvA1 = cast<VPInstruction>(&*std::next(Body->begin(), 12)); 331 EXPECT_EQ(LoadvA0->getOperand(0), CombinedLoadA->getOperand(0)); 332 EXPECT_EQ(LoadvA1->getOperand(0), CombinedLoadA->getOperand(1)); 333 334 VPInstruction *CombinedLoadB = 335 cast<VPInstruction>(CombinedMulAB->getOperand(1)); 336 EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadB->getOpcode()); 337 VPInstruction *LoadvB0 = cast<VPInstruction>(&*std::next(Body->begin(), 4)); 338 VPInstruction *LoadvB1 = cast<VPInstruction>(&*std::next(Body->begin(), 14)); 339 EXPECT_EQ(LoadvB0->getOperand(0), CombinedLoadB->getOperand(0)); 340 EXPECT_EQ(LoadvB1->getOperand(0), CombinedLoadB->getOperand(1)); 341 342 EXPECT_EQ(CombinedMulCD->getOpcode(), Instruction::Mul); 343 344 VPInstruction *CombinedLoadC = 345 cast<VPInstruction>(CombinedMulCD->getOperand(0)); 346 EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadC->getOpcode()); 347 VPInstruction *LoadvC0 = cast<VPInstruction>(&*std::next(Body->begin(), 7)); 348 VPInstruction *LoadvC1 = cast<VPInstruction>(&*std::next(Body->begin(), 17)); 349 EXPECT_EQ(LoadvC0->getOperand(0), CombinedLoadC->getOperand(0)); 350 EXPECT_EQ(LoadvC1->getOperand(0), CombinedLoadC->getOperand(1)); 351 352 VPInstruction *CombinedLoadD = 353 cast<VPInstruction>(CombinedMulCD->getOperand(1)); 354 EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadD->getOpcode()); 355 VPInstruction *LoadvD0 = cast<VPInstruction>(&*std::next(Body->begin(), 9)); 356 VPInstruction *LoadvD1 = cast<VPInstruction>(&*std::next(Body->begin(), 19)); 357 EXPECT_EQ(LoadvD0->getOperand(0), CombinedLoadD->getOperand(0)); 358 EXPECT_EQ(LoadvD1->getOperand(0), CombinedLoadD->getOperand(1)); 359 } 360 361 TEST_F(VPlanSlpTest, testSlpReorder_1) { 362 LLVMContext Ctx; 363 const char *ModuleString = 364 "%struct.Test = type { i32, i32 }\n" 365 "define void @add_x3(%struct.Test* %A, %struct.Test* %B, %struct.Test* " 366 "%C, %struct.Test* %D, %struct.Test* %E) {\n" 367 "entry:\n" 368 " br label %for.body\n" 369 "for.body: ; preds = %for.body, " 370 "%entry\n" 371 " %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n" 372 " %A0 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 " 373 "%indvars.iv, i32 0\n" 374 " %vA0 = load i32, i32* %A0, align 4\n" 375 " %B0 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 " 376 "%indvars.iv, i32 0\n" 377 " %vB0 = load i32, i32* %B0, align 4\n" 378 " %mul11 = mul nsw i32 %vA0, %vB0\n" 379 " %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 380 "%indvars.iv, i32 0\n" 381 " %vC0 = load i32, i32* %C0, align 4\n" 382 " %D0 = getelementptr inbounds %struct.Test, %struct.Test* %D, i64 " 383 "%indvars.iv, i32 0\n" 384 " %vD0 = load i32, i32* %D0, align 4\n" 385 " %mul12 = mul nsw i32 %vC0, %vD0\n" 386 " %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 " 387 "%indvars.iv, i32 1\n" 388 " %vA1 = load i32, i32* %A1, align 4\n" 389 " %B1 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 " 390 "%indvars.iv, i32 1\n" 391 " %vB1 = load i32, i32* %B1, align 4\n" 392 " %mul21 = mul nsw i32 %vA1, %vB1\n" 393 " %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 394 "%indvars.iv, i32 1\n" 395 " %vC1 = load i32, i32* %C1, align 4\n" 396 " %D1 = getelementptr inbounds %struct.Test, %struct.Test* %D, i64 " 397 "%indvars.iv, i32 1\n" 398 " %vD1 = load i32, i32* %D1, align 4\n" 399 " %mul22 = mul nsw i32 %vC1, %vD1\n" 400 " %add1 = add nsw i32 %mul11, %mul12\n" 401 " %add2 = add nsw i32 %mul22, %mul21\n" 402 " %E0 = getelementptr inbounds %struct.Test, %struct.Test* %E, i64 " 403 "%indvars.iv, i32 0\n" 404 " store i32 %add1, i32* %E0, align 4\n" 405 " %E1 = getelementptr inbounds %struct.Test, %struct.Test* %E, i64 " 406 "%indvars.iv, i32 1\n" 407 " store i32 %add2, i32* %E1, align 4\n" 408 " %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n" 409 " %exitcond = icmp eq i64 %indvars.iv.next, 1024\n" 410 " br i1 %exitcond, label %for.cond.cleanup, label %for.body\n" 411 "for.cond.cleanup: ; preds = %for.body\n" 412 " ret void\n" 413 "}\n"; 414 415 Module &M = parseModule(ModuleString); 416 417 Function *F = M.getFunction("add_x3"); 418 BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor(); 419 auto Plan = buildHCFG(LoopHeader); 420 421 VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock(); 422 EXPECT_NE(nullptr, Entry->getSingleSuccessor()); 423 VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock(); 424 425 VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 24)); 426 VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 26)); 427 428 checkReorderExample( 429 Store1, Store2, Body, 430 getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan)); 431 } 432 433 TEST_F(VPlanSlpTest, testSlpReorder_2) { 434 LLVMContext Ctx; 435 const char *ModuleString = 436 "%struct.Test = type { i32, i32 }\n" 437 "define void @add_x3(%struct.Test* %A, %struct.Test* %B, %struct.Test* " 438 "%C, %struct.Test* %D, %struct.Test* %E) {\n" 439 "entry:\n" 440 " br label %for.body\n" 441 "for.body: ; preds = %for.body, " 442 "%entry\n" 443 " %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n" 444 " %A0 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 " 445 "%indvars.iv, i32 0\n" 446 " %vA0 = load i32, i32* %A0, align 4\n" 447 " %B0 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 " 448 "%indvars.iv, i32 0\n" 449 " %vB0 = load i32, i32* %B0, align 4\n" 450 " %mul11 = mul nsw i32 %vA0, %vB0\n" 451 " %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 452 "%indvars.iv, i32 0\n" 453 " %vC0 = load i32, i32* %C0, align 4\n" 454 " %D0 = getelementptr inbounds %struct.Test, %struct.Test* %D, i64 " 455 "%indvars.iv, i32 0\n" 456 " %vD0 = load i32, i32* %D0, align 4\n" 457 " %mul12 = mul nsw i32 %vC0, %vD0\n" 458 " %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 " 459 "%indvars.iv, i32 1\n" 460 " %vA1 = load i32, i32* %A1, align 4\n" 461 " %B1 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 " 462 "%indvars.iv, i32 1\n" 463 " %vB1 = load i32, i32* %B1, align 4\n" 464 " %mul21 = mul nsw i32 %vB1, %vA1\n" 465 " %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 466 "%indvars.iv, i32 1\n" 467 " %vC1 = load i32, i32* %C1, align 4\n" 468 " %D1 = getelementptr inbounds %struct.Test, %struct.Test* %D, i64 " 469 "%indvars.iv, i32 1\n" 470 " %vD1 = load i32, i32* %D1, align 4\n" 471 " %mul22 = mul nsw i32 %vD1, %vC1\n" 472 " %add1 = add nsw i32 %mul11, %mul12\n" 473 " %add2 = add nsw i32 %mul22, %mul21\n" 474 " %E0 = getelementptr inbounds %struct.Test, %struct.Test* %E, i64 " 475 "%indvars.iv, i32 0\n" 476 " store i32 %add1, i32* %E0, align 4\n" 477 " %E1 = getelementptr inbounds %struct.Test, %struct.Test* %E, i64 " 478 "%indvars.iv, i32 1\n" 479 " store i32 %add2, i32* %E1, align 4\n" 480 " %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n" 481 " %exitcond = icmp eq i64 %indvars.iv.next, 1024\n" 482 " br i1 %exitcond, label %for.cond.cleanup, label %for.body\n" 483 "for.cond.cleanup: ; preds = %for.body\n" 484 " ret void\n" 485 "}\n"; 486 487 Module &M = parseModule(ModuleString); 488 489 Function *F = M.getFunction("add_x3"); 490 BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor(); 491 auto Plan = buildHCFG(LoopHeader); 492 493 VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock(); 494 EXPECT_NE(nullptr, Entry->getSingleSuccessor()); 495 VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock(); 496 497 VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 24)); 498 VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 26)); 499 500 checkReorderExample( 501 Store1, Store2, Body, 502 getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan)); 503 } 504 505 TEST_F(VPlanSlpTest, testSlpReorder_3) { 506 LLVMContext Ctx; 507 const char *ModuleString = 508 "%struct.Test = type { i32, i32 }\n" 509 "define void @add_x3(%struct.Test* %A, %struct.Test* %B, %struct.Test* " 510 "%C, %struct.Test* %D, %struct.Test* %E) {\n" 511 "entry:\n" 512 " br label %for.body\n" 513 "for.body: ; preds = %for.body, " 514 "%entry\n" 515 " %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n" 516 " %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 " 517 "%indvars.iv, i32 1\n" 518 " %vA1 = load i32, i32* %A1, align 4\n" 519 " %B0 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 " 520 "%indvars.iv, i32 0\n" 521 " %vB0 = load i32, i32* %B0, align 4\n" 522 " %mul11 = mul nsw i32 %vA1, %vB0\n" 523 " %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 524 "%indvars.iv, i32 0\n" 525 " %vC0 = load i32, i32* %C0, align 4\n" 526 " %D0 = getelementptr inbounds %struct.Test, %struct.Test* %D, i64 " 527 "%indvars.iv, i32 0\n" 528 " %vD0 = load i32, i32* %D0, align 4\n" 529 " %mul12 = mul nsw i32 %vC0, %vD0\n" 530 " %A0 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 " 531 "%indvars.iv, i32 0\n" 532 " %vA0 = load i32, i32* %A0, align 4\n" 533 " %B1 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 " 534 "%indvars.iv, i32 1\n" 535 " %vB1 = load i32, i32* %B1, align 4\n" 536 " %mul21 = mul nsw i32 %vB1, %vA0\n" 537 " %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 538 "%indvars.iv, i32 1\n" 539 " %vC1 = load i32, i32* %C1, align 4\n" 540 " %D1 = getelementptr inbounds %struct.Test, %struct.Test* %D, i64 " 541 "%indvars.iv, i32 1\n" 542 " %vD1 = load i32, i32* %D1, align 4\n" 543 " %mul22 = mul nsw i32 %vD1, %vC1\n" 544 " %add1 = add nsw i32 %mul11, %mul12\n" 545 " %add2 = add nsw i32 %mul22, %mul21\n" 546 " %E0 = getelementptr inbounds %struct.Test, %struct.Test* %E, i64 " 547 "%indvars.iv, i32 0\n" 548 " store i32 %add1, i32* %E0, align 4\n" 549 " %E1 = getelementptr inbounds %struct.Test, %struct.Test* %E, i64 " 550 "%indvars.iv, i32 1\n" 551 " store i32 %add2, i32* %E1, align 4\n" 552 " %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n" 553 " %exitcond = icmp eq i64 %indvars.iv.next, 1024\n" 554 " br i1 %exitcond, label %for.cond.cleanup, label %for.body\n" 555 "for.cond.cleanup: ; preds = %for.body\n" 556 " ret void\n" 557 "}\n"; 558 559 Module &M = parseModule(ModuleString); 560 561 Function *F = M.getFunction("add_x3"); 562 BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor(); 563 auto Plan = buildHCFG(LoopHeader); 564 565 VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock(); 566 EXPECT_NE(nullptr, Entry->getSingleSuccessor()); 567 VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock(); 568 569 VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 24)); 570 VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 26)); 571 572 auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan); 573 VPlanSlp Slp(VPIAI, *Body); 574 SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2}; 575 EXPECT_EQ(nullptr, Slp.buildGraph(StoreRoot)); 576 577 // FIXME Need to select better first value for lane0. 578 EXPECT_FALSE(Slp.isCompletelySLP()); 579 } 580 581 TEST_F(VPlanSlpTest, testSlpReorder_4) { 582 LLVMContext Ctx; 583 const char *ModuleString = 584 "%struct.Test = type { i32, i32 }\n" 585 "define void @add_x3(%struct.Test* %A, %struct.Test* %B, %struct.Test* " 586 "%C, %struct.Test* %D, %struct.Test* %E) {\n" 587 "entry:\n" 588 " br label %for.body\n" 589 "for.body: ; preds = %for.body, " 590 "%entry\n" 591 " %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n" 592 " %A0 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 " 593 "%indvars.iv, i32 0\n" 594 " %vA0 = load i32, i32* %A0, align 4\n" 595 " %B0 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 " 596 "%indvars.iv, i32 0\n" 597 " %vB0 = load i32, i32* %B0, align 4\n" 598 " %mul11 = mul nsw i32 %vA0, %vB0\n" 599 " %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 600 "%indvars.iv, i32 0\n" 601 " %vC0 = load i32, i32* %C0, align 4\n" 602 " %D0 = getelementptr inbounds %struct.Test, %struct.Test* %D, i64 " 603 "%indvars.iv, i32 0\n" 604 " %vD0 = load i32, i32* %D0, align 4\n" 605 " %mul12 = mul nsw i32 %vC0, %vD0\n" 606 " %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 " 607 "%indvars.iv, i32 1\n" 608 " %vA1 = load i32, i32* %A1, align 4\n" 609 " %B1 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 " 610 "%indvars.iv, i32 1\n" 611 " %vB1 = load i32, i32* %B1, align 4\n" 612 " %mul21 = mul nsw i32 %vA1, %vB1\n" 613 " %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 614 "%indvars.iv, i32 1\n" 615 " %vC1 = load i32, i32* %C1, align 4\n" 616 " %D1 = getelementptr inbounds %struct.Test, %struct.Test* %D, i64 " 617 "%indvars.iv, i32 1\n" 618 " %vD1 = load i32, i32* %D1, align 4\n" 619 " %mul22 = mul nsw i32 %vC1, %vD1\n" 620 " %add1 = add nsw i32 %mul11, %mul12\n" 621 " %add2 = add nsw i32 %mul22, %mul21\n" 622 " %E0 = getelementptr inbounds %struct.Test, %struct.Test* %E, i64 " 623 "%indvars.iv, i32 0\n" 624 " store i32 %add1, i32* %E0, align 4\n" 625 " %E1 = getelementptr inbounds %struct.Test, %struct.Test* %E, i64 " 626 "%indvars.iv, i32 1\n" 627 " store i32 %add2, i32* %E1, align 4\n" 628 " %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n" 629 " %exitcond = icmp eq i64 %indvars.iv.next, 1024\n" 630 " br i1 %exitcond, label %for.cond.cleanup, label %for.body\n" 631 "for.cond.cleanup: ; preds = %for.body\n" 632 " ret void\n" 633 "}\n"; 634 635 Module &M = parseModule(ModuleString); 636 637 Function *F = M.getFunction("add_x3"); 638 BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor(); 639 auto Plan = buildHCFG(LoopHeader); 640 641 VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock(); 642 EXPECT_NE(nullptr, Entry->getSingleSuccessor()); 643 VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock(); 644 645 VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 24)); 646 VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 26)); 647 648 checkReorderExample( 649 Store1, Store2, Body, 650 getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan)); 651 } 652 653 // Make sure we do not combine instructions with operands in different BBs. 654 TEST_F(VPlanSlpTest, testInstrsInDifferentBBs) { 655 const char *ModuleString = 656 "%struct.Test = type { i32, i32 }\n" 657 "%struct.Test3 = type { i32, i32, i32 }\n" 658 "%struct.Test4xi8 = type { i8, i8, i8 }\n" 659 "define void @add_x2(%struct.Test* nocapture readonly %A, %struct.Test* " 660 "nocapture readonly %B, %struct.Test* nocapture %C) {\n" 661 "entry:\n" 662 " br label %for.body\n" 663 "for.body: ; preds = %for.body, " 664 "%entry\n" 665 " %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n" 666 " %A0 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 " 667 "%indvars.iv, i32 0\n" 668 " %vA0 = load i32, i32* %A0, align 4\n" 669 " %B0 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 " 670 "%indvars.iv, i32 0\n" 671 " %vB0 = load i32, i32* %B0, align 4\n" 672 " %add0 = add nsw i32 %vA0, %vB0\n" 673 " %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 " 674 "%indvars.iv, i32 1\n" 675 " %vA1 = load i32, i32* %A1, align 4\n" 676 " %B1 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 " 677 "%indvars.iv, i32 1\n" 678 " br label %bb2\n" 679 "bb2:\n" 680 " %vB1 = load i32, i32* %B1, align 4\n" 681 " %add1 = add nsw i32 %vA1, %vB1\n" 682 " %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 683 "%indvars.iv, i32 0\n" 684 " store i32 %add0, i32* %C0, align 4\n" 685 " %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 686 "%indvars.iv, i32 1\n" 687 " store i32 %add1, i32* %C1, align 4\n" 688 " %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n" 689 " %exitcond = icmp eq i64 %indvars.iv.next, 1024\n" 690 " br i1 %exitcond, label %for.cond.cleanup, label %for.body\n" 691 "for.cond.cleanup: ; preds = %for.body\n" 692 " ret void\n" 693 "}\n"; 694 695 Module &M = parseModule(ModuleString); 696 697 Function *F = M.getFunction("add_x2"); 698 BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor(); 699 auto Plan = buildHCFG(LoopHeader); 700 auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan); 701 702 VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock(); 703 EXPECT_NE(nullptr, Entry->getSingleSuccessor()); 704 VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock(); 705 VPBasicBlock *BB2 = Body->getSingleSuccessor()->getEntryBasicBlock(); 706 707 VPInstruction *Store1 = cast<VPInstruction>(&*std::next(BB2->begin(), 3)); 708 VPInstruction *Store2 = cast<VPInstruction>(&*std::next(BB2->begin(), 5)); 709 710 VPlanSlp Slp(VPIAI, *BB2); 711 SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2}; 712 EXPECT_EQ(nullptr, Slp.buildGraph(StoreRoot)); 713 EXPECT_EQ(0u, Slp.getWidestBundleBits()); 714 } 715 716 // Make sure we do not combine instructions with operands in different BBs. 717 TEST_F(VPlanSlpTest, testInstrsInDifferentBBs2) { 718 const char *ModuleString = 719 "%struct.Test = type { i32, i32 }\n" 720 "%struct.Test3 = type { i32, i32, i32 }\n" 721 "%struct.Test4xi8 = type { i8, i8, i8 }\n" 722 "define void @add_x2(%struct.Test* nocapture readonly %A, %struct.Test* " 723 "nocapture readonly %B, %struct.Test* nocapture %C) {\n" 724 "entry:\n" 725 " br label %for.body\n" 726 "for.body: ; preds = %for.body, " 727 "%entry\n" 728 " %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n" 729 " %A0 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 " 730 "%indvars.iv, i32 0\n" 731 " %vA0 = load i32, i32* %A0, align 4\n" 732 " %B0 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 " 733 "%indvars.iv, i32 0\n" 734 " %vB0 = load i32, i32* %B0, align 4\n" 735 " %add0 = add nsw i32 %vA0, %vB0\n" 736 " %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 " 737 "%indvars.iv, i32 1\n" 738 " %vA1 = load i32, i32* %A1, align 4\n" 739 " %B1 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 " 740 "%indvars.iv, i32 1\n" 741 " %vB1 = load i32, i32* %B1, align 4\n" 742 " %add1 = add nsw i32 %vA1, %vB1\n" 743 " br label %bb2\n" 744 "bb2:\n" 745 " %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 746 "%indvars.iv, i32 0\n" 747 " store i32 %add0, i32* %C0, align 4\n" 748 " %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 749 "%indvars.iv, i32 1\n" 750 " store i32 %add1, i32* %C1, align 4\n" 751 " %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n" 752 " %exitcond = icmp eq i64 %indvars.iv.next, 1024\n" 753 " br i1 %exitcond, label %for.cond.cleanup, label %for.body\n" 754 "for.cond.cleanup: ; preds = %for.body\n" 755 " ret void\n" 756 "}\n"; 757 758 Module &M = parseModule(ModuleString); 759 760 Function *F = M.getFunction("add_x2"); 761 BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor(); 762 auto Plan = buildHCFG(LoopHeader); 763 auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan); 764 765 VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock(); 766 EXPECT_NE(nullptr, Entry->getSingleSuccessor()); 767 VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock(); 768 VPBasicBlock *BB2 = Body->getSingleSuccessor()->getEntryBasicBlock(); 769 770 VPInstruction *Store1 = cast<VPInstruction>(&*std::next(BB2->begin(), 1)); 771 VPInstruction *Store2 = cast<VPInstruction>(&*std::next(BB2->begin(), 3)); 772 773 VPlanSlp Slp(VPIAI, *BB2); 774 SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2}; 775 EXPECT_EQ(nullptr, Slp.buildGraph(StoreRoot)); 776 EXPECT_EQ(0u, Slp.getWidestBundleBits()); 777 } 778 779 TEST_F(VPlanSlpTest, testSlpAtomicLoad) { 780 const char *ModuleString = 781 "%struct.Test = type { i32, i32 }\n" 782 "%struct.Test3 = type { i32, i32, i32 }\n" 783 "%struct.Test4xi8 = type { i8, i8, i8 }\n" 784 "define void @add_x2(%struct.Test* nocapture readonly %A, %struct.Test* " 785 "nocapture readonly %B, %struct.Test* nocapture %C) {\n" 786 "entry:\n" 787 " br label %for.body\n" 788 "for.body: ; preds = %for.body, " 789 "%entry\n" 790 " %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n" 791 " %A0 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 " 792 "%indvars.iv, i32 0\n" 793 " %vA0 = load atomic i32, i32* %A0 monotonic, align 4\n" 794 " %B0 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 " 795 "%indvars.iv, i32 0\n" 796 " %vB0 = load i32, i32* %B0, align 4\n" 797 " %add0 = add nsw i32 %vA0, %vB0\n" 798 " %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 " 799 "%indvars.iv, i32 1\n" 800 " %vA1 = load i32, i32* %A1, align 4\n" 801 " %B1 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 " 802 "%indvars.iv, i32 1\n" 803 " %vB1 = load i32, i32* %B1, align 4\n" 804 " %add1 = add nsw i32 %vA1, %vB1\n" 805 " %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 806 "%indvars.iv, i32 0\n" 807 " store i32 %add0, i32* %C0, align 4\n" 808 " %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 809 "%indvars.iv, i32 1\n" 810 " store i32 %add1, i32* %C1, align 4\n" 811 " %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n" 812 " %exitcond = icmp eq i64 %indvars.iv.next, 1024\n" 813 " br i1 %exitcond, label %for.cond.cleanup, label %for.body\n" 814 "for.cond.cleanup: ; preds = %for.body\n" 815 " ret void\n" 816 "}\n"; 817 818 Module &M = parseModule(ModuleString); 819 820 Function *F = M.getFunction("add_x2"); 821 BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor(); 822 auto Plan = buildHCFG(LoopHeader); 823 auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan); 824 825 VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock(); 826 EXPECT_NE(nullptr, Entry->getSingleSuccessor()); 827 VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock(); 828 829 VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 12)); 830 VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 14)); 831 832 VPlanSlp Slp(VPIAI, *Body); 833 SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2}; 834 EXPECT_EQ(nullptr, Slp.buildGraph(StoreRoot)); 835 EXPECT_FALSE(Slp.isCompletelySLP()); 836 } 837 838 TEST_F(VPlanSlpTest, testSlpAtomicStore) { 839 const char *ModuleString = 840 "%struct.Test = type { i32, i32 }\n" 841 "%struct.Test3 = type { i32, i32, i32 }\n" 842 "%struct.Test4xi8 = type { i8, i8, i8 }\n" 843 "define void @add_x2(%struct.Test* nocapture readonly %A, %struct.Test* " 844 "nocapture readonly %B, %struct.Test* nocapture %C) {\n" 845 "entry:\n" 846 " br label %for.body\n" 847 "for.body: ; preds = %for.body, " 848 "%entry\n" 849 " %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n" 850 " %A0 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 " 851 "%indvars.iv, i32 0\n" 852 " %vA0 = load i32, i32* %A0, align 4\n" 853 " %B0 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 " 854 "%indvars.iv, i32 0\n" 855 " %vB0 = load i32, i32* %B0, align 4\n" 856 " %add0 = add nsw i32 %vA0, %vB0\n" 857 " %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 " 858 "%indvars.iv, i32 1\n" 859 " %vA1 = load i32, i32* %A1, align 4\n" 860 " %B1 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 " 861 "%indvars.iv, i32 1\n" 862 " %vB1 = load i32, i32* %B1, align 4\n" 863 " %add1 = add nsw i32 %vA1, %vB1\n" 864 " %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 865 "%indvars.iv, i32 0\n" 866 " store atomic i32 %add0, i32* %C0 monotonic, align 4\n" 867 " %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 868 "%indvars.iv, i32 1\n" 869 " store i32 %add1, i32* %C1, align 4\n" 870 " %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n" 871 " %exitcond = icmp eq i64 %indvars.iv.next, 1024\n" 872 " br i1 %exitcond, label %for.cond.cleanup, label %for.body\n" 873 "for.cond.cleanup: ; preds = %for.body\n" 874 " ret void\n" 875 "}\n"; 876 877 Module &M = parseModule(ModuleString); 878 879 Function *F = M.getFunction("add_x2"); 880 BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor(); 881 auto Plan = buildHCFG(LoopHeader); 882 auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan); 883 884 VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock(); 885 EXPECT_NE(nullptr, Entry->getSingleSuccessor()); 886 VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock(); 887 888 VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 12)); 889 VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 14)); 890 891 VPlanSlp Slp(VPIAI, *Body); 892 SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2}; 893 Slp.buildGraph(StoreRoot); 894 EXPECT_FALSE(Slp.isCompletelySLP()); 895 } 896 897 } // namespace 898 } // namespace llvm 899