1 //===- llvm/unittest/Transforms/Vectorize/VPlanSlpTest.cpp ---------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "../lib/Transforms/Vectorize/VPlan.h" 10 #include "../lib/Transforms/Vectorize/VPlanHCFGBuilder.h" 11 #include "VPlanTestBase.h" 12 #include "llvm/Analysis/TargetLibraryInfo.h" 13 #include "llvm/Analysis/VectorUtils.h" 14 #include "gtest/gtest.h" 15 16 namespace llvm { 17 namespace { 18 19 class VPlanSlpTest : public VPlanTestBase { 20 protected: 21 TargetLibraryInfoImpl TLII; 22 TargetLibraryInfo TLI; 23 DataLayout DL; 24 25 std::unique_ptr<AssumptionCache> AC; 26 std::unique_ptr<ScalarEvolution> SE; 27 std::unique_ptr<AAResults> AARes; 28 std::unique_ptr<BasicAAResult> BasicAA; 29 std::unique_ptr<LoopAccessInfo> LAI; 30 std::unique_ptr<PredicatedScalarEvolution> PSE; 31 std::unique_ptr<InterleavedAccessInfo> IAI; 32 33 VPlanSlpTest() 34 : TLII(), TLI(TLII), 35 DL("e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-" 36 "f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:" 37 "16:32:64-S128") {} 38 39 VPInterleavedAccessInfo getInterleavedAccessInfo(Function &F, Loop *L, 40 VPlan &Plan) { 41 AC.reset(new AssumptionCache(F)); 42 SE.reset(new ScalarEvolution(F, TLI, *AC, *DT, *LI)); 43 BasicAA.reset(new BasicAAResult(DL, F, TLI, *AC, &*DT)); 44 AARes.reset(new AAResults(TLI)); 45 AARes->addAAResult(*BasicAA); 46 PSE.reset(new PredicatedScalarEvolution(*SE, *L)); 47 LAI.reset(new LoopAccessInfo(L, &*SE, &TLI, &*AARes, &*DT, &*LI)); 48 IAI.reset(new InterleavedAccessInfo(*PSE, L, &*DT, &*LI, &*LAI)); 49 IAI->analyzeInterleaving(false); 50 return {Plan, *IAI}; 51 } 52 }; 53 54 TEST_F(VPlanSlpTest, testSlpSimple_2) { 55 const char *ModuleString = 56 "%struct.Test = type { i32, i32 }\n" 57 "%struct.Test3 = type { i32, i32, i32 }\n" 58 "%struct.Test4xi8 = type { i8, i8, i8 }\n" 59 "define void @add_x2(%struct.Test* nocapture readonly %A, %struct.Test* " 60 "nocapture readonly %B, %struct.Test* nocapture %C) {\n" 61 "entry:\n" 62 " br label %for.body\n" 63 "for.body: ; preds = %for.body, " 64 "%entry\n" 65 " %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n" 66 " %A0 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 " 67 "%indvars.iv, i32 0\n" 68 " %vA0 = load i32, i32* %A0, align 4\n" 69 " %B0 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 " 70 "%indvars.iv, i32 0\n" 71 " %vB0 = load i32, i32* %B0, align 4\n" 72 " %add0 = add nsw i32 %vA0, %vB0\n" 73 " %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 " 74 "%indvars.iv, i32 1\n" 75 " %vA1 = load i32, i32* %A1, align 4\n" 76 " %B1 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 " 77 "%indvars.iv, i32 1\n" 78 " %vB1 = load i32, i32* %B1, align 4\n" 79 " %add1 = add nsw i32 %vA1, %vB1\n" 80 " %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 81 "%indvars.iv, i32 0\n" 82 " store i32 %add0, i32* %C0, align 4\n" 83 " %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 84 "%indvars.iv, i32 1\n" 85 " store i32 %add1, i32* %C1, align 4\n" 86 " %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n" 87 " %exitcond = icmp eq i64 %indvars.iv.next, 1024\n" 88 " br i1 %exitcond, label %for.cond.cleanup, label %for.body\n" 89 "for.cond.cleanup: ; preds = %for.body\n" 90 " ret void\n" 91 "}\n"; 92 93 Module &M = parseModule(ModuleString); 94 95 Function *F = M.getFunction("add_x2"); 96 BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor(); 97 auto Plan = buildHCFG(LoopHeader); 98 auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan); 99 100 VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock(); 101 EXPECT_NE(nullptr, Entry->getSingleSuccessor()); 102 VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock(); 103 104 VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 12)); 105 VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 14)); 106 107 VPlanSlp Slp(VPIAI, *Body); 108 SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2}; 109 VPInstruction *CombinedStore = Slp.buildGraph(StoreRoot); 110 EXPECT_EQ(64u, Slp.getWidestBundleBits()); 111 EXPECT_EQ(VPInstruction::SLPStore, CombinedStore->getOpcode()); 112 113 auto *CombinedAdd = cast<VPInstruction>(CombinedStore->getOperand(0)); 114 EXPECT_EQ(Instruction::Add, CombinedAdd->getOpcode()); 115 116 auto *CombinedLoadA = cast<VPInstruction>(CombinedAdd->getOperand(0)); 117 auto *CombinedLoadB = cast<VPInstruction>(CombinedAdd->getOperand(1)); 118 EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadA->getOpcode()); 119 EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadB->getOpcode()); 120 121 delete CombinedStore; 122 delete CombinedAdd; 123 delete CombinedLoadA; 124 delete CombinedLoadB; 125 } 126 127 TEST_F(VPlanSlpTest, testSlpSimple_3) { 128 const char *ModuleString = 129 "%struct.Test = type { i32, i32 }\n" 130 "%struct.Test3 = type { i32, i32, i32 }\n" 131 "%struct.Test4xi8 = type { i8, i8, i8 }\n" 132 "define void @add_x2(%struct.Test* nocapture readonly %A, %struct.Test* " 133 "nocapture readonly %B, %struct.Test* nocapture %C) {\n" 134 "entry:\n" 135 " br label %for.body\n" 136 "for.body: ; preds = %for.body, " 137 "%entry\n" 138 " %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n" 139 " %A0 = getelementptr %struct.Test, %struct.Test* %A, i64 " 140 " %indvars.iv, i32 0\n" 141 " %vA0 = load i32, i32* %A0, align 4\n" 142 " %B0 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 " 143 " %indvars.iv, i32 0\n" 144 " %vB0 = load i32, i32* %B0, align 4\n" 145 " %add0 = add nsw i32 %vA0, %vB0\n" 146 " %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 " 147 " %indvars.iv, i32 1\n" 148 " %vA1 = load i32, i32* %A1, align 4\n" 149 " %B1 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 " 150 " %indvars.iv, i32 1\n" 151 " %vB1 = load i32, i32* %B1, align 4\n" 152 " %add1 = add nsw i32 %vA1, %vB1\n" 153 " %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 154 " %indvars.iv, i32 0\n" 155 " store i32 %add0, i32* %C0, align 4\n" 156 " %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 157 " %indvars.iv, i32 1\n" 158 " store i32 %add1, i32* %C1, align 4\n" 159 " %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n" 160 " %exitcond = icmp eq i64 %indvars.iv.next, 1024\n" 161 " br i1 %exitcond, label %for.cond.cleanup, label %for.body\n" 162 "for.cond.cleanup: ; preds = %for.body\n" 163 " ret void\n" 164 "}\n"; 165 166 Module &M = parseModule(ModuleString); 167 168 Function *F = M.getFunction("add_x2"); 169 BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor(); 170 auto Plan = buildHCFG(LoopHeader); 171 172 VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock(); 173 EXPECT_NE(nullptr, Entry->getSingleSuccessor()); 174 VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock(); 175 176 VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 12)); 177 VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 14)); 178 179 auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan); 180 181 VPlanSlp Slp(VPIAI, *Body); 182 SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2}; 183 VPInstruction *CombinedStore = Slp.buildGraph(StoreRoot); 184 EXPECT_EQ(64u, Slp.getWidestBundleBits()); 185 EXPECT_EQ(VPInstruction::SLPStore, CombinedStore->getOpcode()); 186 187 auto *CombinedAdd = cast<VPInstruction>(CombinedStore->getOperand(0)); 188 EXPECT_EQ(Instruction::Add, CombinedAdd->getOpcode()); 189 190 auto *CombinedLoadA = cast<VPInstruction>(CombinedAdd->getOperand(0)); 191 auto *CombinedLoadB = cast<VPInstruction>(CombinedAdd->getOperand(1)); 192 EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadA->getOpcode()); 193 EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadB->getOpcode()); 194 195 VPInstruction *GetA = cast<VPInstruction>(&*std::next(Body->begin(), 1)); 196 VPInstruction *GetB = cast<VPInstruction>(&*std::next(Body->begin(), 3)); 197 EXPECT_EQ(GetA, CombinedLoadA->getOperand(0)); 198 EXPECT_EQ(GetB, CombinedLoadB->getOperand(0)); 199 200 delete CombinedStore; 201 delete CombinedAdd; 202 delete CombinedLoadA; 203 delete CombinedLoadB; 204 } 205 206 TEST_F(VPlanSlpTest, testSlpReuse_1) { 207 const char *ModuleString = 208 "%struct.Test = type { i32, i32 }\n" 209 "define void @add_x2(%struct.Test* nocapture readonly %A, %struct.Test* " 210 "nocapture readonly %B, %struct.Test* nocapture %C) {\n" 211 "entry:\n" 212 " br label %for.body\n" 213 "for.body: ; preds = %for.body, " 214 "%entry\n" 215 " %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n" 216 " %A0 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 " 217 "%indvars.iv, i32 0\n" 218 " %vA0 = load i32, i32* %A0, align 4\n" 219 " %add0 = add nsw i32 %vA0, %vA0\n" 220 " %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 " 221 "%indvars.iv, i32 1\n" 222 " %vA1 = load i32, i32* %A1, align 4\n" 223 " %add1 = add nsw i32 %vA1, %vA1\n" 224 " %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 225 "%indvars.iv, i32 0\n" 226 " store i32 %add0, i32* %C0, align 4\n" 227 " %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 228 "%indvars.iv, i32 1\n" 229 " store i32 %add1, i32* %C1, align 4\n" 230 " %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n" 231 " %exitcond = icmp eq i64 %indvars.iv.next, 1024\n" 232 " br i1 %exitcond, label %for.cond.cleanup, label %for.body\n" 233 "for.cond.cleanup: ; preds = %for.body\n" 234 " ret void\n" 235 "}\n"; 236 237 Module &M = parseModule(ModuleString); 238 239 Function *F = M.getFunction("add_x2"); 240 BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor(); 241 auto Plan = buildHCFG(LoopHeader); 242 auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan); 243 244 VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock(); 245 EXPECT_NE(nullptr, Entry->getSingleSuccessor()); 246 VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock(); 247 248 VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 8)); 249 VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 10)); 250 251 VPlanSlp Slp(VPIAI, *Body); 252 SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2}; 253 VPInstruction *CombinedStore = Slp.buildGraph(StoreRoot); 254 EXPECT_EQ(64u, Slp.getWidestBundleBits()); 255 EXPECT_EQ(VPInstruction::SLPStore, CombinedStore->getOpcode()); 256 257 auto *CombinedAdd = cast<VPInstruction>(CombinedStore->getOperand(0)); 258 EXPECT_EQ(Instruction::Add, CombinedAdd->getOpcode()); 259 260 auto *CombinedLoadA = cast<VPInstruction>(CombinedAdd->getOperand(0)); 261 EXPECT_EQ(CombinedLoadA, CombinedAdd->getOperand(1)); 262 EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadA->getOpcode()); 263 264 delete CombinedStore; 265 delete CombinedAdd; 266 delete CombinedLoadA; 267 } 268 269 TEST_F(VPlanSlpTest, testSlpReuse_2) { 270 const char *ModuleString = 271 "%struct.Test = type { i32, i32 }\n" 272 "define i32 @add_x2(%struct.Test* nocapture readonly %A, %struct.Test* " 273 "nocapture readonly %B, %struct.Test* nocapture %C) {\n" 274 "entry:\n" 275 " br label %for.body\n" 276 "for.body: ; preds = %for.body, " 277 "%entry\n" 278 " %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n" 279 " %A0 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 " 280 "%indvars.iv, i32 0\n" 281 " %vA0 = load i32, i32* %A0, align 4\n" 282 " %add0 = add nsw i32 %vA0, %vA0\n" 283 " %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 284 "%indvars.iv, i32 0\n" 285 " store i32 %add0, i32* %C0, align 4\n" 286 " %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 " 287 "%indvars.iv, i32 1\n" 288 " %vA1 = load i32, i32* %A1, align 4\n" 289 " %add1 = add nsw i32 %vA1, %vA1\n" 290 " %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 291 "%indvars.iv, i32 1\n" 292 " store i32 %add1, i32* %C1, align 4\n" 293 " %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n" 294 " %exitcond = icmp eq i64 %indvars.iv.next, 1024\n" 295 " br i1 %exitcond, label %for.cond.cleanup, label %for.body\n" 296 "for.cond.cleanup: ; preds = %for.body\n" 297 " ret i32 %vA1\n" 298 "}\n"; 299 300 Module &M = parseModule(ModuleString); 301 302 Function *F = M.getFunction("add_x2"); 303 BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor(); 304 auto Plan = buildHCFG(LoopHeader); 305 auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan); 306 307 VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock(); 308 EXPECT_NE(nullptr, Entry->getSingleSuccessor()); 309 VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock(); 310 311 VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 5)); 312 VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 10)); 313 314 VPlanSlp Slp(VPIAI, *Body); 315 SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2}; 316 Slp.buildGraph(StoreRoot); 317 EXPECT_FALSE(Slp.isCompletelySLP()); 318 } 319 320 static void checkReorderExample(VPInstruction *Store1, VPInstruction *Store2, 321 VPBasicBlock *Body, 322 VPInterleavedAccessInfo &&IAI) { 323 VPlanSlp Slp(IAI, *Body); 324 SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2}; 325 VPInstruction *CombinedStore = Slp.buildGraph(StoreRoot); 326 327 EXPECT_TRUE(Slp.isCompletelySLP()); 328 EXPECT_EQ(CombinedStore->getOpcode(), VPInstruction::SLPStore); 329 330 VPInstruction *CombinedAdd = 331 cast<VPInstruction>(CombinedStore->getOperand(0)); 332 EXPECT_EQ(CombinedAdd->getOpcode(), Instruction::Add); 333 334 VPInstruction *CombinedMulAB = 335 cast<VPInstruction>(CombinedAdd->getOperand(0)); 336 VPInstruction *CombinedMulCD = 337 cast<VPInstruction>(CombinedAdd->getOperand(1)); 338 EXPECT_EQ(CombinedMulAB->getOpcode(), Instruction::Mul); 339 340 VPInstruction *CombinedLoadA = 341 cast<VPInstruction>(CombinedMulAB->getOperand(0)); 342 EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadA->getOpcode()); 343 VPInstruction *LoadvA0 = cast<VPInstruction>(&*std::next(Body->begin(), 2)); 344 VPInstruction *LoadvA1 = cast<VPInstruction>(&*std::next(Body->begin(), 12)); 345 EXPECT_EQ(LoadvA0->getOperand(0), CombinedLoadA->getOperand(0)); 346 EXPECT_EQ(LoadvA1->getOperand(0), CombinedLoadA->getOperand(1)); 347 348 VPInstruction *CombinedLoadB = 349 cast<VPInstruction>(CombinedMulAB->getOperand(1)); 350 EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadB->getOpcode()); 351 VPInstruction *LoadvB0 = cast<VPInstruction>(&*std::next(Body->begin(), 4)); 352 VPInstruction *LoadvB1 = cast<VPInstruction>(&*std::next(Body->begin(), 14)); 353 EXPECT_EQ(LoadvB0->getOperand(0), CombinedLoadB->getOperand(0)); 354 EXPECT_EQ(LoadvB1->getOperand(0), CombinedLoadB->getOperand(1)); 355 356 EXPECT_EQ(CombinedMulCD->getOpcode(), Instruction::Mul); 357 358 VPInstruction *CombinedLoadC = 359 cast<VPInstruction>(CombinedMulCD->getOperand(0)); 360 EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadC->getOpcode()); 361 VPInstruction *LoadvC0 = cast<VPInstruction>(&*std::next(Body->begin(), 7)); 362 VPInstruction *LoadvC1 = cast<VPInstruction>(&*std::next(Body->begin(), 17)); 363 EXPECT_EQ(LoadvC0->getOperand(0), CombinedLoadC->getOperand(0)); 364 EXPECT_EQ(LoadvC1->getOperand(0), CombinedLoadC->getOperand(1)); 365 366 VPInstruction *CombinedLoadD = 367 cast<VPInstruction>(CombinedMulCD->getOperand(1)); 368 EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadD->getOpcode()); 369 VPInstruction *LoadvD0 = cast<VPInstruction>(&*std::next(Body->begin(), 9)); 370 VPInstruction *LoadvD1 = cast<VPInstruction>(&*std::next(Body->begin(), 19)); 371 EXPECT_EQ(LoadvD0->getOperand(0), CombinedLoadD->getOperand(0)); 372 EXPECT_EQ(LoadvD1->getOperand(0), CombinedLoadD->getOperand(1)); 373 374 delete CombinedStore; 375 delete CombinedAdd; 376 delete CombinedMulAB; 377 delete CombinedMulCD; 378 delete CombinedLoadA; 379 delete CombinedLoadB; 380 delete CombinedLoadC; 381 delete CombinedLoadD; 382 } 383 384 TEST_F(VPlanSlpTest, testSlpReorder_1) { 385 LLVMContext Ctx; 386 const char *ModuleString = 387 "%struct.Test = type { i32, i32 }\n" 388 "define void @add_x3(%struct.Test* %A, %struct.Test* %B, %struct.Test* " 389 "%C, %struct.Test* %D, %struct.Test* %E) {\n" 390 "entry:\n" 391 " br label %for.body\n" 392 "for.body: ; preds = %for.body, " 393 "%entry\n" 394 " %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n" 395 " %A0 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 " 396 "%indvars.iv, i32 0\n" 397 " %vA0 = load i32, i32* %A0, align 4\n" 398 " %B0 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 " 399 "%indvars.iv, i32 0\n" 400 " %vB0 = load i32, i32* %B0, align 4\n" 401 " %mul11 = mul nsw i32 %vA0, %vB0\n" 402 " %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 403 "%indvars.iv, i32 0\n" 404 " %vC0 = load i32, i32* %C0, align 4\n" 405 " %D0 = getelementptr inbounds %struct.Test, %struct.Test* %D, i64 " 406 "%indvars.iv, i32 0\n" 407 " %vD0 = load i32, i32* %D0, align 4\n" 408 " %mul12 = mul nsw i32 %vC0, %vD0\n" 409 " %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 " 410 "%indvars.iv, i32 1\n" 411 " %vA1 = load i32, i32* %A1, align 4\n" 412 " %B1 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 " 413 "%indvars.iv, i32 1\n" 414 " %vB1 = load i32, i32* %B1, align 4\n" 415 " %mul21 = mul nsw i32 %vA1, %vB1\n" 416 " %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 417 "%indvars.iv, i32 1\n" 418 " %vC1 = load i32, i32* %C1, align 4\n" 419 " %D1 = getelementptr inbounds %struct.Test, %struct.Test* %D, i64 " 420 "%indvars.iv, i32 1\n" 421 " %vD1 = load i32, i32* %D1, align 4\n" 422 " %mul22 = mul nsw i32 %vC1, %vD1\n" 423 " %add1 = add nsw i32 %mul11, %mul12\n" 424 " %add2 = add nsw i32 %mul22, %mul21\n" 425 " %E0 = getelementptr inbounds %struct.Test, %struct.Test* %E, i64 " 426 "%indvars.iv, i32 0\n" 427 " store i32 %add1, i32* %E0, align 4\n" 428 " %E1 = getelementptr inbounds %struct.Test, %struct.Test* %E, i64 " 429 "%indvars.iv, i32 1\n" 430 " store i32 %add2, i32* %E1, align 4\n" 431 " %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n" 432 " %exitcond = icmp eq i64 %indvars.iv.next, 1024\n" 433 " br i1 %exitcond, label %for.cond.cleanup, label %for.body\n" 434 "for.cond.cleanup: ; preds = %for.body\n" 435 " ret void\n" 436 "}\n"; 437 438 Module &M = parseModule(ModuleString); 439 440 Function *F = M.getFunction("add_x3"); 441 BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor(); 442 auto Plan = buildHCFG(LoopHeader); 443 444 VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock(); 445 EXPECT_NE(nullptr, Entry->getSingleSuccessor()); 446 VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock(); 447 448 VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 24)); 449 VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 26)); 450 451 checkReorderExample( 452 Store1, Store2, Body, 453 getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan)); 454 } 455 456 TEST_F(VPlanSlpTest, testSlpReorder_2) { 457 LLVMContext Ctx; 458 const char *ModuleString = 459 "%struct.Test = type { i32, i32 }\n" 460 "define void @add_x3(%struct.Test* %A, %struct.Test* %B, %struct.Test* " 461 "%C, %struct.Test* %D, %struct.Test* %E) {\n" 462 "entry:\n" 463 " br label %for.body\n" 464 "for.body: ; preds = %for.body, " 465 "%entry\n" 466 " %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n" 467 " %A0 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 " 468 "%indvars.iv, i32 0\n" 469 " %vA0 = load i32, i32* %A0, align 4\n" 470 " %B0 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 " 471 "%indvars.iv, i32 0\n" 472 " %vB0 = load i32, i32* %B0, align 4\n" 473 " %mul11 = mul nsw i32 %vA0, %vB0\n" 474 " %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 475 "%indvars.iv, i32 0\n" 476 " %vC0 = load i32, i32* %C0, align 4\n" 477 " %D0 = getelementptr inbounds %struct.Test, %struct.Test* %D, i64 " 478 "%indvars.iv, i32 0\n" 479 " %vD0 = load i32, i32* %D0, align 4\n" 480 " %mul12 = mul nsw i32 %vC0, %vD0\n" 481 " %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 " 482 "%indvars.iv, i32 1\n" 483 " %vA1 = load i32, i32* %A1, align 4\n" 484 " %B1 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 " 485 "%indvars.iv, i32 1\n" 486 " %vB1 = load i32, i32* %B1, align 4\n" 487 " %mul21 = mul nsw i32 %vB1, %vA1\n" 488 " %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 489 "%indvars.iv, i32 1\n" 490 " %vC1 = load i32, i32* %C1, align 4\n" 491 " %D1 = getelementptr inbounds %struct.Test, %struct.Test* %D, i64 " 492 "%indvars.iv, i32 1\n" 493 " %vD1 = load i32, i32* %D1, align 4\n" 494 " %mul22 = mul nsw i32 %vD1, %vC1\n" 495 " %add1 = add nsw i32 %mul11, %mul12\n" 496 " %add2 = add nsw i32 %mul22, %mul21\n" 497 " %E0 = getelementptr inbounds %struct.Test, %struct.Test* %E, i64 " 498 "%indvars.iv, i32 0\n" 499 " store i32 %add1, i32* %E0, align 4\n" 500 " %E1 = getelementptr inbounds %struct.Test, %struct.Test* %E, i64 " 501 "%indvars.iv, i32 1\n" 502 " store i32 %add2, i32* %E1, align 4\n" 503 " %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n" 504 " %exitcond = icmp eq i64 %indvars.iv.next, 1024\n" 505 " br i1 %exitcond, label %for.cond.cleanup, label %for.body\n" 506 "for.cond.cleanup: ; preds = %for.body\n" 507 " ret void\n" 508 "}\n"; 509 510 Module &M = parseModule(ModuleString); 511 512 Function *F = M.getFunction("add_x3"); 513 BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor(); 514 auto Plan = buildHCFG(LoopHeader); 515 516 VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock(); 517 EXPECT_NE(nullptr, Entry->getSingleSuccessor()); 518 VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock(); 519 520 VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 24)); 521 VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 26)); 522 523 checkReorderExample( 524 Store1, Store2, Body, 525 getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan)); 526 } 527 528 TEST_F(VPlanSlpTest, testSlpReorder_3) { 529 LLVMContext Ctx; 530 const char *ModuleString = 531 "%struct.Test = type { i32, i32 }\n" 532 "define void @add_x3(%struct.Test* %A, %struct.Test* %B, %struct.Test* " 533 "%C, %struct.Test* %D, %struct.Test* %E) {\n" 534 "entry:\n" 535 " br label %for.body\n" 536 "for.body: ; preds = %for.body, " 537 "%entry\n" 538 " %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n" 539 " %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 " 540 "%indvars.iv, i32 1\n" 541 " %vA1 = load i32, i32* %A1, align 4\n" 542 " %B0 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 " 543 "%indvars.iv, i32 0\n" 544 " %vB0 = load i32, i32* %B0, align 4\n" 545 " %mul11 = mul nsw i32 %vA1, %vB0\n" 546 " %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 547 "%indvars.iv, i32 0\n" 548 " %vC0 = load i32, i32* %C0, align 4\n" 549 " %D0 = getelementptr inbounds %struct.Test, %struct.Test* %D, i64 " 550 "%indvars.iv, i32 0\n" 551 " %vD0 = load i32, i32* %D0, align 4\n" 552 " %mul12 = mul nsw i32 %vC0, %vD0\n" 553 " %A0 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 " 554 "%indvars.iv, i32 0\n" 555 " %vA0 = load i32, i32* %A0, align 4\n" 556 " %B1 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 " 557 "%indvars.iv, i32 1\n" 558 " %vB1 = load i32, i32* %B1, align 4\n" 559 " %mul21 = mul nsw i32 %vB1, %vA0\n" 560 " %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 561 "%indvars.iv, i32 1\n" 562 " %vC1 = load i32, i32* %C1, align 4\n" 563 " %D1 = getelementptr inbounds %struct.Test, %struct.Test* %D, i64 " 564 "%indvars.iv, i32 1\n" 565 " %vD1 = load i32, i32* %D1, align 4\n" 566 " %mul22 = mul nsw i32 %vD1, %vC1\n" 567 " %add1 = add nsw i32 %mul11, %mul12\n" 568 " %add2 = add nsw i32 %mul22, %mul21\n" 569 " %E0 = getelementptr inbounds %struct.Test, %struct.Test* %E, i64 " 570 "%indvars.iv, i32 0\n" 571 " store i32 %add1, i32* %E0, align 4\n" 572 " %E1 = getelementptr inbounds %struct.Test, %struct.Test* %E, i64 " 573 "%indvars.iv, i32 1\n" 574 " store i32 %add2, i32* %E1, align 4\n" 575 " %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n" 576 " %exitcond = icmp eq i64 %indvars.iv.next, 1024\n" 577 " br i1 %exitcond, label %for.cond.cleanup, label %for.body\n" 578 "for.cond.cleanup: ; preds = %for.body\n" 579 " ret void\n" 580 "}\n"; 581 582 Module &M = parseModule(ModuleString); 583 584 Function *F = M.getFunction("add_x3"); 585 BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor(); 586 auto Plan = buildHCFG(LoopHeader); 587 588 VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock(); 589 EXPECT_NE(nullptr, Entry->getSingleSuccessor()); 590 VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock(); 591 592 VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 24)); 593 VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 26)); 594 595 auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan); 596 VPlanSlp Slp(VPIAI, *Body); 597 SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2}; 598 EXPECT_EQ(nullptr, Slp.buildGraph(StoreRoot)); 599 600 // FIXME Need to select better first value for lane0. 601 EXPECT_FALSE(Slp.isCompletelySLP()); 602 } 603 604 TEST_F(VPlanSlpTest, testSlpReorder_4) { 605 LLVMContext Ctx; 606 const char *ModuleString = 607 "%struct.Test = type { i32, i32 }\n" 608 "define void @add_x3(%struct.Test* %A, %struct.Test* %B, %struct.Test* " 609 "%C, %struct.Test* %D, %struct.Test* %E) {\n" 610 "entry:\n" 611 " br label %for.body\n" 612 "for.body: ; preds = %for.body, " 613 "%entry\n" 614 " %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n" 615 " %A0 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 " 616 "%indvars.iv, i32 0\n" 617 " %vA0 = load i32, i32* %A0, align 4\n" 618 " %B0 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 " 619 "%indvars.iv, i32 0\n" 620 " %vB0 = load i32, i32* %B0, align 4\n" 621 " %mul11 = mul nsw i32 %vA0, %vB0\n" 622 " %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 623 "%indvars.iv, i32 0\n" 624 " %vC0 = load i32, i32* %C0, align 4\n" 625 " %D0 = getelementptr inbounds %struct.Test, %struct.Test* %D, i64 " 626 "%indvars.iv, i32 0\n" 627 " %vD0 = load i32, i32* %D0, align 4\n" 628 " %mul12 = mul nsw i32 %vC0, %vD0\n" 629 " %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 " 630 "%indvars.iv, i32 1\n" 631 " %vA1 = load i32, i32* %A1, align 4\n" 632 " %B1 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 " 633 "%indvars.iv, i32 1\n" 634 " %vB1 = load i32, i32* %B1, align 4\n" 635 " %mul21 = mul nsw i32 %vA1, %vB1\n" 636 " %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 637 "%indvars.iv, i32 1\n" 638 " %vC1 = load i32, i32* %C1, align 4\n" 639 " %D1 = getelementptr inbounds %struct.Test, %struct.Test* %D, i64 " 640 "%indvars.iv, i32 1\n" 641 " %vD1 = load i32, i32* %D1, align 4\n" 642 " %mul22 = mul nsw i32 %vC1, %vD1\n" 643 " %add1 = add nsw i32 %mul11, %mul12\n" 644 " %add2 = add nsw i32 %mul22, %mul21\n" 645 " %E0 = getelementptr inbounds %struct.Test, %struct.Test* %E, i64 " 646 "%indvars.iv, i32 0\n" 647 " store i32 %add1, i32* %E0, align 4\n" 648 " %E1 = getelementptr inbounds %struct.Test, %struct.Test* %E, i64 " 649 "%indvars.iv, i32 1\n" 650 " store i32 %add2, i32* %E1, align 4\n" 651 " %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n" 652 " %exitcond = icmp eq i64 %indvars.iv.next, 1024\n" 653 " br i1 %exitcond, label %for.cond.cleanup, label %for.body\n" 654 "for.cond.cleanup: ; preds = %for.body\n" 655 " ret void\n" 656 "}\n"; 657 658 Module &M = parseModule(ModuleString); 659 660 Function *F = M.getFunction("add_x3"); 661 BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor(); 662 auto Plan = buildHCFG(LoopHeader); 663 664 VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock(); 665 EXPECT_NE(nullptr, Entry->getSingleSuccessor()); 666 VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock(); 667 668 VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 24)); 669 VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 26)); 670 671 checkReorderExample( 672 Store1, Store2, Body, 673 getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan)); 674 } 675 676 // Make sure we do not combine instructions with operands in different BBs. 677 TEST_F(VPlanSlpTest, testInstrsInDifferentBBs) { 678 const char *ModuleString = 679 "%struct.Test = type { i32, i32 }\n" 680 "%struct.Test3 = type { i32, i32, i32 }\n" 681 "%struct.Test4xi8 = type { i8, i8, i8 }\n" 682 "define void @add_x2(%struct.Test* nocapture readonly %A, %struct.Test* " 683 "nocapture readonly %B, %struct.Test* nocapture %C) {\n" 684 "entry:\n" 685 " br label %for.body\n" 686 "for.body: ; preds = %for.body, " 687 "%entry\n" 688 " %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n" 689 " %A0 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 " 690 "%indvars.iv, i32 0\n" 691 " %vA0 = load i32, i32* %A0, align 4\n" 692 " %B0 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 " 693 "%indvars.iv, i32 0\n" 694 " %vB0 = load i32, i32* %B0, align 4\n" 695 " %add0 = add nsw i32 %vA0, %vB0\n" 696 " %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 " 697 "%indvars.iv, i32 1\n" 698 " %vA1 = load i32, i32* %A1, align 4\n" 699 " %B1 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 " 700 "%indvars.iv, i32 1\n" 701 " br label %bb2\n" 702 "bb2:\n" 703 " %vB1 = load i32, i32* %B1, align 4\n" 704 " %add1 = add nsw i32 %vA1, %vB1\n" 705 " %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 706 "%indvars.iv, i32 0\n" 707 " store i32 %add0, i32* %C0, align 4\n" 708 " %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 709 "%indvars.iv, i32 1\n" 710 " store i32 %add1, i32* %C1, align 4\n" 711 " %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n" 712 " %exitcond = icmp eq i64 %indvars.iv.next, 1024\n" 713 " br i1 %exitcond, label %for.cond.cleanup, label %for.body\n" 714 "for.cond.cleanup: ; preds = %for.body\n" 715 " ret void\n" 716 "}\n"; 717 718 Module &M = parseModule(ModuleString); 719 720 Function *F = M.getFunction("add_x2"); 721 BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor(); 722 auto Plan = buildHCFG(LoopHeader); 723 auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan); 724 725 VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock(); 726 EXPECT_NE(nullptr, Entry->getSingleSuccessor()); 727 VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock(); 728 VPBasicBlock *BB2 = Body->getSingleSuccessor()->getEntryBasicBlock(); 729 730 VPInstruction *Store1 = cast<VPInstruction>(&*std::next(BB2->begin(), 3)); 731 VPInstruction *Store2 = cast<VPInstruction>(&*std::next(BB2->begin(), 5)); 732 733 VPlanSlp Slp(VPIAI, *BB2); 734 SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2}; 735 EXPECT_EQ(nullptr, Slp.buildGraph(StoreRoot)); 736 EXPECT_EQ(0u, Slp.getWidestBundleBits()); 737 } 738 739 // Make sure we do not combine instructions with operands in different BBs. 740 TEST_F(VPlanSlpTest, testInstrsInDifferentBBs2) { 741 const char *ModuleString = 742 "%struct.Test = type { i32, i32 }\n" 743 "%struct.Test3 = type { i32, i32, i32 }\n" 744 "%struct.Test4xi8 = type { i8, i8, i8 }\n" 745 "define void @add_x2(%struct.Test* nocapture readonly %A, %struct.Test* " 746 "nocapture readonly %B, %struct.Test* nocapture %C) {\n" 747 "entry:\n" 748 " br label %for.body\n" 749 "for.body: ; preds = %for.body, " 750 "%entry\n" 751 " %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n" 752 " %A0 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 " 753 "%indvars.iv, i32 0\n" 754 " %vA0 = load i32, i32* %A0, align 4\n" 755 " %B0 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 " 756 "%indvars.iv, i32 0\n" 757 " %vB0 = load i32, i32* %B0, align 4\n" 758 " %add0 = add nsw i32 %vA0, %vB0\n" 759 " %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 " 760 "%indvars.iv, i32 1\n" 761 " %vA1 = load i32, i32* %A1, align 4\n" 762 " %B1 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 " 763 "%indvars.iv, i32 1\n" 764 " %vB1 = load i32, i32* %B1, align 4\n" 765 " %add1 = add nsw i32 %vA1, %vB1\n" 766 " br label %bb2\n" 767 "bb2:\n" 768 " %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 769 "%indvars.iv, i32 0\n" 770 " store i32 %add0, i32* %C0, align 4\n" 771 " %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 772 "%indvars.iv, i32 1\n" 773 " store i32 %add1, i32* %C1, align 4\n" 774 " %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n" 775 " %exitcond = icmp eq i64 %indvars.iv.next, 1024\n" 776 " br i1 %exitcond, label %for.cond.cleanup, label %for.body\n" 777 "for.cond.cleanup: ; preds = %for.body\n" 778 " ret void\n" 779 "}\n"; 780 781 Module &M = parseModule(ModuleString); 782 783 Function *F = M.getFunction("add_x2"); 784 BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor(); 785 auto Plan = buildHCFG(LoopHeader); 786 auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan); 787 788 VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock(); 789 EXPECT_NE(nullptr, Entry->getSingleSuccessor()); 790 VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock(); 791 VPBasicBlock *BB2 = Body->getSingleSuccessor()->getEntryBasicBlock(); 792 793 VPInstruction *Store1 = cast<VPInstruction>(&*std::next(BB2->begin(), 1)); 794 VPInstruction *Store2 = cast<VPInstruction>(&*std::next(BB2->begin(), 3)); 795 796 VPlanSlp Slp(VPIAI, *BB2); 797 SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2}; 798 EXPECT_EQ(nullptr, Slp.buildGraph(StoreRoot)); 799 EXPECT_EQ(0u, Slp.getWidestBundleBits()); 800 } 801 802 TEST_F(VPlanSlpTest, testSlpAtomicLoad) { 803 const char *ModuleString = 804 "%struct.Test = type { i32, i32 }\n" 805 "%struct.Test3 = type { i32, i32, i32 }\n" 806 "%struct.Test4xi8 = type { i8, i8, i8 }\n" 807 "define void @add_x2(%struct.Test* nocapture readonly %A, %struct.Test* " 808 "nocapture readonly %B, %struct.Test* nocapture %C) {\n" 809 "entry:\n" 810 " br label %for.body\n" 811 "for.body: ; preds = %for.body, " 812 "%entry\n" 813 " %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n" 814 " %A0 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 " 815 "%indvars.iv, i32 0\n" 816 " %vA0 = load atomic i32, i32* %A0 monotonic, align 4\n" 817 " %B0 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 " 818 "%indvars.iv, i32 0\n" 819 " %vB0 = load i32, i32* %B0, align 4\n" 820 " %add0 = add nsw i32 %vA0, %vB0\n" 821 " %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 " 822 "%indvars.iv, i32 1\n" 823 " %vA1 = load i32, i32* %A1, align 4\n" 824 " %B1 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 " 825 "%indvars.iv, i32 1\n" 826 " %vB1 = load i32, i32* %B1, align 4\n" 827 " %add1 = add nsw i32 %vA1, %vB1\n" 828 " %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 829 "%indvars.iv, i32 0\n" 830 " store i32 %add0, i32* %C0, align 4\n" 831 " %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 832 "%indvars.iv, i32 1\n" 833 " store i32 %add1, i32* %C1, align 4\n" 834 " %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n" 835 " %exitcond = icmp eq i64 %indvars.iv.next, 1024\n" 836 " br i1 %exitcond, label %for.cond.cleanup, label %for.body\n" 837 "for.cond.cleanup: ; preds = %for.body\n" 838 " ret void\n" 839 "}\n"; 840 841 Module &M = parseModule(ModuleString); 842 843 Function *F = M.getFunction("add_x2"); 844 BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor(); 845 auto Plan = buildHCFG(LoopHeader); 846 auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan); 847 848 VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock(); 849 EXPECT_NE(nullptr, Entry->getSingleSuccessor()); 850 VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock(); 851 852 VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 12)); 853 VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 14)); 854 855 VPlanSlp Slp(VPIAI, *Body); 856 SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2}; 857 EXPECT_EQ(nullptr, Slp.buildGraph(StoreRoot)); 858 EXPECT_FALSE(Slp.isCompletelySLP()); 859 } 860 861 TEST_F(VPlanSlpTest, testSlpAtomicStore) { 862 const char *ModuleString = 863 "%struct.Test = type { i32, i32 }\n" 864 "%struct.Test3 = type { i32, i32, i32 }\n" 865 "%struct.Test4xi8 = type { i8, i8, i8 }\n" 866 "define void @add_x2(%struct.Test* nocapture readonly %A, %struct.Test* " 867 "nocapture readonly %B, %struct.Test* nocapture %C) {\n" 868 "entry:\n" 869 " br label %for.body\n" 870 "for.body: ; preds = %for.body, " 871 "%entry\n" 872 " %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n" 873 " %A0 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 " 874 "%indvars.iv, i32 0\n" 875 " %vA0 = load i32, i32* %A0, align 4\n" 876 " %B0 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 " 877 "%indvars.iv, i32 0\n" 878 " %vB0 = load i32, i32* %B0, align 4\n" 879 " %add0 = add nsw i32 %vA0, %vB0\n" 880 " %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 " 881 "%indvars.iv, i32 1\n" 882 " %vA1 = load i32, i32* %A1, align 4\n" 883 " %B1 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 " 884 "%indvars.iv, i32 1\n" 885 " %vB1 = load i32, i32* %B1, align 4\n" 886 " %add1 = add nsw i32 %vA1, %vB1\n" 887 " %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 888 "%indvars.iv, i32 0\n" 889 " store atomic i32 %add0, i32* %C0 monotonic, align 4\n" 890 " %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 891 "%indvars.iv, i32 1\n" 892 " store i32 %add1, i32* %C1, align 4\n" 893 " %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n" 894 " %exitcond = icmp eq i64 %indvars.iv.next, 1024\n" 895 " br i1 %exitcond, label %for.cond.cleanup, label %for.body\n" 896 "for.cond.cleanup: ; preds = %for.body\n" 897 " ret void\n" 898 "}\n"; 899 900 Module &M = parseModule(ModuleString); 901 902 Function *F = M.getFunction("add_x2"); 903 BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor(); 904 auto Plan = buildHCFG(LoopHeader); 905 auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan); 906 907 VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock(); 908 EXPECT_NE(nullptr, Entry->getSingleSuccessor()); 909 VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock(); 910 911 VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 12)); 912 VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 14)); 913 914 VPlanSlp Slp(VPIAI, *Body); 915 SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2}; 916 Slp.buildGraph(StoreRoot); 917 EXPECT_FALSE(Slp.isCompletelySLP()); 918 } 919 920 } // namespace 921 } // namespace llvm 922