1 //===- llvm/unittest/Transforms/Vectorize/VPlanSlpTest.cpp ---------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "../lib/Transforms/Vectorize/VPlan.h" 10 #include "../lib/Transforms/Vectorize/VPlanHCFGBuilder.h" 11 #include "VPlanTestBase.h" 12 #include "llvm/Analysis/VectorUtils.h" 13 #include "gtest/gtest.h" 14 15 namespace llvm { 16 namespace { 17 18 class VPlanSlpTest : public VPlanTestBase { 19 protected: 20 TargetLibraryInfoImpl TLII; 21 TargetLibraryInfo TLI; 22 DataLayout DL; 23 24 std::unique_ptr<AssumptionCache> AC; 25 std::unique_ptr<ScalarEvolution> SE; 26 std::unique_ptr<AAResults> AARes; 27 std::unique_ptr<BasicAAResult> BasicAA; 28 std::unique_ptr<LoopAccessInfo> LAI; 29 std::unique_ptr<PredicatedScalarEvolution> PSE; 30 std::unique_ptr<InterleavedAccessInfo> IAI; 31 32 VPlanSlpTest() 33 : TLII(), TLI(TLII), 34 DL("e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-" 35 "f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:" 36 "16:32:64-S128") {} 37 38 VPInterleavedAccessInfo getInterleavedAccessInfo(Function &F, Loop *L, 39 VPlan &Plan) { 40 AC.reset(new AssumptionCache(F)); 41 SE.reset(new ScalarEvolution(F, TLI, *AC, *DT, *LI)); 42 BasicAA.reset(new BasicAAResult(DL, F, TLI, *AC, &*DT, &*LI)); 43 AARes.reset(new AAResults(TLI)); 44 AARes->addAAResult(*BasicAA); 45 PSE.reset(new PredicatedScalarEvolution(*SE, *L)); 46 LAI.reset(new LoopAccessInfo(L, &*SE, &TLI, &*AARes, &*DT, &*LI)); 47 IAI.reset(new InterleavedAccessInfo(*PSE, L, &*DT, &*LI, &*LAI)); 48 IAI->analyzeInterleaving(false); 49 return {Plan, *IAI}; 50 } 51 }; 52 53 TEST_F(VPlanSlpTest, testSlpSimple_2) { 54 const char *ModuleString = 55 "%struct.Test = type { i32, i32 }\n" 56 "%struct.Test3 = type { i32, i32, i32 }\n" 57 "%struct.Test4xi8 = type { i8, i8, i8 }\n" 58 "define void @add_x2(%struct.Test* nocapture readonly %A, %struct.Test* " 59 "nocapture readonly %B, %struct.Test* nocapture %C) {\n" 60 "entry:\n" 61 " br label %for.body\n" 62 "for.body: ; preds = %for.body, " 63 "%entry\n" 64 " %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n" 65 " %A0 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 " 66 "%indvars.iv, i32 0\n" 67 " %vA0 = load i32, i32* %A0, align 4\n" 68 " %B0 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 " 69 "%indvars.iv, i32 0\n" 70 " %vB0 = load i32, i32* %B0, align 4\n" 71 " %add0 = add nsw i32 %vA0, %vB0\n" 72 " %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 " 73 "%indvars.iv, i32 1\n" 74 " %vA1 = load i32, i32* %A1, align 4\n" 75 " %B1 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 " 76 "%indvars.iv, i32 1\n" 77 " %vB1 = load i32, i32* %B1, align 4\n" 78 " %add1 = add nsw i32 %vA1, %vB1\n" 79 " %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 80 "%indvars.iv, i32 0\n" 81 " store i32 %add0, i32* %C0, align 4\n" 82 " %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 83 "%indvars.iv, i32 1\n" 84 " store i32 %add1, i32* %C1, align 4\n" 85 " %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n" 86 " %exitcond = icmp eq i64 %indvars.iv.next, 1024\n" 87 " br i1 %exitcond, label %for.cond.cleanup, label %for.body\n" 88 "for.cond.cleanup: ; preds = %for.body\n" 89 " ret void\n" 90 "}\n"; 91 92 Module &M = parseModule(ModuleString); 93 94 Function *F = M.getFunction("add_x2"); 95 BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor(); 96 auto Plan = buildHCFG(LoopHeader); 97 auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan); 98 99 VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock(); 100 EXPECT_NE(nullptr, Entry->getSingleSuccessor()); 101 VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock(); 102 103 VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 12)); 104 VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 14)); 105 106 VPlanSlp Slp(VPIAI, *Body); 107 SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2}; 108 VPInstruction *CombinedStore = Slp.buildGraph(StoreRoot); 109 EXPECT_EQ(64u, Slp.getWidestBundleBits()); 110 EXPECT_EQ(VPInstruction::SLPStore, CombinedStore->getOpcode()); 111 112 auto *CombinedAdd = cast<VPInstruction>(CombinedStore->getOperand(0)); 113 EXPECT_EQ(Instruction::Add, CombinedAdd->getOpcode()); 114 115 auto *CombinedLoadA = cast<VPInstruction>(CombinedAdd->getOperand(0)); 116 auto *CombinedLoadB = cast<VPInstruction>(CombinedAdd->getOperand(1)); 117 EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadA->getOpcode()); 118 EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadB->getOpcode()); 119 } 120 121 TEST_F(VPlanSlpTest, testSlpSimple_3) { 122 const char *ModuleString = 123 "%struct.Test = type { i32, i32 }\n" 124 "%struct.Test3 = type { i32, i32, i32 }\n" 125 "%struct.Test4xi8 = type { i8, i8, i8 }\n" 126 "define void @add_x2(%struct.Test* nocapture readonly %A, %struct.Test* " 127 "nocapture readonly %B, %struct.Test* nocapture %C) {\n" 128 "entry:\n" 129 " br label %for.body\n" 130 "for.body: ; preds = %for.body, " 131 "%entry\n" 132 " %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n" 133 " %A0 = getelementptr %struct.Test, %struct.Test* %A, i64 " 134 " %indvars.iv, i32 0\n" 135 " %vA0 = load i32, i32* %A0, align 4\n" 136 " %B0 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 " 137 " %indvars.iv, i32 0\n" 138 " %vB0 = load i32, i32* %B0, align 4\n" 139 " %add0 = add nsw i32 %vA0, %vB0\n" 140 " %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 " 141 " %indvars.iv, i32 1\n" 142 " %vA1 = load i32, i32* %A1, align 4\n" 143 " %B1 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 " 144 " %indvars.iv, i32 1\n" 145 " %vB1 = load i32, i32* %B1, align 4\n" 146 " %add1 = add nsw i32 %vA1, %vB1\n" 147 " %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 148 " %indvars.iv, i32 0\n" 149 " store i32 %add0, i32* %C0, align 4\n" 150 " %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 151 " %indvars.iv, i32 1\n" 152 " store i32 %add1, i32* %C1, align 4\n" 153 " %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n" 154 " %exitcond = icmp eq i64 %indvars.iv.next, 1024\n" 155 " br i1 %exitcond, label %for.cond.cleanup, label %for.body\n" 156 "for.cond.cleanup: ; preds = %for.body\n" 157 " ret void\n" 158 "}\n"; 159 160 Module &M = parseModule(ModuleString); 161 162 Function *F = M.getFunction("add_x2"); 163 BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor(); 164 auto Plan = buildHCFG(LoopHeader); 165 166 VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock(); 167 EXPECT_NE(nullptr, Entry->getSingleSuccessor()); 168 VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock(); 169 170 VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 12)); 171 VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 14)); 172 173 auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan); 174 175 VPlanSlp Slp(VPIAI, *Body); 176 SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2}; 177 VPInstruction *CombinedStore = Slp.buildGraph(StoreRoot); 178 EXPECT_EQ(64u, Slp.getWidestBundleBits()); 179 EXPECT_EQ(VPInstruction::SLPStore, CombinedStore->getOpcode()); 180 181 auto *CombinedAdd = cast<VPInstruction>(CombinedStore->getOperand(0)); 182 EXPECT_EQ(Instruction::Add, CombinedAdd->getOpcode()); 183 184 auto *CombinedLoadA = cast<VPInstruction>(CombinedAdd->getOperand(0)); 185 auto *CombinedLoadB = cast<VPInstruction>(CombinedAdd->getOperand(1)); 186 EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadA->getOpcode()); 187 EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadB->getOpcode()); 188 189 VPInstruction *GetA = cast<VPInstruction>(&*std::next(Body->begin(), 1)); 190 VPInstruction *GetB = cast<VPInstruction>(&*std::next(Body->begin(), 3)); 191 EXPECT_EQ(GetA, CombinedLoadA->getOperand(0)); 192 EXPECT_EQ(GetB, CombinedLoadB->getOperand(0)); 193 } 194 195 TEST_F(VPlanSlpTest, testSlpReuse_1) { 196 const char *ModuleString = 197 "%struct.Test = type { i32, i32 }\n" 198 "define void @add_x2(%struct.Test* nocapture readonly %A, %struct.Test* " 199 "nocapture readonly %B, %struct.Test* nocapture %C) {\n" 200 "entry:\n" 201 " br label %for.body\n" 202 "for.body: ; preds = %for.body, " 203 "%entry\n" 204 " %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n" 205 " %A0 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 " 206 "%indvars.iv, i32 0\n" 207 " %vA0 = load i32, i32* %A0, align 4\n" 208 " %add0 = add nsw i32 %vA0, %vA0\n" 209 " %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 " 210 "%indvars.iv, i32 1\n" 211 " %vA1 = load i32, i32* %A1, align 4\n" 212 " %add1 = add nsw i32 %vA1, %vA1\n" 213 " %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 214 "%indvars.iv, i32 0\n" 215 " store i32 %add0, i32* %C0, align 4\n" 216 " %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 217 "%indvars.iv, i32 1\n" 218 " store i32 %add1, i32* %C1, align 4\n" 219 " %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n" 220 " %exitcond = icmp eq i64 %indvars.iv.next, 1024\n" 221 " br i1 %exitcond, label %for.cond.cleanup, label %for.body\n" 222 "for.cond.cleanup: ; preds = %for.body\n" 223 " ret void\n" 224 "}\n"; 225 226 Module &M = parseModule(ModuleString); 227 228 Function *F = M.getFunction("add_x2"); 229 BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor(); 230 auto Plan = buildHCFG(LoopHeader); 231 auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan); 232 233 VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock(); 234 EXPECT_NE(nullptr, Entry->getSingleSuccessor()); 235 VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock(); 236 237 VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 8)); 238 VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 10)); 239 240 VPlanSlp Slp(VPIAI, *Body); 241 SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2}; 242 VPInstruction *CombinedStore = Slp.buildGraph(StoreRoot); 243 EXPECT_EQ(64u, Slp.getWidestBundleBits()); 244 EXPECT_EQ(VPInstruction::SLPStore, CombinedStore->getOpcode()); 245 246 auto *CombinedAdd = cast<VPInstruction>(CombinedStore->getOperand(0)); 247 EXPECT_EQ(Instruction::Add, CombinedAdd->getOpcode()); 248 249 auto *CombinedLoadA = cast<VPInstruction>(CombinedAdd->getOperand(0)); 250 EXPECT_EQ(CombinedLoadA, CombinedAdd->getOperand(1)); 251 EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadA->getOpcode()); 252 } 253 254 TEST_F(VPlanSlpTest, testSlpReuse_2) { 255 const char *ModuleString = 256 "%struct.Test = type { i32, i32 }\n" 257 "define i32 @add_x2(%struct.Test* nocapture readonly %A, %struct.Test* " 258 "nocapture readonly %B, %struct.Test* nocapture %C) {\n" 259 "entry:\n" 260 " br label %for.body\n" 261 "for.body: ; preds = %for.body, " 262 "%entry\n" 263 " %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n" 264 " %A0 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 " 265 "%indvars.iv, i32 0\n" 266 " %vA0 = load i32, i32* %A0, align 4\n" 267 " %add0 = add nsw i32 %vA0, %vA0\n" 268 " %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 269 "%indvars.iv, i32 0\n" 270 " store i32 %add0, i32* %C0, align 4\n" 271 " %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 " 272 "%indvars.iv, i32 1\n" 273 " %vA1 = load i32, i32* %A1, align 4\n" 274 " %add1 = add nsw i32 %vA1, %vA1\n" 275 " %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 276 "%indvars.iv, i32 1\n" 277 " store i32 %add1, i32* %C1, align 4\n" 278 " %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n" 279 " %exitcond = icmp eq i64 %indvars.iv.next, 1024\n" 280 " br i1 %exitcond, label %for.cond.cleanup, label %for.body\n" 281 "for.cond.cleanup: ; preds = %for.body\n" 282 " ret i32 %vA1\n" 283 "}\n"; 284 285 Module &M = parseModule(ModuleString); 286 287 Function *F = M.getFunction("add_x2"); 288 BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor(); 289 auto Plan = buildHCFG(LoopHeader); 290 auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan); 291 292 VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock(); 293 EXPECT_NE(nullptr, Entry->getSingleSuccessor()); 294 VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock(); 295 296 VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 5)); 297 VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 10)); 298 299 VPlanSlp Slp(VPIAI, *Body); 300 SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2}; 301 Slp.buildGraph(StoreRoot); 302 EXPECT_FALSE(Slp.isCompletelySLP()); 303 } 304 305 static void checkReorderExample(VPInstruction *Store1, VPInstruction *Store2, 306 VPBasicBlock *Body, 307 VPInterleavedAccessInfo &&IAI) { 308 VPlanSlp Slp(IAI, *Body); 309 SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2}; 310 VPInstruction *CombinedStore = Slp.buildGraph(StoreRoot); 311 312 EXPECT_TRUE(Slp.isCompletelySLP()); 313 EXPECT_EQ(CombinedStore->getOpcode(), VPInstruction::SLPStore); 314 315 VPInstruction *CombinedAdd = 316 cast<VPInstruction>(CombinedStore->getOperand(0)); 317 EXPECT_EQ(CombinedAdd->getOpcode(), Instruction::Add); 318 319 VPInstruction *CombinedMulAB = 320 cast<VPInstruction>(CombinedAdd->getOperand(0)); 321 VPInstruction *CombinedMulCD = 322 cast<VPInstruction>(CombinedAdd->getOperand(1)); 323 EXPECT_EQ(CombinedMulAB->getOpcode(), Instruction::Mul); 324 325 VPInstruction *CombinedLoadA = 326 cast<VPInstruction>(CombinedMulAB->getOperand(0)); 327 EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadA->getOpcode()); 328 VPInstruction *LoadvA0 = cast<VPInstruction>(&*std::next(Body->begin(), 2)); 329 VPInstruction *LoadvA1 = cast<VPInstruction>(&*std::next(Body->begin(), 12)); 330 EXPECT_EQ(LoadvA0->getOperand(0), CombinedLoadA->getOperand(0)); 331 EXPECT_EQ(LoadvA1->getOperand(0), CombinedLoadA->getOperand(1)); 332 333 VPInstruction *CombinedLoadB = 334 cast<VPInstruction>(CombinedMulAB->getOperand(1)); 335 EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadB->getOpcode()); 336 VPInstruction *LoadvB0 = cast<VPInstruction>(&*std::next(Body->begin(), 4)); 337 VPInstruction *LoadvB1 = cast<VPInstruction>(&*std::next(Body->begin(), 14)); 338 EXPECT_EQ(LoadvB0->getOperand(0), CombinedLoadB->getOperand(0)); 339 EXPECT_EQ(LoadvB1->getOperand(0), CombinedLoadB->getOperand(1)); 340 341 EXPECT_EQ(CombinedMulCD->getOpcode(), Instruction::Mul); 342 343 VPInstruction *CombinedLoadC = 344 cast<VPInstruction>(CombinedMulCD->getOperand(0)); 345 EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadC->getOpcode()); 346 VPInstruction *LoadvC0 = cast<VPInstruction>(&*std::next(Body->begin(), 7)); 347 VPInstruction *LoadvC1 = cast<VPInstruction>(&*std::next(Body->begin(), 17)); 348 EXPECT_EQ(LoadvC0->getOperand(0), CombinedLoadC->getOperand(0)); 349 EXPECT_EQ(LoadvC1->getOperand(0), CombinedLoadC->getOperand(1)); 350 351 VPInstruction *CombinedLoadD = 352 cast<VPInstruction>(CombinedMulCD->getOperand(1)); 353 EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadD->getOpcode()); 354 VPInstruction *LoadvD0 = cast<VPInstruction>(&*std::next(Body->begin(), 9)); 355 VPInstruction *LoadvD1 = cast<VPInstruction>(&*std::next(Body->begin(), 19)); 356 EXPECT_EQ(LoadvD0->getOperand(0), CombinedLoadD->getOperand(0)); 357 EXPECT_EQ(LoadvD1->getOperand(0), CombinedLoadD->getOperand(1)); 358 } 359 360 TEST_F(VPlanSlpTest, testSlpReorder_1) { 361 LLVMContext Ctx; 362 const char *ModuleString = 363 "%struct.Test = type { i32, i32 }\n" 364 "define void @add_x3(%struct.Test* %A, %struct.Test* %B, %struct.Test* " 365 "%C, %struct.Test* %D, %struct.Test* %E) {\n" 366 "entry:\n" 367 " br label %for.body\n" 368 "for.body: ; preds = %for.body, " 369 "%entry\n" 370 " %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n" 371 " %A0 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 " 372 "%indvars.iv, i32 0\n" 373 " %vA0 = load i32, i32* %A0, align 4\n" 374 " %B0 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 " 375 "%indvars.iv, i32 0\n" 376 " %vB0 = load i32, i32* %B0, align 4\n" 377 " %mul11 = mul nsw i32 %vA0, %vB0\n" 378 " %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 379 "%indvars.iv, i32 0\n" 380 " %vC0 = load i32, i32* %C0, align 4\n" 381 " %D0 = getelementptr inbounds %struct.Test, %struct.Test* %D, i64 " 382 "%indvars.iv, i32 0\n" 383 " %vD0 = load i32, i32* %D0, align 4\n" 384 " %mul12 = mul nsw i32 %vC0, %vD0\n" 385 " %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 " 386 "%indvars.iv, i32 1\n" 387 " %vA1 = load i32, i32* %A1, align 4\n" 388 " %B1 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 " 389 "%indvars.iv, i32 1\n" 390 " %vB1 = load i32, i32* %B1, align 4\n" 391 " %mul21 = mul nsw i32 %vA1, %vB1\n" 392 " %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 393 "%indvars.iv, i32 1\n" 394 " %vC1 = load i32, i32* %C1, align 4\n" 395 " %D1 = getelementptr inbounds %struct.Test, %struct.Test* %D, i64 " 396 "%indvars.iv, i32 1\n" 397 " %vD1 = load i32, i32* %D1, align 4\n" 398 " %mul22 = mul nsw i32 %vC1, %vD1\n" 399 " %add1 = add nsw i32 %mul11, %mul12\n" 400 " %add2 = add nsw i32 %mul22, %mul21\n" 401 " %E0 = getelementptr inbounds %struct.Test, %struct.Test* %E, i64 " 402 "%indvars.iv, i32 0\n" 403 " store i32 %add1, i32* %E0, align 4\n" 404 " %E1 = getelementptr inbounds %struct.Test, %struct.Test* %E, i64 " 405 "%indvars.iv, i32 1\n" 406 " store i32 %add2, i32* %E1, align 4\n" 407 " %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n" 408 " %exitcond = icmp eq i64 %indvars.iv.next, 1024\n" 409 " br i1 %exitcond, label %for.cond.cleanup, label %for.body\n" 410 "for.cond.cleanup: ; preds = %for.body\n" 411 " ret void\n" 412 "}\n"; 413 414 Module &M = parseModule(ModuleString); 415 416 Function *F = M.getFunction("add_x3"); 417 BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor(); 418 auto Plan = buildHCFG(LoopHeader); 419 420 VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock(); 421 EXPECT_NE(nullptr, Entry->getSingleSuccessor()); 422 VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock(); 423 424 VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 24)); 425 VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 26)); 426 427 checkReorderExample( 428 Store1, Store2, Body, 429 getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan)); 430 } 431 432 TEST_F(VPlanSlpTest, testSlpReorder_2) { 433 LLVMContext Ctx; 434 const char *ModuleString = 435 "%struct.Test = type { i32, i32 }\n" 436 "define void @add_x3(%struct.Test* %A, %struct.Test* %B, %struct.Test* " 437 "%C, %struct.Test* %D, %struct.Test* %E) {\n" 438 "entry:\n" 439 " br label %for.body\n" 440 "for.body: ; preds = %for.body, " 441 "%entry\n" 442 " %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n" 443 " %A0 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 " 444 "%indvars.iv, i32 0\n" 445 " %vA0 = load i32, i32* %A0, align 4\n" 446 " %B0 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 " 447 "%indvars.iv, i32 0\n" 448 " %vB0 = load i32, i32* %B0, align 4\n" 449 " %mul11 = mul nsw i32 %vA0, %vB0\n" 450 " %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 451 "%indvars.iv, i32 0\n" 452 " %vC0 = load i32, i32* %C0, align 4\n" 453 " %D0 = getelementptr inbounds %struct.Test, %struct.Test* %D, i64 " 454 "%indvars.iv, i32 0\n" 455 " %vD0 = load i32, i32* %D0, align 4\n" 456 " %mul12 = mul nsw i32 %vC0, %vD0\n" 457 " %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 " 458 "%indvars.iv, i32 1\n" 459 " %vA1 = load i32, i32* %A1, align 4\n" 460 " %B1 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 " 461 "%indvars.iv, i32 1\n" 462 " %vB1 = load i32, i32* %B1, align 4\n" 463 " %mul21 = mul nsw i32 %vB1, %vA1\n" 464 " %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 465 "%indvars.iv, i32 1\n" 466 " %vC1 = load i32, i32* %C1, align 4\n" 467 " %D1 = getelementptr inbounds %struct.Test, %struct.Test* %D, i64 " 468 "%indvars.iv, i32 1\n" 469 " %vD1 = load i32, i32* %D1, align 4\n" 470 " %mul22 = mul nsw i32 %vD1, %vC1\n" 471 " %add1 = add nsw i32 %mul11, %mul12\n" 472 " %add2 = add nsw i32 %mul22, %mul21\n" 473 " %E0 = getelementptr inbounds %struct.Test, %struct.Test* %E, i64 " 474 "%indvars.iv, i32 0\n" 475 " store i32 %add1, i32* %E0, align 4\n" 476 " %E1 = getelementptr inbounds %struct.Test, %struct.Test* %E, i64 " 477 "%indvars.iv, i32 1\n" 478 " store i32 %add2, i32* %E1, align 4\n" 479 " %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n" 480 " %exitcond = icmp eq i64 %indvars.iv.next, 1024\n" 481 " br i1 %exitcond, label %for.cond.cleanup, label %for.body\n" 482 "for.cond.cleanup: ; preds = %for.body\n" 483 " ret void\n" 484 "}\n"; 485 486 Module &M = parseModule(ModuleString); 487 488 Function *F = M.getFunction("add_x3"); 489 BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor(); 490 auto Plan = buildHCFG(LoopHeader); 491 492 VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock(); 493 EXPECT_NE(nullptr, Entry->getSingleSuccessor()); 494 VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock(); 495 496 VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 24)); 497 VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 26)); 498 499 checkReorderExample( 500 Store1, Store2, Body, 501 getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan)); 502 } 503 504 TEST_F(VPlanSlpTest, testSlpReorder_3) { 505 LLVMContext Ctx; 506 const char *ModuleString = 507 "%struct.Test = type { i32, i32 }\n" 508 "define void @add_x3(%struct.Test* %A, %struct.Test* %B, %struct.Test* " 509 "%C, %struct.Test* %D, %struct.Test* %E) {\n" 510 "entry:\n" 511 " br label %for.body\n" 512 "for.body: ; preds = %for.body, " 513 "%entry\n" 514 " %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n" 515 " %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 " 516 "%indvars.iv, i32 1\n" 517 " %vA1 = load i32, i32* %A1, align 4\n" 518 " %B0 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 " 519 "%indvars.iv, i32 0\n" 520 " %vB0 = load i32, i32* %B0, align 4\n" 521 " %mul11 = mul nsw i32 %vA1, %vB0\n" 522 " %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 523 "%indvars.iv, i32 0\n" 524 " %vC0 = load i32, i32* %C0, align 4\n" 525 " %D0 = getelementptr inbounds %struct.Test, %struct.Test* %D, i64 " 526 "%indvars.iv, i32 0\n" 527 " %vD0 = load i32, i32* %D0, align 4\n" 528 " %mul12 = mul nsw i32 %vC0, %vD0\n" 529 " %A0 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 " 530 "%indvars.iv, i32 0\n" 531 " %vA0 = load i32, i32* %A0, align 4\n" 532 " %B1 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 " 533 "%indvars.iv, i32 1\n" 534 " %vB1 = load i32, i32* %B1, align 4\n" 535 " %mul21 = mul nsw i32 %vB1, %vA0\n" 536 " %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 537 "%indvars.iv, i32 1\n" 538 " %vC1 = load i32, i32* %C1, align 4\n" 539 " %D1 = getelementptr inbounds %struct.Test, %struct.Test* %D, i64 " 540 "%indvars.iv, i32 1\n" 541 " %vD1 = load i32, i32* %D1, align 4\n" 542 " %mul22 = mul nsw i32 %vD1, %vC1\n" 543 " %add1 = add nsw i32 %mul11, %mul12\n" 544 " %add2 = add nsw i32 %mul22, %mul21\n" 545 " %E0 = getelementptr inbounds %struct.Test, %struct.Test* %E, i64 " 546 "%indvars.iv, i32 0\n" 547 " store i32 %add1, i32* %E0, align 4\n" 548 " %E1 = getelementptr inbounds %struct.Test, %struct.Test* %E, i64 " 549 "%indvars.iv, i32 1\n" 550 " store i32 %add2, i32* %E1, align 4\n" 551 " %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n" 552 " %exitcond = icmp eq i64 %indvars.iv.next, 1024\n" 553 " br i1 %exitcond, label %for.cond.cleanup, label %for.body\n" 554 "for.cond.cleanup: ; preds = %for.body\n" 555 " ret void\n" 556 "}\n"; 557 558 Module &M = parseModule(ModuleString); 559 560 Function *F = M.getFunction("add_x3"); 561 BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor(); 562 auto Plan = buildHCFG(LoopHeader); 563 564 VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock(); 565 EXPECT_NE(nullptr, Entry->getSingleSuccessor()); 566 VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock(); 567 568 VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 24)); 569 VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 26)); 570 571 auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan); 572 VPlanSlp Slp(VPIAI, *Body); 573 SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2}; 574 EXPECT_EQ(nullptr, Slp.buildGraph(StoreRoot)); 575 576 // FIXME Need to select better first value for lane0. 577 EXPECT_FALSE(Slp.isCompletelySLP()); 578 } 579 580 TEST_F(VPlanSlpTest, testSlpReorder_4) { 581 LLVMContext Ctx; 582 const char *ModuleString = 583 "%struct.Test = type { i32, i32 }\n" 584 "define void @add_x3(%struct.Test* %A, %struct.Test* %B, %struct.Test* " 585 "%C, %struct.Test* %D, %struct.Test* %E) {\n" 586 "entry:\n" 587 " br label %for.body\n" 588 "for.body: ; preds = %for.body, " 589 "%entry\n" 590 " %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n" 591 " %A0 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 " 592 "%indvars.iv, i32 0\n" 593 " %vA0 = load i32, i32* %A0, align 4\n" 594 " %B0 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 " 595 "%indvars.iv, i32 0\n" 596 " %vB0 = load i32, i32* %B0, align 4\n" 597 " %mul11 = mul nsw i32 %vA0, %vB0\n" 598 " %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 599 "%indvars.iv, i32 0\n" 600 " %vC0 = load i32, i32* %C0, align 4\n" 601 " %D0 = getelementptr inbounds %struct.Test, %struct.Test* %D, i64 " 602 "%indvars.iv, i32 0\n" 603 " %vD0 = load i32, i32* %D0, align 4\n" 604 " %mul12 = mul nsw i32 %vC0, %vD0\n" 605 " %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 " 606 "%indvars.iv, i32 1\n" 607 " %vA1 = load i32, i32* %A1, align 4\n" 608 " %B1 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 " 609 "%indvars.iv, i32 1\n" 610 " %vB1 = load i32, i32* %B1, align 4\n" 611 " %mul21 = mul nsw i32 %vA1, %vB1\n" 612 " %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 613 "%indvars.iv, i32 1\n" 614 " %vC1 = load i32, i32* %C1, align 4\n" 615 " %D1 = getelementptr inbounds %struct.Test, %struct.Test* %D, i64 " 616 "%indvars.iv, i32 1\n" 617 " %vD1 = load i32, i32* %D1, align 4\n" 618 " %mul22 = mul nsw i32 %vC1, %vD1\n" 619 " %add1 = add nsw i32 %mul11, %mul12\n" 620 " %add2 = add nsw i32 %mul22, %mul21\n" 621 " %E0 = getelementptr inbounds %struct.Test, %struct.Test* %E, i64 " 622 "%indvars.iv, i32 0\n" 623 " store i32 %add1, i32* %E0, align 4\n" 624 " %E1 = getelementptr inbounds %struct.Test, %struct.Test* %E, i64 " 625 "%indvars.iv, i32 1\n" 626 " store i32 %add2, i32* %E1, align 4\n" 627 " %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n" 628 " %exitcond = icmp eq i64 %indvars.iv.next, 1024\n" 629 " br i1 %exitcond, label %for.cond.cleanup, label %for.body\n" 630 "for.cond.cleanup: ; preds = %for.body\n" 631 " ret void\n" 632 "}\n"; 633 634 Module &M = parseModule(ModuleString); 635 636 Function *F = M.getFunction("add_x3"); 637 BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor(); 638 auto Plan = buildHCFG(LoopHeader); 639 640 VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock(); 641 EXPECT_NE(nullptr, Entry->getSingleSuccessor()); 642 VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock(); 643 644 VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 24)); 645 VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 26)); 646 647 checkReorderExample( 648 Store1, Store2, Body, 649 getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan)); 650 } 651 652 // Make sure we do not combine instructions with operands in different BBs. 653 TEST_F(VPlanSlpTest, testInstrsInDifferentBBs) { 654 const char *ModuleString = 655 "%struct.Test = type { i32, i32 }\n" 656 "%struct.Test3 = type { i32, i32, i32 }\n" 657 "%struct.Test4xi8 = type { i8, i8, i8 }\n" 658 "define void @add_x2(%struct.Test* nocapture readonly %A, %struct.Test* " 659 "nocapture readonly %B, %struct.Test* nocapture %C) {\n" 660 "entry:\n" 661 " br label %for.body\n" 662 "for.body: ; preds = %for.body, " 663 "%entry\n" 664 " %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n" 665 " %A0 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 " 666 "%indvars.iv, i32 0\n" 667 " %vA0 = load i32, i32* %A0, align 4\n" 668 " %B0 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 " 669 "%indvars.iv, i32 0\n" 670 " %vB0 = load i32, i32* %B0, align 4\n" 671 " %add0 = add nsw i32 %vA0, %vB0\n" 672 " %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 " 673 "%indvars.iv, i32 1\n" 674 " %vA1 = load i32, i32* %A1, align 4\n" 675 " %B1 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 " 676 "%indvars.iv, i32 1\n" 677 " br label %bb2\n" 678 "bb2:\n" 679 " %vB1 = load i32, i32* %B1, align 4\n" 680 " %add1 = add nsw i32 %vA1, %vB1\n" 681 " %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 682 "%indvars.iv, i32 0\n" 683 " store i32 %add0, i32* %C0, align 4\n" 684 " %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 685 "%indvars.iv, i32 1\n" 686 " store i32 %add1, i32* %C1, align 4\n" 687 " %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n" 688 " %exitcond = icmp eq i64 %indvars.iv.next, 1024\n" 689 " br i1 %exitcond, label %for.cond.cleanup, label %for.body\n" 690 "for.cond.cleanup: ; preds = %for.body\n" 691 " ret void\n" 692 "}\n"; 693 694 Module &M = parseModule(ModuleString); 695 696 Function *F = M.getFunction("add_x2"); 697 BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor(); 698 auto Plan = buildHCFG(LoopHeader); 699 auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan); 700 701 VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock(); 702 EXPECT_NE(nullptr, Entry->getSingleSuccessor()); 703 VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock(); 704 VPBasicBlock *BB2 = Body->getSingleSuccessor()->getEntryBasicBlock(); 705 706 VPInstruction *Store1 = cast<VPInstruction>(&*std::next(BB2->begin(), 3)); 707 VPInstruction *Store2 = cast<VPInstruction>(&*std::next(BB2->begin(), 5)); 708 709 VPlanSlp Slp(VPIAI, *BB2); 710 SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2}; 711 EXPECT_EQ(nullptr, Slp.buildGraph(StoreRoot)); 712 EXPECT_EQ(0u, Slp.getWidestBundleBits()); 713 } 714 715 // Make sure we do not combine instructions with operands in different BBs. 716 TEST_F(VPlanSlpTest, testInstrsInDifferentBBs2) { 717 const char *ModuleString = 718 "%struct.Test = type { i32, i32 }\n" 719 "%struct.Test3 = type { i32, i32, i32 }\n" 720 "%struct.Test4xi8 = type { i8, i8, i8 }\n" 721 "define void @add_x2(%struct.Test* nocapture readonly %A, %struct.Test* " 722 "nocapture readonly %B, %struct.Test* nocapture %C) {\n" 723 "entry:\n" 724 " br label %for.body\n" 725 "for.body: ; preds = %for.body, " 726 "%entry\n" 727 " %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n" 728 " %A0 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 " 729 "%indvars.iv, i32 0\n" 730 " %vA0 = load i32, i32* %A0, align 4\n" 731 " %B0 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 " 732 "%indvars.iv, i32 0\n" 733 " %vB0 = load i32, i32* %B0, align 4\n" 734 " %add0 = add nsw i32 %vA0, %vB0\n" 735 " %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 " 736 "%indvars.iv, i32 1\n" 737 " %vA1 = load i32, i32* %A1, align 4\n" 738 " %B1 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 " 739 "%indvars.iv, i32 1\n" 740 " %vB1 = load i32, i32* %B1, align 4\n" 741 " %add1 = add nsw i32 %vA1, %vB1\n" 742 " br label %bb2\n" 743 "bb2:\n" 744 " %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 745 "%indvars.iv, i32 0\n" 746 " store i32 %add0, i32* %C0, align 4\n" 747 " %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 748 "%indvars.iv, i32 1\n" 749 " store i32 %add1, i32* %C1, align 4\n" 750 " %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n" 751 " %exitcond = icmp eq i64 %indvars.iv.next, 1024\n" 752 " br i1 %exitcond, label %for.cond.cleanup, label %for.body\n" 753 "for.cond.cleanup: ; preds = %for.body\n" 754 " ret void\n" 755 "}\n"; 756 757 Module &M = parseModule(ModuleString); 758 759 Function *F = M.getFunction("add_x2"); 760 BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor(); 761 auto Plan = buildHCFG(LoopHeader); 762 auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan); 763 764 VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock(); 765 EXPECT_NE(nullptr, Entry->getSingleSuccessor()); 766 VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock(); 767 VPBasicBlock *BB2 = Body->getSingleSuccessor()->getEntryBasicBlock(); 768 769 VPInstruction *Store1 = cast<VPInstruction>(&*std::next(BB2->begin(), 1)); 770 VPInstruction *Store2 = cast<VPInstruction>(&*std::next(BB2->begin(), 3)); 771 772 VPlanSlp Slp(VPIAI, *BB2); 773 SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2}; 774 EXPECT_EQ(nullptr, Slp.buildGraph(StoreRoot)); 775 EXPECT_EQ(0u, Slp.getWidestBundleBits()); 776 } 777 778 TEST_F(VPlanSlpTest, testSlpAtomicLoad) { 779 const char *ModuleString = 780 "%struct.Test = type { i32, i32 }\n" 781 "%struct.Test3 = type { i32, i32, i32 }\n" 782 "%struct.Test4xi8 = type { i8, i8, i8 }\n" 783 "define void @add_x2(%struct.Test* nocapture readonly %A, %struct.Test* " 784 "nocapture readonly %B, %struct.Test* nocapture %C) {\n" 785 "entry:\n" 786 " br label %for.body\n" 787 "for.body: ; preds = %for.body, " 788 "%entry\n" 789 " %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n" 790 " %A0 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 " 791 "%indvars.iv, i32 0\n" 792 " %vA0 = load atomic i32, i32* %A0 monotonic, align 4\n" 793 " %B0 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 " 794 "%indvars.iv, i32 0\n" 795 " %vB0 = load i32, i32* %B0, align 4\n" 796 " %add0 = add nsw i32 %vA0, %vB0\n" 797 " %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 " 798 "%indvars.iv, i32 1\n" 799 " %vA1 = load i32, i32* %A1, align 4\n" 800 " %B1 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 " 801 "%indvars.iv, i32 1\n" 802 " %vB1 = load i32, i32* %B1, align 4\n" 803 " %add1 = add nsw i32 %vA1, %vB1\n" 804 " %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 805 "%indvars.iv, i32 0\n" 806 " store i32 %add0, i32* %C0, align 4\n" 807 " %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 808 "%indvars.iv, i32 1\n" 809 " store i32 %add1, i32* %C1, align 4\n" 810 " %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n" 811 " %exitcond = icmp eq i64 %indvars.iv.next, 1024\n" 812 " br i1 %exitcond, label %for.cond.cleanup, label %for.body\n" 813 "for.cond.cleanup: ; preds = %for.body\n" 814 " ret void\n" 815 "}\n"; 816 817 Module &M = parseModule(ModuleString); 818 819 Function *F = M.getFunction("add_x2"); 820 BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor(); 821 auto Plan = buildHCFG(LoopHeader); 822 auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan); 823 824 VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock(); 825 EXPECT_NE(nullptr, Entry->getSingleSuccessor()); 826 VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock(); 827 828 VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 12)); 829 VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 14)); 830 831 VPlanSlp Slp(VPIAI, *Body); 832 SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2}; 833 EXPECT_EQ(nullptr, Slp.buildGraph(StoreRoot)); 834 EXPECT_FALSE(Slp.isCompletelySLP()); 835 } 836 837 TEST_F(VPlanSlpTest, testSlpAtomicStore) { 838 const char *ModuleString = 839 "%struct.Test = type { i32, i32 }\n" 840 "%struct.Test3 = type { i32, i32, i32 }\n" 841 "%struct.Test4xi8 = type { i8, i8, i8 }\n" 842 "define void @add_x2(%struct.Test* nocapture readonly %A, %struct.Test* " 843 "nocapture readonly %B, %struct.Test* nocapture %C) {\n" 844 "entry:\n" 845 " br label %for.body\n" 846 "for.body: ; preds = %for.body, " 847 "%entry\n" 848 " %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n" 849 " %A0 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 " 850 "%indvars.iv, i32 0\n" 851 " %vA0 = load i32, i32* %A0, align 4\n" 852 " %B0 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 " 853 "%indvars.iv, i32 0\n" 854 " %vB0 = load i32, i32* %B0, align 4\n" 855 " %add0 = add nsw i32 %vA0, %vB0\n" 856 " %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 " 857 "%indvars.iv, i32 1\n" 858 " %vA1 = load i32, i32* %A1, align 4\n" 859 " %B1 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 " 860 "%indvars.iv, i32 1\n" 861 " %vB1 = load i32, i32* %B1, align 4\n" 862 " %add1 = add nsw i32 %vA1, %vB1\n" 863 " %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 864 "%indvars.iv, i32 0\n" 865 " store atomic i32 %add0, i32* %C0 monotonic, align 4\n" 866 " %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 " 867 "%indvars.iv, i32 1\n" 868 " store i32 %add1, i32* %C1, align 4\n" 869 " %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n" 870 " %exitcond = icmp eq i64 %indvars.iv.next, 1024\n" 871 " br i1 %exitcond, label %for.cond.cleanup, label %for.body\n" 872 "for.cond.cleanup: ; preds = %for.body\n" 873 " ret void\n" 874 "}\n"; 875 876 Module &M = parseModule(ModuleString); 877 878 Function *F = M.getFunction("add_x2"); 879 BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor(); 880 auto Plan = buildHCFG(LoopHeader); 881 auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan); 882 883 VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock(); 884 EXPECT_NE(nullptr, Entry->getSingleSuccessor()); 885 VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock(); 886 887 VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 12)); 888 VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 14)); 889 890 VPlanSlp Slp(VPIAI, *Body); 891 SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2}; 892 Slp.buildGraph(StoreRoot); 893 EXPECT_FALSE(Slp.isCompletelySLP()); 894 } 895 896 } // namespace 897 } // namespace llvm 898