1 //===- llvm/unittest/Transforms/Vectorize/VPlanSlpTest.cpp ---------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "../lib/Transforms/Vectorize/VPlan.h" 10 #include "../lib/Transforms/Vectorize/VPlanHCFGBuilder.h" 11 #include "VPlanTestBase.h" 12 #include "llvm/Analysis/TargetLibraryInfo.h" 13 #include "llvm/Analysis/VectorUtils.h" 14 #include "gtest/gtest.h" 15 16 namespace llvm { 17 namespace { 18 19 class VPlanSlpTest : public VPlanTestIRBase { 20 protected: 21 TargetLibraryInfoImpl TLII; 22 TargetLibraryInfo TLI; 23 DataLayout DL; 24 25 std::unique_ptr<AssumptionCache> AC; 26 std::unique_ptr<ScalarEvolution> SE; 27 std::unique_ptr<AAResults> AARes; 28 std::unique_ptr<BasicAAResult> BasicAA; 29 std::unique_ptr<LoopAccessInfo> LAI; 30 std::unique_ptr<PredicatedScalarEvolution> PSE; 31 std::unique_ptr<InterleavedAccessInfo> IAI; 32 33 VPlanSlpTest() 34 : TLII(), TLI(TLII), 35 DL("e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-" 36 "f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:" 37 "16:32:64-S128") {} 38 39 VPInterleavedAccessInfo getInterleavedAccessInfo(Function &F, Loop *L, 40 VPlan &Plan) { 41 AC.reset(new AssumptionCache(F)); 42 SE.reset(new ScalarEvolution(F, TLI, *AC, *DT, *LI)); 43 BasicAA.reset(new BasicAAResult(DL, F, TLI, *AC, &*DT)); 44 AARes.reset(new AAResults(TLI)); 45 AARes->addAAResult(*BasicAA); 46 PSE.reset(new PredicatedScalarEvolution(*SE, *L)); 47 LAI.reset(new LoopAccessInfo(L, &*SE, nullptr, &TLI, &*AARes, &*DT, &*LI)); 48 IAI.reset(new InterleavedAccessInfo(*PSE, L, &*DT, &*LI, &*LAI)); 49 IAI->analyzeInterleaving(false); 50 return {Plan, *IAI}; 51 } 52 }; 53 54 TEST_F(VPlanSlpTest, testSlpSimple_2) { 55 const char *ModuleString = 56 "%struct.Test = type { i32, i32 }\n" 57 "%struct.Test3 = type { i32, i32, i32 }\n" 58 "%struct.Test4xi8 = type { i8, i8, i8 }\n" 59 "define void @add_x2(ptr nocapture readonly %A, ptr " 60 "nocapture readonly %B, ptr nocapture %C) {\n" 61 "entry:\n" 62 " br label %for.body\n" 63 "for.body: ; preds = %for.body, " 64 "%entry\n" 65 " %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n" 66 " %A0 = getelementptr inbounds %struct.Test, ptr %A, i64 " 67 "%indvars.iv, i32 0\n" 68 " %vA0 = load i32, ptr %A0, align 4\n" 69 " %B0 = getelementptr inbounds %struct.Test, ptr %B, i64 " 70 "%indvars.iv, i32 0\n" 71 " %vB0 = load i32, ptr %B0, align 4\n" 72 " %add0 = add nsw i32 %vA0, %vB0\n" 73 " %A1 = getelementptr inbounds %struct.Test, ptr %A, i64 " 74 "%indvars.iv, i32 1\n" 75 " %vA1 = load i32, ptr %A1, align 4\n" 76 " %B1 = getelementptr inbounds %struct.Test, ptr %B, i64 " 77 "%indvars.iv, i32 1\n" 78 " %vB1 = load i32, ptr %B1, align 4\n" 79 " %add1 = add nsw i32 %vA1, %vB1\n" 80 " %C0 = getelementptr inbounds %struct.Test, ptr %C, i64 " 81 "%indvars.iv, i32 0\n" 82 " store i32 %add0, ptr %C0, align 4\n" 83 " %C1 = getelementptr inbounds %struct.Test, ptr %C, i64 " 84 "%indvars.iv, i32 1\n" 85 " store i32 %add1, ptr %C1, align 4\n" 86 " %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n" 87 " %exitcond = icmp eq i64 %indvars.iv.next, 1024\n" 88 " br i1 %exitcond, label %for.cond.cleanup, label %for.body\n" 89 "for.cond.cleanup: ; preds = %for.body\n" 90 " ret void\n" 91 "}\n"; 92 93 Module &M = parseModule(ModuleString); 94 95 Function *F = M.getFunction("add_x2"); 96 BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor(); 97 auto Plan = buildHCFG(LoopHeader); 98 auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan); 99 100 VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock(); 101 EXPECT_NE(nullptr, Entry->getSingleSuccessor()); 102 VPBasicBlock *Body = Plan->getVectorLoopRegion()->getEntryBasicBlock(); 103 104 VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 12)); 105 VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 14)); 106 107 VPlanSlp Slp(VPIAI, *Body); 108 SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2}; 109 VPInstruction *CombinedStore = Slp.buildGraph(StoreRoot); 110 EXPECT_EQ(64u, Slp.getWidestBundleBits()); 111 EXPECT_EQ(VPInstruction::SLPStore, CombinedStore->getOpcode()); 112 113 auto *CombinedAdd = cast<VPInstruction>(CombinedStore->getOperand(0)); 114 EXPECT_EQ(Instruction::Add, CombinedAdd->getOpcode()); 115 116 auto *CombinedLoadA = cast<VPInstruction>(CombinedAdd->getOperand(0)); 117 auto *CombinedLoadB = cast<VPInstruction>(CombinedAdd->getOperand(1)); 118 EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadA->getOpcode()); 119 EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadB->getOpcode()); 120 121 delete CombinedStore; 122 delete CombinedAdd; 123 delete CombinedLoadA; 124 delete CombinedLoadB; 125 } 126 127 TEST_F(VPlanSlpTest, testSlpSimple_3) { 128 const char *ModuleString = 129 "%struct.Test = type { i32, i32 }\n" 130 "%struct.Test3 = type { i32, i32, i32 }\n" 131 "%struct.Test4xi8 = type { i8, i8, i8 }\n" 132 "define void @add_x2(ptr nocapture readonly %A, ptr " 133 "nocapture readonly %B, ptr nocapture %C) {\n" 134 "entry:\n" 135 " br label %for.body\n" 136 "for.body: ; preds = %for.body, " 137 "%entry\n" 138 " %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n" 139 " %A0 = getelementptr %struct.Test, ptr %A, i64 " 140 " %indvars.iv, i32 0\n" 141 " %vA0 = load i32, ptr %A0, align 4\n" 142 " %B0 = getelementptr inbounds %struct.Test, ptr %B, i64 " 143 " %indvars.iv, i32 0\n" 144 " %vB0 = load i32, ptr %B0, align 4\n" 145 " %add0 = add nsw i32 %vA0, %vB0\n" 146 " %A1 = getelementptr inbounds %struct.Test, ptr %A, i64 " 147 " %indvars.iv, i32 1\n" 148 " %vA1 = load i32, ptr %A1, align 4\n" 149 " %B1 = getelementptr inbounds %struct.Test, ptr %B, i64 " 150 " %indvars.iv, i32 1\n" 151 " %vB1 = load i32, ptr %B1, align 4\n" 152 " %add1 = add nsw i32 %vA1, %vB1\n" 153 " %C0 = getelementptr inbounds %struct.Test, ptr %C, i64 " 154 " %indvars.iv, i32 0\n" 155 " store i32 %add0, ptr %C0, align 4\n" 156 " %C1 = getelementptr inbounds %struct.Test, ptr %C, i64 " 157 " %indvars.iv, i32 1\n" 158 " store i32 %add1, ptr %C1, align 4\n" 159 " %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n" 160 " %exitcond = icmp eq i64 %indvars.iv.next, 1024\n" 161 " br i1 %exitcond, label %for.cond.cleanup, label %for.body\n" 162 "for.cond.cleanup: ; preds = %for.body\n" 163 " ret void\n" 164 "}\n"; 165 166 Module &M = parseModule(ModuleString); 167 168 Function *F = M.getFunction("add_x2"); 169 BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor(); 170 auto Plan = buildHCFG(LoopHeader); 171 172 VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock(); 173 EXPECT_NE(nullptr, Entry->getSingleSuccessor()); 174 VPBasicBlock *Body = Plan->getVectorLoopRegion()->getEntryBasicBlock(); 175 176 VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 12)); 177 VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 14)); 178 179 auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan); 180 181 VPlanSlp Slp(VPIAI, *Body); 182 SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2}; 183 VPInstruction *CombinedStore = Slp.buildGraph(StoreRoot); 184 EXPECT_EQ(64u, Slp.getWidestBundleBits()); 185 EXPECT_EQ(VPInstruction::SLPStore, CombinedStore->getOpcode()); 186 187 auto *CombinedAdd = cast<VPInstruction>(CombinedStore->getOperand(0)); 188 EXPECT_EQ(Instruction::Add, CombinedAdd->getOpcode()); 189 190 auto *CombinedLoadA = cast<VPInstruction>(CombinedAdd->getOperand(0)); 191 auto *CombinedLoadB = cast<VPInstruction>(CombinedAdd->getOperand(1)); 192 EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadA->getOpcode()); 193 EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadB->getOpcode()); 194 195 VPInstruction *GetA = cast<VPInstruction>(&*std::next(Body->begin(), 1)); 196 VPInstruction *GetB = cast<VPInstruction>(&*std::next(Body->begin(), 3)); 197 EXPECT_EQ(GetA, CombinedLoadA->getOperand(0)); 198 EXPECT_EQ(GetB, CombinedLoadB->getOperand(0)); 199 200 delete CombinedStore; 201 delete CombinedAdd; 202 delete CombinedLoadA; 203 delete CombinedLoadB; 204 } 205 206 TEST_F(VPlanSlpTest, testSlpReuse_1) { 207 const char *ModuleString = 208 "%struct.Test = type { i32, i32 }\n" 209 "define void @add_x2(ptr nocapture readonly %A, ptr " 210 "nocapture readonly %B, ptr nocapture %C) {\n" 211 "entry:\n" 212 " br label %for.body\n" 213 "for.body: ; preds = %for.body, " 214 "%entry\n" 215 " %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n" 216 " %A0 = getelementptr inbounds %struct.Test, ptr %A, i64 " 217 "%indvars.iv, i32 0\n" 218 " %vA0 = load i32, ptr %A0, align 4\n" 219 " %add0 = add nsw i32 %vA0, %vA0\n" 220 " %A1 = getelementptr inbounds %struct.Test, ptr %A, i64 " 221 "%indvars.iv, i32 1\n" 222 " %vA1 = load i32, ptr %A1, align 4\n" 223 " %add1 = add nsw i32 %vA1, %vA1\n" 224 " %C0 = getelementptr inbounds %struct.Test, ptr %C, i64 " 225 "%indvars.iv, i32 0\n" 226 " store i32 %add0, ptr %C0, align 4\n" 227 " %C1 = getelementptr inbounds %struct.Test, ptr %C, i64 " 228 "%indvars.iv, i32 1\n" 229 " store i32 %add1, ptr %C1, align 4\n" 230 " %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n" 231 " %exitcond = icmp eq i64 %indvars.iv.next, 1024\n" 232 " br i1 %exitcond, label %for.cond.cleanup, label %for.body\n" 233 "for.cond.cleanup: ; preds = %for.body\n" 234 " ret void\n" 235 "}\n"; 236 237 Module &M = parseModule(ModuleString); 238 239 Function *F = M.getFunction("add_x2"); 240 BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor(); 241 auto Plan = buildHCFG(LoopHeader); 242 auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan); 243 244 VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock(); 245 EXPECT_NE(nullptr, Entry->getSingleSuccessor()); 246 VPBasicBlock *Body = Plan->getVectorLoopRegion()->getEntryBasicBlock(); 247 248 VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 8)); 249 VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 10)); 250 251 VPlanSlp Slp(VPIAI, *Body); 252 SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2}; 253 VPInstruction *CombinedStore = Slp.buildGraph(StoreRoot); 254 EXPECT_EQ(64u, Slp.getWidestBundleBits()); 255 EXPECT_EQ(VPInstruction::SLPStore, CombinedStore->getOpcode()); 256 257 auto *CombinedAdd = cast<VPInstruction>(CombinedStore->getOperand(0)); 258 EXPECT_EQ(Instruction::Add, CombinedAdd->getOpcode()); 259 260 auto *CombinedLoadA = cast<VPInstruction>(CombinedAdd->getOperand(0)); 261 EXPECT_EQ(CombinedLoadA, CombinedAdd->getOperand(1)); 262 EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadA->getOpcode()); 263 264 delete CombinedStore; 265 delete CombinedAdd; 266 delete CombinedLoadA; 267 } 268 269 TEST_F(VPlanSlpTest, testSlpReuse_2) { 270 const char *ModuleString = 271 "%struct.Test = type { i32, i32 }\n" 272 "define void @add_x2(ptr nocapture readonly %A, ptr " 273 "nocapture readonly %B, ptr nocapture %C) {\n" 274 "entry:\n" 275 " br label %for.body\n" 276 "for.body: ; preds = %for.body, " 277 "%entry\n" 278 " %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n" 279 " %A0 = getelementptr inbounds %struct.Test, ptr %A, i64 " 280 "%indvars.iv, i32 0\n" 281 " %vA0 = load i32, ptr %A0, align 4\n" 282 " %add0 = add nsw i32 %vA0, %vA0\n" 283 " %C0 = getelementptr inbounds %struct.Test, ptr %C, i64 " 284 "%indvars.iv, i32 0\n" 285 " store i32 %add0, ptr %C0, align 4\n" 286 " %A1 = getelementptr inbounds %struct.Test, ptr %A, i64 " 287 "%indvars.iv, i32 1\n" 288 " %vA1 = load i32, ptr %A1, align 4\n" 289 " %add1 = add nsw i32 %vA1, %vA1\n" 290 " %C1 = getelementptr inbounds %struct.Test, ptr %C, i64 " 291 "%indvars.iv, i32 1\n" 292 " store i32 %add1, ptr %C1, align 4\n" 293 " %use = add i32 %vA1, 1\n" 294 " %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n" 295 " %exitcond = icmp eq i64 %indvars.iv.next, 1024\n" 296 " br i1 %exitcond, label %for.cond.cleanup, label %for.body\n" 297 "for.cond.cleanup: ; preds = %for.body\n" 298 " ret void\n" 299 "}\n"; 300 301 Module &M = parseModule(ModuleString); 302 303 Function *F = M.getFunction("add_x2"); 304 BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor(); 305 auto Plan = buildHCFG(LoopHeader); 306 auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan); 307 308 VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock(); 309 EXPECT_NE(nullptr, Entry->getSingleSuccessor()); 310 VPBasicBlock *Body = Plan->getVectorLoopRegion()->getEntryBasicBlock(); 311 312 VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 5)); 313 VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 10)); 314 315 VPlanSlp Slp(VPIAI, *Body); 316 SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2}; 317 Slp.buildGraph(StoreRoot); 318 EXPECT_FALSE(Slp.isCompletelySLP()); 319 } 320 321 static void checkReorderExample(VPInstruction *Store1, VPInstruction *Store2, 322 VPBasicBlock *Body, 323 VPInterleavedAccessInfo &&IAI) { 324 VPlanSlp Slp(IAI, *Body); 325 SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2}; 326 VPInstruction *CombinedStore = Slp.buildGraph(StoreRoot); 327 328 EXPECT_TRUE(Slp.isCompletelySLP()); 329 EXPECT_EQ(CombinedStore->getOpcode(), VPInstruction::SLPStore); 330 331 VPInstruction *CombinedAdd = 332 cast<VPInstruction>(CombinedStore->getOperand(0)); 333 EXPECT_EQ(CombinedAdd->getOpcode(), Instruction::Add); 334 335 VPInstruction *CombinedMulAB = 336 cast<VPInstruction>(CombinedAdd->getOperand(0)); 337 VPInstruction *CombinedMulCD = 338 cast<VPInstruction>(CombinedAdd->getOperand(1)); 339 EXPECT_EQ(CombinedMulAB->getOpcode(), Instruction::Mul); 340 341 VPInstruction *CombinedLoadA = 342 cast<VPInstruction>(CombinedMulAB->getOperand(0)); 343 EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadA->getOpcode()); 344 VPInstruction *LoadvA0 = cast<VPInstruction>(&*std::next(Body->begin(), 2)); 345 VPInstruction *LoadvA1 = cast<VPInstruction>(&*std::next(Body->begin(), 12)); 346 EXPECT_EQ(LoadvA0->getOperand(0), CombinedLoadA->getOperand(0)); 347 EXPECT_EQ(LoadvA1->getOperand(0), CombinedLoadA->getOperand(1)); 348 349 VPInstruction *CombinedLoadB = 350 cast<VPInstruction>(CombinedMulAB->getOperand(1)); 351 EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadB->getOpcode()); 352 VPInstruction *LoadvB0 = cast<VPInstruction>(&*std::next(Body->begin(), 4)); 353 VPInstruction *LoadvB1 = cast<VPInstruction>(&*std::next(Body->begin(), 14)); 354 EXPECT_EQ(LoadvB0->getOperand(0), CombinedLoadB->getOperand(0)); 355 EXPECT_EQ(LoadvB1->getOperand(0), CombinedLoadB->getOperand(1)); 356 357 EXPECT_EQ(CombinedMulCD->getOpcode(), Instruction::Mul); 358 359 VPInstruction *CombinedLoadC = 360 cast<VPInstruction>(CombinedMulCD->getOperand(0)); 361 EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadC->getOpcode()); 362 VPInstruction *LoadvC0 = cast<VPInstruction>(&*std::next(Body->begin(), 7)); 363 VPInstruction *LoadvC1 = cast<VPInstruction>(&*std::next(Body->begin(), 17)); 364 EXPECT_EQ(LoadvC0->getOperand(0), CombinedLoadC->getOperand(0)); 365 EXPECT_EQ(LoadvC1->getOperand(0), CombinedLoadC->getOperand(1)); 366 367 VPInstruction *CombinedLoadD = 368 cast<VPInstruction>(CombinedMulCD->getOperand(1)); 369 EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadD->getOpcode()); 370 VPInstruction *LoadvD0 = cast<VPInstruction>(&*std::next(Body->begin(), 9)); 371 VPInstruction *LoadvD1 = cast<VPInstruction>(&*std::next(Body->begin(), 19)); 372 EXPECT_EQ(LoadvD0->getOperand(0), CombinedLoadD->getOperand(0)); 373 EXPECT_EQ(LoadvD1->getOperand(0), CombinedLoadD->getOperand(1)); 374 375 delete CombinedStore; 376 delete CombinedAdd; 377 delete CombinedMulAB; 378 delete CombinedMulCD; 379 delete CombinedLoadA; 380 delete CombinedLoadB; 381 delete CombinedLoadC; 382 delete CombinedLoadD; 383 } 384 385 TEST_F(VPlanSlpTest, testSlpReorder_1) { 386 LLVMContext Ctx; 387 const char *ModuleString = 388 "%struct.Test = type { i32, i32 }\n" 389 "define void @add_x3(ptr %A, ptr %B, ptr " 390 "%C, ptr %D, ptr %E) {\n" 391 "entry:\n" 392 " br label %for.body\n" 393 "for.body: ; preds = %for.body, " 394 "%entry\n" 395 " %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n" 396 " %A0 = getelementptr inbounds %struct.Test, ptr %A, i64 " 397 "%indvars.iv, i32 0\n" 398 " %vA0 = load i32, ptr %A0, align 4\n" 399 " %B0 = getelementptr inbounds %struct.Test, ptr %B, i64 " 400 "%indvars.iv, i32 0\n" 401 " %vB0 = load i32, ptr %B0, align 4\n" 402 " %mul11 = mul nsw i32 %vA0, %vB0\n" 403 " %C0 = getelementptr inbounds %struct.Test, ptr %C, i64 " 404 "%indvars.iv, i32 0\n" 405 " %vC0 = load i32, ptr %C0, align 4\n" 406 " %D0 = getelementptr inbounds %struct.Test, ptr %D, i64 " 407 "%indvars.iv, i32 0\n" 408 " %vD0 = load i32, ptr %D0, align 4\n" 409 " %mul12 = mul nsw i32 %vC0, %vD0\n" 410 " %A1 = getelementptr inbounds %struct.Test, ptr %A, i64 " 411 "%indvars.iv, i32 1\n" 412 " %vA1 = load i32, ptr %A1, align 4\n" 413 " %B1 = getelementptr inbounds %struct.Test, ptr %B, i64 " 414 "%indvars.iv, i32 1\n" 415 " %vB1 = load i32, ptr %B1, align 4\n" 416 " %mul21 = mul nsw i32 %vA1, %vB1\n" 417 " %C1 = getelementptr inbounds %struct.Test, ptr %C, i64 " 418 "%indvars.iv, i32 1\n" 419 " %vC1 = load i32, ptr %C1, align 4\n" 420 " %D1 = getelementptr inbounds %struct.Test, ptr %D, i64 " 421 "%indvars.iv, i32 1\n" 422 " %vD1 = load i32, ptr %D1, align 4\n" 423 " %mul22 = mul nsw i32 %vC1, %vD1\n" 424 " %add1 = add nsw i32 %mul11, %mul12\n" 425 " %add2 = add nsw i32 %mul22, %mul21\n" 426 " %E0 = getelementptr inbounds %struct.Test, ptr %E, i64 " 427 "%indvars.iv, i32 0\n" 428 " store i32 %add1, ptr %E0, align 4\n" 429 " %E1 = getelementptr inbounds %struct.Test, ptr %E, i64 " 430 "%indvars.iv, i32 1\n" 431 " store i32 %add2, ptr %E1, align 4\n" 432 " %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n" 433 " %exitcond = icmp eq i64 %indvars.iv.next, 1024\n" 434 " br i1 %exitcond, label %for.cond.cleanup, label %for.body\n" 435 "for.cond.cleanup: ; preds = %for.body\n" 436 " ret void\n" 437 "}\n"; 438 439 Module &M = parseModule(ModuleString); 440 441 Function *F = M.getFunction("add_x3"); 442 BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor(); 443 auto Plan = buildHCFG(LoopHeader); 444 445 VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock(); 446 EXPECT_NE(nullptr, Entry->getSingleSuccessor()); 447 VPBasicBlock *Body = Plan->getVectorLoopRegion()->getEntryBasicBlock(); 448 449 VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 24)); 450 VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 26)); 451 452 checkReorderExample( 453 Store1, Store2, Body, 454 getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan)); 455 } 456 457 TEST_F(VPlanSlpTest, testSlpReorder_2) { 458 LLVMContext Ctx; 459 const char *ModuleString = 460 "%struct.Test = type { i32, i32 }\n" 461 "define void @add_x3(ptr %A, ptr %B, ptr " 462 "%C, ptr %D, ptr %E) {\n" 463 "entry:\n" 464 " br label %for.body\n" 465 "for.body: ; preds = %for.body, " 466 "%entry\n" 467 " %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n" 468 " %A0 = getelementptr inbounds %struct.Test, ptr %A, i64 " 469 "%indvars.iv, i32 0\n" 470 " %vA0 = load i32, ptr %A0, align 4\n" 471 " %B0 = getelementptr inbounds %struct.Test, ptr %B, i64 " 472 "%indvars.iv, i32 0\n" 473 " %vB0 = load i32, ptr %B0, align 4\n" 474 " %mul11 = mul nsw i32 %vA0, %vB0\n" 475 " %C0 = getelementptr inbounds %struct.Test, ptr %C, i64 " 476 "%indvars.iv, i32 0\n" 477 " %vC0 = load i32, ptr %C0, align 4\n" 478 " %D0 = getelementptr inbounds %struct.Test, ptr %D, i64 " 479 "%indvars.iv, i32 0\n" 480 " %vD0 = load i32, ptr %D0, align 4\n" 481 " %mul12 = mul nsw i32 %vC0, %vD0\n" 482 " %A1 = getelementptr inbounds %struct.Test, ptr %A, i64 " 483 "%indvars.iv, i32 1\n" 484 " %vA1 = load i32, ptr %A1, align 4\n" 485 " %B1 = getelementptr inbounds %struct.Test, ptr %B, i64 " 486 "%indvars.iv, i32 1\n" 487 " %vB1 = load i32, ptr %B1, align 4\n" 488 " %mul21 = mul nsw i32 %vB1, %vA1\n" 489 " %C1 = getelementptr inbounds %struct.Test, ptr %C, i64 " 490 "%indvars.iv, i32 1\n" 491 " %vC1 = load i32, ptr %C1, align 4\n" 492 " %D1 = getelementptr inbounds %struct.Test, ptr %D, i64 " 493 "%indvars.iv, i32 1\n" 494 " %vD1 = load i32, ptr %D1, align 4\n" 495 " %mul22 = mul nsw i32 %vD1, %vC1\n" 496 " %add1 = add nsw i32 %mul11, %mul12\n" 497 " %add2 = add nsw i32 %mul22, %mul21\n" 498 " %E0 = getelementptr inbounds %struct.Test, ptr %E, i64 " 499 "%indvars.iv, i32 0\n" 500 " store i32 %add1, ptr %E0, align 4\n" 501 " %E1 = getelementptr inbounds %struct.Test, ptr %E, i64 " 502 "%indvars.iv, i32 1\n" 503 " store i32 %add2, ptr %E1, align 4\n" 504 " %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n" 505 " %exitcond = icmp eq i64 %indvars.iv.next, 1024\n" 506 " br i1 %exitcond, label %for.cond.cleanup, label %for.body\n" 507 "for.cond.cleanup: ; preds = %for.body\n" 508 " ret void\n" 509 "}\n"; 510 511 Module &M = parseModule(ModuleString); 512 513 Function *F = M.getFunction("add_x3"); 514 BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor(); 515 auto Plan = buildHCFG(LoopHeader); 516 517 VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock(); 518 EXPECT_NE(nullptr, Entry->getSingleSuccessor()); 519 VPBasicBlock *Body = Plan->getVectorLoopRegion()->getEntryBasicBlock(); 520 521 VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 24)); 522 VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 26)); 523 524 checkReorderExample( 525 Store1, Store2, Body, 526 getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan)); 527 } 528 529 TEST_F(VPlanSlpTest, testSlpReorder_3) { 530 LLVMContext Ctx; 531 const char *ModuleString = 532 "%struct.Test = type { i32, i32 }\n" 533 "define void @add_x3(ptr %A, ptr %B, ptr " 534 "%C, ptr %D, ptr %E) {\n" 535 "entry:\n" 536 " br label %for.body\n" 537 "for.body: ; preds = %for.body, " 538 "%entry\n" 539 " %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n" 540 " %A1 = getelementptr inbounds %struct.Test, ptr %A, i64 " 541 "%indvars.iv, i32 1\n" 542 " %vA1 = load i32, ptr %A1, align 4\n" 543 " %B0 = getelementptr inbounds %struct.Test, ptr %B, i64 " 544 "%indvars.iv, i32 0\n" 545 " %vB0 = load i32, ptr %B0, align 4\n" 546 " %mul11 = mul nsw i32 %vA1, %vB0\n" 547 " %C0 = getelementptr inbounds %struct.Test, ptr %C, i64 " 548 "%indvars.iv, i32 0\n" 549 " %vC0 = load i32, ptr %C0, align 4\n" 550 " %D0 = getelementptr inbounds %struct.Test, ptr %D, i64 " 551 "%indvars.iv, i32 0\n" 552 " %vD0 = load i32, ptr %D0, align 4\n" 553 " %mul12 = mul nsw i32 %vC0, %vD0\n" 554 " %A0 = getelementptr inbounds %struct.Test, ptr %A, i64 " 555 "%indvars.iv, i32 0\n" 556 " %vA0 = load i32, ptr %A0, align 4\n" 557 " %B1 = getelementptr inbounds %struct.Test, ptr %B, i64 " 558 "%indvars.iv, i32 1\n" 559 " %vB1 = load i32, ptr %B1, align 4\n" 560 " %mul21 = mul nsw i32 %vB1, %vA0\n" 561 " %C1 = getelementptr inbounds %struct.Test, ptr %C, i64 " 562 "%indvars.iv, i32 1\n" 563 " %vC1 = load i32, ptr %C1, align 4\n" 564 " %D1 = getelementptr inbounds %struct.Test, ptr %D, i64 " 565 "%indvars.iv, i32 1\n" 566 " %vD1 = load i32, ptr %D1, align 4\n" 567 " %mul22 = mul nsw i32 %vD1, %vC1\n" 568 " %add1 = add nsw i32 %mul11, %mul12\n" 569 " %add2 = add nsw i32 %mul22, %mul21\n" 570 " %E0 = getelementptr inbounds %struct.Test, ptr %E, i64 " 571 "%indvars.iv, i32 0\n" 572 " store i32 %add1, ptr %E0, align 4\n" 573 " %E1 = getelementptr inbounds %struct.Test, ptr %E, i64 " 574 "%indvars.iv, i32 1\n" 575 " store i32 %add2, ptr %E1, align 4\n" 576 " %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n" 577 " %exitcond = icmp eq i64 %indvars.iv.next, 1024\n" 578 " br i1 %exitcond, label %for.cond.cleanup, label %for.body\n" 579 "for.cond.cleanup: ; preds = %for.body\n" 580 " ret void\n" 581 "}\n"; 582 583 Module &M = parseModule(ModuleString); 584 585 Function *F = M.getFunction("add_x3"); 586 BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor(); 587 auto Plan = buildHCFG(LoopHeader); 588 589 VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock(); 590 EXPECT_NE(nullptr, Entry->getSingleSuccessor()); 591 VPBasicBlock *Body = Plan->getVectorLoopRegion()->getEntryBasicBlock(); 592 593 VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 24)); 594 VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 26)); 595 596 auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan); 597 VPlanSlp Slp(VPIAI, *Body); 598 SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2}; 599 EXPECT_EQ(nullptr, Slp.buildGraph(StoreRoot)); 600 601 // FIXME Need to select better first value for lane0. 602 EXPECT_FALSE(Slp.isCompletelySLP()); 603 } 604 605 TEST_F(VPlanSlpTest, testSlpReorder_4) { 606 LLVMContext Ctx; 607 const char *ModuleString = 608 "%struct.Test = type { i32, i32 }\n" 609 "define void @add_x3(ptr %A, ptr %B, ptr " 610 "%C, ptr %D, ptr %E) {\n" 611 "entry:\n" 612 " br label %for.body\n" 613 "for.body: ; preds = %for.body, " 614 "%entry\n" 615 " %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n" 616 " %A0 = getelementptr inbounds %struct.Test, ptr %A, i64 " 617 "%indvars.iv, i32 0\n" 618 " %vA0 = load i32, ptr %A0, align 4\n" 619 " %B0 = getelementptr inbounds %struct.Test, ptr %B, i64 " 620 "%indvars.iv, i32 0\n" 621 " %vB0 = load i32, ptr %B0, align 4\n" 622 " %mul11 = mul nsw i32 %vA0, %vB0\n" 623 " %C0 = getelementptr inbounds %struct.Test, ptr %C, i64 " 624 "%indvars.iv, i32 0\n" 625 " %vC0 = load i32, ptr %C0, align 4\n" 626 " %D0 = getelementptr inbounds %struct.Test, ptr %D, i64 " 627 "%indvars.iv, i32 0\n" 628 " %vD0 = load i32, ptr %D0, align 4\n" 629 " %mul12 = mul nsw i32 %vC0, %vD0\n" 630 " %A1 = getelementptr inbounds %struct.Test, ptr %A, i64 " 631 "%indvars.iv, i32 1\n" 632 " %vA1 = load i32, ptr %A1, align 4\n" 633 " %B1 = getelementptr inbounds %struct.Test, ptr %B, i64 " 634 "%indvars.iv, i32 1\n" 635 " %vB1 = load i32, ptr %B1, align 4\n" 636 " %mul21 = mul nsw i32 %vA1, %vB1\n" 637 " %C1 = getelementptr inbounds %struct.Test, ptr %C, i64 " 638 "%indvars.iv, i32 1\n" 639 " %vC1 = load i32, ptr %C1, align 4\n" 640 " %D1 = getelementptr inbounds %struct.Test, ptr %D, i64 " 641 "%indvars.iv, i32 1\n" 642 " %vD1 = load i32, ptr %D1, align 4\n" 643 " %mul22 = mul nsw i32 %vC1, %vD1\n" 644 " %add1 = add nsw i32 %mul11, %mul12\n" 645 " %add2 = add nsw i32 %mul22, %mul21\n" 646 " %E0 = getelementptr inbounds %struct.Test, ptr %E, i64 " 647 "%indvars.iv, i32 0\n" 648 " store i32 %add1, ptr %E0, align 4\n" 649 " %E1 = getelementptr inbounds %struct.Test, ptr %E, i64 " 650 "%indvars.iv, i32 1\n" 651 " store i32 %add2, ptr %E1, align 4\n" 652 " %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n" 653 " %exitcond = icmp eq i64 %indvars.iv.next, 1024\n" 654 " br i1 %exitcond, label %for.cond.cleanup, label %for.body\n" 655 "for.cond.cleanup: ; preds = %for.body\n" 656 " ret void\n" 657 "}\n"; 658 659 Module &M = parseModule(ModuleString); 660 661 Function *F = M.getFunction("add_x3"); 662 BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor(); 663 auto Plan = buildHCFG(LoopHeader); 664 665 VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock(); 666 EXPECT_NE(nullptr, Entry->getSingleSuccessor()); 667 VPBasicBlock *Body = Plan->getVectorLoopRegion()->getEntryBasicBlock(); 668 669 VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 24)); 670 VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 26)); 671 672 checkReorderExample( 673 Store1, Store2, Body, 674 getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan)); 675 } 676 677 // Make sure we do not combine instructions with operands in different BBs. 678 TEST_F(VPlanSlpTest, testInstrsInDifferentBBs) { 679 const char *ModuleString = 680 "%struct.Test = type { i32, i32 }\n" 681 "%struct.Test3 = type { i32, i32, i32 }\n" 682 "%struct.Test4xi8 = type { i8, i8, i8 }\n" 683 "define void @add_x2(ptr nocapture readonly %A, ptr " 684 "nocapture readonly %B, ptr nocapture %C) {\n" 685 "entry:\n" 686 " br label %for.body\n" 687 "for.body: ; preds = %for.body, " 688 "%entry\n" 689 " %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %bb2 ]\n" 690 " %A0 = getelementptr inbounds %struct.Test, ptr %A, i64 " 691 "%indvars.iv, i32 0\n" 692 " %vA0 = load i32, ptr %A0, align 4\n" 693 " %B0 = getelementptr inbounds %struct.Test, ptr %B, i64 " 694 "%indvars.iv, i32 0\n" 695 " %vB0 = load i32, ptr %B0, align 4\n" 696 " %add0 = add nsw i32 %vA0, %vB0\n" 697 " %A1 = getelementptr inbounds %struct.Test, ptr %A, i64 " 698 "%indvars.iv, i32 1\n" 699 " %vA1 = load i32, ptr %A1, align 4\n" 700 " %B1 = getelementptr inbounds %struct.Test, ptr %B, i64 " 701 "%indvars.iv, i32 1\n" 702 " br label %bb2\n" 703 "bb2:\n" 704 " %vB1 = load i32, ptr %B1, align 4\n" 705 " %add1 = add nsw i32 %vA1, %vB1\n" 706 " %C0 = getelementptr inbounds %struct.Test, ptr %C, i64 " 707 "%indvars.iv, i32 0\n" 708 " store i32 %add0, ptr %C0, align 4\n" 709 " %C1 = getelementptr inbounds %struct.Test, ptr %C, i64 " 710 "%indvars.iv, i32 1\n" 711 " store i32 %add1, ptr %C1, align 4\n" 712 " %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n" 713 " %exitcond = icmp eq i64 %indvars.iv.next, 1024\n" 714 " br i1 %exitcond, label %for.cond.cleanup, label %for.body\n" 715 "for.cond.cleanup: ; preds = %for.body\n" 716 " ret void\n" 717 "}\n"; 718 719 Module &M = parseModule(ModuleString); 720 721 Function *F = M.getFunction("add_x2"); 722 BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor(); 723 auto Plan = buildHCFG(LoopHeader); 724 auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan); 725 726 VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock(); 727 EXPECT_NE(nullptr, Entry->getSingleSuccessor()); 728 VPBasicBlock *Body = Plan->getVectorLoopRegion()->getEntryBasicBlock(); 729 VPBasicBlock *BB2 = Body->getSingleSuccessor()->getEntryBasicBlock(); 730 731 VPInstruction *Store1 = cast<VPInstruction>(&*std::next(BB2->begin(), 3)); 732 VPInstruction *Store2 = cast<VPInstruction>(&*std::next(BB2->begin(), 5)); 733 734 VPlanSlp Slp(VPIAI, *BB2); 735 SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2}; 736 EXPECT_EQ(nullptr, Slp.buildGraph(StoreRoot)); 737 EXPECT_EQ(0u, Slp.getWidestBundleBits()); 738 } 739 740 // Make sure we do not combine instructions with operands in different BBs. 741 TEST_F(VPlanSlpTest, testInstrsInDifferentBBs2) { 742 const char *ModuleString = 743 "%struct.Test = type { i32, i32 }\n" 744 "%struct.Test3 = type { i32, i32, i32 }\n" 745 "%struct.Test4xi8 = type { i8, i8, i8 }\n" 746 "define void @add_x2(ptr nocapture readonly %A, ptr " 747 "nocapture readonly %B, ptr nocapture %C) {\n" 748 "entry:\n" 749 " br label %for.body\n" 750 "for.body: ; preds = %for.body, " 751 "%entry\n" 752 " %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %bb2 ]\n" 753 " %A0 = getelementptr inbounds %struct.Test, ptr %A, i64 " 754 "%indvars.iv, i32 0\n" 755 " %vA0 = load i32, ptr %A0, align 4\n" 756 " %B0 = getelementptr inbounds %struct.Test, ptr %B, i64 " 757 "%indvars.iv, i32 0\n" 758 " %vB0 = load i32, ptr %B0, align 4\n" 759 " %add0 = add nsw i32 %vA0, %vB0\n" 760 " %A1 = getelementptr inbounds %struct.Test, ptr %A, i64 " 761 "%indvars.iv, i32 1\n" 762 " %vA1 = load i32, ptr %A1, align 4\n" 763 " %B1 = getelementptr inbounds %struct.Test, ptr %B, i64 " 764 "%indvars.iv, i32 1\n" 765 " %vB1 = load i32, ptr %B1, align 4\n" 766 " %add1 = add nsw i32 %vA1, %vB1\n" 767 " br label %bb2\n" 768 "bb2:\n" 769 " %C0 = getelementptr inbounds %struct.Test, ptr %C, i64 " 770 "%indvars.iv, i32 0\n" 771 " store i32 %add0, ptr %C0, align 4\n" 772 " %C1 = getelementptr inbounds %struct.Test, ptr %C, i64 " 773 "%indvars.iv, i32 1\n" 774 " store i32 %add1, ptr %C1, align 4\n" 775 " %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n" 776 " %exitcond = icmp eq i64 %indvars.iv.next, 1024\n" 777 " br i1 %exitcond, label %for.cond.cleanup, label %for.body\n" 778 "for.cond.cleanup: ; preds = %for.body\n" 779 " ret void\n" 780 "}\n"; 781 782 Module &M = parseModule(ModuleString); 783 784 Function *F = M.getFunction("add_x2"); 785 BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor(); 786 auto Plan = buildHCFG(LoopHeader); 787 auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan); 788 789 VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock(); 790 EXPECT_NE(nullptr, Entry->getSingleSuccessor()); 791 VPBasicBlock *Body = Plan->getVectorLoopRegion()->getEntryBasicBlock(); 792 VPBasicBlock *BB2 = Body->getSingleSuccessor()->getEntryBasicBlock(); 793 794 VPInstruction *Store1 = cast<VPInstruction>(&*std::next(BB2->begin(), 1)); 795 VPInstruction *Store2 = cast<VPInstruction>(&*std::next(BB2->begin(), 3)); 796 797 VPlanSlp Slp(VPIAI, *BB2); 798 SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2}; 799 EXPECT_EQ(nullptr, Slp.buildGraph(StoreRoot)); 800 EXPECT_EQ(0u, Slp.getWidestBundleBits()); 801 } 802 803 TEST_F(VPlanSlpTest, testSlpAtomicLoad) { 804 const char *ModuleString = 805 "%struct.Test = type { i32, i32 }\n" 806 "%struct.Test3 = type { i32, i32, i32 }\n" 807 "%struct.Test4xi8 = type { i8, i8, i8 }\n" 808 "define void @add_x2(ptr nocapture readonly %A, ptr " 809 "nocapture readonly %B, ptr nocapture %C) {\n" 810 "entry:\n" 811 " br label %for.body\n" 812 "for.body: ; preds = %for.body, " 813 "%entry\n" 814 " %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n" 815 " %A0 = getelementptr inbounds %struct.Test, ptr %A, i64 " 816 "%indvars.iv, i32 0\n" 817 " %vA0 = load atomic i32, ptr %A0 monotonic, align 4\n" 818 " %B0 = getelementptr inbounds %struct.Test, ptr %B, i64 " 819 "%indvars.iv, i32 0\n" 820 " %vB0 = load i32, ptr %B0, align 4\n" 821 " %add0 = add nsw i32 %vA0, %vB0\n" 822 " %A1 = getelementptr inbounds %struct.Test, ptr %A, i64 " 823 "%indvars.iv, i32 1\n" 824 " %vA1 = load i32, ptr %A1, align 4\n" 825 " %B1 = getelementptr inbounds %struct.Test, ptr %B, i64 " 826 "%indvars.iv, i32 1\n" 827 " %vB1 = load i32, ptr %B1, align 4\n" 828 " %add1 = add nsw i32 %vA1, %vB1\n" 829 " %C0 = getelementptr inbounds %struct.Test, ptr %C, i64 " 830 "%indvars.iv, i32 0\n" 831 " store i32 %add0, ptr %C0, align 4\n" 832 " %C1 = getelementptr inbounds %struct.Test, ptr %C, i64 " 833 "%indvars.iv, i32 1\n" 834 " store i32 %add1, ptr %C1, align 4\n" 835 " %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n" 836 " %exitcond = icmp eq i64 %indvars.iv.next, 1024\n" 837 " br i1 %exitcond, label %for.cond.cleanup, label %for.body\n" 838 "for.cond.cleanup: ; preds = %for.body\n" 839 " ret void\n" 840 "}\n"; 841 842 Module &M = parseModule(ModuleString); 843 844 Function *F = M.getFunction("add_x2"); 845 BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor(); 846 auto Plan = buildHCFG(LoopHeader); 847 auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan); 848 849 VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock(); 850 EXPECT_NE(nullptr, Entry->getSingleSuccessor()); 851 VPBasicBlock *Body = Plan->getVectorLoopRegion()->getEntryBasicBlock(); 852 853 VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 12)); 854 VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 14)); 855 856 VPlanSlp Slp(VPIAI, *Body); 857 SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2}; 858 EXPECT_EQ(nullptr, Slp.buildGraph(StoreRoot)); 859 EXPECT_FALSE(Slp.isCompletelySLP()); 860 } 861 862 TEST_F(VPlanSlpTest, testSlpAtomicStore) { 863 const char *ModuleString = 864 "%struct.Test = type { i32, i32 }\n" 865 "%struct.Test3 = type { i32, i32, i32 }\n" 866 "%struct.Test4xi8 = type { i8, i8, i8 }\n" 867 "define void @add_x2(ptr nocapture readonly %A, ptr " 868 "nocapture readonly %B, ptr nocapture %C) {\n" 869 "entry:\n" 870 " br label %for.body\n" 871 "for.body: ; preds = %for.body, " 872 "%entry\n" 873 " %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n" 874 " %A0 = getelementptr inbounds %struct.Test, ptr %A, i64 " 875 "%indvars.iv, i32 0\n" 876 " %vA0 = load i32, ptr %A0, align 4\n" 877 " %B0 = getelementptr inbounds %struct.Test, ptr %B, i64 " 878 "%indvars.iv, i32 0\n" 879 " %vB0 = load i32, ptr %B0, align 4\n" 880 " %add0 = add nsw i32 %vA0, %vB0\n" 881 " %A1 = getelementptr inbounds %struct.Test, ptr %A, i64 " 882 "%indvars.iv, i32 1\n" 883 " %vA1 = load i32, ptr %A1, align 4\n" 884 " %B1 = getelementptr inbounds %struct.Test, ptr %B, i64 " 885 "%indvars.iv, i32 1\n" 886 " %vB1 = load i32, ptr %B1, align 4\n" 887 " %add1 = add nsw i32 %vA1, %vB1\n" 888 " %C0 = getelementptr inbounds %struct.Test, ptr %C, i64 " 889 "%indvars.iv, i32 0\n" 890 " store atomic i32 %add0, ptr %C0 monotonic, align 4\n" 891 " %C1 = getelementptr inbounds %struct.Test, ptr %C, i64 " 892 "%indvars.iv, i32 1\n" 893 " store i32 %add1, ptr %C1, align 4\n" 894 " %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n" 895 " %exitcond = icmp eq i64 %indvars.iv.next, 1024\n" 896 " br i1 %exitcond, label %for.cond.cleanup, label %for.body\n" 897 "for.cond.cleanup: ; preds = %for.body\n" 898 " ret void\n" 899 "}\n"; 900 901 Module &M = parseModule(ModuleString); 902 903 Function *F = M.getFunction("add_x2"); 904 BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor(); 905 auto Plan = buildHCFG(LoopHeader); 906 auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan); 907 908 VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock(); 909 EXPECT_NE(nullptr, Entry->getSingleSuccessor()); 910 VPBasicBlock *Body = Plan->getVectorLoopRegion()->getEntryBasicBlock(); 911 912 VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 12)); 913 VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 14)); 914 915 VPlanSlp Slp(VPIAI, *Body); 916 SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2}; 917 Slp.buildGraph(StoreRoot); 918 EXPECT_FALSE(Slp.isCompletelySLP()); 919 } 920 921 } // namespace 922 } // namespace llvm 923