1 //=========- MemTransferLowerTest.cpp - MemTransferLower unit tests -=========// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "llvm/Analysis/CGSCCPassManager.h" 10 #include "llvm/Analysis/ScalarEvolution.h" 11 #include "llvm/Analysis/TargetTransformInfo.h" 12 #include "llvm/AsmParser/Parser.h" 13 #include "llvm/IR/BasicBlock.h" 14 #include "llvm/IR/Function.h" 15 #include "llvm/IR/Instructions.h" 16 #include "llvm/IR/IntrinsicInst.h" 17 #include "llvm/IR/LLVMContext.h" 18 #include "llvm/IR/Module.h" 19 #include "llvm/IR/PassManager.h" 20 #include "llvm/Passes/PassBuilder.h" 21 #include "llvm/Support/Debug.h" 22 #include "llvm/Support/SourceMgr.h" 23 #include "llvm/Testing/Support/Error.h" 24 #include "llvm/Transforms/Utils/LowerMemIntrinsics.h" 25 #include "llvm/Transforms/Vectorize/LoopVectorize.h" 26 27 #include "gtest/gtest-spi.h" 28 #include "gtest/gtest.h" 29 30 using namespace llvm; 31 32 namespace { 33 struct ForwardingPass : public PassInfoMixin<ForwardingPass> { 34 template <typename T> ForwardingPass(T &&Arg) : Func(std::forward<T>(Arg)) {} 35 36 PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM) { 37 return Func(F, FAM); 38 } 39 40 std::function<PreservedAnalyses(Function &, FunctionAnalysisManager &)> Func; 41 }; 42 43 struct MemTransferLowerTest : public testing::Test { 44 PassBuilder PB; 45 LoopAnalysisManager LAM; 46 FunctionAnalysisManager FAM; 47 CGSCCAnalysisManager CGAM; 48 ModuleAnalysisManager MAM; 49 ModulePassManager MPM; 50 LLVMContext Context; 51 std::unique_ptr<Module> M; 52 53 MemTransferLowerTest() { 54 // Register all the basic analyses with the managers. 55 PB.registerModuleAnalyses(MAM); 56 PB.registerCGSCCAnalyses(CGAM); 57 PB.registerFunctionAnalyses(FAM); 58 PB.registerLoopAnalyses(LAM); 59 PB.crossRegisterProxies(LAM, FAM, CGAM, MAM); 60 } 61 62 BasicBlock *getBasicBlockByName(Function &F, StringRef Name) const { 63 for (BasicBlock &BB : F) { 64 if (BB.getName() == Name) 65 return &BB; 66 } 67 return nullptr; 68 } 69 70 Instruction *getInstructionByOpcode(BasicBlock &BB, unsigned Opcode, 71 unsigned Number) const { 72 unsigned CurrNumber = 0; 73 for (Instruction &I : BB) 74 if (I.getOpcode() == Opcode) { 75 ++CurrNumber; 76 if (CurrNumber == Number) 77 return &I; 78 } 79 return nullptr; 80 } 81 82 void ParseAssembly(const char *IR) { 83 SMDiagnostic Error; 84 M = parseAssemblyString(IR, Error, Context); 85 std::string errMsg; 86 raw_string_ostream os(errMsg); 87 Error.print("", os); 88 89 // A failure here means that the test itself is buggy. 90 if (!M) 91 report_fatal_error(errMsg.c_str()); 92 } 93 }; 94 95 // By semantics source and destination of llvm.memcpy.* intrinsic 96 // are either equal or don't overlap. Once the intrinsic is lowered 97 // to a loop it can be hard or impossible to reason about these facts. 98 // For that reason expandMemCpyAsLoop is expected to explicitly mark 99 // loads from source and stores to destination as not aliasing. 100 TEST_F(MemTransferLowerTest, MemCpyKnownLength) { 101 ParseAssembly("declare void @llvm.memcpy.p0i8.p0i8.i64(i8*, i8 *, i64, i1)\n" 102 "define void @foo(i8* %dst, i8* %src, i64 %n) optsize {\n" 103 "entry:\n" 104 " %is_not_equal = icmp ne i8* %dst, %src\n" 105 " br i1 %is_not_equal, label %memcpy, label %exit\n" 106 "memcpy:\n" 107 " call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, " 108 "i64 1024, i1 false)\n" 109 " br label %exit\n" 110 "exit:\n" 111 " ret void\n" 112 "}\n"); 113 114 FunctionPassManager FPM; 115 FPM.addPass(ForwardingPass( 116 [=](Function &F, FunctionAnalysisManager &FAM) -> PreservedAnalyses { 117 TargetTransformInfo TTI(M->getDataLayout()); 118 auto *MemCpyBB = getBasicBlockByName(F, "memcpy"); 119 Instruction *Inst = &MemCpyBB->front(); 120 MemCpyInst *MemCpyI = cast<MemCpyInst>(Inst); 121 auto &SE = FAM.getResult<ScalarEvolutionAnalysis>(F); 122 expandMemCpyAsLoop(MemCpyI, TTI, &SE); 123 auto *CopyLoopBB = getBasicBlockByName(F, "load-store-loop"); 124 Instruction *LoadInst = 125 getInstructionByOpcode(*CopyLoopBB, Instruction::Load, 1); 126 EXPECT_NE(nullptr, LoadInst->getMetadata(LLVMContext::MD_alias_scope)); 127 Instruction *StoreInst = 128 getInstructionByOpcode(*CopyLoopBB, Instruction::Store, 1); 129 EXPECT_NE(nullptr, StoreInst->getMetadata(LLVMContext::MD_noalias)); 130 return PreservedAnalyses::none(); 131 })); 132 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); 133 134 MPM.run(*M, MAM); 135 } 136 137 // This test indirectly checks that loads and stores (generated as a result of 138 // llvm.memcpy lowering) doesn't alias by making sure the loop can be 139 // successfully vectorized without additional runtime checks. 140 TEST_F(MemTransferLowerTest, VecMemCpyKnownLength) { 141 ParseAssembly("declare void @llvm.memcpy.p0i8.p0i8.i64(i8*, i8 *, i64, i1)\n" 142 "define void @foo(i8* %dst, i8* %src, i64 %n) optsize {\n" 143 "entry:\n" 144 " %is_not_equal = icmp ne i8* %dst, %src\n" 145 " br i1 %is_not_equal, label %memcpy, label %exit\n" 146 "memcpy:\n" 147 " call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, " 148 "i64 1024, i1 false)\n" 149 " br label %exit\n" 150 "exit:\n" 151 " ret void\n" 152 "}\n"); 153 154 FunctionPassManager FPM; 155 FPM.addPass(ForwardingPass( 156 [=](Function &F, FunctionAnalysisManager &FAM) -> PreservedAnalyses { 157 TargetTransformInfo TTI(M->getDataLayout()); 158 auto *MemCpyBB = getBasicBlockByName(F, "memcpy"); 159 Instruction *Inst = &MemCpyBB->front(); 160 MemCpyInst *MemCpyI = cast<MemCpyInst>(Inst); 161 auto &SE = FAM.getResult<ScalarEvolutionAnalysis>(F); 162 expandMemCpyAsLoop(MemCpyI, TTI, &SE); 163 return PreservedAnalyses::none(); 164 })); 165 FPM.addPass(LoopVectorizePass(LoopVectorizeOptions())); 166 FPM.addPass(ForwardingPass( 167 [=](Function &F, FunctionAnalysisManager &FAM) -> PreservedAnalyses { 168 auto *TargetBB = getBasicBlockByName(F, "vector.body"); 169 EXPECT_NE(nullptr, TargetBB); 170 return PreservedAnalyses::all(); 171 })); 172 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); 173 174 MPM.run(*M, MAM); 175 } 176 177 TEST_F(MemTransferLowerTest, AtomicMemCpyKnownLength) { 178 ParseAssembly("declare void " 179 "@llvm.memcpy.element.unordered.atomic.p0i32.p0i32.i64(i32*, " 180 "i32 *, i64, i32)\n" 181 "define void @foo(i32* %dst, i32* %src, i64 %n) optsize {\n" 182 "entry:\n" 183 " %is_not_equal = icmp ne i32* %dst, %src\n" 184 " br i1 %is_not_equal, label %memcpy, label %exit\n" 185 "memcpy:\n" 186 " call void " 187 "@llvm.memcpy.element.unordered.atomic.p0i32.p0i32.i64(i32* " 188 "%dst, i32* %src, " 189 "i64 1024, i32 4)\n" 190 " br label %exit\n" 191 "exit:\n" 192 " ret void\n" 193 "}\n"); 194 195 FunctionPassManager FPM; 196 FPM.addPass(ForwardingPass( 197 [=](Function &F, FunctionAnalysisManager &FAM) -> PreservedAnalyses { 198 TargetTransformInfo TTI(M->getDataLayout()); 199 auto *MemCpyBB = getBasicBlockByName(F, "memcpy"); 200 Instruction *Inst = &MemCpyBB->front(); 201 assert(isa<AtomicMemCpyInst>(Inst) && 202 "Expecting llvm.memcpy.p0i8.i64 instructon"); 203 AtomicMemCpyInst *MemCpyI = cast<AtomicMemCpyInst>(Inst); 204 auto &SE = FAM.getResult<ScalarEvolutionAnalysis>(F); 205 expandAtomicMemCpyAsLoop(MemCpyI, TTI, &SE); 206 auto *CopyLoopBB = getBasicBlockByName(F, "load-store-loop"); 207 Instruction *LoadInst = 208 getInstructionByOpcode(*CopyLoopBB, Instruction::Load, 1); 209 EXPECT_TRUE(LoadInst->isAtomic()); 210 EXPECT_NE(LoadInst->getMetadata(LLVMContext::MD_alias_scope), nullptr); 211 Instruction *StoreInst = 212 getInstructionByOpcode(*CopyLoopBB, Instruction::Store, 1); 213 EXPECT_TRUE(StoreInst->isAtomic()); 214 EXPECT_NE(StoreInst->getMetadata(LLVMContext::MD_noalias), nullptr); 215 return PreservedAnalyses::none(); 216 })); 217 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); 218 219 MPM.run(*M, MAM); 220 } 221 222 TEST_F(MemTransferLowerTest, AtomicMemCpyUnKnownLength) { 223 ParseAssembly("declare void " 224 "@llvm.memcpy.element.unordered.atomic.p0i32.p0i32.i64(i32*, " 225 "i32 *, i64, i32)\n" 226 "define void @foo(i32* %dst, i32* %src, i64 %n) optsize {\n" 227 "entry:\n" 228 " %is_not_equal = icmp ne i32* %dst, %src\n" 229 " br i1 %is_not_equal, label %memcpy, label %exit\n" 230 "memcpy:\n" 231 " call void " 232 "@llvm.memcpy.element.unordered.atomic.p0i32.p0i32.i64(i32* " 233 "%dst, i32* %src, " 234 "i64 %n, i32 4)\n" 235 " br label %exit\n" 236 "exit:\n" 237 " ret void\n" 238 "}\n"); 239 240 FunctionPassManager FPM; 241 FPM.addPass(ForwardingPass( 242 [=](Function &F, FunctionAnalysisManager &FAM) -> PreservedAnalyses { 243 TargetTransformInfo TTI(M->getDataLayout()); 244 auto *MemCpyBB = getBasicBlockByName(F, "memcpy"); 245 Instruction *Inst = &MemCpyBB->front(); 246 assert(isa<AtomicMemCpyInst>(Inst) && 247 "Expecting llvm.memcpy.p0i8.i64 instructon"); 248 AtomicMemCpyInst *MemCpyI = cast<AtomicMemCpyInst>(Inst); 249 auto &SE = FAM.getResult<ScalarEvolutionAnalysis>(F); 250 expandAtomicMemCpyAsLoop(MemCpyI, TTI, &SE); 251 auto *CopyLoopBB = getBasicBlockByName(F, "loop-memcpy-expansion"); 252 Instruction *LoadInst = 253 getInstructionByOpcode(*CopyLoopBB, Instruction::Load, 1); 254 EXPECT_TRUE(LoadInst->isAtomic()); 255 EXPECT_NE(LoadInst->getMetadata(LLVMContext::MD_alias_scope), nullptr); 256 Instruction *StoreInst = 257 getInstructionByOpcode(*CopyLoopBB, Instruction::Store, 1); 258 EXPECT_TRUE(StoreInst->isAtomic()); 259 EXPECT_NE(StoreInst->getMetadata(LLVMContext::MD_noalias), nullptr); 260 return PreservedAnalyses::none(); 261 })); 262 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); 263 264 MPM.run(*M, MAM); 265 } 266 } // namespace 267