1 //===- CoroEarly.cpp - Coroutine Early Function Pass ----------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "llvm/Transforms/Coroutines/CoroEarly.h" 10 #include "CoroInternal.h" 11 #include "llvm/IR/DIBuilder.h" 12 #include "llvm/IR/Function.h" 13 #include "llvm/IR/IRBuilder.h" 14 #include "llvm/IR/InstIterator.h" 15 #include "llvm/IR/Module.h" 16 #include "llvm/Transforms/Coroutines/CoroShape.h" 17 18 using namespace llvm; 19 20 #define DEBUG_TYPE "coro-early" 21 22 namespace { 23 // Created on demand if the coro-early pass has work to do. 24 class Lowerer : public coro::LowererBase { 25 IRBuilder<> Builder; 26 PointerType *const AnyResumeFnPtrTy; 27 Constant *NoopCoro = nullptr; 28 29 void lowerResumeOrDestroy(CallBase &CB, CoroSubFnInst::ResumeKind); 30 void lowerCoroPromise(CoroPromiseInst *Intrin); 31 void lowerCoroDone(IntrinsicInst *II); 32 void lowerCoroNoop(IntrinsicInst *II); 33 34 public: 35 Lowerer(Module &M) 36 : LowererBase(M), Builder(Context), 37 AnyResumeFnPtrTy(PointerType::getUnqual(Context)) {} 38 void lowerEarlyIntrinsics(Function &F); 39 }; 40 } 41 42 // Replace a direct call to coro.resume or coro.destroy with an indirect call to 43 // an address returned by coro.subfn.addr intrinsic. This is done so that 44 // CGPassManager recognizes devirtualization when CoroElide pass replaces a call 45 // to coro.subfn.addr with an appropriate function address. 46 void Lowerer::lowerResumeOrDestroy(CallBase &CB, 47 CoroSubFnInst::ResumeKind Index) { 48 Value *ResumeAddr = makeSubFnCall(CB.getArgOperand(0), Index, &CB); 49 CB.setCalledOperand(ResumeAddr); 50 CB.setCallingConv(CallingConv::Fast); 51 } 52 53 // Coroutine promise field is always at the fixed offset from the beginning of 54 // the coroutine frame. i8* coro.promise(i8*, i1 from) intrinsic adds an offset 55 // to a passed pointer to move from coroutine frame to coroutine promise and 56 // vice versa. Since we don't know exactly which coroutine frame it is, we build 57 // a coroutine frame mock up starting with two function pointers, followed by a 58 // properly aligned coroutine promise field. 59 // TODO: Handle the case when coroutine promise alloca has align override. 60 void Lowerer::lowerCoroPromise(CoroPromiseInst *Intrin) { 61 Value *Operand = Intrin->getArgOperand(0); 62 Align Alignment = Intrin->getAlignment(); 63 Type *Int8Ty = Builder.getInt8Ty(); 64 65 auto *SampleStruct = 66 StructType::get(Context, {AnyResumeFnPtrTy, AnyResumeFnPtrTy, Int8Ty}); 67 const DataLayout &DL = TheModule.getDataLayout(); 68 int64_t Offset = alignTo( 69 DL.getStructLayout(SampleStruct)->getElementOffset(2), Alignment); 70 if (Intrin->isFromPromise()) 71 Offset = -Offset; 72 73 Builder.SetInsertPoint(Intrin); 74 Value *Replacement = 75 Builder.CreateConstInBoundsGEP1_32(Int8Ty, Operand, Offset); 76 77 Intrin->replaceAllUsesWith(Replacement); 78 Intrin->eraseFromParent(); 79 } 80 81 // When a coroutine reaches final suspend point, it zeros out ResumeFnAddr in 82 // the coroutine frame (it is UB to resume from a final suspend point). 83 // The llvm.coro.done intrinsic is used to check whether a coroutine is 84 // suspended at the final suspend point or not. 85 void Lowerer::lowerCoroDone(IntrinsicInst *II) { 86 Value *Operand = II->getArgOperand(0); 87 88 // ResumeFnAddr is the first pointer sized element of the coroutine frame. 89 static_assert(coro::Shape::SwitchFieldIndex::Resume == 0, 90 "resume function not at offset zero"); 91 auto *FrameTy = Int8Ptr; 92 93 Builder.SetInsertPoint(II); 94 auto *Load = Builder.CreateLoad(FrameTy, Operand); 95 auto *Cond = Builder.CreateICmpEQ(Load, NullPtr); 96 97 II->replaceAllUsesWith(Cond); 98 II->eraseFromParent(); 99 } 100 101 static void buildDebugInfoForNoopResumeDestroyFunc(Function *NoopFn) { 102 Module &M = *NoopFn->getParent(); 103 if (M.debug_compile_units().empty()) 104 return; 105 106 DICompileUnit *CU = *M.debug_compile_units_begin(); 107 DIBuilder DB(M, /*AllowUnresolved*/ false, CU); 108 std::array<Metadata *, 2> Params{nullptr, nullptr}; 109 auto *SubroutineType = 110 DB.createSubroutineType(DB.getOrCreateTypeArray(Params)); 111 StringRef Name = NoopFn->getName(); 112 auto *SP = DB.createFunction( 113 CU, /*Name=*/Name, /*LinkageName=*/Name, /*File=*/ CU->getFile(), 114 /*LineNo=*/0, SubroutineType, /*ScopeLine=*/0, DINode::FlagArtificial, 115 DISubprogram::SPFlagDefinition); 116 NoopFn->setSubprogram(SP); 117 DB.finalize(); 118 } 119 120 void Lowerer::lowerCoroNoop(IntrinsicInst *II) { 121 if (!NoopCoro) { 122 LLVMContext &C = Builder.getContext(); 123 Module &M = *II->getModule(); 124 125 // Create a noop.frame struct type. 126 auto *FnTy = FunctionType::get(Type::getVoidTy(C), Builder.getPtrTy(0), 127 /*isVarArg=*/false); 128 auto *FnPtrTy = Builder.getPtrTy(0); 129 StructType *FrameTy = 130 StructType::create({FnPtrTy, FnPtrTy}, "NoopCoro.Frame"); 131 132 // Create a Noop function that does nothing. 133 Function *NoopFn = 134 Function::Create(FnTy, GlobalValue::LinkageTypes::PrivateLinkage, 135 "__NoopCoro_ResumeDestroy", &M); 136 NoopFn->setCallingConv(CallingConv::Fast); 137 buildDebugInfoForNoopResumeDestroyFunc(NoopFn); 138 auto *Entry = BasicBlock::Create(C, "entry", NoopFn); 139 ReturnInst::Create(C, Entry); 140 141 // Create a constant struct for the frame. 142 Constant* Values[] = {NoopFn, NoopFn}; 143 Constant* NoopCoroConst = ConstantStruct::get(FrameTy, Values); 144 NoopCoro = new GlobalVariable(M, NoopCoroConst->getType(), /*isConstant=*/true, 145 GlobalVariable::PrivateLinkage, NoopCoroConst, 146 "NoopCoro.Frame.Const"); 147 cast<GlobalVariable>(NoopCoro)->setNoSanitizeMetadata(); 148 } 149 150 Builder.SetInsertPoint(II); 151 auto *NoopCoroVoidPtr = Builder.CreateBitCast(NoopCoro, Int8Ptr); 152 II->replaceAllUsesWith(NoopCoroVoidPtr); 153 II->eraseFromParent(); 154 } 155 156 // Prior to CoroSplit, calls to coro.begin needs to be marked as NoDuplicate, 157 // as CoroSplit assumes there is exactly one coro.begin. After CoroSplit, 158 // NoDuplicate attribute will be removed from coro.begin otherwise, it will 159 // interfere with inlining. 160 static void setCannotDuplicate(CoroIdInst *CoroId) { 161 for (User *U : CoroId->users()) 162 if (auto *CB = dyn_cast<CoroBeginInst>(U)) 163 CB->setCannotDuplicate(); 164 } 165 166 void Lowerer::lowerEarlyIntrinsics(Function &F) { 167 CoroIdInst *CoroId = nullptr; 168 SmallVector<CoroFreeInst *, 4> CoroFrees; 169 bool HasCoroSuspend = false; 170 for (Instruction &I : llvm::make_early_inc_range(instructions(F))) { 171 auto *CB = dyn_cast<CallBase>(&I); 172 if (!CB) 173 continue; 174 175 switch (CB->getIntrinsicID()) { 176 default: 177 continue; 178 case Intrinsic::coro_free: 179 CoroFrees.push_back(cast<CoroFreeInst>(&I)); 180 break; 181 case Intrinsic::coro_suspend: 182 // Make sure that final suspend point is not duplicated as CoroSplit 183 // pass expects that there is at most one final suspend point. 184 if (cast<CoroSuspendInst>(&I)->isFinal()) 185 CB->setCannotDuplicate(); 186 HasCoroSuspend = true; 187 break; 188 case Intrinsic::coro_end_async: 189 case Intrinsic::coro_end: 190 // Make sure that fallthrough coro.end is not duplicated as CoroSplit 191 // pass expects that there is at most one fallthrough coro.end. 192 if (cast<AnyCoroEndInst>(&I)->isFallthrough()) 193 CB->setCannotDuplicate(); 194 break; 195 case Intrinsic::coro_noop: 196 lowerCoroNoop(cast<IntrinsicInst>(&I)); 197 break; 198 case Intrinsic::coro_id: 199 if (auto *CII = cast<CoroIdInst>(&I)) { 200 if (CII->getInfo().isPreSplit()) { 201 assert(F.isPresplitCoroutine() && 202 "The frontend uses Switch-Resumed ABI should emit " 203 "\"presplitcoroutine\" attribute for the coroutine."); 204 setCannotDuplicate(CII); 205 CII->setCoroutineSelf(); 206 CoroId = cast<CoroIdInst>(&I); 207 } 208 } 209 break; 210 case Intrinsic::coro_id_retcon: 211 case Intrinsic::coro_id_retcon_once: 212 case Intrinsic::coro_id_async: 213 F.setPresplitCoroutine(); 214 break; 215 case Intrinsic::coro_resume: 216 lowerResumeOrDestroy(*CB, CoroSubFnInst::ResumeIndex); 217 break; 218 case Intrinsic::coro_destroy: 219 lowerResumeOrDestroy(*CB, CoroSubFnInst::DestroyIndex); 220 break; 221 case Intrinsic::coro_promise: 222 lowerCoroPromise(cast<CoroPromiseInst>(&I)); 223 break; 224 case Intrinsic::coro_done: 225 lowerCoroDone(cast<IntrinsicInst>(&I)); 226 break; 227 } 228 } 229 230 // Make sure that all CoroFree reference the coro.id intrinsic. 231 // Token type is not exposed through coroutine C/C++ builtins to plain C, so 232 // we allow specifying none and fixing it up here. 233 if (CoroId) 234 for (CoroFreeInst *CF : CoroFrees) 235 CF->setArgOperand(0, CoroId); 236 237 // Coroutine suspention could potentially lead to any argument modified 238 // outside of the function, hence arguments should not have noalias 239 // attributes. 240 if (HasCoroSuspend) 241 for (Argument &A : F.args()) 242 if (A.hasNoAliasAttr()) 243 A.removeAttr(Attribute::NoAlias); 244 } 245 246 static bool declaresCoroEarlyIntrinsics(const Module &M) { 247 return coro::declaresIntrinsics( 248 M, {"llvm.coro.id", "llvm.coro.id.retcon", "llvm.coro.id.retcon.once", 249 "llvm.coro.id.async", "llvm.coro.destroy", "llvm.coro.done", 250 "llvm.coro.end", "llvm.coro.end.async", "llvm.coro.noop", 251 "llvm.coro.free", "llvm.coro.promise", "llvm.coro.resume", 252 "llvm.coro.suspend"}); 253 } 254 255 PreservedAnalyses CoroEarlyPass::run(Module &M, ModuleAnalysisManager &) { 256 if (!declaresCoroEarlyIntrinsics(M)) 257 return PreservedAnalyses::all(); 258 259 Lowerer L(M); 260 for (auto &F : M) 261 L.lowerEarlyIntrinsics(F); 262 263 PreservedAnalyses PA; 264 PA.preserveSet<CFGAnalyses>(); 265 return PA; 266 } 267