xref: /llvm-project/clang/lib/CodeGen/CodeGenFunction.cpp (revision 8ace12130526f450c822ca232d1f865b247d7434)
1 //===--- CodeGenFunction.cpp - Emit LLVM Code from ASTs for a Function ----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This coordinates the per-function state used while generating code.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "CodeGenFunction.h"
14 #include "CGBlocks.h"
15 #include "CGCUDARuntime.h"
16 #include "CGCXXABI.h"
17 #include "CGCleanup.h"
18 #include "CGDebugInfo.h"
19 #include "CGOpenMPRuntime.h"
20 #include "CodeGenModule.h"
21 #include "CodeGenPGO.h"
22 #include "TargetInfo.h"
23 #include "clang/AST/ASTContext.h"
24 #include "clang/AST/ASTLambda.h"
25 #include "clang/AST/Attr.h"
26 #include "clang/AST/Decl.h"
27 #include "clang/AST/DeclCXX.h"
28 #include "clang/AST/Expr.h"
29 #include "clang/AST/StmtCXX.h"
30 #include "clang/AST/StmtObjC.h"
31 #include "clang/Basic/Builtins.h"
32 #include "clang/Basic/CodeGenOptions.h"
33 #include "clang/Basic/TargetInfo.h"
34 #include "clang/CodeGen/CGFunctionInfo.h"
35 #include "clang/Frontend/FrontendDiagnostic.h"
36 #include "llvm/ADT/ArrayRef.h"
37 #include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
38 #include "llvm/IR/DataLayout.h"
39 #include "llvm/IR/Dominators.h"
40 #include "llvm/IR/FPEnv.h"
41 #include "llvm/IR/IntrinsicInst.h"
42 #include "llvm/IR/Intrinsics.h"
43 #include "llvm/IR/MDBuilder.h"
44 #include "llvm/IR/Operator.h"
45 #include "llvm/Support/CRC.h"
46 #include "llvm/Transforms/Scalar/LowerExpectIntrinsic.h"
47 #include "llvm/Transforms/Utils/PromoteMemToReg.h"
48 using namespace clang;
49 using namespace CodeGen;
50 
51 /// shouldEmitLifetimeMarkers - Decide whether we need emit the life-time
52 /// markers.
53 static bool shouldEmitLifetimeMarkers(const CodeGenOptions &CGOpts,
54                                       const LangOptions &LangOpts) {
55   if (CGOpts.DisableLifetimeMarkers)
56     return false;
57 
58   // Sanitizers may use markers.
59   if (CGOpts.SanitizeAddressUseAfterScope ||
60       LangOpts.Sanitize.has(SanitizerKind::HWAddress) ||
61       LangOpts.Sanitize.has(SanitizerKind::Memory))
62     return true;
63 
64   // For now, only in optimized builds.
65   return CGOpts.OptimizationLevel != 0;
66 }
67 
68 CodeGenFunction::CodeGenFunction(CodeGenModule &cgm, bool suppressNewContext)
69     : CodeGenTypeCache(cgm), CGM(cgm), Target(cgm.getTarget()),
70       Builder(cgm, cgm.getModule().getContext(), llvm::ConstantFolder(),
71               CGBuilderInserterTy(this)),
72       SanOpts(CGM.getLangOpts().Sanitize), CurFPFeatures(CGM.getLangOpts()),
73       DebugInfo(CGM.getModuleDebugInfo()), PGO(cgm),
74       ShouldEmitLifetimeMarkers(
75           shouldEmitLifetimeMarkers(CGM.getCodeGenOpts(), CGM.getLangOpts())) {
76   if (!suppressNewContext)
77     CGM.getCXXABI().getMangleContext().startNewFunction();
78   EHStack.setCGF(this);
79 
80   SetFastMathFlags(CurFPFeatures);
81   SetFPModel();
82 }
83 
84 CodeGenFunction::~CodeGenFunction() {
85   assert(LifetimeExtendedCleanupStack.empty() && "failed to emit a cleanup");
86 
87   if (getLangOpts().OpenMP && CurFn)
88     CGM.getOpenMPRuntime().functionFinished(*this);
89 
90   // If we have an OpenMPIRBuilder we want to finalize functions (incl.
91   // outlining etc) at some point. Doing it once the function codegen is done
92   // seems to be a reasonable spot. We do it here, as opposed to the deletion
93   // time of the CodeGenModule, because we have to ensure the IR has not yet
94   // been "emitted" to the outside, thus, modifications are still sensible.
95   if (CGM.getLangOpts().OpenMPIRBuilder && CurFn)
96     CGM.getOpenMPRuntime().getOMPBuilder().finalize(CurFn);
97 }
98 
99 // Map the LangOption for exception behavior into
100 // the corresponding enum in the IR.
101 llvm::fp::ExceptionBehavior
102 clang::ToConstrainedExceptMD(LangOptions::FPExceptionModeKind Kind) {
103 
104   switch (Kind) {
105   case LangOptions::FPE_Ignore:  return llvm::fp::ebIgnore;
106   case LangOptions::FPE_MayTrap: return llvm::fp::ebMayTrap;
107   case LangOptions::FPE_Strict:  return llvm::fp::ebStrict;
108   }
109   llvm_unreachable("Unsupported FP Exception Behavior");
110 }
111 
112 void CodeGenFunction::SetFPModel() {
113   llvm::RoundingMode RM = getLangOpts().getFPRoundingMode();
114   auto fpExceptionBehavior = ToConstrainedExceptMD(
115                                getLangOpts().getFPExceptionMode());
116 
117   Builder.setDefaultConstrainedRounding(RM);
118   Builder.setDefaultConstrainedExcept(fpExceptionBehavior);
119   Builder.setIsFPConstrained(fpExceptionBehavior != llvm::fp::ebIgnore ||
120                              RM != llvm::RoundingMode::NearestTiesToEven);
121 }
122 
123 void CodeGenFunction::SetFastMathFlags(FPOptions FPFeatures) {
124   llvm::FastMathFlags FMF;
125   FMF.setAllowReassoc(FPFeatures.getAllowFPReassociate());
126   FMF.setNoNaNs(FPFeatures.getNoHonorNaNs());
127   FMF.setNoInfs(FPFeatures.getNoHonorInfs());
128   FMF.setNoSignedZeros(FPFeatures.getNoSignedZero());
129   FMF.setAllowReciprocal(FPFeatures.getAllowReciprocal());
130   FMF.setApproxFunc(FPFeatures.getAllowApproxFunc());
131   FMF.setAllowContract(FPFeatures.allowFPContractAcrossStatement());
132   Builder.setFastMathFlags(FMF);
133 }
134 
135 CodeGenFunction::CGFPOptionsRAII::CGFPOptionsRAII(CodeGenFunction &CGF,
136                                                   const Expr *E)
137     : CGF(CGF) {
138   ConstructorHelper(E->getFPFeaturesInEffect(CGF.getLangOpts()));
139 }
140 
141 CodeGenFunction::CGFPOptionsRAII::CGFPOptionsRAII(CodeGenFunction &CGF,
142                                                   FPOptions FPFeatures)
143     : CGF(CGF) {
144   ConstructorHelper(FPFeatures);
145 }
146 
147 void CodeGenFunction::CGFPOptionsRAII::ConstructorHelper(FPOptions FPFeatures) {
148   OldFPFeatures = CGF.CurFPFeatures;
149   CGF.CurFPFeatures = FPFeatures;
150 
151   OldExcept = CGF.Builder.getDefaultConstrainedExcept();
152   OldRounding = CGF.Builder.getDefaultConstrainedRounding();
153 
154   if (OldFPFeatures == FPFeatures)
155     return;
156 
157   FMFGuard.emplace(CGF.Builder);
158 
159   llvm::RoundingMode NewRoundingBehavior =
160       static_cast<llvm::RoundingMode>(FPFeatures.getRoundingMode());
161   CGF.Builder.setDefaultConstrainedRounding(NewRoundingBehavior);
162   auto NewExceptionBehavior =
163       ToConstrainedExceptMD(static_cast<LangOptions::FPExceptionModeKind>(
164           FPFeatures.getFPExceptionMode()));
165   CGF.Builder.setDefaultConstrainedExcept(NewExceptionBehavior);
166 
167   CGF.SetFastMathFlags(FPFeatures);
168 
169   assert((CGF.CurFuncDecl == nullptr || CGF.Builder.getIsFPConstrained() ||
170           isa<CXXConstructorDecl>(CGF.CurFuncDecl) ||
171           isa<CXXDestructorDecl>(CGF.CurFuncDecl) ||
172           (NewExceptionBehavior == llvm::fp::ebIgnore &&
173            NewRoundingBehavior == llvm::RoundingMode::NearestTiesToEven)) &&
174          "FPConstrained should be enabled on entire function");
175 
176   auto mergeFnAttrValue = [&](StringRef Name, bool Value) {
177     auto OldValue =
178         CGF.CurFn->getFnAttribute(Name).getValueAsBool();
179     auto NewValue = OldValue & Value;
180     if (OldValue != NewValue)
181       CGF.CurFn->addFnAttr(Name, llvm::toStringRef(NewValue));
182   };
183   mergeFnAttrValue("no-infs-fp-math", FPFeatures.getNoHonorInfs());
184   mergeFnAttrValue("no-nans-fp-math", FPFeatures.getNoHonorNaNs());
185   mergeFnAttrValue("no-signed-zeros-fp-math", FPFeatures.getNoSignedZero());
186   mergeFnAttrValue("unsafe-fp-math", FPFeatures.getAllowFPReassociate() &&
187                                          FPFeatures.getAllowReciprocal() &&
188                                          FPFeatures.getAllowApproxFunc() &&
189                                          FPFeatures.getNoSignedZero());
190 }
191 
192 CodeGenFunction::CGFPOptionsRAII::~CGFPOptionsRAII() {
193   CGF.CurFPFeatures = OldFPFeatures;
194   CGF.Builder.setDefaultConstrainedExcept(OldExcept);
195   CGF.Builder.setDefaultConstrainedRounding(OldRounding);
196 }
197 
198 LValue CodeGenFunction::MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T) {
199   LValueBaseInfo BaseInfo;
200   TBAAAccessInfo TBAAInfo;
201   CharUnits Alignment = CGM.getNaturalTypeAlignment(T, &BaseInfo, &TBAAInfo);
202   return LValue::MakeAddr(Address(V, Alignment), T, getContext(), BaseInfo,
203                           TBAAInfo);
204 }
205 
206 /// Given a value of type T* that may not be to a complete object,
207 /// construct an l-value with the natural pointee alignment of T.
208 LValue
209 CodeGenFunction::MakeNaturalAlignPointeeAddrLValue(llvm::Value *V, QualType T) {
210   LValueBaseInfo BaseInfo;
211   TBAAAccessInfo TBAAInfo;
212   CharUnits Align = CGM.getNaturalTypeAlignment(T, &BaseInfo, &TBAAInfo,
213                                                 /* forPointeeType= */ true);
214   return MakeAddrLValue(Address(V, Align), T, BaseInfo, TBAAInfo);
215 }
216 
217 
218 llvm::Type *CodeGenFunction::ConvertTypeForMem(QualType T) {
219   return CGM.getTypes().ConvertTypeForMem(T);
220 }
221 
222 llvm::Type *CodeGenFunction::ConvertType(QualType T) {
223   return CGM.getTypes().ConvertType(T);
224 }
225 
226 TypeEvaluationKind CodeGenFunction::getEvaluationKind(QualType type) {
227   type = type.getCanonicalType();
228   while (true) {
229     switch (type->getTypeClass()) {
230 #define TYPE(name, parent)
231 #define ABSTRACT_TYPE(name, parent)
232 #define NON_CANONICAL_TYPE(name, parent) case Type::name:
233 #define DEPENDENT_TYPE(name, parent) case Type::name:
234 #define NON_CANONICAL_UNLESS_DEPENDENT_TYPE(name, parent) case Type::name:
235 #include "clang/AST/TypeNodes.inc"
236       llvm_unreachable("non-canonical or dependent type in IR-generation");
237 
238     case Type::Auto:
239     case Type::DeducedTemplateSpecialization:
240       llvm_unreachable("undeduced type in IR-generation");
241 
242     // Various scalar types.
243     case Type::Builtin:
244     case Type::Pointer:
245     case Type::BlockPointer:
246     case Type::LValueReference:
247     case Type::RValueReference:
248     case Type::MemberPointer:
249     case Type::Vector:
250     case Type::ExtVector:
251     case Type::ConstantMatrix:
252     case Type::FunctionProto:
253     case Type::FunctionNoProto:
254     case Type::Enum:
255     case Type::ObjCObjectPointer:
256     case Type::Pipe:
257     case Type::ExtInt:
258       return TEK_Scalar;
259 
260     // Complexes.
261     case Type::Complex:
262       return TEK_Complex;
263 
264     // Arrays, records, and Objective-C objects.
265     case Type::ConstantArray:
266     case Type::IncompleteArray:
267     case Type::VariableArray:
268     case Type::Record:
269     case Type::ObjCObject:
270     case Type::ObjCInterface:
271       return TEK_Aggregate;
272 
273     // We operate on atomic values according to their underlying type.
274     case Type::Atomic:
275       type = cast<AtomicType>(type)->getValueType();
276       continue;
277     }
278     llvm_unreachable("unknown type kind!");
279   }
280 }
281 
282 llvm::DebugLoc CodeGenFunction::EmitReturnBlock() {
283   // For cleanliness, we try to avoid emitting the return block for
284   // simple cases.
285   llvm::BasicBlock *CurBB = Builder.GetInsertBlock();
286 
287   if (CurBB) {
288     assert(!CurBB->getTerminator() && "Unexpected terminated block.");
289 
290     // We have a valid insert point, reuse it if it is empty or there are no
291     // explicit jumps to the return block.
292     if (CurBB->empty() || ReturnBlock.getBlock()->use_empty()) {
293       ReturnBlock.getBlock()->replaceAllUsesWith(CurBB);
294       delete ReturnBlock.getBlock();
295       ReturnBlock = JumpDest();
296     } else
297       EmitBlock(ReturnBlock.getBlock());
298     return llvm::DebugLoc();
299   }
300 
301   // Otherwise, if the return block is the target of a single direct
302   // branch then we can just put the code in that block instead. This
303   // cleans up functions which started with a unified return block.
304   if (ReturnBlock.getBlock()->hasOneUse()) {
305     llvm::BranchInst *BI =
306       dyn_cast<llvm::BranchInst>(*ReturnBlock.getBlock()->user_begin());
307     if (BI && BI->isUnconditional() &&
308         BI->getSuccessor(0) == ReturnBlock.getBlock()) {
309       // Record/return the DebugLoc of the simple 'return' expression to be used
310       // later by the actual 'ret' instruction.
311       llvm::DebugLoc Loc = BI->getDebugLoc();
312       Builder.SetInsertPoint(BI->getParent());
313       BI->eraseFromParent();
314       delete ReturnBlock.getBlock();
315       ReturnBlock = JumpDest();
316       return Loc;
317     }
318   }
319 
320   // FIXME: We are at an unreachable point, there is no reason to emit the block
321   // unless it has uses. However, we still need a place to put the debug
322   // region.end for now.
323 
324   EmitBlock(ReturnBlock.getBlock());
325   return llvm::DebugLoc();
326 }
327 
328 static void EmitIfUsed(CodeGenFunction &CGF, llvm::BasicBlock *BB) {
329   if (!BB) return;
330   if (!BB->use_empty())
331     return CGF.CurFn->getBasicBlockList().push_back(BB);
332   delete BB;
333 }
334 
335 void CodeGenFunction::FinishFunction(SourceLocation EndLoc) {
336   assert(BreakContinueStack.empty() &&
337          "mismatched push/pop in break/continue stack!");
338 
339   bool OnlySimpleReturnStmts = NumSimpleReturnExprs > 0
340     && NumSimpleReturnExprs == NumReturnExprs
341     && ReturnBlock.getBlock()->use_empty();
342   // Usually the return expression is evaluated before the cleanup
343   // code.  If the function contains only a simple return statement,
344   // such as a constant, the location before the cleanup code becomes
345   // the last useful breakpoint in the function, because the simple
346   // return expression will be evaluated after the cleanup code. To be
347   // safe, set the debug location for cleanup code to the location of
348   // the return statement.  Otherwise the cleanup code should be at the
349   // end of the function's lexical scope.
350   //
351   // If there are multiple branches to the return block, the branch
352   // instructions will get the location of the return statements and
353   // all will be fine.
354   if (CGDebugInfo *DI = getDebugInfo()) {
355     if (OnlySimpleReturnStmts)
356       DI->EmitLocation(Builder, LastStopPoint);
357     else
358       DI->EmitLocation(Builder, EndLoc);
359   }
360 
361   // Pop any cleanups that might have been associated with the
362   // parameters.  Do this in whatever block we're currently in; it's
363   // important to do this before we enter the return block or return
364   // edges will be *really* confused.
365   bool HasCleanups = EHStack.stable_begin() != PrologueCleanupDepth;
366   bool HasOnlyLifetimeMarkers =
367       HasCleanups && EHStack.containsOnlyLifetimeMarkers(PrologueCleanupDepth);
368   bool EmitRetDbgLoc = !HasCleanups || HasOnlyLifetimeMarkers;
369   if (HasCleanups) {
370     // Make sure the line table doesn't jump back into the body for
371     // the ret after it's been at EndLoc.
372     Optional<ApplyDebugLocation> AL;
373     if (CGDebugInfo *DI = getDebugInfo()) {
374       if (OnlySimpleReturnStmts)
375         DI->EmitLocation(Builder, EndLoc);
376       else
377         // We may not have a valid end location. Try to apply it anyway, and
378         // fall back to an artificial location if needed.
379         AL = ApplyDebugLocation::CreateDefaultArtificial(*this, EndLoc);
380     }
381 
382     PopCleanupBlocks(PrologueCleanupDepth);
383   }
384 
385   // Emit function epilog (to return).
386   llvm::DebugLoc Loc = EmitReturnBlock();
387 
388   if (ShouldInstrumentFunction()) {
389     if (CGM.getCodeGenOpts().InstrumentFunctions)
390       CurFn->addFnAttr("instrument-function-exit", "__cyg_profile_func_exit");
391     if (CGM.getCodeGenOpts().InstrumentFunctionsAfterInlining)
392       CurFn->addFnAttr("instrument-function-exit-inlined",
393                        "__cyg_profile_func_exit");
394   }
395 
396   // Emit debug descriptor for function end.
397   if (CGDebugInfo *DI = getDebugInfo())
398     DI->EmitFunctionEnd(Builder, CurFn);
399 
400   // Reset the debug location to that of the simple 'return' expression, if any
401   // rather than that of the end of the function's scope '}'.
402   ApplyDebugLocation AL(*this, Loc);
403   EmitFunctionEpilog(*CurFnInfo, EmitRetDbgLoc, EndLoc);
404   EmitEndEHSpec(CurCodeDecl);
405 
406   assert(EHStack.empty() &&
407          "did not remove all scopes from cleanup stack!");
408 
409   // If someone did an indirect goto, emit the indirect goto block at the end of
410   // the function.
411   if (IndirectBranch) {
412     EmitBlock(IndirectBranch->getParent());
413     Builder.ClearInsertionPoint();
414   }
415 
416   // If some of our locals escaped, insert a call to llvm.localescape in the
417   // entry block.
418   if (!EscapedLocals.empty()) {
419     // Invert the map from local to index into a simple vector. There should be
420     // no holes.
421     SmallVector<llvm::Value *, 4> EscapeArgs;
422     EscapeArgs.resize(EscapedLocals.size());
423     for (auto &Pair : EscapedLocals)
424       EscapeArgs[Pair.second] = Pair.first;
425     llvm::Function *FrameEscapeFn = llvm::Intrinsic::getDeclaration(
426         &CGM.getModule(), llvm::Intrinsic::localescape);
427     CGBuilderTy(*this, AllocaInsertPt).CreateCall(FrameEscapeFn, EscapeArgs);
428   }
429 
430   // Remove the AllocaInsertPt instruction, which is just a convenience for us.
431   llvm::Instruction *Ptr = AllocaInsertPt;
432   AllocaInsertPt = nullptr;
433   Ptr->eraseFromParent();
434 
435   // If someone took the address of a label but never did an indirect goto, we
436   // made a zero entry PHI node, which is illegal, zap it now.
437   if (IndirectBranch) {
438     llvm::PHINode *PN = cast<llvm::PHINode>(IndirectBranch->getAddress());
439     if (PN->getNumIncomingValues() == 0) {
440       PN->replaceAllUsesWith(llvm::UndefValue::get(PN->getType()));
441       PN->eraseFromParent();
442     }
443   }
444 
445   EmitIfUsed(*this, EHResumeBlock);
446   EmitIfUsed(*this, TerminateLandingPad);
447   EmitIfUsed(*this, TerminateHandler);
448   EmitIfUsed(*this, UnreachableBlock);
449 
450   for (const auto &FuncletAndParent : TerminateFunclets)
451     EmitIfUsed(*this, FuncletAndParent.second);
452 
453   if (CGM.getCodeGenOpts().EmitDeclMetadata)
454     EmitDeclMetadata();
455 
456   for (const auto &R : DeferredReplacements) {
457     if (llvm::Value *Old = R.first) {
458       Old->replaceAllUsesWith(R.second);
459       cast<llvm::Instruction>(Old)->eraseFromParent();
460     }
461   }
462   DeferredReplacements.clear();
463 
464   // Eliminate CleanupDestSlot alloca by replacing it with SSA values and
465   // PHIs if the current function is a coroutine. We don't do it for all
466   // functions as it may result in slight increase in numbers of instructions
467   // if compiled with no optimizations. We do it for coroutine as the lifetime
468   // of CleanupDestSlot alloca make correct coroutine frame building very
469   // difficult.
470   if (NormalCleanupDest.isValid() && isCoroutine()) {
471     llvm::DominatorTree DT(*CurFn);
472     llvm::PromoteMemToReg(
473         cast<llvm::AllocaInst>(NormalCleanupDest.getPointer()), DT);
474     NormalCleanupDest = Address::invalid();
475   }
476 
477   // Scan function arguments for vector width.
478   for (llvm::Argument &A : CurFn->args())
479     if (auto *VT = dyn_cast<llvm::VectorType>(A.getType()))
480       LargestVectorWidth =
481           std::max((uint64_t)LargestVectorWidth,
482                    VT->getPrimitiveSizeInBits().getKnownMinSize());
483 
484   // Update vector width based on return type.
485   if (auto *VT = dyn_cast<llvm::VectorType>(CurFn->getReturnType()))
486     LargestVectorWidth =
487         std::max((uint64_t)LargestVectorWidth,
488                  VT->getPrimitiveSizeInBits().getKnownMinSize());
489 
490   // Add the required-vector-width attribute. This contains the max width from:
491   // 1. min-vector-width attribute used in the source program.
492   // 2. Any builtins used that have a vector width specified.
493   // 3. Values passed in and out of inline assembly.
494   // 4. Width of vector arguments and return types for this function.
495   // 5. Width of vector aguments and return types for functions called by this
496   //    function.
497   CurFn->addFnAttr("min-legal-vector-width", llvm::utostr(LargestVectorWidth));
498 
499   // Add vscale attribute if appropriate.
500   if (getLangOpts().ArmSveVectorBits) {
501     unsigned VScale = getLangOpts().ArmSveVectorBits / 128;
502     CurFn->addFnAttr(llvm::Attribute::getWithVScaleRangeArgs(getLLVMContext(),
503                                                              VScale, VScale));
504   }
505 
506   // If we generated an unreachable return block, delete it now.
507   if (ReturnBlock.isValid() && ReturnBlock.getBlock()->use_empty()) {
508     Builder.ClearInsertionPoint();
509     ReturnBlock.getBlock()->eraseFromParent();
510   }
511   if (ReturnValue.isValid()) {
512     auto *RetAlloca = dyn_cast<llvm::AllocaInst>(ReturnValue.getPointer());
513     if (RetAlloca && RetAlloca->use_empty()) {
514       RetAlloca->eraseFromParent();
515       ReturnValue = Address::invalid();
516     }
517   }
518 }
519 
520 /// ShouldInstrumentFunction - Return true if the current function should be
521 /// instrumented with __cyg_profile_func_* calls
522 bool CodeGenFunction::ShouldInstrumentFunction() {
523   if (!CGM.getCodeGenOpts().InstrumentFunctions &&
524       !CGM.getCodeGenOpts().InstrumentFunctionsAfterInlining &&
525       !CGM.getCodeGenOpts().InstrumentFunctionEntryBare)
526     return false;
527   if (!CurFuncDecl || CurFuncDecl->hasAttr<NoInstrumentFunctionAttr>())
528     return false;
529   return true;
530 }
531 
532 /// ShouldXRayInstrument - Return true if the current function should be
533 /// instrumented with XRay nop sleds.
534 bool CodeGenFunction::ShouldXRayInstrumentFunction() const {
535   return CGM.getCodeGenOpts().XRayInstrumentFunctions;
536 }
537 
538 /// AlwaysEmitXRayCustomEvents - Return true if we should emit IR for calls to
539 /// the __xray_customevent(...) builtin calls, when doing XRay instrumentation.
540 bool CodeGenFunction::AlwaysEmitXRayCustomEvents() const {
541   return CGM.getCodeGenOpts().XRayInstrumentFunctions &&
542          (CGM.getCodeGenOpts().XRayAlwaysEmitCustomEvents ||
543           CGM.getCodeGenOpts().XRayInstrumentationBundle.Mask ==
544               XRayInstrKind::Custom);
545 }
546 
547 bool CodeGenFunction::AlwaysEmitXRayTypedEvents() const {
548   return CGM.getCodeGenOpts().XRayInstrumentFunctions &&
549          (CGM.getCodeGenOpts().XRayAlwaysEmitTypedEvents ||
550           CGM.getCodeGenOpts().XRayInstrumentationBundle.Mask ==
551               XRayInstrKind::Typed);
552 }
553 
554 llvm::Constant *
555 CodeGenFunction::EncodeAddrForUseInPrologue(llvm::Function *F,
556                                             llvm::Constant *Addr) {
557   // Addresses stored in prologue data can't require run-time fixups and must
558   // be PC-relative. Run-time fixups are undesirable because they necessitate
559   // writable text segments, which are unsafe. And absolute addresses are
560   // undesirable because they break PIE mode.
561 
562   // Add a layer of indirection through a private global. Taking its address
563   // won't result in a run-time fixup, even if Addr has linkonce_odr linkage.
564   auto *GV = new llvm::GlobalVariable(CGM.getModule(), Addr->getType(),
565                                       /*isConstant=*/true,
566                                       llvm::GlobalValue::PrivateLinkage, Addr);
567 
568   // Create a PC-relative address.
569   auto *GOTAsInt = llvm::ConstantExpr::getPtrToInt(GV, IntPtrTy);
570   auto *FuncAsInt = llvm::ConstantExpr::getPtrToInt(F, IntPtrTy);
571   auto *PCRelAsInt = llvm::ConstantExpr::getSub(GOTAsInt, FuncAsInt);
572   return (IntPtrTy == Int32Ty)
573              ? PCRelAsInt
574              : llvm::ConstantExpr::getTrunc(PCRelAsInt, Int32Ty);
575 }
576 
577 llvm::Value *
578 CodeGenFunction::DecodeAddrUsedInPrologue(llvm::Value *F,
579                                           llvm::Value *EncodedAddr) {
580   // Reconstruct the address of the global.
581   auto *PCRelAsInt = Builder.CreateSExt(EncodedAddr, IntPtrTy);
582   auto *FuncAsInt = Builder.CreatePtrToInt(F, IntPtrTy, "func_addr.int");
583   auto *GOTAsInt = Builder.CreateAdd(PCRelAsInt, FuncAsInt, "global_addr.int");
584   auto *GOTAddr = Builder.CreateIntToPtr(GOTAsInt, Int8PtrPtrTy, "global_addr");
585 
586   // Load the original pointer through the global.
587   return Builder.CreateLoad(Address(GOTAddr, getPointerAlign()),
588                             "decoded_addr");
589 }
590 
591 void CodeGenFunction::EmitOpenCLKernelMetadata(const FunctionDecl *FD,
592                                                llvm::Function *Fn)
593 {
594   if (!FD->hasAttr<OpenCLKernelAttr>())
595     return;
596 
597   llvm::LLVMContext &Context = getLLVMContext();
598 
599   CGM.GenOpenCLArgMetadata(Fn, FD, this);
600 
601   if (const VecTypeHintAttr *A = FD->getAttr<VecTypeHintAttr>()) {
602     QualType HintQTy = A->getTypeHint();
603     const ExtVectorType *HintEltQTy = HintQTy->getAs<ExtVectorType>();
604     bool IsSignedInteger =
605         HintQTy->isSignedIntegerType() ||
606         (HintEltQTy && HintEltQTy->getElementType()->isSignedIntegerType());
607     llvm::Metadata *AttrMDArgs[] = {
608         llvm::ConstantAsMetadata::get(llvm::UndefValue::get(
609             CGM.getTypes().ConvertType(A->getTypeHint()))),
610         llvm::ConstantAsMetadata::get(llvm::ConstantInt::get(
611             llvm::IntegerType::get(Context, 32),
612             llvm::APInt(32, (uint64_t)(IsSignedInteger ? 1 : 0))))};
613     Fn->setMetadata("vec_type_hint", llvm::MDNode::get(Context, AttrMDArgs));
614   }
615 
616   if (const WorkGroupSizeHintAttr *A = FD->getAttr<WorkGroupSizeHintAttr>()) {
617     llvm::Metadata *AttrMDArgs[] = {
618         llvm::ConstantAsMetadata::get(Builder.getInt32(A->getXDim())),
619         llvm::ConstantAsMetadata::get(Builder.getInt32(A->getYDim())),
620         llvm::ConstantAsMetadata::get(Builder.getInt32(A->getZDim()))};
621     Fn->setMetadata("work_group_size_hint", llvm::MDNode::get(Context, AttrMDArgs));
622   }
623 
624   if (const ReqdWorkGroupSizeAttr *A = FD->getAttr<ReqdWorkGroupSizeAttr>()) {
625     llvm::Metadata *AttrMDArgs[] = {
626         llvm::ConstantAsMetadata::get(Builder.getInt32(A->getXDim())),
627         llvm::ConstantAsMetadata::get(Builder.getInt32(A->getYDim())),
628         llvm::ConstantAsMetadata::get(Builder.getInt32(A->getZDim()))};
629     Fn->setMetadata("reqd_work_group_size", llvm::MDNode::get(Context, AttrMDArgs));
630   }
631 
632   if (const OpenCLIntelReqdSubGroupSizeAttr *A =
633           FD->getAttr<OpenCLIntelReqdSubGroupSizeAttr>()) {
634     llvm::Metadata *AttrMDArgs[] = {
635         llvm::ConstantAsMetadata::get(Builder.getInt32(A->getSubGroupSize()))};
636     Fn->setMetadata("intel_reqd_sub_group_size",
637                     llvm::MDNode::get(Context, AttrMDArgs));
638   }
639 }
640 
641 /// Determine whether the function F ends with a return stmt.
642 static bool endsWithReturn(const Decl* F) {
643   const Stmt *Body = nullptr;
644   if (auto *FD = dyn_cast_or_null<FunctionDecl>(F))
645     Body = FD->getBody();
646   else if (auto *OMD = dyn_cast_or_null<ObjCMethodDecl>(F))
647     Body = OMD->getBody();
648 
649   if (auto *CS = dyn_cast_or_null<CompoundStmt>(Body)) {
650     auto LastStmt = CS->body_rbegin();
651     if (LastStmt != CS->body_rend())
652       return isa<ReturnStmt>(*LastStmt);
653   }
654   return false;
655 }
656 
657 void CodeGenFunction::markAsIgnoreThreadCheckingAtRuntime(llvm::Function *Fn) {
658   if (SanOpts.has(SanitizerKind::Thread)) {
659     Fn->addFnAttr("sanitize_thread_no_checking_at_run_time");
660     Fn->removeFnAttr(llvm::Attribute::SanitizeThread);
661   }
662 }
663 
664 /// Check if the return value of this function requires sanitization.
665 bool CodeGenFunction::requiresReturnValueCheck() const {
666   return requiresReturnValueNullabilityCheck() ||
667          (SanOpts.has(SanitizerKind::ReturnsNonnullAttribute) && CurCodeDecl &&
668           CurCodeDecl->getAttr<ReturnsNonNullAttr>());
669 }
670 
671 static bool matchesStlAllocatorFn(const Decl *D, const ASTContext &Ctx) {
672   auto *MD = dyn_cast_or_null<CXXMethodDecl>(D);
673   if (!MD || !MD->getDeclName().getAsIdentifierInfo() ||
674       !MD->getDeclName().getAsIdentifierInfo()->isStr("allocate") ||
675       (MD->getNumParams() != 1 && MD->getNumParams() != 2))
676     return false;
677 
678   if (MD->parameters()[0]->getType().getCanonicalType() != Ctx.getSizeType())
679     return false;
680 
681   if (MD->getNumParams() == 2) {
682     auto *PT = MD->parameters()[1]->getType()->getAs<PointerType>();
683     if (!PT || !PT->isVoidPointerType() ||
684         !PT->getPointeeType().isConstQualified())
685       return false;
686   }
687 
688   return true;
689 }
690 
691 /// Return the UBSan prologue signature for \p FD if one is available.
692 static llvm::Constant *getPrologueSignature(CodeGenModule &CGM,
693                                             const FunctionDecl *FD) {
694   if (const auto *MD = dyn_cast<CXXMethodDecl>(FD))
695     if (!MD->isStatic())
696       return nullptr;
697   return CGM.getTargetCodeGenInfo().getUBSanFunctionSignature(CGM);
698 }
699 
700 void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy,
701                                     llvm::Function *Fn,
702                                     const CGFunctionInfo &FnInfo,
703                                     const FunctionArgList &Args,
704                                     SourceLocation Loc,
705                                     SourceLocation StartLoc) {
706   assert(!CurFn &&
707          "Do not use a CodeGenFunction object for more than one function");
708 
709   const Decl *D = GD.getDecl();
710 
711   DidCallStackSave = false;
712   CurCodeDecl = D;
713   if (const auto *FD = dyn_cast_or_null<FunctionDecl>(D))
714     if (FD->usesSEHTry())
715       CurSEHParent = FD;
716   CurFuncDecl = (D ? D->getNonClosureContext() : nullptr);
717   FnRetTy = RetTy;
718   CurFn = Fn;
719   CurFnInfo = &FnInfo;
720   assert(CurFn->isDeclaration() && "Function already has body?");
721 
722   // If this function is ignored for any of the enabled sanitizers,
723   // disable the sanitizer for the function.
724   do {
725 #define SANITIZER(NAME, ID)                                                    \
726   if (SanOpts.empty())                                                         \
727     break;                                                                     \
728   if (SanOpts.has(SanitizerKind::ID))                                          \
729     if (CGM.isInNoSanitizeList(SanitizerKind::ID, Fn, Loc))                    \
730       SanOpts.set(SanitizerKind::ID, false);
731 
732 #include "clang/Basic/Sanitizers.def"
733 #undef SANITIZER
734   } while (0);
735 
736   if (D) {
737     bool NoSanitizeCoverage = false;
738 
739     for (auto Attr : D->specific_attrs<NoSanitizeAttr>()) {
740       // Apply the no_sanitize* attributes to SanOpts.
741       SanitizerMask mask = Attr->getMask();
742       SanOpts.Mask &= ~mask;
743       if (mask & SanitizerKind::Address)
744         SanOpts.set(SanitizerKind::KernelAddress, false);
745       if (mask & SanitizerKind::KernelAddress)
746         SanOpts.set(SanitizerKind::Address, false);
747       if (mask & SanitizerKind::HWAddress)
748         SanOpts.set(SanitizerKind::KernelHWAddress, false);
749       if (mask & SanitizerKind::KernelHWAddress)
750         SanOpts.set(SanitizerKind::HWAddress, false);
751 
752       // SanitizeCoverage is not handled by SanOpts.
753       if (Attr->hasCoverage())
754         NoSanitizeCoverage = true;
755     }
756 
757     if (NoSanitizeCoverage && CGM.getCodeGenOpts().hasSanitizeCoverage())
758       Fn->addFnAttr(llvm::Attribute::NoSanitizeCoverage);
759   }
760 
761   // Apply sanitizer attributes to the function.
762   if (SanOpts.hasOneOf(SanitizerKind::Address | SanitizerKind::KernelAddress))
763     Fn->addFnAttr(llvm::Attribute::SanitizeAddress);
764   if (SanOpts.hasOneOf(SanitizerKind::HWAddress | SanitizerKind::KernelHWAddress))
765     Fn->addFnAttr(llvm::Attribute::SanitizeHWAddress);
766   if (SanOpts.has(SanitizerKind::MemTag))
767     Fn->addFnAttr(llvm::Attribute::SanitizeMemTag);
768   if (SanOpts.has(SanitizerKind::Thread))
769     Fn->addFnAttr(llvm::Attribute::SanitizeThread);
770   if (SanOpts.hasOneOf(SanitizerKind::Memory | SanitizerKind::KernelMemory))
771     Fn->addFnAttr(llvm::Attribute::SanitizeMemory);
772   if (SanOpts.has(SanitizerKind::SafeStack))
773     Fn->addFnAttr(llvm::Attribute::SafeStack);
774   if (SanOpts.has(SanitizerKind::ShadowCallStack))
775     Fn->addFnAttr(llvm::Attribute::ShadowCallStack);
776 
777   // Apply fuzzing attribute to the function.
778   if (SanOpts.hasOneOf(SanitizerKind::Fuzzer | SanitizerKind::FuzzerNoLink))
779     Fn->addFnAttr(llvm::Attribute::OptForFuzzing);
780 
781   // Ignore TSan memory acesses from within ObjC/ObjC++ dealloc, initialize,
782   // .cxx_destruct, __destroy_helper_block_ and all of their calees at run time.
783   if (SanOpts.has(SanitizerKind::Thread)) {
784     if (const auto *OMD = dyn_cast_or_null<ObjCMethodDecl>(D)) {
785       IdentifierInfo *II = OMD->getSelector().getIdentifierInfoForSlot(0);
786       if (OMD->getMethodFamily() == OMF_dealloc ||
787           OMD->getMethodFamily() == OMF_initialize ||
788           (OMD->getSelector().isUnarySelector() && II->isStr(".cxx_destruct"))) {
789         markAsIgnoreThreadCheckingAtRuntime(Fn);
790       }
791     }
792   }
793 
794   // Ignore unrelated casts in STL allocate() since the allocator must cast
795   // from void* to T* before object initialization completes. Don't match on the
796   // namespace because not all allocators are in std::
797   if (D && SanOpts.has(SanitizerKind::CFIUnrelatedCast)) {
798     if (matchesStlAllocatorFn(D, getContext()))
799       SanOpts.Mask &= ~SanitizerKind::CFIUnrelatedCast;
800   }
801 
802   // Ignore null checks in coroutine functions since the coroutines passes
803   // are not aware of how to move the extra UBSan instructions across the split
804   // coroutine boundaries.
805   if (D && SanOpts.has(SanitizerKind::Null))
806     if (const auto *FD = dyn_cast<FunctionDecl>(D))
807       if (FD->getBody() &&
808           FD->getBody()->getStmtClass() == Stmt::CoroutineBodyStmtClass)
809         SanOpts.Mask &= ~SanitizerKind::Null;
810 
811   // Apply xray attributes to the function (as a string, for now)
812   bool AlwaysXRayAttr = false;
813   if (const auto *XRayAttr = D ? D->getAttr<XRayInstrumentAttr>() : nullptr) {
814     if (CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
815             XRayInstrKind::FunctionEntry) ||
816         CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
817             XRayInstrKind::FunctionExit)) {
818       if (XRayAttr->alwaysXRayInstrument() && ShouldXRayInstrumentFunction()) {
819         Fn->addFnAttr("function-instrument", "xray-always");
820         AlwaysXRayAttr = true;
821       }
822       if (XRayAttr->neverXRayInstrument())
823         Fn->addFnAttr("function-instrument", "xray-never");
824       if (const auto *LogArgs = D->getAttr<XRayLogArgsAttr>())
825         if (ShouldXRayInstrumentFunction())
826           Fn->addFnAttr("xray-log-args",
827                         llvm::utostr(LogArgs->getArgumentCount()));
828     }
829   } else {
830     if (ShouldXRayInstrumentFunction() && !CGM.imbueXRayAttrs(Fn, Loc))
831       Fn->addFnAttr(
832           "xray-instruction-threshold",
833           llvm::itostr(CGM.getCodeGenOpts().XRayInstructionThreshold));
834   }
835 
836   if (ShouldXRayInstrumentFunction()) {
837     if (CGM.getCodeGenOpts().XRayIgnoreLoops)
838       Fn->addFnAttr("xray-ignore-loops");
839 
840     if (!CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
841             XRayInstrKind::FunctionExit))
842       Fn->addFnAttr("xray-skip-exit");
843 
844     if (!CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
845             XRayInstrKind::FunctionEntry))
846       Fn->addFnAttr("xray-skip-entry");
847 
848     auto FuncGroups = CGM.getCodeGenOpts().XRayTotalFunctionGroups;
849     if (FuncGroups > 1) {
850       auto FuncName = llvm::makeArrayRef<uint8_t>(
851           CurFn->getName().bytes_begin(), CurFn->getName().bytes_end());
852       auto Group = crc32(FuncName) % FuncGroups;
853       if (Group != CGM.getCodeGenOpts().XRaySelectedFunctionGroup &&
854           !AlwaysXRayAttr)
855         Fn->addFnAttr("function-instrument", "xray-never");
856     }
857   }
858 
859   if (CGM.getCodeGenOpts().getProfileInstr() != CodeGenOptions::ProfileNone)
860     if (CGM.isProfileInstrExcluded(Fn, Loc))
861       Fn->addFnAttr(llvm::Attribute::NoProfile);
862 
863   unsigned Count, Offset;
864   if (const auto *Attr =
865           D ? D->getAttr<PatchableFunctionEntryAttr>() : nullptr) {
866     Count = Attr->getCount();
867     Offset = Attr->getOffset();
868   } else {
869     Count = CGM.getCodeGenOpts().PatchableFunctionEntryCount;
870     Offset = CGM.getCodeGenOpts().PatchableFunctionEntryOffset;
871   }
872   if (Count && Offset <= Count) {
873     Fn->addFnAttr("patchable-function-entry", std::to_string(Count - Offset));
874     if (Offset)
875       Fn->addFnAttr("patchable-function-prefix", std::to_string(Offset));
876   }
877 
878   // Add no-jump-tables value.
879   if (CGM.getCodeGenOpts().NoUseJumpTables)
880     Fn->addFnAttr("no-jump-tables", "true");
881 
882   // Add no-inline-line-tables value.
883   if (CGM.getCodeGenOpts().NoInlineLineTables)
884     Fn->addFnAttr("no-inline-line-tables");
885 
886   // Add profile-sample-accurate value.
887   if (CGM.getCodeGenOpts().ProfileSampleAccurate)
888     Fn->addFnAttr("profile-sample-accurate");
889 
890   if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
891     Fn->addFnAttr("use-sample-profile");
892 
893   if (D && D->hasAttr<CFICanonicalJumpTableAttr>())
894     Fn->addFnAttr("cfi-canonical-jump-table");
895 
896   if (D && D->hasAttr<NoProfileFunctionAttr>())
897     Fn->addFnAttr(llvm::Attribute::NoProfile);
898 
899   if (getLangOpts().OpenCL) {
900     // Add metadata for a kernel function.
901     if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D))
902       EmitOpenCLKernelMetadata(FD, Fn);
903   }
904 
905   // If we are checking function types, emit a function type signature as
906   // prologue data.
907   if (getLangOpts().CPlusPlus && SanOpts.has(SanitizerKind::Function)) {
908     if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D)) {
909       if (llvm::Constant *PrologueSig = getPrologueSignature(CGM, FD)) {
910         // Remove any (C++17) exception specifications, to allow calling e.g. a
911         // noexcept function through a non-noexcept pointer.
912         auto ProtoTy =
913           getContext().getFunctionTypeWithExceptionSpec(FD->getType(),
914                                                         EST_None);
915         llvm::Constant *FTRTTIConst =
916             CGM.GetAddrOfRTTIDescriptor(ProtoTy, /*ForEH=*/true);
917         llvm::Constant *FTRTTIConstEncoded =
918             EncodeAddrForUseInPrologue(Fn, FTRTTIConst);
919         llvm::Constant *PrologueStructElems[] = {PrologueSig,
920                                                  FTRTTIConstEncoded};
921         llvm::Constant *PrologueStructConst =
922             llvm::ConstantStruct::getAnon(PrologueStructElems, /*Packed=*/true);
923         Fn->setPrologueData(PrologueStructConst);
924       }
925     }
926   }
927 
928   // If we're checking nullability, we need to know whether we can check the
929   // return value. Initialize the flag to 'true' and refine it in EmitParmDecl.
930   if (SanOpts.has(SanitizerKind::NullabilityReturn)) {
931     auto Nullability = FnRetTy->getNullability(getContext());
932     if (Nullability && *Nullability == NullabilityKind::NonNull) {
933       if (!(SanOpts.has(SanitizerKind::ReturnsNonnullAttribute) &&
934             CurCodeDecl && CurCodeDecl->getAttr<ReturnsNonNullAttr>()))
935         RetValNullabilityPrecondition =
936             llvm::ConstantInt::getTrue(getLLVMContext());
937     }
938   }
939 
940   // If we're in C++ mode and the function name is "main", it is guaranteed
941   // to be norecurse by the standard (3.6.1.3 "The function main shall not be
942   // used within a program").
943   //
944   // OpenCL C 2.0 v2.2-11 s6.9.i:
945   //     Recursion is not supported.
946   //
947   // SYCL v1.2.1 s3.10:
948   //     kernels cannot include RTTI information, exception classes,
949   //     recursive code, virtual functions or make use of C++ libraries that
950   //     are not compiled for the device.
951   if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D)) {
952     if ((getLangOpts().CPlusPlus && FD->isMain()) || getLangOpts().OpenCL ||
953         getLangOpts().SYCLIsDevice ||
954         (getLangOpts().CUDA && FD->hasAttr<CUDAGlobalAttr>()))
955       Fn->addFnAttr(llvm::Attribute::NoRecurse);
956   }
957 
958   if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D)) {
959     Builder.setIsFPConstrained(FD->hasAttr<StrictFPAttr>());
960     if (FD->hasAttr<StrictFPAttr>())
961       Fn->addFnAttr(llvm::Attribute::StrictFP);
962   }
963 
964   // If a custom alignment is used, force realigning to this alignment on
965   // any main function which certainly will need it.
966   if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D))
967     if ((FD->isMain() || FD->isMSVCRTEntryPoint()) &&
968         CGM.getCodeGenOpts().StackAlignment)
969       Fn->addFnAttr("stackrealign");
970 
971   llvm::BasicBlock *EntryBB = createBasicBlock("entry", CurFn);
972 
973   // Create a marker to make it easy to insert allocas into the entryblock
974   // later.  Don't create this with the builder, because we don't want it
975   // folded.
976   llvm::Value *Undef = llvm::UndefValue::get(Int32Ty);
977   AllocaInsertPt = new llvm::BitCastInst(Undef, Int32Ty, "allocapt", EntryBB);
978 
979   ReturnBlock = getJumpDestInCurrentScope("return");
980 
981   Builder.SetInsertPoint(EntryBB);
982 
983   // If we're checking the return value, allocate space for a pointer to a
984   // precise source location of the checked return statement.
985   if (requiresReturnValueCheck()) {
986     ReturnLocation = CreateDefaultAlignTempAlloca(Int8PtrTy, "return.sloc.ptr");
987     InitTempAlloca(ReturnLocation, llvm::ConstantPointerNull::get(Int8PtrTy));
988   }
989 
990   // Emit subprogram debug descriptor.
991   if (CGDebugInfo *DI = getDebugInfo()) {
992     // Reconstruct the type from the argument list so that implicit parameters,
993     // such as 'this' and 'vtt', show up in the debug info. Preserve the calling
994     // convention.
995     CallingConv CC = CallingConv::CC_C;
996     if (auto *FD = dyn_cast_or_null<FunctionDecl>(D))
997       if (const auto *SrcFnTy = FD->getType()->getAs<FunctionType>())
998         CC = SrcFnTy->getCallConv();
999     SmallVector<QualType, 16> ArgTypes;
1000     for (const VarDecl *VD : Args)
1001       ArgTypes.push_back(VD->getType());
1002     QualType FnType = getContext().getFunctionType(
1003         RetTy, ArgTypes, FunctionProtoType::ExtProtoInfo(CC));
1004     DI->emitFunctionStart(GD, Loc, StartLoc, FnType, CurFn, CurFuncIsThunk);
1005   }
1006 
1007   if (ShouldInstrumentFunction()) {
1008     if (CGM.getCodeGenOpts().InstrumentFunctions)
1009       CurFn->addFnAttr("instrument-function-entry", "__cyg_profile_func_enter");
1010     if (CGM.getCodeGenOpts().InstrumentFunctionsAfterInlining)
1011       CurFn->addFnAttr("instrument-function-entry-inlined",
1012                        "__cyg_profile_func_enter");
1013     if (CGM.getCodeGenOpts().InstrumentFunctionEntryBare)
1014       CurFn->addFnAttr("instrument-function-entry-inlined",
1015                        "__cyg_profile_func_enter_bare");
1016   }
1017 
1018   // Since emitting the mcount call here impacts optimizations such as function
1019   // inlining, we just add an attribute to insert a mcount call in backend.
1020   // The attribute "counting-function" is set to mcount function name which is
1021   // architecture dependent.
1022   if (CGM.getCodeGenOpts().InstrumentForProfiling) {
1023     // Calls to fentry/mcount should not be generated if function has
1024     // the no_instrument_function attribute.
1025     if (!CurFuncDecl || !CurFuncDecl->hasAttr<NoInstrumentFunctionAttr>()) {
1026       if (CGM.getCodeGenOpts().CallFEntry)
1027         Fn->addFnAttr("fentry-call", "true");
1028       else {
1029         Fn->addFnAttr("instrument-function-entry-inlined",
1030                       getTarget().getMCountName());
1031       }
1032       if (CGM.getCodeGenOpts().MNopMCount) {
1033         if (!CGM.getCodeGenOpts().CallFEntry)
1034           CGM.getDiags().Report(diag::err_opt_not_valid_without_opt)
1035             << "-mnop-mcount" << "-mfentry";
1036         Fn->addFnAttr("mnop-mcount");
1037       }
1038 
1039       if (CGM.getCodeGenOpts().RecordMCount) {
1040         if (!CGM.getCodeGenOpts().CallFEntry)
1041           CGM.getDiags().Report(diag::err_opt_not_valid_without_opt)
1042             << "-mrecord-mcount" << "-mfentry";
1043         Fn->addFnAttr("mrecord-mcount");
1044       }
1045     }
1046   }
1047 
1048   if (CGM.getCodeGenOpts().PackedStack) {
1049     if (getContext().getTargetInfo().getTriple().getArch() !=
1050         llvm::Triple::systemz)
1051       CGM.getDiags().Report(diag::err_opt_not_valid_on_target)
1052         << "-mpacked-stack";
1053     Fn->addFnAttr("packed-stack");
1054   }
1055 
1056   if (CGM.getCodeGenOpts().WarnStackSize != UINT_MAX)
1057     Fn->addFnAttr("warn-stack-size",
1058                   std::to_string(CGM.getCodeGenOpts().WarnStackSize));
1059 
1060   if (RetTy->isVoidType()) {
1061     // Void type; nothing to return.
1062     ReturnValue = Address::invalid();
1063 
1064     // Count the implicit return.
1065     if (!endsWithReturn(D))
1066       ++NumReturnExprs;
1067   } else if (CurFnInfo->getReturnInfo().getKind() == ABIArgInfo::Indirect) {
1068     // Indirect return; emit returned value directly into sret slot.
1069     // This reduces code size, and affects correctness in C++.
1070     auto AI = CurFn->arg_begin();
1071     if (CurFnInfo->getReturnInfo().isSRetAfterThis())
1072       ++AI;
1073     ReturnValue = Address(&*AI, CurFnInfo->getReturnInfo().getIndirectAlign());
1074     if (!CurFnInfo->getReturnInfo().getIndirectByVal()) {
1075       ReturnValuePointer =
1076           CreateDefaultAlignTempAlloca(Int8PtrTy, "result.ptr");
1077       Builder.CreateStore(Builder.CreatePointerBitCastOrAddrSpaceCast(
1078                               ReturnValue.getPointer(), Int8PtrTy),
1079                           ReturnValuePointer);
1080     }
1081   } else if (CurFnInfo->getReturnInfo().getKind() == ABIArgInfo::InAlloca &&
1082              !hasScalarEvaluationKind(CurFnInfo->getReturnType())) {
1083     // Load the sret pointer from the argument struct and return into that.
1084     unsigned Idx = CurFnInfo->getReturnInfo().getInAllocaFieldIndex();
1085     llvm::Function::arg_iterator EI = CurFn->arg_end();
1086     --EI;
1087     llvm::Value *Addr = Builder.CreateStructGEP(nullptr, &*EI, Idx);
1088     llvm::Type *Ty =
1089         cast<llvm::GetElementPtrInst>(Addr)->getResultElementType();
1090     ReturnValuePointer = Address(Addr, getPointerAlign());
1091     Addr = Builder.CreateAlignedLoad(Ty, Addr, getPointerAlign(), "agg.result");
1092     ReturnValue = Address(Addr, CGM.getNaturalTypeAlignment(RetTy));
1093   } else {
1094     ReturnValue = CreateIRTemp(RetTy, "retval");
1095 
1096     // Tell the epilog emitter to autorelease the result.  We do this
1097     // now so that various specialized functions can suppress it
1098     // during their IR-generation.
1099     if (getLangOpts().ObjCAutoRefCount &&
1100         !CurFnInfo->isReturnsRetained() &&
1101         RetTy->isObjCRetainableType())
1102       AutoreleaseResult = true;
1103   }
1104 
1105   EmitStartEHSpec(CurCodeDecl);
1106 
1107   PrologueCleanupDepth = EHStack.stable_begin();
1108 
1109   // Emit OpenMP specific initialization of the device functions.
1110   if (getLangOpts().OpenMP && CurCodeDecl)
1111     CGM.getOpenMPRuntime().emitFunctionProlog(*this, CurCodeDecl);
1112 
1113   EmitFunctionProlog(*CurFnInfo, CurFn, Args);
1114 
1115   if (D && isa<CXXMethodDecl>(D) && cast<CXXMethodDecl>(D)->isInstance()) {
1116     CGM.getCXXABI().EmitInstanceFunctionProlog(*this);
1117     const CXXMethodDecl *MD = cast<CXXMethodDecl>(D);
1118     if (MD->getParent()->isLambda() &&
1119         MD->getOverloadedOperator() == OO_Call) {
1120       // We're in a lambda; figure out the captures.
1121       MD->getParent()->getCaptureFields(LambdaCaptureFields,
1122                                         LambdaThisCaptureField);
1123       if (LambdaThisCaptureField) {
1124         // If the lambda captures the object referred to by '*this' - either by
1125         // value or by reference, make sure CXXThisValue points to the correct
1126         // object.
1127 
1128         // Get the lvalue for the field (which is a copy of the enclosing object
1129         // or contains the address of the enclosing object).
1130         LValue ThisFieldLValue = EmitLValueForLambdaField(LambdaThisCaptureField);
1131         if (!LambdaThisCaptureField->getType()->isPointerType()) {
1132           // If the enclosing object was captured by value, just use its address.
1133           CXXThisValue = ThisFieldLValue.getAddress(*this).getPointer();
1134         } else {
1135           // Load the lvalue pointed to by the field, since '*this' was captured
1136           // by reference.
1137           CXXThisValue =
1138               EmitLoadOfLValue(ThisFieldLValue, SourceLocation()).getScalarVal();
1139         }
1140       }
1141       for (auto *FD : MD->getParent()->fields()) {
1142         if (FD->hasCapturedVLAType()) {
1143           auto *ExprArg = EmitLoadOfLValue(EmitLValueForLambdaField(FD),
1144                                            SourceLocation()).getScalarVal();
1145           auto VAT = FD->getCapturedVLAType();
1146           VLASizeMap[VAT->getSizeExpr()] = ExprArg;
1147         }
1148       }
1149     } else {
1150       // Not in a lambda; just use 'this' from the method.
1151       // FIXME: Should we generate a new load for each use of 'this'?  The
1152       // fast register allocator would be happier...
1153       CXXThisValue = CXXABIThisValue;
1154     }
1155 
1156     // Check the 'this' pointer once per function, if it's available.
1157     if (CXXABIThisValue) {
1158       SanitizerSet SkippedChecks;
1159       SkippedChecks.set(SanitizerKind::ObjectSize, true);
1160       QualType ThisTy = MD->getThisType();
1161 
1162       // If this is the call operator of a lambda with no capture-default, it
1163       // may have a static invoker function, which may call this operator with
1164       // a null 'this' pointer.
1165       if (isLambdaCallOperator(MD) &&
1166           MD->getParent()->getLambdaCaptureDefault() == LCD_None)
1167         SkippedChecks.set(SanitizerKind::Null, true);
1168 
1169       EmitTypeCheck(
1170           isa<CXXConstructorDecl>(MD) ? TCK_ConstructorCall : TCK_MemberCall,
1171           Loc, CXXABIThisValue, ThisTy, CXXABIThisAlignment, SkippedChecks);
1172     }
1173   }
1174 
1175   // If any of the arguments have a variably modified type, make sure to
1176   // emit the type size.
1177   for (FunctionArgList::const_iterator i = Args.begin(), e = Args.end();
1178        i != e; ++i) {
1179     const VarDecl *VD = *i;
1180 
1181     // Dig out the type as written from ParmVarDecls; it's unclear whether
1182     // the standard (C99 6.9.1p10) requires this, but we're following the
1183     // precedent set by gcc.
1184     QualType Ty;
1185     if (const ParmVarDecl *PVD = dyn_cast<ParmVarDecl>(VD))
1186       Ty = PVD->getOriginalType();
1187     else
1188       Ty = VD->getType();
1189 
1190     if (Ty->isVariablyModifiedType())
1191       EmitVariablyModifiedType(Ty);
1192   }
1193   // Emit a location at the end of the prologue.
1194   if (CGDebugInfo *DI = getDebugInfo())
1195     DI->EmitLocation(Builder, StartLoc);
1196 
1197   // TODO: Do we need to handle this in two places like we do with
1198   // target-features/target-cpu?
1199   if (CurFuncDecl)
1200     if (const auto *VecWidth = CurFuncDecl->getAttr<MinVectorWidthAttr>())
1201       LargestVectorWidth = VecWidth->getVectorWidth();
1202 }
1203 
1204 void CodeGenFunction::EmitFunctionBody(const Stmt *Body) {
1205   incrementProfileCounter(Body);
1206   if (const CompoundStmt *S = dyn_cast<CompoundStmt>(Body))
1207     EmitCompoundStmtWithoutScope(*S);
1208   else
1209     EmitStmt(Body);
1210 
1211   // This is checked after emitting the function body so we know if there
1212   // are any permitted infinite loops.
1213   if (checkIfFunctionMustProgress())
1214     CurFn->addFnAttr(llvm::Attribute::MustProgress);
1215 }
1216 
1217 /// When instrumenting to collect profile data, the counts for some blocks
1218 /// such as switch cases need to not include the fall-through counts, so
1219 /// emit a branch around the instrumentation code. When not instrumenting,
1220 /// this just calls EmitBlock().
1221 void CodeGenFunction::EmitBlockWithFallThrough(llvm::BasicBlock *BB,
1222                                                const Stmt *S) {
1223   llvm::BasicBlock *SkipCountBB = nullptr;
1224   if (HaveInsertPoint() && CGM.getCodeGenOpts().hasProfileClangInstr()) {
1225     // When instrumenting for profiling, the fallthrough to certain
1226     // statements needs to skip over the instrumentation code so that we
1227     // get an accurate count.
1228     SkipCountBB = createBasicBlock("skipcount");
1229     EmitBranch(SkipCountBB);
1230   }
1231   EmitBlock(BB);
1232   uint64_t CurrentCount = getCurrentProfileCount();
1233   incrementProfileCounter(S);
1234   setCurrentProfileCount(getCurrentProfileCount() + CurrentCount);
1235   if (SkipCountBB)
1236     EmitBlock(SkipCountBB);
1237 }
1238 
1239 /// Tries to mark the given function nounwind based on the
1240 /// non-existence of any throwing calls within it.  We believe this is
1241 /// lightweight enough to do at -O0.
1242 static void TryMarkNoThrow(llvm::Function *F) {
1243   // LLVM treats 'nounwind' on a function as part of the type, so we
1244   // can't do this on functions that can be overwritten.
1245   if (F->isInterposable()) return;
1246 
1247   for (llvm::BasicBlock &BB : *F)
1248     for (llvm::Instruction &I : BB)
1249       if (I.mayThrow())
1250         return;
1251 
1252   F->setDoesNotThrow();
1253 }
1254 
1255 QualType CodeGenFunction::BuildFunctionArgList(GlobalDecl GD,
1256                                                FunctionArgList &Args) {
1257   const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());
1258   QualType ResTy = FD->getReturnType();
1259 
1260   const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(FD);
1261   if (MD && MD->isInstance()) {
1262     if (CGM.getCXXABI().HasThisReturn(GD))
1263       ResTy = MD->getThisType();
1264     else if (CGM.getCXXABI().hasMostDerivedReturn(GD))
1265       ResTy = CGM.getContext().VoidPtrTy;
1266     CGM.getCXXABI().buildThisParam(*this, Args);
1267   }
1268 
1269   // The base version of an inheriting constructor whose constructed base is a
1270   // virtual base is not passed any arguments (because it doesn't actually call
1271   // the inherited constructor).
1272   bool PassedParams = true;
1273   if (const CXXConstructorDecl *CD = dyn_cast<CXXConstructorDecl>(FD))
1274     if (auto Inherited = CD->getInheritedConstructor())
1275       PassedParams =
1276           getTypes().inheritingCtorHasParams(Inherited, GD.getCtorType());
1277 
1278   if (PassedParams) {
1279     for (auto *Param : FD->parameters()) {
1280       Args.push_back(Param);
1281       if (!Param->hasAttr<PassObjectSizeAttr>())
1282         continue;
1283 
1284       auto *Implicit = ImplicitParamDecl::Create(
1285           getContext(), Param->getDeclContext(), Param->getLocation(),
1286           /*Id=*/nullptr, getContext().getSizeType(), ImplicitParamDecl::Other);
1287       SizeArguments[Param] = Implicit;
1288       Args.push_back(Implicit);
1289     }
1290   }
1291 
1292   if (MD && (isa<CXXConstructorDecl>(MD) || isa<CXXDestructorDecl>(MD)))
1293     CGM.getCXXABI().addImplicitStructorParams(*this, ResTy, Args);
1294 
1295   return ResTy;
1296 }
1297 
1298 void CodeGenFunction::GenerateCode(GlobalDecl GD, llvm::Function *Fn,
1299                                    const CGFunctionInfo &FnInfo) {
1300   const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());
1301   CurGD = GD;
1302 
1303   FunctionArgList Args;
1304   QualType ResTy = BuildFunctionArgList(GD, Args);
1305 
1306   // Check if we should generate debug info for this function.
1307   if (FD->hasAttr<NoDebugAttr>())
1308     DebugInfo = nullptr; // disable debug info indefinitely for this function
1309 
1310   // The function might not have a body if we're generating thunks for a
1311   // function declaration.
1312   SourceRange BodyRange;
1313   if (Stmt *Body = FD->getBody())
1314     BodyRange = Body->getSourceRange();
1315   else
1316     BodyRange = FD->getLocation();
1317   CurEHLocation = BodyRange.getEnd();
1318 
1319   // Use the location of the start of the function to determine where
1320   // the function definition is located. By default use the location
1321   // of the declaration as the location for the subprogram. A function
1322   // may lack a declaration in the source code if it is created by code
1323   // gen. (examples: _GLOBAL__I_a, __cxx_global_array_dtor, thunk).
1324   SourceLocation Loc = FD->getLocation();
1325 
1326   // If this is a function specialization then use the pattern body
1327   // as the location for the function.
1328   if (const FunctionDecl *SpecDecl = FD->getTemplateInstantiationPattern())
1329     if (SpecDecl->hasBody(SpecDecl))
1330       Loc = SpecDecl->getLocation();
1331 
1332   Stmt *Body = FD->getBody();
1333 
1334   if (Body) {
1335     // Coroutines always emit lifetime markers.
1336     if (isa<CoroutineBodyStmt>(Body))
1337       ShouldEmitLifetimeMarkers = true;
1338 
1339     // Initialize helper which will detect jumps which can cause invalid
1340     // lifetime markers.
1341     if (ShouldEmitLifetimeMarkers)
1342       Bypasses.Init(Body);
1343   }
1344 
1345   // Emit the standard function prologue.
1346   StartFunction(GD, ResTy, Fn, FnInfo, Args, Loc, BodyRange.getBegin());
1347 
1348   // Save parameters for coroutine function.
1349   if (Body && isa_and_nonnull<CoroutineBodyStmt>(Body))
1350     for (const auto *ParamDecl : FD->parameters())
1351       FnArgs.push_back(ParamDecl);
1352 
1353   // Generate the body of the function.
1354   PGO.assignRegionCounters(GD, CurFn);
1355   if (isa<CXXDestructorDecl>(FD))
1356     EmitDestructorBody(Args);
1357   else if (isa<CXXConstructorDecl>(FD))
1358     EmitConstructorBody(Args);
1359   else if (getLangOpts().CUDA &&
1360            !getLangOpts().CUDAIsDevice &&
1361            FD->hasAttr<CUDAGlobalAttr>())
1362     CGM.getCUDARuntime().emitDeviceStub(*this, Args);
1363   else if (isa<CXXMethodDecl>(FD) &&
1364            cast<CXXMethodDecl>(FD)->isLambdaStaticInvoker()) {
1365     // The lambda static invoker function is special, because it forwards or
1366     // clones the body of the function call operator (but is actually static).
1367     EmitLambdaStaticInvokeBody(cast<CXXMethodDecl>(FD));
1368   } else if (FD->isDefaulted() && isa<CXXMethodDecl>(FD) &&
1369              (cast<CXXMethodDecl>(FD)->isCopyAssignmentOperator() ||
1370               cast<CXXMethodDecl>(FD)->isMoveAssignmentOperator())) {
1371     // Implicit copy-assignment gets the same special treatment as implicit
1372     // copy-constructors.
1373     emitImplicitAssignmentOperatorBody(Args);
1374   } else if (Body) {
1375     EmitFunctionBody(Body);
1376   } else
1377     llvm_unreachable("no definition for emitted function");
1378 
1379   // C++11 [stmt.return]p2:
1380   //   Flowing off the end of a function [...] results in undefined behavior in
1381   //   a value-returning function.
1382   // C11 6.9.1p12:
1383   //   If the '}' that terminates a function is reached, and the value of the
1384   //   function call is used by the caller, the behavior is undefined.
1385   if (getLangOpts().CPlusPlus && !FD->hasImplicitReturnZero() && !SawAsmBlock &&
1386       !FD->getReturnType()->isVoidType() && Builder.GetInsertBlock()) {
1387     bool ShouldEmitUnreachable =
1388         CGM.getCodeGenOpts().StrictReturn ||
1389         !CGM.MayDropFunctionReturn(FD->getASTContext(), FD->getReturnType());
1390     if (SanOpts.has(SanitizerKind::Return)) {
1391       SanitizerScope SanScope(this);
1392       llvm::Value *IsFalse = Builder.getFalse();
1393       EmitCheck(std::make_pair(IsFalse, SanitizerKind::Return),
1394                 SanitizerHandler::MissingReturn,
1395                 EmitCheckSourceLocation(FD->getLocation()), None);
1396     } else if (ShouldEmitUnreachable) {
1397       if (CGM.getCodeGenOpts().OptimizationLevel == 0)
1398         EmitTrapCall(llvm::Intrinsic::trap);
1399     }
1400     if (SanOpts.has(SanitizerKind::Return) || ShouldEmitUnreachable) {
1401       Builder.CreateUnreachable();
1402       Builder.ClearInsertionPoint();
1403     }
1404   }
1405 
1406   // Emit the standard function epilogue.
1407   FinishFunction(BodyRange.getEnd());
1408 
1409   // If we haven't marked the function nothrow through other means, do
1410   // a quick pass now to see if we can.
1411   if (!CurFn->doesNotThrow())
1412     TryMarkNoThrow(CurFn);
1413 }
1414 
1415 /// ContainsLabel - Return true if the statement contains a label in it.  If
1416 /// this statement is not executed normally, it not containing a label means
1417 /// that we can just remove the code.
1418 bool CodeGenFunction::ContainsLabel(const Stmt *S, bool IgnoreCaseStmts) {
1419   // Null statement, not a label!
1420   if (!S) return false;
1421 
1422   // If this is a label, we have to emit the code, consider something like:
1423   // if (0) {  ...  foo:  bar(); }  goto foo;
1424   //
1425   // TODO: If anyone cared, we could track __label__'s, since we know that you
1426   // can't jump to one from outside their declared region.
1427   if (isa<LabelStmt>(S))
1428     return true;
1429 
1430   // If this is a case/default statement, and we haven't seen a switch, we have
1431   // to emit the code.
1432   if (isa<SwitchCase>(S) && !IgnoreCaseStmts)
1433     return true;
1434 
1435   // If this is a switch statement, we want to ignore cases below it.
1436   if (isa<SwitchStmt>(S))
1437     IgnoreCaseStmts = true;
1438 
1439   // Scan subexpressions for verboten labels.
1440   for (const Stmt *SubStmt : S->children())
1441     if (ContainsLabel(SubStmt, IgnoreCaseStmts))
1442       return true;
1443 
1444   return false;
1445 }
1446 
1447 /// containsBreak - Return true if the statement contains a break out of it.
1448 /// If the statement (recursively) contains a switch or loop with a break
1449 /// inside of it, this is fine.
1450 bool CodeGenFunction::containsBreak(const Stmt *S) {
1451   // Null statement, not a label!
1452   if (!S) return false;
1453 
1454   // If this is a switch or loop that defines its own break scope, then we can
1455   // include it and anything inside of it.
1456   if (isa<SwitchStmt>(S) || isa<WhileStmt>(S) || isa<DoStmt>(S) ||
1457       isa<ForStmt>(S))
1458     return false;
1459 
1460   if (isa<BreakStmt>(S))
1461     return true;
1462 
1463   // Scan subexpressions for verboten breaks.
1464   for (const Stmt *SubStmt : S->children())
1465     if (containsBreak(SubStmt))
1466       return true;
1467 
1468   return false;
1469 }
1470 
1471 bool CodeGenFunction::mightAddDeclToScope(const Stmt *S) {
1472   if (!S) return false;
1473 
1474   // Some statement kinds add a scope and thus never add a decl to the current
1475   // scope. Note, this list is longer than the list of statements that might
1476   // have an unscoped decl nested within them, but this way is conservatively
1477   // correct even if more statement kinds are added.
1478   if (isa<IfStmt>(S) || isa<SwitchStmt>(S) || isa<WhileStmt>(S) ||
1479       isa<DoStmt>(S) || isa<ForStmt>(S) || isa<CompoundStmt>(S) ||
1480       isa<CXXForRangeStmt>(S) || isa<CXXTryStmt>(S) ||
1481       isa<ObjCForCollectionStmt>(S) || isa<ObjCAtTryStmt>(S))
1482     return false;
1483 
1484   if (isa<DeclStmt>(S))
1485     return true;
1486 
1487   for (const Stmt *SubStmt : S->children())
1488     if (mightAddDeclToScope(SubStmt))
1489       return true;
1490 
1491   return false;
1492 }
1493 
1494 /// ConstantFoldsToSimpleInteger - If the specified expression does not fold
1495 /// to a constant, or if it does but contains a label, return false.  If it
1496 /// constant folds return true and set the boolean result in Result.
1497 bool CodeGenFunction::ConstantFoldsToSimpleInteger(const Expr *Cond,
1498                                                    bool &ResultBool,
1499                                                    bool AllowLabels) {
1500   llvm::APSInt ResultInt;
1501   if (!ConstantFoldsToSimpleInteger(Cond, ResultInt, AllowLabels))
1502     return false;
1503 
1504   ResultBool = ResultInt.getBoolValue();
1505   return true;
1506 }
1507 
1508 /// ConstantFoldsToSimpleInteger - If the specified expression does not fold
1509 /// to a constant, or if it does but contains a label, return false.  If it
1510 /// constant folds return true and set the folded value.
1511 bool CodeGenFunction::ConstantFoldsToSimpleInteger(const Expr *Cond,
1512                                                    llvm::APSInt &ResultInt,
1513                                                    bool AllowLabels) {
1514   // FIXME: Rename and handle conversion of other evaluatable things
1515   // to bool.
1516   Expr::EvalResult Result;
1517   if (!Cond->EvaluateAsInt(Result, getContext()))
1518     return false;  // Not foldable, not integer or not fully evaluatable.
1519 
1520   llvm::APSInt Int = Result.Val.getInt();
1521   if (!AllowLabels && CodeGenFunction::ContainsLabel(Cond))
1522     return false;  // Contains a label.
1523 
1524   ResultInt = Int;
1525   return true;
1526 }
1527 
1528 /// Determine whether the given condition is an instrumentable condition
1529 /// (i.e. no "&&" or "||").
1530 bool CodeGenFunction::isInstrumentedCondition(const Expr *C) {
1531   // Bypass simplistic logical-NOT operator before determining whether the
1532   // condition contains any other logical operator.
1533   if (const UnaryOperator *UnOp = dyn_cast<UnaryOperator>(C->IgnoreParens()))
1534     if (UnOp->getOpcode() == UO_LNot)
1535       C = UnOp->getSubExpr();
1536 
1537   const BinaryOperator *BOp = dyn_cast<BinaryOperator>(C->IgnoreParens());
1538   return (!BOp || !BOp->isLogicalOp());
1539 }
1540 
1541 /// EmitBranchToCounterBlock - Emit a conditional branch to a new block that
1542 /// increments a profile counter based on the semantics of the given logical
1543 /// operator opcode.  This is used to instrument branch condition coverage for
1544 /// logical operators.
1545 void CodeGenFunction::EmitBranchToCounterBlock(
1546     const Expr *Cond, BinaryOperator::Opcode LOp, llvm::BasicBlock *TrueBlock,
1547     llvm::BasicBlock *FalseBlock, uint64_t TrueCount /* = 0 */,
1548     Stmt::Likelihood LH /* =None */, const Expr *CntrIdx /* = nullptr */) {
1549   // If not instrumenting, just emit a branch.
1550   bool InstrumentRegions = CGM.getCodeGenOpts().hasProfileClangInstr();
1551   if (!InstrumentRegions || !isInstrumentedCondition(Cond))
1552     return EmitBranchOnBoolExpr(Cond, TrueBlock, FalseBlock, TrueCount, LH);
1553 
1554   llvm::BasicBlock *ThenBlock = NULL;
1555   llvm::BasicBlock *ElseBlock = NULL;
1556   llvm::BasicBlock *NextBlock = NULL;
1557 
1558   // Create the block we'll use to increment the appropriate counter.
1559   llvm::BasicBlock *CounterIncrBlock = createBasicBlock("lop.rhscnt");
1560 
1561   // Set block pointers according to Logical-AND (BO_LAnd) semantics. This
1562   // means we need to evaluate the condition and increment the counter on TRUE:
1563   //
1564   // if (Cond)
1565   //   goto CounterIncrBlock;
1566   // else
1567   //   goto FalseBlock;
1568   //
1569   // CounterIncrBlock:
1570   //   Counter++;
1571   //   goto TrueBlock;
1572 
1573   if (LOp == BO_LAnd) {
1574     ThenBlock = CounterIncrBlock;
1575     ElseBlock = FalseBlock;
1576     NextBlock = TrueBlock;
1577   }
1578 
1579   // Set block pointers according to Logical-OR (BO_LOr) semantics. This means
1580   // we need to evaluate the condition and increment the counter on FALSE:
1581   //
1582   // if (Cond)
1583   //   goto TrueBlock;
1584   // else
1585   //   goto CounterIncrBlock;
1586   //
1587   // CounterIncrBlock:
1588   //   Counter++;
1589   //   goto FalseBlock;
1590 
1591   else if (LOp == BO_LOr) {
1592     ThenBlock = TrueBlock;
1593     ElseBlock = CounterIncrBlock;
1594     NextBlock = FalseBlock;
1595   } else {
1596     llvm_unreachable("Expected Opcode must be that of a Logical Operator");
1597   }
1598 
1599   // Emit Branch based on condition.
1600   EmitBranchOnBoolExpr(Cond, ThenBlock, ElseBlock, TrueCount, LH);
1601 
1602   // Emit the block containing the counter increment(s).
1603   EmitBlock(CounterIncrBlock);
1604 
1605   // Increment corresponding counter; if index not provided, use Cond as index.
1606   incrementProfileCounter(CntrIdx ? CntrIdx : Cond);
1607 
1608   // Go to the next block.
1609   EmitBranch(NextBlock);
1610 }
1611 
1612 /// EmitBranchOnBoolExpr - Emit a branch on a boolean condition (e.g. for an if
1613 /// statement) to the specified blocks.  Based on the condition, this might try
1614 /// to simplify the codegen of the conditional based on the branch.
1615 /// \param LH The value of the likelihood attribute on the True branch.
1616 void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond,
1617                                            llvm::BasicBlock *TrueBlock,
1618                                            llvm::BasicBlock *FalseBlock,
1619                                            uint64_t TrueCount,
1620                                            Stmt::Likelihood LH) {
1621   Cond = Cond->IgnoreParens();
1622 
1623   if (const BinaryOperator *CondBOp = dyn_cast<BinaryOperator>(Cond)) {
1624 
1625     // Handle X && Y in a condition.
1626     if (CondBOp->getOpcode() == BO_LAnd) {
1627       // If we have "1 && X", simplify the code.  "0 && X" would have constant
1628       // folded if the case was simple enough.
1629       bool ConstantBool = false;
1630       if (ConstantFoldsToSimpleInteger(CondBOp->getLHS(), ConstantBool) &&
1631           ConstantBool) {
1632         // br(1 && X) -> br(X).
1633         incrementProfileCounter(CondBOp);
1634         return EmitBranchToCounterBlock(CondBOp->getRHS(), BO_LAnd, TrueBlock,
1635                                         FalseBlock, TrueCount, LH);
1636       }
1637 
1638       // If we have "X && 1", simplify the code to use an uncond branch.
1639       // "X && 0" would have been constant folded to 0.
1640       if (ConstantFoldsToSimpleInteger(CondBOp->getRHS(), ConstantBool) &&
1641           ConstantBool) {
1642         // br(X && 1) -> br(X).
1643         return EmitBranchToCounterBlock(CondBOp->getLHS(), BO_LAnd, TrueBlock,
1644                                         FalseBlock, TrueCount, LH, CondBOp);
1645       }
1646 
1647       // Emit the LHS as a conditional.  If the LHS conditional is false, we
1648       // want to jump to the FalseBlock.
1649       llvm::BasicBlock *LHSTrue = createBasicBlock("land.lhs.true");
1650       // The counter tells us how often we evaluate RHS, and all of TrueCount
1651       // can be propagated to that branch.
1652       uint64_t RHSCount = getProfileCount(CondBOp->getRHS());
1653 
1654       ConditionalEvaluation eval(*this);
1655       {
1656         ApplyDebugLocation DL(*this, Cond);
1657         // Propagate the likelihood attribute like __builtin_expect
1658         // __builtin_expect(X && Y, 1) -> X and Y are likely
1659         // __builtin_expect(X && Y, 0) -> only Y is unlikely
1660         EmitBranchOnBoolExpr(CondBOp->getLHS(), LHSTrue, FalseBlock, RHSCount,
1661                              LH == Stmt::LH_Unlikely ? Stmt::LH_None : LH);
1662         EmitBlock(LHSTrue);
1663       }
1664 
1665       incrementProfileCounter(CondBOp);
1666       setCurrentProfileCount(getProfileCount(CondBOp->getRHS()));
1667 
1668       // Any temporaries created here are conditional.
1669       eval.begin(*this);
1670       EmitBranchToCounterBlock(CondBOp->getRHS(), BO_LAnd, TrueBlock,
1671                                FalseBlock, TrueCount, LH);
1672       eval.end(*this);
1673 
1674       return;
1675     }
1676 
1677     if (CondBOp->getOpcode() == BO_LOr) {
1678       // If we have "0 || X", simplify the code.  "1 || X" would have constant
1679       // folded if the case was simple enough.
1680       bool ConstantBool = false;
1681       if (ConstantFoldsToSimpleInteger(CondBOp->getLHS(), ConstantBool) &&
1682           !ConstantBool) {
1683         // br(0 || X) -> br(X).
1684         incrementProfileCounter(CondBOp);
1685         return EmitBranchToCounterBlock(CondBOp->getRHS(), BO_LOr, TrueBlock,
1686                                         FalseBlock, TrueCount, LH);
1687       }
1688 
1689       // If we have "X || 0", simplify the code to use an uncond branch.
1690       // "X || 1" would have been constant folded to 1.
1691       if (ConstantFoldsToSimpleInteger(CondBOp->getRHS(), ConstantBool) &&
1692           !ConstantBool) {
1693         // br(X || 0) -> br(X).
1694         return EmitBranchToCounterBlock(CondBOp->getLHS(), BO_LOr, TrueBlock,
1695                                         FalseBlock, TrueCount, LH, CondBOp);
1696       }
1697 
1698       // Emit the LHS as a conditional.  If the LHS conditional is true, we
1699       // want to jump to the TrueBlock.
1700       llvm::BasicBlock *LHSFalse = createBasicBlock("lor.lhs.false");
1701       // We have the count for entry to the RHS and for the whole expression
1702       // being true, so we can divy up True count between the short circuit and
1703       // the RHS.
1704       uint64_t LHSCount =
1705           getCurrentProfileCount() - getProfileCount(CondBOp->getRHS());
1706       uint64_t RHSCount = TrueCount - LHSCount;
1707 
1708       ConditionalEvaluation eval(*this);
1709       {
1710         // Propagate the likelihood attribute like __builtin_expect
1711         // __builtin_expect(X || Y, 1) -> only Y is likely
1712         // __builtin_expect(X || Y, 0) -> both X and Y are unlikely
1713         ApplyDebugLocation DL(*this, Cond);
1714         EmitBranchOnBoolExpr(CondBOp->getLHS(), TrueBlock, LHSFalse, LHSCount,
1715                              LH == Stmt::LH_Likely ? Stmt::LH_None : LH);
1716         EmitBlock(LHSFalse);
1717       }
1718 
1719       incrementProfileCounter(CondBOp);
1720       setCurrentProfileCount(getProfileCount(CondBOp->getRHS()));
1721 
1722       // Any temporaries created here are conditional.
1723       eval.begin(*this);
1724       EmitBranchToCounterBlock(CondBOp->getRHS(), BO_LOr, TrueBlock, FalseBlock,
1725                                RHSCount, LH);
1726 
1727       eval.end(*this);
1728 
1729       return;
1730     }
1731   }
1732 
1733   if (const UnaryOperator *CondUOp = dyn_cast<UnaryOperator>(Cond)) {
1734     // br(!x, t, f) -> br(x, f, t)
1735     if (CondUOp->getOpcode() == UO_LNot) {
1736       // Negate the count.
1737       uint64_t FalseCount = getCurrentProfileCount() - TrueCount;
1738       // The values of the enum are chosen to make this negation possible.
1739       LH = static_cast<Stmt::Likelihood>(-LH);
1740       // Negate the condition and swap the destination blocks.
1741       return EmitBranchOnBoolExpr(CondUOp->getSubExpr(), FalseBlock, TrueBlock,
1742                                   FalseCount, LH);
1743     }
1744   }
1745 
1746   if (const ConditionalOperator *CondOp = dyn_cast<ConditionalOperator>(Cond)) {
1747     // br(c ? x : y, t, f) -> br(c, br(x, t, f), br(y, t, f))
1748     llvm::BasicBlock *LHSBlock = createBasicBlock("cond.true");
1749     llvm::BasicBlock *RHSBlock = createBasicBlock("cond.false");
1750 
1751     // The ConditionalOperator itself has no likelihood information for its
1752     // true and false branches. This matches the behavior of __builtin_expect.
1753     ConditionalEvaluation cond(*this);
1754     EmitBranchOnBoolExpr(CondOp->getCond(), LHSBlock, RHSBlock,
1755                          getProfileCount(CondOp), Stmt::LH_None);
1756 
1757     // When computing PGO branch weights, we only know the overall count for
1758     // the true block. This code is essentially doing tail duplication of the
1759     // naive code-gen, introducing new edges for which counts are not
1760     // available. Divide the counts proportionally between the LHS and RHS of
1761     // the conditional operator.
1762     uint64_t LHSScaledTrueCount = 0;
1763     if (TrueCount) {
1764       double LHSRatio =
1765           getProfileCount(CondOp) / (double)getCurrentProfileCount();
1766       LHSScaledTrueCount = TrueCount * LHSRatio;
1767     }
1768 
1769     cond.begin(*this);
1770     EmitBlock(LHSBlock);
1771     incrementProfileCounter(CondOp);
1772     {
1773       ApplyDebugLocation DL(*this, Cond);
1774       EmitBranchOnBoolExpr(CondOp->getLHS(), TrueBlock, FalseBlock,
1775                            LHSScaledTrueCount, LH);
1776     }
1777     cond.end(*this);
1778 
1779     cond.begin(*this);
1780     EmitBlock(RHSBlock);
1781     EmitBranchOnBoolExpr(CondOp->getRHS(), TrueBlock, FalseBlock,
1782                          TrueCount - LHSScaledTrueCount, LH);
1783     cond.end(*this);
1784 
1785     return;
1786   }
1787 
1788   if (const CXXThrowExpr *Throw = dyn_cast<CXXThrowExpr>(Cond)) {
1789     // Conditional operator handling can give us a throw expression as a
1790     // condition for a case like:
1791     //   br(c ? throw x : y, t, f) -> br(c, br(throw x, t, f), br(y, t, f)
1792     // Fold this to:
1793     //   br(c, throw x, br(y, t, f))
1794     EmitCXXThrowExpr(Throw, /*KeepInsertionPoint*/false);
1795     return;
1796   }
1797 
1798   // Emit the code with the fully general case.
1799   llvm::Value *CondV;
1800   {
1801     ApplyDebugLocation DL(*this, Cond);
1802     CondV = EvaluateExprAsBool(Cond);
1803   }
1804 
1805   llvm::MDNode *Weights = nullptr;
1806   llvm::MDNode *Unpredictable = nullptr;
1807 
1808   // If the branch has a condition wrapped by __builtin_unpredictable,
1809   // create metadata that specifies that the branch is unpredictable.
1810   // Don't bother if not optimizing because that metadata would not be used.
1811   auto *Call = dyn_cast<CallExpr>(Cond->IgnoreImpCasts());
1812   if (Call && CGM.getCodeGenOpts().OptimizationLevel != 0) {
1813     auto *FD = dyn_cast_or_null<FunctionDecl>(Call->getCalleeDecl());
1814     if (FD && FD->getBuiltinID() == Builtin::BI__builtin_unpredictable) {
1815       llvm::MDBuilder MDHelper(getLLVMContext());
1816       Unpredictable = MDHelper.createUnpredictable();
1817     }
1818   }
1819 
1820   // If there is a Likelihood knowledge for the cond, lower it.
1821   // Note that if not optimizing this won't emit anything.
1822   llvm::Value *NewCondV = emitCondLikelihoodViaExpectIntrinsic(CondV, LH);
1823   if (CondV != NewCondV)
1824     CondV = NewCondV;
1825   else {
1826     // Otherwise, lower profile counts. Note that we do this even at -O0.
1827     uint64_t CurrentCount = std::max(getCurrentProfileCount(), TrueCount);
1828     Weights = createProfileWeights(TrueCount, CurrentCount - TrueCount);
1829   }
1830 
1831   Builder.CreateCondBr(CondV, TrueBlock, FalseBlock, Weights, Unpredictable);
1832 }
1833 
1834 /// ErrorUnsupported - Print out an error that codegen doesn't support the
1835 /// specified stmt yet.
1836 void CodeGenFunction::ErrorUnsupported(const Stmt *S, const char *Type) {
1837   CGM.ErrorUnsupported(S, Type);
1838 }
1839 
1840 /// emitNonZeroVLAInit - Emit the "zero" initialization of a
1841 /// variable-length array whose elements have a non-zero bit-pattern.
1842 ///
1843 /// \param baseType the inner-most element type of the array
1844 /// \param src - a char* pointing to the bit-pattern for a single
1845 /// base element of the array
1846 /// \param sizeInChars - the total size of the VLA, in chars
1847 static void emitNonZeroVLAInit(CodeGenFunction &CGF, QualType baseType,
1848                                Address dest, Address src,
1849                                llvm::Value *sizeInChars) {
1850   CGBuilderTy &Builder = CGF.Builder;
1851 
1852   CharUnits baseSize = CGF.getContext().getTypeSizeInChars(baseType);
1853   llvm::Value *baseSizeInChars
1854     = llvm::ConstantInt::get(CGF.IntPtrTy, baseSize.getQuantity());
1855 
1856   Address begin =
1857     Builder.CreateElementBitCast(dest, CGF.Int8Ty, "vla.begin");
1858   llvm::Value *end = Builder.CreateInBoundsGEP(
1859       begin.getElementType(), begin.getPointer(), sizeInChars, "vla.end");
1860 
1861   llvm::BasicBlock *originBB = CGF.Builder.GetInsertBlock();
1862   llvm::BasicBlock *loopBB = CGF.createBasicBlock("vla-init.loop");
1863   llvm::BasicBlock *contBB = CGF.createBasicBlock("vla-init.cont");
1864 
1865   // Make a loop over the VLA.  C99 guarantees that the VLA element
1866   // count must be nonzero.
1867   CGF.EmitBlock(loopBB);
1868 
1869   llvm::PHINode *cur = Builder.CreatePHI(begin.getType(), 2, "vla.cur");
1870   cur->addIncoming(begin.getPointer(), originBB);
1871 
1872   CharUnits curAlign =
1873     dest.getAlignment().alignmentOfArrayElement(baseSize);
1874 
1875   // memcpy the individual element bit-pattern.
1876   Builder.CreateMemCpy(Address(cur, curAlign), src, baseSizeInChars,
1877                        /*volatile*/ false);
1878 
1879   // Go to the next element.
1880   llvm::Value *next =
1881     Builder.CreateInBoundsGEP(CGF.Int8Ty, cur, baseSizeInChars, "vla.next");
1882 
1883   // Leave if that's the end of the VLA.
1884   llvm::Value *done = Builder.CreateICmpEQ(next, end, "vla-init.isdone");
1885   Builder.CreateCondBr(done, contBB, loopBB);
1886   cur->addIncoming(next, loopBB);
1887 
1888   CGF.EmitBlock(contBB);
1889 }
1890 
1891 void
1892 CodeGenFunction::EmitNullInitialization(Address DestPtr, QualType Ty) {
1893   // Ignore empty classes in C++.
1894   if (getLangOpts().CPlusPlus) {
1895     if (const RecordType *RT = Ty->getAs<RecordType>()) {
1896       if (cast<CXXRecordDecl>(RT->getDecl())->isEmpty())
1897         return;
1898     }
1899   }
1900 
1901   // Cast the dest ptr to the appropriate i8 pointer type.
1902   if (DestPtr.getElementType() != Int8Ty)
1903     DestPtr = Builder.CreateElementBitCast(DestPtr, Int8Ty);
1904 
1905   // Get size and alignment info for this aggregate.
1906   CharUnits size = getContext().getTypeSizeInChars(Ty);
1907 
1908   llvm::Value *SizeVal;
1909   const VariableArrayType *vla;
1910 
1911   // Don't bother emitting a zero-byte memset.
1912   if (size.isZero()) {
1913     // But note that getTypeInfo returns 0 for a VLA.
1914     if (const VariableArrayType *vlaType =
1915           dyn_cast_or_null<VariableArrayType>(
1916                                           getContext().getAsArrayType(Ty))) {
1917       auto VlaSize = getVLASize(vlaType);
1918       SizeVal = VlaSize.NumElts;
1919       CharUnits eltSize = getContext().getTypeSizeInChars(VlaSize.Type);
1920       if (!eltSize.isOne())
1921         SizeVal = Builder.CreateNUWMul(SizeVal, CGM.getSize(eltSize));
1922       vla = vlaType;
1923     } else {
1924       return;
1925     }
1926   } else {
1927     SizeVal = CGM.getSize(size);
1928     vla = nullptr;
1929   }
1930 
1931   // If the type contains a pointer to data member we can't memset it to zero.
1932   // Instead, create a null constant and copy it to the destination.
1933   // TODO: there are other patterns besides zero that we can usefully memset,
1934   // like -1, which happens to be the pattern used by member-pointers.
1935   if (!CGM.getTypes().isZeroInitializable(Ty)) {
1936     // For a VLA, emit a single element, then splat that over the VLA.
1937     if (vla) Ty = getContext().getBaseElementType(vla);
1938 
1939     llvm::Constant *NullConstant = CGM.EmitNullConstant(Ty);
1940 
1941     llvm::GlobalVariable *NullVariable =
1942       new llvm::GlobalVariable(CGM.getModule(), NullConstant->getType(),
1943                                /*isConstant=*/true,
1944                                llvm::GlobalVariable::PrivateLinkage,
1945                                NullConstant, Twine());
1946     CharUnits NullAlign = DestPtr.getAlignment();
1947     NullVariable->setAlignment(NullAlign.getAsAlign());
1948     Address SrcPtr(Builder.CreateBitCast(NullVariable, Builder.getInt8PtrTy()),
1949                    NullAlign);
1950 
1951     if (vla) return emitNonZeroVLAInit(*this, Ty, DestPtr, SrcPtr, SizeVal);
1952 
1953     // Get and call the appropriate llvm.memcpy overload.
1954     Builder.CreateMemCpy(DestPtr, SrcPtr, SizeVal, false);
1955     return;
1956   }
1957 
1958   // Otherwise, just memset the whole thing to zero.  This is legal
1959   // because in LLVM, all default initializers (other than the ones we just
1960   // handled above) are guaranteed to have a bit pattern of all zeros.
1961   Builder.CreateMemSet(DestPtr, Builder.getInt8(0), SizeVal, false);
1962 }
1963 
1964 llvm::BlockAddress *CodeGenFunction::GetAddrOfLabel(const LabelDecl *L) {
1965   // Make sure that there is a block for the indirect goto.
1966   if (!IndirectBranch)
1967     GetIndirectGotoBlock();
1968 
1969   llvm::BasicBlock *BB = getJumpDestForLabel(L).getBlock();
1970 
1971   // Make sure the indirect branch includes all of the address-taken blocks.
1972   IndirectBranch->addDestination(BB);
1973   return llvm::BlockAddress::get(CurFn, BB);
1974 }
1975 
1976 llvm::BasicBlock *CodeGenFunction::GetIndirectGotoBlock() {
1977   // If we already made the indirect branch for indirect goto, return its block.
1978   if (IndirectBranch) return IndirectBranch->getParent();
1979 
1980   CGBuilderTy TmpBuilder(*this, createBasicBlock("indirectgoto"));
1981 
1982   // Create the PHI node that indirect gotos will add entries to.
1983   llvm::Value *DestVal = TmpBuilder.CreatePHI(Int8PtrTy, 0,
1984                                               "indirect.goto.dest");
1985 
1986   // Create the indirect branch instruction.
1987   IndirectBranch = TmpBuilder.CreateIndirectBr(DestVal);
1988   return IndirectBranch->getParent();
1989 }
1990 
1991 /// Computes the length of an array in elements, as well as the base
1992 /// element type and a properly-typed first element pointer.
1993 llvm::Value *CodeGenFunction::emitArrayLength(const ArrayType *origArrayType,
1994                                               QualType &baseType,
1995                                               Address &addr) {
1996   const ArrayType *arrayType = origArrayType;
1997 
1998   // If it's a VLA, we have to load the stored size.  Note that
1999   // this is the size of the VLA in bytes, not its size in elements.
2000   llvm::Value *numVLAElements = nullptr;
2001   if (isa<VariableArrayType>(arrayType)) {
2002     numVLAElements = getVLASize(cast<VariableArrayType>(arrayType)).NumElts;
2003 
2004     // Walk into all VLAs.  This doesn't require changes to addr,
2005     // which has type T* where T is the first non-VLA element type.
2006     do {
2007       QualType elementType = arrayType->getElementType();
2008       arrayType = getContext().getAsArrayType(elementType);
2009 
2010       // If we only have VLA components, 'addr' requires no adjustment.
2011       if (!arrayType) {
2012         baseType = elementType;
2013         return numVLAElements;
2014       }
2015     } while (isa<VariableArrayType>(arrayType));
2016 
2017     // We get out here only if we find a constant array type
2018     // inside the VLA.
2019   }
2020 
2021   // We have some number of constant-length arrays, so addr should
2022   // have LLVM type [M x [N x [...]]]*.  Build a GEP that walks
2023   // down to the first element of addr.
2024   SmallVector<llvm::Value*, 8> gepIndices;
2025 
2026   // GEP down to the array type.
2027   llvm::ConstantInt *zero = Builder.getInt32(0);
2028   gepIndices.push_back(zero);
2029 
2030   uint64_t countFromCLAs = 1;
2031   QualType eltType;
2032 
2033   llvm::ArrayType *llvmArrayType =
2034     dyn_cast<llvm::ArrayType>(addr.getElementType());
2035   while (llvmArrayType) {
2036     assert(isa<ConstantArrayType>(arrayType));
2037     assert(cast<ConstantArrayType>(arrayType)->getSize().getZExtValue()
2038              == llvmArrayType->getNumElements());
2039 
2040     gepIndices.push_back(zero);
2041     countFromCLAs *= llvmArrayType->getNumElements();
2042     eltType = arrayType->getElementType();
2043 
2044     llvmArrayType =
2045       dyn_cast<llvm::ArrayType>(llvmArrayType->getElementType());
2046     arrayType = getContext().getAsArrayType(arrayType->getElementType());
2047     assert((!llvmArrayType || arrayType) &&
2048            "LLVM and Clang types are out-of-synch");
2049   }
2050 
2051   if (arrayType) {
2052     // From this point onwards, the Clang array type has been emitted
2053     // as some other type (probably a packed struct). Compute the array
2054     // size, and just emit the 'begin' expression as a bitcast.
2055     while (arrayType) {
2056       countFromCLAs *=
2057           cast<ConstantArrayType>(arrayType)->getSize().getZExtValue();
2058       eltType = arrayType->getElementType();
2059       arrayType = getContext().getAsArrayType(eltType);
2060     }
2061 
2062     llvm::Type *baseType = ConvertType(eltType);
2063     addr = Builder.CreateElementBitCast(addr, baseType, "array.begin");
2064   } else {
2065     // Create the actual GEP.
2066     addr = Address(Builder.CreateInBoundsGEP(
2067         addr.getElementType(), addr.getPointer(), gepIndices, "array.begin"),
2068         addr.getAlignment());
2069   }
2070 
2071   baseType = eltType;
2072 
2073   llvm::Value *numElements
2074     = llvm::ConstantInt::get(SizeTy, countFromCLAs);
2075 
2076   // If we had any VLA dimensions, factor them in.
2077   if (numVLAElements)
2078     numElements = Builder.CreateNUWMul(numVLAElements, numElements);
2079 
2080   return numElements;
2081 }
2082 
2083 CodeGenFunction::VlaSizePair CodeGenFunction::getVLASize(QualType type) {
2084   const VariableArrayType *vla = getContext().getAsVariableArrayType(type);
2085   assert(vla && "type was not a variable array type!");
2086   return getVLASize(vla);
2087 }
2088 
2089 CodeGenFunction::VlaSizePair
2090 CodeGenFunction::getVLASize(const VariableArrayType *type) {
2091   // The number of elements so far; always size_t.
2092   llvm::Value *numElements = nullptr;
2093 
2094   QualType elementType;
2095   do {
2096     elementType = type->getElementType();
2097     llvm::Value *vlaSize = VLASizeMap[type->getSizeExpr()];
2098     assert(vlaSize && "no size for VLA!");
2099     assert(vlaSize->getType() == SizeTy);
2100 
2101     if (!numElements) {
2102       numElements = vlaSize;
2103     } else {
2104       // It's undefined behavior if this wraps around, so mark it that way.
2105       // FIXME: Teach -fsanitize=undefined to trap this.
2106       numElements = Builder.CreateNUWMul(numElements, vlaSize);
2107     }
2108   } while ((type = getContext().getAsVariableArrayType(elementType)));
2109 
2110   return { numElements, elementType };
2111 }
2112 
2113 CodeGenFunction::VlaSizePair
2114 CodeGenFunction::getVLAElements1D(QualType type) {
2115   const VariableArrayType *vla = getContext().getAsVariableArrayType(type);
2116   assert(vla && "type was not a variable array type!");
2117   return getVLAElements1D(vla);
2118 }
2119 
2120 CodeGenFunction::VlaSizePair
2121 CodeGenFunction::getVLAElements1D(const VariableArrayType *Vla) {
2122   llvm::Value *VlaSize = VLASizeMap[Vla->getSizeExpr()];
2123   assert(VlaSize && "no size for VLA!");
2124   assert(VlaSize->getType() == SizeTy);
2125   return { VlaSize, Vla->getElementType() };
2126 }
2127 
2128 void CodeGenFunction::EmitVariablyModifiedType(QualType type) {
2129   assert(type->isVariablyModifiedType() &&
2130          "Must pass variably modified type to EmitVLASizes!");
2131 
2132   EnsureInsertPoint();
2133 
2134   // We're going to walk down into the type and look for VLA
2135   // expressions.
2136   do {
2137     assert(type->isVariablyModifiedType());
2138 
2139     const Type *ty = type.getTypePtr();
2140     switch (ty->getTypeClass()) {
2141 
2142 #define TYPE(Class, Base)
2143 #define ABSTRACT_TYPE(Class, Base)
2144 #define NON_CANONICAL_TYPE(Class, Base)
2145 #define DEPENDENT_TYPE(Class, Base) case Type::Class:
2146 #define NON_CANONICAL_UNLESS_DEPENDENT_TYPE(Class, Base)
2147 #include "clang/AST/TypeNodes.inc"
2148       llvm_unreachable("unexpected dependent type!");
2149 
2150     // These types are never variably-modified.
2151     case Type::Builtin:
2152     case Type::Complex:
2153     case Type::Vector:
2154     case Type::ExtVector:
2155     case Type::ConstantMatrix:
2156     case Type::Record:
2157     case Type::Enum:
2158     case Type::Elaborated:
2159     case Type::TemplateSpecialization:
2160     case Type::ObjCTypeParam:
2161     case Type::ObjCObject:
2162     case Type::ObjCInterface:
2163     case Type::ObjCObjectPointer:
2164     case Type::ExtInt:
2165       llvm_unreachable("type class is never variably-modified!");
2166 
2167     case Type::Adjusted:
2168       type = cast<AdjustedType>(ty)->getAdjustedType();
2169       break;
2170 
2171     case Type::Decayed:
2172       type = cast<DecayedType>(ty)->getPointeeType();
2173       break;
2174 
2175     case Type::Pointer:
2176       type = cast<PointerType>(ty)->getPointeeType();
2177       break;
2178 
2179     case Type::BlockPointer:
2180       type = cast<BlockPointerType>(ty)->getPointeeType();
2181       break;
2182 
2183     case Type::LValueReference:
2184     case Type::RValueReference:
2185       type = cast<ReferenceType>(ty)->getPointeeType();
2186       break;
2187 
2188     case Type::MemberPointer:
2189       type = cast<MemberPointerType>(ty)->getPointeeType();
2190       break;
2191 
2192     case Type::ConstantArray:
2193     case Type::IncompleteArray:
2194       // Losing element qualification here is fine.
2195       type = cast<ArrayType>(ty)->getElementType();
2196       break;
2197 
2198     case Type::VariableArray: {
2199       // Losing element qualification here is fine.
2200       const VariableArrayType *vat = cast<VariableArrayType>(ty);
2201 
2202       // Unknown size indication requires no size computation.
2203       // Otherwise, evaluate and record it.
2204       if (const Expr *size = vat->getSizeExpr()) {
2205         // It's possible that we might have emitted this already,
2206         // e.g. with a typedef and a pointer to it.
2207         llvm::Value *&entry = VLASizeMap[size];
2208         if (!entry) {
2209           llvm::Value *Size = EmitScalarExpr(size);
2210 
2211           // C11 6.7.6.2p5:
2212           //   If the size is an expression that is not an integer constant
2213           //   expression [...] each time it is evaluated it shall have a value
2214           //   greater than zero.
2215           if (SanOpts.has(SanitizerKind::VLABound) &&
2216               size->getType()->isSignedIntegerType()) {
2217             SanitizerScope SanScope(this);
2218             llvm::Value *Zero = llvm::Constant::getNullValue(Size->getType());
2219             llvm::Constant *StaticArgs[] = {
2220                 EmitCheckSourceLocation(size->getBeginLoc()),
2221                 EmitCheckTypeDescriptor(size->getType())};
2222             EmitCheck(std::make_pair(Builder.CreateICmpSGT(Size, Zero),
2223                                      SanitizerKind::VLABound),
2224                       SanitizerHandler::VLABoundNotPositive, StaticArgs, Size);
2225           }
2226 
2227           // Always zexting here would be wrong if it weren't
2228           // undefined behavior to have a negative bound.
2229           entry = Builder.CreateIntCast(Size, SizeTy, /*signed*/ false);
2230         }
2231       }
2232       type = vat->getElementType();
2233       break;
2234     }
2235 
2236     case Type::FunctionProto:
2237     case Type::FunctionNoProto:
2238       type = cast<FunctionType>(ty)->getReturnType();
2239       break;
2240 
2241     case Type::Paren:
2242     case Type::TypeOf:
2243     case Type::UnaryTransform:
2244     case Type::Attributed:
2245     case Type::SubstTemplateTypeParm:
2246     case Type::MacroQualified:
2247       // Keep walking after single level desugaring.
2248       type = type.getSingleStepDesugaredType(getContext());
2249       break;
2250 
2251     case Type::Typedef:
2252     case Type::Decltype:
2253     case Type::Auto:
2254     case Type::DeducedTemplateSpecialization:
2255       // Stop walking: nothing to do.
2256       return;
2257 
2258     case Type::TypeOfExpr:
2259       // Stop walking: emit typeof expression.
2260       EmitIgnoredExpr(cast<TypeOfExprType>(ty)->getUnderlyingExpr());
2261       return;
2262 
2263     case Type::Atomic:
2264       type = cast<AtomicType>(ty)->getValueType();
2265       break;
2266 
2267     case Type::Pipe:
2268       type = cast<PipeType>(ty)->getElementType();
2269       break;
2270     }
2271   } while (type->isVariablyModifiedType());
2272 }
2273 
2274 Address CodeGenFunction::EmitVAListRef(const Expr* E) {
2275   if (getContext().getBuiltinVaListType()->isArrayType())
2276     return EmitPointerWithAlignment(E);
2277   return EmitLValue(E).getAddress(*this);
2278 }
2279 
2280 Address CodeGenFunction::EmitMSVAListRef(const Expr *E) {
2281   return EmitLValue(E).getAddress(*this);
2282 }
2283 
2284 void CodeGenFunction::EmitDeclRefExprDbgValue(const DeclRefExpr *E,
2285                                               const APValue &Init) {
2286   assert(Init.hasValue() && "Invalid DeclRefExpr initializer!");
2287   if (CGDebugInfo *Dbg = getDebugInfo())
2288     if (CGM.getCodeGenOpts().hasReducedDebugInfo())
2289       Dbg->EmitGlobalVariable(E->getDecl(), Init);
2290 }
2291 
2292 CodeGenFunction::PeepholeProtection
2293 CodeGenFunction::protectFromPeepholes(RValue rvalue) {
2294   // At the moment, the only aggressive peephole we do in IR gen
2295   // is trunc(zext) folding, but if we add more, we can easily
2296   // extend this protection.
2297 
2298   if (!rvalue.isScalar()) return PeepholeProtection();
2299   llvm::Value *value = rvalue.getScalarVal();
2300   if (!isa<llvm::ZExtInst>(value)) return PeepholeProtection();
2301 
2302   // Just make an extra bitcast.
2303   assert(HaveInsertPoint());
2304   llvm::Instruction *inst = new llvm::BitCastInst(value, value->getType(), "",
2305                                                   Builder.GetInsertBlock());
2306 
2307   PeepholeProtection protection;
2308   protection.Inst = inst;
2309   return protection;
2310 }
2311 
2312 void CodeGenFunction::unprotectFromPeepholes(PeepholeProtection protection) {
2313   if (!protection.Inst) return;
2314 
2315   // In theory, we could try to duplicate the peepholes now, but whatever.
2316   protection.Inst->eraseFromParent();
2317 }
2318 
2319 void CodeGenFunction::emitAlignmentAssumption(llvm::Value *PtrValue,
2320                                               QualType Ty, SourceLocation Loc,
2321                                               SourceLocation AssumptionLoc,
2322                                               llvm::Value *Alignment,
2323                                               llvm::Value *OffsetValue) {
2324   if (Alignment->getType() != IntPtrTy)
2325     Alignment =
2326         Builder.CreateIntCast(Alignment, IntPtrTy, false, "casted.align");
2327   if (OffsetValue && OffsetValue->getType() != IntPtrTy)
2328     OffsetValue =
2329         Builder.CreateIntCast(OffsetValue, IntPtrTy, true, "casted.offset");
2330   llvm::Value *TheCheck = nullptr;
2331   if (SanOpts.has(SanitizerKind::Alignment)) {
2332     llvm::Value *PtrIntValue =
2333         Builder.CreatePtrToInt(PtrValue, IntPtrTy, "ptrint");
2334 
2335     if (OffsetValue) {
2336       bool IsOffsetZero = false;
2337       if (const auto *CI = dyn_cast<llvm::ConstantInt>(OffsetValue))
2338         IsOffsetZero = CI->isZero();
2339 
2340       if (!IsOffsetZero)
2341         PtrIntValue = Builder.CreateSub(PtrIntValue, OffsetValue, "offsetptr");
2342     }
2343 
2344     llvm::Value *Zero = llvm::ConstantInt::get(IntPtrTy, 0);
2345     llvm::Value *Mask =
2346         Builder.CreateSub(Alignment, llvm::ConstantInt::get(IntPtrTy, 1));
2347     llvm::Value *MaskedPtr = Builder.CreateAnd(PtrIntValue, Mask, "maskedptr");
2348     TheCheck = Builder.CreateICmpEQ(MaskedPtr, Zero, "maskcond");
2349   }
2350   llvm::Instruction *Assumption = Builder.CreateAlignmentAssumption(
2351       CGM.getDataLayout(), PtrValue, Alignment, OffsetValue);
2352 
2353   if (!SanOpts.has(SanitizerKind::Alignment))
2354     return;
2355   emitAlignmentAssumptionCheck(PtrValue, Ty, Loc, AssumptionLoc, Alignment,
2356                                OffsetValue, TheCheck, Assumption);
2357 }
2358 
2359 void CodeGenFunction::emitAlignmentAssumption(llvm::Value *PtrValue,
2360                                               const Expr *E,
2361                                               SourceLocation AssumptionLoc,
2362                                               llvm::Value *Alignment,
2363                                               llvm::Value *OffsetValue) {
2364   if (auto *CE = dyn_cast<CastExpr>(E))
2365     E = CE->getSubExprAsWritten();
2366   QualType Ty = E->getType();
2367   SourceLocation Loc = E->getExprLoc();
2368 
2369   emitAlignmentAssumption(PtrValue, Ty, Loc, AssumptionLoc, Alignment,
2370                           OffsetValue);
2371 }
2372 
2373 llvm::Value *CodeGenFunction::EmitAnnotationCall(llvm::Function *AnnotationFn,
2374                                                  llvm::Value *AnnotatedVal,
2375                                                  StringRef AnnotationStr,
2376                                                  SourceLocation Location,
2377                                                  const AnnotateAttr *Attr) {
2378   SmallVector<llvm::Value *, 5> Args = {
2379       AnnotatedVal,
2380       Builder.CreateBitCast(CGM.EmitAnnotationString(AnnotationStr), Int8PtrTy),
2381       Builder.CreateBitCast(CGM.EmitAnnotationUnit(Location), Int8PtrTy),
2382       CGM.EmitAnnotationLineNo(Location),
2383   };
2384   if (Attr)
2385     Args.push_back(CGM.EmitAnnotationArgs(Attr));
2386   return Builder.CreateCall(AnnotationFn, Args);
2387 }
2388 
2389 void CodeGenFunction::EmitVarAnnotations(const VarDecl *D, llvm::Value *V) {
2390   assert(D->hasAttr<AnnotateAttr>() && "no annotate attribute");
2391   // FIXME We create a new bitcast for every annotation because that's what
2392   // llvm-gcc was doing.
2393   for (const auto *I : D->specific_attrs<AnnotateAttr>())
2394     EmitAnnotationCall(CGM.getIntrinsic(llvm::Intrinsic::var_annotation),
2395                        Builder.CreateBitCast(V, CGM.Int8PtrTy, V->getName()),
2396                        I->getAnnotation(), D->getLocation(), I);
2397 }
2398 
2399 Address CodeGenFunction::EmitFieldAnnotations(const FieldDecl *D,
2400                                               Address Addr) {
2401   assert(D->hasAttr<AnnotateAttr>() && "no annotate attribute");
2402   llvm::Value *V = Addr.getPointer();
2403   llvm::Type *VTy = V->getType();
2404   llvm::Function *F = CGM.getIntrinsic(llvm::Intrinsic::ptr_annotation,
2405                                     CGM.Int8PtrTy);
2406 
2407   for (const auto *I : D->specific_attrs<AnnotateAttr>()) {
2408     // FIXME Always emit the cast inst so we can differentiate between
2409     // annotation on the first field of a struct and annotation on the struct
2410     // itself.
2411     if (VTy != CGM.Int8PtrTy)
2412       V = Builder.CreateBitCast(V, CGM.Int8PtrTy);
2413     V = EmitAnnotationCall(F, V, I->getAnnotation(), D->getLocation(), I);
2414     V = Builder.CreateBitCast(V, VTy);
2415   }
2416 
2417   return Address(V, Addr.getAlignment());
2418 }
2419 
2420 CodeGenFunction::CGCapturedStmtInfo::~CGCapturedStmtInfo() { }
2421 
2422 CodeGenFunction::SanitizerScope::SanitizerScope(CodeGenFunction *CGF)
2423     : CGF(CGF) {
2424   assert(!CGF->IsSanitizerScope);
2425   CGF->IsSanitizerScope = true;
2426 }
2427 
2428 CodeGenFunction::SanitizerScope::~SanitizerScope() {
2429   CGF->IsSanitizerScope = false;
2430 }
2431 
2432 void CodeGenFunction::InsertHelper(llvm::Instruction *I,
2433                                    const llvm::Twine &Name,
2434                                    llvm::BasicBlock *BB,
2435                                    llvm::BasicBlock::iterator InsertPt) const {
2436   LoopStack.InsertHelper(I);
2437   if (IsSanitizerScope)
2438     CGM.getSanitizerMetadata()->disableSanitizerForInstruction(I);
2439 }
2440 
2441 void CGBuilderInserter::InsertHelper(
2442     llvm::Instruction *I, const llvm::Twine &Name, llvm::BasicBlock *BB,
2443     llvm::BasicBlock::iterator InsertPt) const {
2444   llvm::IRBuilderDefaultInserter::InsertHelper(I, Name, BB, InsertPt);
2445   if (CGF)
2446     CGF->InsertHelper(I, Name, BB, InsertPt);
2447 }
2448 
2449 // Emits an error if we don't have a valid set of target features for the
2450 // called function.
2451 void CodeGenFunction::checkTargetFeatures(const CallExpr *E,
2452                                           const FunctionDecl *TargetDecl) {
2453   return checkTargetFeatures(E->getBeginLoc(), TargetDecl);
2454 }
2455 
2456 // Emits an error if we don't have a valid set of target features for the
2457 // called function.
2458 void CodeGenFunction::checkTargetFeatures(SourceLocation Loc,
2459                                           const FunctionDecl *TargetDecl) {
2460   // Early exit if this is an indirect call.
2461   if (!TargetDecl)
2462     return;
2463 
2464   // Get the current enclosing function if it exists. If it doesn't
2465   // we can't check the target features anyhow.
2466   const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(CurCodeDecl);
2467   if (!FD)
2468     return;
2469 
2470   // Grab the required features for the call. For a builtin this is listed in
2471   // the td file with the default cpu, for an always_inline function this is any
2472   // listed cpu and any listed features.
2473   unsigned BuiltinID = TargetDecl->getBuiltinID();
2474   std::string MissingFeature;
2475   llvm::StringMap<bool> CallerFeatureMap;
2476   CGM.getContext().getFunctionFeatureMap(CallerFeatureMap, FD);
2477   if (BuiltinID) {
2478     StringRef FeatureList(
2479         CGM.getContext().BuiltinInfo.getRequiredFeatures(BuiltinID));
2480     // Return if the builtin doesn't have any required features.
2481     if (FeatureList.empty())
2482       return;
2483     assert(FeatureList.find(' ') == StringRef::npos &&
2484            "Space in feature list");
2485     TargetFeatures TF(CallerFeatureMap);
2486     if (!TF.hasRequiredFeatures(FeatureList))
2487       CGM.getDiags().Report(Loc, diag::err_builtin_needs_feature)
2488           << TargetDecl->getDeclName() << FeatureList;
2489   } else if (!TargetDecl->isMultiVersion() &&
2490              TargetDecl->hasAttr<TargetAttr>()) {
2491     // Get the required features for the callee.
2492 
2493     const TargetAttr *TD = TargetDecl->getAttr<TargetAttr>();
2494     ParsedTargetAttr ParsedAttr =
2495         CGM.getContext().filterFunctionTargetAttrs(TD);
2496 
2497     SmallVector<StringRef, 1> ReqFeatures;
2498     llvm::StringMap<bool> CalleeFeatureMap;
2499     CGM.getContext().getFunctionFeatureMap(CalleeFeatureMap, TargetDecl);
2500 
2501     for (const auto &F : ParsedAttr.Features) {
2502       if (F[0] == '+' && CalleeFeatureMap.lookup(F.substr(1)))
2503         ReqFeatures.push_back(StringRef(F).substr(1));
2504     }
2505 
2506     for (const auto &F : CalleeFeatureMap) {
2507       // Only positive features are "required".
2508       if (F.getValue())
2509         ReqFeatures.push_back(F.getKey());
2510     }
2511     if (!llvm::all_of(ReqFeatures, [&](StringRef Feature) {
2512       if (!CallerFeatureMap.lookup(Feature)) {
2513         MissingFeature = Feature.str();
2514         return false;
2515       }
2516       return true;
2517     }))
2518       CGM.getDiags().Report(Loc, diag::err_function_needs_feature)
2519           << FD->getDeclName() << TargetDecl->getDeclName() << MissingFeature;
2520   }
2521 }
2522 
2523 void CodeGenFunction::EmitSanitizerStatReport(llvm::SanitizerStatKind SSK) {
2524   if (!CGM.getCodeGenOpts().SanitizeStats)
2525     return;
2526 
2527   llvm::IRBuilder<> IRB(Builder.GetInsertBlock(), Builder.GetInsertPoint());
2528   IRB.SetCurrentDebugLocation(Builder.getCurrentDebugLocation());
2529   CGM.getSanStats().create(IRB, SSK);
2530 }
2531 
2532 llvm::Value *
2533 CodeGenFunction::FormResolverCondition(const MultiVersionResolverOption &RO) {
2534   llvm::Value *Condition = nullptr;
2535 
2536   if (!RO.Conditions.Architecture.empty())
2537     Condition = EmitX86CpuIs(RO.Conditions.Architecture);
2538 
2539   if (!RO.Conditions.Features.empty()) {
2540     llvm::Value *FeatureCond = EmitX86CpuSupports(RO.Conditions.Features);
2541     Condition =
2542         Condition ? Builder.CreateAnd(Condition, FeatureCond) : FeatureCond;
2543   }
2544   return Condition;
2545 }
2546 
2547 static void CreateMultiVersionResolverReturn(CodeGenModule &CGM,
2548                                              llvm::Function *Resolver,
2549                                              CGBuilderTy &Builder,
2550                                              llvm::Function *FuncToReturn,
2551                                              bool SupportsIFunc) {
2552   if (SupportsIFunc) {
2553     Builder.CreateRet(FuncToReturn);
2554     return;
2555   }
2556 
2557   llvm::SmallVector<llvm::Value *, 10> Args;
2558   llvm::for_each(Resolver->args(),
2559                  [&](llvm::Argument &Arg) { Args.push_back(&Arg); });
2560 
2561   llvm::CallInst *Result = Builder.CreateCall(FuncToReturn, Args);
2562   Result->setTailCallKind(llvm::CallInst::TCK_MustTail);
2563 
2564   if (Resolver->getReturnType()->isVoidTy())
2565     Builder.CreateRetVoid();
2566   else
2567     Builder.CreateRet(Result);
2568 }
2569 
2570 void CodeGenFunction::EmitMultiVersionResolver(
2571     llvm::Function *Resolver, ArrayRef<MultiVersionResolverOption> Options) {
2572   assert(getContext().getTargetInfo().getTriple().isX86() &&
2573          "Only implemented for x86 targets");
2574 
2575   bool SupportsIFunc = getContext().getTargetInfo().supportsIFunc();
2576 
2577   // Main function's basic block.
2578   llvm::BasicBlock *CurBlock = createBasicBlock("resolver_entry", Resolver);
2579   Builder.SetInsertPoint(CurBlock);
2580   EmitX86CpuInit();
2581 
2582   for (const MultiVersionResolverOption &RO : Options) {
2583     Builder.SetInsertPoint(CurBlock);
2584     llvm::Value *Condition = FormResolverCondition(RO);
2585 
2586     // The 'default' or 'generic' case.
2587     if (!Condition) {
2588       assert(&RO == Options.end() - 1 &&
2589              "Default or Generic case must be last");
2590       CreateMultiVersionResolverReturn(CGM, Resolver, Builder, RO.Function,
2591                                        SupportsIFunc);
2592       return;
2593     }
2594 
2595     llvm::BasicBlock *RetBlock = createBasicBlock("resolver_return", Resolver);
2596     CGBuilderTy RetBuilder(*this, RetBlock);
2597     CreateMultiVersionResolverReturn(CGM, Resolver, RetBuilder, RO.Function,
2598                                      SupportsIFunc);
2599     CurBlock = createBasicBlock("resolver_else", Resolver);
2600     Builder.CreateCondBr(Condition, RetBlock, CurBlock);
2601   }
2602 
2603   // If no generic/default, emit an unreachable.
2604   Builder.SetInsertPoint(CurBlock);
2605   llvm::CallInst *TrapCall = EmitTrapCall(llvm::Intrinsic::trap);
2606   TrapCall->setDoesNotReturn();
2607   TrapCall->setDoesNotThrow();
2608   Builder.CreateUnreachable();
2609   Builder.ClearInsertionPoint();
2610 }
2611 
2612 // Loc - where the diagnostic will point, where in the source code this
2613 //  alignment has failed.
2614 // SecondaryLoc - if present (will be present if sufficiently different from
2615 //  Loc), the diagnostic will additionally point a "Note:" to this location.
2616 //  It should be the location where the __attribute__((assume_aligned))
2617 //  was written e.g.
2618 void CodeGenFunction::emitAlignmentAssumptionCheck(
2619     llvm::Value *Ptr, QualType Ty, SourceLocation Loc,
2620     SourceLocation SecondaryLoc, llvm::Value *Alignment,
2621     llvm::Value *OffsetValue, llvm::Value *TheCheck,
2622     llvm::Instruction *Assumption) {
2623   assert(Assumption && isa<llvm::CallInst>(Assumption) &&
2624          cast<llvm::CallInst>(Assumption)->getCalledOperand() ==
2625              llvm::Intrinsic::getDeclaration(
2626                  Builder.GetInsertBlock()->getParent()->getParent(),
2627                  llvm::Intrinsic::assume) &&
2628          "Assumption should be a call to llvm.assume().");
2629   assert(&(Builder.GetInsertBlock()->back()) == Assumption &&
2630          "Assumption should be the last instruction of the basic block, "
2631          "since the basic block is still being generated.");
2632 
2633   if (!SanOpts.has(SanitizerKind::Alignment))
2634     return;
2635 
2636   // Don't check pointers to volatile data. The behavior here is implementation-
2637   // defined.
2638   if (Ty->getPointeeType().isVolatileQualified())
2639     return;
2640 
2641   // We need to temorairly remove the assumption so we can insert the
2642   // sanitizer check before it, else the check will be dropped by optimizations.
2643   Assumption->removeFromParent();
2644 
2645   {
2646     SanitizerScope SanScope(this);
2647 
2648     if (!OffsetValue)
2649       OffsetValue = Builder.getInt1(0); // no offset.
2650 
2651     llvm::Constant *StaticData[] = {EmitCheckSourceLocation(Loc),
2652                                     EmitCheckSourceLocation(SecondaryLoc),
2653                                     EmitCheckTypeDescriptor(Ty)};
2654     llvm::Value *DynamicData[] = {EmitCheckValue(Ptr),
2655                                   EmitCheckValue(Alignment),
2656                                   EmitCheckValue(OffsetValue)};
2657     EmitCheck({std::make_pair(TheCheck, SanitizerKind::Alignment)},
2658               SanitizerHandler::AlignmentAssumption, StaticData, DynamicData);
2659   }
2660 
2661   // We are now in the (new, empty) "cont" basic block.
2662   // Reintroduce the assumption.
2663   Builder.Insert(Assumption);
2664   // FIXME: Assumption still has it's original basic block as it's Parent.
2665 }
2666 
2667 llvm::DebugLoc CodeGenFunction::SourceLocToDebugLoc(SourceLocation Location) {
2668   if (CGDebugInfo *DI = getDebugInfo())
2669     return DI->SourceLocToDebugLoc(Location);
2670 
2671   return llvm::DebugLoc();
2672 }
2673 
2674 llvm::Value *
2675 CodeGenFunction::emitCondLikelihoodViaExpectIntrinsic(llvm::Value *Cond,
2676                                                       Stmt::Likelihood LH) {
2677   switch (LH) {
2678   case Stmt::LH_None:
2679     return Cond;
2680   case Stmt::LH_Likely:
2681   case Stmt::LH_Unlikely:
2682     // Don't generate llvm.expect on -O0 as the backend won't use it for
2683     // anything.
2684     if (CGM.getCodeGenOpts().OptimizationLevel == 0)
2685       return Cond;
2686     llvm::Type *CondTy = Cond->getType();
2687     assert(CondTy->isIntegerTy(1) && "expecting condition to be a boolean");
2688     llvm::Function *FnExpect =
2689         CGM.getIntrinsic(llvm::Intrinsic::expect, CondTy);
2690     llvm::Value *ExpectedValueOfCond =
2691         llvm::ConstantInt::getBool(CondTy, LH == Stmt::LH_Likely);
2692     return Builder.CreateCall(FnExpect, {Cond, ExpectedValueOfCond},
2693                               Cond->getName() + ".expval");
2694   }
2695   llvm_unreachable("Unknown Likelihood");
2696 }
2697