1 //===----- CGOpenMPRuntime.h - Interface to OpenMP Runtimes -----*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This provides a class for OpenMP runtime code generation. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #ifndef LLVM_CLANG_LIB_CODEGEN_CGOPENMPRUNTIME_H 14 #define LLVM_CLANG_LIB_CODEGEN_CGOPENMPRUNTIME_H 15 16 #include "CGValue.h" 17 #include "clang/AST/DeclOpenMP.h" 18 #include "clang/AST/GlobalDecl.h" 19 #include "clang/AST/Type.h" 20 #include "clang/Basic/OpenMPKinds.h" 21 #include "clang/Basic/SourceLocation.h" 22 #include "llvm/ADT/DenseMap.h" 23 #include "llvm/ADT/PointerIntPair.h" 24 #include "llvm/ADT/SmallPtrSet.h" 25 #include "llvm/ADT/StringMap.h" 26 #include "llvm/ADT/StringSet.h" 27 #include "llvm/Frontend/OpenMP/OMPConstants.h" 28 #include "llvm/Frontend/OpenMP/OMPIRBuilder.h" 29 #include "llvm/IR/Function.h" 30 #include "llvm/IR/ValueHandle.h" 31 #include "llvm/Support/AtomicOrdering.h" 32 33 namespace llvm { 34 class ArrayType; 35 class Constant; 36 class FunctionType; 37 class GlobalVariable; 38 class Type; 39 class Value; 40 class OpenMPIRBuilder; 41 } // namespace llvm 42 43 namespace clang { 44 class Expr; 45 class OMPDependClause; 46 class OMPExecutableDirective; 47 class OMPLoopDirective; 48 class VarDecl; 49 class OMPDeclareReductionDecl; 50 51 namespace CodeGen { 52 class Address; 53 class CodeGenFunction; 54 class CodeGenModule; 55 56 /// A basic class for pre|post-action for advanced codegen sequence for OpenMP 57 /// region. 58 class PrePostActionTy { 59 public: 60 explicit PrePostActionTy() {} 61 virtual void Enter(CodeGenFunction &CGF) {} 62 virtual void Exit(CodeGenFunction &CGF) {} 63 virtual ~PrePostActionTy() {} 64 }; 65 66 /// Class provides a way to call simple version of codegen for OpenMP region, or 67 /// an advanced with possible pre|post-actions in codegen. 68 class RegionCodeGenTy final { 69 intptr_t CodeGen; 70 typedef void (*CodeGenTy)(intptr_t, CodeGenFunction &, PrePostActionTy &); 71 CodeGenTy Callback; 72 mutable PrePostActionTy *PrePostAction; 73 RegionCodeGenTy() = delete; 74 template <typename Callable> 75 static void CallbackFn(intptr_t CodeGen, CodeGenFunction &CGF, 76 PrePostActionTy &Action) { 77 return (*reinterpret_cast<Callable *>(CodeGen))(CGF, Action); 78 } 79 80 public: 81 template <typename Callable> 82 RegionCodeGenTy( 83 Callable &&CodeGen, 84 std::enable_if_t<!std::is_same<std::remove_reference_t<Callable>, 85 RegionCodeGenTy>::value> * = nullptr) 86 : CodeGen(reinterpret_cast<intptr_t>(&CodeGen)), 87 Callback(CallbackFn<std::remove_reference_t<Callable>>), 88 PrePostAction(nullptr) {} 89 void setAction(PrePostActionTy &Action) const { PrePostAction = &Action; } 90 void operator()(CodeGenFunction &CGF) const; 91 }; 92 93 struct OMPTaskDataTy final { 94 SmallVector<const Expr *, 4> PrivateVars; 95 SmallVector<const Expr *, 4> PrivateCopies; 96 SmallVector<const Expr *, 4> FirstprivateVars; 97 SmallVector<const Expr *, 4> FirstprivateCopies; 98 SmallVector<const Expr *, 4> FirstprivateInits; 99 SmallVector<const Expr *, 4> LastprivateVars; 100 SmallVector<const Expr *, 4> LastprivateCopies; 101 SmallVector<const Expr *, 4> ReductionVars; 102 SmallVector<const Expr *, 4> ReductionOrigs; 103 SmallVector<const Expr *, 4> ReductionCopies; 104 SmallVector<const Expr *, 4> ReductionOps; 105 SmallVector<CanonicalDeclPtr<const VarDecl>, 4> PrivateLocals; 106 struct DependData { 107 OpenMPDependClauseKind DepKind = OMPC_DEPEND_unknown; 108 const Expr *IteratorExpr = nullptr; 109 SmallVector<const Expr *, 4> DepExprs; 110 explicit DependData() = default; 111 DependData(OpenMPDependClauseKind DepKind, const Expr *IteratorExpr) 112 : DepKind(DepKind), IteratorExpr(IteratorExpr) {} 113 }; 114 SmallVector<DependData, 4> Dependences; 115 llvm::PointerIntPair<llvm::Value *, 1, bool> Final; 116 llvm::PointerIntPair<llvm::Value *, 1, bool> Schedule; 117 llvm::PointerIntPair<llvm::Value *, 1, bool> Priority; 118 llvm::Value *Reductions = nullptr; 119 unsigned NumberOfParts = 0; 120 bool Tied = true; 121 bool Nogroup = false; 122 bool IsReductionWithTaskMod = false; 123 bool IsWorksharingReduction = false; 124 bool HasNowaitClause = false; 125 bool HasModifier = false; 126 }; 127 128 /// Class intended to support codegen of all kind of the reduction clauses. 129 class ReductionCodeGen { 130 private: 131 /// Data required for codegen of reduction clauses. 132 struct ReductionData { 133 /// Reference to the item shared between tasks to reduce into. 134 const Expr *Shared = nullptr; 135 /// Reference to the original item. 136 const Expr *Ref = nullptr; 137 /// Helper expression for generation of private copy. 138 const Expr *Private = nullptr; 139 /// Helper expression for generation reduction operation. 140 const Expr *ReductionOp = nullptr; 141 ReductionData(const Expr *Shared, const Expr *Ref, const Expr *Private, 142 const Expr *ReductionOp) 143 : Shared(Shared), Ref(Ref), Private(Private), ReductionOp(ReductionOp) { 144 } 145 }; 146 /// List of reduction-based clauses. 147 SmallVector<ReductionData, 4> ClausesData; 148 149 /// List of addresses of shared variables/expressions. 150 SmallVector<std::pair<LValue, LValue>, 4> SharedAddresses; 151 /// List of addresses of original variables/expressions. 152 SmallVector<std::pair<LValue, LValue>, 4> OrigAddresses; 153 /// Sizes of the reduction items in chars. 154 SmallVector<std::pair<llvm::Value *, llvm::Value *>, 4> Sizes; 155 /// Base declarations for the reduction items. 156 SmallVector<const VarDecl *, 4> BaseDecls; 157 158 /// Emits lvalue for shared expression. 159 LValue emitSharedLValue(CodeGenFunction &CGF, const Expr *E); 160 /// Emits upper bound for shared expression (if array section). 161 LValue emitSharedLValueUB(CodeGenFunction &CGF, const Expr *E); 162 /// Performs aggregate initialization. 163 /// \param N Number of reduction item in the common list. 164 /// \param PrivateAddr Address of the corresponding private item. 165 /// \param SharedAddr Address of the original shared variable. 166 /// \param DRD Declare reduction construct used for reduction item. 167 void emitAggregateInitialization(CodeGenFunction &CGF, unsigned N, 168 Address PrivateAddr, Address SharedAddr, 169 const OMPDeclareReductionDecl *DRD); 170 171 public: 172 ReductionCodeGen(ArrayRef<const Expr *> Shareds, ArrayRef<const Expr *> Origs, 173 ArrayRef<const Expr *> Privates, 174 ArrayRef<const Expr *> ReductionOps); 175 /// Emits lvalue for the shared and original reduction item. 176 /// \param N Number of the reduction item. 177 void emitSharedOrigLValue(CodeGenFunction &CGF, unsigned N); 178 /// Emits the code for the variable-modified type, if required. 179 /// \param N Number of the reduction item. 180 void emitAggregateType(CodeGenFunction &CGF, unsigned N); 181 /// Emits the code for the variable-modified type, if required. 182 /// \param N Number of the reduction item. 183 /// \param Size Size of the type in chars. 184 void emitAggregateType(CodeGenFunction &CGF, unsigned N, llvm::Value *Size); 185 /// Performs initialization of the private copy for the reduction item. 186 /// \param N Number of the reduction item. 187 /// \param PrivateAddr Address of the corresponding private item. 188 /// \param DefaultInit Default initialization sequence that should be 189 /// performed if no reduction specific initialization is found. 190 /// \param SharedAddr Address of the original shared variable. 191 void 192 emitInitialization(CodeGenFunction &CGF, unsigned N, Address PrivateAddr, 193 Address SharedAddr, 194 llvm::function_ref<bool(CodeGenFunction &)> DefaultInit); 195 /// Returns true if the private copy requires cleanups. 196 bool needCleanups(unsigned N); 197 /// Emits cleanup code for the reduction item. 198 /// \param N Number of the reduction item. 199 /// \param PrivateAddr Address of the corresponding private item. 200 void emitCleanups(CodeGenFunction &CGF, unsigned N, Address PrivateAddr); 201 /// Adjusts \p PrivatedAddr for using instead of the original variable 202 /// address in normal operations. 203 /// \param N Number of the reduction item. 204 /// \param PrivateAddr Address of the corresponding private item. 205 Address adjustPrivateAddress(CodeGenFunction &CGF, unsigned N, 206 Address PrivateAddr); 207 /// Returns LValue for the reduction item. 208 LValue getSharedLValue(unsigned N) const { return SharedAddresses[N].first; } 209 /// Returns LValue for the original reduction item. 210 LValue getOrigLValue(unsigned N) const { return OrigAddresses[N].first; } 211 /// Returns the size of the reduction item (in chars and total number of 212 /// elements in the item), or nullptr, if the size is a constant. 213 std::pair<llvm::Value *, llvm::Value *> getSizes(unsigned N) const { 214 return Sizes[N]; 215 } 216 /// Returns the base declaration of the reduction item. 217 const VarDecl *getBaseDecl(unsigned N) const { return BaseDecls[N]; } 218 /// Returns the base declaration of the reduction item. 219 const Expr *getRefExpr(unsigned N) const { return ClausesData[N].Ref; } 220 /// Returns true if the initialization of the reduction item uses initializer 221 /// from declare reduction construct. 222 bool usesReductionInitializer(unsigned N) const; 223 /// Return the type of the private item. 224 QualType getPrivateType(unsigned N) const { 225 return cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl()) 226 ->getType(); 227 } 228 }; 229 230 class CGOpenMPRuntime { 231 public: 232 /// Allows to disable automatic handling of functions used in target regions 233 /// as those marked as `omp declare target`. 234 class DisableAutoDeclareTargetRAII { 235 CodeGenModule &CGM; 236 bool SavedShouldMarkAsGlobal = false; 237 238 public: 239 DisableAutoDeclareTargetRAII(CodeGenModule &CGM); 240 ~DisableAutoDeclareTargetRAII(); 241 }; 242 243 /// Manages list of nontemporal decls for the specified directive. 244 class NontemporalDeclsRAII { 245 CodeGenModule &CGM; 246 const bool NeedToPush; 247 248 public: 249 NontemporalDeclsRAII(CodeGenModule &CGM, const OMPLoopDirective &S); 250 ~NontemporalDeclsRAII(); 251 }; 252 253 /// Manages list of nontemporal decls for the specified directive. 254 class UntiedTaskLocalDeclsRAII { 255 CodeGenModule &CGM; 256 const bool NeedToPush; 257 258 public: 259 UntiedTaskLocalDeclsRAII( 260 CodeGenFunction &CGF, 261 const llvm::MapVector<CanonicalDeclPtr<const VarDecl>, 262 std::pair<Address, Address>> &LocalVars); 263 ~UntiedTaskLocalDeclsRAII(); 264 }; 265 266 /// Maps the expression for the lastprivate variable to the global copy used 267 /// to store new value because original variables are not mapped in inner 268 /// parallel regions. Only private copies are captured but we need also to 269 /// store private copy in shared address. 270 /// Also, stores the expression for the private loop counter and it 271 /// threaprivate name. 272 struct LastprivateConditionalData { 273 llvm::MapVector<CanonicalDeclPtr<const Decl>, SmallString<16>> 274 DeclToUniqueName; 275 LValue IVLVal; 276 llvm::Function *Fn = nullptr; 277 bool Disabled = false; 278 }; 279 /// Manages list of lastprivate conditional decls for the specified directive. 280 class LastprivateConditionalRAII { 281 enum class ActionToDo { 282 DoNotPush, 283 PushAsLastprivateConditional, 284 DisableLastprivateConditional, 285 }; 286 CodeGenModule &CGM; 287 ActionToDo Action = ActionToDo::DoNotPush; 288 289 /// Check and try to disable analysis of inner regions for changes in 290 /// lastprivate conditional. 291 void tryToDisableInnerAnalysis(const OMPExecutableDirective &S, 292 llvm::DenseSet<CanonicalDeclPtr<const Decl>> 293 &NeedToAddForLPCsAsDisabled) const; 294 295 LastprivateConditionalRAII(CodeGenFunction &CGF, 296 const OMPExecutableDirective &S); 297 298 public: 299 explicit LastprivateConditionalRAII(CodeGenFunction &CGF, 300 const OMPExecutableDirective &S, 301 LValue IVLVal); 302 static LastprivateConditionalRAII disable(CodeGenFunction &CGF, 303 const OMPExecutableDirective &S); 304 ~LastprivateConditionalRAII(); 305 }; 306 307 llvm::OpenMPIRBuilder &getOMPBuilder() { return OMPBuilder; } 308 309 protected: 310 CodeGenModule &CGM; 311 312 /// An OpenMP-IR-Builder instance. 313 llvm::OpenMPIRBuilder OMPBuilder; 314 315 /// Helper to determine the min/max number of threads/teams for \p D. 316 void computeMinAndMaxThreadsAndTeams( 317 const OMPExecutableDirective &D, CodeGenFunction &CGF, 318 llvm::OpenMPIRBuilder::TargetKernelDefaultAttrs &Attrs); 319 320 /// Helper to emit outlined function for 'target' directive. 321 /// \param D Directive to emit. 322 /// \param ParentName Name of the function that encloses the target region. 323 /// \param OutlinedFn Outlined function value to be defined by this call. 324 /// \param OutlinedFnID Outlined function ID value to be defined by this call. 325 /// \param IsOffloadEntry True if the outlined function is an offload entry. 326 /// \param CodeGen Lambda codegen specific to an accelerator device. 327 /// An outlined function may not be an entry if, e.g. the if clause always 328 /// evaluates to false. 329 virtual void emitTargetOutlinedFunctionHelper(const OMPExecutableDirective &D, 330 StringRef ParentName, 331 llvm::Function *&OutlinedFn, 332 llvm::Constant *&OutlinedFnID, 333 bool IsOffloadEntry, 334 const RegionCodeGenTy &CodeGen); 335 336 /// Returns pointer to ident_t type. 337 llvm::Type *getIdentTyPointerTy(); 338 339 /// Gets thread id value for the current thread. 340 /// 341 llvm::Value *getThreadID(CodeGenFunction &CGF, SourceLocation Loc); 342 343 /// Get the function name of an outlined region. 344 std::string getOutlinedHelperName(StringRef Name) const; 345 std::string getOutlinedHelperName(CodeGenFunction &CGF) const; 346 347 /// Get the function name of a reduction function. 348 std::string getReductionFuncName(StringRef Name) const; 349 350 /// Emits \p Callee function call with arguments \p Args with location \p Loc. 351 void emitCall(CodeGenFunction &CGF, SourceLocation Loc, 352 llvm::FunctionCallee Callee, 353 ArrayRef<llvm::Value *> Args = {}) const; 354 355 /// Emits address of the word in a memory where current thread id is 356 /// stored. 357 virtual Address emitThreadIDAddress(CodeGenFunction &CGF, SourceLocation Loc); 358 359 void setLocThreadIdInsertPt(CodeGenFunction &CGF, 360 bool AtCurrentPoint = false); 361 void clearLocThreadIdInsertPt(CodeGenFunction &CGF); 362 363 /// Check if the default location must be constant. 364 /// Default is false to support OMPT/OMPD. 365 virtual bool isDefaultLocationConstant() const { return false; } 366 367 /// Returns additional flags that can be stored in reserved_2 field of the 368 /// default location. 369 virtual unsigned getDefaultLocationReserved2Flags() const { return 0; } 370 371 /// Returns default flags for the barriers depending on the directive, for 372 /// which this barier is going to be emitted. 373 static unsigned getDefaultFlagsForBarriers(OpenMPDirectiveKind Kind); 374 375 /// Get the LLVM type for the critical name. 376 llvm::ArrayType *getKmpCriticalNameTy() const {return KmpCriticalNameTy;} 377 378 /// Returns corresponding lock object for the specified critical region 379 /// name. If the lock object does not exist it is created, otherwise the 380 /// reference to the existing copy is returned. 381 /// \param CriticalName Name of the critical region. 382 /// 383 llvm::Value *getCriticalRegionLock(StringRef CriticalName); 384 385 protected: 386 /// Map for SourceLocation and OpenMP runtime library debug locations. 387 typedef llvm::DenseMap<SourceLocation, llvm::Value *> OpenMPDebugLocMapTy; 388 OpenMPDebugLocMapTy OpenMPDebugLocMap; 389 /// The type for a microtask which gets passed to __kmpc_fork_call(). 390 /// Original representation is: 391 /// typedef void (kmpc_micro)(kmp_int32 global_tid, kmp_int32 bound_tid,...); 392 llvm::FunctionType *Kmpc_MicroTy = nullptr; 393 /// Stores debug location and ThreadID for the function. 394 struct DebugLocThreadIdTy { 395 llvm::Value *DebugLoc; 396 llvm::Value *ThreadID; 397 /// Insert point for the service instructions. 398 llvm::AssertingVH<llvm::Instruction> ServiceInsertPt = nullptr; 399 }; 400 /// Map of local debug location, ThreadId and functions. 401 typedef llvm::DenseMap<llvm::Function *, DebugLocThreadIdTy> 402 OpenMPLocThreadIDMapTy; 403 OpenMPLocThreadIDMapTy OpenMPLocThreadIDMap; 404 /// Map of UDRs and corresponding combiner/initializer. 405 typedef llvm::DenseMap<const OMPDeclareReductionDecl *, 406 std::pair<llvm::Function *, llvm::Function *>> 407 UDRMapTy; 408 UDRMapTy UDRMap; 409 /// Map of functions and locally defined UDRs. 410 typedef llvm::DenseMap<llvm::Function *, 411 SmallVector<const OMPDeclareReductionDecl *, 4>> 412 FunctionUDRMapTy; 413 FunctionUDRMapTy FunctionUDRMap; 414 /// Map from the user-defined mapper declaration to its corresponding 415 /// functions. 416 llvm::DenseMap<const OMPDeclareMapperDecl *, llvm::Function *> UDMMap; 417 /// Map of functions and their local user-defined mappers. 418 using FunctionUDMMapTy = 419 llvm::DenseMap<llvm::Function *, 420 SmallVector<const OMPDeclareMapperDecl *, 4>>; 421 FunctionUDMMapTy FunctionUDMMap; 422 /// Maps local variables marked as lastprivate conditional to their internal 423 /// types. 424 llvm::DenseMap<llvm::Function *, 425 llvm::DenseMap<CanonicalDeclPtr<const Decl>, 426 std::tuple<QualType, const FieldDecl *, 427 const FieldDecl *, LValue>>> 428 LastprivateConditionalToTypes; 429 /// Maps function to the position of the untied task locals stack. 430 llvm::DenseMap<llvm::Function *, unsigned> FunctionToUntiedTaskStackMap; 431 /// Type kmp_critical_name, originally defined as typedef kmp_int32 432 /// kmp_critical_name[8]; 433 llvm::ArrayType *KmpCriticalNameTy; 434 /// An ordered map of auto-generated variables to their unique names. 435 /// It stores variables with the following names: 1) ".gomp_critical_user_" + 436 /// <critical_section_name> + ".var" for "omp critical" directives; 2) 437 /// <mangled_name_for_global_var> + ".cache." for cache for threadprivate 438 /// variables. 439 llvm::StringMap<llvm::AssertingVH<llvm::GlobalVariable>, 440 llvm::BumpPtrAllocator> InternalVars; 441 /// Type typedef kmp_int32 (* kmp_routine_entry_t)(kmp_int32, void *); 442 llvm::Type *KmpRoutineEntryPtrTy = nullptr; 443 QualType KmpRoutineEntryPtrQTy; 444 /// Type typedef struct kmp_task { 445 /// void * shareds; /**< pointer to block of pointers to 446 /// shared vars */ 447 /// kmp_routine_entry_t routine; /**< pointer to routine to call for 448 /// executing task */ 449 /// kmp_int32 part_id; /**< part id for the task */ 450 /// kmp_routine_entry_t destructors; /* pointer to function to invoke 451 /// deconstructors of firstprivate C++ objects */ 452 /// } kmp_task_t; 453 QualType KmpTaskTQTy; 454 /// Saved kmp_task_t for task directive. 455 QualType SavedKmpTaskTQTy; 456 /// Saved kmp_task_t for taskloop-based directive. 457 QualType SavedKmpTaskloopTQTy; 458 /// Type typedef struct kmp_depend_info { 459 /// kmp_intptr_t base_addr; 460 /// size_t len; 461 /// struct { 462 /// bool in:1; 463 /// bool out:1; 464 /// } flags; 465 /// } kmp_depend_info_t; 466 QualType KmpDependInfoTy; 467 /// Type typedef struct kmp_task_affinity_info { 468 /// kmp_intptr_t base_addr; 469 /// size_t len; 470 /// struct { 471 /// bool flag1 : 1; 472 /// bool flag2 : 1; 473 /// kmp_int32 reserved : 30; 474 /// } flags; 475 /// } kmp_task_affinity_info_t; 476 QualType KmpTaskAffinityInfoTy; 477 /// struct kmp_dim { // loop bounds info casted to kmp_int64 478 /// kmp_int64 lo; // lower 479 /// kmp_int64 up; // upper 480 /// kmp_int64 st; // stride 481 /// }; 482 QualType KmpDimTy; 483 484 bool ShouldMarkAsGlobal = true; 485 /// List of the emitted declarations. 486 llvm::DenseSet<CanonicalDeclPtr<const Decl>> AlreadyEmittedTargetDecls; 487 /// List of the global variables with their addresses that should not be 488 /// emitted for the target. 489 llvm::StringMap<llvm::WeakTrackingVH> EmittedNonTargetVariables; 490 491 /// List of variables that can become declare target implicitly and, thus, 492 /// must be emitted. 493 llvm::SmallDenseSet<const VarDecl *> DeferredGlobalVariables; 494 495 using NontemporalDeclsSet = llvm::SmallDenseSet<CanonicalDeclPtr<const Decl>>; 496 /// Stack for list of declarations in current context marked as nontemporal. 497 /// The set is the union of all current stack elements. 498 llvm::SmallVector<NontemporalDeclsSet, 4> NontemporalDeclsStack; 499 500 using UntiedLocalVarsAddressesMap = 501 llvm::MapVector<CanonicalDeclPtr<const VarDecl>, 502 std::pair<Address, Address>>; 503 llvm::SmallVector<UntiedLocalVarsAddressesMap, 4> UntiedLocalVarsStack; 504 505 /// Stack for list of addresses of declarations in current context marked as 506 /// lastprivate conditional. The set is the union of all current stack 507 /// elements. 508 llvm::SmallVector<LastprivateConditionalData, 4> LastprivateConditionalStack; 509 510 /// Flag for keeping track of weather a requires unified_shared_memory 511 /// directive is present. 512 bool HasRequiresUnifiedSharedMemory = false; 513 514 /// Atomic ordering from the omp requires directive. 515 llvm::AtomicOrdering RequiresAtomicOrdering = llvm::AtomicOrdering::Monotonic; 516 517 /// Flag for keeping track of weather a target region has been emitted. 518 bool HasEmittedTargetRegion = false; 519 520 /// Flag for keeping track of weather a device routine has been emitted. 521 /// Device routines are specific to the 522 bool HasEmittedDeclareTargetRegion = false; 523 524 /// Start scanning from statement \a S and emit all target regions 525 /// found along the way. 526 /// \param S Starting statement. 527 /// \param ParentName Name of the function declaration that is being scanned. 528 void scanForTargetRegionsFunctions(const Stmt *S, StringRef ParentName); 529 530 /// Build type kmp_routine_entry_t (if not built yet). 531 void emitKmpRoutineEntryT(QualType KmpInt32Ty); 532 533 /// Returns pointer to kmpc_micro type. 534 llvm::Type *getKmpc_MicroPointerTy(); 535 536 /// If the specified mangled name is not in the module, create and 537 /// return threadprivate cache object. This object is a pointer's worth of 538 /// storage that's reserved for use by the OpenMP runtime. 539 /// \param VD Threadprivate variable. 540 /// \return Cache variable for the specified threadprivate. 541 llvm::Constant *getOrCreateThreadPrivateCache(const VarDecl *VD); 542 543 /// Set of threadprivate variables with the generated initializer. 544 llvm::StringSet<> ThreadPrivateWithDefinition; 545 546 /// Set of declare target variables with the generated initializer. 547 llvm::StringSet<> DeclareTargetWithDefinition; 548 549 /// Emits initialization code for the threadprivate variables. 550 /// \param VDAddr Address of the global variable \a VD. 551 /// \param Ctor Pointer to a global init function for \a VD. 552 /// \param CopyCtor Pointer to a global copy function for \a VD. 553 /// \param Dtor Pointer to a global destructor function for \a VD. 554 /// \param Loc Location of threadprivate declaration. 555 void emitThreadPrivateVarInit(CodeGenFunction &CGF, Address VDAddr, 556 llvm::Value *Ctor, llvm::Value *CopyCtor, 557 llvm::Value *Dtor, SourceLocation Loc); 558 559 struct TaskResultTy { 560 llvm::Value *NewTask = nullptr; 561 llvm::Function *TaskEntry = nullptr; 562 llvm::Value *NewTaskNewTaskTTy = nullptr; 563 LValue TDBase; 564 const RecordDecl *KmpTaskTQTyRD = nullptr; 565 llvm::Value *TaskDupFn = nullptr; 566 }; 567 /// Emit task region for the task directive. The task region is emitted in 568 /// several steps: 569 /// 1. Emit a call to kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32 570 /// gtid, kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds, 571 /// kmp_routine_entry_t *task_entry). Here task_entry is a pointer to the 572 /// function: 573 /// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) { 574 /// TaskFunction(gtid, tt->part_id, tt->shareds); 575 /// return 0; 576 /// } 577 /// 2. Copy a list of shared variables to field shareds of the resulting 578 /// structure kmp_task_t returned by the previous call (if any). 579 /// 3. Copy a pointer to destructions function to field destructions of the 580 /// resulting structure kmp_task_t. 581 /// \param D Current task directive. 582 /// \param TaskFunction An LLVM function with type void (*)(i32 /*gtid*/, i32 583 /// /*part_id*/, captured_struct */*__context*/); 584 /// \param SharedsTy A type which contains references the shared variables. 585 /// \param Shareds Context with the list of shared variables from the \p 586 /// TaskFunction. 587 /// \param Data Additional data for task generation like tiednsee, final 588 /// state, list of privates etc. 589 TaskResultTy emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc, 590 const OMPExecutableDirective &D, 591 llvm::Function *TaskFunction, QualType SharedsTy, 592 Address Shareds, const OMPTaskDataTy &Data); 593 594 /// Emit update for lastprivate conditional data. 595 void emitLastprivateConditionalUpdate(CodeGenFunction &CGF, LValue IVLVal, 596 StringRef UniqueDeclName, LValue LVal, 597 SourceLocation Loc); 598 599 /// Returns the number of the elements and the address of the depobj 600 /// dependency array. 601 /// \return Number of elements in depobj array and the pointer to the array of 602 /// dependencies. 603 std::pair<llvm::Value *, LValue> getDepobjElements(CodeGenFunction &CGF, 604 LValue DepobjLVal, 605 SourceLocation Loc); 606 607 SmallVector<llvm::Value *, 4> 608 emitDepobjElementsSizes(CodeGenFunction &CGF, QualType &KmpDependInfoTy, 609 const OMPTaskDataTy::DependData &Data); 610 611 void emitDepobjElements(CodeGenFunction &CGF, QualType &KmpDependInfoTy, 612 LValue PosLVal, const OMPTaskDataTy::DependData &Data, 613 Address DependenciesArray); 614 615 public: 616 explicit CGOpenMPRuntime(CodeGenModule &CGM); 617 virtual ~CGOpenMPRuntime() {} 618 virtual void clear(); 619 620 /// Emits object of ident_t type with info for source location. 621 /// \param Flags Flags for OpenMP location. 622 /// \param EmitLoc emit source location with debug-info is off. 623 /// 624 llvm::Value *emitUpdateLocation(CodeGenFunction &CGF, SourceLocation Loc, 625 unsigned Flags = 0, bool EmitLoc = false); 626 627 /// Emit the number of teams for a target directive. Inspect the num_teams 628 /// clause associated with a teams construct combined or closely nested 629 /// with the target directive. 630 /// 631 /// Emit a team of size one for directives such as 'target parallel' that 632 /// have no associated teams construct. 633 /// 634 /// Otherwise, return nullptr. 635 const Expr *getNumTeamsExprForTargetDirective(CodeGenFunction &CGF, 636 const OMPExecutableDirective &D, 637 int32_t &MinTeamsVal, 638 int32_t &MaxTeamsVal); 639 llvm::Value *emitNumTeamsForTargetDirective(CodeGenFunction &CGF, 640 const OMPExecutableDirective &D); 641 642 /// Check for a number of threads upper bound constant value (stored in \p 643 /// UpperBound), or expression (returned). If the value is conditional (via an 644 /// if-clause), store the condition in \p CondExpr. Similarly, a potential 645 /// thread limit expression is stored in \p ThreadLimitExpr. If \p 646 /// UpperBoundOnly is true, no expression evaluation is perfomed. 647 const Expr *getNumThreadsExprForTargetDirective( 648 CodeGenFunction &CGF, const OMPExecutableDirective &D, 649 int32_t &UpperBound, bool UpperBoundOnly, 650 llvm::Value **CondExpr = nullptr, const Expr **ThreadLimitExpr = nullptr); 651 652 /// Emit an expression that denotes the number of threads a target region 653 /// shall use. Will generate "i32 0" to allow the runtime to choose. 654 llvm::Value * 655 emitNumThreadsForTargetDirective(CodeGenFunction &CGF, 656 const OMPExecutableDirective &D); 657 658 /// Return the trip count of loops associated with constructs / 'target teams 659 /// distribute' and 'teams distribute parallel for'. \param SizeEmitter Emits 660 /// the int64 value for the number of iterations of the associated loop. 661 llvm::Value *emitTargetNumIterationsCall( 662 CodeGenFunction &CGF, const OMPExecutableDirective &D, 663 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF, 664 const OMPLoopDirective &D)> 665 SizeEmitter); 666 667 /// Returns true if the current target is a GPU. 668 virtual bool isGPU() const { return false; } 669 670 /// Check if the variable length declaration is delayed: 671 virtual bool isDelayedVariableLengthDecl(CodeGenFunction &CGF, 672 const VarDecl *VD) const { 673 return false; 674 }; 675 676 /// Get call to __kmpc_alloc_shared 677 virtual std::pair<llvm::Value *, llvm::Value *> 678 getKmpcAllocShared(CodeGenFunction &CGF, const VarDecl *VD) { 679 llvm_unreachable("not implemented"); 680 } 681 682 /// Get call to __kmpc_free_shared 683 virtual void getKmpcFreeShared( 684 CodeGenFunction &CGF, 685 const std::pair<llvm::Value *, llvm::Value *> &AddrSizePair) { 686 llvm_unreachable("not implemented"); 687 } 688 689 /// Emits code for OpenMP 'if' clause using specified \a CodeGen 690 /// function. Here is the logic: 691 /// if (Cond) { 692 /// ThenGen(); 693 /// } else { 694 /// ElseGen(); 695 /// } 696 void emitIfClause(CodeGenFunction &CGF, const Expr *Cond, 697 const RegionCodeGenTy &ThenGen, 698 const RegionCodeGenTy &ElseGen); 699 700 /// Checks if the \p Body is the \a CompoundStmt and returns its child 701 /// statement iff there is only one that is not evaluatable at the compile 702 /// time. 703 static const Stmt *getSingleCompoundChild(ASTContext &Ctx, const Stmt *Body); 704 705 /// Get the platform-specific name separator. 706 std::string getName(ArrayRef<StringRef> Parts) const; 707 708 /// Emit code for the specified user defined reduction construct. 709 virtual void emitUserDefinedReduction(CodeGenFunction *CGF, 710 const OMPDeclareReductionDecl *D); 711 /// Get combiner/initializer for the specified user-defined reduction, if any. 712 virtual std::pair<llvm::Function *, llvm::Function *> 713 getUserDefinedReduction(const OMPDeclareReductionDecl *D); 714 715 /// Emit the function for the user defined mapper construct. 716 void emitUserDefinedMapper(const OMPDeclareMapperDecl *D, 717 CodeGenFunction *CGF = nullptr); 718 /// Get the function for the specified user-defined mapper. If it does not 719 /// exist, create one. 720 llvm::Function * 721 getOrCreateUserDefinedMapperFunc(const OMPDeclareMapperDecl *D); 722 723 /// Emits outlined function for the specified OpenMP parallel directive 724 /// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID, 725 /// kmp_int32 BoundID, struct context_vars*). 726 /// \param CGF Reference to current CodeGenFunction. 727 /// \param D OpenMP directive. 728 /// \param ThreadIDVar Variable for thread id in the current OpenMP region. 729 /// \param InnermostKind Kind of innermost directive (for simple directives it 730 /// is a directive itself, for combined - its innermost directive). 731 /// \param CodeGen Code generation sequence for the \a D directive. 732 virtual llvm::Function *emitParallelOutlinedFunction( 733 CodeGenFunction &CGF, const OMPExecutableDirective &D, 734 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, 735 const RegionCodeGenTy &CodeGen); 736 737 /// Emits outlined function for the specified OpenMP teams directive 738 /// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID, 739 /// kmp_int32 BoundID, struct context_vars*). 740 /// \param CGF Reference to current CodeGenFunction. 741 /// \param D OpenMP directive. 742 /// \param ThreadIDVar Variable for thread id in the current OpenMP region. 743 /// \param InnermostKind Kind of innermost directive (for simple directives it 744 /// is a directive itself, for combined - its innermost directive). 745 /// \param CodeGen Code generation sequence for the \a D directive. 746 virtual llvm::Function *emitTeamsOutlinedFunction( 747 CodeGenFunction &CGF, const OMPExecutableDirective &D, 748 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, 749 const RegionCodeGenTy &CodeGen); 750 751 /// Emits outlined function for the OpenMP task directive \a D. This 752 /// outlined function has type void(*)(kmp_int32 ThreadID, struct task_t* 753 /// TaskT). 754 /// \param D OpenMP directive. 755 /// \param ThreadIDVar Variable for thread id in the current OpenMP region. 756 /// \param PartIDVar Variable for partition id in the current OpenMP untied 757 /// task region. 758 /// \param TaskTVar Variable for task_t argument. 759 /// \param InnermostKind Kind of innermost directive (for simple directives it 760 /// is a directive itself, for combined - its innermost directive). 761 /// \param CodeGen Code generation sequence for the \a D directive. 762 /// \param Tied true if task is generated for tied task, false otherwise. 763 /// \param NumberOfParts Number of parts in untied task. Ignored for tied 764 /// tasks. 765 /// 766 virtual llvm::Function *emitTaskOutlinedFunction( 767 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, 768 const VarDecl *PartIDVar, const VarDecl *TaskTVar, 769 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, 770 bool Tied, unsigned &NumberOfParts); 771 772 /// Cleans up references to the objects in finished function. 773 /// 774 virtual void functionFinished(CodeGenFunction &CGF); 775 776 /// Emits code for parallel or serial call of the \a OutlinedFn with 777 /// variables captured in a record which address is stored in \a 778 /// CapturedStruct. 779 /// \param OutlinedFn Outlined function to be run in parallel threads. Type of 780 /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*). 781 /// \param CapturedVars A pointer to the record with the references to 782 /// variables used in \a OutlinedFn function. 783 /// \param IfCond Condition in the associated 'if' clause, if it was 784 /// specified, nullptr otherwise. 785 /// \param NumThreads The value corresponding to the num_threads clause, if 786 /// any, or nullptr. 787 /// 788 virtual void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, 789 llvm::Function *OutlinedFn, 790 ArrayRef<llvm::Value *> CapturedVars, 791 const Expr *IfCond, llvm::Value *NumThreads); 792 793 /// Emits a critical region. 794 /// \param CriticalName Name of the critical region. 795 /// \param CriticalOpGen Generator for the statement associated with the given 796 /// critical region. 797 /// \param Hint Value of the 'hint' clause (optional). 798 virtual void emitCriticalRegion(CodeGenFunction &CGF, StringRef CriticalName, 799 const RegionCodeGenTy &CriticalOpGen, 800 SourceLocation Loc, 801 const Expr *Hint = nullptr); 802 803 /// Emits a master region. 804 /// \param MasterOpGen Generator for the statement associated with the given 805 /// master region. 806 virtual void emitMasterRegion(CodeGenFunction &CGF, 807 const RegionCodeGenTy &MasterOpGen, 808 SourceLocation Loc); 809 810 /// Emits a masked region. 811 /// \param MaskedOpGen Generator for the statement associated with the given 812 /// masked region. 813 virtual void emitMaskedRegion(CodeGenFunction &CGF, 814 const RegionCodeGenTy &MaskedOpGen, 815 SourceLocation Loc, 816 const Expr *Filter = nullptr); 817 818 /// Emits code for a taskyield directive. 819 virtual void emitTaskyieldCall(CodeGenFunction &CGF, SourceLocation Loc); 820 821 /// Emit __kmpc_error call for error directive 822 /// extern void __kmpc_error(ident_t *loc, int severity, const char *message); 823 virtual void emitErrorCall(CodeGenFunction &CGF, SourceLocation Loc, Expr *ME, 824 bool IsFatal); 825 826 /// Emit a taskgroup region. 827 /// \param TaskgroupOpGen Generator for the statement associated with the 828 /// given taskgroup region. 829 virtual void emitTaskgroupRegion(CodeGenFunction &CGF, 830 const RegionCodeGenTy &TaskgroupOpGen, 831 SourceLocation Loc); 832 833 /// Emits a single region. 834 /// \param SingleOpGen Generator for the statement associated with the given 835 /// single region. 836 virtual void emitSingleRegion(CodeGenFunction &CGF, 837 const RegionCodeGenTy &SingleOpGen, 838 SourceLocation Loc, 839 ArrayRef<const Expr *> CopyprivateVars, 840 ArrayRef<const Expr *> DestExprs, 841 ArrayRef<const Expr *> SrcExprs, 842 ArrayRef<const Expr *> AssignmentOps); 843 844 /// Emit an ordered region. 845 /// \param OrderedOpGen Generator for the statement associated with the given 846 /// ordered region. 847 virtual void emitOrderedRegion(CodeGenFunction &CGF, 848 const RegionCodeGenTy &OrderedOpGen, 849 SourceLocation Loc, bool IsThreads); 850 851 /// Emit an implicit/explicit barrier for OpenMP threads. 852 /// \param Kind Directive for which this implicit barrier call must be 853 /// generated. Must be OMPD_barrier for explicit barrier generation. 854 /// \param EmitChecks true if need to emit checks for cancellation barriers. 855 /// \param ForceSimpleCall true simple barrier call must be emitted, false if 856 /// runtime class decides which one to emit (simple or with cancellation 857 /// checks). 858 /// 859 virtual void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc, 860 OpenMPDirectiveKind Kind, 861 bool EmitChecks = true, 862 bool ForceSimpleCall = false); 863 864 /// Check if the specified \a ScheduleKind is static non-chunked. 865 /// This kind of worksharing directive is emitted without outer loop. 866 /// \param ScheduleKind Schedule kind specified in the 'schedule' clause. 867 /// \param Chunked True if chunk is specified in the clause. 868 /// 869 virtual bool isStaticNonchunked(OpenMPScheduleClauseKind ScheduleKind, 870 bool Chunked) const; 871 872 /// Check if the specified \a ScheduleKind is static non-chunked. 873 /// This kind of distribute directive is emitted without outer loop. 874 /// \param ScheduleKind Schedule kind specified in the 'dist_schedule' clause. 875 /// \param Chunked True if chunk is specified in the clause. 876 /// 877 virtual bool isStaticNonchunked(OpenMPDistScheduleClauseKind ScheduleKind, 878 bool Chunked) const; 879 880 /// Check if the specified \a ScheduleKind is static chunked. 881 /// \param ScheduleKind Schedule kind specified in the 'schedule' clause. 882 /// \param Chunked True if chunk is specified in the clause. 883 /// 884 virtual bool isStaticChunked(OpenMPScheduleClauseKind ScheduleKind, 885 bool Chunked) const; 886 887 /// Check if the specified \a ScheduleKind is static non-chunked. 888 /// \param ScheduleKind Schedule kind specified in the 'dist_schedule' clause. 889 /// \param Chunked True if chunk is specified in the clause. 890 /// 891 virtual bool isStaticChunked(OpenMPDistScheduleClauseKind ScheduleKind, 892 bool Chunked) const; 893 894 /// Check if the specified \a ScheduleKind is dynamic. 895 /// This kind of worksharing directive is emitted without outer loop. 896 /// \param ScheduleKind Schedule Kind specified in the 'schedule' clause. 897 /// 898 virtual bool isDynamic(OpenMPScheduleClauseKind ScheduleKind) const; 899 900 /// struct with the values to be passed to the dispatch runtime function 901 struct DispatchRTInput { 902 /// Loop lower bound 903 llvm::Value *LB = nullptr; 904 /// Loop upper bound 905 llvm::Value *UB = nullptr; 906 /// Chunk size specified using 'schedule' clause (nullptr if chunk 907 /// was not specified) 908 llvm::Value *Chunk = nullptr; 909 DispatchRTInput() = default; 910 DispatchRTInput(llvm::Value *LB, llvm::Value *UB, llvm::Value *Chunk) 911 : LB(LB), UB(UB), Chunk(Chunk) {} 912 }; 913 914 /// Call the appropriate runtime routine to initialize it before start 915 /// of loop. 916 917 /// This is used for non static scheduled types and when the ordered 918 /// clause is present on the loop construct. 919 /// Depending on the loop schedule, it is necessary to call some runtime 920 /// routine before start of the OpenMP loop to get the loop upper / lower 921 /// bounds \a LB and \a UB and stride \a ST. 922 /// 923 /// \param CGF Reference to current CodeGenFunction. 924 /// \param Loc Clang source location. 925 /// \param ScheduleKind Schedule kind, specified by the 'schedule' clause. 926 /// \param IVSize Size of the iteration variable in bits. 927 /// \param IVSigned Sign of the iteration variable. 928 /// \param Ordered true if loop is ordered, false otherwise. 929 /// \param DispatchValues struct containing llvm values for lower bound, upper 930 /// bound, and chunk expression. 931 /// For the default (nullptr) value, the chunk 1 will be used. 932 /// 933 virtual void emitForDispatchInit(CodeGenFunction &CGF, SourceLocation Loc, 934 const OpenMPScheduleTy &ScheduleKind, 935 unsigned IVSize, bool IVSigned, bool Ordered, 936 const DispatchRTInput &DispatchValues); 937 938 /// This is used for non static scheduled types and when the ordered 939 /// clause is present on the loop construct. 940 /// 941 /// \param CGF Reference to current CodeGenFunction. 942 /// \param Loc Clang source location. 943 /// 944 virtual void emitForDispatchDeinit(CodeGenFunction &CGF, SourceLocation Loc); 945 946 /// Struct with the values to be passed to the static runtime function 947 struct StaticRTInput { 948 /// Size of the iteration variable in bits. 949 unsigned IVSize = 0; 950 /// Sign of the iteration variable. 951 bool IVSigned = false; 952 /// true if loop is ordered, false otherwise. 953 bool Ordered = false; 954 /// Address of the output variable in which the flag of the last iteration 955 /// is returned. 956 Address IL = Address::invalid(); 957 /// Address of the output variable in which the lower iteration number is 958 /// returned. 959 Address LB = Address::invalid(); 960 /// Address of the output variable in which the upper iteration number is 961 /// returned. 962 Address UB = Address::invalid(); 963 /// Address of the output variable in which the stride value is returned 964 /// necessary to generated the static_chunked scheduled loop. 965 Address ST = Address::invalid(); 966 /// Value of the chunk for the static_chunked scheduled loop. For the 967 /// default (nullptr) value, the chunk 1 will be used. 968 llvm::Value *Chunk = nullptr; 969 StaticRTInput(unsigned IVSize, bool IVSigned, bool Ordered, Address IL, 970 Address LB, Address UB, Address ST, 971 llvm::Value *Chunk = nullptr) 972 : IVSize(IVSize), IVSigned(IVSigned), Ordered(Ordered), IL(IL), LB(LB), 973 UB(UB), ST(ST), Chunk(Chunk) {} 974 }; 975 /// Call the appropriate runtime routine to initialize it before start 976 /// of loop. 977 /// 978 /// This is used only in case of static schedule, when the user did not 979 /// specify a ordered clause on the loop construct. 980 /// Depending on the loop schedule, it is necessary to call some runtime 981 /// routine before start of the OpenMP loop to get the loop upper / lower 982 /// bounds LB and UB and stride ST. 983 /// 984 /// \param CGF Reference to current CodeGenFunction. 985 /// \param Loc Clang source location. 986 /// \param DKind Kind of the directive. 987 /// \param ScheduleKind Schedule kind, specified by the 'schedule' clause. 988 /// \param Values Input arguments for the construct. 989 /// 990 virtual void emitForStaticInit(CodeGenFunction &CGF, SourceLocation Loc, 991 OpenMPDirectiveKind DKind, 992 const OpenMPScheduleTy &ScheduleKind, 993 const StaticRTInput &Values); 994 995 /// 996 /// \param CGF Reference to current CodeGenFunction. 997 /// \param Loc Clang source location. 998 /// \param SchedKind Schedule kind, specified by the 'dist_schedule' clause. 999 /// \param Values Input arguments for the construct. 1000 /// 1001 virtual void emitDistributeStaticInit(CodeGenFunction &CGF, 1002 SourceLocation Loc, 1003 OpenMPDistScheduleClauseKind SchedKind, 1004 const StaticRTInput &Values); 1005 1006 /// Call the appropriate runtime routine to notify that we finished 1007 /// iteration of the ordered loop with the dynamic scheduling. 1008 /// 1009 /// \param CGF Reference to current CodeGenFunction. 1010 /// \param Loc Clang source location. 1011 /// \param IVSize Size of the iteration variable in bits. 1012 /// \param IVSigned Sign of the iteration variable. 1013 /// 1014 virtual void emitForOrderedIterationEnd(CodeGenFunction &CGF, 1015 SourceLocation Loc, unsigned IVSize, 1016 bool IVSigned); 1017 1018 /// Call the appropriate runtime routine to notify that we finished 1019 /// all the work with current loop. 1020 /// 1021 /// \param CGF Reference to current CodeGenFunction. 1022 /// \param Loc Clang source location. 1023 /// \param DKind Kind of the directive for which the static finish is emitted. 1024 /// 1025 virtual void emitForStaticFinish(CodeGenFunction &CGF, SourceLocation Loc, 1026 OpenMPDirectiveKind DKind); 1027 1028 /// Call __kmpc_dispatch_next( 1029 /// ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter, 1030 /// kmp_int[32|64] *p_lower, kmp_int[32|64] *p_upper, 1031 /// kmp_int[32|64] *p_stride); 1032 /// \param IVSize Size of the iteration variable in bits. 1033 /// \param IVSigned Sign of the iteration variable. 1034 /// \param IL Address of the output variable in which the flag of the 1035 /// last iteration is returned. 1036 /// \param LB Address of the output variable in which the lower iteration 1037 /// number is returned. 1038 /// \param UB Address of the output variable in which the upper iteration 1039 /// number is returned. 1040 /// \param ST Address of the output variable in which the stride value is 1041 /// returned. 1042 virtual llvm::Value *emitForNext(CodeGenFunction &CGF, SourceLocation Loc, 1043 unsigned IVSize, bool IVSigned, 1044 Address IL, Address LB, 1045 Address UB, Address ST); 1046 1047 /// Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32 1048 /// global_tid, kmp_int32 num_threads) to generate code for 'num_threads' 1049 /// clause. 1050 /// \param NumThreads An integer value of threads. 1051 virtual void emitNumThreadsClause(CodeGenFunction &CGF, 1052 llvm::Value *NumThreads, 1053 SourceLocation Loc); 1054 1055 /// Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 1056 /// global_tid, int proc_bind) to generate code for 'proc_bind' clause. 1057 virtual void emitProcBindClause(CodeGenFunction &CGF, 1058 llvm::omp::ProcBindKind ProcBind, 1059 SourceLocation Loc); 1060 1061 /// Returns address of the threadprivate variable for the current 1062 /// thread. 1063 /// \param VD Threadprivate variable. 1064 /// \param VDAddr Address of the global variable \a VD. 1065 /// \param Loc Location of the reference to threadprivate var. 1066 /// \return Address of the threadprivate variable for the current thread. 1067 virtual Address getAddrOfThreadPrivate(CodeGenFunction &CGF, 1068 const VarDecl *VD, Address VDAddr, 1069 SourceLocation Loc); 1070 1071 /// Returns the address of the variable marked as declare target with link 1072 /// clause OR as declare target with to clause and unified memory. 1073 virtual ConstantAddress getAddrOfDeclareTargetVar(const VarDecl *VD); 1074 1075 /// Emit a code for initialization of threadprivate variable. It emits 1076 /// a call to runtime library which adds initial value to the newly created 1077 /// threadprivate variable (if it is not constant) and registers destructor 1078 /// for the variable (if any). 1079 /// \param VD Threadprivate variable. 1080 /// \param VDAddr Address of the global variable \a VD. 1081 /// \param Loc Location of threadprivate declaration. 1082 /// \param PerformInit true if initialization expression is not constant. 1083 virtual llvm::Function * 1084 emitThreadPrivateVarDefinition(const VarDecl *VD, Address VDAddr, 1085 SourceLocation Loc, bool PerformInit, 1086 CodeGenFunction *CGF = nullptr); 1087 1088 /// Emit code for handling declare target functions in the runtime. 1089 /// \param FD Declare target function. 1090 /// \param Addr Address of the global \a FD. 1091 /// \param PerformInit true if initialization expression is not constant. 1092 virtual void emitDeclareTargetFunction(const FunctionDecl *FD, 1093 llvm::GlobalValue *GV); 1094 1095 /// Creates artificial threadprivate variable with name \p Name and type \p 1096 /// VarType. 1097 /// \param VarType Type of the artificial threadprivate variable. 1098 /// \param Name Name of the artificial threadprivate variable. 1099 virtual Address getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF, 1100 QualType VarType, 1101 StringRef Name); 1102 1103 /// Emit flush of the variables specified in 'omp flush' directive. 1104 /// \param Vars List of variables to flush. 1105 virtual void emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *> Vars, 1106 SourceLocation Loc, llvm::AtomicOrdering AO); 1107 1108 /// Emit task region for the task directive. The task region is 1109 /// emitted in several steps: 1110 /// 1. Emit a call to kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32 1111 /// gtid, kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds, 1112 /// kmp_routine_entry_t *task_entry). Here task_entry is a pointer to the 1113 /// function: 1114 /// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) { 1115 /// TaskFunction(gtid, tt->part_id, tt->shareds); 1116 /// return 0; 1117 /// } 1118 /// 2. Copy a list of shared variables to field shareds of the resulting 1119 /// structure kmp_task_t returned by the previous call (if any). 1120 /// 3. Copy a pointer to destructions function to field destructions of the 1121 /// resulting structure kmp_task_t. 1122 /// 4. Emit a call to kmp_int32 __kmpc_omp_task(ident_t *, kmp_int32 gtid, 1123 /// kmp_task_t *new_task), where new_task is a resulting structure from 1124 /// previous items. 1125 /// \param D Current task directive. 1126 /// \param TaskFunction An LLVM function with type void (*)(i32 /*gtid*/, i32 1127 /// /*part_id*/, captured_struct */*__context*/); 1128 /// \param SharedsTy A type which contains references the shared variables. 1129 /// \param Shareds Context with the list of shared variables from the \p 1130 /// TaskFunction. 1131 /// \param IfCond Not a nullptr if 'if' clause was specified, nullptr 1132 /// otherwise. 1133 /// \param Data Additional data for task generation like tiednsee, final 1134 /// state, list of privates etc. 1135 virtual void emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc, 1136 const OMPExecutableDirective &D, 1137 llvm::Function *TaskFunction, QualType SharedsTy, 1138 Address Shareds, const Expr *IfCond, 1139 const OMPTaskDataTy &Data); 1140 1141 /// Emit task region for the taskloop directive. The taskloop region is 1142 /// emitted in several steps: 1143 /// 1. Emit a call to kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32 1144 /// gtid, kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds, 1145 /// kmp_routine_entry_t *task_entry). Here task_entry is a pointer to the 1146 /// function: 1147 /// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) { 1148 /// TaskFunction(gtid, tt->part_id, tt->shareds); 1149 /// return 0; 1150 /// } 1151 /// 2. Copy a list of shared variables to field shareds of the resulting 1152 /// structure kmp_task_t returned by the previous call (if any). 1153 /// 3. Copy a pointer to destructions function to field destructions of the 1154 /// resulting structure kmp_task_t. 1155 /// 4. Emit a call to void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t 1156 /// *task, int if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int 1157 /// nogroup, int sched, kmp_uint64 grainsize, void *task_dup ), where new_task 1158 /// is a resulting structure from 1159 /// previous items. 1160 /// \param D Current task directive. 1161 /// \param TaskFunction An LLVM function with type void (*)(i32 /*gtid*/, i32 1162 /// /*part_id*/, captured_struct */*__context*/); 1163 /// \param SharedsTy A type which contains references the shared variables. 1164 /// \param Shareds Context with the list of shared variables from the \p 1165 /// TaskFunction. 1166 /// \param IfCond Not a nullptr if 'if' clause was specified, nullptr 1167 /// otherwise. 1168 /// \param Data Additional data for task generation like tiednsee, final 1169 /// state, list of privates etc. 1170 virtual void emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc, 1171 const OMPLoopDirective &D, 1172 llvm::Function *TaskFunction, 1173 QualType SharedsTy, Address Shareds, 1174 const Expr *IfCond, const OMPTaskDataTy &Data); 1175 1176 /// Emit code for the directive that does not require outlining. 1177 /// 1178 /// \param InnermostKind Kind of innermost directive (for simple directives it 1179 /// is a directive itself, for combined - its innermost directive). 1180 /// \param CodeGen Code generation sequence for the \a D directive. 1181 /// \param HasCancel true if region has inner cancel directive, false 1182 /// otherwise. 1183 virtual void emitInlinedDirective(CodeGenFunction &CGF, 1184 OpenMPDirectiveKind InnermostKind, 1185 const RegionCodeGenTy &CodeGen, 1186 bool HasCancel = false); 1187 1188 /// Emits reduction function. 1189 /// \param ReducerName Name of the function calling the reduction. 1190 /// \param ArgsElemType Array type containing pointers to reduction variables. 1191 /// \param Privates List of private copies for original reduction arguments. 1192 /// \param LHSExprs List of LHS in \a ReductionOps reduction operations. 1193 /// \param RHSExprs List of RHS in \a ReductionOps reduction operations. 1194 /// \param ReductionOps List of reduction operations in form 'LHS binop RHS' 1195 /// or 'operator binop(LHS, RHS)'. 1196 llvm::Function *emitReductionFunction( 1197 StringRef ReducerName, SourceLocation Loc, llvm::Type *ArgsElemType, 1198 ArrayRef<const Expr *> Privates, ArrayRef<const Expr *> LHSExprs, 1199 ArrayRef<const Expr *> RHSExprs, ArrayRef<const Expr *> ReductionOps); 1200 1201 /// Emits single reduction combiner 1202 void emitSingleReductionCombiner(CodeGenFunction &CGF, 1203 const Expr *ReductionOp, 1204 const Expr *PrivateRef, 1205 const DeclRefExpr *LHS, 1206 const DeclRefExpr *RHS); 1207 1208 struct ReductionOptionsTy { 1209 bool WithNowait; 1210 bool SimpleReduction; 1211 OpenMPDirectiveKind ReductionKind; 1212 }; 1213 /// Emit a code for reduction clause. Next code should be emitted for 1214 /// reduction: 1215 /// \code 1216 /// 1217 /// static kmp_critical_name lock = { 0 }; 1218 /// 1219 /// void reduce_func(void *lhs[<n>], void *rhs[<n>]) { 1220 /// ... 1221 /// *(Type<i>*)lhs[i] = RedOp<i>(*(Type<i>*)lhs[i], *(Type<i>*)rhs[i]); 1222 /// ... 1223 /// } 1224 /// 1225 /// ... 1226 /// void *RedList[<n>] = {&<RHSExprs>[0], ..., &<RHSExprs>[<n>-1]}; 1227 /// switch (__kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList), 1228 /// RedList, reduce_func, &<lock>)) { 1229 /// case 1: 1230 /// ... 1231 /// <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]); 1232 /// ... 1233 /// __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>); 1234 /// break; 1235 /// case 2: 1236 /// ... 1237 /// Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i])); 1238 /// ... 1239 /// break; 1240 /// default:; 1241 /// } 1242 /// \endcode 1243 /// 1244 /// \param Privates List of private copies for original reduction arguments. 1245 /// \param LHSExprs List of LHS in \a ReductionOps reduction operations. 1246 /// \param RHSExprs List of RHS in \a ReductionOps reduction operations. 1247 /// \param ReductionOps List of reduction operations in form 'LHS binop RHS' 1248 /// or 'operator binop(LHS, RHS)'. 1249 /// \param Options List of options for reduction codegen: 1250 /// WithNowait true if parent directive has also nowait clause, false 1251 /// otherwise. 1252 /// SimpleReduction Emit reduction operation only. Used for omp simd 1253 /// directive on the host. 1254 /// ReductionKind The kind of reduction to perform. 1255 virtual void emitReduction(CodeGenFunction &CGF, SourceLocation Loc, 1256 ArrayRef<const Expr *> Privates, 1257 ArrayRef<const Expr *> LHSExprs, 1258 ArrayRef<const Expr *> RHSExprs, 1259 ArrayRef<const Expr *> ReductionOps, 1260 ReductionOptionsTy Options); 1261 1262 /// Emit a code for initialization of task reduction clause. Next code 1263 /// should be emitted for reduction: 1264 /// \code 1265 /// 1266 /// _taskred_item_t red_data[n]; 1267 /// ... 1268 /// red_data[i].shar = &shareds[i]; 1269 /// red_data[i].orig = &origs[i]; 1270 /// red_data[i].size = sizeof(origs[i]); 1271 /// red_data[i].f_init = (void*)RedInit<i>; 1272 /// red_data[i].f_fini = (void*)RedDest<i>; 1273 /// red_data[i].f_comb = (void*)RedOp<i>; 1274 /// red_data[i].flags = <Flag_i>; 1275 /// ... 1276 /// void* tg1 = __kmpc_taskred_init(gtid, n, red_data); 1277 /// \endcode 1278 /// For reduction clause with task modifier it emits the next call: 1279 /// \code 1280 /// 1281 /// _taskred_item_t red_data[n]; 1282 /// ... 1283 /// red_data[i].shar = &shareds[i]; 1284 /// red_data[i].orig = &origs[i]; 1285 /// red_data[i].size = sizeof(origs[i]); 1286 /// red_data[i].f_init = (void*)RedInit<i>; 1287 /// red_data[i].f_fini = (void*)RedDest<i>; 1288 /// red_data[i].f_comb = (void*)RedOp<i>; 1289 /// red_data[i].flags = <Flag_i>; 1290 /// ... 1291 /// void* tg1 = __kmpc_taskred_modifier_init(loc, gtid, is_worksharing, n, 1292 /// red_data); 1293 /// \endcode 1294 /// \param LHSExprs List of LHS in \a Data.ReductionOps reduction operations. 1295 /// \param RHSExprs List of RHS in \a Data.ReductionOps reduction operations. 1296 /// \param Data Additional data for task generation like tiedness, final 1297 /// state, list of privates, reductions etc. 1298 virtual llvm::Value *emitTaskReductionInit(CodeGenFunction &CGF, 1299 SourceLocation Loc, 1300 ArrayRef<const Expr *> LHSExprs, 1301 ArrayRef<const Expr *> RHSExprs, 1302 const OMPTaskDataTy &Data); 1303 1304 /// Emits the following code for reduction clause with task modifier: 1305 /// \code 1306 /// __kmpc_task_reduction_modifier_fini(loc, gtid, is_worksharing); 1307 /// \endcode 1308 virtual void emitTaskReductionFini(CodeGenFunction &CGF, SourceLocation Loc, 1309 bool IsWorksharingReduction); 1310 1311 /// Required to resolve existing problems in the runtime. Emits threadprivate 1312 /// variables to store the size of the VLAs/array sections for 1313 /// initializer/combiner/finalizer functions. 1314 /// \param RCG Allows to reuse an existing data for the reductions. 1315 /// \param N Reduction item for which fixups must be emitted. 1316 virtual void emitTaskReductionFixups(CodeGenFunction &CGF, SourceLocation Loc, 1317 ReductionCodeGen &RCG, unsigned N); 1318 1319 /// Get the address of `void *` type of the privatue copy of the reduction 1320 /// item specified by the \p SharedLVal. 1321 /// \param ReductionsPtr Pointer to the reduction data returned by the 1322 /// emitTaskReductionInit function. 1323 /// \param SharedLVal Address of the original reduction item. 1324 virtual Address getTaskReductionItem(CodeGenFunction &CGF, SourceLocation Loc, 1325 llvm::Value *ReductionsPtr, 1326 LValue SharedLVal); 1327 1328 /// Emit code for 'taskwait' directive. 1329 virtual void emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc, 1330 const OMPTaskDataTy &Data); 1331 1332 /// Emit code for 'cancellation point' construct. 1333 /// \param CancelRegion Region kind for which the cancellation point must be 1334 /// emitted. 1335 /// 1336 virtual void emitCancellationPointCall(CodeGenFunction &CGF, 1337 SourceLocation Loc, 1338 OpenMPDirectiveKind CancelRegion); 1339 1340 /// Emit code for 'cancel' construct. 1341 /// \param IfCond Condition in the associated 'if' clause, if it was 1342 /// specified, nullptr otherwise. 1343 /// \param CancelRegion Region kind for which the cancel must be emitted. 1344 /// 1345 virtual void emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc, 1346 const Expr *IfCond, 1347 OpenMPDirectiveKind CancelRegion); 1348 1349 /// Emit outilined function for 'target' directive. 1350 /// \param D Directive to emit. 1351 /// \param ParentName Name of the function that encloses the target region. 1352 /// \param OutlinedFn Outlined function value to be defined by this call. 1353 /// \param OutlinedFnID Outlined function ID value to be defined by this call. 1354 /// \param IsOffloadEntry True if the outlined function is an offload entry. 1355 /// \param CodeGen Code generation sequence for the \a D directive. 1356 /// An outlined function may not be an entry if, e.g. the if clause always 1357 /// evaluates to false. 1358 virtual void emitTargetOutlinedFunction(const OMPExecutableDirective &D, 1359 StringRef ParentName, 1360 llvm::Function *&OutlinedFn, 1361 llvm::Constant *&OutlinedFnID, 1362 bool IsOffloadEntry, 1363 const RegionCodeGenTy &CodeGen); 1364 1365 /// Emit the target offloading code associated with \a D. The emitted 1366 /// code attempts offloading the execution to the device, an the event of 1367 /// a failure it executes the host version outlined in \a OutlinedFn. 1368 /// \param D Directive to emit. 1369 /// \param OutlinedFn Host version of the code to be offloaded. 1370 /// \param OutlinedFnID ID of host version of the code to be offloaded. 1371 /// \param IfCond Expression evaluated in if clause associated with the target 1372 /// directive, or null if no if clause is used. 1373 /// \param Device Expression evaluated in device clause associated with the 1374 /// target directive, or null if no device clause is used and device modifier. 1375 /// \param SizeEmitter Callback to emit number of iterations for loop-based 1376 /// directives. 1377 virtual void emitTargetCall( 1378 CodeGenFunction &CGF, const OMPExecutableDirective &D, 1379 llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond, 1380 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device, 1381 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF, 1382 const OMPLoopDirective &D)> 1383 SizeEmitter); 1384 1385 /// Emit the target regions enclosed in \a GD function definition or 1386 /// the function itself in case it is a valid device function. Returns true if 1387 /// \a GD was dealt with successfully. 1388 /// \param GD Function to scan. 1389 virtual bool emitTargetFunctions(GlobalDecl GD); 1390 1391 /// Emit the global variable if it is a valid device global variable. 1392 /// Returns true if \a GD was dealt with successfully. 1393 /// \param GD Variable declaration to emit. 1394 virtual bool emitTargetGlobalVariable(GlobalDecl GD); 1395 1396 /// Checks if the provided global decl \a GD is a declare target variable and 1397 /// registers it when emitting code for the host. 1398 virtual void registerTargetGlobalVariable(const VarDecl *VD, 1399 llvm::Constant *Addr); 1400 1401 /// Emit the global \a GD if it is meaningful for the target. Returns 1402 /// if it was emitted successfully. 1403 /// \param GD Global to scan. 1404 virtual bool emitTargetGlobal(GlobalDecl GD); 1405 1406 /// Creates all the offload entries in the current compilation unit 1407 /// along with the associated metadata. 1408 void createOffloadEntriesAndInfoMetadata(); 1409 1410 /// Emits code for teams call of the \a OutlinedFn with 1411 /// variables captured in a record which address is stored in \a 1412 /// CapturedStruct. 1413 /// \param OutlinedFn Outlined function to be run by team masters. Type of 1414 /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*). 1415 /// \param CapturedVars A pointer to the record with the references to 1416 /// variables used in \a OutlinedFn function. 1417 /// 1418 virtual void emitTeamsCall(CodeGenFunction &CGF, 1419 const OMPExecutableDirective &D, 1420 SourceLocation Loc, llvm::Function *OutlinedFn, 1421 ArrayRef<llvm::Value *> CapturedVars); 1422 1423 /// Emits call to void __kmpc_push_num_teams(ident_t *loc, kmp_int32 1424 /// global_tid, kmp_int32 num_teams, kmp_int32 thread_limit) to generate code 1425 /// for num_teams clause. 1426 /// \param NumTeams An integer expression of teams. 1427 /// \param ThreadLimit An integer expression of threads. 1428 virtual void emitNumTeamsClause(CodeGenFunction &CGF, const Expr *NumTeams, 1429 const Expr *ThreadLimit, SourceLocation Loc); 1430 1431 /// Emits call to void __kmpc_set_thread_limit(ident_t *loc, kmp_int32 1432 /// global_tid, kmp_int32 thread_limit) to generate code for 1433 /// thread_limit clause on target directive 1434 /// \param ThreadLimit An integer expression of threads. 1435 virtual void emitThreadLimitClause(CodeGenFunction &CGF, 1436 const Expr *ThreadLimit, 1437 SourceLocation Loc); 1438 1439 /// Struct that keeps all the relevant information that should be kept 1440 /// throughout a 'target data' region. 1441 class TargetDataInfo : public llvm::OpenMPIRBuilder::TargetDataInfo { 1442 public: 1443 explicit TargetDataInfo() : llvm::OpenMPIRBuilder::TargetDataInfo() {} 1444 explicit TargetDataInfo(bool RequiresDevicePointerInfo, 1445 bool SeparateBeginEndCalls) 1446 : llvm::OpenMPIRBuilder::TargetDataInfo(RequiresDevicePointerInfo, 1447 SeparateBeginEndCalls) {} 1448 /// Map between the a declaration of a capture and the corresponding new 1449 /// llvm address where the runtime returns the device pointers. 1450 llvm::DenseMap<const ValueDecl *, llvm::Value *> CaptureDeviceAddrMap; 1451 }; 1452 1453 /// Emit the target data mapping code associated with \a D. 1454 /// \param D Directive to emit. 1455 /// \param IfCond Expression evaluated in if clause associated with the 1456 /// target directive, or null if no device clause is used. 1457 /// \param Device Expression evaluated in device clause associated with the 1458 /// target directive, or null if no device clause is used. 1459 /// \param Info A record used to store information that needs to be preserved 1460 /// until the region is closed. 1461 virtual void emitTargetDataCalls(CodeGenFunction &CGF, 1462 const OMPExecutableDirective &D, 1463 const Expr *IfCond, const Expr *Device, 1464 const RegionCodeGenTy &CodeGen, 1465 CGOpenMPRuntime::TargetDataInfo &Info); 1466 1467 /// Emit the data mapping/movement code associated with the directive 1468 /// \a D that should be of the form 'target [{enter|exit} data | update]'. 1469 /// \param D Directive to emit. 1470 /// \param IfCond Expression evaluated in if clause associated with the target 1471 /// directive, or null if no if clause is used. 1472 /// \param Device Expression evaluated in device clause associated with the 1473 /// target directive, or null if no device clause is used. 1474 virtual void emitTargetDataStandAloneCall(CodeGenFunction &CGF, 1475 const OMPExecutableDirective &D, 1476 const Expr *IfCond, 1477 const Expr *Device); 1478 1479 /// Marks function \a Fn with properly mangled versions of vector functions. 1480 /// \param FD Function marked as 'declare simd'. 1481 /// \param Fn LLVM function that must be marked with 'declare simd' 1482 /// attributes. 1483 virtual void emitDeclareSimdFunction(const FunctionDecl *FD, 1484 llvm::Function *Fn); 1485 1486 /// Emit initialization for doacross loop nesting support. 1487 /// \param D Loop-based construct used in doacross nesting construct. 1488 virtual void emitDoacrossInit(CodeGenFunction &CGF, const OMPLoopDirective &D, 1489 ArrayRef<Expr *> NumIterations); 1490 1491 /// Emit code for doacross ordered directive with 'depend' clause. 1492 /// \param C 'depend' clause with 'sink|source' dependency kind. 1493 virtual void emitDoacrossOrdered(CodeGenFunction &CGF, 1494 const OMPDependClause *C); 1495 1496 /// Emit code for doacross ordered directive with 'doacross' clause. 1497 /// \param C 'doacross' clause with 'sink|source' dependence type. 1498 virtual void emitDoacrossOrdered(CodeGenFunction &CGF, 1499 const OMPDoacrossClause *C); 1500 1501 /// Translates the native parameter of outlined function if this is required 1502 /// for target. 1503 /// \param FD Field decl from captured record for the parameter. 1504 /// \param NativeParam Parameter itself. 1505 virtual const VarDecl *translateParameter(const FieldDecl *FD, 1506 const VarDecl *NativeParam) const { 1507 return NativeParam; 1508 } 1509 1510 /// Gets the address of the native argument basing on the address of the 1511 /// target-specific parameter. 1512 /// \param NativeParam Parameter itself. 1513 /// \param TargetParam Corresponding target-specific parameter. 1514 virtual Address getParameterAddress(CodeGenFunction &CGF, 1515 const VarDecl *NativeParam, 1516 const VarDecl *TargetParam) const; 1517 1518 /// Choose default schedule type and chunk value for the 1519 /// dist_schedule clause. 1520 virtual void getDefaultDistScheduleAndChunk(CodeGenFunction &CGF, 1521 const OMPLoopDirective &S, OpenMPDistScheduleClauseKind &ScheduleKind, 1522 llvm::Value *&Chunk) const {} 1523 1524 /// Choose default schedule type and chunk value for the 1525 /// schedule clause. 1526 virtual void getDefaultScheduleAndChunk(CodeGenFunction &CGF, 1527 const OMPLoopDirective &S, OpenMPScheduleClauseKind &ScheduleKind, 1528 const Expr *&ChunkExpr) const; 1529 1530 /// Emits call of the outlined function with the provided arguments, 1531 /// translating these arguments to correct target-specific arguments. 1532 virtual void 1533 emitOutlinedFunctionCall(CodeGenFunction &CGF, SourceLocation Loc, 1534 llvm::FunctionCallee OutlinedFn, 1535 ArrayRef<llvm::Value *> Args = {}) const; 1536 1537 /// Emits OpenMP-specific function prolog. 1538 /// Required for device constructs. 1539 virtual void emitFunctionProlog(CodeGenFunction &CGF, const Decl *D); 1540 1541 /// Gets the OpenMP-specific address of the local variable. 1542 virtual Address getAddressOfLocalVariable(CodeGenFunction &CGF, 1543 const VarDecl *VD); 1544 1545 /// Marks the declaration as already emitted for the device code and returns 1546 /// true, if it was marked already, and false, otherwise. 1547 bool markAsGlobalTarget(GlobalDecl GD); 1548 1549 /// Emit deferred declare target variables marked for deferred emission. 1550 void emitDeferredTargetDecls() const; 1551 1552 /// Adjust some parameters for the target-based directives, like addresses of 1553 /// the variables captured by reference in lambdas. 1554 virtual void 1555 adjustTargetSpecificDataForLambdas(CodeGenFunction &CGF, 1556 const OMPExecutableDirective &D) const; 1557 1558 /// Perform check on requires decl to ensure that target architecture 1559 /// supports unified addressing 1560 virtual void processRequiresDirective(const OMPRequiresDecl *D); 1561 1562 /// Gets default memory ordering as specified in requires directive. 1563 llvm::AtomicOrdering getDefaultMemoryOrdering() const; 1564 1565 /// Checks if the variable has associated OMPAllocateDeclAttr attribute with 1566 /// the predefined allocator and translates it into the corresponding address 1567 /// space. 1568 virtual bool hasAllocateAttributeForGlobalVar(const VarDecl *VD, LangAS &AS); 1569 1570 /// Return whether the unified_shared_memory has been specified. 1571 bool hasRequiresUnifiedSharedMemory() const; 1572 1573 /// Checks if the \p VD variable is marked as nontemporal declaration in 1574 /// current context. 1575 bool isNontemporalDecl(const ValueDecl *VD) const; 1576 1577 /// Create specialized alloca to handle lastprivate conditionals. 1578 Address emitLastprivateConditionalInit(CodeGenFunction &CGF, 1579 const VarDecl *VD); 1580 1581 /// Checks if the provided \p LVal is lastprivate conditional and emits the 1582 /// code to update the value of the original variable. 1583 /// \code 1584 /// lastprivate(conditional: a) 1585 /// ... 1586 /// <type> a; 1587 /// lp_a = ...; 1588 /// #pragma omp critical(a) 1589 /// if (last_iv_a <= iv) { 1590 /// last_iv_a = iv; 1591 /// global_a = lp_a; 1592 /// } 1593 /// \endcode 1594 virtual void checkAndEmitLastprivateConditional(CodeGenFunction &CGF, 1595 const Expr *LHS); 1596 1597 /// Checks if the lastprivate conditional was updated in inner region and 1598 /// writes the value. 1599 /// \code 1600 /// lastprivate(conditional: a) 1601 /// ... 1602 /// <type> a;bool Fired = false; 1603 /// #pragma omp ... shared(a) 1604 /// { 1605 /// lp_a = ...; 1606 /// Fired = true; 1607 /// } 1608 /// if (Fired) { 1609 /// #pragma omp critical(a) 1610 /// if (last_iv_a <= iv) { 1611 /// last_iv_a = iv; 1612 /// global_a = lp_a; 1613 /// } 1614 /// Fired = false; 1615 /// } 1616 /// \endcode 1617 virtual void checkAndEmitSharedLastprivateConditional( 1618 CodeGenFunction &CGF, const OMPExecutableDirective &D, 1619 const llvm::DenseSet<CanonicalDeclPtr<const VarDecl>> &IgnoredDecls); 1620 1621 /// Gets the address of the global copy used for lastprivate conditional 1622 /// update, if any. 1623 /// \param PrivLVal LValue for the private copy. 1624 /// \param VD Original lastprivate declaration. 1625 virtual void emitLastprivateConditionalFinalUpdate(CodeGenFunction &CGF, 1626 LValue PrivLVal, 1627 const VarDecl *VD, 1628 SourceLocation Loc); 1629 1630 /// Emits list of dependecies based on the provided data (array of 1631 /// dependence/expression pairs). 1632 /// \returns Pointer to the first element of the array casted to VoidPtr type. 1633 std::pair<llvm::Value *, Address> 1634 emitDependClause(CodeGenFunction &CGF, 1635 ArrayRef<OMPTaskDataTy::DependData> Dependencies, 1636 SourceLocation Loc); 1637 1638 /// Emits list of dependecies based on the provided data (array of 1639 /// dependence/expression pairs) for depobj construct. In this case, the 1640 /// variable is allocated in dynamically. \returns Pointer to the first 1641 /// element of the array casted to VoidPtr type. 1642 Address emitDepobjDependClause(CodeGenFunction &CGF, 1643 const OMPTaskDataTy::DependData &Dependencies, 1644 SourceLocation Loc); 1645 1646 /// Emits the code to destroy the dependency object provided in depobj 1647 /// directive. 1648 void emitDestroyClause(CodeGenFunction &CGF, LValue DepobjLVal, 1649 SourceLocation Loc); 1650 1651 /// Updates the dependency kind in the specified depobj object. 1652 /// \param DepobjLVal LValue for the main depobj object. 1653 /// \param NewDepKind New dependency kind. 1654 void emitUpdateClause(CodeGenFunction &CGF, LValue DepobjLVal, 1655 OpenMPDependClauseKind NewDepKind, SourceLocation Loc); 1656 1657 /// Initializes user defined allocators specified in the uses_allocators 1658 /// clauses. 1659 void emitUsesAllocatorsInit(CodeGenFunction &CGF, const Expr *Allocator, 1660 const Expr *AllocatorTraits); 1661 1662 /// Destroys user defined allocators specified in the uses_allocators clause. 1663 void emitUsesAllocatorsFini(CodeGenFunction &CGF, const Expr *Allocator); 1664 1665 /// Returns true if the variable is a local variable in untied task. 1666 bool isLocalVarInUntiedTask(CodeGenFunction &CGF, const VarDecl *VD) const; 1667 }; 1668 1669 /// Class supports emissionof SIMD-only code. 1670 class CGOpenMPSIMDRuntime final : public CGOpenMPRuntime { 1671 public: 1672 explicit CGOpenMPSIMDRuntime(CodeGenModule &CGM) : CGOpenMPRuntime(CGM) {} 1673 ~CGOpenMPSIMDRuntime() override {} 1674 1675 /// Emits outlined function for the specified OpenMP parallel directive 1676 /// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID, 1677 /// kmp_int32 BoundID, struct context_vars*). 1678 /// \param CGF Reference to current CodeGenFunction. 1679 /// \param D OpenMP directive. 1680 /// \param ThreadIDVar Variable for thread id in the current OpenMP region. 1681 /// \param InnermostKind Kind of innermost directive (for simple directives it 1682 /// is a directive itself, for combined - its innermost directive). 1683 /// \param CodeGen Code generation sequence for the \a D directive. 1684 llvm::Function *emitParallelOutlinedFunction( 1685 CodeGenFunction &CGF, const OMPExecutableDirective &D, 1686 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, 1687 const RegionCodeGenTy &CodeGen) override; 1688 1689 /// Emits outlined function for the specified OpenMP teams directive 1690 /// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID, 1691 /// kmp_int32 BoundID, struct context_vars*). 1692 /// \param CGF Reference to current CodeGenFunction. 1693 /// \param D OpenMP directive. 1694 /// \param ThreadIDVar Variable for thread id in the current OpenMP region. 1695 /// \param InnermostKind Kind of innermost directive (for simple directives it 1696 /// is a directive itself, for combined - its innermost directive). 1697 /// \param CodeGen Code generation sequence for the \a D directive. 1698 llvm::Function *emitTeamsOutlinedFunction( 1699 CodeGenFunction &CGF, const OMPExecutableDirective &D, 1700 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, 1701 const RegionCodeGenTy &CodeGen) override; 1702 1703 /// Emits outlined function for the OpenMP task directive \a D. This 1704 /// outlined function has type void(*)(kmp_int32 ThreadID, struct task_t* 1705 /// TaskT). 1706 /// \param D OpenMP directive. 1707 /// \param ThreadIDVar Variable for thread id in the current OpenMP region. 1708 /// \param PartIDVar Variable for partition id in the current OpenMP untied 1709 /// task region. 1710 /// \param TaskTVar Variable for task_t argument. 1711 /// \param InnermostKind Kind of innermost directive (for simple directives it 1712 /// is a directive itself, for combined - its innermost directive). 1713 /// \param CodeGen Code generation sequence for the \a D directive. 1714 /// \param Tied true if task is generated for tied task, false otherwise. 1715 /// \param NumberOfParts Number of parts in untied task. Ignored for tied 1716 /// tasks. 1717 /// 1718 llvm::Function *emitTaskOutlinedFunction( 1719 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, 1720 const VarDecl *PartIDVar, const VarDecl *TaskTVar, 1721 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, 1722 bool Tied, unsigned &NumberOfParts) override; 1723 1724 /// Emits code for parallel or serial call of the \a OutlinedFn with 1725 /// variables captured in a record which address is stored in \a 1726 /// CapturedStruct. 1727 /// \param OutlinedFn Outlined function to be run in parallel threads. Type of 1728 /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*). 1729 /// \param CapturedVars A pointer to the record with the references to 1730 /// variables used in \a OutlinedFn function. 1731 /// \param IfCond Condition in the associated 'if' clause, if it was 1732 /// specified, nullptr otherwise. 1733 /// \param NumThreads The value corresponding to the num_threads clause, if 1734 /// any, or nullptr. 1735 /// 1736 void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, 1737 llvm::Function *OutlinedFn, 1738 ArrayRef<llvm::Value *> CapturedVars, 1739 const Expr *IfCond, llvm::Value *NumThreads) override; 1740 1741 /// Emits a critical region. 1742 /// \param CriticalName Name of the critical region. 1743 /// \param CriticalOpGen Generator for the statement associated with the given 1744 /// critical region. 1745 /// \param Hint Value of the 'hint' clause (optional). 1746 void emitCriticalRegion(CodeGenFunction &CGF, StringRef CriticalName, 1747 const RegionCodeGenTy &CriticalOpGen, 1748 SourceLocation Loc, 1749 const Expr *Hint = nullptr) override; 1750 1751 /// Emits a master region. 1752 /// \param MasterOpGen Generator for the statement associated with the given 1753 /// master region. 1754 void emitMasterRegion(CodeGenFunction &CGF, 1755 const RegionCodeGenTy &MasterOpGen, 1756 SourceLocation Loc) override; 1757 1758 /// Emits a masked region. 1759 /// \param MaskedOpGen Generator for the statement associated with the given 1760 /// masked region. 1761 void emitMaskedRegion(CodeGenFunction &CGF, 1762 const RegionCodeGenTy &MaskedOpGen, SourceLocation Loc, 1763 const Expr *Filter = nullptr) override; 1764 1765 /// Emits a masked region. 1766 /// \param MaskedOpGen Generator for the statement associated with the given 1767 /// masked region. 1768 1769 /// Emits code for a taskyield directive. 1770 void emitTaskyieldCall(CodeGenFunction &CGF, SourceLocation Loc) override; 1771 1772 /// Emit a taskgroup region. 1773 /// \param TaskgroupOpGen Generator for the statement associated with the 1774 /// given taskgroup region. 1775 void emitTaskgroupRegion(CodeGenFunction &CGF, 1776 const RegionCodeGenTy &TaskgroupOpGen, 1777 SourceLocation Loc) override; 1778 1779 /// Emits a single region. 1780 /// \param SingleOpGen Generator for the statement associated with the given 1781 /// single region. 1782 void emitSingleRegion(CodeGenFunction &CGF, 1783 const RegionCodeGenTy &SingleOpGen, SourceLocation Loc, 1784 ArrayRef<const Expr *> CopyprivateVars, 1785 ArrayRef<const Expr *> DestExprs, 1786 ArrayRef<const Expr *> SrcExprs, 1787 ArrayRef<const Expr *> AssignmentOps) override; 1788 1789 /// Emit an ordered region. 1790 /// \param OrderedOpGen Generator for the statement associated with the given 1791 /// ordered region. 1792 void emitOrderedRegion(CodeGenFunction &CGF, 1793 const RegionCodeGenTy &OrderedOpGen, 1794 SourceLocation Loc, bool IsThreads) override; 1795 1796 /// Emit an implicit/explicit barrier for OpenMP threads. 1797 /// \param Kind Directive for which this implicit barrier call must be 1798 /// generated. Must be OMPD_barrier for explicit barrier generation. 1799 /// \param EmitChecks true if need to emit checks for cancellation barriers. 1800 /// \param ForceSimpleCall true simple barrier call must be emitted, false if 1801 /// runtime class decides which one to emit (simple or with cancellation 1802 /// checks). 1803 /// 1804 void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc, 1805 OpenMPDirectiveKind Kind, bool EmitChecks = true, 1806 bool ForceSimpleCall = false) override; 1807 1808 /// This is used for non static scheduled types and when the ordered 1809 /// clause is present on the loop construct. 1810 /// Depending on the loop schedule, it is necessary to call some runtime 1811 /// routine before start of the OpenMP loop to get the loop upper / lower 1812 /// bounds \a LB and \a UB and stride \a ST. 1813 /// 1814 /// \param CGF Reference to current CodeGenFunction. 1815 /// \param Loc Clang source location. 1816 /// \param ScheduleKind Schedule kind, specified by the 'schedule' clause. 1817 /// \param IVSize Size of the iteration variable in bits. 1818 /// \param IVSigned Sign of the iteration variable. 1819 /// \param Ordered true if loop is ordered, false otherwise. 1820 /// \param DispatchValues struct containing llvm values for lower bound, upper 1821 /// bound, and chunk expression. 1822 /// For the default (nullptr) value, the chunk 1 will be used. 1823 /// 1824 void emitForDispatchInit(CodeGenFunction &CGF, SourceLocation Loc, 1825 const OpenMPScheduleTy &ScheduleKind, 1826 unsigned IVSize, bool IVSigned, bool Ordered, 1827 const DispatchRTInput &DispatchValues) override; 1828 1829 /// This is used for non static scheduled types and when the ordered 1830 /// clause is present on the loop construct. 1831 /// 1832 /// \param CGF Reference to current CodeGenFunction. 1833 /// \param Loc Clang source location. 1834 /// 1835 void emitForDispatchDeinit(CodeGenFunction &CGF, SourceLocation Loc) override; 1836 1837 /// Call the appropriate runtime routine to initialize it before start 1838 /// of loop. 1839 /// 1840 /// This is used only in case of static schedule, when the user did not 1841 /// specify a ordered clause on the loop construct. 1842 /// Depending on the loop schedule, it is necessary to call some runtime 1843 /// routine before start of the OpenMP loop to get the loop upper / lower 1844 /// bounds LB and UB and stride ST. 1845 /// 1846 /// \param CGF Reference to current CodeGenFunction. 1847 /// \param Loc Clang source location. 1848 /// \param DKind Kind of the directive. 1849 /// \param ScheduleKind Schedule kind, specified by the 'schedule' clause. 1850 /// \param Values Input arguments for the construct. 1851 /// 1852 void emitForStaticInit(CodeGenFunction &CGF, SourceLocation Loc, 1853 OpenMPDirectiveKind DKind, 1854 const OpenMPScheduleTy &ScheduleKind, 1855 const StaticRTInput &Values) override; 1856 1857 /// 1858 /// \param CGF Reference to current CodeGenFunction. 1859 /// \param Loc Clang source location. 1860 /// \param SchedKind Schedule kind, specified by the 'dist_schedule' clause. 1861 /// \param Values Input arguments for the construct. 1862 /// 1863 void emitDistributeStaticInit(CodeGenFunction &CGF, SourceLocation Loc, 1864 OpenMPDistScheduleClauseKind SchedKind, 1865 const StaticRTInput &Values) override; 1866 1867 /// Call the appropriate runtime routine to notify that we finished 1868 /// iteration of the ordered loop with the dynamic scheduling. 1869 /// 1870 /// \param CGF Reference to current CodeGenFunction. 1871 /// \param Loc Clang source location. 1872 /// \param IVSize Size of the iteration variable in bits. 1873 /// \param IVSigned Sign of the iteration variable. 1874 /// 1875 void emitForOrderedIterationEnd(CodeGenFunction &CGF, SourceLocation Loc, 1876 unsigned IVSize, bool IVSigned) override; 1877 1878 /// Call the appropriate runtime routine to notify that we finished 1879 /// all the work with current loop. 1880 /// 1881 /// \param CGF Reference to current CodeGenFunction. 1882 /// \param Loc Clang source location. 1883 /// \param DKind Kind of the directive for which the static finish is emitted. 1884 /// 1885 void emitForStaticFinish(CodeGenFunction &CGF, SourceLocation Loc, 1886 OpenMPDirectiveKind DKind) override; 1887 1888 /// Call __kmpc_dispatch_next( 1889 /// ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter, 1890 /// kmp_int[32|64] *p_lower, kmp_int[32|64] *p_upper, 1891 /// kmp_int[32|64] *p_stride); 1892 /// \param IVSize Size of the iteration variable in bits. 1893 /// \param IVSigned Sign of the iteration variable. 1894 /// \param IL Address of the output variable in which the flag of the 1895 /// last iteration is returned. 1896 /// \param LB Address of the output variable in which the lower iteration 1897 /// number is returned. 1898 /// \param UB Address of the output variable in which the upper iteration 1899 /// number is returned. 1900 /// \param ST Address of the output variable in which the stride value is 1901 /// returned. 1902 llvm::Value *emitForNext(CodeGenFunction &CGF, SourceLocation Loc, 1903 unsigned IVSize, bool IVSigned, Address IL, 1904 Address LB, Address UB, Address ST) override; 1905 1906 /// Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32 1907 /// global_tid, kmp_int32 num_threads) to generate code for 'num_threads' 1908 /// clause. 1909 /// \param NumThreads An integer value of threads. 1910 void emitNumThreadsClause(CodeGenFunction &CGF, llvm::Value *NumThreads, 1911 SourceLocation Loc) override; 1912 1913 /// Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 1914 /// global_tid, int proc_bind) to generate code for 'proc_bind' clause. 1915 void emitProcBindClause(CodeGenFunction &CGF, 1916 llvm::omp::ProcBindKind ProcBind, 1917 SourceLocation Loc) override; 1918 1919 /// Returns address of the threadprivate variable for the current 1920 /// thread. 1921 /// \param VD Threadprivate variable. 1922 /// \param VDAddr Address of the global variable \a VD. 1923 /// \param Loc Location of the reference to threadprivate var. 1924 /// \return Address of the threadprivate variable for the current thread. 1925 Address getAddrOfThreadPrivate(CodeGenFunction &CGF, const VarDecl *VD, 1926 Address VDAddr, SourceLocation Loc) override; 1927 1928 /// Emit a code for initialization of threadprivate variable. It emits 1929 /// a call to runtime library which adds initial value to the newly created 1930 /// threadprivate variable (if it is not constant) and registers destructor 1931 /// for the variable (if any). 1932 /// \param VD Threadprivate variable. 1933 /// \param VDAddr Address of the global variable \a VD. 1934 /// \param Loc Location of threadprivate declaration. 1935 /// \param PerformInit true if initialization expression is not constant. 1936 llvm::Function * 1937 emitThreadPrivateVarDefinition(const VarDecl *VD, Address VDAddr, 1938 SourceLocation Loc, bool PerformInit, 1939 CodeGenFunction *CGF = nullptr) override; 1940 1941 /// Creates artificial threadprivate variable with name \p Name and type \p 1942 /// VarType. 1943 /// \param VarType Type of the artificial threadprivate variable. 1944 /// \param Name Name of the artificial threadprivate variable. 1945 Address getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF, 1946 QualType VarType, 1947 StringRef Name) override; 1948 1949 /// Emit flush of the variables specified in 'omp flush' directive. 1950 /// \param Vars List of variables to flush. 1951 void emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *> Vars, 1952 SourceLocation Loc, llvm::AtomicOrdering AO) override; 1953 1954 /// Emit task region for the task directive. The task region is 1955 /// emitted in several steps: 1956 /// 1. Emit a call to kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32 1957 /// gtid, kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds, 1958 /// kmp_routine_entry_t *task_entry). Here task_entry is a pointer to the 1959 /// function: 1960 /// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) { 1961 /// TaskFunction(gtid, tt->part_id, tt->shareds); 1962 /// return 0; 1963 /// } 1964 /// 2. Copy a list of shared variables to field shareds of the resulting 1965 /// structure kmp_task_t returned by the previous call (if any). 1966 /// 3. Copy a pointer to destructions function to field destructions of the 1967 /// resulting structure kmp_task_t. 1968 /// 4. Emit a call to kmp_int32 __kmpc_omp_task(ident_t *, kmp_int32 gtid, 1969 /// kmp_task_t *new_task), where new_task is a resulting structure from 1970 /// previous items. 1971 /// \param D Current task directive. 1972 /// \param TaskFunction An LLVM function with type void (*)(i32 /*gtid*/, i32 1973 /// /*part_id*/, captured_struct */*__context*/); 1974 /// \param SharedsTy A type which contains references the shared variables. 1975 /// \param Shareds Context with the list of shared variables from the \p 1976 /// TaskFunction. 1977 /// \param IfCond Not a nullptr if 'if' clause was specified, nullptr 1978 /// otherwise. 1979 /// \param Data Additional data for task generation like tiednsee, final 1980 /// state, list of privates etc. 1981 void emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc, 1982 const OMPExecutableDirective &D, 1983 llvm::Function *TaskFunction, QualType SharedsTy, 1984 Address Shareds, const Expr *IfCond, 1985 const OMPTaskDataTy &Data) override; 1986 1987 /// Emit task region for the taskloop directive. The taskloop region is 1988 /// emitted in several steps: 1989 /// 1. Emit a call to kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32 1990 /// gtid, kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds, 1991 /// kmp_routine_entry_t *task_entry). Here task_entry is a pointer to the 1992 /// function: 1993 /// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) { 1994 /// TaskFunction(gtid, tt->part_id, tt->shareds); 1995 /// return 0; 1996 /// } 1997 /// 2. Copy a list of shared variables to field shareds of the resulting 1998 /// structure kmp_task_t returned by the previous call (if any). 1999 /// 3. Copy a pointer to destructions function to field destructions of the 2000 /// resulting structure kmp_task_t. 2001 /// 4. Emit a call to void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t 2002 /// *task, int if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int 2003 /// nogroup, int sched, kmp_uint64 grainsize, void *task_dup ), where new_task 2004 /// is a resulting structure from 2005 /// previous items. 2006 /// \param D Current task directive. 2007 /// \param TaskFunction An LLVM function with type void (*)(i32 /*gtid*/, i32 2008 /// /*part_id*/, captured_struct */*__context*/); 2009 /// \param SharedsTy A type which contains references the shared variables. 2010 /// \param Shareds Context with the list of shared variables from the \p 2011 /// TaskFunction. 2012 /// \param IfCond Not a nullptr if 'if' clause was specified, nullptr 2013 /// otherwise. 2014 /// \param Data Additional data for task generation like tiednsee, final 2015 /// state, list of privates etc. 2016 void emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc, 2017 const OMPLoopDirective &D, llvm::Function *TaskFunction, 2018 QualType SharedsTy, Address Shareds, const Expr *IfCond, 2019 const OMPTaskDataTy &Data) override; 2020 2021 /// Emit a code for reduction clause. Next code should be emitted for 2022 /// reduction: 2023 /// \code 2024 /// 2025 /// static kmp_critical_name lock = { 0 }; 2026 /// 2027 /// void reduce_func(void *lhs[<n>], void *rhs[<n>]) { 2028 /// ... 2029 /// *(Type<i>*)lhs[i] = RedOp<i>(*(Type<i>*)lhs[i], *(Type<i>*)rhs[i]); 2030 /// ... 2031 /// } 2032 /// 2033 /// ... 2034 /// void *RedList[<n>] = {&<RHSExprs>[0], ..., &<RHSExprs>[<n>-1]}; 2035 /// switch (__kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList), 2036 /// RedList, reduce_func, &<lock>)) { 2037 /// case 1: 2038 /// ... 2039 /// <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]); 2040 /// ... 2041 /// __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>); 2042 /// break; 2043 /// case 2: 2044 /// ... 2045 /// Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i])); 2046 /// ... 2047 /// break; 2048 /// default:; 2049 /// } 2050 /// \endcode 2051 /// 2052 /// \param Privates List of private copies for original reduction arguments. 2053 /// \param LHSExprs List of LHS in \a ReductionOps reduction operations. 2054 /// \param RHSExprs List of RHS in \a ReductionOps reduction operations. 2055 /// \param ReductionOps List of reduction operations in form 'LHS binop RHS' 2056 /// or 'operator binop(LHS, RHS)'. 2057 /// \param Options List of options for reduction codegen: 2058 /// WithNowait true if parent directive has also nowait clause, false 2059 /// otherwise. 2060 /// SimpleReduction Emit reduction operation only. Used for omp simd 2061 /// directive on the host. 2062 /// ReductionKind The kind of reduction to perform. 2063 void emitReduction(CodeGenFunction &CGF, SourceLocation Loc, 2064 ArrayRef<const Expr *> Privates, 2065 ArrayRef<const Expr *> LHSExprs, 2066 ArrayRef<const Expr *> RHSExprs, 2067 ArrayRef<const Expr *> ReductionOps, 2068 ReductionOptionsTy Options) override; 2069 2070 /// Emit a code for initialization of task reduction clause. Next code 2071 /// should be emitted for reduction: 2072 /// \code 2073 /// 2074 /// _taskred_item_t red_data[n]; 2075 /// ... 2076 /// red_data[i].shar = &shareds[i]; 2077 /// red_data[i].orig = &origs[i]; 2078 /// red_data[i].size = sizeof(origs[i]); 2079 /// red_data[i].f_init = (void*)RedInit<i>; 2080 /// red_data[i].f_fini = (void*)RedDest<i>; 2081 /// red_data[i].f_comb = (void*)RedOp<i>; 2082 /// red_data[i].flags = <Flag_i>; 2083 /// ... 2084 /// void* tg1 = __kmpc_taskred_init(gtid, n, red_data); 2085 /// \endcode 2086 /// For reduction clause with task modifier it emits the next call: 2087 /// \code 2088 /// 2089 /// _taskred_item_t red_data[n]; 2090 /// ... 2091 /// red_data[i].shar = &shareds[i]; 2092 /// red_data[i].orig = &origs[i]; 2093 /// red_data[i].size = sizeof(origs[i]); 2094 /// red_data[i].f_init = (void*)RedInit<i>; 2095 /// red_data[i].f_fini = (void*)RedDest<i>; 2096 /// red_data[i].f_comb = (void*)RedOp<i>; 2097 /// red_data[i].flags = <Flag_i>; 2098 /// ... 2099 /// void* tg1 = __kmpc_taskred_modifier_init(loc, gtid, is_worksharing, n, 2100 /// red_data); 2101 /// \endcode 2102 /// \param LHSExprs List of LHS in \a Data.ReductionOps reduction operations. 2103 /// \param RHSExprs List of RHS in \a Data.ReductionOps reduction operations. 2104 /// \param Data Additional data for task generation like tiedness, final 2105 /// state, list of privates, reductions etc. 2106 llvm::Value *emitTaskReductionInit(CodeGenFunction &CGF, SourceLocation Loc, 2107 ArrayRef<const Expr *> LHSExprs, 2108 ArrayRef<const Expr *> RHSExprs, 2109 const OMPTaskDataTy &Data) override; 2110 2111 /// Emits the following code for reduction clause with task modifier: 2112 /// \code 2113 /// __kmpc_task_reduction_modifier_fini(loc, gtid, is_worksharing); 2114 /// \endcode 2115 void emitTaskReductionFini(CodeGenFunction &CGF, SourceLocation Loc, 2116 bool IsWorksharingReduction) override; 2117 2118 /// Required to resolve existing problems in the runtime. Emits threadprivate 2119 /// variables to store the size of the VLAs/array sections for 2120 /// initializer/combiner/finalizer functions + emits threadprivate variable to 2121 /// store the pointer to the original reduction item for the custom 2122 /// initializer defined by declare reduction construct. 2123 /// \param RCG Allows to reuse an existing data for the reductions. 2124 /// \param N Reduction item for which fixups must be emitted. 2125 void emitTaskReductionFixups(CodeGenFunction &CGF, SourceLocation Loc, 2126 ReductionCodeGen &RCG, unsigned N) override; 2127 2128 /// Get the address of `void *` type of the privatue copy of the reduction 2129 /// item specified by the \p SharedLVal. 2130 /// \param ReductionsPtr Pointer to the reduction data returned by the 2131 /// emitTaskReductionInit function. 2132 /// \param SharedLVal Address of the original reduction item. 2133 Address getTaskReductionItem(CodeGenFunction &CGF, SourceLocation Loc, 2134 llvm::Value *ReductionsPtr, 2135 LValue SharedLVal) override; 2136 2137 /// Emit code for 'taskwait' directive. 2138 void emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc, 2139 const OMPTaskDataTy &Data) override; 2140 2141 /// Emit code for 'cancellation point' construct. 2142 /// \param CancelRegion Region kind for which the cancellation point must be 2143 /// emitted. 2144 /// 2145 void emitCancellationPointCall(CodeGenFunction &CGF, SourceLocation Loc, 2146 OpenMPDirectiveKind CancelRegion) override; 2147 2148 /// Emit code for 'cancel' construct. 2149 /// \param IfCond Condition in the associated 'if' clause, if it was 2150 /// specified, nullptr otherwise. 2151 /// \param CancelRegion Region kind for which the cancel must be emitted. 2152 /// 2153 void emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc, 2154 const Expr *IfCond, 2155 OpenMPDirectiveKind CancelRegion) override; 2156 2157 /// Emit outilined function for 'target' directive. 2158 /// \param D Directive to emit. 2159 /// \param ParentName Name of the function that encloses the target region. 2160 /// \param OutlinedFn Outlined function value to be defined by this call. 2161 /// \param OutlinedFnID Outlined function ID value to be defined by this call. 2162 /// \param IsOffloadEntry True if the outlined function is an offload entry. 2163 /// \param CodeGen Code generation sequence for the \a D directive. 2164 /// An outlined function may not be an entry if, e.g. the if clause always 2165 /// evaluates to false. 2166 void emitTargetOutlinedFunction(const OMPExecutableDirective &D, 2167 StringRef ParentName, 2168 llvm::Function *&OutlinedFn, 2169 llvm::Constant *&OutlinedFnID, 2170 bool IsOffloadEntry, 2171 const RegionCodeGenTy &CodeGen) override; 2172 2173 /// Emit the target offloading code associated with \a D. The emitted 2174 /// code attempts offloading the execution to the device, an the event of 2175 /// a failure it executes the host version outlined in \a OutlinedFn. 2176 /// \param D Directive to emit. 2177 /// \param OutlinedFn Host version of the code to be offloaded. 2178 /// \param OutlinedFnID ID of host version of the code to be offloaded. 2179 /// \param IfCond Expression evaluated in if clause associated with the target 2180 /// directive, or null if no if clause is used. 2181 /// \param Device Expression evaluated in device clause associated with the 2182 /// target directive, or null if no device clause is used and device modifier. 2183 void emitTargetCall( 2184 CodeGenFunction &CGF, const OMPExecutableDirective &D, 2185 llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond, 2186 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device, 2187 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF, 2188 const OMPLoopDirective &D)> 2189 SizeEmitter) override; 2190 2191 /// Emit the target regions enclosed in \a GD function definition or 2192 /// the function itself in case it is a valid device function. Returns true if 2193 /// \a GD was dealt with successfully. 2194 /// \param GD Function to scan. 2195 bool emitTargetFunctions(GlobalDecl GD) override; 2196 2197 /// Emit the global variable if it is a valid device global variable. 2198 /// Returns true if \a GD was dealt with successfully. 2199 /// \param GD Variable declaration to emit. 2200 bool emitTargetGlobalVariable(GlobalDecl GD) override; 2201 2202 /// Emit the global \a GD if it is meaningful for the target. Returns 2203 /// if it was emitted successfully. 2204 /// \param GD Global to scan. 2205 bool emitTargetGlobal(GlobalDecl GD) override; 2206 2207 /// Emits code for teams call of the \a OutlinedFn with 2208 /// variables captured in a record which address is stored in \a 2209 /// CapturedStruct. 2210 /// \param OutlinedFn Outlined function to be run by team masters. Type of 2211 /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*). 2212 /// \param CapturedVars A pointer to the record with the references to 2213 /// variables used in \a OutlinedFn function. 2214 /// 2215 void emitTeamsCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, 2216 SourceLocation Loc, llvm::Function *OutlinedFn, 2217 ArrayRef<llvm::Value *> CapturedVars) override; 2218 2219 /// Emits call to void __kmpc_push_num_teams(ident_t *loc, kmp_int32 2220 /// global_tid, kmp_int32 num_teams, kmp_int32 thread_limit) to generate code 2221 /// for num_teams clause. 2222 /// \param NumTeams An integer expression of teams. 2223 /// \param ThreadLimit An integer expression of threads. 2224 void emitNumTeamsClause(CodeGenFunction &CGF, const Expr *NumTeams, 2225 const Expr *ThreadLimit, SourceLocation Loc) override; 2226 2227 /// Emit the target data mapping code associated with \a D. 2228 /// \param D Directive to emit. 2229 /// \param IfCond Expression evaluated in if clause associated with the 2230 /// target directive, or null if no device clause is used. 2231 /// \param Device Expression evaluated in device clause associated with the 2232 /// target directive, or null if no device clause is used. 2233 /// \param Info A record used to store information that needs to be preserved 2234 /// until the region is closed. 2235 void emitTargetDataCalls(CodeGenFunction &CGF, 2236 const OMPExecutableDirective &D, const Expr *IfCond, 2237 const Expr *Device, const RegionCodeGenTy &CodeGen, 2238 CGOpenMPRuntime::TargetDataInfo &Info) override; 2239 2240 /// Emit the data mapping/movement code associated with the directive 2241 /// \a D that should be of the form 'target [{enter|exit} data | update]'. 2242 /// \param D Directive to emit. 2243 /// \param IfCond Expression evaluated in if clause associated with the target 2244 /// directive, or null if no if clause is used. 2245 /// \param Device Expression evaluated in device clause associated with the 2246 /// target directive, or null if no device clause is used. 2247 void emitTargetDataStandAloneCall(CodeGenFunction &CGF, 2248 const OMPExecutableDirective &D, 2249 const Expr *IfCond, 2250 const Expr *Device) override; 2251 2252 /// Emit initialization for doacross loop nesting support. 2253 /// \param D Loop-based construct used in doacross nesting construct. 2254 void emitDoacrossInit(CodeGenFunction &CGF, const OMPLoopDirective &D, 2255 ArrayRef<Expr *> NumIterations) override; 2256 2257 /// Emit code for doacross ordered directive with 'depend' clause. 2258 /// \param C 'depend' clause with 'sink|source' dependency kind. 2259 void emitDoacrossOrdered(CodeGenFunction &CGF, 2260 const OMPDependClause *C) override; 2261 2262 /// Emit code for doacross ordered directive with 'doacross' clause. 2263 /// \param C 'doacross' clause with 'sink|source' dependence type. 2264 void emitDoacrossOrdered(CodeGenFunction &CGF, 2265 const OMPDoacrossClause *C) override; 2266 2267 /// Translates the native parameter of outlined function if this is required 2268 /// for target. 2269 /// \param FD Field decl from captured record for the parameter. 2270 /// \param NativeParam Parameter itself. 2271 const VarDecl *translateParameter(const FieldDecl *FD, 2272 const VarDecl *NativeParam) const override; 2273 2274 /// Gets the address of the native argument basing on the address of the 2275 /// target-specific parameter. 2276 /// \param NativeParam Parameter itself. 2277 /// \param TargetParam Corresponding target-specific parameter. 2278 Address getParameterAddress(CodeGenFunction &CGF, const VarDecl *NativeParam, 2279 const VarDecl *TargetParam) const override; 2280 2281 /// Gets the OpenMP-specific address of the local variable. 2282 Address getAddressOfLocalVariable(CodeGenFunction &CGF, 2283 const VarDecl *VD) override { 2284 return Address::invalid(); 2285 } 2286 }; 2287 2288 } // namespace CodeGen 2289 // Utility for openmp doacross clause kind 2290 namespace { 2291 template <typename T> class OMPDoacrossKind { 2292 public: 2293 bool isSink(const T *) { return false; } 2294 bool isSource(const T *) { return false; } 2295 }; 2296 template <> class OMPDoacrossKind<OMPDependClause> { 2297 public: 2298 bool isSink(const OMPDependClause *C) { 2299 return C->getDependencyKind() == OMPC_DEPEND_sink; 2300 } 2301 bool isSource(const OMPDependClause *C) { 2302 return C->getDependencyKind() == OMPC_DEPEND_source; 2303 } 2304 }; 2305 template <> class OMPDoacrossKind<OMPDoacrossClause> { 2306 public: 2307 bool isSource(const OMPDoacrossClause *C) { 2308 return C->getDependenceType() == OMPC_DOACROSS_source || 2309 C->getDependenceType() == OMPC_DOACROSS_source_omp_cur_iteration; 2310 } 2311 bool isSink(const OMPDoacrossClause *C) { 2312 return C->getDependenceType() == OMPC_DOACROSS_sink || 2313 C->getDependenceType() == OMPC_DOACROSS_sink_omp_cur_iteration; 2314 } 2315 }; 2316 } // namespace 2317 } // namespace clang 2318 2319 #endif 2320