Lines Matching defs:task
47 // __kmp_trace_task_stack: print the tied tasks from the task stack in order
51 // thread_data: thread data for task team thread containing stack
102 // __kmp_init_task_stack: initialize the task stack for the first time
107 // thread_data: thread data for task team thread containing stack
125 // __kmp_free_task_stack: free the task stack when thread_data is destroyed.
152 // __kmp_push_task_stack: Push the tied task onto the task stack.
157 // tied_task: the task to push on the stack
183 // Find beginning of this task block
210 // __kmp_pop_task_stack: Pop the tied task from the task stack. Don't return
211 // the task, just check to make sure it matches the ending task passed in.
215 // tied_task: the task popped off the stack
216 // ending_task: the task that is ending (should match popped task)
260 // returns 1 if new task is allowed to execute, 0 otherwise
272 // check if the task is not suspended on barrier
278 // check generation up to the level of the current task
309 // Re-allocates a task deque for a particular thread, copies the content from
401 // __kmp_push_priority_task: Add a task to the team's priority task deque
408 ("__kmp_push_priority_task: T#%d trying to push task %p, pri %d.\n",
411 // Find task queue specific to priority value
448 "TASK_NOT_PUSHED for task %p\n",
452 // expand deque to push the task which is not allowed to execute
464 TCR_4(thread_data->td.td_deque_ntasks) + 1); // Adjust task count
468 "TASK_SUCCESSFULLY_PUSHED: task=%p ntasks=%d head=%u tail=%u\n",
476 // __kmp_push_task: Add a task to the thread's deque
477 static kmp_int32 __kmp_push_task(kmp_int32 gtid, kmp_task_t *task) {
479 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
481 // If we encounter a hidden helper task, and the current thread is not a
482 // hidden helper thread, we have to give the task to any hidden helper thread
487 __kmpc_give_task(task, __kmp_tid_from_gtid(shadow_gtid));
498 ("__kmp_push_task: T#%d trying to push task %p.\n", gtid, taskdata));
501 // untied task needs to increment counter so that the task structure is not
507 ("__kmp_push_task: T#%d untied_count (%d) incremented for task %p\n",
514 "TASK_NOT_PUSHED for task %p\n",
528 if (taskdata->td_flags.priority_specified && task->data2.priority > 0 &&
530 int pri = KMP_MIN(task->data2.priority, __kmp_max_task_priority);
537 // No lock needed since only owner can allocate. If the task is hidden_helper,
552 "TASK_NOT_PUSHED for task %p\n",
560 // expand deque to push the task which is not allowed to execute
565 // Lock the deque for the task push operation
568 // Need to recheck as we can get a proxy task from thread outside of OpenMP
576 "returning TASK_NOT_PUSHED for task %p\n",
580 // expand deque to push the task which is not allowed to execute
595 TCR_4(thread_data->td.td_deque_ntasks) + 1); // Adjust task count
599 "task=%p ntasks=%d head=%u tail=%u\n",
608 // __kmp_pop_current_task_from_thread: set up current task from called thread
628 // __kmp_push_current_task_to_thread: set up current task in called thread for a
632 // team: team for implicit task data
636 // current task of the thread is a parent of the new just created implicit
665 // __kmp_task_start: bookkeeping for a task starting execution
668 // task: task starting execution
669 // current_task: task suspending
670 static void __kmp_task_start(kmp_int32 gtid, kmp_task_t *task,
672 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
676 ("__kmp_task_start(enter): T#%d starting task %p: current_task=%p\n",
681 // mark currently executing task as suspended
682 // TODO: GEH - make sure root team implicit task is initialized properly.
686 // Add task to stack if tied
693 // mark starting task as executing and as current task
708 // before setting work_state, otherwise wrong state is set after end of task
710 KA_TRACE(10, ("__kmp_task_start(exit): T#%d task=%p\n", gtid, taskdata));
718 // Initialize OMPT fields maintained by a task. This will only be called after
721 static inline void __ompt_task_init(kmp_taskdata_t *task, int tid) {
723 task->ompt_task_info.task_data.value = 0;
724 task->ompt_task_info.frame.exit_frame = ompt_data_none;
725 task->ompt_task_info.frame.enter_frame = ompt_data_none;
726 task->ompt_task_info.frame.exit_frame_flags =
728 task->ompt_task_info.frame.enter_frame_flags =
730 task->ompt_task_info.dispatch_chunk.start = 0;
731 task->ompt_task_info.dispatch_chunk.iterations = 0;
735 // Build and trigger task-begin event
736 static inline void __ompt_task_start(kmp_task_t *task,
739 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
745 /* let OMPT know that we're about to run this task */
755 // Build and trigger final task-schedule event
756 static inline void __ompt_task_finish(kmp_task_t *task,
760 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
766 /* let OMPT know that we're returning to the callee task */
776 kmp_task_t *task,
779 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
782 KA_TRACE(10, ("__kmpc_omp_task_begin_if0(enter): T#%d loc=%p task=%p "
787 // untied task needs to increment counter so that the task structure is not
792 "incremented for task %p\n",
797 1; // Execute this task immediately, not deferred.
798 __kmp_task_start(gtid, task, current_task);
816 __ompt_task_start(task, current_task, gtid);
820 KA_TRACE(10, ("__kmpc_omp_task_begin_if0(exit): T#%d loc=%p task=%p,\n", gtid,
827 kmp_task_t *task,
830 __kmpc_omp_task_begin_if0_template<true>(loc_ref, gtid, task, frame_address,
835 // __kmpc_omp_task_begin_if0: report that a given serialized task has started
838 // loc_ref: source location information; points to beginning of task block.
840 // task: task thunk for the started task.
850 kmp_task_t *task) {
854 __kmpc_omp_task_begin_if0_ompt(loc_ref, gtid, task,
860 __kmpc_omp_task_begin_if0_template<false>(loc_ref, gtid, task, NULL, NULL);
864 // __kmpc_omp_task_begin: report that a given task has started execution
866 void __kmpc_omp_task_begin(ident_t *loc_ref, kmp_int32 gtid, kmp_task_t *task) {
871 ("__kmpc_omp_task_begin(enter): T#%d loc=%p task=%p current_task=%p\n",
872 gtid, loc_ref, KMP_TASK_TO_TASKDATA(task), current_task));
874 __kmp_task_start(gtid, task, current_task);
876 KA_TRACE(10, ("__kmpc_omp_task_begin(exit): T#%d loc=%p task=%p,\n", gtid,
877 loc_ref, KMP_TASK_TO_TASKDATA(task)));
882 // __kmp_free_task: free the current task space and the space for shareds
885 // taskdata: task to free
889 KA_TRACE(30, ("__kmp_free_task: T#%d freeing data from task %p\n", gtid,
900 kmp_task_t *task = KMP_TASKDATA_TO_TASK(taskdata);
902 task->data1.destructors = NULL;
903 task->data2.priority = 0;
910 // deallocate the taskdata and shared variable blocks associated with this task
929 // start at one because counts current task and children
934 KA_TRACE(20, ("__kmp_free_task: T#%d freed task %p\n", gtid, taskdata));
937 // __kmp_free_task_and_ancestors: free the current task and ancestors without
941 // taskdata: task to free
960 KA_TRACE(20, ("__kmp_free_task_and_ancestors(enter): T#%d task %p complete "
964 // --- Deallocate my ancestor task ---
971 // Stop checking ancestors at implicit task instead of walking up ancestor
985 "dephash of implicit task %p\n",
987 // cleanup dephash of finished implicit task
1000 20, ("__kmp_free_task_and_ancestors(exit): T#%d task %p has %d children; "
1005 // Only need to keep track of child task counts if any of the following:
1007 // 2. it is a proxy or detachable or hidden helper task
1008 // 3. the children counter of its parent task is greater than 0.
1009 // The reason for the 3rd one is for serialized team that found detached task,
1010 // hidden helper task, T. In this case, the execution of T is still deferred,
1011 // and it is also possible that a regular task depends on T. In this case, if we
1012 // don't track the children, task synchronization will be broken.
1027 // __kmp_task_finish: bookkeeping to do when a task finishes execution
1030 // task: task to be finished
1031 // resumed_task: task to be resumed. (may be NULL if task is serialized)
1037 static void __kmp_task_finish(kmp_int32 gtid, kmp_task_t *task,
1039 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
1050 KA_TRACE(10, ("__kmp_task_finish(enter): T#%d finishing task %p and resuming "
1051 "task %p\n",
1060 // Pop task from stack if tied
1068 // untied task needs to check the counter so that the task structure is not
1073 ("__kmp_task_finish: T#%d untied_count (%d) decremented for task %p\n",
1076 // untied task is not done, to be continued possibly by other thread, do
1080 resumed_task = taskdata->td_parent; // In a serialized task, the resumed
1081 // task is the parent
1084 resumed_task->td_flags.executing = 1; // resume previous task
1085 KA_TRACE(10, ("__kmp_task_finish(exit): T#%d partially done task %p, "
1086 "resuming task %p\n",
1092 // bookkeeping for resuming task:
1099 resumed_task = taskdata->td_parent; // In a serialized task, the resumed
1100 // task is the parent
1104 NULL); // verify that resumed task is passed as argument
1114 kmp_routine_entry_t destr_thunk = task->data1.destructors;
1116 destr_thunk(gtid, task);
1127 // event hasn't been fulfilled yet. Try to detach task.
1131 // task finished execution
1133 taskdata->td_flags.executing = 0; // suspend the finishing task
1136 // For a detached task, which is not completed, we switch back
1140 __ompt_task_finish(task, resumed_task, ompt_task_detach);
1160 __ompt_task_finish(task, resumed_task, ompt_task_switch);
1163 __kmpc_give_task(task, __kmp_tid_from_gtid(gtid));
1170 taskdata->td_flags.complete = 1; // mark the task as completed
1172 taskdata->td_flags.onced = 1; // mark the task as ran once already
1176 // This is not a detached task, we are done here
1178 __ompt_task_finish(task, resumed_task, ompt_task_complete);
1199 // chain with the proxy task as origin
1203 // called. Othertwise, if a task is executed immediately from the
1207 taskdata->td_flags.executing = 0; // suspend the finishing task
1218 20, ("__kmp_task_finish: T#%d finished task %p, %d incomplete children\n",
1221 // Free this task and then ancestor tasks if they have no children.
1224 // it doesn't see the freed task as the current task.
1229 // TODO: GEH - make sure root team implicit task is initialized properly.
1231 resumed_task->td_flags.executing = 1; // resume previous task
1238 // __kmp_free_task, which resets all task parameters such as
1241 // non-TDG implementation because we never reuse a task(data) structure
1247 10, ("__kmp_task_finish(exit): T#%d finished task %p, resuming task %p\n",
1256 kmp_task_t *task) {
1257 KA_TRACE(10, ("__kmpc_omp_task_complete_if0(enter): T#%d loc=%p task=%p\n",
1258 gtid, loc_ref, KMP_TASK_TO_TASKDATA(task)));
1260 // this routine will provide task to resume
1261 __kmp_task_finish<ompt>(gtid, task, NULL);
1263 KA_TRACE(10, ("__kmpc_omp_task_complete_if0(exit): T#%d loc=%p task=%p\n",
1264 gtid, loc_ref, KMP_TASK_TO_TASKDATA(task)));
1282 kmp_task_t *task) {
1283 __kmpc_omp_task_complete_if0_template<true>(loc_ref, gtid, task);
1287 // __kmpc_omp_task_complete_if0: report that a task has completed execution
1289 // loc_ref: source location information; points to end of task block.
1291 // task: task thunk for the completed task.
1293 kmp_task_t *task) {
1296 __kmpc_omp_task_complete_if0_ompt(loc_ref, gtid, task);
1300 __kmpc_omp_task_complete_if0_template<false>(loc_ref, gtid, task);
1304 // __kmpc_omp_task_complete: report that a task has completed execution
1307 kmp_task_t *task) {
1308 KA_TRACE(10, ("__kmpc_omp_task_complete(enter): T#%d loc=%p task=%p\n", gtid,
1309 loc_ref, KMP_TASK_TO_TASKDATA(task)));
1311 __kmp_task_finish<false>(gtid, task,
1312 NULL); // Not sure how to find task to resume
1314 KA_TRACE(10, ("__kmpc_omp_task_complete(exit): T#%d loc=%p task=%p\n", gtid,
1315 loc_ref, KMP_TASK_TO_TASKDATA(task)));
1321 // task for a given thread
1324 // this_thr: thread data structure corresponding to implicit task
1327 // set_curr_task: TRUE if need to push current task to thread
1328 // NOTE: Routine does not set up the implicit task ICVS. This is assumed to
1333 kmp_taskdata_t *task = &team->t.t_implicit_task_taskdata[tid];
1337 ("__kmp_init_implicit_task(enter): T#:%d team=%p task=%p, reinit=%s\n",
1338 tid, team, task, set_curr_task ? "TRUE" : "FALSE"));
1340 task->td_task_id = KMP_GEN_TASK_ID();
1341 task->td_team = team;
1342 // task->td_parent = NULL; // fix for CQ230101 (broken parent task info
1344 task->td_ident = loc_ref;
1345 task->td_taskwait_ident = NULL;
1346 task->td_taskwait_counter = 0;
1347 task->td_taskwait_thread = 0;
1349 task->td_flags.tiedness = TASK_TIED;
1350 task->td_flags.tasktype = TASK_IMPLICIT;
1351 task->td_flags.proxy = TASK_FULL;
1354 task->td_flags.task_serial = 1;
1355 task->td_flags.tasking_ser = (__kmp_tasking_mode == tskm_immediate_exec);
1356 task->td_flags.team_serial = (team->t.t_serialized) ? 1 : 0;
1358 task->td_flags.started = 1;
1359 task->td_flags.executing = 1;
1360 task->td_flags.complete = 0;
1361 task->td_flags.freed = 0;
1363 task->td_flags.onced = 0;
1366 task->td_depnode = NULL;
1367 task->td_last_tied = task;
1368 task->td_allow_completion_event.type = KMP_EVENT_UNINITIALIZED;
1371 KMP_ATOMIC_ST_REL(&task->td_incomplete_child_tasks, 0);
1372 // Not used: don't need to deallocate implicit task
1373 KMP_ATOMIC_ST_REL(&task->td_allocated_child_tasks, 0);
1374 task->td_taskgroup = NULL; // An implicit task does not have taskgroup
1375 task->td_dephash = NULL;
1378 KMP_DEBUG_ASSERT(task->td_incomplete_child_tasks == 0);
1379 KMP_DEBUG_ASSERT(task->td_allocated_child_tasks == 0);
1384 __ompt_task_init(task, tid);
1387 KF_TRACE(10, ("__kmp_init_implicit_task(exit): T#:%d team=%p task=%p\n", tid,
1388 team, task));
1395 // thread: thread data structure corresponding to implicit task
1397 kmp_taskdata_t *task = thread->th.th_current_task;
1398 if (task->td_dephash) {
1400 task->td_flags.complete = 1;
1402 task->td_flags.onced = 1;
1404 children = KMP_ATOMIC_LD_ACQ(&task->td_incomplete_child_tasks);
1405 kmp_tasking_flags_t flags_old = task->td_flags;
1409 if (KMP_COMPARE_AND_STORE_ACQ32(RCAST(kmp_int32 *, &task->td_flags),
1413 "dephash of implicit task %p\n",
1414 thread->th.th_info.ds.ds_gtid, task));
1415 __kmp_dephash_free_entries(thread, task->td_dephash);
1424 // thread: thread data structure corresponding to implicit task
1426 kmp_taskdata_t *task = thread->th.th_current_task;
1427 if (task && task->td_dephash) {
1428 __kmp_dephash_free(thread, task->td_dephash);
1429 task->td_dephash = NULL;
1445 // __kmp_task_alloc: Allocate the taskdata and task data structures for a task
1449 // flags: include tiedness & task type (explicit vs. implicit) of the ''new''
1450 // task encountered. Converted from kmp_int32 to kmp_tasking_flags_t in routine.
1452 // private vars accessed in task.
1454 // in task.
1455 // task_entry: Pointer to task code entry point generated by compiler.
1456 // returns: a pointer to the allocated kmp_task_t structure (task).
1461 kmp_task_t *task;
1476 // If the hidden helper task is not enabled, reset the flag to FALSE.
1494 // Untied task encountered causes the TSC algorithm to check entire deque of
1495 // the victim thread. If no untied task encountered, then checking the head
1513 setup a task team and propagate it to the thread */
1516 ("T#%d creating task team in __kmp_task_alloc for proxy task\n",
1523 /* tasking must be enabled now as the task might not be pushed */
1527 ("T#%d enabling tasking in __kmp_task_alloc for proxy task\n", gtid));
1565 task = KMP_TASKDATA_TO_TASK(taskdata);
1567 // Make sure task & taskdata are aligned appropriately
1570 KMP_DEBUG_ASSERT((((kmp_uintptr_t)task) & (sizeof(double) - 1)) == 0);
1573 KMP_DEBUG_ASSERT((((kmp_uintptr_t)task) & (sizeof(_Quad) - 1)) == 0);
1577 task->shareds = &((char *)taskdata)[shareds_offset];
1579 KMP_DEBUG_ASSERT((((kmp_uintptr_t)task->shareds) & (sizeof(void *) - 1)) ==
1582 task->shareds = NULL;
1584 task->routine = task_entry;
1585 task->part_id = 0; // AC: Always start with 0 part id
1606 // If it is hidden helper task, we need to set the team and task team
1614 // GEH - TODO: fix this to copy parent task's value of tasking_ser flag
1617 // GEH - TODO: fix this to copy parent task's value of team_serial flag
1620 // GEH - Note we serialize the task if the team is serialized to make sure
1636 // start at one because counts current task and children
1639 parent_task->td_taskgroup; // task inherits taskgroup from the parent task
1644 taskdata->td_last_tied = NULL; // will be set when the task is scheduled
1679 KA_TRACE(20, ("__kmp_task_alloc(exit): T#%d created task %p parent=%p\n",
1682 return task;
1716 // target task is untied defined in the specification
1729 @param loc_ref location of the original task directive
1731 @param new_task task thunk allocated by __kmpc_omp_task_alloc() for the ''new
1732 task''
1737 This entry registers the affinity information attached to a task with the task
1747 // __kmp_invoke_task: invoke the specified task
1750 // task: the task to invoke
1751 // current_task: the task to resume after task invocation
1756 __kmp_invoke_task(kmp_int32 gtid, kmp_task_t *task,
1758 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
1762 30, ("__kmp_invoke_task(enter): T#%d invoking task %p, current_task=%p\n",
1764 KMP_DEBUG_ASSERT(task);
1767 // This is a proxy task that was already completed but it needs to run
1771 ("__kmp_invoke_task: T#%d running bottom finish for proxy task %p\n",
1774 __kmp_bottom_half_finish_proxy(gtid, task);
1777 "proxy task %p, resuming task %p\n",
1784 // For untied tasks, the first task executed only calls __kmpc_omp_task and
1788 // Store the threads states and restore them after the task
1801 __kmp_task_start(gtid, task, current_task); // OMPT only if not discarded
1826 // this task belongs to a task group and we need to cancel it
1831 // Invoke the task routine and pass in relevant data.
1862 // OMPT task begin
1865 __ompt_task_start(task, current_task, gtid);
1892 // Time outer level explicit task on barrier for adjusting imbalance time
1898 KMP_FSYNC_ACQUIRED(taskdata); // acquired self (new task)
1904 // executed the task routine once. We must query for the handle completion
1910 if (task->routine != NULL) {
1913 ((void (*)(void *))(*(task->routine)))(task->shareds);
1917 (*(task->routine))(gtid, task);
1924 // Barrier imbalance - adjust arrive time with the task duration
1945 __kmp_task_finish<true>(gtid, task, current_task);
1948 __kmp_task_finish<false>(gtid, task, current_task);
1952 __ompt_task_finish(task, current_task, ompt_task_switch);
1958 ("__kmp_invoke_task(exit): T#%d completed task %p, resuming task %p\n",
1963 // __kmpc_omp_task_parts: Schedule a thread-switchable task for execution
1965 // loc_ref: location of original task pragma (ignored)
1967 // new_task: task thunk allocated by __kmp_omp_task_alloc() for the ''new task''
1969 // TASK_CURRENT_NOT_QUEUED (0) if did not suspend and queue current task to
1971 // TASK_CURRENT_QUEUED (1) if suspended and queued the current task to be
1977 KA_TRACE(10, ("__kmpc_omp_task_parts(enter): T#%d loc=%p task=%p\n", gtid,
1994 /* Should we execute the new task or queue it? For now, let's just always try
1998 { // Execute this task immediately
2007 "loc=%p task=%p, return: TASK_CURRENT_NOT_QUEUED\n",
2018 // __kmp_omp_task: Schedule a non-thread-switchable task for execution
2021 // new_task:non-thread-switchable task thunk allocated by __kmp_omp_task_alloc()
2022 // serialize_immediate: if TRUE then if the task is executed immediately its
2025 // TASK_CURRENT_NOT_QUEUED (0) if did not suspend and queue current task to
2027 // TASK_CURRENT_QUEUED (1) if suspended and queued the current task to be
2057 new_record[i].task = nullptr;
2070 // record a task
2071 if (tdg->record_map[new_taskdata->td_task_id].task == nullptr) {
2072 tdg->record_map[new_taskdata->td_task_id].task = new_task;
2080 /* Should we execute the new task or queue it? For now, let's just always try
2084 { // Execute this task immediately
2108 // non-thread-switchable task from the parent thread only!
2110 // loc_ref: location of original task pragma (ignored)
2112 // new_task: non-thread-switchable task thunk allocated by
2115 // TASK_CURRENT_NOT_QUEUED (0) if did not suspend and queue current task to
2117 // TASK_CURRENT_QUEUED (1) if suspended and queued the current task to be
2127 KA_TRACE(10, ("__kmpc_omp_task(enter): T#%d loc=%p task=%p\n", gtid, loc_ref,
2150 // We are scheduling the continuation of an UNTIED task.
2151 // Scheduling back to the parent task.
2163 "TASK_CURRENT_NOT_QUEUED: loc=%p task=%p\n",
2174 // a taskloop task with the correct OMPT return address
2176 // loc_ref: location of original task pragma (ignored)
2178 // new_task: non-thread-switchable task thunk allocated by
2182 // TASK_CURRENT_NOT_QUEUED (0) if did not suspend and queue current task to
2184 // TASK_CURRENT_QUEUED (1) if suspended and queued the current task to be
2194 KA_TRACE(10, ("__kmpc_omp_task(enter): T#%d loc=%p task=%p\n", gtid, loc_ref,
2215 "TASK_CURRENT_NOT_QUEUED: loc=%p task=%p\n",
2329 KA_TRACE(10, ("__kmpc_omp_taskwait(exit): T#%d task %p finished waiting, "
2346 // __kmpc_omp_taskwait: Wait until all tasks generated by the current task are
2359 // __kmpc_omp_taskyield: switch to a different task
2375 // Should we model this as a task wait or not?
2419 KA_TRACE(10, ("__kmpc_omp_taskyield(exit): T#%d task %p resuming, "
2439 Flags for special info per task reduction item.
2586 Initialize task reduction for the taskgroup.
2615 Initialize task reduction for the taskgroup.
2635 // Copy task reduction data (except for shared pointers).
2728 KMP_ASSERT2(0, "Unknown task reduction item");
2732 // Finalize task reduction.
2774 // Cleanup task reduction data for parallel or worksharing,
2775 // do not touch task private data other threads still working with.
2816 (void *)1) { // wait for task reduction initialization
2835 Initialize task reduction for a parallel or worksharing.
2857 Initialize task reduction for a parallel or worksharing.
2874 Finalize task reduction for a parallel or worksharing.
2913 // __kmpc_end_taskgroup: Wait until all tasks generated by the current task
2943 // mark task as waiting not on a barrier
3003 // finishing task reduction on parallel
3007 // finalize task reduction:
3022 // finishing task reduction on worksharing
3038 // finishing task reduction on taskgroup
3042 // Restore parent taskgroup for the current task
3046 KA_TRACE(10, ("__kmpc_end_taskgroup(exit): T#%d task %p finished waiting\n",
3061 kmp_task_t *task = NULL;
3072 // decrement num_tasks to "reserve" one task to get for execution
3083 // We got a "ticket" to get a "reserved" priority task
3108 // The TSC does not allow to steal victim task
3110 KA_TRACE(20, ("__kmp_get_priority_task(exit #3): T#%d could not get task "
3118 // walk through the deque trying to steal any task
3124 break; // found task to execute
3133 10, ("__kmp_get_priority_task(exit #4): T#%d could not get task from "
3154 task = KMP_TASKDATA_TO_TASK(taskdata);
3155 return task;
3158 // __kmp_remove_my_task: remove a task from my own deque
3162 kmp_task_t *task;
3204 // The TSC does not allow to steal victim task
3207 ("__kmp_remove_my_task(exit #3): T#%d TSC blocks tail task: "
3219 KA_TRACE(10, ("__kmp_remove_my_task(exit #4): T#%d task %p removed: "
3224 task = KMP_TASKDATA_TO_TASK(taskdata);
3225 return task;
3228 // __kmp_steal_task: remove a task from another thread's deque
3236 kmp_task_t *task;
3291 // The TSC does not allow to steal victim task
3300 // walk through victim's deque trying to steal any task
3307 break; // found victim task
3353 ("__kmp_steal_task(exit #5): T#%d stole task %p from T#%d: "
3358 task = KMP_TASKDATA_TO_TASK(taskdata);
3359 return task;
3369 // spinner == NULL means only execute a single task and return.
3378 kmp_task_t *task;
3406 while (1) { // Inner loop to find a task and execute it
3407 task = NULL;
3408 if (task_team->tt.tt_num_task_pri) { // get priority task first
3409 task = __kmp_get_priority_task(gtid, task_team, is_constrained);
3411 if (task == NULL && use_own_tasks) { // check own queue next
3412 task = __kmp_remove_my_task(thread, gtid, task_team, is_constrained);
3414 if ((task == NULL) && (nthreads > 1)) { // Steal a task finally
3454 // There is a slight possibility that it resumes, steals a task
3465 task =
3469 if (task != NULL) { // set last stolen to victim
3483 if (task == NULL)
3486 // Found a task; execute it
3496 __kmp_invoke_task(gtid, task, current_task);
3516 KMP_YIELD(__kmp_library == library_throughput); // Yield before next task
3517 // If execution of a stolen task results in more tasks being placed on our
3520 KA_TRACE(20, ("__kmp_execute_tasks_template: T#%d stolen task spawned "
3528 // The task source has been exhausted. If in final spin loop of barrier,
3561 // If this thread's task team is NULL, primary thread has recognized that
3570 // into infinite loop when a if0 task depends on a hidden helper task
3573 // task so it is in another code path that has the same check.
3658 // __kmp_enable_tasking: Allocate task team and resume threads sleeping at the
3660 // First thread in allocates the task team atomically.
3707 // is used for task stealing) and awakens them if they are.
3727 * Utility routines for "task teams". A task team (kmp_task_t) is kind of
3758 // Lock for task team data structures
3763 // Allocates a task deque for a particular thread, and initialize the necessary
3765 // per task team since task teams are recycled. No lock is needed during
3772 // Initialize last stolen task field to "none"
3783 // Allocate space for task deque, and zero the deque
3792 // Deallocates a task deque for a particular thread. Happens at library
3812 // Allocates a threads_data array for a task team, either by allocating an
3833 // All threads must lock when they encounter the first task of the implicit
3834 // task region to make sure threads_data fields are (re)initialized before
3920 // Deallocates a threads_data array for a task team, including any attached
3956 // Only need to init if task team is isn't active or team size changed
3968 // Allocates a task team associated with a specific team, taking it from
3969 // the global task team free list if possible. Also initializes data
3979 // Take a task team from the task team pool
3991 "task team for team %p\n",
3993 // Allocate a new task team if one is not available. Cannot use
4025 // Frees the task team associated with a specific thread, and adds it
4026 // to the global task team free list.
4031 // Put task team back on free list
4042 // Free all the task teams on the task team free list.
4069 // View the array of two task team pointers as a pair of pointers:
4072 // Serial teams can create a stack of task teams for nested serial teams.
4085 // Serial team pops a task team off the stack
4105 // trying to steal tasks. Wait for each thread to unreference its task team.
4172 // For the serial and root teams, setup the first task team pointer to point
4173 // to task team. The other pointer is a stack of task teams from previous
4192 // If it exists, it is the current task team and shouldn't be touched yet as
4218 } else { // Leave the old task team struct in place for the upcoming region;
4230 // For regular thread, task enabling should be called when the task is going
4251 // __kmp_task_team_sync: Propagation of task team data from team to threads
4263 // It is now safe to propagate the task team pointer from the team struct to
4268 ("__kmp_task_team_sync: Thread T#%d task team switched to task_team "
4302 // Deactivate the old task team, so that the worker threads will stop
4354 // __kmp_give_task puts a task into a given thread queue if:
4359 static bool __kmp_give_task(kmp_info_t *thread, kmp_int32 tid, kmp_task_t *task,
4361 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
4364 KA_TRACE(20, ("__kmp_give_task: trying to give task %p to thread %d.\n",
4377 ("__kmp_give_task: thread %d has no queue while giving task %p.\n",
4386 ("__kmp_give_task: queue is full while giving task %p to thread %d.\n",
4397 // expand deque to push the task which is not allowed to execute
4407 KA_TRACE(30, ("__kmp_give_task: queue is full while giving task %p to "
4430 KA_TRACE(30, ("__kmp_give_task: successfully gave task %p to thread %d.\n",
4444 In order to run the bottom half the task gets queued back into one of the
4452 This creates a second race as the bottom half can free the task before the
4454 td_incomplete_child_task of the proxy task to synchronize the top and bottom
4462 taskdata->td_flags.complete = 1; // mark the task as completed
4470 // Create an imaginary children for this task so the bottom half cannot
4471 // release the task before we have completed the second top half
4511 Execute the completion of a proxy task from a thread of that is part of the
4518 10, ("__kmp_proxy_task_completed(enter): T#%d proxy task %p completing\n",
4528 ("__kmp_proxy_task_completed(exit): T#%d proxy task %p completing\n",
4536 // Enqueue task to complete bottom half completion from a thread within the
4560 // awake at least one thread to execute given task
4575 Execute the completion of a proxy task from a thread that could not belong to
4584 ("__kmp_proxy_task_completed_ooo(enter): proxy task completing ooo %p\n",
4597 ("__kmp_proxy_task_completed_ooo(exit): proxy task completing ooo %p\n",
4602 kmp_task_t *task) {
4603 kmp_taskdata_t *td = KMP_TASK_TO_TASKDATA(task);
4606 td->td_allow_completion_event.ed.task = task;
4614 kmp_task_t *ptask = event->ed.task;
4619 // The associated task might have completed or could be completing at this
4638 // We free ptask afterwards and know the task is finished,
4643 // If the task detached complete the proxy task
4659 // __kmp_task_dup_alloc: Allocate the taskdata and make a copy of source task
4663 // task_src: pointer to source task to be duplicated
4665 // indicating whether we need to update task->td_task_id
4666 // returns: a pointer to the allocated kmp_task_t structure (task).
4672 kmp_task_t *task;
4675 kmp_taskdata_t *parent_task = taskdata_src->td_parent; // same parent task
4679 KA_TRACE(10, ("__kmp_task_dup_alloc(enter): Th %p, source task %p\n", thread,
4682 TASK_FULL); // it should not be proxy task
4696 task = KMP_TASKDATA_TO_TASK(taskdata);
4698 // Initialize new task (only specific fields not affected by memcpy)
4708 if (task->shareds != NULL) { // need setup shareds pointer
4710 task->shareds = &((char *)taskdata)[shareds_offset];
4711 KMP_DEBUG_ASSERT((((kmp_uintptr_t)task->shareds) & (sizeof(void *) - 1)) ==
4716 // task inherits the taskgroup from the parent task
4718 // tied task needs to initialize the td_last_tied at creation,
4723 // Only need to keep track of child task counts if team parallel and tasking
4736 ("__kmp_task_dup_alloc(exit): Th %p, created task %p, parent=%p\n",
4742 return task;
4747 // (used to form taskloop tasks from pattern task)
4748 // Parameters: dest task, src task, lastprivate flag.
4753 // class to encapsulate manipulating loop bounds in a taskloop task.
4757 kmp_task_t *task;
4764 : task(_task), taskdata(KMP_TASK_TO_TASKDATA(task)),
4765 lower_offset((char *)lb - (char *)task),
4766 upper_offset((char *)ub - (char *)task) {
4771 : task(_task), taskdata(KMP_TASK_TO_TASKDATA(_task)),
4778 // Intel task just returns the lower bound normally
4780 retval = *(kmp_int64 *)((char *)task + lower_offset);
4782 // GOMP task has to take into account the sizeof(long)
4784 kmp_int32 *lb = RCAST(kmp_int32 *, task->shareds);
4787 kmp_int64 *lb = RCAST(kmp_int64 *, task->shareds);
4793 retval = *(kmp_int64 *)((char *)task + lower_offset);
4800 // Intel task just returns the upper bound normally
4802 retval = *(kmp_int64 *)((char *)task + upper_offset);
4804 // GOMP task has to take into account the sizeof(long)
4806 kmp_int32 *ub = RCAST(kmp_int32 *, task->shareds) + 1;
4809 kmp_int64 *ub = RCAST(kmp_int64 *, task->shareds) + 1;
4814 retval = *(kmp_int64 *)((char *)task + upper_offset);
4820 // Intel task just sets the lower bound normally
4822 *(kmp_uint64 *)((char *)task + lower_offset) = lb;
4824 // GOMP task has to take into account the sizeof(long)
4826 kmp_uint32 *lower = RCAST(kmp_uint32 *, task->shareds);
4829 kmp_uint64 *lower = RCAST(kmp_uint64 *, task->shareds);
4834 *(kmp_uint64 *)((char *)task + lower_offset) = lb;
4839 // Intel task just sets the upper bound normally
4841 *(kmp_uint64 *)((char *)task + upper_offset) = ub;
4843 // GOMP task has to take into account the sizeof(long)
4845 kmp_uint32 *upper = RCAST(kmp_uint32 *, task->shareds) + 1;
4848 kmp_uint64 *upper = RCAST(kmp_uint64 *, task->shareds) + 1;
4853 *(kmp_uint64 *)((char *)task + upper_offset) = ub;
4862 // task Pattern task, exposes the loop iteration range
4863 // lb Pointer to loop lower bound in task structure
4864 // ub Pointer to loop upper bound in task structure
4868 // grainsize Number of loop iterations per task
4870 // last_chunk Reduction of grainsize for last task
4874 void __kmp_taskloop_linear(ident_t *loc, int gtid, kmp_task_t *task,
4887 kmp_taskloop_bounds_t task_bounds(task, lb, ub);
4905 // Launch num_tasks tasks, assign grainsize iterations each task
4919 // schedule the last task, set lastprivate flag if needed
4936 next_task = __kmp_task_dup_alloc(thread, task, /* taskloop_recur */ 0);
4938 next_task = __kmp_task_dup_alloc(thread, task); // allocate new task
4945 // adjust task-specific bounds
4954 ptask_dup(next_task, task, lastpriv);
4956 ("__kmp_taskloop_linear: T#%d; task #%llu: task %p: lower %lld, "
4963 codeptr_ra); // schedule new task
4971 __kmp_omp_task(gtid, next_task, true); // schedule new task
4975 // free the pattern task and exit
4976 __kmp_task_start(gtid, task, current_task); // make internal bookkeeping
4977 // do not execute the pattern task, just do internal bookkeeping
4978 __kmp_task_finish<false>(gtid, task, current_task);
4981 // Structure to keep taskloop parameters for auxiliary task
4982 // kept in the shareds of the task structure.
4984 kmp_task_t *task;
5010 // Execute part of the taskloop submitted as a task.
5014 kmp_task_t *task = p->task;
5031 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
5032 KMP_DEBUG_ASSERT(task != NULL);
5034 ("__kmp_taskloop_task: T#%d, task %p: %lld tasks, grainsize"
5041 __kmp_taskloop_recur(NULL, gtid, task, lb, ub, st, ub_glob, num_tasks,
5048 __kmp_taskloop_linear(NULL, gtid, task, lb, ub, st, ub_glob, num_tasks,
5059 // Schedule part of the taskloop as a task,
5064 // task Pattern task, exposes the loop iteration range
5065 // lb Pointer to loop lower bound in task structure
5066 // ub Pointer to loop upper bound in task structure
5070 // grainsize Number of loop iterations per task
5072 // last_chunk Reduction of grainsize for last task
5077 void __kmp_taskloop_recur(ident_t *loc, int gtid, kmp_task_t *task,
5087 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
5088 KMP_DEBUG_ASSERT(task != NULL);
5091 ("__kmp_taskloop_recur: T#%d, task %p: %lld tasks, grainsize"
5101 (char *)lb - (char *)task; // remember offset of lb in the task structure
5103 (char *)ub - (char *)task; // remember offset of ub in the task structure
5115 kmp_uint64 n_tsk1 = num_tasks - n_tsk0; // to schedule as a task
5136 // create pattern task for 2nd half of the loop
5138 next_task = __kmp_task_dup_alloc(thread, task,
5141 next_task = __kmp_task_dup_alloc(thread, task); // duplicate the task
5146 ptask_dup(next_task, task, 0);
5149 // create auxiliary task for 2nd half of the loop
5150 // make sure new task has same parent task as the pattern task
5156 // restore current task
5159 p->task = next_task;
5182 // schedule new task with correct return address for OMPT events
5185 __kmp_omp_task(gtid, new_task, true); // schedule new task
5190 __kmp_taskloop_recur(loc, gtid, task, lb, ub, st, ub_glob, n_tsk0, gr_size0,
5197 __kmp_taskloop_linear(loc, gtid, task, lb, ub, st, ub_glob, n_tsk0,
5207 static void __kmp_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int if_val,
5211 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
5212 KMP_DEBUG_ASSERT(task != NULL);
5225 kmp_taskloop_bounds_t task_bounds(task, lb, ub);
5233 0; // reduce grainsize of last task by last_chunk in strict mode
5238 KA_TRACE(20, ("__kmp_taskloop: T#%d, task %p, lb %lld, ub %lld, st %lld, "
5253 // free the pattern task and exit
5254 __kmp_task_start(gtid, task, current_task);
5256 __kmp_task_finish<false>(gtid, task, current_task);
5322 if (if_val == 0) { // if(0) specified, mark task as serial
5324 taskdata->td_flags.tiedness = TASK_TIED; // AC: serial task cannot be untied
5326 __kmp_taskloop_linear(loc, gtid, task, lb, ub, st, ub_glob, num_tasks,
5339 __kmp_taskloop_recur(loc, gtid, task, lb, ub, st, ub_glob, num_tasks,
5350 __kmp_taskloop_linear(loc, gtid, task, lb, ub, st, ub_glob, num_tasks,
5379 @param task Task structure
5381 @param lb Pointer to loop lower bound in task structure
5382 @param ub Pointer to loop upper bound in task structure
5391 void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int if_val,
5396 __kmp_taskloop(loc, gtid, task, if_val, lb, ub, st, nogroup, sched, grainsize,
5405 @param task Task structure
5407 @param lb Pointer to loop lower bound in task structure
5408 @param ub Pointer to loop upper bound in task structure
5418 void __kmpc_taskloop_5(ident_t *loc, int gtid, kmp_task_t *task, int if_val,
5424 __kmp_taskloop(loc, gtid, task, if_val, lb, ub, st, nogroup, sched, grainsize,
5432 @return Returns a pointer to the thread's current task async handle. If no task
5435 Acqurires a pointer to the target async handle from the current task.
5453 @return Returns TRUE if the current task being executed of the given thread has
5454 a task team allocated to it. Otherwise, returns FALSE.
5456 Checks if the current thread has a task team.
5546 kmp_taskdata_t *td = KMP_TASK_TO_TASKDATA(this_record_map[j].task);
5563 // If the parent doesnt have a taskgroup, remove it from the task
5571 __kmp_omp_task(gtid, this_record_map[this_root_tasks[j]].task, true);
5604 this_record_map[i].task = nullptr;
5616 // the beginning of the record process of a task region