Lines Matching +full:wait +full:- +full:state
2 * kmp_wait_release.h -- Wait/Release implementation
5 //===----------------------------------------------------------------------===//
9 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
11 //===----------------------------------------------------------------------===//
20 #include "ompt-specific.h"
24 @defgroup WAIT_RELEASE Wait/Release operations
138 /*! Base class for wait/release volatile flag */
151 this->waiting_threads[0] = thr;
167 if (Sleepable && !(this->sleepLoc))
168 return (traits_type::tcr(*(this->get())) & ~KMP_BARRIER_SLEEP_STATE) ==
171 return traits_type::tcr(*(this->get())) == checker;
182 return traits_type::tcr(*(this->get())) != checker;
185 * Trigger all waiting threads to run by modifying flag to release state. */
187 (void)traits_type::test_then_add4((volatile PtrType *)this->get());
193 if (this->sleepLoc) {
194 this->sleepLoc->store(true);
195 return *(this->get());
197 return traits_type::test_then_or((volatile PtrType *)this->get(),
204 if (this->sleepLoc) {
205 this->sleepLoc->store(false);
208 traits_type::test_then_and((volatile PtrType *)this->get(),
214 if (this->sleepLoc)
215 return this->sleepLoc->load();
220 if (this->sleepLoc)
221 return this->sleepLoc->load();
222 return is_sleeping_val(*(this->get()));
225 if (this->sleepLoc)
226 return this->sleepLoc->load();
227 return is_sleeping_val(*(this->get()));
232 /*! Base class for wait/release atomic flag */
236 std::atomic<PtrType> *loc; /**< Pointer to flag location to wait on */
244 this->waiting_threads[0] = thr;
257 PtrType load() { return loc->load(std::memory_order_acquire); }
259 void store(PtrType val) { loc->store(val, std::memory_order_release); }
262 if (Sleepable && !(this->sleepLoc))
263 return (this->load() & ~KMP_BARRIER_SLEEP_STATE) == checker;
265 return this->load() == checker;
275 bool notdone_check() { return this->load() != checker; }
277 * Trigger all waiting threads to run by modifying flag to release state. */
278 void internal_release() { KMP_ATOMIC_ADD(this->get(), 4); }
283 if (this->sleepLoc) {
284 this->sleepLoc->store(true);
285 return *(this->get());
287 return KMP_ATOMIC_OR(this->get(), KMP_BARRIER_SLEEP_STATE);
293 if (this->sleepLoc) {
294 this->sleepLoc->store(false);
297 KMP_ATOMIC_AND(this->get(), ~KMP_BARRIER_SLEEP_STATE);
302 if (this->sleepLoc)
303 return this->sleepLoc->load();
308 if (this->sleepLoc)
309 return this->sleepLoc->load();
310 return is_sleeping_val(this->load());
313 if (this->sleepLoc)
314 return this->sleepLoc->load();
315 return is_sleeping_val(this->load());
325 int ds_tid = this_thr->th.th_info.ds.ds_tid;
328 this_thr->th.ompt_thread_info.state = ompt_state_overhead;
332 if (this_thr->th.ompt_thread_info.parallel_flags & ompt_parallel_league)
345 int flags = this_thr->th.ompt_thread_info.parallel_flags;
351 // return to idle state
352 this_thr->th.ompt_thread_info.state = ompt_state_idle;
354 this_thr->th.ompt_thread_info.state = ompt_state_overhead;
360 /* Spin wait loop that first does pause/yield, then sleep. A thread that calls
371 volatile void *spin = flag->get();
385 if (flag->done_check()) {
389 th_gtid = this_thr->th.th_info.ds.ds_gtid;
391 kmp_team_t *team = this_thr->th.th_team;
392 if (team && team->t.t_cancel_request == cancel_parallel)
397 KMP_ATOMIC_ST_REL(&this_thr->th.th_blocking, true);
410 In these cases, we don't change the state or trigger events in THIS
414 state := ompt_state_overhead
415 barrier-begin
416 barrier-wait-begin
417 state := ompt_state_wait_barrier
418 call join-barrier-implementation (finally arrive here)
420 call fork-barrier-implementation (finally arrive here)
422 state := ompt_state_overhead
423 barrier-wait-end
424 barrier-end
425 state := ompt_state_work_parallel
429 call fork-barrier-implementation (finally arrive here)
430 {} // worker arrive here with state = ompt_state_idle
434 state := ompt_state_barrier_implicit
435 barrier-begin
436 barrier-wait-begin
437 call join-barrier-implementation (finally arrive here
442 call fork-barrier-implementation (finally arrive here final_spin=TRUE)
444 Worker after task-team is finished:
445 barrier-wait-end
446 barrier-end
447 implicit-task-end
448 idle-begin
449 state := ompt_state_idle
451 Before leaving, if state = ompt_state_idle
452 idle-end
453 state := ompt_state_overhead
459 ompt_entry_state = this_thr->th.ompt_thread_info.state;
463 KMP_MASTER_TID(this_thr->th.th_info.ds.ds_tid)) {
465 if (this_thr->th.th_team)
466 team = this_thr->th.th_team->t.ompt_serialized_team_info;
468 tId = &(team->ompt_task_info.task_data);
473 tId = &(this_thr->th.ompt_thread_info.task_data);
476 this_thr->th.th_task_team == NULL)) {
477 // implicit task is done. Either no taskqueue, or task-team finished
493 (__kmp_zero_bt && !this_thr->th.th_team_bt_set))
498 hibernate = this_thr->th.th_team_bt_intervals;
500 hibernate = this_thr->th.th_team_bt_intervals;
503 /* If the blocktime is nonzero, we want to make sure that we spin wait for
514 hibernate - __kmp_global.g.g_time.dt.t_value));
520 hibernate_goal = KMP_NOW() + this_thr->th.th_team_bt_intervals;
528 // Main wait spin loop
529 while (flag->notdone_check()) {
532 task_team = this_thr->th.th_task_team;
534 1) A newly-created thread is first being released by
541 if (TCR_SYNC_4(task_team->tt.tt_active)) {
543 flag->execute_tasks(
547 this_thr->th.th_reap_state = KMP_SAFE_TO_REAP;
549 KMP_DEBUG_ASSERT(!KMP_MASTER_TID(this_thr->th.th_info.ds.ds_tid));
551 // task-team is done now, other cases should be catched above
555 this_thr->th.th_task_team = NULL;
556 this_thr->th.th_reap_state = KMP_SAFE_TO_REAP;
559 this_thr->th.th_reap_state = KMP_SAFE_TO_REAP;
575 // Check if thread has been signalled to idle state
576 // This indicates that the logical "join-barrier" has finished
577 if (this_thr->th.th_stats->isIdle() &&
583 // Check if the barrier surrounding this wait loop has been cancelled
585 kmp_team_t *team = this_thr->th.th_team;
586 if (team && team->t.t_cancel_request == cancel_parallel)
591 // thread has not released the barrier. We cannot wait here because once the
616 if (task_team != NULL && TCR_4(task_team->tt.tt_found_tasks) &&
628 // Don't suspend if wait loop designated non-sleepable
636 flag->mwait(th_gtid);
642 KMP_ATOMIC_ST_REL(&this_thr->th.th_blocking, false);
644 flag->suspend(th_gtid);
647 KMP_ATOMIC_ST_REL(&this_thr->th.th_blocking, true);
658 this_thr->th.th_reap_state == KMP_SAFE_TO_REAP) {
659 this_thr->th.th_reap_state = KMP_NOT_SAFE_TO_REAP;
665 ompt_state_t ompt_exit_state = this_thr->th.ompt_thread_info.state;
670 ompt_exit_state = this_thr->th.ompt_thread_info.state;
674 this_thr->th.ompt_thread_info.state = ompt_state_overhead;
679 // If we were put into idle state, pop that off the state stack
683 this_thr->th.th_stats->resetIdleFlag();
689 KMP_ATOMIC_ST_REL(&this_thr->th.th_blocking, false);
693 kmp_team_t *team = this_thr->th.th_team;
694 if (team && team->t.t_cancel_request == cancel_parallel) {
698 kmp_task_team_t *task_team = this_thr->th.th_task_team;
700 &(task_team->tt.tt_unfinished_threads);
710 // Set up a monitor on the flag variable causing the calling thread to wait in
711 // a less active state until the flag variable is modified.
718 flag->get()));
720 // User-level mwait is available
726 volatile void *spin = flag->get();
727 void *cacheline = (void *)(kmp_uintptr_t(spin) & ~(CACHE_LINE - 1));
729 if (!flag->done_check()) {
731 th->th.th_active = FALSE;
732 if (th->th.th_active_in_pool) {
733 th->th.th_active_in_pool = FALSE;
737 flag->set_sleeping();
751 if (flag->done_check())
752 flag->unset_sleeping();
754 // if flag changes here, wake-up happens immediately
755 TCW_PTR(th->th.th_sleep_loc, (void *)flag);
756 th->th.th_sleep_loc_type = flag->get_type();
771 if (flag->is_sleeping())
772 flag->unset_sleeping();
773 TCW_PTR(th->th.th_sleep_loc, NULL);
774 th->th.th_sleep_loc_type = flag_unset;
777 th->th.th_active = TRUE;
778 if (TCR_4(th->th.th_in_pool)) {
780 th->th.th_active_in_pool = TRUE;
782 } // Drop out to main wait loop to check flag, handle tasks, etc.
794 int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1;
796 KF_TRACE(20, ("__kmp_release: T#%d releasing flag(%x)\n", gtid, flag->get()));
797 KMP_DEBUG_ASSERT(flag->get());
798 KMP_FSYNC_RELEASING(flag->get_void_p());
800 flag->internal_release();
802 KF_TRACE(100, ("__kmp_release: T#%d set new spin=%d\n", gtid, flag->get(),
803 flag->load()));
808 if (flag->is_any_sleeping()) {
809 for (unsigned int i = 0; i < flag->get_num_waiters(); ++i) {
811 kmp_info_t *waiter = flag->get_waiter(i);
813 int wait_gtid = waiter->th.th_info.ds.ds_gtid;
817 gtid, wait_gtid, flag->get()));
818 flag->resume(wait_gtid); // unsets flag's current_waiter when done
846 bool wait(kmp_info_t *this_thr,
882 bool wait(kmp_info_t *this_thr,
918 bool wait(kmp_info_t *this_thr,
933 // Hierarchical 64-bit on-core barrier instantiation
969 if (this_thr->th.th_bar[bt].bb.wait_flag == KMP_BARRIER_SWITCH_TO_OWN_FLAG)
974 this_thr->th.th_bar[bt].bb.wait_flag = KMP_BARRIER_SWITCHING;
975 kmp_flag_64<> flag(&this_thr->th.th_bar[bt].bb.b_go,
991 void wait(kmp_info_t *this_thr, int final_spin) {
1027 void *flag = CCAST(void *, thr->th.th_sleep_loc);
1028 flag_type type = thr->th.th_sleep_loc_type;