Lines Matching defs:thr_bar
381 kmp_bstate_t *thr_bar;
389 thr_bar = &this_thr->th.th_bar[bt].bb;
492 copy_icvs(&thr_bar->th_fixed_icvs,
513 copy_icvs(&thr_bar->th_fixed_icvs,
562 kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb;
585 team->t.t_id, 0, &thr_bar->b_arrived, thr_bar->b_arrived,
586 thr_bar->b_arrived + KMP_BARRIER_STATE_BUMP));
591 kmp_flag_64<> flag(&thr_bar->b_arrived, other_threads[0]);
663 kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb;
718 gtid, &thr_bar->b_go, KMP_BARRIER_STATE_BUMP));
720 kmp_flag_64<true, false> flag(&thr_bar->b_go, KMP_BARRIER_STATE_BUMP);
724 kmp_flag_64<> flag(&thr_bar->b_go, KMP_BARRIER_STATE_BUMP);
753 TCW_4(thr_bar->b_go, KMP_INIT_BARRIER_STATE);
756 gtid, team->t.t_id, tid, &thr_bar->b_go, KMP_INIT_BARRIER_STATE));
800 kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb;
876 team->t.t_id, parent_tid, &thr_bar->b_arrived, thr_bar->b_arrived,
877 thr_bar->b_arrived + KMP_BARRIER_STATE_BUMP));
883 kmp_flag_64<> flag(&thr_bar->b_arrived, other_threads[parent_tid]);
906 kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb;
917 &thr_bar->b_go, KMP_BARRIER_STATE_BUMP));
919 kmp_flag_64<> flag(&thr_bar->b_go, KMP_BARRIER_STATE_BUMP);
947 TCW_4(thr_bar->b_go, KMP_INIT_BARRIER_STATE);
950 team->t.t_id, tid, &thr_bar->b_go, KMP_INIT_BARRIER_STATE));
1012 kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb;
1036 kmp_flag_64<> p_flag(&thr_bar->b_arrived);
1050 team->t.t_id, parent_tid, &thr_bar->b_arrived,
1051 thr_bar->b_arrived,
1052 thr_bar->b_arrived + KMP_BARRIER_STATE_BUMP));
1131 kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb;
1153 copy_icvs(&thr_bar->th_fixed_icvs,
1159 &thr_bar->b_go, KMP_BARRIER_STATE_BUMP));
1161 kmp_flag_64<> flag(&thr_bar->b_go, KMP_BARRIER_STATE_BUMP);
1188 TCW_4(thr_bar->b_go, KMP_INIT_BARRIER_STATE);
1191 gtid, team->t.t_id, tid, &thr_bar->b_go, KMP_INIT_BARRIER_STATE));
1250 copy_icvs(&child_bar->th_fixed_icvs, &thr_bar->th_fixed_icvs);
1272 &thr_bar->th_fixed_icvs);
1292 kmp_bstate_t *thr_bar,
1296 bool uninitialized = thr_bar->team == NULL;
1297 bool team_changed = team != thr_bar->team;
1298 bool team_sz_changed = nproc != thr_bar->nproc;
1299 bool tid_changed = tid != thr_bar->old_tid;
1303 __kmp_get_hierarchy(nproc, thr_bar);
1307 thr_bar->my_level = thr_bar->depth - 1; // default for primary thread
1308 thr_bar->parent_tid = -1; // default for primary thread
1312 while (d < thr_bar->depth) { // find parent based on level of thread in
1315 if (d == thr_bar->depth - 2) { // reached level right below the primary
1316 thr_bar->parent_tid = 0;
1317 thr_bar->my_level = d;
1319 } else if ((rem = tid % thr_bar->skip_per_level[d + 1]) != 0) {
1322 thr_bar->parent_tid = tid - rem;
1323 thr_bar->my_level = d;
1329 __kmp_type_convert(7 - ((tid - thr_bar->parent_tid) /
1330 (thr_bar->skip_per_level[thr_bar->my_level])),
1331 &(thr_bar->offset));
1332 thr_bar->old_tid = tid;
1333 thr_bar->wait_flag = KMP_BARRIER_NOT_WAITING;
1334 thr_bar->team = team;
1335 thr_bar->parent_bar =
1336 &team->t.t_threads[thr_bar->parent_tid]->th.th_bar[bt].bb;
1339 thr_bar->team = team;
1340 thr_bar->parent_bar =
1341 &team->t.t_threads[thr_bar->parent_tid]->th.th_bar[bt].bb;
1345 thr_bar->nproc = nproc;
1346 thr_bar->leaf_kids = thr_bar->base_leaf_kids;
1347 if (thr_bar->my_level == 0)
1348 thr_bar->leaf_kids = 0;
1349 if (thr_bar->leaf_kids && (kmp_uint32)tid + thr_bar->leaf_kids + 1 > nproc)
1350 __kmp_type_convert(nproc - tid - 1, &(thr_bar->leaf_kids));
1351 thr_bar->leaf_state = 0;
1352 for (int i = 0; i < thr_bar->leaf_kids; ++i)
1353 ((char *)&(thr_bar->leaf_state))[7 - i] = 1;
1363 kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb;
1374 thr_bar->use_oncore_barrier = 1;
1376 thr_bar->use_oncore_barrier = 0; // Do not use oncore barrier when nested
1390 (void)__kmp_init_hierarchical_barrier_thread(bt, thr_bar, nproc, gtid, tid,
1393 if (thr_bar->my_level) { // not a leaf (my_level==0 means leaf)
1398 thr_bar->use_oncore_barrier) {
1399 if (thr_bar->leaf_kids) {
1403 ? thr_bar->b_arrived | thr_bar->leaf_state
1404 : team->t.t_bar[bt].b_arrived | thr_bar->leaf_state;
1408 kmp_flag_64<> flag(&thr_bar->b_arrived, leaf_state);
1413 for (child_tid = tid + 1; child_tid <= tid + thr_bar->leaf_kids;
1426 KMP_TEST_THEN_AND64(&thr_bar->b_arrived, ~(thr_bar->leaf_state));
1429 for (kmp_uint32 d = 1; d < thr_bar->my_level;
1431 kmp_uint32 last = tid + thr_bar->skip_per_level[d + 1],
1432 skip = thr_bar->skip_per_level[d];
1458 for (kmp_uint32 d = 0; d < thr_bar->my_level;
1460 kmp_uint32 last = tid + thr_bar->skip_per_level[d + 1],
1461 skip = thr_bar->skip_per_level[d];
1494 __kmp_gtid_from_tid(thr_bar->parent_tid, team), team->t.t_id,
1495 thr_bar->parent_tid, &thr_bar->b_arrived, thr_bar->b_arrived,
1496 thr_bar->b_arrived + KMP_BARRIER_STATE_BUMP));
1500 if (thr_bar->my_level || __kmp_dflt_blocktime != KMP_MAX_BLOCKTIME ||
1501 !thr_bar->use_oncore_barrier) { // Parent is waiting on my b_arrived
1503 kmp_flag_64<> flag(&thr_bar->b_arrived,
1504 other_threads[thr_bar->parent_tid]);
1508 thr_bar->b_arrived = team->t.t_bar[bt].b_arrived + KMP_BARRIER_STATE_BUMP;
1509 kmp_flag_oncore flag(&thr_bar->parent_bar->b_arrived,
1510 thr_bar->offset + 1);
1511 flag.set_waiter(other_threads[thr_bar->parent_tid]);
1532 kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb;
1544 if (!thr_bar->use_oncore_barrier ||
1545 __kmp_dflt_blocktime != KMP_MAX_BLOCKTIME || thr_bar->my_level != 0 ||
1546 thr_bar->team == NULL) {
1548 thr_bar->wait_flag = KMP_BARRIER_OWN_FLAG;
1549 kmp_flag_64<> flag(&thr_bar->b_go, KMP_BARRIER_STATE_BUMP);
1551 TCW_8(thr_bar->b_go,
1556 thr_bar->wait_flag = KMP_BARRIER_PARENT_FLAG;
1557 kmp_flag_oncore flag(&thr_bar->parent_bar->b_go, KMP_BARRIER_STATE_BUMP,
1558 thr_bar->offset + 1, bt,
1561 if (thr_bar->wait_flag ==
1563 TCW_8(thr_bar->b_go,
1567 &(thr_bar->parent_bar->b_go)))[thr_bar->offset + 1] = 0;
1570 thr_bar->wait_flag = KMP_BARRIER_NOT_WAITING;
1582 gtid, team->t.t_id, tid, &thr_bar->b_go, KMP_INIT_BARRIER_STATE));
1597 thr_bar->use_oncore_barrier = 1;
1599 thr_bar->use_oncore_barrier = 0; // Do not use oncore barrier when nested
1603 unsigned short int old_leaf_kids = thr_bar->leaf_kids;
1604 kmp_uint64 old_leaf_state = thr_bar->leaf_state;
1605 team_change = __kmp_init_hierarchical_barrier_thread(bt, thr_bar, nproc, gtid,
1617 copy_icvs(&thr_bar->th_fixed_icvs,
1620 thr_bar->use_oncore_barrier) { // optimization for inf blocktime
1621 if (!thr_bar->my_level) // I'm a leaf in the hierarchy (my_level==0)
1625 &thr_bar->parent_bar->th_fixed_icvs);
1628 if (thr_bar->my_level) // not a leaf; copy ICVs to my fixed ICVs child can
1630 copy_icvs(&thr_bar->th_fixed_icvs, &thr_bar->parent_bar->th_fixed_icvs);
1633 &thr_bar->parent_bar->th_fixed_icvs);
1639 if (thr_bar->my_level) { // not a leaf
1643 thr_bar->use_oncore_barrier) {
1647 thr_bar->b_go = KMP_BARRIER_STATE_BUMP;
1650 ngo_load(&thr_bar->th_fixed_icvs);
1653 for (child_tid = thr_bar->skip_per_level[1]; child_tid < (int)nproc;
1654 child_tid += thr_bar->skip_per_level[1]) {
1666 ngo_store_go(&child_bar->th_fixed_icvs, &thr_bar->th_fixed_icvs);
1670 TCW_8(thr_bar->b_go,
1673 if (thr_bar->leaf_kids) { // if there are any
1676 old_leaf_kids < thr_bar->leaf_kids) { // some old, some new
1678 thr_bar->b_go |= old_leaf_state;
1681 last = tid + thr_bar->skip_per_level[1];
1701 thr_bar->b_go |= thr_bar->leaf_state;
1705 for (int d = thr_bar->my_level - 1; d >= 0;
1707 last = tid + thr_bar->skip_per_level[d + 1];
1708 kmp_uint32 skip = thr_bar->skip_per_level[d];
1730 &thr_bar->th_fixed_icvs);