Lines Matching defs:thr_bar
381 kmp_bstate_t *thr_bar;
389 thr_bar = &this_thr->th.th_bar[bt].bb;
491 copy_icvs(&thr_bar->th_fixed_icvs,
512 copy_icvs(&thr_bar->th_fixed_icvs,
561 kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb;
584 team->t.t_id, 0, &thr_bar->b_arrived, thr_bar->b_arrived,
585 thr_bar->b_arrived + KMP_BARRIER_STATE_BUMP));
590 kmp_flag_64<> flag(&thr_bar->b_arrived, other_threads[0]);
662 kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb;
717 gtid, &thr_bar->b_go, KMP_BARRIER_STATE_BUMP));
719 kmp_flag_64<true, false> flag(&thr_bar->b_go, KMP_BARRIER_STATE_BUMP);
723 kmp_flag_64<> flag(&thr_bar->b_go, KMP_BARRIER_STATE_BUMP);
752 TCW_4(thr_bar->b_go, KMP_INIT_BARRIER_STATE);
755 gtid, team->t.t_id, tid, &thr_bar->b_go, KMP_INIT_BARRIER_STATE));
799 kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb;
875 team->t.t_id, parent_tid, &thr_bar->b_arrived, thr_bar->b_arrived,
876 thr_bar->b_arrived + KMP_BARRIER_STATE_BUMP));
882 kmp_flag_64<> flag(&thr_bar->b_arrived, other_threads[parent_tid]);
905 kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb;
916 &thr_bar->b_go, KMP_BARRIER_STATE_BUMP));
918 kmp_flag_64<> flag(&thr_bar->b_go, KMP_BARRIER_STATE_BUMP);
946 TCW_4(thr_bar->b_go, KMP_INIT_BARRIER_STATE);
949 team->t.t_id, tid, &thr_bar->b_go, KMP_INIT_BARRIER_STATE));
1011 kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb;
1035 kmp_flag_64<> p_flag(&thr_bar->b_arrived);
1049 team->t.t_id, parent_tid, &thr_bar->b_arrived,
1050 thr_bar->b_arrived,
1051 thr_bar->b_arrived + KMP_BARRIER_STATE_BUMP));
1130 kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb;
1152 copy_icvs(&thr_bar->th_fixed_icvs,
1158 &thr_bar->b_go, KMP_BARRIER_STATE_BUMP));
1160 kmp_flag_64<> flag(&thr_bar->b_go, KMP_BARRIER_STATE_BUMP);
1187 TCW_4(thr_bar->b_go, KMP_INIT_BARRIER_STATE);
1190 gtid, team->t.t_id, tid, &thr_bar->b_go, KMP_INIT_BARRIER_STATE));
1249 copy_icvs(&child_bar->th_fixed_icvs, &thr_bar->th_fixed_icvs);
1271 &thr_bar->th_fixed_icvs);
1291 kmp_bstate_t *thr_bar,
1295 bool uninitialized = thr_bar->team == NULL;
1296 bool team_changed = team != thr_bar->team;
1297 bool team_sz_changed = nproc != thr_bar->nproc;
1298 bool tid_changed = tid != thr_bar->old_tid;
1302 __kmp_get_hierarchy(nproc, thr_bar);
1306 thr_bar->my_level = thr_bar->depth - 1; // default for primary thread
1307 thr_bar->parent_tid = -1; // default for primary thread
1311 while (d < thr_bar->depth) { // find parent based on level of thread in
1314 if (d == thr_bar->depth - 2) { // reached level right below the primary
1315 thr_bar->parent_tid = 0;
1316 thr_bar->my_level = d;
1318 } else if ((rem = tid % thr_bar->skip_per_level[d + 1]) != 0) {
1321 thr_bar->parent_tid = tid - rem;
1322 thr_bar->my_level = d;
1328 __kmp_type_convert(7 - ((tid - thr_bar->parent_tid) /
1329 (thr_bar->skip_per_level[thr_bar->my_level])),
1330 &(thr_bar->offset));
1331 thr_bar->old_tid = tid;
1332 thr_bar->wait_flag = KMP_BARRIER_NOT_WAITING;
1333 thr_bar->team = team;
1334 thr_bar->parent_bar =
1335 &team->t.t_threads[thr_bar->parent_tid]->th.th_bar[bt].bb;
1338 thr_bar->team = team;
1339 thr_bar->parent_bar =
1340 &team->t.t_threads[thr_bar->parent_tid]->th.th_bar[bt].bb;
1344 thr_bar->nproc = nproc;
1345 thr_bar->leaf_kids = thr_bar->base_leaf_kids;
1346 if (thr_bar->my_level == 0)
1347 thr_bar->leaf_kids = 0;
1348 if (thr_bar->leaf_kids && (kmp_uint32)tid + thr_bar->leaf_kids + 1 > nproc)
1349 __kmp_type_convert(nproc - tid - 1, &(thr_bar->leaf_kids));
1350 thr_bar->leaf_state = 0;
1351 for (int i = 0; i < thr_bar->leaf_kids; ++i)
1352 ((char *)&(thr_bar->leaf_state))[7 - i] = 1;
1362 kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb;
1373 thr_bar->use_oncore_barrier = 1;
1375 thr_bar->use_oncore_barrier = 0; // Do not use oncore barrier when nested
1389 (void)__kmp_init_hierarchical_barrier_thread(bt, thr_bar, nproc, gtid, tid,
1392 if (thr_bar->my_level) { // not a leaf (my_level==0 means leaf)
1397 thr_bar->use_oncore_barrier) {
1398 if (thr_bar->leaf_kids) {
1402 ? thr_bar->b_arrived | thr_bar->leaf_state
1403 : team->t.t_bar[bt].b_arrived | thr_bar->leaf_state;
1407 kmp_flag_64<> flag(&thr_bar->b_arrived, leaf_state);
1412 for (child_tid = tid + 1; child_tid <= tid + thr_bar->leaf_kids;
1425 KMP_TEST_THEN_AND64(&thr_bar->b_arrived, ~(thr_bar->leaf_state));
1428 for (kmp_uint32 d = 1; d < thr_bar->my_level;
1430 kmp_uint32 last = tid + thr_bar->skip_per_level[d + 1],
1431 skip = thr_bar->skip_per_level[d];
1457 for (kmp_uint32 d = 0; d < thr_bar->my_level;
1459 kmp_uint32 last = tid + thr_bar->skip_per_level[d + 1],
1460 skip = thr_bar->skip_per_level[d];
1493 __kmp_gtid_from_tid(thr_bar->parent_tid, team), team->t.t_id,
1494 thr_bar->parent_tid, &thr_bar->b_arrived, thr_bar->b_arrived,
1495 thr_bar->b_arrived + KMP_BARRIER_STATE_BUMP));
1499 if (thr_bar->my_level || __kmp_dflt_blocktime != KMP_MAX_BLOCKTIME ||
1500 !thr_bar->use_oncore_barrier) { // Parent is waiting on my b_arrived
1502 kmp_flag_64<> flag(&thr_bar->b_arrived,
1503 other_threads[thr_bar->parent_tid]);
1507 thr_bar->b_arrived = team->t.t_bar[bt].b_arrived + KMP_BARRIER_STATE_BUMP;
1508 kmp_flag_oncore flag(&thr_bar->parent_bar->b_arrived,
1509 thr_bar->offset + 1);
1510 flag.set_waiter(other_threads[thr_bar->parent_tid]);
1531 kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb;
1543 if (!thr_bar->use_oncore_barrier ||
1544 __kmp_dflt_blocktime != KMP_MAX_BLOCKTIME || thr_bar->my_level != 0 ||
1545 thr_bar->team == NULL) {
1547 thr_bar->wait_flag = KMP_BARRIER_OWN_FLAG;
1548 kmp_flag_64<> flag(&thr_bar->b_go, KMP_BARRIER_STATE_BUMP);
1550 TCW_8(thr_bar->b_go,
1555 thr_bar->wait_flag = KMP_BARRIER_PARENT_FLAG;
1556 kmp_flag_oncore flag(&thr_bar->parent_bar->b_go, KMP_BARRIER_STATE_BUMP,
1557 thr_bar->offset + 1, bt,
1560 if (thr_bar->wait_flag ==
1562 TCW_8(thr_bar->b_go,
1566 &(thr_bar->parent_bar->b_go)))[thr_bar->offset + 1] = 0;
1569 thr_bar->wait_flag = KMP_BARRIER_NOT_WAITING;
1581 gtid, team->t.t_id, tid, &thr_bar->b_go, KMP_INIT_BARRIER_STATE));
1596 thr_bar->use_oncore_barrier = 1;
1598 thr_bar->use_oncore_barrier = 0; // Do not use oncore barrier when nested
1602 unsigned short int old_leaf_kids = thr_bar->leaf_kids;
1603 kmp_uint64 old_leaf_state = thr_bar->leaf_state;
1604 team_change = __kmp_init_hierarchical_barrier_thread(bt, thr_bar, nproc, gtid,
1616 copy_icvs(&thr_bar->th_fixed_icvs,
1619 thr_bar->use_oncore_barrier) { // optimization for inf blocktime
1620 if (!thr_bar->my_level) // I'm a leaf in the hierarchy (my_level==0)
1624 &thr_bar->parent_bar->th_fixed_icvs);
1627 if (thr_bar->my_level) // not a leaf; copy ICVs to my fixed ICVs child can
1629 copy_icvs(&thr_bar->th_fixed_icvs, &thr_bar->parent_bar->th_fixed_icvs);
1632 &thr_bar->parent_bar->th_fixed_icvs);
1638 if (thr_bar->my_level) { // not a leaf
1642 thr_bar->use_oncore_barrier) {
1646 thr_bar->b_go = KMP_BARRIER_STATE_BUMP;
1649 ngo_load(&thr_bar->th_fixed_icvs);
1652 for (child_tid = thr_bar->skip_per_level[1]; child_tid < (int)nproc;
1653 child_tid += thr_bar->skip_per_level[1]) {
1665 ngo_store_go(&child_bar->th_fixed_icvs, &thr_bar->th_fixed_icvs);
1669 TCW_8(thr_bar->b_go,
1672 if (thr_bar->leaf_kids) { // if there are any
1675 old_leaf_kids < thr_bar->leaf_kids) { // some old, some new
1677 thr_bar->b_go |= old_leaf_state;
1680 last = tid + thr_bar->skip_per_level[1];
1700 thr_bar->b_go |= thr_bar->leaf_state;
1704 for (int d = thr_bar->my_level - 1; d >= 0;
1706 last = tid + thr_bar->skip_per_level[d + 1];
1707 kmp_uint32 skip = thr_bar->skip_per_level[d];
1729 &thr_bar->th_fixed_icvs);