1 /* Copyright (C) 2005-2019 Free Software Foundation, Inc. 2 Contributed by Richard Henderson <rth@redhat.com>. 3 4 This file is part of the GNU Offloading and Multi Processing Library 5 (libgomp). 6 7 Libgomp is free software; you can redistribute it and/or modify it 8 under the terms of the GNU General Public License as published by 9 the Free Software Foundation; either version 3, or (at your option) 10 any later version. 11 12 Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY 13 WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 14 FOR A PARTICULAR PURPOSE. See the GNU General Public License for 15 more details. 16 17 Under Section 7 of GPL version 3, you are granted additional 18 permissions described in the GCC Runtime Library Exception, version 19 3.1, as published by the Free Software Foundation. 20 21 You should have received a copy of the GNU General Public License and 22 a copy of the GCC Runtime Library Exception along with this program; 23 see the files COPYING3 and COPYING.RUNTIME respectively. If not, see 24 <http://www.gnu.org/licenses/>. */ 25 26 /* This is a Linux specific implementation of a barrier synchronization 27 mechanism for libgomp. This type is private to the library. This 28 implementation uses atomic instructions and the futex syscall. */ 29 30 #include <limits.h> 31 #include "wait.h" 32 33 34 void 35 gomp_barrier_wait_end (gomp_barrier_t *bar, gomp_barrier_state_t state) 36 { 37 if (__builtin_expect (state & BAR_WAS_LAST, 0)) 38 { 39 /* Next time we'll be awaiting TOTAL threads again. */ 40 bar->awaited = bar->total; 41 __atomic_store_n (&bar->generation, bar->generation + BAR_INCR, 42 MEMMODEL_RELEASE); 43 futex_wake ((int *) &bar->generation, INT_MAX); 44 } 45 else 46 { 47 do 48 do_wait ((int *) &bar->generation, state); 49 while (__atomic_load_n (&bar->generation, MEMMODEL_ACQUIRE) == state); 50 } 51 } 52 53 void 54 gomp_barrier_wait (gomp_barrier_t *bar) 55 { 56 gomp_barrier_wait_end (bar, gomp_barrier_wait_start (bar)); 57 } 58 59 /* Like gomp_barrier_wait, except that if the encountering thread 60 is not the last one to hit the barrier, it returns immediately. 61 The intended usage is that a thread which intends to gomp_barrier_destroy 62 this barrier calls gomp_barrier_wait, while all other threads 63 call gomp_barrier_wait_last. When gomp_barrier_wait returns, 64 the barrier can be safely destroyed. */ 65 66 void 67 gomp_barrier_wait_last (gomp_barrier_t *bar) 68 { 69 gomp_barrier_state_t state = gomp_barrier_wait_start (bar); 70 if (state & BAR_WAS_LAST) 71 gomp_barrier_wait_end (bar, state); 72 } 73 74 void 75 gomp_team_barrier_wake (gomp_barrier_t *bar, int count) 76 { 77 futex_wake ((int *) &bar->generation, count == 0 ? INT_MAX : count); 78 } 79 80 void 81 gomp_team_barrier_wait_end (gomp_barrier_t *bar, gomp_barrier_state_t state) 82 { 83 unsigned int generation, gen; 84 85 if (__builtin_expect (state & BAR_WAS_LAST, 0)) 86 { 87 /* Next time we'll be awaiting TOTAL threads again. */ 88 struct gomp_thread *thr = gomp_thread (); 89 struct gomp_team *team = thr->ts.team; 90 91 bar->awaited = bar->total; 92 team->work_share_cancelled = 0; 93 if (__builtin_expect (team->task_count, 0)) 94 { 95 gomp_barrier_handle_tasks (state); 96 state &= ~BAR_WAS_LAST; 97 } 98 else 99 { 100 state &= ~BAR_CANCELLED; 101 state += BAR_INCR - BAR_WAS_LAST; 102 __atomic_store_n (&bar->generation, state, MEMMODEL_RELEASE); 103 futex_wake ((int *) &bar->generation, INT_MAX); 104 return; 105 } 106 } 107 108 generation = state; 109 state &= ~BAR_CANCELLED; 110 do 111 { 112 do_wait ((int *) &bar->generation, generation); 113 gen = __atomic_load_n (&bar->generation, MEMMODEL_ACQUIRE); 114 if (__builtin_expect (gen & BAR_TASK_PENDING, 0)) 115 { 116 gomp_barrier_handle_tasks (state); 117 gen = __atomic_load_n (&bar->generation, MEMMODEL_ACQUIRE); 118 } 119 generation |= gen & BAR_WAITING_FOR_TASK; 120 } 121 while (gen != state + BAR_INCR); 122 } 123 124 void 125 gomp_team_barrier_wait (gomp_barrier_t *bar) 126 { 127 gomp_team_barrier_wait_end (bar, gomp_barrier_wait_start (bar)); 128 } 129 130 void 131 gomp_team_barrier_wait_final (gomp_barrier_t *bar) 132 { 133 gomp_barrier_state_t state = gomp_barrier_wait_final_start (bar); 134 if (__builtin_expect (state & BAR_WAS_LAST, 0)) 135 bar->awaited_final = bar->total; 136 gomp_team_barrier_wait_end (bar, state); 137 } 138 139 bool 140 gomp_team_barrier_wait_cancel_end (gomp_barrier_t *bar, 141 gomp_barrier_state_t state) 142 { 143 unsigned int generation, gen; 144 145 if (__builtin_expect (state & BAR_WAS_LAST, 0)) 146 { 147 /* Next time we'll be awaiting TOTAL threads again. */ 148 /* BAR_CANCELLED should never be set in state here, because 149 cancellation means that at least one of the threads has been 150 cancelled, thus on a cancellable barrier we should never see 151 all threads to arrive. */ 152 struct gomp_thread *thr = gomp_thread (); 153 struct gomp_team *team = thr->ts.team; 154 155 bar->awaited = bar->total; 156 team->work_share_cancelled = 0; 157 if (__builtin_expect (team->task_count, 0)) 158 { 159 gomp_barrier_handle_tasks (state); 160 state &= ~BAR_WAS_LAST; 161 } 162 else 163 { 164 state += BAR_INCR - BAR_WAS_LAST; 165 __atomic_store_n (&bar->generation, state, MEMMODEL_RELEASE); 166 futex_wake ((int *) &bar->generation, INT_MAX); 167 return false; 168 } 169 } 170 171 if (__builtin_expect (state & BAR_CANCELLED, 0)) 172 return true; 173 174 generation = state; 175 do 176 { 177 do_wait ((int *) &bar->generation, generation); 178 gen = __atomic_load_n (&bar->generation, MEMMODEL_ACQUIRE); 179 if (__builtin_expect (gen & BAR_CANCELLED, 0)) 180 return true; 181 if (__builtin_expect (gen & BAR_TASK_PENDING, 0)) 182 { 183 gomp_barrier_handle_tasks (state); 184 gen = __atomic_load_n (&bar->generation, MEMMODEL_ACQUIRE); 185 } 186 generation |= gen & BAR_WAITING_FOR_TASK; 187 } 188 while (gen != state + BAR_INCR); 189 190 return false; 191 } 192 193 bool 194 gomp_team_barrier_wait_cancel (gomp_barrier_t *bar) 195 { 196 return gomp_team_barrier_wait_cancel_end (bar, gomp_barrier_wait_start (bar)); 197 } 198 199 void 200 gomp_team_barrier_cancel (struct gomp_team *team) 201 { 202 gomp_mutex_lock (&team->task_lock); 203 if (team->barrier.generation & BAR_CANCELLED) 204 { 205 gomp_mutex_unlock (&team->task_lock); 206 return; 207 } 208 team->barrier.generation |= BAR_CANCELLED; 209 gomp_mutex_unlock (&team->task_lock); 210 futex_wake ((int *) &team->barrier.generation, INT_MAX); 211 } 212