1 /* Copyright (C) 2005-2020 Free Software Foundation, Inc. 2 Contributed by Richard Henderson <rth@redhat.com>. 3 4 This file is part of the GNU Offloading and Multi Processing Library 5 (libgomp). 6 7 Libgomp is free software; you can redistribute it and/or modify it 8 under the terms of the GNU General Public License as published by 9 the Free Software Foundation; either version 3, or (at your option) 10 any later version. 11 12 Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY 13 WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 14 FOR A PARTICULAR PURPOSE. See the GNU General Public License for 15 more details. 16 17 Under Section 7 of GPL version 3, you are granted additional 18 permissions described in the GCC Runtime Library Exception, version 19 3.1, as published by the Free Software Foundation. 20 21 You should have received a copy of the GNU General Public License and 22 a copy of the GCC Runtime Library Exception along with this program; 23 see the files COPYING3 and COPYING.RUNTIME respectively. If not, see 24 <http://www.gnu.org/licenses/>. */ 25 26 /* This file handles the maintenance of threads in response to team 27 creation and termination. */ 28 29 #include "libgomp.h" 30 #include "pool.h" 31 #include <stdlib.h> 32 #include <string.h> 33 34 #ifdef LIBGOMP_USE_PTHREADS 35 pthread_attr_t gomp_thread_attr; 36 37 /* This key is for the thread destructor. */ 38 pthread_key_t gomp_thread_destructor; 39 40 41 /* This is the libgomp per-thread data structure. */ 42 #if defined HAVE_TLS || defined USE_EMUTLS 43 __thread struct gomp_thread gomp_tls_data; 44 #else 45 pthread_key_t gomp_tls_key; 46 #endif 47 48 49 /* This structure is used to communicate across pthread_create. */ 50 51 struct gomp_thread_start_data 52 { 53 void (*fn) (void *); 54 void *fn_data; 55 struct gomp_team_state ts; 56 struct gomp_task *task; 57 struct gomp_thread_pool *thread_pool; 58 unsigned int place; 59 bool nested; 60 pthread_t handle; 61 }; 62 63 64 /* This function is a pthread_create entry point. This contains the idle 65 loop in which a thread waits to be called up to become part of a team. */ 66 67 static void * 68 gomp_thread_start (void *xdata) 69 { 70 struct gomp_thread_start_data *data = xdata; 71 struct gomp_thread *thr; 72 struct gomp_thread_pool *pool; 73 void (*local_fn) (void *); 74 void *local_data; 75 76 #if defined HAVE_TLS || defined USE_EMUTLS 77 thr = &gomp_tls_data; 78 #else 79 struct gomp_thread local_thr; 80 thr = &local_thr; 81 pthread_setspecific (gomp_tls_key, thr); 82 #endif 83 gomp_sem_init (&thr->release, 0); 84 85 /* Extract what we need from data. */ 86 local_fn = data->fn; 87 local_data = data->fn_data; 88 thr->thread_pool = data->thread_pool; 89 thr->ts = data->ts; 90 thr->task = data->task; 91 thr->place = data->place; 92 #ifdef GOMP_NEEDS_THREAD_HANDLE 93 thr->handle = data->handle; 94 #endif 95 96 thr->ts.team->ordered_release[thr->ts.team_id] = &thr->release; 97 98 /* Make thread pool local. */ 99 pool = thr->thread_pool; 100 101 if (data->nested) 102 { 103 struct gomp_team *team = thr->ts.team; 104 struct gomp_task *task = thr->task; 105 106 gomp_barrier_wait (&team->barrier); 107 108 local_fn (local_data); 109 gomp_team_barrier_wait_final (&team->barrier); 110 gomp_finish_task (task); 111 gomp_barrier_wait_last (&team->barrier); 112 } 113 else 114 { 115 pool->threads[thr->ts.team_id] = thr; 116 117 gomp_simple_barrier_wait (&pool->threads_dock); 118 do 119 { 120 struct gomp_team *team = thr->ts.team; 121 struct gomp_task *task = thr->task; 122 123 local_fn (local_data); 124 gomp_team_barrier_wait_final (&team->barrier); 125 gomp_finish_task (task); 126 127 gomp_simple_barrier_wait (&pool->threads_dock); 128 129 local_fn = thr->fn; 130 local_data = thr->data; 131 thr->fn = NULL; 132 } 133 while (local_fn); 134 } 135 136 gomp_sem_destroy (&thr->release); 137 pthread_detach (pthread_self ()); 138 thr->thread_pool = NULL; 139 thr->task = NULL; 140 return NULL; 141 } 142 #endif 143 144 static inline struct gomp_team * 145 get_last_team (unsigned nthreads) 146 { 147 struct gomp_thread *thr = gomp_thread (); 148 if (thr->ts.team == NULL) 149 { 150 struct gomp_thread_pool *pool = gomp_get_thread_pool (thr, nthreads); 151 struct gomp_team *last_team = pool->last_team; 152 if (last_team != NULL && last_team->nthreads == nthreads) 153 { 154 pool->last_team = NULL; 155 return last_team; 156 } 157 } 158 return NULL; 159 } 160 161 /* Create a new team data structure. */ 162 163 struct gomp_team * 164 gomp_new_team (unsigned nthreads) 165 { 166 struct gomp_team *team; 167 int i; 168 169 team = get_last_team (nthreads); 170 if (team == NULL) 171 { 172 size_t extra = sizeof (team->ordered_release[0]) 173 + sizeof (team->implicit_task[0]); 174 team = team_malloc (sizeof (*team) + nthreads * extra); 175 176 #ifndef HAVE_SYNC_BUILTINS 177 gomp_mutex_init (&team->work_share_list_free_lock); 178 #endif 179 gomp_barrier_init (&team->barrier, nthreads); 180 gomp_mutex_init (&team->task_lock); 181 182 team->nthreads = nthreads; 183 } 184 185 team->work_share_chunk = 8; 186 #ifdef HAVE_SYNC_BUILTINS 187 team->single_count = 0; 188 #endif 189 team->work_shares_to_free = &team->work_shares[0]; 190 gomp_init_work_share (&team->work_shares[0], 0, nthreads); 191 team->work_shares[0].next_alloc = NULL; 192 team->work_share_list_free = NULL; 193 team->work_share_list_alloc = &team->work_shares[1]; 194 for (i = 1; i < 7; i++) 195 team->work_shares[i].next_free = &team->work_shares[i + 1]; 196 team->work_shares[i].next_free = NULL; 197 198 gomp_sem_init (&team->master_release, 0); 199 team->ordered_release = (void *) &team->implicit_task[nthreads]; 200 team->ordered_release[0] = &team->master_release; 201 202 priority_queue_init (&team->task_queue); 203 team->task_count = 0; 204 team->task_queued_count = 0; 205 team->task_running_count = 0; 206 team->work_share_cancelled = 0; 207 team->team_cancelled = 0; 208 209 return team; 210 } 211 212 213 /* Free a team data structure. */ 214 215 static void 216 free_team (struct gomp_team *team) 217 { 218 #ifndef HAVE_SYNC_BUILTINS 219 gomp_mutex_destroy (&team->work_share_list_free_lock); 220 #endif 221 gomp_barrier_destroy (&team->barrier); 222 gomp_mutex_destroy (&team->task_lock); 223 priority_queue_free (&team->task_queue); 224 team_free (team); 225 } 226 227 static void 228 gomp_free_pool_helper (void *thread_pool) 229 { 230 struct gomp_thread *thr = gomp_thread (); 231 struct gomp_thread_pool *pool 232 = (struct gomp_thread_pool *) thread_pool; 233 gomp_simple_barrier_wait_last (&pool->threads_dock); 234 gomp_sem_destroy (&thr->release); 235 thr->thread_pool = NULL; 236 thr->task = NULL; 237 #ifdef LIBGOMP_USE_PTHREADS 238 pthread_detach (pthread_self ()); 239 pthread_exit (NULL); 240 #elif defined(__nvptx__) 241 asm ("exit;"); 242 #elif defined(__AMDGCN__) 243 asm ("s_dcache_wb\n\t" 244 "s_endpgm"); 245 #else 246 #error gomp_free_pool_helper must terminate the thread 247 #endif 248 } 249 250 /* Free a thread pool and release its threads. */ 251 252 void 253 gomp_free_thread (void *arg __attribute__((unused))) 254 { 255 struct gomp_thread *thr = gomp_thread (); 256 struct gomp_thread_pool *pool = thr->thread_pool; 257 if (pool) 258 { 259 if (pool->threads_used > 0) 260 { 261 int i; 262 for (i = 1; i < pool->threads_used; i++) 263 { 264 struct gomp_thread *nthr = pool->threads[i]; 265 nthr->fn = gomp_free_pool_helper; 266 nthr->data = pool; 267 } 268 /* This barrier undocks threads docked on pool->threads_dock. */ 269 gomp_simple_barrier_wait (&pool->threads_dock); 270 /* And this waits till all threads have called gomp_barrier_wait_last 271 in gomp_free_pool_helper. */ 272 gomp_simple_barrier_wait (&pool->threads_dock); 273 /* Now it is safe to destroy the barrier and free the pool. */ 274 gomp_simple_barrier_destroy (&pool->threads_dock); 275 276 #ifdef HAVE_SYNC_BUILTINS 277 __sync_fetch_and_add (&gomp_managed_threads, 278 1L - pool->threads_used); 279 #else 280 gomp_mutex_lock (&gomp_managed_threads_lock); 281 gomp_managed_threads -= pool->threads_used - 1L; 282 gomp_mutex_unlock (&gomp_managed_threads_lock); 283 #endif 284 } 285 if (pool->last_team) 286 free_team (pool->last_team); 287 #ifndef __nvptx__ 288 team_free (pool->threads); 289 team_free (pool); 290 #endif 291 thr->thread_pool = NULL; 292 } 293 if (thr->ts.level == 0 && __builtin_expect (thr->ts.team != NULL, 0)) 294 gomp_team_end (); 295 if (thr->task != NULL) 296 { 297 struct gomp_task *task = thr->task; 298 gomp_end_task (); 299 free (task); 300 } 301 } 302 303 /* Launch a team. */ 304 305 #ifdef LIBGOMP_USE_PTHREADS 306 void 307 gomp_team_start (void (*fn) (void *), void *data, unsigned nthreads, 308 unsigned flags, struct gomp_team *team, 309 struct gomp_taskgroup *taskgroup) 310 { 311 struct gomp_thread_start_data *start_data; 312 struct gomp_thread *thr, *nthr; 313 struct gomp_task *task; 314 struct gomp_task_icv *icv; 315 bool nested; 316 struct gomp_thread_pool *pool; 317 unsigned i, n, old_threads_used = 0; 318 pthread_attr_t thread_attr, *attr; 319 unsigned long nthreads_var; 320 char bind, bind_var; 321 unsigned int s = 0, rest = 0, p = 0, k = 0; 322 unsigned int affinity_count = 0; 323 struct gomp_thread **affinity_thr = NULL; 324 bool force_display = false; 325 326 thr = gomp_thread (); 327 nested = thr->ts.level; 328 pool = thr->thread_pool; 329 task = thr->task; 330 icv = task ? &task->icv : &gomp_global_icv; 331 if (__builtin_expect (gomp_places_list != NULL, 0) && thr->place == 0) 332 { 333 gomp_init_affinity (); 334 if (__builtin_expect (gomp_display_affinity_var, 0) && nthreads == 1) 335 gomp_display_affinity_thread (gomp_thread_self (), &thr->ts, 336 thr->place); 337 } 338 339 /* Always save the previous state, even if this isn't a nested team. 340 In particular, we should save any work share state from an outer 341 orphaned work share construct. */ 342 team->prev_ts = thr->ts; 343 344 thr->ts.team = team; 345 thr->ts.team_id = 0; 346 ++thr->ts.level; 347 if (nthreads > 1) 348 ++thr->ts.active_level; 349 thr->ts.work_share = &team->work_shares[0]; 350 thr->ts.last_work_share = NULL; 351 #ifdef HAVE_SYNC_BUILTINS 352 thr->ts.single_count = 0; 353 #endif 354 thr->ts.static_trip = 0; 355 thr->task = &team->implicit_task[0]; 356 #ifdef GOMP_NEEDS_THREAD_HANDLE 357 thr->handle = pthread_self (); 358 #endif 359 nthreads_var = icv->nthreads_var; 360 if (__builtin_expect (gomp_nthreads_var_list != NULL, 0) 361 && thr->ts.level < gomp_nthreads_var_list_len) 362 nthreads_var = gomp_nthreads_var_list[thr->ts.level]; 363 bind_var = icv->bind_var; 364 if (bind_var != omp_proc_bind_false && (flags & 7) != omp_proc_bind_false) 365 bind_var = flags & 7; 366 bind = bind_var; 367 if (__builtin_expect (gomp_bind_var_list != NULL, 0) 368 && thr->ts.level < gomp_bind_var_list_len) 369 bind_var = gomp_bind_var_list[thr->ts.level]; 370 gomp_init_task (thr->task, task, icv); 371 thr->task->taskgroup = taskgroup; 372 team->implicit_task[0].icv.nthreads_var = nthreads_var; 373 team->implicit_task[0].icv.bind_var = bind_var; 374 375 if (nthreads == 1) 376 return; 377 378 i = 1; 379 380 if (__builtin_expect (gomp_places_list != NULL, 0)) 381 { 382 /* Depending on chosen proc_bind model, set subpartition 383 for the master thread and initialize helper variables 384 P and optionally S, K and/or REST used by later place 385 computation for each additional thread. */ 386 p = thr->place - 1; 387 switch (bind) 388 { 389 case omp_proc_bind_true: 390 case omp_proc_bind_close: 391 if (nthreads > thr->ts.place_partition_len) 392 { 393 /* T > P. S threads will be placed in each place, 394 and the final REM threads placed one by one 395 into the already occupied places. */ 396 s = nthreads / thr->ts.place_partition_len; 397 rest = nthreads % thr->ts.place_partition_len; 398 } 399 else 400 s = 1; 401 k = 1; 402 break; 403 case omp_proc_bind_master: 404 /* Each thread will be bound to master's place. */ 405 break; 406 case omp_proc_bind_spread: 407 if (nthreads <= thr->ts.place_partition_len) 408 { 409 /* T <= P. Each subpartition will have in between s 410 and s+1 places (subpartitions starting at or 411 after rest will have s places, earlier s+1 places), 412 each thread will be bound to the first place in 413 its subpartition (except for the master thread 414 that can be bound to another place in its 415 subpartition). */ 416 s = thr->ts.place_partition_len / nthreads; 417 rest = thr->ts.place_partition_len % nthreads; 418 rest = (s + 1) * rest + thr->ts.place_partition_off; 419 if (p < rest) 420 { 421 p -= (p - thr->ts.place_partition_off) % (s + 1); 422 thr->ts.place_partition_len = s + 1; 423 } 424 else 425 { 426 p -= (p - rest) % s; 427 thr->ts.place_partition_len = s; 428 } 429 thr->ts.place_partition_off = p; 430 } 431 else 432 { 433 /* T > P. Each subpartition will have just a single 434 place and we'll place between s and s+1 435 threads into each subpartition. */ 436 s = nthreads / thr->ts.place_partition_len; 437 rest = nthreads % thr->ts.place_partition_len; 438 thr->ts.place_partition_off = p; 439 thr->ts.place_partition_len = 1; 440 k = 1; 441 } 442 break; 443 } 444 } 445 else 446 bind = omp_proc_bind_false; 447 448 /* We only allow the reuse of idle threads for non-nested PARALLEL 449 regions. This appears to be implied by the semantics of 450 threadprivate variables, but perhaps that's reading too much into 451 things. Certainly it does prevent any locking problems, since 452 only the initial program thread will modify gomp_threads. */ 453 if (!nested) 454 { 455 old_threads_used = pool->threads_used; 456 457 if (nthreads <= old_threads_used) 458 n = nthreads; 459 else if (old_threads_used == 0) 460 { 461 n = 0; 462 gomp_simple_barrier_init (&pool->threads_dock, nthreads); 463 } 464 else 465 { 466 n = old_threads_used; 467 468 /* Increase the barrier threshold to make sure all new 469 threads arrive before the team is released. */ 470 gomp_simple_barrier_reinit (&pool->threads_dock, nthreads); 471 } 472 473 /* Not true yet, but soon will be. We're going to release all 474 threads from the dock, and those that aren't part of the 475 team will exit. */ 476 pool->threads_used = nthreads; 477 478 /* If necessary, expand the size of the gomp_threads array. It is 479 expected that changes in the number of threads are rare, thus we 480 make no effort to expand gomp_threads_size geometrically. */ 481 if (nthreads >= pool->threads_size) 482 { 483 pool->threads_size = nthreads + 1; 484 pool->threads 485 = gomp_realloc (pool->threads, 486 pool->threads_size 487 * sizeof (struct gomp_thread *)); 488 /* Add current (master) thread to threads[]. */ 489 pool->threads[0] = thr; 490 } 491 492 /* Release existing idle threads. */ 493 for (; i < n; ++i) 494 { 495 unsigned int place_partition_off = thr->ts.place_partition_off; 496 unsigned int place_partition_len = thr->ts.place_partition_len; 497 unsigned int place = 0; 498 if (__builtin_expect (gomp_places_list != NULL, 0)) 499 { 500 switch (bind) 501 { 502 case omp_proc_bind_true: 503 case omp_proc_bind_close: 504 if (k == s) 505 { 506 ++p; 507 if (p == (team->prev_ts.place_partition_off 508 + team->prev_ts.place_partition_len)) 509 p = team->prev_ts.place_partition_off; 510 k = 1; 511 if (i == nthreads - rest) 512 s = 1; 513 } 514 else 515 ++k; 516 break; 517 case omp_proc_bind_master: 518 break; 519 case omp_proc_bind_spread: 520 if (k == 0) 521 { 522 /* T <= P. */ 523 if (p < rest) 524 p += s + 1; 525 else 526 p += s; 527 if (p == (team->prev_ts.place_partition_off 528 + team->prev_ts.place_partition_len)) 529 p = team->prev_ts.place_partition_off; 530 place_partition_off = p; 531 if (p < rest) 532 place_partition_len = s + 1; 533 else 534 place_partition_len = s; 535 } 536 else 537 { 538 /* T > P. */ 539 if (k == s) 540 { 541 ++p; 542 if (p == (team->prev_ts.place_partition_off 543 + team->prev_ts.place_partition_len)) 544 p = team->prev_ts.place_partition_off; 545 k = 1; 546 if (i == nthreads - rest) 547 s = 1; 548 } 549 else 550 ++k; 551 place_partition_off = p; 552 place_partition_len = 1; 553 } 554 break; 555 } 556 if (affinity_thr != NULL 557 || (bind != omp_proc_bind_true 558 && pool->threads[i]->place != p + 1) 559 || pool->threads[i]->place <= place_partition_off 560 || pool->threads[i]->place > (place_partition_off 561 + place_partition_len)) 562 { 563 unsigned int l; 564 force_display = true; 565 if (affinity_thr == NULL) 566 { 567 unsigned int j; 568 569 if (team->prev_ts.place_partition_len > 64) 570 affinity_thr 571 = gomp_malloc (team->prev_ts.place_partition_len 572 * sizeof (struct gomp_thread *)); 573 else 574 affinity_thr 575 = gomp_alloca (team->prev_ts.place_partition_len 576 * sizeof (struct gomp_thread *)); 577 memset (affinity_thr, '\0', 578 team->prev_ts.place_partition_len 579 * sizeof (struct gomp_thread *)); 580 for (j = i; j < old_threads_used; j++) 581 { 582 if (pool->threads[j]->place 583 > team->prev_ts.place_partition_off 584 && (pool->threads[j]->place 585 <= (team->prev_ts.place_partition_off 586 + team->prev_ts.place_partition_len))) 587 { 588 l = pool->threads[j]->place - 1 589 - team->prev_ts.place_partition_off; 590 pool->threads[j]->data = affinity_thr[l]; 591 affinity_thr[l] = pool->threads[j]; 592 } 593 pool->threads[j] = NULL; 594 } 595 if (nthreads > old_threads_used) 596 memset (&pool->threads[old_threads_used], 597 '\0', ((nthreads - old_threads_used) 598 * sizeof (struct gomp_thread *))); 599 n = nthreads; 600 affinity_count = old_threads_used - i; 601 } 602 if (affinity_count == 0) 603 break; 604 l = p; 605 if (affinity_thr[l - team->prev_ts.place_partition_off] 606 == NULL) 607 { 608 if (bind != omp_proc_bind_true) 609 continue; 610 for (l = place_partition_off; 611 l < place_partition_off + place_partition_len; 612 l++) 613 if (affinity_thr[l - team->prev_ts.place_partition_off] 614 != NULL) 615 break; 616 if (l == place_partition_off + place_partition_len) 617 continue; 618 } 619 nthr = affinity_thr[l - team->prev_ts.place_partition_off]; 620 affinity_thr[l - team->prev_ts.place_partition_off] 621 = (struct gomp_thread *) nthr->data; 622 affinity_count--; 623 pool->threads[i] = nthr; 624 } 625 else 626 nthr = pool->threads[i]; 627 place = p + 1; 628 } 629 else 630 nthr = pool->threads[i]; 631 nthr->ts.team = team; 632 nthr->ts.work_share = &team->work_shares[0]; 633 nthr->ts.last_work_share = NULL; 634 nthr->ts.team_id = i; 635 nthr->ts.level = team->prev_ts.level + 1; 636 nthr->ts.active_level = thr->ts.active_level; 637 nthr->ts.place_partition_off = place_partition_off; 638 nthr->ts.place_partition_len = place_partition_len; 639 #ifdef HAVE_SYNC_BUILTINS 640 nthr->ts.single_count = 0; 641 #endif 642 nthr->ts.static_trip = 0; 643 nthr->task = &team->implicit_task[i]; 644 nthr->place = place; 645 gomp_init_task (nthr->task, task, icv); 646 team->implicit_task[i].icv.nthreads_var = nthreads_var; 647 team->implicit_task[i].icv.bind_var = bind_var; 648 nthr->task->taskgroup = taskgroup; 649 nthr->fn = fn; 650 nthr->data = data; 651 team->ordered_release[i] = &nthr->release; 652 } 653 654 if (__builtin_expect (affinity_thr != NULL, 0)) 655 { 656 /* If AFFINITY_THR is non-NULL just because we had to 657 permute some threads in the pool, but we've managed 658 to find exactly as many old threads as we'd find 659 without affinity, we don't need to handle this 660 specially anymore. */ 661 if (nthreads <= old_threads_used 662 ? (affinity_count == old_threads_used - nthreads) 663 : (i == old_threads_used)) 664 { 665 if (team->prev_ts.place_partition_len > 64) 666 free (affinity_thr); 667 affinity_thr = NULL; 668 affinity_count = 0; 669 } 670 else 671 { 672 i = 1; 673 /* We are going to compute the places/subpartitions 674 again from the beginning. So, we need to reinitialize 675 vars modified by the switch (bind) above inside 676 of the loop, to the state they had after the initial 677 switch (bind). */ 678 switch (bind) 679 { 680 case omp_proc_bind_true: 681 case omp_proc_bind_close: 682 if (nthreads > thr->ts.place_partition_len) 683 /* T > P. S has been changed, so needs 684 to be recomputed. */ 685 s = nthreads / thr->ts.place_partition_len; 686 k = 1; 687 p = thr->place - 1; 688 break; 689 case omp_proc_bind_master: 690 /* No vars have been changed. */ 691 break; 692 case omp_proc_bind_spread: 693 p = thr->ts.place_partition_off; 694 if (k != 0) 695 { 696 /* T > P. */ 697 s = nthreads / team->prev_ts.place_partition_len; 698 k = 1; 699 } 700 break; 701 } 702 703 /* Increase the barrier threshold to make sure all new 704 threads and all the threads we're going to let die 705 arrive before the team is released. */ 706 if (affinity_count) 707 gomp_simple_barrier_reinit (&pool->threads_dock, 708 nthreads + affinity_count); 709 } 710 } 711 712 if (i == nthreads) 713 goto do_release; 714 715 } 716 717 if (__builtin_expect (nthreads + affinity_count > old_threads_used, 0)) 718 { 719 long diff = (long) (nthreads + affinity_count) - (long) old_threads_used; 720 721 if (old_threads_used == 0) 722 --diff; 723 724 #ifdef HAVE_SYNC_BUILTINS 725 __sync_fetch_and_add (&gomp_managed_threads, diff); 726 #else 727 gomp_mutex_lock (&gomp_managed_threads_lock); 728 gomp_managed_threads += diff; 729 gomp_mutex_unlock (&gomp_managed_threads_lock); 730 #endif 731 } 732 733 attr = &gomp_thread_attr; 734 if (__builtin_expect (gomp_places_list != NULL, 0)) 735 { 736 size_t stacksize; 737 pthread_attr_init (&thread_attr); 738 if (! pthread_attr_getstacksize (&gomp_thread_attr, &stacksize)) 739 pthread_attr_setstacksize (&thread_attr, stacksize); 740 attr = &thread_attr; 741 } 742 743 start_data = gomp_alloca (sizeof (struct gomp_thread_start_data) 744 * (nthreads - i)); 745 746 /* Launch new threads. */ 747 for (; i < nthreads; ++i) 748 { 749 int err; 750 751 start_data->ts.place_partition_off = thr->ts.place_partition_off; 752 start_data->ts.place_partition_len = thr->ts.place_partition_len; 753 start_data->place = 0; 754 if (__builtin_expect (gomp_places_list != NULL, 0)) 755 { 756 switch (bind) 757 { 758 case omp_proc_bind_true: 759 case omp_proc_bind_close: 760 if (k == s) 761 { 762 ++p; 763 if (p == (team->prev_ts.place_partition_off 764 + team->prev_ts.place_partition_len)) 765 p = team->prev_ts.place_partition_off; 766 k = 1; 767 if (i == nthreads - rest) 768 s = 1; 769 } 770 else 771 ++k; 772 break; 773 case omp_proc_bind_master: 774 break; 775 case omp_proc_bind_spread: 776 if (k == 0) 777 { 778 /* T <= P. */ 779 if (p < rest) 780 p += s + 1; 781 else 782 p += s; 783 if (p == (team->prev_ts.place_partition_off 784 + team->prev_ts.place_partition_len)) 785 p = team->prev_ts.place_partition_off; 786 start_data->ts.place_partition_off = p; 787 if (p < rest) 788 start_data->ts.place_partition_len = s + 1; 789 else 790 start_data->ts.place_partition_len = s; 791 } 792 else 793 { 794 /* T > P. */ 795 if (k == s) 796 { 797 ++p; 798 if (p == (team->prev_ts.place_partition_off 799 + team->prev_ts.place_partition_len)) 800 p = team->prev_ts.place_partition_off; 801 k = 1; 802 if (i == nthreads - rest) 803 s = 1; 804 } 805 else 806 ++k; 807 start_data->ts.place_partition_off = p; 808 start_data->ts.place_partition_len = 1; 809 } 810 break; 811 } 812 start_data->place = p + 1; 813 if (affinity_thr != NULL && pool->threads[i] != NULL) 814 continue; 815 gomp_init_thread_affinity (attr, p); 816 } 817 818 start_data->fn = fn; 819 start_data->fn_data = data; 820 start_data->ts.team = team; 821 start_data->ts.work_share = &team->work_shares[0]; 822 start_data->ts.last_work_share = NULL; 823 start_data->ts.team_id = i; 824 start_data->ts.level = team->prev_ts.level + 1; 825 start_data->ts.active_level = thr->ts.active_level; 826 #ifdef HAVE_SYNC_BUILTINS 827 start_data->ts.single_count = 0; 828 #endif 829 start_data->ts.static_trip = 0; 830 start_data->task = &team->implicit_task[i]; 831 gomp_init_task (start_data->task, task, icv); 832 team->implicit_task[i].icv.nthreads_var = nthreads_var; 833 team->implicit_task[i].icv.bind_var = bind_var; 834 start_data->task->taskgroup = taskgroup; 835 start_data->thread_pool = pool; 836 start_data->nested = nested; 837 838 attr = gomp_adjust_thread_attr (attr, &thread_attr); 839 err = pthread_create (&start_data->handle, attr, gomp_thread_start, 840 start_data); 841 start_data++; 842 if (err != 0) 843 gomp_fatal ("Thread creation failed: %s", strerror (err)); 844 } 845 846 if (__builtin_expect (attr == &thread_attr, 0)) 847 pthread_attr_destroy (&thread_attr); 848 849 do_release: 850 if (nested) 851 gomp_barrier_wait (&team->barrier); 852 else 853 gomp_simple_barrier_wait (&pool->threads_dock); 854 855 /* Decrease the barrier threshold to match the number of threads 856 that should arrive back at the end of this team. The extra 857 threads should be exiting. Note that we arrange for this test 858 to never be true for nested teams. If AFFINITY_COUNT is non-zero, 859 the barrier as well as gomp_managed_threads was temporarily 860 set to NTHREADS + AFFINITY_COUNT. For NTHREADS < OLD_THREADS_COUNT, 861 AFFINITY_COUNT if non-zero will be always at least 862 OLD_THREADS_COUNT - NTHREADS. */ 863 if (__builtin_expect (nthreads < old_threads_used, 0) 864 || __builtin_expect (affinity_count, 0)) 865 { 866 long diff = (long) nthreads - (long) old_threads_used; 867 868 if (affinity_count) 869 diff = -affinity_count; 870 871 gomp_simple_barrier_reinit (&pool->threads_dock, nthreads); 872 873 #ifdef HAVE_SYNC_BUILTINS 874 __sync_fetch_and_add (&gomp_managed_threads, diff); 875 #else 876 gomp_mutex_lock (&gomp_managed_threads_lock); 877 gomp_managed_threads += diff; 878 gomp_mutex_unlock (&gomp_managed_threads_lock); 879 #endif 880 } 881 if (__builtin_expect (gomp_display_affinity_var, 0)) 882 { 883 if (nested 884 || nthreads != old_threads_used 885 || force_display) 886 { 887 gomp_display_affinity_thread (gomp_thread_self (), &thr->ts, 888 thr->place); 889 if (nested) 890 { 891 start_data -= nthreads - 1; 892 for (i = 1; i < nthreads; ++i) 893 { 894 gomp_display_affinity_thread ( 895 #ifdef LIBGOMP_USE_PTHREADS 896 start_data->handle, 897 #else 898 gomp_thread_self (), 899 #endif 900 &start_data->ts, 901 start_data->place); 902 start_data++; 903 } 904 } 905 else 906 { 907 for (i = 1; i < nthreads; ++i) 908 { 909 gomp_thread_handle handle 910 = gomp_thread_to_pthread_t (pool->threads[i]); 911 gomp_display_affinity_thread (handle, &pool->threads[i]->ts, 912 pool->threads[i]->place); 913 } 914 } 915 } 916 } 917 if (__builtin_expect (affinity_thr != NULL, 0) 918 && team->prev_ts.place_partition_len > 64) 919 free (affinity_thr); 920 } 921 #endif 922 923 924 /* Terminate the current team. This is only to be called by the master 925 thread. We assume that we must wait for the other threads. */ 926 927 void 928 gomp_team_end (void) 929 { 930 struct gomp_thread *thr = gomp_thread (); 931 struct gomp_team *team = thr->ts.team; 932 933 /* This barrier handles all pending explicit threads. 934 As #pragma omp cancel parallel might get awaited count in 935 team->barrier in a inconsistent state, we need to use a different 936 counter here. */ 937 gomp_team_barrier_wait_final (&team->barrier); 938 if (__builtin_expect (team->team_cancelled, 0)) 939 { 940 struct gomp_work_share *ws = team->work_shares_to_free; 941 do 942 { 943 struct gomp_work_share *next_ws = gomp_ptrlock_get (&ws->next_ws); 944 if (next_ws == NULL) 945 gomp_ptrlock_set (&ws->next_ws, ws); 946 gomp_fini_work_share (ws); 947 ws = next_ws; 948 } 949 while (ws != NULL); 950 } 951 else 952 gomp_fini_work_share (thr->ts.work_share); 953 954 gomp_end_task (); 955 thr->ts = team->prev_ts; 956 957 if (__builtin_expect (thr->ts.level != 0, 0)) 958 { 959 #ifdef HAVE_SYNC_BUILTINS 960 __sync_fetch_and_add (&gomp_managed_threads, 1L - team->nthreads); 961 #else 962 gomp_mutex_lock (&gomp_managed_threads_lock); 963 gomp_managed_threads -= team->nthreads - 1L; 964 gomp_mutex_unlock (&gomp_managed_threads_lock); 965 #endif 966 /* This barrier has gomp_barrier_wait_last counterparts 967 and ensures the team can be safely destroyed. */ 968 gomp_barrier_wait (&team->barrier); 969 } 970 971 if (__builtin_expect (team->work_shares[0].next_alloc != NULL, 0)) 972 { 973 struct gomp_work_share *ws = team->work_shares[0].next_alloc; 974 do 975 { 976 struct gomp_work_share *next_ws = ws->next_alloc; 977 free (ws); 978 ws = next_ws; 979 } 980 while (ws != NULL); 981 } 982 gomp_sem_destroy (&team->master_release); 983 984 if (__builtin_expect (thr->ts.team != NULL, 0) 985 || __builtin_expect (team->nthreads == 1, 0)) 986 free_team (team); 987 else 988 { 989 struct gomp_thread_pool *pool = thr->thread_pool; 990 if (pool->last_team) 991 free_team (pool->last_team); 992 pool->last_team = team; 993 gomp_release_thread_pool (pool); 994 } 995 } 996 997 #ifdef LIBGOMP_USE_PTHREADS 998 999 /* Constructors for this file. */ 1000 1001 static void __attribute__((constructor)) 1002 initialize_team (void) 1003 { 1004 #if !defined HAVE_TLS && !defined USE_EMUTLS 1005 static struct gomp_thread initial_thread_tls_data; 1006 1007 pthread_key_create (&gomp_tls_key, NULL); 1008 pthread_setspecific (gomp_tls_key, &initial_thread_tls_data); 1009 #endif 1010 1011 if (pthread_key_create (&gomp_thread_destructor, gomp_free_thread) != 0) 1012 gomp_fatal ("could not create thread pool destructor."); 1013 } 1014 1015 static void __attribute__((destructor)) 1016 team_destructor (void) 1017 { 1018 /* Without this dlclose on libgomp could lead to subsequent 1019 crashes. */ 1020 pthread_key_delete (gomp_thread_destructor); 1021 } 1022 1023 /* Similar to gomp_free_pool_helper, but don't detach itself, 1024 gomp_pause_host will pthread_join those threads. */ 1025 1026 static void 1027 gomp_pause_pool_helper (void *thread_pool) 1028 { 1029 struct gomp_thread *thr = gomp_thread (); 1030 struct gomp_thread_pool *pool 1031 = (struct gomp_thread_pool *) thread_pool; 1032 gomp_simple_barrier_wait_last (&pool->threads_dock); 1033 gomp_sem_destroy (&thr->release); 1034 thr->thread_pool = NULL; 1035 thr->task = NULL; 1036 pthread_exit (NULL); 1037 } 1038 1039 /* Free a thread pool and release its threads. Return non-zero on 1040 failure. */ 1041 1042 int 1043 gomp_pause_host (void) 1044 { 1045 struct gomp_thread *thr = gomp_thread (); 1046 struct gomp_thread_pool *pool = thr->thread_pool; 1047 if (thr->ts.level) 1048 return -1; 1049 if (pool) 1050 { 1051 if (pool->threads_used > 0) 1052 { 1053 int i; 1054 pthread_t *thrs 1055 = gomp_alloca (sizeof (pthread_t) * pool->threads_used); 1056 for (i = 1; i < pool->threads_used; i++) 1057 { 1058 struct gomp_thread *nthr = pool->threads[i]; 1059 nthr->fn = gomp_pause_pool_helper; 1060 nthr->data = pool; 1061 thrs[i] = gomp_thread_to_pthread_t (nthr); 1062 } 1063 /* This barrier undocks threads docked on pool->threads_dock. */ 1064 gomp_simple_barrier_wait (&pool->threads_dock); 1065 /* And this waits till all threads have called gomp_barrier_wait_last 1066 in gomp_pause_pool_helper. */ 1067 gomp_simple_barrier_wait (&pool->threads_dock); 1068 /* Now it is safe to destroy the barrier and free the pool. */ 1069 gomp_simple_barrier_destroy (&pool->threads_dock); 1070 1071 #ifdef HAVE_SYNC_BUILTINS 1072 __sync_fetch_and_add (&gomp_managed_threads, 1073 1L - pool->threads_used); 1074 #else 1075 gomp_mutex_lock (&gomp_managed_threads_lock); 1076 gomp_managed_threads -= pool->threads_used - 1L; 1077 gomp_mutex_unlock (&gomp_managed_threads_lock); 1078 #endif 1079 for (i = 1; i < pool->threads_used; i++) 1080 pthread_join (thrs[i], NULL); 1081 } 1082 if (pool->last_team) 1083 free_team (pool->last_team); 1084 #ifndef __nvptx__ 1085 team_free (pool->threads); 1086 team_free (pool); 1087 #endif 1088 thr->thread_pool = NULL; 1089 } 1090 return 0; 1091 } 1092 #endif 1093 1094 struct gomp_task_icv * 1095 gomp_new_icv (void) 1096 { 1097 struct gomp_thread *thr = gomp_thread (); 1098 struct gomp_task *task = gomp_malloc (sizeof (struct gomp_task)); 1099 gomp_init_task (task, NULL, &gomp_global_icv); 1100 thr->task = task; 1101 #ifdef LIBGOMP_USE_PTHREADS 1102 pthread_setspecific (gomp_thread_destructor, thr); 1103 #endif 1104 return &task->icv; 1105 } 1106