1 /* Copyright (C) 2007-2015 Free Software Foundation, Inc. 2 Contributed by Richard Henderson <rth@redhat.com>. 3 4 This file is part of the GNU Offloading and Multi Processing Library 5 (libgomp). 6 7 Libgomp is free software; you can redistribute it and/or modify it 8 under the terms of the GNU General Public License as published by 9 the Free Software Foundation; either version 3, or (at your option) 10 any later version. 11 12 Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY 13 WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 14 FOR A PARTICULAR PURPOSE. See the GNU General Public License for 15 more details. 16 17 Under Section 7 of GPL version 3, you are granted additional 18 permissions described in the GCC Runtime Library Exception, version 19 3.1, as published by the Free Software Foundation. 20 21 You should have received a copy of the GNU General Public License and 22 a copy of the GCC Runtime Library Exception along with this program; 23 see the files COPYING3 and COPYING.RUNTIME respectively. If not, see 24 <http://www.gnu.org/licenses/>. */ 25 26 /* This file handles the maintainence of tasks in response to task 27 creation and termination. */ 28 29 #include "libgomp.h" 30 #include <stdlib.h> 31 #include <string.h> 32 33 typedef struct gomp_task_depend_entry *hash_entry_type; 34 35 static inline void * 36 htab_alloc (size_t size) 37 { 38 return gomp_malloc (size); 39 } 40 41 static inline void 42 htab_free (void *ptr) 43 { 44 free (ptr); 45 } 46 47 #include "hashtab.h" 48 49 static inline hashval_t 50 htab_hash (hash_entry_type element) 51 { 52 return hash_pointer (element->addr); 53 } 54 55 static inline bool 56 htab_eq (hash_entry_type x, hash_entry_type y) 57 { 58 return x->addr == y->addr; 59 } 60 61 /* Create a new task data structure. */ 62 63 void 64 gomp_init_task (struct gomp_task *task, struct gomp_task *parent_task, 65 struct gomp_task_icv *prev_icv) 66 { 67 task->parent = parent_task; 68 task->icv = *prev_icv; 69 task->kind = GOMP_TASK_IMPLICIT; 70 task->taskwait = NULL; 71 task->in_tied_task = false; 72 task->final_task = false; 73 task->copy_ctors_done = false; 74 task->parent_depends_on = false; 75 task->children = NULL; 76 task->taskgroup = NULL; 77 task->dependers = NULL; 78 task->depend_hash = NULL; 79 task->depend_count = 0; 80 } 81 82 /* Clean up a task, after completing it. */ 83 84 void 85 gomp_end_task (void) 86 { 87 struct gomp_thread *thr = gomp_thread (); 88 struct gomp_task *task = thr->task; 89 90 gomp_finish_task (task); 91 thr->task = task->parent; 92 } 93 94 static inline void 95 gomp_clear_parent (struct gomp_task *children) 96 { 97 struct gomp_task *task = children; 98 99 if (task) 100 do 101 { 102 task->parent = NULL; 103 task = task->next_child; 104 } 105 while (task != children); 106 } 107 108 static void gomp_task_maybe_wait_for_dependencies (void **depend); 109 110 /* Called when encountering an explicit task directive. If IF_CLAUSE is 111 false, then we must not delay in executing the task. If UNTIED is true, 112 then the task may be executed by any member of the team. */ 113 114 void 115 GOMP_task (void (*fn) (void *), void *data, void (*cpyfn) (void *, void *), 116 long arg_size, long arg_align, bool if_clause, unsigned flags, 117 void **depend) 118 { 119 struct gomp_thread *thr = gomp_thread (); 120 struct gomp_team *team = thr->ts.team; 121 122 #ifdef HAVE_BROKEN_POSIX_SEMAPHORES 123 /* If pthread_mutex_* is used for omp_*lock*, then each task must be 124 tied to one thread all the time. This means UNTIED tasks must be 125 tied and if CPYFN is non-NULL IF(0) must be forced, as CPYFN 126 might be running on different thread than FN. */ 127 if (cpyfn) 128 if_clause = false; 129 if (flags & 1) 130 flags &= ~1; 131 #endif 132 133 /* If parallel or taskgroup has been cancelled, don't start new tasks. */ 134 if (team 135 && (gomp_team_barrier_cancelled (&team->barrier) 136 || (thr->task->taskgroup && thr->task->taskgroup->cancelled))) 137 return; 138 139 if (!if_clause || team == NULL 140 || (thr->task && thr->task->final_task) 141 || team->task_count > 64 * team->nthreads) 142 { 143 struct gomp_task task; 144 145 /* If there are depend clauses and earlier deferred sibling tasks 146 with depend clauses, check if there isn't a dependency. If there 147 is, we need to wait for them. There is no need to handle 148 depend clauses for non-deferred tasks other than this, because 149 the parent task is suspended until the child task finishes and thus 150 it can't start further child tasks. */ 151 if ((flags & 8) && thr->task && thr->task->depend_hash) 152 gomp_task_maybe_wait_for_dependencies (depend); 153 154 gomp_init_task (&task, thr->task, gomp_icv (false)); 155 task.kind = GOMP_TASK_IFFALSE; 156 task.final_task = (thr->task && thr->task->final_task) || (flags & 2); 157 if (thr->task) 158 { 159 task.in_tied_task = thr->task->in_tied_task; 160 task.taskgroup = thr->task->taskgroup; 161 } 162 thr->task = &task; 163 if (__builtin_expect (cpyfn != NULL, 0)) 164 { 165 char buf[arg_size + arg_align - 1]; 166 char *arg = (char *) (((uintptr_t) buf + arg_align - 1) 167 & ~(uintptr_t) (arg_align - 1)); 168 cpyfn (arg, data); 169 fn (arg); 170 } 171 else 172 fn (data); 173 /* Access to "children" is normally done inside a task_lock 174 mutex region, but the only way this particular task.children 175 can be set is if this thread's task work function (fn) 176 creates children. So since the setter is *this* thread, we 177 need no barriers here when testing for non-NULL. We can have 178 task.children set by the current thread then changed by a 179 child thread, but seeing a stale non-NULL value is not a 180 problem. Once past the task_lock acquisition, this thread 181 will see the real value of task.children. */ 182 if (task.children != NULL) 183 { 184 gomp_mutex_lock (&team->task_lock); 185 gomp_clear_parent (task.children); 186 gomp_mutex_unlock (&team->task_lock); 187 } 188 gomp_end_task (); 189 } 190 else 191 { 192 struct gomp_task *task; 193 struct gomp_task *parent = thr->task; 194 struct gomp_taskgroup *taskgroup = parent->taskgroup; 195 char *arg; 196 bool do_wake; 197 size_t depend_size = 0; 198 199 if (flags & 8) 200 depend_size = ((uintptr_t) depend[0] 201 * sizeof (struct gomp_task_depend_entry)); 202 task = gomp_malloc (sizeof (*task) + depend_size 203 + arg_size + arg_align - 1); 204 arg = (char *) (((uintptr_t) (task + 1) + depend_size + arg_align - 1) 205 & ~(uintptr_t) (arg_align - 1)); 206 gomp_init_task (task, parent, gomp_icv (false)); 207 task->kind = GOMP_TASK_IFFALSE; 208 task->in_tied_task = parent->in_tied_task; 209 task->taskgroup = taskgroup; 210 thr->task = task; 211 if (cpyfn) 212 { 213 cpyfn (arg, data); 214 task->copy_ctors_done = true; 215 } 216 else 217 memcpy (arg, data, arg_size); 218 thr->task = parent; 219 task->kind = GOMP_TASK_WAITING; 220 task->fn = fn; 221 task->fn_data = arg; 222 task->final_task = (flags & 2) >> 1; 223 gomp_mutex_lock (&team->task_lock); 224 /* If parallel or taskgroup has been cancelled, don't start new 225 tasks. */ 226 if (__builtin_expect ((gomp_team_barrier_cancelled (&team->barrier) 227 || (taskgroup && taskgroup->cancelled)) 228 && !task->copy_ctors_done, 0)) 229 { 230 gomp_mutex_unlock (&team->task_lock); 231 gomp_finish_task (task); 232 free (task); 233 return; 234 } 235 if (taskgroup) 236 taskgroup->num_children++; 237 if (depend_size) 238 { 239 size_t ndepend = (uintptr_t) depend[0]; 240 size_t nout = (uintptr_t) depend[1]; 241 size_t i; 242 hash_entry_type ent; 243 244 task->depend_count = ndepend; 245 task->num_dependees = 0; 246 if (parent->depend_hash == NULL) 247 parent->depend_hash 248 = htab_create (2 * ndepend > 12 ? 2 * ndepend : 12); 249 for (i = 0; i < ndepend; i++) 250 { 251 task->depend[i].addr = depend[2 + i]; 252 task->depend[i].next = NULL; 253 task->depend[i].prev = NULL; 254 task->depend[i].task = task; 255 task->depend[i].is_in = i >= nout; 256 task->depend[i].redundant = false; 257 task->depend[i].redundant_out = false; 258 259 hash_entry_type *slot 260 = htab_find_slot (&parent->depend_hash, &task->depend[i], 261 INSERT); 262 hash_entry_type out = NULL, last = NULL; 263 if (*slot) 264 { 265 /* If multiple depends on the same task are the 266 same, all but the first one are redundant. 267 As inout/out come first, if any of them is 268 inout/out, it will win, which is the right 269 semantics. */ 270 if ((*slot)->task == task) 271 { 272 task->depend[i].redundant = true; 273 continue; 274 } 275 for (ent = *slot; ent; ent = ent->next) 276 { 277 if (ent->redundant_out) 278 break; 279 280 last = ent; 281 282 /* depend(in:...) doesn't depend on earlier 283 depend(in:...). */ 284 if (i >= nout && ent->is_in) 285 continue; 286 287 if (!ent->is_in) 288 out = ent; 289 290 struct gomp_task *tsk = ent->task; 291 if (tsk->dependers == NULL) 292 { 293 tsk->dependers 294 = gomp_malloc (sizeof (struct gomp_dependers_vec) 295 + 6 * sizeof (struct gomp_task *)); 296 tsk->dependers->n_elem = 1; 297 tsk->dependers->allocated = 6; 298 tsk->dependers->elem[0] = task; 299 task->num_dependees++; 300 continue; 301 } 302 /* We already have some other dependency on tsk 303 from earlier depend clause. */ 304 else if (tsk->dependers->n_elem 305 && (tsk->dependers->elem[tsk->dependers->n_elem 306 - 1] 307 == task)) 308 continue; 309 else if (tsk->dependers->n_elem 310 == tsk->dependers->allocated) 311 { 312 tsk->dependers->allocated 313 = tsk->dependers->allocated * 2 + 2; 314 tsk->dependers 315 = gomp_realloc (tsk->dependers, 316 sizeof (struct gomp_dependers_vec) 317 + (tsk->dependers->allocated 318 * sizeof (struct gomp_task *))); 319 } 320 tsk->dependers->elem[tsk->dependers->n_elem++] = task; 321 task->num_dependees++; 322 } 323 task->depend[i].next = *slot; 324 (*slot)->prev = &task->depend[i]; 325 } 326 *slot = &task->depend[i]; 327 328 /* There is no need to store more than one depend({,in}out:) 329 task per address in the hash table chain for the purpose 330 of creation of deferred tasks, because each out 331 depends on all earlier outs, thus it is enough to record 332 just the last depend({,in}out:). For depend(in:), we need 333 to keep all of the previous ones not terminated yet, because 334 a later depend({,in}out:) might need to depend on all of 335 them. So, if the new task's clause is depend({,in}out:), 336 we know there is at most one other depend({,in}out:) clause 337 in the list (out). For non-deferred tasks we want to see 338 all outs, so they are moved to the end of the chain, 339 after first redundant_out entry all following entries 340 should be redundant_out. */ 341 if (!task->depend[i].is_in && out) 342 { 343 if (out != last) 344 { 345 out->next->prev = out->prev; 346 out->prev->next = out->next; 347 out->next = last->next; 348 out->prev = last; 349 last->next = out; 350 if (out->next) 351 out->next->prev = out; 352 } 353 out->redundant_out = true; 354 } 355 } 356 if (task->num_dependees) 357 { 358 gomp_mutex_unlock (&team->task_lock); 359 return; 360 } 361 } 362 if (parent->children) 363 { 364 task->next_child = parent->children; 365 task->prev_child = parent->children->prev_child; 366 task->next_child->prev_child = task; 367 task->prev_child->next_child = task; 368 } 369 else 370 { 371 task->next_child = task; 372 task->prev_child = task; 373 } 374 parent->children = task; 375 if (taskgroup) 376 { 377 if (taskgroup->children) 378 { 379 task->next_taskgroup = taskgroup->children; 380 task->prev_taskgroup = taskgroup->children->prev_taskgroup; 381 task->next_taskgroup->prev_taskgroup = task; 382 task->prev_taskgroup->next_taskgroup = task; 383 } 384 else 385 { 386 task->next_taskgroup = task; 387 task->prev_taskgroup = task; 388 } 389 taskgroup->children = task; 390 } 391 if (team->task_queue) 392 { 393 task->next_queue = team->task_queue; 394 task->prev_queue = team->task_queue->prev_queue; 395 task->next_queue->prev_queue = task; 396 task->prev_queue->next_queue = task; 397 } 398 else 399 { 400 task->next_queue = task; 401 task->prev_queue = task; 402 team->task_queue = task; 403 } 404 ++team->task_count; 405 ++team->task_queued_count; 406 gomp_team_barrier_set_task_pending (&team->barrier); 407 do_wake = team->task_running_count + !parent->in_tied_task 408 < team->nthreads; 409 gomp_mutex_unlock (&team->task_lock); 410 if (do_wake) 411 gomp_team_barrier_wake (&team->barrier, 1); 412 } 413 } 414 415 static inline bool 416 gomp_task_run_pre (struct gomp_task *child_task, struct gomp_task *parent, 417 struct gomp_taskgroup *taskgroup, struct gomp_team *team) 418 { 419 if (parent) 420 { 421 if (parent->children == child_task) 422 parent->children = child_task->next_child; 423 if (__builtin_expect (child_task->parent_depends_on, 0) 424 && parent->taskwait->last_parent_depends_on == child_task) 425 { 426 if (child_task->prev_child->kind == GOMP_TASK_WAITING 427 && child_task->prev_child->parent_depends_on) 428 parent->taskwait->last_parent_depends_on = child_task->prev_child; 429 else 430 parent->taskwait->last_parent_depends_on = NULL; 431 } 432 } 433 if (taskgroup && taskgroup->children == child_task) 434 taskgroup->children = child_task->next_taskgroup; 435 child_task->prev_queue->next_queue = child_task->next_queue; 436 child_task->next_queue->prev_queue = child_task->prev_queue; 437 if (team->task_queue == child_task) 438 { 439 if (child_task->next_queue != child_task) 440 team->task_queue = child_task->next_queue; 441 else 442 team->task_queue = NULL; 443 } 444 child_task->kind = GOMP_TASK_TIED; 445 if (--team->task_queued_count == 0) 446 gomp_team_barrier_clear_task_pending (&team->barrier); 447 if ((gomp_team_barrier_cancelled (&team->barrier) 448 || (taskgroup && taskgroup->cancelled)) 449 && !child_task->copy_ctors_done) 450 return true; 451 return false; 452 } 453 454 static void 455 gomp_task_run_post_handle_depend_hash (struct gomp_task *child_task) 456 { 457 struct gomp_task *parent = child_task->parent; 458 size_t i; 459 460 for (i = 0; i < child_task->depend_count; i++) 461 if (!child_task->depend[i].redundant) 462 { 463 if (child_task->depend[i].next) 464 child_task->depend[i].next->prev = child_task->depend[i].prev; 465 if (child_task->depend[i].prev) 466 child_task->depend[i].prev->next = child_task->depend[i].next; 467 else 468 { 469 hash_entry_type *slot 470 = htab_find_slot (&parent->depend_hash, &child_task->depend[i], 471 NO_INSERT); 472 if (*slot != &child_task->depend[i]) 473 abort (); 474 if (child_task->depend[i].next) 475 *slot = child_task->depend[i].next; 476 else 477 htab_clear_slot (parent->depend_hash, slot); 478 } 479 } 480 } 481 482 static size_t 483 gomp_task_run_post_handle_dependers (struct gomp_task *child_task, 484 struct gomp_team *team) 485 { 486 struct gomp_task *parent = child_task->parent; 487 size_t i, count = child_task->dependers->n_elem, ret = 0; 488 for (i = 0; i < count; i++) 489 { 490 struct gomp_task *task = child_task->dependers->elem[i]; 491 if (--task->num_dependees != 0) 492 continue; 493 494 struct gomp_taskgroup *taskgroup = task->taskgroup; 495 if (parent) 496 { 497 if (parent->children) 498 { 499 /* If parent is in gomp_task_maybe_wait_for_dependencies 500 and it doesn't need to wait for this task, put it after 501 all ready to run tasks it needs to wait for. */ 502 if (parent->taskwait && parent->taskwait->last_parent_depends_on 503 && !task->parent_depends_on) 504 { 505 struct gomp_task *last_parent_depends_on 506 = parent->taskwait->last_parent_depends_on; 507 task->next_child = last_parent_depends_on->next_child; 508 task->prev_child = last_parent_depends_on; 509 } 510 else 511 { 512 task->next_child = parent->children; 513 task->prev_child = parent->children->prev_child; 514 parent->children = task; 515 } 516 task->next_child->prev_child = task; 517 task->prev_child->next_child = task; 518 } 519 else 520 { 521 task->next_child = task; 522 task->prev_child = task; 523 parent->children = task; 524 } 525 if (parent->taskwait) 526 { 527 if (parent->taskwait->in_taskwait) 528 { 529 parent->taskwait->in_taskwait = false; 530 gomp_sem_post (&parent->taskwait->taskwait_sem); 531 } 532 else if (parent->taskwait->in_depend_wait) 533 { 534 parent->taskwait->in_depend_wait = false; 535 gomp_sem_post (&parent->taskwait->taskwait_sem); 536 } 537 if (parent->taskwait->last_parent_depends_on == NULL 538 && task->parent_depends_on) 539 parent->taskwait->last_parent_depends_on = task; 540 } 541 } 542 if (taskgroup) 543 { 544 if (taskgroup->children) 545 { 546 task->next_taskgroup = taskgroup->children; 547 task->prev_taskgroup = taskgroup->children->prev_taskgroup; 548 task->next_taskgroup->prev_taskgroup = task; 549 task->prev_taskgroup->next_taskgroup = task; 550 } 551 else 552 { 553 task->next_taskgroup = task; 554 task->prev_taskgroup = task; 555 } 556 taskgroup->children = task; 557 if (taskgroup->in_taskgroup_wait) 558 { 559 taskgroup->in_taskgroup_wait = false; 560 gomp_sem_post (&taskgroup->taskgroup_sem); 561 } 562 } 563 if (team->task_queue) 564 { 565 task->next_queue = team->task_queue; 566 task->prev_queue = team->task_queue->prev_queue; 567 task->next_queue->prev_queue = task; 568 task->prev_queue->next_queue = task; 569 } 570 else 571 { 572 task->next_queue = task; 573 task->prev_queue = task; 574 team->task_queue = task; 575 } 576 ++team->task_count; 577 ++team->task_queued_count; 578 ++ret; 579 } 580 free (child_task->dependers); 581 child_task->dependers = NULL; 582 if (ret > 1) 583 gomp_team_barrier_set_task_pending (&team->barrier); 584 return ret; 585 } 586 587 static inline size_t 588 gomp_task_run_post_handle_depend (struct gomp_task *child_task, 589 struct gomp_team *team) 590 { 591 if (child_task->depend_count == 0) 592 return 0; 593 594 /* If parent is gone already, the hash table is freed and nothing 595 will use the hash table anymore, no need to remove anything from it. */ 596 if (child_task->parent != NULL) 597 gomp_task_run_post_handle_depend_hash (child_task); 598 599 if (child_task->dependers == NULL) 600 return 0; 601 602 return gomp_task_run_post_handle_dependers (child_task, team); 603 } 604 605 static inline void 606 gomp_task_run_post_remove_parent (struct gomp_task *child_task) 607 { 608 struct gomp_task *parent = child_task->parent; 609 if (parent == NULL) 610 return; 611 if (__builtin_expect (child_task->parent_depends_on, 0) 612 && --parent->taskwait->n_depend == 0 613 && parent->taskwait->in_depend_wait) 614 { 615 parent->taskwait->in_depend_wait = false; 616 gomp_sem_post (&parent->taskwait->taskwait_sem); 617 } 618 child_task->prev_child->next_child = child_task->next_child; 619 child_task->next_child->prev_child = child_task->prev_child; 620 if (parent->children != child_task) 621 return; 622 if (child_task->next_child != child_task) 623 parent->children = child_task->next_child; 624 else 625 { 626 /* We access task->children in GOMP_taskwait 627 outside of the task lock mutex region, so 628 need a release barrier here to ensure memory 629 written by child_task->fn above is flushed 630 before the NULL is written. */ 631 __atomic_store_n (&parent->children, NULL, MEMMODEL_RELEASE); 632 if (parent->taskwait && parent->taskwait->in_taskwait) 633 { 634 parent->taskwait->in_taskwait = false; 635 gomp_sem_post (&parent->taskwait->taskwait_sem); 636 } 637 } 638 } 639 640 static inline void 641 gomp_task_run_post_remove_taskgroup (struct gomp_task *child_task) 642 { 643 struct gomp_taskgroup *taskgroup = child_task->taskgroup; 644 if (taskgroup == NULL) 645 return; 646 child_task->prev_taskgroup->next_taskgroup = child_task->next_taskgroup; 647 child_task->next_taskgroup->prev_taskgroup = child_task->prev_taskgroup; 648 if (taskgroup->num_children > 1) 649 --taskgroup->num_children; 650 else 651 { 652 /* We access taskgroup->num_children in GOMP_taskgroup_end 653 outside of the task lock mutex region, so 654 need a release barrier here to ensure memory 655 written by child_task->fn above is flushed 656 before the NULL is written. */ 657 __atomic_store_n (&taskgroup->num_children, 0, MEMMODEL_RELEASE); 658 } 659 if (taskgroup->children != child_task) 660 return; 661 if (child_task->next_taskgroup != child_task) 662 taskgroup->children = child_task->next_taskgroup; 663 else 664 { 665 taskgroup->children = NULL; 666 if (taskgroup->in_taskgroup_wait) 667 { 668 taskgroup->in_taskgroup_wait = false; 669 gomp_sem_post (&taskgroup->taskgroup_sem); 670 } 671 } 672 } 673 674 void 675 gomp_barrier_handle_tasks (gomp_barrier_state_t state) 676 { 677 struct gomp_thread *thr = gomp_thread (); 678 struct gomp_team *team = thr->ts.team; 679 struct gomp_task *task = thr->task; 680 struct gomp_task *child_task = NULL; 681 struct gomp_task *to_free = NULL; 682 int do_wake = 0; 683 684 gomp_mutex_lock (&team->task_lock); 685 if (gomp_barrier_last_thread (state)) 686 { 687 if (team->task_count == 0) 688 { 689 gomp_team_barrier_done (&team->barrier, state); 690 gomp_mutex_unlock (&team->task_lock); 691 gomp_team_barrier_wake (&team->barrier, 0); 692 return; 693 } 694 gomp_team_barrier_set_waiting_for_tasks (&team->barrier); 695 } 696 697 while (1) 698 { 699 bool cancelled = false; 700 if (team->task_queue != NULL) 701 { 702 child_task = team->task_queue; 703 cancelled = gomp_task_run_pre (child_task, child_task->parent, 704 child_task->taskgroup, team); 705 if (__builtin_expect (cancelled, 0)) 706 { 707 if (to_free) 708 { 709 gomp_finish_task (to_free); 710 free (to_free); 711 to_free = NULL; 712 } 713 goto finish_cancelled; 714 } 715 team->task_running_count++; 716 child_task->in_tied_task = true; 717 } 718 gomp_mutex_unlock (&team->task_lock); 719 if (do_wake) 720 { 721 gomp_team_barrier_wake (&team->barrier, do_wake); 722 do_wake = 0; 723 } 724 if (to_free) 725 { 726 gomp_finish_task (to_free); 727 free (to_free); 728 to_free = NULL; 729 } 730 if (child_task) 731 { 732 thr->task = child_task; 733 child_task->fn (child_task->fn_data); 734 thr->task = task; 735 } 736 else 737 return; 738 gomp_mutex_lock (&team->task_lock); 739 if (child_task) 740 { 741 finish_cancelled:; 742 size_t new_tasks 743 = gomp_task_run_post_handle_depend (child_task, team); 744 gomp_task_run_post_remove_parent (child_task); 745 gomp_clear_parent (child_task->children); 746 gomp_task_run_post_remove_taskgroup (child_task); 747 to_free = child_task; 748 child_task = NULL; 749 if (!cancelled) 750 team->task_running_count--; 751 if (new_tasks > 1) 752 { 753 do_wake = team->nthreads - team->task_running_count; 754 if (do_wake > new_tasks) 755 do_wake = new_tasks; 756 } 757 if (--team->task_count == 0 758 && gomp_team_barrier_waiting_for_tasks (&team->barrier)) 759 { 760 gomp_team_barrier_done (&team->barrier, state); 761 gomp_mutex_unlock (&team->task_lock); 762 gomp_team_barrier_wake (&team->barrier, 0); 763 gomp_mutex_lock (&team->task_lock); 764 } 765 } 766 } 767 } 768 769 /* Called when encountering a taskwait directive. */ 770 771 void 772 GOMP_taskwait (void) 773 { 774 struct gomp_thread *thr = gomp_thread (); 775 struct gomp_team *team = thr->ts.team; 776 struct gomp_task *task = thr->task; 777 struct gomp_task *child_task = NULL; 778 struct gomp_task *to_free = NULL; 779 struct gomp_taskwait taskwait; 780 int do_wake = 0; 781 782 /* The acquire barrier on load of task->children here synchronizes 783 with the write of a NULL in gomp_task_run_post_remove_parent. It is 784 not necessary that we synchronize with other non-NULL writes at 785 this point, but we must ensure that all writes to memory by a 786 child thread task work function are seen before we exit from 787 GOMP_taskwait. */ 788 if (task == NULL 789 || __atomic_load_n (&task->children, MEMMODEL_ACQUIRE) == NULL) 790 return; 791 792 memset (&taskwait, 0, sizeof (taskwait)); 793 gomp_mutex_lock (&team->task_lock); 794 while (1) 795 { 796 bool cancelled = false; 797 if (task->children == NULL) 798 { 799 bool destroy_taskwait = task->taskwait != NULL; 800 task->taskwait = NULL; 801 gomp_mutex_unlock (&team->task_lock); 802 if (to_free) 803 { 804 gomp_finish_task (to_free); 805 free (to_free); 806 } 807 if (destroy_taskwait) 808 gomp_sem_destroy (&taskwait.taskwait_sem); 809 return; 810 } 811 if (task->children->kind == GOMP_TASK_WAITING) 812 { 813 child_task = task->children; 814 cancelled 815 = gomp_task_run_pre (child_task, task, child_task->taskgroup, 816 team); 817 if (__builtin_expect (cancelled, 0)) 818 { 819 if (to_free) 820 { 821 gomp_finish_task (to_free); 822 free (to_free); 823 to_free = NULL; 824 } 825 goto finish_cancelled; 826 } 827 } 828 else 829 { 830 /* All tasks we are waiting for are already running 831 in other threads. Wait for them. */ 832 if (task->taskwait == NULL) 833 { 834 taskwait.in_depend_wait = false; 835 gomp_sem_init (&taskwait.taskwait_sem, 0); 836 task->taskwait = &taskwait; 837 } 838 taskwait.in_taskwait = true; 839 } 840 gomp_mutex_unlock (&team->task_lock); 841 if (do_wake) 842 { 843 gomp_team_barrier_wake (&team->barrier, do_wake); 844 do_wake = 0; 845 } 846 if (to_free) 847 { 848 gomp_finish_task (to_free); 849 free (to_free); 850 to_free = NULL; 851 } 852 if (child_task) 853 { 854 thr->task = child_task; 855 child_task->fn (child_task->fn_data); 856 thr->task = task; 857 } 858 else 859 gomp_sem_wait (&taskwait.taskwait_sem); 860 gomp_mutex_lock (&team->task_lock); 861 if (child_task) 862 { 863 finish_cancelled:; 864 size_t new_tasks 865 = gomp_task_run_post_handle_depend (child_task, team); 866 child_task->prev_child->next_child = child_task->next_child; 867 child_task->next_child->prev_child = child_task->prev_child; 868 if (task->children == child_task) 869 { 870 if (child_task->next_child != child_task) 871 task->children = child_task->next_child; 872 else 873 task->children = NULL; 874 } 875 gomp_clear_parent (child_task->children); 876 gomp_task_run_post_remove_taskgroup (child_task); 877 to_free = child_task; 878 child_task = NULL; 879 team->task_count--; 880 if (new_tasks > 1) 881 { 882 do_wake = team->nthreads - team->task_running_count 883 - !task->in_tied_task; 884 if (do_wake > new_tasks) 885 do_wake = new_tasks; 886 } 887 } 888 } 889 } 890 891 /* This is like GOMP_taskwait, but we only wait for tasks that the 892 upcoming task depends on. */ 893 894 static void 895 gomp_task_maybe_wait_for_dependencies (void **depend) 896 { 897 struct gomp_thread *thr = gomp_thread (); 898 struct gomp_task *task = thr->task; 899 struct gomp_team *team = thr->ts.team; 900 struct gomp_task_depend_entry elem, *ent = NULL; 901 struct gomp_taskwait taskwait; 902 struct gomp_task *last_parent_depends_on = NULL; 903 size_t ndepend = (uintptr_t) depend[0]; 904 size_t nout = (uintptr_t) depend[1]; 905 size_t i; 906 size_t num_awaited = 0; 907 struct gomp_task *child_task = NULL; 908 struct gomp_task *to_free = NULL; 909 int do_wake = 0; 910 911 gomp_mutex_lock (&team->task_lock); 912 for (i = 0; i < ndepend; i++) 913 { 914 elem.addr = depend[i + 2]; 915 ent = htab_find (task->depend_hash, &elem); 916 for (; ent; ent = ent->next) 917 if (i >= nout && ent->is_in) 918 continue; 919 else 920 { 921 struct gomp_task *tsk = ent->task; 922 if (!tsk->parent_depends_on) 923 { 924 tsk->parent_depends_on = true; 925 ++num_awaited; 926 if (tsk->num_dependees == 0 && tsk->kind == GOMP_TASK_WAITING) 927 { 928 /* If a task we need to wait for is not already 929 running and is ready to be scheduled, move it 930 to front, so that we run it as soon as possible. */ 931 if (last_parent_depends_on) 932 { 933 tsk->prev_child->next_child = tsk->next_child; 934 tsk->next_child->prev_child = tsk->prev_child; 935 tsk->prev_child = last_parent_depends_on; 936 tsk->next_child = last_parent_depends_on->next_child; 937 tsk->prev_child->next_child = tsk; 938 tsk->next_child->prev_child = tsk; 939 } 940 else if (tsk != task->children) 941 { 942 tsk->prev_child->next_child = tsk->next_child; 943 tsk->next_child->prev_child = tsk->prev_child; 944 tsk->prev_child = task->children; 945 tsk->next_child = task->children->next_child; 946 task->children = tsk; 947 tsk->prev_child->next_child = tsk; 948 tsk->next_child->prev_child = tsk; 949 } 950 last_parent_depends_on = tsk; 951 } 952 } 953 } 954 } 955 if (num_awaited == 0) 956 { 957 gomp_mutex_unlock (&team->task_lock); 958 return; 959 } 960 961 memset (&taskwait, 0, sizeof (taskwait)); 962 taskwait.n_depend = num_awaited; 963 taskwait.last_parent_depends_on = last_parent_depends_on; 964 gomp_sem_init (&taskwait.taskwait_sem, 0); 965 task->taskwait = &taskwait; 966 967 while (1) 968 { 969 bool cancelled = false; 970 if (taskwait.n_depend == 0) 971 { 972 task->taskwait = NULL; 973 gomp_mutex_unlock (&team->task_lock); 974 if (to_free) 975 { 976 gomp_finish_task (to_free); 977 free (to_free); 978 } 979 gomp_sem_destroy (&taskwait.taskwait_sem); 980 return; 981 } 982 if (task->children->kind == GOMP_TASK_WAITING) 983 { 984 child_task = task->children; 985 cancelled 986 = gomp_task_run_pre (child_task, task, child_task->taskgroup, 987 team); 988 if (__builtin_expect (cancelled, 0)) 989 { 990 if (to_free) 991 { 992 gomp_finish_task (to_free); 993 free (to_free); 994 to_free = NULL; 995 } 996 goto finish_cancelled; 997 } 998 } 999 else 1000 /* All tasks we are waiting for are already running 1001 in other threads. Wait for them. */ 1002 taskwait.in_depend_wait = true; 1003 gomp_mutex_unlock (&team->task_lock); 1004 if (do_wake) 1005 { 1006 gomp_team_barrier_wake (&team->barrier, do_wake); 1007 do_wake = 0; 1008 } 1009 if (to_free) 1010 { 1011 gomp_finish_task (to_free); 1012 free (to_free); 1013 to_free = NULL; 1014 } 1015 if (child_task) 1016 { 1017 thr->task = child_task; 1018 child_task->fn (child_task->fn_data); 1019 thr->task = task; 1020 } 1021 else 1022 gomp_sem_wait (&taskwait.taskwait_sem); 1023 gomp_mutex_lock (&team->task_lock); 1024 if (child_task) 1025 { 1026 finish_cancelled:; 1027 size_t new_tasks 1028 = gomp_task_run_post_handle_depend (child_task, team); 1029 if (child_task->parent_depends_on) 1030 --taskwait.n_depend; 1031 child_task->prev_child->next_child = child_task->next_child; 1032 child_task->next_child->prev_child = child_task->prev_child; 1033 if (task->children == child_task) 1034 { 1035 if (child_task->next_child != child_task) 1036 task->children = child_task->next_child; 1037 else 1038 task->children = NULL; 1039 } 1040 gomp_clear_parent (child_task->children); 1041 gomp_task_run_post_remove_taskgroup (child_task); 1042 to_free = child_task; 1043 child_task = NULL; 1044 team->task_count--; 1045 if (new_tasks > 1) 1046 { 1047 do_wake = team->nthreads - team->task_running_count 1048 - !task->in_tied_task; 1049 if (do_wake > new_tasks) 1050 do_wake = new_tasks; 1051 } 1052 } 1053 } 1054 } 1055 1056 /* Called when encountering a taskyield directive. */ 1057 1058 void 1059 GOMP_taskyield (void) 1060 { 1061 /* Nothing at the moment. */ 1062 } 1063 1064 void 1065 GOMP_taskgroup_start (void) 1066 { 1067 struct gomp_thread *thr = gomp_thread (); 1068 struct gomp_team *team = thr->ts.team; 1069 struct gomp_task *task = thr->task; 1070 struct gomp_taskgroup *taskgroup; 1071 1072 /* If team is NULL, all tasks are executed as 1073 GOMP_TASK_IFFALSE tasks and thus all children tasks of 1074 taskgroup and their descendant tasks will be finished 1075 by the time GOMP_taskgroup_end is called. */ 1076 if (team == NULL) 1077 return; 1078 taskgroup = gomp_malloc (sizeof (struct gomp_taskgroup)); 1079 taskgroup->prev = task->taskgroup; 1080 taskgroup->children = NULL; 1081 taskgroup->in_taskgroup_wait = false; 1082 taskgroup->cancelled = false; 1083 taskgroup->num_children = 0; 1084 gomp_sem_init (&taskgroup->taskgroup_sem, 0); 1085 task->taskgroup = taskgroup; 1086 } 1087 1088 void 1089 GOMP_taskgroup_end (void) 1090 { 1091 struct gomp_thread *thr = gomp_thread (); 1092 struct gomp_team *team = thr->ts.team; 1093 struct gomp_task *task = thr->task; 1094 struct gomp_taskgroup *taskgroup; 1095 struct gomp_task *child_task = NULL; 1096 struct gomp_task *to_free = NULL; 1097 int do_wake = 0; 1098 1099 if (team == NULL) 1100 return; 1101 taskgroup = task->taskgroup; 1102 1103 /* The acquire barrier on load of taskgroup->num_children here 1104 synchronizes with the write of 0 in gomp_task_run_post_remove_taskgroup. 1105 It is not necessary that we synchronize with other non-0 writes at 1106 this point, but we must ensure that all writes to memory by a 1107 child thread task work function are seen before we exit from 1108 GOMP_taskgroup_end. */ 1109 if (__atomic_load_n (&taskgroup->num_children, MEMMODEL_ACQUIRE) == 0) 1110 goto finish; 1111 1112 gomp_mutex_lock (&team->task_lock); 1113 while (1) 1114 { 1115 bool cancelled = false; 1116 if (taskgroup->children == NULL) 1117 { 1118 if (taskgroup->num_children) 1119 { 1120 if (task->children == NULL) 1121 goto do_wait; 1122 child_task = task->children; 1123 } 1124 else 1125 { 1126 gomp_mutex_unlock (&team->task_lock); 1127 if (to_free) 1128 { 1129 gomp_finish_task (to_free); 1130 free (to_free); 1131 } 1132 goto finish; 1133 } 1134 } 1135 else 1136 child_task = taskgroup->children; 1137 if (child_task->kind == GOMP_TASK_WAITING) 1138 { 1139 cancelled 1140 = gomp_task_run_pre (child_task, child_task->parent, taskgroup, 1141 team); 1142 if (__builtin_expect (cancelled, 0)) 1143 { 1144 if (to_free) 1145 { 1146 gomp_finish_task (to_free); 1147 free (to_free); 1148 to_free = NULL; 1149 } 1150 goto finish_cancelled; 1151 } 1152 } 1153 else 1154 { 1155 child_task = NULL; 1156 do_wait: 1157 /* All tasks we are waiting for are already running 1158 in other threads. Wait for them. */ 1159 taskgroup->in_taskgroup_wait = true; 1160 } 1161 gomp_mutex_unlock (&team->task_lock); 1162 if (do_wake) 1163 { 1164 gomp_team_barrier_wake (&team->barrier, do_wake); 1165 do_wake = 0; 1166 } 1167 if (to_free) 1168 { 1169 gomp_finish_task (to_free); 1170 free (to_free); 1171 to_free = NULL; 1172 } 1173 if (child_task) 1174 { 1175 thr->task = child_task; 1176 child_task->fn (child_task->fn_data); 1177 thr->task = task; 1178 } 1179 else 1180 gomp_sem_wait (&taskgroup->taskgroup_sem); 1181 gomp_mutex_lock (&team->task_lock); 1182 if (child_task) 1183 { 1184 finish_cancelled:; 1185 size_t new_tasks 1186 = gomp_task_run_post_handle_depend (child_task, team); 1187 gomp_task_run_post_remove_parent (child_task); 1188 gomp_clear_parent (child_task->children); 1189 gomp_task_run_post_remove_taskgroup (child_task); 1190 to_free = child_task; 1191 child_task = NULL; 1192 team->task_count--; 1193 if (new_tasks > 1) 1194 { 1195 do_wake = team->nthreads - team->task_running_count 1196 - !task->in_tied_task; 1197 if (do_wake > new_tasks) 1198 do_wake = new_tasks; 1199 } 1200 } 1201 } 1202 1203 finish: 1204 task->taskgroup = taskgroup->prev; 1205 gomp_sem_destroy (&taskgroup->taskgroup_sem); 1206 free (taskgroup); 1207 } 1208 1209 int 1210 omp_in_final (void) 1211 { 1212 struct gomp_thread *thr = gomp_thread (); 1213 return thr->task && thr->task->final_task; 1214 } 1215 1216 ialias (omp_in_final) 1217