1 /* Copyright (C) 2005-2015 Free Software Foundation, Inc. 2 Contributed by Richard Henderson <rth@redhat.com>. 3 4 This file is part of the GNU Offloading and Multi Processing Library 5 (libgomp). 6 7 Libgomp is free software; you can redistribute it and/or modify it 8 under the terms of the GNU General Public License as published by 9 the Free Software Foundation; either version 3, or (at your option) 10 any later version. 11 12 Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY 13 WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 14 FOR A PARTICULAR PURPOSE. See the GNU General Public License for 15 more details. 16 17 Under Section 7 of GPL version 3, you are granted additional 18 permissions described in the GCC Runtime Library Exception, version 19 3.1, as published by the Free Software Foundation. 20 21 You should have received a copy of the GNU General Public License and 22 a copy of the GCC Runtime Library Exception along with this program; 23 see the files COPYING3 and COPYING.RUNTIME respectively. If not, see 24 <http://www.gnu.org/licenses/>. */ 25 26 /* This file contains data types and function declarations that are not 27 part of the official OpenACC or OpenMP user interfaces. There are 28 declarations in here that are part of the GNU Offloading and Multi 29 Processing ABI, in that the compiler is required to know about them 30 and use them. 31 32 The convention is that the all caps prefix "GOMP" is used group items 33 that are part of the external ABI, and the lower case prefix "gomp" 34 is used group items that are completely private to the library. */ 35 36 #ifndef LIBGOMP_H 37 #define LIBGOMP_H 1 38 39 #include "config.h" 40 #include "gstdint.h" 41 #include "libgomp-plugin.h" 42 43 #include <pthread.h> 44 #include <stdbool.h> 45 #include <stdlib.h> 46 #include <stdarg.h> 47 48 #ifdef HAVE_ATTRIBUTE_VISIBILITY 49 # pragma GCC visibility push(hidden) 50 #endif 51 52 /* If we were a C++ library, we'd get this from <std/atomic>. */ 53 enum memmodel 54 { 55 MEMMODEL_RELAXED = 0, 56 MEMMODEL_CONSUME = 1, 57 MEMMODEL_ACQUIRE = 2, 58 MEMMODEL_RELEASE = 3, 59 MEMMODEL_ACQ_REL = 4, 60 MEMMODEL_SEQ_CST = 5 61 }; 62 63 #include "sem.h" 64 #include "mutex.h" 65 #include "bar.h" 66 #include "ptrlock.h" 67 68 69 /* This structure contains the data to control one work-sharing construct, 70 either a LOOP (FOR/DO) or a SECTIONS. */ 71 72 enum gomp_schedule_type 73 { 74 GFS_RUNTIME, 75 GFS_STATIC, 76 GFS_DYNAMIC, 77 GFS_GUIDED, 78 GFS_AUTO 79 }; 80 81 struct gomp_work_share 82 { 83 /* This member records the SCHEDULE clause to be used for this construct. 84 The user specification of "runtime" will already have been resolved. 85 If this is a SECTIONS construct, this value will always be DYNAMIC. */ 86 enum gomp_schedule_type sched; 87 88 int mode; 89 90 union { 91 struct { 92 /* This is the chunk_size argument to the SCHEDULE clause. */ 93 long chunk_size; 94 95 /* This is the iteration end point. If this is a SECTIONS construct, 96 this is the number of contained sections. */ 97 long end; 98 99 /* This is the iteration step. If this is a SECTIONS construct, this 100 is always 1. */ 101 long incr; 102 }; 103 104 struct { 105 /* The same as above, but for the unsigned long long loop variants. */ 106 unsigned long long chunk_size_ull; 107 unsigned long long end_ull; 108 unsigned long long incr_ull; 109 }; 110 }; 111 112 /* This is a circular queue that details which threads will be allowed 113 into the ordered region and in which order. When a thread allocates 114 iterations on which it is going to work, it also registers itself at 115 the end of the array. When a thread reaches the ordered region, it 116 checks to see if it is the one at the head of the queue. If not, it 117 blocks on its RELEASE semaphore. */ 118 unsigned *ordered_team_ids; 119 120 /* This is the number of threads that have registered themselves in 121 the circular queue ordered_team_ids. */ 122 unsigned ordered_num_used; 123 124 /* This is the team_id of the currently acknowledged owner of the ordered 125 section, or -1u if the ordered section has not been acknowledged by 126 any thread. This is distinguished from the thread that is *allowed* 127 to take the section next. */ 128 unsigned ordered_owner; 129 130 /* This is the index into the circular queue ordered_team_ids of the 131 current thread that's allowed into the ordered reason. */ 132 unsigned ordered_cur; 133 134 /* This is a chain of allocated gomp_work_share blocks, valid only 135 in the first gomp_work_share struct in the block. */ 136 struct gomp_work_share *next_alloc; 137 138 /* The above fields are written once during workshare initialization, 139 or related to ordered worksharing. Make sure the following fields 140 are in a different cache line. */ 141 142 /* This lock protects the update of the following members. */ 143 gomp_mutex_t lock __attribute__((aligned (64))); 144 145 /* This is the count of the number of threads that have exited the work 146 share construct. If the construct was marked nowait, they have moved on 147 to other work; otherwise they're blocked on a barrier. The last member 148 of the team to exit the work share construct must deallocate it. */ 149 unsigned threads_completed; 150 151 union { 152 /* This is the next iteration value to be allocated. In the case of 153 GFS_STATIC loops, this the iteration start point and never changes. */ 154 long next; 155 156 /* The same, but with unsigned long long type. */ 157 unsigned long long next_ull; 158 159 /* This is the returned data structure for SINGLE COPYPRIVATE. */ 160 void *copyprivate; 161 }; 162 163 union { 164 /* Link to gomp_work_share struct for next work sharing construct 165 encountered after this one. */ 166 gomp_ptrlock_t next_ws; 167 168 /* gomp_work_share structs are chained in the free work share cache 169 through this. */ 170 struct gomp_work_share *next_free; 171 }; 172 173 /* If only few threads are in the team, ordered_team_ids can point 174 to this array which fills the padding at the end of this struct. */ 175 unsigned inline_ordered_team_ids[0]; 176 }; 177 178 /* This structure contains all of the thread-local data associated with 179 a thread team. This is the data that must be saved when a thread 180 encounters a nested PARALLEL construct. */ 181 182 struct gomp_team_state 183 { 184 /* This is the team of which the thread is currently a member. */ 185 struct gomp_team *team; 186 187 /* This is the work share construct which this thread is currently 188 processing. Recall that with NOWAIT, not all threads may be 189 processing the same construct. */ 190 struct gomp_work_share *work_share; 191 192 /* This is the previous work share construct or NULL if there wasn't any. 193 When all threads are done with the current work sharing construct, 194 the previous one can be freed. The current one can't, as its 195 next_ws field is used. */ 196 struct gomp_work_share *last_work_share; 197 198 /* This is the ID of this thread within the team. This value is 199 guaranteed to be between 0 and N-1, where N is the number of 200 threads in the team. */ 201 unsigned team_id; 202 203 /* Nesting level. */ 204 unsigned level; 205 206 /* Active nesting level. Only active parallel regions are counted. */ 207 unsigned active_level; 208 209 /* Place-partition-var, offset and length into gomp_places_list array. */ 210 unsigned place_partition_off; 211 unsigned place_partition_len; 212 213 #ifdef HAVE_SYNC_BUILTINS 214 /* Number of single stmts encountered. */ 215 unsigned long single_count; 216 #endif 217 218 /* For GFS_RUNTIME loops that resolved to GFS_STATIC, this is the 219 trip number through the loop. So first time a particular loop 220 is encountered this number is 0, the second time through the loop 221 is 1, etc. This is unused when the compiler knows in advance that 222 the loop is statically scheduled. */ 223 unsigned long static_trip; 224 }; 225 226 struct target_mem_desc; 227 228 /* These are the OpenMP 4.0 Internal Control Variables described in 229 section 2.3.1. Those described as having one copy per task are 230 stored within the structure; those described as having one copy 231 for the whole program are (naturally) global variables. */ 232 233 struct gomp_task_icv 234 { 235 unsigned long nthreads_var; 236 enum gomp_schedule_type run_sched_var; 237 int run_sched_modifier; 238 int default_device_var; 239 unsigned int thread_limit_var; 240 bool dyn_var; 241 bool nest_var; 242 char bind_var; 243 /* Internal ICV. */ 244 struct target_mem_desc *target_data; 245 }; 246 247 extern struct gomp_task_icv gomp_global_icv; 248 #ifndef HAVE_SYNC_BUILTINS 249 extern gomp_mutex_t gomp_managed_threads_lock; 250 #endif 251 extern unsigned long gomp_max_active_levels_var; 252 extern bool gomp_cancel_var; 253 extern unsigned long long gomp_spin_count_var, gomp_throttled_spin_count_var; 254 extern unsigned long gomp_available_cpus, gomp_managed_threads; 255 extern unsigned long *gomp_nthreads_var_list, gomp_nthreads_var_list_len; 256 extern char *gomp_bind_var_list; 257 extern unsigned long gomp_bind_var_list_len; 258 extern void **gomp_places_list; 259 extern unsigned long gomp_places_list_len; 260 extern int gomp_debug_var; 261 extern int goacc_device_num; 262 extern char *goacc_device_type; 263 264 enum gomp_task_kind 265 { 266 GOMP_TASK_IMPLICIT, 267 GOMP_TASK_IFFALSE, 268 GOMP_TASK_WAITING, 269 GOMP_TASK_TIED 270 }; 271 272 struct gomp_task; 273 struct gomp_taskgroup; 274 struct htab; 275 276 struct gomp_task_depend_entry 277 { 278 void *addr; 279 struct gomp_task_depend_entry *next; 280 struct gomp_task_depend_entry *prev; 281 struct gomp_task *task; 282 bool is_in; 283 bool redundant; 284 bool redundant_out; 285 }; 286 287 struct gomp_dependers_vec 288 { 289 size_t n_elem; 290 size_t allocated; 291 struct gomp_task *elem[]; 292 }; 293 294 /* Used when in GOMP_taskwait or in gomp_task_maybe_wait_for_dependencies. */ 295 296 struct gomp_taskwait 297 { 298 bool in_taskwait; 299 bool in_depend_wait; 300 size_t n_depend; 301 struct gomp_task *last_parent_depends_on; 302 gomp_sem_t taskwait_sem; 303 }; 304 305 /* This structure describes a "task" to be run by a thread. */ 306 307 struct gomp_task 308 { 309 struct gomp_task *parent; 310 struct gomp_task *children; 311 struct gomp_task *next_child; 312 struct gomp_task *prev_child; 313 struct gomp_task *next_queue; 314 struct gomp_task *prev_queue; 315 struct gomp_task *next_taskgroup; 316 struct gomp_task *prev_taskgroup; 317 struct gomp_taskgroup *taskgroup; 318 struct gomp_dependers_vec *dependers; 319 struct htab *depend_hash; 320 struct gomp_taskwait *taskwait; 321 size_t depend_count; 322 size_t num_dependees; 323 struct gomp_task_icv icv; 324 void (*fn) (void *); 325 void *fn_data; 326 enum gomp_task_kind kind; 327 bool in_tied_task; 328 bool final_task; 329 bool copy_ctors_done; 330 bool parent_depends_on; 331 struct gomp_task_depend_entry depend[]; 332 }; 333 334 struct gomp_taskgroup 335 { 336 struct gomp_taskgroup *prev; 337 struct gomp_task *children; 338 bool in_taskgroup_wait; 339 bool cancelled; 340 gomp_sem_t taskgroup_sem; 341 size_t num_children; 342 }; 343 344 /* This structure describes a "team" of threads. These are the threads 345 that are spawned by a PARALLEL constructs, as well as the work sharing 346 constructs that the team encounters. */ 347 348 struct gomp_team 349 { 350 /* This is the number of threads in the current team. */ 351 unsigned nthreads; 352 353 /* This is number of gomp_work_share structs that have been allocated 354 as a block last time. */ 355 unsigned work_share_chunk; 356 357 /* This is the saved team state that applied to a master thread before 358 the current thread was created. */ 359 struct gomp_team_state prev_ts; 360 361 /* This semaphore should be used by the master thread instead of its 362 "native" semaphore in the thread structure. Required for nested 363 parallels, as the master is a member of two teams. */ 364 gomp_sem_t master_release; 365 366 /* This points to an array with pointers to the release semaphore 367 of the threads in the team. */ 368 gomp_sem_t **ordered_release; 369 370 /* List of work shares on which gomp_fini_work_share hasn't been 371 called yet. If the team hasn't been cancelled, this should be 372 equal to each thr->ts.work_share, but otherwise it can be a possibly 373 long list of workshares. */ 374 struct gomp_work_share *work_shares_to_free; 375 376 /* List of gomp_work_share structs chained through next_free fields. 377 This is populated and taken off only by the first thread in the 378 team encountering a new work sharing construct, in a critical 379 section. */ 380 struct gomp_work_share *work_share_list_alloc; 381 382 /* List of gomp_work_share structs freed by free_work_share. New 383 entries are atomically added to the start of the list, and 384 alloc_work_share can safely only move all but the first entry 385 to work_share_list alloc, as free_work_share can happen concurrently 386 with alloc_work_share. */ 387 struct gomp_work_share *work_share_list_free; 388 389 #ifdef HAVE_SYNC_BUILTINS 390 /* Number of simple single regions encountered by threads in this 391 team. */ 392 unsigned long single_count; 393 #else 394 /* Mutex protecting addition of workshares to work_share_list_free. */ 395 gomp_mutex_t work_share_list_free_lock; 396 #endif 397 398 /* This barrier is used for most synchronization of the team. */ 399 gomp_barrier_t barrier; 400 401 /* Initial work shares, to avoid allocating any gomp_work_share 402 structs in the common case. */ 403 struct gomp_work_share work_shares[8]; 404 405 gomp_mutex_t task_lock; 406 struct gomp_task *task_queue; 407 /* Number of all GOMP_TASK_{WAITING,TIED} tasks in the team. */ 408 unsigned int task_count; 409 /* Number of GOMP_TASK_WAITING tasks currently waiting to be scheduled. */ 410 unsigned int task_queued_count; 411 /* Number of GOMP_TASK_{WAITING,TIED} tasks currently running 412 directly in gomp_barrier_handle_tasks; tasks spawned 413 from e.g. GOMP_taskwait or GOMP_taskgroup_end don't count, even when 414 that is called from a task run from gomp_barrier_handle_tasks. 415 task_running_count should be always <= team->nthreads, 416 and if current task isn't in_tied_task, then it will be 417 even < team->nthreads. */ 418 unsigned int task_running_count; 419 int work_share_cancelled; 420 int team_cancelled; 421 422 /* This array contains structures for implicit tasks. */ 423 struct gomp_task implicit_task[]; 424 }; 425 426 /* This structure contains all data that is private to libgomp and is 427 allocated per thread. */ 428 429 struct gomp_thread 430 { 431 /* This is the function that the thread should run upon launch. */ 432 void (*fn) (void *data); 433 void *data; 434 435 /* This is the current team state for this thread. The ts.team member 436 is NULL only if the thread is idle. */ 437 struct gomp_team_state ts; 438 439 /* This is the task that the thread is currently executing. */ 440 struct gomp_task *task; 441 442 /* This semaphore is used for ordered loops. */ 443 gomp_sem_t release; 444 445 /* Place this thread is bound to plus one, or zero if not bound 446 to any place. */ 447 unsigned int place; 448 449 /* User pthread thread pool */ 450 struct gomp_thread_pool *thread_pool; 451 }; 452 453 454 struct gomp_thread_pool 455 { 456 /* This array manages threads spawned from the top level, which will 457 return to the idle loop once the current PARALLEL construct ends. */ 458 struct gomp_thread **threads; 459 unsigned threads_size; 460 unsigned threads_used; 461 struct gomp_team *last_team; 462 /* Number of threads running in this contention group. */ 463 unsigned long threads_busy; 464 465 /* This barrier holds and releases threads waiting in threads. */ 466 gomp_barrier_t threads_dock; 467 }; 468 469 enum gomp_cancel_kind 470 { 471 GOMP_CANCEL_PARALLEL = 1, 472 GOMP_CANCEL_LOOP = 2, 473 GOMP_CANCEL_FOR = GOMP_CANCEL_LOOP, 474 GOMP_CANCEL_DO = GOMP_CANCEL_LOOP, 475 GOMP_CANCEL_SECTIONS = 4, 476 GOMP_CANCEL_TASKGROUP = 8 477 }; 478 479 /* ... and here is that TLS data. */ 480 481 #if defined HAVE_TLS || defined USE_EMUTLS 482 extern __thread struct gomp_thread gomp_tls_data; 483 static inline struct gomp_thread *gomp_thread (void) 484 { 485 return &gomp_tls_data; 486 } 487 #else 488 extern pthread_key_t gomp_tls_key; 489 static inline struct gomp_thread *gomp_thread (void) 490 { 491 return pthread_getspecific (gomp_tls_key); 492 } 493 #endif 494 495 extern struct gomp_task_icv *gomp_new_icv (void); 496 497 /* Here's how to access the current copy of the ICVs. */ 498 499 static inline struct gomp_task_icv *gomp_icv (bool write) 500 { 501 struct gomp_task *task = gomp_thread ()->task; 502 if (task) 503 return &task->icv; 504 else if (write) 505 return gomp_new_icv (); 506 else 507 return &gomp_global_icv; 508 } 509 510 /* The attributes to be used during thread creation. */ 511 extern pthread_attr_t gomp_thread_attr; 512 513 /* Function prototypes. */ 514 515 /* affinity.c */ 516 517 extern void gomp_init_affinity (void); 518 extern void gomp_init_thread_affinity (pthread_attr_t *, unsigned int); 519 extern void **gomp_affinity_alloc (unsigned long, bool); 520 extern void gomp_affinity_init_place (void *); 521 extern bool gomp_affinity_add_cpus (void *, unsigned long, unsigned long, 522 long, bool); 523 extern bool gomp_affinity_remove_cpu (void *, unsigned long); 524 extern bool gomp_affinity_copy_place (void *, void *, long); 525 extern bool gomp_affinity_same_place (void *, void *); 526 extern bool gomp_affinity_finalize_place_list (bool); 527 extern bool gomp_affinity_init_level (int, unsigned long, bool); 528 extern void gomp_affinity_print_place (void *); 529 530 /* alloc.c */ 531 532 extern void *gomp_malloc (size_t) __attribute__((malloc)); 533 extern void *gomp_malloc_cleared (size_t) __attribute__((malloc)); 534 extern void *gomp_realloc (void *, size_t); 535 536 /* Avoid conflicting prototypes of alloca() in system headers by using 537 GCC's builtin alloca(). */ 538 #define gomp_alloca(x) __builtin_alloca(x) 539 540 /* error.c */ 541 542 extern void gomp_vdebug (int, const char *, va_list); 543 extern void gomp_debug (int, const char *, ...) 544 __attribute__ ((format (printf, 2, 3))); 545 #define gomp_vdebug(KIND, FMT, VALIST) \ 546 do { \ 547 if (__builtin_expect (gomp_debug_var, 0)) \ 548 (gomp_vdebug) ((KIND), (FMT), (VALIST)); \ 549 } while (0) 550 #define gomp_debug(KIND, ...) \ 551 do { \ 552 if (__builtin_expect (gomp_debug_var, 0)) \ 553 (gomp_debug) ((KIND), __VA_ARGS__); \ 554 } while (0) 555 extern void gomp_verror (const char *, va_list); 556 extern void gomp_error (const char *, ...) 557 __attribute__ ((format (printf, 1, 2))); 558 extern void gomp_vfatal (const char *, va_list) 559 __attribute__ ((noreturn)); 560 extern void gomp_fatal (const char *, ...) 561 __attribute__ ((noreturn, format (printf, 1, 2))); 562 563 /* iter.c */ 564 565 extern int gomp_iter_static_next (long *, long *); 566 extern bool gomp_iter_dynamic_next_locked (long *, long *); 567 extern bool gomp_iter_guided_next_locked (long *, long *); 568 569 #ifdef HAVE_SYNC_BUILTINS 570 extern bool gomp_iter_dynamic_next (long *, long *); 571 extern bool gomp_iter_guided_next (long *, long *); 572 #endif 573 574 /* iter_ull.c */ 575 576 extern int gomp_iter_ull_static_next (unsigned long long *, 577 unsigned long long *); 578 extern bool gomp_iter_ull_dynamic_next_locked (unsigned long long *, 579 unsigned long long *); 580 extern bool gomp_iter_ull_guided_next_locked (unsigned long long *, 581 unsigned long long *); 582 583 #if defined HAVE_SYNC_BUILTINS && defined __LP64__ 584 extern bool gomp_iter_ull_dynamic_next (unsigned long long *, 585 unsigned long long *); 586 extern bool gomp_iter_ull_guided_next (unsigned long long *, 587 unsigned long long *); 588 #endif 589 590 /* ordered.c */ 591 592 extern void gomp_ordered_first (void); 593 extern void gomp_ordered_last (void); 594 extern void gomp_ordered_next (void); 595 extern void gomp_ordered_static_init (void); 596 extern void gomp_ordered_static_next (void); 597 extern void gomp_ordered_sync (void); 598 599 /* parallel.c */ 600 601 extern unsigned gomp_resolve_num_threads (unsigned, unsigned); 602 603 /* proc.c (in config/) */ 604 605 extern void gomp_init_num_threads (void); 606 extern unsigned gomp_dynamic_max_threads (void); 607 608 /* task.c */ 609 610 extern void gomp_init_task (struct gomp_task *, struct gomp_task *, 611 struct gomp_task_icv *); 612 extern void gomp_end_task (void); 613 extern void gomp_barrier_handle_tasks (gomp_barrier_state_t); 614 615 static void inline 616 gomp_finish_task (struct gomp_task *task) 617 { 618 if (__builtin_expect (task->depend_hash != NULL, 0)) 619 free (task->depend_hash); 620 } 621 622 /* team.c */ 623 624 extern struct gomp_team *gomp_new_team (unsigned); 625 extern void gomp_team_start (void (*) (void *), void *, unsigned, 626 unsigned, struct gomp_team *); 627 extern void gomp_team_end (void); 628 extern void gomp_free_thread (void *); 629 630 /* target.c */ 631 632 extern void gomp_init_targets_once (void); 633 extern int gomp_get_num_devices (void); 634 635 typedef struct splay_tree_node_s *splay_tree_node; 636 typedef struct splay_tree_s *splay_tree; 637 typedef struct splay_tree_key_s *splay_tree_key; 638 639 struct target_mem_desc { 640 /* Reference count. */ 641 uintptr_t refcount; 642 /* All the splay nodes allocated together. */ 643 splay_tree_node array; 644 /* Start of the target region. */ 645 uintptr_t tgt_start; 646 /* End of the targer region. */ 647 uintptr_t tgt_end; 648 /* Handle to free. */ 649 void *to_free; 650 /* Previous target_mem_desc. */ 651 struct target_mem_desc *prev; 652 /* Number of items in following list. */ 653 size_t list_count; 654 655 /* Corresponding target device descriptor. */ 656 struct gomp_device_descr *device_descr; 657 658 /* List of splay keys to remove (or decrease refcount) 659 at the end of region. */ 660 splay_tree_key list[]; 661 }; 662 663 struct splay_tree_key_s { 664 /* Address of the host object. */ 665 uintptr_t host_start; 666 /* Address immediately after the host object. */ 667 uintptr_t host_end; 668 /* Descriptor of the target memory. */ 669 struct target_mem_desc *tgt; 670 /* Offset from tgt->tgt_start to the start of the target object. */ 671 uintptr_t tgt_offset; 672 /* Reference count. */ 673 uintptr_t refcount; 674 /* Asynchronous reference count. */ 675 uintptr_t async_refcount; 676 /* True if data should be copied from device to host at the end. */ 677 bool copy_from; 678 }; 679 680 #include "splay-tree.h" 681 682 typedef struct acc_dispatch_t 683 { 684 /* This is a linked list of data mapped using the 685 acc_map_data/acc_unmap_data or "acc enter data"/"acc exit data" pragmas. 686 Unlike mapped_data in the goacc_thread struct, unmapping can 687 happen out-of-order with respect to mapping. */ 688 /* This is guarded by the lock in the "outer" struct gomp_device_descr. */ 689 struct target_mem_desc *data_environ; 690 691 /* Execute. */ 692 void (*exec_func) (void (*) (void *), size_t, void **, void **, size_t *, 693 unsigned short *, int, int, int, int, void *); 694 695 /* Async cleanup callback registration. */ 696 void (*register_async_cleanup_func) (void *); 697 698 /* Asynchronous routines. */ 699 int (*async_test_func) (int); 700 int (*async_test_all_func) (void); 701 void (*async_wait_func) (int); 702 void (*async_wait_async_func) (int, int); 703 void (*async_wait_all_func) (void); 704 void (*async_wait_all_async_func) (int); 705 void (*async_set_async_func) (int); 706 707 /* Create/destroy TLS data. */ 708 void *(*create_thread_data_func) (int); 709 void (*destroy_thread_data_func) (void *); 710 711 /* NVIDIA target specific routines. */ 712 struct { 713 void *(*get_current_device_func) (void); 714 void *(*get_current_context_func) (void); 715 void *(*get_stream_func) (int); 716 int (*set_stream_func) (int, void *); 717 } cuda; 718 } acc_dispatch_t; 719 720 /* This structure describes accelerator device. 721 It contains name of the corresponding libgomp plugin, function handlers for 722 interaction with the device, ID-number of the device, and information about 723 mapped memory. */ 724 struct gomp_device_descr 725 { 726 /* Immutable data, which is only set during initialization, and which is not 727 guarded by the lock. */ 728 729 /* The name of the device. */ 730 const char *name; 731 732 /* Capabilities of device (supports OpenACC, OpenMP). */ 733 unsigned int capabilities; 734 735 /* This is the ID number of device among devices of the same type. */ 736 int target_id; 737 738 /* This is the TYPE of device. */ 739 enum offload_target_type type; 740 741 /* Function handlers. */ 742 const char *(*get_name_func) (void); 743 unsigned int (*get_caps_func) (void); 744 int (*get_type_func) (void); 745 int (*get_num_devices_func) (void); 746 void (*init_device_func) (int); 747 void (*fini_device_func) (int); 748 int (*load_image_func) (int, void *, struct addr_pair **); 749 void (*unload_image_func) (int, void *); 750 void *(*alloc_func) (int, size_t); 751 void (*free_func) (int, void *); 752 void *(*dev2host_func) (int, void *, const void *, size_t); 753 void *(*host2dev_func) (int, void *, const void *, size_t); 754 void (*run_func) (int, void *, void *); 755 756 /* Splay tree containing information about mapped memory regions. */ 757 struct splay_tree_s mem_map; 758 759 /* Mutex for the mutable data. */ 760 gomp_mutex_t lock; 761 762 /* Set to true when device is initialized. */ 763 bool is_initialized; 764 765 /* OpenACC-specific data and functions. */ 766 /* This is mutable because of its mutable data_environ and target_data 767 members. */ 768 acc_dispatch_t openacc; 769 }; 770 771 extern void gomp_acc_insert_pointer (size_t, void **, size_t *, void *); 772 extern void gomp_acc_remove_pointer (void *, bool, int, int); 773 774 extern struct target_mem_desc *gomp_map_vars (struct gomp_device_descr *, 775 size_t, void **, void **, 776 size_t *, void *, bool, bool); 777 extern void gomp_copy_from_async (struct target_mem_desc *); 778 extern void gomp_unmap_vars (struct target_mem_desc *, bool); 779 extern void gomp_init_device (struct gomp_device_descr *); 780 extern void gomp_free_memmap (struct splay_tree_s *); 781 extern void gomp_fini_device (struct gomp_device_descr *); 782 783 /* work.c */ 784 785 extern void gomp_init_work_share (struct gomp_work_share *, bool, unsigned); 786 extern void gomp_fini_work_share (struct gomp_work_share *); 787 extern bool gomp_work_share_start (bool); 788 extern void gomp_work_share_end (void); 789 extern bool gomp_work_share_end_cancel (void); 790 extern void gomp_work_share_end_nowait (void); 791 792 static inline void 793 gomp_work_share_init_done (void) 794 { 795 struct gomp_thread *thr = gomp_thread (); 796 if (__builtin_expect (thr->ts.last_work_share != NULL, 1)) 797 gomp_ptrlock_set (&thr->ts.last_work_share->next_ws, thr->ts.work_share); 798 } 799 800 #ifdef HAVE_ATTRIBUTE_VISIBILITY 801 # pragma GCC visibility pop 802 #endif 803 804 /* Now that we're back to default visibility, include the globals. */ 805 #include "libgomp_g.h" 806 807 /* Include omp.h by parts. */ 808 #include "omp-lock.h" 809 #define _LIBGOMP_OMP_LOCK_DEFINED 1 810 #include "omp.h.in" 811 812 #if !defined (HAVE_ATTRIBUTE_VISIBILITY) \ 813 || !defined (HAVE_ATTRIBUTE_ALIAS) \ 814 || !defined (HAVE_AS_SYMVER_DIRECTIVE) \ 815 || !defined (PIC) \ 816 || !defined (HAVE_SYMVER_SYMBOL_RENAMING_RUNTIME_SUPPORT) 817 # undef LIBGOMP_GNU_SYMBOL_VERSIONING 818 #endif 819 820 #ifdef LIBGOMP_GNU_SYMBOL_VERSIONING 821 extern void gomp_init_lock_30 (omp_lock_t *) __GOMP_NOTHROW; 822 extern void gomp_destroy_lock_30 (omp_lock_t *) __GOMP_NOTHROW; 823 extern void gomp_set_lock_30 (omp_lock_t *) __GOMP_NOTHROW; 824 extern void gomp_unset_lock_30 (omp_lock_t *) __GOMP_NOTHROW; 825 extern int gomp_test_lock_30 (omp_lock_t *) __GOMP_NOTHROW; 826 extern void gomp_init_nest_lock_30 (omp_nest_lock_t *) __GOMP_NOTHROW; 827 extern void gomp_destroy_nest_lock_30 (omp_nest_lock_t *) __GOMP_NOTHROW; 828 extern void gomp_set_nest_lock_30 (omp_nest_lock_t *) __GOMP_NOTHROW; 829 extern void gomp_unset_nest_lock_30 (omp_nest_lock_t *) __GOMP_NOTHROW; 830 extern int gomp_test_nest_lock_30 (omp_nest_lock_t *) __GOMP_NOTHROW; 831 832 extern void gomp_init_lock_25 (omp_lock_25_t *) __GOMP_NOTHROW; 833 extern void gomp_destroy_lock_25 (omp_lock_25_t *) __GOMP_NOTHROW; 834 extern void gomp_set_lock_25 (omp_lock_25_t *) __GOMP_NOTHROW; 835 extern void gomp_unset_lock_25 (omp_lock_25_t *) __GOMP_NOTHROW; 836 extern int gomp_test_lock_25 (omp_lock_25_t *) __GOMP_NOTHROW; 837 extern void gomp_init_nest_lock_25 (omp_nest_lock_25_t *) __GOMP_NOTHROW; 838 extern void gomp_destroy_nest_lock_25 (omp_nest_lock_25_t *) __GOMP_NOTHROW; 839 extern void gomp_set_nest_lock_25 (omp_nest_lock_25_t *) __GOMP_NOTHROW; 840 extern void gomp_unset_nest_lock_25 (omp_nest_lock_25_t *) __GOMP_NOTHROW; 841 extern int gomp_test_nest_lock_25 (omp_nest_lock_25_t *) __GOMP_NOTHROW; 842 843 # define strong_alias(fn, al) \ 844 extern __typeof (fn) al __attribute__ ((alias (#fn))); 845 # define omp_lock_symver(fn) \ 846 __asm (".symver g" #fn "_30, " #fn "@@OMP_3.0"); \ 847 __asm (".symver g" #fn "_25, " #fn "@OMP_1.0"); 848 #else 849 # define gomp_init_lock_30 omp_init_lock 850 # define gomp_destroy_lock_30 omp_destroy_lock 851 # define gomp_set_lock_30 omp_set_lock 852 # define gomp_unset_lock_30 omp_unset_lock 853 # define gomp_test_lock_30 omp_test_lock 854 # define gomp_init_nest_lock_30 omp_init_nest_lock 855 # define gomp_destroy_nest_lock_30 omp_destroy_nest_lock 856 # define gomp_set_nest_lock_30 omp_set_nest_lock 857 # define gomp_unset_nest_lock_30 omp_unset_nest_lock 858 # define gomp_test_nest_lock_30 omp_test_nest_lock 859 #endif 860 861 #ifdef HAVE_ATTRIBUTE_VISIBILITY 862 # define attribute_hidden __attribute__ ((visibility ("hidden"))) 863 #else 864 # define attribute_hidden 865 #endif 866 867 #ifdef HAVE_ATTRIBUTE_ALIAS 868 # define ialias_ulp ialias_str1(__USER_LABEL_PREFIX__) 869 # define ialias_str1(x) ialias_str2(x) 870 # define ialias_str2(x) #x 871 # define ialias(fn) \ 872 extern __typeof (fn) gomp_ialias_##fn \ 873 __attribute__ ((alias (#fn))) attribute_hidden; 874 # define ialias_redirect(fn) \ 875 extern __typeof (fn) fn __asm__ (ialias_ulp "gomp_ialias_" #fn) attribute_hidden; 876 # define ialias_call(fn) gomp_ialias_ ## fn 877 #else 878 # define ialias(fn) 879 # define ialias_redirect(fn) 880 # define ialias_call(fn) fn 881 #endif 882 883 #endif /* LIBGOMP_H */ 884