1 /* Copyright (C) 2013-2016 Free Software Foundation, Inc. 2 3 Contributed by Mentor Embedded. 4 5 This file is part of the GNU Offloading and Multi Processing Library 6 (libgomp). 7 8 Libgomp is free software; you can redistribute it and/or modify it 9 under the terms of the GNU General Public License as published by 10 the Free Software Foundation; either version 3, or (at your option) 11 any later version. 12 13 Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY 14 WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 15 FOR A PARTICULAR PURPOSE. See the GNU General Public License for 16 more details. 17 18 Under Section 7 of GPL version 3, you are granted additional 19 permissions described in the GCC Runtime Library Exception, version 20 3.1, as published by the Free Software Foundation. 21 22 You should have received a copy of the GNU General Public License and 23 a copy of the GCC Runtime Library Exception along with this program; 24 see the files COPYING3 and COPYING.RUNTIME respectively. If not, see 25 <http://www.gnu.org/licenses/>. */ 26 27 /* This file handles OpenACC constructs. */ 28 29 #include "openacc.h" 30 #include "libgomp.h" 31 #include "libgomp_g.h" 32 #include "gomp-constants.h" 33 #include "oacc-int.h" 34 #ifdef HAVE_INTTYPES_H 35 # include <inttypes.h> /* For PRIu64. */ 36 #endif 37 #include <string.h> 38 #include <stdarg.h> 39 #include <assert.h> 40 41 static int 42 find_pset (int pos, size_t mapnum, unsigned short *kinds) 43 { 44 if (pos + 1 >= mapnum) 45 return 0; 46 47 unsigned char kind = kinds[pos+1] & 0xff; 48 49 return kind == GOMP_MAP_TO_PSET; 50 } 51 52 static void goacc_wait (int async, int num_waits, va_list *ap); 53 54 55 /* Launch a possibly offloaded function on DEVICE. FN is the host fn 56 address. MAPNUM, HOSTADDRS, SIZES & KINDS describe the memory 57 blocks to be copied to/from the device. Varadic arguments are 58 keyed optional parameters terminated with a zero. */ 59 60 void 61 GOACC_parallel_keyed (int device, void (*fn) (void *), 62 size_t mapnum, void **hostaddrs, size_t *sizes, 63 unsigned short *kinds, ...) 64 { 65 bool host_fallback = device == GOMP_DEVICE_HOST_FALLBACK; 66 va_list ap; 67 struct goacc_thread *thr; 68 struct gomp_device_descr *acc_dev; 69 struct target_mem_desc *tgt; 70 void **devaddrs; 71 unsigned int i; 72 struct splay_tree_key_s k; 73 splay_tree_key tgt_fn_key; 74 void (*tgt_fn); 75 int async = GOMP_ASYNC_SYNC; 76 unsigned dims[GOMP_DIM_MAX]; 77 unsigned tag; 78 79 #ifdef HAVE_INTTYPES_H 80 gomp_debug (0, "%s: mapnum=%"PRIu64", hostaddrs=%p, size=%p, kinds=%p\n", 81 __FUNCTION__, (uint64_t) mapnum, hostaddrs, sizes, kinds); 82 #else 83 gomp_debug (0, "%s: mapnum=%lu, hostaddrs=%p, sizes=%p, kinds=%p\n", 84 __FUNCTION__, (unsigned long) mapnum, hostaddrs, sizes, kinds); 85 #endif 86 goacc_lazy_initialize (); 87 88 thr = goacc_thread (); 89 acc_dev = thr->dev; 90 91 /* Host fallback if "if" clause is false or if the current device is set to 92 the host. */ 93 if (host_fallback) 94 { 95 goacc_save_and_set_bind (acc_device_host); 96 fn (hostaddrs); 97 goacc_restore_bind (); 98 return; 99 } 100 else if (acc_device_type (acc_dev->type) == acc_device_host) 101 { 102 fn (hostaddrs); 103 return; 104 } 105 106 /* Default: let the runtime choose. */ 107 for (i = 0; i != GOMP_DIM_MAX; i++) 108 dims[i] = 0; 109 110 va_start (ap, kinds); 111 /* TODO: This will need amending when device_type is implemented. */ 112 while ((tag = va_arg (ap, unsigned)) != 0) 113 { 114 if (GOMP_LAUNCH_DEVICE (tag)) 115 gomp_fatal ("device_type '%d' offload parameters, libgomp is too old", 116 GOMP_LAUNCH_DEVICE (tag)); 117 118 switch (GOMP_LAUNCH_CODE (tag)) 119 { 120 case GOMP_LAUNCH_DIM: 121 { 122 unsigned mask = GOMP_LAUNCH_OP (tag); 123 124 for (i = 0; i != GOMP_DIM_MAX; i++) 125 if (mask & GOMP_DIM_MASK (i)) 126 dims[i] = va_arg (ap, unsigned); 127 } 128 break; 129 130 case GOMP_LAUNCH_ASYNC: 131 { 132 /* Small constant values are encoded in the operand. */ 133 async = GOMP_LAUNCH_OP (tag); 134 135 if (async == GOMP_LAUNCH_OP_MAX) 136 async = va_arg (ap, unsigned); 137 break; 138 } 139 140 case GOMP_LAUNCH_WAIT: 141 { 142 unsigned num_waits = GOMP_LAUNCH_OP (tag); 143 144 if (num_waits) 145 goacc_wait (async, num_waits, &ap); 146 break; 147 } 148 149 default: 150 gomp_fatal ("unrecognized offload code '%d'," 151 " libgomp is too old", GOMP_LAUNCH_CODE (tag)); 152 } 153 } 154 va_end (ap); 155 156 acc_dev->openacc.async_set_async_func (async); 157 158 if (!(acc_dev->capabilities & GOMP_OFFLOAD_CAP_NATIVE_EXEC)) 159 { 160 k.host_start = (uintptr_t) fn; 161 k.host_end = k.host_start + 1; 162 gomp_mutex_lock (&acc_dev->lock); 163 tgt_fn_key = splay_tree_lookup (&acc_dev->mem_map, &k); 164 gomp_mutex_unlock (&acc_dev->lock); 165 166 if (tgt_fn_key == NULL) 167 gomp_fatal ("target function wasn't mapped"); 168 169 tgt_fn = (void (*)) tgt_fn_key->tgt_offset; 170 } 171 else 172 tgt_fn = (void (*)) fn; 173 174 tgt = gomp_map_vars (acc_dev, mapnum, hostaddrs, NULL, sizes, kinds, true, 175 GOMP_MAP_VARS_OPENACC); 176 177 devaddrs = gomp_alloca (sizeof (void *) * mapnum); 178 for (i = 0; i < mapnum; i++) 179 devaddrs[i] = (void *) (tgt->list[i].key->tgt->tgt_start 180 + tgt->list[i].key->tgt_offset); 181 182 acc_dev->openacc.exec_func (tgt_fn, mapnum, hostaddrs, devaddrs, 183 async, dims, tgt); 184 185 /* If running synchronously, unmap immediately. */ 186 if (async < acc_async_noval) 187 gomp_unmap_vars (tgt, true); 188 else 189 { 190 gomp_copy_from_async (tgt); 191 acc_dev->openacc.register_async_cleanup_func (tgt); 192 } 193 194 acc_dev->openacc.async_set_async_func (acc_async_sync); 195 } 196 197 /* Legacy entry point, only provide host execution. */ 198 199 void 200 GOACC_parallel (int device, void (*fn) (void *), 201 size_t mapnum, void **hostaddrs, size_t *sizes, 202 unsigned short *kinds, 203 int num_gangs, int num_workers, int vector_length, 204 int async, int num_waits, ...) 205 { 206 goacc_save_and_set_bind (acc_device_host); 207 fn (hostaddrs); 208 goacc_restore_bind (); 209 } 210 211 void 212 GOACC_data_start (int device, size_t mapnum, 213 void **hostaddrs, size_t *sizes, unsigned short *kinds) 214 { 215 bool host_fallback = device == GOMP_DEVICE_HOST_FALLBACK; 216 struct target_mem_desc *tgt; 217 218 #ifdef HAVE_INTTYPES_H 219 gomp_debug (0, "%s: mapnum=%"PRIu64", hostaddrs=%p, size=%p, kinds=%p\n", 220 __FUNCTION__, (uint64_t) mapnum, hostaddrs, sizes, kinds); 221 #else 222 gomp_debug (0, "%s: mapnum=%lu, hostaddrs=%p, sizes=%p, kinds=%p\n", 223 __FUNCTION__, (unsigned long) mapnum, hostaddrs, sizes, kinds); 224 #endif 225 226 goacc_lazy_initialize (); 227 228 struct goacc_thread *thr = goacc_thread (); 229 struct gomp_device_descr *acc_dev = thr->dev; 230 231 /* Host fallback or 'do nothing'. */ 232 if ((acc_dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM) 233 || host_fallback) 234 { 235 tgt = gomp_map_vars (NULL, 0, NULL, NULL, NULL, NULL, true, 236 GOMP_MAP_VARS_OPENACC); 237 tgt->prev = thr->mapped_data; 238 thr->mapped_data = tgt; 239 240 return; 241 } 242 243 gomp_debug (0, " %s: prepare mappings\n", __FUNCTION__); 244 tgt = gomp_map_vars (acc_dev, mapnum, hostaddrs, NULL, sizes, kinds, true, 245 GOMP_MAP_VARS_OPENACC); 246 gomp_debug (0, " %s: mappings prepared\n", __FUNCTION__); 247 tgt->prev = thr->mapped_data; 248 thr->mapped_data = tgt; 249 } 250 251 void 252 GOACC_data_end (void) 253 { 254 struct goacc_thread *thr = goacc_thread (); 255 struct target_mem_desc *tgt = thr->mapped_data; 256 257 gomp_debug (0, " %s: restore mappings\n", __FUNCTION__); 258 thr->mapped_data = tgt->prev; 259 gomp_unmap_vars (tgt, true); 260 gomp_debug (0, " %s: mappings restored\n", __FUNCTION__); 261 } 262 263 void 264 GOACC_enter_exit_data (int device, size_t mapnum, 265 void **hostaddrs, size_t *sizes, unsigned short *kinds, 266 int async, int num_waits, ...) 267 { 268 struct goacc_thread *thr; 269 struct gomp_device_descr *acc_dev; 270 bool host_fallback = device == GOMP_DEVICE_HOST_FALLBACK; 271 bool data_enter = false; 272 size_t i; 273 274 goacc_lazy_initialize (); 275 276 thr = goacc_thread (); 277 acc_dev = thr->dev; 278 279 if ((acc_dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM) 280 || host_fallback) 281 return; 282 283 if (num_waits) 284 { 285 va_list ap; 286 287 va_start (ap, num_waits); 288 goacc_wait (async, num_waits, &ap); 289 va_end (ap); 290 } 291 292 acc_dev->openacc.async_set_async_func (async); 293 294 /* Determine if this is an "acc enter data". */ 295 for (i = 0; i < mapnum; ++i) 296 { 297 unsigned char kind = kinds[i] & 0xff; 298 299 if (kind == GOMP_MAP_POINTER || kind == GOMP_MAP_TO_PSET) 300 continue; 301 302 if (kind == GOMP_MAP_FORCE_ALLOC 303 || kind == GOMP_MAP_FORCE_PRESENT 304 || kind == GOMP_MAP_FORCE_TO) 305 { 306 data_enter = true; 307 break; 308 } 309 310 if (kind == GOMP_MAP_DELETE 311 || kind == GOMP_MAP_FORCE_FROM) 312 break; 313 314 gomp_fatal (">>>> GOACC_enter_exit_data UNHANDLED kind 0x%.2x", 315 kind); 316 } 317 318 if (data_enter) 319 { 320 for (i = 0; i < mapnum; i++) 321 { 322 unsigned char kind = kinds[i] & 0xff; 323 324 /* Scan for PSETs. */ 325 int psets = find_pset (i, mapnum, kinds); 326 327 if (!psets) 328 { 329 switch (kind) 330 { 331 case GOMP_MAP_POINTER: 332 gomp_acc_insert_pointer (1, &hostaddrs[i], &sizes[i], 333 &kinds[i]); 334 break; 335 case GOMP_MAP_FORCE_ALLOC: 336 acc_create (hostaddrs[i], sizes[i]); 337 break; 338 case GOMP_MAP_FORCE_PRESENT: 339 acc_present_or_copyin (hostaddrs[i], sizes[i]); 340 break; 341 case GOMP_MAP_FORCE_TO: 342 acc_present_or_copyin (hostaddrs[i], sizes[i]); 343 break; 344 default: 345 gomp_fatal (">>>> GOACC_enter_exit_data UNHANDLED kind 0x%.2x", 346 kind); 347 break; 348 } 349 } 350 else 351 { 352 gomp_acc_insert_pointer (3, &hostaddrs[i], &sizes[i], &kinds[i]); 353 /* Increment 'i' by two because OpenACC requires fortran 354 arrays to be contiguous, so each PSET is associated with 355 one of MAP_FORCE_ALLOC/MAP_FORCE_PRESET/MAP_FORCE_TO, and 356 one MAP_POINTER. */ 357 i += 2; 358 } 359 } 360 } 361 else 362 for (i = 0; i < mapnum; ++i) 363 { 364 unsigned char kind = kinds[i] & 0xff; 365 366 int psets = find_pset (i, mapnum, kinds); 367 368 if (!psets) 369 { 370 switch (kind) 371 { 372 case GOMP_MAP_POINTER: 373 gomp_acc_remove_pointer (hostaddrs[i], (kinds[i] & 0xff) 374 == GOMP_MAP_FORCE_FROM, 375 async, 1); 376 break; 377 case GOMP_MAP_DELETE: 378 acc_delete (hostaddrs[i], sizes[i]); 379 break; 380 case GOMP_MAP_FORCE_FROM: 381 acc_copyout (hostaddrs[i], sizes[i]); 382 break; 383 default: 384 gomp_fatal (">>>> GOACC_enter_exit_data UNHANDLED kind 0x%.2x", 385 kind); 386 break; 387 } 388 } 389 else 390 { 391 gomp_acc_remove_pointer (hostaddrs[i], (kinds[i] & 0xff) 392 == GOMP_MAP_FORCE_FROM, async, 3); 393 /* See the above comment. */ 394 i += 2; 395 } 396 } 397 398 acc_dev->openacc.async_set_async_func (acc_async_sync); 399 } 400 401 static void 402 goacc_wait (int async, int num_waits, va_list *ap) 403 { 404 struct goacc_thread *thr = goacc_thread (); 405 struct gomp_device_descr *acc_dev = thr->dev; 406 407 while (num_waits--) 408 { 409 int qid = va_arg (*ap, int); 410 411 if (acc_async_test (qid)) 412 continue; 413 414 if (async == acc_async_sync) 415 acc_wait (qid); 416 else if (qid == async) 417 ;/* If we're waiting on the same asynchronous queue as we're 418 launching on, the queue itself will order work as 419 required, so there's no need to wait explicitly. */ 420 else 421 acc_dev->openacc.async_wait_async_func (qid, async); 422 } 423 } 424 425 void 426 GOACC_update (int device, size_t mapnum, 427 void **hostaddrs, size_t *sizes, unsigned short *kinds, 428 int async, int num_waits, ...) 429 { 430 bool host_fallback = device == GOMP_DEVICE_HOST_FALLBACK; 431 size_t i; 432 433 goacc_lazy_initialize (); 434 435 struct goacc_thread *thr = goacc_thread (); 436 struct gomp_device_descr *acc_dev = thr->dev; 437 438 if ((acc_dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM) 439 || host_fallback) 440 return; 441 442 if (num_waits) 443 { 444 va_list ap; 445 446 va_start (ap, num_waits); 447 goacc_wait (async, num_waits, &ap); 448 va_end (ap); 449 } 450 451 acc_dev->openacc.async_set_async_func (async); 452 453 for (i = 0; i < mapnum; ++i) 454 { 455 unsigned char kind = kinds[i] & 0xff; 456 457 switch (kind) 458 { 459 case GOMP_MAP_POINTER: 460 case GOMP_MAP_TO_PSET: 461 break; 462 463 case GOMP_MAP_FORCE_TO: 464 acc_update_device (hostaddrs[i], sizes[i]); 465 break; 466 467 case GOMP_MAP_FORCE_FROM: 468 acc_update_self (hostaddrs[i], sizes[i]); 469 break; 470 471 default: 472 gomp_fatal (">>>> GOACC_update UNHANDLED kind 0x%.2x", kind); 473 break; 474 } 475 } 476 477 acc_dev->openacc.async_set_async_func (acc_async_sync); 478 } 479 480 void 481 GOACC_wait (int async, int num_waits, ...) 482 { 483 if (num_waits) 484 { 485 va_list ap; 486 487 va_start (ap, num_waits); 488 goacc_wait (async, num_waits, &ap); 489 va_end (ap); 490 } 491 else if (async == acc_async_sync) 492 acc_wait_all (); 493 else if (async == acc_async_noval) 494 goacc_thread ()->dev->openacc.async_wait_all_async_func (acc_async_noval); 495 } 496 497 int 498 GOACC_get_num_threads (void) 499 { 500 return 1; 501 } 502 503 int 504 GOACC_get_thread_num (void) 505 { 506 return 0; 507 } 508 509 void 510 GOACC_declare (int device, size_t mapnum, 511 void **hostaddrs, size_t *sizes, unsigned short *kinds) 512 { 513 int i; 514 515 for (i = 0; i < mapnum; i++) 516 { 517 unsigned char kind = kinds[i] & 0xff; 518 519 if (kind == GOMP_MAP_POINTER || kind == GOMP_MAP_TO_PSET) 520 continue; 521 522 switch (kind) 523 { 524 case GOMP_MAP_FORCE_ALLOC: 525 case GOMP_MAP_FORCE_FROM: 526 case GOMP_MAP_FORCE_TO: 527 case GOMP_MAP_POINTER: 528 case GOMP_MAP_DELETE: 529 GOACC_enter_exit_data (device, 1, &hostaddrs[i], &sizes[i], 530 &kinds[i], 0, 0); 531 break; 532 533 case GOMP_MAP_FORCE_DEVICEPTR: 534 break; 535 536 case GOMP_MAP_ALLOC: 537 if (!acc_is_present (hostaddrs[i], sizes[i])) 538 GOACC_enter_exit_data (device, 1, &hostaddrs[i], &sizes[i], 539 &kinds[i], 0, 0); 540 break; 541 542 case GOMP_MAP_TO: 543 GOACC_enter_exit_data (device, 1, &hostaddrs[i], &sizes[i], 544 &kinds[i], 0, 0); 545 546 break; 547 548 case GOMP_MAP_FROM: 549 kinds[i] = GOMP_MAP_FORCE_FROM; 550 GOACC_enter_exit_data (device, 1, &hostaddrs[i], &sizes[i], 551 &kinds[i], 0, 0); 552 break; 553 554 case GOMP_MAP_FORCE_PRESENT: 555 if (!acc_is_present (hostaddrs[i], sizes[i])) 556 gomp_fatal ("[%p,%ld] is not mapped", hostaddrs[i], 557 (unsigned long) sizes[i]); 558 break; 559 560 default: 561 assert (0); 562 break; 563 } 564 } 565 } 566