1 //===-- OpenMP/OMPT/Callback.cpp - OpenMP Tooling Callback implementation -===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // Implementation of OMPT callback interfaces for target independent layer 10 // 11 //===----------------------------------------------------------------------===// 12 13 #ifdef OMPT_SUPPORT 14 15 #include <cstdlib> 16 #include <cstring> 17 #include <memory> 18 19 #include "Shared/Debug.h" 20 21 #include "OpenMP/OMPT/Callback.h" 22 #include "OpenMP/OMPT/Connector.h" 23 #include "OpenMP/OMPT/Interface.h" 24 25 #include "llvm/Support/DynamicLibrary.h" 26 27 #undef DEBUG_PREFIX 28 #define DEBUG_PREFIX "OMPT" 29 30 // Define OMPT callback functions (bound to actual callbacks later on) 31 #define defineOmptCallback(Name, Type, Code) \ 32 Name##_t llvm::omp::target::ompt::Name##_fn = nullptr; 33 FOREACH_OMPT_NOEMI_EVENT(defineOmptCallback) 34 FOREACH_OMPT_EMI_EVENT(defineOmptCallback) 35 #undef defineOmptCallback 36 37 using namespace llvm::omp::target::ompt; 38 39 /// Forward declaration 40 class LibomptargetRtlFinalizer; 41 42 /// Object that will maintain the RTL finalizer from the plugin 43 LibomptargetRtlFinalizer *LibraryFinalizer = nullptr; 44 45 thread_local Interface llvm::omp::target::ompt::RegionInterface; 46 47 thread_local void *llvm::omp::target::ompt::ReturnAddress = nullptr; 48 49 bool llvm::omp::target::ompt::Initialized = false; 50 51 ompt_get_callback_t llvm::omp::target::ompt::lookupCallbackByCode = nullptr; 52 ompt_function_lookup_t llvm::omp::target::ompt::lookupCallbackByName = nullptr; 53 ompt_get_target_task_data_t ompt_get_target_task_data_fn = nullptr; 54 ompt_get_task_data_t ompt_get_task_data_fn = nullptr; 55 56 /// Unique correlation id 57 static std::atomic<uint64_t> IdCounter(1); 58 59 /// Used to create a new correlation id 60 static uint64_t createId() { return IdCounter.fetch_add(1); } 61 62 /// Create a new correlation id and update the operations id 63 static uint64_t createOpId() { 64 uint64_t NewId = createId(); 65 RegionInterface.setHostOpId(NewId); 66 return NewId; 67 } 68 69 /// Create a new correlation id and update the target region id 70 static uint64_t createRegionId() { 71 uint64_t NewId = createId(); 72 RegionInterface.setTargetDataValue(NewId); 73 return NewId; 74 } 75 76 void Interface::beginTargetDataAlloc(int64_t DeviceId, void *HstPtrBegin, 77 void **TgtPtrBegin, size_t Size, 78 void *Code) { 79 beginTargetDataOperation(); 80 if (ompt_callback_target_data_op_emi_fn) { 81 // HostOpId will be set by the tool. Invoke the tool supplied data op EMI 82 // callback 83 ompt_callback_target_data_op_emi_fn( 84 ompt_scope_begin, TargetTaskData, &TargetData, &HostOpId, 85 ompt_target_data_alloc, HstPtrBegin, 86 /*SrcDeviceNum=*/omp_get_initial_device(), *TgtPtrBegin, 87 /*TgtDeviceNum=*/DeviceId, Size, Code); 88 } else if (ompt_callback_target_data_op_fn) { 89 // HostOpId is set by the runtime 90 HostOpId = createOpId(); 91 // Invoke the tool supplied data op callback 92 ompt_callback_target_data_op_fn( 93 TargetData.value, HostOpId, ompt_target_data_alloc, HstPtrBegin, 94 /*SrcDeviceNum=*/omp_get_initial_device(), *TgtPtrBegin, 95 /*TgtDeviceNum=*/DeviceId, Size, Code); 96 } 97 } 98 99 void Interface::endTargetDataAlloc(int64_t DeviceId, void *HstPtrBegin, 100 void **TgtPtrBegin, size_t Size, 101 void *Code) { 102 // Only EMI callback handles end scope 103 if (ompt_callback_target_data_op_emi_fn) { 104 // HostOpId will be set by the tool. Invoke the tool supplied data op EMI 105 // callback 106 ompt_callback_target_data_op_emi_fn( 107 ompt_scope_end, TargetTaskData, &TargetData, &HostOpId, 108 ompt_target_data_alloc, HstPtrBegin, 109 /*SrcDeviceNum=*/omp_get_initial_device(), *TgtPtrBegin, 110 /*TgtDeviceNum=*/DeviceId, Size, Code); 111 } 112 endTargetDataOperation(); 113 } 114 115 void Interface::beginTargetDataSubmit(int64_t SrcDeviceId, void *SrcPtrBegin, 116 int64_t DstDeviceId, void *DstPtrBegin, 117 size_t Size, void *Code) { 118 beginTargetDataOperation(); 119 if (ompt_callback_target_data_op_emi_fn) { 120 // HostOpId will be set by the tool. Invoke the tool supplied data op EMI 121 // callback 122 ompt_callback_target_data_op_emi_fn( 123 ompt_scope_begin, TargetTaskData, &TargetData, &HostOpId, 124 ompt_target_data_transfer_to_device, SrcPtrBegin, SrcDeviceId, 125 DstPtrBegin, DstDeviceId, Size, Code); 126 } else if (ompt_callback_target_data_op_fn) { 127 // HostOpId is set by the runtime 128 HostOpId = createOpId(); 129 // Invoke the tool supplied data op callback 130 ompt_callback_target_data_op_fn( 131 TargetData.value, HostOpId, ompt_target_data_transfer_to_device, 132 SrcPtrBegin, SrcDeviceId, DstPtrBegin, DstDeviceId, Size, Code); 133 } 134 } 135 136 void Interface::endTargetDataSubmit(int64_t SrcDeviceId, void *SrcPtrBegin, 137 int64_t DstDeviceId, void *DstPtrBegin, 138 size_t Size, void *Code) { 139 // Only EMI callback handles end scope 140 if (ompt_callback_target_data_op_emi_fn) { 141 // HostOpId will be set by the tool. Invoke the tool supplied data op EMI 142 // callback 143 ompt_callback_target_data_op_emi_fn( 144 ompt_scope_end, TargetTaskData, &TargetData, &HostOpId, 145 ompt_target_data_transfer_to_device, SrcPtrBegin, SrcDeviceId, 146 DstPtrBegin, DstDeviceId, Size, Code); 147 } 148 endTargetDataOperation(); 149 } 150 151 void Interface::beginTargetDataDelete(int64_t DeviceId, void *TgtPtrBegin, 152 void *Code) { 153 beginTargetDataOperation(); 154 if (ompt_callback_target_data_op_emi_fn) { 155 // HostOpId will be set by the tool. Invoke the tool supplied data op EMI 156 // callback 157 ompt_callback_target_data_op_emi_fn( 158 ompt_scope_begin, TargetTaskData, &TargetData, &HostOpId, 159 ompt_target_data_delete, TgtPtrBegin, DeviceId, 160 /*TgtPtrBegin=*/nullptr, /*TgtDeviceNum=*/-1, /*Bytes=*/0, Code); 161 } else if (ompt_callback_target_data_op_fn) { 162 // HostOpId is set by the runtime 163 HostOpId = createOpId(); 164 // Invoke the tool supplied data op callback 165 ompt_callback_target_data_op_fn(TargetData.value, HostOpId, 166 ompt_target_data_delete, TgtPtrBegin, 167 DeviceId, /*TgtPtrBegin=*/nullptr, 168 /*TgtDeviceNum=*/-1, /*Bytes=*/0, Code); 169 } 170 } 171 172 void Interface::endTargetDataDelete(int64_t DeviceId, void *TgtPtrBegin, 173 void *Code) { 174 // Only EMI callback handles end scope 175 if (ompt_callback_target_data_op_emi_fn) { 176 // HostOpId will be set by the tool. Invoke the tool supplied data op EMI 177 // callback 178 ompt_callback_target_data_op_emi_fn( 179 ompt_scope_end, TargetTaskData, &TargetData, &HostOpId, 180 ompt_target_data_delete, TgtPtrBegin, DeviceId, 181 /*TgtPtrBegin=*/nullptr, /*TgtDeviceNum=*/-1, /*Bytes=*/0, Code); 182 } 183 endTargetDataOperation(); 184 } 185 186 void Interface::beginTargetDataRetrieve(int64_t SrcDeviceId, void *SrcPtrBegin, 187 int64_t DstDeviceId, void *DstPtrBegin, 188 size_t Size, void *Code) { 189 beginTargetDataOperation(); 190 if (ompt_callback_target_data_op_emi_fn) { 191 // HostOpId will be set by the tool. Invoke the tool supplied data op EMI 192 // callback 193 ompt_callback_target_data_op_emi_fn( 194 ompt_scope_begin, TargetTaskData, &TargetData, &HostOpId, 195 ompt_target_data_transfer_from_device, SrcPtrBegin, SrcDeviceId, 196 DstPtrBegin, DstDeviceId, Size, Code); 197 } else if (ompt_callback_target_data_op_fn) { 198 // HostOpId is set by the runtime 199 HostOpId = createOpId(); 200 // Invoke the tool supplied data op callback 201 ompt_callback_target_data_op_fn( 202 TargetData.value, HostOpId, ompt_target_data_transfer_from_device, 203 SrcPtrBegin, SrcDeviceId, DstPtrBegin, DstDeviceId, Size, Code); 204 } 205 } 206 207 void Interface::endTargetDataRetrieve(int64_t SrcDeviceId, void *SrcPtrBegin, 208 int64_t DstDeviceId, void *DstPtrBegin, 209 size_t Size, void *Code) { 210 // Only EMI callback handles end scope 211 if (ompt_callback_target_data_op_emi_fn) { 212 // HostOpId will be set by the tool. Invoke the tool supplied data op EMI 213 // callback 214 ompt_callback_target_data_op_emi_fn( 215 ompt_scope_end, TargetTaskData, &TargetData, &HostOpId, 216 ompt_target_data_transfer_from_device, SrcPtrBegin, SrcDeviceId, 217 DstPtrBegin, DstDeviceId, Size, Code); 218 } 219 endTargetDataOperation(); 220 } 221 222 void Interface::beginTargetSubmit(unsigned int NumTeams) { 223 if (ompt_callback_target_submit_emi_fn) { 224 // HostOpId is set by the tool. Invoke the tool supplied target submit EMI 225 // callback 226 ompt_callback_target_submit_emi_fn(ompt_scope_begin, &TargetData, &HostOpId, 227 NumTeams); 228 } else if (ompt_callback_target_submit_fn) { 229 // HostOpId is set by the runtime 230 HostOpId = createOpId(); 231 ompt_callback_target_submit_fn(TargetData.value, HostOpId, NumTeams); 232 } 233 } 234 235 void Interface::endTargetSubmit(unsigned int NumTeams) { 236 // Only EMI callback handles end scope 237 if (ompt_callback_target_submit_emi_fn) { 238 // HostOpId is set by the tool. Invoke the tool supplied target submit EMI 239 // callback 240 ompt_callback_target_submit_emi_fn(ompt_scope_end, &TargetData, &HostOpId, 241 NumTeams); 242 } 243 } 244 245 void Interface::beginTargetDataEnter(int64_t DeviceId, void *Code) { 246 beginTargetRegion(); 247 if (ompt_callback_target_emi_fn) { 248 // Invoke the tool supplied target EMI callback 249 ompt_callback_target_emi_fn(ompt_target_enter_data, ompt_scope_begin, 250 DeviceId, TaskData, TargetTaskData, &TargetData, 251 Code); 252 } else if (ompt_callback_target_fn) { 253 // Invoke the tool supplied target callback 254 ompt_callback_target_fn(ompt_target_enter_data, ompt_scope_begin, DeviceId, 255 TaskData, TargetData.value, Code); 256 } 257 } 258 259 void Interface::endTargetDataEnter(int64_t DeviceId, void *Code) { 260 if (ompt_callback_target_emi_fn) { 261 // Invoke the tool supplied target EMI callback 262 ompt_callback_target_emi_fn(ompt_target_enter_data, ompt_scope_end, 263 DeviceId, TaskData, TargetTaskData, &TargetData, 264 Code); 265 } else if (ompt_callback_target_fn) { 266 // Invoke the tool supplied target callback 267 ompt_callback_target_fn(ompt_target_enter_data, ompt_scope_end, DeviceId, 268 TaskData, TargetData.value, Code); 269 } 270 endTargetRegion(); 271 } 272 273 void Interface::beginTargetDataExit(int64_t DeviceId, void *Code) { 274 beginTargetRegion(); 275 if (ompt_callback_target_emi_fn) { 276 // Invoke the tool supplied target EMI callback 277 ompt_callback_target_emi_fn(ompt_target_exit_data, ompt_scope_begin, 278 DeviceId, TaskData, TargetTaskData, &TargetData, 279 Code); 280 } else if (ompt_callback_target_fn) { 281 TargetData.value = createRegionId(); 282 // Invoke the tool supplied target callback 283 ompt_callback_target_fn(ompt_target_exit_data, ompt_scope_begin, DeviceId, 284 TaskData, TargetData.value, Code); 285 } 286 } 287 288 void Interface::endTargetDataExit(int64_t DeviceId, void *Code) { 289 if (ompt_callback_target_emi_fn) { 290 // Invoke the tool supplied target EMI callback 291 ompt_callback_target_emi_fn(ompt_target_exit_data, ompt_scope_end, DeviceId, 292 TaskData, TargetTaskData, &TargetData, Code); 293 } else if (ompt_callback_target_fn) { 294 // Invoke the tool supplied target callback 295 ompt_callback_target_fn(ompt_target_exit_data, ompt_scope_end, DeviceId, 296 TaskData, TargetData.value, Code); 297 } 298 endTargetRegion(); 299 } 300 301 void Interface::beginTargetUpdate(int64_t DeviceId, void *Code) { 302 beginTargetRegion(); 303 if (ompt_callback_target_emi_fn) { 304 // Invoke the tool supplied target EMI callback 305 ompt_callback_target_emi_fn(ompt_target_update, ompt_scope_begin, DeviceId, 306 TaskData, TargetTaskData, &TargetData, Code); 307 } else if (ompt_callback_target_fn) { 308 TargetData.value = createRegionId(); 309 // Invoke the tool supplied target callback 310 ompt_callback_target_fn(ompt_target_update, ompt_scope_begin, DeviceId, 311 TaskData, TargetData.value, Code); 312 } 313 } 314 315 void Interface::endTargetUpdate(int64_t DeviceId, void *Code) { 316 if (ompt_callback_target_emi_fn) { 317 // Invoke the tool supplied target EMI callback 318 ompt_callback_target_emi_fn(ompt_target_update, ompt_scope_end, DeviceId, 319 TaskData, TargetTaskData, &TargetData, Code); 320 } else if (ompt_callback_target_fn) { 321 // Invoke the tool supplied target callback 322 ompt_callback_target_fn(ompt_target_update, ompt_scope_end, DeviceId, 323 TaskData, TargetData.value, Code); 324 } 325 endTargetRegion(); 326 } 327 328 void Interface::beginTargetAssociatePointer(int64_t DeviceId, void *HstPtrBegin, 329 void *TgtPtrBegin, size_t Size, 330 void *Code) { 331 beginTargetDataOperation(); 332 if (ompt_callback_target_data_op_emi_fn) { 333 ompt_callback_target_data_op_emi_fn( 334 ompt_scope_begin, TargetTaskData, &TargetData, &HostOpId, 335 ompt_target_data_associate, HstPtrBegin, omp_get_initial_device(), 336 TgtPtrBegin, DeviceId, Size, Code); 337 } else if (ompt_callback_target_data_op_fn) { 338 HostOpId = createOpId(); 339 ompt_callback_target_data_op_fn( 340 TargetData.value, HostOpId, ompt_target_data_associate, HstPtrBegin, 341 omp_get_initial_device(), TgtPtrBegin, DeviceId, Size, Code); 342 } 343 } 344 345 void Interface::endTargetAssociatePointer(int64_t DeviceId, void *HstPtrBegin, 346 void *TgtPtrBegin, size_t Size, 347 void *Code) { 348 if (ompt_callback_target_data_op_emi_fn) { 349 ompt_callback_target_data_op_emi_fn( 350 ompt_scope_end, TargetTaskData, &TargetData, &HostOpId, 351 ompt_target_data_associate, HstPtrBegin, omp_get_initial_device(), 352 TgtPtrBegin, DeviceId, Size, Code); 353 } 354 } 355 356 void Interface::beginTargetDisassociatePointer(int64_t DeviceId, 357 void *HstPtrBegin, 358 void *TgtPtrBegin, size_t Size, 359 void *Code) { 360 beginTargetDataOperation(); 361 if (ompt_callback_target_data_op_emi_fn) { 362 ompt_callback_target_data_op_emi_fn( 363 ompt_scope_begin, TargetTaskData, &TargetData, &HostOpId, 364 ompt_target_data_disassociate, HstPtrBegin, omp_get_initial_device(), 365 TgtPtrBegin, DeviceId, Size, Code); 366 } else if (ompt_callback_target_data_op_fn) { 367 HostOpId = createOpId(); 368 ompt_callback_target_data_op_fn( 369 TargetData.value, HostOpId, ompt_target_data_disassociate, HstPtrBegin, 370 omp_get_initial_device(), TgtPtrBegin, DeviceId, Size, Code); 371 } 372 } 373 void Interface::endTargetDisassociatePointer(int64_t DeviceId, 374 void *HstPtrBegin, 375 void *TgtPtrBegin, size_t Size, 376 void *Code) { 377 if (ompt_callback_target_data_op_emi_fn) { 378 ompt_callback_target_data_op_emi_fn( 379 ompt_scope_end, TargetTaskData, &TargetData, &HostOpId, 380 ompt_target_data_disassociate, HstPtrBegin, omp_get_initial_device(), 381 TgtPtrBegin, DeviceId, Size, Code); 382 } 383 } 384 385 void Interface::beginTarget(int64_t DeviceId, void *Code) { 386 beginTargetRegion(); 387 if (ompt_callback_target_emi_fn) { 388 // Invoke the tool supplied target EMI callback 389 ompt_callback_target_emi_fn(ompt_target, ompt_scope_begin, DeviceId, 390 TaskData, TargetTaskData, &TargetData, Code); 391 } else if (ompt_callback_target_fn) { 392 TargetData.value = createRegionId(); 393 // Invoke the tool supplied target callback 394 ompt_callback_target_fn(ompt_target, ompt_scope_begin, DeviceId, TaskData, 395 TargetData.value, Code); 396 } 397 } 398 399 void Interface::endTarget(int64_t DeviceId, void *Code) { 400 if (ompt_callback_target_emi_fn) { 401 // Invoke the tool supplied target EMI callback 402 ompt_callback_target_emi_fn(ompt_target, ompt_scope_end, DeviceId, TaskData, 403 TargetTaskData, &TargetData, Code); 404 } else if (ompt_callback_target_fn) { 405 // Invoke the tool supplied target callback 406 ompt_callback_target_fn(ompt_target, ompt_scope_end, DeviceId, TaskData, 407 TargetData.value, Code); 408 } 409 endTargetRegion(); 410 } 411 412 void Interface::beginTargetDataOperation() { 413 DP("in ompt_target_region_begin (TargetRegionId = %lu)\n", TargetData.value); 414 } 415 416 void Interface::endTargetDataOperation() { 417 DP("in ompt_target_region_end (TargetRegionId = %lu)\n", TargetData.value); 418 } 419 420 void Interface::beginTargetRegion() { 421 // Set up task state 422 assert(ompt_get_task_data_fn && "Calling a null task data function"); 423 TaskData = ompt_get_task_data_fn(); 424 // Set up target task state 425 assert(ompt_get_target_task_data_fn && 426 "Calling a null target task data function"); 427 TargetTaskData = ompt_get_target_task_data_fn(); 428 // Target state will be set later 429 TargetData = ompt_data_none; 430 } 431 432 void Interface::endTargetRegion() { 433 TaskData = 0; 434 TargetTaskData = 0; 435 TargetData = ompt_data_none; 436 } 437 438 /// Used to maintain the finalization functions that are received 439 /// from the plugins during connect. 440 /// Note: Currently, there are no plugin-specific finalizations, so each plugin 441 /// will call the same (empty) function. 442 class LibomptargetRtlFinalizer { 443 public: 444 LibomptargetRtlFinalizer() {} 445 446 void registerRtl(ompt_finalize_t FinalizationFunction) { 447 if (FinalizationFunction) { 448 RtlFinalizationFunctions.emplace_back(FinalizationFunction); 449 } 450 } 451 452 void finalize() { 453 for (auto FinalizationFunction : RtlFinalizationFunctions) 454 FinalizationFunction(/*tool_data=*/nullptr); 455 RtlFinalizationFunctions.clear(); 456 } 457 458 private: 459 llvm::SmallVector<ompt_finalize_t> RtlFinalizationFunctions; 460 }; 461 462 int llvm::omp::target::ompt::initializeLibrary(ompt_function_lookup_t lookup, 463 int initial_device_num, 464 ompt_data_t *tool_data) { 465 DP("Executing initializeLibrary\n"); 466 #define bindOmptFunctionName(OmptFunction, DestinationFunction) \ 467 if (lookup) \ 468 DestinationFunction = (OmptFunction##_t)lookup(#OmptFunction); \ 469 DP("initializeLibrary bound %s=%p\n", #DestinationFunction, \ 470 ((void *)(uint64_t)DestinationFunction)); 471 472 bindOmptFunctionName(ompt_get_callback, lookupCallbackByCode); 473 bindOmptFunctionName(ompt_get_task_data, ompt_get_task_data_fn); 474 bindOmptFunctionName(ompt_get_target_task_data, ompt_get_target_task_data_fn); 475 #undef bindOmptFunctionName 476 477 // Store pointer of 'ompt_libomp_target_fn_lookup' for use by libomptarget 478 lookupCallbackByName = lookup; 479 480 assert(lookupCallbackByCode && "lookupCallbackByCode should be non-null"); 481 assert(lookupCallbackByName && "lookupCallbackByName should be non-null"); 482 assert(ompt_get_task_data_fn && "ompt_get_task_data_fn should be non-null"); 483 assert(ompt_get_target_task_data_fn && 484 "ompt_get_target_task_data_fn should be non-null"); 485 assert(LibraryFinalizer == nullptr && 486 "LibraryFinalizer should not be initialized yet"); 487 488 LibraryFinalizer = new LibomptargetRtlFinalizer(); 489 490 Initialized = true; 491 492 return 0; 493 } 494 495 void llvm::omp::target::ompt::finalizeLibrary(ompt_data_t *data) { 496 DP("Executing finalizeLibrary\n"); 497 // Before disabling OMPT, call the (plugin) finalizations that were registered 498 // with this library 499 LibraryFinalizer->finalize(); 500 delete LibraryFinalizer; 501 Initialized = false; 502 } 503 504 void llvm::omp::target::ompt::connectLibrary() { 505 DP("Entering connectLibrary\n"); 506 // Connect with libomp 507 static OmptLibraryConnectorTy LibompConnector("libomp"); 508 static ompt_start_tool_result_t OmptResult; 509 510 // Initialize OmptResult with the init and fini functions that will be 511 // called by the connector 512 OmptResult.initialize = ompt::initializeLibrary; 513 OmptResult.finalize = ompt::finalizeLibrary; 514 OmptResult.tool_data.value = 0; 515 516 // Now call connect that causes the above init/fini functions to be called 517 LibompConnector.connect(&OmptResult); 518 519 #define bindOmptCallback(Name, Type, Code) \ 520 if (lookupCallbackByCode) \ 521 lookupCallbackByCode( \ 522 (ompt_callbacks_t)(Code), \ 523 (ompt_callback_t *)&(llvm::omp::target::ompt::Name##_fn)); 524 FOREACH_OMPT_NOEMI_EVENT(bindOmptCallback) 525 FOREACH_OMPT_EMI_EVENT(bindOmptCallback) 526 #undef bindOmptCallback 527 528 DP("Exiting connectLibrary\n"); 529 } 530 531 #endif // OMPT_SUPPORT 532