xref: /llvm-project/offload/src/OpenMP/Mapping.cpp (revision 81d20d861e48f5202c9f79b47dee244674fb9121)
1 //===-- OpenMP/Mapping.cpp - OpenMP/OpenACC pointer mapping impl. ---------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 //===----------------------------------------------------------------------===//
10 
11 #include "OpenMP/Mapping.h"
12 
13 #include "PluginManager.h"
14 #include "Shared/Debug.h"
15 #include "Shared/Requirements.h"
16 #include "device.h"
17 
18 /// Dump a table of all the host-target pointer pairs on failure
dumpTargetPointerMappings(const ident_t * Loc,DeviceTy & Device,bool toStdOut)19 void dumpTargetPointerMappings(const ident_t *Loc, DeviceTy &Device,
20                                bool toStdOut) {
21   MappingInfoTy::HDTTMapAccessorTy HDTTMap =
22       Device.getMappingInfo().HostDataToTargetMap.getExclusiveAccessor();
23   if (HDTTMap->empty()) {
24     DUMP_INFO(toStdOut, OMP_INFOTYPE_ALL, Device.DeviceID,
25               "OpenMP Host-Device pointer mappings table empty\n");
26     return;
27   }
28 
29   SourceInfo Kernel(Loc);
30   DUMP_INFO(toStdOut, OMP_INFOTYPE_ALL, Device.DeviceID,
31             "OpenMP Host-Device pointer mappings after block at %s:%d:%d:\n",
32             Kernel.getFilename(), Kernel.getLine(), Kernel.getColumn());
33   DUMP_INFO(toStdOut, OMP_INFOTYPE_ALL, Device.DeviceID,
34             "%-18s %-18s %s %s %s %s\n", "Host Ptr", "Target Ptr", "Size (B)",
35             "DynRefCount", "HoldRefCount", "Declaration");
36   for (const auto &It : *HDTTMap) {
37     HostDataToTargetTy &HDTT = *It.HDTT;
38     SourceInfo Info(HDTT.HstPtrName);
39     DUMP_INFO(toStdOut, OMP_INFOTYPE_ALL, Device.DeviceID,
40               DPxMOD " " DPxMOD " %-8" PRIuPTR " %-11s %-12s %s at %s:%d:%d\n",
41               DPxPTR(HDTT.HstPtrBegin), DPxPTR(HDTT.TgtPtrBegin),
42               HDTT.HstPtrEnd - HDTT.HstPtrBegin,
43               HDTT.dynRefCountToStr().c_str(), HDTT.holdRefCountToStr().c_str(),
44               Info.getName(), Info.getFilename(), Info.getLine(),
45               Info.getColumn());
46   }
47 }
48 
associatePtr(void * HstPtrBegin,void * TgtPtrBegin,int64_t Size)49 int MappingInfoTy::associatePtr(void *HstPtrBegin, void *TgtPtrBegin,
50                                 int64_t Size) {
51   HDTTMapAccessorTy HDTTMap = HostDataToTargetMap.getExclusiveAccessor();
52 
53   // Check if entry exists
54   auto It = HDTTMap->find(HstPtrBegin);
55   if (It != HDTTMap->end()) {
56     HostDataToTargetTy &HDTT = *It->HDTT;
57     std::lock_guard<HostDataToTargetTy> LG(HDTT);
58     // Mapping already exists
59     bool IsValid = HDTT.HstPtrEnd == (uintptr_t)HstPtrBegin + Size &&
60                    HDTT.TgtPtrBegin == (uintptr_t)TgtPtrBegin;
61     if (IsValid) {
62       DP("Attempt to re-associate the same device ptr+offset with the same "
63          "host ptr, nothing to do\n");
64       return OFFLOAD_SUCCESS;
65     }
66     REPORT("Not allowed to re-associate a different device ptr+offset with "
67            "the same host ptr\n");
68     return OFFLOAD_FAIL;
69   }
70 
71   // Mapping does not exist, allocate it with refCount=INF
72   const HostDataToTargetTy &NewEntry =
73       *HDTTMap
74            ->emplace(new HostDataToTargetTy(
75                /*HstPtrBase=*/(uintptr_t)HstPtrBegin,
76                /*HstPtrBegin=*/(uintptr_t)HstPtrBegin,
77                /*HstPtrEnd=*/(uintptr_t)HstPtrBegin + Size,
78                /*TgtAllocBegin=*/(uintptr_t)TgtPtrBegin,
79                /*TgtPtrBegin=*/(uintptr_t)TgtPtrBegin,
80                /*UseHoldRefCount=*/false, /*Name=*/nullptr,
81                /*IsRefCountINF=*/true))
82            .first->HDTT;
83   DP("Creating new map entry: HstBase=" DPxMOD ", HstBegin=" DPxMOD
84      ", HstEnd=" DPxMOD ", TgtBegin=" DPxMOD ", DynRefCount=%s, "
85      "HoldRefCount=%s\n",
86      DPxPTR(NewEntry.HstPtrBase), DPxPTR(NewEntry.HstPtrBegin),
87      DPxPTR(NewEntry.HstPtrEnd), DPxPTR(NewEntry.TgtPtrBegin),
88      NewEntry.dynRefCountToStr().c_str(), NewEntry.holdRefCountToStr().c_str());
89   (void)NewEntry;
90 
91   // Notify the plugin about the new mapping.
92   return Device.notifyDataMapped(HstPtrBegin, Size);
93 }
94 
disassociatePtr(void * HstPtrBegin)95 int MappingInfoTy::disassociatePtr(void *HstPtrBegin) {
96   HDTTMapAccessorTy HDTTMap = HostDataToTargetMap.getExclusiveAccessor();
97 
98   auto It = HDTTMap->find(HstPtrBegin);
99   if (It == HDTTMap->end()) {
100     REPORT("Association not found\n");
101     return OFFLOAD_FAIL;
102   }
103   // Mapping exists
104   HostDataToTargetTy &HDTT = *It->HDTT;
105   std::lock_guard<HostDataToTargetTy> LG(HDTT);
106 
107   if (HDTT.getHoldRefCount()) {
108     // This is based on OpenACC 3.1, sec 3.2.33 "acc_unmap_data", L3656-3657:
109     // "It is an error to call acc_unmap_data if the structured reference
110     // count for the pointer is not zero."
111     REPORT("Trying to disassociate a pointer with a non-zero hold reference "
112            "count\n");
113     return OFFLOAD_FAIL;
114   }
115 
116   if (HDTT.isDynRefCountInf()) {
117     DP("Association found, removing it\n");
118     void *Event = HDTT.getEvent();
119     delete &HDTT;
120     if (Event)
121       Device.destroyEvent(Event);
122     HDTTMap->erase(It);
123     return Device.notifyDataUnmapped(HstPtrBegin);
124   }
125 
126   REPORT("Trying to disassociate a pointer which was not mapped via "
127          "omp_target_associate_ptr\n");
128   return OFFLOAD_FAIL;
129 }
130 
lookupMapping(HDTTMapAccessorTy & HDTTMap,void * HstPtrBegin,int64_t Size,HostDataToTargetTy * OwnedTPR)131 LookupResult MappingInfoTy::lookupMapping(HDTTMapAccessorTy &HDTTMap,
132                                           void *HstPtrBegin, int64_t Size,
133                                           HostDataToTargetTy *OwnedTPR) {
134 
135   uintptr_t HP = (uintptr_t)HstPtrBegin;
136   LookupResult LR;
137 
138   DP("Looking up mapping(HstPtrBegin=" DPxMOD ", Size=%" PRId64 ")...\n",
139      DPxPTR(HP), Size);
140 
141   if (HDTTMap->empty())
142     return LR;
143 
144   auto Upper = HDTTMap->upper_bound(HP);
145 
146   if (Size == 0) {
147     // specification v5.1 Pointer Initialization for Device Data Environments
148     // upper_bound satisfies
149     //   std::prev(upper)->HDTT.HstPtrBegin <= hp < upper->HDTT.HstPtrBegin
150     if (Upper != HDTTMap->begin()) {
151       LR.TPR.setEntry(std::prev(Upper)->HDTT, OwnedTPR);
152       // the left side of extended address range is satisified.
153       // hp >= LR.TPR.getEntry()->HstPtrBegin || hp >=
154       // LR.TPR.getEntry()->HstPtrBase
155       LR.Flags.IsContained = HP < LR.TPR.getEntry()->HstPtrEnd ||
156                              HP < LR.TPR.getEntry()->HstPtrBase;
157     }
158 
159     if (!LR.Flags.IsContained && Upper != HDTTMap->end()) {
160       LR.TPR.setEntry(Upper->HDTT, OwnedTPR);
161       // the right side of extended address range is satisified.
162       // hp < LR.TPR.getEntry()->HstPtrEnd || hp < LR.TPR.getEntry()->HstPtrBase
163       LR.Flags.IsContained = HP >= LR.TPR.getEntry()->HstPtrBase;
164     }
165   } else {
166     // check the left bin
167     if (Upper != HDTTMap->begin()) {
168       LR.TPR.setEntry(std::prev(Upper)->HDTT, OwnedTPR);
169       // Is it contained?
170       LR.Flags.IsContained = HP >= LR.TPR.getEntry()->HstPtrBegin &&
171                              HP < LR.TPR.getEntry()->HstPtrEnd &&
172                              (HP + Size) <= LR.TPR.getEntry()->HstPtrEnd;
173       // Does it extend beyond the mapped region?
174       LR.Flags.ExtendsAfter = HP < LR.TPR.getEntry()->HstPtrEnd &&
175                               (HP + Size) > LR.TPR.getEntry()->HstPtrEnd;
176     }
177 
178     // check the right bin
179     if (!(LR.Flags.IsContained || LR.Flags.ExtendsAfter) &&
180         Upper != HDTTMap->end()) {
181       LR.TPR.setEntry(Upper->HDTT, OwnedTPR);
182       // Does it extend into an already mapped region?
183       LR.Flags.ExtendsBefore = HP < LR.TPR.getEntry()->HstPtrBegin &&
184                                (HP + Size) > LR.TPR.getEntry()->HstPtrBegin;
185       // Does it extend beyond the mapped region?
186       LR.Flags.ExtendsAfter = HP < LR.TPR.getEntry()->HstPtrEnd &&
187                               (HP + Size) > LR.TPR.getEntry()->HstPtrEnd;
188     }
189 
190     if (LR.Flags.ExtendsBefore) {
191       DP("WARNING: Pointer is not mapped but section extends into already "
192          "mapped data\n");
193     }
194     if (LR.Flags.ExtendsAfter) {
195       DP("WARNING: Pointer is already mapped but section extends beyond mapped "
196          "region\n");
197     }
198   }
199 
200   return LR;
201 }
202 
getTargetPointer(HDTTMapAccessorTy & HDTTMap,void * HstPtrBegin,void * HstPtrBase,int64_t TgtPadding,int64_t Size,map_var_info_t HstPtrName,bool HasFlagTo,bool HasFlagAlways,bool IsImplicit,bool UpdateRefCount,bool HasCloseModifier,bool HasPresentModifier,bool HasHoldModifier,AsyncInfoTy & AsyncInfo,HostDataToTargetTy * OwnedTPR,bool ReleaseHDTTMap)203 TargetPointerResultTy MappingInfoTy::getTargetPointer(
204     HDTTMapAccessorTy &HDTTMap, void *HstPtrBegin, void *HstPtrBase,
205     int64_t TgtPadding, int64_t Size, map_var_info_t HstPtrName, bool HasFlagTo,
206     bool HasFlagAlways, bool IsImplicit, bool UpdateRefCount,
207     bool HasCloseModifier, bool HasPresentModifier, bool HasHoldModifier,
208     AsyncInfoTy &AsyncInfo, HostDataToTargetTy *OwnedTPR, bool ReleaseHDTTMap) {
209 
210   LookupResult LR = lookupMapping(HDTTMap, HstPtrBegin, Size, OwnedTPR);
211   LR.TPR.Flags.IsPresent = true;
212 
213   // Release the mapping table lock only after the entry is locked by
214   // attaching it to TPR. Once TPR is destroyed it will release the lock
215   // on entry. If it is returned the lock will move to the returned object.
216   // If LR.Entry is already owned/locked we avoid trying to lock it again.
217 
218   // Check if the pointer is contained.
219   // If a variable is mapped to the device manually by the user - which would
220   // lead to the IsContained flag to be true - then we must ensure that the
221   // device address is returned even under unified memory conditions.
222   if (LR.Flags.IsContained ||
223       ((LR.Flags.ExtendsBefore || LR.Flags.ExtendsAfter) && IsImplicit)) {
224     const char *RefCountAction;
225     if (UpdateRefCount) {
226       // After this, reference count >= 1. If the reference count was 0 but the
227       // entry was still there we can reuse the data on the device and avoid a
228       // new submission.
229       LR.TPR.getEntry()->incRefCount(HasHoldModifier);
230       RefCountAction = " (incremented)";
231     } else {
232       // It might have been allocated with the parent, but it's still new.
233       LR.TPR.Flags.IsNewEntry = LR.TPR.getEntry()->getTotalRefCount() == 1;
234       RefCountAction = " (update suppressed)";
235     }
236     const char *DynRefCountAction = HasHoldModifier ? "" : RefCountAction;
237     const char *HoldRefCountAction = HasHoldModifier ? RefCountAction : "";
238     uintptr_t Ptr = LR.TPR.getEntry()->TgtPtrBegin +
239                     ((uintptr_t)HstPtrBegin - LR.TPR.getEntry()->HstPtrBegin);
240     INFO(OMP_INFOTYPE_MAPPING_EXISTS, Device.DeviceID,
241          "Mapping exists%s with HstPtrBegin=" DPxMOD ", TgtPtrBegin=" DPxMOD
242          ", Size=%" PRId64 ", DynRefCount=%s%s, HoldRefCount=%s%s, Name=%s\n",
243          (IsImplicit ? " (implicit)" : ""), DPxPTR(HstPtrBegin), DPxPTR(Ptr),
244          Size, LR.TPR.getEntry()->dynRefCountToStr().c_str(), DynRefCountAction,
245          LR.TPR.getEntry()->holdRefCountToStr().c_str(), HoldRefCountAction,
246          (HstPtrName) ? getNameFromMapping(HstPtrName).c_str() : "unknown");
247     LR.TPR.TargetPointer = (void *)Ptr;
248   } else if ((LR.Flags.ExtendsBefore || LR.Flags.ExtendsAfter) && !IsImplicit) {
249     // Explicit extension of mapped data - not allowed.
250     MESSAGE("explicit extension not allowed: host address specified is " DPxMOD
251             " (%" PRId64
252             " bytes), but device allocation maps to host at " DPxMOD
253             " (%" PRId64 " bytes)",
254             DPxPTR(HstPtrBegin), Size, DPxPTR(LR.TPR.getEntry()->HstPtrBegin),
255             LR.TPR.getEntry()->HstPtrEnd - LR.TPR.getEntry()->HstPtrBegin);
256     if (HasPresentModifier)
257       MESSAGE("device mapping required by 'present' map type modifier does not "
258               "exist for host address " DPxMOD " (%" PRId64 " bytes)",
259               DPxPTR(HstPtrBegin), Size);
260   } else if ((PM->getRequirements() & OMP_REQ_UNIFIED_SHARED_MEMORY &&
261               !HasCloseModifier) ||
262              (PM->getRequirements() & OMPX_REQ_AUTO_ZERO_COPY)) {
263 
264     // If unified shared memory is active, implicitly mapped variables that are
265     // not privatized use host address. Any explicitly mapped variables also use
266     // host address where correctness is not impeded. In all other cases maps
267     // are respected.
268     // In addition to the mapping rules above, the close map modifier forces the
269     // mapping of the variable to the device.
270     if (Size) {
271       INFO(OMP_INFOTYPE_MAPPING_CHANGED, Device.DeviceID,
272            "Return HstPtrBegin " DPxMOD " Size=%" PRId64 " for unified shared "
273            "memory\n",
274            DPxPTR((uintptr_t)HstPtrBegin), Size);
275       DP("Return HstPtrBegin " DPxMOD " Size=%" PRId64 " for unified shared "
276          "memory\n",
277          DPxPTR((uintptr_t)HstPtrBegin), Size);
278       LR.TPR.Flags.IsPresent = false;
279       LR.TPR.Flags.IsHostPointer = true;
280       LR.TPR.TargetPointer = HstPtrBegin;
281     }
282   } else if (HasPresentModifier) {
283     DP("Mapping required by 'present' map type modifier does not exist for "
284        "HstPtrBegin=" DPxMOD ", Size=%" PRId64 "\n",
285        DPxPTR(HstPtrBegin), Size);
286     MESSAGE("device mapping required by 'present' map type modifier does not "
287             "exist for host address " DPxMOD " (%" PRId64 " bytes)",
288             DPxPTR(HstPtrBegin), Size);
289   } else if (Size) {
290     // If it is not contained and Size > 0, we should create a new entry for it.
291     LR.TPR.Flags.IsNewEntry = true;
292     uintptr_t TgtAllocBegin =
293         (uintptr_t)Device.allocData(TgtPadding + Size, HstPtrBegin);
294     uintptr_t TgtPtrBegin = TgtAllocBegin + TgtPadding;
295     // Release the mapping table lock only after the entry is locked by
296     // attaching it to TPR.
297     LR.TPR.setEntry(HDTTMap
298                         ->emplace(new HostDataToTargetTy(
299                             (uintptr_t)HstPtrBase, (uintptr_t)HstPtrBegin,
300                             (uintptr_t)HstPtrBegin + Size, TgtAllocBegin,
301                             TgtPtrBegin, HasHoldModifier, HstPtrName))
302                         .first->HDTT);
303     INFO(OMP_INFOTYPE_MAPPING_CHANGED, Device.DeviceID,
304          "Creating new map entry with HstPtrBase=" DPxMOD
305          ", HstPtrBegin=" DPxMOD ", TgtAllocBegin=" DPxMOD
306          ", TgtPtrBegin=" DPxMOD
307          ", Size=%ld, DynRefCount=%s, HoldRefCount=%s, Name=%s\n",
308          DPxPTR(HstPtrBase), DPxPTR(HstPtrBegin), DPxPTR(TgtAllocBegin),
309          DPxPTR(TgtPtrBegin), Size,
310          LR.TPR.getEntry()->dynRefCountToStr().c_str(),
311          LR.TPR.getEntry()->holdRefCountToStr().c_str(),
312          (HstPtrName) ? getNameFromMapping(HstPtrName).c_str() : "unknown");
313     LR.TPR.TargetPointer = (void *)TgtPtrBegin;
314 
315     // Notify the plugin about the new mapping.
316     if (Device.notifyDataMapped(HstPtrBegin, Size))
317       return TargetPointerResultTy{};
318   } else {
319     // This entry is not present and we did not create a new entry for it.
320     LR.TPR.Flags.IsPresent = false;
321   }
322 
323   // All mapping table modifications have been made. If the user requested it we
324   // give up the lock.
325   if (ReleaseHDTTMap)
326     HDTTMap.destroy();
327 
328   // If the target pointer is valid, and we need to transfer data, issue the
329   // data transfer.
330   if (LR.TPR.TargetPointer && !LR.TPR.Flags.IsHostPointer && HasFlagTo &&
331       (LR.TPR.Flags.IsNewEntry || HasFlagAlways) && Size != 0) {
332     DP("Moving %" PRId64 " bytes (hst:" DPxMOD ") -> (tgt:" DPxMOD ")\n", Size,
333        DPxPTR(HstPtrBegin), DPxPTR(LR.TPR.TargetPointer));
334 
335     int Ret = Device.submitData(LR.TPR.TargetPointer, HstPtrBegin, Size,
336                                 AsyncInfo, LR.TPR.getEntry());
337     if (Ret != OFFLOAD_SUCCESS) {
338       REPORT("Copying data to device failed.\n");
339       // We will also return nullptr if the data movement fails because that
340       // pointer points to a corrupted memory region so it doesn't make any
341       // sense to continue to use it.
342       LR.TPR.TargetPointer = nullptr;
343     } else if (LR.TPR.getEntry()->addEventIfNecessary(Device, AsyncInfo) !=
344                OFFLOAD_SUCCESS)
345       return TargetPointerResultTy{};
346   } else {
347     // If not a host pointer and no present modifier, we need to wait for the
348     // event if it exists.
349     // Note: Entry might be nullptr because of zero length array section.
350     if (LR.TPR.getEntry() && !LR.TPR.Flags.IsHostPointer &&
351         !HasPresentModifier) {
352       void *Event = LR.TPR.getEntry()->getEvent();
353       if (Event) {
354         int Ret = Device.waitEvent(Event, AsyncInfo);
355         if (Ret != OFFLOAD_SUCCESS) {
356           // If it fails to wait for the event, we need to return nullptr in
357           // case of any data race.
358           REPORT("Failed to wait for event " DPxMOD ".\n", DPxPTR(Event));
359           return TargetPointerResultTy{};
360         }
361       }
362     }
363   }
364 
365   return std::move(LR.TPR);
366 }
367 
getTgtPtrBegin(void * HstPtrBegin,int64_t Size,bool UpdateRefCount,bool UseHoldRefCount,bool MustContain,bool ForceDelete,bool FromDataEnd)368 TargetPointerResultTy MappingInfoTy::getTgtPtrBegin(
369     void *HstPtrBegin, int64_t Size, bool UpdateRefCount, bool UseHoldRefCount,
370     bool MustContain, bool ForceDelete, bool FromDataEnd) {
371   HDTTMapAccessorTy HDTTMap = HostDataToTargetMap.getExclusiveAccessor();
372 
373   LookupResult LR = lookupMapping(HDTTMap, HstPtrBegin, Size);
374 
375   LR.TPR.Flags.IsPresent = true;
376 
377   if (LR.Flags.IsContained ||
378       (!MustContain && (LR.Flags.ExtendsBefore || LR.Flags.ExtendsAfter))) {
379     LR.TPR.Flags.IsLast =
380         LR.TPR.getEntry()->decShouldRemove(UseHoldRefCount, ForceDelete);
381 
382     if (ForceDelete) {
383       LR.TPR.getEntry()->resetRefCount(UseHoldRefCount);
384       assert(LR.TPR.Flags.IsLast ==
385                  LR.TPR.getEntry()->decShouldRemove(UseHoldRefCount) &&
386              "expected correct IsLast prediction for reset");
387     }
388 
389     // Increment the number of threads that is using the entry on a
390     // targetDataEnd, tracking the number of possible "deleters". A thread may
391     // come to own the entry deletion even if it was not the last one querying
392     // for it. Thus, we must track every query on targetDataEnds to ensure only
393     // the last thread that holds a reference to an entry actually deletes it.
394     if (FromDataEnd)
395       LR.TPR.getEntry()->incDataEndThreadCount();
396 
397     const char *RefCountAction;
398     if (!UpdateRefCount) {
399       RefCountAction = " (update suppressed)";
400     } else if (LR.TPR.Flags.IsLast) {
401       LR.TPR.getEntry()->decRefCount(UseHoldRefCount);
402       assert(LR.TPR.getEntry()->getTotalRefCount() == 0 &&
403              "Expected zero reference count when deletion is scheduled");
404       if (ForceDelete)
405         RefCountAction = " (reset, delayed deletion)";
406       else
407         RefCountAction = " (decremented, delayed deletion)";
408     } else {
409       LR.TPR.getEntry()->decRefCount(UseHoldRefCount);
410       RefCountAction = " (decremented)";
411     }
412     const char *DynRefCountAction = UseHoldRefCount ? "" : RefCountAction;
413     const char *HoldRefCountAction = UseHoldRefCount ? RefCountAction : "";
414     uintptr_t TP = LR.TPR.getEntry()->TgtPtrBegin +
415                    ((uintptr_t)HstPtrBegin - LR.TPR.getEntry()->HstPtrBegin);
416     INFO(OMP_INFOTYPE_MAPPING_EXISTS, Device.DeviceID,
417          "Mapping exists with HstPtrBegin=" DPxMOD ", TgtPtrBegin=" DPxMOD ", "
418          "Size=%" PRId64 ", DynRefCount=%s%s, HoldRefCount=%s%s\n",
419          DPxPTR(HstPtrBegin), DPxPTR(TP), Size,
420          LR.TPR.getEntry()->dynRefCountToStr().c_str(), DynRefCountAction,
421          LR.TPR.getEntry()->holdRefCountToStr().c_str(), HoldRefCountAction);
422     LR.TPR.TargetPointer = (void *)TP;
423   } else if (PM->getRequirements() & OMP_REQ_UNIFIED_SHARED_MEMORY ||
424              PM->getRequirements() & OMPX_REQ_AUTO_ZERO_COPY) {
425     // If the value isn't found in the mapping and unified shared memory
426     // is on then it means we have stumbled upon a value which we need to
427     // use directly from the host.
428     DP("Get HstPtrBegin " DPxMOD " Size=%" PRId64 " for unified shared "
429        "memory\n",
430        DPxPTR((uintptr_t)HstPtrBegin), Size);
431     LR.TPR.Flags.IsPresent = false;
432     LR.TPR.Flags.IsHostPointer = true;
433     LR.TPR.TargetPointer = HstPtrBegin;
434   } else {
435     // OpenMP Specification v5.2: if a matching list item is not found, the
436     // pointer retains its original value as per firstprivate semantics.
437     LR.TPR.Flags.IsPresent = false;
438     LR.TPR.Flags.IsHostPointer = false;
439     LR.TPR.TargetPointer = HstPtrBegin;
440   }
441 
442   return std::move(LR.TPR);
443 }
444 
445 // Return the target pointer begin (where the data will be moved).
getTgtPtrBegin(HDTTMapAccessorTy & HDTTMap,void * HstPtrBegin,int64_t Size)446 void *MappingInfoTy::getTgtPtrBegin(HDTTMapAccessorTy &HDTTMap,
447                                     void *HstPtrBegin, int64_t Size) {
448   uintptr_t HP = (uintptr_t)HstPtrBegin;
449   LookupResult LR = lookupMapping(HDTTMap, HstPtrBegin, Size);
450   if (LR.Flags.IsContained || LR.Flags.ExtendsBefore || LR.Flags.ExtendsAfter) {
451     uintptr_t TP =
452         LR.TPR.getEntry()->TgtPtrBegin + (HP - LR.TPR.getEntry()->HstPtrBegin);
453     return (void *)TP;
454   }
455 
456   return NULL;
457 }
458 
eraseMapEntry(HDTTMapAccessorTy & HDTTMap,HostDataToTargetTy * Entry,int64_t Size)459 int MappingInfoTy::eraseMapEntry(HDTTMapAccessorTy &HDTTMap,
460                                  HostDataToTargetTy *Entry, int64_t Size) {
461   assert(Entry && "Trying to delete a null entry from the HDTT map.");
462   assert(Entry->getTotalRefCount() == 0 &&
463          Entry->getDataEndThreadCount() == 0 &&
464          "Trying to delete entry that is in use or owned by another thread.");
465 
466   INFO(OMP_INFOTYPE_MAPPING_CHANGED, Device.DeviceID,
467        "Removing map entry with HstPtrBegin=" DPxMOD ", TgtPtrBegin=" DPxMOD
468        ", Size=%" PRId64 ", Name=%s\n",
469        DPxPTR(Entry->HstPtrBegin), DPxPTR(Entry->TgtPtrBegin), Size,
470        (Entry->HstPtrName) ? getNameFromMapping(Entry->HstPtrName).c_str()
471                            : "unknown");
472 
473   if (HDTTMap->erase(Entry) == 0) {
474     REPORT("Trying to remove a non-existent map entry\n");
475     return OFFLOAD_FAIL;
476   }
477 
478   return OFFLOAD_SUCCESS;
479 }
480 
deallocTgtPtrAndEntry(HostDataToTargetTy * Entry,int64_t Size)481 int MappingInfoTy::deallocTgtPtrAndEntry(HostDataToTargetTy *Entry,
482                                          int64_t Size) {
483   assert(Entry && "Trying to deallocate a null entry.");
484 
485   DP("Deleting tgt data " DPxMOD " of size %" PRId64 " by freeing allocation "
486      "starting at " DPxMOD "\n",
487      DPxPTR(Entry->TgtPtrBegin), Size, DPxPTR(Entry->TgtAllocBegin));
488 
489   void *Event = Entry->getEvent();
490   if (Event && Device.destroyEvent(Event) != OFFLOAD_SUCCESS) {
491     REPORT("Failed to destroy event " DPxMOD "\n", DPxPTR(Event));
492     return OFFLOAD_FAIL;
493   }
494 
495   int Ret = Device.deleteData((void *)Entry->TgtAllocBegin);
496 
497   // Notify the plugin about the unmapped memory.
498   Ret |= Device.notifyDataUnmapped((void *)Entry->HstPtrBegin);
499 
500   delete Entry;
501 
502   return Ret;
503 }
504 
printCopyInfoImpl(int DeviceId,bool H2D,void * SrcPtrBegin,void * DstPtrBegin,int64_t Size,HostDataToTargetTy * HT)505 static void printCopyInfoImpl(int DeviceId, bool H2D, void *SrcPtrBegin,
506                               void *DstPtrBegin, int64_t Size,
507                               HostDataToTargetTy *HT) {
508 
509   INFO(OMP_INFOTYPE_DATA_TRANSFER, DeviceId,
510        "Copying data from %s to %s, %sPtr=" DPxMOD ", %sPtr=" DPxMOD
511        ", Size=%" PRId64 ", Name=%s\n",
512        H2D ? "host" : "device", H2D ? "device" : "host", H2D ? "Hst" : "Tgt",
513        DPxPTR(H2D ? SrcPtrBegin : DstPtrBegin), H2D ? "Tgt" : "Hst",
514        DPxPTR(H2D ? DstPtrBegin : SrcPtrBegin), Size,
515        (HT && HT->HstPtrName) ? getNameFromMapping(HT->HstPtrName).c_str()
516                               : "unknown");
517 }
518 
printCopyInfo(void * TgtPtrBegin,void * HstPtrBegin,int64_t Size,bool H2D,HostDataToTargetTy * Entry,MappingInfoTy::HDTTMapAccessorTy * HDTTMapPtr)519 void MappingInfoTy::printCopyInfo(
520     void *TgtPtrBegin, void *HstPtrBegin, int64_t Size, bool H2D,
521     HostDataToTargetTy *Entry, MappingInfoTy::HDTTMapAccessorTy *HDTTMapPtr) {
522   auto HDTTMap =
523       HostDataToTargetMap.getExclusiveAccessor(!!Entry || !!HDTTMapPtr);
524   LookupResult LR;
525   if (!Entry) {
526     LR = lookupMapping(HDTTMapPtr ? *HDTTMapPtr : HDTTMap, HstPtrBegin, Size);
527     Entry = LR.TPR.getEntry();
528   }
529   printCopyInfoImpl(Device.DeviceID, H2D, HstPtrBegin, TgtPtrBegin, Size,
530                     Entry);
531 }
532