1330d8983SJohannes Doerfert //===-- OpenMP/Mapping.cpp - OpenMP/OpenACC pointer mapping impl. ---------===//
2330d8983SJohannes Doerfert //
3330d8983SJohannes Doerfert // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4330d8983SJohannes Doerfert // See https://llvm.org/LICENSE.txt for license information.
5330d8983SJohannes Doerfert // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6330d8983SJohannes Doerfert //
7330d8983SJohannes Doerfert //===----------------------------------------------------------------------===//
8330d8983SJohannes Doerfert //
9330d8983SJohannes Doerfert //===----------------------------------------------------------------------===//
10330d8983SJohannes Doerfert
11330d8983SJohannes Doerfert #include "OpenMP/Mapping.h"
12330d8983SJohannes Doerfert
13330d8983SJohannes Doerfert #include "PluginManager.h"
14330d8983SJohannes Doerfert #include "Shared/Debug.h"
15330d8983SJohannes Doerfert #include "Shared/Requirements.h"
16330d8983SJohannes Doerfert #include "device.h"
17330d8983SJohannes Doerfert
18330d8983SJohannes Doerfert /// Dump a table of all the host-target pointer pairs on failure
dumpTargetPointerMappings(const ident_t * Loc,DeviceTy & Device,bool toStdOut)19330d8983SJohannes Doerfert void dumpTargetPointerMappings(const ident_t *Loc, DeviceTy &Device,
20330d8983SJohannes Doerfert bool toStdOut) {
21330d8983SJohannes Doerfert MappingInfoTy::HDTTMapAccessorTy HDTTMap =
22330d8983SJohannes Doerfert Device.getMappingInfo().HostDataToTargetMap.getExclusiveAccessor();
23330d8983SJohannes Doerfert if (HDTTMap->empty()) {
24330d8983SJohannes Doerfert DUMP_INFO(toStdOut, OMP_INFOTYPE_ALL, Device.DeviceID,
25330d8983SJohannes Doerfert "OpenMP Host-Device pointer mappings table empty\n");
26330d8983SJohannes Doerfert return;
27330d8983SJohannes Doerfert }
28330d8983SJohannes Doerfert
29330d8983SJohannes Doerfert SourceInfo Kernel(Loc);
30330d8983SJohannes Doerfert DUMP_INFO(toStdOut, OMP_INFOTYPE_ALL, Device.DeviceID,
31330d8983SJohannes Doerfert "OpenMP Host-Device pointer mappings after block at %s:%d:%d:\n",
32330d8983SJohannes Doerfert Kernel.getFilename(), Kernel.getLine(), Kernel.getColumn());
33330d8983SJohannes Doerfert DUMP_INFO(toStdOut, OMP_INFOTYPE_ALL, Device.DeviceID,
34330d8983SJohannes Doerfert "%-18s %-18s %s %s %s %s\n", "Host Ptr", "Target Ptr", "Size (B)",
35330d8983SJohannes Doerfert "DynRefCount", "HoldRefCount", "Declaration");
36330d8983SJohannes Doerfert for (const auto &It : *HDTTMap) {
37330d8983SJohannes Doerfert HostDataToTargetTy &HDTT = *It.HDTT;
38330d8983SJohannes Doerfert SourceInfo Info(HDTT.HstPtrName);
39330d8983SJohannes Doerfert DUMP_INFO(toStdOut, OMP_INFOTYPE_ALL, Device.DeviceID,
40330d8983SJohannes Doerfert DPxMOD " " DPxMOD " %-8" PRIuPTR " %-11s %-12s %s at %s:%d:%d\n",
41330d8983SJohannes Doerfert DPxPTR(HDTT.HstPtrBegin), DPxPTR(HDTT.TgtPtrBegin),
42330d8983SJohannes Doerfert HDTT.HstPtrEnd - HDTT.HstPtrBegin,
43330d8983SJohannes Doerfert HDTT.dynRefCountToStr().c_str(), HDTT.holdRefCountToStr().c_str(),
44330d8983SJohannes Doerfert Info.getName(), Info.getFilename(), Info.getLine(),
45330d8983SJohannes Doerfert Info.getColumn());
46330d8983SJohannes Doerfert }
47330d8983SJohannes Doerfert }
48330d8983SJohannes Doerfert
associatePtr(void * HstPtrBegin,void * TgtPtrBegin,int64_t Size)49330d8983SJohannes Doerfert int MappingInfoTy::associatePtr(void *HstPtrBegin, void *TgtPtrBegin,
50330d8983SJohannes Doerfert int64_t Size) {
51330d8983SJohannes Doerfert HDTTMapAccessorTy HDTTMap = HostDataToTargetMap.getExclusiveAccessor();
52330d8983SJohannes Doerfert
53330d8983SJohannes Doerfert // Check if entry exists
54330d8983SJohannes Doerfert auto It = HDTTMap->find(HstPtrBegin);
55330d8983SJohannes Doerfert if (It != HDTTMap->end()) {
56330d8983SJohannes Doerfert HostDataToTargetTy &HDTT = *It->HDTT;
57330d8983SJohannes Doerfert std::lock_guard<HostDataToTargetTy> LG(HDTT);
58330d8983SJohannes Doerfert // Mapping already exists
59330d8983SJohannes Doerfert bool IsValid = HDTT.HstPtrEnd == (uintptr_t)HstPtrBegin + Size &&
60330d8983SJohannes Doerfert HDTT.TgtPtrBegin == (uintptr_t)TgtPtrBegin;
61330d8983SJohannes Doerfert if (IsValid) {
62330d8983SJohannes Doerfert DP("Attempt to re-associate the same device ptr+offset with the same "
63330d8983SJohannes Doerfert "host ptr, nothing to do\n");
64330d8983SJohannes Doerfert return OFFLOAD_SUCCESS;
65330d8983SJohannes Doerfert }
66330d8983SJohannes Doerfert REPORT("Not allowed to re-associate a different device ptr+offset with "
67330d8983SJohannes Doerfert "the same host ptr\n");
68330d8983SJohannes Doerfert return OFFLOAD_FAIL;
69330d8983SJohannes Doerfert }
70330d8983SJohannes Doerfert
71330d8983SJohannes Doerfert // Mapping does not exist, allocate it with refCount=INF
72330d8983SJohannes Doerfert const HostDataToTargetTy &NewEntry =
73330d8983SJohannes Doerfert *HDTTMap
74330d8983SJohannes Doerfert ->emplace(new HostDataToTargetTy(
75330d8983SJohannes Doerfert /*HstPtrBase=*/(uintptr_t)HstPtrBegin,
76330d8983SJohannes Doerfert /*HstPtrBegin=*/(uintptr_t)HstPtrBegin,
77330d8983SJohannes Doerfert /*HstPtrEnd=*/(uintptr_t)HstPtrBegin + Size,
78330d8983SJohannes Doerfert /*TgtAllocBegin=*/(uintptr_t)TgtPtrBegin,
79330d8983SJohannes Doerfert /*TgtPtrBegin=*/(uintptr_t)TgtPtrBegin,
80330d8983SJohannes Doerfert /*UseHoldRefCount=*/false, /*Name=*/nullptr,
81330d8983SJohannes Doerfert /*IsRefCountINF=*/true))
82330d8983SJohannes Doerfert .first->HDTT;
83330d8983SJohannes Doerfert DP("Creating new map entry: HstBase=" DPxMOD ", HstBegin=" DPxMOD
84330d8983SJohannes Doerfert ", HstEnd=" DPxMOD ", TgtBegin=" DPxMOD ", DynRefCount=%s, "
85330d8983SJohannes Doerfert "HoldRefCount=%s\n",
86330d8983SJohannes Doerfert DPxPTR(NewEntry.HstPtrBase), DPxPTR(NewEntry.HstPtrBegin),
87330d8983SJohannes Doerfert DPxPTR(NewEntry.HstPtrEnd), DPxPTR(NewEntry.TgtPtrBegin),
88330d8983SJohannes Doerfert NewEntry.dynRefCountToStr().c_str(), NewEntry.holdRefCountToStr().c_str());
89330d8983SJohannes Doerfert (void)NewEntry;
90330d8983SJohannes Doerfert
91330d8983SJohannes Doerfert // Notify the plugin about the new mapping.
92330d8983SJohannes Doerfert return Device.notifyDataMapped(HstPtrBegin, Size);
93330d8983SJohannes Doerfert }
94330d8983SJohannes Doerfert
disassociatePtr(void * HstPtrBegin)95330d8983SJohannes Doerfert int MappingInfoTy::disassociatePtr(void *HstPtrBegin) {
96330d8983SJohannes Doerfert HDTTMapAccessorTy HDTTMap = HostDataToTargetMap.getExclusiveAccessor();
97330d8983SJohannes Doerfert
98330d8983SJohannes Doerfert auto It = HDTTMap->find(HstPtrBegin);
99330d8983SJohannes Doerfert if (It == HDTTMap->end()) {
100330d8983SJohannes Doerfert REPORT("Association not found\n");
101330d8983SJohannes Doerfert return OFFLOAD_FAIL;
102330d8983SJohannes Doerfert }
103330d8983SJohannes Doerfert // Mapping exists
104330d8983SJohannes Doerfert HostDataToTargetTy &HDTT = *It->HDTT;
105330d8983SJohannes Doerfert std::lock_guard<HostDataToTargetTy> LG(HDTT);
106330d8983SJohannes Doerfert
107330d8983SJohannes Doerfert if (HDTT.getHoldRefCount()) {
108330d8983SJohannes Doerfert // This is based on OpenACC 3.1, sec 3.2.33 "acc_unmap_data", L3656-3657:
109330d8983SJohannes Doerfert // "It is an error to call acc_unmap_data if the structured reference
110330d8983SJohannes Doerfert // count for the pointer is not zero."
111330d8983SJohannes Doerfert REPORT("Trying to disassociate a pointer with a non-zero hold reference "
112330d8983SJohannes Doerfert "count\n");
113330d8983SJohannes Doerfert return OFFLOAD_FAIL;
114330d8983SJohannes Doerfert }
115330d8983SJohannes Doerfert
116330d8983SJohannes Doerfert if (HDTT.isDynRefCountInf()) {
117330d8983SJohannes Doerfert DP("Association found, removing it\n");
118330d8983SJohannes Doerfert void *Event = HDTT.getEvent();
119330d8983SJohannes Doerfert delete &HDTT;
120330d8983SJohannes Doerfert if (Event)
121330d8983SJohannes Doerfert Device.destroyEvent(Event);
122330d8983SJohannes Doerfert HDTTMap->erase(It);
123330d8983SJohannes Doerfert return Device.notifyDataUnmapped(HstPtrBegin);
124330d8983SJohannes Doerfert }
125330d8983SJohannes Doerfert
126330d8983SJohannes Doerfert REPORT("Trying to disassociate a pointer which was not mapped via "
127330d8983SJohannes Doerfert "omp_target_associate_ptr\n");
128330d8983SJohannes Doerfert return OFFLOAD_FAIL;
129330d8983SJohannes Doerfert }
130330d8983SJohannes Doerfert
lookupMapping(HDTTMapAccessorTy & HDTTMap,void * HstPtrBegin,int64_t Size,HostDataToTargetTy * OwnedTPR)131330d8983SJohannes Doerfert LookupResult MappingInfoTy::lookupMapping(HDTTMapAccessorTy &HDTTMap,
132330d8983SJohannes Doerfert void *HstPtrBegin, int64_t Size,
133330d8983SJohannes Doerfert HostDataToTargetTy *OwnedTPR) {
134330d8983SJohannes Doerfert
135330d8983SJohannes Doerfert uintptr_t HP = (uintptr_t)HstPtrBegin;
136330d8983SJohannes Doerfert LookupResult LR;
137330d8983SJohannes Doerfert
138330d8983SJohannes Doerfert DP("Looking up mapping(HstPtrBegin=" DPxMOD ", Size=%" PRId64 ")...\n",
139330d8983SJohannes Doerfert DPxPTR(HP), Size);
140330d8983SJohannes Doerfert
141330d8983SJohannes Doerfert if (HDTTMap->empty())
142330d8983SJohannes Doerfert return LR;
143330d8983SJohannes Doerfert
144330d8983SJohannes Doerfert auto Upper = HDTTMap->upper_bound(HP);
145330d8983SJohannes Doerfert
146330d8983SJohannes Doerfert if (Size == 0) {
147330d8983SJohannes Doerfert // specification v5.1 Pointer Initialization for Device Data Environments
148330d8983SJohannes Doerfert // upper_bound satisfies
149330d8983SJohannes Doerfert // std::prev(upper)->HDTT.HstPtrBegin <= hp < upper->HDTT.HstPtrBegin
150330d8983SJohannes Doerfert if (Upper != HDTTMap->begin()) {
151330d8983SJohannes Doerfert LR.TPR.setEntry(std::prev(Upper)->HDTT, OwnedTPR);
152330d8983SJohannes Doerfert // the left side of extended address range is satisified.
153330d8983SJohannes Doerfert // hp >= LR.TPR.getEntry()->HstPtrBegin || hp >=
154330d8983SJohannes Doerfert // LR.TPR.getEntry()->HstPtrBase
155330d8983SJohannes Doerfert LR.Flags.IsContained = HP < LR.TPR.getEntry()->HstPtrEnd ||
156330d8983SJohannes Doerfert HP < LR.TPR.getEntry()->HstPtrBase;
157330d8983SJohannes Doerfert }
158330d8983SJohannes Doerfert
159330d8983SJohannes Doerfert if (!LR.Flags.IsContained && Upper != HDTTMap->end()) {
160330d8983SJohannes Doerfert LR.TPR.setEntry(Upper->HDTT, OwnedTPR);
161330d8983SJohannes Doerfert // the right side of extended address range is satisified.
162330d8983SJohannes Doerfert // hp < LR.TPR.getEntry()->HstPtrEnd || hp < LR.TPR.getEntry()->HstPtrBase
163330d8983SJohannes Doerfert LR.Flags.IsContained = HP >= LR.TPR.getEntry()->HstPtrBase;
164330d8983SJohannes Doerfert }
165330d8983SJohannes Doerfert } else {
166330d8983SJohannes Doerfert // check the left bin
167330d8983SJohannes Doerfert if (Upper != HDTTMap->begin()) {
168330d8983SJohannes Doerfert LR.TPR.setEntry(std::prev(Upper)->HDTT, OwnedTPR);
169330d8983SJohannes Doerfert // Is it contained?
170330d8983SJohannes Doerfert LR.Flags.IsContained = HP >= LR.TPR.getEntry()->HstPtrBegin &&
171330d8983SJohannes Doerfert HP < LR.TPR.getEntry()->HstPtrEnd &&
172330d8983SJohannes Doerfert (HP + Size) <= LR.TPR.getEntry()->HstPtrEnd;
173330d8983SJohannes Doerfert // Does it extend beyond the mapped region?
174330d8983SJohannes Doerfert LR.Flags.ExtendsAfter = HP < LR.TPR.getEntry()->HstPtrEnd &&
175330d8983SJohannes Doerfert (HP + Size) > LR.TPR.getEntry()->HstPtrEnd;
176330d8983SJohannes Doerfert }
177330d8983SJohannes Doerfert
178330d8983SJohannes Doerfert // check the right bin
179330d8983SJohannes Doerfert if (!(LR.Flags.IsContained || LR.Flags.ExtendsAfter) &&
180330d8983SJohannes Doerfert Upper != HDTTMap->end()) {
181330d8983SJohannes Doerfert LR.TPR.setEntry(Upper->HDTT, OwnedTPR);
182330d8983SJohannes Doerfert // Does it extend into an already mapped region?
183330d8983SJohannes Doerfert LR.Flags.ExtendsBefore = HP < LR.TPR.getEntry()->HstPtrBegin &&
184330d8983SJohannes Doerfert (HP + Size) > LR.TPR.getEntry()->HstPtrBegin;
185330d8983SJohannes Doerfert // Does it extend beyond the mapped region?
186330d8983SJohannes Doerfert LR.Flags.ExtendsAfter = HP < LR.TPR.getEntry()->HstPtrEnd &&
187330d8983SJohannes Doerfert (HP + Size) > LR.TPR.getEntry()->HstPtrEnd;
188330d8983SJohannes Doerfert }
189330d8983SJohannes Doerfert
190330d8983SJohannes Doerfert if (LR.Flags.ExtendsBefore) {
191330d8983SJohannes Doerfert DP("WARNING: Pointer is not mapped but section extends into already "
192330d8983SJohannes Doerfert "mapped data\n");
193330d8983SJohannes Doerfert }
194330d8983SJohannes Doerfert if (LR.Flags.ExtendsAfter) {
195330d8983SJohannes Doerfert DP("WARNING: Pointer is already mapped but section extends beyond mapped "
196330d8983SJohannes Doerfert "region\n");
197330d8983SJohannes Doerfert }
198330d8983SJohannes Doerfert }
199330d8983SJohannes Doerfert
200330d8983SJohannes Doerfert return LR;
201330d8983SJohannes Doerfert }
202330d8983SJohannes Doerfert
getTargetPointer(HDTTMapAccessorTy & HDTTMap,void * HstPtrBegin,void * HstPtrBase,int64_t TgtPadding,int64_t Size,map_var_info_t HstPtrName,bool HasFlagTo,bool HasFlagAlways,bool IsImplicit,bool UpdateRefCount,bool HasCloseModifier,bool HasPresentModifier,bool HasHoldModifier,AsyncInfoTy & AsyncInfo,HostDataToTargetTy * OwnedTPR,bool ReleaseHDTTMap)203330d8983SJohannes Doerfert TargetPointerResultTy MappingInfoTy::getTargetPointer(
204330d8983SJohannes Doerfert HDTTMapAccessorTy &HDTTMap, void *HstPtrBegin, void *HstPtrBase,
205330d8983SJohannes Doerfert int64_t TgtPadding, int64_t Size, map_var_info_t HstPtrName, bool HasFlagTo,
206330d8983SJohannes Doerfert bool HasFlagAlways, bool IsImplicit, bool UpdateRefCount,
207330d8983SJohannes Doerfert bool HasCloseModifier, bool HasPresentModifier, bool HasHoldModifier,
208330d8983SJohannes Doerfert AsyncInfoTy &AsyncInfo, HostDataToTargetTy *OwnedTPR, bool ReleaseHDTTMap) {
209330d8983SJohannes Doerfert
210330d8983SJohannes Doerfert LookupResult LR = lookupMapping(HDTTMap, HstPtrBegin, Size, OwnedTPR);
211330d8983SJohannes Doerfert LR.TPR.Flags.IsPresent = true;
212330d8983SJohannes Doerfert
213330d8983SJohannes Doerfert // Release the mapping table lock only after the entry is locked by
214330d8983SJohannes Doerfert // attaching it to TPR. Once TPR is destroyed it will release the lock
215330d8983SJohannes Doerfert // on entry. If it is returned the lock will move to the returned object.
216330d8983SJohannes Doerfert // If LR.Entry is already owned/locked we avoid trying to lock it again.
217330d8983SJohannes Doerfert
218330d8983SJohannes Doerfert // Check if the pointer is contained.
219330d8983SJohannes Doerfert // If a variable is mapped to the device manually by the user - which would
220330d8983SJohannes Doerfert // lead to the IsContained flag to be true - then we must ensure that the
221330d8983SJohannes Doerfert // device address is returned even under unified memory conditions.
222330d8983SJohannes Doerfert if (LR.Flags.IsContained ||
223330d8983SJohannes Doerfert ((LR.Flags.ExtendsBefore || LR.Flags.ExtendsAfter) && IsImplicit)) {
224330d8983SJohannes Doerfert const char *RefCountAction;
225330d8983SJohannes Doerfert if (UpdateRefCount) {
226330d8983SJohannes Doerfert // After this, reference count >= 1. If the reference count was 0 but the
227330d8983SJohannes Doerfert // entry was still there we can reuse the data on the device and avoid a
228330d8983SJohannes Doerfert // new submission.
229330d8983SJohannes Doerfert LR.TPR.getEntry()->incRefCount(HasHoldModifier);
230330d8983SJohannes Doerfert RefCountAction = " (incremented)";
231330d8983SJohannes Doerfert } else {
232330d8983SJohannes Doerfert // It might have been allocated with the parent, but it's still new.
233330d8983SJohannes Doerfert LR.TPR.Flags.IsNewEntry = LR.TPR.getEntry()->getTotalRefCount() == 1;
234330d8983SJohannes Doerfert RefCountAction = " (update suppressed)";
235330d8983SJohannes Doerfert }
236330d8983SJohannes Doerfert const char *DynRefCountAction = HasHoldModifier ? "" : RefCountAction;
237330d8983SJohannes Doerfert const char *HoldRefCountAction = HasHoldModifier ? RefCountAction : "";
238330d8983SJohannes Doerfert uintptr_t Ptr = LR.TPR.getEntry()->TgtPtrBegin +
239330d8983SJohannes Doerfert ((uintptr_t)HstPtrBegin - LR.TPR.getEntry()->HstPtrBegin);
240330d8983SJohannes Doerfert INFO(OMP_INFOTYPE_MAPPING_EXISTS, Device.DeviceID,
241330d8983SJohannes Doerfert "Mapping exists%s with HstPtrBegin=" DPxMOD ", TgtPtrBegin=" DPxMOD
242330d8983SJohannes Doerfert ", Size=%" PRId64 ", DynRefCount=%s%s, HoldRefCount=%s%s, Name=%s\n",
243330d8983SJohannes Doerfert (IsImplicit ? " (implicit)" : ""), DPxPTR(HstPtrBegin), DPxPTR(Ptr),
244330d8983SJohannes Doerfert Size, LR.TPR.getEntry()->dynRefCountToStr().c_str(), DynRefCountAction,
245330d8983SJohannes Doerfert LR.TPR.getEntry()->holdRefCountToStr().c_str(), HoldRefCountAction,
246330d8983SJohannes Doerfert (HstPtrName) ? getNameFromMapping(HstPtrName).c_str() : "unknown");
247330d8983SJohannes Doerfert LR.TPR.TargetPointer = (void *)Ptr;
248330d8983SJohannes Doerfert } else if ((LR.Flags.ExtendsBefore || LR.Flags.ExtendsAfter) && !IsImplicit) {
249330d8983SJohannes Doerfert // Explicit extension of mapped data - not allowed.
250330d8983SJohannes Doerfert MESSAGE("explicit extension not allowed: host address specified is " DPxMOD
251330d8983SJohannes Doerfert " (%" PRId64
252330d8983SJohannes Doerfert " bytes), but device allocation maps to host at " DPxMOD
253330d8983SJohannes Doerfert " (%" PRId64 " bytes)",
254330d8983SJohannes Doerfert DPxPTR(HstPtrBegin), Size, DPxPTR(LR.TPR.getEntry()->HstPtrBegin),
255330d8983SJohannes Doerfert LR.TPR.getEntry()->HstPtrEnd - LR.TPR.getEntry()->HstPtrBegin);
256330d8983SJohannes Doerfert if (HasPresentModifier)
257330d8983SJohannes Doerfert MESSAGE("device mapping required by 'present' map type modifier does not "
258330d8983SJohannes Doerfert "exist for host address " DPxMOD " (%" PRId64 " bytes)",
259330d8983SJohannes Doerfert DPxPTR(HstPtrBegin), Size);
260330d8983SJohannes Doerfert } else if ((PM->getRequirements() & OMP_REQ_UNIFIED_SHARED_MEMORY &&
261330d8983SJohannes Doerfert !HasCloseModifier) ||
262330d8983SJohannes Doerfert (PM->getRequirements() & OMPX_REQ_AUTO_ZERO_COPY)) {
263330d8983SJohannes Doerfert
264330d8983SJohannes Doerfert // If unified shared memory is active, implicitly mapped variables that are
265330d8983SJohannes Doerfert // not privatized use host address. Any explicitly mapped variables also use
266330d8983SJohannes Doerfert // host address where correctness is not impeded. In all other cases maps
267330d8983SJohannes Doerfert // are respected.
268330d8983SJohannes Doerfert // In addition to the mapping rules above, the close map modifier forces the
269330d8983SJohannes Doerfert // mapping of the variable to the device.
270330d8983SJohannes Doerfert if (Size) {
271330d8983SJohannes Doerfert INFO(OMP_INFOTYPE_MAPPING_CHANGED, Device.DeviceID,
272330d8983SJohannes Doerfert "Return HstPtrBegin " DPxMOD " Size=%" PRId64 " for unified shared "
273330d8983SJohannes Doerfert "memory\n",
274330d8983SJohannes Doerfert DPxPTR((uintptr_t)HstPtrBegin), Size);
275330d8983SJohannes Doerfert DP("Return HstPtrBegin " DPxMOD " Size=%" PRId64 " for unified shared "
276330d8983SJohannes Doerfert "memory\n",
277330d8983SJohannes Doerfert DPxPTR((uintptr_t)HstPtrBegin), Size);
278330d8983SJohannes Doerfert LR.TPR.Flags.IsPresent = false;
279330d8983SJohannes Doerfert LR.TPR.Flags.IsHostPointer = true;
280330d8983SJohannes Doerfert LR.TPR.TargetPointer = HstPtrBegin;
281330d8983SJohannes Doerfert }
282330d8983SJohannes Doerfert } else if (HasPresentModifier) {
283330d8983SJohannes Doerfert DP("Mapping required by 'present' map type modifier does not exist for "
284330d8983SJohannes Doerfert "HstPtrBegin=" DPxMOD ", Size=%" PRId64 "\n",
285330d8983SJohannes Doerfert DPxPTR(HstPtrBegin), Size);
286330d8983SJohannes Doerfert MESSAGE("device mapping required by 'present' map type modifier does not "
287330d8983SJohannes Doerfert "exist for host address " DPxMOD " (%" PRId64 " bytes)",
288330d8983SJohannes Doerfert DPxPTR(HstPtrBegin), Size);
289330d8983SJohannes Doerfert } else if (Size) {
290330d8983SJohannes Doerfert // If it is not contained and Size > 0, we should create a new entry for it.
291330d8983SJohannes Doerfert LR.TPR.Flags.IsNewEntry = true;
292330d8983SJohannes Doerfert uintptr_t TgtAllocBegin =
293330d8983SJohannes Doerfert (uintptr_t)Device.allocData(TgtPadding + Size, HstPtrBegin);
294330d8983SJohannes Doerfert uintptr_t TgtPtrBegin = TgtAllocBegin + TgtPadding;
295330d8983SJohannes Doerfert // Release the mapping table lock only after the entry is locked by
296330d8983SJohannes Doerfert // attaching it to TPR.
297330d8983SJohannes Doerfert LR.TPR.setEntry(HDTTMap
298330d8983SJohannes Doerfert ->emplace(new HostDataToTargetTy(
299330d8983SJohannes Doerfert (uintptr_t)HstPtrBase, (uintptr_t)HstPtrBegin,
300330d8983SJohannes Doerfert (uintptr_t)HstPtrBegin + Size, TgtAllocBegin,
301330d8983SJohannes Doerfert TgtPtrBegin, HasHoldModifier, HstPtrName))
302330d8983SJohannes Doerfert .first->HDTT);
303330d8983SJohannes Doerfert INFO(OMP_INFOTYPE_MAPPING_CHANGED, Device.DeviceID,
304330d8983SJohannes Doerfert "Creating new map entry with HstPtrBase=" DPxMOD
305330d8983SJohannes Doerfert ", HstPtrBegin=" DPxMOD ", TgtAllocBegin=" DPxMOD
306330d8983SJohannes Doerfert ", TgtPtrBegin=" DPxMOD
307330d8983SJohannes Doerfert ", Size=%ld, DynRefCount=%s, HoldRefCount=%s, Name=%s\n",
308330d8983SJohannes Doerfert DPxPTR(HstPtrBase), DPxPTR(HstPtrBegin), DPxPTR(TgtAllocBegin),
309330d8983SJohannes Doerfert DPxPTR(TgtPtrBegin), Size,
310330d8983SJohannes Doerfert LR.TPR.getEntry()->dynRefCountToStr().c_str(),
311330d8983SJohannes Doerfert LR.TPR.getEntry()->holdRefCountToStr().c_str(),
312330d8983SJohannes Doerfert (HstPtrName) ? getNameFromMapping(HstPtrName).c_str() : "unknown");
313330d8983SJohannes Doerfert LR.TPR.TargetPointer = (void *)TgtPtrBegin;
314330d8983SJohannes Doerfert
315330d8983SJohannes Doerfert // Notify the plugin about the new mapping.
316330d8983SJohannes Doerfert if (Device.notifyDataMapped(HstPtrBegin, Size))
317*81d20d86SJoseph Huber return TargetPointerResultTy{};
318330d8983SJohannes Doerfert } else {
319330d8983SJohannes Doerfert // This entry is not present and we did not create a new entry for it.
320330d8983SJohannes Doerfert LR.TPR.Flags.IsPresent = false;
321330d8983SJohannes Doerfert }
322330d8983SJohannes Doerfert
323330d8983SJohannes Doerfert // All mapping table modifications have been made. If the user requested it we
324330d8983SJohannes Doerfert // give up the lock.
325330d8983SJohannes Doerfert if (ReleaseHDTTMap)
326330d8983SJohannes Doerfert HDTTMap.destroy();
327330d8983SJohannes Doerfert
328330d8983SJohannes Doerfert // If the target pointer is valid, and we need to transfer data, issue the
329330d8983SJohannes Doerfert // data transfer.
330330d8983SJohannes Doerfert if (LR.TPR.TargetPointer && !LR.TPR.Flags.IsHostPointer && HasFlagTo &&
331330d8983SJohannes Doerfert (LR.TPR.Flags.IsNewEntry || HasFlagAlways) && Size != 0) {
332330d8983SJohannes Doerfert DP("Moving %" PRId64 " bytes (hst:" DPxMOD ") -> (tgt:" DPxMOD ")\n", Size,
333330d8983SJohannes Doerfert DPxPTR(HstPtrBegin), DPxPTR(LR.TPR.TargetPointer));
334330d8983SJohannes Doerfert
335330d8983SJohannes Doerfert int Ret = Device.submitData(LR.TPR.TargetPointer, HstPtrBegin, Size,
336330d8983SJohannes Doerfert AsyncInfo, LR.TPR.getEntry());
337330d8983SJohannes Doerfert if (Ret != OFFLOAD_SUCCESS) {
338330d8983SJohannes Doerfert REPORT("Copying data to device failed.\n");
339330d8983SJohannes Doerfert // We will also return nullptr if the data movement fails because that
340330d8983SJohannes Doerfert // pointer points to a corrupted memory region so it doesn't make any
341330d8983SJohannes Doerfert // sense to continue to use it.
342330d8983SJohannes Doerfert LR.TPR.TargetPointer = nullptr;
343330d8983SJohannes Doerfert } else if (LR.TPR.getEntry()->addEventIfNecessary(Device, AsyncInfo) !=
344330d8983SJohannes Doerfert OFFLOAD_SUCCESS)
345*81d20d86SJoseph Huber return TargetPointerResultTy{};
346330d8983SJohannes Doerfert } else {
347330d8983SJohannes Doerfert // If not a host pointer and no present modifier, we need to wait for the
348330d8983SJohannes Doerfert // event if it exists.
349330d8983SJohannes Doerfert // Note: Entry might be nullptr because of zero length array section.
350330d8983SJohannes Doerfert if (LR.TPR.getEntry() && !LR.TPR.Flags.IsHostPointer &&
351330d8983SJohannes Doerfert !HasPresentModifier) {
352330d8983SJohannes Doerfert void *Event = LR.TPR.getEntry()->getEvent();
353330d8983SJohannes Doerfert if (Event) {
354330d8983SJohannes Doerfert int Ret = Device.waitEvent(Event, AsyncInfo);
355330d8983SJohannes Doerfert if (Ret != OFFLOAD_SUCCESS) {
356330d8983SJohannes Doerfert // If it fails to wait for the event, we need to return nullptr in
357330d8983SJohannes Doerfert // case of any data race.
358330d8983SJohannes Doerfert REPORT("Failed to wait for event " DPxMOD ".\n", DPxPTR(Event));
359*81d20d86SJoseph Huber return TargetPointerResultTy{};
360330d8983SJohannes Doerfert }
361330d8983SJohannes Doerfert }
362330d8983SJohannes Doerfert }
363330d8983SJohannes Doerfert }
364330d8983SJohannes Doerfert
365330d8983SJohannes Doerfert return std::move(LR.TPR);
366330d8983SJohannes Doerfert }
367330d8983SJohannes Doerfert
getTgtPtrBegin(void * HstPtrBegin,int64_t Size,bool UpdateRefCount,bool UseHoldRefCount,bool MustContain,bool ForceDelete,bool FromDataEnd)368330d8983SJohannes Doerfert TargetPointerResultTy MappingInfoTy::getTgtPtrBegin(
369330d8983SJohannes Doerfert void *HstPtrBegin, int64_t Size, bool UpdateRefCount, bool UseHoldRefCount,
370330d8983SJohannes Doerfert bool MustContain, bool ForceDelete, bool FromDataEnd) {
371330d8983SJohannes Doerfert HDTTMapAccessorTy HDTTMap = HostDataToTargetMap.getExclusiveAccessor();
372330d8983SJohannes Doerfert
373330d8983SJohannes Doerfert LookupResult LR = lookupMapping(HDTTMap, HstPtrBegin, Size);
374330d8983SJohannes Doerfert
375330d8983SJohannes Doerfert LR.TPR.Flags.IsPresent = true;
376330d8983SJohannes Doerfert
377330d8983SJohannes Doerfert if (LR.Flags.IsContained ||
378330d8983SJohannes Doerfert (!MustContain && (LR.Flags.ExtendsBefore || LR.Flags.ExtendsAfter))) {
379330d8983SJohannes Doerfert LR.TPR.Flags.IsLast =
380330d8983SJohannes Doerfert LR.TPR.getEntry()->decShouldRemove(UseHoldRefCount, ForceDelete);
381330d8983SJohannes Doerfert
382330d8983SJohannes Doerfert if (ForceDelete) {
383330d8983SJohannes Doerfert LR.TPR.getEntry()->resetRefCount(UseHoldRefCount);
384330d8983SJohannes Doerfert assert(LR.TPR.Flags.IsLast ==
385330d8983SJohannes Doerfert LR.TPR.getEntry()->decShouldRemove(UseHoldRefCount) &&
386330d8983SJohannes Doerfert "expected correct IsLast prediction for reset");
387330d8983SJohannes Doerfert }
388330d8983SJohannes Doerfert
389330d8983SJohannes Doerfert // Increment the number of threads that is using the entry on a
390330d8983SJohannes Doerfert // targetDataEnd, tracking the number of possible "deleters". A thread may
391330d8983SJohannes Doerfert // come to own the entry deletion even if it was not the last one querying
392330d8983SJohannes Doerfert // for it. Thus, we must track every query on targetDataEnds to ensure only
393330d8983SJohannes Doerfert // the last thread that holds a reference to an entry actually deletes it.
394330d8983SJohannes Doerfert if (FromDataEnd)
395330d8983SJohannes Doerfert LR.TPR.getEntry()->incDataEndThreadCount();
396330d8983SJohannes Doerfert
397330d8983SJohannes Doerfert const char *RefCountAction;
398330d8983SJohannes Doerfert if (!UpdateRefCount) {
399330d8983SJohannes Doerfert RefCountAction = " (update suppressed)";
400330d8983SJohannes Doerfert } else if (LR.TPR.Flags.IsLast) {
401330d8983SJohannes Doerfert LR.TPR.getEntry()->decRefCount(UseHoldRefCount);
402330d8983SJohannes Doerfert assert(LR.TPR.getEntry()->getTotalRefCount() == 0 &&
403330d8983SJohannes Doerfert "Expected zero reference count when deletion is scheduled");
404330d8983SJohannes Doerfert if (ForceDelete)
405330d8983SJohannes Doerfert RefCountAction = " (reset, delayed deletion)";
406330d8983SJohannes Doerfert else
407330d8983SJohannes Doerfert RefCountAction = " (decremented, delayed deletion)";
408330d8983SJohannes Doerfert } else {
409330d8983SJohannes Doerfert LR.TPR.getEntry()->decRefCount(UseHoldRefCount);
410330d8983SJohannes Doerfert RefCountAction = " (decremented)";
411330d8983SJohannes Doerfert }
412330d8983SJohannes Doerfert const char *DynRefCountAction = UseHoldRefCount ? "" : RefCountAction;
413330d8983SJohannes Doerfert const char *HoldRefCountAction = UseHoldRefCount ? RefCountAction : "";
414330d8983SJohannes Doerfert uintptr_t TP = LR.TPR.getEntry()->TgtPtrBegin +
415330d8983SJohannes Doerfert ((uintptr_t)HstPtrBegin - LR.TPR.getEntry()->HstPtrBegin);
416330d8983SJohannes Doerfert INFO(OMP_INFOTYPE_MAPPING_EXISTS, Device.DeviceID,
417330d8983SJohannes Doerfert "Mapping exists with HstPtrBegin=" DPxMOD ", TgtPtrBegin=" DPxMOD ", "
418330d8983SJohannes Doerfert "Size=%" PRId64 ", DynRefCount=%s%s, HoldRefCount=%s%s\n",
419330d8983SJohannes Doerfert DPxPTR(HstPtrBegin), DPxPTR(TP), Size,
420330d8983SJohannes Doerfert LR.TPR.getEntry()->dynRefCountToStr().c_str(), DynRefCountAction,
421330d8983SJohannes Doerfert LR.TPR.getEntry()->holdRefCountToStr().c_str(), HoldRefCountAction);
422330d8983SJohannes Doerfert LR.TPR.TargetPointer = (void *)TP;
423330d8983SJohannes Doerfert } else if (PM->getRequirements() & OMP_REQ_UNIFIED_SHARED_MEMORY ||
424330d8983SJohannes Doerfert PM->getRequirements() & OMPX_REQ_AUTO_ZERO_COPY) {
425330d8983SJohannes Doerfert // If the value isn't found in the mapping and unified shared memory
426330d8983SJohannes Doerfert // is on then it means we have stumbled upon a value which we need to
427330d8983SJohannes Doerfert // use directly from the host.
428330d8983SJohannes Doerfert DP("Get HstPtrBegin " DPxMOD " Size=%" PRId64 " for unified shared "
429330d8983SJohannes Doerfert "memory\n",
430330d8983SJohannes Doerfert DPxPTR((uintptr_t)HstPtrBegin), Size);
431330d8983SJohannes Doerfert LR.TPR.Flags.IsPresent = false;
432330d8983SJohannes Doerfert LR.TPR.Flags.IsHostPointer = true;
433330d8983SJohannes Doerfert LR.TPR.TargetPointer = HstPtrBegin;
434330d8983SJohannes Doerfert } else {
435330d8983SJohannes Doerfert // OpenMP Specification v5.2: if a matching list item is not found, the
436330d8983SJohannes Doerfert // pointer retains its original value as per firstprivate semantics.
437330d8983SJohannes Doerfert LR.TPR.Flags.IsPresent = false;
438330d8983SJohannes Doerfert LR.TPR.Flags.IsHostPointer = false;
439330d8983SJohannes Doerfert LR.TPR.TargetPointer = HstPtrBegin;
440330d8983SJohannes Doerfert }
441330d8983SJohannes Doerfert
442330d8983SJohannes Doerfert return std::move(LR.TPR);
443330d8983SJohannes Doerfert }
444330d8983SJohannes Doerfert
445330d8983SJohannes Doerfert // Return the target pointer begin (where the data will be moved).
getTgtPtrBegin(HDTTMapAccessorTy & HDTTMap,void * HstPtrBegin,int64_t Size)446330d8983SJohannes Doerfert void *MappingInfoTy::getTgtPtrBegin(HDTTMapAccessorTy &HDTTMap,
447330d8983SJohannes Doerfert void *HstPtrBegin, int64_t Size) {
448330d8983SJohannes Doerfert uintptr_t HP = (uintptr_t)HstPtrBegin;
449330d8983SJohannes Doerfert LookupResult LR = lookupMapping(HDTTMap, HstPtrBegin, Size);
450330d8983SJohannes Doerfert if (LR.Flags.IsContained || LR.Flags.ExtendsBefore || LR.Flags.ExtendsAfter) {
451330d8983SJohannes Doerfert uintptr_t TP =
452330d8983SJohannes Doerfert LR.TPR.getEntry()->TgtPtrBegin + (HP - LR.TPR.getEntry()->HstPtrBegin);
453330d8983SJohannes Doerfert return (void *)TP;
454330d8983SJohannes Doerfert }
455330d8983SJohannes Doerfert
456330d8983SJohannes Doerfert return NULL;
457330d8983SJohannes Doerfert }
458330d8983SJohannes Doerfert
eraseMapEntry(HDTTMapAccessorTy & HDTTMap,HostDataToTargetTy * Entry,int64_t Size)459330d8983SJohannes Doerfert int MappingInfoTy::eraseMapEntry(HDTTMapAccessorTy &HDTTMap,
460330d8983SJohannes Doerfert HostDataToTargetTy *Entry, int64_t Size) {
461330d8983SJohannes Doerfert assert(Entry && "Trying to delete a null entry from the HDTT map.");
462330d8983SJohannes Doerfert assert(Entry->getTotalRefCount() == 0 &&
463330d8983SJohannes Doerfert Entry->getDataEndThreadCount() == 0 &&
464330d8983SJohannes Doerfert "Trying to delete entry that is in use or owned by another thread.");
465330d8983SJohannes Doerfert
466330d8983SJohannes Doerfert INFO(OMP_INFOTYPE_MAPPING_CHANGED, Device.DeviceID,
467330d8983SJohannes Doerfert "Removing map entry with HstPtrBegin=" DPxMOD ", TgtPtrBegin=" DPxMOD
468330d8983SJohannes Doerfert ", Size=%" PRId64 ", Name=%s\n",
469330d8983SJohannes Doerfert DPxPTR(Entry->HstPtrBegin), DPxPTR(Entry->TgtPtrBegin), Size,
470330d8983SJohannes Doerfert (Entry->HstPtrName) ? getNameFromMapping(Entry->HstPtrName).c_str()
471330d8983SJohannes Doerfert : "unknown");
472330d8983SJohannes Doerfert
473330d8983SJohannes Doerfert if (HDTTMap->erase(Entry) == 0) {
474330d8983SJohannes Doerfert REPORT("Trying to remove a non-existent map entry\n");
475330d8983SJohannes Doerfert return OFFLOAD_FAIL;
476330d8983SJohannes Doerfert }
477330d8983SJohannes Doerfert
478330d8983SJohannes Doerfert return OFFLOAD_SUCCESS;
479330d8983SJohannes Doerfert }
480330d8983SJohannes Doerfert
deallocTgtPtrAndEntry(HostDataToTargetTy * Entry,int64_t Size)481330d8983SJohannes Doerfert int MappingInfoTy::deallocTgtPtrAndEntry(HostDataToTargetTy *Entry,
482330d8983SJohannes Doerfert int64_t Size) {
483330d8983SJohannes Doerfert assert(Entry && "Trying to deallocate a null entry.");
484330d8983SJohannes Doerfert
485330d8983SJohannes Doerfert DP("Deleting tgt data " DPxMOD " of size %" PRId64 " by freeing allocation "
486330d8983SJohannes Doerfert "starting at " DPxMOD "\n",
487330d8983SJohannes Doerfert DPxPTR(Entry->TgtPtrBegin), Size, DPxPTR(Entry->TgtAllocBegin));
488330d8983SJohannes Doerfert
489330d8983SJohannes Doerfert void *Event = Entry->getEvent();
490330d8983SJohannes Doerfert if (Event && Device.destroyEvent(Event) != OFFLOAD_SUCCESS) {
491330d8983SJohannes Doerfert REPORT("Failed to destroy event " DPxMOD "\n", DPxPTR(Event));
492330d8983SJohannes Doerfert return OFFLOAD_FAIL;
493330d8983SJohannes Doerfert }
494330d8983SJohannes Doerfert
495330d8983SJohannes Doerfert int Ret = Device.deleteData((void *)Entry->TgtAllocBegin);
496330d8983SJohannes Doerfert
497330d8983SJohannes Doerfert // Notify the plugin about the unmapped memory.
498330d8983SJohannes Doerfert Ret |= Device.notifyDataUnmapped((void *)Entry->HstPtrBegin);
499330d8983SJohannes Doerfert
500330d8983SJohannes Doerfert delete Entry;
501330d8983SJohannes Doerfert
502330d8983SJohannes Doerfert return Ret;
503330d8983SJohannes Doerfert }
504330d8983SJohannes Doerfert
printCopyInfoImpl(int DeviceId,bool H2D,void * SrcPtrBegin,void * DstPtrBegin,int64_t Size,HostDataToTargetTy * HT)505330d8983SJohannes Doerfert static void printCopyInfoImpl(int DeviceId, bool H2D, void *SrcPtrBegin,
506330d8983SJohannes Doerfert void *DstPtrBegin, int64_t Size,
507330d8983SJohannes Doerfert HostDataToTargetTy *HT) {
508330d8983SJohannes Doerfert
509330d8983SJohannes Doerfert INFO(OMP_INFOTYPE_DATA_TRANSFER, DeviceId,
510330d8983SJohannes Doerfert "Copying data from %s to %s, %sPtr=" DPxMOD ", %sPtr=" DPxMOD
511330d8983SJohannes Doerfert ", Size=%" PRId64 ", Name=%s\n",
512330d8983SJohannes Doerfert H2D ? "host" : "device", H2D ? "device" : "host", H2D ? "Hst" : "Tgt",
513330d8983SJohannes Doerfert DPxPTR(H2D ? SrcPtrBegin : DstPtrBegin), H2D ? "Tgt" : "Hst",
514330d8983SJohannes Doerfert DPxPTR(H2D ? DstPtrBegin : SrcPtrBegin), Size,
515330d8983SJohannes Doerfert (HT && HT->HstPtrName) ? getNameFromMapping(HT->HstPtrName).c_str()
516330d8983SJohannes Doerfert : "unknown");
517330d8983SJohannes Doerfert }
518330d8983SJohannes Doerfert
printCopyInfo(void * TgtPtrBegin,void * HstPtrBegin,int64_t Size,bool H2D,HostDataToTargetTy * Entry,MappingInfoTy::HDTTMapAccessorTy * HDTTMapPtr)519330d8983SJohannes Doerfert void MappingInfoTy::printCopyInfo(
520330d8983SJohannes Doerfert void *TgtPtrBegin, void *HstPtrBegin, int64_t Size, bool H2D,
521330d8983SJohannes Doerfert HostDataToTargetTy *Entry, MappingInfoTy::HDTTMapAccessorTy *HDTTMapPtr) {
522330d8983SJohannes Doerfert auto HDTTMap =
523330d8983SJohannes Doerfert HostDataToTargetMap.getExclusiveAccessor(!!Entry || !!HDTTMapPtr);
524330d8983SJohannes Doerfert LookupResult LR;
525330d8983SJohannes Doerfert if (!Entry) {
526330d8983SJohannes Doerfert LR = lookupMapping(HDTTMapPtr ? *HDTTMapPtr : HDTTMap, HstPtrBegin, Size);
527330d8983SJohannes Doerfert Entry = LR.TPR.getEntry();
528330d8983SJohannes Doerfert }
529330d8983SJohannes Doerfert printCopyInfoImpl(Device.DeviceID, H2D, HstPtrBegin, TgtPtrBegin, Size,
530330d8983SJohannes Doerfert Entry);
531330d8983SJohannes Doerfert }
532