1 //===---------- ExecutorSharedMemoryMapperService.cpp -----------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "llvm/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.h"
10 
11 #include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h"
12 #include "llvm/Support/Process.h"
13 #include "llvm/Support/WindowsError.h"
14 
15 #include <sstream>
16 
17 #if defined(LLVM_ON_UNIX)
18 #include <errno.h>
19 #include <fcntl.h>
20 #include <sys/mman.h>
21 #if defined(__MVS__)
22 #include "llvm/Support/BLAKE3.h"
23 #include <sys/shm.h>
24 #endif
25 #include <unistd.h>
26 #endif
27 
28 namespace llvm {
29 namespace orc {
30 namespace rt_bootstrap {
31 
32 #if defined(_WIN32)
33 static DWORD getWindowsProtectionFlags(MemProt MP) {
34   if (MP == MemProt::Read)
35     return PAGE_READONLY;
36   if (MP == MemProt::Write ||
37       MP == (MemProt::Write | MemProt::Read)) {
38     // Note: PAGE_WRITE is not supported by VirtualProtect
39     return PAGE_READWRITE;
40   }
41   if (MP == (MemProt::Read | MemProt::Exec))
42     return PAGE_EXECUTE_READ;
43   if (MP == (MemProt::Read | MemProt::Write | MemProt::Exec))
44     return PAGE_EXECUTE_READWRITE;
45   if (MP == MemProt::Exec)
46     return PAGE_EXECUTE;
47 
48   return PAGE_NOACCESS;
49 }
50 #endif
51 
52 Expected<std::pair<ExecutorAddr, std::string>>
53 ExecutorSharedMemoryMapperService::reserve(uint64_t Size) {
54 #if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32)
55 
56 #if defined(LLVM_ON_UNIX)
57 
58   std::string SharedMemoryName;
59   {
60     std::stringstream SharedMemoryNameStream;
61     SharedMemoryNameStream << "/jitlink_" << sys::Process::getProcessId() << '_'
62                            << (++SharedMemoryCount);
63     SharedMemoryName = SharedMemoryNameStream.str();
64   }
65 
66 #if defined(__MVS__)
67   ArrayRef<uint8_t> Data(
68       reinterpret_cast<const uint8_t *>(SharedMemoryName.c_str()),
69       SharedMemoryName.size());
70   auto HashedName = BLAKE3::hash<sizeof(key_t)>(Data);
71   key_t Key = *reinterpret_cast<key_t *>(HashedName.data());
72   int SharedMemoryId =
73       shmget(Key, Size, IPC_CREAT | IPC_EXCL | __IPC_SHAREAS | 0700);
74   if (SharedMemoryId < 0)
75     return errorCodeToError(errnoAsErrorCode());
76 
77   void *Addr = shmat(SharedMemoryId, nullptr, 0);
78   if (Addr == reinterpret_cast<void *>(-1))
79     return errorCodeToError(errnoAsErrorCode());
80 #else
81   int SharedMemoryFile =
82       shm_open(SharedMemoryName.c_str(), O_RDWR | O_CREAT | O_EXCL, 0700);
83   if (SharedMemoryFile < 0)
84     return errorCodeToError(errnoAsErrorCode());
85 
86   // by default size is 0
87   if (ftruncate(SharedMemoryFile, Size) < 0)
88     return errorCodeToError(errnoAsErrorCode());
89 
90   void *Addr = mmap(nullptr, Size, PROT_NONE, MAP_SHARED, SharedMemoryFile, 0);
91   if (Addr == MAP_FAILED)
92     return errorCodeToError(errnoAsErrorCode());
93 
94   close(SharedMemoryFile);
95 #endif
96 
97 #elif defined(_WIN32)
98 
99   std::string SharedMemoryName;
100   {
101     std::stringstream SharedMemoryNameStream;
102     SharedMemoryNameStream << "jitlink_" << sys::Process::getProcessId() << '_'
103                            << (++SharedMemoryCount);
104     SharedMemoryName = SharedMemoryNameStream.str();
105   }
106 
107   std::wstring WideSharedMemoryName(SharedMemoryName.begin(),
108                                     SharedMemoryName.end());
109   HANDLE SharedMemoryFile = CreateFileMappingW(
110       INVALID_HANDLE_VALUE, NULL, PAGE_EXECUTE_READWRITE, Size >> 32,
111       Size & 0xffffffff, WideSharedMemoryName.c_str());
112   if (!SharedMemoryFile)
113     return errorCodeToError(mapWindowsError(GetLastError()));
114 
115   void *Addr = MapViewOfFile(SharedMemoryFile,
116                              FILE_MAP_ALL_ACCESS | FILE_MAP_EXECUTE, 0, 0, 0);
117   if (!Addr) {
118     CloseHandle(SharedMemoryFile);
119     return errorCodeToError(mapWindowsError(GetLastError()));
120   }
121 
122 #endif
123 
124   {
125     std::lock_guard<std::mutex> Lock(Mutex);
126     Reservations[Addr].Size = Size;
127 #if defined(_WIN32)
128     Reservations[Addr].SharedMemoryFile = SharedMemoryFile;
129 #endif
130   }
131 
132   return std::make_pair(ExecutorAddr::fromPtr(Addr),
133                         std::move(SharedMemoryName));
134 #else
135   return make_error<StringError>(
136       "SharedMemoryMapper is not supported on this platform yet",
137       inconvertibleErrorCode());
138 #endif
139 }
140 
141 Expected<ExecutorAddr> ExecutorSharedMemoryMapperService::initialize(
142     ExecutorAddr Reservation, tpctypes::SharedMemoryFinalizeRequest &FR) {
143 #if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32)
144 
145   ExecutorAddr MinAddr(~0ULL);
146 
147   // Contents are already in place
148   for (auto &Segment : FR.Segments) {
149     if (Segment.Addr < MinAddr)
150       MinAddr = Segment.Addr;
151 
152 #if defined(LLVM_ON_UNIX)
153 
154 #if defined(__MVS__)
155       // TODO Is it possible to change the protection level?
156 #else
157     int NativeProt = 0;
158     if ((Segment.RAG.Prot & MemProt::Read) == MemProt::Read)
159       NativeProt |= PROT_READ;
160     if ((Segment.RAG.Prot & MemProt::Write) == MemProt::Write)
161       NativeProt |= PROT_WRITE;
162     if ((Segment.RAG.Prot & MemProt::Exec) == MemProt::Exec)
163       NativeProt |= PROT_EXEC;
164 
165     if (mprotect(Segment.Addr.toPtr<void *>(), Segment.Size, NativeProt))
166       return errorCodeToError(errnoAsErrorCode());
167 #endif
168 
169 #elif defined(_WIN32)
170 
171     DWORD NativeProt = getWindowsProtectionFlags(Segment.RAG.Prot);
172 
173     if (!VirtualProtect(Segment.Addr.toPtr<void *>(), Segment.Size, NativeProt,
174                         &NativeProt))
175       return errorCodeToError(mapWindowsError(GetLastError()));
176 
177 #endif
178 
179     if ((Segment.RAG.Prot & MemProt::Exec) == MemProt::Exec)
180       sys::Memory::InvalidateInstructionCache(Segment.Addr.toPtr<void *>(),
181                                               Segment.Size);
182   }
183 
184   // Run finalization actions and get deinitlization action list.
185   auto DeinitializeActions = shared::runFinalizeActions(FR.Actions);
186   if (!DeinitializeActions) {
187     return DeinitializeActions.takeError();
188   }
189 
190   {
191     std::lock_guard<std::mutex> Lock(Mutex);
192     Allocations[MinAddr].DeinitializationActions =
193         std::move(*DeinitializeActions);
194     Reservations[Reservation.toPtr<void *>()].Allocations.push_back(MinAddr);
195   }
196 
197   return MinAddr;
198 
199 #else
200   return make_error<StringError>(
201       "SharedMemoryMapper is not supported on this platform yet",
202       inconvertibleErrorCode());
203 #endif
204 }
205 
206 Error ExecutorSharedMemoryMapperService::deinitialize(
207     const std::vector<ExecutorAddr> &Bases) {
208   Error AllErr = Error::success();
209 
210   {
211     std::lock_guard<std::mutex> Lock(Mutex);
212 
213     for (auto Base : llvm::reverse(Bases)) {
214       if (Error Err = shared::runDeallocActions(
215               Allocations[Base].DeinitializationActions)) {
216         AllErr = joinErrors(std::move(AllErr), std::move(Err));
217       }
218 
219       // Remove the allocation from the allocation list of its reservation
220       for (auto &Reservation : Reservations) {
221         auto AllocationIt = llvm::find(Reservation.second.Allocations, Base);
222         if (AllocationIt != Reservation.second.Allocations.end()) {
223           Reservation.second.Allocations.erase(AllocationIt);
224           break;
225         }
226       }
227 
228       Allocations.erase(Base);
229     }
230   }
231 
232   return AllErr;
233 }
234 
235 Error ExecutorSharedMemoryMapperService::release(
236     const std::vector<ExecutorAddr> &Bases) {
237 #if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32)
238   Error Err = Error::success();
239 
240   for (auto Base : Bases) {
241     std::vector<ExecutorAddr> AllocAddrs;
242     size_t Size;
243 
244 #if defined(_WIN32)
245     HANDLE SharedMemoryFile;
246 #endif
247 
248     {
249       std::lock_guard<std::mutex> Lock(Mutex);
250       auto &R = Reservations[Base.toPtr<void *>()];
251       Size = R.Size;
252 
253 #if defined(_WIN32)
254       SharedMemoryFile = R.SharedMemoryFile;
255 #endif
256 
257       AllocAddrs.swap(R.Allocations);
258     }
259 
260     // deinitialize sub allocations
261     if (Error E = deinitialize(AllocAddrs))
262       Err = joinErrors(std::move(Err), std::move(E));
263 
264 #if defined(LLVM_ON_UNIX)
265 
266 #if defined(__MVS__)
267     (void)Size;
268 
269     if (shmdt(Base.toPtr<void *>()) < 0)
270       Err = joinErrors(std::move(Err), errorCodeToError(errnoAsErrorCode()));
271 #else
272     if (munmap(Base.toPtr<void *>(), Size) != 0)
273       Err = joinErrors(std::move(Err), errorCodeToError(errnoAsErrorCode()));
274 #endif
275 
276 #elif defined(_WIN32)
277     (void)Size;
278 
279     if (!UnmapViewOfFile(Base.toPtr<void *>()))
280       Err = joinErrors(std::move(Err),
281                        errorCodeToError(mapWindowsError(GetLastError())));
282 
283     CloseHandle(SharedMemoryFile);
284 
285 #endif
286 
287     std::lock_guard<std::mutex> Lock(Mutex);
288     Reservations.erase(Base.toPtr<void *>());
289   }
290 
291   return Err;
292 #else
293   return make_error<StringError>(
294       "SharedMemoryMapper is not supported on this platform yet",
295       inconvertibleErrorCode());
296 #endif
297 }
298 
299 Error ExecutorSharedMemoryMapperService::shutdown() {
300   if (Reservations.empty())
301     return Error::success();
302 
303   std::vector<ExecutorAddr> ReservationAddrs;
304   ReservationAddrs.reserve(Reservations.size());
305   for (const auto &R : Reservations)
306     ReservationAddrs.push_back(ExecutorAddr::fromPtr(R.getFirst()));
307 
308   return release(std::move(ReservationAddrs));
309 }
310 
311 void ExecutorSharedMemoryMapperService::addBootstrapSymbols(
312     StringMap<ExecutorAddr> &M) {
313   M[rt::ExecutorSharedMemoryMapperServiceInstanceName] =
314       ExecutorAddr::fromPtr(this);
315   M[rt::ExecutorSharedMemoryMapperServiceReserveWrapperName] =
316       ExecutorAddr::fromPtr(&reserveWrapper);
317   M[rt::ExecutorSharedMemoryMapperServiceInitializeWrapperName] =
318       ExecutorAddr::fromPtr(&initializeWrapper);
319   M[rt::ExecutorSharedMemoryMapperServiceDeinitializeWrapperName] =
320       ExecutorAddr::fromPtr(&deinitializeWrapper);
321   M[rt::ExecutorSharedMemoryMapperServiceReleaseWrapperName] =
322       ExecutorAddr::fromPtr(&releaseWrapper);
323 }
324 
325 llvm::orc::shared::CWrapperFunctionResult
326 ExecutorSharedMemoryMapperService::reserveWrapper(const char *ArgData,
327                                                   size_t ArgSize) {
328   return shared::WrapperFunction<
329              rt::SPSExecutorSharedMemoryMapperServiceReserveSignature>::
330       handle(ArgData, ArgSize,
331              shared::makeMethodWrapperHandler(
332                  &ExecutorSharedMemoryMapperService::reserve))
333           .release();
334 }
335 
336 llvm::orc::shared::CWrapperFunctionResult
337 ExecutorSharedMemoryMapperService::initializeWrapper(const char *ArgData,
338                                                      size_t ArgSize) {
339   return shared::WrapperFunction<
340              rt::SPSExecutorSharedMemoryMapperServiceInitializeSignature>::
341       handle(ArgData, ArgSize,
342              shared::makeMethodWrapperHandler(
343                  &ExecutorSharedMemoryMapperService::initialize))
344           .release();
345 }
346 
347 llvm::orc::shared::CWrapperFunctionResult
348 ExecutorSharedMemoryMapperService::deinitializeWrapper(const char *ArgData,
349                                                        size_t ArgSize) {
350   return shared::WrapperFunction<
351              rt::SPSExecutorSharedMemoryMapperServiceDeinitializeSignature>::
352       handle(ArgData, ArgSize,
353              shared::makeMethodWrapperHandler(
354                  &ExecutorSharedMemoryMapperService::deinitialize))
355           .release();
356 }
357 
358 llvm::orc::shared::CWrapperFunctionResult
359 ExecutorSharedMemoryMapperService::releaseWrapper(const char *ArgData,
360                                                   size_t ArgSize) {
361   return shared::WrapperFunction<
362              rt::SPSExecutorSharedMemoryMapperServiceReleaseSignature>::
363       handle(ArgData, ArgSize,
364              shared::makeMethodWrapperHandler(
365                  &ExecutorSharedMemoryMapperService::release))
366           .release();
367 }
368 
369 } // namespace rt_bootstrap
370 } // end namespace orc
371 } // end namespace llvm
372