1 //===---------- ExecutorSharedMemoryMapperService.cpp -----------*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "llvm/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.h" 10 #include "llvm/Config/llvm-config.h" // for LLVM_ON_UNIX 11 #include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h" 12 #include "llvm/Support/Process.h" 13 #include "llvm/Support/WindowsError.h" 14 #include <sstream> 15 16 #if defined(LLVM_ON_UNIX) 17 #include <errno.h> 18 #include <fcntl.h> 19 #include <sys/mman.h> 20 #if defined(__MVS__) 21 #include "llvm/Support/BLAKE3.h" 22 #include <sys/shm.h> 23 #endif 24 #include <unistd.h> 25 #endif 26 27 namespace llvm { 28 namespace orc { 29 namespace rt_bootstrap { 30 31 #if defined(_WIN32) 32 static DWORD getWindowsProtectionFlags(MemProt MP) { 33 if (MP == MemProt::Read) 34 return PAGE_READONLY; 35 if (MP == MemProt::Write || 36 MP == (MemProt::Write | MemProt::Read)) { 37 // Note: PAGE_WRITE is not supported by VirtualProtect 38 return PAGE_READWRITE; 39 } 40 if (MP == (MemProt::Read | MemProt::Exec)) 41 return PAGE_EXECUTE_READ; 42 if (MP == (MemProt::Read | MemProt::Write | MemProt::Exec)) 43 return PAGE_EXECUTE_READWRITE; 44 if (MP == MemProt::Exec) 45 return PAGE_EXECUTE; 46 47 return PAGE_NOACCESS; 48 } 49 #endif 50 51 Expected<std::pair<ExecutorAddr, std::string>> 52 ExecutorSharedMemoryMapperService::reserve(uint64_t Size) { 53 #if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32) 54 55 #if defined(LLVM_ON_UNIX) 56 57 std::string SharedMemoryName; 58 { 59 std::stringstream SharedMemoryNameStream; 60 SharedMemoryNameStream << "/jitlink_" << sys::Process::getProcessId() << '_' 61 << (++SharedMemoryCount); 62 SharedMemoryName = SharedMemoryNameStream.str(); 63 } 64 65 #if defined(__MVS__) 66 ArrayRef<uint8_t> Data( 67 reinterpret_cast<const uint8_t *>(SharedMemoryName.c_str()), 68 SharedMemoryName.size()); 69 auto HashedName = BLAKE3::hash<sizeof(key_t)>(Data); 70 key_t Key = *reinterpret_cast<key_t *>(HashedName.data()); 71 int SharedMemoryId = 72 shmget(Key, Size, IPC_CREAT | IPC_EXCL | __IPC_SHAREAS | 0700); 73 if (SharedMemoryId < 0) 74 return errorCodeToError(errnoAsErrorCode()); 75 76 void *Addr = shmat(SharedMemoryId, nullptr, 0); 77 if (Addr == reinterpret_cast<void *>(-1)) 78 return errorCodeToError(errnoAsErrorCode()); 79 #else 80 int SharedMemoryFile = 81 shm_open(SharedMemoryName.c_str(), O_RDWR | O_CREAT | O_EXCL, 0700); 82 if (SharedMemoryFile < 0) 83 return errorCodeToError(errnoAsErrorCode()); 84 85 // by default size is 0 86 if (ftruncate(SharedMemoryFile, Size) < 0) 87 return errorCodeToError(errnoAsErrorCode()); 88 89 void *Addr = mmap(nullptr, Size, PROT_NONE, MAP_SHARED, SharedMemoryFile, 0); 90 if (Addr == MAP_FAILED) 91 return errorCodeToError(errnoAsErrorCode()); 92 93 close(SharedMemoryFile); 94 #endif 95 96 #elif defined(_WIN32) 97 98 std::string SharedMemoryName; 99 { 100 std::stringstream SharedMemoryNameStream; 101 SharedMemoryNameStream << "jitlink_" << sys::Process::getProcessId() << '_' 102 << (++SharedMemoryCount); 103 SharedMemoryName = SharedMemoryNameStream.str(); 104 } 105 106 std::wstring WideSharedMemoryName(SharedMemoryName.begin(), 107 SharedMemoryName.end()); 108 HANDLE SharedMemoryFile = CreateFileMappingW( 109 INVALID_HANDLE_VALUE, NULL, PAGE_EXECUTE_READWRITE, Size >> 32, 110 Size & 0xffffffff, WideSharedMemoryName.c_str()); 111 if (!SharedMemoryFile) 112 return errorCodeToError(mapWindowsError(GetLastError())); 113 114 void *Addr = MapViewOfFile(SharedMemoryFile, 115 FILE_MAP_ALL_ACCESS | FILE_MAP_EXECUTE, 0, 0, 0); 116 if (!Addr) { 117 CloseHandle(SharedMemoryFile); 118 return errorCodeToError(mapWindowsError(GetLastError())); 119 } 120 121 #endif 122 123 { 124 std::lock_guard<std::mutex> Lock(Mutex); 125 Reservations[Addr].Size = Size; 126 #if defined(_WIN32) 127 Reservations[Addr].SharedMemoryFile = SharedMemoryFile; 128 #endif 129 } 130 131 return std::make_pair(ExecutorAddr::fromPtr(Addr), 132 std::move(SharedMemoryName)); 133 #else 134 return make_error<StringError>( 135 "SharedMemoryMapper is not supported on this platform yet", 136 inconvertibleErrorCode()); 137 #endif 138 } 139 140 Expected<ExecutorAddr> ExecutorSharedMemoryMapperService::initialize( 141 ExecutorAddr Reservation, tpctypes::SharedMemoryFinalizeRequest &FR) { 142 #if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32) 143 144 ExecutorAddr MinAddr(~0ULL); 145 146 // Contents are already in place 147 for (auto &Segment : FR.Segments) { 148 if (Segment.Addr < MinAddr) 149 MinAddr = Segment.Addr; 150 151 #if defined(LLVM_ON_UNIX) 152 153 #if defined(__MVS__) 154 // TODO Is it possible to change the protection level? 155 #else 156 int NativeProt = 0; 157 if ((Segment.RAG.Prot & MemProt::Read) == MemProt::Read) 158 NativeProt |= PROT_READ; 159 if ((Segment.RAG.Prot & MemProt::Write) == MemProt::Write) 160 NativeProt |= PROT_WRITE; 161 if ((Segment.RAG.Prot & MemProt::Exec) == MemProt::Exec) 162 NativeProt |= PROT_EXEC; 163 164 if (mprotect(Segment.Addr.toPtr<void *>(), Segment.Size, NativeProt)) 165 return errorCodeToError(errnoAsErrorCode()); 166 #endif 167 168 #elif defined(_WIN32) 169 170 DWORD NativeProt = getWindowsProtectionFlags(Segment.RAG.Prot); 171 172 if (!VirtualProtect(Segment.Addr.toPtr<void *>(), Segment.Size, NativeProt, 173 &NativeProt)) 174 return errorCodeToError(mapWindowsError(GetLastError())); 175 176 #endif 177 178 if ((Segment.RAG.Prot & MemProt::Exec) == MemProt::Exec) 179 sys::Memory::InvalidateInstructionCache(Segment.Addr.toPtr<void *>(), 180 Segment.Size); 181 } 182 183 // Run finalization actions and get deinitlization action list. 184 auto DeinitializeActions = shared::runFinalizeActions(FR.Actions); 185 if (!DeinitializeActions) { 186 return DeinitializeActions.takeError(); 187 } 188 189 { 190 std::lock_guard<std::mutex> Lock(Mutex); 191 Allocations[MinAddr].DeinitializationActions = 192 std::move(*DeinitializeActions); 193 Reservations[Reservation.toPtr<void *>()].Allocations.push_back(MinAddr); 194 } 195 196 return MinAddr; 197 198 #else 199 return make_error<StringError>( 200 "SharedMemoryMapper is not supported on this platform yet", 201 inconvertibleErrorCode()); 202 #endif 203 } 204 205 Error ExecutorSharedMemoryMapperService::deinitialize( 206 const std::vector<ExecutorAddr> &Bases) { 207 Error AllErr = Error::success(); 208 209 { 210 std::lock_guard<std::mutex> Lock(Mutex); 211 212 for (auto Base : llvm::reverse(Bases)) { 213 if (Error Err = shared::runDeallocActions( 214 Allocations[Base].DeinitializationActions)) { 215 AllErr = joinErrors(std::move(AllErr), std::move(Err)); 216 } 217 218 // Remove the allocation from the allocation list of its reservation 219 for (auto &Reservation : Reservations) { 220 auto AllocationIt = llvm::find(Reservation.second.Allocations, Base); 221 if (AllocationIt != Reservation.second.Allocations.end()) { 222 Reservation.second.Allocations.erase(AllocationIt); 223 break; 224 } 225 } 226 227 Allocations.erase(Base); 228 } 229 } 230 231 return AllErr; 232 } 233 234 Error ExecutorSharedMemoryMapperService::release( 235 const std::vector<ExecutorAddr> &Bases) { 236 #if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32) 237 Error Err = Error::success(); 238 239 for (auto Base : Bases) { 240 std::vector<ExecutorAddr> AllocAddrs; 241 size_t Size; 242 243 #if defined(_WIN32) 244 HANDLE SharedMemoryFile; 245 #endif 246 247 { 248 std::lock_guard<std::mutex> Lock(Mutex); 249 auto &R = Reservations[Base.toPtr<void *>()]; 250 Size = R.Size; 251 252 #if defined(_WIN32) 253 SharedMemoryFile = R.SharedMemoryFile; 254 #endif 255 256 AllocAddrs.swap(R.Allocations); 257 } 258 259 // deinitialize sub allocations 260 if (Error E = deinitialize(AllocAddrs)) 261 Err = joinErrors(std::move(Err), std::move(E)); 262 263 #if defined(LLVM_ON_UNIX) 264 265 #if defined(__MVS__) 266 (void)Size; 267 268 if (shmdt(Base.toPtr<void *>()) < 0) 269 Err = joinErrors(std::move(Err), errorCodeToError(errnoAsErrorCode())); 270 #else 271 if (munmap(Base.toPtr<void *>(), Size) != 0) 272 Err = joinErrors(std::move(Err), errorCodeToError(errnoAsErrorCode())); 273 #endif 274 275 #elif defined(_WIN32) 276 (void)Size; 277 278 if (!UnmapViewOfFile(Base.toPtr<void *>())) 279 Err = joinErrors(std::move(Err), 280 errorCodeToError(mapWindowsError(GetLastError()))); 281 282 CloseHandle(SharedMemoryFile); 283 284 #endif 285 286 std::lock_guard<std::mutex> Lock(Mutex); 287 Reservations.erase(Base.toPtr<void *>()); 288 } 289 290 return Err; 291 #else 292 return make_error<StringError>( 293 "SharedMemoryMapper is not supported on this platform yet", 294 inconvertibleErrorCode()); 295 #endif 296 } 297 298 Error ExecutorSharedMemoryMapperService::shutdown() { 299 if (Reservations.empty()) 300 return Error::success(); 301 302 std::vector<ExecutorAddr> ReservationAddrs; 303 ReservationAddrs.reserve(Reservations.size()); 304 for (const auto &R : Reservations) 305 ReservationAddrs.push_back(ExecutorAddr::fromPtr(R.getFirst())); 306 307 return release(std::move(ReservationAddrs)); 308 } 309 310 void ExecutorSharedMemoryMapperService::addBootstrapSymbols( 311 StringMap<ExecutorAddr> &M) { 312 M[rt::ExecutorSharedMemoryMapperServiceInstanceName] = 313 ExecutorAddr::fromPtr(this); 314 M[rt::ExecutorSharedMemoryMapperServiceReserveWrapperName] = 315 ExecutorAddr::fromPtr(&reserveWrapper); 316 M[rt::ExecutorSharedMemoryMapperServiceInitializeWrapperName] = 317 ExecutorAddr::fromPtr(&initializeWrapper); 318 M[rt::ExecutorSharedMemoryMapperServiceDeinitializeWrapperName] = 319 ExecutorAddr::fromPtr(&deinitializeWrapper); 320 M[rt::ExecutorSharedMemoryMapperServiceReleaseWrapperName] = 321 ExecutorAddr::fromPtr(&releaseWrapper); 322 } 323 324 llvm::orc::shared::CWrapperFunctionResult 325 ExecutorSharedMemoryMapperService::reserveWrapper(const char *ArgData, 326 size_t ArgSize) { 327 return shared::WrapperFunction< 328 rt::SPSExecutorSharedMemoryMapperServiceReserveSignature>:: 329 handle(ArgData, ArgSize, 330 shared::makeMethodWrapperHandler( 331 &ExecutorSharedMemoryMapperService::reserve)) 332 .release(); 333 } 334 335 llvm::orc::shared::CWrapperFunctionResult 336 ExecutorSharedMemoryMapperService::initializeWrapper(const char *ArgData, 337 size_t ArgSize) { 338 return shared::WrapperFunction< 339 rt::SPSExecutorSharedMemoryMapperServiceInitializeSignature>:: 340 handle(ArgData, ArgSize, 341 shared::makeMethodWrapperHandler( 342 &ExecutorSharedMemoryMapperService::initialize)) 343 .release(); 344 } 345 346 llvm::orc::shared::CWrapperFunctionResult 347 ExecutorSharedMemoryMapperService::deinitializeWrapper(const char *ArgData, 348 size_t ArgSize) { 349 return shared::WrapperFunction< 350 rt::SPSExecutorSharedMemoryMapperServiceDeinitializeSignature>:: 351 handle(ArgData, ArgSize, 352 shared::makeMethodWrapperHandler( 353 &ExecutorSharedMemoryMapperService::deinitialize)) 354 .release(); 355 } 356 357 llvm::orc::shared::CWrapperFunctionResult 358 ExecutorSharedMemoryMapperService::releaseWrapper(const char *ArgData, 359 size_t ArgSize) { 360 return shared::WrapperFunction< 361 rt::SPSExecutorSharedMemoryMapperServiceReleaseSignature>:: 362 handle(ArgData, ArgSize, 363 shared::makeMethodWrapperHandler( 364 &ExecutorSharedMemoryMapperService::release)) 365 .release(); 366 } 367 368 } // namespace rt_bootstrap 369 } // end namespace orc 370 } // end namespace llvm 371