1 //===- SectionMemoryManager.cpp - Memory manager for MCJIT/RtDyld *- C++ -*-==// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements the section-based memory manager used by the MCJIT 10 // execution engine and RuntimeDyld 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "llvm/ExecutionEngine/SectionMemoryManager.h" 15 #include "llvm/Config/config.h" 16 #include "llvm/Support/Process.h" 17 18 namespace llvm { 19 20 uint8_t *SectionMemoryManager::allocateDataSection(uintptr_t Size, 21 unsigned Alignment, 22 unsigned SectionID, 23 StringRef SectionName, 24 bool IsReadOnly) { 25 if (IsReadOnly) 26 return allocateSection(SectionMemoryManager::AllocationPurpose::ROData, 27 Size, Alignment); 28 return allocateSection(SectionMemoryManager::AllocationPurpose::RWData, Size, 29 Alignment); 30 } 31 32 uint8_t *SectionMemoryManager::allocateCodeSection(uintptr_t Size, 33 unsigned Alignment, 34 unsigned SectionID, 35 StringRef SectionName) { 36 return allocateSection(SectionMemoryManager::AllocationPurpose::Code, Size, 37 Alignment); 38 } 39 40 uint8_t *SectionMemoryManager::allocateSection( 41 SectionMemoryManager::AllocationPurpose Purpose, uintptr_t Size, 42 unsigned Alignment) { 43 if (!Alignment) 44 Alignment = 16; 45 46 assert(!(Alignment & (Alignment - 1)) && "Alignment must be a power of two."); 47 48 uintptr_t RequiredSize = Alignment * ((Size + Alignment - 1) / Alignment + 1); 49 uintptr_t Addr = 0; 50 51 MemoryGroup &MemGroup = [&]() -> MemoryGroup & { 52 switch (Purpose) { 53 case AllocationPurpose::Code: 54 return CodeMem; 55 case AllocationPurpose::ROData: 56 return RODataMem; 57 case AllocationPurpose::RWData: 58 return RWDataMem; 59 } 60 llvm_unreachable("Unknown SectionMemoryManager::AllocationPurpose"); 61 }(); 62 63 // Look in the list of free memory regions and use a block there if one 64 // is available. 65 for (FreeMemBlock &FreeMB : MemGroup.FreeMem) { 66 if (FreeMB.Free.allocatedSize() >= RequiredSize) { 67 Addr = (uintptr_t)FreeMB.Free.base(); 68 uintptr_t EndOfBlock = Addr + FreeMB.Free.allocatedSize(); 69 // Align the address. 70 Addr = (Addr + Alignment - 1) & ~(uintptr_t)(Alignment - 1); 71 72 if (FreeMB.PendingPrefixIndex == (unsigned)-1) { 73 // The part of the block we're giving out to the user is now pending 74 MemGroup.PendingMem.push_back(sys::MemoryBlock((void *)Addr, Size)); 75 76 // Remember this pending block, such that future allocations can just 77 // modify it rather than creating a new one 78 FreeMB.PendingPrefixIndex = MemGroup.PendingMem.size() - 1; 79 } else { 80 sys::MemoryBlock &PendingMB = 81 MemGroup.PendingMem[FreeMB.PendingPrefixIndex]; 82 PendingMB = sys::MemoryBlock(PendingMB.base(), 83 Addr + Size - (uintptr_t)PendingMB.base()); 84 } 85 86 // Remember how much free space is now left in this block 87 FreeMB.Free = 88 sys::MemoryBlock((void *)(Addr + Size), EndOfBlock - Addr - Size); 89 return (uint8_t *)Addr; 90 } 91 } 92 93 // No pre-allocated free block was large enough. Allocate a new memory region. 94 // Note that all sections get allocated as read-write. The permissions will 95 // be updated later based on memory group. 96 // 97 // FIXME: It would be useful to define a default allocation size (or add 98 // it as a constructor parameter) to minimize the number of allocations. 99 // 100 // FIXME: Initialize the Near member for each memory group to avoid 101 // interleaving. 102 std::error_code ec; 103 sys::MemoryBlock MB = MMapper->allocateMappedMemory( 104 Purpose, RequiredSize, &MemGroup.Near, 105 sys::Memory::MF_READ | sys::Memory::MF_WRITE, ec); 106 if (ec) { 107 // FIXME: Add error propagation to the interface. 108 return nullptr; 109 } 110 111 // Save this address as the basis for our next request 112 MemGroup.Near = MB; 113 114 // Copy the address to all the other groups, if they have not 115 // been initialized. 116 if (CodeMem.Near.base() == nullptr) 117 CodeMem.Near = MB; 118 if (RODataMem.Near.base() == nullptr) 119 RODataMem.Near = MB; 120 if (RWDataMem.Near.base() == nullptr) 121 RWDataMem.Near = MB; 122 123 // Remember that we allocated this memory 124 MemGroup.AllocatedMem.push_back(MB); 125 Addr = (uintptr_t)MB.base(); 126 uintptr_t EndOfBlock = Addr + MB.allocatedSize(); 127 128 // Align the address. 129 Addr = (Addr + Alignment - 1) & ~(uintptr_t)(Alignment - 1); 130 131 // The part of the block we're giving out to the user is now pending 132 MemGroup.PendingMem.push_back(sys::MemoryBlock((void *)Addr, Size)); 133 134 // The allocateMappedMemory may allocate much more memory than we need. In 135 // this case, we store the unused memory as a free memory block. 136 unsigned FreeSize = EndOfBlock - Addr - Size; 137 if (FreeSize > 16) { 138 FreeMemBlock FreeMB; 139 FreeMB.Free = sys::MemoryBlock((void *)(Addr + Size), FreeSize); 140 FreeMB.PendingPrefixIndex = (unsigned)-1; 141 MemGroup.FreeMem.push_back(FreeMB); 142 } 143 144 // Return aligned address 145 return (uint8_t *)Addr; 146 } 147 148 bool SectionMemoryManager::finalizeMemory(std::string *ErrMsg) { 149 // FIXME: Should in-progress permissions be reverted if an error occurs? 150 std::error_code ec; 151 152 // Make code memory executable. 153 ec = applyMemoryGroupPermissions(CodeMem, 154 sys::Memory::MF_READ | sys::Memory::MF_EXEC); 155 if (ec) { 156 if (ErrMsg) { 157 *ErrMsg = ec.message(); 158 } 159 return true; 160 } 161 162 // Make read-only data memory read-only. 163 ec = applyMemoryGroupPermissions(RODataMem, sys::Memory::MF_READ); 164 if (ec) { 165 if (ErrMsg) { 166 *ErrMsg = ec.message(); 167 } 168 return true; 169 } 170 171 // Read-write data memory already has the correct permissions 172 173 // Some platforms with separate data cache and instruction cache require 174 // explicit cache flush, otherwise JIT code manipulations (like resolved 175 // relocations) will get to the data cache but not to the instruction cache. 176 invalidateInstructionCache(); 177 178 return false; 179 } 180 181 static sys::MemoryBlock trimBlockToPageSize(sys::MemoryBlock M) { 182 static const size_t PageSize = sys::Process::getPageSizeEstimate(); 183 184 size_t StartOverlap = 185 (PageSize - ((uintptr_t)M.base() % PageSize)) % PageSize; 186 187 size_t TrimmedSize = M.allocatedSize(); 188 TrimmedSize -= StartOverlap; 189 TrimmedSize -= TrimmedSize % PageSize; 190 191 sys::MemoryBlock Trimmed((void *)((uintptr_t)M.base() + StartOverlap), 192 TrimmedSize); 193 194 assert(((uintptr_t)Trimmed.base() % PageSize) == 0); 195 assert((Trimmed.allocatedSize() % PageSize) == 0); 196 assert(M.base() <= Trimmed.base() && 197 Trimmed.allocatedSize() <= M.allocatedSize()); 198 199 return Trimmed; 200 } 201 202 std::error_code 203 SectionMemoryManager::applyMemoryGroupPermissions(MemoryGroup &MemGroup, 204 unsigned Permissions) { 205 for (sys::MemoryBlock &MB : MemGroup.PendingMem) 206 if (std::error_code EC = MMapper->protectMappedMemory(MB, Permissions)) 207 return EC; 208 209 MemGroup.PendingMem.clear(); 210 211 // Now go through free blocks and trim any of them that don't span the entire 212 // page because one of the pending blocks may have overlapped it. 213 for (FreeMemBlock &FreeMB : MemGroup.FreeMem) { 214 FreeMB.Free = trimBlockToPageSize(FreeMB.Free); 215 // We cleared the PendingMem list, so all these pointers are now invalid 216 FreeMB.PendingPrefixIndex = (unsigned)-1; 217 } 218 219 // Remove all blocks which are now empty 220 erase_if(MemGroup.FreeMem, [](FreeMemBlock &FreeMB) { 221 return FreeMB.Free.allocatedSize() == 0; 222 }); 223 224 return std::error_code(); 225 } 226 227 void SectionMemoryManager::invalidateInstructionCache() { 228 for (sys::MemoryBlock &Block : CodeMem.PendingMem) 229 sys::Memory::InvalidateInstructionCache(Block.base(), 230 Block.allocatedSize()); 231 } 232 233 SectionMemoryManager::~SectionMemoryManager() { 234 for (MemoryGroup *Group : {&CodeMem, &RWDataMem, &RODataMem}) { 235 for (sys::MemoryBlock &Block : Group->AllocatedMem) 236 MMapper->releaseMappedMemory(Block); 237 } 238 } 239 240 SectionMemoryManager::MemoryMapper::~MemoryMapper() = default; 241 242 void SectionMemoryManager::anchor() {} 243 244 namespace { 245 // Trivial implementation of SectionMemoryManager::MemoryMapper that just calls 246 // into sys::Memory. 247 class DefaultMMapper final : public SectionMemoryManager::MemoryMapper { 248 public: 249 sys::MemoryBlock 250 allocateMappedMemory(SectionMemoryManager::AllocationPurpose Purpose, 251 size_t NumBytes, const sys::MemoryBlock *const NearBlock, 252 unsigned Flags, std::error_code &EC) override { 253 return sys::Memory::allocateMappedMemory(NumBytes, NearBlock, Flags, EC); 254 } 255 256 std::error_code protectMappedMemory(const sys::MemoryBlock &Block, 257 unsigned Flags) override { 258 return sys::Memory::protectMappedMemory(Block, Flags); 259 } 260 261 std::error_code releaseMappedMemory(sys::MemoryBlock &M) override { 262 return sys::Memory::releaseMappedMemory(M); 263 } 264 }; 265 } // namespace 266 267 SectionMemoryManager::SectionMemoryManager(MemoryMapper *UnownedMM) 268 : MMapper(UnownedMM), OwnedMMapper(nullptr) { 269 if (!MMapper) { 270 OwnedMMapper = std::make_unique<DefaultMMapper>(); 271 MMapper = OwnedMMapper.get(); 272 } 273 } 274 275 } // namespace llvm 276