1 //===- llvm/unittest/Support/MemoryBufferTest.cpp - MemoryBuffer tests ----===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements unit tests for the MemoryBuffer support class. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "llvm/Support/MemoryBuffer.h" 14 #include "llvm/ADT/ScopeExit.h" 15 #include "llvm/Config/llvm-config.h" // for LLVM_ENABLE_THREADS, LLVM_ON_UNIX 16 #include "llvm/Support/FileSystem.h" 17 #include "llvm/Support/FileUtilities.h" 18 #include "llvm/Support/Process.h" 19 #include "llvm/Support/SmallVectorMemoryBuffer.h" 20 #include "llvm/Support/raw_ostream.h" 21 #include "llvm/Testing/Support/Error.h" 22 #include "gtest/gtest.h" 23 #if LLVM_ENABLE_THREADS 24 #include <thread> 25 #endif 26 #if LLVM_ON_UNIX 27 #include <unistd.h> 28 #endif 29 #if _WIN32 30 #include <windows.h> 31 #endif 32 33 using namespace llvm; 34 35 #define ASSERT_NO_ERROR(x) \ 36 if (std::error_code ASSERT_NO_ERROR_ec = x) { \ 37 SmallString<128> MessageStorage; \ 38 raw_svector_ostream Message(MessageStorage); \ 39 Message << #x ": did not return errc::success.\n" \ 40 << "error number: " << ASSERT_NO_ERROR_ec.value() << "\n" \ 41 << "error message: " << ASSERT_NO_ERROR_ec.message() << "\n"; \ 42 GTEST_FATAL_FAILURE_(MessageStorage.c_str()); \ 43 } else { \ 44 } 45 46 #define ASSERT_ERROR(x) \ 47 if (!x) { \ 48 SmallString<128> MessageStorage; \ 49 raw_svector_ostream Message(MessageStorage); \ 50 Message << #x ": did not return a failure error code.\n"; \ 51 GTEST_FATAL_FAILURE_(MessageStorage.c_str()); \ 52 } 53 54 namespace { 55 56 class MemoryBufferTest : public testing::Test { 57 protected: 58 MemoryBufferTest() 59 : data("this is some data") 60 { } 61 62 void SetUp() override {} 63 64 /// Common testing for different modes of getOpenFileSlice. 65 /// Creates a temporary file with known contents, and uses 66 /// MemoryBuffer::getOpenFileSlice to map it. 67 /// If \p Reopen is true, the file is closed after creating and reopened 68 /// anew before using MemoryBuffer. 69 void testGetOpenFileSlice(bool Reopen); 70 71 typedef std::unique_ptr<MemoryBuffer> OwningBuffer; 72 73 std::string data; 74 }; 75 76 TEST_F(MemoryBufferTest, get) { 77 // Default name and null-terminator flag 78 OwningBuffer MB1(MemoryBuffer::getMemBuffer(data)); 79 EXPECT_NE(nullptr, MB1.get()); 80 81 // RequiresNullTerminator = false 82 OwningBuffer MB2(MemoryBuffer::getMemBuffer(data, "one", false)); 83 EXPECT_NE(nullptr, MB2.get()); 84 85 // RequiresNullTerminator = true 86 OwningBuffer MB3(MemoryBuffer::getMemBuffer(data, "two", true)); 87 EXPECT_NE(nullptr, MB3.get()); 88 89 // verify all 3 buffers point to the same address 90 EXPECT_EQ(MB1->getBufferStart(), MB2->getBufferStart()); 91 EXPECT_EQ(MB2->getBufferStart(), MB3->getBufferStart()); 92 93 // verify the original data is unmodified after deleting the buffers 94 MB1.reset(); 95 MB2.reset(); 96 MB3.reset(); 97 EXPECT_EQ("this is some data", data); 98 } 99 100 TEST_F(MemoryBufferTest, getOpenFile) { 101 int FD; 102 SmallString<64> TestPath; 103 ASSERT_EQ(sys::fs::createTemporaryFile("MemoryBufferTest_getOpenFile", "temp", 104 FD, TestPath), 105 std::error_code()); 106 107 FileRemover Cleanup(TestPath); 108 raw_fd_ostream OF(FD, /*shouldClose*/ true); 109 OF << "12345678"; 110 OF.close(); 111 112 { 113 Expected<sys::fs::file_t> File = sys::fs::openNativeFileForRead(TestPath); 114 ASSERT_THAT_EXPECTED(File, Succeeded()); 115 auto OnExit = 116 make_scope_exit([&] { ASSERT_NO_ERROR(sys::fs::closeFile(*File)); }); 117 ErrorOr<OwningBuffer> MB = MemoryBuffer::getOpenFile(*File, TestPath, 6); 118 ASSERT_NO_ERROR(MB.getError()); 119 EXPECT_EQ("123456", MB.get()->getBuffer()); 120 } 121 { 122 Expected<sys::fs::file_t> File = sys::fs::openNativeFileForWrite( 123 TestPath, sys::fs::CD_OpenExisting, sys::fs::OF_None); 124 ASSERT_THAT_EXPECTED(File, Succeeded()); 125 auto OnExit = 126 make_scope_exit([&] { ASSERT_NO_ERROR(sys::fs::closeFile(*File)); }); 127 ASSERT_ERROR(MemoryBuffer::getOpenFile(*File, TestPath, 6).getError()); 128 } 129 } 130 131 TEST_F(MemoryBufferTest, NullTerminator4K) { 132 // Test that a file with size that is a multiple of the page size can be null 133 // terminated correctly by MemoryBuffer. 134 int TestFD; 135 SmallString<64> TestPath; 136 sys::fs::createTemporaryFile("MemoryBufferTest_NullTerminator4K", "temp", 137 TestFD, TestPath); 138 FileRemover Cleanup(TestPath); 139 raw_fd_ostream OF(TestFD, true, /*unbuffered=*/true); 140 for (unsigned i = 0; i < 4096 / 16; ++i) { 141 OF << "0123456789abcdef"; 142 } 143 OF.close(); 144 145 ErrorOr<OwningBuffer> MB = MemoryBuffer::getFile(TestPath.c_str()); 146 std::error_code EC = MB.getError(); 147 ASSERT_FALSE(EC); 148 149 const char *BufData = MB.get()->getBufferStart(); 150 EXPECT_EQ('f', BufData[4095]); 151 EXPECT_EQ('\0', BufData[4096]); 152 } 153 154 TEST_F(MemoryBufferTest, copy) { 155 // copy with no name 156 OwningBuffer MBC1(MemoryBuffer::getMemBufferCopy(data)); 157 EXPECT_NE(nullptr, MBC1.get()); 158 159 // copy with a name 160 OwningBuffer MBC2(MemoryBuffer::getMemBufferCopy(data, "copy")); 161 EXPECT_NE(nullptr, MBC2.get()); 162 163 // verify the two copies do not point to the same place 164 EXPECT_NE(MBC1->getBufferStart(), MBC2->getBufferStart()); 165 166 // check that copies from defaulted stringrefs don't trigger UB. 167 OwningBuffer MBC3(MemoryBuffer::getMemBufferCopy(StringRef{})); 168 EXPECT_NE(nullptr, MBC3.get()); 169 } 170 171 #if LLVM_ENABLE_THREADS 172 TEST_F(MemoryBufferTest, createFromPipe) { 173 sys::fs::file_t pipes[2]; 174 #if LLVM_ON_UNIX 175 ASSERT_EQ(::pipe(pipes), 0) << strerror(errno); 176 #else 177 ASSERT_TRUE(::CreatePipe(&pipes[0], &pipes[1], nullptr, 0)) 178 << ::GetLastError(); 179 #endif 180 auto ReadCloser = make_scope_exit([&] { sys::fs::closeFile(pipes[0]); }); 181 std::thread Writer([&] { 182 auto WriteCloser = make_scope_exit([&] { sys::fs::closeFile(pipes[1]); }); 183 for (unsigned i = 0; i < 5; ++i) { 184 std::this_thread::sleep_for(std::chrono::milliseconds(10)); 185 #if LLVM_ON_UNIX 186 ASSERT_EQ(::write(pipes[1], "foo", 3), 3) << strerror(errno); 187 #else 188 DWORD Written; 189 ASSERT_TRUE(::WriteFile(pipes[1], "foo", 3, &Written, nullptr)) 190 << ::GetLastError(); 191 ASSERT_EQ(Written, 3u); 192 #endif 193 } 194 }); 195 ErrorOr<OwningBuffer> MB = 196 MemoryBuffer::getOpenFile(pipes[0], "pipe", /*FileSize*/ -1); 197 Writer.join(); 198 ASSERT_NO_ERROR(MB.getError()); 199 EXPECT_EQ(MB.get()->getBuffer(), "foofoofoofoofoo"); 200 } 201 #endif 202 203 TEST_F(MemoryBufferTest, make_new) { 204 // 0-sized buffer 205 OwningBuffer Zero(WritableMemoryBuffer::getNewUninitMemBuffer(0)); 206 EXPECT_NE(nullptr, Zero.get()); 207 208 // uninitialized buffer with no name 209 OwningBuffer One(WritableMemoryBuffer::getNewUninitMemBuffer(321)); 210 EXPECT_NE(nullptr, One.get()); 211 212 // uninitialized buffer with name 213 OwningBuffer Two(WritableMemoryBuffer::getNewUninitMemBuffer(123, "bla")); 214 EXPECT_NE(nullptr, Two.get()); 215 216 // 0-initialized buffer with no name 217 OwningBuffer Three(WritableMemoryBuffer::getNewMemBuffer(321, data)); 218 EXPECT_NE(nullptr, Three.get()); 219 for (size_t i = 0; i < 321; ++i) 220 EXPECT_EQ(0, Three->getBufferStart()[0]); 221 222 // 0-initialized buffer with name 223 OwningBuffer Four(WritableMemoryBuffer::getNewMemBuffer(123, "zeros")); 224 EXPECT_NE(nullptr, Four.get()); 225 for (size_t i = 0; i < 123; ++i) 226 EXPECT_EQ(0, Four->getBufferStart()[0]); 227 228 // uninitialized buffer with rollover size 229 OwningBuffer Five( 230 WritableMemoryBuffer::getNewUninitMemBuffer(SIZE_MAX, "huge")); 231 EXPECT_EQ(nullptr, Five.get()); 232 } 233 234 TEST_F(MemoryBufferTest, getNewAligned) { 235 auto CheckAlignment = [](size_t AlignmentValue) { 236 Align Alignment(AlignmentValue); 237 OwningBuffer AlignedBuffer = 238 WritableMemoryBuffer::getNewUninitMemBuffer(0, "", Alignment); 239 EXPECT_TRUE(isAddrAligned(Alignment, AlignedBuffer->getBufferStart())); 240 }; 241 242 // Test allocation with different alignments. 243 CheckAlignment(16); 244 CheckAlignment(32); 245 CheckAlignment(64); 246 CheckAlignment(128); 247 CheckAlignment(256); 248 } 249 250 void MemoryBufferTest::testGetOpenFileSlice(bool Reopen) { 251 // Test that MemoryBuffer::getOpenFile works properly when no null 252 // terminator is requested and the size is large enough to trigger 253 // the usage of memory mapping. 254 int TestFD; 255 SmallString<64> TestPath; 256 // Create a temporary file and write data into it. 257 sys::fs::createTemporaryFile("prefix", "temp", TestFD, TestPath); 258 FileRemover Cleanup(TestPath); 259 // OF is responsible for closing the file; If the file is not 260 // reopened, it will be unbuffered so that the results are 261 // immediately visible through the fd. 262 raw_fd_ostream OF(TestFD, true, !Reopen); 263 for (int i = 0; i < 60000; ++i) { 264 OF << "0123456789"; 265 } 266 267 if (Reopen) { 268 OF.close(); 269 EXPECT_FALSE(sys::fs::openFileForRead(TestPath.c_str(), TestFD)); 270 } 271 272 ErrorOr<OwningBuffer> Buf = MemoryBuffer::getOpenFileSlice( 273 sys::fs::convertFDToNativeFile(TestFD), TestPath.c_str(), 274 40000, // Size 275 80000 // Offset 276 ); 277 278 std::error_code EC = Buf.getError(); 279 EXPECT_FALSE(EC); 280 281 StringRef BufData = Buf.get()->getBuffer(); 282 EXPECT_EQ(BufData.size(), 40000U); 283 EXPECT_EQ(BufData[0], '0'); 284 EXPECT_EQ(BufData[9], '9'); 285 } 286 287 TEST_F(MemoryBufferTest, getOpenFileNoReopen) { 288 testGetOpenFileSlice(false); 289 } 290 291 TEST_F(MemoryBufferTest, getOpenFileReopened) { 292 testGetOpenFileSlice(true); 293 } 294 295 TEST_F(MemoryBufferTest, slice) { 296 // Create a file that is six pages long with different data on each page. 297 int FD; 298 SmallString<64> TestPath; 299 sys::fs::createTemporaryFile("MemoryBufferTest_Slice", "temp", FD, TestPath); 300 FileRemover Cleanup(TestPath); 301 raw_fd_ostream OF(FD, true, /*unbuffered=*/true); 302 for (unsigned i = 0; i < 0x2000 / 8; ++i) { 303 OF << "12345678"; 304 } 305 for (unsigned i = 0; i < 0x2000 / 8; ++i) { 306 OF << "abcdefgh"; 307 } 308 for (unsigned i = 0; i < 0x2000 / 8; ++i) { 309 OF << "ABCDEFGH"; 310 } 311 OF.close(); 312 313 // Try offset of one page. 314 ErrorOr<OwningBuffer> MB = MemoryBuffer::getFileSlice(TestPath.str(), 315 0x4000, 0x1000); 316 std::error_code EC = MB.getError(); 317 ASSERT_FALSE(EC); 318 EXPECT_EQ(0x4000UL, MB.get()->getBufferSize()); 319 320 StringRef BufData = MB.get()->getBuffer(); 321 EXPECT_TRUE(BufData.substr(0x0000, 8) == "12345678"); 322 EXPECT_TRUE(BufData.substr(0x0FF8, 8) == "12345678"); 323 EXPECT_TRUE(BufData.substr(0x1000, 8) == "abcdefgh"); 324 EXPECT_TRUE(BufData.substr(0x2FF8, 8) == "abcdefgh"); 325 EXPECT_TRUE(BufData.substr(0x3000, 8) == "ABCDEFGH"); 326 EXPECT_TRUE(BufData.substr(0x3FF8, 8) == "ABCDEFGH"); 327 328 // Try non-page aligned. 329 ErrorOr<OwningBuffer> MB2 = MemoryBuffer::getFileSlice(TestPath.str(), 330 0x3000, 0x0800); 331 EC = MB2.getError(); 332 ASSERT_FALSE(EC); 333 EXPECT_EQ(0x3000UL, MB2.get()->getBufferSize()); 334 335 StringRef BufData2 = MB2.get()->getBuffer(); 336 EXPECT_TRUE(BufData2.substr(0x0000, 8) == "12345678"); 337 EXPECT_TRUE(BufData2.substr(0x17F8, 8) == "12345678"); 338 EXPECT_TRUE(BufData2.substr(0x1800, 8) == "abcdefgh"); 339 EXPECT_TRUE(BufData2.substr(0x2FF8, 8) == "abcdefgh"); 340 } 341 342 TEST_F(MemoryBufferTest, writableSlice) { 343 // Create a file initialized with some data 344 int FD; 345 SmallString<64> TestPath; 346 sys::fs::createTemporaryFile("MemoryBufferTest_WritableSlice", "temp", FD, 347 TestPath); 348 FileRemover Cleanup(TestPath); 349 raw_fd_ostream OF(FD, true); 350 for (unsigned i = 0; i < 0x1000; ++i) 351 OF << "0123456789abcdef"; 352 OF.close(); 353 354 { 355 auto MBOrError = 356 WritableMemoryBuffer::getFileSlice(TestPath.str(), 0x6000, 0x2000); 357 ASSERT_FALSE(MBOrError.getError()); 358 // Write some data. It should be mapped private, so that upon completion 359 // the original file contents are not modified. 360 WritableMemoryBuffer &MB = **MBOrError; 361 ASSERT_EQ(0x6000u, MB.getBufferSize()); 362 char *Start = MB.getBufferStart(); 363 ASSERT_EQ(MB.getBufferEnd(), MB.getBufferStart() + MB.getBufferSize()); 364 ::memset(Start, 'x', MB.getBufferSize()); 365 } 366 367 auto MBOrError = MemoryBuffer::getFile(TestPath); 368 ASSERT_FALSE(MBOrError.getError()); 369 auto &MB = **MBOrError; 370 ASSERT_EQ(0x10000u, MB.getBufferSize()); 371 for (size_t i = 0; i < MB.getBufferSize(); i += 0x10) 372 EXPECT_EQ("0123456789abcdef", MB.getBuffer().substr(i, 0x10)) << "i: " << i; 373 } 374 375 TEST_F(MemoryBufferTest, writeThroughFile) { 376 // Create a file initialized with some data 377 int FD; 378 SmallString<64> TestPath; 379 sys::fs::createTemporaryFile("MemoryBufferTest_WriteThrough", "temp", FD, 380 TestPath); 381 FileRemover Cleanup(TestPath); 382 raw_fd_ostream OF(FD, true); 383 OF << "0123456789abcdef"; 384 OF.close(); 385 { 386 auto MBOrError = WriteThroughMemoryBuffer::getFile(TestPath); 387 ASSERT_FALSE(MBOrError.getError()); 388 // Write some data. It should be mapped readwrite, so that upon completion 389 // the original file contents are modified. 390 WriteThroughMemoryBuffer &MB = **MBOrError; 391 ASSERT_EQ(16u, MB.getBufferSize()); 392 char *Start = MB.getBufferStart(); 393 ASSERT_EQ(MB.getBufferEnd(), MB.getBufferStart() + MB.getBufferSize()); 394 ::memset(Start, 'x', MB.getBufferSize()); 395 } 396 397 auto MBOrError = MemoryBuffer::getFile(TestPath); 398 ASSERT_FALSE(MBOrError.getError()); 399 auto &MB = **MBOrError; 400 ASSERT_EQ(16u, MB.getBufferSize()); 401 EXPECT_EQ("xxxxxxxxxxxxxxxx", MB.getBuffer()); 402 } 403 404 TEST_F(MemoryBufferTest, mmapVolatileNoNull) { 405 // Verify that `MemoryBuffer::getOpenFile` will use mmap when 406 // `RequiresNullTerminator = false`, `IsVolatile = true`, and the file is 407 // large enough to use mmap. 408 // 409 // This is done because Clang should use this mode to open module files, and 410 // falling back to malloc for them causes a huge memory usage increase. 411 412 int FD; 413 SmallString<64> TestPath; 414 ASSERT_NO_ERROR(sys::fs::createTemporaryFile( 415 "MemoryBufferTest_mmapVolatileNoNull", "temp", FD, TestPath)); 416 FileRemover Cleanup(TestPath); 417 raw_fd_ostream OF(FD, true); 418 // Create a file large enough to mmap. 4 pages should be enough. 419 unsigned PageSize = sys::Process::getPageSizeEstimate(); 420 unsigned FileWrites = (PageSize * 4) / 8; 421 for (unsigned i = 0; i < FileWrites; ++i) 422 OF << "01234567"; 423 OF.close(); 424 425 Expected<sys::fs::file_t> File = sys::fs::openNativeFileForRead(TestPath); 426 ASSERT_THAT_EXPECTED(File, Succeeded()); 427 auto OnExit = 428 make_scope_exit([&] { ASSERT_NO_ERROR(sys::fs::closeFile(*File)); }); 429 430 auto MBOrError = MemoryBuffer::getOpenFile(*File, TestPath, 431 /*FileSize=*/-1, /*RequiresNullTerminator=*/false, /*IsVolatile=*/true); 432 ASSERT_NO_ERROR(MBOrError.getError()) 433 OwningBuffer MB = std::move(*MBOrError); 434 EXPECT_EQ(MB->getBufferKind(), MemoryBuffer::MemoryBuffer_MMap); 435 EXPECT_EQ(MB->getBufferSize(), std::size_t(FileWrites * 8)); 436 EXPECT_TRUE(MB->getBuffer().starts_with("01234567")); 437 } 438 439 // Test that SmallVector without a null terminator gets one. 440 TEST(SmallVectorMemoryBufferTest, WithoutNullTerminatorRequiresNullTerminator) { 441 SmallString<0> Data("some data"); 442 443 SmallVectorMemoryBuffer MB(std::move(Data), 444 /*RequiresNullTerminator=*/true); 445 EXPECT_EQ(MB.getBufferSize(), 9u); 446 EXPECT_EQ(MB.getBufferEnd()[0], '\0'); 447 } 448 449 // Test that SmallVector with a null terminator keeps it. 450 TEST(SmallVectorMemoryBufferTest, WithNullTerminatorRequiresNullTerminator) { 451 SmallString<0> Data("some data"); 452 Data.push_back('\0'); 453 Data.pop_back(); 454 455 SmallVectorMemoryBuffer MB(std::move(Data), 456 /*RequiresNullTerminator=*/true); 457 EXPECT_EQ(MB.getBufferSize(), 9u); 458 EXPECT_EQ(MB.getBufferEnd()[0], '\0'); 459 } 460 461 } // namespace 462