1 //===- llvm/unittest/Support/MemoryBufferTest.cpp - MemoryBuffer tests ----===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements unit tests for the MemoryBuffer support class. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "llvm/Support/MemoryBuffer.h" 14 #include "llvm/Support/SmallVectorMemoryBuffer.h" 15 #include "llvm/ADT/ScopeExit.h" 16 #include "llvm/Support/FileSystem.h" 17 #include "llvm/Support/FileUtilities.h" 18 #include "llvm/Support/Process.h" 19 #include "llvm/Support/raw_ostream.h" 20 #include "llvm/Testing/Support/Error.h" 21 #include "gtest/gtest.h" 22 #if LLVM_ENABLE_THREADS 23 #include <thread> 24 #endif 25 #if LLVM_ON_UNIX 26 #include <unistd.h> 27 #endif 28 #if _WIN32 29 #include <windows.h> 30 #endif 31 32 using namespace llvm; 33 34 #define ASSERT_NO_ERROR(x) \ 35 if (std::error_code ASSERT_NO_ERROR_ec = x) { \ 36 SmallString<128> MessageStorage; \ 37 raw_svector_ostream Message(MessageStorage); \ 38 Message << #x ": did not return errc::success.\n" \ 39 << "error number: " << ASSERT_NO_ERROR_ec.value() << "\n" \ 40 << "error message: " << ASSERT_NO_ERROR_ec.message() << "\n"; \ 41 GTEST_FATAL_FAILURE_(MessageStorage.c_str()); \ 42 } else { \ 43 } 44 45 #define ASSERT_ERROR(x) \ 46 if (!x) { \ 47 SmallString<128> MessageStorage; \ 48 raw_svector_ostream Message(MessageStorage); \ 49 Message << #x ": did not return a failure error code.\n"; \ 50 GTEST_FATAL_FAILURE_(MessageStorage.c_str()); \ 51 } 52 53 namespace { 54 55 class MemoryBufferTest : public testing::Test { 56 protected: 57 MemoryBufferTest() 58 : data("this is some data") 59 { } 60 61 void SetUp() override {} 62 63 /// Common testing for different modes of getOpenFileSlice. 64 /// Creates a temporary file with known contents, and uses 65 /// MemoryBuffer::getOpenFileSlice to map it. 66 /// If \p Reopen is true, the file is closed after creating and reopened 67 /// anew before using MemoryBuffer. 68 void testGetOpenFileSlice(bool Reopen); 69 70 typedef std::unique_ptr<MemoryBuffer> OwningBuffer; 71 72 std::string data; 73 }; 74 75 TEST_F(MemoryBufferTest, get) { 76 // Default name and null-terminator flag 77 OwningBuffer MB1(MemoryBuffer::getMemBuffer(data)); 78 EXPECT_NE(nullptr, MB1.get()); 79 80 // RequiresNullTerminator = false 81 OwningBuffer MB2(MemoryBuffer::getMemBuffer(data, "one", false)); 82 EXPECT_NE(nullptr, MB2.get()); 83 84 // RequiresNullTerminator = true 85 OwningBuffer MB3(MemoryBuffer::getMemBuffer(data, "two", true)); 86 EXPECT_NE(nullptr, MB3.get()); 87 88 // verify all 3 buffers point to the same address 89 EXPECT_EQ(MB1->getBufferStart(), MB2->getBufferStart()); 90 EXPECT_EQ(MB2->getBufferStart(), MB3->getBufferStart()); 91 92 // verify the original data is unmodified after deleting the buffers 93 MB1.reset(); 94 MB2.reset(); 95 MB3.reset(); 96 EXPECT_EQ("this is some data", data); 97 } 98 99 TEST_F(MemoryBufferTest, getOpenFile) { 100 int FD; 101 SmallString<64> TestPath; 102 ASSERT_EQ(sys::fs::createTemporaryFile("MemoryBufferTest_getOpenFile", "temp", 103 FD, TestPath), 104 std::error_code()); 105 106 FileRemover Cleanup(TestPath); 107 raw_fd_ostream OF(FD, /*shouldClose*/ true); 108 OF << "12345678"; 109 OF.close(); 110 111 { 112 Expected<sys::fs::file_t> File = sys::fs::openNativeFileForRead(TestPath); 113 ASSERT_THAT_EXPECTED(File, Succeeded()); 114 auto OnExit = 115 make_scope_exit([&] { ASSERT_NO_ERROR(sys::fs::closeFile(*File)); }); 116 ErrorOr<OwningBuffer> MB = MemoryBuffer::getOpenFile(*File, TestPath, 6); 117 ASSERT_NO_ERROR(MB.getError()); 118 EXPECT_EQ("123456", MB.get()->getBuffer()); 119 } 120 { 121 Expected<sys::fs::file_t> File = sys::fs::openNativeFileForWrite( 122 TestPath, sys::fs::CD_OpenExisting, sys::fs::OF_None); 123 ASSERT_THAT_EXPECTED(File, Succeeded()); 124 auto OnExit = 125 make_scope_exit([&] { ASSERT_NO_ERROR(sys::fs::closeFile(*File)); }); 126 ASSERT_ERROR(MemoryBuffer::getOpenFile(*File, TestPath, 6).getError()); 127 } 128 } 129 130 TEST_F(MemoryBufferTest, NullTerminator4K) { 131 // Test that a file with size that is a multiple of the page size can be null 132 // terminated correctly by MemoryBuffer. 133 int TestFD; 134 SmallString<64> TestPath; 135 sys::fs::createTemporaryFile("MemoryBufferTest_NullTerminator4K", "temp", 136 TestFD, TestPath); 137 FileRemover Cleanup(TestPath); 138 raw_fd_ostream OF(TestFD, true, /*unbuffered=*/true); 139 for (unsigned i = 0; i < 4096 / 16; ++i) { 140 OF << "0123456789abcdef"; 141 } 142 OF.close(); 143 144 ErrorOr<OwningBuffer> MB = MemoryBuffer::getFile(TestPath.c_str()); 145 std::error_code EC = MB.getError(); 146 ASSERT_FALSE(EC); 147 148 const char *BufData = MB.get()->getBufferStart(); 149 EXPECT_EQ('f', BufData[4095]); 150 EXPECT_EQ('\0', BufData[4096]); 151 } 152 153 TEST_F(MemoryBufferTest, copy) { 154 // copy with no name 155 OwningBuffer MBC1(MemoryBuffer::getMemBufferCopy(data)); 156 EXPECT_NE(nullptr, MBC1.get()); 157 158 // copy with a name 159 OwningBuffer MBC2(MemoryBuffer::getMemBufferCopy(data, "copy")); 160 EXPECT_NE(nullptr, MBC2.get()); 161 162 // verify the two copies do not point to the same place 163 EXPECT_NE(MBC1->getBufferStart(), MBC2->getBufferStart()); 164 165 // check that copies from defaulted stringrefs don't trigger UB. 166 OwningBuffer MBC3(MemoryBuffer::getMemBufferCopy(StringRef{})); 167 EXPECT_NE(nullptr, MBC3.get()); 168 } 169 170 #if LLVM_ENABLE_THREADS 171 TEST_F(MemoryBufferTest, createFromPipe) { 172 sys::fs::file_t pipes[2]; 173 #if LLVM_ON_UNIX 174 ASSERT_EQ(::pipe(pipes), 0) << strerror(errno); 175 #else 176 ASSERT_TRUE(::CreatePipe(&pipes[0], &pipes[1], nullptr, 0)) 177 << ::GetLastError(); 178 #endif 179 auto ReadCloser = make_scope_exit([&] { sys::fs::closeFile(pipes[0]); }); 180 std::thread Writer([&] { 181 auto WriteCloser = make_scope_exit([&] { sys::fs::closeFile(pipes[1]); }); 182 for (unsigned i = 0; i < 5; ++i) { 183 std::this_thread::sleep_for(std::chrono::milliseconds(10)); 184 #if LLVM_ON_UNIX 185 ASSERT_EQ(::write(pipes[1], "foo", 3), 3) << strerror(errno); 186 #else 187 DWORD Written; 188 ASSERT_TRUE(::WriteFile(pipes[1], "foo", 3, &Written, nullptr)) 189 << ::GetLastError(); 190 ASSERT_EQ(Written, 3u); 191 #endif 192 } 193 }); 194 ErrorOr<OwningBuffer> MB = 195 MemoryBuffer::getOpenFile(pipes[0], "pipe", /*FileSize*/ -1); 196 Writer.join(); 197 ASSERT_NO_ERROR(MB.getError()); 198 EXPECT_EQ(MB.get()->getBuffer(), "foofoofoofoofoo"); 199 } 200 #endif 201 202 TEST_F(MemoryBufferTest, make_new) { 203 // 0-sized buffer 204 OwningBuffer Zero(WritableMemoryBuffer::getNewUninitMemBuffer(0)); 205 EXPECT_NE(nullptr, Zero.get()); 206 207 // uninitialized buffer with no name 208 OwningBuffer One(WritableMemoryBuffer::getNewUninitMemBuffer(321)); 209 EXPECT_NE(nullptr, One.get()); 210 211 // uninitialized buffer with name 212 OwningBuffer Two(WritableMemoryBuffer::getNewUninitMemBuffer(123, "bla")); 213 EXPECT_NE(nullptr, Two.get()); 214 215 // 0-initialized buffer with no name 216 OwningBuffer Three(WritableMemoryBuffer::getNewMemBuffer(321, data)); 217 EXPECT_NE(nullptr, Three.get()); 218 for (size_t i = 0; i < 321; ++i) 219 EXPECT_EQ(0, Three->getBufferStart()[0]); 220 221 // 0-initialized buffer with name 222 OwningBuffer Four(WritableMemoryBuffer::getNewMemBuffer(123, "zeros")); 223 EXPECT_NE(nullptr, Four.get()); 224 for (size_t i = 0; i < 123; ++i) 225 EXPECT_EQ(0, Four->getBufferStart()[0]); 226 227 // uninitialized buffer with rollover size 228 OwningBuffer Five( 229 WritableMemoryBuffer::getNewUninitMemBuffer(SIZE_MAX, "huge")); 230 EXPECT_EQ(nullptr, Five.get()); 231 } 232 233 TEST_F(MemoryBufferTest, getNewAligned) { 234 auto CheckAlignment = [](size_t AlignmentValue) { 235 Align Alignment(AlignmentValue); 236 OwningBuffer AlignedBuffer = 237 WritableMemoryBuffer::getNewUninitMemBuffer(0, "", Alignment); 238 EXPECT_TRUE(isAddrAligned(Alignment, AlignedBuffer->getBufferStart())); 239 }; 240 241 // Test allocation with different alignments. 242 CheckAlignment(16); 243 CheckAlignment(32); 244 CheckAlignment(64); 245 CheckAlignment(128); 246 CheckAlignment(256); 247 } 248 249 void MemoryBufferTest::testGetOpenFileSlice(bool Reopen) { 250 // Test that MemoryBuffer::getOpenFile works properly when no null 251 // terminator is requested and the size is large enough to trigger 252 // the usage of memory mapping. 253 int TestFD; 254 SmallString<64> TestPath; 255 // Create a temporary file and write data into it. 256 sys::fs::createTemporaryFile("prefix", "temp", TestFD, TestPath); 257 FileRemover Cleanup(TestPath); 258 // OF is responsible for closing the file; If the file is not 259 // reopened, it will be unbuffered so that the results are 260 // immediately visible through the fd. 261 raw_fd_ostream OF(TestFD, true, !Reopen); 262 for (int i = 0; i < 60000; ++i) { 263 OF << "0123456789"; 264 } 265 266 if (Reopen) { 267 OF.close(); 268 EXPECT_FALSE(sys::fs::openFileForRead(TestPath.c_str(), TestFD)); 269 } 270 271 ErrorOr<OwningBuffer> Buf = MemoryBuffer::getOpenFileSlice( 272 sys::fs::convertFDToNativeFile(TestFD), TestPath.c_str(), 273 40000, // Size 274 80000 // Offset 275 ); 276 277 std::error_code EC = Buf.getError(); 278 EXPECT_FALSE(EC); 279 280 StringRef BufData = Buf.get()->getBuffer(); 281 EXPECT_EQ(BufData.size(), 40000U); 282 EXPECT_EQ(BufData[0], '0'); 283 EXPECT_EQ(BufData[9], '9'); 284 } 285 286 TEST_F(MemoryBufferTest, getOpenFileNoReopen) { 287 testGetOpenFileSlice(false); 288 } 289 290 TEST_F(MemoryBufferTest, getOpenFileReopened) { 291 testGetOpenFileSlice(true); 292 } 293 294 TEST_F(MemoryBufferTest, slice) { 295 // Create a file that is six pages long with different data on each page. 296 int FD; 297 SmallString<64> TestPath; 298 sys::fs::createTemporaryFile("MemoryBufferTest_Slice", "temp", FD, TestPath); 299 FileRemover Cleanup(TestPath); 300 raw_fd_ostream OF(FD, true, /*unbuffered=*/true); 301 for (unsigned i = 0; i < 0x2000 / 8; ++i) { 302 OF << "12345678"; 303 } 304 for (unsigned i = 0; i < 0x2000 / 8; ++i) { 305 OF << "abcdefgh"; 306 } 307 for (unsigned i = 0; i < 0x2000 / 8; ++i) { 308 OF << "ABCDEFGH"; 309 } 310 OF.close(); 311 312 // Try offset of one page. 313 ErrorOr<OwningBuffer> MB = MemoryBuffer::getFileSlice(TestPath.str(), 314 0x4000, 0x1000); 315 std::error_code EC = MB.getError(); 316 ASSERT_FALSE(EC); 317 EXPECT_EQ(0x4000UL, MB.get()->getBufferSize()); 318 319 StringRef BufData = MB.get()->getBuffer(); 320 EXPECT_TRUE(BufData.substr(0x0000, 8) == "12345678"); 321 EXPECT_TRUE(BufData.substr(0x0FF8, 8) == "12345678"); 322 EXPECT_TRUE(BufData.substr(0x1000, 8) == "abcdefgh"); 323 EXPECT_TRUE(BufData.substr(0x2FF8, 8) == "abcdefgh"); 324 EXPECT_TRUE(BufData.substr(0x3000, 8) == "ABCDEFGH"); 325 EXPECT_TRUE(BufData.substr(0x3FF8, 8) == "ABCDEFGH"); 326 327 // Try non-page aligned. 328 ErrorOr<OwningBuffer> MB2 = MemoryBuffer::getFileSlice(TestPath.str(), 329 0x3000, 0x0800); 330 EC = MB2.getError(); 331 ASSERT_FALSE(EC); 332 EXPECT_EQ(0x3000UL, MB2.get()->getBufferSize()); 333 334 StringRef BufData2 = MB2.get()->getBuffer(); 335 EXPECT_TRUE(BufData2.substr(0x0000, 8) == "12345678"); 336 EXPECT_TRUE(BufData2.substr(0x17F8, 8) == "12345678"); 337 EXPECT_TRUE(BufData2.substr(0x1800, 8) == "abcdefgh"); 338 EXPECT_TRUE(BufData2.substr(0x2FF8, 8) == "abcdefgh"); 339 } 340 341 TEST_F(MemoryBufferTest, writableSlice) { 342 // Create a file initialized with some data 343 int FD; 344 SmallString<64> TestPath; 345 sys::fs::createTemporaryFile("MemoryBufferTest_WritableSlice", "temp", FD, 346 TestPath); 347 FileRemover Cleanup(TestPath); 348 raw_fd_ostream OF(FD, true); 349 for (unsigned i = 0; i < 0x1000; ++i) 350 OF << "0123456789abcdef"; 351 OF.close(); 352 353 { 354 auto MBOrError = 355 WritableMemoryBuffer::getFileSlice(TestPath.str(), 0x6000, 0x2000); 356 ASSERT_FALSE(MBOrError.getError()); 357 // Write some data. It should be mapped private, so that upon completion 358 // the original file contents are not modified. 359 WritableMemoryBuffer &MB = **MBOrError; 360 ASSERT_EQ(0x6000u, MB.getBufferSize()); 361 char *Start = MB.getBufferStart(); 362 ASSERT_EQ(MB.getBufferEnd(), MB.getBufferStart() + MB.getBufferSize()); 363 ::memset(Start, 'x', MB.getBufferSize()); 364 } 365 366 auto MBOrError = MemoryBuffer::getFile(TestPath); 367 ASSERT_FALSE(MBOrError.getError()); 368 auto &MB = **MBOrError; 369 ASSERT_EQ(0x10000u, MB.getBufferSize()); 370 for (size_t i = 0; i < MB.getBufferSize(); i += 0x10) 371 EXPECT_EQ("0123456789abcdef", MB.getBuffer().substr(i, 0x10)) << "i: " << i; 372 } 373 374 TEST_F(MemoryBufferTest, writeThroughFile) { 375 // Create a file initialized with some data 376 int FD; 377 SmallString<64> TestPath; 378 sys::fs::createTemporaryFile("MemoryBufferTest_WriteThrough", "temp", FD, 379 TestPath); 380 FileRemover Cleanup(TestPath); 381 raw_fd_ostream OF(FD, true); 382 OF << "0123456789abcdef"; 383 OF.close(); 384 { 385 auto MBOrError = WriteThroughMemoryBuffer::getFile(TestPath); 386 ASSERT_FALSE(MBOrError.getError()); 387 // Write some data. It should be mapped readwrite, so that upon completion 388 // the original file contents are modified. 389 WriteThroughMemoryBuffer &MB = **MBOrError; 390 ASSERT_EQ(16u, MB.getBufferSize()); 391 char *Start = MB.getBufferStart(); 392 ASSERT_EQ(MB.getBufferEnd(), MB.getBufferStart() + MB.getBufferSize()); 393 ::memset(Start, 'x', MB.getBufferSize()); 394 } 395 396 auto MBOrError = MemoryBuffer::getFile(TestPath); 397 ASSERT_FALSE(MBOrError.getError()); 398 auto &MB = **MBOrError; 399 ASSERT_EQ(16u, MB.getBufferSize()); 400 EXPECT_EQ("xxxxxxxxxxxxxxxx", MB.getBuffer()); 401 } 402 403 TEST_F(MemoryBufferTest, mmapVolatileNoNull) { 404 // Verify that `MemoryBuffer::getOpenFile` will use mmap when 405 // `RequiresNullTerminator = false`, `IsVolatile = true`, and the file is 406 // large enough to use mmap. 407 // 408 // This is done because Clang should use this mode to open module files, and 409 // falling back to malloc for them causes a huge memory usage increase. 410 411 int FD; 412 SmallString<64> TestPath; 413 ASSERT_NO_ERROR(sys::fs::createTemporaryFile( 414 "MemoryBufferTest_mmapVolatileNoNull", "temp", FD, TestPath)); 415 FileRemover Cleanup(TestPath); 416 raw_fd_ostream OF(FD, true); 417 // Create a file large enough to mmap. 4 pages should be enough. 418 unsigned PageSize = sys::Process::getPageSizeEstimate(); 419 unsigned FileWrites = (PageSize * 4) / 8; 420 for (unsigned i = 0; i < FileWrites; ++i) 421 OF << "01234567"; 422 OF.close(); 423 424 Expected<sys::fs::file_t> File = sys::fs::openNativeFileForRead(TestPath); 425 ASSERT_THAT_EXPECTED(File, Succeeded()); 426 auto OnExit = 427 make_scope_exit([&] { ASSERT_NO_ERROR(sys::fs::closeFile(*File)); }); 428 429 auto MBOrError = MemoryBuffer::getOpenFile(*File, TestPath, 430 /*FileSize=*/-1, /*RequiresNullTerminator=*/false, /*IsVolatile=*/true); 431 ASSERT_NO_ERROR(MBOrError.getError()) 432 OwningBuffer MB = std::move(*MBOrError); 433 EXPECT_EQ(MB->getBufferKind(), MemoryBuffer::MemoryBuffer_MMap); 434 EXPECT_EQ(MB->getBufferSize(), std::size_t(FileWrites * 8)); 435 EXPECT_TRUE(MB->getBuffer().starts_with("01234567")); 436 } 437 438 // Test that SmallVector without a null terminator gets one. 439 TEST(SmallVectorMemoryBufferTest, WithoutNullTerminatorRequiresNullTerminator) { 440 SmallString<0> Data("some data"); 441 442 SmallVectorMemoryBuffer MB(std::move(Data), 443 /*RequiresNullTerminator=*/true); 444 EXPECT_EQ(MB.getBufferSize(), 9u); 445 EXPECT_EQ(MB.getBufferEnd()[0], '\0'); 446 } 447 448 // Test that SmallVector with a null terminator keeps it. 449 TEST(SmallVectorMemoryBufferTest, WithNullTerminatorRequiresNullTerminator) { 450 SmallString<0> Data("some data"); 451 Data.push_back('\0'); 452 Data.pop_back(); 453 454 SmallVectorMemoryBuffer MB(std::move(Data), 455 /*RequiresNullTerminator=*/true); 456 EXPECT_EQ(MB.getBufferSize(), 9u); 457 EXPECT_EQ(MB.getBufferEnd()[0], '\0'); 458 } 459 460 } // namespace 461