1 /* 2 * Copyright (c) Meta Platforms, Inc. and affiliates. 3 * All rights reserved. 4 * 5 * This source code is licensed under both the BSD-style license (found in the 6 * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 * in the COPYING file in the root directory of this source tree). 8 */ 9 #include "Pzstd.h" 10 extern "C" { 11 #include "datagen.h" 12 } 13 #include "test/RoundTrip.h" 14 #include "utils/ScopeGuard.h" 15 16 #include <cstddef> 17 #include <cstdio> 18 #include <gtest/gtest.h> 19 #include <memory> 20 #include <random> 21 22 using namespace std; 23 using namespace pzstd; 24 25 TEST(Pzstd, SmallSizes) { 26 unsigned seed = std::random_device{}(); 27 std::fprintf(stderr, "Pzstd.SmallSizes seed: %u\n", seed); 28 std::mt19937 gen(seed); 29 30 for (unsigned len = 1; len < 256; ++len) { 31 if (len % 16 == 0) { 32 std::fprintf(stderr, "%u / 16\n", len / 16); 33 } 34 std::string inputFile = std::tmpnam(nullptr); 35 auto guard = makeScopeGuard([&] { std::remove(inputFile.c_str()); }); 36 { 37 static uint8_t buf[256]; 38 RDG_genBuffer(buf, len, 0.5, 0.0, gen()); 39 auto fd = std::fopen(inputFile.c_str(), "wb"); 40 auto written = std::fwrite(buf, 1, len, fd); 41 std::fclose(fd); 42 ASSERT_EQ(written, len); 43 } 44 for (unsigned numThreads = 1; numThreads <= 2; ++numThreads) { 45 for (unsigned level = 1; level <= 4; level *= 4) { 46 auto errorGuard = makeScopeGuard([&] { 47 std::fprintf(stderr, "# threads: %u\n", numThreads); 48 std::fprintf(stderr, "compression level: %u\n", level); 49 }); 50 Options options; 51 options.overwrite = true; 52 options.inputFiles = {inputFile}; 53 options.numThreads = numThreads; 54 options.compressionLevel = level; 55 options.verbosity = 1; 56 ASSERT_TRUE(roundTrip(options)); 57 errorGuard.dismiss(); 58 } 59 } 60 } 61 } 62 63 TEST(Pzstd, LargeSizes) { 64 unsigned seed = std::random_device{}(); 65 std::fprintf(stderr, "Pzstd.LargeSizes seed: %u\n", seed); 66 std::mt19937 gen(seed); 67 68 for (unsigned len = 1 << 20; len <= (1 << 24); len *= 2) { 69 std::string inputFile = std::tmpnam(nullptr); 70 auto guard = makeScopeGuard([&] { std::remove(inputFile.c_str()); }); 71 { 72 std::unique_ptr<uint8_t[]> buf(new uint8_t[len]); 73 RDG_genBuffer(buf.get(), len, 0.5, 0.0, gen()); 74 auto fd = std::fopen(inputFile.c_str(), "wb"); 75 auto written = std::fwrite(buf.get(), 1, len, fd); 76 std::fclose(fd); 77 ASSERT_EQ(written, len); 78 } 79 for (unsigned numThreads = 1; numThreads <= 16; numThreads *= 4) { 80 for (unsigned level = 1; level <= 4; level *= 4) { 81 auto errorGuard = makeScopeGuard([&] { 82 std::fprintf(stderr, "# threads: %u\n", numThreads); 83 std::fprintf(stderr, "compression level: %u\n", level); 84 }); 85 Options options; 86 options.overwrite = true; 87 options.inputFiles = {inputFile}; 88 options.numThreads = std::min(numThreads, options.numThreads); 89 options.compressionLevel = level; 90 options.verbosity = 1; 91 ASSERT_TRUE(roundTrip(options)); 92 errorGuard.dismiss(); 93 } 94 } 95 } 96 } 97 98 TEST(Pzstd, DISABLED_ExtremelyLargeSize) { 99 unsigned seed = std::random_device{}(); 100 std::fprintf(stderr, "Pzstd.ExtremelyLargeSize seed: %u\n", seed); 101 std::mt19937 gen(seed); 102 103 std::string inputFile = std::tmpnam(nullptr); 104 auto guard = makeScopeGuard([&] { std::remove(inputFile.c_str()); }); 105 106 { 107 // Write 4GB + 64 MB 108 constexpr size_t kLength = 1 << 26; 109 std::unique_ptr<uint8_t[]> buf(new uint8_t[kLength]); 110 auto fd = std::fopen(inputFile.c_str(), "wb"); 111 auto closeGuard = makeScopeGuard([&] { std::fclose(fd); }); 112 for (size_t i = 0; i < (1 << 6) + 1; ++i) { 113 RDG_genBuffer(buf.get(), kLength, 0.5, 0.0, gen()); 114 auto written = std::fwrite(buf.get(), 1, kLength, fd); 115 if (written != kLength) { 116 std::fprintf(stderr, "Failed to write file, skipping test\n"); 117 return; 118 } 119 } 120 } 121 122 Options options; 123 options.overwrite = true; 124 options.inputFiles = {inputFile}; 125 options.compressionLevel = 1; 126 if (options.numThreads == 0) { 127 options.numThreads = 1; 128 } 129 ASSERT_TRUE(roundTrip(options)); 130 } 131 132 TEST(Pzstd, ExtremelyCompressible) { 133 std::string inputFile = std::tmpnam(nullptr); 134 auto guard = makeScopeGuard([&] { std::remove(inputFile.c_str()); }); 135 { 136 std::unique_ptr<uint8_t[]> buf(new uint8_t[10000]); 137 std::memset(buf.get(), 'a', 10000); 138 auto fd = std::fopen(inputFile.c_str(), "wb"); 139 auto written = std::fwrite(buf.get(), 1, 10000, fd); 140 std::fclose(fd); 141 ASSERT_EQ(written, 10000); 142 } 143 Options options; 144 options.overwrite = true; 145 options.inputFiles = {inputFile}; 146 options.numThreads = 1; 147 options.compressionLevel = 1; 148 ASSERT_TRUE(roundTrip(options)); 149 } 150