1 #include <stddef.h> 2 #include <stdint.h> 3 #include <stdlib.h> // malloc 4 #include <stdio.h> 5 #include <assert.h> 6 #include <string.h> 7 8 #include "../zstd_seekable.h" 9 10 11 /* ZSTD_seekable_customFile implementation that reads/seeks a buffer while keeping track of total bytes read */ 12 typedef struct { 13 const void *ptr; 14 size_t size; 15 size_t pos; 16 size_t totalRead; 17 } buffWrapperWithTotal_t; 18 19 static int readBuffWithTotal(void* opaque, void* buffer, size_t n) 20 { 21 buffWrapperWithTotal_t* const buff = (buffWrapperWithTotal_t*)opaque; 22 assert(buff != NULL); 23 if (buff->pos + n > buff->size) return -1; 24 memcpy(buffer, (const char*)buff->ptr + buff->pos, n); 25 buff->pos += n; 26 buff->totalRead += n; 27 return 0; 28 } 29 30 static int seekBuffWithTotal(void* opaque, long long offset, int origin) 31 { 32 buffWrapperWithTotal_t* const buff = (buffWrapperWithTotal_t*) opaque; 33 unsigned long long newOffset; 34 assert(buff != NULL); 35 switch (origin) { 36 case SEEK_SET: 37 assert(offset >= 0); 38 newOffset = (unsigned long long)offset; 39 break; 40 case SEEK_CUR: 41 newOffset = (unsigned long long)((long long)buff->pos + offset); 42 break; 43 case SEEK_END: 44 newOffset = (unsigned long long)((long long)buff->size + offset); 45 break; 46 default: 47 assert(0); /* not possible */ 48 } 49 if (newOffset > buff->size) { 50 return -1; 51 } 52 buff->pos = newOffset; 53 return 0; 54 } 55 56 /* Basic unit tests for zstd seekable format */ 57 int main(int argc, const char** argv) 58 { 59 unsigned testNb = 1; 60 (void)argc; (void)argv; 61 printf("Beginning zstd seekable format tests...\n"); 62 63 printf("Test %u - simple round trip: ", testNb++); 64 { size_t const inSize = 4000; 65 void* const inBuffer = malloc(inSize); 66 assert(inBuffer != NULL); 67 68 size_t const seekCapacity = 5000; 69 void* const seekBuffer = malloc(seekCapacity); 70 assert(seekBuffer != NULL); 71 size_t seekSize; 72 73 size_t const outCapacity = inSize; 74 void* const outBuffer = malloc(outCapacity); 75 assert(outBuffer != NULL); 76 77 ZSTD_seekable_CStream* const zscs = ZSTD_seekable_createCStream(); 78 assert(zscs != NULL); 79 80 { size_t const initStatus = ZSTD_seekable_initCStream(zscs, 9, 0 /* checksumFlag */, (unsigned)inSize /* maxFrameSize */); 81 assert(!ZSTD_isError(initStatus)); 82 } 83 84 { ZSTD_outBuffer outb = { .dst=seekBuffer, .pos=0, .size=seekCapacity }; 85 ZSTD_inBuffer inb = { .src=inBuffer, .pos=0, .size=inSize }; 86 87 size_t const cStatus = ZSTD_seekable_compressStream(zscs, &outb, &inb); 88 assert(!ZSTD_isError(cStatus)); 89 assert(inb.pos == inb.size); 90 91 size_t const endStatus = ZSTD_seekable_endStream(zscs, &outb); 92 assert(!ZSTD_isError(endStatus)); 93 seekSize = outb.pos; 94 } 95 96 ZSTD_seekable* const stream = ZSTD_seekable_create(); 97 assert(stream != NULL); 98 { size_t const initStatus = ZSTD_seekable_initBuff(stream, seekBuffer, seekSize); 99 assert(!ZSTD_isError(initStatus)); } 100 101 { size_t const decStatus = ZSTD_seekable_decompress(stream, outBuffer, outCapacity, 0); 102 assert(decStatus == inSize); } 103 104 /* unit test ZSTD_seekTable functions */ 105 ZSTD_seekTable* const zst = ZSTD_seekTable_create_fromSeekable(stream); 106 assert(zst != NULL); 107 108 unsigned const nbFrames = ZSTD_seekTable_getNumFrames(zst); 109 assert(nbFrames > 0); 110 111 unsigned long long const frame0Offset = ZSTD_seekTable_getFrameCompressedOffset(zst, 0); 112 assert(frame0Offset == 0); 113 114 unsigned long long const content0Offset = ZSTD_seekTable_getFrameDecompressedOffset(zst, 0); 115 assert(content0Offset == 0); 116 117 size_t const cSize = ZSTD_seekTable_getFrameCompressedSize(zst, 0); 118 assert(!ZSTD_isError(cSize)); 119 assert(cSize <= seekCapacity); 120 121 size_t const origSize = ZSTD_seekTable_getFrameDecompressedSize(zst, 0); 122 assert(origSize == inSize); 123 124 unsigned const fo1idx = ZSTD_seekTable_offsetToFrameIndex(zst, 1); 125 assert(fo1idx == 0); 126 127 free(inBuffer); 128 free(seekBuffer); 129 free(outBuffer); 130 ZSTD_seekable_freeCStream(zscs); 131 ZSTD_seekTable_free(zst); 132 ZSTD_seekable_free(stream); 133 } 134 printf("Success!\n"); 135 136 137 printf("Test %u - check that seekable decompress does not hang: ", testNb++); 138 { /* Github issue #2335 */ 139 const size_t compressed_size = 17; 140 const uint8_t compressed_data[17] = { 141 '^', 142 '*', 143 'M', 144 '\x18', 145 '\t', 146 '\x00', 147 '\x00', 148 '\x00', 149 '\x00', 150 '\x00', 151 '\x00', 152 '\x00', 153 (uint8_t)('\x03'), 154 (uint8_t)('\xb1'), 155 (uint8_t)('\xea'), 156 (uint8_t)('\x92'), 157 (uint8_t)('\x8f'), 158 }; 159 const size_t uncompressed_size = 32; 160 uint8_t uncompressed_data[32]; 161 162 ZSTD_seekable* const stream = ZSTD_seekable_create(); 163 assert(stream != NULL); 164 { size_t const status = ZSTD_seekable_initBuff(stream, compressed_data, compressed_size); 165 if (ZSTD_isError(status)) { 166 ZSTD_seekable_free(stream); 167 goto _test_error; 168 } } 169 170 /* Should return an error, but not hang */ 171 { const size_t offset = 2; 172 size_t const status = ZSTD_seekable_decompress(stream, uncompressed_data, uncompressed_size, offset); 173 if (!ZSTD_isError(status)) { 174 ZSTD_seekable_free(stream); 175 goto _test_error; 176 } } 177 178 ZSTD_seekable_free(stream); 179 } 180 printf("Success!\n"); 181 182 printf("Test %u - check #2 that seekable decompress does not hang: ", testNb++); 183 { /* Github issue #FIXME */ 184 const size_t compressed_size = 27; 185 const uint8_t compressed_data[27] = { 186 (uint8_t)'\x28', 187 (uint8_t)'\xb5', 188 (uint8_t)'\x2f', 189 (uint8_t)'\xfd', 190 (uint8_t)'\x00', 191 (uint8_t)'\x32', 192 (uint8_t)'\x91', 193 (uint8_t)'\x00', 194 (uint8_t)'\x00', 195 (uint8_t)'\x00', 196 (uint8_t)'\x5e', 197 (uint8_t)'\x2a', 198 (uint8_t)'\x4d', 199 (uint8_t)'\x18', 200 (uint8_t)'\x09', 201 (uint8_t)'\x00', 202 (uint8_t)'\x00', 203 (uint8_t)'\x00', 204 (uint8_t)'\x00', 205 (uint8_t)'\x00', 206 (uint8_t)'\x00', 207 (uint8_t)'\x00', 208 (uint8_t)'\x00', 209 (uint8_t)'\xb1', 210 (uint8_t)'\xea', 211 (uint8_t)'\x92', 212 (uint8_t)'\x8f', 213 }; 214 const size_t uncompressed_size = 400; 215 uint8_t uncompressed_data[400]; 216 217 ZSTD_seekable* stream = ZSTD_seekable_create(); 218 size_t status = ZSTD_seekable_initBuff(stream, compressed_data, compressed_size); 219 if (ZSTD_isError(status)) { 220 ZSTD_seekable_free(stream); 221 goto _test_error; 222 } 223 224 const size_t offset = 2; 225 /* Should return an error, but not hang */ 226 status = ZSTD_seekable_decompress(stream, uncompressed_data, uncompressed_size, offset); 227 if (!ZSTD_isError(status)) { 228 ZSTD_seekable_free(stream); 229 goto _test_error; 230 } 231 232 ZSTD_seekable_free(stream); 233 } 234 printf("Success!\n"); 235 236 237 printf("Test %u - check ZSTD magic in compressing empty string: ", testNb++); 238 { // compressing empty string should return a zstd header 239 size_t const capacity = 255; 240 char* inBuffer = malloc(capacity); 241 assert(inBuffer != NULL); 242 inBuffer[0] = '\0'; 243 void* const outBuffer = malloc(capacity); 244 assert(outBuffer != NULL); 245 246 ZSTD_seekable_CStream *s = ZSTD_seekable_createCStream(); 247 ZSTD_seekable_initCStream(s, 1, 1, 255); 248 249 ZSTD_inBuffer input = { .src=inBuffer, .pos=0, .size=0 }; 250 ZSTD_outBuffer output = { .dst=outBuffer, .pos=0, .size=capacity }; 251 252 ZSTD_seekable_compressStream(s, &output, &input); 253 ZSTD_seekable_endStream(s, &output); 254 255 if((((char*)output.dst)[0] != '\x28') | (((char*)output.dst)[1] != '\xb5') | (((char*)output.dst)[2] != '\x2f') | (((char*)output.dst)[3] != '\xfd')) { 256 printf("%#02x %#02x %#02x %#02x\n", ((char*)output.dst)[0], ((char*)output.dst)[1] , ((char*)output.dst)[2] , ((char*)output.dst)[3] ); 257 258 free(inBuffer); 259 free(outBuffer); 260 ZSTD_seekable_freeCStream(s); 261 goto _test_error; 262 } 263 264 free(inBuffer); 265 free(outBuffer); 266 ZSTD_seekable_freeCStream(s); 267 } 268 printf("Success!\n"); 269 270 271 printf("Test %u - multiple decompress calls: ", testNb++); 272 { char const inBuffer[] = "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt"; 273 size_t const inSize = sizeof(inBuffer); 274 275 size_t const seekCapacity = 5000; 276 void* const seekBuffer = malloc(seekCapacity); 277 assert(seekBuffer != NULL); 278 size_t seekSize; 279 280 size_t const outCapacity = inSize; 281 char* const outBuffer = malloc(outCapacity); 282 assert(outBuffer != NULL); 283 284 ZSTD_seekable_CStream* const zscs = ZSTD_seekable_createCStream(); 285 assert(zscs != NULL); 286 287 /* compress test data with a small frame size to ensure multiple frames in the output */ 288 unsigned const maxFrameSize = 40; 289 { size_t const initStatus = ZSTD_seekable_initCStream(zscs, 9, 0 /* checksumFlag */, maxFrameSize); 290 assert(!ZSTD_isError(initStatus)); 291 } 292 293 { ZSTD_outBuffer outb = { .dst=seekBuffer, .pos=0, .size=seekCapacity }; 294 ZSTD_inBuffer inb = { .src=inBuffer, .pos=0, .size=inSize }; 295 296 while (inb.pos < inb.size) { 297 size_t const cStatus = ZSTD_seekable_compressStream(zscs, &outb, &inb); 298 assert(!ZSTD_isError(cStatus)); 299 } 300 301 size_t const endStatus = ZSTD_seekable_endStream(zscs, &outb); 302 assert(!ZSTD_isError(endStatus)); 303 seekSize = outb.pos; 304 } 305 306 ZSTD_seekable* const stream = ZSTD_seekable_create(); 307 assert(stream != NULL); 308 buffWrapperWithTotal_t buffWrapper = {seekBuffer, seekSize, 0, 0}; 309 { ZSTD_seekable_customFile srcFile = {&buffWrapper, &readBuffWithTotal, &seekBuffWithTotal}; 310 size_t const initStatus = ZSTD_seekable_initAdvanced(stream, srcFile); 311 assert(!ZSTD_isError(initStatus)); } 312 313 /* Perform a series of small reads and seeks (repeatedly read 1 byte and skip 1 byte) 314 and check that we didn't reread input data unnecessarily */ 315 size_t pos; 316 for (pos = 0; pos < inSize; pos += 2) { 317 size_t const decStatus = ZSTD_seekable_decompress(stream, outBuffer, 1, pos); 318 if (decStatus != 1 || outBuffer[0] != inBuffer[pos]) { 319 goto _test_error; 320 } 321 } 322 if (buffWrapper.totalRead > seekSize) { 323 /* We read more than the compressed size, meaning there were some rereads. 324 This is unneeded because we only seeked forward. */ 325 printf("Too much data read: %zu read, with compressed size %zu\n", buffWrapper.totalRead, seekSize); 326 goto _test_error; 327 } 328 329 /* Perform some reads and seeks to ensure correctness */ 330 struct { 331 size_t offset; 332 size_t size; 333 } const tests[] = { /* Assume the frame size is 40 */ 334 {20, 40}, /* read partial data from two frames */ 335 {60, 10}, /* continue reading from the same offset */ 336 {50, 20}, /* seek backward within the same frame */ 337 {10, 10}, /* seek backward to a different frame */ 338 {25, 10}, /* seek forward within the same frame */ 339 {60, 10}, /* seek forward to a different frame */ 340 }; 341 size_t idx; 342 for (idx = 0; idx < sizeof(tests) / sizeof(tests[0]); idx++) { 343 size_t const decStatus = ZSTD_seekable_decompress(stream, outBuffer, tests[idx].size, tests[idx].offset); 344 if (decStatus != tests[idx].size || memcmp(outBuffer, inBuffer + tests[idx].offset, tests[idx].size) != 0) { 345 goto _test_error; 346 } 347 } 348 349 free(seekBuffer); 350 free(outBuffer); 351 ZSTD_seekable_freeCStream(zscs); 352 ZSTD_seekable_free(stream); 353 } 354 printf("Success!\n"); 355 356 /* TODO: Add more tests */ 357 printf("Finished tests\n"); 358 return 0; 359 360 _test_error: 361 printf("test failed! Exiting..\n"); 362 return 1; 363 } 364