1 /**********************************************************************
2 Copyright(c) 2011-2016 Intel Corporation All rights reserved.
3
4 Redistribution and use in source and binary forms, with or without
5 modification, are permitted provided that the following conditions
6 are met:
7 * Redistributions of source code must retain the above copyright
8 notice, this list of conditions and the following disclaimer.
9 * Redistributions in binary form must reproduce the above copyright
10 notice, this list of conditions and the following disclaimer in
11 the documentation and/or other materials provided with the
12 distribution.
13 * Neither the name of Intel Corporation nor the names of its
14 contributors may be used to endorse or promote products derived
15 from this software without specific prior written permission.
16
17 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 **********************************************************************/
29
30 #define _FILE_OFFSET_BITS 64
31 #include <stdio.h>
32 #include <stdlib.h>
33 #include <string.h>
34 #include <assert.h>
35 #include <stdarg.h>
36 #include <inttypes.h>
37
38 #include "igzip_lib.h"
39 #include "checksum_test_ref.h"
40 #include "inflate_std_vects.h"
41 #include <math.h>
42 #include "test.h"
43 #include "unaligned.h"
44
45 #ifdef HAVE_GETOPT
46 #include <getopt.h>
47 #endif
48
49 #ifndef RANDOMS
50 #define RANDOMS 0x40
51 #endif
52 #ifndef TEST_SEED
53 #define TEST_SEED 0x1234
54 #endif
55
56 #define MAX_BITS_COUNT 20
57 #define MIN_BITS_COUNT 8
58
59 #define IBUF_SIZE (1024 * 1024)
60
61 #define MAX_LARGE_COMP_BUF_SIZE (1024 * 1024)
62
63 #define PAGE_SIZE 4 * 1024
64
65 #define MAX_FILE_SIZE 0x7fff8fff
66
67 #define str1 "Short test string"
68 #define str2 \
69 "one two three four five six seven eight nine ten eleven twelve " \
70 "thirteen fourteen fifteen sixteen"
71
72 #define TYPE0_HDR_SIZE 5 /* Size of a type 0 blocks header in bytes */
73 #define TYPE0_MAX_SIZE 65535 /* Max length of a type 0 block in bytes (excludes the header) */
74
75 #define MAX_LOOPS 20
76 /* Defines for the possible error conditions */
77 enum IGZIP_TEST_ERROR_CODES {
78 IGZIP_COMP_OK = 0,
79
80 MALLOC_FAILED,
81 FILE_READ_FAILED,
82
83 COMPRESS_INCORRECT_STATE,
84 COMPRESS_INPUT_STREAM_INTEGRITY_ERROR,
85 COMPRESS_OUTPUT_STREAM_INTEGRITY_ERROR,
86 COMPRESS_END_OF_STREAM_NOT_SET,
87 COMPRESS_ALL_INPUT_FAIL,
88 COMPRESS_OUT_BUFFER_OVERFLOW,
89 COMPRESS_LOOP_COUNT_OVERFLOW,
90 COMPRESS_GENERAL_ERROR,
91
92 INFLATE_END_OF_INPUT,
93 INFLATE_INVALID_BLOCK_HEADER,
94 INFLATE_INVALID_SYMBOL,
95 INFLATE_OUT_BUFFER_OVERFLOW,
96 INFLATE_LEFTOVER_INPUT,
97 INFLATE_INCORRECT_OUTPUT_SIZE,
98 INFLATE_INVALID_LOOK_BACK_DISTANCE,
99 INFLATE_INPUT_STREAM_INTEGRITY_ERROR,
100 INFLATE_OUTPUT_STREAM_INTEGRITY_ERROR,
101 INVALID_GZIP_HEADER,
102 INCORRECT_GZIP_TRAILER,
103 INVALID_ZLIB_HEADER,
104 INCORRECT_ZLIB_TRAILER,
105
106 UNSUPPORTED_METHOD,
107
108 INFLATE_GENERAL_ERROR,
109
110 INVALID_FLUSH_ERROR,
111
112 OVERFLOW_TEST_ERROR,
113 RESULT_ERROR
114 };
115
116 static const int hdr_bytes = 300;
117
118 static const uint32_t gzip_trl_bytes = 8;
119 static const uint32_t zlib_trl_bytes = 4;
120 static const int gzip_extra_bytes = 18; /* gzip_hdr_bytes + gzip_trl_bytes */
121 static const int zlib_extra_bytes = 6; /* zlib_hdr_bytes + zlib_trl_bytes */
122
123 int inflate_type = 0;
124
125 struct isal_hufftables *hufftables = NULL;
126 struct isal_hufftables *hufftables_subset = NULL;
127
128 #define HISTORY_SIZE 32 * 1024
129 #define MIN_LENGTH 3
130 #define MIN_DIST 1
131
132 struct test_options {
133 int test_seed;
134 int randoms;
135 int do_large_test;
136 int verbose;
137 };
138
139 struct test_options options;
140
141 void
init_options(void)142 init_options(void)
143 {
144 options.test_seed = TEST_SEED;
145 options.randoms = RANDOMS;
146 options.do_large_test = 1;
147 #ifdef TEST_VERBOSE
148 options.verbose = 1;
149 #else
150 options.verbose = 0;
151 #endif
152 }
153
154 void
usage(void)155 usage(void)
156 {
157 fprintf(stderr, "Usage: igzip_rand_test [options] [FILES]\n"
158 " -h help, print this message\n"
159 " -l turn off large input test\n"
160 " -r <iter> number of randoms for each test\n"
161 " -s <seed> set rand() test seed\n"
162 " -v enable verbose test log\n");
163 exit(0);
164 }
165
166 size_t
parse_options(int argc,char * argv[])167 parse_options(int argc, char *argv[])
168 {
169 init_options();
170 #ifdef HAVE_GETOPT
171 int c;
172 char optstring[] = "hlr:s:v";
173 while ((c = getopt(argc, argv, optstring)) != -1) {
174 switch (c) {
175 case 'l':
176 options.do_large_test = 0;
177 break;
178 case 'r':
179 options.randoms = atoi(optarg);
180 break;
181 case 's':
182 options.test_seed = atoi(optarg);
183 break;
184 case 'v':
185 options.verbose = 1;
186 break;
187 case 'h':
188 default:
189 usage();
190 break;
191 }
192 }
193 return optind;
194 #else
195 return 1;
196 #endif
197 }
198
199 /* Create random compressible data. This is achieved by randomly choosing a
200 * random character, or to repeat previous data in the stream for a random
201 * length and look back distance. The probability of a random character or a
202 * repeat being chosen is semi-randomly chosen by setting max_repeat_data to be
203 * differing values */
204 void
create_rand_repeat_data(uint8_t * data,int size)205 create_rand_repeat_data(uint8_t *data, int size)
206 {
207 uint32_t next_data;
208 uint8_t *data_start = data;
209 uint32_t length, distance;
210 uint32_t symbol_count = rand() % 255 + 1, swaps_left, tmp;
211 uint32_t max_repeat_data = symbol_count;
212 uint8_t symbols[256], *symbols_next, swap_val;
213
214 /* An array of the powers of 2 (except the final element which is 0) */
215 const uint32_t power_of_2_array[] = {
216 0x00000001, 0x00000002, 0x00000004, 0x00000008, 0x00000010, 0x00000020, 0x00000040,
217 0x00000080, 0x00000100, 0x00000200, 0x00000400, 0x00000800, 0x00001000, 0x00002000,
218 0x00004000, 0x00008000, 0x00010000, 0x00020000, 0x00040000, 0x00080000, 0x00100000,
219 0x00200000, 0x00400000, 0x00800000, 0x01000000, 0x02000000, 0x04000000, 0x08000000,
220 0x10000000, 0x20000000, 0x40000000, 0x00000000
221 };
222
223 uint32_t power = rand() % sizeof(power_of_2_array) / sizeof(uint32_t);
224
225 if (symbol_count > 128) {
226 memset(symbols, 1, sizeof(symbols));
227 swap_val = 0;
228 swaps_left = 256 - symbol_count;
229 } else {
230 memset(symbols, 0, sizeof(symbols));
231 swap_val = 1;
232 swaps_left = symbol_count;
233 }
234
235 while (swaps_left > 0) {
236 tmp = rand() % 256;
237 if (symbols[tmp] != swap_val) {
238 symbols[tmp] = swap_val;
239 swaps_left--;
240 }
241 }
242
243 symbols_next = symbols;
244 for (tmp = 0; tmp < 256; tmp++) {
245 if (symbols[tmp]) {
246 *symbols_next = tmp;
247 symbols_next++;
248 }
249 }
250
251 max_repeat_data += power_of_2_array[power];
252
253 if (size > 0) {
254 size--;
255 *data++ = rand();
256 }
257
258 while (size > 0) {
259 next_data = rand() % max_repeat_data;
260 if (next_data < symbol_count) {
261 *data++ = symbols[next_data];
262 size--;
263 } else if (size < 3) {
264 *data++ = symbols[rand() % symbol_count];
265 size--;
266 } else {
267 length = (rand() % 256) + MIN_LENGTH;
268 if (length > size)
269 length = (rand() % (size - 2)) + MIN_LENGTH;
270
271 distance = (rand() % HISTORY_SIZE) + MIN_DIST;
272 if (distance > data - data_start)
273 distance = (rand() % (data - data_start)) + MIN_DIST;
274
275 size -= length;
276 if (distance <= length) {
277 while (length-- > 0) {
278 *data = *(data - distance);
279 data++;
280 }
281 } else {
282 memcpy(data, data - distance, length);
283 data += length;
284 }
285 }
286 }
287 }
288
289 void
create_rand_dict(uint8_t * dict,uint32_t dict_len,uint8_t * buf,uint32_t buf_len)290 create_rand_dict(uint8_t *dict, uint32_t dict_len, uint8_t *buf, uint32_t buf_len)
291 {
292 uint32_t dict_chunk_size, buf_chunk_size;
293 while (dict_len > 0) {
294 dict_chunk_size = rand() % IGZIP_K;
295 dict_chunk_size = (dict_len >= dict_chunk_size) ? dict_chunk_size : dict_len;
296
297 buf_chunk_size = rand() % IGZIP_K;
298 buf_chunk_size = (buf_len >= buf_chunk_size) ? buf_chunk_size : buf_len;
299
300 if (rand() % 3 == 0 && buf_len >= dict_len)
301 memcpy(dict, buf, dict_chunk_size);
302 else
303 create_rand_repeat_data(dict, dict_chunk_size);
304
305 dict_len -= dict_chunk_size;
306 dict += dict_chunk_size;
307 buf_len -= buf_chunk_size;
308 buf += buf_chunk_size;
309 }
310 }
311
312 int
get_rand_data_length(void)313 get_rand_data_length(void)
314 {
315 int max_mask = (1 << ((rand() % (MAX_BITS_COUNT - MIN_BITS_COUNT)) + MIN_BITS_COUNT)) - 1;
316 return rand() & max_mask;
317 }
318
319 int
get_rand_level(void)320 get_rand_level(void)
321 {
322 return ISAL_DEF_MIN_LEVEL + rand() % (ISAL_DEF_MAX_LEVEL - ISAL_DEF_MIN_LEVEL + 1);
323 }
324
325 int
get_rand_level_buf_size(int level)326 get_rand_level_buf_size(int level)
327 {
328 int size;
329 switch (level) {
330 case 3:
331 size = rand() % IBUF_SIZE + ISAL_DEF_LVL3_MIN;
332 break;
333 case 2:
334 size = rand() % IBUF_SIZE + ISAL_DEF_LVL2_MIN;
335 break;
336 case 1:
337 default:
338 size = rand() % IBUF_SIZE + ISAL_DEF_LVL1_MIN;
339 }
340 return size;
341 }
342
343 void
print_error(int error_code)344 print_error(int error_code)
345 {
346 switch (error_code) {
347 case IGZIP_COMP_OK:
348 break;
349 case MALLOC_FAILED:
350 printf("error: failed to allocate memory\n");
351 break;
352 case FILE_READ_FAILED:
353 printf("error: failed to read in file\n");
354 break;
355 case COMPRESS_INCORRECT_STATE:
356 printf("error: incorrect stream internal state\n");
357 break;
358 case COMPRESS_INPUT_STREAM_INTEGRITY_ERROR:
359 printf("error: inconsistent stream input buffer\n");
360 break;
361 case COMPRESS_OUTPUT_STREAM_INTEGRITY_ERROR:
362 printf("error: inconsistent stream output buffer\n");
363 break;
364 case COMPRESS_END_OF_STREAM_NOT_SET:
365 printf("error: end of stream not set\n");
366 break;
367 case COMPRESS_ALL_INPUT_FAIL:
368 printf("error: not all input data compressed\n");
369 break;
370 case COMPRESS_OUT_BUFFER_OVERFLOW:
371 printf("error: output buffer overflow while compressing data\n");
372 break;
373 case COMPRESS_GENERAL_ERROR:
374 printf("error: compression failed\n");
375 break;
376 case INFLATE_END_OF_INPUT:
377 printf("error: did not decompress all input\n");
378 break;
379 case INFLATE_INVALID_BLOCK_HEADER:
380 printf("error: invalid header\n");
381 break;
382 case INFLATE_INVALID_SYMBOL:
383 printf("error: invalid symbol found when decompressing input\n");
384 break;
385 case INFLATE_OUT_BUFFER_OVERFLOW:
386 printf("error: output buffer overflow while decompressing data\n");
387 break;
388 case INFLATE_GENERAL_ERROR:
389 printf("error: decompression failed\n");
390 break;
391 case INFLATE_LEFTOVER_INPUT:
392 printf("error: the trailer of igzip output contains junk\n");
393 break;
394 case INFLATE_INCORRECT_OUTPUT_SIZE:
395 printf("error: incorrect amount of data was decompressed\n");
396 break;
397 case INFLATE_INVALID_LOOK_BACK_DISTANCE:
398 printf("error: invalid look back distance found while decompressing\n");
399 break;
400 case INFLATE_INPUT_STREAM_INTEGRITY_ERROR:
401 printf("error: inconsistent input buffer\n");
402 break;
403 case INFLATE_OUTPUT_STREAM_INTEGRITY_ERROR:
404 printf("error: inconsistent output buffer\n");
405 break;
406 case INVALID_GZIP_HEADER:
407 printf("error: incorrect gzip header found when inflating data\n");
408 break;
409 case INCORRECT_GZIP_TRAILER:
410 printf("error: incorrect gzip trailer found when inflating data\n");
411 break;
412 case INVALID_ZLIB_HEADER:
413 printf("error: incorrect zlib header found when inflating data\n");
414 break;
415 case INCORRECT_ZLIB_TRAILER:
416 printf("error: incorrect zlib trailer found when inflating data\n");
417 break;
418 case UNSUPPORTED_METHOD:
419 printf("error: invalid compression method in wrapper header\n");
420 break;
421 case INVALID_FLUSH_ERROR:
422 printf("error: invalid flush did not cause compression to error\n");
423 break;
424 case RESULT_ERROR:
425 printf("error: decompressed data is not the same as the compressed data\n");
426 break;
427 case OVERFLOW_TEST_ERROR:
428 printf("error: overflow undetected\n");
429 break;
430 default:
431 printf("error: unknown error code\n");
432 }
433 }
434
435 void
print_uint8_t(uint8_t * array,uint64_t length)436 print_uint8_t(uint8_t *array, uint64_t length)
437 {
438 const int line_size = 16;
439 int i;
440
441 printf("Length = %" PRIu64 "", length);
442 for (i = 0; i < length; i++) {
443 if ((i % line_size) == 0)
444 printf("\n0x%08x\t", i);
445 else
446 printf(" ");
447 printf("0x%02x,", array[i]);
448 }
449 printf("\n");
450 }
451
452 void
log_print(char * format,...)453 log_print(char *format, ...)
454 {
455 va_list args;
456 va_start(args, format);
457
458 if (options.verbose)
459 vfprintf(stdout, format, args);
460
461 va_end(args);
462 }
463
464 void
log_uint8_t(uint8_t * array,uint64_t length)465 log_uint8_t(uint8_t *array, uint64_t length)
466 {
467 if (options.verbose)
468 print_uint8_t(array, length);
469 }
470
471 void
log_error(int error_code)472 log_error(int error_code)
473 {
474 if (options.verbose)
475 print_error(error_code);
476 }
477
478 uint32_t
check_gzip_trl(uint64_t gzip_trl,uint32_t inflate_crc,uint8_t * uncompress_buf,uint32_t uncompress_len)479 check_gzip_trl(uint64_t gzip_trl, uint32_t inflate_crc, uint8_t *uncompress_buf,
480 uint32_t uncompress_len)
481 {
482 uint64_t trl, ret = 0;
483 uint32_t crc;
484
485 crc = crc32_gzip_refl_ref(0, uncompress_buf, uncompress_len);
486 trl = ((uint64_t) uncompress_len << 32) | crc;
487
488 if (crc != inflate_crc || trl != gzip_trl)
489 ret = INCORRECT_GZIP_TRAILER;
490
491 return ret;
492 }
493
494 uint32_t
check_zlib_trl(uint32_t zlib_trl,uint32_t inflate_adler,uint8_t * uncompress_buf,uint32_t uncompress_len)495 check_zlib_trl(uint32_t zlib_trl, uint32_t inflate_adler, uint8_t *uncompress_buf,
496 uint32_t uncompress_len)
497 {
498 uint32_t trl, ret = 0;
499 uint32_t adler;
500
501 adler = adler_ref(1, uncompress_buf, uncompress_len);
502
503 trl = (adler >> 24) | ((adler >> 8) & 0xFF00) | (adler << 24) | ((adler & 0xFF00) << 8);
504
505 if (adler != inflate_adler || trl != zlib_trl) {
506 ret = INCORRECT_ZLIB_TRAILER;
507 }
508
509 return ret;
510 }
511
512 int
inflate_stateless_pass(uint8_t * compress_buf,uint64_t compress_len,uint8_t * uncompress_buf,uint32_t * uncompress_len,uint32_t gzip_flag)513 inflate_stateless_pass(uint8_t *compress_buf, uint64_t compress_len, uint8_t *uncompress_buf,
514 uint32_t *uncompress_len, uint32_t gzip_flag)
515 {
516 struct inflate_state state;
517 int ret = 0, offset = 0;
518 struct isal_gzip_header gz_hdr;
519 struct isal_zlib_header z_hdr;
520
521 state.next_in = compress_buf;
522 state.avail_in = compress_len;
523 state.next_out = uncompress_buf;
524 state.avail_out = *uncompress_len;
525
526 if (gzip_flag == IGZIP_GZIP) {
527 if (rand() % 2 == 0) {
528 memset(&gz_hdr, 0, sizeof(gz_hdr));
529 isal_inflate_reset(&state);
530 state.tmp_in_size = 0;
531 gzip_flag = ISAL_GZIP_NO_HDR_VER;
532
533 if (isal_read_gzip_header(&state, &gz_hdr) != 0)
534 return INVALID_GZIP_HEADER;
535 }
536 } else if (gzip_flag == IGZIP_ZLIB) {
537 if (rand() % 2 == 0) {
538 memset(&z_hdr, 0, sizeof(z_hdr));
539 isal_inflate_reset(&state);
540 gzip_flag = ISAL_ZLIB_NO_HDR_VER;
541 if (isal_read_zlib_header(&state, &z_hdr) != 0)
542 return INVALID_ZLIB_HEADER;
543 }
544 }
545
546 state.crc_flag = gzip_flag;
547
548 ret = isal_inflate_stateless(&state);
549
550 *uncompress_len = state.total_out;
551
552 if (gzip_flag) {
553 if (gzip_flag == IGZIP_GZIP || gzip_flag == IGZIP_GZIP_NO_HDR ||
554 gzip_flag == ISAL_GZIP_NO_HDR_VER) {
555 if (gzip_flag == IGZIP_GZIP || gzip_flag == ISAL_GZIP_NO_HDR_VER)
556 offset = gzip_trl_bytes;
557
558 if (!ret)
559 ret = check_gzip_trl(load_le_u64(state.next_in - offset), state.crc,
560 uncompress_buf, *uncompress_len);
561 else if (ret == ISAL_INCORRECT_CHECKSUM)
562 ret = INCORRECT_GZIP_TRAILER;
563 state.avail_in -= (gzip_trl_bytes - offset);
564 } else if (gzip_flag == IGZIP_ZLIB || gzip_flag == IGZIP_ZLIB_NO_HDR ||
565 gzip_flag == ISAL_ZLIB_NO_HDR_VER) {
566 if (gzip_flag == IGZIP_ZLIB || gzip_flag == ISAL_ZLIB_NO_HDR_VER)
567 offset = zlib_trl_bytes;
568
569 if (!ret)
570 ret = check_zlib_trl(load_le_u32(state.next_in - offset), state.crc,
571 uncompress_buf, *uncompress_len);
572 else if (ret == ISAL_INCORRECT_CHECKSUM)
573 ret = INCORRECT_ZLIB_TRAILER;
574 state.avail_in -= (zlib_trl_bytes - offset);
575 }
576 }
577
578 if (ret == 0 && state.avail_in != 0)
579 ret = INFLATE_LEFTOVER_INPUT;
580
581 return ret;
582 }
583
584 /* Check if that the state of the data stream is consistent */
585 int
inflate_state_valid_check(struct inflate_state * state,uint8_t * in_buf,uint32_t in_size,uint8_t * out_buf,uint32_t out_size,uint32_t in_processed,uint32_t out_processed,uint32_t data_size)586 inflate_state_valid_check(struct inflate_state *state, uint8_t *in_buf, uint32_t in_size,
587 uint8_t *out_buf, uint32_t out_size, uint32_t in_processed,
588 uint32_t out_processed, uint32_t data_size)
589 {
590 uint32_t in_buffer_size, total_out, out_buffer_size;
591
592 in_buffer_size = (in_size == 0) ? 0 : state->next_in - in_buf + state->avail_in;
593
594 /* Check for a consistent amount of data processed */
595 if (in_buffer_size != in_size)
596 return INFLATE_INPUT_STREAM_INTEGRITY_ERROR;
597
598 total_out = (out_size == 0) ? out_processed : out_processed + (state->next_out - out_buf);
599 out_buffer_size = (out_size == 0) ? 0 : state->next_out - out_buf + state->avail_out;
600
601 /* Check for a consistent amount of data compressed */
602 if (total_out != state->total_out || out_buffer_size != out_size)
603 return INFLATE_OUTPUT_STREAM_INTEGRITY_ERROR;
604
605 return 0;
606 }
607
608 /* Performs compression with checks to discover and verify the state of the
609 * stream
610 * state: inflate data structure which has been initialized to use
611 * in_buf and out_buf as the buffers
612 * compress_len: size of all input compressed data
613 * data_size: size of all available output buffers
614 * in_buf: next buffer of data to be inflated
615 * in_size: size of in_buf
616 * out_buf: next out put buffer where data is stored
617 * out_size: size of out_buf
618 * in_processed: the amount of input data which has been loaded into buffers
619 * to be inflated, this includes the data in in_buf
620 * out_processed: the amount of output data which has been decompressed and stored,
621 * this does not include the data in the current out_buf
622 */
623 int
isal_inflate_with_checks(struct inflate_state * state,uint32_t compress_len,uint32_t data_size,uint8_t * in_buf,uint32_t in_size,uint32_t in_processed,uint8_t * out_buf,uint32_t out_size,uint32_t out_processed)624 isal_inflate_with_checks(struct inflate_state *state, uint32_t compress_len, uint32_t data_size,
625 uint8_t *in_buf, uint32_t in_size, uint32_t in_processed, uint8_t *out_buf,
626 uint32_t out_size, uint32_t out_processed)
627 {
628 int ret, stream_check = 0;
629
630 ret = isal_inflate(state);
631
632 /* Verify the stream is in a valid state when no errors occurred */
633 if (ret >= 0) {
634 stream_check = inflate_state_valid_check(state, in_buf, in_size, out_buf, out_size,
635 in_processed, out_processed, data_size);
636 }
637
638 if (stream_check != 0)
639 return stream_check;
640
641 return ret;
642 }
643
644 int
inflate_multi_pass(uint8_t * compress_buf,uint64_t compress_len,uint8_t * uncompress_buf,uint32_t * uncompress_len,uint32_t gzip_flag,uint8_t * dict,uint32_t dict_len,uint32_t hist_bits)645 inflate_multi_pass(uint8_t *compress_buf, uint64_t compress_len, uint8_t *uncompress_buf,
646 uint32_t *uncompress_len, uint32_t gzip_flag, uint8_t *dict, uint32_t dict_len,
647 uint32_t hist_bits)
648 {
649 struct inflate_state *state = NULL;
650 int ret = 0;
651 uint8_t *comp_tmp = NULL, *uncomp_tmp = NULL;
652 uint32_t comp_tmp_size = 0, uncomp_tmp_size = 0;
653 uint32_t comp_processed = 0, uncomp_processed = 0;
654 int32_t read_in_old = 0;
655 uint32_t reset_test_flag = 0;
656
657 state = malloc(sizeof(struct inflate_state));
658 if (state == NULL) {
659 printf("Failed to allocate memory\n");
660 exit(0);
661 }
662
663 create_rand_repeat_data((uint8_t *) state, sizeof(*state));
664 isal_inflate_init(state);
665
666 if (rand() % 4 == 0) {
667 /* Test reset */
668 reset_test_flag = 1;
669 create_rand_repeat_data((uint8_t *) state, sizeof(*state));
670 }
671
672 if (gzip_flag == IGZIP_GZIP_NO_HDR) {
673 if (rand() % 2 == 0)
674 compress_len -= gzip_trl_bytes;
675 else
676 gzip_flag = ISAL_GZIP_NO_HDR_VER;
677 } else if (gzip_flag == IGZIP_ZLIB_NO_HDR) {
678 if (rand() % 2 == 0)
679 compress_len -= zlib_trl_bytes;
680 else
681 gzip_flag = ISAL_ZLIB_NO_HDR_VER;
682 }
683
684 state->next_in = NULL;
685 state->next_out = NULL;
686 state->avail_in = 0;
687 state->avail_out = 0;
688 state->crc_flag = gzip_flag;
689 state->hist_bits = hist_bits;
690
691 if (reset_test_flag)
692 isal_inflate_reset(state);
693
694 if (dict != NULL)
695 isal_inflate_set_dict(state, dict, dict_len);
696
697 while (1) {
698 if (state->avail_in == 0) {
699 comp_tmp_size = rand() % (compress_len + 1);
700
701 if (comp_tmp_size >= compress_len - comp_processed)
702 comp_tmp_size = compress_len - comp_processed;
703
704 if (comp_tmp_size != 0) {
705 if (comp_tmp != NULL)
706 free(comp_tmp);
707
708 comp_tmp = malloc(comp_tmp_size);
709
710 if (comp_tmp == NULL) {
711 printf("Failed to allocate memory\n");
712 ret = MALLOC_FAILED;
713 goto exit_in_multi_pass;
714 }
715
716 memcpy(comp_tmp, compress_buf + comp_processed, comp_tmp_size);
717 comp_processed += comp_tmp_size;
718
719 state->next_in = comp_tmp;
720 state->avail_in = comp_tmp_size;
721 }
722 }
723
724 if (state->avail_out == 0) {
725 /* Save uncompressed data into uncompress_buf */
726 if (uncomp_tmp != NULL) {
727 memcpy(uncompress_buf + uncomp_processed, uncomp_tmp,
728 uncomp_tmp_size);
729 uncomp_processed += uncomp_tmp_size;
730 }
731
732 uncomp_tmp_size = rand() % (*uncompress_len + 1);
733
734 /* Limit size of buffer to be smaller than maximum */
735 if (uncomp_tmp_size > *uncompress_len - uncomp_processed)
736 uncomp_tmp_size = *uncompress_len - uncomp_processed;
737
738 if (uncomp_tmp_size != 0) {
739
740 if (uncomp_tmp != NULL) {
741 fflush(0);
742 free(uncomp_tmp);
743 }
744
745 uncomp_tmp = malloc(uncomp_tmp_size);
746 if (uncomp_tmp == NULL) {
747 printf("Failed to allocate memory\n");
748 ret = MALLOC_FAILED;
749 goto exit_in_multi_pass;
750 }
751 memset(uncomp_tmp, 0, uncomp_tmp_size);
752
753 state->avail_out = uncomp_tmp_size;
754 state->next_out = uncomp_tmp;
755 }
756 }
757
758 log_print("Pre inflate\n");
759 log_print("compressed_size = 0x%05lx, in_processed = 0x%05x, in_size = 0x%05x, "
760 "avail_in = 0x%05x\n",
761 compress_len, comp_processed, comp_tmp_size, state->avail_in);
762 log_print("data_size = 0x%05x, out_processed = 0x%05x, out_size = 0x%05x, "
763 "avail_out = 0x%05x, total_out = 0x%05x\n",
764 *uncompress_len, uncomp_processed, uncomp_tmp_size, state->avail_out,
765 state->total_out);
766
767 ret = isal_inflate_with_checks(state, compress_len, *uncompress_len, comp_tmp,
768 comp_tmp_size, comp_processed, uncomp_tmp,
769 uncomp_tmp_size, uncomp_processed);
770
771 log_print("Post inflate\n");
772 log_print("compressed_size = 0x%05lx, in_processed = 0x%05x, in_size = 0x%05x, "
773 "avail_in = 0x%05x\n",
774 compress_len, comp_processed, comp_tmp_size, state->avail_in);
775 log_print("data_size = 0x%05x, out_processed = 0x%05x, out_size = 0x%05x, "
776 "avail_out = 0x%05x, total_out = 0x%05x\n",
777 *uncompress_len, uncomp_processed, uncomp_tmp_size, state->avail_out,
778 state->total_out);
779
780 if (state->block_state == ISAL_BLOCK_FINISH || ret != 0) {
781 memcpy(uncompress_buf + uncomp_processed, uncomp_tmp, uncomp_tmp_size);
782 *uncompress_len = state->total_out;
783 break;
784 }
785
786 if (*uncompress_len - uncomp_processed == 0 && state->avail_out == 0 &&
787 state->tmp_out_valid - state->tmp_out_processed > 0) {
788 ret = ISAL_OUT_OVERFLOW;
789 break;
790 }
791
792 if (compress_len - comp_processed == 0 && state->avail_in == 0 &&
793 (state->block_state != ISAL_BLOCK_INPUT_DONE) &&
794 state->tmp_out_valid - state->tmp_out_processed == 0) {
795 if (state->read_in_length == read_in_old) {
796 ret = ISAL_END_INPUT;
797 break;
798 }
799 read_in_old = state->read_in_length;
800 }
801 }
802
803 if (gzip_flag) {
804 if (!ret) {
805 if (gzip_flag == IGZIP_GZIP || gzip_flag == IGZIP_GZIP_NO_HDR ||
806 gzip_flag == ISAL_GZIP_NO_HDR_VER) {
807 if (gzip_flag == ISAL_GZIP_NO_HDR_VER || gzip_flag == IGZIP_GZIP)
808 compress_len -= gzip_trl_bytes;
809 ret = check_gzip_trl(load_le_u64(compress_buf + compress_len),
810 state->crc, uncompress_buf, *uncompress_len);
811 } else if (gzip_flag == IGZIP_ZLIB_NO_HDR) {
812 if (gzip_flag == IGZIP_ZLIB || gzip_flag == ISAL_ZLIB_NO_HDR_VER)
813 compress_len -= zlib_trl_bytes;
814 ret = check_zlib_trl(load_le_u32(compress_buf + compress_len),
815 state->crc, uncompress_buf, *uncompress_len);
816 }
817 }
818 }
819 if (ret == 0 && state->avail_in != 0)
820 ret = INFLATE_LEFTOVER_INPUT;
821
822 exit_in_multi_pass:
823 if (comp_tmp != NULL) {
824 free(comp_tmp);
825 comp_tmp = NULL;
826 }
827
828 if (uncomp_tmp != NULL) {
829 free(uncomp_tmp);
830 uncomp_tmp = NULL;
831 }
832
833 free(state);
834 return ret;
835 }
836
837 int
inflate_ret_to_code(int ret)838 inflate_ret_to_code(int ret)
839 {
840 switch (ret) {
841 case ISAL_DECOMP_OK:
842 return 0;
843 case ISAL_END_INPUT:
844 return INFLATE_END_OF_INPUT;
845 case ISAL_OUT_OVERFLOW:
846 return INFLATE_OUT_BUFFER_OVERFLOW;
847 case ISAL_INVALID_BLOCK:
848 return INFLATE_INVALID_BLOCK_HEADER;
849 case ISAL_INVALID_SYMBOL:
850 return INFLATE_INVALID_SYMBOL;
851 case ISAL_INVALID_LOOKBACK:
852 return INFLATE_INVALID_LOOK_BACK_DISTANCE;
853 default:
854 return INFLATE_GENERAL_ERROR;
855 }
856 }
857
858 /* Inflate the compressed data and check that the decompressed data agrees with the input data */
859 int
inflate_check(uint8_t * z_buf,uint32_t z_size,uint8_t * in_buf,uint32_t in_size,uint32_t gzip_flag,uint8_t * dict,uint32_t dict_len,uint32_t hist_bits)860 inflate_check(uint8_t *z_buf, uint32_t z_size, uint8_t *in_buf, uint32_t in_size,
861 uint32_t gzip_flag, uint8_t *dict, uint32_t dict_len, uint32_t hist_bits)
862 {
863 /* Test inflate with reference inflate */
864
865 int ret = 0;
866 uint32_t test_size = in_size;
867 uint8_t *test_buf = NULL;
868 int mem_result = 0;
869 int gzip_trl_result = 0;
870
871 if (in_size > 0) {
872 assert(in_buf != NULL);
873 test_buf = malloc(test_size);
874 if (test_buf == NULL)
875 return MALLOC_FAILED;
876 }
877
878 if (test_buf != NULL)
879 memset(test_buf, 0xff, test_size);
880
881 if (inflate_type == 0 && dict == NULL) {
882 ret = inflate_stateless_pass(z_buf, z_size, test_buf, &test_size, gzip_flag);
883 inflate_type = 1;
884 } else {
885 ret = inflate_multi_pass(z_buf, z_size, test_buf, &test_size, gzip_flag, dict,
886 dict_len, hist_bits);
887 inflate_type = 0;
888 }
889
890 if (test_buf != NULL)
891 mem_result = memcmp(in_buf, test_buf, in_size);
892
893 if (options.verbose && mem_result) {
894 int i;
895 for (i = 0; i < in_size; i++) {
896 if (in_buf[i] != test_buf[i]) {
897 log_print("First incorrect data at 0x%x of 0x%x, 0x%x != 0x%x\n", i,
898 in_size, in_buf[i], test_buf[i]);
899 break;
900 }
901 }
902 }
903
904 if (test_buf != NULL)
905 free(test_buf);
906 switch (ret) {
907 case 0:
908 break;
909 case ISAL_END_INPUT:
910 return INFLATE_END_OF_INPUT;
911 break;
912 case ISAL_INVALID_BLOCK:
913 return INFLATE_INVALID_BLOCK_HEADER;
914 break;
915 case ISAL_INVALID_SYMBOL:
916 return INFLATE_INVALID_SYMBOL;
917 break;
918 case ISAL_OUT_OVERFLOW:
919 return INFLATE_OUT_BUFFER_OVERFLOW;
920 break;
921 case ISAL_INVALID_LOOKBACK:
922 return INFLATE_INVALID_LOOK_BACK_DISTANCE;
923 break;
924 case INFLATE_LEFTOVER_INPUT:
925 return INFLATE_LEFTOVER_INPUT;
926 break;
927 case INCORRECT_GZIP_TRAILER:
928 gzip_trl_result = INCORRECT_GZIP_TRAILER;
929 break;
930 case INCORRECT_ZLIB_TRAILER:
931 gzip_trl_result = INCORRECT_ZLIB_TRAILER;
932 break;
933 case ISAL_INCORRECT_CHECKSUM:
934 if (gzip_flag == IGZIP_GZIP || gzip_flag == IGZIP_GZIP_NO_HDR ||
935 gzip_flag == ISAL_GZIP_NO_HDR_VER)
936 gzip_trl_result = INCORRECT_GZIP_TRAILER;
937 else if (gzip_flag == IGZIP_ZLIB || gzip_flag == IGZIP_ZLIB_NO_HDR ||
938 gzip_flag == ISAL_ZLIB_NO_HDR_VER)
939 gzip_trl_result = INCORRECT_GZIP_TRAILER;
940 break;
941 case ISAL_UNSUPPORTED_METHOD:
942 return UNSUPPORTED_METHOD;
943 case INFLATE_INPUT_STREAM_INTEGRITY_ERROR:
944 return INFLATE_INPUT_STREAM_INTEGRITY_ERROR;
945 break;
946 case INFLATE_OUTPUT_STREAM_INTEGRITY_ERROR:
947 return INFLATE_OUTPUT_STREAM_INTEGRITY_ERROR;
948 break;
949 default:
950 return INFLATE_GENERAL_ERROR;
951 break;
952 }
953
954 if (test_size != in_size)
955 return INFLATE_INCORRECT_OUTPUT_SIZE;
956
957 if (mem_result)
958 return RESULT_ERROR;
959
960 if (gzip_trl_result == INCORRECT_GZIP_TRAILER)
961 return INCORRECT_GZIP_TRAILER;
962
963 else if (gzip_trl_result == INCORRECT_ZLIB_TRAILER)
964 return INCORRECT_ZLIB_TRAILER;
965
966 return 0;
967 }
968
969 /* Check if that the state of the data stream is consistent */
970 int
stream_valid_check(struct isal_zstream * stream,uint8_t * in_buf,uint32_t in_size,uint8_t * out_buf,uint32_t out_size,uint32_t in_processed,uint32_t out_processed,uint32_t data_size)971 stream_valid_check(struct isal_zstream *stream, uint8_t *in_buf, uint32_t in_size, uint8_t *out_buf,
972 uint32_t out_size, uint32_t in_processed, uint32_t out_processed,
973 uint32_t data_size)
974 {
975 uint32_t total_in, in_buffer_size, total_out, out_buffer_size;
976
977 total_in = (in_size == 0) ? in_processed
978 : (in_processed - in_size) + (stream->next_in - in_buf);
979 in_buffer_size = (in_size == 0) ? 0 : stream->next_in - in_buf + stream->avail_in;
980
981 /* Check for a consistent amount of data processed */
982 if (total_in != stream->total_in || in_buffer_size != in_size)
983 return COMPRESS_INPUT_STREAM_INTEGRITY_ERROR;
984
985 total_out = (out_size == 0) ? out_processed : out_processed + (stream->next_out - out_buf);
986 out_buffer_size = (out_size == 0) ? 0 : stream->next_out - out_buf + stream->avail_out;
987
988 /* Check for a consistent amount of data compressed */
989 if (total_out != stream->total_out || out_buffer_size != out_size) {
990 return COMPRESS_OUTPUT_STREAM_INTEGRITY_ERROR;
991 }
992
993 return 0;
994 }
995
996 /* Performs compression with checks to discover and verify the state of the
997 * stream
998 * stream: compress data structure which has been initialized to use
999 * in_buf and out_buf as the buffers
1000 * data_size: size of all input data
1001 * compressed_size: size of all available output buffers
1002 * in_buf: next buffer of data to be compressed
1003 * in_size: size of in_buf
1004 * out_buf: next out put buffer where data is stored
1005 * out_size: size of out_buf
1006 * in_processed: the amount of input data which has been loaded into buffers
1007 * to be compressed, this includes the data in in_buf
1008 * out_processed: the amount of output data which has been compressed and stored,
1009 * this does not include the data in the current out_buf
1010 */
1011 int
isal_deflate_with_checks(struct isal_zstream * stream,uint32_t data_size,uint32_t compressed_size,uint8_t * in_buf,uint32_t in_size,uint32_t in_processed,uint8_t * out_buf,uint32_t out_size,uint32_t out_processed)1012 isal_deflate_with_checks(struct isal_zstream *stream, uint32_t data_size, uint32_t compressed_size,
1013 uint8_t *in_buf, uint32_t in_size, uint32_t in_processed, uint8_t *out_buf,
1014 uint32_t out_size, uint32_t out_processed)
1015 {
1016 int ret, stream_check;
1017 struct isal_zstate *state = &stream->internal_state;
1018
1019 log_print("Pre compression\n");
1020 log_print("data_size = 0x%05x, in_processed = 0x%05x, in_size = 0x%05x, avail_in "
1021 "= 0x%05x, total_in = 0x%05x\n",
1022 data_size, in_processed, in_size, stream->avail_in, stream->total_in);
1023 log_print("compressed_size = 0x%05x, out_processed = 0x%05x, out_size = 0x%05x, avail_out "
1024 "= 0x%05x, total_out = 0x%05x\n",
1025 compressed_size, out_processed, out_size, stream->avail_out, stream->total_out);
1026
1027 ret = isal_deflate(stream);
1028
1029 log_print("Post compression\n");
1030 log_print("data_size = 0x%05x, in_processed = 0x%05x, in_size = 0x%05x, avail_in "
1031 "= 0x%05x, total_in = 0x%05x\n",
1032 data_size, in_processed, in_size, stream->avail_in, stream->total_in);
1033 log_print("compressed_size = 0x%05x, out_processed = 0x%05x, out_size = 0x%05x, avail_out "
1034 "= 0x%05x, total_out = 0x%05x\n",
1035 compressed_size, out_processed, out_size, stream->avail_out, stream->total_out);
1036 log_print("\n\n");
1037
1038 /* Verify the stream is in a valid state */
1039 stream_check = stream_valid_check(stream, in_buf, in_size, out_buf, out_size, in_processed,
1040 out_processed, data_size);
1041
1042 if (stream_check != 0)
1043 return stream_check;
1044
1045 if (ret != IGZIP_COMP_OK)
1046 return COMPRESS_GENERAL_ERROR;
1047
1048 /* Check if the compression is completed */
1049 if (state->state != ZSTATE_END)
1050 if (compressed_size - out_processed - (out_size - stream->avail_out) <= 0)
1051 return COMPRESS_OUT_BUFFER_OVERFLOW;
1052
1053 return ret;
1054 }
1055
1056 void
set_random_hufftable(struct isal_zstream * stream,int level,uint8_t * data,uint32_t data_size)1057 set_random_hufftable(struct isal_zstream *stream, int level, uint8_t *data, uint32_t data_size)
1058 {
1059 struct isal_hufftables *huff = hufftables;
1060 struct isal_huff_histogram hist;
1061 if (level == 0 || rand() % 16 == 0) {
1062 if (rand() % 8 == 0) {
1063 huff = hufftables_subset;
1064 memset(&hist, 0, sizeof(hist));
1065 isal_update_histogram(data, data_size, &hist);
1066 isal_create_hufftables_subset(huff, &hist);
1067 }
1068
1069 isal_deflate_set_hufftables(stream, huff, rand() % 4);
1070 }
1071 }
1072
1073 /* Compress the input data into the output buffer where the input buffer and
1074 * output buffer are randomly segmented to test state information for the
1075 * compression*/
1076 int
compress_multi_pass(uint8_t * data,uint32_t data_size,uint8_t * compressed_buf,uint32_t * compressed_size,uint32_t flush_type,uint32_t gzip_flag,uint32_t level,uint8_t * dict,uint32_t dict_len,uint32_t hist_bits)1077 compress_multi_pass(uint8_t *data, uint32_t data_size, uint8_t *compressed_buf,
1078 uint32_t *compressed_size, uint32_t flush_type, uint32_t gzip_flag,
1079 uint32_t level, uint8_t *dict, uint32_t dict_len, uint32_t hist_bits)
1080 {
1081 int ret = IGZIP_COMP_OK;
1082 uint8_t *in_buf = NULL, *out_buf = NULL;
1083 uint32_t in_size = 0, out_size = 0;
1084 uint32_t in_processed = 0, out_processed = 0;
1085 struct isal_zstream *stream;
1086 struct isal_zstate *state;
1087 uint32_t loop_count = 0;
1088 uint32_t level_buf_size;
1089 uint8_t *level_buf = NULL;
1090 struct isal_hufftables *huff_tmp;
1091 uint32_t reset_test_flag = 0;
1092 uint8_t tmp_symbol;
1093 int no_mod = 0;
1094 struct isal_dict dict_str;
1095
1096 log_print("Starting Compress Multi Pass\n");
1097
1098 stream = malloc(sizeof(*stream));
1099 if (stream == NULL)
1100 return MALLOC_FAILED;
1101 state = &stream->internal_state;
1102
1103 create_rand_repeat_data((uint8_t *) stream, sizeof(*stream));
1104
1105 isal_deflate_init(stream);
1106
1107 if (state->state != ZSTATE_NEW_HDR) {
1108 ret = COMPRESS_INCORRECT_STATE;
1109 goto exit_comp_multi_pass;
1110 }
1111
1112 if (rand() % 4 == 0) {
1113 /* Test reset */
1114 reset_test_flag = 1;
1115 huff_tmp = stream->hufftables;
1116 create_rand_repeat_data((uint8_t *) stream, sizeof(*stream));
1117
1118 /* Restore variables not necessarily set by user */
1119 stream->hufftables = huff_tmp;
1120 stream->end_of_stream = 0;
1121 stream->level = 0;
1122 stream->level_buf = NULL;
1123 stream->level_buf_size = 0;
1124 }
1125
1126 stream->flush = flush_type;
1127 stream->end_of_stream = 0;
1128
1129 /* These are set here to allow the loop to run correctly */
1130 stream->avail_in = 0;
1131 stream->avail_out = 0;
1132 stream->gzip_flag = gzip_flag;
1133 stream->level = level;
1134 stream->hist_bits = hist_bits;
1135
1136 if (level >= 1) {
1137 level_buf_size = get_rand_level_buf_size(stream->level);
1138 level_buf = malloc(level_buf_size);
1139 if (level_buf == NULL) {
1140 free(stream);
1141 return MALLOC_FAILED;
1142 }
1143 create_rand_repeat_data(level_buf, level_buf_size);
1144 stream->level_buf = level_buf;
1145 stream->level_buf_size = level_buf_size;
1146 }
1147
1148 if (reset_test_flag)
1149 isal_deflate_reset(stream);
1150
1151 if (dict != NULL) {
1152 if (rand() % 2 == 0)
1153 isal_deflate_set_dict(stream, dict, dict_len);
1154 else {
1155 memset(&dict_str, 0, sizeof(dict_str));
1156 isal_deflate_process_dict(stream, &dict_str, dict, dict_len);
1157 isal_deflate_reset_dict(stream, &dict_str);
1158 }
1159 }
1160
1161 while (1) {
1162 loop_count++;
1163
1164 /* Setup in buffer for next round of compression */
1165 if (stream->avail_in == 0) {
1166 if (flush_type == NO_FLUSH || state->state == ZSTATE_NEW_HDR) {
1167 /* Randomly choose size of the next out buffer */
1168 in_size = rand() % (data_size + 1);
1169
1170 /* Limit size of buffer to be smaller than maximum */
1171 if (in_size >= data_size - in_processed) {
1172 in_size = data_size - in_processed;
1173 stream->end_of_stream = 1;
1174 }
1175
1176 if (in_size != 0) {
1177 if (in_buf != NULL)
1178 free(in_buf);
1179
1180 in_buf = malloc(in_size);
1181 if (in_buf == NULL) {
1182 ret = MALLOC_FAILED;
1183 goto exit_comp_multi_pass;
1184 }
1185 memcpy(in_buf, data + in_processed, in_size);
1186 in_processed += in_size;
1187
1188 stream->avail_in = in_size;
1189 stream->next_in = in_buf;
1190 }
1191 }
1192 } else {
1193 /* Randomly modify data after next in */
1194 if (rand() % 4 == 0 && !no_mod) {
1195
1196 tmp_symbol = rand();
1197 log_print("Modifying data at index 0x%x from 0x%x to 0x%x before "
1198 "recalling isal_deflate\n",
1199 in_processed - stream->avail_in,
1200 data[in_processed - stream->avail_in], tmp_symbol);
1201 *stream->next_in = tmp_symbol;
1202 data[in_processed - stream->avail_in] = tmp_symbol;
1203 }
1204 }
1205
1206 /* Setup out buffer for next round of compression */
1207 if (stream->avail_out == 0) {
1208 /* Save compressed data into compressed_buf */
1209 if (out_buf != NULL) {
1210 memcpy(compressed_buf + out_processed, out_buf,
1211 out_size - stream->avail_out);
1212 out_processed += out_size - stream->avail_out;
1213 }
1214
1215 /* Randomly choose size of the next out buffer */
1216 out_size = rand() % (*compressed_size + 1);
1217
1218 /* Limit size of buffer to be smaller than maximum */
1219 if (out_size > *compressed_size - out_processed)
1220 out_size = *compressed_size - out_processed;
1221
1222 if (out_size != 0) {
1223 if (out_buf != NULL)
1224 free(out_buf);
1225
1226 out_buf = malloc(out_size);
1227 if (out_buf == NULL) {
1228 ret = MALLOC_FAILED;
1229 goto exit_comp_multi_pass;
1230 }
1231
1232 stream->avail_out = out_size;
1233 stream->next_out = out_buf;
1234 }
1235 }
1236
1237 if (state->state == ZSTATE_NEW_HDR) {
1238 set_random_hufftable(stream, level, data, data_size);
1239 if (stream->hufftables == hufftables_subset)
1240 no_mod = 1;
1241 else
1242 no_mod = 0;
1243 }
1244
1245 ret = isal_deflate_with_checks(stream, data_size, *compressed_size, in_buf, in_size,
1246 in_processed, out_buf, out_size, out_processed);
1247
1248 if (ret) {
1249 if (ret == COMPRESS_OUT_BUFFER_OVERFLOW || ret == COMPRESS_INCORRECT_STATE)
1250 memcpy(compressed_buf + out_processed, out_buf, out_size);
1251 break;
1252 }
1253
1254 /* Check if the compression is completed */
1255 if (state->state == ZSTATE_END) {
1256 memcpy(compressed_buf + out_processed, out_buf, out_size);
1257 *compressed_size = stream->total_out;
1258 break;
1259 }
1260 }
1261
1262 exit_comp_multi_pass:
1263 if (stream != NULL)
1264 free(stream);
1265 if (level_buf != NULL)
1266 free(level_buf);
1267 if (in_buf != NULL)
1268 free(in_buf);
1269 if (out_buf != NULL)
1270 free(out_buf);
1271
1272 if (ret == COMPRESS_OUT_BUFFER_OVERFLOW && flush_type == SYNC_FLUSH &&
1273 loop_count >= MAX_LOOPS)
1274 ret = COMPRESS_LOOP_COUNT_OVERFLOW;
1275
1276 return ret;
1277 }
1278
1279 /* Compress the input data into the outbuffer in one call to isal_deflate */
1280 int
compress_single_pass(uint8_t * data,uint32_t data_size,uint8_t * compressed_buf,uint32_t * compressed_size,uint32_t flush_type,uint32_t gzip_flag,uint32_t level,uint8_t * dict,uint32_t dict_len,uint32_t hist_bits)1281 compress_single_pass(uint8_t *data, uint32_t data_size, uint8_t *compressed_buf,
1282 uint32_t *compressed_size, uint32_t flush_type, uint32_t gzip_flag,
1283 uint32_t level, uint8_t *dict, uint32_t dict_len, uint32_t hist_bits)
1284 {
1285 int ret = IGZIP_COMP_OK;
1286 struct isal_zstream stream;
1287 struct isal_zstate *state = &stream.internal_state;
1288 uint32_t level_buf_size;
1289 uint8_t *level_buf = NULL;
1290 struct isal_hufftables *huff_tmp;
1291 uint32_t reset_test_flag = 0;
1292 struct isal_dict dict_str;
1293
1294 log_print("Starting Compress Single Pass\n");
1295
1296 create_rand_repeat_data((uint8_t *) &stream, sizeof(stream));
1297
1298 isal_deflate_init(&stream);
1299
1300 set_random_hufftable(&stream, level, data, data_size);
1301
1302 if (state->state != ZSTATE_NEW_HDR)
1303 return COMPRESS_INCORRECT_STATE;
1304
1305 if (rand() % 4 == 0) {
1306 /* Test reset */
1307 reset_test_flag = 1;
1308 huff_tmp = stream.hufftables;
1309 create_rand_repeat_data((uint8_t *) &stream, sizeof(stream));
1310
1311 /* Restore variables not necessarily set by user */
1312 stream.hufftables = huff_tmp;
1313 stream.end_of_stream = 0;
1314 stream.level_buf = NULL;
1315 stream.level_buf_size = 0;
1316 }
1317
1318 stream.flush = flush_type;
1319 stream.avail_in = data_size;
1320 stream.next_in = data;
1321 stream.avail_out = *compressed_size;
1322 stream.next_out = compressed_buf;
1323 stream.end_of_stream = 1;
1324 stream.gzip_flag = gzip_flag;
1325 stream.level = level;
1326 stream.hist_bits = hist_bits;
1327
1328 if (level >= 1) {
1329 level_buf_size = get_rand_level_buf_size(stream.level);
1330 level_buf = malloc(level_buf_size);
1331 if (level_buf == NULL)
1332 return MALLOC_FAILED;
1333 create_rand_repeat_data(level_buf, level_buf_size);
1334 stream.level_buf = level_buf;
1335 stream.level_buf_size = level_buf_size;
1336 }
1337
1338 if (reset_test_flag)
1339 isal_deflate_reset(&stream);
1340
1341 if (dict != NULL) {
1342 if (rand() % 2 == 0)
1343 isal_deflate_set_dict(&stream, dict, dict_len);
1344 else {
1345 memset(&dict_str, 0, sizeof(dict_str));
1346 isal_deflate_process_dict(&stream, &dict_str, dict, dict_len);
1347 isal_deflate_reset_dict(&stream, &dict_str);
1348 }
1349 }
1350
1351 ret = isal_deflate_with_checks(&stream, data_size, *compressed_size, data, data_size,
1352 data_size, compressed_buf, *compressed_size, 0);
1353
1354 if (level_buf != NULL)
1355 free(level_buf);
1356
1357 /* Check if the compression is completed */
1358 if (state->state == ZSTATE_END)
1359 *compressed_size = stream.total_out;
1360 else if (flush_type == SYNC_FLUSH && stream.avail_out < 16)
1361 ret = COMPRESS_OUT_BUFFER_OVERFLOW;
1362
1363 return ret;
1364 }
1365
1366 /* Compress the input data repeatedly into the outbuffer
1367 * Compresses and verifies in place to decrease memory usage
1368 */
1369 int
compress_ver_rep_buf(uint8_t * data,uint32_t data_size,uint64_t data_rep_size,uint8_t * compressed_buf,uint32_t compressed_size,uint8_t * decomp_buf,uint32_t decomp_buf_size,uint32_t flush_type,uint32_t gzip_flag,uint32_t level)1370 compress_ver_rep_buf(uint8_t *data, uint32_t data_size, uint64_t data_rep_size,
1371 uint8_t *compressed_buf, uint32_t compressed_size, uint8_t *decomp_buf,
1372 uint32_t decomp_buf_size, uint32_t flush_type, uint32_t gzip_flag,
1373 uint32_t level)
1374 {
1375 int ret = IGZIP_COMP_OK;
1376 struct isal_zstream stream;
1377 struct inflate_state state;
1378 uint32_t level_buf_size;
1379 uint8_t *level_buf = NULL;
1380 uint64_t data_remaining = data_rep_size;
1381 uint64_t data_verified = 0;
1382 uint32_t index;
1383 uint32_t out_size, cmp_size;
1384 uint32_t avail_out_start;
1385
1386 log_print("Starting Compress and Verify Repeated Buffer\n");
1387
1388 create_rand_repeat_data((uint8_t *) &stream, sizeof(stream));
1389
1390 /* Setup compression stream */
1391 isal_deflate_init(&stream);
1392 stream.avail_in = 0;
1393 stream.next_in = NULL;
1394 stream.avail_out = 0;
1395 stream.next_out = NULL;
1396
1397 set_random_hufftable(&stream, level, data, data_size);
1398 stream.flush = flush_type;
1399 stream.end_of_stream = 0;
1400 stream.gzip_flag = gzip_flag;
1401 stream.level = level;
1402
1403 if (level >= 1) {
1404 level_buf_size = get_rand_level_buf_size(stream.level);
1405 level_buf = malloc(level_buf_size);
1406 if (level_buf == NULL)
1407 return MALLOC_FAILED;
1408 create_rand_repeat_data(level_buf, level_buf_size);
1409 stream.level_buf = level_buf;
1410 stream.level_buf_size = level_buf_size;
1411 }
1412
1413 /* Setup decompression stream */
1414 create_rand_repeat_data((uint8_t *) &state, sizeof(state));
1415 isal_inflate_init(&state);
1416 state.crc_flag = gzip_flag;
1417
1418 while (data_remaining || stream.avail_in) {
1419 /* Compress the input buffer */
1420 if (stream.next_out == NULL) {
1421 stream.avail_out = compressed_size;
1422 stream.next_out = compressed_buf;
1423 }
1424
1425 while (stream.avail_out > 0 && (data_remaining || stream.avail_in)) {
1426 if (stream.avail_in == 0) {
1427 stream.avail_in = data_size;
1428 if (data_size >= data_remaining) {
1429 stream.avail_in = data_remaining;
1430 stream.end_of_stream = 1;
1431 }
1432
1433 stream.next_in = data;
1434 data_remaining -= stream.avail_in;
1435 }
1436
1437 ret = isal_deflate(&stream);
1438
1439 if (ret)
1440 return COMPRESS_GENERAL_ERROR;
1441 }
1442
1443 /* Verify the compressed buffer */
1444 state.next_in = compressed_buf;
1445 state.avail_in = compressed_size;
1446 state.next_out = NULL;
1447 state.avail_out = 0;
1448 create_rand_repeat_data(decomp_buf, decomp_buf_size);
1449
1450 while (state.avail_out == 0) {
1451 state.next_out = decomp_buf;
1452 state.avail_out = decomp_buf_size;
1453
1454 /* Force decoding to stop when avail_out rolls over */
1455 if ((1ULL << 32) - state.total_out < decomp_buf_size)
1456 state.avail_out = (1ULL << 32) - state.total_out;
1457
1458 avail_out_start = state.avail_out;
1459
1460 ret = isal_inflate(&state);
1461 if (ret)
1462 return inflate_ret_to_code(ret);
1463
1464 /* Check data accuracy */
1465 index = data_verified % data_size;
1466 out_size = avail_out_start - state.avail_out;
1467 cmp_size = (out_size > data_size - index) ? data_size - index : out_size;
1468 ret |= memcmp(decomp_buf, data + index, cmp_size);
1469 out_size -= cmp_size;
1470 cmp_size = (out_size > index) ? index : out_size;
1471 ret |= memcmp(decomp_buf + data_size - index, data, cmp_size);
1472 out_size -= cmp_size;
1473 ret |= memcmp(decomp_buf, decomp_buf + data_size, out_size);
1474 if (ret)
1475 return RESULT_ERROR;
1476
1477 data_verified += avail_out_start - state.avail_out;
1478 }
1479 stream.next_out = NULL;
1480 }
1481
1482 if (level_buf != NULL)
1483 free(level_buf);
1484
1485 return ret;
1486 }
1487
1488 /* Statelessly compress the input buffer into the output buffer */
1489 int
compress_stateless(uint8_t * data,uint32_t data_size,uint8_t * compressed_buf,uint32_t * compressed_size,uint32_t flush_type,uint32_t gzip_flag,uint32_t level,uint32_t hist_bits)1490 compress_stateless(uint8_t *data, uint32_t data_size, uint8_t *compressed_buf,
1491 uint32_t *compressed_size, uint32_t flush_type, uint32_t gzip_flag,
1492 uint32_t level, uint32_t hist_bits)
1493 {
1494 int ret = IGZIP_COMP_OK;
1495 struct isal_zstream stream;
1496 uint32_t level_buf_size;
1497 uint8_t *level_buf = NULL;
1498 struct isal_hufftables *huff_tmp;
1499 uint32_t reset_test_flag = 0;
1500
1501 create_rand_repeat_data((uint8_t *) &stream, sizeof(stream));
1502
1503 isal_deflate_stateless_init(&stream);
1504
1505 set_random_hufftable(&stream, level, data, data_size);
1506
1507 if (rand() % 4 == 0) {
1508 /* Test reset */
1509 reset_test_flag = 1;
1510 huff_tmp = stream.hufftables;
1511 create_rand_repeat_data((uint8_t *) &stream, sizeof(stream));
1512
1513 /* Restore variables not necessarily set by user */
1514 stream.hufftables = huff_tmp;
1515 stream.end_of_stream = 0;
1516 stream.level_buf = NULL;
1517 stream.level_buf_size = 0;
1518 }
1519
1520 stream.avail_in = data_size;
1521 stream.next_in = data;
1522 stream.flush = flush_type;
1523 if (flush_type != NO_FLUSH)
1524 stream.end_of_stream = 1;
1525 stream.avail_out = *compressed_size;
1526 stream.next_out = compressed_buf;
1527 stream.gzip_flag = gzip_flag;
1528 stream.level = level;
1529 stream.hist_bits = hist_bits;
1530
1531 if (level == 1) {
1532 /* This is to test case where level buf uses already existing
1533 * internal buffers */
1534 level_buf_size = rand() % IBUF_SIZE;
1535
1536 if (level_buf_size >= ISAL_DEF_LVL1_MIN) {
1537 level_buf = malloc(level_buf_size);
1538 if (level_buf == NULL)
1539 return MALLOC_FAILED;
1540 create_rand_repeat_data(level_buf, level_buf_size);
1541 stream.level_buf = level_buf;
1542 stream.level_buf_size = level_buf_size;
1543 }
1544 } else if (level > 1) {
1545 level_buf_size = get_rand_level_buf_size(level);
1546 level_buf = malloc(level_buf_size);
1547 if (level_buf == NULL)
1548 return MALLOC_FAILED;
1549 create_rand_repeat_data(level_buf, level_buf_size);
1550 stream.level_buf = level_buf;
1551 stream.level_buf_size = level_buf_size;
1552 }
1553
1554 if (reset_test_flag)
1555 isal_deflate_reset(&stream);
1556
1557 ret = isal_deflate_stateless(&stream);
1558
1559 if (level_buf != NULL)
1560 free(level_buf);
1561
1562 /* verify the stream */
1563 if (stream.next_in - data != stream.total_in ||
1564 stream.total_in + stream.avail_in != data_size)
1565 return COMPRESS_INPUT_STREAM_INTEGRITY_ERROR;
1566
1567 if (stream.next_out - compressed_buf != stream.total_out ||
1568 stream.total_out + stream.avail_out != *compressed_size) {
1569 return COMPRESS_OUTPUT_STREAM_INTEGRITY_ERROR;
1570 }
1571
1572 if (ret != IGZIP_COMP_OK) {
1573 if (ret == STATELESS_OVERFLOW)
1574 return COMPRESS_OUT_BUFFER_OVERFLOW;
1575 else if (ret == INVALID_FLUSH)
1576 return INVALID_FLUSH_ERROR;
1577 else {
1578 printf("Return due to ret = %d with level = %d or %d\n", ret, level,
1579 stream.level);
1580 return COMPRESS_GENERAL_ERROR;
1581 }
1582 }
1583
1584 if (!stream.end_of_stream) {
1585 return COMPRESS_END_OF_STREAM_NOT_SET;
1586 }
1587
1588 if (stream.avail_in != 0)
1589 return COMPRESS_ALL_INPUT_FAIL;
1590
1591 *compressed_size = stream.total_out;
1592
1593 return ret;
1594 }
1595
1596 /* Statelessly compress the input buffer into the output buffer */
1597 int
compress_stateless_full_flush(uint8_t * data,uint32_t data_size,uint8_t * compressed_buf,uint32_t * compressed_size,uint32_t level,uint32_t hist_bits)1598 compress_stateless_full_flush(uint8_t *data, uint32_t data_size, uint8_t *compressed_buf,
1599 uint32_t *compressed_size, uint32_t level, uint32_t hist_bits)
1600 {
1601 int ret = IGZIP_COMP_OK;
1602 uint8_t *in_buf = NULL, *level_buf = NULL, *out_buf = compressed_buf;
1603 uint32_t in_size = 0, level_buf_size;
1604 uint32_t in_processed = 00;
1605 struct isal_zstream stream;
1606 uint32_t loop_count = 0;
1607 struct isal_hufftables *huff_tmp;
1608 uint32_t reset_test_flag = 0;
1609
1610 log_print("Starting Stateless Compress Full Flush\n");
1611
1612 create_rand_repeat_data((uint8_t *) &stream, sizeof(stream));
1613
1614 isal_deflate_stateless_init(&stream);
1615
1616 if (rand() % 4 == 0) {
1617 /* Test reset */
1618 reset_test_flag = 1;
1619 huff_tmp = stream.hufftables;
1620 create_rand_repeat_data((uint8_t *) &stream, sizeof(stream));
1621
1622 /* Restore variables not necessarily set by user */
1623 stream.hufftables = huff_tmp;
1624 stream.end_of_stream = 0;
1625 stream.level_buf = NULL;
1626 stream.level_buf_size = 0;
1627 stream.gzip_flag = 0;
1628 }
1629
1630 stream.flush = FULL_FLUSH;
1631 stream.end_of_stream = 0;
1632 stream.avail_out = *compressed_size;
1633 stream.next_out = compressed_buf;
1634 stream.level = level;
1635 stream.hist_bits = hist_bits;
1636
1637 if (level == 1) {
1638 /* This is to test case where level_buf uses already existing
1639 * internal buffers */
1640 level_buf_size = rand() % IBUF_SIZE;
1641
1642 if (level_buf_size >= ISAL_DEF_LVL1_MIN) {
1643 level_buf = malloc(level_buf_size);
1644 if (level_buf == NULL)
1645 return MALLOC_FAILED;
1646 create_rand_repeat_data(level_buf, level_buf_size);
1647 stream.level_buf = level_buf;
1648 stream.level_buf_size = level_buf_size;
1649 }
1650 } else if (level > 1) {
1651 level_buf_size = get_rand_level_buf_size(level);
1652 level_buf = malloc(level_buf_size);
1653 if (level_buf == NULL)
1654 return MALLOC_FAILED;
1655 create_rand_repeat_data(level_buf, level_buf_size);
1656 stream.level_buf = level_buf;
1657 stream.level_buf_size = level_buf_size;
1658 }
1659
1660 if (reset_test_flag)
1661 isal_deflate_reset(&stream);
1662
1663 while (1) {
1664 loop_count++;
1665
1666 /* Randomly choose size of the next out buffer */
1667 in_size = rand() % (data_size + 1);
1668
1669 /* Limit size of buffer to be smaller than maximum */
1670 if (in_size >= data_size - in_processed) {
1671 in_size = data_size - in_processed;
1672 stream.end_of_stream = 1;
1673 }
1674
1675 stream.avail_in = in_size;
1676
1677 if (in_size != 0) {
1678 if (in_buf != NULL)
1679 free(in_buf);
1680
1681 in_buf = malloc(in_size);
1682 if (in_buf == NULL) {
1683 ret = MALLOC_FAILED;
1684 break;
1685 }
1686 memcpy(in_buf, data + in_processed, in_size);
1687 in_processed += in_size;
1688
1689 stream.next_in = in_buf;
1690 }
1691
1692 out_buf = stream.next_out;
1693
1694 if (stream.internal_state.state == ZSTATE_NEW_HDR)
1695 set_random_hufftable(&stream, level, data, data_size);
1696
1697 ret = isal_deflate_stateless(&stream);
1698
1699 assert(stream.internal_state.bitbuf.m_bit_count == 0);
1700
1701 assert(compressed_buf == stream.next_out - stream.total_out);
1702 if (ret)
1703 break;
1704
1705 /* Verify that blocks are independent */
1706 ret = inflate_check(out_buf, stream.next_out - out_buf, in_buf, in_size, 0, NULL, 0,
1707 hist_bits);
1708
1709 if (ret == INFLATE_INVALID_LOOK_BACK_DISTANCE) {
1710 break;
1711 } else
1712 ret = 0;
1713
1714 /* Check if the compression is completed */
1715 if (in_processed == data_size) {
1716 *compressed_size = stream.total_out;
1717 break;
1718 }
1719 }
1720
1721 if (level_buf != NULL)
1722 free(level_buf);
1723
1724 if (in_buf != NULL)
1725 free(in_buf);
1726
1727 if (ret == STATELESS_OVERFLOW && loop_count >= MAX_LOOPS)
1728 ret = COMPRESS_LOOP_COUNT_OVERFLOW;
1729
1730 return ret;
1731 }
1732
1733 /* Compress the input data into the output buffer where the input buffer and
1734 * is randomly segmented to test for independence of blocks in full flush
1735 * compression*/
1736 int
compress_full_flush(uint8_t * data,uint32_t data_size,uint8_t * compressed_buf,uint32_t * compressed_size,uint32_t gzip_flag,uint32_t level)1737 compress_full_flush(uint8_t *data, uint32_t data_size, uint8_t *compressed_buf,
1738 uint32_t *compressed_size, uint32_t gzip_flag, uint32_t level)
1739 {
1740 int ret = IGZIP_COMP_OK;
1741 uint8_t *in_buf = NULL, *out_buf = compressed_buf, *level_buf = NULL;
1742 uint32_t in_size = 0, level_buf_size;
1743 uint32_t in_processed = 00;
1744 struct isal_zstream stream;
1745 struct isal_zstate *state = &stream.internal_state;
1746 uint32_t loop_count = 0;
1747 struct isal_hufftables *huff_tmp;
1748 uint32_t reset_test_flag = 0;
1749
1750 log_print("Starting Compress Full Flush\n");
1751
1752 create_rand_repeat_data((uint8_t *) &stream, sizeof(stream));
1753
1754 isal_deflate_init(&stream);
1755
1756 if (state->state != ZSTATE_NEW_HDR)
1757 return COMPRESS_INCORRECT_STATE;
1758
1759 if (rand() % 4 == 0) {
1760 /* Test reset */
1761 reset_test_flag = 1;
1762 huff_tmp = stream.hufftables;
1763 create_rand_repeat_data((uint8_t *) &stream, sizeof(stream));
1764
1765 /* Restore variables not necessarily set by user */
1766 stream.hufftables = huff_tmp;
1767 stream.end_of_stream = 0;
1768 stream.level_buf = NULL;
1769 stream.level_buf_size = 0;
1770 stream.hist_bits = 0;
1771 }
1772
1773 stream.flush = FULL_FLUSH;
1774 stream.end_of_stream = 0;
1775 stream.avail_out = *compressed_size;
1776 stream.next_out = compressed_buf;
1777 stream.total_out = 0;
1778 stream.gzip_flag = gzip_flag;
1779 stream.level = level;
1780
1781 if (level >= 1) {
1782 level_buf_size = get_rand_level_buf_size(stream.level);
1783 if (level_buf_size >= ISAL_DEF_LVL1_MIN) {
1784 level_buf = malloc(level_buf_size);
1785 if (level_buf == NULL)
1786 return MALLOC_FAILED;
1787 create_rand_repeat_data(level_buf, level_buf_size);
1788 stream.level_buf = level_buf;
1789 stream.level_buf_size = level_buf_size;
1790 }
1791 }
1792
1793 if (reset_test_flag)
1794 isal_deflate_reset(&stream);
1795
1796 while (1) {
1797 loop_count++;
1798
1799 /* Setup in buffer for next round of compression */
1800 if (state->state == ZSTATE_NEW_HDR) {
1801 /* Randomly choose size of the next out buffer */
1802 in_size = rand() % (data_size + 1);
1803
1804 /* Limit size of buffer to be smaller than maximum */
1805 if (in_size >= data_size - in_processed) {
1806 in_size = data_size - in_processed;
1807 stream.end_of_stream = 1;
1808 }
1809
1810 stream.avail_in = in_size;
1811
1812 if (in_size != 0) {
1813 if (in_buf != NULL)
1814 free(in_buf);
1815
1816 in_buf = malloc(in_size);
1817 if (in_buf == NULL) {
1818 ret = MALLOC_FAILED;
1819 break;
1820 }
1821 memcpy(in_buf, data + in_processed, in_size);
1822 in_processed += in_size;
1823
1824 stream.next_in = in_buf;
1825 }
1826
1827 out_buf = stream.next_out;
1828 }
1829
1830 if (state->state == ZSTATE_NEW_HDR)
1831 set_random_hufftable(&stream, level, data, data_size);
1832
1833 ret = isal_deflate(&stream);
1834
1835 if (ret)
1836 break;
1837
1838 /* Verify that blocks are independent */
1839 if (state->state == ZSTATE_NEW_HDR || state->state == ZSTATE_END) {
1840 ret = inflate_check(out_buf, stream.next_out - out_buf, in_buf, in_size, 0,
1841 NULL, 0, 0);
1842
1843 if (ret == INFLATE_INVALID_LOOK_BACK_DISTANCE)
1844 break;
1845 else
1846 ret = 0;
1847 }
1848
1849 /* Check if the compression is completed */
1850 if (state->state == ZSTATE_END) {
1851 *compressed_size = stream.total_out;
1852 break;
1853 }
1854 }
1855
1856 if (level_buf != NULL)
1857 free(level_buf);
1858
1859 if (in_buf != NULL)
1860 free(in_buf);
1861
1862 if (ret == COMPRESS_OUT_BUFFER_OVERFLOW && loop_count >= MAX_LOOPS)
1863 ret = COMPRESS_LOOP_COUNT_OVERFLOW;
1864
1865 return ret;
1866 }
1867
1868 /*Compress the input buffer into the output buffer, but switch the flush type in
1869 * the middle of the compression to test what happens*/
1870 int
compress_swap_flush(uint8_t * data,uint32_t data_size,uint8_t * compressed_buf,uint32_t * compressed_size,uint32_t flush_type,int level,uint32_t gzip_flag)1871 compress_swap_flush(uint8_t *data, uint32_t data_size, uint8_t *compressed_buf,
1872 uint32_t *compressed_size, uint32_t flush_type, int level, uint32_t gzip_flag)
1873 {
1874 int ret = IGZIP_COMP_OK;
1875 struct isal_zstream stream;
1876 struct isal_zstate *state = &stream.internal_state;
1877 uint32_t partial_size;
1878 struct isal_hufftables *huff_tmp;
1879 uint32_t reset_test_flag = 0;
1880 uint32_t level_buf_size;
1881 uint8_t *level_buf = NULL;
1882
1883 log_print("Starting Compress Swap Flush\n");
1884
1885 isal_deflate_init(&stream);
1886
1887 set_random_hufftable(&stream, 0, data, data_size);
1888
1889 if (state->state != ZSTATE_NEW_HDR)
1890 return COMPRESS_INCORRECT_STATE;
1891
1892 if (rand() % 4 == 0) {
1893 /* Test reset */
1894 reset_test_flag = 1;
1895 huff_tmp = stream.hufftables;
1896 create_rand_repeat_data((uint8_t *) &stream, sizeof(stream));
1897
1898 /* Restore variables not necessarily set by user */
1899 stream.hufftables = huff_tmp;
1900 stream.end_of_stream = 0;
1901 stream.level = 0;
1902 stream.level_buf = NULL;
1903 stream.level_buf_size = 0;
1904 }
1905
1906 partial_size = rand() % (data_size + 1);
1907
1908 stream.flush = flush_type;
1909 stream.avail_in = partial_size;
1910 stream.next_in = data;
1911 stream.avail_out = *compressed_size;
1912 stream.next_out = compressed_buf;
1913 stream.end_of_stream = 0;
1914 stream.gzip_flag = gzip_flag;
1915 if (level) {
1916 stream.level = level;
1917 level_buf_size = get_rand_level_buf_size(stream.level);
1918 level_buf = malloc(level_buf_size);
1919 if (level_buf == NULL)
1920 return MALLOC_FAILED;
1921 create_rand_repeat_data(level_buf, level_buf_size);
1922 stream.level_buf = level_buf;
1923 stream.level_buf_size = level_buf_size;
1924 }
1925
1926 if (reset_test_flag)
1927 isal_deflate_reset(&stream);
1928
1929 ret = isal_deflate_with_checks(&stream, data_size, *compressed_size, data, partial_size,
1930 partial_size, compressed_buf, *compressed_size, 0);
1931
1932 if (ret)
1933 return ret;
1934
1935 if (state->state == ZSTATE_NEW_HDR)
1936 set_random_hufftable(&stream, 0, data, data_size);
1937
1938 flush_type = rand() % 3;
1939
1940 stream.flush = flush_type;
1941 stream.avail_in = data_size - partial_size;
1942 stream.next_in = data + partial_size;
1943 stream.end_of_stream = 1;
1944
1945 ret = isal_deflate_with_checks(&stream, data_size, *compressed_size, data + partial_size,
1946 data_size - partial_size, data_size, compressed_buf,
1947 *compressed_size, 0);
1948
1949 if (ret == COMPRESS_GENERAL_ERROR)
1950 return INVALID_FLUSH_ERROR;
1951
1952 *compressed_size = stream.total_out;
1953
1954 if (stream.level_buf != NULL)
1955 free(stream.level_buf);
1956
1957 return ret;
1958 }
1959
1960 /* Test deflate_stateless */
1961 int
test_compress_stateless(uint8_t * in_data,uint32_t in_size,uint32_t flush_type)1962 test_compress_stateless(uint8_t *in_data, uint32_t in_size, uint32_t flush_type)
1963 {
1964 int ret = IGZIP_COMP_OK;
1965 uint32_t z_size, overflow, gzip_flag, level, hist_bits;
1966 uint8_t *z_buf = NULL;
1967 uint8_t *in_buf = NULL;
1968
1969 gzip_flag = rand() % 5;
1970 hist_bits = rand() % 16;
1971 level = get_rand_level();
1972
1973 if (in_size != 0) {
1974 in_buf = malloc(in_size);
1975
1976 if (in_buf == NULL)
1977 return MALLOC_FAILED;
1978
1979 memcpy(in_buf, in_data, in_size);
1980 }
1981
1982 /* Test non-overflow case where a type 0 block is not written */
1983 z_size = 2 * in_size + hdr_bytes;
1984 if (gzip_flag == IGZIP_GZIP)
1985 z_size += gzip_extra_bytes;
1986 else if (gzip_flag == IGZIP_GZIP_NO_HDR)
1987 z_size += gzip_trl_bytes;
1988 else if (gzip_flag == IGZIP_ZLIB)
1989 z_size += zlib_extra_bytes;
1990 else if (gzip_flag == IGZIP_ZLIB_NO_HDR)
1991 z_size += zlib_trl_bytes;
1992
1993 z_buf = malloc(z_size);
1994
1995 if (z_buf == NULL) {
1996 ret = MALLOC_FAILED;
1997 goto exit_comp_stateless;
1998 }
1999
2000 create_rand_repeat_data(z_buf, z_size);
2001
2002 /* If flush type is invalid */
2003 if (flush_type != NO_FLUSH && flush_type != FULL_FLUSH) {
2004 ret = compress_stateless(in_buf, in_size, z_buf, &z_size, flush_type, gzip_flag,
2005 level, hist_bits);
2006
2007 if (ret != INVALID_FLUSH_ERROR)
2008 print_error(ret);
2009 else
2010 ret = 0;
2011
2012 if (z_buf != NULL)
2013 free(z_buf);
2014
2015 if (in_buf != NULL)
2016 free(in_buf);
2017
2018 return ret;
2019 }
2020
2021 /* Else test valid flush type */
2022 ret = compress_stateless(in_buf, in_size, z_buf, &z_size, flush_type, gzip_flag, level,
2023 hist_bits);
2024
2025 if (!ret)
2026 ret = inflate_check(z_buf, z_size, in_buf, in_size, gzip_flag, NULL, 0, hist_bits);
2027
2028 if (options.verbose && ret) {
2029 log_print("Compressed array at level %d with gzip flag %d, flush type %d, and "
2030 "window bits %d: ",
2031 level, gzip_flag, flush_type, hist_bits);
2032 log_uint8_t(z_buf, z_size);
2033 log_print("\n");
2034 log_print("Data: ");
2035 log_uint8_t(in_buf, in_size);
2036 }
2037
2038 if (z_buf != NULL)
2039 free(z_buf);
2040
2041 print_error(ret);
2042 if (ret)
2043 return ret;
2044
2045 /*Test non-overflow case where a type 0 block is possible to be written */
2046 z_size = TYPE0_HDR_SIZE * ((in_size + TYPE0_MAX_SIZE - 1) / TYPE0_MAX_SIZE) + in_size;
2047
2048 if (gzip_flag == IGZIP_GZIP)
2049 z_size += gzip_extra_bytes;
2050 else if (gzip_flag == IGZIP_GZIP_NO_HDR)
2051 z_size += gzip_trl_bytes;
2052 else if (gzip_flag == IGZIP_ZLIB)
2053 z_size += zlib_extra_bytes;
2054 else if (gzip_flag == IGZIP_ZLIB_NO_HDR)
2055 z_size += zlib_trl_bytes;
2056
2057 if (z_size <= gzip_extra_bytes)
2058 z_size += TYPE0_HDR_SIZE;
2059
2060 if (z_size < 8)
2061 z_size = 8;
2062
2063 z_buf = malloc(z_size);
2064
2065 if (z_buf == NULL) {
2066 ret = MALLOC_FAILED;
2067 goto exit_comp_stateless;
2068 }
2069
2070 create_rand_repeat_data(z_buf, z_size);
2071
2072 ret = compress_stateless(in_buf, in_size, z_buf, &z_size, flush_type, gzip_flag, level,
2073 hist_bits);
2074 if (!ret)
2075 ret = inflate_check(z_buf, z_size, in_buf, in_size, gzip_flag, NULL, 0, hist_bits);
2076 if (ret) {
2077 log_print("Compressed array at level %d with gzip flag %d, flush type %d, and "
2078 "hist_bits %d: ",
2079 level, gzip_flag, flush_type, hist_bits);
2080 log_uint8_t(z_buf, z_size);
2081 log_print("\n");
2082 log_print("Data: ");
2083 log_uint8_t(in_buf, in_size);
2084 }
2085
2086 if (!ret) {
2087 free(z_buf);
2088 z_buf = NULL;
2089
2090 /* Test random overflow case */
2091 z_size = rand() % z_size;
2092
2093 if (z_size > in_size)
2094 z_size = rand() & in_size;
2095
2096 if (z_size > 0) {
2097 z_buf = malloc(z_size);
2098
2099 if (z_buf == NULL) {
2100 ret = MALLOC_FAILED;
2101 goto exit_comp_stateless;
2102 }
2103 }
2104
2105 overflow = compress_stateless(in_buf, in_size, z_buf, &z_size, flush_type,
2106 gzip_flag, level, hist_bits);
2107
2108 if (overflow != COMPRESS_OUT_BUFFER_OVERFLOW) {
2109 if (overflow == 0)
2110 ret = inflate_check(z_buf, z_size, in_buf, in_size, gzip_flag, NULL,
2111 0, hist_bits);
2112
2113 if (overflow != 0 || ret != 0) {
2114 log_print("overflow error = %d\n", overflow);
2115 log_error(overflow);
2116 log_print("inflate ret = %d\n", ret);
2117 log_error(ret);
2118
2119 log_print("Compressed array at level %d with gzip flag %d, flush "
2120 "type %d, and hist_bits %d: ",
2121 level, gzip_flag, flush_type, hist_bits);
2122
2123 log_uint8_t(z_buf, z_size);
2124 log_print("\n");
2125 log_print("Data: ");
2126 log_uint8_t(in_buf, in_size);
2127
2128 printf("Failed on compress single pass overflow\n");
2129 print_error(ret);
2130 ret = OVERFLOW_TEST_ERROR;
2131 }
2132 }
2133 }
2134
2135 print_error(ret);
2136 if (ret) {
2137 if (z_buf != NULL) {
2138 free(z_buf);
2139 z_buf = NULL;
2140 }
2141 if (in_buf != NULL)
2142 free(in_buf);
2143 return ret;
2144 }
2145
2146 if (flush_type == FULL_FLUSH) {
2147 if (z_buf != NULL)
2148 free(z_buf);
2149
2150 z_size = 2 * in_size + MAX_LOOPS * (hdr_bytes + 5);
2151
2152 z_buf = malloc(z_size);
2153
2154 if (z_buf == NULL) {
2155 ret = MALLOC_FAILED;
2156 goto exit_comp_stateless;
2157 }
2158
2159 create_rand_repeat_data(z_buf, z_size);
2160
2161 /* Else test valid flush type */
2162 ret = compress_stateless_full_flush(in_buf, in_size, z_buf, &z_size, level,
2163 hist_bits);
2164
2165 if (!ret)
2166 ret = inflate_check(z_buf, z_size, in_buf, in_size, 0, NULL, 0, hist_bits);
2167 else if (ret == COMPRESS_LOOP_COUNT_OVERFLOW)
2168 ret = 0;
2169
2170 print_error(ret);
2171
2172 if (ret) {
2173 log_print("Compressed array at level %d with gzip flag %d, flush type %d, "
2174 "and hist_bits %d: ",
2175 level, gzip_flag, FULL_FLUSH, hist_bits);
2176 log_uint8_t(z_buf, z_size);
2177 log_print("\n");
2178 log_print("Data: ");
2179 log_uint8_t(in_buf, in_size);
2180 }
2181 }
2182
2183 exit_comp_stateless:
2184 if (z_buf != NULL)
2185 free(z_buf);
2186
2187 if (in_buf != NULL)
2188 free(in_buf);
2189
2190 return ret;
2191 }
2192
2193 /* Test deflate */
2194 int
test_compress(uint8_t * in_buf,uint32_t in_size,uint32_t flush_type)2195 test_compress(uint8_t *in_buf, uint32_t in_size, uint32_t flush_type)
2196 {
2197 int ret = IGZIP_COMP_OK, fin_ret = IGZIP_COMP_OK;
2198 uint32_t overflow = 0, gzip_flag, level, hist_bits;
2199 uint32_t z_size = 0, z_size_max = 0, z_compressed_size, dict_len = 0;
2200 uint8_t *z_buf = NULL, *dict = NULL;
2201
2202 /* Test a non overflow case */
2203 if (flush_type == NO_FLUSH)
2204 z_size_max = 2 * in_size + hdr_bytes + 2;
2205 else if (flush_type == SYNC_FLUSH || flush_type == FULL_FLUSH)
2206 z_size_max = 2 * in_size + MAX_LOOPS * (hdr_bytes + 5);
2207 else {
2208 printf("Invalid Flush Parameter\n");
2209 return COMPRESS_GENERAL_ERROR;
2210 }
2211
2212 gzip_flag = rand() % 5;
2213 hist_bits = rand() % 16;
2214 level = get_rand_level();
2215
2216 z_size = z_size_max;
2217
2218 if (gzip_flag == IGZIP_GZIP)
2219 z_size += gzip_extra_bytes;
2220 else if (gzip_flag == IGZIP_GZIP_NO_HDR)
2221 z_size += gzip_trl_bytes;
2222 else if (gzip_flag == IGZIP_ZLIB)
2223 z_size += zlib_extra_bytes;
2224 else if (gzip_flag == IGZIP_ZLIB_NO_HDR)
2225 z_size += zlib_trl_bytes;
2226
2227 z_buf = malloc(z_size);
2228 if (z_buf == NULL) {
2229 print_error(MALLOC_FAILED);
2230 return MALLOC_FAILED;
2231 }
2232 create_rand_repeat_data(z_buf, z_size);
2233
2234 if (rand() % 8 == 0) {
2235 dict_len = (rand() % IGZIP_HIST_SIZE) + 1;
2236 dict = malloc(dict_len);
2237 if (dict == NULL) {
2238 print_error(MALLOC_FAILED);
2239 fin_ret = MALLOC_FAILED;
2240 goto test_compress_cleanup;
2241 }
2242 create_rand_dict(dict, dict_len, z_buf, z_size);
2243 }
2244
2245 ret = compress_single_pass(in_buf, in_size, z_buf, &z_size, flush_type, gzip_flag, level,
2246 dict, dict_len, hist_bits);
2247
2248 if (!ret)
2249 ret = inflate_check(z_buf, z_size, in_buf, in_size, gzip_flag, dict, dict_len,
2250 hist_bits);
2251
2252 if (ret) {
2253 log_print("Compressed array at level %d with gzip flag %d, flush type %d, and "
2254 "hist_bits %d: ",
2255 level, gzip_flag, flush_type, hist_bits);
2256 log_uint8_t(z_buf, z_size);
2257 log_print("\n");
2258 if (dict != NULL) {
2259 log_print("Using Dictionary: ");
2260 log_uint8_t(dict, dict_len);
2261 log_print("\n");
2262 }
2263 log_print("Data: ");
2264 log_uint8_t(in_buf, in_size);
2265
2266 printf("Failed on compress single pass\n");
2267 print_error(ret);
2268 }
2269
2270 if (dict != NULL) {
2271 free(dict);
2272 dict = NULL;
2273 dict_len = 0;
2274 }
2275
2276 fin_ret |= ret;
2277 if (ret)
2278 goto test_compress_cleanup;
2279
2280 z_compressed_size = z_size;
2281 z_size = z_size_max;
2282 create_rand_repeat_data(z_buf, z_size_max);
2283
2284 if (rand() % 8 == 0) {
2285 dict_len = (rand() % IGZIP_HIST_SIZE) + 1;
2286 dict = malloc(dict_len);
2287 if (dict == NULL) {
2288 print_error(MALLOC_FAILED);
2289 return MALLOC_FAILED;
2290 }
2291 create_rand_dict(dict, dict_len, z_buf, z_size);
2292 }
2293
2294 ret = compress_multi_pass(in_buf, in_size, z_buf, &z_size, flush_type, gzip_flag, level,
2295 dict, dict_len, hist_bits);
2296
2297 if (!ret)
2298 ret = inflate_check(z_buf, z_size, in_buf, in_size, gzip_flag, dict, dict_len,
2299 hist_bits);
2300
2301 if (ret) {
2302 log_print("Compressed array at level %d with gzip flag %d, flush type %d and "
2303 "hist_bits %d: ",
2304 level, gzip_flag, flush_type, hist_bits);
2305 log_uint8_t(z_buf, z_size);
2306 log_print("\n");
2307 if (dict != NULL) {
2308 log_print("Using Dictionary: ");
2309 log_uint8_t(dict, dict_len);
2310 log_print("\n");
2311 }
2312 log_print("Data: ");
2313 log_uint8_t(in_buf, in_size);
2314
2315 printf("Failed on compress multi pass\n");
2316 print_error(ret);
2317 }
2318
2319 if (dict != NULL) {
2320 free(dict);
2321 dict = NULL;
2322 dict_len = 0;
2323 }
2324
2325 fin_ret |= ret;
2326 if (ret)
2327 goto test_compress_cleanup;
2328
2329 ret = 0;
2330
2331 /* Test random overflow case */
2332 if (flush_type == SYNC_FLUSH && z_compressed_size > in_size)
2333 z_compressed_size = in_size + 1;
2334
2335 z_size = rand() % z_compressed_size;
2336 create_rand_repeat_data(z_buf, z_size);
2337
2338 overflow = compress_single_pass(in_buf, in_size, z_buf, &z_size, flush_type, gzip_flag,
2339 level, dict, dict_len, hist_bits);
2340
2341 if (overflow != COMPRESS_OUT_BUFFER_OVERFLOW) {
2342 if (overflow == 0)
2343 ret = inflate_check(z_buf, z_size, in_buf, in_size, gzip_flag, dict,
2344 dict_len, hist_bits);
2345
2346 /* Rarely single pass overflow will compresses data
2347 * better than the initial run. This is to stop that
2348 * case from erroring. */
2349 if (overflow != 0 || ret != 0) {
2350 log_print("overflow error = %d\n", overflow);
2351 log_error(overflow);
2352 log_print("inflate ret = %d\n", ret);
2353 log_error(ret);
2354
2355 log_print("Compressed array at level %d with gzip flag %d, flush type %d, "
2356 "and hist_bits %d: ",
2357 level, gzip_flag, flush_type, hist_bits);
2358 log_uint8_t(z_buf, z_size);
2359 log_print("\n");
2360 log_print("Data: ");
2361 log_uint8_t(in_buf, in_size);
2362
2363 printf("Failed on compress single pass overflow\n");
2364 print_error(ret);
2365 ret = OVERFLOW_TEST_ERROR;
2366 }
2367 }
2368
2369 fin_ret |= ret;
2370 if (ret)
2371 goto test_compress_cleanup;
2372
2373 if (flush_type == NO_FLUSH) {
2374 create_rand_repeat_data(z_buf, z_size);
2375
2376 overflow = compress_multi_pass(in_buf, in_size, z_buf, &z_size, flush_type,
2377 gzip_flag, level, dict, dict_len, hist_bits);
2378
2379 if (overflow != COMPRESS_OUT_BUFFER_OVERFLOW) {
2380 if (overflow == 0)
2381 ret = inflate_check(z_buf, z_size, in_buf, in_size, gzip_flag, dict,
2382 dict_len, hist_bits);
2383
2384 /* Rarely multi pass overflow will compresses data
2385 * better than the initial run. This is to stop that
2386 * case from erroring */
2387 if (overflow != 0 || ret != 0) {
2388 log_print("overflow error = %d\n", overflow);
2389 log_error(overflow);
2390 log_print("inflate ret = %d\n", ret);
2391 log_error(ret);
2392 log_print("Compressed array at level %d with gzip flag %d, flush "
2393 "type %d, and hist_bits %d: ",
2394 level, gzip_flag, flush_type, hist_bits);
2395 log_uint8_t(z_buf, z_size);
2396 log_print("\n");
2397 log_print("Data: ");
2398 log_uint8_t(in_buf, in_size);
2399
2400 printf("Failed on compress multi pass overflow\n");
2401 print_error(ret);
2402 ret = OVERFLOW_TEST_ERROR;
2403 }
2404 }
2405 fin_ret |= ret;
2406 }
2407
2408 test_compress_cleanup:
2409 free(z_buf);
2410
2411 return fin_ret;
2412 }
2413
2414 /* Test swapping flush types in the middle of compression */
2415 int
test_flush(uint8_t * in_buf,uint32_t in_size)2416 test_flush(uint8_t *in_buf, uint32_t in_size)
2417 {
2418 int fin_ret = IGZIP_COMP_OK, ret;
2419 uint32_t z_size, flush_type = 0, gzip_flag, level;
2420 uint8_t *z_buf = NULL;
2421
2422 gzip_flag = rand() % 5;
2423 level = get_rand_level();
2424
2425 z_size = 2 * in_size + 2 * hdr_bytes + 8;
2426 if (gzip_flag == IGZIP_GZIP)
2427 z_size += gzip_extra_bytes;
2428 else if (gzip_flag == IGZIP_GZIP_NO_HDR)
2429 z_size += gzip_trl_bytes;
2430 else if (gzip_flag == IGZIP_ZLIB)
2431 z_size += zlib_extra_bytes;
2432 else if (gzip_flag == IGZIP_ZLIB_NO_HDR)
2433 z_size += zlib_trl_bytes;
2434
2435 z_buf = malloc(z_size);
2436
2437 if (z_buf == NULL)
2438 return MALLOC_FAILED;
2439
2440 create_rand_repeat_data(z_buf, z_size);
2441
2442 while (flush_type < 3)
2443 flush_type = rand() & 0xFFFF;
2444
2445 /* Test invalid flush */
2446 ret = compress_single_pass(in_buf, in_size, z_buf, &z_size, flush_type, gzip_flag, level,
2447 NULL, 0, 0);
2448
2449 if (ret == COMPRESS_GENERAL_ERROR)
2450 ret = 0;
2451 else {
2452 printf("Failed when passing invalid flush parameter\n");
2453 ret = INVALID_FLUSH_ERROR;
2454 }
2455
2456 fin_ret |= ret;
2457 print_error(ret);
2458
2459 create_rand_repeat_data(z_buf, z_size);
2460
2461 /* Test swapping flush type */
2462 ret = compress_swap_flush(in_buf, in_size, z_buf, &z_size, rand() % 3, level, gzip_flag);
2463
2464 if (!ret)
2465 ret = inflate_check(z_buf, z_size, in_buf, in_size, gzip_flag, NULL, 0, 0);
2466
2467 if (ret) {
2468 log_print("Compressed array at level %d with gzip flag %d: ", level, gzip_flag);
2469 log_uint8_t(z_buf, z_size);
2470 log_print("\n");
2471 log_print("Data: ");
2472 log_uint8_t(in_buf, in_size);
2473
2474 printf("Failed on swapping flush type\n");
2475 print_error(ret);
2476 }
2477
2478 fin_ret |= ret;
2479 print_error(ret);
2480
2481 free(z_buf);
2482
2483 return fin_ret;
2484 }
2485
2486 /* Test there are no length distance pairs across full flushes */
2487 int
test_full_flush(uint8_t * in_buf,uint32_t in_size)2488 test_full_flush(uint8_t *in_buf, uint32_t in_size)
2489 {
2490 int ret = IGZIP_COMP_OK;
2491 uint32_t z_size, gzip_flag, level;
2492 uint8_t *z_buf = NULL;
2493
2494 gzip_flag = rand() % 5;
2495 level = get_rand_level();
2496 z_size = 2 * in_size + MAX_LOOPS * (hdr_bytes + 5);
2497
2498 if (gzip_flag == IGZIP_GZIP)
2499 z_size += gzip_extra_bytes;
2500 else if (gzip_flag == IGZIP_GZIP_NO_HDR)
2501 z_size += gzip_trl_bytes;
2502 else if (gzip_flag == IGZIP_ZLIB)
2503 z_size += zlib_extra_bytes;
2504 else if (gzip_flag == IGZIP_ZLIB_NO_HDR)
2505 z_size += zlib_trl_bytes;
2506
2507 z_buf = malloc(z_size);
2508 if (z_buf == NULL) {
2509 print_error(MALLOC_FAILED);
2510 return MALLOC_FAILED;
2511 }
2512
2513 create_rand_repeat_data(z_buf, z_size);
2514
2515 ret = compress_full_flush(in_buf, in_size, z_buf, &z_size, gzip_flag, level);
2516
2517 if (!ret)
2518 ret = inflate_check(z_buf, z_size, in_buf, in_size, gzip_flag, NULL, 0, 0);
2519
2520 if (ret) {
2521 log_print("Compressed array at level %d with gzip flag %d and flush type %d: ",
2522 level, gzip_flag, FULL_FLUSH);
2523 log_uint8_t(z_buf, z_size);
2524 log_print("\n");
2525 log_print("Data: ");
2526 log_uint8_t(in_buf, in_size);
2527
2528 printf("Failed on compress multi pass\n");
2529 print_error(ret);
2530 }
2531
2532 free(z_buf);
2533
2534 return ret;
2535 }
2536
2537 int
test_inflate(struct vect_result * in_vector)2538 test_inflate(struct vect_result *in_vector)
2539 {
2540 int ret = IGZIP_COMP_OK;
2541 uint8_t *compress_buf = in_vector->vector, *out_buf = NULL;
2542 uint64_t compress_len = in_vector->vector_length;
2543 uint32_t out_size = 0;
2544
2545 out_size = 10 * in_vector->vector_length;
2546 out_buf = malloc(out_size);
2547 if (out_buf == NULL)
2548 return MALLOC_FAILED;
2549
2550 ret = inflate_stateless_pass(compress_buf, compress_len, out_buf, &out_size, 0);
2551
2552 if (ret == INFLATE_LEFTOVER_INPUT)
2553 ret = ISAL_DECOMP_OK;
2554
2555 if (ret != in_vector->expected_error)
2556 printf("Inflate return value incorrect, %d != %d\n", ret,
2557 in_vector->expected_error);
2558 else
2559 ret = IGZIP_COMP_OK;
2560
2561 if (!ret) {
2562 ret = inflate_multi_pass(compress_buf, compress_len, out_buf, &out_size, 0, NULL, 0,
2563 0);
2564
2565 if (ret == INFLATE_LEFTOVER_INPUT)
2566 ret = ISAL_DECOMP_OK;
2567
2568 if (ret != in_vector->expected_error)
2569 printf("Inflate return value incorrect, %d != %d\n", ret,
2570 in_vector->expected_error);
2571 else
2572 ret = IGZIP_COMP_OK;
2573 }
2574
2575 free(out_buf);
2576
2577 return ret;
2578 }
2579
2580 int
test_large(uint8_t * in_buf,uint32_t in_size,uint64_t large_size)2581 test_large(uint8_t *in_buf, uint32_t in_size, uint64_t large_size)
2582 {
2583
2584 int ret = IGZIP_COMP_OK;
2585 uint32_t gzip_flag, level;
2586 uint32_t z_size = 0, z_size_max = 0, tmp_buf_size;
2587 uint8_t *z_buf = NULL, *tmp_buf = NULL;
2588 int flush_type = NO_FLUSH;
2589
2590 /* Test a non overflow case */
2591 z_size_max = MAX_LARGE_COMP_BUF_SIZE;
2592
2593 gzip_flag = rand() % 5;
2594 level = get_rand_level();
2595
2596 z_size = z_size_max;
2597 z_buf = malloc(z_size);
2598 if (z_buf == NULL) {
2599 print_error(MALLOC_FAILED);
2600 return MALLOC_FAILED;
2601 }
2602 create_rand_repeat_data(z_buf, z_size);
2603
2604 tmp_buf_size = IBUF_SIZE;
2605 tmp_buf = malloc(tmp_buf_size);
2606 if (tmp_buf == NULL) {
2607 print_error(MALLOC_FAILED);
2608 free(z_buf);
2609 return MALLOC_FAILED;
2610 }
2611
2612 ret = compress_ver_rep_buf(in_buf, in_size, large_size, z_buf, z_size, tmp_buf,
2613 tmp_buf_size, flush_type, gzip_flag, level);
2614
2615 if (ret)
2616 print_error(ret);
2617
2618 if (z_buf != NULL) {
2619 free(z_buf);
2620 z_buf = NULL;
2621 }
2622
2623 if (tmp_buf != NULL) {
2624 free(tmp_buf);
2625 tmp_buf = NULL;
2626 }
2627
2628 return ret;
2629 }
2630
2631 /* Run multiple compression tests on data stored in a file */
2632 int
test_compress_file(char * file_name)2633 test_compress_file(char *file_name)
2634 {
2635 int ret = IGZIP_COMP_OK;
2636 uint64_t in_size;
2637 uint8_t *in_buf = NULL;
2638 FILE *in_file = NULL;
2639
2640 in_file = fopen(file_name, "rb");
2641 if (!in_file) {
2642 printf("Failed to open file %s\n", file_name);
2643 return FILE_READ_FAILED;
2644 }
2645
2646 in_size = get_filesize(in_file);
2647 if (in_size > MAX_FILE_SIZE)
2648 in_size = MAX_FILE_SIZE;
2649
2650 if (in_size != 0) {
2651 in_buf = malloc(in_size);
2652 if (in_buf == NULL) {
2653 printf("Failed to allocate in_buf for test_compress_file\n");
2654 ret = MALLOC_FAILED;
2655 goto exit_comp_file;
2656 }
2657 if (fread(in_buf, 1, in_size, in_file) != in_size) {
2658 printf("Failed to read in_buf from test_compress_file\n");
2659 ret = FILE_READ_FAILED;
2660 goto exit_comp_file;
2661 }
2662 }
2663
2664 ret |= test_compress_stateless(in_buf, in_size, NO_FLUSH);
2665 if (!ret)
2666 ret |= test_compress_stateless(in_buf, in_size, SYNC_FLUSH);
2667 if (!ret)
2668 ret |= test_compress_stateless(in_buf, in_size, FULL_FLUSH);
2669 if (!ret)
2670 ret |= test_compress(in_buf, in_size, NO_FLUSH);
2671 if (!ret)
2672 ret |= test_compress(in_buf, in_size, SYNC_FLUSH);
2673 if (!ret)
2674 ret |= test_compress(in_buf, in_size, FULL_FLUSH);
2675 if (!ret)
2676 ret |= test_flush(in_buf, in_size);
2677
2678 if (ret)
2679 printf("Failed on file %s\n", file_name);
2680
2681 exit_comp_file:
2682 if (in_buf != NULL)
2683 free(in_buf);
2684
2685 fclose(in_file);
2686 return ret;
2687 }
2688
2689 int
create_custom_hufftables(struct isal_hufftables * hufftables_custom,int file_count,char * files[])2690 create_custom_hufftables(struct isal_hufftables *hufftables_custom, int file_count, char *files[])
2691 {
2692 long int file_length;
2693 struct isal_huff_histogram histogram;
2694 FILE *file;
2695 int i;
2696
2697 memset(&histogram, 0, sizeof(histogram));
2698
2699 for (i = 0; i < file_count; i++) {
2700 uint8_t *stream = NULL;
2701
2702 printf("Processing %s\n", files[i]);
2703 file = fopen(files[i], "r");
2704 if (file == NULL) {
2705 printf("Error opening file\n");
2706 return 1;
2707 }
2708 fseek(file, 0, SEEK_END);
2709 file_length = ftell(file);
2710 fseek(file, 0, SEEK_SET);
2711 file_length -= ftell(file);
2712
2713 if (file_length > 0) {
2714 stream = malloc(file_length);
2715 if (stream == NULL) {
2716 printf("Failed to allocate memory to read in file\n");
2717 fclose(file);
2718 return 1;
2719 }
2720 } else {
2721 printf("Zero file length: %s\n", files[i]);
2722 fclose(file);
2723 return 1;
2724 }
2725
2726 if (file_length > 0)
2727 if (fread(stream, 1, file_length, file) != file_length) {
2728 printf("Error occurred when reading file\n");
2729 fclose(file);
2730 free(stream);
2731 stream = NULL;
2732 return 1;
2733 }
2734
2735 /* Create a histogram of frequency of symbols found in stream to
2736 * generate the huffman tree.*/
2737 isal_update_histogram(stream, file_length, &histogram);
2738
2739 fclose(file);
2740 if (stream != NULL)
2741 free(stream);
2742 }
2743
2744 return isal_create_hufftables(hufftables_custom, &histogram);
2745 }
2746
2747 int
main(int argc,char * argv[])2748 main(int argc, char *argv[])
2749 {
2750 int i = 0, j = 0, ret = 0, fin_ret = IGZIP_COMP_OK;
2751 uint32_t in_size = 0, offset = 0;
2752 uint8_t *in_buf = NULL;
2753 struct isal_hufftables hufftables_custom, hufftables_sub;
2754 uint64_t iterations, large_buf_size;
2755 size_t argv_index;
2756 char **input_files;
2757 size_t file_count;
2758
2759 argv_index = parse_options(argc, argv);
2760
2761 input_files = &argv[argv_index];
2762 file_count = argc - argv_index;
2763
2764 if (options.verbose) {
2765 setbuf(stdout, NULL);
2766
2767 printf("Window Size: %d K\n", IGZIP_HIST_SIZE / 1024);
2768 printf("Test Seed : %d\n", options.test_seed);
2769 printf("Randoms : %d\n", options.randoms);
2770 }
2771 srand(options.test_seed);
2772
2773 hufftables_subset = &hufftables_sub;
2774 if (file_count > 0) {
2775 ret = create_custom_hufftables(&hufftables_custom, file_count, input_files);
2776 if (ret == 0)
2777 hufftables = &hufftables_custom;
2778 else {
2779 printf("Failed to generate custom hufftable");
2780 return -1;
2781 }
2782 }
2783
2784 in_buf = malloc(IBUF_SIZE);
2785
2786 if (in_buf == NULL) {
2787 fprintf(stderr, "Can't allocate in_buf memory\n");
2788 return -1;
2789 }
2790
2791 memset(in_buf, 0, IBUF_SIZE);
2792
2793 if (file_count > 0) {
2794 printf("igzip_rand_test files: ");
2795
2796 for (i = 0; i < file_count; i++) {
2797 ret = test_compress_file(input_files[i]);
2798 if (ret)
2799 break;
2800 }
2801
2802 if (options.verbose)
2803 printf("................");
2804 printf("%s\n", ret ? "Fail" : "Pass");
2805 if (ret)
2806 goto main_exit;
2807 }
2808
2809 printf("igzip_rand_test stateless: ");
2810
2811 ret = test_compress_stateless((uint8_t *) str1, sizeof(str1), NO_FLUSH);
2812 if (ret)
2813 goto exit_stateless_no_flush;
2814
2815 ret = test_compress_stateless((uint8_t *) str2, sizeof(str2), NO_FLUSH);
2816 if (ret)
2817 goto exit_stateless_no_flush;
2818
2819 for (i = 0; i < options.randoms; i++) {
2820 in_size = get_rand_data_length();
2821 offset = rand() % (IBUF_SIZE + 1 - in_size);
2822 in_buf += offset;
2823
2824 create_rand_repeat_data(in_buf, in_size);
2825
2826 ret = test_compress_stateless(in_buf, in_size, NO_FLUSH);
2827
2828 in_buf -= offset;
2829
2830 if (options.verbose && (i % (options.randoms / 16) == 0))
2831 printf(".");
2832
2833 if (ret)
2834 goto exit_stateless_no_flush;
2835 }
2836
2837 for (i = 0; i < options.randoms / 16; i++) {
2838 create_rand_repeat_data(in_buf, PAGE_SIZE);
2839 ret |= test_compress_stateless(in_buf, PAGE_SIZE, NO_FLUSH); // good for efence
2840 if (ret)
2841 goto exit_stateless_no_flush;
2842 }
2843
2844 ret = test_compress_stateless((uint8_t *) str1, sizeof(str1), SYNC_FLUSH);
2845 if (ret)
2846 goto exit_stateless_no_flush;
2847
2848 ret = test_compress_stateless((uint8_t *) str2, sizeof(str2), SYNC_FLUSH);
2849 if (ret)
2850 goto exit_stateless_no_flush;
2851
2852 for (i = 0; i < 16; i++) {
2853 in_size = get_rand_data_length();
2854 offset = rand() % (IBUF_SIZE + 1 - in_size);
2855 in_buf += offset;
2856
2857 create_rand_repeat_data(in_buf, in_size);
2858
2859 ret |= test_compress_stateless(in_buf, in_size, SYNC_FLUSH);
2860
2861 in_buf -= offset;
2862
2863 if (ret)
2864 break;
2865 }
2866
2867 exit_stateless_no_flush:
2868 printf("%s\n", ret ? "Fail" : "Pass");
2869 if (ret)
2870 goto main_exit;
2871
2872 printf("igzip_rand_test stateless FULL_FLUSH: ");
2873
2874 ret = test_compress_stateless((uint8_t *) str1, sizeof(str1), FULL_FLUSH);
2875 if (ret)
2876 goto exit_stateless_full_flush;
2877
2878 ret = test_compress_stateless((uint8_t *) str2, sizeof(str2), FULL_FLUSH);
2879 if (ret)
2880 goto exit_stateless_full_flush;
2881
2882 for (i = 0; i < options.randoms; i++) {
2883 in_size = get_rand_data_length();
2884 offset = rand() % (IBUF_SIZE + 1 - in_size);
2885 in_buf += offset;
2886
2887 create_rand_repeat_data(in_buf, in_size);
2888
2889 ret = test_compress_stateless(in_buf, in_size, FULL_FLUSH);
2890
2891 in_buf -= offset;
2892
2893 if (options.verbose && (i % (options.randoms / 16) == 0))
2894 printf(".");
2895
2896 if (ret)
2897 goto exit_stateless_full_flush;
2898 }
2899
2900 for (i = 0; i < options.randoms / 16; i++) {
2901 create_rand_repeat_data(in_buf, PAGE_SIZE);
2902 ret = test_compress_stateless(in_buf, PAGE_SIZE, FULL_FLUSH); // good for efence
2903 if (ret)
2904 break;
2905 }
2906
2907 exit_stateless_full_flush:
2908 printf("%s\n", ret ? "Fail" : "Pass");
2909 if (ret)
2910 goto main_exit;
2911
2912 printf("igzip_rand_test stateful NO_FLUSH: ");
2913
2914 memcpy(in_buf, str1, sizeof(str1));
2915 ret = test_compress(in_buf, sizeof(str1), NO_FLUSH);
2916 if (ret)
2917 goto exit_stateful_no_flush;
2918
2919 memcpy(in_buf, str2, sizeof(str2));
2920 ret |= test_compress(in_buf, sizeof(str2), NO_FLUSH);
2921 if (ret)
2922 goto exit_stateful_no_flush;
2923
2924 for (i = 0; i < options.randoms; i++) {
2925 in_size = get_rand_data_length();
2926 offset = rand() % (IBUF_SIZE + 1 - in_size);
2927 in_buf += offset;
2928
2929 create_rand_repeat_data(in_buf, in_size);
2930
2931 ret |= test_compress(in_buf, in_size, NO_FLUSH);
2932
2933 in_buf -= offset;
2934 if (options.verbose && (i % (options.randoms / 16) == 0))
2935 printf(".");
2936
2937 if (ret)
2938 break;
2939 }
2940
2941 exit_stateful_no_flush:
2942 printf("%s\n", ret ? "Fail" : "Pass");
2943 if (ret)
2944 goto main_exit;
2945
2946 printf("igzip_rand_test stateful SYNC_FLUSH: ");
2947
2948 memcpy(in_buf, str1, sizeof(str1));
2949 ret = test_compress(in_buf, sizeof(str1), SYNC_FLUSH);
2950 if (ret)
2951 goto exit_stateful_sync_flush;
2952
2953 memcpy(in_buf, str2, sizeof(str2));
2954 ret |= test_compress(in_buf, sizeof(str2), SYNC_FLUSH);
2955 if (ret)
2956 goto exit_stateful_sync_flush;
2957
2958 for (i = 0; i < options.randoms; i++) {
2959 in_size = get_rand_data_length();
2960 offset = rand() % (IBUF_SIZE + 1 - in_size);
2961 in_buf += offset;
2962
2963 create_rand_repeat_data(in_buf, in_size);
2964
2965 ret |= test_compress(in_buf, in_size, SYNC_FLUSH);
2966
2967 in_buf -= offset;
2968
2969 if (options.verbose && (i % (options.randoms / 16) == 0))
2970 printf(".");
2971 if (ret)
2972 break;
2973 }
2974
2975 exit_stateful_sync_flush:
2976 printf("%s\n", ret ? "Fail" : "Pass");
2977 if (ret)
2978 goto main_exit;
2979
2980 printf("igzip_rand_test stateful FULL_FLUSH: ");
2981
2982 memcpy(in_buf, str1, sizeof(str1));
2983 ret = test_compress(in_buf, sizeof(str1), FULL_FLUSH);
2984 if (ret)
2985 goto exit_stateful_full_flush;
2986
2987 memcpy(in_buf, str2, sizeof(str2));
2988 ret |= test_compress(in_buf, sizeof(str2), FULL_FLUSH);
2989 if (ret)
2990 goto exit_stateful_full_flush;
2991
2992 for (i = 0; i < options.randoms; i++) {
2993 in_size = get_rand_data_length();
2994 offset = rand() % (IBUF_SIZE + 1 - in_size);
2995 in_buf += offset;
2996
2997 create_rand_repeat_data(in_buf, in_size);
2998
2999 ret |= test_compress(in_buf, in_size, FULL_FLUSH);
3000
3001 in_buf -= offset;
3002
3003 if (options.verbose && (i % (options.randoms / 16) == 0))
3004 printf(".");
3005 if (ret)
3006 goto exit_stateful_full_flush;
3007 }
3008
3009 for (i = 0; i < options.randoms / 8; i++) {
3010 in_size = get_rand_data_length();
3011 offset = rand() % (IBUF_SIZE + 1 - in_size);
3012 in_buf += offset;
3013
3014 create_rand_repeat_data(in_buf, in_size);
3015
3016 ret |= test_full_flush(in_buf, in_size);
3017
3018 in_buf -= offset;
3019
3020 if (ret)
3021 break;
3022 }
3023
3024 exit_stateful_full_flush:
3025 printf("%s\n", ret ? "Fail" : "Pass");
3026 if (ret)
3027 goto main_exit;
3028
3029 printf("igzip_rand_test stateful Change Flush: ");
3030
3031 ret = test_flush((uint8_t *) str1, sizeof(str1));
3032 if (ret)
3033 goto exit_stateful_change_flush;
3034
3035 ret |= test_flush((uint8_t *) str2, sizeof(str2));
3036 if (ret)
3037 goto exit_stateful_change_flush;
3038
3039 for (i = 0; i < options.randoms / 4; i++) {
3040 in_size = get_rand_data_length();
3041 offset = rand() % (IBUF_SIZE + 1 - in_size);
3042 in_buf += offset;
3043
3044 create_rand_repeat_data(in_buf, in_size);
3045
3046 ret |= test_flush(in_buf, in_size);
3047
3048 in_buf -= offset;
3049
3050 if (options.verbose && (i % ((options.randoms / 4) / 16) == 0))
3051 printf(".");
3052 if (ret)
3053 break;
3054 }
3055
3056 exit_stateful_change_flush:
3057 printf("%s\n", ret ? "Fail" : "Pass");
3058 if (ret)
3059 goto main_exit;
3060
3061 if (options.do_large_test) {
3062 printf("igzip_rand_test large input ");
3063
3064 iterations = options.randoms / 256 + 1;
3065 for (i = 0; i < iterations; i++) {
3066 in_size = rand() % (32 * 1024) + 16 * 1024;
3067 offset = rand() % (IBUF_SIZE + 1 - in_size);
3068 in_buf += offset;
3069
3070 large_buf_size = 1;
3071 large_buf_size <<= 32;
3072 large_buf_size += rand() % (1024 * 1024) + 1;
3073 create_rand_repeat_data(in_buf, in_size);
3074
3075 ret = test_large(in_buf, in_size, large_buf_size);
3076
3077 if (ret)
3078 goto exit_large_test;
3079
3080 in_buf -= offset;
3081
3082 if (options.verbose) {
3083 if (iterations < 16) {
3084 for (j = 0; j < 16 / iterations; j++)
3085 printf(".");
3086 } else if (i % (iterations / 16) == 0)
3087 printf(".");
3088 }
3089 }
3090
3091 if (options.verbose && iterations < 16) {
3092 for (j = (16 / iterations) * iterations; j < 16; j++)
3093 printf(".");
3094 }
3095
3096 exit_large_test:
3097 printf("%s\n", ret ? "Fail" : "Pass");
3098 if (ret)
3099 goto main_exit;
3100 }
3101
3102 printf("igzip_rand_test inflate Std Vectors: ");
3103
3104 for (i = 0; i < sizeof(std_vect_array) / sizeof(struct vect_result); i++) {
3105 ret = test_inflate(&std_vect_array[i]);
3106 if (ret)
3107 break;
3108 }
3109
3110 if (options.verbose)
3111 printf("................");
3112 printf("%s\n", ret ? "Fail" : "Pass");
3113
3114 main_exit:
3115 fin_ret |= ret;
3116 printf("igzip rand test finished: %s\n",
3117 (fin_ret != IGZIP_COMP_OK) ? "Some tests failed" : "All tests passed");
3118
3119 free(in_buf);
3120
3121 return (fin_ret == IGZIP_COMP_OK) ? EXIT_SUCCESS : EXIT_FAILURE;
3122 }
3123