1 /**********************************************************************
2 Copyright(c) 2011-2018 Intel Corporation All rights reserved.
3
4 Redistribution and use in source and binary forms, with or without
5 modification, are permitted provided that the following conditions
6 are met:
7 * Redistributions of source code must retain the above copyright
8 notice, this list of conditions and the following disclaimer.
9 * Redistributions in binary form must reproduce the above copyright
10 notice, this list of conditions and the following disclaimer in
11 the documentation and/or other materials provided with the
12 distribution.
13 * Neither the name of Intel Corporation nor the names of its
14 contributors may be used to endorse or promote products derived
15 from this software without specific prior written permission.
16
17 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 **********************************************************************/
29
30 #define _FILE_OFFSET_BITS 64
31 #include <stdio.h>
32 #include <stdlib.h>
33 #include <assert.h>
34 #include <string.h>
35 #include <getopt.h>
36 #include <sys/stat.h>
37 #include <utime.h>
38 #include <unistd.h>
39 #include <stdbool.h>
40 #include <stdarg.h>
41 #include <errno.h>
42 #include "igzip_lib.h" /* Normally you use isa-l.h instead for external programs */
43
44 #if defined(HAVE_THREADS)
45 #include <pthread.h>
46 #include "crc.h"
47 #endif
48
49 #if !defined(VERSION)
50 #if defined(ISAL_VERSION)
51 #define VERSION ISAL_VERSION
52 #else
53 #define VERSION "unknown version"
54 #endif
55 #endif
56
57 #define BAD_OPTION 1
58 #define BAD_LEVEL 1
59 #define FILE_EXISTS 0
60 #define MALLOC_FAILED -1
61 #define FILE_OPEN_ERROR -2
62 #define FILE_READ_ERROR -3
63 #define FILE_WRITE_ERROR -4
64
65 #define BUF_SIZE 1024
66 #define BLOCK_SIZE (1024 * 1024)
67
68 #define MAX_FILEPATH_BUF 4096
69
70 #define UNIX 3
71
72 #define NAME_DEFAULT 0
73 #define NO_NAME 1
74 #define YES_NAME 2
75
76 #define NO_TEST 0
77 #define TEST 1
78
79 #define LEVEL_DEFAULT 2
80 #define DEFAULT_SUFFIX_LEN 3
81 char *default_suffixes[] = { ".gz", ".z" };
82 int default_suffixes_lens[] = { 3, 2 };
83
84 char stdin_file_name[] = "-";
85 int stdin_file_name_len = 1;
86
87 enum compression_modes { COMPRESS_MODE, DECOMPRESS_MODE };
88
89 enum long_only_opt_val { RM };
90
91 enum log_types { INFORM, WARN, ERROR, VERBOSE };
92
93 int level_size_buf[10] = {
94 #ifdef ISAL_DEF_LVL0_DEFAULT
95 ISAL_DEF_LVL0_DEFAULT,
96 #else
97 0,
98 #endif
99 #ifdef ISAL_DEF_LVL1_DEFAULT
100 ISAL_DEF_LVL1_DEFAULT,
101 #else
102 0,
103 #endif
104 #ifdef ISAL_DEF_LVL2_DEFAULT
105 ISAL_DEF_LVL2_DEFAULT,
106 #else
107 0,
108 #endif
109 #ifdef ISAL_DEF_LVL3_DEFAULT
110 ISAL_DEF_LVL3_DEFAULT,
111 #else
112 0,
113 #endif
114 #ifdef ISAL_DEF_LVL4_DEFAULT
115 ISAL_DEF_LVL4_DEFAULT,
116 #else
117 0,
118 #endif
119 #ifdef ISAL_DEF_LVL5_DEFAULT
120 ISAL_DEF_LVL5_DEFAULT,
121 #else
122 0,
123 #endif
124 #ifdef ISAL_DEF_LVL6_DEFAULT
125 ISAL_DEF_LVL6_DEFAULT,
126 #else
127 0,
128 #endif
129 #ifdef ISAL_DEF_LVL7_DEFAULT
130 ISAL_DEF_LVL7_DEFAULT,
131 #else
132 0,
133 #endif
134 #ifdef ISAL_DEF_LVL8_DEFAULT
135 ISAL_DEF_LVL8_DEFAULT,
136 #else
137 0,
138 #endif
139 #ifdef ISAL_DEF_LVL9_DEFAULT
140 ISAL_DEF_LVL9_DEFAULT,
141 #else
142 0,
143 #endif
144 };
145
146 struct cli_options {
147 char *infile_name;
148 size_t infile_name_len;
149 char *outfile_name;
150 size_t outfile_name_len;
151 char *suffix;
152 size_t suffix_len;
153 int level;
154 int mode;
155 int use_stdout;
156 int remove;
157 int force;
158 int quiet_level;
159 int verbose_level;
160 int name;
161 int test;
162 int threads;
163 uint8_t *in_buf;
164 uint8_t *out_buf;
165 uint8_t *level_buf;
166 size_t in_buf_size;
167 size_t out_buf_size;
168 size_t level_buf_size;
169 };
170
171 struct cli_options global_options;
172
173 void
init_options(struct cli_options * options)174 init_options(struct cli_options *options)
175 {
176 options->infile_name = NULL;
177 options->infile_name_len = 0;
178 options->outfile_name = NULL;
179 options->outfile_name_len = 0;
180 options->suffix = NULL;
181 options->suffix_len = 0;
182 options->level = LEVEL_DEFAULT;
183 options->mode = COMPRESS_MODE;
184 options->use_stdout = false;
185 options->remove = false;
186 options->force = false;
187 options->quiet_level = 0;
188 options->verbose_level = 0;
189 options->name = NAME_DEFAULT;
190 options->test = NO_TEST;
191 options->in_buf = NULL;
192 options->out_buf = NULL;
193 options->level_buf = NULL;
194 options->in_buf_size = 0;
195 options->out_buf_size = 0;
196 options->level_buf_size = 0;
197 options->threads = 1;
198 };
199
200 int
is_interactive(void)201 is_interactive(void)
202 {
203 int ret;
204 ret = !global_options.force && !global_options.quiet_level && isatty(fileno(stdin));
205 return ret;
206 }
207
208 size_t
get_filesize(FILE * fp)209 get_filesize(FILE *fp)
210 {
211 size_t file_size;
212 fpos_t pos, pos_curr;
213
214 fgetpos(fp, &pos_curr); /* Save current position */
215 #if defined(_WIN32) || defined(_WIN64)
216 _fseeki64(fp, 0, SEEK_END);
217 #else
218 fseeko(fp, 0, SEEK_END);
219 #endif
220 fgetpos(fp, &pos);
221 file_size = *(size_t *) &pos;
222 fsetpos(fp, &pos_curr); /* Restore position */
223
224 return file_size;
225 }
226
227 int
get_posix_filetime(FILE * fp,uint32_t * time)228 get_posix_filetime(FILE *fp, uint32_t *time)
229 {
230 struct stat file_stats;
231 const int ret = fstat(fileno(fp), &file_stats);
232 if (time != NULL && ret == 0)
233 *time = file_stats.st_mtime;
234 return ret;
235 }
236
237 uint32_t
set_filetime(char * file_name,uint32_t posix_time)238 set_filetime(char *file_name, uint32_t posix_time)
239 {
240 struct utimbuf new_time;
241 new_time.actime = posix_time;
242 new_time.modtime = posix_time;
243 return utime(file_name, &new_time);
244 }
245
246 void
log_print(int log_type,char * format,...)247 log_print(int log_type, char *format, ...)
248 {
249 va_list args;
250 va_start(args, format);
251
252 switch (log_type) {
253 case INFORM:
254 vfprintf(stdout, format, args);
255 break;
256 case WARN:
257 if (global_options.quiet_level <= 0)
258 vfprintf(stderr, format, args);
259 break;
260 case ERROR:
261 if (global_options.quiet_level <= 1)
262 vfprintf(stderr, format, args);
263 break;
264 case VERBOSE:
265 if (global_options.verbose_level > 0)
266 vfprintf(stderr, format, args);
267 break;
268 }
269
270 va_end(args);
271 }
272
273 int
usage(int exit_code)274 usage(int exit_code)
275 {
276 int log_type = exit_code ? WARN : INFORM;
277 log_print(log_type,
278 "Usage: igzip [options] [infiles]\n\n"
279 "Options:\n"
280 " -h, --help help, print this message\n"
281 " -# use compression level # with 0 <= # <= %d\n"
282 " -o <file> output file\n"
283 " -c, --stdout write to stdout\n"
284 " -d, --decompress decompress file\n"
285 " -z, --compress compress file (default)\n"
286 " -f, --force overwrite output without prompting\n"
287 " --rm remove source files after successful (de)compression\n"
288 " -k, --keep keep source files (default)\n"
289 " -S, --suffix <.suf> suffix to use while (de)compressing\n"
290 " -V, --version show version number\n"
291 " -v, --verbose verbose mode\n"
292 " -N, --name save/use file name and timestamp in compress/decompress\n"
293 " -n, --no-name do not save/use file name and timestamp in "
294 "compress/decompress\n"
295 " -t, --test test compressed file integrity\n"
296 " -T, --threads <n> use n threads to compress if enabled\n"
297 " -q, --quiet suppress warnings\n\n"
298 "with no infile, or when infile is - , read standard input\n\n",
299 ISAL_DEF_MAX_LEVEL);
300
301 exit(exit_code);
302 }
303
304 void
print_version(void)305 print_version(void)
306 {
307 log_print(INFORM, "igzip command line interface %s\n", VERSION);
308 }
309
310 void *
malloc_safe(size_t size)311 malloc_safe(size_t size)
312 {
313 void *ptr = NULL;
314 if (size == 0)
315 return ptr;
316
317 ptr = malloc(size);
318 if (ptr == NULL) {
319 log_print(ERROR, "igzip: Failed to allocate memory\n");
320 exit(MALLOC_FAILED);
321 }
322
323 return ptr;
324 }
325
326 FILE *
fopen_safe(const char * file_name,const char * mode)327 fopen_safe(const char *file_name, const char *mode)
328 {
329 FILE *file;
330
331 /* Assumes write mode always starts with w */
332 if (mode[0] == 'w') {
333 if (access(file_name, F_OK) == 0) {
334 int answer = 0, tmp;
335
336 log_print(WARN, "igzip: %s already exists;", file_name);
337 if (is_interactive()) {
338 log_print(WARN, " do you wish to overwrite (y/n)?");
339 answer = getchar();
340
341 tmp = answer;
342 while (tmp != '\n' && tmp != EOF)
343 tmp = getchar();
344
345 if (answer != 'y' && answer != 'Y') {
346 log_print(WARN, " not overwritten\n");
347 return NULL;
348 }
349 } else if (!global_options.force) {
350 log_print(WARN, " not overwritten\n");
351 return NULL;
352 }
353 }
354 }
355
356 file = fopen(file_name, mode);
357 if (!file) {
358 const char *error_str = strerror(errno);
359
360 log_print(ERROR, "igzip: Failed to open %s : %s\n", file_name, error_str);
361 return NULL;
362 }
363
364 return file;
365 }
366
367 size_t
fread_safe(void * buf,size_t word_size,size_t buf_size,FILE * in,char * file_name)368 fread_safe(void *buf, size_t word_size, size_t buf_size, FILE *in, char *file_name)
369 {
370 size_t read_size;
371 read_size = fread(buf, word_size, buf_size, in);
372 if (ferror(in)) {
373 log_print(ERROR, "igzip: Error encountered while reading file %s\n", file_name);
374 exit(FILE_READ_ERROR);
375 }
376 return read_size;
377 }
378
379 size_t
fwrite_safe(void * buf,size_t word_size,size_t buf_size,FILE * out,char * file_name)380 fwrite_safe(void *buf, size_t word_size, size_t buf_size, FILE *out, char *file_name)
381 {
382 size_t write_size;
383 write_size = fwrite(buf, word_size, buf_size, out);
384 if (ferror(out)) {
385 log_print(ERROR, "igzip: Error encountered while writing to file %s\n", file_name);
386 exit(FILE_WRITE_ERROR);
387 }
388 return write_size;
389 }
390
391 void
open_in_file(FILE ** in,char * infile_name)392 open_in_file(FILE **in, char *infile_name)
393 {
394 *in = NULL;
395 if (infile_name == NULL)
396 *in = stdin;
397 else
398 *in = fopen_safe(infile_name, "rb");
399 }
400
401 void
open_out_file(FILE ** out,char * outfile_name)402 open_out_file(FILE **out, char *outfile_name)
403 {
404 *out = NULL;
405 if (global_options.use_stdout)
406 *out = stdout;
407 else if (outfile_name != NULL)
408 *out = fopen_safe(outfile_name, "wb");
409 else if (!isatty(fileno(stdout)) || global_options.force)
410 *out = stdout;
411 else {
412 log_print(WARN, "igzip: No output location. Use -c to output to terminal\n");
413 exit(FILE_OPEN_ERROR);
414 }
415 }
416
417 #if defined(HAVE_THREADS)
418
419 #define MAX_THREADS 8
420 #define MAX_JOBQUEUE 16 /* must be a power of 2 */
421
422 enum job_status { JOB_UNALLOCATED = 0, JOB_ALLOCATED, JOB_SUCCESS, JOB_FAIL };
423
424 struct thread_job {
425 uint8_t *next_in;
426 uint32_t avail_in;
427 uint8_t *next_out;
428 uint32_t avail_out;
429 uint32_t total_out;
430 uint32_t type;
431 uint32_t status;
432 };
433 struct thread_pool {
434 pthread_t threads[MAX_THREADS];
435 struct thread_job job[MAX_JOBQUEUE];
436 pthread_mutex_t mutex;
437 pthread_cond_t cond;
438 int head;
439 int tail;
440 int queue;
441 int shutdown;
442 };
443
444 // Globals for thread pool
445 struct thread_pool pool;
446
447 static inline int
pool_has_space(void)448 pool_has_space(void)
449 {
450 return ((pool.head + 1) % MAX_JOBQUEUE) != pool.tail;
451 }
452
453 static inline int
pool_has_work(void)454 pool_has_work(void)
455 {
456 return (pool.queue != pool.head);
457 }
458
459 int
pool_get_work(void)460 pool_get_work(void)
461 {
462 assert(pool.queue != pool.head);
463 pool.queue = (pool.queue + 1) % MAX_JOBQUEUE;
464 return pool.queue;
465 }
466
467 int
pool_put_work(struct isal_zstream * stream)468 pool_put_work(struct isal_zstream *stream)
469 {
470 pthread_mutex_lock(&pool.mutex);
471 if (!pool_has_space() || pool.shutdown) {
472 pthread_mutex_unlock(&pool.mutex);
473 return 1;
474 }
475 int idx = (pool.head + 1) % MAX_JOBQUEUE;
476 pool.job[idx].next_in = stream->next_in;
477 pool.job[idx].avail_in = stream->avail_in;
478 pool.job[idx].next_out = stream->next_out;
479 pool.job[idx].avail_out = stream->avail_out;
480 pool.job[idx].status = JOB_ALLOCATED;
481 pool.job[idx].type = stream->end_of_stream == 0 ? 0 : 1;
482 pool.head = idx;
483 pthread_cond_signal(&pool.cond);
484 pthread_mutex_unlock(&pool.mutex);
485 return 0;
486 }
487
488 void *
thread_worker(void * none)489 thread_worker(void *none)
490 {
491 struct isal_zstream wstream;
492 int check;
493 int work_idx;
494 int level = global_options.level;
495 int level_size = level_size_buf[level];
496 uint8_t *level_buf = malloc_safe(level_size);
497 log_print(VERBOSE, "Start worker\n");
498
499 while (!pool.shutdown) {
500 pthread_mutex_lock(&pool.mutex);
501 while (!pool_has_work() && !pool.shutdown) {
502 pthread_cond_wait(&pool.cond, &pool.mutex);
503 }
504 if (pool.shutdown) {
505 pthread_mutex_unlock(&pool.mutex);
506 continue;
507 }
508
509 work_idx = pool_get_work();
510 pthread_cond_signal(&pool.cond);
511 pthread_mutex_unlock(&pool.mutex);
512
513 isal_deflate_stateless_init(&wstream);
514 wstream.next_in = pool.job[work_idx].next_in;
515 wstream.next_out = pool.job[work_idx].next_out;
516 wstream.avail_in = pool.job[work_idx].avail_in;
517 wstream.avail_out = pool.job[work_idx].avail_out;
518 wstream.end_of_stream = pool.job[work_idx].type;
519 wstream.flush = FULL_FLUSH;
520 wstream.level = global_options.level;
521 wstream.level_buf = level_buf;
522 wstream.level_buf_size = level_size;
523
524 check = isal_deflate_stateless(&wstream);
525 log_print(VERBOSE, "Worker finished job %d, out=%d\n", work_idx, wstream.total_out);
526
527 pool.job[work_idx].total_out = wstream.total_out;
528 pool.job[work_idx].status = JOB_SUCCESS + check; // complete or fail
529 if (check)
530 break;
531 }
532 free(level_buf);
533 log_print(VERBOSE, "Worker quit\n");
534 pthread_exit(NULL);
535 }
536
537 int
pool_create(void)538 pool_create(void)
539 {
540 int i;
541 int nthreads = global_options.threads - 1;
542 pool.head = 0;
543 pool.tail = 0;
544 pool.queue = 0;
545 pool.shutdown = 0;
546 for (i = 0; i < nthreads; i++)
547 pthread_create(&pool.threads[i], NULL, thread_worker, NULL);
548
549 log_print(VERBOSE, "Created %d pool threads\n", nthreads);
550 return 0;
551 }
552
553 void
pool_quit(void)554 pool_quit(void)
555 {
556 int i;
557 pthread_mutex_lock(&pool.mutex);
558 pool.shutdown = 1;
559 pthread_mutex_unlock(&pool.mutex);
560 pthread_cond_broadcast(&pool.cond);
561 for (i = 0; i < global_options.threads - 1; i++)
562 pthread_join(pool.threads[i], NULL);
563 log_print(VERBOSE, "Deleted %d pool threads\n", i);
564 }
565
566 #endif // defined(HAVE_THREADS)
567
568 int
compress_file(void)569 compress_file(void)
570 {
571 FILE *in = NULL, *out = NULL;
572 unsigned char *inbuf = NULL, *outbuf = NULL, *level_buf = NULL;
573 size_t inbuf_size, outbuf_size;
574 int level_size = 0;
575 struct isal_zstream stream;
576 struct isal_gzip_header gz_hdr;
577 int ret, success = 0;
578
579 char *infile_name = global_options.infile_name;
580 char *outfile_name = global_options.outfile_name;
581 char *allocated_name = NULL;
582 char *suffix = global_options.suffix;
583 size_t infile_name_len = global_options.infile_name_len;
584 size_t outfile_name_len = global_options.outfile_name_len;
585 size_t suffix_len = global_options.suffix_len;
586
587 int level = global_options.level;
588
589 if (suffix == NULL) {
590 suffix = default_suffixes[0];
591 suffix_len = default_suffixes_lens[0];
592 }
593
594 if (infile_name_len == stdin_file_name_len && infile_name != NULL &&
595 memcmp(infile_name, stdin_file_name, infile_name_len) == 0) {
596 infile_name = NULL;
597 infile_name_len = 0;
598 }
599
600 if (outfile_name == NULL && infile_name != NULL && !global_options.use_stdout) {
601 outfile_name_len = infile_name_len + suffix_len;
602 allocated_name = malloc_safe(outfile_name_len + 1);
603 outfile_name = allocated_name;
604 strncpy(outfile_name, infile_name, infile_name_len + 1);
605 strncat(outfile_name, suffix, suffix_len);
606 }
607
608 open_in_file(&in, infile_name);
609 if (in == NULL)
610 goto compress_file_cleanup;
611
612 if (infile_name_len != 0 && infile_name_len == outfile_name_len && infile_name != NULL &&
613 outfile_name != NULL && strncmp(infile_name, outfile_name, infile_name_len) == 0) {
614 log_print(ERROR, "igzip: Error input and output file names must differ\n");
615 goto compress_file_cleanup;
616 }
617
618 open_out_file(&out, outfile_name);
619 if (out == NULL)
620 goto compress_file_cleanup;
621
622 inbuf_size = global_options.in_buf_size;
623 outbuf_size = global_options.out_buf_size;
624
625 inbuf = global_options.in_buf;
626 outbuf = global_options.out_buf;
627 level_size = global_options.level_buf_size;
628 level_buf = global_options.level_buf;
629
630 isal_gzip_header_init(&gz_hdr);
631 if (global_options.name == NAME_DEFAULT || global_options.name == YES_NAME) {
632 if (get_posix_filetime(in, &gz_hdr.time) != 0)
633 goto compress_file_cleanup;
634 gz_hdr.name = infile_name;
635 }
636 gz_hdr.os = UNIX;
637 gz_hdr.name_buf_len = infile_name_len + 1;
638
639 isal_deflate_init(&stream);
640 stream.avail_in = 0;
641 stream.flush = NO_FLUSH;
642 stream.level = level;
643 stream.level_buf = level_buf;
644 stream.level_buf_size = level_size;
645 stream.gzip_flag = IGZIP_GZIP_NO_HDR;
646 stream.next_out = outbuf;
647 stream.avail_out = outbuf_size;
648
649 isal_write_gzip_header(&stream, &gz_hdr);
650
651 if (global_options.threads > 1) {
652 #if defined(HAVE_THREADS)
653 int q;
654 int end_of_stream = 0;
655 uint32_t crc = 0;
656 uint64_t total_in = 0;
657
658 // Write the header
659 fwrite_safe(outbuf, 1, stream.total_out, out, outfile_name);
660
661 do {
662 size_t nread;
663 size_t inbuf_used = 0;
664 size_t outbuf_used = 0;
665 uint8_t *iptr = inbuf;
666 uint8_t *optr = outbuf;
667
668 for (q = 0; q < MAX_JOBQUEUE - 1; q++) {
669 inbuf_used += BLOCK_SIZE;
670 outbuf_used += 2 * BLOCK_SIZE;
671 if (inbuf_used > inbuf_size || outbuf_used > outbuf_size)
672 break;
673
674 nread = fread_safe(iptr, 1, BLOCK_SIZE, in, infile_name);
675 crc = crc32_gzip_refl(crc, iptr, nread);
676 end_of_stream = feof(in);
677 total_in += nread;
678 stream.next_in = iptr;
679 stream.next_out = optr;
680 stream.avail_in = nread;
681 stream.avail_out = 2 * BLOCK_SIZE;
682 stream.end_of_stream = end_of_stream;
683 ret = pool_put_work(&stream);
684 if (ret || end_of_stream)
685 break;
686
687 iptr += BLOCK_SIZE;
688 optr += 2 * BLOCK_SIZE;
689 }
690
691 while (pool.tail != pool.head) { // Unprocessed jobs
692 int t = (pool.tail + 1) % MAX_JOBQUEUE;
693 if (pool.job[t].status >= JOB_SUCCESS) { // Finished next
694 if (pool.job[t].status > JOB_SUCCESS) {
695 success = 0;
696 log_print(ERROR,
697 "igzip: Error encountered while "
698 "compressing file %s\n",
699 infile_name);
700 goto compress_file_cleanup;
701 }
702 fwrite_safe(pool.job[t].next_out, 1, pool.job[t].total_out,
703 out, outfile_name);
704
705 pool.job[t].total_out = 0;
706 pool.job[t].status = 0;
707 pool.tail = t;
708 pthread_cond_broadcast(&pool.cond);
709 }
710 // Pick up a job while we wait
711 pthread_mutex_lock(&pool.mutex);
712 if (!pool_has_work()) {
713 pthread_mutex_unlock(&pool.mutex);
714 continue;
715 }
716
717 int work_idx = pool_get_work();
718 pthread_cond_signal(&pool.cond);
719 pthread_mutex_unlock(&pool.mutex);
720
721 isal_deflate_stateless_init(&stream);
722 stream.next_in = pool.job[work_idx].next_in;
723 stream.next_out = pool.job[work_idx].next_out;
724 stream.avail_in = pool.job[work_idx].avail_in;
725 stream.avail_out = pool.job[work_idx].avail_out;
726 stream.end_of_stream = pool.job[work_idx].type;
727 stream.flush = FULL_FLUSH;
728 stream.level = global_options.level;
729 stream.level_buf = level_buf;
730 stream.level_buf_size = level_size;
731 int check = isal_deflate_stateless(&stream);
732 log_print(VERBOSE, "Self finished job %d, out=%d\n", work_idx,
733 stream.total_out);
734 pool.job[work_idx].total_out = stream.total_out;
735 pool.job[work_idx].status = JOB_SUCCESS + check; // complete or fail
736 }
737 } while (!end_of_stream);
738
739 // Write gzip trailer
740 fwrite_safe(&crc, sizeof(uint32_t), 1, out, outfile_name);
741 fwrite_safe(&total_in, sizeof(uint32_t), 1, out, outfile_name);
742
743 #else // No compiled threading support but asked for threads > 1
744 assert(1);
745 #endif
746 } else { // Single thread
747 do {
748 if (stream.avail_in == 0) {
749 stream.next_in = inbuf;
750 stream.avail_in =
751 fread_safe(stream.next_in, 1, inbuf_size, in, infile_name);
752 stream.end_of_stream = feof(in);
753 }
754
755 if (stream.next_out == NULL) {
756 stream.next_out = outbuf;
757 stream.avail_out = outbuf_size;
758 }
759
760 ret = isal_deflate(&stream);
761
762 if (ret != ISAL_DECOMP_OK) {
763 log_print(ERROR,
764 "igzip: Error encountered while compressing file %s\n",
765 infile_name);
766 goto compress_file_cleanup;
767 }
768
769 fwrite_safe(outbuf, 1, stream.next_out - outbuf, out, outfile_name);
770 stream.next_out = NULL;
771
772 } while (!feof(in) || stream.avail_out == 0);
773 }
774
775 success = 1;
776
777 compress_file_cleanup:
778 if (out != NULL && out != stdout)
779 fclose(out);
780
781 if (in != NULL && in != stdin) {
782 fclose(in);
783 if (success && global_options.remove)
784 remove(infile_name);
785 }
786
787 if (allocated_name != NULL)
788 free(allocated_name);
789
790 return (success == 0);
791 }
792
793 int
decompress_file(void)794 decompress_file(void)
795 {
796 FILE *in = NULL, *out = NULL;
797 unsigned char *inbuf = NULL, *outbuf = NULL;
798 size_t inbuf_size, outbuf_size;
799 struct inflate_state state;
800 struct isal_gzip_header gz_hdr;
801 const int terminal = 0, implicit = 1, stripped = 2;
802 int ret = 0, success = 0, outfile_type = terminal;
803
804 char *infile_name = global_options.infile_name;
805 char *outfile_name = global_options.outfile_name;
806 char *allocated_name = NULL;
807 char *suffix = global_options.suffix;
808 size_t infile_name_len = global_options.infile_name_len;
809 size_t outfile_name_len = global_options.outfile_name_len;
810 size_t suffix_len = global_options.suffix_len;
811 int suffix_index = 0;
812 uint32_t file_time;
813
814 // Allocate mem and setup to hold gzip header info
815 if (infile_name_len == stdin_file_name_len && infile_name != NULL &&
816 memcmp(infile_name, stdin_file_name, infile_name_len) == 0) {
817 infile_name = NULL;
818 infile_name_len = 0;
819 }
820
821 if (outfile_name == NULL && !global_options.use_stdout) {
822 if (infile_name != NULL) {
823 outfile_type = stripped;
824 while (suffix_index <
825 sizeof(default_suffixes) / sizeof(*default_suffixes)) {
826 if (suffix == NULL) {
827 suffix = default_suffixes[suffix_index];
828 suffix_len = default_suffixes_lens[suffix_index];
829 suffix_index++;
830 }
831
832 outfile_name_len = infile_name_len - suffix_len;
833 if (infile_name_len >= suffix_len &&
834 memcmp(infile_name + outfile_name_len, suffix, suffix_len) == 0)
835 break;
836 suffix = NULL;
837 suffix_len = 0;
838 }
839
840 if (suffix == NULL && global_options.test == NO_TEST) {
841 log_print(ERROR, "igzip: %s: unknown suffix -- ignored\n",
842 infile_name);
843 return 1;
844 }
845 }
846 if (global_options.name == YES_NAME) {
847 outfile_name_len = 0;
848 outfile_type = implicit;
849 }
850 if (outfile_type != terminal) {
851 allocated_name = malloc_safe(outfile_name_len >= MAX_FILEPATH_BUF
852 ? outfile_name_len + 1
853 : MAX_FILEPATH_BUF);
854 outfile_name = allocated_name;
855 }
856 }
857
858 open_in_file(&in, infile_name);
859 if (in == NULL)
860 goto decompress_file_cleanup;
861
862 if (get_posix_filetime(in, &file_time) != 0)
863 goto decompress_file_cleanup;
864
865 inbuf_size = global_options.in_buf_size;
866 outbuf_size = global_options.out_buf_size;
867 inbuf = global_options.in_buf;
868 outbuf = global_options.out_buf;
869
870 isal_gzip_header_init(&gz_hdr);
871 if (outfile_type == implicit) {
872 gz_hdr.name = outfile_name;
873 gz_hdr.name_buf_len = MAX_FILEPATH_BUF;
874 }
875
876 isal_inflate_init(&state);
877 state.crc_flag = ISAL_GZIP_NO_HDR_VER;
878 state.next_in = inbuf;
879 state.avail_in = fread_safe(state.next_in, 1, inbuf_size, in, infile_name);
880
881 // Actually read and save the header info
882 ret = isal_read_gzip_header(&state, &gz_hdr);
883 if (ret != ISAL_DECOMP_OK) {
884 log_print(ERROR, "igzip: Error invalid gzip header found for file %s\n",
885 infile_name);
886 goto decompress_file_cleanup;
887 }
888
889 if (outfile_type == implicit)
890 file_time = gz_hdr.time;
891
892 if (outfile_name != NULL && infile_name != NULL &&
893 (outfile_type == stripped || (outfile_type == implicit && outfile_name[0] == 0))) {
894 outfile_name_len = infile_name_len - suffix_len;
895 memcpy(outfile_name, infile_name, outfile_name_len);
896 outfile_name[outfile_name_len] = 0;
897 }
898
899 if (infile_name_len != 0 && infile_name_len == outfile_name_len && infile_name != NULL &&
900 outfile_name != NULL && strncmp(infile_name, outfile_name, infile_name_len) == 0) {
901 log_print(ERROR, "igzip: Error input and output file names must differ\n");
902 goto decompress_file_cleanup;
903 }
904
905 if (global_options.test == NO_TEST) {
906 open_out_file(&out, outfile_name);
907 if (out == NULL)
908 goto decompress_file_cleanup;
909 }
910
911 // Start reading in compressed data and decompress
912 do {
913 if (state.avail_in == 0) {
914 state.next_in = inbuf;
915 state.avail_in = fread_safe(state.next_in, 1, inbuf_size, in, infile_name);
916 }
917
918 state.next_out = outbuf;
919 state.avail_out = outbuf_size;
920
921 ret = isal_inflate(&state);
922 if (ret != ISAL_DECOMP_OK) {
923 log_print(ERROR, "igzip: Error encountered while decompressing file %s\n",
924 infile_name);
925 goto decompress_file_cleanup;
926 }
927
928 if (out != NULL)
929 fwrite_safe(outbuf, 1, state.next_out - outbuf, out, outfile_name);
930
931 } while (state.block_state != ISAL_BLOCK_FINISH // while not done
932 && (!feof(in) || state.avail_out == 0) // and work to do
933 );
934
935 // Add the following to look for and decode additional concatenated files
936 if (!feof(in) && state.avail_in == 0) {
937 state.next_in = inbuf;
938 state.avail_in = fread_safe(state.next_in, 1, inbuf_size, in, infile_name);
939 }
940
941 while (state.avail_in > 0 && state.next_in[0] == 31) {
942 // Look for magic numbers for gzip header. Follows the gzread() decision
943 // whether to treat as trailing junk
944 if (state.avail_in > 1 && state.next_in[1] != 139)
945 break;
946
947 isal_inflate_reset(&state);
948 state.crc_flag = ISAL_GZIP; // Let isal_inflate() process extra headers
949 do {
950 if (state.avail_in == 0 && !feof(in)) {
951 state.next_in = inbuf;
952 state.avail_in =
953 fread_safe(state.next_in, 1, inbuf_size, in, infile_name);
954 }
955
956 state.next_out = outbuf;
957 state.avail_out = outbuf_size;
958
959 ret = isal_inflate(&state);
960 if (ret != ISAL_DECOMP_OK) {
961 log_print(ERROR,
962 "igzip: Error while decompressing extra concatenated"
963 "gzip files on %s\n",
964 infile_name);
965 goto decompress_file_cleanup;
966 }
967
968 if (out != NULL)
969 fwrite_safe(outbuf, 1, state.next_out - outbuf, out, outfile_name);
970
971 } while (state.block_state != ISAL_BLOCK_FINISH &&
972 (!feof(in) || state.avail_out == 0));
973
974 if (!feof(in) && state.avail_in == 0) {
975 state.next_in = inbuf;
976 state.avail_in = fread_safe(state.next_in, 1, inbuf_size, in, infile_name);
977 }
978 }
979
980 if (state.block_state != ISAL_BLOCK_FINISH)
981 log_print(ERROR, "igzip: Error %s does not contain a complete gzip file\n",
982 infile_name);
983 else
984 success = 1;
985
986 decompress_file_cleanup:
987 if (out != NULL && out != stdout) {
988 fclose(out);
989 if (success)
990 set_filetime(outfile_name, file_time);
991 }
992
993 if (in != NULL && in != stdin) {
994 fclose(in);
995 if (success && global_options.remove)
996 remove(infile_name);
997 }
998
999 if (allocated_name != NULL)
1000 free(allocated_name);
1001
1002 return (success == 0);
1003 }
1004
1005 int
main(int argc,char * argv[])1006 main(int argc, char *argv[])
1007 {
1008 int c;
1009 char optstring[] = "hcdz0123456789o:S:kfqVvNntT:";
1010 int long_only_flag;
1011 int ret = 0;
1012 int bad_option = 0;
1013 int bad_level = 0;
1014 int bad_c = 0;
1015
1016 struct option long_options[] = { { "help", no_argument, NULL, 'h' },
1017 { "stdout", no_argument, NULL, 'c' },
1018 { "to-stdout", no_argument, NULL, 'c' },
1019 { "compress", no_argument, NULL, 'z' },
1020 { "decompress", no_argument, NULL, 'd' },
1021 { "uncompress", no_argument, NULL, 'd' },
1022 { "keep", no_argument, NULL, 'k' },
1023 { "rm", no_argument, &long_only_flag, RM },
1024 { "suffix", no_argument, NULL, 'S' },
1025 { "fast", no_argument, NULL, '1' },
1026 { "best", no_argument, NULL, '0' + ISAL_DEF_MAX_LEVEL },
1027 { "force", no_argument, NULL, 'f' },
1028 { "quiet", no_argument, NULL, 'q' },
1029 { "version", no_argument, NULL, 'V' },
1030 { "verbose", no_argument, NULL, 'v' },
1031 { "no-name", no_argument, NULL, 'n' },
1032 { "name", no_argument, NULL, 'N' },
1033 { "test", no_argument, NULL, 't' },
1034 { "threads", required_argument, NULL, 'T' },
1035 /* Possible future extensions
1036 {"recursive, no_argument, NULL, 'r'},
1037 {"list", no_argument, NULL, 'l'},
1038 {"benchmark", optional_argument, NULL, 'b'},
1039 {"benchmark_end", required_argument, NULL, 'e'},
1040 */
1041 { 0, 0, 0, 0 } };
1042
1043 init_options(&global_options);
1044
1045 opterr = 0;
1046 while ((c = getopt_long(argc, argv, optstring, long_options, NULL)) != -1) {
1047 if (c >= '0' && c <= '9') {
1048 if (c > '0' + ISAL_DEF_MAX_LEVEL)
1049 bad_level = 1;
1050 else
1051 global_options.level = c - '0';
1052
1053 continue;
1054 }
1055
1056 switch (c) {
1057 case 0:
1058 switch (long_only_flag) {
1059 case RM:
1060 global_options.remove = true;
1061 break;
1062 default:
1063 bad_option = 1;
1064 bad_c = c;
1065 break;
1066 }
1067 break;
1068 case 'o':
1069 global_options.outfile_name = optarg;
1070 global_options.outfile_name_len = strlen(global_options.outfile_name);
1071 break;
1072 case 'c':
1073 global_options.use_stdout = true;
1074 break;
1075 case 'z':
1076 global_options.mode = COMPRESS_MODE;
1077 break;
1078 case 'd':
1079 global_options.mode = DECOMPRESS_MODE;
1080 break;
1081 case 'S':
1082 global_options.suffix = optarg;
1083 global_options.suffix_len = strlen(global_options.suffix);
1084 break;
1085 case 'k':
1086 global_options.remove = false;
1087 break;
1088 case 'f':
1089 global_options.force = true;
1090 break;
1091 case 'q':
1092 global_options.quiet_level++;
1093 break;
1094 case 'v':
1095 global_options.verbose_level++;
1096 break;
1097 case 'V':
1098 print_version();
1099 return 0;
1100 case 'N':
1101 global_options.name = YES_NAME;
1102 break;
1103 case 'n':
1104 global_options.name = NO_NAME;
1105 break;
1106 case 't':
1107 global_options.test = TEST;
1108 global_options.mode = DECOMPRESS_MODE;
1109 break;
1110 case 'T':
1111 #if defined(HAVE_THREADS)
1112 c = atoi(optarg);
1113 c = c > MAX_THREADS ? MAX_THREADS : c;
1114 c = c < 1 ? 1 : c;
1115 global_options.threads = c;
1116 #endif
1117 break;
1118 case 'h':
1119 usage(0);
1120 default:
1121 bad_option = 1;
1122 bad_c = optopt;
1123 break;
1124 }
1125 }
1126
1127 if (bad_option) {
1128 log_print(ERROR, "igzip: invalid option ");
1129 if (bad_c)
1130 log_print(ERROR, "-%c\n", bad_c);
1131
1132 else
1133 log_print(ERROR, ("\n"));
1134
1135 usage(BAD_OPTION);
1136 }
1137
1138 if (bad_level) {
1139 log_print(ERROR, "igzip: invalid compression level\n");
1140 usage(BAD_LEVEL);
1141 }
1142
1143 if (global_options.outfile_name && optind < argc - 1) {
1144 log_print(ERROR,
1145 "igzip: An output file may be specified with only one input file\n");
1146 return 0;
1147 }
1148
1149 global_options.in_buf_size = BLOCK_SIZE;
1150 global_options.out_buf_size = BLOCK_SIZE;
1151
1152 #if defined(HAVE_THREADS)
1153 if (global_options.threads > 1) {
1154 global_options.in_buf_size += (BLOCK_SIZE * MAX_JOBQUEUE);
1155 global_options.out_buf_size += (BLOCK_SIZE * MAX_JOBQUEUE * 2);
1156 pool_create();
1157 }
1158 #endif
1159 global_options.in_buf = malloc_safe(global_options.in_buf_size);
1160 global_options.out_buf = malloc_safe(global_options.out_buf_size);
1161 global_options.level_buf_size = level_size_buf[global_options.level];
1162 global_options.level_buf = malloc_safe(global_options.level_buf_size);
1163
1164 if (global_options.mode == COMPRESS_MODE) {
1165 if (optind >= argc)
1166 ret |= compress_file();
1167 while (optind < argc) {
1168 global_options.infile_name = argv[optind];
1169 global_options.infile_name_len = strlen(global_options.infile_name);
1170 ret |= compress_file();
1171 optind++;
1172 }
1173
1174 } else if (global_options.mode == DECOMPRESS_MODE) {
1175 if (optind >= argc)
1176 ret |= decompress_file();
1177 while (optind < argc) {
1178 global_options.infile_name = argv[optind];
1179 global_options.infile_name_len = strlen(global_options.infile_name);
1180 ret |= decompress_file();
1181 optind++;
1182 }
1183 }
1184 #if defined(HAVE_THREADS)
1185 if (global_options.threads > 1)
1186 pool_quit();
1187 #endif
1188
1189 free(global_options.in_buf);
1190 free(global_options.out_buf);
1191 free(global_options.level_buf);
1192 return ret;
1193 }
1194