1 /********************************************************************** 2 Copyright(c) 2011-2018 Intel Corporation All rights reserved. 3 4 Redistribution and use in source and binary forms, with or without 5 modification, are permitted provided that the following conditions 6 are met: 7 * Redistributions of source code must retain the above copyright 8 notice, this list of conditions and the following disclaimer. 9 * Redistributions in binary form must reproduce the above copyright 10 notice, this list of conditions and the following disclaimer in 11 the documentation and/or other materials provided with the 12 distribution. 13 * Neither the name of Intel Corporation nor the names of its 14 contributors may be used to endorse or promote products derived 15 from this software without specific prior written permission. 16 17 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 **********************************************************************/ 29 30 #define _FILE_OFFSET_BITS 64 31 #include <stdio.h> 32 #include <stdlib.h> 33 #include <assert.h> 34 #include <string.h> 35 #include <getopt.h> 36 #include <sys/stat.h> 37 #include <utime.h> 38 #include <unistd.h> 39 #include <stdbool.h> 40 #include <stdarg.h> 41 #include "igzip_lib.h" /* Normally you use isa-l.h instead for external programs */ 42 43 #if defined (HAVE_THREADS) 44 # include <pthread.h> 45 # include "crc.h" 46 #endif 47 48 #if !defined (VERSION) 49 # if defined (ISAL_VERSION) 50 # define VERSION ISAL_VERSION 51 # else 52 # define VERSION "unknown version" 53 # endif 54 #endif 55 56 #define BAD_OPTION 1 57 #define BAD_LEVEL 1 58 #define FILE_EXISTS 0 59 #define MALLOC_FAILED -1 60 #define FILE_OPEN_ERROR -2 61 #define FILE_READ_ERROR -3 62 #define FILE_WRITE_ERROR -4 63 64 #define BUF_SIZE 1024 65 #define BLOCK_SIZE (1024 * 1024) 66 67 #define MAX_FILEPATH_BUF 4096 68 69 #define UNIX 3 70 71 #define NAME_DEFAULT 0 72 #define NO_NAME 1 73 #define YES_NAME 2 74 75 #define NO_TEST 0 76 #define TEST 1 77 78 #define LEVEL_DEFAULT 2 79 #define DEFAULT_SUFFIX_LEN 3 80 char *default_suffixes[] = { ".gz", ".z" }; 81 int default_suffixes_lens[] = { 3, 2 }; 82 83 char stdin_file_name[] = "-"; 84 int stdin_file_name_len = 1; 85 86 enum compression_modes { 87 COMPRESS_MODE, 88 DECOMPRESS_MODE 89 }; 90 91 enum long_only_opt_val { 92 RM 93 }; 94 95 enum log_types { 96 INFORM, 97 WARN, 98 ERROR, 99 VERBOSE 100 }; 101 102 int level_size_buf[10] = { 103 #ifdef ISAL_DEF_LVL0_DEFAULT 104 ISAL_DEF_LVL0_DEFAULT, 105 #else 106 0, 107 #endif 108 #ifdef ISAL_DEF_LVL1_DEFAULT 109 ISAL_DEF_LVL1_DEFAULT, 110 #else 111 0, 112 #endif 113 #ifdef ISAL_DEF_LVL2_DEFAULT 114 ISAL_DEF_LVL2_DEFAULT, 115 #else 116 0, 117 #endif 118 #ifdef ISAL_DEF_LVL3_DEFAULT 119 ISAL_DEF_LVL3_DEFAULT, 120 #else 121 0, 122 #endif 123 #ifdef ISAL_DEF_LVL4_DEFAULT 124 ISAL_DEF_LVL4_DEFAULT, 125 #else 126 0, 127 #endif 128 #ifdef ISAL_DEF_LVL5_DEFAULT 129 ISAL_DEF_LVL5_DEFAULT, 130 #else 131 0, 132 #endif 133 #ifdef ISAL_DEF_LVL6_DEFAULT 134 ISAL_DEF_LVL6_DEFAULT, 135 #else 136 0, 137 #endif 138 #ifdef ISAL_DEF_LVL7_DEFAULT 139 ISAL_DEF_LVL7_DEFAULT, 140 #else 141 0, 142 #endif 143 #ifdef ISAL_DEF_LVL8_DEFAULT 144 ISAL_DEF_LVL8_DEFAULT, 145 #else 146 0, 147 #endif 148 #ifdef ISAL_DEF_LVL9_DEFAULT 149 ISAL_DEF_LVL9_DEFAULT, 150 #else 151 0, 152 #endif 153 }; 154 155 struct cli_options { 156 char *infile_name; 157 size_t infile_name_len; 158 char *outfile_name; 159 size_t outfile_name_len; 160 char *suffix; 161 size_t suffix_len; 162 int level; 163 int mode; 164 int use_stdout; 165 int remove; 166 int force; 167 int quiet_level; 168 int verbose_level; 169 int name; 170 int test; 171 int threads; 172 uint8_t *in_buf; 173 uint8_t *out_buf; 174 uint8_t *level_buf; 175 size_t in_buf_size; 176 size_t out_buf_size; 177 size_t level_buf_size; 178 }; 179 180 struct cli_options global_options; 181 182 void init_options(struct cli_options *options) 183 { 184 options->infile_name = NULL; 185 options->infile_name_len = 0; 186 options->outfile_name = NULL; 187 options->outfile_name_len = 0; 188 options->suffix = NULL; 189 options->suffix_len = 0; 190 options->level = LEVEL_DEFAULT; 191 options->mode = COMPRESS_MODE; 192 options->use_stdout = false; 193 options->remove = false; 194 options->force = false; 195 options->quiet_level = 0; 196 options->verbose_level = 0; 197 options->name = NAME_DEFAULT; 198 options->test = NO_TEST; 199 options->in_buf = NULL; 200 options->out_buf = NULL; 201 options->level_buf = NULL; 202 options->in_buf_size = 0; 203 options->out_buf_size = 0; 204 options->level_buf_size = 0; 205 options->threads = 1; 206 }; 207 208 int is_interactive(void) 209 { 210 int ret; 211 ret = !global_options.force && !global_options.quiet_level && isatty(fileno(stdin)); 212 return ret; 213 } 214 215 size_t get_filesize(FILE * fp) 216 { 217 size_t file_size; 218 fpos_t pos, pos_curr; 219 220 fgetpos(fp, &pos_curr); /* Save current position */ 221 #if defined(_WIN32) || defined(_WIN64) 222 _fseeki64(fp, 0, SEEK_END); 223 #else 224 fseeko(fp, 0, SEEK_END); 225 #endif 226 fgetpos(fp, &pos); 227 file_size = *(size_t *)&pos; 228 fsetpos(fp, &pos_curr); /* Restore position */ 229 230 return file_size; 231 } 232 233 uint32_t get_posix_filetime(FILE * fp) 234 { 235 struct stat file_stats; 236 fstat(fileno(fp), &file_stats); 237 return file_stats.st_mtime; 238 } 239 240 uint32_t set_filetime(char *file_name, uint32_t posix_time) 241 { 242 struct utimbuf new_time; 243 new_time.actime = posix_time; 244 new_time.modtime = posix_time; 245 return utime(file_name, &new_time); 246 } 247 248 void log_print(int log_type, char *format, ...) 249 { 250 va_list args; 251 va_start(args, format); 252 253 switch (log_type) { 254 case INFORM: 255 vfprintf(stdout, format, args); 256 break; 257 case WARN: 258 if (global_options.quiet_level <= 0) 259 vfprintf(stderr, format, args); 260 break; 261 case ERROR: 262 if (global_options.quiet_level <= 1) 263 vfprintf(stderr, format, args); 264 break; 265 case VERBOSE: 266 if (global_options.verbose_level > 0) 267 vfprintf(stderr, format, args); 268 break; 269 } 270 271 va_end(args); 272 } 273 274 int usage(int exit_code) 275 { 276 int log_type = exit_code ? WARN : INFORM; 277 log_print(log_type, 278 "Usage: igzip [options] [infiles]\n\n" 279 "Options:\n" 280 " -h, --help help, print this message\n" 281 " -# use compression level # with 0 <= # <= %d\n" 282 " -o <file> output file\n" 283 " -c, --stdout write to stdout\n" 284 " -d, --decompress decompress file\n" 285 " -z, --compress compress file (default)\n" 286 " -f, --force overwrite output without prompting\n" 287 " --rm remove source files after successful (de)compression\n" 288 " -k, --keep keep source files (default)\n" 289 " -S, --suffix <.suf> suffix to use while (de)compressing\n" 290 " -V, --version show version number\n" 291 " -v, --verbose verbose mode\n" 292 " -N, --name save/use file name and timestamp in compress/decompress\n" 293 " -n, --no-name do not save/use file name and timestamp in compress/decompress\n" 294 " -t, --test test compressed file integrity\n" 295 " -T, --threads <n> use n threads to compress if enabled\n" 296 " -q, --quiet suppress warnings\n\n" 297 "with no infile, or when infile is - , read standard input\n\n", 298 ISAL_DEF_MAX_LEVEL); 299 300 exit(exit_code); 301 } 302 303 void print_version(void) 304 { 305 log_print(INFORM, "igzip command line interface %s\n", VERSION); 306 } 307 308 void *malloc_safe(size_t size) 309 { 310 void *ptr = NULL; 311 if (size == 0) 312 return ptr; 313 314 ptr = malloc(size); 315 if (ptr == NULL) { 316 log_print(ERROR, "igzip: Failed to allocate memory\n"); 317 exit(MALLOC_FAILED); 318 } 319 320 return ptr; 321 } 322 323 FILE *fopen_safe(char *file_name, char *mode) 324 { 325 FILE *file; 326 int answer = 0, tmp; 327 328 /* Assumes write mode always starts with w */ 329 if (mode[0] == 'w') { 330 if (access(file_name, F_OK) == 0) { 331 log_print(WARN, "igzip: %s already exists;", file_name); 332 if (is_interactive()) { 333 log_print(WARN, " do you wish to overwrite (y/n)?"); 334 answer = getchar(); 335 336 tmp = answer; 337 while (tmp != '\n' && tmp != EOF) 338 tmp = getchar(); 339 340 if (answer != 'y' && answer != 'Y') { 341 log_print(WARN, " not overwritten\n"); 342 return NULL; 343 } 344 } else if (!global_options.force) { 345 log_print(WARN, " not overwritten\n"); 346 return NULL; 347 } 348 349 if (access(file_name, W_OK) != 0) { 350 log_print(ERROR, "igzip: %s: Permission denied\n", file_name); 351 return NULL; 352 } 353 } 354 } 355 356 /* Assumes read mode always starts with r */ 357 if (mode[0] == 'r') { 358 if (access(file_name, F_OK) != 0) { 359 log_print(ERROR, "igzip: %s does not exist\n", file_name); 360 return NULL; 361 } 362 363 if (access(file_name, R_OK) != 0) { 364 log_print(ERROR, "igzip: %s: Permission denied\n", file_name); 365 return NULL; 366 } 367 } 368 369 file = fopen(file_name, mode); 370 if (!file) { 371 log_print(ERROR, "igzip: Failed to open %s\n", file_name); 372 return NULL; 373 } 374 375 return file; 376 } 377 378 size_t fread_safe(void *buf, size_t word_size, size_t buf_size, FILE * in, char *file_name) 379 { 380 size_t read_size; 381 read_size = fread(buf, word_size, buf_size, in); 382 if (ferror(in)) { 383 log_print(ERROR, "igzip: Error encountered while reading file %s\n", 384 file_name); 385 exit(FILE_READ_ERROR); 386 } 387 return read_size; 388 } 389 390 size_t fwrite_safe(void *buf, size_t word_size, size_t buf_size, FILE * out, char *file_name) 391 { 392 size_t write_size; 393 write_size = fwrite(buf, word_size, buf_size, out); 394 if (ferror(out)) { 395 log_print(ERROR, "igzip: Error encountered while writing to file %s\n", 396 file_name); 397 exit(FILE_WRITE_ERROR); 398 } 399 return write_size; 400 } 401 402 void open_in_file(FILE ** in, char *infile_name) 403 { 404 *in = NULL; 405 if (infile_name == NULL) 406 *in = stdin; 407 else 408 *in = fopen_safe(infile_name, "rb"); 409 } 410 411 void open_out_file(FILE ** out, char *outfile_name) 412 { 413 *out = NULL; 414 if (global_options.use_stdout) 415 *out = stdout; 416 else if (outfile_name != NULL) 417 *out = fopen_safe(outfile_name, "wb"); 418 else if (!isatty(fileno(stdout)) || global_options.force) 419 *out = stdout; 420 else { 421 log_print(WARN, "igzip: No output location. Use -c to output to terminal\n"); 422 exit(FILE_OPEN_ERROR); 423 } 424 } 425 426 #if defined(HAVE_THREADS) 427 428 #define MAX_THREADS 8 429 #define MAX_JOBQUEUE 16 /* must be a power of 2 */ 430 431 enum job_status { 432 JOB_UNALLOCATED = 0, 433 JOB_ALLOCATED, 434 JOB_SUCCESS, 435 JOB_FAIL 436 }; 437 438 struct thread_job { 439 uint8_t *next_in; 440 uint32_t avail_in; 441 uint8_t *next_out; 442 uint32_t avail_out; 443 uint32_t total_out; 444 uint32_t type; 445 uint32_t status; 446 }; 447 struct thread_pool { 448 pthread_t threads[MAX_THREADS]; 449 struct thread_job job[MAX_JOBQUEUE]; 450 pthread_mutex_t mutex; 451 pthread_cond_t cond; 452 int head; 453 int tail; 454 int queue; 455 int shutdown; 456 }; 457 458 // Globals for thread pool 459 struct thread_pool pool; 460 461 static inline int pool_has_space() 462 { 463 return ((pool.head + 1) % MAX_JOBQUEUE) != pool.tail; 464 } 465 466 static inline int pool_has_work() 467 { 468 return (pool.queue != pool.head); 469 } 470 471 int pool_get_work() 472 { 473 assert(pool.queue != pool.head); 474 pool.queue = (pool.queue + 1) % MAX_JOBQUEUE; 475 return pool.queue; 476 } 477 478 int pool_put_work(struct isal_zstream *stream) 479 { 480 pthread_mutex_lock(&pool.mutex); 481 if (!pool_has_space() || pool.shutdown) { 482 pthread_mutex_unlock(&pool.mutex); 483 return 1; 484 } 485 int idx = (pool.head + 1) % MAX_JOBQUEUE; 486 pool.job[idx].next_in = stream->next_in; 487 pool.job[idx].avail_in = stream->avail_in; 488 pool.job[idx].next_out = stream->next_out; 489 pool.job[idx].avail_out = stream->avail_out; 490 pool.job[idx].status = JOB_ALLOCATED; 491 pool.job[idx].type = stream->end_of_stream == 0 ? 0 : 1; 492 pool.head = idx; 493 pthread_cond_signal(&pool.cond); 494 pthread_mutex_unlock(&pool.mutex); 495 return 0; 496 } 497 498 void *thread_worker(void *none) 499 { 500 struct isal_zstream wstream; 501 int check; 502 int work_idx; 503 int level = global_options.level; 504 int level_size = level_size_buf[level]; 505 uint8_t *level_buf = malloc_safe(level_size); 506 log_print(VERBOSE, "Start worker\n"); 507 508 while (!pool.shutdown) { 509 pthread_mutex_lock(&pool.mutex); 510 while (!pool_has_work() && !pool.shutdown) { 511 pthread_cond_wait(&pool.cond, &pool.mutex); 512 } 513 if (pool.shutdown) { 514 pthread_mutex_unlock(&pool.mutex); 515 continue; 516 } 517 518 work_idx = pool_get_work(); 519 pthread_cond_signal(&pool.cond); 520 pthread_mutex_unlock(&pool.mutex); 521 522 isal_deflate_stateless_init(&wstream); 523 wstream.next_in = pool.job[work_idx].next_in; 524 wstream.next_out = pool.job[work_idx].next_out; 525 wstream.avail_in = pool.job[work_idx].avail_in; 526 wstream.avail_out = pool.job[work_idx].avail_out; 527 wstream.end_of_stream = pool.job[work_idx].type; 528 wstream.flush = FULL_FLUSH; 529 wstream.level = global_options.level; 530 wstream.level_buf = level_buf; 531 wstream.level_buf_size = level_size; 532 533 check = isal_deflate_stateless(&wstream); 534 log_print(VERBOSE, "Worker finished job %d, out=%d\n", 535 work_idx, wstream.total_out); 536 537 pool.job[work_idx].total_out = wstream.total_out; 538 pool.job[work_idx].status = JOB_SUCCESS + check; // complete or fail 539 if (check) 540 break; 541 } 542 free(level_buf); 543 log_print(VERBOSE, "Worker quit\n"); 544 pthread_exit(NULL); 545 } 546 547 int pool_create() 548 { 549 int i; 550 int nthreads = global_options.threads - 1; 551 pool.head = 0; 552 pool.tail = 0; 553 pool.queue = 0; 554 pool.shutdown = 0; 555 for (i = 0; i < nthreads; i++) 556 pthread_create(&pool.threads[i], NULL, thread_worker, NULL); 557 558 log_print(VERBOSE, "Created %d pool threads\n", nthreads); 559 return 0; 560 } 561 562 void pool_quit() 563 { 564 int i; 565 pthread_mutex_lock(&pool.mutex); 566 pool.shutdown = 1; 567 pthread_mutex_unlock(&pool.mutex); 568 pthread_cond_broadcast(&pool.cond); 569 for (i = 0; i < global_options.threads - 1; i++) 570 pthread_join(pool.threads[i], NULL); 571 log_print(VERBOSE, "Deleted %d pool threads\n", i); 572 } 573 574 #endif // defined(HAVE_THREADS) 575 576 int compress_file(void) 577 { 578 FILE *in = NULL, *out = NULL; 579 unsigned char *inbuf = NULL, *outbuf = NULL, *level_buf = NULL; 580 size_t inbuf_size, outbuf_size; 581 int level_size = 0; 582 struct isal_zstream stream; 583 struct isal_gzip_header gz_hdr; 584 int ret, success = 0; 585 586 char *infile_name = global_options.infile_name; 587 char *outfile_name = global_options.outfile_name; 588 char *allocated_name = NULL; 589 char *suffix = global_options.suffix; 590 size_t infile_name_len = global_options.infile_name_len; 591 size_t outfile_name_len = global_options.outfile_name_len; 592 size_t suffix_len = global_options.suffix_len; 593 594 int level = global_options.level; 595 596 if (suffix == NULL) { 597 suffix = default_suffixes[0]; 598 suffix_len = default_suffixes_lens[0]; 599 } 600 601 if (infile_name_len == stdin_file_name_len && 602 infile_name != NULL && 603 memcmp(infile_name, stdin_file_name, infile_name_len) == 0) { 604 infile_name = NULL; 605 infile_name_len = 0; 606 } 607 608 if (outfile_name == NULL && infile_name != NULL && !global_options.use_stdout) { 609 outfile_name_len = infile_name_len + suffix_len; 610 allocated_name = malloc_safe(outfile_name_len + 1); 611 outfile_name = allocated_name; 612 strncpy(outfile_name, infile_name, infile_name_len + 1); 613 strncat(outfile_name, suffix, outfile_name_len + 1); 614 } 615 616 open_in_file(&in, infile_name); 617 if (in == NULL) 618 goto compress_file_cleanup; 619 620 if (infile_name_len != 0 && infile_name_len == outfile_name_len 621 && infile_name != NULL && outfile_name != NULL 622 && strncmp(infile_name, outfile_name, infile_name_len) == 0) { 623 log_print(ERROR, "igzip: Error input and output file names must differ\n"); 624 goto compress_file_cleanup; 625 } 626 627 open_out_file(&out, outfile_name); 628 if (out == NULL) 629 goto compress_file_cleanup; 630 631 inbuf_size = global_options.in_buf_size; 632 outbuf_size = global_options.out_buf_size; 633 634 inbuf = global_options.in_buf; 635 outbuf = global_options.out_buf; 636 level_size = global_options.level_buf_size; 637 level_buf = global_options.level_buf; 638 639 isal_gzip_header_init(&gz_hdr); 640 if (global_options.name == NAME_DEFAULT || global_options.name == YES_NAME) { 641 gz_hdr.time = get_posix_filetime(in); 642 gz_hdr.name = infile_name; 643 } 644 gz_hdr.os = UNIX; 645 gz_hdr.name_buf_len = infile_name_len + 1; 646 647 isal_deflate_init(&stream); 648 stream.avail_in = 0; 649 stream.flush = NO_FLUSH; 650 stream.level = level; 651 stream.level_buf = level_buf; 652 stream.level_buf_size = level_size; 653 stream.gzip_flag = IGZIP_GZIP_NO_HDR; 654 stream.next_out = outbuf; 655 stream.avail_out = outbuf_size; 656 657 isal_write_gzip_header(&stream, &gz_hdr); 658 659 if (global_options.threads > 1) { 660 #if defined(HAVE_THREADS) 661 int q; 662 int end_of_stream = 0; 663 uint32_t crc = 0; 664 uint64_t total_in = 0; 665 666 // Write the header 667 fwrite_safe(outbuf, 1, stream.total_out, out, outfile_name); 668 669 do { 670 size_t nread; 671 size_t inbuf_used = 0; 672 size_t outbuf_used = 0; 673 uint8_t *iptr = inbuf; 674 uint8_t *optr = outbuf; 675 676 for (q = 0; q < MAX_JOBQUEUE - 1; q++) { 677 inbuf_used += BLOCK_SIZE; 678 outbuf_used += 2 * BLOCK_SIZE; 679 if (inbuf_used > inbuf_size || outbuf_used > outbuf_size) 680 break; 681 682 nread = fread_safe(iptr, 1, BLOCK_SIZE, in, infile_name); 683 crc = crc32_gzip_refl(crc, iptr, nread); 684 end_of_stream = feof(in); 685 total_in += nread; 686 stream.next_in = iptr; 687 stream.next_out = optr; 688 stream.avail_in = nread; 689 stream.avail_out = 2 * BLOCK_SIZE; 690 stream.end_of_stream = end_of_stream; 691 ret = pool_put_work(&stream); 692 if (ret || end_of_stream) 693 break; 694 695 iptr += BLOCK_SIZE; 696 optr += 2 * BLOCK_SIZE; 697 } 698 699 while (pool.tail != pool.head) { // Unprocessed jobs 700 int t = (pool.tail + 1) % MAX_JOBQUEUE; 701 if (pool.job[t].status >= JOB_SUCCESS) { // Finished next 702 if (pool.job[t].status > JOB_SUCCESS) { 703 success = 0; 704 log_print(ERROR, 705 "igzip: Error encountered while compressing file %s\n", 706 infile_name); 707 goto compress_file_cleanup; 708 } 709 fwrite_safe(pool.job[t].next_out, 1, 710 pool.job[t].total_out, out, outfile_name); 711 712 pool.job[t].total_out = 0; 713 pool.job[t].status = 0; 714 pool.tail = t; 715 pthread_cond_broadcast(&pool.cond); 716 } 717 // Pick up a job while we wait 718 pthread_mutex_lock(&pool.mutex); 719 if (!pool_has_work()) { 720 pthread_mutex_unlock(&pool.mutex); 721 continue; 722 } 723 724 int work_idx = pool_get_work(); 725 pthread_cond_signal(&pool.cond); 726 pthread_mutex_unlock(&pool.mutex); 727 728 isal_deflate_stateless_init(&stream); 729 stream.next_in = pool.job[work_idx].next_in; 730 stream.next_out = pool.job[work_idx].next_out; 731 stream.avail_in = pool.job[work_idx].avail_in; 732 stream.avail_out = pool.job[work_idx].avail_out; 733 stream.end_of_stream = pool.job[work_idx].type; 734 stream.flush = FULL_FLUSH; 735 stream.level = global_options.level; 736 stream.level_buf = level_buf; 737 stream.level_buf_size = level_size; 738 int check = isal_deflate_stateless(&stream); 739 log_print(VERBOSE, "Self finished job %d, out=%d\n", 740 work_idx, stream.total_out); 741 pool.job[work_idx].total_out = stream.total_out; 742 pool.job[work_idx].status = JOB_SUCCESS + check; // complete or fail 743 } 744 } while (!end_of_stream); 745 746 // Write gzip trailer 747 fwrite_safe(&crc, sizeof(uint32_t), 1, out, outfile_name); 748 fwrite_safe(&total_in, sizeof(uint32_t), 1, out, outfile_name); 749 750 #else // No compiled threading support but asked for threads > 1 751 assert(1); 752 #endif 753 } else { // Single thread 754 do { 755 if (stream.avail_in == 0) { 756 stream.next_in = inbuf; 757 stream.avail_in = 758 fread_safe(stream.next_in, 1, inbuf_size, in, infile_name); 759 stream.end_of_stream = feof(in); 760 } 761 762 if (stream.next_out == NULL) { 763 stream.next_out = outbuf; 764 stream.avail_out = outbuf_size; 765 } 766 767 ret = isal_deflate(&stream); 768 769 if (ret != ISAL_DECOMP_OK) { 770 log_print(ERROR, 771 "igzip: Error encountered while compressing file %s\n", 772 infile_name); 773 goto compress_file_cleanup; 774 } 775 776 fwrite_safe(outbuf, 1, stream.next_out - outbuf, out, outfile_name); 777 stream.next_out = NULL; 778 779 } while (!feof(in) || stream.avail_out == 0); 780 } 781 782 success = 1; 783 784 compress_file_cleanup: 785 if (out != NULL && out != stdout) 786 fclose(out); 787 788 if (in != NULL && in != stdin) { 789 fclose(in); 790 if (success && global_options.remove) 791 remove(infile_name); 792 } 793 794 if (allocated_name != NULL) 795 free(allocated_name); 796 797 return (success == 0); 798 } 799 800 int decompress_file(void) 801 { 802 FILE *in = NULL, *out = NULL; 803 unsigned char *inbuf = NULL, *outbuf = NULL; 804 size_t inbuf_size, outbuf_size; 805 struct inflate_state state; 806 struct isal_gzip_header gz_hdr; 807 const int terminal = 0, implicit = 1, stripped = 2; 808 int ret = 0, success = 0, outfile_type = terminal; 809 810 char *infile_name = global_options.infile_name; 811 char *outfile_name = global_options.outfile_name; 812 char *allocated_name = NULL; 813 char *suffix = global_options.suffix; 814 size_t infile_name_len = global_options.infile_name_len; 815 size_t outfile_name_len = global_options.outfile_name_len; 816 size_t suffix_len = global_options.suffix_len; 817 int suffix_index = 0; 818 uint32_t file_time; 819 820 // Allocate mem and setup to hold gzip header info 821 if (infile_name_len == stdin_file_name_len && 822 infile_name != NULL && 823 memcmp(infile_name, stdin_file_name, infile_name_len) == 0) { 824 infile_name = NULL; 825 infile_name_len = 0; 826 } 827 828 if (outfile_name == NULL && !global_options.use_stdout) { 829 if (infile_name != NULL) { 830 outfile_type = stripped; 831 while (suffix_index < 832 sizeof(default_suffixes) / sizeof(*default_suffixes)) { 833 if (suffix == NULL) { 834 suffix = default_suffixes[suffix_index]; 835 suffix_len = default_suffixes_lens[suffix_index]; 836 suffix_index++; 837 } 838 839 outfile_name_len = infile_name_len - suffix_len; 840 if (infile_name_len >= suffix_len 841 && memcmp(infile_name + outfile_name_len, suffix, 842 suffix_len) == 0) 843 break; 844 suffix = NULL; 845 suffix_len = 0; 846 } 847 848 if (suffix == NULL && global_options.test == NO_TEST) { 849 log_print(ERROR, "igzip: %s: unknown suffix -- ignored\n", 850 infile_name); 851 return 1; 852 } 853 } 854 if (global_options.name == YES_NAME) { 855 outfile_name_len = 0; 856 outfile_type = implicit; 857 } 858 if (outfile_type != terminal) { 859 allocated_name = malloc_safe(outfile_name_len >= 860 MAX_FILEPATH_BUF ? outfile_name_len + 861 1 : MAX_FILEPATH_BUF); 862 outfile_name = allocated_name; 863 } 864 } 865 866 open_in_file(&in, infile_name); 867 if (in == NULL) 868 goto decompress_file_cleanup; 869 870 file_time = get_posix_filetime(in); 871 872 inbuf_size = global_options.in_buf_size; 873 outbuf_size = global_options.out_buf_size; 874 inbuf = global_options.in_buf; 875 outbuf = global_options.out_buf; 876 877 isal_gzip_header_init(&gz_hdr); 878 if (outfile_type == implicit) { 879 gz_hdr.name = outfile_name; 880 gz_hdr.name_buf_len = MAX_FILEPATH_BUF; 881 } 882 883 isal_inflate_init(&state); 884 state.crc_flag = ISAL_GZIP_NO_HDR_VER; 885 state.next_in = inbuf; 886 state.avail_in = fread_safe(state.next_in, 1, inbuf_size, in, infile_name); 887 888 // Actually read and save the header info 889 ret = isal_read_gzip_header(&state, &gz_hdr); 890 if (ret != ISAL_DECOMP_OK) { 891 log_print(ERROR, "igzip: Error invalid gzip header found for file %s\n", 892 infile_name); 893 goto decompress_file_cleanup; 894 } 895 896 if (outfile_type == implicit) 897 file_time = gz_hdr.time; 898 899 if (outfile_name != NULL && infile_name != NULL 900 && (outfile_type == stripped 901 || (outfile_type == implicit && outfile_name[0] == 0))) { 902 outfile_name_len = infile_name_len - suffix_len; 903 memcpy(outfile_name, infile_name, outfile_name_len); 904 outfile_name[outfile_name_len] = 0; 905 } 906 907 if (infile_name_len != 0 && infile_name_len == outfile_name_len 908 && infile_name != NULL && outfile_name != NULL 909 && strncmp(infile_name, outfile_name, infile_name_len) == 0) { 910 log_print(ERROR, "igzip: Error input and output file names must differ\n"); 911 goto decompress_file_cleanup; 912 } 913 914 if (global_options.test == NO_TEST) { 915 open_out_file(&out, outfile_name); 916 if (out == NULL) 917 goto decompress_file_cleanup; 918 } 919 920 // Start reading in compressed data and decompress 921 do { 922 if (state.avail_in == 0) { 923 state.next_in = inbuf; 924 state.avail_in = 925 fread_safe(state.next_in, 1, inbuf_size, in, infile_name); 926 } 927 928 state.next_out = outbuf; 929 state.avail_out = outbuf_size; 930 931 ret = isal_inflate(&state); 932 if (ret != ISAL_DECOMP_OK) { 933 log_print(ERROR, 934 "igzip: Error encountered while decompressing file %s\n", 935 infile_name); 936 goto decompress_file_cleanup; 937 } 938 939 if (out != NULL) 940 fwrite_safe(outbuf, 1, state.next_out - outbuf, out, outfile_name); 941 942 } while (state.block_state != ISAL_BLOCK_FINISH // while not done 943 && (!feof(in) || state.avail_out == 0) // and work to do 944 ); 945 946 // Add the following to look for and decode additional concatenated files 947 if (!feof(in) && state.avail_in == 0) { 948 state.next_in = inbuf; 949 state.avail_in = fread_safe(state.next_in, 1, inbuf_size, in, infile_name); 950 } 951 952 while (state.avail_in > 0 && state.next_in[0] == 31) { 953 // Look for magic numbers for gzip header. Follows the gzread() decision 954 // whether to treat as trailing junk 955 if (state.avail_in > 1 && state.next_in[1] != 139) 956 break; 957 958 isal_inflate_reset(&state); 959 state.crc_flag = ISAL_GZIP; // Let isal_inflate() process extra headers 960 do { 961 if (state.avail_in == 0 && !feof(in)) { 962 state.next_in = inbuf; 963 state.avail_in = 964 fread_safe(state.next_in, 1, inbuf_size, in, infile_name); 965 } 966 967 state.next_out = outbuf; 968 state.avail_out = outbuf_size; 969 970 ret = isal_inflate(&state); 971 if (ret != ISAL_DECOMP_OK) { 972 log_print(ERROR, 973 "igzip: Error while decompressing extra concatenated" 974 "gzip files on %s\n", infile_name); 975 goto decompress_file_cleanup; 976 } 977 978 if (out != NULL) 979 fwrite_safe(outbuf, 1, state.next_out - outbuf, out, 980 outfile_name); 981 982 } while (state.block_state != ISAL_BLOCK_FINISH 983 && (!feof(in) || state.avail_out == 0)); 984 985 if (!feof(in) && state.avail_in == 0) { 986 state.next_in = inbuf; 987 state.avail_in = 988 fread_safe(state.next_in, 1, inbuf_size, in, infile_name); 989 } 990 } 991 992 if (state.block_state != ISAL_BLOCK_FINISH) 993 log_print(ERROR, "igzip: Error %s does not contain a complete gzip file\n", 994 infile_name); 995 else 996 success = 1; 997 998 decompress_file_cleanup: 999 if (out != NULL && out != stdout) { 1000 fclose(out); 1001 if (success) 1002 set_filetime(outfile_name, file_time); 1003 } 1004 1005 if (in != NULL && in != stdin) { 1006 fclose(in); 1007 if (success && global_options.remove) 1008 remove(infile_name); 1009 } 1010 1011 if (allocated_name != NULL) 1012 free(allocated_name); 1013 1014 return (success == 0); 1015 } 1016 1017 int main(int argc, char *argv[]) 1018 { 1019 int c; 1020 char optstring[] = "hcdz0123456789o:S:kfqVvNntT:"; 1021 int long_only_flag; 1022 int ret = 0; 1023 int bad_option = 0; 1024 int bad_level = 0; 1025 int bad_c = 0; 1026 1027 struct option long_options[] = { 1028 {"help", no_argument, NULL, 'h'}, 1029 {"stdout", no_argument, NULL, 'c'}, 1030 {"to-stdout", no_argument, NULL, 'c'}, 1031 {"compress", no_argument, NULL, 'z'}, 1032 {"decompress", no_argument, NULL, 'd'}, 1033 {"uncompress", no_argument, NULL, 'd'}, 1034 {"keep", no_argument, NULL, 'k'}, 1035 {"rm", no_argument, &long_only_flag, RM}, 1036 {"suffix", no_argument, NULL, 'S'}, 1037 {"fast", no_argument, NULL, '1'}, 1038 {"best", no_argument, NULL, '0' + ISAL_DEF_MAX_LEVEL}, 1039 {"force", no_argument, NULL, 'f'}, 1040 {"quiet", no_argument, NULL, 'q'}, 1041 {"version", no_argument, NULL, 'V'}, 1042 {"verbose", no_argument, NULL, 'v'}, 1043 {"no-name", no_argument, NULL, 'n'}, 1044 {"name", no_argument, NULL, 'N'}, 1045 {"test", no_argument, NULL, 't'}, 1046 {"threads", required_argument, NULL, 'T'}, 1047 /* Possible future extensions 1048 {"recursive, no_argument, NULL, 'r'}, 1049 {"list", no_argument, NULL, 'l'}, 1050 {"benchmark", optional_argument, NULL, 'b'}, 1051 {"benchmark_end", required_argument, NULL, 'e'}, 1052 */ 1053 {0, 0, 0, 0} 1054 }; 1055 1056 init_options(&global_options); 1057 1058 opterr = 0; 1059 while ((c = getopt_long(argc, argv, optstring, long_options, NULL)) != -1) { 1060 if (c >= '0' && c <= '9') { 1061 if (c > '0' + ISAL_DEF_MAX_LEVEL) 1062 bad_level = 1; 1063 else 1064 global_options.level = c - '0'; 1065 1066 continue; 1067 } 1068 1069 switch (c) { 1070 case 0: 1071 switch (long_only_flag) { 1072 case RM: 1073 global_options.remove = true; 1074 break; 1075 default: 1076 bad_option = 1; 1077 bad_c = c; 1078 break; 1079 } 1080 break; 1081 case 'o': 1082 global_options.outfile_name = optarg; 1083 global_options.outfile_name_len = strlen(global_options.outfile_name); 1084 break; 1085 case 'c': 1086 global_options.use_stdout = true; 1087 break; 1088 case 'z': 1089 global_options.mode = COMPRESS_MODE; 1090 break; 1091 case 'd': 1092 global_options.mode = DECOMPRESS_MODE; 1093 break; 1094 case 'S': 1095 global_options.suffix = optarg; 1096 global_options.suffix_len = strlen(global_options.suffix); 1097 break; 1098 case 'k': 1099 global_options.remove = false; 1100 break; 1101 case 'f': 1102 global_options.force = true; 1103 break; 1104 case 'q': 1105 global_options.quiet_level++; 1106 break; 1107 case 'v': 1108 global_options.verbose_level++; 1109 break; 1110 case 'V': 1111 print_version(); 1112 return 0; 1113 case 'N': 1114 global_options.name = YES_NAME; 1115 break; 1116 case 'n': 1117 global_options.name = NO_NAME; 1118 break; 1119 case 't': 1120 global_options.test = TEST; 1121 global_options.mode = DECOMPRESS_MODE; 1122 break; 1123 case 'T': 1124 #if defined(HAVE_THREADS) 1125 c = atoi(optarg); 1126 c = c > MAX_THREADS ? MAX_THREADS : c; 1127 c = c < 1 ? 1 : c; 1128 global_options.threads = c; 1129 #endif 1130 break; 1131 case 'h': 1132 usage(0); 1133 default: 1134 bad_option = 1; 1135 bad_c = optopt; 1136 break; 1137 } 1138 } 1139 1140 if (bad_option) { 1141 log_print(ERROR, "igzip: invalid option "); 1142 if (bad_c) 1143 log_print(ERROR, "-%c\n", bad_c); 1144 1145 else 1146 log_print(ERROR, ("\n")); 1147 1148 usage(BAD_OPTION); 1149 } 1150 1151 if (bad_level) { 1152 log_print(ERROR, "igzip: invalid compression level\n"); 1153 usage(BAD_LEVEL); 1154 } 1155 1156 if (global_options.outfile_name && optind < argc - 1) { 1157 log_print(ERROR, 1158 "igzip: An output file may be specified with only one input file\n"); 1159 return 0; 1160 } 1161 1162 global_options.in_buf_size = BLOCK_SIZE; 1163 global_options.out_buf_size = BLOCK_SIZE; 1164 1165 #if defined(HAVE_THREADS) 1166 if (global_options.threads > 1) { 1167 global_options.in_buf_size += (BLOCK_SIZE * MAX_JOBQUEUE); 1168 global_options.out_buf_size += (BLOCK_SIZE * MAX_JOBQUEUE * 2); 1169 pool_create(); 1170 } 1171 #endif 1172 global_options.in_buf = malloc_safe(global_options.in_buf_size); 1173 global_options.out_buf = malloc_safe(global_options.out_buf_size); 1174 global_options.level_buf_size = level_size_buf[global_options.level]; 1175 global_options.level_buf = malloc_safe(global_options.level_buf_size); 1176 1177 if (global_options.mode == COMPRESS_MODE) { 1178 if (optind >= argc) 1179 ret |= compress_file(); 1180 while (optind < argc) { 1181 global_options.infile_name = argv[optind]; 1182 global_options.infile_name_len = strlen(global_options.infile_name); 1183 ret |= compress_file(); 1184 optind++; 1185 } 1186 1187 } else if (global_options.mode == DECOMPRESS_MODE) { 1188 if (optind >= argc) 1189 ret |= decompress_file(); 1190 while (optind < argc) { 1191 global_options.infile_name = argv[optind]; 1192 global_options.infile_name_len = strlen(global_options.infile_name); 1193 ret |= decompress_file(); 1194 optind++; 1195 } 1196 } 1197 #if defined(HAVE_THREADS) 1198 if (global_options.threads > 1) 1199 pool_quit(); 1200 #endif 1201 1202 free(global_options.in_buf); 1203 free(global_options.out_buf); 1204 free(global_options.level_buf); 1205 return ret; 1206 } 1207