1 /*- 2 * Copyright (c) 2003-2010 Tim Kientzle 3 * Copyright (c) 2009-2012 Michihiro NAKAJIMA 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR 16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18 * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, 19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 */ 26 27 #include "archive_platform.h" 28 29 #ifdef HAVE_ERRNO_H 30 #include <errno.h> 31 #endif 32 #ifdef HAVE_STDLIB_H 33 #include <stdlib.h> 34 #endif 35 #ifdef HAVE_STRING_H 36 #include <string.h> 37 #endif 38 #include <time.h> 39 #ifdef HAVE_LZMA_H 40 #include <lzma.h> 41 #endif 42 43 #include "archive.h" 44 #include "archive_endian.h" 45 #include "archive_private.h" 46 #include "archive_write_private.h" 47 48 #if ARCHIVE_VERSION_NUMBER < 4000000 49 int 50 archive_write_set_compression_lzip(struct archive *a) 51 { 52 __archive_write_filters_free(a); 53 return (archive_write_add_filter_lzip(a)); 54 } 55 56 int 57 archive_write_set_compression_lzma(struct archive *a) 58 { 59 __archive_write_filters_free(a); 60 return (archive_write_add_filter_lzma(a)); 61 } 62 63 int 64 archive_write_set_compression_xz(struct archive *a) 65 { 66 __archive_write_filters_free(a); 67 return (archive_write_add_filter_xz(a)); 68 } 69 70 #endif 71 72 #ifndef HAVE_LZMA_H 73 int 74 archive_write_add_filter_xz(struct archive *a) 75 { 76 archive_set_error(a, ARCHIVE_ERRNO_MISC, 77 "xz compression not supported on this platform"); 78 return (ARCHIVE_FATAL); 79 } 80 81 int 82 archive_write_add_filter_lzma(struct archive *a) 83 { 84 archive_set_error(a, ARCHIVE_ERRNO_MISC, 85 "lzma compression not supported on this platform"); 86 return (ARCHIVE_FATAL); 87 } 88 89 int 90 archive_write_add_filter_lzip(struct archive *a) 91 { 92 archive_set_error(a, ARCHIVE_ERRNO_MISC, 93 "lzma compression not supported on this platform"); 94 return (ARCHIVE_FATAL); 95 } 96 #else 97 /* Don't compile this if we don't have liblzma. */ 98 99 struct private_data { 100 int compression_level; 101 uint32_t threads; 102 lzma_stream stream; 103 lzma_filter lzmafilters[2]; 104 lzma_options_lzma lzma_opt; 105 int64_t total_in; 106 unsigned char *compressed; 107 size_t compressed_buffer_size; 108 int64_t total_out; 109 /* the CRC32 value of uncompressed data for lzip */ 110 uint32_t crc32; 111 }; 112 113 static int archive_compressor_xz_options(struct archive_write_filter *, 114 const char *, const char *); 115 static int archive_compressor_xz_open(struct archive_write_filter *); 116 static int archive_compressor_xz_write(struct archive_write_filter *, 117 const void *, size_t); 118 static int archive_compressor_xz_close(struct archive_write_filter *); 119 static int archive_compressor_xz_free(struct archive_write_filter *); 120 static int drive_compressor(struct archive_write_filter *, 121 struct private_data *, int finishing); 122 123 struct option_value { 124 uint32_t dict_size; 125 uint32_t nice_len; 126 lzma_match_finder mf; 127 }; 128 static const struct option_value option_values[] = { 129 { 1 << 16, 32, LZMA_MF_HC3}, 130 { 1 << 20, 32, LZMA_MF_HC3}, 131 { 3 << 19, 32, LZMA_MF_HC4}, 132 { 1 << 21, 32, LZMA_MF_BT4}, 133 { 3 << 20, 32, LZMA_MF_BT4}, 134 { 1 << 22, 32, LZMA_MF_BT4}, 135 { 1 << 23, 64, LZMA_MF_BT4}, 136 { 1 << 24, 64, LZMA_MF_BT4}, 137 { 3 << 23, 64, LZMA_MF_BT4}, 138 { 1 << 25, 64, LZMA_MF_BT4} 139 }; 140 141 static int 142 common_setup(struct archive_write_filter *f) 143 { 144 struct private_data *data; 145 struct archive_write *a = (struct archive_write *)f->archive; 146 data = calloc(1, sizeof(*data)); 147 if (data == NULL) { 148 archive_set_error(&a->archive, ENOMEM, "Out of memory"); 149 return (ARCHIVE_FATAL); 150 } 151 f->data = data; 152 data->compression_level = LZMA_PRESET_DEFAULT; 153 data->threads = 1; 154 f->open = &archive_compressor_xz_open; 155 f->close = archive_compressor_xz_close; 156 f->free = archive_compressor_xz_free; 157 f->options = &archive_compressor_xz_options; 158 return (ARCHIVE_OK); 159 } 160 161 /* 162 * Add an xz compression filter to this write handle. 163 */ 164 int 165 archive_write_add_filter_xz(struct archive *_a) 166 { 167 struct archive_write_filter *f; 168 int r; 169 170 archive_check_magic(_a, ARCHIVE_WRITE_MAGIC, 171 ARCHIVE_STATE_NEW, "archive_write_add_filter_xz"); 172 f = __archive_write_allocate_filter(_a); 173 r = common_setup(f); 174 if (r == ARCHIVE_OK) { 175 f->code = ARCHIVE_FILTER_XZ; 176 f->name = "xz"; 177 } 178 return (r); 179 } 180 181 /* LZMA is handled identically, we just need a different compression 182 * code set. (The liblzma setup looks at the code to determine 183 * the one place that XZ and LZMA require different handling.) */ 184 int 185 archive_write_add_filter_lzma(struct archive *_a) 186 { 187 struct archive_write_filter *f; 188 int r; 189 190 archive_check_magic(_a, ARCHIVE_WRITE_MAGIC, 191 ARCHIVE_STATE_NEW, "archive_write_add_filter_lzma"); 192 f = __archive_write_allocate_filter(_a); 193 r = common_setup(f); 194 if (r == ARCHIVE_OK) { 195 f->code = ARCHIVE_FILTER_LZMA; 196 f->name = "lzma"; 197 } 198 return (r); 199 } 200 201 int 202 archive_write_add_filter_lzip(struct archive *_a) 203 { 204 struct archive_write_filter *f; 205 int r; 206 207 archive_check_magic(_a, ARCHIVE_WRITE_MAGIC, 208 ARCHIVE_STATE_NEW, "archive_write_add_filter_lzip"); 209 f = __archive_write_allocate_filter(_a); 210 r = common_setup(f); 211 if (r == ARCHIVE_OK) { 212 f->code = ARCHIVE_FILTER_LZIP; 213 f->name = "lzip"; 214 } 215 return (r); 216 } 217 218 static int 219 archive_compressor_xz_init_stream(struct archive_write_filter *f, 220 struct private_data *data) 221 { 222 static const lzma_stream lzma_stream_init_data = LZMA_STREAM_INIT; 223 int ret; 224 #ifdef HAVE_LZMA_STREAM_ENCODER_MT 225 lzma_mt mt_options; 226 #endif 227 228 data->stream = lzma_stream_init_data; 229 data->stream.next_out = data->compressed; 230 data->stream.avail_out = data->compressed_buffer_size; 231 if (f->code == ARCHIVE_FILTER_XZ) { 232 #ifdef HAVE_LZMA_STREAM_ENCODER_MT 233 if (data->threads != 1) { 234 memset(&mt_options, 0, sizeof(mt_options)); 235 mt_options.threads = data->threads; 236 mt_options.timeout = 300; 237 mt_options.filters = data->lzmafilters; 238 mt_options.check = LZMA_CHECK_CRC64; 239 ret = lzma_stream_encoder_mt(&(data->stream), 240 &mt_options); 241 } else 242 #endif 243 ret = lzma_stream_encoder(&(data->stream), 244 data->lzmafilters, LZMA_CHECK_CRC64); 245 } else if (f->code == ARCHIVE_FILTER_LZMA) { 246 ret = lzma_alone_encoder(&(data->stream), &data->lzma_opt); 247 } else { /* ARCHIVE_FILTER_LZIP */ 248 int dict_size = data->lzma_opt.dict_size; 249 int ds, log2dic, wedges; 250 251 /* Calculate a coded dictionary size */ 252 if (dict_size < (1 << 12) || dict_size > (1 << 29)) { 253 archive_set_error(f->archive, ARCHIVE_ERRNO_MISC, 254 "Unacceptable dictionary size for lzip: %d", 255 dict_size); 256 return (ARCHIVE_FATAL); 257 } 258 for (log2dic = 29; log2dic >= 12; log2dic--) { 259 if (dict_size & (1 << log2dic)) 260 break; 261 } 262 if (dict_size > (1 << log2dic)) { 263 log2dic++; 264 wedges = 265 ((1 << log2dic) - dict_size) / (1 << (log2dic - 4)); 266 } else 267 wedges = 0; 268 ds = ((wedges << 5) & 0xe0) | (log2dic & 0x1f); 269 270 data->crc32 = 0; 271 /* Make a header */ 272 data->compressed[0] = 0x4C; 273 data->compressed[1] = 0x5A; 274 data->compressed[2] = 0x49; 275 data->compressed[3] = 0x50; 276 data->compressed[4] = 1;/* Version */ 277 data->compressed[5] = (unsigned char)ds; 278 data->stream.next_out += 6; 279 data->stream.avail_out -= 6; 280 281 ret = lzma_raw_encoder(&(data->stream), data->lzmafilters); 282 } 283 if (ret == LZMA_OK) 284 return (ARCHIVE_OK); 285 286 switch (ret) { 287 case LZMA_MEM_ERROR: 288 archive_set_error(f->archive, ENOMEM, 289 "Internal error initializing compression library: " 290 "Cannot allocate memory"); 291 break; 292 default: 293 archive_set_error(f->archive, ARCHIVE_ERRNO_MISC, 294 "Internal error initializing compression library: " 295 "It's a bug in liblzma"); 296 break; 297 } 298 return (ARCHIVE_FATAL); 299 } 300 301 /* 302 * Setup callback. 303 */ 304 static int 305 archive_compressor_xz_open(struct archive_write_filter *f) 306 { 307 struct private_data *data = f->data; 308 int ret; 309 310 if (data->compressed == NULL) { 311 size_t bs = 65536, bpb; 312 if (f->archive->magic == ARCHIVE_WRITE_MAGIC) { 313 /* Buffer size should be a multiple number of the bytes 314 * per block for performance. */ 315 bpb = archive_write_get_bytes_per_block(f->archive); 316 if (bpb > bs) 317 bs = bpb; 318 else if (bpb != 0) 319 bs -= bs % bpb; 320 } 321 data->compressed_buffer_size = bs; 322 data->compressed = malloc(data->compressed_buffer_size); 323 if (data->compressed == NULL) { 324 archive_set_error(f->archive, ENOMEM, 325 "Can't allocate data for compression buffer"); 326 return (ARCHIVE_FATAL); 327 } 328 } 329 330 f->write = archive_compressor_xz_write; 331 332 /* Initialize compression library. */ 333 if (f->code == ARCHIVE_FILTER_LZIP) { 334 const struct option_value *val = 335 &option_values[data->compression_level]; 336 337 data->lzma_opt.dict_size = val->dict_size; 338 data->lzma_opt.preset_dict = NULL; 339 data->lzma_opt.preset_dict_size = 0; 340 data->lzma_opt.lc = LZMA_LC_DEFAULT; 341 data->lzma_opt.lp = LZMA_LP_DEFAULT; 342 data->lzma_opt.pb = LZMA_PB_DEFAULT; 343 data->lzma_opt.mode = 344 data->compression_level<= 2? LZMA_MODE_FAST:LZMA_MODE_NORMAL; 345 data->lzma_opt.nice_len = val->nice_len; 346 data->lzma_opt.mf = val->mf; 347 data->lzma_opt.depth = 0; 348 data->lzmafilters[0].id = LZMA_FILTER_LZMA1; 349 data->lzmafilters[0].options = &data->lzma_opt; 350 data->lzmafilters[1].id = LZMA_VLI_UNKNOWN;/* Terminate */ 351 } else { 352 if (lzma_lzma_preset(&data->lzma_opt, data->compression_level)) { 353 archive_set_error(f->archive, ARCHIVE_ERRNO_MISC, 354 "Internal error initializing compression library"); 355 } 356 data->lzmafilters[0].id = LZMA_FILTER_LZMA2; 357 data->lzmafilters[0].options = &data->lzma_opt; 358 data->lzmafilters[1].id = LZMA_VLI_UNKNOWN;/* Terminate */ 359 } 360 ret = archive_compressor_xz_init_stream(f, data); 361 if (ret == LZMA_OK) { 362 f->data = data; 363 return (0); 364 } 365 return (ARCHIVE_FATAL); 366 } 367 368 /* 369 * Set write options. 370 */ 371 static int 372 archive_compressor_xz_options(struct archive_write_filter *f, 373 const char *key, const char *value) 374 { 375 struct private_data *data = (struct private_data *)f->data; 376 377 if (strcmp(key, "compression-level") == 0) { 378 if (value == NULL || !(value[0] >= '0' && value[0] <= '9') || 379 value[1] != '\0') 380 return (ARCHIVE_WARN); 381 data->compression_level = value[0] - '0'; 382 if (data->compression_level > 9) 383 data->compression_level = 9; 384 return (ARCHIVE_OK); 385 } else if (strcmp(key, "threads") == 0) { 386 char *endptr; 387 388 if (value == NULL) 389 return (ARCHIVE_WARN); 390 errno = 0; 391 data->threads = (int)strtoul(value, &endptr, 10); 392 if (errno != 0 || *endptr != '\0') { 393 data->threads = 1; 394 return (ARCHIVE_WARN); 395 } 396 if (data->threads == 0) { 397 #ifdef HAVE_LZMA_STREAM_ENCODER_MT 398 data->threads = lzma_cputhreads(); 399 #else 400 data->threads = 1; 401 #endif 402 } 403 return (ARCHIVE_OK); 404 } 405 406 /* Note: The "warn" return is just to inform the options 407 * supervisor that we didn't handle it. It will generate 408 * a suitable error if no one used this option. */ 409 return (ARCHIVE_WARN); 410 } 411 412 /* 413 * Write data to the compressed stream. 414 */ 415 static int 416 archive_compressor_xz_write(struct archive_write_filter *f, 417 const void *buff, size_t length) 418 { 419 struct private_data *data = (struct private_data *)f->data; 420 int ret; 421 422 /* Update statistics */ 423 data->total_in += length; 424 if (f->code == ARCHIVE_FILTER_LZIP) 425 data->crc32 = lzma_crc32(buff, length, data->crc32); 426 427 /* Compress input data to output buffer */ 428 data->stream.next_in = buff; 429 data->stream.avail_in = length; 430 if ((ret = drive_compressor(f, data, 0)) != ARCHIVE_OK) 431 return (ret); 432 433 return (ARCHIVE_OK); 434 } 435 436 437 /* 438 * Finish the compression... 439 */ 440 static int 441 archive_compressor_xz_close(struct archive_write_filter *f) 442 { 443 struct private_data *data = (struct private_data *)f->data; 444 int ret; 445 446 ret = drive_compressor(f, data, 1); 447 if (ret == ARCHIVE_OK) { 448 data->total_out += 449 data->compressed_buffer_size - data->stream.avail_out; 450 ret = __archive_write_filter(f->next_filter, 451 data->compressed, 452 data->compressed_buffer_size - data->stream.avail_out); 453 if (f->code == ARCHIVE_FILTER_LZIP && ret == ARCHIVE_OK) { 454 archive_le32enc(data->compressed, data->crc32); 455 archive_le64enc(data->compressed+4, data->total_in); 456 archive_le64enc(data->compressed+12, data->total_out + 20); 457 ret = __archive_write_filter(f->next_filter, 458 data->compressed, 20); 459 } 460 } 461 lzma_end(&(data->stream)); 462 return ret; 463 } 464 465 static int 466 archive_compressor_xz_free(struct archive_write_filter *f) 467 { 468 struct private_data *data = (struct private_data *)f->data; 469 free(data->compressed); 470 free(data); 471 f->data = NULL; 472 return (ARCHIVE_OK); 473 } 474 475 /* 476 * Utility function to push input data through compressor, 477 * writing full output blocks as necessary. 478 * 479 * Note that this handles both the regular write case (finishing == 480 * false) and the end-of-archive case (finishing == true). 481 */ 482 static int 483 drive_compressor(struct archive_write_filter *f, 484 struct private_data *data, int finishing) 485 { 486 int ret; 487 488 for (;;) { 489 if (data->stream.avail_out == 0) { 490 data->total_out += data->compressed_buffer_size; 491 ret = __archive_write_filter(f->next_filter, 492 data->compressed, 493 data->compressed_buffer_size); 494 if (ret != ARCHIVE_OK) 495 return (ARCHIVE_FATAL); 496 data->stream.next_out = data->compressed; 497 data->stream.avail_out = data->compressed_buffer_size; 498 } 499 500 /* If there's nothing to do, we're done. */ 501 if (!finishing && data->stream.avail_in == 0) 502 return (ARCHIVE_OK); 503 504 ret = lzma_code(&(data->stream), 505 finishing ? LZMA_FINISH : LZMA_RUN ); 506 507 switch (ret) { 508 case LZMA_OK: 509 /* In non-finishing case, check if compressor 510 * consumed everything */ 511 if (!finishing && data->stream.avail_in == 0) 512 return (ARCHIVE_OK); 513 /* In finishing case, this return always means 514 * there's more work */ 515 break; 516 case LZMA_STREAM_END: 517 /* This return can only occur in finishing case. */ 518 if (finishing) 519 return (ARCHIVE_OK); 520 archive_set_error(f->archive, ARCHIVE_ERRNO_MISC, 521 "lzma compression data error"); 522 return (ARCHIVE_FATAL); 523 case LZMA_MEMLIMIT_ERROR: 524 archive_set_error(f->archive, ENOMEM, 525 "lzma compression error: " 526 "%ju MiB would have been needed", 527 (uintmax_t)((lzma_memusage(&(data->stream)) 528 + 1024 * 1024 -1) 529 / (1024 * 1024))); 530 return (ARCHIVE_FATAL); 531 default: 532 /* Any other return value indicates an error. */ 533 archive_set_error(f->archive, ARCHIVE_ERRNO_MISC, 534 "lzma compression failed:" 535 " lzma_code() call returned status %d", 536 ret); 537 return (ARCHIVE_FATAL); 538 } 539 } 540 } 541 542 #endif /* HAVE_LZMA_H */ 543