1 /* $OpenBSD: zopen.c,v 1.3 2016/09/04 15:45:27 tedu Exp $ */ 2 /* $NetBSD: zopen.c,v 1.5 1995/03/26 09:44:53 glass Exp $ */ 3 4 /*- 5 * Copyright (c) 1985, 1986, 1992, 1993 6 * The Regents of the University of California. All rights reserved. 7 * 8 * This code is derived from software contributed to Berkeley by 9 * Diomidis Spinellis and James A. Woods, derived from original 10 * work by Spencer Thomas and Joseph Orost. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions and the following disclaimer. 17 * 2. Redistributions in binary form must reproduce the above copyright 18 * notice, this list of conditions and the following disclaimer in the 19 * documentation and/or other materials provided with the distribution. 20 * 3. Neither the name of the University nor the names of its contributors 21 * may be used to endorse or promote products derived from this software 22 * without specific prior written permission. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 27 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 34 * SUCH DAMAGE. 35 * 36 * From: @(#)zopen.c 8.1 (Berkeley) 6/27/93 37 */ 38 39 /*- 40 * fcompress.c - File compression ala IEEE Computer, June 1984. 41 * 42 * Compress authors: 43 * Spencer W. Thomas (decvax!utah-cs!thomas) 44 * Jim McKie (decvax!mcvax!jim) 45 * Steve Davies (decvax!vax135!petsd!peora!srd) 46 * Ken Turkowski (decvax!decwrl!turtlevax!ken) 47 * James A. Woods (decvax!ihnp4!ames!jaw) 48 * Joe Orost (decvax!vax135!petsd!joe) 49 * 50 * Cleaned up and converted to library returning I/O streams by 51 * Diomidis Spinellis <dds@doc.ic.ac.uk>. 52 * 53 * zopen(filename, mode, bits) 54 * Returns a FILE * that can be used for read or write. The modes 55 * supported are only "r" and "w". Seeking is not allowed. On 56 * reading the file is decompressed, on writing it is compressed. 57 * The output is compatible with compress(1) with 16 bit tables. 58 * Any file produced by compress(1) can be read. 59 */ 60 61 #include <sys/stat.h> 62 63 #include <ctype.h> 64 #include <errno.h> 65 #include <signal.h> 66 #include <stdio.h> 67 #include <stdlib.h> 68 #include <string.h> 69 #include <unistd.h> 70 #include <fcntl.h> 71 #include "compress.h" 72 73 #define MINIMUM(a, b) (((a) < (b)) ? (a) : (b)) 74 75 #define BITS 16 /* Default bits. */ 76 #define HSIZE 69001 /* 95% occupancy */ 77 #define ZBUFSIZ 8192 /* I/O buffer size */ 78 79 /* A code_int must be able to hold 2**BITS values of type int, and also -1. */ 80 typedef long code_int; 81 typedef long count_int; 82 83 static const u_char z_magic[] = 84 {'\037', '\235'}; /* 1F 9D */ 85 86 #define BIT_MASK 0x1f /* Defines for third byte of header. */ 87 #define BLOCK_MASK 0x80 88 89 /* 90 * Masks 0x40 and 0x20 are free. I think 0x20 should mean that there is 91 * a fourth header byte (for expansion). 92 */ 93 #define INIT_BITS 9 /* Initial number of bits/code. */ 94 95 #define MAXCODE(n_bits) ((1 << (n_bits)) - 1) 96 97 struct s_zstate { 98 int zs_fd; /* File stream for I/O */ 99 char zs_mode; /* r or w */ 100 enum { 101 S_START, S_MAGIC, S_MIDDLE, S_EOF 102 } zs_state; /* State of computation */ 103 int zs_n_bits; /* Number of bits/code. */ 104 int zs_maxbits; /* User settable max # bits/code. */ 105 code_int zs_maxcode; /* Maximum code, given n_bits. */ 106 code_int zs_maxmaxcode; /* Should NEVER generate this code. */ 107 count_int zs_htab[HSIZE]; 108 u_short zs_codetab[HSIZE]; 109 code_int zs_hsize; /* For dynamic table sizing. */ 110 code_int zs_free_ent; /* First unused entry. */ 111 /* 112 * Block compression parameters -- after all codes are used up, 113 * and compression rate changes, start over. 114 */ 115 int zs_block_compress; 116 int zs_clear_flg; 117 long zs_ratio; 118 count_int zs_checkpoint; 119 long zs_in_count; /* Length of input. */ 120 long zs_bytes_out; /* Length of output. */ 121 long zs_out_count; /* # of codes output (for debugging).*/ 122 u_char zs_buf[ZBUFSIZ]; /* I/O buffer */ 123 u_char *zs_bp; /* Current I/O window in the zs_buf */ 124 int zs_offset; /* Number of bits in the zs_buf */ 125 union { 126 struct { 127 long zs_fcode; 128 code_int zs_ent; 129 code_int zs_hsize_reg; 130 int zs_hshift; 131 } w; /* Write parameters */ 132 struct { 133 u_char *zs_stackp, *zs_ebp; 134 int zs_finchar; 135 code_int zs_code, zs_oldcode, zs_incode; 136 int zs_size; 137 } r; /* Read parameters */ 138 } u; 139 }; 140 141 /* Definitions to retain old variable names */ 142 #define zs_fcode u.w.zs_fcode 143 #define zs_ent u.w.zs_ent 144 #define zs_hsize_reg u.w.zs_hsize_reg 145 #define zs_hshift u.w.zs_hshift 146 #define zs_stackp u.r.zs_stackp 147 #define zs_finchar u.r.zs_finchar 148 #define zs_code u.r.zs_code 149 #define zs_oldcode u.r.zs_oldcode 150 #define zs_incode u.r.zs_incode 151 #define zs_size u.r.zs_size 152 #define zs_ebp u.r.zs_ebp 153 154 /* 155 * To save much memory, we overlay the table used by compress() with those 156 * used by decompress(). The tab_prefix table is the same size and type as 157 * the codetab. The tab_suffix table needs 2**BITS characters. We get this 158 * from the beginning of htab. The output stack uses the rest of htab, and 159 * contains characters. There is plenty of room for any possible stack 160 * (stack used to be 8000 characters). 161 */ 162 163 #define htabof(i) zs->zs_htab[i] 164 #define codetabof(i) zs->zs_codetab[i] 165 166 #define tab_prefixof(i) codetabof(i) 167 #define tab_suffixof(i) ((u_char *)(zs->zs_htab))[i] 168 #define de_stack ((u_char *)&tab_suffixof(1 << BITS)) 169 170 #define CHECK_GAP 10000 /* Ratio check interval. */ 171 172 /* 173 * the next two codes should not be changed lightly, as they must not 174 * lie within the contiguous general code space. 175 */ 176 #define FIRST 257 /* First free entry. */ 177 #define CLEAR 256 /* Table clear output code. */ 178 179 static int cl_block(struct s_zstate *); 180 static void cl_hash(struct s_zstate *, count_int); 181 static code_int getcode(struct s_zstate *); 182 static int output(struct s_zstate *, code_int); 183 184 /*- 185 * Algorithm from "A Technique for High Performance Data Compression", 186 * Terry A. Welch, IEEE Computer Vol 17, No 6 (June 1984), pp 8-19. 187 * 188 * Algorithm: 189 * Modified Lempel-Ziv method (LZW). Basically finds common 190 * substrings and replaces them with a variable size code. This is 191 * deterministic, and can be done on the fly. Thus, the decompression 192 * procedure needs no input table, but tracks the way the table was built. 193 */ 194 195 /*- 196 * compress write 197 * 198 * Algorithm: use open addressing double hashing (no chaining) on the 199 * prefix code / next character combination. We do a variant of Knuth's 200 * algorithm D (vol. 3, sec. 6.4) along with G. Knott's relatively-prime 201 * secondary probe. Here, the modular division first probe is gives way 202 * to a faster exclusive-or manipulation. Also do block compression with 203 * an adaptive reset, whereby the code table is cleared when the compression 204 * ratio decreases, but after the table fills. The variable-length output 205 * codes are re-sized at this point, and a special CLEAR code is generated 206 * for the decompressor. Late addition: construct the table according to 207 * file size for noticeable speed improvement on small files. Please direct 208 * questions about this implementation to ames!jaw. 209 */ 210 int 211 zwrite(void *cookie, const char *wbp, int num) 212 { 213 code_int i; 214 int c, disp; 215 struct s_zstate *zs; 216 const u_char *bp; 217 u_char tmp; 218 int count; 219 220 zs = cookie; 221 count = num; 222 bp = (u_char *)wbp; 223 switch (zs->zs_state) { 224 case S_MAGIC: 225 return -1; 226 case S_EOF: 227 return 0; 228 case S_START: 229 zs->zs_state = S_MIDDLE; 230 231 zs->zs_maxmaxcode = 1L << zs->zs_maxbits; 232 if (write(zs->zs_fd, z_magic, sizeof(z_magic)) != 233 sizeof(z_magic)) 234 return (-1); 235 tmp = (u_char)(zs->zs_maxbits | zs->zs_block_compress); 236 if (write(zs->zs_fd, &tmp, sizeof(tmp)) != sizeof(tmp)) 237 return (-1); 238 239 zs->zs_bp = zs->zs_buf; 240 zs->zs_offset = 0; 241 zs->zs_bytes_out = 3; /* Includes 3-byte header mojo. */ 242 zs->zs_out_count = 0; 243 zs->zs_clear_flg = 0; 244 zs->zs_ratio = 0; 245 zs->zs_in_count = 1; 246 zs->zs_checkpoint = CHECK_GAP; 247 zs->zs_maxcode = MAXCODE(zs->zs_n_bits = INIT_BITS); 248 zs->zs_free_ent = ((zs->zs_block_compress) ? FIRST : 256); 249 250 zs->zs_ent = *bp++; 251 --count; 252 253 zs->zs_hshift = 0; 254 for (zs->zs_fcode = (long)zs->zs_hsize; zs->zs_fcode < 65536L; 255 zs->zs_fcode *= 2L) 256 zs->zs_hshift++; 257 /* Set hash code range bound. */ 258 zs->zs_hshift = 8 - zs->zs_hshift; 259 260 zs->zs_hsize_reg = zs->zs_hsize; 261 /* Clear hash table. */ 262 cl_hash(zs, (count_int)zs->zs_hsize_reg); 263 264 case S_MIDDLE: 265 for (i = 0; count-- > 0;) { 266 c = *bp++; 267 zs->zs_in_count++; 268 zs->zs_fcode = (long)(((long)c << zs->zs_maxbits) + 269 zs->zs_ent); 270 /* Xor hashing. */ 271 i = ((c << zs->zs_hshift) ^ zs->zs_ent); 272 273 if (htabof(i) == zs->zs_fcode) { 274 zs->zs_ent = codetabof(i); 275 continue; 276 } else if ((long)htabof(i) < 0) /* Empty slot. */ 277 goto nomatch; 278 /* Secondary hash (after G. Knott). */ 279 disp = zs->zs_hsize_reg - i; 280 if (i == 0) 281 disp = 1; 282 probe: if ((i -= disp) < 0) 283 i += zs->zs_hsize_reg; 284 285 if (htabof(i) == zs->zs_fcode) { 286 zs->zs_ent = codetabof(i); 287 continue; 288 } 289 if ((long)htabof(i) >= 0) 290 goto probe; 291 nomatch: if (output(zs, (code_int) zs->zs_ent) == -1) 292 return (-1); 293 zs->zs_out_count++; 294 zs->zs_ent = c; 295 if (zs->zs_free_ent < zs->zs_maxmaxcode) { 296 /* code -> hashtable */ 297 codetabof(i) = zs->zs_free_ent++; 298 htabof(i) = zs->zs_fcode; 299 } else if ((count_int)zs->zs_in_count >= 300 zs->zs_checkpoint && zs->zs_block_compress) { 301 if (cl_block(zs) == -1) 302 return (-1); 303 } 304 } 305 } 306 return (num); 307 } 308 309 int 310 z_close(void *cookie, struct z_info *info, const char *name, struct stat *sb) 311 { 312 struct s_zstate *zs; 313 int rval; 314 315 zs = cookie; 316 if (zs->zs_mode == 'w') { /* Put out the final code. */ 317 if (output(zs, (code_int) zs->zs_ent) == -1) { 318 (void)close(zs->zs_fd); 319 free(zs); 320 return (-1); 321 } 322 zs->zs_out_count++; 323 if (output(zs, (code_int) - 1) == -1) { 324 (void)close(zs->zs_fd); 325 free(zs); 326 return (-1); 327 } 328 } 329 330 if (info != NULL) { 331 info->mtime = 0; 332 info->crc = (u_int32_t)-1; 333 info->hlen = 0; 334 info->total_in = (off_t)zs->zs_in_count; 335 info->total_out = (off_t)zs->zs_bytes_out; 336 } 337 338 rval = close(zs->zs_fd); 339 free(zs); 340 return (rval); 341 } 342 343 static int 344 zclose(void *cookie) 345 { 346 return z_close(cookie, NULL, NULL, NULL); 347 } 348 349 /*- 350 * Output the given code. 351 * Inputs: 352 * code: A n_bits-bit integer. If == -1, then EOF. This assumes 353 * that n_bits =< (long)wordsize - 1. 354 * Outputs: 355 * Outputs code to the file. 356 * Assumptions: 357 * Chars are 8 bits long. 358 * Algorithm: 359 * Maintain a BITS character long buffer (so that 8 codes will 360 * fit in it exactly). Use the VAX insv instruction to insert each 361 * code in turn. When the buffer fills up empty it and start over. 362 */ 363 364 static const u_char lmask[9] = 365 {0xff, 0xfe, 0xfc, 0xf8, 0xf0, 0xe0, 0xc0, 0x80, 0x00}; 366 static const u_char rmask[9] = 367 {0x00, 0x01, 0x03, 0x07, 0x0f, 0x1f, 0x3f, 0x7f, 0xff}; 368 369 static int 370 output(struct s_zstate *zs, code_int ocode) 371 { 372 int bits; 373 374 if (ocode >= 0) { 375 int r_off; 376 u_char *bp; 377 378 /* Get to the first byte. */ 379 bp = zs->zs_bp + (zs->zs_offset >> 3); 380 r_off = zs->zs_offset & 7; 381 bits = zs->zs_n_bits; 382 383 /* 384 * Since ocode is always >= 8 bits, only need to mask the first 385 * hunk on the left. 386 */ 387 *bp = (*bp & rmask[r_off]) | ((ocode << r_off) & lmask[r_off]); 388 bp++; 389 bits -= (8 - r_off); 390 ocode >>= 8 - r_off; 391 /* Get any 8 bit parts in the middle (<=1 for up to 16 bits) */ 392 if (bits >= 8) { 393 *bp++ = ocode; 394 ocode >>= 8; 395 bits -= 8; 396 } 397 /* Last bits. */ 398 if (bits) 399 *bp = ocode; 400 zs->zs_offset += zs->zs_n_bits; 401 if (zs->zs_offset == (zs->zs_n_bits << 3)) { 402 zs->zs_bp += zs->zs_n_bits; 403 zs->zs_offset = 0; 404 } 405 /* 406 * If the next entry is going to be too big for the ocode size, 407 * then increase it, if possible. 408 */ 409 if (zs->zs_free_ent > zs->zs_maxcode || 410 (zs->zs_clear_flg > 0)) { 411 /* 412 * Write the whole buffer, because the input side won't 413 * discover the size increase until after it has read it 414 */ 415 if (zs->zs_offset > 0) { 416 zs->zs_bp += zs->zs_n_bits; 417 zs->zs_offset = 0; 418 } 419 420 if (zs->zs_clear_flg) { 421 zs->zs_maxcode = 422 MAXCODE(zs->zs_n_bits = INIT_BITS); 423 zs->zs_clear_flg = 0; 424 } else { 425 zs->zs_n_bits++; 426 if (zs->zs_n_bits == zs->zs_maxbits) 427 zs->zs_maxcode = zs->zs_maxmaxcode; 428 else 429 zs->zs_maxcode = 430 MAXCODE(zs->zs_n_bits); 431 } 432 } 433 434 if (zs->zs_bp + zs->zs_n_bits > &zs->zs_buf[ZBUFSIZ]) { 435 bits = zs->zs_bp - zs->zs_buf; 436 if (write(zs->zs_fd, zs->zs_buf, bits) != bits) 437 return (-1); 438 zs->zs_bytes_out += bits; 439 if (zs->zs_offset > 0) 440 fprintf (stderr, "zs_offset != 0\n"); 441 zs->zs_bp = zs->zs_buf; 442 } 443 } else { 444 /* At EOF, write the rest of the buffer. */ 445 if (zs->zs_offset > 0) 446 zs->zs_bp += (zs->zs_offset + 7) / 8; 447 if (zs->zs_bp > zs->zs_buf) { 448 bits = zs->zs_bp - zs->zs_buf; 449 if (write(zs->zs_fd, zs->zs_buf, bits) != bits) 450 return (-1); 451 zs->zs_bytes_out += bits; 452 } 453 zs->zs_offset = 0; 454 zs->zs_bp = zs->zs_buf; 455 } 456 return (0); 457 } 458 459 /* Table clear for block compress. */ 460 static int 461 cl_block(struct s_zstate *zs) 462 { 463 long rat; 464 465 zs->zs_checkpoint = zs->zs_in_count + CHECK_GAP; 466 467 if (zs->zs_in_count > 0x007fffff) { /* Shift will overflow. */ 468 rat = zs->zs_bytes_out >> 8; 469 if (rat == 0) /* Don't divide by zero. */ 470 rat = 0x7fffffff; 471 else 472 rat = zs->zs_in_count / rat; 473 } else { 474 /* 8 fractional bits. */ 475 rat = (zs->zs_in_count << 8) / zs->zs_bytes_out; 476 } 477 if (rat > zs->zs_ratio) 478 zs->zs_ratio = rat; 479 else { 480 zs->zs_ratio = 0; 481 cl_hash(zs, (count_int) zs->zs_hsize); 482 zs->zs_free_ent = FIRST; 483 zs->zs_clear_flg = 1; 484 if (output(zs, (code_int) CLEAR) == -1) 485 return (-1); 486 } 487 return (0); 488 } 489 490 /* Reset code table. */ 491 static void 492 cl_hash(struct s_zstate *zs, count_int cl_hsize) 493 { 494 count_int *htab_p; 495 long i, m1; 496 497 m1 = -1; 498 htab_p = zs->zs_htab + cl_hsize; 499 i = cl_hsize - 16; 500 do { /* Might use Sys V memset(3) here. */ 501 *(htab_p - 16) = m1; 502 *(htab_p - 15) = m1; 503 *(htab_p - 14) = m1; 504 *(htab_p - 13) = m1; 505 *(htab_p - 12) = m1; 506 *(htab_p - 11) = m1; 507 *(htab_p - 10) = m1; 508 *(htab_p - 9) = m1; 509 *(htab_p - 8) = m1; 510 *(htab_p - 7) = m1; 511 *(htab_p - 6) = m1; 512 *(htab_p - 5) = m1; 513 *(htab_p - 4) = m1; 514 *(htab_p - 3) = m1; 515 *(htab_p - 2) = m1; 516 *(htab_p - 1) = m1; 517 htab_p -= 16; 518 } while ((i -= 16) >= 0); 519 for (i += 16; i > 0; i--) 520 *--htab_p = m1; 521 } 522 523 FILE * 524 zopen(const char *name, const char *mode, int bits) 525 { 526 FILE *fp; 527 int fd; 528 void *cookie; 529 if ((fd = open(name, (*mode=='r'? O_RDONLY:O_WRONLY|O_CREAT), 530 S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH)) == -1) 531 return NULL; 532 if ((cookie = z_open(fd, mode, NULL, bits, 0, 0)) == NULL) { 533 close(fd); 534 return NULL; 535 } 536 if ((fp = funopen(cookie, NULL, 537 (*mode == 'w'?zwrite:NULL), NULL, zclose)) == NULL) { 538 close(fd); 539 free(cookie); 540 return NULL; 541 } 542 return fp; 543 } 544 545 void * 546 z_open(int fd, const char *mode, char *name, int bits, 547 u_int32_t mtime, int gotmagic) 548 { 549 struct s_zstate *zs; 550 551 if ((mode[0] != 'r' && mode[0] != 'w') || mode[1] != '\0' || 552 bits < 0 || bits > BITS) { 553 errno = EINVAL; 554 return (NULL); 555 } 556 557 if ((zs = calloc(1, sizeof(struct s_zstate))) == NULL) 558 return (NULL); 559 560 /* User settable max # bits/code. */ 561 zs->zs_maxbits = bits ? bits : BITS; 562 /* Should NEVER generate this code. */ 563 zs->zs_maxmaxcode = 1 << zs->zs_maxbits; 564 zs->zs_hsize = HSIZE; /* For dynamic table sizing. */ 565 zs->zs_free_ent = 0; /* First unused entry. */ 566 zs->zs_block_compress = BLOCK_MASK; 567 zs->zs_clear_flg = 0; 568 zs->zs_ratio = 0; 569 zs->zs_checkpoint = CHECK_GAP; 570 zs->zs_in_count = 0; /* Length of input. */ 571 zs->zs_out_count = 0; /* # of codes output (for debugging).*/ 572 zs->zs_state = gotmagic ? S_MAGIC : S_START; 573 zs->zs_offset = 0; 574 zs->zs_size = 0; 575 zs->zs_mode = mode[0]; 576 zs->zs_bp = zs->zs_ebp = zs->zs_buf; 577 578 zs->zs_fd = fd; 579 return zs; 580 } 581