1 /********************************************************************** 2 Copyright(c) 2011-2016 Intel Corporation All rights reserved. 3 4 Redistribution and use in source and binary forms, with or without 5 modification, are permitted provided that the following conditions 6 are met: 7 * Redistributions of source code must retain the above copyright 8 notice, this list of conditions and the following disclaimer. 9 * Redistributions in binary form must reproduce the above copyright 10 notice, this list of conditions and the following disclaimer in 11 the documentation and/or other materials provided with the 12 distribution. 13 * Neither the name of Intel Corporation nor the names of its 14 contributors may be used to endorse or promote products derived 15 from this software without specific prior written permission. 16 17 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 **********************************************************************/ 29 30 #define ASM 31 32 #include <assert.h> 33 #include <string.h> 34 #include <wchar.h> 35 #ifdef _WIN32 36 #include <intrin.h> 37 #endif 38 39 #define MAX_WRITE_BITS_SIZE 8 40 #define FORCE_FLUSH 64 41 #define MIN_OBUF_SIZE 224 42 #define NON_EMPTY_BLOCK_SIZE 6 43 #define MAX_SYNC_FLUSH_SIZE NON_EMPTY_BLOCK_SIZE + MAX_WRITE_BITS_SIZE 44 45 #include "huffman.h" 46 #include "bitbuf2.h" 47 #include "igzip_lib.h" 48 #include "crc.h" 49 #include "repeated_char_result.h" 50 #include "huff_codes.h" 51 #include "encode_df.h" 52 #include "igzip_level_buf_structs.h" 53 #include "igzip_checksums.h" 54 #include "igzip_wrapper.h" 55 #include "unaligned.h" 56 57 extern void 58 isal_deflate_hash_lvl0(uint16_t *, uint32_t, uint32_t, uint8_t *, uint32_t); 59 extern void 60 isal_deflate_hash_lvl1(uint16_t *, uint32_t, uint32_t, uint8_t *, uint32_t); 61 extern void 62 isal_deflate_hash_lvl2(uint16_t *, uint32_t, uint32_t, uint8_t *, uint32_t); 63 extern void 64 isal_deflate_hash_lvl3(uint16_t *, uint32_t, uint32_t, uint8_t *, uint32_t); 65 extern const uint8_t gzip_hdr[]; 66 extern const uint32_t gzip_hdr_bytes; 67 extern const uint32_t gzip_trl_bytes; 68 extern const uint8_t zlib_hdr[]; 69 extern const uint32_t zlib_hdr_bytes; 70 extern const uint32_t zlib_trl_bytes; 71 extern const struct isal_hufftables hufftables_default; 72 extern const struct isal_hufftables hufftables_static; 73 74 static uint32_t 75 write_stored_block(struct isal_zstream *stream); 76 77 static int 78 write_stream_header_stateless(struct isal_zstream *stream); 79 static void 80 write_stream_header(struct isal_zstream *stream); 81 static int 82 write_deflate_header_stateless(struct isal_zstream *stream); 83 static int 84 write_deflate_header_unaligned_stateless(struct isal_zstream *stream); 85 86 #define TYPE0_HDR_LEN 4 87 #define TYPE0_BLK_HDR_LEN 5 88 #define TYPE0_MAX_BLK_LEN 65535 89 90 void 91 isal_deflate_body(struct isal_zstream *stream); 92 void 93 isal_deflate_finish(struct isal_zstream *stream); 94 95 void 96 isal_deflate_icf_body(struct isal_zstream *stream); 97 void 98 isal_deflate_icf_finish_lvl1(struct isal_zstream *stream); 99 void 100 isal_deflate_icf_finish_lvl2(struct isal_zstream *stream); 101 void 102 isal_deflate_icf_finish_lvl3(struct isal_zstream *stream); 103 /*****************************************************************/ 104 105 /* Forward declarations */ 106 static inline void 107 reset_match_history(struct isal_zstream *stream); 108 static void 109 write_header(struct isal_zstream *stream, uint8_t *deflate_hdr, uint32_t deflate_hdr_count, 110 uint32_t extra_bits_count, uint32_t next_state, uint32_t toggle_end_of_stream); 111 static void 112 write_trailer(struct isal_zstream *stream); 113 114 /*****************************************************************/ 115 116 // isal_adler32_bam1 - adler with (B | A minus 1) storage 117 118 uint32_t 119 isal_adler32_bam1(uint32_t adler32, const unsigned char *start, uint64_t length) 120 { 121 uint64_t a; 122 123 /* Internally the checksum is being stored as B | (A-1) so crc and 124 * addler have same init value */ 125 a = adler32 & 0xffff; 126 a = (a == ADLER_MOD - 1) ? 0 : a + 1; 127 adler32 = isal_adler32((adler32 & 0xffff0000) | a, start, length); 128 a = (adler32 & 0xffff); 129 a = (a == 0) ? ADLER_MOD - 1 : a - 1; 130 131 return (adler32 & 0xffff0000) | a; 132 } 133 134 static void 135 update_checksum(struct isal_zstream *stream, uint8_t *start_in, uint64_t length) 136 { 137 struct isal_zstate *state = &stream->internal_state; 138 switch (stream->gzip_flag) { 139 case IGZIP_GZIP: 140 case IGZIP_GZIP_NO_HDR: 141 state->crc = crc32_gzip_refl(state->crc, start_in, length); 142 break; 143 case IGZIP_ZLIB: 144 case IGZIP_ZLIB_NO_HDR: 145 state->crc = isal_adler32_bam1(state->crc, start_in, length); 146 break; 147 } 148 } 149 150 static void 151 sync_flush(struct isal_zstream *stream) 152 { 153 struct isal_zstate *state = &stream->internal_state; 154 uint64_t bits_to_write = 0xFFFF0000, bits_len; 155 uint64_t bytes; 156 int flush_size; 157 158 if (stream->avail_out >= 8) { 159 set_buf(&state->bitbuf, stream->next_out, stream->avail_out); 160 161 flush_size = (-(state->bitbuf.m_bit_count + 3)) % 8; 162 163 bits_to_write <<= flush_size + 3; 164 bits_len = 32 + flush_size + 3; 165 166 state->state = ZSTATE_NEW_HDR; 167 state->has_eob = 0; 168 169 write_bits(&state->bitbuf, bits_to_write, bits_len); 170 171 bytes = buffer_used(&state->bitbuf); 172 stream->next_out = buffer_ptr(&state->bitbuf); 173 stream->avail_out -= bytes; 174 stream->total_out += bytes; 175 176 if (stream->flush == FULL_FLUSH) { 177 /* Clear match history so there are no cross 178 * block length distance pairs */ 179 state->has_hist = IGZIP_NO_HIST; 180 } 181 } 182 } 183 184 static void 185 flush_write_buffer(struct isal_zstream *stream) 186 { 187 struct isal_zstate *state = &stream->internal_state; 188 int bytes = 0; 189 if (stream->avail_out >= 8) { 190 set_buf(&state->bitbuf, stream->next_out, stream->avail_out); 191 flush(&state->bitbuf); 192 stream->next_out = buffer_ptr(&state->bitbuf); 193 bytes = buffer_used(&state->bitbuf); 194 stream->avail_out -= bytes; 195 stream->total_out += bytes; 196 state->state = ZSTATE_NEW_HDR; 197 } 198 } 199 200 static void 201 flush_icf_block(struct isal_zstream *stream) 202 { 203 struct isal_zstate *state = &stream->internal_state; 204 struct level_buf *level_buf = (struct level_buf *) stream->level_buf; 205 struct BitBuf2 *write_buf = &state->bitbuf; 206 struct deflate_icf *icf_buf_encoded_next; 207 208 set_buf(write_buf, stream->next_out, stream->avail_out); 209 210 icf_buf_encoded_next = 211 encode_deflate_icf(level_buf->icf_buf_start + state->count, level_buf->icf_buf_next, 212 write_buf, &level_buf->encode_tables); 213 214 state->count = icf_buf_encoded_next - level_buf->icf_buf_start; 215 stream->next_out = buffer_ptr(write_buf); 216 stream->total_out += buffer_used(write_buf); 217 stream->avail_out -= buffer_used(write_buf); 218 219 if (level_buf->icf_buf_next <= icf_buf_encoded_next) { 220 state->count = 0; 221 if (stream->avail_in == 0 && stream->end_of_stream) 222 state->state = ZSTATE_TRL; 223 else if (stream->avail_in == 0 && stream->flush != NO_FLUSH) 224 state->state = ZSTATE_SYNC_FLUSH; 225 else 226 state->state = ZSTATE_NEW_HDR; 227 } 228 } 229 230 static int 231 check_level_req(struct isal_zstream *stream) 232 { 233 if (stream->level == 0) 234 return 0; 235 236 if (stream->level_buf == NULL) 237 return ISAL_INVALID_LEVEL_BUF; 238 239 switch (stream->level) { 240 case 3: 241 if (stream->level_buf_size < ISAL_DEF_LVL3_MIN) 242 return ISAL_INVALID_LEVEL; 243 break; 244 245 case 2: 246 if (stream->level_buf_size < ISAL_DEF_LVL2_MIN) 247 return ISAL_INVALID_LEVEL; 248 break; 249 case 1: 250 if (stream->level_buf_size < ISAL_DEF_LVL1_MIN) 251 return ISAL_INVALID_LEVEL; 252 break; 253 default: 254 return ISAL_INVALID_LEVEL; 255 } 256 257 return 0; 258 } 259 260 static int 261 init_hash8k_buf(struct isal_zstream *stream) 262 { 263 struct isal_zstate *state = &stream->internal_state; 264 struct level_buf *level_buf = (struct level_buf *) stream->level_buf; 265 state->has_level_buf_init = 1; 266 return sizeof(struct level_buf) - MAX_LVL_BUF_SIZE + sizeof(level_buf->hash8k); 267 } 268 269 static int 270 init_hash_hist_buf(struct isal_zstream *stream) 271 { 272 struct isal_zstate *state = &stream->internal_state; 273 struct level_buf *level_buf = (struct level_buf *) stream->level_buf; 274 state->has_level_buf_init = 1; 275 return sizeof(struct level_buf) - MAX_LVL_BUF_SIZE + sizeof(level_buf->hash_hist); 276 } 277 278 static int 279 init_hash_map_buf(struct isal_zstream *stream) 280 { 281 struct isal_zstate *state = &stream->internal_state; 282 struct level_buf *level_buf = (struct level_buf *) stream->level_buf; 283 if (!state->has_level_buf_init) { 284 level_buf->hash_map.matches_next = level_buf->hash_map.matches; 285 level_buf->hash_map.matches_end = level_buf->hash_map.matches; 286 } 287 state->has_level_buf_init = 1; 288 289 return sizeof(struct level_buf) - MAX_LVL_BUF_SIZE + sizeof(level_buf->hash_map); 290 } 291 292 /* returns the size of the level specific buffer */ 293 static int 294 init_lvlX_buf(struct isal_zstream *stream) 295 { 296 switch (stream->level) { 297 case 3: 298 return init_hash_map_buf(stream); 299 case 2: 300 return init_hash_hist_buf(stream); 301 default: 302 return init_hash8k_buf(stream); 303 } 304 } 305 306 static void 307 init_new_icf_block(struct isal_zstream *stream) 308 { 309 struct isal_zstate *state = &stream->internal_state; 310 struct level_buf *level_buf = (struct level_buf *) stream->level_buf; 311 int level_struct_size; 312 313 level_struct_size = init_lvlX_buf(stream); 314 315 state->block_next = state->block_end; 316 level_buf->icf_buf_start = (struct deflate_icf *) (stream->level_buf + level_struct_size); 317 318 level_buf->icf_buf_next = level_buf->icf_buf_start; 319 level_buf->icf_buf_avail_out = 320 stream->level_buf_size - level_struct_size - sizeof(struct deflate_icf); 321 322 memset(&level_buf->hist, 0, sizeof(struct isal_mod_hist)); 323 state->state = ZSTATE_BODY; 324 } 325 326 static int 327 are_buffers_empty_hashX(struct isal_zstream *stream) 328 { 329 return !stream->avail_in; 330 } 331 332 static int 333 are_buffers_empty_hash_map(struct isal_zstream *stream) 334 { 335 struct level_buf *level_buf = (struct level_buf *) stream->level_buf; 336 337 return (!stream->avail_in && 338 level_buf->hash_map.matches_next >= level_buf->hash_map.matches_end); 339 } 340 341 static int 342 are_buffers_empty(struct isal_zstream *stream) 343 { 344 345 switch (stream->level) { 346 case 3: 347 return are_buffers_empty_hash_map(stream); 348 case 2: 349 return are_buffers_empty_hashX(stream); 350 default: 351 return are_buffers_empty_hashX(stream); 352 } 353 } 354 355 static void 356 create_icf_block_hdr(struct isal_zstream *stream, uint8_t *start_in) 357 { 358 struct isal_zstate *state = &stream->internal_state; 359 struct level_buf *level_buf = (struct level_buf *) stream->level_buf; 360 struct BitBuf2 *write_buf = &state->bitbuf; 361 struct BitBuf2 write_buf_tmp; 362 uint32_t out_size = stream->avail_out; 363 uint32_t avail_output, block_start_offset; 364 uint8_t *end_out = stream->next_out + out_size; 365 uint64_t cur_in_processed; 366 uint64_t bit_count; 367 uint64_t block_in_size = state->block_end - state->block_next; 368 uint64_t block_size; 369 int buffer_header = 0; 370 371 memcpy(&write_buf_tmp, write_buf, sizeof(struct BitBuf2)); 372 373 /* Calculate the bytes required to store a type 0 block. Need to account 374 * for bits stored in the bitbuf. Since 3 bits correspond to the deflate 375 * type 0 header, we need to add one byte more when the number of bits 376 * is at least 6 mod 8. */ 377 block_size = (TYPE0_BLK_HDR_LEN) * 378 ((block_in_size + TYPE0_MAX_BLK_LEN - 1) / TYPE0_MAX_BLK_LEN) + 379 block_in_size; 380 block_size = block_size ? block_size : TYPE0_BLK_HDR_LEN; 381 block_size += (write_buf->m_bit_count + 2) / 8; 382 383 /* Write EOB in icf_buf */ 384 level_buf->hist.ll_hist[256] = 1; 385 level_buf->icf_buf_next->lit_len = 0x100; 386 level_buf->icf_buf_next->lit_dist = NULL_DIST_SYM; 387 level_buf->icf_buf_next->dist_extra = 0; 388 level_buf->icf_buf_next++; 389 390 state->has_eob_hdr = (stream->end_of_stream && are_buffers_empty(stream)) ? 1 : 0; 391 392 if (end_out - stream->next_out >= ISAL_DEF_MAX_HDR_SIZE) { 393 /* Assumes ISAL_DEF_MAX_HDR_SIZE is large enough to contain a 394 * max length header and a gzip header */ 395 if (stream->gzip_flag == IGZIP_GZIP || stream->gzip_flag == IGZIP_ZLIB) 396 write_stream_header_stateless(stream); 397 set_buf(write_buf, stream->next_out, stream->avail_out); 398 buffer_header = 0; 399 400 } else { 401 /* Start writing into temporary buffer */ 402 set_buf(write_buf, level_buf->deflate_hdr, ISAL_DEF_MAX_HDR_SIZE); 403 buffer_header = 1; 404 } 405 406 bit_count = create_hufftables_icf(write_buf, &level_buf->encode_tables, &level_buf->hist, 407 state->has_eob_hdr); 408 409 /* Assumes that type 0 block has size less than 4G */ 410 block_start_offset = (stream->total_in - state->block_next); 411 cur_in_processed = stream->next_in - start_in; 412 avail_output = 413 stream->avail_out + sizeof(state->buffer) - (stream->total_in - state->block_end); 414 415 if (bit_count / 8 >= block_size && cur_in_processed >= block_start_offset && 416 block_size <= avail_output) { 417 /* Reset stream for writing out a type0 block */ 418 state->has_eob_hdr = 0; 419 memcpy(write_buf, &write_buf_tmp, sizeof(struct BitBuf2)); 420 state->state = ZSTATE_TYPE0_HDR; 421 422 } else if (buffer_header) { 423 /* Setup stream to write out a buffered header */ 424 level_buf->deflate_hdr_count = buffer_used(write_buf); 425 level_buf->deflate_hdr_extra_bits = write_buf->m_bit_count; 426 flush(write_buf); 427 memcpy(write_buf, &write_buf_tmp, sizeof(struct BitBuf2)); 428 write_buf->m_bits = 0; 429 write_buf->m_bit_count = 0; 430 state->state = ZSTATE_HDR; 431 432 } else { 433 stream->next_out = buffer_ptr(write_buf); 434 stream->total_out += buffer_used(write_buf); 435 stream->avail_out -= buffer_used(write_buf); 436 state->state = ZSTATE_FLUSH_ICF_BUFFER; 437 } 438 } 439 440 static void 441 isal_deflate_pass(struct isal_zstream *stream) 442 { 443 struct isal_zstate *state = &stream->internal_state; 444 struct isal_hufftables *hufftables = stream->hufftables; 445 uint8_t *start_in = stream->next_in; 446 447 if (state->state == ZSTATE_NEW_HDR || state->state == ZSTATE_HDR) { 448 if (state->count == 0) 449 /* Assume the final header is being written since the header 450 * stored in hufftables is the final header. */ 451 state->has_eob_hdr = 1; 452 write_header(stream, hufftables->deflate_hdr, hufftables->deflate_hdr_count, 453 hufftables->deflate_hdr_extra_bits, ZSTATE_BODY, 454 !stream->end_of_stream); 455 } 456 457 if (state->state == ZSTATE_BODY) 458 isal_deflate_body(stream); 459 460 if (state->state == ZSTATE_FLUSH_READ_BUFFER) 461 isal_deflate_finish(stream); 462 if (state->state == ZSTATE_SYNC_FLUSH) 463 sync_flush(stream); 464 465 if (state->state == ZSTATE_FLUSH_WRITE_BUFFER) 466 flush_write_buffer(stream); 467 468 if (stream->gzip_flag) 469 update_checksum(stream, start_in, stream->next_in - start_in); 470 471 if (state->state == ZSTATE_TRL) 472 write_trailer(stream); 473 } 474 475 static void 476 isal_deflate_icf_finish(struct isal_zstream *stream) 477 { 478 switch (stream->level) { 479 case 3: 480 isal_deflate_icf_finish_lvl3(stream); 481 break; 482 case 2: 483 isal_deflate_icf_finish_lvl2(stream); 484 break; 485 default: 486 isal_deflate_icf_finish_lvl1(stream); 487 } 488 } 489 490 static void 491 isal_deflate_icf_pass(struct isal_zstream *stream, uint8_t *inbuf_start) 492 { 493 uint8_t *start_in = stream->next_in; 494 struct isal_zstate *state = &stream->internal_state; 495 struct level_buf *level_buf = (struct level_buf *) stream->level_buf; 496 497 do { 498 if (state->state == ZSTATE_NEW_HDR) 499 init_new_icf_block(stream); 500 501 if (state->state == ZSTATE_BODY) 502 isal_deflate_icf_body(stream); 503 504 if (state->state == ZSTATE_FLUSH_READ_BUFFER) 505 isal_deflate_icf_finish(stream); 506 507 if (state->state == ZSTATE_CREATE_HDR) 508 create_icf_block_hdr(stream, inbuf_start); 509 510 if (state->state == ZSTATE_HDR) 511 /* Note that the header may be prepended by the 512 * remaining bits in the previous block, as such the 513 * toggle header flag cannot be used */ 514 write_header(stream, level_buf->deflate_hdr, level_buf->deflate_hdr_count, 515 level_buf->deflate_hdr_extra_bits, ZSTATE_FLUSH_ICF_BUFFER, 0); 516 517 if (state->state == ZSTATE_FLUSH_ICF_BUFFER) 518 flush_icf_block(stream); 519 520 if (state->state == ZSTATE_TYPE0_HDR || state->state == ZSTATE_TYPE0_BODY) { 521 if (stream->gzip_flag == IGZIP_GZIP || stream->gzip_flag == IGZIP_ZLIB) 522 write_stream_header(stream); 523 write_stored_block(stream); 524 } 525 526 } while (state->state == ZSTATE_NEW_HDR); 527 528 if (state->state == ZSTATE_SYNC_FLUSH) 529 sync_flush(stream); 530 531 if (state->state == ZSTATE_FLUSH_WRITE_BUFFER) 532 flush_write_buffer(stream); 533 534 if (stream->gzip_flag) 535 update_checksum(stream, start_in, stream->next_in - start_in); 536 537 if (state->state == ZSTATE_TRL) 538 write_trailer(stream); 539 } 540 541 static void 542 isal_deflate_int(struct isal_zstream *stream, uint8_t *start_in) 543 { 544 struct isal_zstate *state = &stream->internal_state; 545 uint32_t size; 546 547 /* Move data from temporary output buffer to output buffer */ 548 if (state->state >= ZSTATE_TMP_OFFSET) { 549 size = state->tmp_out_end - state->tmp_out_start; 550 if (size > stream->avail_out) 551 size = stream->avail_out; 552 memcpy(stream->next_out, state->tmp_out_buff + state->tmp_out_start, size); 553 stream->next_out += size; 554 stream->avail_out -= size; 555 stream->total_out += size; 556 state->tmp_out_start += size; 557 558 if (state->tmp_out_start == state->tmp_out_end) 559 state->state -= ZSTATE_TMP_OFFSET; 560 561 if (stream->avail_out == 0 || 562 state->state == ZSTATE_END 563 // or do not write out empty blocks since the outbuffer was processed 564 || (state->state == ZSTATE_NEW_HDR && stream->avail_out == 0)) 565 return; 566 } 567 assert(state->tmp_out_start == state->tmp_out_end); 568 569 if (stream->level == 0) 570 isal_deflate_pass(stream); 571 else 572 isal_deflate_icf_pass(stream, start_in); 573 574 /* Fill temporary output buffer then complete filling output buffer */ 575 if (stream->avail_out > 0 && stream->avail_out < 8 && state->state != ZSTATE_NEW_HDR) { 576 uint8_t *next_out; 577 uint32_t avail_out; 578 uint32_t total_out; 579 580 next_out = stream->next_out; 581 avail_out = stream->avail_out; 582 total_out = stream->total_out; 583 584 stream->next_out = state->tmp_out_buff; 585 stream->avail_out = sizeof(state->tmp_out_buff); 586 stream->total_out = 0; 587 588 if (stream->level == 0) 589 isal_deflate_pass(stream); 590 else 591 isal_deflate_icf_pass(stream, start_in); 592 593 state->tmp_out_start = 0; 594 state->tmp_out_end = stream->total_out; 595 596 stream->next_out = next_out; 597 stream->avail_out = avail_out; 598 stream->total_out = total_out; 599 if (state->tmp_out_end) { 600 size = state->tmp_out_end; 601 if (size > stream->avail_out) 602 size = stream->avail_out; 603 memcpy(stream->next_out, state->tmp_out_buff, size); 604 stream->next_out += size; 605 stream->avail_out -= size; 606 stream->total_out += size; 607 state->tmp_out_start += size; 608 if (state->tmp_out_start != state->tmp_out_end) 609 state->state += ZSTATE_TMP_OFFSET; 610 } 611 } 612 } 613 614 static void 615 write_constant_compressed_stateless(struct isal_zstream *stream, uint32_t repeated_length) 616 { 617 /* Assumes repeated_length is at least 1. 618 * Assumes the input end_of_stream is either 0 or 1. */ 619 struct isal_zstate *state = &stream->internal_state; 620 uint32_t rep_bits = ((repeated_length - 1) / 258) * 2; 621 uint32_t rep_bytes = rep_bits / 8; 622 uint32_t rep_extra = (repeated_length - 1) % 258; 623 uint32_t bytes; 624 uint32_t repeated_char = *stream->next_in; 625 uint8_t *start_in = stream->next_in; 626 627 /* Guarantee there is enough space for the header even in the worst case */ 628 if (stream->avail_out < HEADER_LENGTH + MAX_FIXUP_CODE_LENGTH + rep_bytes + 8) 629 return; 630 631 /* Assumes the repeated char is either 0 or 0xFF. */ 632 memcpy(stream->next_out, repeated_char_header[repeated_char & 1], HEADER_LENGTH); 633 634 if (stream->avail_in == repeated_length && stream->end_of_stream > 0) { 635 stream->next_out[0] |= 1; 636 state->has_eob_hdr = 1; 637 state->has_eob = 1; 638 state->state = ZSTATE_TRL; 639 } else { 640 state->state = ZSTATE_NEW_HDR; 641 } 642 643 memset(stream->next_out + HEADER_LENGTH, 0, rep_bytes); 644 stream->avail_out -= HEADER_LENGTH + rep_bytes; 645 stream->next_out += HEADER_LENGTH + rep_bytes; 646 stream->total_out += HEADER_LENGTH + rep_bytes; 647 648 set_buf(&state->bitbuf, stream->next_out, stream->avail_out); 649 650 /* These two lines are basically a modified version of init. */ 651 state->bitbuf.m_bits = 0; 652 state->bitbuf.m_bit_count = rep_bits % 8; 653 654 /* Add smaller repeat codes as necessary. Code280 can describe repeat 655 * lengths of 115-130 bits. Code10 can describe repeat lengths of 10 656 * bits. If more than 230 bits, fill code with two code280s. Else if 657 * more than 115 repeats, fill with code10s until one code280 can 658 * finish the rest of the repeats. Else, fill with code10s and 659 * literals */ 660 if (rep_extra > 115) { 661 while (rep_extra > 130 && rep_extra < 230) { 662 write_bits(&state->bitbuf, CODE_10, CODE_10_LENGTH); 663 rep_extra -= 10; 664 } 665 666 if (rep_extra >= 230) { 667 write_bits(&state->bitbuf, 668 CODE_280 | ((rep_extra / 2 - 115) << CODE_280_LENGTH), 669 CODE_280_TOTAL_LENGTH); 670 rep_extra -= rep_extra / 2; 671 } 672 673 write_bits(&state->bitbuf, CODE_280 | ((rep_extra - 115) << CODE_280_LENGTH), 674 CODE_280_TOTAL_LENGTH); 675 676 } else { 677 while (rep_extra >= 10) { 678 679 write_bits(&state->bitbuf, CODE_10, CODE_10_LENGTH); 680 rep_extra -= 10; 681 } 682 683 for (; rep_extra > 0; rep_extra--) 684 write_bits(&state->bitbuf, CODE_LIT, CODE_LIT_LENGTH); 685 } 686 687 write_bits(&state->bitbuf, END_OF_BLOCK, END_OF_BLOCK_LEN); 688 689 stream->next_in += repeated_length; 690 stream->avail_in -= repeated_length; 691 stream->total_in += repeated_length; 692 state->block_end += repeated_length; 693 694 bytes = buffer_used(&state->bitbuf); 695 stream->next_out = buffer_ptr(&state->bitbuf); 696 stream->avail_out -= bytes; 697 stream->total_out += bytes; 698 699 if (stream->gzip_flag) 700 update_checksum(stream, start_in, stream->next_in - start_in); 701 702 return; 703 } 704 705 static int 706 detect_repeated_char_length(uint8_t *in, uint32_t length) 707 { 708 /* This currently assumes the first 8 bytes are the same character. 709 * This won't work effectively if the input stream isn't aligned well. */ 710 uint8_t *p_8, *end = in + length; 711 uint64_t *p_64 = (uint64_t *) in; 712 uint64_t w = *p_64; 713 uint8_t c = (uint8_t) w; 714 715 for (; (p_64 <= (uint64_t *) (end - 8)) && (w == *p_64); p_64++) 716 ; 717 718 p_8 = (uint8_t *) p_64; 719 720 for (; (p_8 < end) && (c == *p_8); p_8++) 721 ; 722 723 return p_8 - in; 724 } 725 726 static int 727 isal_deflate_int_stateless(struct isal_zstream *stream) 728 { 729 uint32_t repeat_length; 730 struct isal_zstate *state = &stream->internal_state; 731 732 if (stream->gzip_flag == IGZIP_GZIP || stream->gzip_flag == IGZIP_ZLIB) 733 if (write_stream_header_stateless(stream)) 734 return STATELESS_OVERFLOW; 735 736 if (stream->avail_in >= 8 && (load_native_u64(stream->next_in) == 0 || 737 load_native_u64(stream->next_in) == ~(uint64_t) 0)) { 738 repeat_length = detect_repeated_char_length(stream->next_in, stream->avail_in); 739 740 if (stream->avail_in == repeat_length || repeat_length >= MIN_REPEAT_LEN) 741 write_constant_compressed_stateless(stream, repeat_length); 742 } 743 744 if (stream->level == 0) { 745 if (state->state == ZSTATE_NEW_HDR || state->state == ZSTATE_HDR) { 746 write_deflate_header_unaligned_stateless(stream); 747 if (state->state == ZSTATE_NEW_HDR || state->state == ZSTATE_HDR) 748 return STATELESS_OVERFLOW; 749 750 reset_match_history(stream); 751 } 752 753 isal_deflate_pass(stream); 754 755 } else if (stream->level <= ISAL_DEF_MAX_LEVEL) { 756 if (state->state == ZSTATE_NEW_HDR || state->state == ZSTATE_HDR) 757 reset_match_history(stream); 758 759 state->count = 0; 760 isal_deflate_icf_pass(stream, stream->next_in); 761 } 762 763 if (state->state == ZSTATE_END || 764 (state->state == ZSTATE_NEW_HDR && stream->flush == FULL_FLUSH)) 765 return COMP_OK; 766 else 767 return STATELESS_OVERFLOW; 768 } 769 770 static void 771 write_type0_header(struct isal_zstream *stream) 772 { 773 struct isal_zstate *state = &stream->internal_state; 774 uint64_t stored_blk_hdr; 775 uint32_t copy_size; 776 uint32_t memcpy_len, avail_in; 777 uint32_t block_in_size = state->block_end - state->block_next; 778 uint32_t block_next_offset; 779 struct BitBuf2 *bitbuf = &stream->internal_state.bitbuf; 780 781 if (block_in_size > TYPE0_MAX_BLK_LEN) { 782 stored_blk_hdr = 0xFFFF; 783 copy_size = TYPE0_MAX_BLK_LEN; 784 } else { 785 stored_blk_hdr = ~block_in_size; 786 stored_blk_hdr <<= 16; 787 stored_blk_hdr |= (block_in_size & 0xFFFF); 788 copy_size = block_in_size; 789 790 /* Handle BFINAL bit */ 791 block_next_offset = stream->total_in - state->block_next; 792 avail_in = stream->avail_in + block_next_offset; 793 if (stream->end_of_stream && avail_in == block_in_size) 794 stream->internal_state.has_eob_hdr = 1; 795 } 796 797 if (bitbuf->m_bit_count == 0 && stream->avail_out >= TYPE0_HDR_LEN + 1) { 798 stored_blk_hdr = stored_blk_hdr << 8; 799 stored_blk_hdr |= stream->internal_state.has_eob_hdr; 800 memcpy_len = TYPE0_HDR_LEN + 1; 801 stored_blk_hdr = to_le64(stored_blk_hdr); 802 memcpy(stream->next_out, &stored_blk_hdr, memcpy_len); 803 } else if (stream->avail_out >= 8) { 804 set_buf(bitbuf, stream->next_out, stream->avail_out); 805 write_bits_flush(bitbuf, stream->internal_state.has_eob_hdr, 3); 806 stream->next_out = buffer_ptr(bitbuf); 807 stream->total_out += buffer_used(bitbuf); 808 stream->avail_out -= buffer_used(bitbuf); 809 memcpy_len = TYPE0_HDR_LEN; 810 stored_blk_hdr = to_le64(stored_blk_hdr); 811 memcpy(stream->next_out, &stored_blk_hdr, memcpy_len); 812 } else { 813 stream->internal_state.has_eob_hdr = 0; 814 return; 815 } 816 817 stream->next_out += memcpy_len; 818 stream->avail_out -= memcpy_len; 819 stream->total_out += memcpy_len; 820 stream->internal_state.state = ZSTATE_TYPE0_BODY; 821 822 stream->internal_state.count = copy_size; 823 } 824 825 static uint32_t 826 write_stored_block(struct isal_zstream *stream) 827 { 828 uint32_t copy_size, avail_in, block_next_offset; 829 uint8_t *next_in; 830 struct isal_zstate *state = &stream->internal_state; 831 832 do { 833 if (state->state == ZSTATE_TYPE0_HDR) { 834 write_type0_header(stream); 835 if (state->state == ZSTATE_TYPE0_HDR) 836 break; 837 } 838 839 assert(state->count <= state->block_end - state->block_next); 840 copy_size = state->count; 841 842 block_next_offset = stream->total_in - state->block_next; 843 next_in = stream->next_in - block_next_offset; 844 avail_in = stream->avail_in + block_next_offset; 845 846 if (copy_size > stream->avail_out || copy_size > avail_in) { 847 state->count = copy_size; 848 copy_size = (stream->avail_out <= avail_in) ? stream->avail_out : avail_in; 849 850 memcpy(stream->next_out, next_in, copy_size); 851 state->count -= copy_size; 852 } else { 853 memcpy(stream->next_out, next_in, copy_size); 854 855 state->count = 0; 856 state->state = ZSTATE_TYPE0_HDR; 857 } 858 859 state->block_next += copy_size; 860 stream->next_out += copy_size; 861 stream->avail_out -= copy_size; 862 stream->total_out += copy_size; 863 864 if (state->block_next == state->block_end) { 865 state->state = state->has_eob_hdr ? ZSTATE_TRL : ZSTATE_NEW_HDR; 866 if (stream->flush == FULL_FLUSH && state->state == ZSTATE_NEW_HDR && 867 are_buffers_empty(stream)) { 868 /* Clear match history so there are no cross 869 * block length distance pairs */ 870 reset_match_history(stream); 871 } 872 } 873 } while (state->state == ZSTATE_TYPE0_HDR); 874 875 return state->block_end - state->block_next; 876 } 877 878 static inline void 879 reset_match_history(struct isal_zstream *stream) 880 { 881 struct isal_zstate *state = &stream->internal_state; 882 struct level_buf *level_buf = (struct level_buf *) stream->level_buf; 883 uint16_t *hash_table; 884 uint32_t hash_table_size; 885 886 hash_table_size = 2 * (state->hash_mask + 1); 887 888 switch (stream->level) { 889 case 3: 890 hash_table = level_buf->lvl3.hash_table; 891 break; 892 case 2: 893 hash_table = level_buf->lvl2.hash_table; 894 break; 895 case 1: 896 hash_table = level_buf->lvl1.hash_table; 897 break; 898 default: 899 hash_table = state->head; 900 } 901 902 state->has_hist = IGZIP_NO_HIST; 903 904 /* There is definitely more than 16 bytes in the hash table. Set this 905 * minimum to avoid a wmemset of size 0 */ 906 if (hash_table_size <= sizeof(wchar_t)) 907 hash_table_size = sizeof(wchar_t); 908 909 if (sizeof(wchar_t) == 2) { 910 uint16_t hash_init_val; 911 912 hash_init_val = stream->total_in & 0xffff; 913 wmemset((wchar_t *) hash_table, hash_init_val, hash_table_size / sizeof(wchar_t)); 914 915 } else if (sizeof(wchar_t) == 4) { 916 uint32_t hash_init_val; 917 int rep_bits; 918 919 hash_init_val = stream->total_in & 0xffff; 920 for (rep_bits = sizeof(uint16_t) * 8; rep_bits < sizeof(wchar_t) * 8; rep_bits *= 2) 921 hash_init_val |= hash_init_val << rep_bits; 922 923 wmemset((wchar_t *) hash_table, hash_init_val, hash_table_size / sizeof(wchar_t)); 924 } else { 925 if ((stream->total_in & 0xFFFF) == 0) 926 memset(hash_table, 0, hash_table_size); 927 else { 928 int i; 929 for (i = 0; i < hash_table_size / 2; i++) { 930 hash_table[i] = (uint16_t) (stream->total_in); 931 } 932 } 933 } 934 } 935 936 static void inline set_dist_mask(struct isal_zstream *stream) 937 { 938 struct isal_zstate *state = &stream->internal_state; 939 uint32_t hist_size; 940 941 if (stream->hist_bits > ISAL_DEF_MAX_HIST_BITS || stream->hist_bits == 0) 942 stream->hist_bits = ISAL_DEF_MAX_HIST_BITS; 943 944 hist_size = (1 << (stream->hist_bits)); 945 state->dist_mask = hist_size - 1; 946 947 if (IGZIP_HIST_SIZE < ISAL_DEF_HIST_SIZE && state->dist_mask > IGZIP_HIST_SIZE - 1) 948 state->dist_mask = IGZIP_HIST_SIZE - 1; 949 } 950 951 static void inline set_hash_mask(struct isal_zstream *stream) 952 { 953 struct isal_zstate *state = &stream->internal_state; 954 955 switch (stream->level) { 956 case 3: 957 state->hash_mask = LVL3_HASH_MASK; 958 break; 959 case 2: 960 state->hash_mask = LVL2_HASH_MASK; 961 break; 962 case 1: 963 state->hash_mask = LVL1_HASH_MASK; 964 break; 965 case 0: 966 state->hash_mask = LVL0_HASH_MASK; 967 } 968 } 969 970 void 971 isal_deflate_init(struct isal_zstream *stream) 972 { 973 struct isal_zstate *state = &stream->internal_state; 974 975 stream->total_in = 0; 976 stream->total_out = 0; 977 stream->hufftables = (struct isal_hufftables *) &hufftables_default; 978 stream->level = 0; 979 stream->level_buf = NULL; 980 stream->level_buf_size = 0; 981 stream->end_of_stream = 0; 982 stream->flush = NO_FLUSH; 983 stream->gzip_flag = 0; 984 stream->hist_bits = 0; 985 986 state->block_next = 0; 987 state->block_end = 0; 988 state->b_bytes_valid = 0; 989 state->b_bytes_processed = 0; 990 state->total_in_start = 0; 991 state->has_wrap_hdr = 0; 992 state->has_eob = 0; 993 state->has_eob_hdr = 0; 994 state->has_hist = IGZIP_NO_HIST; 995 state->has_level_buf_init = 0; 996 state->state = ZSTATE_NEW_HDR; 997 state->count = 0; 998 999 state->tmp_out_start = 0; 1000 state->tmp_out_end = 0; 1001 1002 init(&state->bitbuf); 1003 1004 state->crc = 0; 1005 1006 return; 1007 } 1008 1009 void 1010 isal_deflate_reset(struct isal_zstream *stream) 1011 { 1012 struct isal_zstate *state = &stream->internal_state; 1013 1014 stream->total_in = 0; 1015 stream->total_out = 0; 1016 1017 state->block_next = 0; 1018 state->block_end = 0; 1019 state->b_bytes_valid = 0; 1020 state->b_bytes_processed = 0; 1021 state->total_in_start = 0; 1022 state->has_wrap_hdr = 0; 1023 state->has_eob = 0; 1024 state->has_level_buf_init = 0; 1025 state->has_eob_hdr = 0; 1026 state->has_hist = IGZIP_NO_HIST; 1027 state->state = ZSTATE_NEW_HDR; 1028 state->count = 0; 1029 1030 state->tmp_out_start = 0; 1031 state->tmp_out_end = 0; 1032 1033 init(&state->bitbuf); 1034 1035 state->crc = 0; 1036 } 1037 1038 void 1039 isal_gzip_header_init(struct isal_gzip_header *gz_hdr) 1040 { 1041 gz_hdr->text = 0; 1042 gz_hdr->time = 0; 1043 gz_hdr->xflags = 0; 1044 gz_hdr->os = 0xff; 1045 gz_hdr->extra = NULL; 1046 gz_hdr->extra_buf_len = 0; 1047 gz_hdr->extra_len = 0; 1048 gz_hdr->name = NULL; 1049 gz_hdr->name_buf_len = 0; 1050 gz_hdr->comment = NULL; 1051 gz_hdr->comment_buf_len = 0; 1052 gz_hdr->hcrc = 0; 1053 gz_hdr->flags = 0; 1054 } 1055 1056 void 1057 isal_zlib_header_init(struct isal_zlib_header *z_hdr) 1058 { 1059 z_hdr->info = 0; 1060 z_hdr->level = 0; 1061 z_hdr->dict_id = 0; 1062 z_hdr->dict_flag = 0; 1063 } 1064 1065 uint32_t 1066 isal_write_gzip_header(struct isal_zstream *stream, struct isal_gzip_header *gz_hdr) 1067 { 1068 uint32_t flags = 0, hcrc, hdr_size = GZIP_HDR_BASE; 1069 uint8_t *out_buf = stream->next_out, *out_buf_start = stream->next_out; 1070 uint32_t name_len = 0, comment_len = 0; 1071 1072 if (gz_hdr->text) 1073 flags |= TEXT_FLAG; 1074 if (gz_hdr->extra) { 1075 flags |= EXTRA_FLAG; 1076 hdr_size += GZIP_EXTRA_LEN + gz_hdr->extra_len; 1077 } 1078 if (gz_hdr->name) { 1079 flags |= NAME_FLAG; 1080 name_len = strnlen(gz_hdr->name, gz_hdr->name_buf_len); 1081 if (name_len < gz_hdr->name_buf_len) 1082 name_len++; 1083 hdr_size += name_len; 1084 } 1085 if (gz_hdr->comment) { 1086 flags |= COMMENT_FLAG; 1087 comment_len = strnlen(gz_hdr->comment, gz_hdr->comment_buf_len); 1088 if (comment_len < gz_hdr->comment_buf_len) 1089 comment_len++; 1090 hdr_size += comment_len; 1091 } 1092 if (gz_hdr->hcrc) { 1093 flags |= HCRC_FLAG; 1094 hdr_size += GZIP_HCRC_LEN; 1095 } 1096 1097 if (stream->avail_out < hdr_size) 1098 return hdr_size; 1099 1100 out_buf[0] = 0x1f; 1101 out_buf[1] = 0x8b; 1102 out_buf[2] = DEFLATE_METHOD; 1103 out_buf[3] = flags; 1104 store_le_u32(out_buf + 4, gz_hdr->time); 1105 out_buf[8] = gz_hdr->xflags; 1106 out_buf[9] = gz_hdr->os; 1107 1108 out_buf += GZIP_HDR_BASE; 1109 if (flags & EXTRA_FLAG) { 1110 store_le_u16(out_buf, gz_hdr->extra_len); 1111 out_buf += GZIP_EXTRA_LEN; 1112 1113 memcpy(out_buf, gz_hdr->extra, gz_hdr->extra_len); 1114 out_buf += gz_hdr->extra_len; 1115 } 1116 1117 if (flags & NAME_FLAG) { 1118 memcpy(out_buf, gz_hdr->name, name_len); 1119 out_buf += name_len; 1120 } 1121 1122 if (flags & COMMENT_FLAG) { 1123 memcpy(out_buf, gz_hdr->comment, comment_len); 1124 out_buf += comment_len; 1125 } 1126 1127 if (flags & HCRC_FLAG) { 1128 hcrc = crc32_gzip_refl(0, out_buf_start, out_buf - out_buf_start); 1129 store_le_u16(out_buf, hcrc); 1130 out_buf += GZIP_HCRC_LEN; 1131 } 1132 1133 stream->next_out += hdr_size; 1134 stream->total_out += hdr_size; 1135 stream->avail_out -= hdr_size; 1136 1137 return ISAL_DECOMP_OK; 1138 } 1139 1140 uint32_t 1141 isal_write_zlib_header(struct isal_zstream *stream, struct isal_zlib_header *z_hdr) 1142 { 1143 uint32_t cmf, flg, dict_flag = 0, hdr_size = ZLIB_HDR_BASE; 1144 uint8_t *out_buf = stream->next_out; 1145 1146 if (z_hdr->dict_flag) { 1147 dict_flag = ZLIB_DICT_FLAG; 1148 hdr_size = ZLIB_HDR_BASE + ZLIB_DICT_LEN; 1149 } 1150 1151 if (stream->avail_out < hdr_size) 1152 return hdr_size; 1153 1154 cmf = DEFLATE_METHOD | (z_hdr->info << 4); 1155 flg = (z_hdr->level << 6) | dict_flag; 1156 1157 flg += 31 - ((256 * cmf + flg) % 31); 1158 1159 out_buf[0] = cmf; 1160 out_buf[1] = flg; 1161 1162 if (dict_flag) 1163 store_le_u32(out_buf + 2, z_hdr->dict_id); 1164 1165 stream->next_out += hdr_size; 1166 stream->total_out += hdr_size; 1167 stream->avail_out -= hdr_size; 1168 1169 return ISAL_DECOMP_OK; 1170 } 1171 1172 int 1173 isal_deflate_set_hufftables(struct isal_zstream *stream, struct isal_hufftables *hufftables, 1174 int type) 1175 { 1176 if (stream->internal_state.state != ZSTATE_NEW_HDR) 1177 return ISAL_INVALID_OPERATION; 1178 1179 switch (type) { 1180 case IGZIP_HUFFTABLE_DEFAULT: 1181 stream->hufftables = (struct isal_hufftables *) &hufftables_default; 1182 break; 1183 case IGZIP_HUFFTABLE_STATIC: 1184 stream->hufftables = (struct isal_hufftables *) &hufftables_static; 1185 break; 1186 case IGZIP_HUFFTABLE_CUSTOM: 1187 if (hufftables != NULL) { 1188 stream->hufftables = hufftables; 1189 break; 1190 } 1191 default: 1192 return ISAL_INVALID_OPERATION; 1193 } 1194 1195 return COMP_OK; 1196 } 1197 1198 void 1199 isal_deflate_stateless_init(struct isal_zstream *stream) 1200 { 1201 stream->total_in = 0; 1202 stream->total_out = 0; 1203 stream->hufftables = (struct isal_hufftables *) &hufftables_default; 1204 stream->level = 0; 1205 stream->level_buf = NULL; 1206 stream->level_buf_size = 0; 1207 stream->end_of_stream = 0; 1208 stream->flush = NO_FLUSH; 1209 stream->gzip_flag = 0; 1210 stream->hist_bits = 0; 1211 stream->internal_state.has_wrap_hdr = 0; 1212 stream->internal_state.state = ZSTATE_NEW_HDR; 1213 return; 1214 } 1215 1216 void 1217 isal_deflate_hash(struct isal_zstream *stream, uint8_t *dict, uint32_t dict_len) 1218 { 1219 /* Reset history to prevent out of bounds matches this works because 1220 * dictionary must set at least 1 element in the history */ 1221 struct level_buf *level_buf = (struct level_buf *) stream->level_buf; 1222 uint32_t hash_mask = stream->internal_state.hash_mask; 1223 1224 switch (stream->level) { 1225 case 3: 1226 memset(level_buf->lvl3.hash_table, -1, sizeof(level_buf->lvl3.hash_table)); 1227 isal_deflate_hash_lvl3(level_buf->lvl3.hash_table, hash_mask, stream->total_in, 1228 dict, dict_len); 1229 break; 1230 1231 case 2: 1232 memset(level_buf->lvl2.hash_table, -1, sizeof(level_buf->lvl2.hash_table)); 1233 isal_deflate_hash_lvl2(level_buf->lvl2.hash_table, hash_mask, stream->total_in, 1234 dict, dict_len); 1235 break; 1236 case 1: 1237 memset(level_buf->lvl1.hash_table, -1, sizeof(level_buf->lvl1.hash_table)); 1238 isal_deflate_hash_lvl1(level_buf->lvl1.hash_table, hash_mask, stream->total_in, 1239 dict, dict_len); 1240 break; 1241 default: 1242 memset(stream->internal_state.head, -1, sizeof(stream->internal_state.head)); 1243 isal_deflate_hash_lvl0(stream->internal_state.head, hash_mask, stream->total_in, 1244 dict, dict_len); 1245 } 1246 1247 stream->internal_state.has_hist = IGZIP_HIST; 1248 } 1249 1250 int 1251 isal_deflate_process_dict(struct isal_zstream *stream, struct isal_dict *dict, uint8_t *dict_data, 1252 uint32_t dict_len) 1253 { 1254 if ((dict == NULL) || (dict_len == 0) || (dict->level > ISAL_DEF_MAX_LEVEL)) 1255 return ISAL_INVALID_STATE; 1256 1257 if (dict_len > IGZIP_HIST_SIZE) { 1258 dict_data = dict_data + dict_len - IGZIP_HIST_SIZE; 1259 dict_len = IGZIP_HIST_SIZE; 1260 } 1261 1262 dict->level = stream->level; 1263 dict->hist_size = dict_len; 1264 memcpy(dict->history, dict_data, dict_len); 1265 memset(dict->hashtable, -1, sizeof(dict->hashtable)); 1266 1267 switch (stream->level) { 1268 case 3: 1269 dict->hash_size = IGZIP_LVL3_HASH_SIZE; 1270 isal_deflate_hash_lvl3(dict->hashtable, LVL3_HASH_MASK, 0, dict_data, dict_len); 1271 break; 1272 1273 case 2: 1274 dict->hash_size = IGZIP_LVL2_HASH_SIZE; 1275 isal_deflate_hash_lvl2(dict->hashtable, LVL2_HASH_MASK, 0, dict_data, dict_len); 1276 break; 1277 case 1: 1278 dict->hash_size = IGZIP_LVL1_HASH_SIZE; 1279 isal_deflate_hash_lvl1(dict->hashtable, LVL1_HASH_MASK, 0, dict_data, dict_len); 1280 break; 1281 default: 1282 dict->hash_size = IGZIP_LVL0_HASH_SIZE; 1283 isal_deflate_hash_lvl0(dict->hashtable, LVL0_HASH_MASK, 0, dict_data, dict_len); 1284 } 1285 return COMP_OK; 1286 } 1287 1288 int 1289 isal_deflate_reset_dict(struct isal_zstream *stream, struct isal_dict *dict) 1290 { 1291 struct isal_zstate *state = &stream->internal_state; 1292 struct level_buf *level_buf = (struct level_buf *) stream->level_buf; 1293 int ret; 1294 1295 if ((state->state != ZSTATE_NEW_HDR) || 1296 (state->b_bytes_processed != state->b_bytes_valid) || (dict->level != stream->level) || 1297 (dict->hist_size == 0) || (dict->hist_size > IGZIP_HIST_SIZE) || 1298 (dict->hash_size > IGZIP_LVL3_HASH_SIZE)) 1299 return ISAL_INVALID_STATE; 1300 1301 ret = check_level_req(stream); 1302 if (ret) 1303 return ret; 1304 1305 memcpy(state->buffer, dict->history, dict->hist_size); 1306 state->b_bytes_processed = dict->hist_size; 1307 state->b_bytes_valid = dict->hist_size; 1308 state->has_hist = IGZIP_DICT_HASH_SET; 1309 1310 switch (stream->level) { 1311 case 3: 1312 memcpy(level_buf->lvl3.hash_table, dict->hashtable, 1313 sizeof(level_buf->lvl3.hash_table)); 1314 break; 1315 1316 case 2: 1317 memcpy(level_buf->lvl2.hash_table, dict->hashtable, 1318 sizeof(level_buf->lvl2.hash_table)); 1319 break; 1320 case 1: 1321 memcpy(level_buf->lvl1.hash_table, dict->hashtable, 1322 sizeof(level_buf->lvl1.hash_table)); 1323 break; 1324 default: 1325 memcpy(stream->internal_state.head, dict->hashtable, 1326 sizeof(stream->internal_state.head)); 1327 } 1328 1329 return COMP_OK; 1330 } 1331 1332 int 1333 isal_deflate_set_dict(struct isal_zstream *stream, uint8_t *dict, uint32_t dict_len) 1334 { 1335 struct isal_zstate *state = &stream->internal_state; 1336 1337 if (state->state != ZSTATE_NEW_HDR || state->b_bytes_processed != state->b_bytes_valid) 1338 return ISAL_INVALID_STATE; 1339 1340 if (dict_len <= 0) 1341 return COMP_OK; 1342 1343 if (dict_len > IGZIP_HIST_SIZE) { 1344 dict = dict + dict_len - IGZIP_HIST_SIZE; 1345 dict_len = IGZIP_HIST_SIZE; 1346 } 1347 1348 memcpy(state->buffer, dict, dict_len); 1349 state->b_bytes_processed = dict_len; 1350 state->b_bytes_valid = dict_len; 1351 1352 state->has_hist = IGZIP_DICT_HIST; 1353 1354 return COMP_OK; 1355 } 1356 1357 int 1358 isal_deflate_stateless(struct isal_zstream *stream) 1359 { 1360 struct isal_zstate *state = &stream->internal_state; 1361 uint8_t *next_in = stream->next_in; 1362 const uint32_t avail_in = stream->avail_in; 1363 const uint32_t total_in = stream->total_in; 1364 1365 uint8_t *next_out = stream->next_out; 1366 const uint32_t avail_out = stream->avail_out; 1367 const uint32_t total_out = stream->total_out; 1368 const uint32_t gzip_flag = stream->gzip_flag; 1369 const uint32_t has_wrap_hdr = state->has_wrap_hdr; 1370 1371 int level_check; 1372 uint64_t stored_len; 1373 1374 /* Final block has already been written */ 1375 state->block_next = stream->total_in; 1376 state->block_end = stream->total_in; 1377 state->has_eob_hdr = 0; 1378 init(&state->bitbuf); 1379 state->state = ZSTATE_NEW_HDR; 1380 state->crc = 0; 1381 state->has_level_buf_init = 0; 1382 set_dist_mask(stream); 1383 1384 if (stream->flush == NO_FLUSH) 1385 stream->end_of_stream = 1; 1386 1387 if (stream->flush != NO_FLUSH && stream->flush != FULL_FLUSH) 1388 return INVALID_FLUSH; 1389 1390 level_check = check_level_req(stream); 1391 if (level_check) { 1392 if (stream->level == 1 && stream->level_buf == NULL) { 1393 /* Default to internal buffer if invalid size is supplied */ 1394 stream->level_buf = state->buffer; 1395 stream->level_buf_size = sizeof(state->buffer) + sizeof(state->head); 1396 } else 1397 return level_check; 1398 } 1399 1400 set_hash_mask(stream); 1401 1402 if (state->hash_mask > 2 * avail_in) 1403 state->hash_mask = (1 << bsr(avail_in)) - 1; 1404 1405 if (avail_in == 0) 1406 stored_len = TYPE0_BLK_HDR_LEN; 1407 else { 1408 stored_len = TYPE0_BLK_HDR_LEN * 1409 ((avail_in + TYPE0_MAX_BLK_LEN - 1) / TYPE0_MAX_BLK_LEN); 1410 stored_len += avail_in; 1411 } 1412 1413 /* 1414 at least 1 byte compressed data in the case of empty dynamic block which only 1415 contains the EOB 1416 */ 1417 if (stream->gzip_flag == IGZIP_GZIP) 1418 stored_len += gzip_hdr_bytes + gzip_trl_bytes; 1419 else if (stream->gzip_flag == IGZIP_GZIP_NO_HDR) 1420 stored_len += gzip_trl_bytes; 1421 1422 else if (stream->gzip_flag == IGZIP_ZLIB) 1423 stored_len += zlib_hdr_bytes + zlib_trl_bytes; 1424 1425 else if (stream->gzip_flag == IGZIP_ZLIB_NO_HDR) 1426 stored_len += zlib_trl_bytes; 1427 1428 if (avail_out >= stored_len) 1429 stream->avail_out = stored_len; 1430 1431 if (isal_deflate_int_stateless(stream) == COMP_OK) { 1432 if (avail_out >= stored_len) 1433 stream->avail_out += avail_out - stored_len; 1434 return COMP_OK; 1435 } else { 1436 if (avail_out >= stored_len) 1437 stream->avail_out += avail_out - stored_len; 1438 if (stream->flush == FULL_FLUSH) { 1439 reset_match_history(stream); 1440 } 1441 stream->internal_state.has_eob_hdr = 0; 1442 } 1443 1444 if (avail_out < stored_len) 1445 return STATELESS_OVERFLOW; 1446 1447 stream->next_in = next_in + avail_in; 1448 stream->avail_in = 0; 1449 stream->total_in = avail_in; 1450 1451 state->block_next = stream->total_in - avail_in; 1452 state->block_end = stream->total_in; 1453 1454 stream->next_out = next_out; 1455 stream->avail_out = avail_out; 1456 stream->total_out = total_out; 1457 1458 stream->gzip_flag = gzip_flag; 1459 state->has_wrap_hdr = has_wrap_hdr; 1460 init(&stream->internal_state.bitbuf); 1461 stream->internal_state.count = 0; 1462 1463 if (stream->gzip_flag == IGZIP_GZIP || stream->gzip_flag == IGZIP_ZLIB) 1464 write_stream_header_stateless(stream); 1465 1466 stream->internal_state.state = ZSTATE_TYPE0_HDR; 1467 1468 write_stored_block(stream); 1469 1470 stream->total_in = total_in + avail_in; 1471 1472 if (stream->gzip_flag) { 1473 stream->internal_state.crc = 0; 1474 update_checksum(stream, next_in, avail_in); 1475 } 1476 1477 if (stream->end_of_stream) 1478 write_trailer(stream); 1479 1480 return COMP_OK; 1481 } 1482 1483 static inline uint32_t 1484 get_hist_size(struct isal_zstream *stream, uint8_t *start_in, int32_t buf_hist_start) 1485 { 1486 struct isal_zstate *state = &stream->internal_state; 1487 uint32_t history_size; 1488 uint32_t buffered_history; 1489 uint32_t buffered_size = state->b_bytes_valid - state->b_bytes_processed; 1490 uint32_t input_history; 1491 1492 buffered_history = (state->has_hist) ? state->b_bytes_processed - buf_hist_start : 0; 1493 input_history = stream->next_in - start_in; 1494 1495 /* Calculate history required for deflate window */ 1496 history_size = (buffered_history >= input_history) ? buffered_history : input_history; 1497 if (history_size > IGZIP_HIST_SIZE) 1498 history_size = IGZIP_HIST_SIZE; 1499 1500 /* Calculate history required based on internal state */ 1501 if (state->state == ZSTATE_TYPE0_HDR || state->state == ZSTATE_TYPE0_BODY || 1502 state->state == ZSTATE_TMP_TYPE0_HDR || state->state == ZSTATE_TMP_TYPE0_BODY) { 1503 if (stream->total_in - state->block_next > history_size) { 1504 history_size = (stream->total_in - state->block_next); 1505 } 1506 } else if (stream->avail_in + buffered_size == 0 && 1507 (stream->end_of_stream || stream->flush == FULL_FLUSH)) { 1508 history_size = 0; 1509 } 1510 return history_size; 1511 } 1512 1513 int 1514 isal_deflate(struct isal_zstream *stream) 1515 { 1516 struct isal_zstate *state = &stream->internal_state; 1517 int ret = COMP_OK; 1518 uint8_t *next_in, *start_in, *buf_start_in, *next_in_pre; 1519 uint32_t avail_in, total_start, hist_size, future_size; 1520 uint32_t in_size, in_size_initial, out_size, out_size_initial; 1521 uint32_t processed, buffered_size = state->b_bytes_valid - state->b_bytes_processed; 1522 uint32_t flush_type = stream->flush; 1523 uint32_t end_of_stream = stream->end_of_stream; 1524 uint32_t size = 0; 1525 int32_t buf_hist_start = 0; 1526 uint8_t *copy_down_src = NULL; 1527 uint64_t copy_down_size = 0, copy_start_offset; 1528 int internal; 1529 1530 if (stream->flush >= 3) 1531 return INVALID_FLUSH; 1532 1533 ret = check_level_req(stream); 1534 if (ret) 1535 return ret; 1536 1537 start_in = stream->next_in; 1538 total_start = stream->total_in; 1539 1540 hist_size = get_hist_size(stream, start_in, buf_hist_start); 1541 1542 if (state->has_hist == IGZIP_NO_HIST) { 1543 set_dist_mask(stream); 1544 set_hash_mask(stream); 1545 if (state->hash_mask > 2 * stream->avail_in && 1546 (stream->flush == FULL_FLUSH || stream->end_of_stream)) 1547 state->hash_mask = (1 << bsr(2 * stream->avail_in)) - 1; 1548 stream->total_in -= buffered_size; 1549 reset_match_history(stream); 1550 stream->total_in += buffered_size; 1551 buf_hist_start = state->b_bytes_processed; 1552 1553 } else if (state->has_hist == IGZIP_DICT_HIST) { 1554 set_dist_mask(stream); 1555 set_hash_mask(stream); 1556 isal_deflate_hash(stream, state->buffer, state->b_bytes_processed); 1557 } else if (state->has_hist == IGZIP_DICT_HASH_SET) { 1558 set_dist_mask(stream); 1559 set_hash_mask(stream); 1560 } 1561 1562 in_size = stream->avail_in + buffered_size; 1563 out_size = stream->total_out; 1564 do { 1565 in_size_initial = in_size; 1566 out_size_initial = out_size; 1567 buf_start_in = start_in; 1568 internal = 0; 1569 1570 /* Setup to compress from internal buffer if insufficient history */ 1571 if (stream->total_in - total_start < hist_size + buffered_size) { 1572 /* On entry there should always be sufficient history bufferd */ 1573 /* assert(state->b_bytes_processed >= hist_size); */ 1574 1575 internal = 1; 1576 /* Shift down internal buffer if it contains more data 1577 * than required */ 1578 if (state->b_bytes_processed > hist_size) { 1579 copy_start_offset = state->b_bytes_processed - hist_size; 1580 1581 copy_down_src = &state->buffer[copy_start_offset]; 1582 copy_down_size = state->b_bytes_valid - copy_start_offset; 1583 memmove(state->buffer, copy_down_src, copy_down_size); 1584 1585 state->b_bytes_valid -= copy_down_src - state->buffer; 1586 state->b_bytes_processed -= copy_down_src - state->buffer; 1587 buf_hist_start -= copy_down_src - state->buffer; 1588 if (buf_hist_start < 0) 1589 buf_hist_start = 0; 1590 } 1591 1592 size = stream->avail_in; 1593 if (size > sizeof(state->buffer) - state->b_bytes_valid) 1594 size = sizeof(state->buffer) - state->b_bytes_valid; 1595 1596 memcpy(&state->buffer[state->b_bytes_valid], stream->next_in, size); 1597 1598 stream->next_in += size; 1599 stream->avail_in -= size; 1600 stream->total_in += size; 1601 state->b_bytes_valid += size; 1602 buffered_size += size; 1603 1604 /* Save off next_in and avail_in if compression is 1605 * performed in internal buffer, total_in can be 1606 * recovered from knowledge of the size of the buffered 1607 * input */ 1608 next_in = stream->next_in; 1609 avail_in = stream->avail_in; 1610 1611 /* If not much data is buffered and there is no need to 1612 * flush the buffer, just continue rather than attempt 1613 * to compress */ 1614 if (avail_in == 0 && buffered_size <= IGZIP_HIST_SIZE && 1615 stream->total_in - buffered_size - state->block_next <= 1616 IGZIP_HIST_SIZE && 1617 !stream->end_of_stream && stream->flush == NO_FLUSH) 1618 continue; 1619 1620 if (avail_in) { 1621 stream->flush = NO_FLUSH; 1622 stream->end_of_stream = 0; 1623 } 1624 1625 stream->next_in = &state->buffer[state->b_bytes_processed]; 1626 stream->avail_in = buffered_size; 1627 stream->total_in -= buffered_size; 1628 1629 buf_start_in = state->buffer; 1630 1631 } else if (buffered_size) { 1632 /* The user provided buffer has sufficient data, reset 1633 * the user supplied buffer to included any data already 1634 * buffered */ 1635 stream->next_in -= buffered_size; 1636 stream->avail_in += buffered_size; 1637 stream->total_in -= buffered_size; 1638 state->b_bytes_valid = 0; 1639 state->b_bytes_processed = 0; 1640 buffered_size = 0; 1641 } 1642 1643 next_in_pre = stream->next_in; 1644 isal_deflate_int(stream, buf_start_in); 1645 processed = stream->next_in - next_in_pre; 1646 hist_size = get_hist_size(stream, buf_start_in, buf_hist_start); 1647 1648 /* Restore compression to unbuffered input when compressing to internal buffer */ 1649 if (internal) { 1650 state->b_bytes_processed += processed; 1651 buffered_size -= processed; 1652 1653 stream->flush = flush_type; 1654 stream->end_of_stream = end_of_stream; 1655 stream->total_in += buffered_size; 1656 1657 stream->next_in = next_in; 1658 stream->avail_in = avail_in; 1659 } 1660 1661 in_size = stream->avail_in + buffered_size; 1662 out_size = stream->total_out; 1663 1664 } while (internal && stream->avail_in > 0 && stream->avail_out > 0 && 1665 (in_size_initial != in_size || out_size_initial != out_size)); 1666 1667 /* Buffer history if data was pulled from the external buffer and future 1668 * calls to deflate will be required */ 1669 if (!internal && (state->state != ZSTATE_END && state->state != ZSTATE_TRL)) { 1670 /* If the external buffer was used, sufficient history must 1671 * exist in the user input buffer */ 1672 /* assert(stream->total_in - total_start >= */ 1673 /* hist_size + buffered_size); */ 1674 1675 stream->next_in -= buffered_size; 1676 stream->avail_in += buffered_size; 1677 stream->total_in -= buffered_size; 1678 1679 memmove(state->buffer, stream->next_in - hist_size, hist_size); 1680 state->b_bytes_processed = hist_size; 1681 state->b_bytes_valid = hist_size; 1682 buffered_size = 0; 1683 } 1684 1685 /* Buffer input data if it is necessary for continued execution */ 1686 if (stream->avail_in > 0 && (stream->avail_out > 0 || stream->level == 3)) { 1687 /* Determine how much data to buffer */ 1688 future_size = sizeof(state->buffer) - state->b_bytes_valid; 1689 if (stream->avail_in < future_size) 1690 /* Buffer all data if it fits as it will need to be buffered 1691 * on the next call anyways*/ 1692 future_size = stream->avail_in; 1693 else if (ISAL_LOOK_AHEAD < future_size) 1694 /* Buffer a minimum look ahead required for level 3 */ 1695 future_size = ISAL_LOOK_AHEAD; 1696 1697 memcpy(&state->buffer[state->b_bytes_valid], stream->next_in, future_size); 1698 1699 state->b_bytes_valid += future_size; 1700 buffered_size += future_size; 1701 stream->next_in += future_size; 1702 stream->total_in += future_size; 1703 stream->avail_in -= future_size; 1704 } 1705 1706 return ret; 1707 } 1708 1709 // Helper function to avoid code duplication. 1710 static void 1711 _zlib_header_in_buffer(struct isal_zstream *stream, uint8_t *buffer) 1712 { 1713 uint8_t hist_bits, info, level, cmf, flg; 1714 uint8_t dict_flag = 0; 1715 1716 if (stream->hist_bits == 0) // default hist_bits 1717 hist_bits = ISAL_DEF_MAX_HIST_BITS; 1718 else 1719 hist_bits = stream->hist_bits; 1720 if (hist_bits > 8) 1721 info = hist_bits - 8; 1722 else 1723 info = 0; // For low window sizes ensure correct cmf flag. 1724 if (stream->level == 0) 1725 level = 0; // Fastest algorithm 1726 else 1727 level = 1; // ISA-L levels 1-3 are fast algorithms. 1728 1729 cmf = DEFLATE_METHOD | (info << 4); 1730 flg = (level << 6) | dict_flag; 1731 flg += 31 - ((256 * cmf + flg) % 31); 1732 buffer[0] = cmf; 1733 buffer[1] = flg; 1734 return; 1735 } 1736 1737 static int 1738 write_stream_header_stateless(struct isal_zstream *stream) 1739 { 1740 uint32_t hdr_bytes; 1741 // Create a 10-byte buffer. Since the gzip header is almost fixed (9 of 10 1742 // bytes are fixed) use it to initialize the buffer. 1743 uint8_t buffer[10] = { 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff }; 1744 uint32_t next_flag; 1745 1746 if (stream->internal_state.has_wrap_hdr) 1747 return COMP_OK; 1748 1749 if (stream->gzip_flag == IGZIP_ZLIB) { 1750 hdr_bytes = zlib_hdr_bytes; 1751 _zlib_header_in_buffer(stream, buffer); 1752 next_flag = IGZIP_ZLIB_NO_HDR; 1753 } else { 1754 hdr_bytes = gzip_hdr_bytes; 1755 if (stream->level == 0) 1756 buffer[8] = 0x04; // Fastest algorithm in xfl flag 1757 next_flag = IGZIP_GZIP_NO_HDR; 1758 } 1759 1760 if (hdr_bytes >= stream->avail_out) 1761 return STATELESS_OVERFLOW; 1762 1763 stream->avail_out -= hdr_bytes; 1764 stream->total_out += hdr_bytes; 1765 1766 memcpy(stream->next_out, buffer, hdr_bytes); 1767 1768 stream->next_out += hdr_bytes; 1769 stream->internal_state.has_wrap_hdr = 1; 1770 stream->gzip_flag = next_flag; 1771 1772 return COMP_OK; 1773 } 1774 1775 static void 1776 write_stream_header(struct isal_zstream *stream) 1777 { 1778 struct isal_zstate *state = &stream->internal_state; 1779 int bytes_to_write; 1780 uint32_t hdr_bytes; 1781 // Create a 10-byte buffer. Since the gzip header is almost fixed (9 of 10 1782 // bytes are fixed) use it to initialize the buffer. 1783 uint8_t buffer[10] = { 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff }; 1784 1785 if (stream->internal_state.has_wrap_hdr) 1786 return; 1787 1788 if (stream->gzip_flag == IGZIP_ZLIB) { 1789 hdr_bytes = zlib_hdr_bytes; 1790 _zlib_header_in_buffer(stream, buffer); 1791 } else { 1792 if (stream->level == 0) 1793 buffer[8] = 0x04; // Fastest algorithm in xfl flag 1794 hdr_bytes = gzip_hdr_bytes; 1795 } 1796 1797 bytes_to_write = hdr_bytes; 1798 bytes_to_write -= state->count; 1799 1800 if (bytes_to_write > stream->avail_out) 1801 bytes_to_write = stream->avail_out; 1802 1803 memcpy(stream->next_out, buffer + state->count, bytes_to_write); 1804 state->count += bytes_to_write; 1805 1806 if (state->count == hdr_bytes) { 1807 state->count = 0; 1808 state->has_wrap_hdr = 1; 1809 } 1810 1811 stream->avail_out -= bytes_to_write; 1812 stream->total_out += bytes_to_write; 1813 stream->next_out += bytes_to_write; 1814 } 1815 1816 static int 1817 write_deflate_header_stateless(struct isal_zstream *stream) 1818 { 1819 struct isal_zstate *state = &stream->internal_state; 1820 struct isal_hufftables *hufftables = stream->hufftables; 1821 uint64_t hdr_extra_bits = hufftables->deflate_hdr[hufftables->deflate_hdr_count]; 1822 uint32_t count; 1823 1824 if (hufftables->deflate_hdr_count + 8 >= stream->avail_out) 1825 return STATELESS_OVERFLOW; 1826 1827 memcpy(stream->next_out, hufftables->deflate_hdr, hufftables->deflate_hdr_count); 1828 1829 if (stream->end_of_stream == 0) { 1830 if (hufftables->deflate_hdr_count > 0) 1831 *stream->next_out -= 1; 1832 else 1833 hdr_extra_bits -= 1; 1834 } else 1835 state->has_eob_hdr = 1; 1836 1837 stream->avail_out -= hufftables->deflate_hdr_count; 1838 stream->total_out += hufftables->deflate_hdr_count; 1839 stream->next_out += hufftables->deflate_hdr_count; 1840 1841 set_buf(&state->bitbuf, stream->next_out, stream->avail_out); 1842 1843 write_bits(&state->bitbuf, hdr_extra_bits, hufftables->deflate_hdr_extra_bits); 1844 1845 count = buffer_used(&state->bitbuf); 1846 stream->next_out = buffer_ptr(&state->bitbuf); 1847 stream->avail_out -= count; 1848 stream->total_out += count; 1849 1850 state->state = ZSTATE_BODY; 1851 1852 return COMP_OK; 1853 } 1854 1855 static int 1856 write_deflate_header_unaligned_stateless(struct isal_zstream *stream) 1857 { 1858 struct isal_zstate *state = &stream->internal_state; 1859 struct isal_hufftables *hufftables = stream->hufftables; 1860 unsigned int count; 1861 uint64_t bit_count; 1862 uint8_t *header_next; 1863 uint8_t *header_end; 1864 uint64_t header_bits; 1865 1866 if (state->bitbuf.m_bit_count == 0) 1867 return write_deflate_header_stateless(stream); 1868 1869 if (hufftables->deflate_hdr_count + 16 >= stream->avail_out) 1870 return STATELESS_OVERFLOW; 1871 1872 set_buf(&state->bitbuf, stream->next_out, stream->avail_out); 1873 1874 header_next = hufftables->deflate_hdr; 1875 header_end = header_next + 1876 (hufftables->deflate_hdr_count / sizeof(header_bits)) * sizeof(header_bits); 1877 1878 header_bits = load_le_u64(header_next); 1879 1880 if (stream->end_of_stream == 0) 1881 header_bits--; 1882 else 1883 state->has_eob_hdr = 1; 1884 1885 header_next += sizeof(header_bits); 1886 1887 /* Write out Complete Header bits */ 1888 for (; header_next <= header_end; header_next += sizeof(header_bits)) { 1889 write_bits(&state->bitbuf, header_bits, 32); 1890 header_bits >>= 32; 1891 write_bits(&state->bitbuf, header_bits, 32); 1892 header_bits = load_le_u64(header_next); 1893 } 1894 bit_count = (hufftables->deflate_hdr_count & 0x7) * 8 + hufftables->deflate_hdr_extra_bits; 1895 1896 if (bit_count > MAX_BITBUF_BIT_WRITE) { 1897 write_bits(&state->bitbuf, header_bits, MAX_BITBUF_BIT_WRITE); 1898 header_bits >>= MAX_BITBUF_BIT_WRITE; 1899 bit_count -= MAX_BITBUF_BIT_WRITE; 1900 } 1901 1902 write_bits(&state->bitbuf, header_bits, bit_count); 1903 1904 /* check_space flushes extra bytes in bitbuf. Required because 1905 * write_bits_always fails when the next commit makes the buffer 1906 * length exceed 64 bits */ 1907 check_space(&state->bitbuf, FORCE_FLUSH); 1908 1909 count = buffer_used(&state->bitbuf); 1910 stream->next_out = buffer_ptr(&state->bitbuf); 1911 stream->avail_out -= count; 1912 stream->total_out += count; 1913 1914 state->state = ZSTATE_BODY; 1915 1916 return COMP_OK; 1917 } 1918 1919 /* Toggle end of stream only works when deflate header is aligned */ 1920 static void 1921 write_header(struct isal_zstream *stream, uint8_t *deflate_hdr, uint32_t deflate_hdr_count, 1922 uint32_t extra_bits_count, uint32_t next_state, uint32_t toggle_end_of_stream) 1923 { 1924 struct isal_zstate *state = &stream->internal_state; 1925 uint32_t hdr_extra_bits = deflate_hdr[deflate_hdr_count]; 1926 uint32_t count; 1927 state->state = ZSTATE_HDR; 1928 1929 if (state->bitbuf.m_bit_count != 0) { 1930 if (stream->avail_out < 8) 1931 return; 1932 set_buf(&state->bitbuf, stream->next_out, stream->avail_out); 1933 flush(&state->bitbuf); 1934 count = buffer_used(&state->bitbuf); 1935 stream->next_out = buffer_ptr(&state->bitbuf); 1936 stream->avail_out -= count; 1937 stream->total_out += count; 1938 } 1939 1940 if (stream->gzip_flag == IGZIP_GZIP || stream->gzip_flag == IGZIP_ZLIB) 1941 write_stream_header(stream); 1942 1943 count = deflate_hdr_count - state->count; 1944 1945 if (count != 0) { 1946 if (count > stream->avail_out) 1947 count = stream->avail_out; 1948 1949 memcpy(stream->next_out, deflate_hdr + state->count, count); 1950 1951 if (toggle_end_of_stream && state->count == 0 && count > 0) { 1952 /* Assumes the final block bit is the first bit */ 1953 *stream->next_out ^= 1; 1954 state->has_eob_hdr = !state->has_eob_hdr; 1955 } 1956 1957 stream->next_out += count; 1958 stream->avail_out -= count; 1959 stream->total_out += count; 1960 state->count += count; 1961 1962 count = deflate_hdr_count - state->count; 1963 } else if (toggle_end_of_stream && deflate_hdr_count == 0) { 1964 /* Assumes the final block bit is the first bit */ 1965 hdr_extra_bits ^= 1; 1966 state->has_eob_hdr = !state->has_eob_hdr; 1967 } 1968 1969 if ((count == 0) && (stream->avail_out >= 8)) { 1970 1971 set_buf(&state->bitbuf, stream->next_out, stream->avail_out); 1972 1973 write_bits(&state->bitbuf, hdr_extra_bits, extra_bits_count); 1974 1975 state->state = next_state; 1976 state->count = 0; 1977 1978 count = buffer_used(&state->bitbuf); 1979 stream->next_out = buffer_ptr(&state->bitbuf); 1980 stream->avail_out -= count; 1981 stream->total_out += count; 1982 } 1983 } 1984 1985 static void 1986 write_trailer(struct isal_zstream *stream) 1987 { 1988 struct isal_zstate *state = &stream->internal_state; 1989 unsigned int bytes = 0; 1990 uint32_t crc = state->crc; 1991 1992 set_buf(&state->bitbuf, stream->next_out, stream->avail_out); 1993 1994 if (!state->has_eob_hdr) { 1995 /* If the final header has not been written, write a 1996 * final block. This block is a static huffman block 1997 * which only contains the end of block symbol. The code 1998 * that happens to do this is the fist 10 bits of 1999 * 0x003 */ 2000 if (stream->avail_out < 8) 2001 return; 2002 2003 state->has_eob_hdr = 1; 2004 write_bits(&state->bitbuf, 0x003, 10); 2005 if (is_full(&state->bitbuf)) { 2006 stream->next_out = buffer_ptr(&state->bitbuf); 2007 bytes = buffer_used(&state->bitbuf); 2008 stream->avail_out -= bytes; 2009 stream->total_out += bytes; 2010 return; 2011 } 2012 } 2013 2014 if (state->bitbuf.m_bit_count) { 2015 /* the flush() will pad to the next byte and write up to 8 bytes 2016 * to the output stream/buffer. 2017 */ 2018 if (stream->avail_out < 8) 2019 return; 2020 2021 flush(&state->bitbuf); 2022 } 2023 2024 stream->next_out = buffer_ptr(&state->bitbuf); 2025 bytes = buffer_used(&state->bitbuf); 2026 2027 switch (stream->gzip_flag) { 2028 case IGZIP_GZIP: 2029 case IGZIP_GZIP_NO_HDR: 2030 if (stream->avail_out - bytes >= gzip_trl_bytes) { 2031 store_le_u64(stream->next_out, ((uint64_t) stream->total_in << 32) | crc); 2032 stream->next_out += gzip_trl_bytes; 2033 bytes += gzip_trl_bytes; 2034 state->state = ZSTATE_END; 2035 } 2036 break; 2037 2038 case IGZIP_ZLIB: 2039 case IGZIP_ZLIB_NO_HDR: 2040 if (stream->avail_out - bytes >= zlib_trl_bytes) { 2041 store_be_u32(stream->next_out, 2042 (crc & 0xFFFF0000) | ((crc & 0xFFFF) + 1) % ADLER_MOD); 2043 stream->next_out += zlib_trl_bytes; 2044 bytes += zlib_trl_bytes; 2045 state->state = ZSTATE_END; 2046 } 2047 break; 2048 2049 default: 2050 state->state = ZSTATE_END; 2051 } 2052 2053 stream->avail_out -= bytes; 2054 stream->total_out += bytes; 2055 } 2056