1 /**********************************************************************
2 Copyright(c) 2011-2016 Intel Corporation All rights reserved.
3
4 Redistribution and use in source and binary forms, with or without
5 modification, are permitted provided that the following conditions
6 are met:
7 * Redistributions of source code must retain the above copyright
8 notice, this list of conditions and the following disclaimer.
9 * Redistributions in binary form must reproduce the above copyright
10 notice, this list of conditions and the following disclaimer in
11 the documentation and/or other materials provided with the
12 distribution.
13 * Neither the name of Intel Corporation nor the names of its
14 contributors may be used to endorse or promote products derived
15 from this software without specific prior written permission.
16
17 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 **********************************************************************/
29
30 #define ASM
31
32 #include <assert.h>
33 #include <string.h>
34 #include <wchar.h>
35 #ifdef _WIN32
36 #include <intrin.h>
37 #endif
38
39 #define MAX_WRITE_BITS_SIZE 8
40 #define FORCE_FLUSH 64
41 #define MIN_OBUF_SIZE 224
42 #define NON_EMPTY_BLOCK_SIZE 6
43 #define MAX_SYNC_FLUSH_SIZE NON_EMPTY_BLOCK_SIZE + MAX_WRITE_BITS_SIZE
44
45 #include "huffman.h"
46 #include "bitbuf2.h"
47 #include "igzip_lib.h"
48 #include "crc.h"
49 #include "repeated_char_result.h"
50 #include "huff_codes.h"
51 #include "encode_df.h"
52 #include "igzip_level_buf_structs.h"
53 #include "igzip_checksums.h"
54 #include "igzip_wrapper.h"
55 #include "unaligned.h"
56
57 extern void
58 isal_deflate_hash_lvl0(uint16_t *, uint32_t, uint32_t, uint8_t *, uint32_t);
59 extern void
60 isal_deflate_hash_lvl1(uint16_t *, uint32_t, uint32_t, uint8_t *, uint32_t);
61 extern void
62 isal_deflate_hash_lvl2(uint16_t *, uint32_t, uint32_t, uint8_t *, uint32_t);
63 extern void
64 isal_deflate_hash_lvl3(uint16_t *, uint32_t, uint32_t, uint8_t *, uint32_t);
65 extern const uint8_t gzip_hdr[];
66 extern const uint32_t gzip_hdr_bytes;
67 extern const uint32_t gzip_trl_bytes;
68 extern const uint8_t zlib_hdr[];
69 extern const uint32_t zlib_hdr_bytes;
70 extern const uint32_t zlib_trl_bytes;
71 extern const struct isal_hufftables hufftables_default;
72 extern const struct isal_hufftables hufftables_static;
73
74 static uint32_t
75 write_stored_block(struct isal_zstream *stream);
76
77 static int
78 write_stream_header_stateless(struct isal_zstream *stream);
79 static void
80 write_stream_header(struct isal_zstream *stream);
81 static int
82 write_deflate_header_stateless(struct isal_zstream *stream);
83 static int
84 write_deflate_header_unaligned_stateless(struct isal_zstream *stream);
85
86 #define TYPE0_HDR_LEN 4
87 #define TYPE0_BLK_HDR_LEN 5
88 #define TYPE0_MAX_BLK_LEN 65535
89
90 void
91 isal_deflate_body(struct isal_zstream *stream);
92 void
93 isal_deflate_finish(struct isal_zstream *stream);
94
95 void
96 isal_deflate_icf_body(struct isal_zstream *stream);
97 void
98 isal_deflate_icf_finish_lvl1(struct isal_zstream *stream);
99 void
100 isal_deflate_icf_finish_lvl2(struct isal_zstream *stream);
101 void
102 isal_deflate_icf_finish_lvl3(struct isal_zstream *stream);
103 /*****************************************************************/
104
105 /* Forward declarations */
106 static inline void
107 reset_match_history(struct isal_zstream *stream);
108 static void
109 write_header(struct isal_zstream *stream, uint8_t *deflate_hdr, uint32_t deflate_hdr_count,
110 uint32_t extra_bits_count, uint32_t next_state, uint32_t toggle_end_of_stream);
111 static void
112 write_trailer(struct isal_zstream *stream);
113
114 /*****************************************************************/
115
116 // isal_adler32_bam1 - adler with (B | A minus 1) storage
117
118 uint32_t
isal_adler32_bam1(uint32_t adler32,const unsigned char * start,uint64_t length)119 isal_adler32_bam1(uint32_t adler32, const unsigned char *start, uint64_t length)
120 {
121 uint64_t a;
122
123 /* Internally the checksum is being stored as B | (A-1) so crc and
124 * addler have same init value */
125 a = adler32 & 0xffff;
126 a = (a == ADLER_MOD - 1) ? 0 : a + 1;
127 adler32 = isal_adler32((adler32 & 0xffff0000) | a, start, length);
128 a = (adler32 & 0xffff);
129 a = (a == 0) ? ADLER_MOD - 1 : a - 1;
130
131 return (adler32 & 0xffff0000) | a;
132 }
133
134 static void
update_checksum(struct isal_zstream * stream,uint8_t * start_in,uint64_t length)135 update_checksum(struct isal_zstream *stream, uint8_t *start_in, uint64_t length)
136 {
137 struct isal_zstate *state = &stream->internal_state;
138 switch (stream->gzip_flag) {
139 case IGZIP_GZIP:
140 case IGZIP_GZIP_NO_HDR:
141 state->crc = crc32_gzip_refl(state->crc, start_in, length);
142 break;
143 case IGZIP_ZLIB:
144 case IGZIP_ZLIB_NO_HDR:
145 state->crc = isal_adler32_bam1(state->crc, start_in, length);
146 break;
147 }
148 }
149
150 static void
sync_flush(struct isal_zstream * stream)151 sync_flush(struct isal_zstream *stream)
152 {
153 struct isal_zstate *state = &stream->internal_state;
154 uint64_t bits_to_write = 0xFFFF0000, bits_len;
155 uint64_t bytes;
156 int flush_size;
157
158 if (stream->avail_out >= 8) {
159 set_buf(&state->bitbuf, stream->next_out, stream->avail_out);
160
161 flush_size = (-(state->bitbuf.m_bit_count + 3)) % 8;
162
163 bits_to_write <<= flush_size + 3;
164 bits_len = 32 + flush_size + 3;
165
166 state->state = ZSTATE_NEW_HDR;
167 state->has_eob = 0;
168
169 write_bits(&state->bitbuf, bits_to_write, bits_len);
170
171 bytes = buffer_used(&state->bitbuf);
172 stream->next_out = buffer_ptr(&state->bitbuf);
173 stream->avail_out -= bytes;
174 stream->total_out += bytes;
175
176 if (stream->flush == FULL_FLUSH) {
177 /* Clear match history so there are no cross
178 * block length distance pairs */
179 state->has_hist = IGZIP_NO_HIST;
180 }
181 }
182 }
183
184 static void
flush_write_buffer(struct isal_zstream * stream)185 flush_write_buffer(struct isal_zstream *stream)
186 {
187 struct isal_zstate *state = &stream->internal_state;
188 int bytes = 0;
189 if (stream->avail_out >= 8) {
190 set_buf(&state->bitbuf, stream->next_out, stream->avail_out);
191 flush(&state->bitbuf);
192 stream->next_out = buffer_ptr(&state->bitbuf);
193 bytes = buffer_used(&state->bitbuf);
194 stream->avail_out -= bytes;
195 stream->total_out += bytes;
196 state->state = ZSTATE_NEW_HDR;
197 }
198 }
199
200 static void
flush_icf_block(struct isal_zstream * stream)201 flush_icf_block(struct isal_zstream *stream)
202 {
203 struct isal_zstate *state = &stream->internal_state;
204 struct level_buf *level_buf = (struct level_buf *) stream->level_buf;
205 struct BitBuf2 *write_buf = &state->bitbuf;
206 struct deflate_icf *icf_buf_encoded_next;
207
208 set_buf(write_buf, stream->next_out, stream->avail_out);
209
210 icf_buf_encoded_next =
211 encode_deflate_icf(level_buf->icf_buf_start + state->count, level_buf->icf_buf_next,
212 write_buf, &level_buf->encode_tables);
213
214 state->count = icf_buf_encoded_next - level_buf->icf_buf_start;
215 stream->next_out = buffer_ptr(write_buf);
216 stream->total_out += buffer_used(write_buf);
217 stream->avail_out -= buffer_used(write_buf);
218
219 if (level_buf->icf_buf_next <= icf_buf_encoded_next) {
220 state->count = 0;
221 if (stream->avail_in == 0 && stream->end_of_stream)
222 state->state = ZSTATE_TRL;
223 else if (stream->avail_in == 0 && stream->flush != NO_FLUSH)
224 state->state = ZSTATE_SYNC_FLUSH;
225 else
226 state->state = ZSTATE_NEW_HDR;
227 }
228 }
229
230 static int
check_level_req(struct isal_zstream * stream)231 check_level_req(struct isal_zstream *stream)
232 {
233 if (stream->level == 0)
234 return 0;
235
236 if (stream->level_buf == NULL)
237 return ISAL_INVALID_LEVEL_BUF;
238
239 switch (stream->level) {
240 case 3:
241 if (stream->level_buf_size < ISAL_DEF_LVL3_MIN)
242 return ISAL_INVALID_LEVEL;
243 break;
244
245 case 2:
246 if (stream->level_buf_size < ISAL_DEF_LVL2_MIN)
247 return ISAL_INVALID_LEVEL;
248 break;
249 case 1:
250 if (stream->level_buf_size < ISAL_DEF_LVL1_MIN)
251 return ISAL_INVALID_LEVEL;
252 break;
253 default:
254 return ISAL_INVALID_LEVEL;
255 }
256
257 return 0;
258 }
259
260 static int
init_hash8k_buf(struct isal_zstream * stream)261 init_hash8k_buf(struct isal_zstream *stream)
262 {
263 struct isal_zstate *state = &stream->internal_state;
264 struct level_buf *level_buf = (struct level_buf *) stream->level_buf;
265 state->has_level_buf_init = 1;
266 return sizeof(struct level_buf) - MAX_LVL_BUF_SIZE + sizeof(level_buf->hash8k);
267 }
268
269 static int
init_hash_hist_buf(struct isal_zstream * stream)270 init_hash_hist_buf(struct isal_zstream *stream)
271 {
272 struct isal_zstate *state = &stream->internal_state;
273 struct level_buf *level_buf = (struct level_buf *) stream->level_buf;
274 state->has_level_buf_init = 1;
275 return sizeof(struct level_buf) - MAX_LVL_BUF_SIZE + sizeof(level_buf->hash_hist);
276 }
277
278 static int
init_hash_map_buf(struct isal_zstream * stream)279 init_hash_map_buf(struct isal_zstream *stream)
280 {
281 struct isal_zstate *state = &stream->internal_state;
282 struct level_buf *level_buf = (struct level_buf *) stream->level_buf;
283 if (!state->has_level_buf_init) {
284 level_buf->hash_map.matches_next = level_buf->hash_map.matches;
285 level_buf->hash_map.matches_end = level_buf->hash_map.matches;
286 }
287 state->has_level_buf_init = 1;
288
289 return sizeof(struct level_buf) - MAX_LVL_BUF_SIZE + sizeof(level_buf->hash_map);
290 }
291
292 /* returns the size of the level specific buffer */
293 static int
init_lvlX_buf(struct isal_zstream * stream)294 init_lvlX_buf(struct isal_zstream *stream)
295 {
296 switch (stream->level) {
297 case 3:
298 return init_hash_map_buf(stream);
299 case 2:
300 return init_hash_hist_buf(stream);
301 default:
302 return init_hash8k_buf(stream);
303 }
304 }
305
306 static void
init_new_icf_block(struct isal_zstream * stream)307 init_new_icf_block(struct isal_zstream *stream)
308 {
309 struct isal_zstate *state = &stream->internal_state;
310 struct level_buf *level_buf = (struct level_buf *) stream->level_buf;
311 int level_struct_size;
312
313 level_struct_size = init_lvlX_buf(stream);
314
315 state->block_next = state->block_end;
316 level_buf->icf_buf_start = (struct deflate_icf *) (stream->level_buf + level_struct_size);
317
318 level_buf->icf_buf_next = level_buf->icf_buf_start;
319 level_buf->icf_buf_avail_out =
320 stream->level_buf_size - level_struct_size - sizeof(struct deflate_icf);
321
322 memset(&level_buf->hist, 0, sizeof(struct isal_mod_hist));
323 state->state = ZSTATE_BODY;
324 }
325
326 static int
are_buffers_empty_hashX(struct isal_zstream * stream)327 are_buffers_empty_hashX(struct isal_zstream *stream)
328 {
329 return !stream->avail_in;
330 }
331
332 static int
are_buffers_empty_hash_map(struct isal_zstream * stream)333 are_buffers_empty_hash_map(struct isal_zstream *stream)
334 {
335 struct level_buf *level_buf = (struct level_buf *) stream->level_buf;
336
337 return (!stream->avail_in &&
338 level_buf->hash_map.matches_next >= level_buf->hash_map.matches_end);
339 }
340
341 static int
are_buffers_empty(struct isal_zstream * stream)342 are_buffers_empty(struct isal_zstream *stream)
343 {
344
345 switch (stream->level) {
346 case 3:
347 return are_buffers_empty_hash_map(stream);
348 case 2:
349 return are_buffers_empty_hashX(stream);
350 default:
351 return are_buffers_empty_hashX(stream);
352 }
353 }
354
355 static void
create_icf_block_hdr(struct isal_zstream * stream,uint8_t * start_in)356 create_icf_block_hdr(struct isal_zstream *stream, uint8_t *start_in)
357 {
358 struct isal_zstate *state = &stream->internal_state;
359 struct level_buf *level_buf = (struct level_buf *) stream->level_buf;
360 struct BitBuf2 *write_buf = &state->bitbuf;
361 struct BitBuf2 write_buf_tmp;
362 uint32_t out_size = stream->avail_out;
363 uint32_t avail_output, block_start_offset;
364 uint8_t *end_out = stream->next_out + out_size;
365 uint64_t cur_in_processed;
366 uint64_t bit_count;
367 uint64_t block_in_size = state->block_end - state->block_next;
368 uint64_t block_size;
369 int buffer_header = 0;
370
371 memcpy(&write_buf_tmp, write_buf, sizeof(struct BitBuf2));
372
373 /* Calculate the bytes required to store a type 0 block. Need to account
374 * for bits stored in the bitbuf. Since 3 bits correspond to the deflate
375 * type 0 header, we need to add one byte more when the number of bits
376 * is at least 6 mod 8. */
377 block_size = (TYPE0_BLK_HDR_LEN) *
378 ((block_in_size + TYPE0_MAX_BLK_LEN - 1) / TYPE0_MAX_BLK_LEN) +
379 block_in_size;
380 block_size = block_size ? block_size : TYPE0_BLK_HDR_LEN;
381 block_size += (write_buf->m_bit_count + 2) / 8;
382
383 /* Write EOB in icf_buf */
384 level_buf->hist.ll_hist[256] = 1;
385 level_buf->icf_buf_next->lit_len = 0x100;
386 level_buf->icf_buf_next->lit_dist = NULL_DIST_SYM;
387 level_buf->icf_buf_next->dist_extra = 0;
388 level_buf->icf_buf_next++;
389
390 state->has_eob_hdr = (stream->end_of_stream && are_buffers_empty(stream)) ? 1 : 0;
391
392 if (end_out - stream->next_out >= ISAL_DEF_MAX_HDR_SIZE) {
393 /* Assumes ISAL_DEF_MAX_HDR_SIZE is large enough to contain a
394 * max length header and a gzip header */
395 if (stream->gzip_flag == IGZIP_GZIP || stream->gzip_flag == IGZIP_ZLIB)
396 write_stream_header_stateless(stream);
397 set_buf(write_buf, stream->next_out, stream->avail_out);
398 buffer_header = 0;
399
400 } else {
401 /* Start writing into temporary buffer */
402 set_buf(write_buf, level_buf->deflate_hdr, ISAL_DEF_MAX_HDR_SIZE);
403 buffer_header = 1;
404 }
405
406 bit_count = create_hufftables_icf(write_buf, &level_buf->encode_tables, &level_buf->hist,
407 state->has_eob_hdr);
408
409 /* Assumes that type 0 block has size less than 4G */
410 block_start_offset = (stream->total_in - state->block_next);
411 cur_in_processed = stream->next_in - start_in;
412 avail_output =
413 stream->avail_out + sizeof(state->buffer) - (stream->total_in - state->block_end);
414
415 if (bit_count / 8 >= block_size && cur_in_processed >= block_start_offset &&
416 block_size <= avail_output) {
417 /* Reset stream for writing out a type0 block */
418 state->has_eob_hdr = 0;
419 memcpy(write_buf, &write_buf_tmp, sizeof(struct BitBuf2));
420 state->state = ZSTATE_TYPE0_HDR;
421
422 } else if (buffer_header) {
423 /* Setup stream to write out a buffered header */
424 level_buf->deflate_hdr_count = buffer_used(write_buf);
425 level_buf->deflate_hdr_extra_bits = write_buf->m_bit_count;
426 flush(write_buf);
427 memcpy(write_buf, &write_buf_tmp, sizeof(struct BitBuf2));
428 write_buf->m_bits = 0;
429 write_buf->m_bit_count = 0;
430 state->state = ZSTATE_HDR;
431
432 } else {
433 stream->next_out = buffer_ptr(write_buf);
434 stream->total_out += buffer_used(write_buf);
435 stream->avail_out -= buffer_used(write_buf);
436 state->state = ZSTATE_FLUSH_ICF_BUFFER;
437 }
438 }
439
440 static void
isal_deflate_pass(struct isal_zstream * stream)441 isal_deflate_pass(struct isal_zstream *stream)
442 {
443 struct isal_zstate *state = &stream->internal_state;
444 struct isal_hufftables *hufftables = stream->hufftables;
445 uint8_t *start_in = stream->next_in;
446
447 if (state->state == ZSTATE_NEW_HDR || state->state == ZSTATE_HDR) {
448 if (state->count == 0)
449 /* Assume the final header is being written since the header
450 * stored in hufftables is the final header. */
451 state->has_eob_hdr = 1;
452 write_header(stream, hufftables->deflate_hdr, hufftables->deflate_hdr_count,
453 hufftables->deflate_hdr_extra_bits, ZSTATE_BODY,
454 !stream->end_of_stream);
455 }
456
457 if (state->state == ZSTATE_BODY)
458 isal_deflate_body(stream);
459
460 if (state->state == ZSTATE_FLUSH_READ_BUFFER)
461 isal_deflate_finish(stream);
462 if (state->state == ZSTATE_SYNC_FLUSH)
463 sync_flush(stream);
464
465 if (state->state == ZSTATE_FLUSH_WRITE_BUFFER)
466 flush_write_buffer(stream);
467
468 if (stream->gzip_flag)
469 update_checksum(stream, start_in, stream->next_in - start_in);
470
471 if (state->state == ZSTATE_TRL)
472 write_trailer(stream);
473 }
474
475 static void
isal_deflate_icf_finish(struct isal_zstream * stream)476 isal_deflate_icf_finish(struct isal_zstream *stream)
477 {
478 switch (stream->level) {
479 case 3:
480 isal_deflate_icf_finish_lvl3(stream);
481 break;
482 case 2:
483 isal_deflate_icf_finish_lvl2(stream);
484 break;
485 default:
486 isal_deflate_icf_finish_lvl1(stream);
487 }
488 }
489
490 static void
isal_deflate_icf_pass(struct isal_zstream * stream,uint8_t * inbuf_start)491 isal_deflate_icf_pass(struct isal_zstream *stream, uint8_t *inbuf_start)
492 {
493 uint8_t *start_in = stream->next_in;
494 struct isal_zstate *state = &stream->internal_state;
495 struct level_buf *level_buf = (struct level_buf *) stream->level_buf;
496
497 do {
498 if (state->state == ZSTATE_NEW_HDR)
499 init_new_icf_block(stream);
500
501 if (state->state == ZSTATE_BODY)
502 isal_deflate_icf_body(stream);
503
504 if (state->state == ZSTATE_FLUSH_READ_BUFFER)
505 isal_deflate_icf_finish(stream);
506
507 if (state->state == ZSTATE_CREATE_HDR)
508 create_icf_block_hdr(stream, inbuf_start);
509
510 if (state->state == ZSTATE_HDR)
511 /* Note that the header may be prepended by the
512 * remaining bits in the previous block, as such the
513 * toggle header flag cannot be used */
514 write_header(stream, level_buf->deflate_hdr, level_buf->deflate_hdr_count,
515 level_buf->deflate_hdr_extra_bits, ZSTATE_FLUSH_ICF_BUFFER, 0);
516
517 if (state->state == ZSTATE_FLUSH_ICF_BUFFER)
518 flush_icf_block(stream);
519
520 if (state->state == ZSTATE_TYPE0_HDR || state->state == ZSTATE_TYPE0_BODY) {
521 if (stream->gzip_flag == IGZIP_GZIP || stream->gzip_flag == IGZIP_ZLIB)
522 write_stream_header(stream);
523 write_stored_block(stream);
524 }
525
526 } while (state->state == ZSTATE_NEW_HDR);
527
528 if (state->state == ZSTATE_SYNC_FLUSH)
529 sync_flush(stream);
530
531 if (state->state == ZSTATE_FLUSH_WRITE_BUFFER)
532 flush_write_buffer(stream);
533
534 if (stream->gzip_flag)
535 update_checksum(stream, start_in, stream->next_in - start_in);
536
537 if (state->state == ZSTATE_TRL)
538 write_trailer(stream);
539 }
540
541 static void
isal_deflate_int(struct isal_zstream * stream,uint8_t * start_in)542 isal_deflate_int(struct isal_zstream *stream, uint8_t *start_in)
543 {
544 struct isal_zstate *state = &stream->internal_state;
545 uint32_t size;
546
547 /* Move data from temporary output buffer to output buffer */
548 if (state->state >= ZSTATE_TMP_OFFSET) {
549 size = state->tmp_out_end - state->tmp_out_start;
550 if (size > stream->avail_out)
551 size = stream->avail_out;
552 memcpy(stream->next_out, state->tmp_out_buff + state->tmp_out_start, size);
553 stream->next_out += size;
554 stream->avail_out -= size;
555 stream->total_out += size;
556 state->tmp_out_start += size;
557
558 if (state->tmp_out_start == state->tmp_out_end)
559 state->state -= ZSTATE_TMP_OFFSET;
560
561 if (stream->avail_out == 0 ||
562 state->state == ZSTATE_END
563 // or do not write out empty blocks since the outbuffer was processed
564 || (state->state == ZSTATE_NEW_HDR && stream->avail_out == 0))
565 return;
566 }
567 assert(state->tmp_out_start == state->tmp_out_end);
568
569 if (stream->level == 0)
570 isal_deflate_pass(stream);
571 else
572 isal_deflate_icf_pass(stream, start_in);
573
574 /* Fill temporary output buffer then complete filling output buffer */
575 if (stream->avail_out > 0 && stream->avail_out < 8 && state->state != ZSTATE_NEW_HDR) {
576 uint8_t *next_out;
577 uint32_t avail_out;
578 uint32_t total_out;
579
580 next_out = stream->next_out;
581 avail_out = stream->avail_out;
582 total_out = stream->total_out;
583
584 stream->next_out = state->tmp_out_buff;
585 stream->avail_out = sizeof(state->tmp_out_buff);
586 stream->total_out = 0;
587
588 if (stream->level == 0)
589 isal_deflate_pass(stream);
590 else
591 isal_deflate_icf_pass(stream, start_in);
592
593 state->tmp_out_start = 0;
594 state->tmp_out_end = stream->total_out;
595
596 stream->next_out = next_out;
597 stream->avail_out = avail_out;
598 stream->total_out = total_out;
599 if (state->tmp_out_end) {
600 size = state->tmp_out_end;
601 if (size > stream->avail_out)
602 size = stream->avail_out;
603 memcpy(stream->next_out, state->tmp_out_buff, size);
604 stream->next_out += size;
605 stream->avail_out -= size;
606 stream->total_out += size;
607 state->tmp_out_start += size;
608 if (state->tmp_out_start != state->tmp_out_end)
609 state->state += ZSTATE_TMP_OFFSET;
610 }
611 }
612 }
613
614 static void
write_constant_compressed_stateless(struct isal_zstream * stream,uint32_t repeated_length)615 write_constant_compressed_stateless(struct isal_zstream *stream, uint32_t repeated_length)
616 {
617 /* Assumes repeated_length is at least 1.
618 * Assumes the input end_of_stream is either 0 or 1. */
619 struct isal_zstate *state = &stream->internal_state;
620 uint32_t rep_bits = ((repeated_length - 1) / 258) * 2;
621 uint32_t rep_bytes = rep_bits / 8;
622 uint32_t rep_extra = (repeated_length - 1) % 258;
623 uint32_t bytes;
624 uint32_t repeated_char = *stream->next_in;
625 uint8_t *start_in = stream->next_in;
626
627 /* Guarantee there is enough space for the header even in the worst case */
628 if (stream->avail_out < HEADER_LENGTH + MAX_FIXUP_CODE_LENGTH + rep_bytes + 8)
629 return;
630
631 /* Assumes the repeated char is either 0 or 0xFF. */
632 memcpy(stream->next_out, repeated_char_header[repeated_char & 1], HEADER_LENGTH);
633
634 if (stream->avail_in == repeated_length && stream->end_of_stream > 0) {
635 stream->next_out[0] |= 1;
636 state->has_eob_hdr = 1;
637 state->has_eob = 1;
638 state->state = ZSTATE_TRL;
639 } else {
640 state->state = ZSTATE_NEW_HDR;
641 }
642
643 memset(stream->next_out + HEADER_LENGTH, 0, rep_bytes);
644 stream->avail_out -= HEADER_LENGTH + rep_bytes;
645 stream->next_out += HEADER_LENGTH + rep_bytes;
646 stream->total_out += HEADER_LENGTH + rep_bytes;
647
648 set_buf(&state->bitbuf, stream->next_out, stream->avail_out);
649
650 /* These two lines are basically a modified version of init. */
651 state->bitbuf.m_bits = 0;
652 state->bitbuf.m_bit_count = rep_bits % 8;
653
654 /* Add smaller repeat codes as necessary. Code280 can describe repeat
655 * lengths of 115-130 bits. Code10 can describe repeat lengths of 10
656 * bits. If more than 230 bits, fill code with two code280s. Else if
657 * more than 115 repeats, fill with code10s until one code280 can
658 * finish the rest of the repeats. Else, fill with code10s and
659 * literals */
660 if (rep_extra > 115) {
661 while (rep_extra > 130 && rep_extra < 230) {
662 write_bits(&state->bitbuf, CODE_10, CODE_10_LENGTH);
663 rep_extra -= 10;
664 }
665
666 if (rep_extra >= 230) {
667 write_bits(&state->bitbuf,
668 CODE_280 | ((rep_extra / 2 - 115) << CODE_280_LENGTH),
669 CODE_280_TOTAL_LENGTH);
670 rep_extra -= rep_extra / 2;
671 }
672
673 write_bits(&state->bitbuf, CODE_280 | ((rep_extra - 115) << CODE_280_LENGTH),
674 CODE_280_TOTAL_LENGTH);
675
676 } else {
677 while (rep_extra >= 10) {
678
679 write_bits(&state->bitbuf, CODE_10, CODE_10_LENGTH);
680 rep_extra -= 10;
681 }
682
683 for (; rep_extra > 0; rep_extra--)
684 write_bits(&state->bitbuf, CODE_LIT, CODE_LIT_LENGTH);
685 }
686
687 write_bits(&state->bitbuf, END_OF_BLOCK, END_OF_BLOCK_LEN);
688
689 stream->next_in += repeated_length;
690 stream->avail_in -= repeated_length;
691 stream->total_in += repeated_length;
692 state->block_end += repeated_length;
693
694 bytes = buffer_used(&state->bitbuf);
695 stream->next_out = buffer_ptr(&state->bitbuf);
696 stream->avail_out -= bytes;
697 stream->total_out += bytes;
698
699 if (stream->gzip_flag)
700 update_checksum(stream, start_in, stream->next_in - start_in);
701
702 return;
703 }
704
705 static int
detect_repeated_char_length(uint8_t * in,uint32_t length)706 detect_repeated_char_length(uint8_t *in, uint32_t length)
707 {
708 /* This currently assumes the first 8 bytes are the same character.
709 * This won't work effectively if the input stream isn't aligned well. */
710 uint8_t *p_8, *end = in + length;
711 uint64_t *p_64 = (uint64_t *) in;
712 uint64_t w = *p_64;
713 uint8_t c = (uint8_t) w;
714
715 for (; (p_64 <= (uint64_t *) (end - 8)) && (w == *p_64); p_64++)
716 ;
717
718 p_8 = (uint8_t *) p_64;
719
720 for (; (p_8 < end) && (c == *p_8); p_8++)
721 ;
722
723 return p_8 - in;
724 }
725
726 static int
isal_deflate_int_stateless(struct isal_zstream * stream)727 isal_deflate_int_stateless(struct isal_zstream *stream)
728 {
729 uint32_t repeat_length;
730 struct isal_zstate *state = &stream->internal_state;
731
732 if (stream->gzip_flag == IGZIP_GZIP || stream->gzip_flag == IGZIP_ZLIB)
733 if (write_stream_header_stateless(stream))
734 return STATELESS_OVERFLOW;
735
736 if (stream->avail_in >= 8 && (load_native_u64(stream->next_in) == 0 ||
737 load_native_u64(stream->next_in) == ~(uint64_t) 0)) {
738 repeat_length = detect_repeated_char_length(stream->next_in, stream->avail_in);
739
740 if (stream->avail_in == repeat_length || repeat_length >= MIN_REPEAT_LEN)
741 write_constant_compressed_stateless(stream, repeat_length);
742 }
743
744 if (stream->level == 0) {
745 if (state->state == ZSTATE_NEW_HDR || state->state == ZSTATE_HDR) {
746 write_deflate_header_unaligned_stateless(stream);
747 if (state->state == ZSTATE_NEW_HDR || state->state == ZSTATE_HDR)
748 return STATELESS_OVERFLOW;
749
750 reset_match_history(stream);
751 }
752
753 isal_deflate_pass(stream);
754
755 } else if (stream->level <= ISAL_DEF_MAX_LEVEL) {
756 if (state->state == ZSTATE_NEW_HDR || state->state == ZSTATE_HDR)
757 reset_match_history(stream);
758
759 state->count = 0;
760 isal_deflate_icf_pass(stream, stream->next_in);
761 }
762
763 if (state->state == ZSTATE_END ||
764 (state->state == ZSTATE_NEW_HDR && stream->flush == FULL_FLUSH))
765 return COMP_OK;
766 else
767 return STATELESS_OVERFLOW;
768 }
769
770 static void
write_type0_header(struct isal_zstream * stream)771 write_type0_header(struct isal_zstream *stream)
772 {
773 struct isal_zstate *state = &stream->internal_state;
774 uint64_t stored_blk_hdr;
775 uint32_t copy_size;
776 uint32_t memcpy_len, avail_in;
777 uint32_t block_in_size = state->block_end - state->block_next;
778 uint32_t block_next_offset;
779 struct BitBuf2 *bitbuf = &stream->internal_state.bitbuf;
780
781 if (block_in_size > TYPE0_MAX_BLK_LEN) {
782 stored_blk_hdr = 0xFFFF;
783 copy_size = TYPE0_MAX_BLK_LEN;
784 } else {
785 stored_blk_hdr = ~block_in_size;
786 stored_blk_hdr <<= 16;
787 stored_blk_hdr |= (block_in_size & 0xFFFF);
788 copy_size = block_in_size;
789
790 /* Handle BFINAL bit */
791 block_next_offset = stream->total_in - state->block_next;
792 avail_in = stream->avail_in + block_next_offset;
793 if (stream->end_of_stream && avail_in == block_in_size)
794 stream->internal_state.has_eob_hdr = 1;
795 }
796
797 if (bitbuf->m_bit_count == 0 && stream->avail_out >= TYPE0_HDR_LEN + 1) {
798 stored_blk_hdr = stored_blk_hdr << 8;
799 stored_blk_hdr |= stream->internal_state.has_eob_hdr;
800 memcpy_len = TYPE0_HDR_LEN + 1;
801 stored_blk_hdr = to_le64(stored_blk_hdr);
802 memcpy(stream->next_out, &stored_blk_hdr, memcpy_len);
803 } else if (stream->avail_out >= 8) {
804 set_buf(bitbuf, stream->next_out, stream->avail_out);
805 write_bits_flush(bitbuf, stream->internal_state.has_eob_hdr, 3);
806 stream->next_out = buffer_ptr(bitbuf);
807 stream->total_out += buffer_used(bitbuf);
808 stream->avail_out -= buffer_used(bitbuf);
809 memcpy_len = TYPE0_HDR_LEN;
810 stored_blk_hdr = to_le64(stored_blk_hdr);
811 memcpy(stream->next_out, &stored_blk_hdr, memcpy_len);
812 } else {
813 stream->internal_state.has_eob_hdr = 0;
814 return;
815 }
816
817 stream->next_out += memcpy_len;
818 stream->avail_out -= memcpy_len;
819 stream->total_out += memcpy_len;
820 stream->internal_state.state = ZSTATE_TYPE0_BODY;
821
822 stream->internal_state.count = copy_size;
823 }
824
825 static uint32_t
write_stored_block(struct isal_zstream * stream)826 write_stored_block(struct isal_zstream *stream)
827 {
828 uint32_t copy_size, avail_in, block_next_offset;
829 uint8_t *next_in;
830 struct isal_zstate *state = &stream->internal_state;
831
832 do {
833 if (state->state == ZSTATE_TYPE0_HDR) {
834 write_type0_header(stream);
835 if (state->state == ZSTATE_TYPE0_HDR)
836 break;
837 }
838
839 assert(state->count <= state->block_end - state->block_next);
840 copy_size = state->count;
841
842 block_next_offset = stream->total_in - state->block_next;
843 next_in = stream->next_in - block_next_offset;
844 avail_in = stream->avail_in + block_next_offset;
845
846 if (copy_size > stream->avail_out || copy_size > avail_in) {
847 state->count = copy_size;
848 copy_size = (stream->avail_out <= avail_in) ? stream->avail_out : avail_in;
849
850 memcpy(stream->next_out, next_in, copy_size);
851 state->count -= copy_size;
852 } else {
853 memcpy(stream->next_out, next_in, copy_size);
854
855 state->count = 0;
856 state->state = ZSTATE_TYPE0_HDR;
857 }
858
859 state->block_next += copy_size;
860 stream->next_out += copy_size;
861 stream->avail_out -= copy_size;
862 stream->total_out += copy_size;
863
864 if (state->block_next == state->block_end) {
865 state->state = state->has_eob_hdr ? ZSTATE_TRL : ZSTATE_NEW_HDR;
866 if (stream->flush == FULL_FLUSH && state->state == ZSTATE_NEW_HDR &&
867 are_buffers_empty(stream)) {
868 /* Clear match history so there are no cross
869 * block length distance pairs */
870 reset_match_history(stream);
871 }
872 }
873 } while (state->state == ZSTATE_TYPE0_HDR);
874
875 return state->block_end - state->block_next;
876 }
877
878 static inline void
reset_match_history(struct isal_zstream * stream)879 reset_match_history(struct isal_zstream *stream)
880 {
881 struct isal_zstate *state = &stream->internal_state;
882 struct level_buf *level_buf = (struct level_buf *) stream->level_buf;
883 uint16_t *hash_table;
884 uint32_t hash_table_size;
885
886 hash_table_size = 2 * (state->hash_mask + 1);
887
888 switch (stream->level) {
889 case 3:
890 hash_table = level_buf->lvl3.hash_table;
891 break;
892 case 2:
893 hash_table = level_buf->lvl2.hash_table;
894 break;
895 case 1:
896 hash_table = level_buf->lvl1.hash_table;
897 break;
898 default:
899 hash_table = state->head;
900 }
901
902 state->has_hist = IGZIP_NO_HIST;
903
904 /* There is definitely more than 16 bytes in the hash table. Set this
905 * minimum to avoid a wmemset of size 0 */
906 if (hash_table_size <= sizeof(wchar_t))
907 hash_table_size = sizeof(wchar_t);
908
909 if (sizeof(wchar_t) == 2) {
910 uint16_t hash_init_val;
911
912 hash_init_val = stream->total_in & 0xffff;
913 wmemset((wchar_t *) hash_table, hash_init_val, hash_table_size / sizeof(wchar_t));
914
915 } else if (sizeof(wchar_t) == 4) {
916 uint32_t hash_init_val;
917 int rep_bits;
918
919 hash_init_val = stream->total_in & 0xffff;
920 for (rep_bits = sizeof(uint16_t) * 8; rep_bits < sizeof(wchar_t) * 8; rep_bits *= 2)
921 hash_init_val |= hash_init_val << rep_bits;
922
923 wmemset((wchar_t *) hash_table, hash_init_val, hash_table_size / sizeof(wchar_t));
924 } else {
925 if ((stream->total_in & 0xFFFF) == 0)
926 memset(hash_table, 0, hash_table_size);
927 else {
928 int i;
929 for (i = 0; i < hash_table_size / 2; i++) {
930 hash_table[i] = (uint16_t) (stream->total_in);
931 }
932 }
933 }
934 }
935
set_dist_mask(struct isal_zstream * stream)936 static void inline set_dist_mask(struct isal_zstream *stream)
937 {
938 struct isal_zstate *state = &stream->internal_state;
939 uint32_t hist_size;
940
941 if (stream->hist_bits > ISAL_DEF_MAX_HIST_BITS || stream->hist_bits == 0)
942 stream->hist_bits = ISAL_DEF_MAX_HIST_BITS;
943
944 hist_size = (1 << (stream->hist_bits));
945 state->dist_mask = hist_size - 1;
946
947 if (IGZIP_HIST_SIZE < ISAL_DEF_HIST_SIZE && state->dist_mask > IGZIP_HIST_SIZE - 1)
948 state->dist_mask = IGZIP_HIST_SIZE - 1;
949 }
950
set_hash_mask(struct isal_zstream * stream)951 static void inline set_hash_mask(struct isal_zstream *stream)
952 {
953 struct isal_zstate *state = &stream->internal_state;
954
955 switch (stream->level) {
956 case 3:
957 state->hash_mask = LVL3_HASH_MASK;
958 break;
959 case 2:
960 state->hash_mask = LVL2_HASH_MASK;
961 break;
962 case 1:
963 state->hash_mask = LVL1_HASH_MASK;
964 break;
965 case 0:
966 state->hash_mask = LVL0_HASH_MASK;
967 }
968 }
969
970 void
isal_deflate_init(struct isal_zstream * stream)971 isal_deflate_init(struct isal_zstream *stream)
972 {
973 struct isal_zstate *state = &stream->internal_state;
974
975 stream->total_in = 0;
976 stream->total_out = 0;
977 stream->hufftables = (struct isal_hufftables *) &hufftables_default;
978 stream->level = 0;
979 stream->level_buf = NULL;
980 stream->level_buf_size = 0;
981 stream->end_of_stream = 0;
982 stream->flush = NO_FLUSH;
983 stream->gzip_flag = 0;
984 stream->hist_bits = 0;
985
986 state->block_next = 0;
987 state->block_end = 0;
988 state->b_bytes_valid = 0;
989 state->b_bytes_processed = 0;
990 state->total_in_start = 0;
991 state->has_wrap_hdr = 0;
992 state->has_eob = 0;
993 state->has_eob_hdr = 0;
994 state->has_hist = IGZIP_NO_HIST;
995 state->has_level_buf_init = 0;
996 state->state = ZSTATE_NEW_HDR;
997 state->count = 0;
998
999 state->tmp_out_start = 0;
1000 state->tmp_out_end = 0;
1001
1002 init(&state->bitbuf);
1003
1004 state->crc = 0;
1005
1006 return;
1007 }
1008
1009 void
isal_deflate_reset(struct isal_zstream * stream)1010 isal_deflate_reset(struct isal_zstream *stream)
1011 {
1012 struct isal_zstate *state = &stream->internal_state;
1013
1014 stream->total_in = 0;
1015 stream->total_out = 0;
1016
1017 state->block_next = 0;
1018 state->block_end = 0;
1019 state->b_bytes_valid = 0;
1020 state->b_bytes_processed = 0;
1021 state->total_in_start = 0;
1022 state->has_wrap_hdr = 0;
1023 state->has_eob = 0;
1024 state->has_level_buf_init = 0;
1025 state->has_eob_hdr = 0;
1026 state->has_hist = IGZIP_NO_HIST;
1027 state->state = ZSTATE_NEW_HDR;
1028 state->count = 0;
1029
1030 state->tmp_out_start = 0;
1031 state->tmp_out_end = 0;
1032
1033 init(&state->bitbuf);
1034
1035 state->crc = 0;
1036 }
1037
1038 void
isal_gzip_header_init(struct isal_gzip_header * gz_hdr)1039 isal_gzip_header_init(struct isal_gzip_header *gz_hdr)
1040 {
1041 gz_hdr->text = 0;
1042 gz_hdr->time = 0;
1043 gz_hdr->xflags = 0;
1044 gz_hdr->os = 0xff;
1045 gz_hdr->extra = NULL;
1046 gz_hdr->extra_buf_len = 0;
1047 gz_hdr->extra_len = 0;
1048 gz_hdr->name = NULL;
1049 gz_hdr->name_buf_len = 0;
1050 gz_hdr->comment = NULL;
1051 gz_hdr->comment_buf_len = 0;
1052 gz_hdr->hcrc = 0;
1053 gz_hdr->flags = 0;
1054 }
1055
1056 void
isal_zlib_header_init(struct isal_zlib_header * z_hdr)1057 isal_zlib_header_init(struct isal_zlib_header *z_hdr)
1058 {
1059 z_hdr->info = 0;
1060 z_hdr->level = 0;
1061 z_hdr->dict_id = 0;
1062 z_hdr->dict_flag = 0;
1063 }
1064
1065 uint32_t
isal_write_gzip_header(struct isal_zstream * stream,struct isal_gzip_header * gz_hdr)1066 isal_write_gzip_header(struct isal_zstream *stream, struct isal_gzip_header *gz_hdr)
1067 {
1068 uint32_t flags = 0, hcrc, hdr_size = GZIP_HDR_BASE;
1069 uint8_t *out_buf = stream->next_out, *out_buf_start = stream->next_out;
1070 uint32_t name_len = 0, comment_len = 0;
1071
1072 if (gz_hdr->text)
1073 flags |= TEXT_FLAG;
1074 if (gz_hdr->extra) {
1075 flags |= EXTRA_FLAG;
1076 hdr_size += GZIP_EXTRA_LEN + gz_hdr->extra_len;
1077 }
1078 if (gz_hdr->name) {
1079 flags |= NAME_FLAG;
1080 name_len = strnlen(gz_hdr->name, gz_hdr->name_buf_len);
1081 if (name_len < gz_hdr->name_buf_len)
1082 name_len++;
1083 hdr_size += name_len;
1084 }
1085 if (gz_hdr->comment) {
1086 flags |= COMMENT_FLAG;
1087 comment_len = strnlen(gz_hdr->comment, gz_hdr->comment_buf_len);
1088 if (comment_len < gz_hdr->comment_buf_len)
1089 comment_len++;
1090 hdr_size += comment_len;
1091 }
1092 if (gz_hdr->hcrc) {
1093 flags |= HCRC_FLAG;
1094 hdr_size += GZIP_HCRC_LEN;
1095 }
1096
1097 if (stream->avail_out < hdr_size)
1098 return hdr_size;
1099
1100 out_buf[0] = 0x1f;
1101 out_buf[1] = 0x8b;
1102 out_buf[2] = DEFLATE_METHOD;
1103 out_buf[3] = flags;
1104 store_le_u32(out_buf + 4, gz_hdr->time);
1105 out_buf[8] = gz_hdr->xflags;
1106 out_buf[9] = gz_hdr->os;
1107
1108 out_buf += GZIP_HDR_BASE;
1109 if (flags & EXTRA_FLAG) {
1110 store_le_u16(out_buf, gz_hdr->extra_len);
1111 out_buf += GZIP_EXTRA_LEN;
1112
1113 memcpy(out_buf, gz_hdr->extra, gz_hdr->extra_len);
1114 out_buf += gz_hdr->extra_len;
1115 }
1116
1117 if (flags & NAME_FLAG) {
1118 memcpy(out_buf, gz_hdr->name, name_len);
1119 out_buf += name_len;
1120 }
1121
1122 if (flags & COMMENT_FLAG) {
1123 memcpy(out_buf, gz_hdr->comment, comment_len);
1124 out_buf += comment_len;
1125 }
1126
1127 if (flags & HCRC_FLAG) {
1128 hcrc = crc32_gzip_refl(0, out_buf_start, out_buf - out_buf_start);
1129 store_le_u16(out_buf, hcrc);
1130 out_buf += GZIP_HCRC_LEN;
1131 }
1132
1133 stream->next_out += hdr_size;
1134 stream->total_out += hdr_size;
1135 stream->avail_out -= hdr_size;
1136
1137 return ISAL_DECOMP_OK;
1138 }
1139
1140 uint32_t
isal_write_zlib_header(struct isal_zstream * stream,struct isal_zlib_header * z_hdr)1141 isal_write_zlib_header(struct isal_zstream *stream, struct isal_zlib_header *z_hdr)
1142 {
1143 uint32_t cmf, flg, dict_flag = 0, hdr_size = ZLIB_HDR_BASE;
1144 uint8_t *out_buf = stream->next_out;
1145
1146 if (z_hdr->dict_flag) {
1147 dict_flag = ZLIB_DICT_FLAG;
1148 hdr_size = ZLIB_HDR_BASE + ZLIB_DICT_LEN;
1149 }
1150
1151 if (stream->avail_out < hdr_size)
1152 return hdr_size;
1153
1154 cmf = DEFLATE_METHOD | (z_hdr->info << 4);
1155 flg = (z_hdr->level << 6) | dict_flag;
1156
1157 flg += 31 - ((256 * cmf + flg) % 31);
1158
1159 out_buf[0] = cmf;
1160 out_buf[1] = flg;
1161
1162 if (dict_flag)
1163 store_le_u32(out_buf + 2, z_hdr->dict_id);
1164
1165 stream->next_out += hdr_size;
1166 stream->total_out += hdr_size;
1167 stream->avail_out -= hdr_size;
1168
1169 return ISAL_DECOMP_OK;
1170 }
1171
1172 int
isal_deflate_set_hufftables(struct isal_zstream * stream,struct isal_hufftables * hufftables,int type)1173 isal_deflate_set_hufftables(struct isal_zstream *stream, struct isal_hufftables *hufftables,
1174 int type)
1175 {
1176 if (stream->internal_state.state != ZSTATE_NEW_HDR)
1177 return ISAL_INVALID_OPERATION;
1178
1179 switch (type) {
1180 case IGZIP_HUFFTABLE_DEFAULT:
1181 stream->hufftables = (struct isal_hufftables *) &hufftables_default;
1182 break;
1183 case IGZIP_HUFFTABLE_STATIC:
1184 stream->hufftables = (struct isal_hufftables *) &hufftables_static;
1185 break;
1186 case IGZIP_HUFFTABLE_CUSTOM:
1187 if (hufftables != NULL) {
1188 stream->hufftables = hufftables;
1189 break;
1190 }
1191 default:
1192 return ISAL_INVALID_OPERATION;
1193 }
1194
1195 return COMP_OK;
1196 }
1197
1198 void
isal_deflate_stateless_init(struct isal_zstream * stream)1199 isal_deflate_stateless_init(struct isal_zstream *stream)
1200 {
1201 stream->total_in = 0;
1202 stream->total_out = 0;
1203 stream->hufftables = (struct isal_hufftables *) &hufftables_default;
1204 stream->level = 0;
1205 stream->level_buf = NULL;
1206 stream->level_buf_size = 0;
1207 stream->end_of_stream = 0;
1208 stream->flush = NO_FLUSH;
1209 stream->gzip_flag = 0;
1210 stream->hist_bits = 0;
1211 stream->internal_state.has_wrap_hdr = 0;
1212 stream->internal_state.state = ZSTATE_NEW_HDR;
1213 return;
1214 }
1215
1216 void
isal_deflate_hash(struct isal_zstream * stream,uint8_t * dict,uint32_t dict_len)1217 isal_deflate_hash(struct isal_zstream *stream, uint8_t *dict, uint32_t dict_len)
1218 {
1219 /* Reset history to prevent out of bounds matches this works because
1220 * dictionary must set at least 1 element in the history */
1221 struct level_buf *level_buf = (struct level_buf *) stream->level_buf;
1222 uint32_t hash_mask = stream->internal_state.hash_mask;
1223
1224 switch (stream->level) {
1225 case 3:
1226 memset(level_buf->lvl3.hash_table, -1, sizeof(level_buf->lvl3.hash_table));
1227 isal_deflate_hash_lvl3(level_buf->lvl3.hash_table, hash_mask, stream->total_in,
1228 dict, dict_len);
1229 break;
1230
1231 case 2:
1232 memset(level_buf->lvl2.hash_table, -1, sizeof(level_buf->lvl2.hash_table));
1233 isal_deflate_hash_lvl2(level_buf->lvl2.hash_table, hash_mask, stream->total_in,
1234 dict, dict_len);
1235 break;
1236 case 1:
1237 memset(level_buf->lvl1.hash_table, -1, sizeof(level_buf->lvl1.hash_table));
1238 isal_deflate_hash_lvl1(level_buf->lvl1.hash_table, hash_mask, stream->total_in,
1239 dict, dict_len);
1240 break;
1241 default:
1242 memset(stream->internal_state.head, -1, sizeof(stream->internal_state.head));
1243 isal_deflate_hash_lvl0(stream->internal_state.head, hash_mask, stream->total_in,
1244 dict, dict_len);
1245 }
1246
1247 stream->internal_state.has_hist = IGZIP_HIST;
1248 }
1249
1250 int
isal_deflate_process_dict(struct isal_zstream * stream,struct isal_dict * dict,uint8_t * dict_data,uint32_t dict_len)1251 isal_deflate_process_dict(struct isal_zstream *stream, struct isal_dict *dict, uint8_t *dict_data,
1252 uint32_t dict_len)
1253 {
1254 if ((dict == NULL) || (dict_len == 0) || (dict->level > ISAL_DEF_MAX_LEVEL))
1255 return ISAL_INVALID_STATE;
1256
1257 if (dict_len > IGZIP_HIST_SIZE) {
1258 dict_data = dict_data + dict_len - IGZIP_HIST_SIZE;
1259 dict_len = IGZIP_HIST_SIZE;
1260 }
1261
1262 dict->level = stream->level;
1263 dict->hist_size = dict_len;
1264 memcpy(dict->history, dict_data, dict_len);
1265 memset(dict->hashtable, -1, sizeof(dict->hashtable));
1266
1267 switch (stream->level) {
1268 case 3:
1269 dict->hash_size = IGZIP_LVL3_HASH_SIZE;
1270 isal_deflate_hash_lvl3(dict->hashtable, LVL3_HASH_MASK, 0, dict_data, dict_len);
1271 break;
1272
1273 case 2:
1274 dict->hash_size = IGZIP_LVL2_HASH_SIZE;
1275 isal_deflate_hash_lvl2(dict->hashtable, LVL2_HASH_MASK, 0, dict_data, dict_len);
1276 break;
1277 case 1:
1278 dict->hash_size = IGZIP_LVL1_HASH_SIZE;
1279 isal_deflate_hash_lvl1(dict->hashtable, LVL1_HASH_MASK, 0, dict_data, dict_len);
1280 break;
1281 default:
1282 dict->hash_size = IGZIP_LVL0_HASH_SIZE;
1283 isal_deflate_hash_lvl0(dict->hashtable, LVL0_HASH_MASK, 0, dict_data, dict_len);
1284 }
1285 return COMP_OK;
1286 }
1287
1288 int
isal_deflate_reset_dict(struct isal_zstream * stream,struct isal_dict * dict)1289 isal_deflate_reset_dict(struct isal_zstream *stream, struct isal_dict *dict)
1290 {
1291 struct isal_zstate *state = &stream->internal_state;
1292 struct level_buf *level_buf = (struct level_buf *) stream->level_buf;
1293 int ret;
1294
1295 if ((state->state != ZSTATE_NEW_HDR) ||
1296 (state->b_bytes_processed != state->b_bytes_valid) || (dict->level != stream->level) ||
1297 (dict->hist_size == 0) || (dict->hist_size > IGZIP_HIST_SIZE) ||
1298 (dict->hash_size > IGZIP_LVL3_HASH_SIZE))
1299 return ISAL_INVALID_STATE;
1300
1301 ret = check_level_req(stream);
1302 if (ret)
1303 return ret;
1304
1305 memcpy(state->buffer, dict->history, dict->hist_size);
1306 state->b_bytes_processed = dict->hist_size;
1307 state->b_bytes_valid = dict->hist_size;
1308 state->has_hist = IGZIP_DICT_HASH_SET;
1309
1310 switch (stream->level) {
1311 case 3:
1312 memcpy(level_buf->lvl3.hash_table, dict->hashtable,
1313 sizeof(level_buf->lvl3.hash_table));
1314 break;
1315
1316 case 2:
1317 memcpy(level_buf->lvl2.hash_table, dict->hashtable,
1318 sizeof(level_buf->lvl2.hash_table));
1319 break;
1320 case 1:
1321 memcpy(level_buf->lvl1.hash_table, dict->hashtable,
1322 sizeof(level_buf->lvl1.hash_table));
1323 break;
1324 default:
1325 memcpy(stream->internal_state.head, dict->hashtable,
1326 sizeof(stream->internal_state.head));
1327 }
1328
1329 return COMP_OK;
1330 }
1331
1332 int
isal_deflate_set_dict(struct isal_zstream * stream,uint8_t * dict,uint32_t dict_len)1333 isal_deflate_set_dict(struct isal_zstream *stream, uint8_t *dict, uint32_t dict_len)
1334 {
1335 struct isal_zstate *state = &stream->internal_state;
1336
1337 if (state->state != ZSTATE_NEW_HDR || state->b_bytes_processed != state->b_bytes_valid)
1338 return ISAL_INVALID_STATE;
1339
1340 if (dict_len <= 0)
1341 return COMP_OK;
1342
1343 if (dict_len > IGZIP_HIST_SIZE) {
1344 dict = dict + dict_len - IGZIP_HIST_SIZE;
1345 dict_len = IGZIP_HIST_SIZE;
1346 }
1347
1348 memcpy(state->buffer, dict, dict_len);
1349 state->b_bytes_processed = dict_len;
1350 state->b_bytes_valid = dict_len;
1351
1352 state->has_hist = IGZIP_DICT_HIST;
1353
1354 return COMP_OK;
1355 }
1356
1357 int
isal_deflate_stateless(struct isal_zstream * stream)1358 isal_deflate_stateless(struct isal_zstream *stream)
1359 {
1360 struct isal_zstate *state = &stream->internal_state;
1361 uint8_t *next_in = stream->next_in;
1362 const uint32_t avail_in = stream->avail_in;
1363 const uint32_t total_in = stream->total_in;
1364
1365 uint8_t *next_out = stream->next_out;
1366 const uint32_t avail_out = stream->avail_out;
1367 const uint32_t total_out = stream->total_out;
1368 const uint32_t gzip_flag = stream->gzip_flag;
1369 const uint32_t has_wrap_hdr = state->has_wrap_hdr;
1370
1371 int level_check;
1372 uint64_t stored_len;
1373
1374 /* Final block has already been written */
1375 state->block_next = stream->total_in;
1376 state->block_end = stream->total_in;
1377 state->has_eob_hdr = 0;
1378 init(&state->bitbuf);
1379 state->state = ZSTATE_NEW_HDR;
1380 state->crc = 0;
1381 state->has_level_buf_init = 0;
1382 set_dist_mask(stream);
1383
1384 if (stream->flush == NO_FLUSH)
1385 stream->end_of_stream = 1;
1386
1387 if (stream->flush != NO_FLUSH && stream->flush != FULL_FLUSH)
1388 return INVALID_FLUSH;
1389
1390 level_check = check_level_req(stream);
1391 if (level_check) {
1392 if (stream->level == 1 && stream->level_buf == NULL) {
1393 /* Default to internal buffer if invalid size is supplied */
1394 stream->level_buf = state->buffer;
1395 stream->level_buf_size = sizeof(state->buffer) + sizeof(state->head);
1396 } else
1397 return level_check;
1398 }
1399
1400 set_hash_mask(stream);
1401
1402 if (state->hash_mask > 2 * avail_in)
1403 state->hash_mask = (1 << bsr(avail_in)) - 1;
1404
1405 if (avail_in == 0)
1406 stored_len = TYPE0_BLK_HDR_LEN;
1407 else {
1408 stored_len = TYPE0_BLK_HDR_LEN *
1409 ((avail_in + TYPE0_MAX_BLK_LEN - 1) / TYPE0_MAX_BLK_LEN);
1410 stored_len += avail_in;
1411 }
1412
1413 /*
1414 at least 1 byte compressed data in the case of empty dynamic block which only
1415 contains the EOB
1416 */
1417 if (stream->gzip_flag == IGZIP_GZIP)
1418 stored_len += gzip_hdr_bytes + gzip_trl_bytes;
1419 else if (stream->gzip_flag == IGZIP_GZIP_NO_HDR)
1420 stored_len += gzip_trl_bytes;
1421
1422 else if (stream->gzip_flag == IGZIP_ZLIB)
1423 stored_len += zlib_hdr_bytes + zlib_trl_bytes;
1424
1425 else if (stream->gzip_flag == IGZIP_ZLIB_NO_HDR)
1426 stored_len += zlib_trl_bytes;
1427
1428 if (avail_out >= stored_len)
1429 stream->avail_out = stored_len;
1430
1431 if (isal_deflate_int_stateless(stream) == COMP_OK) {
1432 if (avail_out >= stored_len)
1433 stream->avail_out += avail_out - stored_len;
1434 return COMP_OK;
1435 } else {
1436 if (avail_out >= stored_len)
1437 stream->avail_out += avail_out - stored_len;
1438 if (stream->flush == FULL_FLUSH) {
1439 reset_match_history(stream);
1440 }
1441 stream->internal_state.has_eob_hdr = 0;
1442 }
1443
1444 if (avail_out < stored_len)
1445 return STATELESS_OVERFLOW;
1446
1447 stream->next_in = next_in + avail_in;
1448 stream->avail_in = 0;
1449 stream->total_in = avail_in;
1450
1451 state->block_next = stream->total_in - avail_in;
1452 state->block_end = stream->total_in;
1453
1454 stream->next_out = next_out;
1455 stream->avail_out = avail_out;
1456 stream->total_out = total_out;
1457
1458 stream->gzip_flag = gzip_flag;
1459 state->has_wrap_hdr = has_wrap_hdr;
1460 init(&stream->internal_state.bitbuf);
1461 stream->internal_state.count = 0;
1462
1463 if (stream->gzip_flag == IGZIP_GZIP || stream->gzip_flag == IGZIP_ZLIB)
1464 write_stream_header_stateless(stream);
1465
1466 stream->internal_state.state = ZSTATE_TYPE0_HDR;
1467
1468 write_stored_block(stream);
1469
1470 stream->total_in = total_in + avail_in;
1471
1472 if (stream->gzip_flag) {
1473 stream->internal_state.crc = 0;
1474 update_checksum(stream, next_in, avail_in);
1475 }
1476
1477 if (stream->end_of_stream)
1478 write_trailer(stream);
1479
1480 return COMP_OK;
1481 }
1482
1483 static inline uint32_t
get_hist_size(struct isal_zstream * stream,uint8_t * start_in,int32_t buf_hist_start)1484 get_hist_size(struct isal_zstream *stream, uint8_t *start_in, int32_t buf_hist_start)
1485 {
1486 struct isal_zstate *state = &stream->internal_state;
1487 uint32_t history_size;
1488 uint32_t buffered_history;
1489 uint32_t buffered_size = state->b_bytes_valid - state->b_bytes_processed;
1490 uint32_t input_history;
1491
1492 buffered_history = (state->has_hist) ? state->b_bytes_processed - buf_hist_start : 0;
1493 input_history = stream->next_in - start_in;
1494
1495 /* Calculate history required for deflate window */
1496 history_size = (buffered_history >= input_history) ? buffered_history : input_history;
1497 if (history_size > IGZIP_HIST_SIZE)
1498 history_size = IGZIP_HIST_SIZE;
1499
1500 /* Calculate history required based on internal state */
1501 if (state->state == ZSTATE_TYPE0_HDR || state->state == ZSTATE_TYPE0_BODY ||
1502 state->state == ZSTATE_TMP_TYPE0_HDR || state->state == ZSTATE_TMP_TYPE0_BODY) {
1503 if (stream->total_in - state->block_next > history_size) {
1504 history_size = (stream->total_in - state->block_next);
1505 }
1506 } else if (stream->avail_in + buffered_size == 0 &&
1507 (stream->end_of_stream || stream->flush == FULL_FLUSH)) {
1508 history_size = 0;
1509 }
1510 return history_size;
1511 }
1512
1513 int
isal_deflate(struct isal_zstream * stream)1514 isal_deflate(struct isal_zstream *stream)
1515 {
1516 struct isal_zstate *state = &stream->internal_state;
1517 int ret = COMP_OK;
1518 uint8_t *next_in, *start_in, *buf_start_in, *next_in_pre;
1519 uint32_t avail_in, total_start, hist_size, future_size;
1520 uint32_t in_size, in_size_initial, out_size, out_size_initial;
1521 uint32_t processed, buffered_size = state->b_bytes_valid - state->b_bytes_processed;
1522 uint32_t flush_type = stream->flush;
1523 uint32_t end_of_stream = stream->end_of_stream;
1524 uint32_t size = 0;
1525 int32_t buf_hist_start = 0;
1526 uint8_t *copy_down_src = NULL;
1527 uint64_t copy_down_size = 0, copy_start_offset;
1528 int internal;
1529
1530 if (stream->flush >= 3)
1531 return INVALID_FLUSH;
1532
1533 ret = check_level_req(stream);
1534 if (ret)
1535 return ret;
1536
1537 start_in = stream->next_in;
1538 total_start = stream->total_in;
1539
1540 hist_size = get_hist_size(stream, start_in, buf_hist_start);
1541
1542 if (state->has_hist == IGZIP_NO_HIST) {
1543 set_dist_mask(stream);
1544 set_hash_mask(stream);
1545 if (state->hash_mask > 2 * stream->avail_in &&
1546 (stream->flush == FULL_FLUSH || stream->end_of_stream))
1547 state->hash_mask = (1 << bsr(2 * stream->avail_in)) - 1;
1548 stream->total_in -= buffered_size;
1549 reset_match_history(stream);
1550 stream->total_in += buffered_size;
1551 buf_hist_start = state->b_bytes_processed;
1552
1553 } else if (state->has_hist == IGZIP_DICT_HIST) {
1554 set_dist_mask(stream);
1555 set_hash_mask(stream);
1556 isal_deflate_hash(stream, state->buffer, state->b_bytes_processed);
1557 } else if (state->has_hist == IGZIP_DICT_HASH_SET) {
1558 set_dist_mask(stream);
1559 set_hash_mask(stream);
1560 }
1561
1562 in_size = stream->avail_in + buffered_size;
1563 out_size = stream->total_out;
1564 do {
1565 in_size_initial = in_size;
1566 out_size_initial = out_size;
1567 buf_start_in = start_in;
1568 internal = 0;
1569
1570 /* Setup to compress from internal buffer if insufficient history */
1571 if (stream->total_in - total_start < hist_size + buffered_size) {
1572 /* On entry there should always be sufficient history bufferd */
1573 /* assert(state->b_bytes_processed >= hist_size); */
1574
1575 internal = 1;
1576 /* Shift down internal buffer if it contains more data
1577 * than required */
1578 if (state->b_bytes_processed > hist_size) {
1579 copy_start_offset = state->b_bytes_processed - hist_size;
1580
1581 copy_down_src = &state->buffer[copy_start_offset];
1582 copy_down_size = state->b_bytes_valid - copy_start_offset;
1583 memmove(state->buffer, copy_down_src, copy_down_size);
1584
1585 state->b_bytes_valid -= copy_down_src - state->buffer;
1586 state->b_bytes_processed -= copy_down_src - state->buffer;
1587 buf_hist_start -= copy_down_src - state->buffer;
1588 if (buf_hist_start < 0)
1589 buf_hist_start = 0;
1590 }
1591
1592 size = stream->avail_in;
1593 if (size > sizeof(state->buffer) - state->b_bytes_valid)
1594 size = sizeof(state->buffer) - state->b_bytes_valid;
1595
1596 memcpy(&state->buffer[state->b_bytes_valid], stream->next_in, size);
1597
1598 stream->next_in += size;
1599 stream->avail_in -= size;
1600 stream->total_in += size;
1601 state->b_bytes_valid += size;
1602 buffered_size += size;
1603
1604 /* Save off next_in and avail_in if compression is
1605 * performed in internal buffer, total_in can be
1606 * recovered from knowledge of the size of the buffered
1607 * input */
1608 next_in = stream->next_in;
1609 avail_in = stream->avail_in;
1610
1611 /* If not much data is buffered and there is no need to
1612 * flush the buffer, just continue rather than attempt
1613 * to compress */
1614 if (avail_in == 0 && buffered_size <= IGZIP_HIST_SIZE &&
1615 stream->total_in - buffered_size - state->block_next <=
1616 IGZIP_HIST_SIZE &&
1617 !stream->end_of_stream && stream->flush == NO_FLUSH)
1618 continue;
1619
1620 if (avail_in) {
1621 stream->flush = NO_FLUSH;
1622 stream->end_of_stream = 0;
1623 }
1624
1625 stream->next_in = &state->buffer[state->b_bytes_processed];
1626 stream->avail_in = buffered_size;
1627 stream->total_in -= buffered_size;
1628
1629 buf_start_in = state->buffer;
1630
1631 } else if (buffered_size) {
1632 /* The user provided buffer has sufficient data, reset
1633 * the user supplied buffer to included any data already
1634 * buffered */
1635 stream->next_in -= buffered_size;
1636 stream->avail_in += buffered_size;
1637 stream->total_in -= buffered_size;
1638 state->b_bytes_valid = 0;
1639 state->b_bytes_processed = 0;
1640 buffered_size = 0;
1641 }
1642
1643 next_in_pre = stream->next_in;
1644 isal_deflate_int(stream, buf_start_in);
1645 processed = stream->next_in - next_in_pre;
1646 hist_size = get_hist_size(stream, buf_start_in, buf_hist_start);
1647
1648 /* Restore compression to unbuffered input when compressing to internal buffer */
1649 if (internal) {
1650 state->b_bytes_processed += processed;
1651 buffered_size -= processed;
1652
1653 stream->flush = flush_type;
1654 stream->end_of_stream = end_of_stream;
1655 stream->total_in += buffered_size;
1656
1657 stream->next_in = next_in;
1658 stream->avail_in = avail_in;
1659 }
1660
1661 in_size = stream->avail_in + buffered_size;
1662 out_size = stream->total_out;
1663
1664 } while (internal && stream->avail_in > 0 && stream->avail_out > 0 &&
1665 (in_size_initial != in_size || out_size_initial != out_size));
1666
1667 /* Buffer history if data was pulled from the external buffer and future
1668 * calls to deflate will be required */
1669 if (!internal && (state->state != ZSTATE_END && state->state != ZSTATE_TRL)) {
1670 /* If the external buffer was used, sufficient history must
1671 * exist in the user input buffer */
1672 /* assert(stream->total_in - total_start >= */
1673 /* hist_size + buffered_size); */
1674
1675 stream->next_in -= buffered_size;
1676 stream->avail_in += buffered_size;
1677 stream->total_in -= buffered_size;
1678
1679 memmove(state->buffer, stream->next_in - hist_size, hist_size);
1680 state->b_bytes_processed = hist_size;
1681 state->b_bytes_valid = hist_size;
1682 buffered_size = 0;
1683 }
1684
1685 /* Buffer input data if it is necessary for continued execution */
1686 if (stream->avail_in > 0 && (stream->avail_out > 0 || stream->level == 3)) {
1687 /* Determine how much data to buffer */
1688 future_size = sizeof(state->buffer) - state->b_bytes_valid;
1689 if (stream->avail_in < future_size)
1690 /* Buffer all data if it fits as it will need to be buffered
1691 * on the next call anyways*/
1692 future_size = stream->avail_in;
1693 else if (ISAL_LOOK_AHEAD < future_size)
1694 /* Buffer a minimum look ahead required for level 3 */
1695 future_size = ISAL_LOOK_AHEAD;
1696
1697 memcpy(&state->buffer[state->b_bytes_valid], stream->next_in, future_size);
1698
1699 state->b_bytes_valid += future_size;
1700 buffered_size += future_size;
1701 stream->next_in += future_size;
1702 stream->total_in += future_size;
1703 stream->avail_in -= future_size;
1704 }
1705
1706 return ret;
1707 }
1708
1709 // Helper function to avoid code duplication.
1710 static void
_zlib_header_in_buffer(struct isal_zstream * stream,uint8_t * buffer)1711 _zlib_header_in_buffer(struct isal_zstream *stream, uint8_t *buffer)
1712 {
1713 uint8_t hist_bits, info, level, cmf, flg;
1714 uint8_t dict_flag = 0;
1715
1716 if (stream->hist_bits == 0) // default hist_bits
1717 hist_bits = ISAL_DEF_MAX_HIST_BITS;
1718 else
1719 hist_bits = stream->hist_bits;
1720 if (hist_bits > 8)
1721 info = hist_bits - 8;
1722 else
1723 info = 0; // For low window sizes ensure correct cmf flag.
1724 if (stream->level == 0)
1725 level = 0; // Fastest algorithm
1726 else
1727 level = 1; // ISA-L levels 1-3 are fast algorithms.
1728
1729 cmf = DEFLATE_METHOD | (info << 4);
1730 flg = (level << 6) | dict_flag;
1731 flg += 31 - ((256 * cmf + flg) % 31);
1732 buffer[0] = cmf;
1733 buffer[1] = flg;
1734 return;
1735 }
1736
1737 static int
write_stream_header_stateless(struct isal_zstream * stream)1738 write_stream_header_stateless(struct isal_zstream *stream)
1739 {
1740 uint32_t hdr_bytes;
1741 // Create a 10-byte buffer. Since the gzip header is almost fixed (9 of 10
1742 // bytes are fixed) use it to initialize the buffer.
1743 uint8_t buffer[10] = { 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff };
1744 uint32_t next_flag;
1745
1746 if (stream->internal_state.has_wrap_hdr)
1747 return COMP_OK;
1748
1749 if (stream->gzip_flag == IGZIP_ZLIB) {
1750 hdr_bytes = zlib_hdr_bytes;
1751 _zlib_header_in_buffer(stream, buffer);
1752 next_flag = IGZIP_ZLIB_NO_HDR;
1753 } else {
1754 hdr_bytes = gzip_hdr_bytes;
1755 if (stream->level == 0)
1756 buffer[8] = 0x04; // Fastest algorithm in xfl flag
1757 next_flag = IGZIP_GZIP_NO_HDR;
1758 }
1759
1760 if (hdr_bytes >= stream->avail_out)
1761 return STATELESS_OVERFLOW;
1762
1763 stream->avail_out -= hdr_bytes;
1764 stream->total_out += hdr_bytes;
1765
1766 memcpy(stream->next_out, buffer, hdr_bytes);
1767
1768 stream->next_out += hdr_bytes;
1769 stream->internal_state.has_wrap_hdr = 1;
1770 stream->gzip_flag = next_flag;
1771
1772 return COMP_OK;
1773 }
1774
1775 static void
write_stream_header(struct isal_zstream * stream)1776 write_stream_header(struct isal_zstream *stream)
1777 {
1778 struct isal_zstate *state = &stream->internal_state;
1779 int bytes_to_write;
1780 uint32_t hdr_bytes;
1781 // Create a 10-byte buffer. Since the gzip header is almost fixed (9 of 10
1782 // bytes are fixed) use it to initialize the buffer.
1783 uint8_t buffer[10] = { 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff };
1784
1785 if (stream->internal_state.has_wrap_hdr)
1786 return;
1787
1788 if (stream->gzip_flag == IGZIP_ZLIB) {
1789 hdr_bytes = zlib_hdr_bytes;
1790 _zlib_header_in_buffer(stream, buffer);
1791 } else {
1792 if (stream->level == 0)
1793 buffer[8] = 0x04; // Fastest algorithm in xfl flag
1794 hdr_bytes = gzip_hdr_bytes;
1795 }
1796
1797 bytes_to_write = hdr_bytes;
1798 bytes_to_write -= state->count;
1799
1800 if (bytes_to_write > stream->avail_out)
1801 bytes_to_write = stream->avail_out;
1802
1803 memcpy(stream->next_out, buffer + state->count, bytes_to_write);
1804 state->count += bytes_to_write;
1805
1806 if (state->count == hdr_bytes) {
1807 state->count = 0;
1808 state->has_wrap_hdr = 1;
1809 }
1810
1811 stream->avail_out -= bytes_to_write;
1812 stream->total_out += bytes_to_write;
1813 stream->next_out += bytes_to_write;
1814 }
1815
1816 static int
write_deflate_header_stateless(struct isal_zstream * stream)1817 write_deflate_header_stateless(struct isal_zstream *stream)
1818 {
1819 struct isal_zstate *state = &stream->internal_state;
1820 struct isal_hufftables *hufftables = stream->hufftables;
1821 uint64_t hdr_extra_bits = hufftables->deflate_hdr[hufftables->deflate_hdr_count];
1822 uint32_t count;
1823
1824 if (hufftables->deflate_hdr_count + 8 >= stream->avail_out)
1825 return STATELESS_OVERFLOW;
1826
1827 memcpy(stream->next_out, hufftables->deflate_hdr, hufftables->deflate_hdr_count);
1828
1829 if (stream->end_of_stream == 0) {
1830 if (hufftables->deflate_hdr_count > 0)
1831 *stream->next_out -= 1;
1832 else
1833 hdr_extra_bits -= 1;
1834 } else
1835 state->has_eob_hdr = 1;
1836
1837 stream->avail_out -= hufftables->deflate_hdr_count;
1838 stream->total_out += hufftables->deflate_hdr_count;
1839 stream->next_out += hufftables->deflate_hdr_count;
1840
1841 set_buf(&state->bitbuf, stream->next_out, stream->avail_out);
1842
1843 write_bits(&state->bitbuf, hdr_extra_bits, hufftables->deflate_hdr_extra_bits);
1844
1845 count = buffer_used(&state->bitbuf);
1846 stream->next_out = buffer_ptr(&state->bitbuf);
1847 stream->avail_out -= count;
1848 stream->total_out += count;
1849
1850 state->state = ZSTATE_BODY;
1851
1852 return COMP_OK;
1853 }
1854
1855 static int
write_deflate_header_unaligned_stateless(struct isal_zstream * stream)1856 write_deflate_header_unaligned_stateless(struct isal_zstream *stream)
1857 {
1858 struct isal_zstate *state = &stream->internal_state;
1859 struct isal_hufftables *hufftables = stream->hufftables;
1860 unsigned int count;
1861 uint64_t bit_count;
1862 uint8_t *header_next;
1863 uint8_t *header_end;
1864 uint64_t header_bits;
1865
1866 if (state->bitbuf.m_bit_count == 0)
1867 return write_deflate_header_stateless(stream);
1868
1869 if (hufftables->deflate_hdr_count + 16 >= stream->avail_out)
1870 return STATELESS_OVERFLOW;
1871
1872 set_buf(&state->bitbuf, stream->next_out, stream->avail_out);
1873
1874 header_next = hufftables->deflate_hdr;
1875 header_end = header_next +
1876 (hufftables->deflate_hdr_count / sizeof(header_bits)) * sizeof(header_bits);
1877
1878 header_bits = load_le_u64(header_next);
1879
1880 if (stream->end_of_stream == 0)
1881 header_bits--;
1882 else
1883 state->has_eob_hdr = 1;
1884
1885 header_next += sizeof(header_bits);
1886
1887 /* Write out Complete Header bits */
1888 for (; header_next <= header_end; header_next += sizeof(header_bits)) {
1889 write_bits(&state->bitbuf, header_bits, 32);
1890 header_bits >>= 32;
1891 write_bits(&state->bitbuf, header_bits, 32);
1892 header_bits = load_le_u64(header_next);
1893 }
1894 bit_count = (hufftables->deflate_hdr_count & 0x7) * 8 + hufftables->deflate_hdr_extra_bits;
1895
1896 if (bit_count > MAX_BITBUF_BIT_WRITE) {
1897 write_bits(&state->bitbuf, header_bits, MAX_BITBUF_BIT_WRITE);
1898 header_bits >>= MAX_BITBUF_BIT_WRITE;
1899 bit_count -= MAX_BITBUF_BIT_WRITE;
1900 }
1901
1902 write_bits(&state->bitbuf, header_bits, bit_count);
1903
1904 /* check_space flushes extra bytes in bitbuf. Required because
1905 * write_bits_always fails when the next commit makes the buffer
1906 * length exceed 64 bits */
1907 check_space(&state->bitbuf, FORCE_FLUSH);
1908
1909 count = buffer_used(&state->bitbuf);
1910 stream->next_out = buffer_ptr(&state->bitbuf);
1911 stream->avail_out -= count;
1912 stream->total_out += count;
1913
1914 state->state = ZSTATE_BODY;
1915
1916 return COMP_OK;
1917 }
1918
1919 /* Toggle end of stream only works when deflate header is aligned */
1920 static void
write_header(struct isal_zstream * stream,uint8_t * deflate_hdr,uint32_t deflate_hdr_count,uint32_t extra_bits_count,uint32_t next_state,uint32_t toggle_end_of_stream)1921 write_header(struct isal_zstream *stream, uint8_t *deflate_hdr, uint32_t deflate_hdr_count,
1922 uint32_t extra_bits_count, uint32_t next_state, uint32_t toggle_end_of_stream)
1923 {
1924 struct isal_zstate *state = &stream->internal_state;
1925 uint32_t hdr_extra_bits = deflate_hdr[deflate_hdr_count];
1926 uint32_t count;
1927 state->state = ZSTATE_HDR;
1928
1929 if (state->bitbuf.m_bit_count != 0) {
1930 if (stream->avail_out < 8)
1931 return;
1932 set_buf(&state->bitbuf, stream->next_out, stream->avail_out);
1933 flush(&state->bitbuf);
1934 count = buffer_used(&state->bitbuf);
1935 stream->next_out = buffer_ptr(&state->bitbuf);
1936 stream->avail_out -= count;
1937 stream->total_out += count;
1938 }
1939
1940 if (stream->gzip_flag == IGZIP_GZIP || stream->gzip_flag == IGZIP_ZLIB)
1941 write_stream_header(stream);
1942
1943 count = deflate_hdr_count - state->count;
1944
1945 if (count != 0) {
1946 if (count > stream->avail_out)
1947 count = stream->avail_out;
1948
1949 memcpy(stream->next_out, deflate_hdr + state->count, count);
1950
1951 if (toggle_end_of_stream && state->count == 0 && count > 0) {
1952 /* Assumes the final block bit is the first bit */
1953 *stream->next_out ^= 1;
1954 state->has_eob_hdr = !state->has_eob_hdr;
1955 }
1956
1957 stream->next_out += count;
1958 stream->avail_out -= count;
1959 stream->total_out += count;
1960 state->count += count;
1961
1962 count = deflate_hdr_count - state->count;
1963 } else if (toggle_end_of_stream && deflate_hdr_count == 0) {
1964 /* Assumes the final block bit is the first bit */
1965 hdr_extra_bits ^= 1;
1966 state->has_eob_hdr = !state->has_eob_hdr;
1967 }
1968
1969 if ((count == 0) && (stream->avail_out >= 8)) {
1970
1971 set_buf(&state->bitbuf, stream->next_out, stream->avail_out);
1972
1973 write_bits(&state->bitbuf, hdr_extra_bits, extra_bits_count);
1974
1975 state->state = next_state;
1976 state->count = 0;
1977
1978 count = buffer_used(&state->bitbuf);
1979 stream->next_out = buffer_ptr(&state->bitbuf);
1980 stream->avail_out -= count;
1981 stream->total_out += count;
1982 }
1983 }
1984
1985 static void
write_trailer(struct isal_zstream * stream)1986 write_trailer(struct isal_zstream *stream)
1987 {
1988 struct isal_zstate *state = &stream->internal_state;
1989 unsigned int bytes = 0;
1990 uint32_t crc = state->crc;
1991
1992 set_buf(&state->bitbuf, stream->next_out, stream->avail_out);
1993
1994 if (!state->has_eob_hdr) {
1995 /* If the final header has not been written, write a
1996 * final block. This block is a static huffman block
1997 * which only contains the end of block symbol. The code
1998 * that happens to do this is the fist 10 bits of
1999 * 0x003 */
2000 if (stream->avail_out < 8)
2001 return;
2002
2003 state->has_eob_hdr = 1;
2004 write_bits(&state->bitbuf, 0x003, 10);
2005 if (is_full(&state->bitbuf)) {
2006 stream->next_out = buffer_ptr(&state->bitbuf);
2007 bytes = buffer_used(&state->bitbuf);
2008 stream->avail_out -= bytes;
2009 stream->total_out += bytes;
2010 return;
2011 }
2012 }
2013
2014 if (state->bitbuf.m_bit_count) {
2015 /* the flush() will pad to the next byte and write up to 8 bytes
2016 * to the output stream/buffer.
2017 */
2018 if (stream->avail_out < 8)
2019 return;
2020
2021 flush(&state->bitbuf);
2022 }
2023
2024 stream->next_out = buffer_ptr(&state->bitbuf);
2025 bytes = buffer_used(&state->bitbuf);
2026
2027 switch (stream->gzip_flag) {
2028 case IGZIP_GZIP:
2029 case IGZIP_GZIP_NO_HDR:
2030 if (stream->avail_out - bytes >= gzip_trl_bytes) {
2031 store_le_u64(stream->next_out, ((uint64_t) stream->total_in << 32) | crc);
2032 stream->next_out += gzip_trl_bytes;
2033 bytes += gzip_trl_bytes;
2034 state->state = ZSTATE_END;
2035 }
2036 break;
2037
2038 case IGZIP_ZLIB:
2039 case IGZIP_ZLIB_NO_HDR:
2040 if (stream->avail_out - bytes >= zlib_trl_bytes) {
2041 store_be_u32(stream->next_out,
2042 (crc & 0xFFFF0000) | ((crc & 0xFFFF) + 1) % ADLER_MOD);
2043 stream->next_out += zlib_trl_bytes;
2044 bytes += zlib_trl_bytes;
2045 state->state = ZSTATE_END;
2046 }
2047 break;
2048
2049 default:
2050 state->state = ZSTATE_END;
2051 }
2052
2053 stream->avail_out -= bytes;
2054 stream->total_out += bytes;
2055 }
2056