1 #include <stdint.h>
2 #include "igzip_lib.h"
3 #include "huffman.h"
4 #include "huff_codes.h"
5 #include "encode_df.h"
6 #include "igzip_level_buf_structs.h"
7 #include "unaligned.h"
8
9 static inline void
write_deflate_icf(struct deflate_icf * icf,uint32_t lit_len,uint32_t lit_dist,uint32_t extra_bits)10 write_deflate_icf(struct deflate_icf *icf, uint32_t lit_len, uint32_t lit_dist, uint32_t extra_bits)
11 {
12 icf->lit_len = lit_len;
13 icf->lit_dist = lit_dist;
14 icf->dist_extra = extra_bits;
15 }
16
17 static inline void
update_state(struct isal_zstream * stream,uint8_t * start_in,uint8_t * next_in,uint8_t * end_in,struct deflate_icf * start_out,struct deflate_icf * next_out,struct deflate_icf * end_out)18 update_state(struct isal_zstream *stream, uint8_t *start_in, uint8_t *next_in, uint8_t *end_in,
19 struct deflate_icf *start_out, struct deflate_icf *next_out,
20 struct deflate_icf *end_out)
21 {
22 struct level_buf *level_buf = (struct level_buf *) stream->level_buf;
23
24 if (next_in - start_in > 0)
25 stream->internal_state.has_hist = IGZIP_HIST;
26
27 stream->next_in = next_in;
28 stream->total_in += next_in - start_in;
29 stream->internal_state.block_end = stream->total_in;
30 stream->avail_in = end_in - next_in;
31
32 level_buf->icf_buf_next = next_out;
33 level_buf->icf_buf_avail_out = end_out - next_out;
34 }
35
36 void
isal_deflate_icf_body_hash_hist_base(struct isal_zstream * stream)37 isal_deflate_icf_body_hash_hist_base(struct isal_zstream *stream)
38 {
39 uint32_t literal, hash;
40 uint8_t *start_in, *next_in, *end_in, *end, *next_hash;
41 struct deflate_icf *start_out, *next_out, *end_out;
42 uint16_t match_length;
43 uint32_t dist;
44 uint32_t code, code2, extra_bits;
45 struct isal_zstate *state = &stream->internal_state;
46 struct level_buf *level_buf = (struct level_buf *) stream->level_buf;
47 uint16_t *last_seen = level_buf->hash_hist.hash_table;
48 uint8_t *file_start = (uint8_t *) ((uintptr_t) stream->next_in - stream->total_in);
49 uint32_t hist_size = state->dist_mask;
50 uint32_t hash_mask = state->hash_mask;
51
52 if (stream->avail_in == 0) {
53 if (stream->end_of_stream || stream->flush != NO_FLUSH)
54 state->state = ZSTATE_FLUSH_READ_BUFFER;
55 return;
56 }
57
58 start_in = stream->next_in;
59 end_in = start_in + stream->avail_in;
60 next_in = start_in;
61
62 start_out = ((struct level_buf *) stream->level_buf)->icf_buf_next;
63 end_out = start_out + ((struct level_buf *) stream->level_buf)->icf_buf_avail_out /
64 sizeof(struct deflate_icf);
65 next_out = start_out;
66
67 while (next_in + ISAL_LOOK_AHEAD < end_in) {
68
69 if (next_out >= end_out) {
70 state->state = ZSTATE_CREATE_HDR;
71 update_state(stream, start_in, next_in, end_in, start_out, next_out,
72 end_out);
73 return;
74 }
75
76 literal = load_le_u32(next_in);
77 hash = compute_hash(literal) & hash_mask;
78 dist = (next_in - file_start - last_seen[hash]) & 0xFFFF;
79 last_seen[hash] = (uint64_t) (next_in - file_start);
80
81 /* The -1 are to handle the case when dist = 0 */
82 if (dist - 1 < hist_size) {
83 assert(dist != 0);
84
85 match_length = compare258(next_in - dist, next_in, 258);
86
87 if (match_length >= SHORTEST_MATCH) {
88 next_hash = next_in;
89 #ifdef ISAL_LIMIT_HASH_UPDATE
90 end = next_hash + 3;
91 #else
92 end = next_hash + match_length;
93 #endif
94 next_hash++;
95
96 for (; next_hash < end; next_hash++) {
97 literal = load_le_u32(next_hash);
98 hash = compute_hash(literal) & hash_mask;
99 last_seen[hash] = (uint64_t) (next_hash - file_start);
100 }
101
102 get_len_icf_code(match_length, &code);
103 get_dist_icf_code(dist, &code2, &extra_bits);
104
105 level_buf->hist.ll_hist[code]++;
106 level_buf->hist.d_hist[code2]++;
107
108 write_deflate_icf(next_out, code, code2, extra_bits);
109 next_out++;
110 next_in += match_length;
111
112 continue;
113 }
114 }
115
116 get_lit_icf_code(literal & 0xFF, &code);
117 level_buf->hist.ll_hist[code]++;
118 write_deflate_icf(next_out, code, NULL_DIST_SYM, 0);
119 next_out++;
120 next_in++;
121 }
122
123 update_state(stream, start_in, next_in, end_in, start_out, next_out, end_out);
124
125 assert(stream->avail_in <= ISAL_LOOK_AHEAD);
126 if (stream->end_of_stream || stream->flush != NO_FLUSH)
127 state->state = ZSTATE_FLUSH_READ_BUFFER;
128
129 return;
130 }
131
132 void
isal_deflate_icf_finish_hash_hist_base(struct isal_zstream * stream)133 isal_deflate_icf_finish_hash_hist_base(struct isal_zstream *stream)
134 {
135 uint32_t literal = 0, hash;
136 uint8_t *start_in, *next_in, *end_in, *end, *next_hash;
137 struct deflate_icf *start_out, *next_out, *end_out;
138 uint16_t match_length;
139 uint32_t dist;
140 uint32_t code, code2, extra_bits;
141 struct isal_zstate *state = &stream->internal_state;
142 struct level_buf *level_buf = (struct level_buf *) stream->level_buf;
143 uint16_t *last_seen = level_buf->hash_hist.hash_table;
144 uint8_t *file_start = (uint8_t *) ((uintptr_t) stream->next_in - stream->total_in);
145 uint32_t hist_size = state->dist_mask;
146 uint32_t hash_mask = state->hash_mask;
147
148 start_in = stream->next_in;
149 end_in = start_in + stream->avail_in;
150 next_in = start_in;
151
152 start_out = ((struct level_buf *) stream->level_buf)->icf_buf_next;
153 end_out = start_out + ((struct level_buf *) stream->level_buf)->icf_buf_avail_out /
154 sizeof(struct deflate_icf);
155 next_out = start_out;
156
157 if (stream->avail_in == 0) {
158 if (stream->end_of_stream || stream->flush != NO_FLUSH)
159 state->state = ZSTATE_CREATE_HDR;
160 return;
161 }
162
163 while (next_in + 3 < end_in) {
164 if (next_out >= end_out) {
165 state->state = ZSTATE_CREATE_HDR;
166 update_state(stream, start_in, next_in, end_in, start_out, next_out,
167 end_out);
168 return;
169 }
170
171 literal = load_le_u32(next_in);
172 hash = compute_hash(literal) & hash_mask;
173 dist = (next_in - file_start - last_seen[hash]) & 0xFFFF;
174 last_seen[hash] = (uint64_t) (next_in - file_start);
175
176 if (dist - 1 < hist_size) { /* The -1 are to handle the case when dist = 0 */
177 match_length = compare258(next_in - dist, next_in, end_in - next_in);
178
179 if (match_length >= SHORTEST_MATCH) {
180 next_hash = next_in;
181 #ifdef ISAL_LIMIT_HASH_UPDATE
182 end = next_hash + 3;
183 #else
184 end = next_hash + match_length;
185 #endif
186 next_hash++;
187
188 for (; next_hash < end - 3; next_hash++) {
189 literal = load_le_u32(next_hash);
190 hash = compute_hash(literal) & hash_mask;
191 last_seen[hash] = (uint64_t) (next_hash - file_start);
192 }
193
194 get_len_icf_code(match_length, &code);
195 get_dist_icf_code(dist, &code2, &extra_bits);
196
197 level_buf->hist.ll_hist[code]++;
198 level_buf->hist.d_hist[code2]++;
199
200 write_deflate_icf(next_out, code, code2, extra_bits);
201
202 next_out++;
203 next_in += match_length;
204
205 continue;
206 }
207 }
208
209 get_lit_icf_code(literal & 0xFF, &code);
210 level_buf->hist.ll_hist[code]++;
211 write_deflate_icf(next_out, code, NULL_DIST_SYM, 0);
212 next_out++;
213 next_in++;
214 }
215
216 while (next_in < end_in) {
217 if (next_out >= end_out) {
218 state->state = ZSTATE_CREATE_HDR;
219 update_state(stream, start_in, next_in, end_in, start_out, next_out,
220 end_out);
221 return;
222 }
223
224 literal = *next_in;
225 get_lit_icf_code(literal & 0xFF, &code);
226 level_buf->hist.ll_hist[code]++;
227 write_deflate_icf(next_out, code, NULL_DIST_SYM, 0);
228 next_out++;
229 next_in++;
230 }
231
232 if (next_in == end_in) {
233 if (stream->end_of_stream || stream->flush != NO_FLUSH)
234 state->state = ZSTATE_CREATE_HDR;
235 }
236
237 update_state(stream, start_in, next_in, end_in, start_out, next_out, end_out);
238
239 return;
240 }
241
242 void
isal_deflate_icf_finish_hash_map_base(struct isal_zstream * stream)243 isal_deflate_icf_finish_hash_map_base(struct isal_zstream *stream)
244 {
245 uint32_t literal = 0, hash;
246 uint8_t *start_in, *next_in, *end_in, *end, *next_hash;
247 struct deflate_icf *start_out, *next_out, *end_out;
248 uint16_t match_length;
249 uint32_t dist;
250 uint32_t code, code2, extra_bits;
251 struct isal_zstate *state = &stream->internal_state;
252 struct level_buf *level_buf = (struct level_buf *) stream->level_buf;
253 uint16_t *last_seen = level_buf->hash_map.hash_table;
254 uint8_t *file_start = (uint8_t *) ((uintptr_t) stream->next_in - stream->total_in);
255 uint32_t hist_size = state->dist_mask;
256 uint32_t hash_mask = state->hash_mask;
257
258 start_in = stream->next_in;
259 end_in = start_in + stream->avail_in;
260 next_in = start_in;
261
262 start_out = level_buf->icf_buf_next;
263 end_out = start_out + level_buf->icf_buf_avail_out / sizeof(struct deflate_icf);
264 next_out = start_out;
265
266 if (stream->avail_in == 0) {
267 if (stream->end_of_stream || stream->flush != NO_FLUSH)
268 state->state = ZSTATE_CREATE_HDR;
269 return;
270 }
271
272 while (next_in + 3 < end_in) {
273 if (next_out >= end_out) {
274 state->state = ZSTATE_CREATE_HDR;
275 update_state(stream, start_in, next_in, end_in, start_out, next_out,
276 end_out);
277 return;
278 }
279
280 literal = load_le_u32(next_in);
281 hash = compute_hash_mad(literal) & hash_mask;
282 dist = (next_in - file_start - last_seen[hash]) & 0xFFFF;
283 last_seen[hash] = (uint64_t) (next_in - file_start);
284
285 if (dist - 1 < hist_size) { /* The -1 are to handle the case when dist = 0 */
286 match_length = compare258(next_in - dist, next_in, end_in - next_in);
287
288 if (match_length >= SHORTEST_MATCH) {
289 next_hash = next_in;
290 #ifdef ISAL_LIMIT_HASH_UPDATE
291 end = next_hash + 3;
292 #else
293 end = next_hash + match_length;
294 #endif
295 next_hash++;
296
297 for (; next_hash < end - 3; next_hash++) {
298 literal = load_le_u32(next_hash);
299 hash = compute_hash_mad(literal) & hash_mask;
300 last_seen[hash] = (uint64_t) (next_hash - file_start);
301 }
302
303 get_len_icf_code(match_length, &code);
304 get_dist_icf_code(dist, &code2, &extra_bits);
305
306 level_buf->hist.ll_hist[code]++;
307 level_buf->hist.d_hist[code2]++;
308
309 write_deflate_icf(next_out, code, code2, extra_bits);
310
311 next_out++;
312 next_in += match_length;
313
314 continue;
315 }
316 }
317
318 get_lit_icf_code(literal & 0xFF, &code);
319 level_buf->hist.ll_hist[code]++;
320 write_deflate_icf(next_out, code, NULL_DIST_SYM, 0);
321 next_out++;
322 next_in++;
323 }
324
325 while (next_in < end_in) {
326 if (next_out >= end_out) {
327 state->state = ZSTATE_CREATE_HDR;
328 update_state(stream, start_in, next_in, end_in, start_out, next_out,
329 end_out);
330 return;
331 }
332
333 literal = *next_in;
334 get_lit_icf_code(literal & 0xFF, &code);
335 level_buf->hist.ll_hist[code]++;
336 write_deflate_icf(next_out, code, NULL_DIST_SYM, 0);
337 next_out++;
338 next_in++;
339 }
340
341 if (next_in == end_in) {
342 if (stream->end_of_stream || stream->flush != NO_FLUSH)
343 state->state = ZSTATE_CREATE_HDR;
344 }
345
346 update_state(stream, start_in, next_in, end_in, start_out, next_out, end_out);
347
348 return;
349 }
350
351 void
isal_deflate_hash_mad_base(uint16_t * hash_table,uint32_t hash_mask,uint32_t current_index,uint8_t * dict,uint32_t dict_len)352 isal_deflate_hash_mad_base(uint16_t *hash_table, uint32_t hash_mask, uint32_t current_index,
353 uint8_t *dict, uint32_t dict_len)
354 {
355 uint8_t *next_in = dict;
356 uint8_t *end_in = dict + dict_len - SHORTEST_MATCH;
357 uint32_t literal;
358 uint32_t hash;
359 uint16_t index = current_index - dict_len;
360
361 while (next_in <= end_in) {
362 literal = load_le_u32(next_in);
363 hash = compute_hash_mad(literal) & hash_mask;
364 hash_table[hash] = index;
365 index++;
366 next_in++;
367 }
368 }
369