1 /* LTO IL compression streams.
2
3 Copyright (C) 2009-2022 Free Software Foundation, Inc.
4 Contributed by Simon Baldwin <simonb@google.com>
5
6 This file is part of GCC.
7
8 GCC is free software; you can redistribute it and/or modify it
9 under the terms of the GNU General Public License as published by
10 the Free Software Foundation; either version 3, or (at your option)
11 any later version.
12
13 GCC is distributed in the hope that it will be useful, but WITHOUT
14 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
15 or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
16 License for more details.
17
18 You should have received a copy of the GNU General Public License
19 along with GCC; see the file COPYING3. If not see
20 <http://www.gnu.org/licenses/>. */
21
22 #include "config.h"
23 #include "system.h"
24 #include "coretypes.h"
25 #include "backend.h"
26 #include "tree.h"
27 #include "gimple.h"
28 #include "cgraph.h"
29 #include "lto-streamer.h"
30 /* zlib.h includes other system headers. Those headers may test feature
31 test macros. config.h may define feature test macros. For this reason,
32 zlib.h needs to be included after, rather than before, config.h and
33 system.h. */
34 #include <zlib.h>
35 #include "lto-compress.h"
36 #include "timevar.h"
37
38 #ifdef HAVE_ZSTD_H
39 #include <zstd.h>
40 #endif
41
42 /* Compression stream structure, holds the flush callback and opaque token,
43 the buffered data, and a note of whether compressing or uncompressing. */
44
45 struct lto_compression_stream
46 {
47 void (*callback) (const char *, unsigned, void *);
48 void *opaque;
49 char *buffer;
50 size_t bytes;
51 size_t allocation;
52 bool is_compression;
53 };
54
55 /* Overall compression constants for zlib. */
56
57 static const size_t Z_BUFFER_LENGTH = 4096;
58 static const size_t MIN_STREAM_ALLOCATION = 1024;
59
60 /* For zlib, allocate SIZE count of ITEMS and return the address, OPAQUE
61 is unused. */
62
63 static void *
lto_zalloc(void * opaque,unsigned items,unsigned size)64 lto_zalloc (void *opaque, unsigned items, unsigned size)
65 {
66 gcc_assert (opaque == Z_NULL);
67 return xmalloc (items * size);
68 }
69
70 /* For zlib, free memory at ADDRESS, OPAQUE is unused. */
71
72 static void
lto_zfree(void * opaque,void * address)73 lto_zfree (void *opaque, void *address)
74 {
75 gcc_assert (opaque == Z_NULL);
76 free (address);
77 }
78
79 /* Return a zlib compression level that zlib will not reject. Normalizes
80 the compression level from the command line flag, clamping non-default
81 values to the appropriate end of their valid range. */
82
83 static int
lto_normalized_zlib_level(void)84 lto_normalized_zlib_level (void)
85 {
86 int level = flag_lto_compression_level;
87
88 if (level != Z_DEFAULT_COMPRESSION)
89 {
90 if (level < Z_NO_COMPRESSION)
91 level = Z_NO_COMPRESSION;
92 else if (level > Z_BEST_COMPRESSION)
93 level = Z_BEST_COMPRESSION;
94 }
95
96 return level;
97 }
98
99 /* Free the buffer and memory associated with STREAM. */
100
101 static void
lto_destroy_compression_stream(struct lto_compression_stream * stream)102 lto_destroy_compression_stream (struct lto_compression_stream *stream)
103 {
104 free (stream->buffer);
105 free (stream);
106 }
107
108 #ifdef HAVE_ZSTD_H
109 /* Return a zstd compression level that zstd will not reject. Normalizes
110 the compression level from the command line flag, clamping non-default
111 values to the appropriate end of their valid range. */
112
113 static int
lto_normalized_zstd_level(void)114 lto_normalized_zstd_level (void)
115 {
116 int level = flag_lto_compression_level;
117
118 if (level < 0)
119 level = 0;
120 else if (level > ZSTD_maxCLevel ())
121 level = ZSTD_maxCLevel ();
122
123 return level;
124 }
125
126 /* Compress STREAM using ZSTD algorithm. */
127
128 static void
lto_compression_zstd(struct lto_compression_stream * stream)129 lto_compression_zstd (struct lto_compression_stream *stream)
130 {
131 unsigned char *cursor = (unsigned char *) stream->buffer;
132 size_t size = stream->bytes;
133
134 timevar_push (TV_IPA_LTO_COMPRESS);
135 size_t const outbuf_length = ZSTD_compressBound (size);
136 char *outbuf = (char *) xmalloc (outbuf_length);
137
138 size_t const csize = ZSTD_compress (outbuf, outbuf_length, cursor, size,
139 lto_normalized_zstd_level ());
140
141 if (ZSTD_isError (csize))
142 internal_error ("compressed stream: %s", ZSTD_getErrorName (csize));
143
144 lto_stats.num_compressed_il_bytes += csize;
145 stream->callback (outbuf, csize, NULL);
146
147 lto_destroy_compression_stream (stream);
148 free (outbuf);
149 timevar_pop (TV_IPA_LTO_COMPRESS);
150 }
151
152 /* Uncompress STREAM using ZSTD algorithm. */
153
154 static void
lto_uncompression_zstd(struct lto_compression_stream * stream)155 lto_uncompression_zstd (struct lto_compression_stream *stream)
156 {
157 unsigned char *cursor = (unsigned char *) stream->buffer;
158 size_t size = stream->bytes;
159
160 timevar_push (TV_IPA_LTO_DECOMPRESS);
161 unsigned long long const rsize = ZSTD_getFrameContentSize (cursor, size);
162 if (rsize == ZSTD_CONTENTSIZE_ERROR)
163 internal_error ("original not compressed with zstd");
164 else if (rsize == ZSTD_CONTENTSIZE_UNKNOWN)
165 internal_error ("original size unknown");
166
167 char *outbuf = (char *) xmalloc (rsize);
168 size_t const dsize = ZSTD_decompress (outbuf, rsize, cursor, size);
169
170 if (ZSTD_isError (dsize))
171 internal_error ("decompressed stream: %s", ZSTD_getErrorName (dsize));
172
173 lto_stats.num_uncompressed_il_bytes += dsize;
174 stream->callback (outbuf, dsize, stream->opaque);
175
176 lto_destroy_compression_stream (stream);
177 free (outbuf);
178 timevar_pop (TV_IPA_LTO_DECOMPRESS);
179 }
180
181 #endif
182
183 /* Create a new compression stream, with CALLBACK flush function passed
184 OPAQUE token, IS_COMPRESSION indicates if compressing or uncompressing. */
185
186 static struct lto_compression_stream *
lto_new_compression_stream(void (* callback)(const char *,unsigned,void *),void * opaque,bool is_compression)187 lto_new_compression_stream (void (*callback) (const char *, unsigned, void *),
188 void *opaque, bool is_compression)
189 {
190 struct lto_compression_stream *stream
191 = (struct lto_compression_stream *) xmalloc (sizeof (*stream));
192
193 memset (stream, 0, sizeof (*stream));
194 stream->callback = callback;
195 stream->opaque = opaque;
196 stream->is_compression = is_compression;
197
198 return stream;
199 }
200
201 /* Append NUM_CHARS from address BASE to STREAM. */
202
203 static void
lto_append_to_compression_stream(struct lto_compression_stream * stream,const char * base,size_t num_chars)204 lto_append_to_compression_stream (struct lto_compression_stream *stream,
205 const char *base, size_t num_chars)
206 {
207 size_t required = stream->bytes + num_chars;
208
209 if (stream->allocation < required)
210 {
211 if (stream->allocation == 0)
212 stream->allocation = MIN_STREAM_ALLOCATION;
213 while (stream->allocation < required)
214 stream->allocation *= 2;
215
216 stream->buffer = (char *) xrealloc (stream->buffer, stream->allocation);
217 }
218
219 memcpy (stream->buffer + stream->bytes, base, num_chars);
220 stream->bytes += num_chars;
221 }
222
223 /* Return a new compression stream, with CALLBACK flush function passed
224 OPAQUE token. */
225
226 struct lto_compression_stream *
lto_start_compression(void (* callback)(const char *,unsigned,void *),void * opaque)227 lto_start_compression (void (*callback) (const char *, unsigned, void *),
228 void *opaque)
229 {
230 return lto_new_compression_stream (callback, opaque, true);
231 }
232
233 /* Append NUM_CHARS from address BASE to STREAM. */
234
235 void
lto_compress_block(struct lto_compression_stream * stream,const char * base,size_t num_chars)236 lto_compress_block (struct lto_compression_stream *stream,
237 const char *base, size_t num_chars)
238 {
239 gcc_assert (stream->is_compression);
240
241 lto_append_to_compression_stream (stream, base, num_chars);
242 lto_stats.num_output_il_bytes += num_chars;
243 }
244
245 static void ATTRIBUTE_UNUSED
lto_compression_zlib(struct lto_compression_stream * stream)246 lto_compression_zlib (struct lto_compression_stream *stream)
247 {
248 unsigned char *cursor = (unsigned char *) stream->buffer;
249 size_t remaining = stream->bytes;
250 const size_t outbuf_length = Z_BUFFER_LENGTH;
251 unsigned char *outbuf = (unsigned char *) xmalloc (outbuf_length);
252 z_stream out_stream;
253 int status;
254
255 gcc_assert (stream->is_compression);
256
257 timevar_push (TV_IPA_LTO_COMPRESS);
258
259 out_stream.next_out = outbuf;
260 out_stream.avail_out = outbuf_length;
261 out_stream.next_in = cursor;
262 out_stream.avail_in = remaining;
263 out_stream.zalloc = lto_zalloc;
264 out_stream.zfree = lto_zfree;
265 out_stream.opaque = Z_NULL;
266
267 status = deflateInit (&out_stream, lto_normalized_zlib_level ());
268 if (status != Z_OK)
269 internal_error ("compressed stream: %s", zError (status));
270
271 do
272 {
273 size_t in_bytes, out_bytes;
274
275 status = deflate (&out_stream, Z_FINISH);
276 if (status != Z_OK && status != Z_STREAM_END)
277 internal_error ("compressed stream: %s", zError (status));
278
279 in_bytes = remaining - out_stream.avail_in;
280 out_bytes = outbuf_length - out_stream.avail_out;
281
282 stream->callback ((const char *) outbuf, out_bytes, stream->opaque);
283 lto_stats.num_compressed_il_bytes += out_bytes;
284
285 cursor += in_bytes;
286 remaining -= in_bytes;
287
288 out_stream.next_out = outbuf;
289 out_stream.avail_out = outbuf_length;
290 out_stream.next_in = cursor;
291 out_stream.avail_in = remaining;
292 }
293 while (status != Z_STREAM_END);
294
295 status = deflateEnd (&out_stream);
296 if (status != Z_OK)
297 internal_error ("compressed stream: %s", zError (status));
298
299 lto_destroy_compression_stream (stream);
300 free (outbuf);
301 timevar_pop (TV_IPA_LTO_COMPRESS);
302 }
303
304 void
lto_end_compression(struct lto_compression_stream * stream)305 lto_end_compression (struct lto_compression_stream *stream)
306 {
307 #ifdef HAVE_ZSTD_H
308 lto_compression_zstd (stream);
309 #else
310 lto_compression_zlib (stream);
311 #endif
312 }
313
314 /* Return a new uncompression stream, with CALLBACK flush function passed
315 OPAQUE token. */
316
317 struct lto_compression_stream *
lto_start_uncompression(void (* callback)(const char *,unsigned,void *),void * opaque)318 lto_start_uncompression (void (*callback) (const char *, unsigned, void *),
319 void *opaque)
320 {
321 return lto_new_compression_stream (callback, opaque, false);
322 }
323
324 /* Append NUM_CHARS from address BASE to STREAM. */
325
326 void
lto_uncompress_block(struct lto_compression_stream * stream,const char * base,size_t num_chars)327 lto_uncompress_block (struct lto_compression_stream *stream,
328 const char *base, size_t num_chars)
329 {
330 gcc_assert (!stream->is_compression);
331
332 lto_append_to_compression_stream (stream, base, num_chars);
333 lto_stats.num_input_il_bytes += num_chars;
334 }
335
336 static void
lto_uncompression_zlib(struct lto_compression_stream * stream)337 lto_uncompression_zlib (struct lto_compression_stream *stream)
338 {
339 unsigned char *cursor = (unsigned char *) stream->buffer;
340 size_t remaining = stream->bytes;
341 const size_t outbuf_length = Z_BUFFER_LENGTH;
342 unsigned char *outbuf = (unsigned char *) xmalloc (outbuf_length);
343
344 gcc_assert (!stream->is_compression);
345 timevar_push (TV_IPA_LTO_DECOMPRESS);
346
347 while (remaining > 0)
348 {
349 z_stream in_stream;
350 size_t out_bytes;
351 int status;
352
353 in_stream.next_out = outbuf;
354 in_stream.avail_out = outbuf_length;
355 in_stream.next_in = cursor;
356 in_stream.avail_in = remaining;
357 in_stream.zalloc = lto_zalloc;
358 in_stream.zfree = lto_zfree;
359 in_stream.opaque = Z_NULL;
360
361 status = inflateInit (&in_stream);
362 if (status != Z_OK)
363 internal_error ("compressed stream: %s", zError (status));
364
365 do
366 {
367 size_t in_bytes;
368
369 status = inflate (&in_stream, Z_SYNC_FLUSH);
370 if (status != Z_OK && status != Z_STREAM_END)
371 internal_error ("compressed stream: %s", zError (status));
372
373 in_bytes = remaining - in_stream.avail_in;
374 out_bytes = outbuf_length - in_stream.avail_out;
375
376 stream->callback ((const char *) outbuf, out_bytes, stream->opaque);
377 lto_stats.num_uncompressed_il_bytes += out_bytes;
378
379 cursor += in_bytes;
380 remaining -= in_bytes;
381
382 in_stream.next_out = outbuf;
383 in_stream.avail_out = outbuf_length;
384 in_stream.next_in = cursor;
385 in_stream.avail_in = remaining;
386 }
387 while (!(status == Z_STREAM_END && out_bytes == 0));
388
389 status = inflateEnd (&in_stream);
390 if (status != Z_OK)
391 internal_error ("compressed stream: %s", zError (status));
392 }
393
394 lto_destroy_compression_stream (stream);
395 free (outbuf);
396 timevar_pop (TV_IPA_LTO_DECOMPRESS);
397 }
398
399 void
lto_end_uncompression(struct lto_compression_stream * stream,lto_compression compression)400 lto_end_uncompression (struct lto_compression_stream *stream,
401 lto_compression compression)
402 {
403 #ifdef HAVE_ZSTD_H
404 if (compression == ZSTD)
405 {
406 lto_uncompression_zstd (stream);
407 return;
408 }
409 #endif
410 if (compression == ZSTD)
411 internal_error ("compiler does not support ZSTD LTO compression");
412
413 lto_uncompression_zlib (stream);
414 }
415