1c4a807edSdjm /*
2*ab19a69eSdjm * Copyright (c) 2020-2022 Yubico AB. All rights reserved.
3c4a807edSdjm * Use of this source code is governed by a BSD-style
4c4a807edSdjm * license that can be found in the LICENSE file.
5c4a807edSdjm */
6c4a807edSdjm
7c4a807edSdjm #include <zlib.h>
8c4a807edSdjm #include "fido.h"
9c4a807edSdjm
10c4a807edSdjm #define BOUND (1024UL * 1024UL)
11c4a807edSdjm
12*ab19a69eSdjm /* zlib inflate (raw + headers) */
13c4a807edSdjm static int
rfc1950_inflate(fido_blob_t * out,const fido_blob_t * in,size_t origsiz)14*ab19a69eSdjm rfc1950_inflate(fido_blob_t *out, const fido_blob_t *in, size_t origsiz)
15c4a807edSdjm {
16c4a807edSdjm u_long ilen, olen;
17*ab19a69eSdjm int z;
18c4a807edSdjm
19c4a807edSdjm memset(out, 0, sizeof(*out));
20*ab19a69eSdjm
21c4a807edSdjm if (in->len > ULONG_MAX || (ilen = (u_long)in->len) > BOUND ||
22*ab19a69eSdjm origsiz > ULONG_MAX || (olen = (u_long)origsiz) > BOUND) {
23*ab19a69eSdjm fido_log_debug("%s: in->len=%zu, origsiz=%zu", __func__,
24*ab19a69eSdjm in->len, origsiz);
25c4a807edSdjm return FIDO_ERR_INVALID_ARGUMENT;
26*ab19a69eSdjm }
27*ab19a69eSdjm
28c4a807edSdjm if ((out->ptr = calloc(1, olen)) == NULL)
29c4a807edSdjm return FIDO_ERR_INTERNAL;
30c4a807edSdjm out->len = olen;
31*ab19a69eSdjm
32*ab19a69eSdjm if ((z = uncompress(out->ptr, &olen, in->ptr, ilen)) != Z_OK ||
33*ab19a69eSdjm olen > SIZE_MAX || olen != out->len) {
34*ab19a69eSdjm fido_log_debug("%s: uncompress: %d, olen=%lu, out->len=%zu",
35*ab19a69eSdjm __func__, z, olen, out->len);
36c4a807edSdjm fido_blob_reset(out);
37c4a807edSdjm return FIDO_ERR_COMPRESS;
38c4a807edSdjm }
39c4a807edSdjm
40c4a807edSdjm return FIDO_OK;
41c4a807edSdjm }
42c4a807edSdjm
43*ab19a69eSdjm /* raw inflate */
44*ab19a69eSdjm static int
rfc1951_inflate(fido_blob_t * out,const fido_blob_t * in,size_t origsiz)45*ab19a69eSdjm rfc1951_inflate(fido_blob_t *out, const fido_blob_t *in, size_t origsiz)
46*ab19a69eSdjm {
47*ab19a69eSdjm z_stream zs;
48*ab19a69eSdjm u_int ilen, olen;
49*ab19a69eSdjm int r, z;
50*ab19a69eSdjm
51*ab19a69eSdjm memset(&zs, 0, sizeof(zs));
52*ab19a69eSdjm memset(out, 0, sizeof(*out));
53*ab19a69eSdjm
54*ab19a69eSdjm if (in->len > UINT_MAX || (ilen = (u_int)in->len) > BOUND ||
55*ab19a69eSdjm origsiz > UINT_MAX || (olen = (u_int)origsiz) > BOUND) {
56*ab19a69eSdjm fido_log_debug("%s: in->len=%zu, origsiz=%zu", __func__,
57*ab19a69eSdjm in->len, origsiz);
58*ab19a69eSdjm return FIDO_ERR_INVALID_ARGUMENT;
59*ab19a69eSdjm }
60*ab19a69eSdjm if ((z = inflateInit2(&zs, -MAX_WBITS)) != Z_OK) {
61*ab19a69eSdjm fido_log_debug("%s: inflateInit2: %d", __func__, z);
62*ab19a69eSdjm return FIDO_ERR_COMPRESS;
63*ab19a69eSdjm }
64*ab19a69eSdjm
65*ab19a69eSdjm if ((out->ptr = calloc(1, olen)) == NULL) {
66*ab19a69eSdjm r = FIDO_ERR_INTERNAL;
67*ab19a69eSdjm goto fail;
68*ab19a69eSdjm }
69*ab19a69eSdjm out->len = olen;
70*ab19a69eSdjm zs.next_in = in->ptr;
71*ab19a69eSdjm zs.avail_in = ilen;
72*ab19a69eSdjm zs.next_out = out->ptr;
73*ab19a69eSdjm zs.avail_out = olen;
74*ab19a69eSdjm
75*ab19a69eSdjm if ((z = inflate(&zs, Z_FINISH)) != Z_STREAM_END) {
76*ab19a69eSdjm fido_log_debug("%s: inflate: %d", __func__, z);
77*ab19a69eSdjm r = FIDO_ERR_COMPRESS;
78*ab19a69eSdjm goto fail;
79*ab19a69eSdjm }
80*ab19a69eSdjm if (zs.avail_out != 0) {
81*ab19a69eSdjm fido_log_debug("%s: %u != 0", __func__, zs.avail_out);
82*ab19a69eSdjm r = FIDO_ERR_COMPRESS;
83*ab19a69eSdjm goto fail;
84*ab19a69eSdjm }
85*ab19a69eSdjm
86*ab19a69eSdjm r = FIDO_OK;
87*ab19a69eSdjm fail:
88*ab19a69eSdjm if ((z = inflateEnd(&zs)) != Z_OK) {
89*ab19a69eSdjm fido_log_debug("%s: inflateEnd: %d", __func__, z);
90*ab19a69eSdjm r = FIDO_ERR_COMPRESS;
91*ab19a69eSdjm }
92*ab19a69eSdjm if (r != FIDO_OK)
93*ab19a69eSdjm fido_blob_reset(out);
94*ab19a69eSdjm
95*ab19a69eSdjm return r;
96*ab19a69eSdjm }
97*ab19a69eSdjm
98*ab19a69eSdjm /* raw deflate */
99*ab19a69eSdjm static int
rfc1951_deflate(fido_blob_t * out,const fido_blob_t * in)100*ab19a69eSdjm rfc1951_deflate(fido_blob_t *out, const fido_blob_t *in)
101*ab19a69eSdjm {
102*ab19a69eSdjm z_stream zs;
103*ab19a69eSdjm u_int ilen, olen;
104*ab19a69eSdjm int r, z;
105*ab19a69eSdjm
106*ab19a69eSdjm memset(&zs, 0, sizeof(zs));
107*ab19a69eSdjm memset(out, 0, sizeof(*out));
108*ab19a69eSdjm
109*ab19a69eSdjm if (in->len > UINT_MAX || (ilen = (u_int)in->len) > BOUND) {
110*ab19a69eSdjm fido_log_debug("%s: in->len=%zu", __func__, in->len);
111*ab19a69eSdjm return FIDO_ERR_INVALID_ARGUMENT;
112*ab19a69eSdjm }
113*ab19a69eSdjm if ((z = deflateInit2(&zs, Z_DEFAULT_COMPRESSION, Z_DEFLATED,
114*ab19a69eSdjm -MAX_WBITS, 8, Z_DEFAULT_STRATEGY)) != Z_OK) {
115*ab19a69eSdjm fido_log_debug("%s: deflateInit2: %d", __func__, z);
116*ab19a69eSdjm return FIDO_ERR_COMPRESS;
117*ab19a69eSdjm }
118*ab19a69eSdjm
119*ab19a69eSdjm olen = BOUND;
120*ab19a69eSdjm if ((out->ptr = calloc(1, olen)) == NULL) {
121*ab19a69eSdjm r = FIDO_ERR_INTERNAL;
122*ab19a69eSdjm goto fail;
123*ab19a69eSdjm }
124*ab19a69eSdjm out->len = olen;
125*ab19a69eSdjm zs.next_in = in->ptr;
126*ab19a69eSdjm zs.avail_in = ilen;
127*ab19a69eSdjm zs.next_out = out->ptr;
128*ab19a69eSdjm zs.avail_out = olen;
129*ab19a69eSdjm
130*ab19a69eSdjm if ((z = deflate(&zs, Z_FINISH)) != Z_STREAM_END) {
131*ab19a69eSdjm fido_log_debug("%s: inflate: %d", __func__, z);
132*ab19a69eSdjm r = FIDO_ERR_COMPRESS;
133*ab19a69eSdjm goto fail;
134*ab19a69eSdjm }
135*ab19a69eSdjm if (zs.avail_out >= out->len) {
136*ab19a69eSdjm fido_log_debug("%s: %u > %zu", __func__, zs.avail_out,
137*ab19a69eSdjm out->len);
138*ab19a69eSdjm r = FIDO_ERR_COMPRESS;
139*ab19a69eSdjm goto fail;
140*ab19a69eSdjm }
141*ab19a69eSdjm out->len -= zs.avail_out;
142*ab19a69eSdjm
143*ab19a69eSdjm r = FIDO_OK;
144*ab19a69eSdjm fail:
145*ab19a69eSdjm if ((z = deflateEnd(&zs)) != Z_OK) {
146*ab19a69eSdjm fido_log_debug("%s: deflateEnd: %d", __func__, z);
147*ab19a69eSdjm r = FIDO_ERR_COMPRESS;
148*ab19a69eSdjm }
149*ab19a69eSdjm if (r != FIDO_OK)
150*ab19a69eSdjm fido_blob_reset(out);
151*ab19a69eSdjm
152*ab19a69eSdjm return r;
153*ab19a69eSdjm }
154*ab19a69eSdjm
155c4a807edSdjm int
fido_compress(fido_blob_t * out,const fido_blob_t * in)156c4a807edSdjm fido_compress(fido_blob_t *out, const fido_blob_t *in)
157c4a807edSdjm {
158*ab19a69eSdjm return rfc1951_deflate(out, in);
159c4a807edSdjm }
160c4a807edSdjm
161c4a807edSdjm int
fido_uncompress(fido_blob_t * out,const fido_blob_t * in,size_t origsiz)162c4a807edSdjm fido_uncompress(fido_blob_t *out, const fido_blob_t *in, size_t origsiz)
163c4a807edSdjm {
164*ab19a69eSdjm if (rfc1950_inflate(out, in, origsiz) == FIDO_OK)
165*ab19a69eSdjm return FIDO_OK; /* backwards compat with libfido2 < 1.11 */
166*ab19a69eSdjm return rfc1951_inflate(out, in, origsiz);
167c4a807edSdjm }
168