xref: /netbsd-src/external/ibm-public/postfix/dist/src/tls/tls_fprint.c (revision c48c605c14fd8622b523d1d6a3f0c0bad133ea89)
1 /*	$NetBSD: tls_fprint.c,v 1.4 2023/12/23 20:30:45 christos Exp $	*/
2 
3 /*++
4 /* NAME
5 /*	tls_fprint 3
6 /* SUMMARY
7 /*	Digests fingerprints and all that.
8 /* SYNOPSIS
9 /*	#include <tls.h>
10 /*
11 /*	EVP_MD *tls_digest_byname(const char *mdalg, EVP_MD_CTX **mdctxPtr)
12 /*	const char *mdalg;
13 /*	EVP_MD_CTX **mdctxPtr;
14 /*
15 /*	char	*tls_serverid_digest(TLScontext, props, ciphers)
16 /*	TLS_SESS_STATE *TLScontext;
17 /*	const TLS_CLIENT_START_PROPS *props;
18 /*	const char *ciphers;
19 /*
20 /*	char	*tls_digest_encode(md_buf, md_len)
21 /*	const unsigned char *md_buf;
22 /*	const char *md_len;
23 /*
24 /*	char	*tls_cert_fprint(peercert, mdalg)
25 /*	X509	*peercert;
26 /*	const char *mdalg;
27 /*
28 /*	char	*tls_pkey_fprint(peercert, mdalg)
29 /*	X509	*peercert;
30 /*	const char *mdalg;
31 /* DESCRIPTION
32 /*	tls_digest_byname() constructs, and optionally returns, an EVP_MD_CTX
33 /*	handle for performing digest operations with the algorithm named by the
34 /*	mdalg parameter.  The return value is non-null on success, and holds a
35 /*	digest algorithm handle.  If the mdctxPtr argument is non-null the
36 /*	created context is returned to the caller, who is then responsible for
37 /*	deleting it by calling EVP_MD_ctx_free() once it is no longer needed.
38 /*
39 /*	tls_digest_encode() converts a binary message digest to a hex ASCII
40 /*	format with ':' separators between each pair of hex digits.
41 /*	The return value is dynamically allocated with mymalloc(),
42 /*	and the caller must eventually free it with myfree().
43 /*
44 /*	tls_cert_fprint() returns a fingerprint of the given
45 /*	certificate using the requested message digest, formatted
46 /*	with tls_digest_encode(). Panics if the
47 /*	(previously verified) digest algorithm is not found. The return
48 /*	value is dynamically allocated with mymalloc(), and the caller
49 /*	must eventually free it with myfree().
50 /*
51 /*	tls_pkey_fprint() returns a public-key fingerprint; in all
52 /*	other respects the function behaves as tls_cert_fprint().
53 /*	The var_tls_bc_pkey_fprint variable enables an incorrect
54 /*	algorithm that was used in Postfix versions 2.9.[0-5].
55 /*	The return value is dynamically allocated with mymalloc(),
56 /*	and the caller must eventually free it with myfree().
57 /*
58 /*	tls_serverid_digest() suffixes props->serverid computed by the SMTP
59 /*	client with "&" plus a digest of additional parameters needed to ensure
60 /*	that re-used sessions are more likely to be reused and that they will
61 /*	satisfy all protocol and security requirements.  The return value is
62 /*	dynamically allocated with mymalloc(), and the caller must eventually
63 /*	free it with myfree().
64 /*
65 /*	Arguments:
66 /* .IP mdalg
67 /*	A digest algorithm name, such as "sha256".
68 /* .IP peercert
69 /*	Server or client X.509 certificate.
70 /* .IP md_buf
71 /*	The raw binary digest.
72 /* .IP md_len
73 /*	The digest length in bytes.
74 /* .IP mdalg
75 /*	Name of a message digest algorithm suitable for computing secure
76 /*	(1st pre-image resistant) message digests of certificates. For now,
77 /*	md5, sha1, or member of SHA-2 family if supported by OpenSSL.
78 /* .IP mdctxPtr
79 /*	Pointer to an (EVP_MD_CTX *) handle, or NULL if only probing for
80 /*	algorithm support without immediate use in mind.
81 /* .IP buf
82 /*	Input data for the message digest algorithm mdalg.
83 /* .IP len
84 /*	The length of the input data.
85 /* .IP props
86 /*	The client start properties for the session, which contains the
87 /*	initial serverid from the SMTP client and the DANE verification
88 /*	parameters.
89 /* .IP protomask
90 /*	The mask of protocol exclusions.
91 /* .IP ciphers
92 /*	The SSL client cipherlist.
93 /* LICENSE
94 /* .ad
95 /* .fi
96 /*	This software is free. You can do with it whatever you want.
97 /*	The original author kindly requests that you acknowledge
98 /*	the use of his software.
99 /* AUTHOR(S)
100 /*	Wietse Venema
101 /*	IBM T.J. Watson Research
102 /*	P.O. Box 704
103 /*	Yorktown Heights, NY 10598, USA
104 /*
105 /*	Viktor Dukhovni
106 /*--*/
107 
108 /* System library. */
109 
110 #include <sys_defs.h>
111 #include <ctype.h>
112 
113 #ifdef USE_TLS
114 #include <string.h>
115 
116 /* Utility library. */
117 
118 #include <msg.h>
119 #include <mymalloc.h>
120 #include <stringops.h>
121 
122 /* Global library. */
123 
124 #include <mail_params.h>
125 
126 /* TLS library. */
127 
128 #define TLS_INTERNAL
129 #include <tls.h>
130 
131 /* Application-specific. */
132 
133 static const char hexcodes[] = "0123456789ABCDEF";
134 
135 #define CHECK_OK_AND(stillok) (ok = ok && (stillok))
136 #define CHECK_OK_AND_DIGEST_OBJECT(m, p) \
137 	CHECK_OK_AND_DIGEST_DATA((m), (unsigned char *)(p), sizeof(*(p)))
138 #define CHECK_OK_AND_DIGEST_DATA(m, p, l) CHECK_OK_AND(digest_bytes((m), (p), (l)))
139 #define CHECK_OK_AND_DIGEST_CHARS(m, s) CHECK_OK_AND(digest_chars((m), (s)))
140 
141 /* digest_bytes - hash octet string of given length */
142 
digest_bytes(EVP_MD_CTX * ctx,const unsigned char * buf,size_t len)143 static int digest_bytes(EVP_MD_CTX *ctx, const unsigned char *buf, size_t len)
144 {
145     return (EVP_DigestUpdate(ctx, buf, len));
146 }
147 
148 /* digest_chars - hash string including trailing NUL */
149 
digest_chars(EVP_MD_CTX * ctx,const char * s)150 static int digest_chars(EVP_MD_CTX *ctx, const char *s)
151 {
152     return (EVP_DigestUpdate(ctx, s, strlen(s) + 1));
153 }
154 
155 /* tlsa_cmp - compare TLSA RRs for sorting to canonical order */
156 
tlsa_cmp(const void * a,const void * b)157 static int tlsa_cmp(const void *a, const void *b)
158 {
159     TLS_TLSA *p = *(TLS_TLSA **) a;
160     TLS_TLSA *q = *(TLS_TLSA **) b;
161     int     d;
162 
163     if ((d = (int) p->usage - (int) q->usage) != 0)
164 	return d;
165     if ((d = (int) p->selector - (int) q->selector) != 0)
166 	return d;
167     if ((d = (int) p->mtype - (int) q->mtype) != 0)
168 	return d;
169     if ((d = (int) p->length - (int) q->length) != 0)
170 	return d;
171     return (memcmp(p->data, q->data, p->length));
172 }
173 
174 /* tls_digest_tlsa - fold in digest of TLSA records */
175 
tls_digest_tlsa(EVP_MD_CTX * mdctx,TLS_TLSA * tlsa)176 static int tls_digest_tlsa(EVP_MD_CTX *mdctx, TLS_TLSA *tlsa)
177 {
178     TLS_TLSA *p;
179     TLS_TLSA **arr;
180     int     ok = 1;
181     int     n;
182     int     i;
183 
184     for (n = 0, p = tlsa; p != 0; p = p->next)
185 	++n;
186     arr = (TLS_TLSA **) mymalloc(n * sizeof(*arr));
187     for (i = 0, p = tlsa; p; p = p->next)
188 	arr[i++] = (void *) p;
189     qsort(arr, n, sizeof(arr[0]), tlsa_cmp);
190 
191     CHECK_OK_AND_DIGEST_OBJECT(mdctx, &n);
192     for (i = 0; i < n; ++i) {
193 	CHECK_OK_AND_DIGEST_OBJECT(mdctx, &arr[i]->usage);
194 	CHECK_OK_AND_DIGEST_OBJECT(mdctx, &arr[i]->selector);
195 	CHECK_OK_AND_DIGEST_OBJECT(mdctx, &arr[i]->mtype);
196 	CHECK_OK_AND_DIGEST_OBJECT(mdctx, &arr[i]->length);
197 	CHECK_OK_AND_DIGEST_DATA(mdctx, arr[i]->data, arr[i]->length);
198     }
199     myfree((void *) arr);
200     return (ok);
201 }
202 
203 /* tls_digest_byname - test availability or prepare to use digest */
204 
tls_digest_byname(const char * mdalg,EVP_MD_CTX ** mdctxPtr)205 const EVP_MD *tls_digest_byname(const char *mdalg, EVP_MD_CTX **mdctxPtr)
206 {
207     const EVP_MD *md;
208     EVP_MD_CTX *mdctx = NULL;
209     int     ok = 1;
210 
211     /*
212      * In OpenSSL 3.0, because of dynamically variable algorithm providers,
213      * there is a time-of-check/time-of-use issue that means that abstract
214      * algorithm handles returned by EVP_get_digestbyname() can (and not
215      * infrequently do) return ultimately unusable algorithms, to check for
216      * actual availability, one needs to use the new EVP_MD_fetch() API, or
217      * indirectly check usability by creating a concrete context. We take the
218      * latter approach here (works for 1.1.1 without #ifdef).
219      *
220      * Note that EVP_MD_CTX_{create,destroy} were renamed to, respectively,
221      * EVP_MD_CTX_{new,free} in OpenSSL 1.1.0.
222      */
223     CHECK_OK_AND(md = EVP_get_digestbyname(mdalg));
224 
225     /*
226      * Sanity check: Newer shared libraries could (hypothetical ABI break)
227      * allow larger digests, we avoid such poison algorithms.
228      */
229     CHECK_OK_AND(EVP_MD_size(md) <= EVP_MAX_MD_SIZE);
230     CHECK_OK_AND(mdctx = EVP_MD_CTX_new());
231     CHECK_OK_AND(EVP_DigestInit_ex(mdctx, md, NULL));
232 
233 
234     if (ok && mdctxPtr != 0)
235 	*mdctxPtr = mdctx;
236     else
237 	EVP_MD_CTX_free(mdctx);
238     return (ok ? md : 0);
239 }
240 
241 /* tls_serverid_digest - suffix props->serverid with parameter digest */
242 
tls_serverid_digest(TLS_SESS_STATE * TLScontext,const TLS_CLIENT_START_PROPS * props,const char * ciphers)243 char   *tls_serverid_digest(TLS_SESS_STATE *TLScontext,
244 			            const TLS_CLIENT_START_PROPS *props,
245 			            const char *ciphers)
246 {
247     EVP_MD_CTX *mdctx;
248     const char *mdalg;
249     unsigned char md_buf[EVP_MAX_MD_SIZE];
250     unsigned int md_len;
251     int     ok = 1;
252     int     i;
253     long    sslversion;
254     VSTRING *result;
255 
256     /*
257      * Try to use sha256: our serverid choice should be strong enough to
258      * resist 2nd-preimage attacks with a difficulty comparable to that of
259      * DANE TLSA digests.  Failing that, we compute serverid digests with the
260      * default digest, but DANE requires sha256 and sha512, so if we must
261      * fall back to our default digest, DANE support won't be available.  We
262      * panic if the fallback algorithm is not available, as it was verified
263      * available in tls_client_init() and must not simply vanish.  Our
264      * provider set is not expected to change once the OpenSSL library is
265      * initialized.
266      */
267     if (tls_digest_byname(mdalg = LN_sha256, &mdctx) == 0
268 	&& tls_digest_byname(mdalg = props->mdalg, &mdctx) == 0)
269 	msg_panic("digest algorithm \"%s\" not found", props->mdalg);
270 
271     /* Salt the session lookup key with the OpenSSL runtime version. */
272     sslversion = OpenSSL_version_num();
273 
274     CHECK_OK_AND_DIGEST_CHARS(mdctx, props->helo ? props->helo : "");
275     CHECK_OK_AND_DIGEST_OBJECT(mdctx, &sslversion);
276     CHECK_OK_AND_DIGEST_CHARS(mdctx, props->protocols);
277     CHECK_OK_AND_DIGEST_CHARS(mdctx, ciphers);
278 
279     /*
280      * Ensure separation of caches for sessions where DANE trust
281      * configuration succeeded from those where it did not.  The latter
282      * should always see a certificate validation failure, both on initial
283      * handshake and on resumption.
284      */
285     CHECK_OK_AND_DIGEST_OBJECT(mdctx, &TLScontext->must_fail);
286 
287     /*
288      * DNS-based or synthetic DANE trust settings are potentially used at all
289      * levels above "encrypt".
290      */
291     if (TLScontext->level > TLS_LEV_ENCRYPT
292 	&& props->dane && props->dane->tlsa) {
293 	CHECK_OK_AND(tls_digest_tlsa(mdctx, props->dane->tlsa));
294     } else {
295 	int     none = 0;		/* Record a TLSA RR count of zero */
296 
297 	CHECK_OK_AND_DIGEST_OBJECT(mdctx, &none);
298     }
299 
300     /*
301      * Include the chosen SNI name, which can affect server certificate
302      * selection.
303      */
304     if (TLScontext->level > TLS_LEV_ENCRYPT && TLScontext->peer_sni)
305 	CHECK_OK_AND_DIGEST_CHARS(mdctx, TLScontext->peer_sni);
306     else
307 	CHECK_OK_AND_DIGEST_CHARS(mdctx, "");
308 
309     CHECK_OK_AND(EVP_DigestFinal_ex(mdctx, md_buf, &md_len));
310     EVP_MD_CTX_destroy(mdctx);
311     if (!ok)
312 	msg_fatal("error computing %s message digest", mdalg);
313 
314     /* Check for OpenSSL contract violation */
315     if (md_len > EVP_MAX_MD_SIZE)
316 	msg_panic("unexpectedly large %s digest size: %u", mdalg, md_len);
317 
318     /*
319      * Append the digest to the serverid.  We don't compare this digest to
320      * any user-specified fingerprints.  Therefore, we don't need to use a
321      * colon-separated format, which saves space in the TLS session cache and
322      * makes logging of session cache lookup keys more readable.
323      *
324      * This does however duplicate a few lines of code from the digest encoder
325      * for colon-separated cert and pkey fingerprints. If that is a
326      * compelling reason to consolidate, we could use that and append the
327      * result.
328      */
329     result = vstring_alloc(strlen(props->serverid) + 1 + 2 * md_len);
330     vstring_strcpy(result, props->serverid);
331     VSTRING_ADDCH(result, '&');
332     for (i = 0; i < md_len; i++) {
333 	VSTRING_ADDCH(result, hexcodes[(md_buf[i] & 0xf0) >> 4U]);
334 	VSTRING_ADDCH(result, hexcodes[(md_buf[i] & 0x0f)]);
335     }
336     VSTRING_TERMINATE(result);
337     return (vstring_export(result));
338 }
339 
340 /* tls_digest_encode - encode message digest binary blob as xx:xx:... */
341 
tls_digest_encode(const unsigned char * md_buf,int md_len)342 char   *tls_digest_encode(const unsigned char *md_buf, int md_len)
343 {
344     int     i;
345     char   *result = mymalloc(md_len * 3);
346 
347     /* Check for contract violation */
348     if (md_len > EVP_MAX_MD_SIZE || md_len >= INT_MAX / 3)
349 	msg_panic("unexpectedly large message digest size: %u", md_len);
350 
351     /* No risk of overruns, len is bounded by OpenSSL digest length */
352     for (i = 0; i < md_len; i++) {
353 	result[i * 3] = hexcodes[(md_buf[i] & 0xf0) >> 4U];
354 	result[(i * 3) + 1] = hexcodes[(md_buf[i] & 0x0f)];
355 	result[(i * 3) + 2] = (i + 1 != md_len) ? ':' : '\0';
356     }
357     return (result);
358 }
359 
360 /* tls_data_fprint - compute and encode digest of binary object */
361 
tls_data_fprint(const unsigned char * buf,int len,const char * mdalg)362 static char *tls_data_fprint(const unsigned char *buf, int len, const char *mdalg)
363 {
364     EVP_MD_CTX *mdctx = NULL;
365     unsigned char md_buf[EVP_MAX_MD_SIZE];
366     unsigned int md_len;
367     int     ok = 1;
368 
369     /* Previously available in "init" routine. */
370     if (tls_digest_byname(mdalg, &mdctx) == 0)
371 	msg_panic("digest algorithm \"%s\" not found", mdalg);
372 
373     CHECK_OK_AND_DIGEST_DATA(mdctx, buf, len);
374     CHECK_OK_AND(EVP_DigestFinal_ex(mdctx, md_buf, &md_len));
375     EVP_MD_CTX_destroy(mdctx);
376     if (!ok)
377 	msg_fatal("error computing %s message digest", mdalg);
378 
379     return (tls_digest_encode(md_buf, md_len));
380 }
381 
382 /* tls_cert_fprint - extract certificate fingerprint */
383 
tls_cert_fprint(X509 * peercert,const char * mdalg)384 char   *tls_cert_fprint(X509 *peercert, const char *mdalg)
385 {
386     int     len;
387     unsigned char *buf;
388     unsigned char *buf2;
389     char   *result;
390 
391     len = i2d_X509(peercert, NULL);
392     buf2 = buf = mymalloc(len);
393     i2d_X509(peercert, &buf2);
394     if (buf2 - buf != len)
395 	msg_panic("i2d_X509 invalid result length");
396 
397     result = tls_data_fprint(buf, len, mdalg);
398     myfree(buf);
399 
400     return (result);
401 }
402 
403 /* tls_pkey_fprint - extract public key fingerprint from certificate */
404 
tls_pkey_fprint(X509 * peercert,const char * mdalg)405 char   *tls_pkey_fprint(X509 *peercert, const char *mdalg)
406 {
407     if (var_tls_bc_pkey_fprint) {
408 	const char *myname = "tls_pkey_fprint";
409 	ASN1_BIT_STRING *key;
410 	char   *result;
411 
412 	key = X509_get0_pubkey_bitstr(peercert);
413 	if (key == 0)
414 	    msg_fatal("%s: error extracting legacy public-key fingerprint: %m",
415 		      myname);
416 
417 	result = tls_data_fprint(key->data, key->length, mdalg);
418 	return (result);
419     } else {
420 	int     len;
421 	unsigned char *buf;
422 	unsigned char *buf2;
423 	char   *result;
424 
425 	len = i2d_X509_PUBKEY(X509_get_X509_PUBKEY(peercert), NULL);
426 	buf2 = buf = mymalloc(len);
427 	i2d_X509_PUBKEY(X509_get_X509_PUBKEY(peercert), &buf2);
428 	if (buf2 - buf != len)
429 	    msg_panic("i2d_X509_PUBKEY invalid result length");
430 
431 	result = tls_data_fprint(buf, len, mdalg);
432 	myfree(buf);
433 	return (result);
434     }
435 }
436 
437 #endif
438