xref: /netbsd-src/external/ibm-public/postfix/dist/src/tls/tls_fprint.c (revision d16b7486a53dcb8072b60ec6fcb4373a2d0c27b7)
1 /*	$NetBSD: tls_fprint.c,v 1.3 2022/10/08 16:12:50 christos Exp $	*/
2 
3 /*++
4 /* NAME
5 /*	tls_fprint 3
6 /* SUMMARY
7 /*	Digests fingerprints and all that.
8 /* SYNOPSIS
9 /*	#include <tls.h>
10 /*
11 /*	char	*tls_serverid_digest(TLScontext, props, ciphers)
12 /*	TLS_SESS_STATE *TLScontext;
13 /*	const TLS_CLIENT_START_PROPS *props;
14 /*	const char *ciphers;
15 /*
16 /*	char	*tls_digest_encode(md_buf, md_len)
17 /*	const unsigned char *md_buf;
18 /*	const char *md_len;
19 /*
20 /*	char	*tls_cert_fprint(peercert, mdalg)
21 /*	X509	*peercert;
22 /*	const char *mdalg;
23 /*
24 /*	char	*tls_pkey_fprint(peercert, mdalg)
25 /*	X509	*peercert;
26 /*	const char *mdalg;
27 /* DESCRIPTION
28 /*	tls_digest_encode() converts a binary message digest to a hex ASCII
29 /*	format with ':' separators between each pair of hex digits.
30 /*	The return value is dynamically allocated with mymalloc(),
31 /*	and the caller must eventually free it with myfree().
32 /*
33 /*	tls_cert_fprint() returns a fingerprint of the given
34 /*	certificate using the requested message digest, formatted
35 /*	with tls_digest_encode(). Panics if the
36 /*	(previously verified) digest algorithm is not found. The return
37 /*	value is dynamically allocated with mymalloc(), and the caller
38 /*	must eventually free it with myfree().
39 /*
40 /*	tls_pkey_fprint() returns a public-key fingerprint; in all
41 /*	other respects the function behaves as tls_cert_fprint().
42 /*	The var_tls_bc_pkey_fprint variable enables an incorrect
43 /*	algorithm that was used in Postfix versions 2.9.[0-5].
44 /*	The return value is dynamically allocated with mymalloc(),
45 /*	and the caller must eventually free it with myfree().
46 /*
47 /*	tls_serverid_digest() suffixes props->serverid computed by the SMTP
48 /*	client with "&" plus a digest of additional parameters needed to ensure
49 /*	that re-used sessions are more likely to be reused and that they will
50 /*	satisfy all protocol and security requirements.  The return value is
51 /*	dynamically allocated with mymalloc(), and the caller must eventually
52 /*	free it with myfree().
53 /*
54 /*	Arguments:
55 /* .IP peercert
56 /*	Server or client X.509 certificate.
57 /* .IP md_buf
58 /*	The raw binary digest.
59 /* .IP md_len
60 /*	The digest length in bytes.
61 /* .IP mdalg
62 /*	Name of a message digest algorithm suitable for computing secure
63 /*	(1st pre-image resistant) message digests of certificates. For now,
64 /*	md5, sha1, or member of SHA-2 family if supported by OpenSSL.
65 /* .IP buf
66 /*	Input data for the message digest algorithm mdalg.
67 /* .IP len
68 /*	The length of the input data.
69 /* .IP props
70 /*	The client start properties for the session, which contains the
71 /*	initial serverid from the SMTP client and the DANE verification
72 /*	parameters.
73 /* .IP protomask
74 /*	The mask of protocol exclusions.
75 /* .IP ciphers
76 /*	The SSL client cipherlist.
77 /* LICENSE
78 /* .ad
79 /* .fi
80 /*	This software is free. You can do with it whatever you want.
81 /*	The original author kindly requests that you acknowledge
82 /*	the use of his software.
83 /* AUTHOR(S)
84 /*	Wietse Venema
85 /*	IBM T.J. Watson Research
86 /*	P.O. Box 704
87 /*	Yorktown Heights, NY 10598, USA
88 /*
89 /*	Viktor Dukhovni
90 /*--*/
91 
92 /* System library. */
93 
94 #include <sys_defs.h>
95 #include <ctype.h>
96 
97 #ifdef USE_TLS
98 #include <string.h>
99 
100 /* Utility library. */
101 
102 #include <msg.h>
103 #include <mymalloc.h>
104 #include <stringops.h>
105 
106 /* Global library. */
107 
108 #include <mail_params.h>
109 
110 /* TLS library. */
111 
112 #define TLS_INTERNAL
113 #include <tls.h>
114 
115 /* Application-specific. */
116 
117 static const char hexcodes[] = "0123456789ABCDEF";
118 
119 #define checkok(ret)	(ok &= ((ret) ? 1 : 0))
120 #define digest_object(p) digest_data((unsigned char *)(p), sizeof(*(p)))
121 #define digest_data(p, l) checkok(digest_bytes(mdctx, (p), (l)))
122 #define digest_string(s) checkok(digest_chars(mdctx, (s)))
123 #define digest_dane(tlsa) checkok(tls_digest_tlsa(mdctx, tlsa))
124 
125 /* digest_bytes - hash octet string of given length */
126 
127 static int digest_bytes(EVP_MD_CTX *ctx, const unsigned char *buf, size_t len)
128 {
129     return (EVP_DigestUpdate(ctx, buf, len));
130 }
131 
132 /* digest_chars - hash string including trailing NUL */
133 
134 static int digest_chars(EVP_MD_CTX *ctx, const char *s)
135 {
136     return (EVP_DigestUpdate(ctx, s, strlen(s) + 1));
137 }
138 
139 /* tlsa_cmp - compare TLSA RRs for sorting to canonical order */
140 
141 static int tlsa_cmp(const void *a, const void *b)
142 {
143     TLS_TLSA *p = *(TLS_TLSA **) a;
144     TLS_TLSA *q = *(TLS_TLSA **) b;
145     int     d;
146 
147     if ((d = (int) p->usage - (int) q->usage) != 0)
148 	return d;
149     if ((d = (int) p->selector - (int) q->selector) != 0)
150 	return d;
151     if ((d = (int) p->mtype - (int) q->mtype) != 0)
152 	return d;
153     if ((d = (int) p->length - (int) q->length) != 0)
154 	return d;
155     return (memcmp(p->data, q->data, p->length));
156 }
157 
158 /* tls_digest_tlsa - fold in digest of TLSA records */
159 
160 static int tls_digest_tlsa(EVP_MD_CTX *mdctx, TLS_TLSA *tlsa)
161 {
162     TLS_TLSA *p;
163     TLS_TLSA **arr;
164     int     ok = 1;
165     int     n;
166     int     i;
167 
168     for (n = 0, p = tlsa; p != 0; p = p->next)
169 	++n;
170     arr = (TLS_TLSA **) mymalloc(n * sizeof(*arr));
171     for (i = 0, p = tlsa; p; p = p->next)
172 	arr[i++] = (void *) p;
173     qsort(arr, n, sizeof(arr[0]), tlsa_cmp);
174 
175     digest_object(&n);
176     for (i = 0; i < n; ++i) {
177 	digest_object(&arr[i]->usage);
178 	digest_object(&arr[i]->selector);
179 	digest_object(&arr[i]->mtype);
180 	digest_object(&arr[i]->length);
181 	digest_data(arr[i]->data, arr[i]->length);
182     }
183     myfree((void *) arr);
184     return (ok);
185 }
186 
187 /* tls_serverid_digest - suffix props->serverid with parameter digest */
188 
189 char   *tls_serverid_digest(TLS_SESS_STATE *TLScontext,
190 			            const TLS_CLIENT_START_PROPS *props,
191 			            const char *ciphers)
192 {
193     EVP_MD_CTX *mdctx;
194     const EVP_MD *md;
195     const char *mdalg;
196     unsigned char md_buf[EVP_MAX_MD_SIZE];
197     unsigned int md_len;
198     int     ok = 1;
199     int     i;
200     long    sslversion;
201     VSTRING *result;
202 
203     /*
204      * Try to use sha256: our serverid choice should be strong enough to
205      * resist 2nd-preimage attacks with a difficulty comparable to that of
206      * DANE TLSA digests.  Failing that, we compute serverid digests with the
207      * default digest, but DANE requires sha256 and sha512, so if we must
208      * fall back to our default digest, DANE support won't be available.  We
209      * panic if the fallback algorithm is not available, as it was verified
210      * available in tls_client_init() and must not simply vanish.
211      */
212     if ((md = EVP_get_digestbyname(mdalg = "sha256")) == 0
213 	&& (md = EVP_get_digestbyname(mdalg = props->mdalg)) == 0)
214 	msg_panic("digest algorithm \"%s\" not found", mdalg);
215 
216     /* Salt the session lookup key with the OpenSSL runtime version. */
217     sslversion = OpenSSL_version_num();
218 
219     mdctx = EVP_MD_CTX_create();
220     checkok(EVP_DigestInit_ex(mdctx, md, NULL));
221     digest_string(props->helo ? props->helo : "");
222     digest_object(&sslversion);
223     digest_string(props->protocols);
224     digest_string(ciphers);
225 
226     /*
227      * Ensure separation of caches for sessions where DANE trust
228      * configuration succeeded from those where it did not.  The latter
229      * should always see a certificate validation failure, both on initial
230      * handshake and on resumption.
231      */
232     digest_object(&TLScontext->must_fail);
233 
234     /*
235      * DNS-based or synthetic DANE trust settings are potentially used at all
236      * levels above "encrypt".
237      */
238     if (TLScontext->level > TLS_LEV_ENCRYPT
239 	&& props->dane && props->dane->tlsa) {
240 	digest_dane(props->dane->tlsa);
241     } else {
242 	int     none = 0;		/* Record a TLSA RR count of zero */
243 
244 	digest_object(&none);
245     }
246 
247     /*
248      * Include the chosen SNI name, which can affect server certificate
249      * selection.
250      */
251     if (TLScontext->level > TLS_LEV_ENCRYPT && TLScontext->peer_sni)
252 	digest_string(TLScontext->peer_sni);
253     else
254 	digest_string("");
255 
256     checkok(EVP_DigestFinal_ex(mdctx, md_buf, &md_len));
257     EVP_MD_CTX_destroy(mdctx);
258     if (!ok)
259 	msg_fatal("error computing %s message digest", mdalg);
260 
261     /* Check for OpenSSL contract violation */
262     if (md_len > EVP_MAX_MD_SIZE)
263 	msg_panic("unexpectedly large %s digest size: %u", mdalg, md_len);
264 
265     /*
266      * Append the digest to the serverid.  We don't compare this digest to
267      * any user-specified fingerprints.  Therefore, we don't need to use a
268      * colon-separated format, which saves space in the TLS session cache and
269      * makes logging of session cache lookup keys more readable.
270      *
271      * This does however duplicate a few lines of code from the digest encoder
272      * for colon-separated cert and pkey fingerprints. If that is a
273      * compelling reason to consolidate, we could use that and append the
274      * result.
275      */
276     result = vstring_alloc(strlen(props->serverid) + 1 + 2 * md_len);
277     vstring_strcpy(result, props->serverid);
278     VSTRING_ADDCH(result, '&');
279     for (i = 0; i < md_len; i++) {
280 	VSTRING_ADDCH(result, hexcodes[(md_buf[i] & 0xf0) >> 4U]);
281 	VSTRING_ADDCH(result, hexcodes[(md_buf[i] & 0x0f)]);
282     }
283     VSTRING_TERMINATE(result);
284     return (vstring_export(result));
285 }
286 
287 /* tls_digest_encode - encode message digest binary blob as xx:xx:... */
288 
289 char   *tls_digest_encode(const unsigned char *md_buf, int md_len)
290 {
291     int     i;
292     char   *result = mymalloc(md_len * 3);
293 
294     /* Check for contract violation */
295     if (md_len > EVP_MAX_MD_SIZE || md_len >= INT_MAX / 3)
296 	msg_panic("unexpectedly large message digest size: %u", md_len);
297 
298     /* No risk of overruns, len is bounded by OpenSSL digest length */
299     for (i = 0; i < md_len; i++) {
300 	result[i * 3] = hexcodes[(md_buf[i] & 0xf0) >> 4U];
301 	result[(i * 3) + 1] = hexcodes[(md_buf[i] & 0x0f)];
302 	result[(i * 3) + 2] = (i + 1 != md_len) ? ':' : '\0';
303     }
304     return (result);
305 }
306 
307 /* tls_data_fprint - compute and encode digest of binary object */
308 
309 static char *tls_data_fprint(const unsigned char *buf, int len, const char *mdalg)
310 {
311     EVP_MD_CTX *mdctx;
312     const EVP_MD *md;
313     unsigned char md_buf[EVP_MAX_MD_SIZE];
314     unsigned int md_len;
315     int     ok = 1;
316 
317     /* Previously available in "init" routine. */
318     if ((md = EVP_get_digestbyname(mdalg)) == 0)
319 	msg_panic("digest algorithm \"%s\" not found", mdalg);
320 
321     mdctx = EVP_MD_CTX_create();
322     checkok(EVP_DigestInit_ex(mdctx, md, NULL));
323     digest_data(buf, len);
324     checkok(EVP_DigestFinal_ex(mdctx, md_buf, &md_len));
325     EVP_MD_CTX_destroy(mdctx);
326     if (!ok)
327 	msg_fatal("error computing %s message digest", mdalg);
328 
329     return (tls_digest_encode(md_buf, md_len));
330 }
331 
332 /* tls_cert_fprint - extract certificate fingerprint */
333 
334 char   *tls_cert_fprint(X509 *peercert, const char *mdalg)
335 {
336     int     len;
337     unsigned char *buf;
338     unsigned char *buf2;
339     char   *result;
340 
341     len = i2d_X509(peercert, NULL);
342     buf2 = buf = mymalloc(len);
343     i2d_X509(peercert, &buf2);
344     if (buf2 - buf != len)
345 	msg_panic("i2d_X509 invalid result length");
346 
347     result = tls_data_fprint(buf, len, mdalg);
348     myfree(buf);
349 
350     return (result);
351 }
352 
353 /* tls_pkey_fprint - extract public key fingerprint from certificate */
354 
355 char   *tls_pkey_fprint(X509 *peercert, const char *mdalg)
356 {
357     if (var_tls_bc_pkey_fprint) {
358 	const char *myname = "tls_pkey_fprint";
359 	ASN1_BIT_STRING *key;
360 	char   *result;
361 
362 	key = X509_get0_pubkey_bitstr(peercert);
363 	if (key == 0)
364 	    msg_fatal("%s: error extracting legacy public-key fingerprint: %m",
365 		      myname);
366 
367 	result = tls_data_fprint(key->data, key->length, mdalg);
368 	return (result);
369     } else {
370 	int     len;
371 	unsigned char *buf;
372 	unsigned char *buf2;
373 	char   *result;
374 
375 	len = i2d_X509_PUBKEY(X509_get_X509_PUBKEY(peercert), NULL);
376 	buf2 = buf = mymalloc(len);
377 	i2d_X509_PUBKEY(X509_get_X509_PUBKEY(peercert), &buf2);
378 	if (buf2 - buf != len)
379 	    msg_panic("i2d_X509_PUBKEY invalid result length");
380 
381 	result = tls_data_fprint(buf, len, mdalg);
382 	myfree(buf);
383 	return (result);
384     }
385 }
386 
387 #endif
388