xref: /netbsd-src/external/ibm-public/postfix/dist/src/tls/tls_fprint.c (revision bdc22b2e01993381dcefeff2bc9b56ca75a4235c)
1 /*	$NetBSD: tls_fprint.c,v 1.2 2017/02/14 01:16:48 christos Exp $	*/
2 
3 /*++
4 /* NAME
5 /*	tls_fprint 3
6 /* SUMMARY
7 /*	Digests fingerprints and all that.
8 /* SYNOPSIS
9 /*	#include <tls.h>
10 /*
11 /*	char	*tls_serverid_digest(props, protomask, ciphers)
12 /*	const TLS_CLIENT_START_PROPS *props;
13 /*	long	protomask;
14 /*	const char *ciphers;
15 /*
16 /*	char	*tls_digest_encode(md_buf, md_len)
17 /*	const unsigned char *md_buf;
18 /*	const char *md_len;
19 /*
20 /*	char	*tls_data_fprint(buf, len, mdalg)
21 /*	const char *buf;
22 /*	int	len;
23 /*	const char *mdalg;
24 /*
25 /*	char	*tls_cert_fprint(peercert, mdalg)
26 /*	X509	*peercert;
27 /*	const char *mdalg;
28 /*
29 /*	char	*tls_pkey_fprint(peercert, mdalg)
30 /*	X509	*peercert;
31 /*	const char *mdalg;
32 /* DESCRIPTION
33 /*	tls_digest_encode() converts a binary message digest to a hex ASCII
34 /*	format with ':' separators between each pair of hex digits.
35 /*	The return value is dynamically allocated with mymalloc(),
36 /*	and the caller must eventually free it with myfree().
37 /*
38 /*	tls_data_fprint() digests unstructured data, and encodes the digested
39 /*	result via tls_digest_encode().  The return value is dynamically
40 /*	allocated with mymalloc(), and the caller must eventually free it
41 /*	with myfree().
42 /*
43 /*	tls_cert_fprint() returns a fingerprint of the the given
44 /*	certificate using the requested message digest, formatted
45 /*	with tls_digest_encode(). Panics if the
46 /*	(previously verified) digest algorithm is not found. The return
47 /*	value is dynamically allocated with mymalloc(), and the caller
48 /*	must eventually free it with myfree().
49 /*
50 /*	tls_pkey_fprint() returns a public-key fingerprint; in all
51 /*	other respects the function behaves as tls_cert_fprint().
52 /*	The var_tls_bc_pkey_fprint variable enables an incorrect
53 /*	algorithm that was used in Postfix versions 2.9.[0-5].
54 /*	The return value is dynamically allocated with mymalloc(),
55 /*	and the caller must eventually free it with myfree().
56 /*
57 /*	tls_serverid_digest() suffixes props->serverid computed by the SMTP
58 /*	client with "&" plus a digest of additional parameters
59 /*	needed to ensure that re-used sessions are more likely to
60 /*	be reused and that they will satisfy all protocol and
61 /*	security requirements.
62 /*	The return value is dynamically allocated with mymalloc(),
63 /*	and the caller must eventually free it with myfree().
64 /*
65 /*	Arguments:
66 /* .IP peercert
67 /*	Server or client X.509 certificate.
68 /* .IP md_buf
69 /*	The raw binary digest.
70 /* .IP md_len
71 /*	The digest length in bytes.
72 /* .IP mdalg
73 /*	Name of a message digest algorithm suitable for computing secure
74 /*	(1st pre-image resistant) message digests of certificates. For now,
75 /*	md5, sha1, or member of SHA-2 family if supported by OpenSSL.
76 /* .IP buf
77 /*	Input data for the message digest algorithm mdalg.
78 /* .IP len
79 /*	The length of the input data.
80 /* .IP props
81 /*	The client start properties for the session, which contains the
82 /*	initial serverid from the SMTP client and the DANE verification
83 /*	parameters.
84 /* .IP protomask
85 /*	The mask of protocol exclusions.
86 /* .IP ciphers
87 /*	The SSL client cipherlist.
88 /* LICENSE
89 /* .ad
90 /* .fi
91 /*	This software is free. You can do with it whatever you want.
92 /*	The original author kindly requests that you acknowledge
93 /*	the use of his software.
94 /* AUTHOR(S)
95 /*	Wietse Venema
96 /*	IBM T.J. Watson Research
97 /*	P.O. Box 704
98 /*	Yorktown Heights, NY 10598, USA
99 /*
100 /*	Viktor Dukhovni
101 /*--*/
102 
103 /* System library. */
104 
105 #include <sys_defs.h>
106 #include <ctype.h>
107 
108 #ifdef USE_TLS
109 #include <string.h>
110 
111 /* Utility library. */
112 
113 #include <msg.h>
114 #include <mymalloc.h>
115 #include <stringops.h>
116 
117 /* Global library. */
118 
119 #include <mail_params.h>
120 
121 /* TLS library. */
122 
123 #define TLS_INTERNAL
124 #include <tls.h>
125 
126 /* Application-specific. */
127 
128 static const char hexcodes[] = "0123456789ABCDEF";
129 
130 #define checkok(ret)	(ok &= ((ret) ? 1 : 0))
131 #define digest_data(p, l) checkok(EVP_DigestUpdate(mdctx, (char *)(p), (l)))
132 #define digest_object(p) digest_data((p), sizeof(*(p)))
133 #define digest_string(s) digest_data((s), strlen(s)+1)
134 
135 #define digest_dane(dane, memb) do { \
136 	if ((dane)->memb != 0) \
137 	    checkok(digest_tlsa_usage(mdctx, (dane)->memb, #memb)); \
138     } while (0)
139 
140 #define digest_tlsa_argv(tlsa, memb) do { \
141 	if ((tlsa)->memb) { \
142 	    digest_string(#memb); \
143 	    for (dgst = (tlsa)->memb->argv; *dgst; ++dgst) \
144 		digest_string(*dgst); \
145 	} \
146     } while (0)
147 
148 /* digest_tlsa_usage - digest TA or EE match list sorted by alg and value */
149 
150 static int digest_tlsa_usage(EVP_MD_CTX * mdctx, TLS_TLSA *tlsa,
151 			             const char *usage)
152 {
153     char  **dgst;
154     int     ok = 1;
155 
156     for (digest_string(usage); tlsa; tlsa = tlsa->next) {
157 	digest_string(tlsa->mdalg);
158 	digest_tlsa_argv(tlsa, pkeys);
159 	digest_tlsa_argv(tlsa, certs);
160     }
161     return (ok);
162 }
163 
164 /* tls_serverid_digest - suffix props->serverid with parameter digest */
165 
166 char   *tls_serverid_digest(const TLS_CLIENT_START_PROPS *props, long protomask,
167 			            const char *ciphers)
168 {
169     EVP_MD_CTX *mdctx;
170     const EVP_MD *md;
171     const char *mdalg;
172     unsigned char md_buf[EVP_MAX_MD_SIZE];
173     unsigned int md_len;
174     int     ok = 1;
175     int     i;
176     long    sslversion;
177     VSTRING *result;
178 
179     /*
180      * Try to use sha256: our serverid choice should be strong enough to
181      * resist 2nd-preimage attacks with a difficulty comparable to that of
182      * DANE TLSA digests.  Failing that, we compute serverid digests with the
183      * default digest, but DANE requires sha256 and sha512, so if we must
184      * fall back to our default digest, DANE support won't be available.  We
185      * panic if the fallback algorithm is not available, as it was verified
186      * available in tls_client_init() and must not simply vanish.
187      */
188     if ((md = EVP_get_digestbyname(mdalg = "sha256")) == 0
189 	&& (md = EVP_get_digestbyname(mdalg = props->mdalg)) == 0)
190 	msg_panic("digest algorithm \"%s\" not found", mdalg);
191 
192     /* Salt the session lookup key with the OpenSSL runtime version. */
193     sslversion = OpenSSL_version_num();
194 
195     mdctx = EVP_MD_CTX_create();
196     checkok(EVP_DigestInit_ex(mdctx, md, NULL));
197     digest_string(props->helo ? props->helo : "");
198     digest_object(&sslversion);
199     digest_object(&protomask);
200     digest_string(ciphers);
201 
202     /*
203      * All we get from the session cache is a single bit telling us whether
204      * the certificate is trusted or not, but we need to know whether the
205      * trust is CA-based (in that case we must do name checks) or whether it
206      * is a direct end-point match.  We mustn't confuse the two, so it is
207      * best to process only TA trust in the verify callback and check the EE
208      * trust after. This works since re-used sessions always have access to
209      * the leaf certificate, while only the original session has the leaf and
210      * the full trust chain.
211      *
212      * Only the trust anchor matchlist is hashed into the session key. The end
213      * entity certs are not used to determine whether a certificate is
214      * trusted or not, rather these are rechecked against the leaf cert
215      * outside the verification callback, each time a session is created or
216      * reused.
217      *
218      * Therefore, the security context of the session does not depend on the EE
219      * matching data, which is checked separately each time.  So we exclude
220      * the EE part of the DANE structure from the serverid digest.
221      *
222      * If the security level is "dane", we send SNI information to the peer.
223      * This may cause it to respond with a non-default certificate.  Since
224      * certificates for sessions with no or different SNI data may not match,
225      * we must include the SNI name in the session id.
226      */
227     if (props->dane) {
228 	digest_dane(props->dane, ta);
229 #if 0
230 	digest_dane(props->dane, ee);		/* See above */
231 #endif
232 	digest_string(TLS_DANE_BASED(props->tls_level) ? props->host : "");
233     }
234     checkok(EVP_DigestFinal_ex(mdctx, md_buf, &md_len));
235     EVP_MD_CTX_destroy(mdctx);
236     if (!ok)
237 	msg_fatal("error computing %s message digest", mdalg);
238 
239     /* Check for OpenSSL contract violation */
240     if (md_len > EVP_MAX_MD_SIZE)
241 	msg_panic("unexpectedly large %s digest size: %u", mdalg, md_len);
242 
243     /*
244      * Append the digest to the serverid.  We don't compare this digest to
245      * any user-specified fingerprints.  Therefore, we don't need to use a
246      * colon-separated format, which saves space in the TLS session cache and
247      * makes logging of session cache lookup keys more readable.
248      *
249      * This does however duplicate a few lines of code from the digest encoder
250      * for colon-separated cert and pkey fingerprints. If that is a
251      * compelling reason to consolidate, we could use that and append the
252      * result.
253      */
254     result = vstring_alloc(strlen(props->serverid) + 1 + 2 * md_len);
255     vstring_strcpy(result, props->serverid);
256     VSTRING_ADDCH(result, '&');
257     for (i = 0; i < md_len; i++) {
258 	VSTRING_ADDCH(result, hexcodes[(md_buf[i] & 0xf0) >> 4U]);
259 	VSTRING_ADDCH(result, hexcodes[(md_buf[i] & 0x0f)]);
260     }
261     VSTRING_TERMINATE(result);
262     return (vstring_export(result));
263 }
264 
265 /* tls_digest_encode - encode message digest binary blob as xx:xx:... */
266 
267 char   *tls_digest_encode(const unsigned char *md_buf, int md_len)
268 {
269     int     i;
270     char   *result = mymalloc(md_len * 3);
271 
272     /* Check for contract violation */
273     if (md_len > EVP_MAX_MD_SIZE || md_len >= INT_MAX / 3)
274 	msg_panic("unexpectedly large message digest size: %u", md_len);
275 
276     /* No risk of overrunes, len is bounded by OpenSSL digest length */
277     for (i = 0; i < md_len; i++) {
278 	result[i * 3] = hexcodes[(md_buf[i] & 0xf0) >> 4U];
279 	result[(i * 3) + 1] = hexcodes[(md_buf[i] & 0x0f)];
280 	result[(i * 3) + 2] = (i + 1 != md_len) ? ':' : '\0';
281     }
282     return (result);
283 }
284 
285 /* tls_data_fprint - compute and encode digest of binary object */
286 
287 char   *tls_data_fprint(const char *buf, int len, const char *mdalg)
288 {
289     EVP_MD_CTX *mdctx;
290     const EVP_MD *md;
291     unsigned char md_buf[EVP_MAX_MD_SIZE];
292     unsigned int md_len;
293     int     ok = 1;
294 
295     /* Previously available in "init" routine. */
296     if ((md = EVP_get_digestbyname(mdalg)) == 0)
297 	msg_panic("digest algorithm \"%s\" not found", mdalg);
298 
299     mdctx = EVP_MD_CTX_create();
300     checkok(EVP_DigestInit_ex(mdctx, md, NULL));
301     digest_data(buf, len);
302     checkok(EVP_DigestFinal_ex(mdctx, md_buf, &md_len));
303     EVP_MD_CTX_destroy(mdctx);
304     if (!ok)
305 	msg_fatal("error computing %s message digest", mdalg);
306 
307     return (tls_digest_encode(md_buf, md_len));
308 }
309 
310 /* tls_cert_fprint - extract certificate fingerprint */
311 
312 char   *tls_cert_fprint(X509 *peercert, const char *mdalg)
313 {
314     int     len;
315     char   *buf;
316     char   *buf2;
317     char   *result;
318 
319     len = i2d_X509(peercert, NULL);
320     buf2 = buf = mymalloc(len);
321     i2d_X509(peercert, (unsigned char **) &buf2);
322     if (buf2 - buf != len)
323 	msg_panic("i2d_X509 invalid result length");
324 
325     result = tls_data_fprint(buf, len, mdalg);
326     myfree(buf);
327 
328     return (result);
329 }
330 
331 /* tls_pkey_fprint - extract public key fingerprint from certificate */
332 
333 char   *tls_pkey_fprint(X509 *peercert, const char *mdalg)
334 {
335     if (var_tls_bc_pkey_fprint) {
336 	const char *myname = "tls_pkey_fprint";
337 	ASN1_BIT_STRING *key;
338 	char   *result;
339 
340 	key = X509_get0_pubkey_bitstr(peercert);
341 	if (key == 0)
342 	    msg_fatal("%s: error extracting legacy public-key fingerprint: %m",
343 		      myname);
344 
345 	result = tls_data_fprint((char *) key->data, key->length, mdalg);
346 	return (result);
347     } else {
348 	int     len;
349 	char   *buf;
350 	char   *buf2;
351 	char   *result;
352 
353 	len = i2d_X509_PUBKEY(X509_get_X509_PUBKEY(peercert), NULL);
354 	buf2 = buf = mymalloc(len);
355 	i2d_X509_PUBKEY(X509_get_X509_PUBKEY(peercert), (unsigned char **) &buf2);
356 	if (buf2 - buf != len)
357 	    msg_panic("i2d_X509_PUBKEY invalid result length");
358 
359 	result = tls_data_fprint(buf, len, mdalg);
360 	myfree(buf);
361 	return (result);
362     }
363 }
364 
365 #endif
366