xref: /onnv-gate/usr/src/cmd/sendmail/src/tls.c (revision 1658:e3b42aa6425a)
10Sstevel@tonic-gate /*
20Sstevel@tonic-gate  * Copyright (c) 2000-2005 Sendmail, Inc. and its suppliers.
30Sstevel@tonic-gate  *	All rights reserved.
40Sstevel@tonic-gate  *
50Sstevel@tonic-gate  * By using this file, you agree to the terms and conditions set
60Sstevel@tonic-gate  * forth in the LICENSE file which can be found at the top level of
70Sstevel@tonic-gate  * the sendmail distribution.
80Sstevel@tonic-gate  *
90Sstevel@tonic-gate  */
100Sstevel@tonic-gate 
110Sstevel@tonic-gate #pragma ident	"%Z%%M%	%I%	%E% SMI"
120Sstevel@tonic-gate 
130Sstevel@tonic-gate #include <sendmail.h>
140Sstevel@tonic-gate 
15*1658Sjbeck SM_RCSID("@(#)$Id: tls.c,v 8.102 2006/03/02 19:18:27 ca Exp $")
160Sstevel@tonic-gate 
170Sstevel@tonic-gate #if STARTTLS
180Sstevel@tonic-gate #  include <openssl/err.h>
190Sstevel@tonic-gate #  include <openssl/bio.h>
200Sstevel@tonic-gate #  include <openssl/pem.h>
210Sstevel@tonic-gate #  ifndef HASURANDOMDEV
220Sstevel@tonic-gate #   include <openssl/rand.h>
230Sstevel@tonic-gate #  endif /* ! HASURANDOMDEV */
240Sstevel@tonic-gate # if !TLS_NO_RSA
250Sstevel@tonic-gate static RSA *rsa_tmp = NULL;	/* temporary RSA key */
260Sstevel@tonic-gate static RSA *tmp_rsa_key __P((SSL *, int, int));
270Sstevel@tonic-gate # endif /* !TLS_NO_RSA */
280Sstevel@tonic-gate #  if !defined(OPENSSL_VERSION_NUMBER) || OPENSSL_VERSION_NUMBER < 0x00907000L
290Sstevel@tonic-gate static int	tls_verify_cb __P((X509_STORE_CTX *));
300Sstevel@tonic-gate #  else /* !defined() || OPENSSL_VERSION_NUMBER < 0x00907000L */
310Sstevel@tonic-gate static int	tls_verify_cb __P((X509_STORE_CTX *, void *));
320Sstevel@tonic-gate #  endif /* !defined() || OPENSSL_VERSION_NUMBER < 0x00907000L */
330Sstevel@tonic-gate 
340Sstevel@tonic-gate # if OPENSSL_VERSION_NUMBER > 0x00907000L
350Sstevel@tonic-gate static int x509_verify_cb __P((int, X509_STORE_CTX *));
360Sstevel@tonic-gate # endif /* OPENSSL_VERSION_NUMBER > 0x00907000L */
370Sstevel@tonic-gate 
380Sstevel@tonic-gate # if !defined(OPENSSL_VERSION_NUMBER) || OPENSSL_VERSION_NUMBER < 0x00907000L
390Sstevel@tonic-gate #  define CONST097
400Sstevel@tonic-gate # else /* !defined() || OPENSSL_VERSION_NUMBER < 0x00907000L */
410Sstevel@tonic-gate #  define CONST097 const
420Sstevel@tonic-gate # endif /* !defined() || OPENSSL_VERSION_NUMBER < 0x00907000L */
430Sstevel@tonic-gate static void	apps_ssl_info_cb __P((CONST097 SSL *, int , int));
440Sstevel@tonic-gate static bool	tls_ok_f __P((char *, char *, int));
450Sstevel@tonic-gate static bool	tls_safe_f __P((char *, long, bool));
460Sstevel@tonic-gate static int	tls_verify_log __P((int, X509_STORE_CTX *, char *));
470Sstevel@tonic-gate 
480Sstevel@tonic-gate # if !NO_DH
490Sstevel@tonic-gate static DH *get_dh512 __P((void));
500Sstevel@tonic-gate 
510Sstevel@tonic-gate static unsigned char dh512_p[] =
520Sstevel@tonic-gate {
530Sstevel@tonic-gate 	0xDA,0x58,0x3C,0x16,0xD9,0x85,0x22,0x89,0xD0,0xE4,0xAF,0x75,
540Sstevel@tonic-gate 	0x6F,0x4C,0xCA,0x92,0xDD,0x4B,0xE5,0x33,0xB8,0x04,0xFB,0x0F,
550Sstevel@tonic-gate 	0xED,0x94,0xEF,0x9C,0x8A,0x44,0x03,0xED,0x57,0x46,0x50,0xD3,
560Sstevel@tonic-gate 	0x69,0x99,0xDB,0x29,0xD7,0x76,0x27,0x6B,0xA2,0xD3,0xD4,0x12,
570Sstevel@tonic-gate 	0xE2,0x18,0xF4,0xDD,0x1E,0x08,0x4C,0xF6,0xD8,0x00,0x3E,0x7C,
580Sstevel@tonic-gate 	0x47,0x74,0xE8,0x33
590Sstevel@tonic-gate };
600Sstevel@tonic-gate static unsigned char dh512_g[] =
610Sstevel@tonic-gate {
620Sstevel@tonic-gate 	0x02
630Sstevel@tonic-gate };
640Sstevel@tonic-gate 
650Sstevel@tonic-gate static DH *
660Sstevel@tonic-gate get_dh512()
670Sstevel@tonic-gate {
680Sstevel@tonic-gate 	DH *dh = NULL;
690Sstevel@tonic-gate 
700Sstevel@tonic-gate 	if ((dh = DH_new()) == NULL)
710Sstevel@tonic-gate 		return NULL;
720Sstevel@tonic-gate 	dh->p = BN_bin2bn(dh512_p, sizeof(dh512_p), NULL);
730Sstevel@tonic-gate 	dh->g = BN_bin2bn(dh512_g, sizeof(dh512_g), NULL);
740Sstevel@tonic-gate 	if ((dh->p == NULL) || (dh->g == NULL))
750Sstevel@tonic-gate 		return NULL;
760Sstevel@tonic-gate 	return dh;
770Sstevel@tonic-gate }
780Sstevel@tonic-gate # endif /* !NO_DH */
790Sstevel@tonic-gate 
800Sstevel@tonic-gate 
810Sstevel@tonic-gate /*
820Sstevel@tonic-gate **  TLS_RAND_INIT -- initialize STARTTLS random generator
830Sstevel@tonic-gate **
840Sstevel@tonic-gate **	Parameters:
850Sstevel@tonic-gate **		randfile -- name of file with random data
860Sstevel@tonic-gate **		logl -- loglevel
870Sstevel@tonic-gate **
880Sstevel@tonic-gate **	Returns:
890Sstevel@tonic-gate **		success/failure
900Sstevel@tonic-gate **
910Sstevel@tonic-gate **	Side Effects:
920Sstevel@tonic-gate **		initializes PRNG for tls library.
930Sstevel@tonic-gate */
940Sstevel@tonic-gate 
950Sstevel@tonic-gate # define MIN_RAND_BYTES	128	/* 1024 bits */
960Sstevel@tonic-gate 
970Sstevel@tonic-gate # define RF_OK		0	/* randfile OK */
980Sstevel@tonic-gate # define RF_MISS	1	/* randfile == NULL || *randfile == '\0' */
990Sstevel@tonic-gate # define RF_UNKNOWN	2	/* unknown prefix for randfile */
1000Sstevel@tonic-gate 
1010Sstevel@tonic-gate # define RI_NONE	0	/* no init yet */
1020Sstevel@tonic-gate # define RI_SUCCESS	1	/* init was successful */
1030Sstevel@tonic-gate # define RI_FAIL	2	/* init failed */
1040Sstevel@tonic-gate 
1050Sstevel@tonic-gate static bool	tls_rand_init __P((char *, int));
1060Sstevel@tonic-gate 
1070Sstevel@tonic-gate static bool
1080Sstevel@tonic-gate tls_rand_init(randfile, logl)
1090Sstevel@tonic-gate 	char *randfile;
1100Sstevel@tonic-gate 	int logl;
1110Sstevel@tonic-gate {
1120Sstevel@tonic-gate # ifndef HASURANDOMDEV
1130Sstevel@tonic-gate 	/* not required if /dev/urandom exists, OpenSSL does it internally */
1140Sstevel@tonic-gate 
1150Sstevel@tonic-gate 	bool ok;
1160Sstevel@tonic-gate 	int randdef;
1170Sstevel@tonic-gate 	static int done = RI_NONE;
1180Sstevel@tonic-gate 
1190Sstevel@tonic-gate 	/*
1200Sstevel@tonic-gate 	**  initialize PRNG
1210Sstevel@tonic-gate 	*/
1220Sstevel@tonic-gate 
1230Sstevel@tonic-gate 	/* did we try this before? if yes: return old value */
1240Sstevel@tonic-gate 	if (done != RI_NONE)
1250Sstevel@tonic-gate 		return done == RI_SUCCESS;
1260Sstevel@tonic-gate 
1270Sstevel@tonic-gate 	/* set default values */
1280Sstevel@tonic-gate 	ok = false;
1290Sstevel@tonic-gate 	done = RI_FAIL;
1300Sstevel@tonic-gate 	randdef = (randfile == NULL || *randfile == '\0') ? RF_MISS : RF_OK;
1310Sstevel@tonic-gate #   if EGD
1320Sstevel@tonic-gate 	if (randdef == RF_OK && sm_strncasecmp(randfile, "egd:", 4) == 0)
1330Sstevel@tonic-gate 	{
1340Sstevel@tonic-gate 		randfile += 4;
1350Sstevel@tonic-gate 		if (RAND_egd(randfile) < 0)
1360Sstevel@tonic-gate 		{
1370Sstevel@tonic-gate 			sm_syslog(LOG_WARNING, NOQID,
1380Sstevel@tonic-gate 				  "STARTTLS: RAND_egd(%s) failed: random number generator not seeded",
1390Sstevel@tonic-gate 				   randfile);
1400Sstevel@tonic-gate 		}
1410Sstevel@tonic-gate 		else
1420Sstevel@tonic-gate 			ok = true;
1430Sstevel@tonic-gate 	}
1440Sstevel@tonic-gate 	else
1450Sstevel@tonic-gate #   endif /* EGD */
1460Sstevel@tonic-gate 	if (randdef == RF_OK && sm_strncasecmp(randfile, "file:", 5) == 0)
1470Sstevel@tonic-gate 	{
1480Sstevel@tonic-gate 		int fd;
1490Sstevel@tonic-gate 		long sff;
1500Sstevel@tonic-gate 		struct stat st;
1510Sstevel@tonic-gate 
1520Sstevel@tonic-gate 		randfile += 5;
1530Sstevel@tonic-gate 		sff = SFF_SAFEDIRPATH | SFF_NOWLINK
1540Sstevel@tonic-gate 		      | SFF_NOGWFILES | SFF_NOWWFILES
1550Sstevel@tonic-gate 		      | SFF_NOGRFILES | SFF_NOWRFILES
1560Sstevel@tonic-gate 		      | SFF_MUSTOWN | SFF_ROOTOK | SFF_OPENASROOT;
1570Sstevel@tonic-gate 		if (DontLockReadFiles)
1580Sstevel@tonic-gate 			sff |= SFF_NOLOCK;
1590Sstevel@tonic-gate 		if ((fd = safeopen(randfile, O_RDONLY, 0, sff)) >= 0)
1600Sstevel@tonic-gate 		{
1610Sstevel@tonic-gate 			if (fstat(fd, &st) < 0)
1620Sstevel@tonic-gate 			{
1630Sstevel@tonic-gate 				if (LogLevel > logl)
1640Sstevel@tonic-gate 					sm_syslog(LOG_ERR, NOQID,
1650Sstevel@tonic-gate 						  "STARTTLS: can't fstat(%s)",
1660Sstevel@tonic-gate 						  randfile);
1670Sstevel@tonic-gate 			}
1680Sstevel@tonic-gate 			else
1690Sstevel@tonic-gate 			{
1700Sstevel@tonic-gate 				bool use, problem;
1710Sstevel@tonic-gate 
1720Sstevel@tonic-gate 				use = true;
1730Sstevel@tonic-gate 				problem = false;
1740Sstevel@tonic-gate 
1750Sstevel@tonic-gate 				/* max. age of file: 10 minutes */
1760Sstevel@tonic-gate 				if (st.st_mtime + 600 < curtime())
1770Sstevel@tonic-gate 				{
1780Sstevel@tonic-gate 					use = bitnset(DBS_INSUFFICIENTENTROPY,
1790Sstevel@tonic-gate 						      DontBlameSendmail);
1800Sstevel@tonic-gate 					problem = true;
1810Sstevel@tonic-gate 					if (LogLevel > logl)
1820Sstevel@tonic-gate 						sm_syslog(LOG_ERR, NOQID,
1830Sstevel@tonic-gate 							  "STARTTLS: RandFile %s too old: %s",
1840Sstevel@tonic-gate 							  randfile,
1850Sstevel@tonic-gate 							  use ? "unsafe" :
1860Sstevel@tonic-gate 								"unusable");
1870Sstevel@tonic-gate 				}
1880Sstevel@tonic-gate 				if (use && st.st_size < MIN_RAND_BYTES)
1890Sstevel@tonic-gate 				{
1900Sstevel@tonic-gate 					use = bitnset(DBS_INSUFFICIENTENTROPY,
1910Sstevel@tonic-gate 						      DontBlameSendmail);
1920Sstevel@tonic-gate 					problem = true;
1930Sstevel@tonic-gate 					if (LogLevel > logl)
1940Sstevel@tonic-gate 						sm_syslog(LOG_ERR, NOQID,
1950Sstevel@tonic-gate 							  "STARTTLS: size(%s) < %d: %s",
1960Sstevel@tonic-gate 							  randfile,
1970Sstevel@tonic-gate 							  MIN_RAND_BYTES,
1980Sstevel@tonic-gate 							  use ? "unsafe" :
1990Sstevel@tonic-gate 								"unusable");
2000Sstevel@tonic-gate 				}
2010Sstevel@tonic-gate 				if (use)
2020Sstevel@tonic-gate 					ok = RAND_load_file(randfile, -1) >=
2030Sstevel@tonic-gate 					     MIN_RAND_BYTES;
2040Sstevel@tonic-gate 				if (use && !ok)
2050Sstevel@tonic-gate 				{
2060Sstevel@tonic-gate 					if (LogLevel > logl)
2070Sstevel@tonic-gate 						sm_syslog(LOG_WARNING, NOQID,
2080Sstevel@tonic-gate 							  "STARTTLS: RAND_load_file(%s) failed: random number generator not seeded",
2090Sstevel@tonic-gate 							  randfile);
2100Sstevel@tonic-gate 				}
2110Sstevel@tonic-gate 				if (problem)
2120Sstevel@tonic-gate 					ok = false;
2130Sstevel@tonic-gate 			}
2140Sstevel@tonic-gate 			if (ok || bitnset(DBS_INSUFFICIENTENTROPY,
2150Sstevel@tonic-gate 					  DontBlameSendmail))
2160Sstevel@tonic-gate 			{
2170Sstevel@tonic-gate 				/* add this even if fstat() failed */
2180Sstevel@tonic-gate 				RAND_seed((void *) &st, sizeof st);
2190Sstevel@tonic-gate 			}
2200Sstevel@tonic-gate 			(void) close(fd);
2210Sstevel@tonic-gate 		}
2220Sstevel@tonic-gate 		else
2230Sstevel@tonic-gate 		{
2240Sstevel@tonic-gate 			if (LogLevel > logl)
2250Sstevel@tonic-gate 				sm_syslog(LOG_WARNING, NOQID,
2260Sstevel@tonic-gate 					  "STARTTLS: Warning: safeopen(%s) failed",
2270Sstevel@tonic-gate 					  randfile);
2280Sstevel@tonic-gate 		}
2290Sstevel@tonic-gate 	}
2300Sstevel@tonic-gate 	else if (randdef == RF_OK)
2310Sstevel@tonic-gate 	{
2320Sstevel@tonic-gate 		if (LogLevel > logl)
2330Sstevel@tonic-gate 			sm_syslog(LOG_WARNING, NOQID,
2340Sstevel@tonic-gate 				  "STARTTLS: Error: no proper random file definition %s",
2350Sstevel@tonic-gate 				  randfile);
2360Sstevel@tonic-gate 		randdef = RF_UNKNOWN;
2370Sstevel@tonic-gate 	}
2380Sstevel@tonic-gate 	if (randdef == RF_MISS)
2390Sstevel@tonic-gate 	{
2400Sstevel@tonic-gate 		if (LogLevel > logl)
2410Sstevel@tonic-gate 			sm_syslog(LOG_WARNING, NOQID,
2420Sstevel@tonic-gate 				  "STARTTLS: Error: missing random file definition");
2430Sstevel@tonic-gate 	}
2440Sstevel@tonic-gate 	if (!ok && bitnset(DBS_INSUFFICIENTENTROPY, DontBlameSendmail))
2450Sstevel@tonic-gate 	{
2460Sstevel@tonic-gate 		int i;
2470Sstevel@tonic-gate 		long r;
2480Sstevel@tonic-gate 		unsigned char buf[MIN_RAND_BYTES];
2490Sstevel@tonic-gate 
2500Sstevel@tonic-gate 		/* assert((MIN_RAND_BYTES % sizeof(long)) == 0); */
2510Sstevel@tonic-gate 		for (i = 0; i <= sizeof(buf) - sizeof(long); i += sizeof(long))
2520Sstevel@tonic-gate 		{
2530Sstevel@tonic-gate 			r = get_random();
2540Sstevel@tonic-gate 			(void) memcpy(buf + i, (void *) &r, sizeof(long));
2550Sstevel@tonic-gate 		}
2560Sstevel@tonic-gate 		RAND_seed(buf, sizeof buf);
2570Sstevel@tonic-gate 		if (LogLevel > logl)
2580Sstevel@tonic-gate 			sm_syslog(LOG_WARNING, NOQID,
2590Sstevel@tonic-gate 				  "STARTTLS: Warning: random number generator not properly seeded");
2600Sstevel@tonic-gate 		ok = true;
2610Sstevel@tonic-gate 	}
2620Sstevel@tonic-gate 	done = ok ? RI_SUCCESS : RI_FAIL;
2630Sstevel@tonic-gate 	return ok;
2640Sstevel@tonic-gate # else /* ! HASURANDOMDEV */
2650Sstevel@tonic-gate 	return true;
2660Sstevel@tonic-gate # endif /* ! HASURANDOMDEV */
2670Sstevel@tonic-gate }
2680Sstevel@tonic-gate /*
2690Sstevel@tonic-gate **  INIT_TLS_LIBRARY -- Calls functions which setup TLS library for global use.
2700Sstevel@tonic-gate **
2710Sstevel@tonic-gate **	Parameters:
2720Sstevel@tonic-gate **		none.
2730Sstevel@tonic-gate **
2740Sstevel@tonic-gate **	Returns:
2750Sstevel@tonic-gate **		succeeded?
2760Sstevel@tonic-gate */
2770Sstevel@tonic-gate 
2780Sstevel@tonic-gate bool
2790Sstevel@tonic-gate init_tls_library()
2800Sstevel@tonic-gate {
2810Sstevel@tonic-gate 	/* basic TLS initialization, ignore result for now */
2820Sstevel@tonic-gate 	SSL_library_init();
2830Sstevel@tonic-gate 	SSL_load_error_strings();
2840Sstevel@tonic-gate # if 0
2850Sstevel@tonic-gate 	/* this is currently a macro for SSL_library_init */
2860Sstevel@tonic-gate 	SSLeay_add_ssl_algorithms();
2870Sstevel@tonic-gate # endif /* 0 */
2880Sstevel@tonic-gate 
2890Sstevel@tonic-gate 	return tls_rand_init(RandFile, 7);
2900Sstevel@tonic-gate }
2910Sstevel@tonic-gate /*
2920Sstevel@tonic-gate **  TLS_SET_VERIFY -- request client certificate?
2930Sstevel@tonic-gate **
2940Sstevel@tonic-gate **	Parameters:
2950Sstevel@tonic-gate **		ctx -- TLS context
2960Sstevel@tonic-gate **		ssl -- TLS structure
2970Sstevel@tonic-gate **		vrfy -- require certificate?
2980Sstevel@tonic-gate **
2990Sstevel@tonic-gate **	Returns:
3000Sstevel@tonic-gate **		none.
3010Sstevel@tonic-gate **
3020Sstevel@tonic-gate **	Side Effects:
3030Sstevel@tonic-gate **		Sets verification state for TLS
3040Sstevel@tonic-gate **
3050Sstevel@tonic-gate # if TLS_VRFY_PER_CTX
3060Sstevel@tonic-gate **	Notice:
3070Sstevel@tonic-gate **		This is per TLS context, not per TLS structure;
3080Sstevel@tonic-gate **		the former is global, the latter per connection.
3090Sstevel@tonic-gate **		It would be nice to do this per connection, but this
3100Sstevel@tonic-gate **		doesn't work in the current TLS libraries :-(
3110Sstevel@tonic-gate # endif * TLS_VRFY_PER_CTX *
3120Sstevel@tonic-gate */
3130Sstevel@tonic-gate 
3140Sstevel@tonic-gate void
3150Sstevel@tonic-gate tls_set_verify(ctx, ssl, vrfy)
3160Sstevel@tonic-gate 	SSL_CTX *ctx;
3170Sstevel@tonic-gate 	SSL *ssl;
3180Sstevel@tonic-gate 	bool vrfy;
3190Sstevel@tonic-gate {
3200Sstevel@tonic-gate # if !TLS_VRFY_PER_CTX
3210Sstevel@tonic-gate 	SSL_set_verify(ssl, vrfy ? SSL_VERIFY_PEER : SSL_VERIFY_NONE, NULL);
3220Sstevel@tonic-gate # else /* !TLS_VRFY_PER_CTX */
3230Sstevel@tonic-gate 	SSL_CTX_set_verify(ctx, vrfy ? SSL_VERIFY_PEER : SSL_VERIFY_NONE,
3240Sstevel@tonic-gate 			NULL);
3250Sstevel@tonic-gate # endif /* !TLS_VRFY_PER_CTX */
3260Sstevel@tonic-gate }
3270Sstevel@tonic-gate 
3280Sstevel@tonic-gate /*
3290Sstevel@tonic-gate **  status in initialization
3300Sstevel@tonic-gate **  these flags keep track of the status of the initialization
3310Sstevel@tonic-gate **  i.e., whether a file exists (_EX) and whether it can be used (_OK)
3320Sstevel@tonic-gate **  [due to permissions]
3330Sstevel@tonic-gate */
3340Sstevel@tonic-gate 
3350Sstevel@tonic-gate # define TLS_S_NONE	0x00000000	/* none yet */
3360Sstevel@tonic-gate # define TLS_S_CERT_EX	0x00000001	/* cert file exists */
3370Sstevel@tonic-gate # define TLS_S_CERT_OK	0x00000002	/* cert file is ok */
3380Sstevel@tonic-gate # define TLS_S_KEY_EX	0x00000004	/* key file exists */
3390Sstevel@tonic-gate # define TLS_S_KEY_OK	0x00000008	/* key file is ok */
3400Sstevel@tonic-gate # define TLS_S_CERTP_EX	0x00000010	/* CA cert path exists */
3410Sstevel@tonic-gate # define TLS_S_CERTP_OK	0x00000020	/* CA cert path is ok */
3420Sstevel@tonic-gate # define TLS_S_CERTF_EX	0x00000040	/* CA cert file exists */
3430Sstevel@tonic-gate # define TLS_S_CERTF_OK	0x00000080	/* CA cert file is ok */
3440Sstevel@tonic-gate # define TLS_S_CRLF_EX	0x00000100	/* CRL file exists */
3450Sstevel@tonic-gate # define TLS_S_CRLF_OK	0x00000200	/* CRL file is ok */
3460Sstevel@tonic-gate 
3470Sstevel@tonic-gate # if _FFR_TLS_1
3480Sstevel@tonic-gate #  define TLS_S_CERT2_EX	0x00001000	/* 2nd cert file exists */
3490Sstevel@tonic-gate #  define TLS_S_CERT2_OK	0x00002000	/* 2nd cert file is ok */
3500Sstevel@tonic-gate #  define TLS_S_KEY2_EX	0x00004000	/* 2nd key file exists */
3510Sstevel@tonic-gate #  define TLS_S_KEY2_OK	0x00008000	/* 2nd key file is ok */
3520Sstevel@tonic-gate # endif /* _FFR_TLS_1 */
3530Sstevel@tonic-gate 
3540Sstevel@tonic-gate # define TLS_S_DH_OK	0x00200000	/* DH cert is ok */
3550Sstevel@tonic-gate # define TLS_S_DHPAR_EX	0x00400000	/* DH param file exists */
3560Sstevel@tonic-gate # define TLS_S_DHPAR_OK	0x00800000	/* DH param file is ok to use */
3570Sstevel@tonic-gate 
3580Sstevel@tonic-gate /* Type of variable */
3590Sstevel@tonic-gate # define TLS_T_OTHER	0
3600Sstevel@tonic-gate # define TLS_T_SRV	1
3610Sstevel@tonic-gate # define TLS_T_CLT	2
3620Sstevel@tonic-gate 
3630Sstevel@tonic-gate /*
3640Sstevel@tonic-gate **  TLS_OK_F -- can var be an absolute filename?
3650Sstevel@tonic-gate **
3660Sstevel@tonic-gate **	Parameters:
3670Sstevel@tonic-gate **		var -- filename
3680Sstevel@tonic-gate **		fn -- what is the filename used for?
3690Sstevel@tonic-gate **		type -- type of variable
3700Sstevel@tonic-gate **
3710Sstevel@tonic-gate **	Returns:
3720Sstevel@tonic-gate **		ok?
3730Sstevel@tonic-gate */
3740Sstevel@tonic-gate 
3750Sstevel@tonic-gate static bool
3760Sstevel@tonic-gate tls_ok_f(var, fn, type)
3770Sstevel@tonic-gate 	char *var;
3780Sstevel@tonic-gate 	char *fn;
3790Sstevel@tonic-gate 	int type;
3800Sstevel@tonic-gate {
3810Sstevel@tonic-gate 	/* must be absolute pathname */
3820Sstevel@tonic-gate 	if (var != NULL && *var == '/')
3830Sstevel@tonic-gate 		return true;
3840Sstevel@tonic-gate 	if (LogLevel > 12)
3850Sstevel@tonic-gate 		sm_syslog(LOG_WARNING, NOQID, "STARTTLS: %s%s missing",
3860Sstevel@tonic-gate 			  type == TLS_T_SRV ? "Server" :
3870Sstevel@tonic-gate 			  (type == TLS_T_CLT ? "Client" : ""), fn);
3880Sstevel@tonic-gate 	return false;
3890Sstevel@tonic-gate }
3900Sstevel@tonic-gate /*
3910Sstevel@tonic-gate **  TLS_SAFE_F -- is a file safe to use?
3920Sstevel@tonic-gate **
3930Sstevel@tonic-gate **	Parameters:
3940Sstevel@tonic-gate **		var -- filename
3950Sstevel@tonic-gate **		sff -- flags for safefile()
3960Sstevel@tonic-gate **		srv -- server side?
3970Sstevel@tonic-gate **
3980Sstevel@tonic-gate **	Returns:
3990Sstevel@tonic-gate **		ok?
4000Sstevel@tonic-gate */
4010Sstevel@tonic-gate 
4020Sstevel@tonic-gate static bool
4030Sstevel@tonic-gate tls_safe_f(var, sff, srv)
4040Sstevel@tonic-gate 	char *var;
4050Sstevel@tonic-gate 	long sff;
4060Sstevel@tonic-gate 	bool srv;
4070Sstevel@tonic-gate {
4080Sstevel@tonic-gate 	int ret;
4090Sstevel@tonic-gate 
4100Sstevel@tonic-gate 	if ((ret = safefile(var, RunAsUid, RunAsGid, RunAsUserName, sff,
4110Sstevel@tonic-gate 			    S_IRUSR, NULL)) == 0)
4120Sstevel@tonic-gate 		return true;
4130Sstevel@tonic-gate 	if (LogLevel > 7)
4140Sstevel@tonic-gate 		sm_syslog(LOG_WARNING, NOQID, "STARTTLS=%s: file %s unsafe: %s",
4150Sstevel@tonic-gate 			  srv ? "server" : "client", var, sm_errstring(ret));
4160Sstevel@tonic-gate 	return false;
4170Sstevel@tonic-gate }
4180Sstevel@tonic-gate 
4190Sstevel@tonic-gate /*
4200Sstevel@tonic-gate **  TLS_OK_F -- macro to simplify calls to tls_ok_f
4210Sstevel@tonic-gate **
4220Sstevel@tonic-gate **	Parameters:
4230Sstevel@tonic-gate **		var -- filename
4240Sstevel@tonic-gate **		fn -- what is the filename used for?
4250Sstevel@tonic-gate **		req -- is the file required?
4260Sstevel@tonic-gate **		st -- status bit to set if ok
4270Sstevel@tonic-gate **		type -- type of variable
4280Sstevel@tonic-gate **
4290Sstevel@tonic-gate **	Side Effects:
4300Sstevel@tonic-gate **		uses r, ok; may change ok and status.
4310Sstevel@tonic-gate **
4320Sstevel@tonic-gate */
4330Sstevel@tonic-gate 
4340Sstevel@tonic-gate # define TLS_OK_F(var, fn, req, st, type) if (ok) \
4350Sstevel@tonic-gate 	{ \
4360Sstevel@tonic-gate 		r = tls_ok_f(var, fn, type); \
4370Sstevel@tonic-gate 		if (r) \
4380Sstevel@tonic-gate 			status |= st; \
4390Sstevel@tonic-gate 		else if (req) \
4400Sstevel@tonic-gate 			ok = false; \
4410Sstevel@tonic-gate 	}
4420Sstevel@tonic-gate 
4430Sstevel@tonic-gate /*
4440Sstevel@tonic-gate **  TLS_UNR -- macro to return whether a file should be unreadable
4450Sstevel@tonic-gate **
4460Sstevel@tonic-gate **	Parameters:
4470Sstevel@tonic-gate **		bit -- flag to test
4480Sstevel@tonic-gate **		req -- flags
4490Sstevel@tonic-gate **
4500Sstevel@tonic-gate **	Returns:
4510Sstevel@tonic-gate **		0/SFF_NORFILES
4520Sstevel@tonic-gate */
4530Sstevel@tonic-gate # define TLS_UNR(bit, req)	(bitset(bit, req) ? SFF_NORFILES : 0)
4540Sstevel@tonic-gate # define TLS_OUNR(bit, req)	(bitset(bit, req) ? SFF_NOWRFILES : 0)
4550Sstevel@tonic-gate # define TLS_KEYSFF(req)	\
4560Sstevel@tonic-gate 	(bitnset(DBS_GROUPREADABLEKEYFILE, DontBlameSendmail) ?	\
4570Sstevel@tonic-gate 		TLS_OUNR(TLS_I_KEY_OUNR, req) :			\
4580Sstevel@tonic-gate 		TLS_UNR(TLS_I_KEY_UNR, req))
4590Sstevel@tonic-gate 
4600Sstevel@tonic-gate /*
4610Sstevel@tonic-gate **  TLS_SAFE_F -- macro to simplify calls to tls_safe_f
4620Sstevel@tonic-gate **
4630Sstevel@tonic-gate **	Parameters:
4640Sstevel@tonic-gate **		var -- filename
4650Sstevel@tonic-gate **		sff -- flags for safefile()
4660Sstevel@tonic-gate **		req -- is the file required?
4670Sstevel@tonic-gate **		ex -- does the file exist?
4680Sstevel@tonic-gate **		st -- status bit to set if ok
4690Sstevel@tonic-gate **		srv -- server side?
4700Sstevel@tonic-gate **
4710Sstevel@tonic-gate **	Side Effects:
4720Sstevel@tonic-gate **		uses r, ok, ex; may change ok and status.
4730Sstevel@tonic-gate **
4740Sstevel@tonic-gate */
4750Sstevel@tonic-gate 
4760Sstevel@tonic-gate # define TLS_SAFE_F(var, sff, req, ex, st, srv) if (ex && ok) \
4770Sstevel@tonic-gate 	{ \
4780Sstevel@tonic-gate 		r = tls_safe_f(var, sff, srv); \
4790Sstevel@tonic-gate 		if (r) \
4800Sstevel@tonic-gate 			status |= st;	\
4810Sstevel@tonic-gate 		else if (req) \
4820Sstevel@tonic-gate 			ok = false;	\
4830Sstevel@tonic-gate 	}
4840Sstevel@tonic-gate 
4850Sstevel@tonic-gate /*
4860Sstevel@tonic-gate **  INITTLS -- initialize TLS
4870Sstevel@tonic-gate **
4880Sstevel@tonic-gate **	Parameters:
4890Sstevel@tonic-gate **		ctx -- pointer to context
4900Sstevel@tonic-gate **		req -- requirements for initialization (see sendmail.h)
4910Sstevel@tonic-gate **		srv -- server side?
4920Sstevel@tonic-gate **		certfile -- filename of certificate
4930Sstevel@tonic-gate **		keyfile -- filename of private key
4940Sstevel@tonic-gate **		cacertpath -- path to CAs
4950Sstevel@tonic-gate **		cacertfile -- file with CA(s)
4960Sstevel@tonic-gate **		dhparam -- parameters for DH
4970Sstevel@tonic-gate **
4980Sstevel@tonic-gate **	Returns:
4990Sstevel@tonic-gate **		succeeded?
5000Sstevel@tonic-gate */
5010Sstevel@tonic-gate 
502*1658Sjbeck /*
503*1658Sjbeck **  The session_id_context identifies the service that created a session.
504*1658Sjbeck **  This information is used to distinguish between multiple TLS-based
505*1658Sjbeck **  servers running on the same server. We use the name of the mail system.
506*1658Sjbeck **  Note: the session cache is not persistent.
507*1658Sjbeck */
508*1658Sjbeck 
509*1658Sjbeck static char server_session_id_context[] = "sendmail8";
510*1658Sjbeck 
5110Sstevel@tonic-gate bool
5120Sstevel@tonic-gate inittls(ctx, req, srv, certfile, keyfile, cacertpath, cacertfile, dhparam)
5130Sstevel@tonic-gate 	SSL_CTX **ctx;
5140Sstevel@tonic-gate 	unsigned long req;
5150Sstevel@tonic-gate 	bool srv;
5160Sstevel@tonic-gate 	char *certfile, *keyfile, *cacertpath, *cacertfile, *dhparam;
5170Sstevel@tonic-gate {
5180Sstevel@tonic-gate # if !NO_DH
5190Sstevel@tonic-gate 	static DH *dh = NULL;
5200Sstevel@tonic-gate # endif /* !NO_DH */
5210Sstevel@tonic-gate 	int r;
5220Sstevel@tonic-gate 	bool ok;
5230Sstevel@tonic-gate 	long sff, status;
5240Sstevel@tonic-gate 	char *who;
5250Sstevel@tonic-gate # if _FFR_TLS_1
5260Sstevel@tonic-gate 	char *cf2, *kf2;
5270Sstevel@tonic-gate # endif /* _FFR_TLS_1 */
5280Sstevel@tonic-gate #  if SM_CONF_SHM
5290Sstevel@tonic-gate 	extern int ShmId;
5300Sstevel@tonic-gate #  endif /* SM_CONF_SHM */
5310Sstevel@tonic-gate # if OPENSSL_VERSION_NUMBER > 0x00907000L
5320Sstevel@tonic-gate 	BIO *crl_file;
5330Sstevel@tonic-gate 	X509_CRL *crl;
5340Sstevel@tonic-gate 	X509_STORE *store;
5350Sstevel@tonic-gate # endif /* OPENSSL_VERSION_NUMBER > 0x00907000L */
5360Sstevel@tonic-gate 
5370Sstevel@tonic-gate 	status = TLS_S_NONE;
5380Sstevel@tonic-gate 	who = srv ? "server" : "client";
5390Sstevel@tonic-gate 	if (ctx == NULL)
5400Sstevel@tonic-gate 		syserr("STARTTLS=%s, inittls: ctx == NULL", who);
5410Sstevel@tonic-gate 
5420Sstevel@tonic-gate 	/* already initialized? (we could re-init...) */
5430Sstevel@tonic-gate 	if (*ctx != NULL)
5440Sstevel@tonic-gate 		return true;
5450Sstevel@tonic-gate 	ok = true;
5460Sstevel@tonic-gate 
5470Sstevel@tonic-gate # if _FFR_TLS_1
5480Sstevel@tonic-gate 	/*
5490Sstevel@tonic-gate 	**  look for a second filename: it must be separated by a ','
5500Sstevel@tonic-gate 	**  no blanks allowed (they won't be skipped).
5510Sstevel@tonic-gate 	**  we change a global variable here! this change will be undone
5520Sstevel@tonic-gate 	**  before return from the function but only if it returns true.
5530Sstevel@tonic-gate 	**  this isn't a problem since in a failure case this function
5540Sstevel@tonic-gate 	**  won't be called again with the same (overwritten) values.
5550Sstevel@tonic-gate 	**  otherwise each return must be replaced with a goto endinittls.
5560Sstevel@tonic-gate 	*/
5570Sstevel@tonic-gate 
5580Sstevel@tonic-gate 	cf2 = NULL;
5590Sstevel@tonic-gate 	kf2 = NULL;
5600Sstevel@tonic-gate 	if (certfile != NULL && (cf2 = strchr(certfile, ',')) != NULL)
5610Sstevel@tonic-gate 	{
5620Sstevel@tonic-gate 		*cf2++ = '\0';
5630Sstevel@tonic-gate 		if (keyfile != NULL && (kf2 = strchr(keyfile, ',')) != NULL)
5640Sstevel@tonic-gate 			*kf2++ = '\0';
5650Sstevel@tonic-gate 	}
5660Sstevel@tonic-gate # endif /* _FFR_TLS_1 */
5670Sstevel@tonic-gate 
5680Sstevel@tonic-gate 	/*
5690Sstevel@tonic-gate 	**  Check whether files/paths are defined
5700Sstevel@tonic-gate 	*/
5710Sstevel@tonic-gate 
5720Sstevel@tonic-gate 	TLS_OK_F(certfile, "CertFile", bitset(TLS_I_CERT_EX, req),
5730Sstevel@tonic-gate 		 TLS_S_CERT_EX, srv ? TLS_T_SRV : TLS_T_CLT);
5740Sstevel@tonic-gate 	TLS_OK_F(keyfile, "KeyFile", bitset(TLS_I_KEY_EX, req),
5750Sstevel@tonic-gate 		 TLS_S_KEY_EX, srv ? TLS_T_SRV : TLS_T_CLT);
5760Sstevel@tonic-gate 	TLS_OK_F(cacertpath, "CACertPath", bitset(TLS_I_CERTP_EX, req),
5770Sstevel@tonic-gate 		 TLS_S_CERTP_EX, TLS_T_OTHER);
5780Sstevel@tonic-gate 	TLS_OK_F(cacertfile, "CACertFile", bitset(TLS_I_CERTF_EX, req),
5790Sstevel@tonic-gate 		 TLS_S_CERTF_EX, TLS_T_OTHER);
5800Sstevel@tonic-gate 
5810Sstevel@tonic-gate # if OPENSSL_VERSION_NUMBER > 0x00907000L
5820Sstevel@tonic-gate 	TLS_OK_F(CRLFile, "CRLFile", bitset(TLS_I_CRLF_EX, req),
5830Sstevel@tonic-gate 		 TLS_S_CRLF_EX, TLS_T_OTHER);
5840Sstevel@tonic-gate # endif /* OPENSSL_VERSION_NUMBER > 0x00907000L */
5850Sstevel@tonic-gate 
5860Sstevel@tonic-gate # if _FFR_TLS_1
5870Sstevel@tonic-gate 	/*
5880Sstevel@tonic-gate 	**  if the second file is specified it must exist
5890Sstevel@tonic-gate 	**  XXX: it is possible here to define only one of those files
5900Sstevel@tonic-gate 	*/
5910Sstevel@tonic-gate 
5920Sstevel@tonic-gate 	if (cf2 != NULL)
5930Sstevel@tonic-gate 	{
5940Sstevel@tonic-gate 		TLS_OK_F(cf2, "CertFile", bitset(TLS_I_CERT_EX, req),
5950Sstevel@tonic-gate 			 TLS_S_CERT2_EX, srv ? TLS_T_SRV : TLS_T_CLT);
5960Sstevel@tonic-gate 	}
5970Sstevel@tonic-gate 	if (kf2 != NULL)
5980Sstevel@tonic-gate 	{
5990Sstevel@tonic-gate 		TLS_OK_F(kf2, "KeyFile", bitset(TLS_I_KEY_EX, req),
6000Sstevel@tonic-gate 			 TLS_S_KEY2_EX, srv ? TLS_T_SRV : TLS_T_CLT);
6010Sstevel@tonic-gate 	}
6020Sstevel@tonic-gate # endif /* _FFR_TLS_1 */
6030Sstevel@tonic-gate 
6040Sstevel@tonic-gate 	/*
6050Sstevel@tonic-gate 	**  valid values for dhparam are (only the first char is checked)
6060Sstevel@tonic-gate 	**  none	no parameters: don't use DH
6070Sstevel@tonic-gate 	**  512		generate 512 bit parameters (fixed)
6080Sstevel@tonic-gate 	**  1024	generate 1024 bit parameters
6090Sstevel@tonic-gate 	**  /file/name	read parameters from /file/name
6100Sstevel@tonic-gate 	**  default is: 1024 for server, 512 for client (OK? XXX)
6110Sstevel@tonic-gate 	*/
6120Sstevel@tonic-gate 
6130Sstevel@tonic-gate 	if (bitset(TLS_I_TRY_DH, req))
6140Sstevel@tonic-gate 	{
6150Sstevel@tonic-gate 		if (dhparam != NULL)
6160Sstevel@tonic-gate 		{
6170Sstevel@tonic-gate 			char c = *dhparam;
6180Sstevel@tonic-gate 
6190Sstevel@tonic-gate 			if (c == '1')
6200Sstevel@tonic-gate 				req |= TLS_I_DH1024;
6210Sstevel@tonic-gate 			else if (c == '5')
6220Sstevel@tonic-gate 				req |= TLS_I_DH512;
6230Sstevel@tonic-gate 			else if (c != 'n' && c != 'N' && c != '/')
6240Sstevel@tonic-gate 			{
6250Sstevel@tonic-gate 				if (LogLevel > 12)
6260Sstevel@tonic-gate 					sm_syslog(LOG_WARNING, NOQID,
6270Sstevel@tonic-gate 						  "STARTTLS=%s, error: illegal value '%s' for DHParam",
6280Sstevel@tonic-gate 						  who, dhparam);
6290Sstevel@tonic-gate 				dhparam = NULL;
6300Sstevel@tonic-gate 			}
6310Sstevel@tonic-gate 		}
6320Sstevel@tonic-gate 		if (dhparam == NULL)
6330Sstevel@tonic-gate 			dhparam = srv ? "1" : "5";
6340Sstevel@tonic-gate 		else if (*dhparam == '/')
6350Sstevel@tonic-gate 		{
6360Sstevel@tonic-gate 			TLS_OK_F(dhparam, "DHParameters",
6370Sstevel@tonic-gate 				 bitset(TLS_I_DHPAR_EX, req),
6380Sstevel@tonic-gate 				 TLS_S_DHPAR_EX, TLS_T_OTHER);
6390Sstevel@tonic-gate 		}
6400Sstevel@tonic-gate 	}
6410Sstevel@tonic-gate 	if (!ok)
6420Sstevel@tonic-gate 		return ok;
6430Sstevel@tonic-gate 
6440Sstevel@tonic-gate 	/* certfile etc. must be "safe". */
6450Sstevel@tonic-gate 	sff = SFF_REGONLY | SFF_SAFEDIRPATH | SFF_NOWLINK
6460Sstevel@tonic-gate 	     | SFF_NOGWFILES | SFF_NOWWFILES
6470Sstevel@tonic-gate 	     | SFF_MUSTOWN | SFF_ROOTOK | SFF_OPENASROOT;
6480Sstevel@tonic-gate 	if (DontLockReadFiles)
6490Sstevel@tonic-gate 		sff |= SFF_NOLOCK;
6500Sstevel@tonic-gate 
6510Sstevel@tonic-gate 	TLS_SAFE_F(certfile, sff | TLS_UNR(TLS_I_CERT_UNR, req),
6520Sstevel@tonic-gate 		   bitset(TLS_I_CERT_EX, req),
6530Sstevel@tonic-gate 		   bitset(TLS_S_CERT_EX, status), TLS_S_CERT_OK, srv);
6540Sstevel@tonic-gate 	TLS_SAFE_F(keyfile, sff | TLS_KEYSFF(req),
6550Sstevel@tonic-gate 		   bitset(TLS_I_KEY_EX, req),
6560Sstevel@tonic-gate 		   bitset(TLS_S_KEY_EX, status), TLS_S_KEY_OK, srv);
6570Sstevel@tonic-gate 	TLS_SAFE_F(cacertfile, sff | TLS_UNR(TLS_I_CERTF_UNR, req),
6580Sstevel@tonic-gate 		   bitset(TLS_I_CERTF_EX, req),
6590Sstevel@tonic-gate 		   bitset(TLS_S_CERTF_EX, status), TLS_S_CERTF_OK, srv);
6600Sstevel@tonic-gate 	TLS_SAFE_F(dhparam, sff | TLS_UNR(TLS_I_DHPAR_UNR, req),
6610Sstevel@tonic-gate 		   bitset(TLS_I_DHPAR_EX, req),
6620Sstevel@tonic-gate 		   bitset(TLS_S_DHPAR_EX, status), TLS_S_DHPAR_OK, srv);
6630Sstevel@tonic-gate # if OPENSSL_VERSION_NUMBER > 0x00907000L
6640Sstevel@tonic-gate 	TLS_SAFE_F(CRLFile, sff | TLS_UNR(TLS_I_CRLF_UNR, req),
6650Sstevel@tonic-gate 		   bitset(TLS_I_CRLF_EX, req),
6660Sstevel@tonic-gate 		   bitset(TLS_S_CRLF_EX, status), TLS_S_CRLF_OK, srv);
6670Sstevel@tonic-gate # endif /* OPENSSL_VERSION_NUMBER > 0x00907000L */
6680Sstevel@tonic-gate 	if (!ok)
6690Sstevel@tonic-gate 		return ok;
6700Sstevel@tonic-gate # if _FFR_TLS_1
6710Sstevel@tonic-gate 	if (cf2 != NULL)
6720Sstevel@tonic-gate 	{
6730Sstevel@tonic-gate 		TLS_SAFE_F(cf2, sff | TLS_UNR(TLS_I_CERT_UNR, req),
6740Sstevel@tonic-gate 			   bitset(TLS_I_CERT_EX, req),
6750Sstevel@tonic-gate 			   bitset(TLS_S_CERT2_EX, status), TLS_S_CERT2_OK, srv);
6760Sstevel@tonic-gate 	}
6770Sstevel@tonic-gate 	if (kf2 != NULL)
6780Sstevel@tonic-gate 	{
6790Sstevel@tonic-gate 		TLS_SAFE_F(kf2, sff | TLS_KEYSFF(req),
6800Sstevel@tonic-gate 			   bitset(TLS_I_KEY_EX, req),
6810Sstevel@tonic-gate 			   bitset(TLS_S_KEY2_EX, status), TLS_S_KEY2_OK, srv);
6820Sstevel@tonic-gate 	}
6830Sstevel@tonic-gate # endif /* _FFR_TLS_1 */
6840Sstevel@tonic-gate 
6850Sstevel@tonic-gate 	/* create a method and a new context */
6860Sstevel@tonic-gate 	if ((*ctx = SSL_CTX_new(srv ? SSLv23_server_method() :
6870Sstevel@tonic-gate 				      SSLv23_client_method())) == NULL)
6880Sstevel@tonic-gate 	{
6890Sstevel@tonic-gate 		if (LogLevel > 7)
6900Sstevel@tonic-gate 			sm_syslog(LOG_WARNING, NOQID,
6910Sstevel@tonic-gate 				  "STARTTLS=%s, error: SSL_CTX_new(SSLv23_%s_method()) failed",
6920Sstevel@tonic-gate 				  who, who);
6930Sstevel@tonic-gate 		if (LogLevel > 9)
6940Sstevel@tonic-gate 			tlslogerr(who);
6950Sstevel@tonic-gate 		return false;
6960Sstevel@tonic-gate 	}
6970Sstevel@tonic-gate 
6980Sstevel@tonic-gate # if OPENSSL_VERSION_NUMBER > 0x00907000L
6990Sstevel@tonic-gate 	if (CRLFile != NULL)
7000Sstevel@tonic-gate 	{
7010Sstevel@tonic-gate 		/* get a pointer to the current certificate validation store */
7020Sstevel@tonic-gate 		store = SSL_CTX_get_cert_store(*ctx);	/* does not fail */
7030Sstevel@tonic-gate 		crl_file = BIO_new(BIO_s_file_internal());
7040Sstevel@tonic-gate 		if (crl_file != NULL)
7050Sstevel@tonic-gate 		{
7060Sstevel@tonic-gate 			if (BIO_read_filename(crl_file, CRLFile) >= 0)
7070Sstevel@tonic-gate 			{
7080Sstevel@tonic-gate 				crl = PEM_read_bio_X509_CRL(crl_file, NULL,
7090Sstevel@tonic-gate 							NULL, NULL);
7100Sstevel@tonic-gate 				BIO_free(crl_file);
7110Sstevel@tonic-gate 				X509_STORE_add_crl(store, crl);
7120Sstevel@tonic-gate 				X509_CRL_free(crl);
7130Sstevel@tonic-gate 				X509_STORE_set_flags(store,
7140Sstevel@tonic-gate 					X509_V_FLAG_CRL_CHECK|X509_V_FLAG_CRL_CHECK_ALL);
7150Sstevel@tonic-gate 				X509_STORE_set_verify_cb_func(store,
7160Sstevel@tonic-gate 						x509_verify_cb);
7170Sstevel@tonic-gate 			}
7180Sstevel@tonic-gate 			else
7190Sstevel@tonic-gate 			{
7200Sstevel@tonic-gate 				if (LogLevel > 9)
7210Sstevel@tonic-gate 				{
7220Sstevel@tonic-gate 					sm_syslog(LOG_WARNING, NOQID,
7230Sstevel@tonic-gate 						  "STARTTLS=%s, error: PEM_read_bio_X509_CRL(%s)=failed",
7240Sstevel@tonic-gate 						  who, CRLFile);
7250Sstevel@tonic-gate 				}
7260Sstevel@tonic-gate 
7270Sstevel@tonic-gate 				/* avoid memory leaks */
7280Sstevel@tonic-gate 				BIO_free(crl_file);
7290Sstevel@tonic-gate 				return false;
7300Sstevel@tonic-gate 			}
7310Sstevel@tonic-gate 
7320Sstevel@tonic-gate 		}
7330Sstevel@tonic-gate 		else if (LogLevel > 9)
7340Sstevel@tonic-gate 			sm_syslog(LOG_WARNING, NOQID,
7350Sstevel@tonic-gate 				  "STARTTLS=%s, error: BIO_new=failed", who);
7360Sstevel@tonic-gate 	}
7370Sstevel@tonic-gate #  if _FFR_CRLPATH
7380Sstevel@tonic-gate 	if (CRLPath != NULL)
7390Sstevel@tonic-gate 	{
7400Sstevel@tonic-gate 		X509_LOOKUP *lookup;
7410Sstevel@tonic-gate 
7420Sstevel@tonic-gate 		lookup = X509_STORE_add_lookup(store, X509_LOOKUP_hash_dir());
7430Sstevel@tonic-gate 		if (lookup == NULL)
7440Sstevel@tonic-gate 		{
7450Sstevel@tonic-gate 			if (LogLevel > 9)
7460Sstevel@tonic-gate 			{
7470Sstevel@tonic-gate 				sm_syslog(LOG_WARNING, NOQID,
7480Sstevel@tonic-gate 					  "STARTTLS=%s, error: X509_STORE_add_lookup(hash)=failed",
7490Sstevel@tonic-gate 					  who, CRLFile);
7500Sstevel@tonic-gate 			}
7510Sstevel@tonic-gate 			return false;
7520Sstevel@tonic-gate 		}
7530Sstevel@tonic-gate 		X509_LOOKUP_add_dir(lookup, CRLPath, X509_FILETYPE_PEM);
7540Sstevel@tonic-gate 		X509_STORE_set_flags(store,
7550Sstevel@tonic-gate 			X509_V_FLAG_CRL_CHECK|X509_V_FLAG_CRL_CHECK_ALL);
7560Sstevel@tonic-gate 	}
7570Sstevel@tonic-gate #  endif /* _FFR_CRLPATH */
7580Sstevel@tonic-gate # endif /* OPENSSL_VERSION_NUMBER > 0x00907000L */
7590Sstevel@tonic-gate 
7600Sstevel@tonic-gate # if TLS_NO_RSA
7610Sstevel@tonic-gate 	/* turn off backward compatibility, required for no-rsa */
7620Sstevel@tonic-gate 	SSL_CTX_set_options(*ctx, SSL_OP_NO_SSLv2);
7630Sstevel@tonic-gate # endif /* TLS_NO_RSA */
7640Sstevel@tonic-gate 
7650Sstevel@tonic-gate 
7660Sstevel@tonic-gate # if !TLS_NO_RSA
7670Sstevel@tonic-gate 	/*
7680Sstevel@tonic-gate 	**  Create a temporary RSA key
7690Sstevel@tonic-gate 	**  XXX  Maybe we shouldn't create this always (even though it
7700Sstevel@tonic-gate 	**  is only at startup).
7710Sstevel@tonic-gate 	**  It is a time-consuming operation and it is not always necessary.
7720Sstevel@tonic-gate 	**  maybe we should do it only on demand...
7730Sstevel@tonic-gate 	*/
7740Sstevel@tonic-gate 
7750Sstevel@tonic-gate 	if (bitset(TLS_I_RSA_TMP, req)
7760Sstevel@tonic-gate #   if SM_CONF_SHM
7770Sstevel@tonic-gate 	    && ShmId != SM_SHM_NO_ID &&
7780Sstevel@tonic-gate 	    (rsa_tmp = RSA_generate_key(RSA_KEYLENGTH, RSA_F4, NULL,
7790Sstevel@tonic-gate 					NULL)) == NULL
7800Sstevel@tonic-gate #   else /* SM_CONF_SHM */
7810Sstevel@tonic-gate 	    && 0	/* no shared memory: no need to generate key now */
7820Sstevel@tonic-gate #   endif /* SM_CONF_SHM */
7830Sstevel@tonic-gate 	   )
7840Sstevel@tonic-gate 	{
7850Sstevel@tonic-gate 		if (LogLevel > 7)
7860Sstevel@tonic-gate 		{
7870Sstevel@tonic-gate 			sm_syslog(LOG_WARNING, NOQID,
7880Sstevel@tonic-gate 				  "STARTTLS=%s, error: RSA_generate_key failed",
7890Sstevel@tonic-gate 				  who);
7900Sstevel@tonic-gate 			if (LogLevel > 9)
7910Sstevel@tonic-gate 				tlslogerr(who);
7920Sstevel@tonic-gate 		}
7930Sstevel@tonic-gate 		return false;
7940Sstevel@tonic-gate 	}
7950Sstevel@tonic-gate # endif /* !TLS_NO_RSA */
7960Sstevel@tonic-gate 
7970Sstevel@tonic-gate 	/*
7980Sstevel@tonic-gate 	**  load private key
7990Sstevel@tonic-gate 	**  XXX change this for DSA-only version
8000Sstevel@tonic-gate 	*/
8010Sstevel@tonic-gate 
8020Sstevel@tonic-gate 	if (bitset(TLS_S_KEY_OK, status) &&
8030Sstevel@tonic-gate 	    SSL_CTX_use_PrivateKey_file(*ctx, keyfile,
8040Sstevel@tonic-gate 					 SSL_FILETYPE_PEM) <= 0)
8050Sstevel@tonic-gate 	{
8060Sstevel@tonic-gate 		if (LogLevel > 7)
8070Sstevel@tonic-gate 		{
8080Sstevel@tonic-gate 			sm_syslog(LOG_WARNING, NOQID,
8090Sstevel@tonic-gate 				  "STARTTLS=%s, error: SSL_CTX_use_PrivateKey_file(%s) failed",
8100Sstevel@tonic-gate 				  who, keyfile);
8110Sstevel@tonic-gate 			if (LogLevel > 9)
8120Sstevel@tonic-gate 				tlslogerr(who);
8130Sstevel@tonic-gate 		}
8140Sstevel@tonic-gate 		if (bitset(TLS_I_USE_KEY, req))
8150Sstevel@tonic-gate 			return false;
8160Sstevel@tonic-gate 	}
8170Sstevel@tonic-gate 
8180Sstevel@tonic-gate 	/* get the certificate file */
8190Sstevel@tonic-gate 	if (bitset(TLS_S_CERT_OK, status) &&
8200Sstevel@tonic-gate 	    SSL_CTX_use_certificate_file(*ctx, certfile,
8210Sstevel@tonic-gate 					 SSL_FILETYPE_PEM) <= 0)
8220Sstevel@tonic-gate 	{
8230Sstevel@tonic-gate 		if (LogLevel > 7)
8240Sstevel@tonic-gate 		{
8250Sstevel@tonic-gate 			sm_syslog(LOG_WARNING, NOQID,
8260Sstevel@tonic-gate 				  "STARTTLS=%s, error: SSL_CTX_use_certificate_file(%s) failed",
8270Sstevel@tonic-gate 				  who, certfile);
8280Sstevel@tonic-gate 			if (LogLevel > 9)
8290Sstevel@tonic-gate 				tlslogerr(who);
8300Sstevel@tonic-gate 		}
8310Sstevel@tonic-gate 		if (bitset(TLS_I_USE_CERT, req))
8320Sstevel@tonic-gate 			return false;
8330Sstevel@tonic-gate 	}
8340Sstevel@tonic-gate 
8350Sstevel@tonic-gate 	/* check the private key */
8360Sstevel@tonic-gate 	if (bitset(TLS_S_KEY_OK, status) &&
8370Sstevel@tonic-gate 	    (r = SSL_CTX_check_private_key(*ctx)) <= 0)
8380Sstevel@tonic-gate 	{
8390Sstevel@tonic-gate 		/* Private key does not match the certificate public key */
8400Sstevel@tonic-gate 		if (LogLevel > 5)
8410Sstevel@tonic-gate 		{
8420Sstevel@tonic-gate 			sm_syslog(LOG_WARNING, NOQID,
8430Sstevel@tonic-gate 				  "STARTTLS=%s, error: SSL_CTX_check_private_key failed(%s): %d",
8440Sstevel@tonic-gate 				  who, keyfile, r);
8450Sstevel@tonic-gate 			if (LogLevel > 9)
8460Sstevel@tonic-gate 				tlslogerr(who);
8470Sstevel@tonic-gate 		}
8480Sstevel@tonic-gate 		if (bitset(TLS_I_USE_KEY, req))
8490Sstevel@tonic-gate 			return false;
8500Sstevel@tonic-gate 	}
8510Sstevel@tonic-gate 
8520Sstevel@tonic-gate # if _FFR_TLS_1
8530Sstevel@tonic-gate 	/* XXX this code is pretty much duplicated from above! */
8540Sstevel@tonic-gate 
8550Sstevel@tonic-gate 	/* load private key */
8560Sstevel@tonic-gate 	if (bitset(TLS_S_KEY2_OK, status) &&
8570Sstevel@tonic-gate 	    SSL_CTX_use_PrivateKey_file(*ctx, kf2, SSL_FILETYPE_PEM) <= 0)
8580Sstevel@tonic-gate 	{
8590Sstevel@tonic-gate 		if (LogLevel > 7)
8600Sstevel@tonic-gate 		{
8610Sstevel@tonic-gate 			sm_syslog(LOG_WARNING, NOQID,
8620Sstevel@tonic-gate 				  "STARTTLS=%s, error: SSL_CTX_use_PrivateKey_file(%s) failed",
8630Sstevel@tonic-gate 				  who, kf2);
8640Sstevel@tonic-gate 			if (LogLevel > 9)
8650Sstevel@tonic-gate 				tlslogerr(who);
8660Sstevel@tonic-gate 		}
8670Sstevel@tonic-gate 	}
8680Sstevel@tonic-gate 
8690Sstevel@tonic-gate 	/* get the certificate file */
8700Sstevel@tonic-gate 	if (bitset(TLS_S_CERT2_OK, status) &&
8710Sstevel@tonic-gate 	    SSL_CTX_use_certificate_file(*ctx, cf2, SSL_FILETYPE_PEM) <= 0)
8720Sstevel@tonic-gate 	{
8730Sstevel@tonic-gate 		if (LogLevel > 7)
8740Sstevel@tonic-gate 		{
8750Sstevel@tonic-gate 			sm_syslog(LOG_WARNING, NOQID,
8760Sstevel@tonic-gate 				  "STARTTLS=%s, error: SSL_CTX_use_certificate_file(%s) failed",
8770Sstevel@tonic-gate 				  who, cf2);
8780Sstevel@tonic-gate 			if (LogLevel > 9)
8790Sstevel@tonic-gate 				tlslogerr(who);
8800Sstevel@tonic-gate 		}
8810Sstevel@tonic-gate 	}
8820Sstevel@tonic-gate 
8830Sstevel@tonic-gate 	/* also check the private key */
8840Sstevel@tonic-gate 	if (bitset(TLS_S_KEY2_OK, status) &&
8850Sstevel@tonic-gate 	    (r = SSL_CTX_check_private_key(*ctx)) <= 0)
8860Sstevel@tonic-gate 	{
8870Sstevel@tonic-gate 		/* Private key does not match the certificate public key */
8880Sstevel@tonic-gate 		if (LogLevel > 5)
8890Sstevel@tonic-gate 		{
8900Sstevel@tonic-gate 			sm_syslog(LOG_WARNING, NOQID,
8910Sstevel@tonic-gate 				  "STARTTLS=%s, error: SSL_CTX_check_private_key 2 failed: %d",
8920Sstevel@tonic-gate 				  who, r);
8930Sstevel@tonic-gate 			if (LogLevel > 9)
8940Sstevel@tonic-gate 				tlslogerr(who);
8950Sstevel@tonic-gate 		}
8960Sstevel@tonic-gate 	}
8970Sstevel@tonic-gate # endif /* _FFR_TLS_1 */
8980Sstevel@tonic-gate 
8990Sstevel@tonic-gate 	/* SSL_CTX_set_quiet_shutdown(*ctx, 1); violation of standard? */
9000Sstevel@tonic-gate 	SSL_CTX_set_options(*ctx, SSL_OP_ALL);	/* XXX bug compatibility? */
9010Sstevel@tonic-gate 
9020Sstevel@tonic-gate # if !NO_DH
9030Sstevel@tonic-gate 	/* Diffie-Hellman initialization */
9040Sstevel@tonic-gate 	if (bitset(TLS_I_TRY_DH, req))
9050Sstevel@tonic-gate 	{
9060Sstevel@tonic-gate 		if (bitset(TLS_S_DHPAR_OK, status))
9070Sstevel@tonic-gate 		{
9080Sstevel@tonic-gate 			BIO *bio;
9090Sstevel@tonic-gate 
9100Sstevel@tonic-gate 			if ((bio = BIO_new_file(dhparam, "r")) != NULL)
9110Sstevel@tonic-gate 			{
9120Sstevel@tonic-gate 				dh = PEM_read_bio_DHparams(bio, NULL, NULL, NULL);
9130Sstevel@tonic-gate 				BIO_free(bio);
9140Sstevel@tonic-gate 				if (dh == NULL && LogLevel > 7)
9150Sstevel@tonic-gate 				{
9160Sstevel@tonic-gate 					unsigned long err;
9170Sstevel@tonic-gate 
9180Sstevel@tonic-gate 					err = ERR_get_error();
9190Sstevel@tonic-gate 					sm_syslog(LOG_WARNING, NOQID,
9200Sstevel@tonic-gate 						  "STARTTLS=%s, error: cannot read DH parameters(%s): %s",
9210Sstevel@tonic-gate 						  who, dhparam,
9220Sstevel@tonic-gate 						  ERR_error_string(err, NULL));
9230Sstevel@tonic-gate 					if (LogLevel > 9)
9240Sstevel@tonic-gate 						tlslogerr(who);
9250Sstevel@tonic-gate 				}
9260Sstevel@tonic-gate 			}
9270Sstevel@tonic-gate 			else
9280Sstevel@tonic-gate 			{
9290Sstevel@tonic-gate 				if (LogLevel > 5)
9300Sstevel@tonic-gate 				{
9310Sstevel@tonic-gate 					sm_syslog(LOG_WARNING, NOQID,
9320Sstevel@tonic-gate 						  "STARTTLS=%s, error: BIO_new_file(%s) failed",
9330Sstevel@tonic-gate 						  who, dhparam);
9340Sstevel@tonic-gate 					if (LogLevel > 9)
9350Sstevel@tonic-gate 						tlslogerr(who);
9360Sstevel@tonic-gate 				}
9370Sstevel@tonic-gate 			}
9380Sstevel@tonic-gate 		}
9390Sstevel@tonic-gate 		if (dh == NULL && bitset(TLS_I_DH1024, req))
9400Sstevel@tonic-gate 		{
9410Sstevel@tonic-gate 			DSA *dsa;
9420Sstevel@tonic-gate 
9430Sstevel@tonic-gate 			/* this takes a while! (7-130s on a 450MHz AMD K6-2) */
9440Sstevel@tonic-gate 			dsa = DSA_generate_parameters(1024, NULL, 0, NULL,
9450Sstevel@tonic-gate 						      NULL, 0, NULL);
9460Sstevel@tonic-gate 			dh = DSA_dup_DH(dsa);
9470Sstevel@tonic-gate 			DSA_free(dsa);
9480Sstevel@tonic-gate 		}
9490Sstevel@tonic-gate 		else
9500Sstevel@tonic-gate 		if (dh == NULL && bitset(TLS_I_DH512, req))
9510Sstevel@tonic-gate 			dh = get_dh512();
9520Sstevel@tonic-gate 
9530Sstevel@tonic-gate 		if (dh == NULL)
9540Sstevel@tonic-gate 		{
9550Sstevel@tonic-gate 			if (LogLevel > 9)
9560Sstevel@tonic-gate 			{
9570Sstevel@tonic-gate 				unsigned long err;
9580Sstevel@tonic-gate 
9590Sstevel@tonic-gate 				err = ERR_get_error();
9600Sstevel@tonic-gate 				sm_syslog(LOG_WARNING, NOQID,
9610Sstevel@tonic-gate 					  "STARTTLS=%s, error: cannot read or set DH parameters(%s): %s",
9620Sstevel@tonic-gate 					  who, dhparam,
9630Sstevel@tonic-gate 					  ERR_error_string(err, NULL));
9640Sstevel@tonic-gate 			}
9650Sstevel@tonic-gate 			if (bitset(TLS_I_REQ_DH, req))
9660Sstevel@tonic-gate 				return false;
9670Sstevel@tonic-gate 		}
9680Sstevel@tonic-gate 		else
9690Sstevel@tonic-gate 		{
9700Sstevel@tonic-gate 			SSL_CTX_set_tmp_dh(*ctx, dh);
9710Sstevel@tonic-gate 
9720Sstevel@tonic-gate 			/* important to avoid small subgroup attacks */
9730Sstevel@tonic-gate 			SSL_CTX_set_options(*ctx, SSL_OP_SINGLE_DH_USE);
9740Sstevel@tonic-gate 			if (LogLevel > 13)
9750Sstevel@tonic-gate 				sm_syslog(LOG_INFO, NOQID,
9760Sstevel@tonic-gate 					  "STARTTLS=%s, Diffie-Hellman init, key=%d bit (%c)",
9770Sstevel@tonic-gate 					  who, 8 * DH_size(dh), *dhparam);
9780Sstevel@tonic-gate 			DH_free(dh);
9790Sstevel@tonic-gate 		}
9800Sstevel@tonic-gate 	}
9810Sstevel@tonic-gate # endif /* !NO_DH */
9820Sstevel@tonic-gate 
9830Sstevel@tonic-gate 
9840Sstevel@tonic-gate 	/* XXX do we need this cache here? */
9850Sstevel@tonic-gate 	if (bitset(TLS_I_CACHE, req))
986*1658Sjbeck 	{
987*1658Sjbeck 		SSL_CTX_sess_set_cache_size(*ctx, 1);
988*1658Sjbeck 		SSL_CTX_set_timeout(*ctx, 1);
989*1658Sjbeck 		SSL_CTX_set_session_id_context(*ctx,
990*1658Sjbeck 			(void *) &server_session_id_context,
991*1658Sjbeck 			sizeof(server_session_id_context));
992*1658Sjbeck 		(void) SSL_CTX_set_session_cache_mode(*ctx,
993*1658Sjbeck 				SSL_SESS_CACHE_SERVER);
994*1658Sjbeck 	}
995*1658Sjbeck 	else
996*1658Sjbeck 	{
997*1658Sjbeck 		(void) SSL_CTX_set_session_cache_mode(*ctx,
998*1658Sjbeck 				SSL_SESS_CACHE_OFF);
999*1658Sjbeck 	}
10000Sstevel@tonic-gate 
10010Sstevel@tonic-gate 	/* load certificate locations and default CA paths */
10020Sstevel@tonic-gate 	if (bitset(TLS_S_CERTP_EX, status) && bitset(TLS_S_CERTF_EX, status))
10030Sstevel@tonic-gate 	{
10040Sstevel@tonic-gate 		if ((r = SSL_CTX_load_verify_locations(*ctx, cacertfile,
10050Sstevel@tonic-gate 						       cacertpath)) == 1)
10060Sstevel@tonic-gate 		{
10070Sstevel@tonic-gate # if !TLS_NO_RSA
10080Sstevel@tonic-gate 			if (bitset(TLS_I_RSA_TMP, req))
10090Sstevel@tonic-gate 				SSL_CTX_set_tmp_rsa_callback(*ctx, tmp_rsa_key);
10100Sstevel@tonic-gate # endif /* !TLS_NO_RSA */
10110Sstevel@tonic-gate 
10120Sstevel@tonic-gate 			/*
10130Sstevel@tonic-gate 			**  We have to install our own verify callback:
10140Sstevel@tonic-gate 			**  SSL_VERIFY_PEER requests a client cert but even
10150Sstevel@tonic-gate 			**  though *FAIL_IF* isn't set, the connection
10160Sstevel@tonic-gate 			**  will be aborted if the client presents a cert
10170Sstevel@tonic-gate 			**  that is not "liked" (can't be verified?) by
10180Sstevel@tonic-gate 			**  the TLS library :-(
10190Sstevel@tonic-gate 			*/
10200Sstevel@tonic-gate 
10210Sstevel@tonic-gate 			/*
10220Sstevel@tonic-gate 			**  XXX currently we could call tls_set_verify()
10230Sstevel@tonic-gate 			**  but we hope that that function will later on
10240Sstevel@tonic-gate 			**  only set the mode per connection.
10250Sstevel@tonic-gate 			*/
10260Sstevel@tonic-gate 			SSL_CTX_set_verify(*ctx,
10270Sstevel@tonic-gate 				bitset(TLS_I_NO_VRFY, req) ? SSL_VERIFY_NONE
10280Sstevel@tonic-gate 							   : SSL_VERIFY_PEER,
10290Sstevel@tonic-gate 				NULL);
10300Sstevel@tonic-gate 
10310Sstevel@tonic-gate 			/* install verify callback */
10320Sstevel@tonic-gate 			SSL_CTX_set_cert_verify_callback(*ctx, tls_verify_cb,
10330Sstevel@tonic-gate 							 NULL);
10340Sstevel@tonic-gate 			SSL_CTX_set_client_CA_list(*ctx,
10350Sstevel@tonic-gate 				SSL_load_client_CA_file(cacertfile));
10360Sstevel@tonic-gate 		}
10370Sstevel@tonic-gate 		else
10380Sstevel@tonic-gate 		{
10390Sstevel@tonic-gate 			/*
10400Sstevel@tonic-gate 			**  can't load CA data; do we care?
10410Sstevel@tonic-gate 			**  the data is necessary to authenticate the client,
10420Sstevel@tonic-gate 			**  which in turn would be necessary
10430Sstevel@tonic-gate 			**  if we want to allow relaying based on it.
10440Sstevel@tonic-gate 			*/
10450Sstevel@tonic-gate 			if (LogLevel > 5)
10460Sstevel@tonic-gate 			{
10470Sstevel@tonic-gate 				sm_syslog(LOG_WARNING, NOQID,
10480Sstevel@tonic-gate 					  "STARTTLS=%s, error: load verify locs %s, %s failed: %d",
10490Sstevel@tonic-gate 					  who, cacertpath, cacertfile, r);
10500Sstevel@tonic-gate 				if (LogLevel > 9)
10510Sstevel@tonic-gate 					tlslogerr(who);
10520Sstevel@tonic-gate 			}
10530Sstevel@tonic-gate 			if (bitset(TLS_I_VRFY_LOC, req))
10540Sstevel@tonic-gate 				return false;
10550Sstevel@tonic-gate 		}
10560Sstevel@tonic-gate 	}
10570Sstevel@tonic-gate 
10580Sstevel@tonic-gate 	/* XXX: make this dependent on an option? */
10590Sstevel@tonic-gate 	if (tTd(96, 9))
10600Sstevel@tonic-gate 		SSL_CTX_set_info_callback(*ctx, apps_ssl_info_cb);
10610Sstevel@tonic-gate 
10620Sstevel@tonic-gate # if _FFR_TLS_1
10630Sstevel@tonic-gate 	/* install our own cipher list */
10640Sstevel@tonic-gate 	if (CipherList != NULL && *CipherList != '\0')
10650Sstevel@tonic-gate 	{
10660Sstevel@tonic-gate 		if (SSL_CTX_set_cipher_list(*ctx, CipherList) <= 0)
10670Sstevel@tonic-gate 		{
10680Sstevel@tonic-gate 			if (LogLevel > 7)
10690Sstevel@tonic-gate 			{
10700Sstevel@tonic-gate 				sm_syslog(LOG_WARNING, NOQID,
10710Sstevel@tonic-gate 					  "STARTTLS=%s, error: SSL_CTX_set_cipher_list(%s) failed, list ignored",
10720Sstevel@tonic-gate 					  who, CipherList);
10730Sstevel@tonic-gate 
10740Sstevel@tonic-gate 				if (LogLevel > 9)
10750Sstevel@tonic-gate 					tlslogerr(who);
10760Sstevel@tonic-gate 			}
10770Sstevel@tonic-gate 			/* failure if setting to this list is required? */
10780Sstevel@tonic-gate 		}
10790Sstevel@tonic-gate 	}
10800Sstevel@tonic-gate # endif /* _FFR_TLS_1 */
10810Sstevel@tonic-gate 	if (LogLevel > 12)
10820Sstevel@tonic-gate 		sm_syslog(LOG_INFO, NOQID, "STARTTLS=%s, init=%d", who, ok);
10830Sstevel@tonic-gate 
10840Sstevel@tonic-gate # if _FFR_TLS_1
10850Sstevel@tonic-gate #  if 0
10860Sstevel@tonic-gate 	/*
10870Sstevel@tonic-gate 	**  this label is required if we want to have a "clean" exit
10880Sstevel@tonic-gate 	**  see the comments above at the initialization of cf2
10890Sstevel@tonic-gate 	*/
10900Sstevel@tonic-gate 
10910Sstevel@tonic-gate     endinittls:
10920Sstevel@tonic-gate #  endif /* 0 */
10930Sstevel@tonic-gate 
10940Sstevel@tonic-gate 	/* undo damage to global variables */
10950Sstevel@tonic-gate 	if (cf2 != NULL)
10960Sstevel@tonic-gate 		*--cf2 = ',';
10970Sstevel@tonic-gate 	if (kf2 != NULL)
10980Sstevel@tonic-gate 		*--kf2 = ',';
10990Sstevel@tonic-gate # endif /* _FFR_TLS_1 */
11000Sstevel@tonic-gate 
11010Sstevel@tonic-gate 	return ok;
11020Sstevel@tonic-gate }
11030Sstevel@tonic-gate /*
11040Sstevel@tonic-gate **  TLS_GET_INFO -- get information about TLS connection
11050Sstevel@tonic-gate **
11060Sstevel@tonic-gate **	Parameters:
11070Sstevel@tonic-gate **		ssl -- TLS connection structure
11080Sstevel@tonic-gate **		srv -- server or client
11090Sstevel@tonic-gate **		host -- hostname of other side
11100Sstevel@tonic-gate **		mac -- macro storage
11110Sstevel@tonic-gate **		certreq -- did we ask for a cert?
11120Sstevel@tonic-gate **
11130Sstevel@tonic-gate **	Returns:
11140Sstevel@tonic-gate **		result of authentication.
11150Sstevel@tonic-gate **
11160Sstevel@tonic-gate **	Side Effects:
11170Sstevel@tonic-gate **		sets macros: {cipher}, {tls_version}, {verify},
11180Sstevel@tonic-gate **		{cipher_bits}, {alg_bits}, {cert}, {cert_subject},
11190Sstevel@tonic-gate **		{cert_issuer}, {cn_subject}, {cn_issuer}
11200Sstevel@tonic-gate */
11210Sstevel@tonic-gate 
11220Sstevel@tonic-gate int
11230Sstevel@tonic-gate tls_get_info(ssl, srv, host, mac, certreq)
11240Sstevel@tonic-gate 	SSL *ssl;
11250Sstevel@tonic-gate 	bool srv;
11260Sstevel@tonic-gate 	char *host;
11270Sstevel@tonic-gate 	MACROS_T *mac;
11280Sstevel@tonic-gate 	bool certreq;
11290Sstevel@tonic-gate {
11300Sstevel@tonic-gate 	SSL_CIPHER *c;
11310Sstevel@tonic-gate 	int b, r;
11320Sstevel@tonic-gate 	long verifyok;
11330Sstevel@tonic-gate 	char *s, *who;
11340Sstevel@tonic-gate 	char bitstr[16];
11350Sstevel@tonic-gate 	X509 *cert;
11360Sstevel@tonic-gate 
11370Sstevel@tonic-gate 	c = SSL_get_current_cipher(ssl);
11380Sstevel@tonic-gate 
11390Sstevel@tonic-gate 	/* cast is just workaround for compiler warning */
11400Sstevel@tonic-gate 	macdefine(mac, A_TEMP, macid("{cipher}"),
11410Sstevel@tonic-gate 		  (char *) SSL_CIPHER_get_name(c));
11420Sstevel@tonic-gate 	b = SSL_CIPHER_get_bits(c, &r);
11430Sstevel@tonic-gate 	(void) sm_snprintf(bitstr, sizeof bitstr, "%d", b);
11440Sstevel@tonic-gate 	macdefine(mac, A_TEMP, macid("{cipher_bits}"), bitstr);
11450Sstevel@tonic-gate 	(void) sm_snprintf(bitstr, sizeof bitstr, "%d", r);
11460Sstevel@tonic-gate 	macdefine(mac, A_TEMP, macid("{alg_bits}"), bitstr);
11470Sstevel@tonic-gate 	s = SSL_CIPHER_get_version(c);
11480Sstevel@tonic-gate 	if (s == NULL)
11490Sstevel@tonic-gate 		s = "UNKNOWN";
11500Sstevel@tonic-gate 	macdefine(mac, A_TEMP, macid("{tls_version}"), s);
11510Sstevel@tonic-gate 
11520Sstevel@tonic-gate 	who = srv ? "server" : "client";
11530Sstevel@tonic-gate 	cert = SSL_get_peer_certificate(ssl);
11540Sstevel@tonic-gate 	verifyok = SSL_get_verify_result(ssl);
11550Sstevel@tonic-gate 	if (LogLevel > 14)
11560Sstevel@tonic-gate 		sm_syslog(LOG_INFO, NOQID,
11570Sstevel@tonic-gate 			  "STARTTLS=%s, get_verify: %ld get_peer: 0x%lx",
11580Sstevel@tonic-gate 			  who, verifyok, (unsigned long) cert);
11590Sstevel@tonic-gate 	if (cert != NULL)
11600Sstevel@tonic-gate 	{
11610Sstevel@tonic-gate 		unsigned int n;
11620Sstevel@tonic-gate 		unsigned char md[EVP_MAX_MD_SIZE];
11630Sstevel@tonic-gate 		char buf[MAXNAME];
11640Sstevel@tonic-gate 
11650Sstevel@tonic-gate 		X509_NAME_oneline(X509_get_subject_name(cert),
11660Sstevel@tonic-gate 				  buf, sizeof buf);
11670Sstevel@tonic-gate 		macdefine(mac, A_TEMP, macid("{cert_subject}"),
11680Sstevel@tonic-gate 			 xtextify(buf, "<>\")"));
11690Sstevel@tonic-gate 		X509_NAME_oneline(X509_get_issuer_name(cert),
11700Sstevel@tonic-gate 				  buf, sizeof buf);
11710Sstevel@tonic-gate 		macdefine(mac, A_TEMP, macid("{cert_issuer}"),
11720Sstevel@tonic-gate 			 xtextify(buf, "<>\")"));
11730Sstevel@tonic-gate 		X509_NAME_get_text_by_NID(X509_get_subject_name(cert),
11740Sstevel@tonic-gate 					  NID_commonName, buf, sizeof buf);
11750Sstevel@tonic-gate 		macdefine(mac, A_TEMP, macid("{cn_subject}"),
11760Sstevel@tonic-gate 			 xtextify(buf, "<>\")"));
11770Sstevel@tonic-gate 		X509_NAME_get_text_by_NID(X509_get_issuer_name(cert),
11780Sstevel@tonic-gate 					  NID_commonName, buf, sizeof buf);
11790Sstevel@tonic-gate 		macdefine(mac, A_TEMP, macid("{cn_issuer}"),
11800Sstevel@tonic-gate 			 xtextify(buf, "<>\")"));
11810Sstevel@tonic-gate 		n = 0;
11820Sstevel@tonic-gate 		if (X509_digest(cert, EVP_md5(), md, &n) != 0 && n > 0)
11830Sstevel@tonic-gate 		{
11840Sstevel@tonic-gate 			char md5h[EVP_MAX_MD_SIZE * 3];
11850Sstevel@tonic-gate 			static const char hexcodes[] = "0123456789ABCDEF";
11860Sstevel@tonic-gate 
11870Sstevel@tonic-gate 			SM_ASSERT((n * 3) + 2 < sizeof(md5h));
11880Sstevel@tonic-gate 			for (r = 0; r < (int) n; r++)
11890Sstevel@tonic-gate 			{
11900Sstevel@tonic-gate 				md5h[r * 3] = hexcodes[(md[r] & 0xf0) >> 4];
11910Sstevel@tonic-gate 				md5h[(r * 3) + 1] = hexcodes[(md[r] & 0x0f)];
11920Sstevel@tonic-gate 				md5h[(r * 3) + 2] = ':';
11930Sstevel@tonic-gate 			}
11940Sstevel@tonic-gate 			md5h[(n * 3) - 1] = '\0';
11950Sstevel@tonic-gate 			macdefine(mac, A_TEMP, macid("{cert_md5}"), md5h);
11960Sstevel@tonic-gate 		}
11970Sstevel@tonic-gate 		else
11980Sstevel@tonic-gate 			macdefine(mac, A_TEMP, macid("{cert_md5}"), "");
11990Sstevel@tonic-gate 	}
12000Sstevel@tonic-gate 	else
12010Sstevel@tonic-gate 	{
12020Sstevel@tonic-gate 		macdefine(mac, A_PERM, macid("{cert_subject}"), "");
12030Sstevel@tonic-gate 		macdefine(mac, A_PERM, macid("{cert_issuer}"), "");
12040Sstevel@tonic-gate 		macdefine(mac, A_PERM, macid("{cn_subject}"), "");
12050Sstevel@tonic-gate 		macdefine(mac, A_PERM, macid("{cn_issuer}"), "");
12060Sstevel@tonic-gate 		macdefine(mac, A_TEMP, macid("{cert_md5}"), "");
12070Sstevel@tonic-gate 	}
12080Sstevel@tonic-gate 	switch (verifyok)
12090Sstevel@tonic-gate 	{
12100Sstevel@tonic-gate 	  case X509_V_OK:
12110Sstevel@tonic-gate 		if (cert != NULL)
12120Sstevel@tonic-gate 		{
12130Sstevel@tonic-gate 			s = "OK";
12140Sstevel@tonic-gate 			r = TLS_AUTH_OK;
12150Sstevel@tonic-gate 		}
12160Sstevel@tonic-gate 		else
12170Sstevel@tonic-gate 		{
12180Sstevel@tonic-gate 			s = certreq ? "NO" : "NOT",
12190Sstevel@tonic-gate 			r = TLS_AUTH_NO;
12200Sstevel@tonic-gate 		}
12210Sstevel@tonic-gate 		break;
12220Sstevel@tonic-gate 	  default:
12230Sstevel@tonic-gate 		s = "FAIL";
12240Sstevel@tonic-gate 		r = TLS_AUTH_FAIL;
12250Sstevel@tonic-gate 		break;
12260Sstevel@tonic-gate 	}
12270Sstevel@tonic-gate 	macdefine(mac, A_PERM, macid("{verify}"), s);
12280Sstevel@tonic-gate 	if (cert != NULL)
12290Sstevel@tonic-gate 		X509_free(cert);
12300Sstevel@tonic-gate 
12310Sstevel@tonic-gate 	/* do some logging */
12320Sstevel@tonic-gate 	if (LogLevel > 8)
12330Sstevel@tonic-gate 	{
12340Sstevel@tonic-gate 		char *vers, *s1, *s2, *cbits, *algbits;
12350Sstevel@tonic-gate 
12360Sstevel@tonic-gate 		vers = macget(mac, macid("{tls_version}"));
12370Sstevel@tonic-gate 		cbits = macget(mac, macid("{cipher_bits}"));
12380Sstevel@tonic-gate 		algbits = macget(mac, macid("{alg_bits}"));
12390Sstevel@tonic-gate 		s1 = macget(mac, macid("{verify}"));
12400Sstevel@tonic-gate 		s2 = macget(mac, macid("{cipher}"));
12410Sstevel@tonic-gate 
12420Sstevel@tonic-gate 		/* XXX: maybe cut off ident info? */
12430Sstevel@tonic-gate 		sm_syslog(LOG_INFO, NOQID,
12440Sstevel@tonic-gate 			  "STARTTLS=%s, relay=%.100s, version=%.16s, verify=%.16s, cipher=%.64s, bits=%.6s/%.6s",
12450Sstevel@tonic-gate 			  who,
12460Sstevel@tonic-gate 			  host == NULL ? "local" : host,
12470Sstevel@tonic-gate 			  vers, s1, s2, /* sm_snprintf() can deal with NULL */
12480Sstevel@tonic-gate 			  algbits == NULL ? "0" : algbits,
12490Sstevel@tonic-gate 			  cbits == NULL ? "0" : cbits);
12500Sstevel@tonic-gate 		if (LogLevel > 11)
12510Sstevel@tonic-gate 		{
12520Sstevel@tonic-gate 			/*
12530Sstevel@tonic-gate 			**  Maybe run xuntextify on the strings?
12540Sstevel@tonic-gate 			**  That is easier to read but makes it maybe a bit
12550Sstevel@tonic-gate 			**  more complicated to figure out the right values
12560Sstevel@tonic-gate 			**  for the access map...
12570Sstevel@tonic-gate 			*/
12580Sstevel@tonic-gate 
12590Sstevel@tonic-gate 			s1 = macget(mac, macid("{cert_subject}"));
12600Sstevel@tonic-gate 			s2 = macget(mac, macid("{cert_issuer}"));
12610Sstevel@tonic-gate 			sm_syslog(LOG_INFO, NOQID,
12620Sstevel@tonic-gate 				  "STARTTLS=%s, cert-subject=%.256s, cert-issuer=%.256s, verifymsg=%s",
12630Sstevel@tonic-gate 				  who, s1, s2,
12640Sstevel@tonic-gate 				  X509_verify_cert_error_string(verifyok));
12650Sstevel@tonic-gate 		}
12660Sstevel@tonic-gate 	}
12670Sstevel@tonic-gate 	return r;
12680Sstevel@tonic-gate }
12690Sstevel@tonic-gate /*
12700Sstevel@tonic-gate **  ENDTLS -- shutdown secure connection
12710Sstevel@tonic-gate **
12720Sstevel@tonic-gate **	Parameters:
12730Sstevel@tonic-gate **		ssl -- SSL connection information.
12740Sstevel@tonic-gate **		side -- server/client (for logging).
12750Sstevel@tonic-gate **
12760Sstevel@tonic-gate **	Returns:
12770Sstevel@tonic-gate **		success? (EX_* code)
12780Sstevel@tonic-gate */
12790Sstevel@tonic-gate 
12800Sstevel@tonic-gate int
12810Sstevel@tonic-gate endtls(ssl, side)
12820Sstevel@tonic-gate 	SSL *ssl;
12830Sstevel@tonic-gate 	char *side;
12840Sstevel@tonic-gate {
12850Sstevel@tonic-gate 	int ret = EX_OK;
12860Sstevel@tonic-gate 
12870Sstevel@tonic-gate 	if (ssl != NULL)
12880Sstevel@tonic-gate 	{
12890Sstevel@tonic-gate 		int r;
12900Sstevel@tonic-gate 
12910Sstevel@tonic-gate 		if ((r = SSL_shutdown(ssl)) < 0)
12920Sstevel@tonic-gate 		{
12930Sstevel@tonic-gate 			if (LogLevel > 11)
12940Sstevel@tonic-gate 			{
12950Sstevel@tonic-gate 				sm_syslog(LOG_WARNING, NOQID,
12960Sstevel@tonic-gate 					  "STARTTLS=%s, SSL_shutdown failed: %d",
12970Sstevel@tonic-gate 					  side, r);
12980Sstevel@tonic-gate 				tlslogerr(side);
12990Sstevel@tonic-gate 			}
13000Sstevel@tonic-gate 			ret = EX_SOFTWARE;
13010Sstevel@tonic-gate 		}
13020Sstevel@tonic-gate # if !defined(OPENSSL_VERSION_NUMBER) || OPENSSL_VERSION_NUMBER > 0x0090602fL
13030Sstevel@tonic-gate 
13040Sstevel@tonic-gate 		/*
13050Sstevel@tonic-gate 		**  Bug in OpenSSL (at least up to 0.9.6b):
13060Sstevel@tonic-gate 		**  From: Lutz.Jaenicke@aet.TU-Cottbus.DE
13070Sstevel@tonic-gate 		**  Message-ID: <20010723152244.A13122@serv01.aet.tu-cottbus.de>
13080Sstevel@tonic-gate 		**  To: openssl-users@openssl.org
13090Sstevel@tonic-gate 		**  Subject: Re: SSL_shutdown() woes (fwd)
13100Sstevel@tonic-gate 		**
13110Sstevel@tonic-gate 		**  The side sending the shutdown alert first will
13120Sstevel@tonic-gate 		**  not care about the answer of the peer but will
13130Sstevel@tonic-gate 		**  immediately return with a return value of "0"
13140Sstevel@tonic-gate 		**  (ssl/s3_lib.c:ssl3_shutdown()). SSL_get_error will evaluate
13150Sstevel@tonic-gate 		**  the value of "0" and as the shutdown alert of the peer was
13160Sstevel@tonic-gate 		**  not received (actually, the program did not even wait for
13170Sstevel@tonic-gate 		**  the answer), an SSL_ERROR_SYSCALL is flagged, because this
13180Sstevel@tonic-gate 		**  is the default rule in case everything else does not apply.
13190Sstevel@tonic-gate 		**
13200Sstevel@tonic-gate 		**  For your server the problem is different, because it
13210Sstevel@tonic-gate 		**  receives the shutdown first (setting SSL_RECEIVED_SHUTDOWN),
13220Sstevel@tonic-gate 		**  then sends its response (SSL_SENT_SHUTDOWN), so for the
13230Sstevel@tonic-gate 		**  server the shutdown was successfull.
13240Sstevel@tonic-gate 		**
13250Sstevel@tonic-gate 		**  As is by know, you would have to call SSL_shutdown() once
13260Sstevel@tonic-gate 		**  and ignore an SSL_ERROR_SYSCALL returned. Then call
13270Sstevel@tonic-gate 		**  SSL_shutdown() again to actually get the server's response.
13280Sstevel@tonic-gate 		**
13290Sstevel@tonic-gate 		**  In the last discussion, Bodo Moeller concluded that a
13300Sstevel@tonic-gate 		**  rewrite of the shutdown code would be necessary, but
13310Sstevel@tonic-gate 		**  probably with another API, as the change would not be
13320Sstevel@tonic-gate 		**  compatible to the way it is now.  Things do not become
13330Sstevel@tonic-gate 		**  easier as other programs do not follow the shutdown
13340Sstevel@tonic-gate 		**  guidelines anyway, so that a lot error conditions and
13350Sstevel@tonic-gate 		**  compitibility issues would have to be caught.
13360Sstevel@tonic-gate 		**
13370Sstevel@tonic-gate 		**  For now the recommondation is to ignore the error message.
13380Sstevel@tonic-gate 		*/
13390Sstevel@tonic-gate 
13400Sstevel@tonic-gate 		else if (r == 0)
13410Sstevel@tonic-gate 		{
13420Sstevel@tonic-gate 			if (LogLevel > 15)
13430Sstevel@tonic-gate 			{
13440Sstevel@tonic-gate 				sm_syslog(LOG_WARNING, NOQID,
13450Sstevel@tonic-gate 					  "STARTTLS=%s, SSL_shutdown not done",
13460Sstevel@tonic-gate 					  side);
13470Sstevel@tonic-gate 				tlslogerr(side);
13480Sstevel@tonic-gate 			}
13490Sstevel@tonic-gate 			ret = EX_SOFTWARE;
13500Sstevel@tonic-gate 		}
13510Sstevel@tonic-gate # endif /* !defined(OPENSSL_VERSION_NUMBER) || OPENSSL_VERSION_NUMBER > 0x0090602fL */
13520Sstevel@tonic-gate 		SSL_free(ssl);
13530Sstevel@tonic-gate 		ssl = NULL;
13540Sstevel@tonic-gate 	}
13550Sstevel@tonic-gate 	return ret;
13560Sstevel@tonic-gate }
13570Sstevel@tonic-gate 
13580Sstevel@tonic-gate # if !TLS_NO_RSA
13590Sstevel@tonic-gate /*
13600Sstevel@tonic-gate **  TMP_RSA_KEY -- return temporary RSA key
13610Sstevel@tonic-gate **
13620Sstevel@tonic-gate **	Parameters:
13630Sstevel@tonic-gate **		s -- TLS connection structure
13640Sstevel@tonic-gate **		export --
13650Sstevel@tonic-gate **		keylength --
13660Sstevel@tonic-gate **
13670Sstevel@tonic-gate **	Returns:
13680Sstevel@tonic-gate **		temporary RSA key.
13690Sstevel@tonic-gate */
13700Sstevel@tonic-gate 
13710Sstevel@tonic-gate #   ifndef MAX_RSA_TMP_CNT
13720Sstevel@tonic-gate #    define MAX_RSA_TMP_CNT	1000	/* XXX better value? */
13730Sstevel@tonic-gate #   endif /* ! MAX_RSA_TMP_CNT */
13740Sstevel@tonic-gate 
13750Sstevel@tonic-gate /* ARGUSED0 */
13760Sstevel@tonic-gate static RSA *
13770Sstevel@tonic-gate tmp_rsa_key(s, export, keylength)
13780Sstevel@tonic-gate 	SSL *s;
13790Sstevel@tonic-gate 	int export;
13800Sstevel@tonic-gate 	int keylength;
13810Sstevel@tonic-gate {
13820Sstevel@tonic-gate #   if SM_CONF_SHM
13830Sstevel@tonic-gate 	extern int ShmId;
13840Sstevel@tonic-gate 	extern int *PRSATmpCnt;
13850Sstevel@tonic-gate 
13860Sstevel@tonic-gate 	if (ShmId != SM_SHM_NO_ID && rsa_tmp != NULL &&
13870Sstevel@tonic-gate 	    ++(*PRSATmpCnt) < MAX_RSA_TMP_CNT)
13880Sstevel@tonic-gate 		return rsa_tmp;
13890Sstevel@tonic-gate #   endif /* SM_CONF_SHM */
13900Sstevel@tonic-gate 
13910Sstevel@tonic-gate 	if (rsa_tmp != NULL)
13920Sstevel@tonic-gate 		RSA_free(rsa_tmp);
13930Sstevel@tonic-gate 	rsa_tmp = RSA_generate_key(RSA_KEYLENGTH, RSA_F4, NULL, NULL);
13940Sstevel@tonic-gate 	if (rsa_tmp == NULL)
13950Sstevel@tonic-gate 	{
13960Sstevel@tonic-gate 		if (LogLevel > 0)
13970Sstevel@tonic-gate 			sm_syslog(LOG_ERR, NOQID,
13980Sstevel@tonic-gate 				  "STARTTLS=server, tmp_rsa_key: RSA_generate_key failed!");
13990Sstevel@tonic-gate 	}
14000Sstevel@tonic-gate 	else
14010Sstevel@tonic-gate 	{
14020Sstevel@tonic-gate #   if SM_CONF_SHM
14030Sstevel@tonic-gate #    if 0
14040Sstevel@tonic-gate 		/*
14050Sstevel@tonic-gate 		**  XXX we can't (yet) share the new key...
14060Sstevel@tonic-gate 		**	The RSA structure contains pointers hence it can't be
14070Sstevel@tonic-gate 		**	easily kept in shared memory.  It must be transformed
14080Sstevel@tonic-gate 		**	into a continous memory region first, then stored,
14090Sstevel@tonic-gate 		**	and later read out again (each time re-transformed).
14100Sstevel@tonic-gate 		*/
14110Sstevel@tonic-gate 
14120Sstevel@tonic-gate 		if (ShmId != SM_SHM_NO_ID)
14130Sstevel@tonic-gate 			*PRSATmpCnt = 0;
14140Sstevel@tonic-gate #    endif /* 0 */
14150Sstevel@tonic-gate #   endif /* SM_CONF_SHM */
14160Sstevel@tonic-gate 		if (LogLevel > 9)
14170Sstevel@tonic-gate 			sm_syslog(LOG_ERR, NOQID,
14180Sstevel@tonic-gate 				  "STARTTLS=server, tmp_rsa_key: new temp RSA key");
14190Sstevel@tonic-gate 	}
14200Sstevel@tonic-gate 	return rsa_tmp;
14210Sstevel@tonic-gate }
14220Sstevel@tonic-gate # endif /* !TLS_NO_RSA */
14230Sstevel@tonic-gate /*
14240Sstevel@tonic-gate **  APPS_SSL_INFO_CB -- info callback for TLS connections
14250Sstevel@tonic-gate **
14260Sstevel@tonic-gate **	Parameters:
14270Sstevel@tonic-gate **		s -- TLS connection structure
14280Sstevel@tonic-gate **		where -- state in handshake
14290Sstevel@tonic-gate **		ret -- return code of last operation
14300Sstevel@tonic-gate **
14310Sstevel@tonic-gate **	Returns:
14320Sstevel@tonic-gate **		none.
14330Sstevel@tonic-gate */
14340Sstevel@tonic-gate 
14350Sstevel@tonic-gate static void
14360Sstevel@tonic-gate apps_ssl_info_cb(s, where, ret)
14370Sstevel@tonic-gate 	CONST097 SSL *s;
14380Sstevel@tonic-gate 	int where;
14390Sstevel@tonic-gate 	int ret;
14400Sstevel@tonic-gate {
14410Sstevel@tonic-gate 	int w;
14420Sstevel@tonic-gate 	char *str;
14430Sstevel@tonic-gate 	BIO *bio_err = NULL;
14440Sstevel@tonic-gate 
14450Sstevel@tonic-gate 	if (LogLevel > 14)
14460Sstevel@tonic-gate 		sm_syslog(LOG_INFO, NOQID,
14470Sstevel@tonic-gate 			  "STARTTLS: info_callback where=0x%x, ret=%d",
14480Sstevel@tonic-gate 			  where, ret);
14490Sstevel@tonic-gate 
14500Sstevel@tonic-gate 	w = where & ~SSL_ST_MASK;
14510Sstevel@tonic-gate 	if (bio_err == NULL)
14520Sstevel@tonic-gate 		bio_err = BIO_new_fp(stderr, BIO_NOCLOSE);
14530Sstevel@tonic-gate 
14540Sstevel@tonic-gate 	if (bitset(SSL_ST_CONNECT, w))
14550Sstevel@tonic-gate 		str = "SSL_connect";
14560Sstevel@tonic-gate 	else if (bitset(SSL_ST_ACCEPT, w))
14570Sstevel@tonic-gate 		str = "SSL_accept";
14580Sstevel@tonic-gate 	else
14590Sstevel@tonic-gate 		str = "undefined";
14600Sstevel@tonic-gate 
14610Sstevel@tonic-gate 	if (bitset(SSL_CB_LOOP, where))
14620Sstevel@tonic-gate 	{
14630Sstevel@tonic-gate 		if (LogLevel > 12)
14640Sstevel@tonic-gate 			sm_syslog(LOG_NOTICE, NOQID,
14650Sstevel@tonic-gate 				"STARTTLS: %s:%s",
14660Sstevel@tonic-gate 				str, SSL_state_string_long(s));
14670Sstevel@tonic-gate 	}
14680Sstevel@tonic-gate 	else if (bitset(SSL_CB_ALERT, where))
14690Sstevel@tonic-gate 	{
14700Sstevel@tonic-gate 		str = bitset(SSL_CB_READ, where) ? "read" : "write";
14710Sstevel@tonic-gate 		if (LogLevel > 12)
14720Sstevel@tonic-gate 			sm_syslog(LOG_NOTICE, NOQID,
14730Sstevel@tonic-gate 				"STARTTLS: SSL3 alert %s:%s:%s",
14740Sstevel@tonic-gate 				str, SSL_alert_type_string_long(ret),
14750Sstevel@tonic-gate 				SSL_alert_desc_string_long(ret));
14760Sstevel@tonic-gate 	}
14770Sstevel@tonic-gate 	else if (bitset(SSL_CB_EXIT, where))
14780Sstevel@tonic-gate 	{
14790Sstevel@tonic-gate 		if (ret == 0)
14800Sstevel@tonic-gate 		{
14810Sstevel@tonic-gate 			if (LogLevel > 7)
14820Sstevel@tonic-gate 				sm_syslog(LOG_WARNING, NOQID,
14830Sstevel@tonic-gate 					"STARTTLS: %s:failed in %s",
14840Sstevel@tonic-gate 					str, SSL_state_string_long(s));
14850Sstevel@tonic-gate 		}
14860Sstevel@tonic-gate 		else if (ret < 0)
14870Sstevel@tonic-gate 		{
14880Sstevel@tonic-gate 			if (LogLevel > 7)
14890Sstevel@tonic-gate 				sm_syslog(LOG_WARNING, NOQID,
14900Sstevel@tonic-gate 					"STARTTLS: %s:error in %s",
14910Sstevel@tonic-gate 					str, SSL_state_string_long(s));
14920Sstevel@tonic-gate 		}
14930Sstevel@tonic-gate 	}
14940Sstevel@tonic-gate }
14950Sstevel@tonic-gate /*
14960Sstevel@tonic-gate **  TLS_VERIFY_LOG -- log verify error for TLS certificates
14970Sstevel@tonic-gate **
14980Sstevel@tonic-gate **	Parameters:
14990Sstevel@tonic-gate **		ok -- verify ok?
15000Sstevel@tonic-gate **		ctx -- x509 context
15010Sstevel@tonic-gate **
15020Sstevel@tonic-gate **	Returns:
15030Sstevel@tonic-gate **		0 -- fatal error
15040Sstevel@tonic-gate **		1 -- ok
15050Sstevel@tonic-gate */
15060Sstevel@tonic-gate 
15070Sstevel@tonic-gate static int
15080Sstevel@tonic-gate tls_verify_log(ok, ctx, name)
15090Sstevel@tonic-gate 	int ok;
15100Sstevel@tonic-gate 	X509_STORE_CTX *ctx;
15110Sstevel@tonic-gate 	char *name;
15120Sstevel@tonic-gate {
15130Sstevel@tonic-gate 	SSL *ssl;
15140Sstevel@tonic-gate 	X509 *cert;
15150Sstevel@tonic-gate 	int reason, depth;
15160Sstevel@tonic-gate 	char buf[512];
15170Sstevel@tonic-gate 
15180Sstevel@tonic-gate 	cert = X509_STORE_CTX_get_current_cert(ctx);
15190Sstevel@tonic-gate 	reason = X509_STORE_CTX_get_error(ctx);
15200Sstevel@tonic-gate 	depth = X509_STORE_CTX_get_error_depth(ctx);
15210Sstevel@tonic-gate 	ssl = (SSL *) X509_STORE_CTX_get_ex_data(ctx,
15220Sstevel@tonic-gate 			SSL_get_ex_data_X509_STORE_CTX_idx());
15230Sstevel@tonic-gate 
15240Sstevel@tonic-gate 	if (ssl == NULL)
15250Sstevel@tonic-gate 	{
15260Sstevel@tonic-gate 		/* internal error */
15270Sstevel@tonic-gate 		sm_syslog(LOG_ERR, NOQID,
15280Sstevel@tonic-gate 			  "STARTTLS: internal error: tls_verify_cb: ssl == NULL");
15290Sstevel@tonic-gate 		return 0;
15300Sstevel@tonic-gate 	}
15310Sstevel@tonic-gate 
15320Sstevel@tonic-gate 	X509_NAME_oneline(X509_get_subject_name(cert), buf, sizeof buf);
15330Sstevel@tonic-gate 	sm_syslog(LOG_INFO, NOQID,
15340Sstevel@tonic-gate 		  "STARTTLS: %s cert verify: depth=%d %s, state=%d, reason=%s",
15350Sstevel@tonic-gate 		  name, depth, buf, ok, X509_verify_cert_error_string(reason));
15360Sstevel@tonic-gate 	return 1;
15370Sstevel@tonic-gate }
15380Sstevel@tonic-gate 
15390Sstevel@tonic-gate /*
15400Sstevel@tonic-gate **  TLS_VERIFY_CB -- verify callback for TLS certificates
15410Sstevel@tonic-gate **
15420Sstevel@tonic-gate **	Parameters:
15430Sstevel@tonic-gate **		ctx -- x509 context
15440Sstevel@tonic-gate **
15450Sstevel@tonic-gate **	Returns:
15460Sstevel@tonic-gate **		accept connection?
15470Sstevel@tonic-gate **		currently: always yes.
15480Sstevel@tonic-gate */
15490Sstevel@tonic-gate 
15500Sstevel@tonic-gate static int
15510Sstevel@tonic-gate #  if !defined(OPENSSL_VERSION_NUMBER) || OPENSSL_VERSION_NUMBER < 0x00907000L
15520Sstevel@tonic-gate tls_verify_cb(ctx)
15530Sstevel@tonic-gate 	X509_STORE_CTX *ctx;
15540Sstevel@tonic-gate #  else /* !defined() || OPENSSL_VERSION_NUMBER < 0x00907000L */
15550Sstevel@tonic-gate tls_verify_cb(ctx, unused)
15560Sstevel@tonic-gate 	X509_STORE_CTX *ctx;
15570Sstevel@tonic-gate 	void *unused;
15580Sstevel@tonic-gate #  endif /* !defined() || OPENSSL_VERSION_NUMBER < 0x00907000L */
15590Sstevel@tonic-gate {
15600Sstevel@tonic-gate 	int ok;
15610Sstevel@tonic-gate 
15620Sstevel@tonic-gate 	ok = X509_verify_cert(ctx);
15630Sstevel@tonic-gate 	if (ok == 0)
15640Sstevel@tonic-gate 	{
15650Sstevel@tonic-gate 		if (LogLevel > 13)
15660Sstevel@tonic-gate 			return tls_verify_log(ok, ctx, "TLS");
15670Sstevel@tonic-gate 		return 1;	/* override it */
15680Sstevel@tonic-gate 	}
15690Sstevel@tonic-gate 	return ok;
15700Sstevel@tonic-gate }
15710Sstevel@tonic-gate /*
15720Sstevel@tonic-gate **  TLSLOGERR -- log the errors from the TLS error stack
15730Sstevel@tonic-gate **
15740Sstevel@tonic-gate **	Parameters:
15750Sstevel@tonic-gate **		who -- server/client (for logging).
15760Sstevel@tonic-gate **
15770Sstevel@tonic-gate **	Returns:
15780Sstevel@tonic-gate **		none.
15790Sstevel@tonic-gate */
15800Sstevel@tonic-gate 
15810Sstevel@tonic-gate void
15820Sstevel@tonic-gate tlslogerr(who)
1583*1658Sjbeck 	const char *who;
15840Sstevel@tonic-gate {
15850Sstevel@tonic-gate 	unsigned long l;
15860Sstevel@tonic-gate 	int line, flags;
15870Sstevel@tonic-gate 	unsigned long es;
15880Sstevel@tonic-gate 	char *file, *data;
15890Sstevel@tonic-gate 	char buf[256];
15900Sstevel@tonic-gate #  define CP (const char **)
15910Sstevel@tonic-gate 
15920Sstevel@tonic-gate 	es = CRYPTO_thread_id();
15930Sstevel@tonic-gate 	while ((l = ERR_get_error_line_data(CP &file, &line, CP &data, &flags))
15940Sstevel@tonic-gate 		!= 0)
15950Sstevel@tonic-gate 	{
15960Sstevel@tonic-gate 		sm_syslog(LOG_WARNING, NOQID,
15970Sstevel@tonic-gate 			  "STARTTLS=%s: %lu:%s:%s:%d:%s", who, es,
15980Sstevel@tonic-gate 			  ERR_error_string(l, buf),
15990Sstevel@tonic-gate 			  file, line,
16000Sstevel@tonic-gate 			  bitset(ERR_TXT_STRING, flags) ? data : "");
16010Sstevel@tonic-gate 	}
16020Sstevel@tonic-gate }
16030Sstevel@tonic-gate 
16040Sstevel@tonic-gate # if OPENSSL_VERSION_NUMBER > 0x00907000L
16050Sstevel@tonic-gate /*
16060Sstevel@tonic-gate **  X509_VERIFY_CB -- verify callback
16070Sstevel@tonic-gate **
16080Sstevel@tonic-gate **	Parameters:
16090Sstevel@tonic-gate **		ctx -- x509 context
16100Sstevel@tonic-gate **
16110Sstevel@tonic-gate **	Returns:
16120Sstevel@tonic-gate **		accept connection?
16130Sstevel@tonic-gate **		currently: always yes.
16140Sstevel@tonic-gate */
16150Sstevel@tonic-gate 
16160Sstevel@tonic-gate static int
16170Sstevel@tonic-gate x509_verify_cb(ok, ctx)
16180Sstevel@tonic-gate 	int ok;
16190Sstevel@tonic-gate 	X509_STORE_CTX *ctx;
16200Sstevel@tonic-gate {
16210Sstevel@tonic-gate 	if (ok == 0)
16220Sstevel@tonic-gate 	{
16230Sstevel@tonic-gate 		if (LogLevel > 13)
16240Sstevel@tonic-gate 			tls_verify_log(ok, ctx, "x509");
16250Sstevel@tonic-gate 		if (ctx->error == X509_V_ERR_UNABLE_TO_GET_CRL)
16260Sstevel@tonic-gate 		{
16270Sstevel@tonic-gate 			ctx->error = 0;
16280Sstevel@tonic-gate 			return 1;	/* override it */
16290Sstevel@tonic-gate 		}
16300Sstevel@tonic-gate 	}
16310Sstevel@tonic-gate 	return ok;
16320Sstevel@tonic-gate }
16330Sstevel@tonic-gate # endif /* OPENSSL_VERSION_NUMBER > 0x00907000L */
16340Sstevel@tonic-gate #endif /* STARTTLS */
1635