xref: /onnv-gate/usr/src/common/openssl/ssl/s2_enc.c (revision 2139:6243c3338933)
10Sstevel@tonic-gate /* ssl/s2_enc.c */
20Sstevel@tonic-gate /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
30Sstevel@tonic-gate  * All rights reserved.
40Sstevel@tonic-gate  *
50Sstevel@tonic-gate  * This package is an SSL implementation written
60Sstevel@tonic-gate  * by Eric Young (eay@cryptsoft.com).
70Sstevel@tonic-gate  * The implementation was written so as to conform with Netscapes SSL.
80Sstevel@tonic-gate  *
90Sstevel@tonic-gate  * This library is free for commercial and non-commercial use as long as
100Sstevel@tonic-gate  * the following conditions are aheared to.  The following conditions
110Sstevel@tonic-gate  * apply to all code found in this distribution, be it the RC4, RSA,
120Sstevel@tonic-gate  * lhash, DES, etc., code; not just the SSL code.  The SSL documentation
130Sstevel@tonic-gate  * included with this distribution is covered by the same copyright terms
140Sstevel@tonic-gate  * except that the holder is Tim Hudson (tjh@cryptsoft.com).
150Sstevel@tonic-gate  *
160Sstevel@tonic-gate  * Copyright remains Eric Young's, and as such any Copyright notices in
170Sstevel@tonic-gate  * the code are not to be removed.
180Sstevel@tonic-gate  * If this package is used in a product, Eric Young should be given attribution
190Sstevel@tonic-gate  * as the author of the parts of the library used.
200Sstevel@tonic-gate  * This can be in the form of a textual message at program startup or
210Sstevel@tonic-gate  * in documentation (online or textual) provided with the package.
220Sstevel@tonic-gate  *
230Sstevel@tonic-gate  * Redistribution and use in source and binary forms, with or without
240Sstevel@tonic-gate  * modification, are permitted provided that the following conditions
250Sstevel@tonic-gate  * are met:
260Sstevel@tonic-gate  * 1. Redistributions of source code must retain the copyright
270Sstevel@tonic-gate  *    notice, this list of conditions and the following disclaimer.
280Sstevel@tonic-gate  * 2. Redistributions in binary form must reproduce the above copyright
290Sstevel@tonic-gate  *    notice, this list of conditions and the following disclaimer in the
300Sstevel@tonic-gate  *    documentation and/or other materials provided with the distribution.
310Sstevel@tonic-gate  * 3. All advertising materials mentioning features or use of this software
320Sstevel@tonic-gate  *    must display the following acknowledgement:
330Sstevel@tonic-gate  *    "This product includes cryptographic software written by
340Sstevel@tonic-gate  *     Eric Young (eay@cryptsoft.com)"
350Sstevel@tonic-gate  *    The word 'cryptographic' can be left out if the rouines from the library
360Sstevel@tonic-gate  *    being used are not cryptographic related :-).
370Sstevel@tonic-gate  * 4. If you include any Windows specific code (or a derivative thereof) from
380Sstevel@tonic-gate  *    the apps directory (application code) you must include an acknowledgement:
390Sstevel@tonic-gate  *    "This product includes software written by Tim Hudson (tjh@cryptsoft.com)"
400Sstevel@tonic-gate  *
410Sstevel@tonic-gate  * THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND
420Sstevel@tonic-gate  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
430Sstevel@tonic-gate  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
440Sstevel@tonic-gate  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
450Sstevel@tonic-gate  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
460Sstevel@tonic-gate  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
470Sstevel@tonic-gate  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
480Sstevel@tonic-gate  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
490Sstevel@tonic-gate  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
500Sstevel@tonic-gate  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
510Sstevel@tonic-gate  * SUCH DAMAGE.
520Sstevel@tonic-gate  *
530Sstevel@tonic-gate  * The licence and distribution terms for any publically available version or
540Sstevel@tonic-gate  * derivative of this code cannot be changed.  i.e. this code cannot simply be
550Sstevel@tonic-gate  * copied and put under another distribution licence
560Sstevel@tonic-gate  * [including the GNU Public Licence.]
570Sstevel@tonic-gate  */
580Sstevel@tonic-gate 
590Sstevel@tonic-gate #include "ssl_locl.h"
600Sstevel@tonic-gate #ifndef OPENSSL_NO_SSL2
610Sstevel@tonic-gate #include <stdio.h>
620Sstevel@tonic-gate 
ssl2_enc_init(SSL * s,int client)630Sstevel@tonic-gate int ssl2_enc_init(SSL *s, int client)
640Sstevel@tonic-gate 	{
650Sstevel@tonic-gate 	/* Max number of bytes needed */
660Sstevel@tonic-gate 	EVP_CIPHER_CTX *rs,*ws;
670Sstevel@tonic-gate 	const EVP_CIPHER *c;
680Sstevel@tonic-gate 	const EVP_MD *md;
690Sstevel@tonic-gate 	int num;
700Sstevel@tonic-gate 
710Sstevel@tonic-gate 	if (!ssl_cipher_get_evp(s->session,&c,&md,NULL))
720Sstevel@tonic-gate 		{
730Sstevel@tonic-gate 		ssl2_return_error(s,SSL2_PE_NO_CIPHER);
740Sstevel@tonic-gate 		SSLerr(SSL_F_SSL2_ENC_INIT,SSL_R_PROBLEMS_MAPPING_CIPHER_FUNCTIONS);
750Sstevel@tonic-gate 		return(0);
760Sstevel@tonic-gate 		}
770Sstevel@tonic-gate 
780Sstevel@tonic-gate 	s->read_hash=md;
790Sstevel@tonic-gate 	s->write_hash=md;
800Sstevel@tonic-gate 
810Sstevel@tonic-gate 	if ((s->enc_read_ctx == NULL) &&
820Sstevel@tonic-gate 		((s->enc_read_ctx=(EVP_CIPHER_CTX *)
830Sstevel@tonic-gate 		OPENSSL_malloc(sizeof(EVP_CIPHER_CTX))) == NULL))
840Sstevel@tonic-gate 		goto err;
850Sstevel@tonic-gate 	if ((s->enc_write_ctx == NULL) &&
860Sstevel@tonic-gate 		((s->enc_write_ctx=(EVP_CIPHER_CTX *)
870Sstevel@tonic-gate 		OPENSSL_malloc(sizeof(EVP_CIPHER_CTX))) == NULL))
880Sstevel@tonic-gate 		goto err;
890Sstevel@tonic-gate 
900Sstevel@tonic-gate 	rs= s->enc_read_ctx;
910Sstevel@tonic-gate 	ws= s->enc_write_ctx;
920Sstevel@tonic-gate 
930Sstevel@tonic-gate 	EVP_CIPHER_CTX_init(rs);
940Sstevel@tonic-gate 	EVP_CIPHER_CTX_init(ws);
950Sstevel@tonic-gate 
960Sstevel@tonic-gate 	num=c->key_len;
970Sstevel@tonic-gate 	s->s2->key_material_length=num*2;
980Sstevel@tonic-gate 	OPENSSL_assert(s->s2->key_material_length <= sizeof s->s2->key_material);
990Sstevel@tonic-gate 
1000Sstevel@tonic-gate 	if (ssl2_generate_key_material(s) <= 0)
1010Sstevel@tonic-gate 		return 0;
1020Sstevel@tonic-gate 
103*2139Sjp161948 	OPENSSL_assert(c->iv_len <= (int)sizeof(s->session->key_arg));
1040Sstevel@tonic-gate 	EVP_EncryptInit_ex(ws,c,NULL,&(s->s2->key_material[(client)?num:0]),
1050Sstevel@tonic-gate 		s->session->key_arg);
1060Sstevel@tonic-gate 	EVP_DecryptInit_ex(rs,c,NULL,&(s->s2->key_material[(client)?0:num]),
1070Sstevel@tonic-gate 		s->session->key_arg);
1080Sstevel@tonic-gate 	s->s2->read_key=  &(s->s2->key_material[(client)?0:num]);
1090Sstevel@tonic-gate 	s->s2->write_key= &(s->s2->key_material[(client)?num:0]);
1100Sstevel@tonic-gate 	return(1);
1110Sstevel@tonic-gate err:
1120Sstevel@tonic-gate 	SSLerr(SSL_F_SSL2_ENC_INIT,ERR_R_MALLOC_FAILURE);
1130Sstevel@tonic-gate 	return(0);
1140Sstevel@tonic-gate 	}
1150Sstevel@tonic-gate 
1160Sstevel@tonic-gate /* read/writes from s->s2->mac_data using length for encrypt and
1170Sstevel@tonic-gate  * decrypt.  It sets s->s2->padding and s->[rw]length
1180Sstevel@tonic-gate  * if we are encrypting */
ssl2_enc(SSL * s,int send)1190Sstevel@tonic-gate void ssl2_enc(SSL *s, int send)
1200Sstevel@tonic-gate 	{
1210Sstevel@tonic-gate 	EVP_CIPHER_CTX *ds;
1220Sstevel@tonic-gate 	unsigned long l;
1230Sstevel@tonic-gate 	int bs;
1240Sstevel@tonic-gate 
1250Sstevel@tonic-gate 	if (send)
1260Sstevel@tonic-gate 		{
1270Sstevel@tonic-gate 		ds=s->enc_write_ctx;
1280Sstevel@tonic-gate 		l=s->s2->wlength;
1290Sstevel@tonic-gate 		}
1300Sstevel@tonic-gate 	else
1310Sstevel@tonic-gate 		{
1320Sstevel@tonic-gate 		ds=s->enc_read_ctx;
1330Sstevel@tonic-gate 		l=s->s2->rlength;
1340Sstevel@tonic-gate 		}
1350Sstevel@tonic-gate 
1360Sstevel@tonic-gate 	/* check for NULL cipher */
1370Sstevel@tonic-gate 	if (ds == NULL) return;
1380Sstevel@tonic-gate 
1390Sstevel@tonic-gate 
1400Sstevel@tonic-gate 	bs=ds->cipher->block_size;
1410Sstevel@tonic-gate 	/* This should be using (bs-1) and bs instead of 7 and 8, but
1420Sstevel@tonic-gate 	 * what the hell. */
1430Sstevel@tonic-gate 	if (bs == 8)
1440Sstevel@tonic-gate 		l=(l+7)/8*8;
1450Sstevel@tonic-gate 
1460Sstevel@tonic-gate 	EVP_Cipher(ds,s->s2->mac_data,s->s2->mac_data,l);
1470Sstevel@tonic-gate 	}
1480Sstevel@tonic-gate 
ssl2_mac(SSL * s,unsigned char * md,int send)1490Sstevel@tonic-gate void ssl2_mac(SSL *s, unsigned char *md, int send)
1500Sstevel@tonic-gate 	{
1510Sstevel@tonic-gate 	EVP_MD_CTX c;
1520Sstevel@tonic-gate 	unsigned char sequence[4],*p,*sec,*act;
1530Sstevel@tonic-gate 	unsigned long seq;
1540Sstevel@tonic-gate 	unsigned int len;
1550Sstevel@tonic-gate 
1560Sstevel@tonic-gate 	if (send)
1570Sstevel@tonic-gate 		{
1580Sstevel@tonic-gate 		seq=s->s2->write_sequence;
1590Sstevel@tonic-gate 		sec=s->s2->write_key;
1600Sstevel@tonic-gate 		len=s->s2->wact_data_length;
1610Sstevel@tonic-gate 		act=s->s2->wact_data;
1620Sstevel@tonic-gate 		}
1630Sstevel@tonic-gate 	else
1640Sstevel@tonic-gate 		{
1650Sstevel@tonic-gate 		seq=s->s2->read_sequence;
1660Sstevel@tonic-gate 		sec=s->s2->read_key;
1670Sstevel@tonic-gate 		len=s->s2->ract_data_length;
1680Sstevel@tonic-gate 		act=s->s2->ract_data;
1690Sstevel@tonic-gate 		}
1700Sstevel@tonic-gate 
1710Sstevel@tonic-gate 	p= &(sequence[0]);
1720Sstevel@tonic-gate 	l2n(seq,p);
1730Sstevel@tonic-gate 
1740Sstevel@tonic-gate 	/* There has to be a MAC algorithm. */
1750Sstevel@tonic-gate 	EVP_MD_CTX_init(&c);
1760Sstevel@tonic-gate 	EVP_DigestInit_ex(&c, s->read_hash, NULL);
1770Sstevel@tonic-gate 	EVP_DigestUpdate(&c,sec,
1780Sstevel@tonic-gate 		EVP_CIPHER_CTX_key_length(s->enc_read_ctx));
1790Sstevel@tonic-gate 	EVP_DigestUpdate(&c,act,len);
1800Sstevel@tonic-gate 	/* the above line also does the pad data */
1810Sstevel@tonic-gate 	EVP_DigestUpdate(&c,sequence,4);
1820Sstevel@tonic-gate 	EVP_DigestFinal_ex(&c,md,NULL);
1830Sstevel@tonic-gate 	EVP_MD_CTX_cleanup(&c);
1840Sstevel@tonic-gate 	}
1850Sstevel@tonic-gate #else /* !OPENSSL_NO_SSL2 */
1860Sstevel@tonic-gate 
1870Sstevel@tonic-gate # if PEDANTIC
1880Sstevel@tonic-gate static void *dummy=&dummy;
1890Sstevel@tonic-gate # endif
1900Sstevel@tonic-gate 
1910Sstevel@tonic-gate #endif
192