xref: /onnv-gate/usr/src/lib/libresolv2/common/dst/hmac_link.c (revision 11038:74b12212b8a2)
10Sstevel@tonic-gate /*
2*11038SRao.Shoaib@Sun.COM  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
30Sstevel@tonic-gate  * Use is subject to license terms.
40Sstevel@tonic-gate  */
50Sstevel@tonic-gate 
60Sstevel@tonic-gate 
70Sstevel@tonic-gate #ifdef HMAC_MD5
80Sstevel@tonic-gate #ifndef LINT
9*11038SRao.Shoaib@Sun.COM static const char rcsid[] = "$Header: /proj/cvs/prod/libbind/dst/hmac_link.c,v 1.8 2007/09/24 17:18:25 each Exp $";
100Sstevel@tonic-gate #endif
110Sstevel@tonic-gate /*
120Sstevel@tonic-gate  * Portions Copyright (c) 1995-1998 by Trusted Information Systems, Inc.
130Sstevel@tonic-gate  *
140Sstevel@tonic-gate  * Permission to use, copy modify, and distribute this software for any
150Sstevel@tonic-gate  * purpose with or without fee is hereby granted, provided that the above
160Sstevel@tonic-gate  * copyright notice and this permission notice appear in all copies.
170Sstevel@tonic-gate  *
180Sstevel@tonic-gate  * THE SOFTWARE IS PROVIDED "AS IS" AND TRUSTED INFORMATION SYSTEMS
190Sstevel@tonic-gate  * DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL
200Sstevel@tonic-gate  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS.  IN NO EVENT SHALL
210Sstevel@tonic-gate  * TRUSTED INFORMATION SYSTEMS BE LIABLE FOR ANY SPECIAL, DIRECT,
220Sstevel@tonic-gate  * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
230Sstevel@tonic-gate  * FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
240Sstevel@tonic-gate  * NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
250Sstevel@tonic-gate  * WITH THE USE OR PERFORMANCE OF THE SOFTWARE.
260Sstevel@tonic-gate  */
270Sstevel@tonic-gate 
28*11038SRao.Shoaib@Sun.COM /*%
290Sstevel@tonic-gate  * This file contains an implementation of the HMAC-MD5 algorithm.
300Sstevel@tonic-gate  */
310Sstevel@tonic-gate #include "port_before.h"
320Sstevel@tonic-gate 
330Sstevel@tonic-gate #include <stdio.h>
340Sstevel@tonic-gate #include <unistd.h>
350Sstevel@tonic-gate #include <stdlib.h>
360Sstevel@tonic-gate #include <string.h>
370Sstevel@tonic-gate #include <memory.h>
380Sstevel@tonic-gate #include <sys/param.h>
390Sstevel@tonic-gate #include <sys/time.h>
400Sstevel@tonic-gate #include <netinet/in.h>
410Sstevel@tonic-gate #include <arpa/nameser.h>
420Sstevel@tonic-gate #include <resolv.h>
430Sstevel@tonic-gate 
440Sstevel@tonic-gate #include "dst_internal.h"
45*11038SRao.Shoaib@Sun.COM 
460Sstevel@tonic-gate #ifdef USE_MD5
47*11038SRao.Shoaib@Sun.COM # ifndef HAVE_MD5
48*11038SRao.Shoaib@Sun.COM #  include "md5.h"
49*11038SRao.Shoaib@Sun.COM # else
50*11038SRao.Shoaib@Sun.COM #  ifdef SOLARIS2
51*11038SRao.Shoaib@Sun.COM #   include <sys/md5.h>
52*11038SRao.Shoaib@Sun.COM #  endif
53*11038SRao.Shoaib@Sun.COM # endif
540Sstevel@tonic-gate # ifndef _MD5_H_
55*11038SRao.Shoaib@Sun.COM #  define _MD5_H_ 1	/*%< make sure we do not include rsaref md5.h file */
560Sstevel@tonic-gate # endif
570Sstevel@tonic-gate #endif
580Sstevel@tonic-gate 
590Sstevel@tonic-gate #include "port_after.h"
600Sstevel@tonic-gate 
610Sstevel@tonic-gate 
620Sstevel@tonic-gate #define HMAC_LEN	64
630Sstevel@tonic-gate #define HMAC_IPAD	0x36
640Sstevel@tonic-gate #define HMAC_OPAD	0x5c
650Sstevel@tonic-gate #define MD5_LEN		16
660Sstevel@tonic-gate 
670Sstevel@tonic-gate 
680Sstevel@tonic-gate typedef struct hmackey {
690Sstevel@tonic-gate 	u_char hk_ipad[64], hk_opad[64];
700Sstevel@tonic-gate } HMAC_Key;
710Sstevel@tonic-gate 
720Sstevel@tonic-gate 
730Sstevel@tonic-gate /**************************************************************************
740Sstevel@tonic-gate  * dst_hmac_md5_sign
750Sstevel@tonic-gate  *     Call HMAC signing functions to sign a block of data.
760Sstevel@tonic-gate  *     There are three steps to signing, INIT (initialize structures),
770Sstevel@tonic-gate  *     UPDATE (hash (more) data), FINAL (generate a signature).  This
780Sstevel@tonic-gate  *     routine performs one or more of these steps.
790Sstevel@tonic-gate  * Parameters
800Sstevel@tonic-gate  *     mode	SIG_MODE_INIT, SIG_MODE_UPDATE and/or SIG_MODE_FINAL.
810Sstevel@tonic-gate  *     priv_key    key to use for signing.
820Sstevel@tonic-gate  *     context   the context to be used in this digest
830Sstevel@tonic-gate  *     data	data to be signed.
840Sstevel@tonic-gate  *     len	 length in bytes of data.
850Sstevel@tonic-gate  *     signature   location to store signature.
860Sstevel@tonic-gate  *     sig_len     size of the signature location
870Sstevel@tonic-gate  * returns
880Sstevel@tonic-gate  *	N  Success on SIG_MODE_FINAL = returns signature length in bytes
890Sstevel@tonic-gate  *	0  Success on SIG_MODE_INIT  and UPDATE
900Sstevel@tonic-gate  *	 <0  Failure
910Sstevel@tonic-gate  */
920Sstevel@tonic-gate 
930Sstevel@tonic-gate static int
dst_hmac_md5_sign(const int mode,DST_KEY * d_key,void ** context,const u_char * data,const int len,u_char * signature,const int sig_len)940Sstevel@tonic-gate dst_hmac_md5_sign(const int mode, DST_KEY *d_key, void **context,
950Sstevel@tonic-gate 		  const u_char *data, const int len,
960Sstevel@tonic-gate 		  u_char *signature, const int sig_len)
970Sstevel@tonic-gate {
980Sstevel@tonic-gate 	HMAC_Key *key;
990Sstevel@tonic-gate 	int sign_len = 0;
1000Sstevel@tonic-gate 	MD5_CTX *ctx = NULL;
1010Sstevel@tonic-gate 
102*11038SRao.Shoaib@Sun.COM 	if (d_key == NULL || d_key->dk_KEY_struct == NULL)
103*11038SRao.Shoaib@Sun.COM 		return (-1);
104*11038SRao.Shoaib@Sun.COM 
1050Sstevel@tonic-gate 	if (mode & SIG_MODE_INIT)
1060Sstevel@tonic-gate 		ctx = (MD5_CTX *) malloc(sizeof(*ctx));
1070Sstevel@tonic-gate 	else if (context)
1080Sstevel@tonic-gate 		ctx = (MD5_CTX *) *context;
1090Sstevel@tonic-gate 	if (ctx == NULL)
1100Sstevel@tonic-gate 		return (-1);
1110Sstevel@tonic-gate 
1120Sstevel@tonic-gate 	key = (HMAC_Key *) d_key->dk_KEY_struct;
1130Sstevel@tonic-gate 
1140Sstevel@tonic-gate 	if (mode & SIG_MODE_INIT) {
1150Sstevel@tonic-gate 		MD5Init(ctx);
1160Sstevel@tonic-gate 		MD5Update(ctx, key->hk_ipad, HMAC_LEN);
1170Sstevel@tonic-gate 	}
1180Sstevel@tonic-gate 
1190Sstevel@tonic-gate 	if ((mode & SIG_MODE_UPDATE) && (data && len > 0))
1200Sstevel@tonic-gate 		MD5Update(ctx, data, len);
1210Sstevel@tonic-gate 
1220Sstevel@tonic-gate 	if (mode & SIG_MODE_FINAL) {
1230Sstevel@tonic-gate 		if (signature == NULL || sig_len < MD5_LEN)
1240Sstevel@tonic-gate 			return (SIGN_FINAL_FAILURE);
1250Sstevel@tonic-gate 		MD5Final(signature, ctx);
1260Sstevel@tonic-gate 
1270Sstevel@tonic-gate 		/* perform outer MD5 */
1280Sstevel@tonic-gate 		MD5Init(ctx);
1290Sstevel@tonic-gate 		MD5Update(ctx, key->hk_opad, HMAC_LEN);
1300Sstevel@tonic-gate 		MD5Update(ctx, signature, MD5_LEN);
1310Sstevel@tonic-gate 		MD5Final(signature, ctx);
1320Sstevel@tonic-gate 		sign_len = MD5_LEN;
1330Sstevel@tonic-gate 		SAFE_FREE(ctx);
1340Sstevel@tonic-gate 	}
1350Sstevel@tonic-gate 	else {
1360Sstevel@tonic-gate 		if (context == NULL)
1370Sstevel@tonic-gate 			return (-1);
1380Sstevel@tonic-gate 		*context = (void *) ctx;
1390Sstevel@tonic-gate 	}
1400Sstevel@tonic-gate 	return (sign_len);
1410Sstevel@tonic-gate }
1420Sstevel@tonic-gate 
1430Sstevel@tonic-gate 
1440Sstevel@tonic-gate /**************************************************************************
1450Sstevel@tonic-gate  * dst_hmac_md5_verify()
1460Sstevel@tonic-gate  *     Calls HMAC verification routines.  There are three steps to
1470Sstevel@tonic-gate  *     verification, INIT (initialize structures), UPDATE (hash (more) data),
1480Sstevel@tonic-gate  *     FINAL (generate a signature).  This routine performs one or more of
1490Sstevel@tonic-gate  *     these steps.
1500Sstevel@tonic-gate  * Parameters
1510Sstevel@tonic-gate  *     mode	SIG_MODE_INIT, SIG_MODE_UPDATE and/or SIG_MODE_FINAL.
1520Sstevel@tonic-gate  *     dkey	key to use for verify.
1530Sstevel@tonic-gate  *     data	data signed.
1540Sstevel@tonic-gate  *     len	 length in bytes of data.
1550Sstevel@tonic-gate  *     signature   signature.
1560Sstevel@tonic-gate  *     sig_len     length in bytes of signature.
1570Sstevel@tonic-gate  * returns
1580Sstevel@tonic-gate  *     0  Success
1590Sstevel@tonic-gate  *    <0  Failure
1600Sstevel@tonic-gate  */
1610Sstevel@tonic-gate 
1620Sstevel@tonic-gate static int
dst_hmac_md5_verify(const int mode,DST_KEY * d_key,void ** context,const u_char * data,const int len,const u_char * signature,const int sig_len)1630Sstevel@tonic-gate dst_hmac_md5_verify(const int mode, DST_KEY *d_key, void **context,
1640Sstevel@tonic-gate 		const u_char *data, const int len,
1650Sstevel@tonic-gate 		const u_char *signature, const int sig_len)
1660Sstevel@tonic-gate {
1670Sstevel@tonic-gate 	HMAC_Key *key;
1680Sstevel@tonic-gate 	MD5_CTX *ctx = NULL;
1690Sstevel@tonic-gate 
170*11038SRao.Shoaib@Sun.COM 	if (d_key == NULL || d_key->dk_KEY_struct == NULL)
171*11038SRao.Shoaib@Sun.COM 		return (-1);
172*11038SRao.Shoaib@Sun.COM 
1730Sstevel@tonic-gate 	if (mode & SIG_MODE_INIT)
1740Sstevel@tonic-gate 		ctx = (MD5_CTX *) malloc(sizeof(*ctx));
1750Sstevel@tonic-gate 	else if (context)
1760Sstevel@tonic-gate 		ctx = (MD5_CTX *) *context;
1770Sstevel@tonic-gate 	if (ctx == NULL)
1780Sstevel@tonic-gate 		return (-1);
1790Sstevel@tonic-gate 
1800Sstevel@tonic-gate 	key = (HMAC_Key *) d_key->dk_KEY_struct;
1810Sstevel@tonic-gate 	if (mode & SIG_MODE_INIT) {
1820Sstevel@tonic-gate 		MD5Init(ctx);
1830Sstevel@tonic-gate 		MD5Update(ctx, key->hk_ipad, HMAC_LEN);
1840Sstevel@tonic-gate 	}
1850Sstevel@tonic-gate 	if ((mode & SIG_MODE_UPDATE) && (data && len > 0))
1860Sstevel@tonic-gate 		MD5Update(ctx, data, len);
1870Sstevel@tonic-gate 
1880Sstevel@tonic-gate 	if (mode & SIG_MODE_FINAL) {
1890Sstevel@tonic-gate 		u_char digest[MD5_LEN];
1900Sstevel@tonic-gate 		if (signature == NULL || key == NULL || sig_len != MD5_LEN)
1910Sstevel@tonic-gate 			return (VERIFY_FINAL_FAILURE);
1920Sstevel@tonic-gate 		MD5Final(digest, ctx);
1930Sstevel@tonic-gate 
1940Sstevel@tonic-gate 		/* perform outer MD5 */
1950Sstevel@tonic-gate 		MD5Init(ctx);
1960Sstevel@tonic-gate 		MD5Update(ctx, key->hk_opad, HMAC_LEN);
1970Sstevel@tonic-gate 		MD5Update(ctx, digest, MD5_LEN);
1980Sstevel@tonic-gate 		MD5Final(digest, ctx);
1990Sstevel@tonic-gate 
2000Sstevel@tonic-gate 		SAFE_FREE(ctx);
2010Sstevel@tonic-gate 		if (memcmp(digest, signature, MD5_LEN) != 0)
2020Sstevel@tonic-gate 			return (VERIFY_FINAL_FAILURE);
2030Sstevel@tonic-gate 	}
2040Sstevel@tonic-gate 	else {
2050Sstevel@tonic-gate 		if (context == NULL)
2060Sstevel@tonic-gate 			return (-1);
2070Sstevel@tonic-gate 		*context = (void *) ctx;
2080Sstevel@tonic-gate 	}
2090Sstevel@tonic-gate 	return (0);
2100Sstevel@tonic-gate }
2110Sstevel@tonic-gate 
2120Sstevel@tonic-gate 
2130Sstevel@tonic-gate /**************************************************************************
2140Sstevel@tonic-gate  * dst_buffer_to_hmac_md5
2150Sstevel@tonic-gate  *     Converts key from raw data to an HMAC Key
2160Sstevel@tonic-gate  *     This function gets in a pointer to the data
2170Sstevel@tonic-gate  * Parameters
2180Sstevel@tonic-gate  *     hkey	the HMAC key to be filled in
2190Sstevel@tonic-gate  *     key	the key in raw format
2200Sstevel@tonic-gate  *     keylen	the length of the key
2210Sstevel@tonic-gate  * Return
2220Sstevel@tonic-gate  *	0	Success
2230Sstevel@tonic-gate  *	<0	Failure
2240Sstevel@tonic-gate  */
2250Sstevel@tonic-gate static int
dst_buffer_to_hmac_md5(DST_KEY * dkey,const u_char * key,const int keylen)2260Sstevel@tonic-gate dst_buffer_to_hmac_md5(DST_KEY *dkey, const u_char *key, const int keylen)
2270Sstevel@tonic-gate {
2280Sstevel@tonic-gate 	int i;
2290Sstevel@tonic-gate 	HMAC_Key *hkey = NULL;
2300Sstevel@tonic-gate 	MD5_CTX ctx;
2310Sstevel@tonic-gate 	int local_keylen = keylen;
232*11038SRao.Shoaib@Sun.COM 	u_char tk[MD5_LEN];
2330Sstevel@tonic-gate 
2340Sstevel@tonic-gate 	if (dkey == NULL || key == NULL || keylen < 0)
2350Sstevel@tonic-gate 		return (-1);
2360Sstevel@tonic-gate 
2370Sstevel@tonic-gate 	if ((hkey = (HMAC_Key *) malloc(sizeof(HMAC_Key))) == NULL)
2380Sstevel@tonic-gate 		  return (-2);
2390Sstevel@tonic-gate 
2400Sstevel@tonic-gate 	memset(hkey->hk_ipad, 0, sizeof(hkey->hk_ipad));
2410Sstevel@tonic-gate 	memset(hkey->hk_opad, 0, sizeof(hkey->hk_opad));
2420Sstevel@tonic-gate 
2430Sstevel@tonic-gate 	/* if key is longer than HMAC_LEN bytes reset it to key=MD5(key) */
2440Sstevel@tonic-gate 	if (keylen > HMAC_LEN) {
2450Sstevel@tonic-gate 		MD5Init(&ctx);
2460Sstevel@tonic-gate 		MD5Update(&ctx, key, keylen);
2470Sstevel@tonic-gate 		MD5Final(tk, &ctx);
2480Sstevel@tonic-gate 		memset((void *) &ctx, 0, sizeof(ctx));
2490Sstevel@tonic-gate 		key = tk;
2500Sstevel@tonic-gate 		local_keylen = MD5_LEN;
2510Sstevel@tonic-gate 	}
2520Sstevel@tonic-gate 	/* start out by storing key in pads */
2530Sstevel@tonic-gate 	memcpy(hkey->hk_ipad, key, local_keylen);
2540Sstevel@tonic-gate 	memcpy(hkey->hk_opad, key, local_keylen);
2550Sstevel@tonic-gate 
2560Sstevel@tonic-gate 	/* XOR key with hk_ipad and opad values */
2570Sstevel@tonic-gate 	for (i = 0; i < HMAC_LEN; i++) {
2580Sstevel@tonic-gate 		hkey->hk_ipad[i] ^= HMAC_IPAD;
2590Sstevel@tonic-gate 		hkey->hk_opad[i] ^= HMAC_OPAD;
2600Sstevel@tonic-gate 	}
2610Sstevel@tonic-gate 	dkey->dk_key_size = local_keylen;
2620Sstevel@tonic-gate 	dkey->dk_KEY_struct = (void *) hkey;
2630Sstevel@tonic-gate 	return (1);
2640Sstevel@tonic-gate }
2650Sstevel@tonic-gate 
2660Sstevel@tonic-gate 
2670Sstevel@tonic-gate /**************************************************************************
2680Sstevel@tonic-gate  *  dst_hmac_md5_key_to_file_format
2690Sstevel@tonic-gate  *	Encodes an HMAC Key into the portable file format.
2700Sstevel@tonic-gate  *  Parameters
2710Sstevel@tonic-gate  *	hkey      HMAC KEY structure
2720Sstevel@tonic-gate  *	buff      output buffer
2730Sstevel@tonic-gate  *	buff_len  size of output buffer
2740Sstevel@tonic-gate  *  Return
2750Sstevel@tonic-gate  *	0  Failure - null input hkey
2760Sstevel@tonic-gate  *     -1  Failure - not enough space in output area
2770Sstevel@tonic-gate  *	N  Success - Length of data returned in buff
2780Sstevel@tonic-gate  */
2790Sstevel@tonic-gate 
2800Sstevel@tonic-gate static int
dst_hmac_md5_key_to_file_format(const DST_KEY * dkey,char * buff,const int buff_len)2810Sstevel@tonic-gate dst_hmac_md5_key_to_file_format(const DST_KEY *dkey, char *buff,
282*11038SRao.Shoaib@Sun.COM 			        const int buff_len)
2830Sstevel@tonic-gate {
2840Sstevel@tonic-gate 	char *bp;
285*11038SRao.Shoaib@Sun.COM 	int len, i, key_len;
2860Sstevel@tonic-gate 	u_char key[HMAC_LEN];
2870Sstevel@tonic-gate 	HMAC_Key *hkey;
2880Sstevel@tonic-gate 
2890Sstevel@tonic-gate 	if (dkey == NULL || dkey->dk_KEY_struct == NULL)
2900Sstevel@tonic-gate 		return (0);
291*11038SRao.Shoaib@Sun.COM 	/*
292*11038SRao.Shoaib@Sun.COM 	 * Using snprintf() would be so much simpler here.
293*11038SRao.Shoaib@Sun.COM 	 */
294*11038SRao.Shoaib@Sun.COM 	if (buff == NULL ||
295*11038SRao.Shoaib@Sun.COM 	    buff_len <= (int)(strlen(key_file_fmt_str) +
296*11038SRao.Shoaib@Sun.COM 			      strlen(KEY_FILE_FORMAT) + 4))
297*11038SRao.Shoaib@Sun.COM 		return (-1);	/*%< no OR not enough space in output area */
2980Sstevel@tonic-gate 	hkey = (HMAC_Key *) dkey->dk_KEY_struct;
299*11038SRao.Shoaib@Sun.COM 	memset(buff, 0, buff_len);	/*%< just in case */
3000Sstevel@tonic-gate 	/* write file header */
3010Sstevel@tonic-gate 	sprintf(buff, key_file_fmt_str, KEY_FILE_FORMAT, KEY_HMAC_MD5, "HMAC");
3020Sstevel@tonic-gate 
303*11038SRao.Shoaib@Sun.COM 	bp = buff + strlen(buff);
3040Sstevel@tonic-gate 
3050Sstevel@tonic-gate 	memset(key, 0, HMAC_LEN);
3060Sstevel@tonic-gate 	for (i = 0; i < HMAC_LEN; i++)
3070Sstevel@tonic-gate 		key[i] = hkey->hk_ipad[i] ^ HMAC_IPAD;
3080Sstevel@tonic-gate 	for (i = HMAC_LEN - 1; i >= 0; i--)
3090Sstevel@tonic-gate 		if (key[i] != 0)
3100Sstevel@tonic-gate 			break;
3110Sstevel@tonic-gate 	key_len = i + 1;
3120Sstevel@tonic-gate 
313*11038SRao.Shoaib@Sun.COM 	if (buff_len - (bp - buff) < 6)
314*11038SRao.Shoaib@Sun.COM 		return (-1);
3150Sstevel@tonic-gate 	strcat(bp, "Key: ");
3160Sstevel@tonic-gate 	bp += strlen("Key: ");
3170Sstevel@tonic-gate 
318*11038SRao.Shoaib@Sun.COM 	len = b64_ntop(key, key_len, bp, buff_len - (bp - buff));
3190Sstevel@tonic-gate 	if (len < 0)
3200Sstevel@tonic-gate 		return (-1);
3210Sstevel@tonic-gate 	bp += len;
322*11038SRao.Shoaib@Sun.COM 	if (buff_len - (bp - buff) < 2)
323*11038SRao.Shoaib@Sun.COM 		return (-1);
3240Sstevel@tonic-gate 	*(bp++) = '\n';
3250Sstevel@tonic-gate 	*bp = '\0';
3260Sstevel@tonic-gate 
327*11038SRao.Shoaib@Sun.COM 	return (bp - buff);
3280Sstevel@tonic-gate }
3290Sstevel@tonic-gate 
3300Sstevel@tonic-gate 
3310Sstevel@tonic-gate /**************************************************************************
3320Sstevel@tonic-gate  * dst_hmac_md5_key_from_file_format
3330Sstevel@tonic-gate  *     Converts contents of a key file into an HMAC key.
3340Sstevel@tonic-gate  * Parameters
3350Sstevel@tonic-gate  *     hkey    structure to put key into
3360Sstevel@tonic-gate  *     buff       buffer containing the encoded key
3370Sstevel@tonic-gate  *     buff_len   the length of the buffer
3380Sstevel@tonic-gate  * Return
3390Sstevel@tonic-gate  *     n >= 0 Foot print of the key converted
3400Sstevel@tonic-gate  *     n <  0 Error in conversion
3410Sstevel@tonic-gate  */
3420Sstevel@tonic-gate 
3430Sstevel@tonic-gate static int
dst_hmac_md5_key_from_file_format(DST_KEY * dkey,const char * buff,const int buff_len)3440Sstevel@tonic-gate dst_hmac_md5_key_from_file_format(DST_KEY *dkey, const char *buff,
3450Sstevel@tonic-gate 			      const int buff_len)
3460Sstevel@tonic-gate {
3470Sstevel@tonic-gate 	const char *p = buff, *eol;
3480Sstevel@tonic-gate 	u_char key[HMAC_LEN+1];	/* b64_pton needs more than 64 bytes do decode
349*11038SRao.Shoaib@Sun.COM 				 * it should probably be fixed rather than doing
350*11038SRao.Shoaib@Sun.COM 				 * this
351*11038SRao.Shoaib@Sun.COM 				 */
3520Sstevel@tonic-gate 	u_char *tmp;
3530Sstevel@tonic-gate 	int key_len, len;
3540Sstevel@tonic-gate 
3550Sstevel@tonic-gate 	if (dkey == NULL)
3560Sstevel@tonic-gate 		return (-2);
3570Sstevel@tonic-gate 	if (buff == NULL || buff_len < 0)
3580Sstevel@tonic-gate 		return (-1);
3590Sstevel@tonic-gate 
3600Sstevel@tonic-gate 	memset(key, 0, sizeof(key));
3610Sstevel@tonic-gate 
3620Sstevel@tonic-gate 	if (!dst_s_verify_str(&p, "Key: "))
3630Sstevel@tonic-gate 		return (-3);
3640Sstevel@tonic-gate 
3650Sstevel@tonic-gate 	eol = strchr(p, '\n');
3660Sstevel@tonic-gate 	if (eol == NULL)
3670Sstevel@tonic-gate 		return (-4);
3680Sstevel@tonic-gate 	len = eol - p;
3690Sstevel@tonic-gate 	tmp = malloc(len + 2);
370*11038SRao.Shoaib@Sun.COM 	if (tmp == NULL)
371*11038SRao.Shoaib@Sun.COM 		return (-5);
3720Sstevel@tonic-gate 	memcpy(tmp, p, len);
3730Sstevel@tonic-gate 	*(tmp + len) = 0x0;
374*11038SRao.Shoaib@Sun.COM 	key_len = b64_pton((char *)tmp, key, HMAC_LEN+1);	/*%< see above */
3750Sstevel@tonic-gate 	SAFE_FREE2(tmp, len + 2);
3760Sstevel@tonic-gate 
3770Sstevel@tonic-gate 	if (dst_buffer_to_hmac_md5(dkey, key, key_len) < 0) {
3780Sstevel@tonic-gate 		return (-6);
3790Sstevel@tonic-gate 	}
3800Sstevel@tonic-gate 	return (0);
3810Sstevel@tonic-gate }
3820Sstevel@tonic-gate 
383*11038SRao.Shoaib@Sun.COM /*%
3840Sstevel@tonic-gate  * dst_hmac_md5_to_dns_key()
3850Sstevel@tonic-gate  *         function to extract hmac key from DST_KEY structure
3860Sstevel@tonic-gate  * intput:
3870Sstevel@tonic-gate  *      in_key:  HMAC-MD5 key
3880Sstevel@tonic-gate  * output:
3890Sstevel@tonic-gate  *	out_str: buffer to write ot
3900Sstevel@tonic-gate  *      out_len: size of output buffer
3910Sstevel@tonic-gate  * returns:
3920Sstevel@tonic-gate  *      number of bytes written to output buffer
3930Sstevel@tonic-gate  */
3940Sstevel@tonic-gate static int
dst_hmac_md5_to_dns_key(const DST_KEY * in_key,u_char * out_str,const int out_len)3950Sstevel@tonic-gate dst_hmac_md5_to_dns_key(const DST_KEY *in_key, u_char *out_str,
3960Sstevel@tonic-gate 			const int out_len)
3970Sstevel@tonic-gate {
3980Sstevel@tonic-gate 
3990Sstevel@tonic-gate 	HMAC_Key *hkey;
4000Sstevel@tonic-gate 	int i;
4010Sstevel@tonic-gate 
4020Sstevel@tonic-gate 	if (in_key == NULL || in_key->dk_KEY_struct == NULL ||
4030Sstevel@tonic-gate 	    out_len <= in_key->dk_key_size || out_str == NULL)
4040Sstevel@tonic-gate 		return (-1);
4050Sstevel@tonic-gate 
4060Sstevel@tonic-gate 	hkey = (HMAC_Key *) in_key->dk_KEY_struct;
4070Sstevel@tonic-gate 	for (i = 0; i < in_key->dk_key_size; i++)
4080Sstevel@tonic-gate 		out_str[i] = hkey->hk_ipad[i] ^ HMAC_IPAD;
4090Sstevel@tonic-gate 	return (i);
4100Sstevel@tonic-gate }
4110Sstevel@tonic-gate 
4120Sstevel@tonic-gate /**************************************************************************
4130Sstevel@tonic-gate  *  dst_hmac_md5_compare_keys
4140Sstevel@tonic-gate  *	Compare two keys for equality.
4150Sstevel@tonic-gate  *  Return
4160Sstevel@tonic-gate  *	0	  The keys are equal
4170Sstevel@tonic-gate  *	NON-ZERO   The keys are not equal
4180Sstevel@tonic-gate  */
4190Sstevel@tonic-gate 
4200Sstevel@tonic-gate static int
dst_hmac_md5_compare_keys(const DST_KEY * key1,const DST_KEY * key2)4210Sstevel@tonic-gate dst_hmac_md5_compare_keys(const DST_KEY *key1, const DST_KEY *key2)
4220Sstevel@tonic-gate {
4230Sstevel@tonic-gate 	HMAC_Key *hkey1 = (HMAC_Key *) key1->dk_KEY_struct;
4240Sstevel@tonic-gate 	HMAC_Key *hkey2 = (HMAC_Key *) key2->dk_KEY_struct;
4250Sstevel@tonic-gate 	return memcmp(hkey1->hk_ipad, hkey2->hk_ipad, HMAC_LEN);
4260Sstevel@tonic-gate }
4270Sstevel@tonic-gate 
4280Sstevel@tonic-gate /**************************************************************************
4290Sstevel@tonic-gate  * dst_hmac_md5_free_key_structure
4300Sstevel@tonic-gate  *     Frees all (none) dynamically allocated structures in hkey
4310Sstevel@tonic-gate  */
4320Sstevel@tonic-gate 
4330Sstevel@tonic-gate static void *
dst_hmac_md5_free_key_structure(void * key)4340Sstevel@tonic-gate dst_hmac_md5_free_key_structure(void *key)
4350Sstevel@tonic-gate {
4360Sstevel@tonic-gate 	HMAC_Key *hkey = key;
4370Sstevel@tonic-gate 	SAFE_FREE(hkey);
4380Sstevel@tonic-gate 	return (NULL);
4390Sstevel@tonic-gate }
4400Sstevel@tonic-gate 
4410Sstevel@tonic-gate 
4420Sstevel@tonic-gate /***************************************************************************
4430Sstevel@tonic-gate  * dst_hmac_md5_generate_key
4440Sstevel@tonic-gate  *     Creates a HMAC key of size size with a maximum size of 63 bytes
4450Sstevel@tonic-gate  *     generating a HMAC key larger than 63 bytes makes no sense as that key
4460Sstevel@tonic-gate  *     is digested before use.
4470Sstevel@tonic-gate  */
4480Sstevel@tonic-gate 
4490Sstevel@tonic-gate static int
dst_hmac_md5_generate_key(DST_KEY * key,const int nothing)4500Sstevel@tonic-gate dst_hmac_md5_generate_key(DST_KEY *key, const int nothing)
4510Sstevel@tonic-gate {
452*11038SRao.Shoaib@Sun.COM 	(void)key;
453*11038SRao.Shoaib@Sun.COM 	(void)nothing;
454*11038SRao.Shoaib@Sun.COM 	return (-1);
4550Sstevel@tonic-gate }
4560Sstevel@tonic-gate 
457*11038SRao.Shoaib@Sun.COM /*%
4580Sstevel@tonic-gate  * dst_hmac_md5_init()  Function to answer set up function pointers for HMAC
4590Sstevel@tonic-gate  *	   related functions
4600Sstevel@tonic-gate  */
4610Sstevel@tonic-gate int
dst_hmac_md5_init()4620Sstevel@tonic-gate dst_hmac_md5_init()
4630Sstevel@tonic-gate {
4640Sstevel@tonic-gate 	if (dst_t_func[KEY_HMAC_MD5] != NULL)
4650Sstevel@tonic-gate 		return (1);
4660Sstevel@tonic-gate 	dst_t_func[KEY_HMAC_MD5] = malloc(sizeof(struct dst_func));
4670Sstevel@tonic-gate 	if (dst_t_func[KEY_HMAC_MD5] == NULL)
4680Sstevel@tonic-gate 		return (0);
4690Sstevel@tonic-gate 	memset(dst_t_func[KEY_HMAC_MD5], 0, sizeof(struct dst_func));
4700Sstevel@tonic-gate 	dst_t_func[KEY_HMAC_MD5]->sign = dst_hmac_md5_sign;
4710Sstevel@tonic-gate 	dst_t_func[KEY_HMAC_MD5]->verify = dst_hmac_md5_verify;
4720Sstevel@tonic-gate 	dst_t_func[KEY_HMAC_MD5]->compare = dst_hmac_md5_compare_keys;
4730Sstevel@tonic-gate 	dst_t_func[KEY_HMAC_MD5]->generate = dst_hmac_md5_generate_key;
4740Sstevel@tonic-gate 	dst_t_func[KEY_HMAC_MD5]->destroy = dst_hmac_md5_free_key_structure;
4750Sstevel@tonic-gate 	dst_t_func[KEY_HMAC_MD5]->to_dns_key = dst_hmac_md5_to_dns_key;
4760Sstevel@tonic-gate 	dst_t_func[KEY_HMAC_MD5]->from_dns_key = dst_buffer_to_hmac_md5;
4770Sstevel@tonic-gate 	dst_t_func[KEY_HMAC_MD5]->to_file_fmt = dst_hmac_md5_key_to_file_format;
4780Sstevel@tonic-gate 	dst_t_func[KEY_HMAC_MD5]->from_file_fmt = dst_hmac_md5_key_from_file_format;
4790Sstevel@tonic-gate 	return (1);
4800Sstevel@tonic-gate }
4810Sstevel@tonic-gate 
4820Sstevel@tonic-gate #else
483*11038SRao.Shoaib@Sun.COM #define	dst_hmac_md5_init	__dst_hmac_md5_init
484*11038SRao.Shoaib@Sun.COM 
4850Sstevel@tonic-gate int
dst_hmac_md5_init()4860Sstevel@tonic-gate dst_hmac_md5_init(){
4870Sstevel@tonic-gate 	return (0);
4880Sstevel@tonic-gate }
4890Sstevel@tonic-gate #endif
4900Sstevel@tonic-gate 
491*11038SRao.Shoaib@Sun.COM /*! \file */
492