xref: /onnv-gate/usr/src/uts/common/rpc/sec_gss/rpcsec_gss_misc.c (revision 0:68f95e015346)
1*0Sstevel@tonic-gate /*
2*0Sstevel@tonic-gate  * CDDL HEADER START
3*0Sstevel@tonic-gate  *
4*0Sstevel@tonic-gate  * The contents of this file are subject to the terms of the
5*0Sstevel@tonic-gate  * Common Development and Distribution License, Version 1.0 only
6*0Sstevel@tonic-gate  * (the "License").  You may not use this file except in compliance
7*0Sstevel@tonic-gate  * with the License.
8*0Sstevel@tonic-gate  *
9*0Sstevel@tonic-gate  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10*0Sstevel@tonic-gate  * or http://www.opensolaris.org/os/licensing.
11*0Sstevel@tonic-gate  * See the License for the specific language governing permissions
12*0Sstevel@tonic-gate  * and limitations under the License.
13*0Sstevel@tonic-gate  *
14*0Sstevel@tonic-gate  * When distributing Covered Code, include this CDDL HEADER in each
15*0Sstevel@tonic-gate  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16*0Sstevel@tonic-gate  * If applicable, add the following below this CDDL HEADER, with the
17*0Sstevel@tonic-gate  * fields enclosed by brackets "[]" replaced with your own identifying
18*0Sstevel@tonic-gate  * information: Portions Copyright [yyyy] [name of copyright owner]
19*0Sstevel@tonic-gate  *
20*0Sstevel@tonic-gate  * CDDL HEADER END
21*0Sstevel@tonic-gate  */
22*0Sstevel@tonic-gate /*
23*0Sstevel@tonic-gate  * Copyright 1996,1997,1999,2002-2003 Sun Microsystems, Inc.
24*0Sstevel@tonic-gate  * All rights reserved.  Use is subject to license terms.
25*0Sstevel@tonic-gate  */
26*0Sstevel@tonic-gate 
27*0Sstevel@tonic-gate #pragma ident	"%Z%%M%	%I%	%E% SMI"
28*0Sstevel@tonic-gate 
29*0Sstevel@tonic-gate /*
30*0Sstevel@tonic-gate  * Copyright 1993 OpenVision Technologies, Inc., All Rights Reserved.
31*0Sstevel@tonic-gate  *
32*0Sstevel@tonic-gate  * $Header:
33*0Sstevel@tonic-gate  * /afs/gza.com/product/secure/rel-eng/src/1.1/rpc/RCS/auth_gssapi_misc.c,v 1.10
34*0Sstevel@tonic-gate  * 1994/10/27 12:39:23 jik Exp $
35*0Sstevel@tonic-gate  */
36*0Sstevel@tonic-gate 
37*0Sstevel@tonic-gate #include <sys/param.h>
38*0Sstevel@tonic-gate #include <sys/types.h>
39*0Sstevel@tonic-gate #include <sys/stream.h>
40*0Sstevel@tonic-gate #include <sys/strsubr.h>
41*0Sstevel@tonic-gate #include <sys/cmn_err.h>
42*0Sstevel@tonic-gate #include <gssapi/gssapi.h>
43*0Sstevel@tonic-gate #include <rpc/rpc.h>
44*0Sstevel@tonic-gate #include <rpc/rpcsec_defs.h>
45*0Sstevel@tonic-gate 
46*0Sstevel@tonic-gate /*
47*0Sstevel@tonic-gate  * The initial allocation size for dynamic allocation.
48*0Sstevel@tonic-gate  */
49*0Sstevel@tonic-gate #define	CKU_INITSIZE    2048
50*0Sstevel@tonic-gate 
51*0Sstevel@tonic-gate /*
52*0Sstevel@tonic-gate  * The size of additional allocations, if required.  It is larger to
53*0Sstevel@tonic-gate  * reduce the number of actual allocations.
54*0Sstevel@tonic-gate  */
55*0Sstevel@tonic-gate #define	CKU_ALLOCSIZE   8192
56*0Sstevel@tonic-gate 
57*0Sstevel@tonic-gate 
58*0Sstevel@tonic-gate /*
59*0Sstevel@tonic-gate  * Miscellaneous XDR routines.
60*0Sstevel@tonic-gate  */
61*0Sstevel@tonic-gate bool_t
__xdr_gss_buf(xdrs,buf)62*0Sstevel@tonic-gate __xdr_gss_buf(xdrs, buf)
63*0Sstevel@tonic-gate 	XDR		*xdrs;
64*0Sstevel@tonic-gate 	gss_buffer_t	buf;
65*0Sstevel@tonic-gate {
66*0Sstevel@tonic-gate 	uint_t cast_len, bound_len;
67*0Sstevel@tonic-gate 
68*0Sstevel@tonic-gate 	/*
69*0Sstevel@tonic-gate 	 * We go through this contortion because size_t is a now a ulong,
70*0Sstevel@tonic-gate 	 * GSS-API uses ulongs.
71*0Sstevel@tonic-gate 	 */
72*0Sstevel@tonic-gate 
73*0Sstevel@tonic-gate 	if (xdrs->x_op != XDR_DECODE) {
74*0Sstevel@tonic-gate 		bound_len = cast_len = (uint_t)buf->length;
75*0Sstevel@tonic-gate 	} else {
76*0Sstevel@tonic-gate 		bound_len = (uint_t)-1;
77*0Sstevel@tonic-gate 	}
78*0Sstevel@tonic-gate 
79*0Sstevel@tonic-gate 	if (xdr_bytes(xdrs, (char **)&buf->value, &cast_len,
80*0Sstevel@tonic-gate 	    bound_len) == TRUE) {
81*0Sstevel@tonic-gate 		if (xdrs->x_op == XDR_DECODE)
82*0Sstevel@tonic-gate 			buf->length = cast_len;
83*0Sstevel@tonic-gate 
84*0Sstevel@tonic-gate 		return (TRUE);
85*0Sstevel@tonic-gate 	}
86*0Sstevel@tonic-gate 
87*0Sstevel@tonic-gate 	return (FALSE);
88*0Sstevel@tonic-gate }
89*0Sstevel@tonic-gate 
90*0Sstevel@tonic-gate bool_t
__xdr_rpc_gss_creds(xdrs,creds)91*0Sstevel@tonic-gate __xdr_rpc_gss_creds(xdrs, creds)
92*0Sstevel@tonic-gate 	XDR			*xdrs;
93*0Sstevel@tonic-gate 	rpc_gss_creds		*creds;
94*0Sstevel@tonic-gate {
95*0Sstevel@tonic-gate 	if (!xdr_u_int(xdrs, (uint_t *)&creds->version) ||
96*0Sstevel@tonic-gate 				!xdr_u_int(xdrs, (uint_t *)&creds->gss_proc) ||
97*0Sstevel@tonic-gate 				!xdr_u_int(xdrs, (uint_t *)&creds->seq_num) ||
98*0Sstevel@tonic-gate 				!xdr_u_int(xdrs, (uint_t *)&creds->service) ||
99*0Sstevel@tonic-gate 				!__xdr_gss_buf(xdrs, &creds->ctx_handle))
100*0Sstevel@tonic-gate 		return (FALSE);
101*0Sstevel@tonic-gate 	return (TRUE);
102*0Sstevel@tonic-gate }
103*0Sstevel@tonic-gate 
104*0Sstevel@tonic-gate bool_t
__xdr_rpc_gss_init_arg(xdrs,init_arg)105*0Sstevel@tonic-gate __xdr_rpc_gss_init_arg(xdrs, init_arg)
106*0Sstevel@tonic-gate 	XDR			*xdrs;
107*0Sstevel@tonic-gate 	rpc_gss_init_arg	*init_arg;
108*0Sstevel@tonic-gate {
109*0Sstevel@tonic-gate 	if (!__xdr_gss_buf(xdrs, init_arg))
110*0Sstevel@tonic-gate 		return (FALSE);
111*0Sstevel@tonic-gate 	return (TRUE);
112*0Sstevel@tonic-gate }
113*0Sstevel@tonic-gate 
114*0Sstevel@tonic-gate bool_t
__xdr_rpc_gss_init_res(xdrs,init_res)115*0Sstevel@tonic-gate __xdr_rpc_gss_init_res(xdrs, init_res)
116*0Sstevel@tonic-gate 	XDR			*xdrs;
117*0Sstevel@tonic-gate 	rpc_gss_init_res	*init_res;
118*0Sstevel@tonic-gate {
119*0Sstevel@tonic-gate 	if (!__xdr_gss_buf(xdrs, &init_res->ctx_handle) ||
120*0Sstevel@tonic-gate 			!xdr_u_int(xdrs, (uint_t *)&init_res->gss_major) ||
121*0Sstevel@tonic-gate 			!xdr_u_int(xdrs, (uint_t *)&init_res->gss_minor) ||
122*0Sstevel@tonic-gate 			!xdr_u_int(xdrs, (uint_t *)&init_res->seq_window) ||
123*0Sstevel@tonic-gate 			!__xdr_gss_buf(xdrs, &init_res->token))
124*0Sstevel@tonic-gate 		return (FALSE);
125*0Sstevel@tonic-gate 	return (TRUE);
126*0Sstevel@tonic-gate }
127*0Sstevel@tonic-gate 
128*0Sstevel@tonic-gate /*
129*0Sstevel@tonic-gate  * Generic routine to wrap data used by client and server sides.
130*0Sstevel@tonic-gate  */
131*0Sstevel@tonic-gate bool_t
__rpc_gss_wrap_data(service,qop,context,seq_num,out_xdrs,xdr_func,xdr_ptr)132*0Sstevel@tonic-gate __rpc_gss_wrap_data(service, qop, context, seq_num, out_xdrs,
133*0Sstevel@tonic-gate 			xdr_func, xdr_ptr)
134*0Sstevel@tonic-gate 	OM_uint32		qop;
135*0Sstevel@tonic-gate 	rpc_gss_service_t	service;
136*0Sstevel@tonic-gate 	gss_ctx_id_t		context;
137*0Sstevel@tonic-gate 	uint_t			seq_num;
138*0Sstevel@tonic-gate 	XDR			*out_xdrs;
139*0Sstevel@tonic-gate 	bool_t			(*xdr_func)();
140*0Sstevel@tonic-gate 	caddr_t			xdr_ptr;
141*0Sstevel@tonic-gate {
142*0Sstevel@tonic-gate 	OM_uint32		major, minor;
143*0Sstevel@tonic-gate 	gss_buffer_desc		in_buf, out_buf;
144*0Sstevel@tonic-gate 	XDR			temp_xdrs;
145*0Sstevel@tonic-gate 	char			*mp;
146*0Sstevel@tonic-gate /* EXPORT DELETE START */
147*0Sstevel@tonic-gate 	bool_t			conf_state;
148*0Sstevel@tonic-gate /* EXPORT DELETE END */
149*0Sstevel@tonic-gate 	bool_t			ret = FALSE;
150*0Sstevel@tonic-gate 	int			size;
151*0Sstevel@tonic-gate 
152*0Sstevel@tonic-gate 	/*
153*0Sstevel@tonic-gate 	 * Create a temporary XDR/buffer to hold the data to be wrapped.
154*0Sstevel@tonic-gate 	 * We need an extra bit for the sequence number serialized first.
155*0Sstevel@tonic-gate 	 */
156*0Sstevel@tonic-gate 	size = xdr_sizeof(xdr_func, xdr_ptr) + BYTES_PER_XDR_UNIT;
157*0Sstevel@tonic-gate 	mp = kmem_alloc(size, KM_SLEEP);
158*0Sstevel@tonic-gate 	out_buf.length = 0;
159*0Sstevel@tonic-gate 
160*0Sstevel@tonic-gate 	xdrmem_create(&temp_xdrs, mp, size, XDR_ENCODE);
161*0Sstevel@tonic-gate 
162*0Sstevel@tonic-gate 	/*
163*0Sstevel@tonic-gate 	 * serialize the sequence number into tmp memory
164*0Sstevel@tonic-gate 	 */
165*0Sstevel@tonic-gate 	if (!xdr_u_int(&temp_xdrs, &seq_num))
166*0Sstevel@tonic-gate 		goto fail;
167*0Sstevel@tonic-gate 
168*0Sstevel@tonic-gate 	/*
169*0Sstevel@tonic-gate 	 * serialize the arguments into tmp memory
170*0Sstevel@tonic-gate 	 */
171*0Sstevel@tonic-gate 	if (!(*xdr_func)(&temp_xdrs, xdr_ptr))
172*0Sstevel@tonic-gate 		goto fail;
173*0Sstevel@tonic-gate 
174*0Sstevel@tonic-gate 	/*
175*0Sstevel@tonic-gate 	 * Data to be wrapped goes in in_buf.  If privacy is used,
176*0Sstevel@tonic-gate 	 * out_buf will have wrapped data (in_buf will no longer be
177*0Sstevel@tonic-gate 	 * needed).  If integrity is used, out_buf will have checksum
178*0Sstevel@tonic-gate 	 * which will follow the data in in_buf.
179*0Sstevel@tonic-gate 	 */
180*0Sstevel@tonic-gate 	in_buf.length = xdr_getpos(&temp_xdrs);
181*0Sstevel@tonic-gate 	in_buf.value = (char *)temp_xdrs.x_base;
182*0Sstevel@tonic-gate 
183*0Sstevel@tonic-gate 	switch (service) {
184*0Sstevel@tonic-gate 	case rpc_gss_svc_privacy:
185*0Sstevel@tonic-gate 
186*0Sstevel@tonic-gate /* EXPORT DELETE START */
187*0Sstevel@tonic-gate 		if ((major = kgss_seal(&minor, context, TRUE, qop, &in_buf,
188*0Sstevel@tonic-gate 				&conf_state, &out_buf)) != GSS_S_COMPLETE) {
189*0Sstevel@tonic-gate 			RPCGSS_LOG1(1, "rpc_gss_wrap: kgss_seal failed."
190*0Sstevel@tonic-gate 				"major = %x, minor = %x", major, minor);
191*0Sstevel@tonic-gate 			goto fail;
192*0Sstevel@tonic-gate 		}
193*0Sstevel@tonic-gate 		in_buf.length = 0;	/* in_buf not needed */
194*0Sstevel@tonic-gate 		if (!conf_state)
195*0Sstevel@tonic-gate /* EXPORT DELETE END */
196*0Sstevel@tonic-gate 			goto fail;
197*0Sstevel@tonic-gate /* EXPORT DELETE START */
198*0Sstevel@tonic-gate 		break;
199*0Sstevel@tonic-gate /* EXPORT DELETE END */
200*0Sstevel@tonic-gate 	case rpc_gss_svc_integrity:
201*0Sstevel@tonic-gate 		if ((major = kgss_sign(&minor, context, qop, &in_buf,
202*0Sstevel@tonic-gate 				&out_buf)) != GSS_S_COMPLETE) {
203*0Sstevel@tonic-gate 			RPCGSS_LOG1(1, "rpc_gss_wrap: kgss_sign failed."
204*0Sstevel@tonic-gate 				"major = %x, minor = %x", major, minor);
205*0Sstevel@tonic-gate 			goto fail;
206*0Sstevel@tonic-gate 		}
207*0Sstevel@tonic-gate 		break;
208*0Sstevel@tonic-gate 	default:
209*0Sstevel@tonic-gate 		goto fail;
210*0Sstevel@tonic-gate 	}
211*0Sstevel@tonic-gate 
212*0Sstevel@tonic-gate 	/*
213*0Sstevel@tonic-gate 	 * write out in_buf and out_buf as needed
214*0Sstevel@tonic-gate 	 */
215*0Sstevel@tonic-gate 	if (in_buf.length != 0) {
216*0Sstevel@tonic-gate 		if (!__xdr_gss_buf(out_xdrs, &in_buf))
217*0Sstevel@tonic-gate 			goto fail;
218*0Sstevel@tonic-gate 	}
219*0Sstevel@tonic-gate 
220*0Sstevel@tonic-gate 	if (!__xdr_gss_buf(out_xdrs, &out_buf))
221*0Sstevel@tonic-gate 		goto fail;
222*0Sstevel@tonic-gate 	ret = TRUE;
223*0Sstevel@tonic-gate fail:
224*0Sstevel@tonic-gate 	kmem_free(mp, size);
225*0Sstevel@tonic-gate 	if (out_buf.length != 0)
226*0Sstevel@tonic-gate 		(void) gss_release_buffer(&minor, &out_buf);
227*0Sstevel@tonic-gate 	return (ret);
228*0Sstevel@tonic-gate }
229*0Sstevel@tonic-gate 
230*0Sstevel@tonic-gate /*
231*0Sstevel@tonic-gate  * Generic routine to unwrap data used by client and server sides.
232*0Sstevel@tonic-gate  */
233*0Sstevel@tonic-gate bool_t
__rpc_gss_unwrap_data(service,context,seq_num,qop_check,in_xdrs,xdr_func,xdr_ptr)234*0Sstevel@tonic-gate __rpc_gss_unwrap_data(service, context, seq_num, qop_check, in_xdrs,
235*0Sstevel@tonic-gate 			xdr_func, xdr_ptr)
236*0Sstevel@tonic-gate 	rpc_gss_service_t	service;
237*0Sstevel@tonic-gate 	gss_ctx_id_t		context;
238*0Sstevel@tonic-gate 	uint_t			seq_num;
239*0Sstevel@tonic-gate 	OM_uint32		qop_check;
240*0Sstevel@tonic-gate 	XDR			*in_xdrs;
241*0Sstevel@tonic-gate 	bool_t			(*xdr_func)();
242*0Sstevel@tonic-gate 	caddr_t			xdr_ptr;
243*0Sstevel@tonic-gate {
244*0Sstevel@tonic-gate 	gss_buffer_desc		in_buf, out_buf;
245*0Sstevel@tonic-gate 	XDR			temp_xdrs;
246*0Sstevel@tonic-gate 	uint_t			seq_num2;
247*0Sstevel@tonic-gate 	bool_t			conf = FALSE;
248*0Sstevel@tonic-gate 	OM_uint32		major = GSS_S_COMPLETE, minor = 0;
249*0Sstevel@tonic-gate 	int			qop = 0;
250*0Sstevel@tonic-gate 
251*0Sstevel@tonic-gate 	in_buf.value = NULL;
252*0Sstevel@tonic-gate 	out_buf.value = NULL;
253*0Sstevel@tonic-gate 
254*0Sstevel@tonic-gate 	/*
255*0Sstevel@tonic-gate 	 * Pull out wrapped data.  For privacy service, this is the
256*0Sstevel@tonic-gate 	 * encrypted data.  For integrity service, this is the data
257*0Sstevel@tonic-gate 	 * followed by a checksum.
258*0Sstevel@tonic-gate 	 */
259*0Sstevel@tonic-gate 	if (!__xdr_gss_buf(in_xdrs, &in_buf)) {
260*0Sstevel@tonic-gate 		return (FALSE);
261*0Sstevel@tonic-gate 	}
262*0Sstevel@tonic-gate 
263*0Sstevel@tonic-gate 	if (service == rpc_gss_svc_privacy) {
264*0Sstevel@tonic-gate 		major = GSS_S_FAILURE;
265*0Sstevel@tonic-gate /* EXPORT DELETE START */
266*0Sstevel@tonic-gate 		major = kgss_unseal(&minor, context, &in_buf, &out_buf, &conf,
267*0Sstevel@tonic-gate 					&qop);
268*0Sstevel@tonic-gate /* EXPORT DELETE END */
269*0Sstevel@tonic-gate 		kmem_free(in_buf.value, in_buf.length);
270*0Sstevel@tonic-gate 		if (major != GSS_S_COMPLETE) {
271*0Sstevel@tonic-gate 			RPCGSS_LOG1(1, "rpc_gss_unwrap: kgss_unseal failed."
272*0Sstevel@tonic-gate 				"major = %x, minor = %x", major, minor);
273*0Sstevel@tonic-gate 			return (FALSE);
274*0Sstevel@tonic-gate 		}
275*0Sstevel@tonic-gate 		/*
276*0Sstevel@tonic-gate 		 * Keep the returned token (unencrypted data) in in_buf.
277*0Sstevel@tonic-gate 		 */
278*0Sstevel@tonic-gate 		in_buf.length = out_buf.length;
279*0Sstevel@tonic-gate 		in_buf.value = out_buf.value;
280*0Sstevel@tonic-gate 
281*0Sstevel@tonic-gate 		/*
282*0Sstevel@tonic-gate 		 * If privacy was not used, or if QOP is not what we are
283*0Sstevel@tonic-gate 		 * expecting, fail.
284*0Sstevel@tonic-gate 		 */
285*0Sstevel@tonic-gate 		if (!conf || qop != qop_check)
286*0Sstevel@tonic-gate 			goto fail;
287*0Sstevel@tonic-gate 
288*0Sstevel@tonic-gate 	} else if (service == rpc_gss_svc_integrity) {
289*0Sstevel@tonic-gate 		if (!__xdr_gss_buf(in_xdrs, &out_buf)) {
290*0Sstevel@tonic-gate 			return (FALSE);
291*0Sstevel@tonic-gate 		}
292*0Sstevel@tonic-gate 		major = kgss_verify(&minor, context, &in_buf, &out_buf,
293*0Sstevel@tonic-gate 				&qop);
294*0Sstevel@tonic-gate 		kmem_free(out_buf.value, out_buf.length);
295*0Sstevel@tonic-gate 		if (major != GSS_S_COMPLETE) {
296*0Sstevel@tonic-gate 			kmem_free(in_buf.value, in_buf.length);
297*0Sstevel@tonic-gate 			RPCGSS_LOG1(1, "rpc_gss_unwrap: kgss_verify failed."
298*0Sstevel@tonic-gate 				"major = %x, minor = %x", major, minor);
299*0Sstevel@tonic-gate 			return (FALSE);
300*0Sstevel@tonic-gate 		}
301*0Sstevel@tonic-gate 
302*0Sstevel@tonic-gate 		/*
303*0Sstevel@tonic-gate 		 * If QOP is not what we are expecting, fail.
304*0Sstevel@tonic-gate 		 */
305*0Sstevel@tonic-gate 		if (qop != qop_check)
306*0Sstevel@tonic-gate 			goto fail;
307*0Sstevel@tonic-gate 	}
308*0Sstevel@tonic-gate 
309*0Sstevel@tonic-gate 	xdrmem_create(&temp_xdrs, in_buf.value, in_buf.length, XDR_DECODE);
310*0Sstevel@tonic-gate 
311*0Sstevel@tonic-gate 	/*
312*0Sstevel@tonic-gate 	 * The data consists of the sequence number followed by the
313*0Sstevel@tonic-gate 	 * arguments.  Make sure sequence number is what we are
314*0Sstevel@tonic-gate 	 * expecting (i.e., the value in the header).
315*0Sstevel@tonic-gate 	 */
316*0Sstevel@tonic-gate 	if (!xdr_u_int(&temp_xdrs, &seq_num2))
317*0Sstevel@tonic-gate 		goto fail;
318*0Sstevel@tonic-gate 	if (seq_num2 != seq_num)
319*0Sstevel@tonic-gate 		goto fail;
320*0Sstevel@tonic-gate 
321*0Sstevel@tonic-gate 	/*
322*0Sstevel@tonic-gate 	 * Deserialize the arguments into xdr_ptr, and release in_buf.
323*0Sstevel@tonic-gate 	 */
324*0Sstevel@tonic-gate 	if (!(*xdr_func)(&temp_xdrs, xdr_ptr)) {
325*0Sstevel@tonic-gate 		goto fail;
326*0Sstevel@tonic-gate 	}
327*0Sstevel@tonic-gate 
328*0Sstevel@tonic-gate 	if (service == rpc_gss_svc_privacy)
329*0Sstevel@tonic-gate 		(void) gss_release_buffer(&minor, &in_buf);
330*0Sstevel@tonic-gate 	else
331*0Sstevel@tonic-gate 		kmem_free(in_buf.value, in_buf.length);
332*0Sstevel@tonic-gate 	XDR_DESTROY(&temp_xdrs);
333*0Sstevel@tonic-gate 	return (TRUE);
334*0Sstevel@tonic-gate fail:
335*0Sstevel@tonic-gate 	XDR_DESTROY(&temp_xdrs);
336*0Sstevel@tonic-gate 	if (service == rpc_gss_svc_privacy)
337*0Sstevel@tonic-gate 		(void) gss_release_buffer(&minor, &in_buf);
338*0Sstevel@tonic-gate 	else
339*0Sstevel@tonic-gate 		kmem_free(in_buf.value, in_buf.length);
340*0Sstevel@tonic-gate 	return (FALSE);
341*0Sstevel@tonic-gate }
342