xref: /freebsd-src/sys/contrib/openzfs/module/os/freebsd/zfs/crypto_os.c (revision ce4dcb97ca433b2a2f03fbae957dae0ff16f6f51)
1eda14cbcSMatt Macy /*
2eda14cbcSMatt Macy  * Copyright (c) 2005-2010 Pawel Jakub Dawidek <pjd@FreeBSD.org>
3eda14cbcSMatt Macy  * Copyright (c) 2018 Sean Eric Fagan <sef@ixsystems.com>
4eda14cbcSMatt Macy  * All rights reserved.
5eda14cbcSMatt Macy  *
6eda14cbcSMatt Macy  * Redistribution and use in source and binary forms, with or without
7eda14cbcSMatt Macy  * modification, are permitted provided that the following conditions
8eda14cbcSMatt Macy  * are met:
9eda14cbcSMatt Macy  * 1. Redistributions of source code must retain the above copyright
10eda14cbcSMatt Macy  *    notice, this list of conditions and the following disclaimer.
11eda14cbcSMatt Macy  * 2. Redistributions in binary form must reproduce the above copyright
12eda14cbcSMatt Macy  *    notice, this list of conditions and the following disclaimer in the
13eda14cbcSMatt Macy  *    documentation and/or other materials provided with the distribution.
14eda14cbcSMatt Macy  *
15eda14cbcSMatt Macy  * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
16eda14cbcSMatt Macy  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17eda14cbcSMatt Macy  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18eda14cbcSMatt Macy  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
19eda14cbcSMatt Macy  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20eda14cbcSMatt Macy  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21eda14cbcSMatt Macy  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22eda14cbcSMatt Macy  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23eda14cbcSMatt Macy  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24eda14cbcSMatt Macy  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25eda14cbcSMatt Macy  * SUCH DAMAGE.
26eda14cbcSMatt Macy  *
27eda14cbcSMatt Macy  * Portions of this file are derived from sys/geom/eli/g_eli_hmac.c
28eda14cbcSMatt Macy  */
29eda14cbcSMatt Macy 
30eda14cbcSMatt Macy #include <sys/types.h>
31eda14cbcSMatt Macy #include <sys/errno.h>
32eda14cbcSMatt Macy 
33eda14cbcSMatt Macy #ifdef _KERNEL
34eda14cbcSMatt Macy #include <sys/libkern.h>
35eda14cbcSMatt Macy #include <sys/malloc.h>
36eda14cbcSMatt Macy #include <sys/sysctl.h>
37eda14cbcSMatt Macy #include <opencrypto/cryptodev.h>
38eda14cbcSMatt Macy #include <opencrypto/xform.h>
39eda14cbcSMatt Macy #endif
40eda14cbcSMatt Macy 
41eda14cbcSMatt Macy #include <sys/zio_crypt.h>
42eda14cbcSMatt Macy #include <sys/fs/zfs.h>
43eda14cbcSMatt Macy #include <sys/zio.h>
44eda14cbcSMatt Macy 
45eda14cbcSMatt Macy #include <sys/freebsd_crypto.h>
46eda14cbcSMatt Macy 
47eda14cbcSMatt Macy #define	SHA512_HMAC_BLOCK_SIZE	128
48eda14cbcSMatt Macy 
49eda14cbcSMatt Macy static int crypt_sessions = 0;
50eda14cbcSMatt Macy SYSCTL_DECL(_vfs_zfs);
51eda14cbcSMatt Macy SYSCTL_INT(_vfs_zfs, OID_AUTO, crypt_sessions, CTLFLAG_RD,
52eda14cbcSMatt Macy 	&crypt_sessions, 0, "Number of cryptographic sessions created");
53eda14cbcSMatt Macy 
54eda14cbcSMatt Macy void
55eda14cbcSMatt Macy crypto_mac_init(struct hmac_ctx *ctx, const crypto_key_t *c_key)
56eda14cbcSMatt Macy {
57eda14cbcSMatt Macy 	uint8_t k_ipad[SHA512_HMAC_BLOCK_SIZE],
58eda14cbcSMatt Macy 	    k_opad[SHA512_HMAC_BLOCK_SIZE],
59eda14cbcSMatt Macy 	    key[SHA512_HMAC_BLOCK_SIZE];
60eda14cbcSMatt Macy 	SHA512_CTX lctx;
61eda14cbcSMatt Macy 	int i;
62eda14cbcSMatt Macy 	size_t cl_bytes = CRYPTO_BITS2BYTES(c_key->ck_length);
63eda14cbcSMatt Macy 
64eda14cbcSMatt Macy 	/*
65eda14cbcSMatt Macy 	 * This code is based on the similar code in geom/eli/g_eli_hmac.c
66eda14cbcSMatt Macy 	 */
67da5137abSMartin Matuska 	memset(key, 0, sizeof (key));
68eda14cbcSMatt Macy 	if (c_key->ck_length  == 0)
69eda14cbcSMatt Macy 		/* do nothing */;
70eda14cbcSMatt Macy 	else if (cl_bytes <= SHA512_HMAC_BLOCK_SIZE)
71da5137abSMartin Matuska 		memcpy(key, c_key->ck_data, cl_bytes);
72eda14cbcSMatt Macy 	else {
73eda14cbcSMatt Macy 		/*
74eda14cbcSMatt Macy 		 * If key is longer than 128 bytes reset it to
75eda14cbcSMatt Macy 		 * key = SHA512(key).
76eda14cbcSMatt Macy 		 */
77eda14cbcSMatt Macy 		SHA512_Init(&lctx);
78eda14cbcSMatt Macy 		SHA512_Update(&lctx, c_key->ck_data, cl_bytes);
79eda14cbcSMatt Macy 		SHA512_Final(key, &lctx);
80eda14cbcSMatt Macy 	}
81eda14cbcSMatt Macy 
82eda14cbcSMatt Macy 	/* XOR key with ipad and opad values. */
83eda14cbcSMatt Macy 	for (i = 0; i < sizeof (key); i++) {
84eda14cbcSMatt Macy 		k_ipad[i] = key[i] ^ 0x36;
85eda14cbcSMatt Macy 		k_opad[i] = key[i] ^ 0x5c;
86eda14cbcSMatt Macy 	}
87da5137abSMartin Matuska 	memset(key, 0, sizeof (key));
88eda14cbcSMatt Macy 
89eda14cbcSMatt Macy 	/* Start inner SHA512. */
90eda14cbcSMatt Macy 	SHA512_Init(&ctx->innerctx);
91eda14cbcSMatt Macy 	SHA512_Update(&ctx->innerctx, k_ipad, sizeof (k_ipad));
92da5137abSMartin Matuska 	memset(k_ipad, 0, sizeof (k_ipad));
93eda14cbcSMatt Macy 	/* Start outer SHA512. */
94eda14cbcSMatt Macy 	SHA512_Init(&ctx->outerctx);
95eda14cbcSMatt Macy 	SHA512_Update(&ctx->outerctx, k_opad, sizeof (k_opad));
96da5137abSMartin Matuska 	memset(k_opad, 0, sizeof (k_opad));
97eda14cbcSMatt Macy }
98eda14cbcSMatt Macy 
99eda14cbcSMatt Macy void
100eda14cbcSMatt Macy crypto_mac_update(struct hmac_ctx *ctx, const void *data, size_t datasize)
101eda14cbcSMatt Macy {
102eda14cbcSMatt Macy 	SHA512_Update(&ctx->innerctx, data, datasize);
103eda14cbcSMatt Macy }
104eda14cbcSMatt Macy 
105eda14cbcSMatt Macy void
106eda14cbcSMatt Macy crypto_mac_final(struct hmac_ctx *ctx, void *md, size_t mdsize)
107eda14cbcSMatt Macy {
108eda14cbcSMatt Macy 	uint8_t digest[SHA512_DIGEST_LENGTH];
109eda14cbcSMatt Macy 
110eda14cbcSMatt Macy 	/* Complete inner hash */
111eda14cbcSMatt Macy 	SHA512_Final(digest, &ctx->innerctx);
112eda14cbcSMatt Macy 
113eda14cbcSMatt Macy 	/* Complete outer hash */
114eda14cbcSMatt Macy 	SHA512_Update(&ctx->outerctx, digest, sizeof (digest));
115eda14cbcSMatt Macy 	SHA512_Final(digest, &ctx->outerctx);
116eda14cbcSMatt Macy 
117da5137abSMartin Matuska 	memset(ctx, 0, sizeof (*ctx));
118eda14cbcSMatt Macy 	/* mdsize == 0 means "Give me the whole hash!" */
119eda14cbcSMatt Macy 	if (mdsize == 0)
120eda14cbcSMatt Macy 		mdsize = SHA512_DIGEST_LENGTH;
121da5137abSMartin Matuska 	memcpy(md, digest, mdsize);
122da5137abSMartin Matuska 	memset(digest, 0, sizeof (digest));
123eda14cbcSMatt Macy }
124eda14cbcSMatt Macy 
125eda14cbcSMatt Macy void
126eda14cbcSMatt Macy crypto_mac(const crypto_key_t *key, const void *in_data, size_t in_data_size,
127eda14cbcSMatt Macy     void *out_data, size_t out_data_size)
128eda14cbcSMatt Macy {
129eda14cbcSMatt Macy 	struct hmac_ctx ctx;
130eda14cbcSMatt Macy 
131eda14cbcSMatt Macy 	crypto_mac_init(&ctx, key);
132eda14cbcSMatt Macy 	crypto_mac_update(&ctx, in_data, in_data_size);
133eda14cbcSMatt Macy 	crypto_mac_final(&ctx, out_data, out_data_size);
134eda14cbcSMatt Macy }
135eda14cbcSMatt Macy 
136eda14cbcSMatt Macy static int
137eda14cbcSMatt Macy freebsd_zfs_crypt_done(struct cryptop *crp)
138eda14cbcSMatt Macy {
139eda14cbcSMatt Macy 	freebsd_crypt_session_t *ses;
140eda14cbcSMatt Macy 
141eda14cbcSMatt Macy 	ses = crp->crp_opaque;
142eda14cbcSMatt Macy 	mtx_lock(&ses->fs_lock);
143eda14cbcSMatt Macy 	ses->fs_done = true;
144eda14cbcSMatt Macy 	mtx_unlock(&ses->fs_lock);
145eda14cbcSMatt Macy 	wakeup(crp);
146eda14cbcSMatt Macy 	return (0);
147eda14cbcSMatt Macy }
148eda14cbcSMatt Macy 
149*1f1e2261SMartin Matuska static int
150*1f1e2261SMartin Matuska freebsd_zfs_crypt_done_sync(struct cryptop *crp)
151*1f1e2261SMartin Matuska {
152*1f1e2261SMartin Matuska 
153*1f1e2261SMartin Matuska 	return (0);
154*1f1e2261SMartin Matuska }
155*1f1e2261SMartin Matuska 
156eda14cbcSMatt Macy void
157eda14cbcSMatt Macy freebsd_crypt_freesession(freebsd_crypt_session_t *sess)
158eda14cbcSMatt Macy {
159eda14cbcSMatt Macy 	mtx_destroy(&sess->fs_lock);
160eda14cbcSMatt Macy 	crypto_freesession(sess->fs_sid);
161da5137abSMartin Matuska 	memset(sess, 0, sizeof (*sess));
162eda14cbcSMatt Macy }
163eda14cbcSMatt Macy 
164eda14cbcSMatt Macy static int
165eda14cbcSMatt Macy zfs_crypto_dispatch(freebsd_crypt_session_t *session, struct cryptop *crp)
166eda14cbcSMatt Macy {
167eda14cbcSMatt Macy 	int error;
168eda14cbcSMatt Macy 
169eda14cbcSMatt Macy 	crp->crp_opaque = session;
170eda14cbcSMatt Macy 	for (;;) {
171*1f1e2261SMartin Matuska #if __FreeBSD_version < 1400004
172*1f1e2261SMartin Matuska 		boolean_t async = ((crypto_ses2caps(crp->crp_session) &
173*1f1e2261SMartin Matuska 		    CRYPTOCAP_F_SYNC) == 0);
174*1f1e2261SMartin Matuska #else
175*1f1e2261SMartin Matuska 		boolean_t async = !CRYPTO_SESS_SYNC(crp->crp_session);
176*1f1e2261SMartin Matuska #endif
177*1f1e2261SMartin Matuska 		crp->crp_callback = async ? freebsd_zfs_crypt_done :
178*1f1e2261SMartin Matuska 		    freebsd_zfs_crypt_done_sync;
179eda14cbcSMatt Macy 		error = crypto_dispatch(crp);
180*1f1e2261SMartin Matuska 		if (error == 0) {
181*1f1e2261SMartin Matuska 			if (async) {
182eda14cbcSMatt Macy 				mtx_lock(&session->fs_lock);
183*1f1e2261SMartin Matuska 				while (session->fs_done == false) {
18416038816SMartin Matuska 					msleep(crp, &session->fs_lock, 0,
18516038816SMartin Matuska 					    "zfs_crypto", 0);
186*1f1e2261SMartin Matuska 				}
187eda14cbcSMatt Macy 				mtx_unlock(&session->fs_lock);
188*1f1e2261SMartin Matuska 			}
189eda14cbcSMatt Macy 			error = crp->crp_etype;
190*1f1e2261SMartin Matuska 		}
191*1f1e2261SMartin Matuska 
192*1f1e2261SMartin Matuska 		if (error == ENOMEM) {
193*1f1e2261SMartin Matuska 			pause("zcrnomem", 1);
194*1f1e2261SMartin Matuska 		} else if (error != EAGAIN) {
195eda14cbcSMatt Macy 			break;
196eda14cbcSMatt Macy 		}
197eda14cbcSMatt Macy 		crp->crp_etype = 0;
198eda14cbcSMatt Macy 		crp->crp_flags &= ~CRYPTO_F_DONE;
199eda14cbcSMatt Macy 		session->fs_done = false;
200eda14cbcSMatt Macy 	}
201eda14cbcSMatt Macy 	return (error);
202eda14cbcSMatt Macy }
203eda14cbcSMatt Macy static void
204eda14cbcSMatt Macy freebsd_crypt_uio_debug_log(boolean_t encrypt,
205eda14cbcSMatt Macy     freebsd_crypt_session_t *input_sessionp,
206e92ffd9bSMartin Matuska     const struct zio_crypt_info *c_info,
207184c1b94SMartin Matuska     zfs_uio_t *data_uio,
208eda14cbcSMatt Macy     crypto_key_t *key,
209eda14cbcSMatt Macy     uint8_t *ivbuf,
210eda14cbcSMatt Macy     size_t datalen,
211eda14cbcSMatt Macy     size_t auth_len)
212eda14cbcSMatt Macy {
213eda14cbcSMatt Macy #ifdef FCRYPTO_DEBUG
214eda14cbcSMatt Macy 	struct cryptodesc *crd;
215eda14cbcSMatt Macy 	uint8_t *p = NULL;
216eda14cbcSMatt Macy 	size_t total = 0;
217eda14cbcSMatt Macy 
218c03c5b1cSMartin Matuska 	printf("%s(%s, %p, { %s, %d, %d, %s }, %p, { %p, %u }, "
219eda14cbcSMatt Macy 	    "%p, %u, %u)\n",
220eda14cbcSMatt Macy 	    __FUNCTION__, encrypt ? "encrypt" : "decrypt", input_sessionp,
221eda14cbcSMatt Macy 	    c_info->ci_algname, c_info->ci_crypt_type,
222eda14cbcSMatt Macy 	    (unsigned int)c_info->ci_keylen, c_info->ci_name,
223c03c5b1cSMartin Matuska 	    data_uio, key->ck_data,
224eda14cbcSMatt Macy 	    (unsigned int)key->ck_length,
225eda14cbcSMatt Macy 	    ivbuf, (unsigned int)datalen, (unsigned int)auth_len);
226eda14cbcSMatt Macy 	printf("\tkey = { ");
227eda14cbcSMatt Macy 	for (int i = 0; i < key->ck_length / 8; i++) {
228eda14cbcSMatt Macy 		uint8_t *b = (uint8_t *)key->ck_data;
229eda14cbcSMatt Macy 		printf("%02x ", b[i]);
230eda14cbcSMatt Macy 	}
231eda14cbcSMatt Macy 	printf("}\n");
232184c1b94SMartin Matuska 	for (int i = 0; i < zfs_uio_iovcnt(data_uio); i++) {
233eda14cbcSMatt Macy 		printf("\tiovec #%d: <%p, %u>\n", i,
234184c1b94SMartin Matuska 		    zfs_uio_iovbase(data_uio, i),
235184c1b94SMartin Matuska 		    (unsigned int)zfs_uio_iovlen(data_uio, i));
236184c1b94SMartin Matuska 		total += zfs_uio_iovlen(data_uio, i);
237eda14cbcSMatt Macy 	}
238184c1b94SMartin Matuska 	zfs_uio_resid(data_uio) = total;
239eda14cbcSMatt Macy #endif
240eda14cbcSMatt Macy }
241eda14cbcSMatt Macy /*
242eda14cbcSMatt Macy  * Create a new cryptographic session.  This should
243eda14cbcSMatt Macy  * happen every time the key changes (including when
244eda14cbcSMatt Macy  * it's first loaded).
245eda14cbcSMatt Macy  */
246eda14cbcSMatt Macy int
247eda14cbcSMatt Macy freebsd_crypt_newsession(freebsd_crypt_session_t *sessp,
248e92ffd9bSMartin Matuska     const struct zio_crypt_info *c_info, crypto_key_t *key)
249eda14cbcSMatt Macy {
250da5137abSMartin Matuska 	struct crypto_session_params csp = {0};
251eda14cbcSMatt Macy 	int error = 0;
252eda14cbcSMatt Macy 
253eda14cbcSMatt Macy #ifdef FCRYPTO_DEBUG
254c03c5b1cSMartin Matuska 	printf("%s(%p, { %s, %d, %d, %s }, { %p, %u })\n",
255eda14cbcSMatt Macy 	    __FUNCTION__, sessp,
256eda14cbcSMatt Macy 	    c_info->ci_algname, c_info->ci_crypt_type,
257eda14cbcSMatt Macy 	    (unsigned int)c_info->ci_keylen, c_info->ci_name,
258c03c5b1cSMartin Matuska 	    key->ck_data, (unsigned int)key->ck_length);
259eda14cbcSMatt Macy 	printf("\tkey = { ");
260eda14cbcSMatt Macy 	for (int i = 0; i < key->ck_length / 8; i++) {
261eda14cbcSMatt Macy 		uint8_t *b = (uint8_t *)key->ck_data;
262eda14cbcSMatt Macy 		printf("%02x ", b[i]);
263eda14cbcSMatt Macy 	}
264eda14cbcSMatt Macy 	printf("}\n");
265eda14cbcSMatt Macy #endif
266eda14cbcSMatt Macy 	csp.csp_mode = CSP_MODE_AEAD;
267eda14cbcSMatt Macy 	csp.csp_cipher_key = key->ck_data;
268eda14cbcSMatt Macy 	csp.csp_cipher_klen = key->ck_length / 8;
269eda14cbcSMatt Macy 	switch (c_info->ci_crypt_type) {
270eda14cbcSMatt Macy 		case ZC_TYPE_GCM:
271eda14cbcSMatt Macy 		csp.csp_cipher_alg = CRYPTO_AES_NIST_GCM_16;
272eda14cbcSMatt Macy 		csp.csp_ivlen = AES_GCM_IV_LEN;
273eda14cbcSMatt Macy 		switch (key->ck_length/8) {
274eda14cbcSMatt Macy 		case AES_128_GMAC_KEY_LEN:
275eda14cbcSMatt Macy 		case AES_192_GMAC_KEY_LEN:
276eda14cbcSMatt Macy 		case AES_256_GMAC_KEY_LEN:
277eda14cbcSMatt Macy 			break;
278eda14cbcSMatt Macy 		default:
279eda14cbcSMatt Macy 			error = EINVAL;
280eda14cbcSMatt Macy 			goto bad;
281eda14cbcSMatt Macy 		}
282eda14cbcSMatt Macy 		break;
283eda14cbcSMatt Macy 	case ZC_TYPE_CCM:
284eda14cbcSMatt Macy 		csp.csp_cipher_alg = CRYPTO_AES_CCM_16;
285eda14cbcSMatt Macy 		csp.csp_ivlen = AES_CCM_IV_LEN;
286eda14cbcSMatt Macy 		switch (key->ck_length/8) {
287eda14cbcSMatt Macy 		case AES_128_CBC_MAC_KEY_LEN:
288eda14cbcSMatt Macy 		case AES_192_CBC_MAC_KEY_LEN:
289eda14cbcSMatt Macy 		case AES_256_CBC_MAC_KEY_LEN:
290eda14cbcSMatt Macy 			break;
291eda14cbcSMatt Macy 		default:
292eda14cbcSMatt Macy 			error = EINVAL;
293eda14cbcSMatt Macy 			goto bad;
294eda14cbcSMatt Macy 			break;
295eda14cbcSMatt Macy 		}
296eda14cbcSMatt Macy 		break;
297eda14cbcSMatt Macy 	default:
298eda14cbcSMatt Macy 		error = ENOTSUP;
299eda14cbcSMatt Macy 		goto bad;
300eda14cbcSMatt Macy 	}
301ba27dd8bSMartin Matuska 
302ba27dd8bSMartin Matuska 	/*
303ba27dd8bSMartin Matuska 	 * Disable the use of hardware drivers on FreeBSD 13 and later since
304ba27dd8bSMartin Matuska 	 * common crypto offload drivers impose constraints on AES-GCM AAD
305ba27dd8bSMartin Matuska 	 * lengths that make them unusable for ZFS, and we currently do not have
306ba27dd8bSMartin Matuska 	 * a mechanism to fall back to a software driver for requests not
307ba27dd8bSMartin Matuska 	 * handled by a hardware driver.
308ba27dd8bSMartin Matuska 	 *
309ba27dd8bSMartin Matuska 	 * On 12 we continue to permit the use of hardware drivers since
310ba27dd8bSMartin Matuska 	 * CPU-accelerated drivers such as aesni(4) register themselves as
311ba27dd8bSMartin Matuska 	 * hardware drivers.
312ba27dd8bSMartin Matuska 	 */
313ba27dd8bSMartin Matuska 	error = crypto_newsession(&sessp->fs_sid, &csp, CRYPTOCAP_F_SOFTWARE);
314eda14cbcSMatt Macy 	mtx_init(&sessp->fs_lock, "FreeBSD Cryptographic Session Lock",
315eda14cbcSMatt Macy 	    NULL, MTX_DEF);
316eda14cbcSMatt Macy 	crypt_sessions++;
317eda14cbcSMatt Macy bad:
318eda14cbcSMatt Macy #ifdef FCRYPTO_DEBUG
319eda14cbcSMatt Macy 	if (error)
320eda14cbcSMatt Macy 		printf("%s: returning error %d\n", __FUNCTION__, error);
321eda14cbcSMatt Macy #endif
322eda14cbcSMatt Macy 	return (error);
323eda14cbcSMatt Macy }
324eda14cbcSMatt Macy 
325eda14cbcSMatt Macy int
326eda14cbcSMatt Macy freebsd_crypt_uio(boolean_t encrypt,
327eda14cbcSMatt Macy     freebsd_crypt_session_t *input_sessionp,
328e92ffd9bSMartin Matuska     const struct zio_crypt_info *c_info,
329184c1b94SMartin Matuska     zfs_uio_t *data_uio,
330eda14cbcSMatt Macy     crypto_key_t *key,
331eda14cbcSMatt Macy     uint8_t *ivbuf,
332eda14cbcSMatt Macy     size_t datalen,
333eda14cbcSMatt Macy     size_t auth_len)
334eda14cbcSMatt Macy {
335eda14cbcSMatt Macy 	struct cryptop *crp;
336eda14cbcSMatt Macy 	freebsd_crypt_session_t *session = NULL;
337eda14cbcSMatt Macy 	int error = 0;
338eda14cbcSMatt Macy 	size_t total = 0;
339eda14cbcSMatt Macy 
340eda14cbcSMatt Macy 	freebsd_crypt_uio_debug_log(encrypt, input_sessionp, c_info, data_uio,
341eda14cbcSMatt Macy 	    key, ivbuf, datalen, auth_len);
342184c1b94SMartin Matuska 	for (int i = 0; i < zfs_uio_iovcnt(data_uio); i++)
343184c1b94SMartin Matuska 		total += zfs_uio_iovlen(data_uio, i);
344184c1b94SMartin Matuska 	zfs_uio_resid(data_uio) = total;
345eda14cbcSMatt Macy 	if (input_sessionp == NULL) {
346eda14cbcSMatt Macy 		session = kmem_zalloc(sizeof (*session), KM_SLEEP);
347eda14cbcSMatt Macy 		error = freebsd_crypt_newsession(session, c_info, key);
348eda14cbcSMatt Macy 		if (error)
349eda14cbcSMatt Macy 			goto out;
350eda14cbcSMatt Macy 	} else
351eda14cbcSMatt Macy 		session = input_sessionp;
352eda14cbcSMatt Macy 
353eda14cbcSMatt Macy 	crp = crypto_getreq(session->fs_sid, M_WAITOK);
354eda14cbcSMatt Macy 	if (encrypt) {
355eda14cbcSMatt Macy 		crp->crp_op = CRYPTO_OP_ENCRYPT |
356eda14cbcSMatt Macy 		    CRYPTO_OP_COMPUTE_DIGEST;
357eda14cbcSMatt Macy 	} else {
358eda14cbcSMatt Macy 		crp->crp_op = CRYPTO_OP_DECRYPT |
359eda14cbcSMatt Macy 		    CRYPTO_OP_VERIFY_DIGEST;
360eda14cbcSMatt Macy 	}
361eda14cbcSMatt Macy 	crp->crp_flags = CRYPTO_F_CBIFSYNC | CRYPTO_F_IV_SEPARATE;
362184c1b94SMartin Matuska 	crypto_use_uio(crp, GET_UIO_STRUCT(data_uio));
363eda14cbcSMatt Macy 
364eda14cbcSMatt Macy 	crp->crp_aad_start = 0;
365eda14cbcSMatt Macy 	crp->crp_aad_length = auth_len;
366eda14cbcSMatt Macy 	crp->crp_payload_start = auth_len;
367eda14cbcSMatt Macy 	crp->crp_payload_length = datalen;
368eda14cbcSMatt Macy 	crp->crp_digest_start = auth_len + datalen;
369eda14cbcSMatt Macy 
370da5137abSMartin Matuska 	memcpy(crp->crp_iv, ivbuf, ZIO_DATA_IV_LEN);
371eda14cbcSMatt Macy 	error = zfs_crypto_dispatch(session, crp);
372eda14cbcSMatt Macy 	crypto_freereq(crp);
373eda14cbcSMatt Macy out:
374eda14cbcSMatt Macy #ifdef FCRYPTO_DEBUG
375eda14cbcSMatt Macy 	if (error)
376eda14cbcSMatt Macy 		printf("%s: returning error %d\n", __FUNCTION__, error);
377eda14cbcSMatt Macy #endif
378eda14cbcSMatt Macy 	if (input_sessionp == NULL) {
379eda14cbcSMatt Macy 		freebsd_crypt_freesession(session);
380eda14cbcSMatt Macy 		kmem_free(session, sizeof (*session));
381eda14cbcSMatt Macy 	}
382eda14cbcSMatt Macy 	return (error);
383eda14cbcSMatt Macy }
384