xref: /netbsd-src/sys/dev/pci/ubsec.c (revision f3cfa6f6ce31685c6c4a758bc430e69eb99f50a4)
1 /*	$NetBSD: ubsec.c,v 1.46 2018/12/22 14:07:53 maxv Exp $	*/
2 /* $FreeBSD: src/sys/dev/ubsec/ubsec.c,v 1.6.2.6 2003/01/23 21:06:43 sam Exp $ */
3 /*	$OpenBSD: ubsec.c,v 1.143 2009/03/27 13:31:30 reyk Exp$	*/
4 
5 /*
6  * Copyright (c) 2000 Jason L. Wright (jason@thought.net)
7  * Copyright (c) 2000 Theo de Raadt (deraadt@openbsd.org)
8  * Copyright (c) 2001 Patrik Lindergren (patrik@ipunplugged.com)
9  *
10  * Redistribution and use in source and binary forms, with or without
11  * modification, are permitted provided that the following conditions
12  * are met:
13  * 1. Redistributions of source code must retain the above copyright
14  *    notice, this list of conditions and the following disclaimer.
15  * 2. Redistributions in binary form must reproduce the above copyright
16  *    notice, this list of conditions and the following disclaimer in the
17  *    documentation and/or other materials provided with the distribution.
18  *
19  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
20  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
21  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
22  * DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
23  * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
24  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
25  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
27  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
28  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29  * POSSIBILITY OF SUCH DAMAGE.
30  *
31  * Effort sponsored in part by the Defense Advanced Research Projects
32  * Agency (DARPA) and Air Force Research Laboratory, Air Force
33  * Materiel Command, USAF, under agreement number F30602-01-2-0537.
34  *
35  */
36 
37 #include <sys/cdefs.h>
38 __KERNEL_RCSID(0, "$NetBSD: ubsec.c,v 1.46 2018/12/22 14:07:53 maxv Exp $");
39 
40 #undef UBSEC_DEBUG
41 
42 /*
43  * uBsec 5[56]01, 58xx hardware crypto accelerator
44  */
45 
46 #include <sys/param.h>
47 #include <sys/systm.h>
48 #include <sys/proc.h>
49 #include <sys/endian.h>
50 #ifdef __NetBSD__
51   #define UBSEC_NO_RNG	/* hangs on attach */
52   #define letoh16 htole16
53   #define letoh32 htole32
54 #endif
55 #include <sys/errno.h>
56 #include <sys/malloc.h>
57 #include <sys/kernel.h>
58 #include <sys/mbuf.h>
59 #include <sys/device.h>
60 #include <sys/module.h>
61 #include <sys/queue.h>
62 #include <sys/sysctl.h>
63 
64 #include <opencrypto/cryptodev.h>
65 #include <opencrypto/xform.h>
66 #ifdef __OpenBSD__
67  #include <dev/rndvar.h>
68  #include <sys/md5k.h>
69 #else
70  #include <sys/cprng.h>
71  #include <sys/md5.h>
72  #include <sys/rndpool.h>
73  #include <sys/rndsource.h>
74 #endif
75 #include <sys/sha1.h>
76 
77 #include <dev/pci/pcireg.h>
78 #include <dev/pci/pcivar.h>
79 #include <dev/pci/pcidevs.h>
80 
81 #include <dev/pci/ubsecreg.h>
82 #include <dev/pci/ubsecvar.h>
83 
84 /*
85  * Prototypes and count for the pci_device structure
86  */
87 static	int  ubsec_probe(device_t, cfdata_t, void *);
88 static	void ubsec_attach(device_t, device_t, void *);
89 static	int  ubsec_detach(device_t, int);
90 static	int  ubsec_sysctl_init(void);
91 static	void ubsec_reset_board(struct ubsec_softc *);
92 static	void ubsec_init_board(struct ubsec_softc *);
93 static	void ubsec_init_pciregs(struct pci_attach_args *pa);
94 static	void ubsec_cleanchip(struct ubsec_softc *);
95 static	void ubsec_totalreset(struct ubsec_softc *);
96 static	int  ubsec_free_q(struct ubsec_softc*, struct ubsec_q *);
97 
98 #ifdef __OpenBSD__
99 struct cfattach ubsec_ca = {
100 	sizeof(struct ubsec_softc), ubsec_probe, ubsec_attach,
101 };
102 
103 struct cfdriver ubsec_cd = {
104 	0, "ubsec", DV_DULL
105 };
106 #else
107 CFATTACH_DECL_NEW(ubsec, sizeof(struct ubsec_softc), ubsec_probe, ubsec_attach,
108 	      ubsec_detach, NULL);
109 extern struct cfdriver ubsec_cd;
110 #endif
111 
112 /* patchable */
113 #ifdef	UBSEC_DEBUG
114 extern int ubsec_debug;
115 int ubsec_debug=1;
116 #endif
117 
118 static	int	ubsec_intr(void *);
119 static	int	ubsec_newsession(void*, u_int32_t *, struct cryptoini *);
120 static	int	ubsec_freesession(void*, u_int64_t);
121 static	int	ubsec_process(void*, struct cryptop *, int hint);
122 static	void	ubsec_callback(struct ubsec_softc *, struct ubsec_q *);
123 static	void	ubsec_feed(struct ubsec_softc *);
124 static	void	ubsec_mcopy(struct mbuf *, struct mbuf *, int, int);
125 static	void	ubsec_callback2(struct ubsec_softc *, struct ubsec_q2 *);
126 static	void	ubsec_feed2(struct ubsec_softc *);
127 static	void	ubsec_feed4(struct ubsec_softc *);
128 #ifndef UBSEC_NO_RNG
129 static  void	ubsec_rng(void *);
130 static  void	ubsec_rng_locked(void *);
131 static  void	ubsec_rng_get(size_t, void *);
132 #endif /* UBSEC_NO_RNG */
133 static	int 	ubsec_dma_malloc(struct ubsec_softc *, bus_size_t,
134 				 struct ubsec_dma_alloc *, int);
135 static	void	ubsec_dma_free(struct ubsec_softc *, struct ubsec_dma_alloc *);
136 static	int	ubsec_dmamap_aligned(bus_dmamap_t);
137 
138 static	int	ubsec_kprocess(void*, struct cryptkop *, int);
139 static	int	ubsec_kprocess_modexp_sw(struct ubsec_softc *,
140 					 struct cryptkop *, int);
141 static	int	ubsec_kprocess_modexp_hw(struct ubsec_softc *,
142 					 struct cryptkop *, int);
143 static	int	ubsec_kprocess_rsapriv(struct ubsec_softc *,
144 				       struct cryptkop *, int);
145 static	void	ubsec_kfree(struct ubsec_softc *, struct ubsec_q2 *);
146 static	int	ubsec_ksigbits(struct crparam *);
147 static	void	ubsec_kshift_r(u_int, u_int8_t *, u_int, u_int8_t *, u_int);
148 static	void	ubsec_kshift_l(u_int, u_int8_t *, u_int, u_int8_t *, u_int);
149 
150 #ifdef UBSEC_DEBUG
151 static void	ubsec_dump_pb(volatile struct ubsec_pktbuf *);
152 static void	ubsec_dump_mcr(struct ubsec_mcr *);
153 static	void	ubsec_dump_ctx2(volatile struct ubsec_ctx_keyop *);
154 #endif
155 
156 #define	READ_REG(sc,r) \
157 	bus_space_read_4((sc)->sc_st, (sc)->sc_sh, (r))
158 
159 #define WRITE_REG(sc,reg,val) \
160 	bus_space_write_4((sc)->sc_st, (sc)->sc_sh, reg, val)
161 
162 #define	SWAP32(x) (x) = htole32(ntohl((x)))
163 #ifndef HTOLE32
164  #define	HTOLE32(x) (x) = htole32(x)
165 #endif
166 
167 struct ubsec_stats ubsecstats;
168 
169 static struct sysctllog *ubsec_sysctllog;
170 
171 /*
172  * ubsec_maxbatch controls the number of crypto ops to voluntarily
173  * collect into one submission to the hardware.  This batching happens
174  * when ops are dispatched from the crypto subsystem with a hint that
175  * more are to follow immediately.  These ops must also not be marked
176  * with a ``no delay'' flag.
177  */
178 static	int ubsec_maxbatch = 1;
179 
180 /*
181  * ubsec_maxaggr controls the number of crypto ops to submit to the
182  * hardware as a unit.  This aggregation reduces the number of interrupts
183  * to the host at the expense of increased latency (for all but the last
184  * operation).  For network traffic setting this to one yields the highest
185  * performance but at the expense of more interrupt processing.
186  */
187 static	int ubsec_maxaggr = 1;
188 
189 static const struct ubsec_product {
190 	pci_vendor_id_t		ubsec_vendor;
191 	pci_product_id_t	ubsec_product;
192 	int			ubsec_flags;
193 	int			ubsec_statmask;
194 	int			ubsec_maxaggr;
195 	const char		*ubsec_name;
196 } ubsec_products[] = {
197 	{ PCI_VENDOR_BLUESTEEL,	PCI_PRODUCT_BLUESTEEL_5501,
198 	  0,
199 	  BS_STAT_MCR1_DONE | BS_STAT_DMAERR,
200 	  UBS_MIN_AGGR,
201 	  "Bluesteel 5501"
202 	},
203 	{ PCI_VENDOR_BLUESTEEL,	PCI_PRODUCT_BLUESTEEL_5601,
204 	  UBS_FLAGS_KEY | UBS_FLAGS_RNG,
205 	  BS_STAT_MCR1_DONE | BS_STAT_DMAERR,
206 	  UBS_MIN_AGGR,
207 	  "Bluesteel 5601"
208 	},
209 
210 	{ PCI_VENDOR_BROADCOM,	PCI_PRODUCT_BROADCOM_5801,
211 	  0,
212 	  BS_STAT_MCR1_DONE | BS_STAT_DMAERR,
213 	  UBS_MIN_AGGR,
214 	  "Broadcom BCM5801"
215 	},
216 
217 	{ PCI_VENDOR_BROADCOM,	PCI_PRODUCT_BROADCOM_5802,
218 	  UBS_FLAGS_KEY | UBS_FLAGS_RNG,
219 	  BS_STAT_MCR1_DONE | BS_STAT_DMAERR,
220 	  UBS_MIN_AGGR,
221 	  "Broadcom BCM5802"
222 	},
223 
224 	{ PCI_VENDOR_BROADCOM,	PCI_PRODUCT_BROADCOM_5805,
225 	  UBS_FLAGS_KEY | UBS_FLAGS_RNG,
226 	  BS_STAT_MCR1_DONE | BS_STAT_DMAERR,
227 	  UBS_MIN_AGGR,
228 	  "Broadcom BCM5805"
229 	},
230 
231 	{ PCI_VENDOR_BROADCOM,	PCI_PRODUCT_BROADCOM_5820,
232 	  UBS_FLAGS_KEY | UBS_FLAGS_RNG | UBS_FLAGS_LONGCTX |
233 	      UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY,
234 	  BS_STAT_MCR1_DONE | BS_STAT_DMAERR,
235 	  UBS_MIN_AGGR,
236 	  "Broadcom BCM5820"
237 	},
238 
239 	{ PCI_VENDOR_BROADCOM,	PCI_PRODUCT_BROADCOM_5821,
240 	  UBS_FLAGS_KEY | UBS_FLAGS_RNG | UBS_FLAGS_LONGCTX |
241 	      UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY,
242 	  BS_STAT_MCR1_DONE | BS_STAT_DMAERR |
243 	      BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY,
244 	  UBS_MIN_AGGR,
245 	  "Broadcom BCM5821"
246 	},
247 	{ PCI_VENDOR_SUN,	PCI_PRODUCT_SUN_SCA1K,
248 	  UBS_FLAGS_KEY | UBS_FLAGS_RNG | UBS_FLAGS_LONGCTX |
249 	      UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY,
250 	  BS_STAT_MCR1_DONE | BS_STAT_DMAERR |
251 	      BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY,
252 	  UBS_MIN_AGGR,
253 	  "Sun Crypto Accelerator 1000"
254 	},
255 	{ PCI_VENDOR_SUN,	PCI_PRODUCT_SUN_5821,
256 	  UBS_FLAGS_KEY | UBS_FLAGS_RNG | UBS_FLAGS_LONGCTX |
257 	      UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY,
258 	  BS_STAT_MCR1_DONE | BS_STAT_DMAERR |
259 	      BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY,
260 	  UBS_MIN_AGGR,
261 	  "Broadcom BCM5821 (Sun)"
262 	},
263 
264 	{ PCI_VENDOR_BROADCOM,	PCI_PRODUCT_BROADCOM_5822,
265 	  UBS_FLAGS_KEY | UBS_FLAGS_RNG | UBS_FLAGS_LONGCTX |
266 	      UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY,
267 	  BS_STAT_MCR1_DONE | BS_STAT_DMAERR |
268 	      BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY,
269 	  UBS_MIN_AGGR,
270 	  "Broadcom BCM5822"
271 	},
272 
273 	{ PCI_VENDOR_BROADCOM,	PCI_PRODUCT_BROADCOM_5823,
274 	  UBS_FLAGS_KEY | UBS_FLAGS_RNG | UBS_FLAGS_LONGCTX |
275 	      UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY | UBS_FLAGS_AES,
276 	  BS_STAT_MCR1_DONE | BS_STAT_DMAERR |
277 	      BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY,
278 	  UBS_MIN_AGGR,
279 	  "Broadcom BCM5823"
280 	},
281 
282 	{ PCI_VENDOR_BROADCOM,	PCI_PRODUCT_BROADCOM_5825,
283 	  UBS_FLAGS_KEY | UBS_FLAGS_RNG | UBS_FLAGS_LONGCTX |
284 	      UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY | UBS_FLAGS_AES,
285 	  BS_STAT_MCR1_DONE | BS_STAT_DMAERR |
286 	      BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY,
287 	  UBS_MIN_AGGR,
288 	  "Broadcom BCM5825"
289 	},
290 
291 	{ PCI_VENDOR_BROADCOM,	PCI_PRODUCT_BROADCOM_5860,
292 	  UBS_FLAGS_MULTIMCR | UBS_FLAGS_HWNORM |
293 	      UBS_FLAGS_LONGCTX |
294 	      UBS_FLAGS_RNG | UBS_FLAGS_RNG4 |
295 	      UBS_FLAGS_KEY | UBS_FLAGS_BIGKEY | UBS_FLAGS_AES,
296 	  BS_STAT_MCR1_DONE | BS_STAT_DMAERR |
297 	      BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY |
298 	      BS_STAT_MCR3_ALLEMPTY | BS_STAT_MCR4_ALLEMPTY,
299 	  UBS_MAX_AGGR,
300 	  "Broadcom BCM5860"
301 	},
302 
303 	{ PCI_VENDOR_BROADCOM,	PCI_PRODUCT_BROADCOM_5861,
304 	  UBS_FLAGS_MULTIMCR | UBS_FLAGS_HWNORM |
305 	      UBS_FLAGS_LONGCTX |
306 	      UBS_FLAGS_RNG | UBS_FLAGS_RNG4 |
307 	      UBS_FLAGS_KEY | UBS_FLAGS_BIGKEY | UBS_FLAGS_AES,
308 	  BS_STAT_MCR1_DONE | BS_STAT_DMAERR |
309 	      BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY |
310 	      BS_STAT_MCR3_ALLEMPTY | BS_STAT_MCR4_ALLEMPTY,
311 	  UBS_MAX_AGGR,
312 	  "Broadcom BCM5861"
313 	},
314 
315 	{ PCI_VENDOR_BROADCOM,	PCI_PRODUCT_BROADCOM_5862,
316 	  UBS_FLAGS_MULTIMCR | UBS_FLAGS_HWNORM |
317 	      UBS_FLAGS_LONGCTX |
318 	      UBS_FLAGS_RNG | UBS_FLAGS_RNG4 |
319 	      UBS_FLAGS_KEY | UBS_FLAGS_BIGKEY | UBS_FLAGS_AES,
320 	  BS_STAT_MCR1_DONE | BS_STAT_DMAERR |
321 	      BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY |
322 	      BS_STAT_MCR3_ALLEMPTY | BS_STAT_MCR4_ALLEMPTY,
323 	  UBS_MAX_AGGR,
324 	  "Broadcom BCM5862"
325 	},
326 
327 	{ 0,			0,
328 	  0,
329 	  0,
330 	  0,
331 	  NULL
332 	}
333 };
334 
335 static const struct ubsec_product *
336 ubsec_lookup(const struct pci_attach_args *pa)
337 {
338 	const struct ubsec_product *up;
339 
340 	for (up = ubsec_products; up->ubsec_name != NULL; up++) {
341 		if (PCI_VENDOR(pa->pa_id) == up->ubsec_vendor &&
342 		    PCI_PRODUCT(pa->pa_id) == up->ubsec_product)
343 			return (up);
344 	}
345 	return (NULL);
346 }
347 
348 static int
349 ubsec_probe(device_t parent, cfdata_t match, void *aux)
350 {
351 	struct pci_attach_args *pa = (struct pci_attach_args *)aux;
352 
353 	if (ubsec_lookup(pa) != NULL)
354 		return (1);
355 
356 	return (0);
357 }
358 
359 static void
360 ubsec_attach(device_t parent, device_t self, void *aux)
361 {
362 	struct ubsec_softc *sc = device_private(self);
363 	struct pci_attach_args *pa = aux;
364 	const struct ubsec_product *up;
365 	pci_chipset_tag_t pc = pa->pa_pc;
366 	pci_intr_handle_t ih;
367 	const char *intrstr = NULL;
368 	pcireg_t memtype;
369 	struct ubsec_dma *dmap;
370 	u_int32_t cmd, i;
371 	char intrbuf[PCI_INTRSTR_LEN];
372 
373 	sc->sc_dev = self;
374 	sc->sc_pct = pc;
375 
376 	up = ubsec_lookup(pa);
377 	if (up == NULL) {
378 		printf("\n");
379 		panic("ubsec_attach: impossible");
380 	}
381 
382 	pci_aprint_devinfo_fancy(pa, "Crypto processor", up->ubsec_name, 1);
383 
384 	SIMPLEQ_INIT(&sc->sc_queue);
385 	SIMPLEQ_INIT(&sc->sc_qchip);
386 	SIMPLEQ_INIT(&sc->sc_queue2);
387 	SIMPLEQ_INIT(&sc->sc_qchip2);
388 	SIMPLEQ_INIT(&sc->sc_queue4);
389 	SIMPLEQ_INIT(&sc->sc_qchip4);
390 	SIMPLEQ_INIT(&sc->sc_q2free);
391 
392 	sc->sc_flags = up->ubsec_flags;
393 	sc->sc_statmask = up->ubsec_statmask;
394 	sc->sc_maxaggr = up->ubsec_maxaggr;
395 
396 	cmd = pci_conf_read(pc, pa->pa_tag, PCI_COMMAND_STATUS_REG);
397 	cmd |= PCI_COMMAND_MASTER_ENABLE;
398 	pci_conf_write(pc, pa->pa_tag, PCI_COMMAND_STATUS_REG, cmd);
399 
400 	memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, BS_BAR);
401 	if (pci_mapreg_map(pa, BS_BAR, memtype, 0,
402 	    &sc->sc_st, &sc->sc_sh, NULL, &sc->sc_memsize)) {
403 		aprint_error_dev(self, "can't find mem space");
404 		return;
405 	}
406 
407 	sc->sc_dmat = pa->pa_dmat;
408 
409 	if (pci_intr_map(pa, &ih)) {
410 		aprint_error_dev(self, "couldn't map interrupt\n");
411 		return;
412 	}
413 	intrstr = pci_intr_string(pc, ih, intrbuf, sizeof(intrbuf));
414 	sc->sc_ih = pci_intr_establish_xname(pc, ih, IPL_NET, ubsec_intr, sc,
415 	    device_xname(self));
416 	if (sc->sc_ih == NULL) {
417 		aprint_error_dev(self, "couldn't establish interrupt");
418 		if (intrstr != NULL)
419 			aprint_error(" at %s", intrstr);
420 		aprint_error("\n");
421 		return;
422 	}
423 	aprint_normal_dev(self, "interrupting at %s\n", intrstr);
424 
425 	sc->sc_cid = crypto_get_driverid(0);
426 	if (sc->sc_cid < 0) {
427 		aprint_error_dev(self, "couldn't get crypto driver id\n");
428 		pci_intr_disestablish(pc, sc->sc_ih);
429 		return;
430 	}
431 
432 	sc->sc_rng_need = RND_POOLBITS / NBBY;
433 	mutex_init(&sc->sc_mtx, MUTEX_DEFAULT, IPL_VM);
434 
435 	SIMPLEQ_INIT(&sc->sc_freequeue);
436 	dmap = sc->sc_dmaa;
437 	for (i = 0; i < UBS_MAX_NQUEUE; i++, dmap++) {
438 		struct ubsec_q *q;
439 
440 		q = (struct ubsec_q *)malloc(sizeof(struct ubsec_q),
441 		    M_DEVBUF, M_ZERO|M_NOWAIT);
442 		if (q == NULL) {
443 			aprint_error_dev(self,
444 			    "can't allocate queue buffers\n");
445 			break;
446 		}
447 
448 		if (ubsec_dma_malloc(sc, sizeof(struct ubsec_dmachunk),
449 		    &dmap->d_alloc, 0)) {
450 			aprint_error_dev(self, "can't allocate dma buffers\n");
451 			free(q, M_DEVBUF);
452 			break;
453 		}
454 		dmap->d_dma = (struct ubsec_dmachunk *)dmap->d_alloc.dma_vaddr;
455 
456 		q->q_dma = dmap;
457 		sc->sc_queuea[i] = q;
458 
459 		SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
460 	}
461 
462 	crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0,
463 	    ubsec_newsession, ubsec_freesession, ubsec_process, sc);
464 	crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0,
465 	    ubsec_newsession, ubsec_freesession, ubsec_process, sc);
466 	crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC_96, 0, 0,
467 	    ubsec_newsession, ubsec_freesession, ubsec_process, sc);
468 	crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC_96, 0, 0,
469 	    ubsec_newsession, ubsec_freesession, ubsec_process, sc);
470 	if (sc->sc_flags & UBS_FLAGS_AES) {
471 		crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0,
472 		    ubsec_newsession, ubsec_freesession, ubsec_process, sc);
473 	}
474 
475 	/*
476 	 * Reset Broadcom chip
477 	 */
478 	ubsec_reset_board(sc);
479 
480 	/*
481 	 * Init Broadcom specific PCI settings
482 	 */
483 	ubsec_init_pciregs(pa);
484 
485 	/*
486 	 * Init Broadcom chip
487 	 */
488 	ubsec_init_board(sc);
489 
490 #ifndef UBSEC_NO_RNG
491 	if (sc->sc_flags & UBS_FLAGS_RNG) {
492 		if (sc->sc_flags & UBS_FLAGS_RNG4)
493 			sc->sc_statmask |= BS_STAT_MCR4_DONE;
494 		else
495 			sc->sc_statmask |= BS_STAT_MCR2_DONE;
496 
497 		if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr),
498 		    &sc->sc_rng.rng_q.q_mcr, 0))
499 			goto skip_rng;
500 
501 		if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_rngbypass),
502 		    &sc->sc_rng.rng_q.q_ctx, 0)) {
503 			ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr);
504 			goto skip_rng;
505 		}
506 
507 		if (ubsec_dma_malloc(sc, sizeof(u_int32_t) *
508 		    UBSEC_RNG_BUFSIZ, &sc->sc_rng.rng_buf, 0)) {
509 			ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_ctx);
510 			ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr);
511 			goto skip_rng;
512 		}
513 
514 		rndsource_setcb(&sc->sc_rnd_source, ubsec_rng_get, sc);
515 		rnd_attach_source(&sc->sc_rnd_source, device_xname(sc->sc_dev),
516 				  RND_TYPE_RNG,
517 				  RND_FLAG_COLLECT_VALUE|RND_FLAG_HASCB);
518 		if (hz >= 100)
519 			sc->sc_rnghz = hz / 100;
520 		else
521 			sc->sc_rnghz = 1;
522 #ifdef __OpenBSD__
523 		timeout_set(&sc->sc_rngto, ubsec_rng, sc);
524 		timeout_add(&sc->sc_rngto, sc->sc_rnghz);
525 #else
526 		callout_init(&sc->sc_rngto, 0);
527 		callout_setfunc(&sc->sc_rngto, ubsec_rng, sc);
528 		callout_schedule(&sc->sc_rngto, sc->sc_rnghz);
529 #endif
530  skip_rng:
531 		if (sc->sc_rnghz)
532 			aprint_normal_dev(self,
533 			    "random number generator enabled\n");
534 		else
535 			aprint_error_dev(self,
536 			    "WARNING: random number generator disabled\n");
537 	}
538 #endif /* UBSEC_NO_RNG */
539 
540 	if (sc->sc_flags & UBS_FLAGS_KEY) {
541 		sc->sc_statmask |= BS_STAT_MCR2_DONE;
542 
543 		crypto_kregister(sc->sc_cid, CRK_MOD_EXP, 0,
544 				 ubsec_kprocess, sc);
545 #if 0
546 		crypto_kregister(sc->sc_cid, CRK_MOD_EXP_CRT, 0,
547 				 ubsec_kprocess, sc);
548 #endif
549 	}
550 }
551 
552 static int
553 ubsec_detach(device_t self, int flags)
554 {
555 	struct ubsec_softc *sc = device_private(self);
556 	struct ubsec_q *q, *qtmp;
557 	volatile u_int32_t ctrl;
558 
559 	/* disable interrupts */
560 	/* XXX wait/abort current ops? where is DMAERR enabled? */
561 	ctrl = READ_REG(sc, BS_CTRL);
562 
563 	ctrl &= ~(BS_CTRL_MCR2INT | BS_CTRL_MCR1INT | BS_CTRL_DMAERR);
564 	if (sc->sc_flags & UBS_FLAGS_MULTIMCR)
565 		ctrl &= ~BS_CTRL_MCR4INT;
566 
567 	WRITE_REG(sc, BS_CTRL, ctrl);
568 
569 #ifndef UBSEC_NO_RNG
570 	if (sc->sc_flags & UBS_FLAGS_RNG) {
571 		callout_halt(&sc->sc_rngto, NULL);
572 		ubsec_dma_free(sc, &sc->sc_rng.rng_buf);
573 		ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_ctx);
574 		ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr);
575 		rnd_detach_source(&sc->sc_rnd_source);
576 	}
577 #endif /* UBSEC_NO_RNG */
578 
579 	crypto_unregister_all(sc->sc_cid);
580 
581 	mutex_spin_enter(&sc->sc_mtx);
582 
583 	ubsec_totalreset(sc);  /* XXX leaves the chip running */
584 
585 	SIMPLEQ_FOREACH_SAFE(q, &sc->sc_freequeue, q_next, qtmp) {
586 		ubsec_dma_free(sc, &q->q_dma->d_alloc);
587 		if (q->q_src_map != NULL)
588 			bus_dmamap_destroy(sc->sc_dmat, q->q_src_map);
589 		if (q->q_cached_dst_map != NULL)
590 			bus_dmamap_destroy(sc->sc_dmat, q->q_cached_dst_map);
591 		free(q, M_DEVBUF);
592 	}
593 
594 	mutex_spin_exit(&sc->sc_mtx);
595 
596 	if (sc->sc_ih != NULL) {
597 		pci_intr_disestablish(sc->sc_pct, sc->sc_ih);
598 		sc->sc_ih = NULL;
599 	}
600 
601 	if (sc->sc_memsize != 0) {
602 		bus_space_unmap(sc->sc_st, sc->sc_sh, sc->sc_memsize);
603 		sc->sc_memsize = 0;
604 	}
605 
606 	return 0;
607 }
608 
609 MODULE(MODULE_CLASS_DRIVER, ubsec, "pci,opencrypto");
610 
611 #ifdef _MODULE
612 #include "ioconf.c"
613 #endif
614 
615 static int
616 ubsec_modcmd(modcmd_t cmd, void *data)
617 {
618 	int error = 0;
619 
620 	switch (cmd) {
621 	case MODULE_CMD_INIT:
622 #ifdef _MODULE
623 		error = config_init_component(cfdriver_ioconf_ubsec,
624 		    cfattach_ioconf_ubsec, cfdata_ioconf_ubsec);
625 #endif
626 		if (error == 0)
627 			error = ubsec_sysctl_init();
628 		return error;
629 	case MODULE_CMD_FINI:
630 		if (ubsec_sysctllog != NULL)
631 			sysctl_teardown(&ubsec_sysctllog);
632 #ifdef _MODULE
633 		error = config_fini_component(cfdriver_ioconf_ubsec,
634 		    cfattach_ioconf_ubsec, cfdata_ioconf_ubsec);
635 #endif
636 		return error;
637 	default:
638 		return ENOTTY;
639 	}
640 }
641 
642 static int
643 ubsec_sysctl_init(void)
644 {
645 	const struct sysctlnode *node = NULL;
646 
647 	ubsec_sysctllog = NULL;
648 
649 	sysctl_createv(&ubsec_sysctllog, 0, NULL, &node,
650 		CTLFLAG_PERMANENT,
651 		CTLTYPE_NODE, "ubsec",
652 		SYSCTL_DESCR("ubsec opetions"),
653 		NULL, 0, NULL, 0,
654 		CTL_HW, CTL_CREATE, CTL_EOL);
655 	sysctl_createv(&ubsec_sysctllog, 0, &node, NULL,
656 		CTLFLAG_PERMANENT | CTLFLAG_READWRITE,
657 		CTLTYPE_INT, "maxbatch",
658 		SYSCTL_DESCR("max ops to batch w/o interrupt"),
659 		NULL, 0, &ubsec_maxbatch, 0,
660 		CTL_CREATE, CTL_EOL);
661 	sysctl_createv(&ubsec_sysctllog, 0, &node, NULL,
662 		CTLFLAG_PERMANENT | CTLFLAG_READWRITE,
663 		CTLTYPE_INT, "maxaggr",
664 		SYSCTL_DESCR("max ops to aggregate under one interrupt"),
665 		NULL, 0, &ubsec_maxaggr, 0,
666 		CTL_CREATE, CTL_EOL);
667 
668 	return 0;
669 }
670 
671 /*
672  * UBSEC Interrupt routine
673  */
674 static int
675 ubsec_intr(void *arg)
676 {
677 	struct ubsec_softc *sc = arg;
678 	volatile u_int32_t stat;
679 	struct ubsec_q *q;
680 	struct ubsec_dma *dmap;
681 	int flags;
682 	int npkts = 0, i;
683 
684 	mutex_spin_enter(&sc->sc_mtx);
685 	stat = READ_REG(sc, BS_STAT);
686 	stat &= sc->sc_statmask;
687 	if (stat == 0) {
688 		mutex_spin_exit(&sc->sc_mtx);
689 		return (0);
690 	}
691 
692 	WRITE_REG(sc, BS_STAT, stat);		/* IACK */
693 
694 	/*
695 	 * Check to see if we have any packets waiting for us
696 	 */
697 	if ((stat & BS_STAT_MCR1_DONE)) {
698 		while (!SIMPLEQ_EMPTY(&sc->sc_qchip)) {
699 			q = SIMPLEQ_FIRST(&sc->sc_qchip);
700 			dmap = q->q_dma;
701 
702 			if ((dmap->d_dma->d_mcr.mcr_flags
703 			    & htole16(UBS_MCR_DONE)) == 0)
704 				break;
705 
706 			q = SIMPLEQ_FIRST(&sc->sc_qchip);
707 			SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, /*q,*/ q_next);
708 
709 			npkts = q->q_nstacked_mcrs;
710 			sc->sc_nqchip -= 1+npkts;
711 			/*
712 			 * search for further sc_qchip ubsec_q's that share
713 			 * the same MCR, and complete them too, they must be
714 			 * at the top.
715 			 */
716 			for (i = 0; i < npkts; i++) {
717 				if(q->q_stacked_mcr[i])
718 					ubsec_callback(sc, q->q_stacked_mcr[i]);
719 				else
720 					break;
721 			}
722 			ubsec_callback(sc, q);
723 		}
724 
725 		/*
726 		 * Don't send any more packet to chip if there has been
727 		 * a DMAERR.
728 		 */
729 		if (!(stat & BS_STAT_DMAERR))
730 			ubsec_feed(sc);
731 	}
732 
733 	/*
734 	 * Check to see if we have any key setups/rng's waiting for us
735 	 */
736 	if ((sc->sc_flags & (UBS_FLAGS_KEY|UBS_FLAGS_RNG)) &&
737 	    (stat & BS_STAT_MCR2_DONE)) {
738 		struct ubsec_q2 *q2;
739 		struct ubsec_mcr *mcr;
740 
741 		while (!SIMPLEQ_EMPTY(&sc->sc_qchip2)) {
742 			q2 = SIMPLEQ_FIRST(&sc->sc_qchip2);
743 
744 			bus_dmamap_sync(sc->sc_dmat, q2->q_mcr.dma_map,
745 			    0, q2->q_mcr.dma_map->dm_mapsize,
746 			    BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
747 
748 			mcr = (struct ubsec_mcr *)q2->q_mcr.dma_vaddr;
749 
750 			/* A bug in new devices requires to swap this field */
751 			if (sc->sc_flags & UBS_FLAGS_MULTIMCR)
752 				flags = htole16(mcr->mcr_flags);
753 			else
754 				flags = mcr->mcr_flags;
755 			if ((flags & htole16(UBS_MCR_DONE)) == 0) {
756 				bus_dmamap_sync(sc->sc_dmat,
757 				    q2->q_mcr.dma_map, 0,
758 				    q2->q_mcr.dma_map->dm_mapsize,
759 				    BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
760 				break;
761 			}
762 			q2 = SIMPLEQ_FIRST(&sc->sc_qchip2);
763 			SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip2, /*q2,*/ q_next);
764 			ubsec_callback2(sc, q2);
765 			/*
766 			 * Don't send any more packet to chip if there has been
767 			 * a DMAERR.
768 			 */
769 			if (!(stat & BS_STAT_DMAERR))
770 				ubsec_feed2(sc);
771 		}
772 	}
773 	if ((sc->sc_flags & UBS_FLAGS_RNG4) && (stat & BS_STAT_MCR4_DONE)) {
774 		struct ubsec_q2 *q2;
775 		struct ubsec_mcr *mcr;
776 
777 		while (!SIMPLEQ_EMPTY(&sc->sc_qchip4)) {
778 			q2 = SIMPLEQ_FIRST(&sc->sc_qchip4);
779 
780 			bus_dmamap_sync(sc->sc_dmat, q2->q_mcr.dma_map,
781 			    0, q2->q_mcr.dma_map->dm_mapsize,
782 			    BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
783 
784 			mcr = (struct ubsec_mcr *)q2->q_mcr.dma_vaddr;
785 
786 			/* A bug in new devices requires to swap this field */
787 			flags = htole16(mcr->mcr_flags);
788 
789 			if ((flags & htole16(UBS_MCR_DONE)) == 0) {
790 				bus_dmamap_sync(sc->sc_dmat,
791 				    q2->q_mcr.dma_map, 0,
792 				    q2->q_mcr.dma_map->dm_mapsize,
793 				    BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
794 				break;
795 			}
796 			SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip4, q_next);
797 			ubsec_callback2(sc, q2);
798 			/*
799 			 * Don't send any more packet to chip if there has been
800 			 * a DMAERR.
801 			 */
802 			if (!(stat & BS_STAT_DMAERR))
803 				ubsec_feed4(sc);
804 		}
805 	}
806 
807 	/*
808 	 * Check to see if we got any DMA Error
809 	 */
810 	if (stat & BS_STAT_DMAERR) {
811 #ifdef UBSEC_DEBUG
812 		if (ubsec_debug) {
813 			volatile u_int32_t a = READ_REG(sc, BS_ERR);
814 
815 			printf("%s: dmaerr %s@%08x\n", device_xname(sc->sc_dev),
816 			    (a & BS_ERR_READ) ? "read" : "write",
817 			       a & BS_ERR_ADDR);
818 		}
819 #endif /* UBSEC_DEBUG */
820 		ubsecstats.hst_dmaerr++;
821 		ubsec_totalreset(sc);
822 		ubsec_feed(sc);
823 	}
824 
825 	if (sc->sc_needwakeup) {		/* XXX check high watermark */
826 		int wkeup = sc->sc_needwakeup & (CRYPTO_SYMQ|CRYPTO_ASYMQ);
827 #ifdef UBSEC_DEBUG
828 		if (ubsec_debug)
829 			printf("%s: wakeup crypto (%x)\n",
830 			    device_xname(sc->sc_dev), sc->sc_needwakeup);
831 #endif /* UBSEC_DEBUG */
832 		sc->sc_needwakeup &= ~wkeup;
833 		crypto_unblock(sc->sc_cid, wkeup);
834 	}
835 	mutex_spin_exit(&sc->sc_mtx);
836 	return (1);
837 }
838 
839 /*
840  * ubsec_feed() - aggregate and post requests to chip
841  * OpenBSD comments:
842  *		  It is assumed that the caller set splnet()
843  */
844 static void
845 ubsec_feed(struct ubsec_softc *sc)
846 {
847 	struct ubsec_q *q, *q2;
848 	int npkts, i;
849 	void *v;
850 	u_int32_t stat;
851 #ifdef UBSEC_DEBUG
852 	static int max;
853 #endif /* UBSEC_DEBUG */
854 
855 	npkts = sc->sc_nqueue;
856 	if (npkts > ubsecstats.hst_maxqueue)
857 		ubsecstats.hst_maxqueue = npkts;
858 	if (npkts < 2)
859 		goto feed1;
860 
861 	/*
862 	 * Decide how many ops to combine in a single MCR.  We cannot
863 	 * aggregate more than UBS_MAX_AGGR because this is the number
864 	 * of slots defined in the data structure.  Otherwise we clamp
865 	 * based on the tunable parameter ubsec_maxaggr.  Note that
866 	 * aggregation can happen in two ways: either by batching ops
867 	 * from above or because the h/w backs up and throttles us.
868 	 * Aggregating ops reduces the number of interrupts to the host
869 	 * but also (potentially) increases the latency for processing
870 	 * completed ops as we only get an interrupt when all aggregated
871 	 * ops have completed.
872 	 */
873 	if (npkts > sc->sc_maxaggr)
874 		npkts = sc->sc_maxaggr;
875 	if (npkts > ubsec_maxaggr)
876 		npkts = ubsec_maxaggr;
877 	if (npkts > ubsecstats.hst_maxbatch)
878 		ubsecstats.hst_maxbatch = npkts;
879 	if (npkts < 2)
880 		goto feed1;
881 	ubsecstats.hst_totbatch += npkts-1;
882 
883 	if ((stat = READ_REG(sc, BS_STAT))
884 	    & (BS_STAT_MCR1_FULL | BS_STAT_DMAERR)) {
885 		if (stat & BS_STAT_DMAERR) {
886 			ubsec_totalreset(sc);
887 			ubsecstats.hst_dmaerr++;
888 		} else {
889 			ubsecstats.hst_mcr1full++;
890 		}
891 		return;
892 	}
893 
894 #ifdef UBSEC_DEBUG
895 	if (ubsec_debug)
896 	    printf("merging %d records\n", npkts);
897 	/* XXX temporary aggregation statistics reporting code */
898 	if (max < npkts) {
899 		max = npkts;
900 		printf("%s: new max aggregate %d\n", device_xname(sc->sc_dev),
901 		    max);
902 	}
903 #endif /* UBSEC_DEBUG */
904 
905 	q = SIMPLEQ_FIRST(&sc->sc_queue);
906 	SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, /*q,*/ q_next);
907 	--sc->sc_nqueue;
908 
909 	bus_dmamap_sync(sc->sc_dmat, q->q_src_map,
910 	    0, q->q_src_map->dm_mapsize, BUS_DMASYNC_PREWRITE);
911 	if (q->q_dst_map != NULL)
912 		bus_dmamap_sync(sc->sc_dmat, q->q_dst_map,
913 		    0, q->q_dst_map->dm_mapsize, BUS_DMASYNC_PREREAD);
914 
915 	q->q_nstacked_mcrs = npkts - 1;		/* Number of packets stacked */
916 
917 	for (i = 0; i < q->q_nstacked_mcrs; i++) {
918 		q2 = SIMPLEQ_FIRST(&sc->sc_queue);
919 		bus_dmamap_sync(sc->sc_dmat, q2->q_src_map,
920 		    0, q2->q_src_map->dm_mapsize, BUS_DMASYNC_PREWRITE);
921 		if (q2->q_dst_map != NULL)
922 			bus_dmamap_sync(sc->sc_dmat, q2->q_dst_map,
923 			    0, q2->q_dst_map->dm_mapsize, BUS_DMASYNC_PREREAD);
924 		q2= SIMPLEQ_FIRST(&sc->sc_queue);
925 		SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, /*q2,*/ q_next);
926 		--sc->sc_nqueue;
927 
928 		v = ((void *)&q2->q_dma->d_dma->d_mcr);
929 		v = (char*)v + (sizeof(struct ubsec_mcr) -
930 				 sizeof(struct ubsec_mcr_add));
931 		memcpy(&q->q_dma->d_dma->d_mcradd[i], v,
932 		    sizeof(struct ubsec_mcr_add));
933 		q->q_stacked_mcr[i] = q2;
934 	}
935 	q->q_dma->d_dma->d_mcr.mcr_pkts = htole16(npkts);
936 	SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next);
937 	sc->sc_nqchip += npkts;
938 	if (sc->sc_nqchip > ubsecstats.hst_maxqchip)
939 		ubsecstats.hst_maxqchip = sc->sc_nqchip;
940 	bus_dmamap_sync(sc->sc_dmat, q->q_dma->d_alloc.dma_map,
941 	    0, q->q_dma->d_alloc.dma_map->dm_mapsize,
942 	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
943 	WRITE_REG(sc, BS_MCR1, q->q_dma->d_alloc.dma_paddr +
944 	    offsetof(struct ubsec_dmachunk, d_mcr));
945 	return;
946 
947 feed1:
948 	while (!SIMPLEQ_EMPTY(&sc->sc_queue)) {
949 		if ((stat = READ_REG(sc, BS_STAT))
950 		    & (BS_STAT_MCR1_FULL | BS_STAT_DMAERR)) {
951 			if (stat & BS_STAT_DMAERR) {
952 				ubsec_totalreset(sc);
953 				ubsecstats.hst_dmaerr++;
954 			} else {
955 				ubsecstats.hst_mcr1full++;
956 			}
957 			break;
958 		}
959 
960 		q = SIMPLEQ_FIRST(&sc->sc_queue);
961 
962 		bus_dmamap_sync(sc->sc_dmat, q->q_src_map,
963 		    0, q->q_src_map->dm_mapsize, BUS_DMASYNC_PREWRITE);
964 		if (q->q_dst_map != NULL)
965 			bus_dmamap_sync(sc->sc_dmat, q->q_dst_map,
966 			    0, q->q_dst_map->dm_mapsize, BUS_DMASYNC_PREREAD);
967 		bus_dmamap_sync(sc->sc_dmat, q->q_dma->d_alloc.dma_map,
968 		    0, q->q_dma->d_alloc.dma_map->dm_mapsize,
969 		    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
970 
971 		WRITE_REG(sc, BS_MCR1, q->q_dma->d_alloc.dma_paddr +
972 		    offsetof(struct ubsec_dmachunk, d_mcr));
973 #ifdef UBSEC_DEBUG
974 		if (ubsec_debug)
975 			printf("feed: q->chip %p %08x stat %08x\n",
976  		    	       q, (u_int32_t)q->q_dma->d_alloc.dma_paddr,
977 			       stat);
978 #endif /* UBSEC_DEBUG */
979 		q = SIMPLEQ_FIRST(&sc->sc_queue);
980 		SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, /*q,*/ q_next);
981 		--sc->sc_nqueue;
982 		SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next);
983 		sc->sc_nqchip++;
984 	}
985 	if (sc->sc_nqchip > ubsecstats.hst_maxqchip)
986 		ubsecstats.hst_maxqchip = sc->sc_nqchip;
987 }
988 
989 /*
990  * Allocate a new 'session' and return an encoded session id.  'sidp'
991  * contains our registration id, and should contain an encoded session
992  * id on successful allocation.
993  */
994 static int
995 ubsec_newsession(void *arg, u_int32_t *sidp, struct cryptoini *cri)
996 {
997 	struct cryptoini *c, *encini = NULL, *macini = NULL;
998 	struct ubsec_softc *sc;
999 	struct ubsec_session *ses = NULL;
1000 	MD5_CTX md5ctx;
1001 	SHA1_CTX sha1ctx;
1002 	int i, sesn;
1003 
1004 	sc = arg;
1005 	KASSERT(sc != NULL /*, ("ubsec_newsession: null softc")*/);
1006 
1007 	if (sidp == NULL || cri == NULL || sc == NULL)
1008 		return (EINVAL);
1009 
1010 	for (c = cri; c != NULL; c = c->cri_next) {
1011 		if (c->cri_alg == CRYPTO_MD5_HMAC_96 ||
1012 		    c->cri_alg == CRYPTO_SHA1_HMAC_96) {
1013 			if (macini)
1014 				return (EINVAL);
1015 			macini = c;
1016 		} else if (c->cri_alg == CRYPTO_DES_CBC ||
1017 		    c->cri_alg == CRYPTO_3DES_CBC ||
1018 		    c->cri_alg == CRYPTO_AES_CBC) {
1019 			if (encini)
1020 				return (EINVAL);
1021 			encini = c;
1022 		} else
1023 			return (EINVAL);
1024 	}
1025 	if (encini == NULL && macini == NULL)
1026 		return (EINVAL);
1027 
1028 	if (encini && encini->cri_alg == CRYPTO_AES_CBC) {
1029 		switch (encini->cri_klen) {
1030 		case 128:
1031 		case 192:
1032 		case 256:
1033 			break;
1034 		default:
1035 			return (EINVAL);
1036 		}
1037 	}
1038 
1039 	if (sc->sc_sessions == NULL) {
1040 		ses = sc->sc_sessions = (struct ubsec_session *)malloc(
1041 		    sizeof(struct ubsec_session), M_DEVBUF, M_NOWAIT);
1042 		if (ses == NULL)
1043 			return (ENOMEM);
1044 		sesn = 0;
1045 		sc->sc_nsessions = 1;
1046 	} else {
1047 		for (sesn = 0; sesn < sc->sc_nsessions; sesn++) {
1048 			if (sc->sc_sessions[sesn].ses_used == 0) {
1049 				ses = &sc->sc_sessions[sesn];
1050 				break;
1051 			}
1052 		}
1053 
1054 		if (ses == NULL) {
1055 			sesn = sc->sc_nsessions;
1056 			ses = (struct ubsec_session *)malloc((sesn + 1) *
1057 			    sizeof(struct ubsec_session), M_DEVBUF, M_NOWAIT);
1058 			if (ses == NULL)
1059 				return (ENOMEM);
1060 			memcpy(ses, sc->sc_sessions, sesn *
1061 			    sizeof(struct ubsec_session));
1062 			memset(sc->sc_sessions, 0, sesn *
1063 			    sizeof(struct ubsec_session));
1064 			free(sc->sc_sessions, M_DEVBUF);
1065 			sc->sc_sessions = ses;
1066 			ses = &sc->sc_sessions[sesn];
1067 			sc->sc_nsessions++;
1068 		}
1069 	}
1070 
1071 	memset(ses, 0, sizeof(struct ubsec_session));
1072 	ses->ses_used = 1;
1073 	if (encini) {
1074 		/* get an IV, network byte order */
1075 #ifdef __NetBSD__
1076 		cprng_fast(ses->ses_iv, sizeof(ses->ses_iv));
1077 #else
1078 		get_random_bytes(ses->ses_iv, sizeof(ses->ses_iv));
1079 #endif
1080 
1081 		/* Go ahead and compute key in ubsec's byte order */
1082 		if (encini->cri_alg == CRYPTO_AES_CBC) {
1083 			memcpy(ses->ses_key, encini->cri_key,
1084 			    encini->cri_klen / 8);
1085 		}
1086 		if (encini->cri_alg == CRYPTO_DES_CBC) {
1087 			memcpy(&ses->ses_key[0], encini->cri_key, 8);
1088 			memcpy(&ses->ses_key[2], encini->cri_key, 8);
1089 			memcpy(&ses->ses_key[4], encini->cri_key, 8);
1090 		} else
1091 			memcpy(ses->ses_key, encini->cri_key, 24);
1092 
1093 		SWAP32(ses->ses_key[0]);
1094 		SWAP32(ses->ses_key[1]);
1095 		SWAP32(ses->ses_key[2]);
1096 		SWAP32(ses->ses_key[3]);
1097 		SWAP32(ses->ses_key[4]);
1098 		SWAP32(ses->ses_key[5]);
1099 	}
1100 
1101 	if (macini) {
1102 		for (i = 0; i < macini->cri_klen / 8; i++)
1103 			macini->cri_key[i] ^= HMAC_IPAD_VAL;
1104 
1105 		if (macini->cri_alg == CRYPTO_MD5_HMAC_96) {
1106 			MD5Init(&md5ctx);
1107 			MD5Update(&md5ctx, macini->cri_key,
1108 			    macini->cri_klen / 8);
1109 			MD5Update(&md5ctx, hmac_ipad_buffer,
1110 			    HMAC_BLOCK_LEN - (macini->cri_klen / 8));
1111 			memcpy(ses->ses_hminner, md5ctx.state,
1112 			    sizeof(md5ctx.state));
1113 		} else {
1114 			SHA1Init(&sha1ctx);
1115 			SHA1Update(&sha1ctx, macini->cri_key,
1116 			    macini->cri_klen / 8);
1117 			SHA1Update(&sha1ctx, hmac_ipad_buffer,
1118 			    HMAC_BLOCK_LEN - (macini->cri_klen / 8));
1119 			memcpy(ses->ses_hminner, sha1ctx.state,
1120 			    sizeof(sha1ctx.state));
1121 		}
1122 
1123 		for (i = 0; i < macini->cri_klen / 8; i++)
1124 			macini->cri_key[i] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL);
1125 
1126 		if (macini->cri_alg == CRYPTO_MD5_HMAC_96) {
1127 			MD5Init(&md5ctx);
1128 			MD5Update(&md5ctx, macini->cri_key,
1129 			    macini->cri_klen / 8);
1130 			MD5Update(&md5ctx, hmac_opad_buffer,
1131 			    HMAC_BLOCK_LEN - (macini->cri_klen / 8));
1132 			memcpy(ses->ses_hmouter, md5ctx.state,
1133 			    sizeof(md5ctx.state));
1134 		} else {
1135 			SHA1Init(&sha1ctx);
1136 			SHA1Update(&sha1ctx, macini->cri_key,
1137 			    macini->cri_klen / 8);
1138 			SHA1Update(&sha1ctx, hmac_opad_buffer,
1139 			    HMAC_BLOCK_LEN - (macini->cri_klen / 8));
1140 			memcpy(ses->ses_hmouter, sha1ctx.state,
1141 			    sizeof(sha1ctx.state));
1142 		}
1143 
1144 		for (i = 0; i < macini->cri_klen / 8; i++)
1145 			macini->cri_key[i] ^= HMAC_OPAD_VAL;
1146 	}
1147 
1148 	*sidp = UBSEC_SID(device_unit(sc->sc_dev), sesn);
1149 	return (0);
1150 }
1151 
1152 /*
1153  * Deallocate a session.
1154  */
1155 static int
1156 ubsec_freesession(void *arg, u_int64_t tid)
1157 {
1158 	struct ubsec_softc *sc;
1159 	int session;
1160 	u_int32_t sid = ((u_int32_t) tid) & 0xffffffff;
1161 
1162 	sc = arg;
1163 	KASSERT(sc != NULL /*, ("ubsec_freesession: null softc")*/);
1164 
1165 	session = UBSEC_SESSION(sid);
1166 	if (session >= sc->sc_nsessions)
1167 		return (EINVAL);
1168 
1169 	memset(&sc->sc_sessions[session], 0, sizeof(sc->sc_sessions[session]));
1170 	return (0);
1171 }
1172 
1173 #ifdef __FreeBSD__ /* Ugly gratuitous changes to bus_dma */
1174 static void
1175 ubsec_op_cb(void *arg, bus_dma_segment_t *seg, int nsegs, bus_size_t mapsize,
1176     int error)
1177 {
1178 	struct ubsec_operand *op = arg;
1179 
1180 	KASSERT(nsegs <= UBS_MAX_SCATTER
1181 		/*, ("Too many DMA segments returned when mapping operand")*/);
1182 #ifdef UBSEC_DEBUG
1183 	if (ubsec_debug)
1184 		printf("ubsec_op_cb: mapsize %u nsegs %d\n",
1185 			(u_int) mapsize, nsegs);
1186 #endif
1187 	op->mapsize = mapsize;
1188 	op->nsegs = nsegs;
1189 	memcpy(op->segs, seg, nsegs * sizeof (seg[0]));
1190 }
1191 #endif
1192 
1193 static int
1194 ubsec_process(void *arg, struct cryptop *crp, int hint)
1195 {
1196 	struct ubsec_q *q = NULL;
1197 #ifdef	__OpenBSD__
1198 	int card;
1199 #endif
1200 	int err = 0, i, j, nicealign;
1201 	struct ubsec_softc *sc;
1202 	struct cryptodesc *crd1, *crd2, *maccrd, *enccrd;
1203 	int encoffset = 0, macoffset = 0, cpskip, cpoffset;
1204 	int sskip, dskip, stheend, dtheend;
1205 	int16_t coffset;
1206 	struct ubsec_session *ses, key;
1207 	struct ubsec_dma *dmap = NULL;
1208 	u_int16_t flags = 0;
1209 	int ivlen = 0, keylen = 0;
1210 
1211 	sc = arg;
1212 	KASSERT(sc != NULL /*, ("ubsec_process: null softc")*/);
1213 
1214 	if (crp == NULL || crp->crp_callback == NULL || sc == NULL) {
1215 		ubsecstats.hst_invalid++;
1216 		return (EINVAL);
1217 	}
1218 	if (UBSEC_SESSION(crp->crp_sid) >= sc->sc_nsessions) {
1219 		ubsecstats.hst_badsession++;
1220 		return (EINVAL);
1221 	}
1222 
1223 	mutex_spin_enter(&sc->sc_mtx);
1224 
1225 	if (SIMPLEQ_EMPTY(&sc->sc_freequeue)) {
1226 		ubsecstats.hst_queuefull++;
1227 		sc->sc_needwakeup |= CRYPTO_SYMQ;
1228 		mutex_spin_exit(&sc->sc_mtx);
1229 		return(ERESTART);
1230 	}
1231 
1232 	q = SIMPLEQ_FIRST(&sc->sc_freequeue);
1233 	SIMPLEQ_REMOVE_HEAD(&sc->sc_freequeue, /*q,*/ q_next);
1234 	mutex_spin_exit(&sc->sc_mtx);
1235 
1236 	dmap = q->q_dma; /* Save dma pointer */
1237 	/* don't lose the cached dmamaps q_src_map and q_cached_dst_map */
1238 	memset(q, 0, offsetof(struct ubsec_q, q_src_map));
1239 	memset(&key, 0, sizeof(key));
1240 
1241 	q->q_sesn = UBSEC_SESSION(crp->crp_sid);
1242 	q->q_dma = dmap;
1243 	ses = &sc->sc_sessions[q->q_sesn];
1244 
1245 	if (crp->crp_flags & CRYPTO_F_IMBUF) {
1246 		q->q_src_m = (struct mbuf *)crp->crp_buf;
1247 		q->q_dst_m = (struct mbuf *)crp->crp_buf;
1248 	} else if (crp->crp_flags & CRYPTO_F_IOV) {
1249 		q->q_src_io = (struct uio *)crp->crp_buf;
1250 		q->q_dst_io = (struct uio *)crp->crp_buf;
1251 	} else {
1252 		ubsecstats.hst_badflags++;
1253 		err = EINVAL;
1254 		goto errout;	/* XXX we don't handle contiguous blocks! */
1255 	}
1256 
1257 	memset(&dmap->d_dma->d_mcr, 0, sizeof(struct ubsec_mcr));
1258 
1259 	dmap->d_dma->d_mcr.mcr_pkts = htole16(1);
1260 	dmap->d_dma->d_mcr.mcr_flags = 0;
1261 	q->q_crp = crp;
1262 
1263 	crd1 = crp->crp_desc;
1264 	if (crd1 == NULL) {
1265 		ubsecstats.hst_nodesc++;
1266 		err = EINVAL;
1267 		goto errout;
1268 	}
1269 	crd2 = crd1->crd_next;
1270 
1271 	if (crd2 == NULL) {
1272 		if (crd1->crd_alg == CRYPTO_MD5_HMAC_96 ||
1273 		    crd1->crd_alg == CRYPTO_SHA1_HMAC_96) {
1274 			maccrd = crd1;
1275 			enccrd = NULL;
1276 		} else if (crd1->crd_alg == CRYPTO_DES_CBC ||
1277 		    crd1->crd_alg == CRYPTO_3DES_CBC ||
1278 		    crd1->crd_alg == CRYPTO_AES_CBC) {
1279 			maccrd = NULL;
1280 			enccrd = crd1;
1281 		} else {
1282 			ubsecstats.hst_badalg++;
1283 			err = EINVAL;
1284 			goto errout;
1285 		}
1286 	} else {
1287 		if ((crd1->crd_alg == CRYPTO_MD5_HMAC_96 ||
1288 		    crd1->crd_alg == CRYPTO_SHA1_HMAC_96) &&
1289 		    (crd2->crd_alg == CRYPTO_DES_CBC ||
1290 		    crd2->crd_alg == CRYPTO_3DES_CBC ||
1291 		    crd2->crd_alg == CRYPTO_AES_CBC) &&
1292 		    ((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) {
1293 			maccrd = crd1;
1294 			enccrd = crd2;
1295 		} else if ((crd1->crd_alg == CRYPTO_DES_CBC ||
1296 		    crd1->crd_alg == CRYPTO_3DES_CBC ||
1297 		    crd1->crd_alg == CRYPTO_AES_CBC) &&
1298 		    (crd2->crd_alg == CRYPTO_MD5_HMAC_96 ||
1299 		    crd2->crd_alg == CRYPTO_SHA1_HMAC_96) &&
1300 		    (crd1->crd_flags & CRD_F_ENCRYPT)) {
1301 			enccrd = crd1;
1302 			maccrd = crd2;
1303 		} else {
1304 			/*
1305 			 * We cannot order the ubsec as requested
1306 			 */
1307 			ubsecstats.hst_badalg++;
1308 			err = EINVAL;
1309 			goto errout;
1310 		}
1311 	}
1312 
1313 	if (enccrd) {
1314 		if (enccrd->crd_alg == CRYPTO_AES_CBC) {
1315 			if ((sc->sc_flags & UBS_FLAGS_AES) == 0) {
1316 				/*
1317 				 * We cannot order the ubsec as requested
1318 				 */
1319 				ubsecstats.hst_badalg++;
1320 				err = EINVAL;
1321 				goto errout;
1322 			}
1323 			flags |= htole16(UBS_PKTCTX_ENC_AES);
1324 			switch (enccrd->crd_klen) {
1325 			case 128:
1326 			case 192:
1327 			case 256:
1328 				keylen = enccrd->crd_klen / 8;
1329 				break;
1330 			default:
1331 				err = EINVAL;
1332 				goto errout;
1333 			}
1334 			ivlen = 16;
1335 		} else {
1336 			flags |= htole16(UBS_PKTCTX_ENC_3DES);
1337 			ivlen = 8;
1338 			keylen = 24;
1339 		}
1340 
1341 		encoffset = enccrd->crd_skip;
1342 
1343 		if (enccrd->crd_flags & CRD_F_ENCRYPT) {
1344 			q->q_flags |= UBSEC_QFLAGS_COPYOUTIV;
1345 
1346 			if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
1347 				memcpy(key.ses_iv, enccrd->crd_iv, ivlen);
1348 			else {
1349 				for (i = 0; i < (ivlen / 4); i++)
1350 					key.ses_iv[i] = ses->ses_iv[i];
1351 			}
1352 
1353 			if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) {
1354 				if (crp->crp_flags & CRYPTO_F_IMBUF)
1355 					m_copyback(q->q_src_m,
1356 					    enccrd->crd_inject,
1357 					    ivlen, (void *)key.ses_iv);
1358 				else if (crp->crp_flags & CRYPTO_F_IOV)
1359 					cuio_copyback(q->q_src_io,
1360 					    enccrd->crd_inject,
1361 					    ivlen, (void *)key.ses_iv);
1362 			}
1363 		} else {
1364 			flags |= htole16(UBS_PKTCTX_INBOUND);
1365 
1366 			if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
1367 				memcpy(key.ses_iv, enccrd->crd_iv, ivlen);
1368 			else if (crp->crp_flags & CRYPTO_F_IMBUF)
1369 				m_copydata(q->q_src_m, enccrd->crd_inject,
1370 				    ivlen, (void *)key.ses_iv);
1371 			else if (crp->crp_flags & CRYPTO_F_IOV)
1372 				cuio_copydata(q->q_src_io,
1373 				    enccrd->crd_inject, 8,
1374 				    (void *)key.ses_iv);
1375 		}
1376 
1377 		for (i = 0; i < (keylen / 4); i++)
1378 			key.ses_key[i] = ses->ses_key[i];
1379 		for (i = 0; i < (ivlen / 4); i++)
1380 			SWAP32(key.ses_iv[i]);
1381 	}
1382 
1383 	if (maccrd) {
1384 		macoffset = maccrd->crd_skip;
1385 
1386 		if (maccrd->crd_alg == CRYPTO_MD5_HMAC_96)
1387 			flags |= htole16(UBS_PKTCTX_AUTH_MD5);
1388 		else
1389 			flags |= htole16(UBS_PKTCTX_AUTH_SHA1);
1390 
1391 		for (i = 0; i < 5; i++) {
1392 			key.ses_hminner[i] = ses->ses_hminner[i];
1393 			key.ses_hmouter[i] = ses->ses_hmouter[i];
1394 
1395 			HTOLE32(key.ses_hminner[i]);
1396 			HTOLE32(key.ses_hmouter[i]);
1397 		}
1398 	}
1399 
1400 	if (enccrd && maccrd) {
1401 		/*
1402 		 * ubsec cannot handle packets where the end of encryption
1403 		 * and authentication are not the same, or where the
1404 		 * encrypted part begins before the authenticated part.
1405 		 */
1406 		if ((encoffset + enccrd->crd_len) !=
1407 		    (macoffset + maccrd->crd_len)) {
1408 			ubsecstats.hst_lenmismatch++;
1409 			err = EINVAL;
1410 			goto errout;
1411 		}
1412 		if (enccrd->crd_skip < maccrd->crd_skip) {
1413 			ubsecstats.hst_skipmismatch++;
1414 			err = EINVAL;
1415 			goto errout;
1416 		}
1417 		sskip = maccrd->crd_skip;
1418 		cpskip = dskip = enccrd->crd_skip;
1419 		stheend = maccrd->crd_len;
1420 		dtheend = enccrd->crd_len;
1421 		coffset = enccrd->crd_skip - maccrd->crd_skip;
1422 		cpoffset = cpskip + dtheend;
1423 #ifdef UBSEC_DEBUG
1424 		if (ubsec_debug) {
1425 			printf("mac: skip %d, len %d, inject %d\n",
1426 			    maccrd->crd_skip, maccrd->crd_len,
1427 			    maccrd->crd_inject);
1428 			printf("enc: skip %d, len %d, inject %d\n",
1429 			    enccrd->crd_skip, enccrd->crd_len,
1430 			    enccrd->crd_inject);
1431 			printf("src: skip %d, len %d\n", sskip, stheend);
1432 			printf("dst: skip %d, len %d\n", dskip, dtheend);
1433 			printf("ubs: coffset %d, pktlen %d, cpskip %d, cpoffset %d\n",
1434 			       coffset, stheend, cpskip, cpoffset);
1435 		}
1436 #endif
1437 	} else {
1438 		cpskip = dskip = sskip = macoffset + encoffset;
1439 		dtheend = stheend = (enccrd)?enccrd->crd_len:maccrd->crd_len;
1440 		cpoffset = cpskip + dtheend;
1441 		coffset = 0;
1442 	}
1443 
1444 	if (q->q_src_map == NULL) {
1445 		/* XXX FIXME: jonathan asks, what the heck's that 0xfff0?  */
1446 		if (bus_dmamap_create(sc->sc_dmat, 0xfff0, UBS_MAX_SCATTER,
1447 			0xfff0, 0, BUS_DMA_NOWAIT, &q->q_src_map) != 0) {
1448 			err = ENOMEM;
1449 			goto errout;
1450 		}
1451 	}
1452 	if (crp->crp_flags & CRYPTO_F_IMBUF) {
1453 		if (bus_dmamap_load_mbuf(sc->sc_dmat, q->q_src_map,
1454 		    q->q_src_m, BUS_DMA_NOWAIT) != 0) {
1455 			ubsecstats.hst_noload++;
1456 			err = ENOMEM;
1457 			goto errout;
1458 		}
1459 	} else if (crp->crp_flags & CRYPTO_F_IOV) {
1460 		if (bus_dmamap_load_uio(sc->sc_dmat, q->q_src_map,
1461 		    q->q_src_io, BUS_DMA_NOWAIT) != 0) {
1462 			ubsecstats.hst_noload++;
1463 			err = ENOMEM;
1464 			goto errout;
1465 		}
1466 	}
1467 	nicealign = ubsec_dmamap_aligned(q->q_src_map);
1468 
1469 	dmap->d_dma->d_mcr.mcr_pktlen = htole16(stheend);
1470 
1471 #ifdef UBSEC_DEBUG
1472 	if (ubsec_debug)
1473 		printf("src skip: %d nicealign: %u\n", sskip, nicealign);
1474 #endif
1475 	for (i = j = 0; i < q->q_src_map->dm_nsegs; i++) {
1476 		struct ubsec_pktbuf *pb;
1477 		bus_size_t packl = q->q_src_map->dm_segs[i].ds_len;
1478 		bus_addr_t packp = q->q_src_map->dm_segs[i].ds_addr;
1479 
1480 		if (sskip >= packl) {
1481 			sskip -= packl;
1482 			continue;
1483 		}
1484 
1485 		packl -= sskip;
1486 		packp += sskip;
1487 		sskip = 0;
1488 
1489 		if (packl > 0xfffc) {
1490 			err = EIO;
1491 			goto errout;
1492 		}
1493 
1494 		if (j == 0)
1495 			pb = &dmap->d_dma->d_mcr.mcr_ipktbuf;
1496 		else
1497 			pb = &dmap->d_dma->d_sbuf[j - 1];
1498 
1499 		pb->pb_addr = htole32(packp);
1500 
1501 		if (stheend) {
1502 			if (packl > stheend) {
1503 				pb->pb_len = htole32(stheend);
1504 				stheend = 0;
1505 			} else {
1506 				pb->pb_len = htole32(packl);
1507 				stheend -= packl;
1508 			}
1509 		} else
1510 			pb->pb_len = htole32(packl);
1511 
1512 		if ((i + 1) == q->q_src_map->dm_nsegs)
1513 			pb->pb_next = 0;
1514 		else
1515 			pb->pb_next = htole32(dmap->d_alloc.dma_paddr +
1516 			    offsetof(struct ubsec_dmachunk, d_sbuf[j]));
1517 		j++;
1518 	}
1519 
1520 	if (enccrd == NULL && maccrd != NULL) {
1521 		dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr = 0;
1522 		dmap->d_dma->d_mcr.mcr_opktbuf.pb_len = 0;
1523 		dmap->d_dma->d_mcr.mcr_opktbuf.pb_next = htole32(dmap->d_alloc.dma_paddr +
1524 		    offsetof(struct ubsec_dmachunk, d_macbuf[0]));
1525 #ifdef UBSEC_DEBUG
1526 		if (ubsec_debug)
1527 			printf("opkt: %x %x %x\n",
1528 	 		    dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr,
1529 	 		    dmap->d_dma->d_mcr.mcr_opktbuf.pb_len,
1530 	 		    dmap->d_dma->d_mcr.mcr_opktbuf.pb_next);
1531 
1532 #endif
1533 	} else {
1534 		if (crp->crp_flags & CRYPTO_F_IOV) {
1535 			if (!nicealign) {
1536 				ubsecstats.hst_iovmisaligned++;
1537 				err = EINVAL;
1538 				goto errout;
1539 			}
1540 			if (q->q_dst_map == NULL) {
1541 				if (q->q_cached_dst_map == NULL) {
1542 					/*
1543 					 * XXX: ``what the heck's that''
1544 					 * 0xfff0?
1545 					 */
1546 					if (bus_dmamap_create(sc->sc_dmat,
1547 					    0xfff0, UBS_MAX_SCATTER, 0xfff0, 0,
1548 					    BUS_DMA_NOWAIT,
1549 					    &q->q_cached_dst_map) != 0) {
1550 						ubsecstats.hst_nomap++;
1551 						err = ENOMEM;
1552 						goto errout;
1553 					}
1554 				}
1555 				q->q_dst_map = q->q_cached_dst_map;
1556 			}
1557 			if (bus_dmamap_load_uio(sc->sc_dmat, q->q_dst_map,
1558 			    q->q_dst_io, BUS_DMA_NOWAIT) != 0) {
1559 				ubsecstats.hst_noload++;
1560 				err = ENOMEM;
1561 				goto errout;
1562 			}
1563 		} else if (crp->crp_flags & CRYPTO_F_IMBUF) {
1564 			if (nicealign) {
1565 				q->q_dst_m = q->q_src_m;
1566 				q->q_dst_map = q->q_src_map;
1567 			} else {
1568 				int totlen, len;
1569 				struct mbuf *m, *top, **mp;
1570 
1571 				ubsecstats.hst_unaligned++;
1572 				totlen = q->q_src_map->dm_mapsize;
1573 				if (q->q_src_m->m_flags & M_PKTHDR) {
1574 					len = MHLEN;
1575 					MGETHDR(m, M_DONTWAIT, MT_DATA);
1576 					/*XXX FIXME: m_dup_pkthdr */
1577 					if (m && 1 /*!m_dup_pkthdr(m, q->q_src_m, M_DONTWAIT)*/) {
1578 						m_free(m);
1579 						m = NULL;
1580 					}
1581 				} else {
1582 					len = MLEN;
1583 					MGET(m, M_DONTWAIT, MT_DATA);
1584 				}
1585 				if (m == NULL) {
1586 					ubsecstats.hst_nombuf++;
1587 					err = sc->sc_nqueue ? ERESTART : ENOMEM;
1588 					goto errout;
1589 				}
1590 				if (len == MHLEN)
1591 				  /*XXX was M_DUP_PKTHDR*/
1592 				  m_copy_pkthdr(m, q->q_src_m);
1593 				if (totlen >= MINCLSIZE) {
1594 					MCLGET(m, M_DONTWAIT);
1595 					if ((m->m_flags & M_EXT) == 0) {
1596 						m_free(m);
1597 						ubsecstats.hst_nomcl++;
1598 						err = sc->sc_nqueue
1599 						    ? ERESTART : ENOMEM;
1600 						goto errout;
1601 					}
1602 					len = MCLBYTES;
1603 				}
1604 				m->m_len = len;
1605 				top = NULL;
1606 				mp = &top;
1607 
1608 				while (totlen > 0) {
1609 					if (top) {
1610 						MGET(m, M_DONTWAIT, MT_DATA);
1611 						if (m == NULL) {
1612 							m_freem(top);
1613 							ubsecstats.hst_nombuf++;
1614 							err = sc->sc_nqueue ? ERESTART : ENOMEM;
1615 							goto errout;
1616 						}
1617 						len = MLEN;
1618 					}
1619 					if (top && totlen >= MINCLSIZE) {
1620 						MCLGET(m, M_DONTWAIT);
1621 						if ((m->m_flags & M_EXT) == 0) {
1622 							*mp = m;
1623 							m_freem(top);
1624 							ubsecstats.hst_nomcl++;
1625 							err = sc->sc_nqueue ? ERESTART : ENOMEM;
1626 							goto errout;
1627 						}
1628 						len = MCLBYTES;
1629 					}
1630 					m->m_len = len = uimin(totlen, len);
1631 					totlen -= len;
1632 					*mp = m;
1633 					mp = &m->m_next;
1634 				}
1635 				q->q_dst_m = top;
1636 				ubsec_mcopy(q->q_src_m, q->q_dst_m,
1637 				    cpskip, cpoffset);
1638 				if (q->q_dst_map == NULL) {
1639 					if (q->q_cached_dst_map == NULL) {
1640 						/* XXX again, what the heck is that 0xfff0? */
1641 						if (bus_dmamap_create(sc->sc_dmat, 0xfff0,
1642 						    UBS_MAX_SCATTER, 0xfff0, 0, BUS_DMA_NOWAIT,
1643 						    &q->q_cached_dst_map) != 0) {
1644 							ubsecstats.hst_nomap++;
1645 							err = ENOMEM;
1646 							goto errout;
1647 						}
1648 					}
1649 					q->q_dst_map = q->q_cached_dst_map;
1650 				}
1651 				if (bus_dmamap_load_mbuf(sc->sc_dmat,
1652 				    q->q_dst_map, q->q_dst_m,
1653 				    BUS_DMA_NOWAIT) != 0) {
1654 					ubsecstats.hst_noload++;
1655 					err = ENOMEM;
1656 					goto errout;
1657 				}
1658 			}
1659 		} else {
1660 			ubsecstats.hst_badflags++;
1661 			err = EINVAL;
1662 			goto errout;
1663 		}
1664 
1665 #ifdef UBSEC_DEBUG
1666 		if (ubsec_debug)
1667 			printf("dst skip: %d\n", dskip);
1668 #endif
1669 		for (i = j = 0; i < q->q_dst_map->dm_nsegs; i++) {
1670 			struct ubsec_pktbuf *pb;
1671 			bus_size_t packl = q->q_dst_map->dm_segs[i].ds_len;
1672 			bus_addr_t packp = q->q_dst_map->dm_segs[i].ds_addr;
1673 
1674 			if (dskip >= packl) {
1675 				dskip -= packl;
1676 				continue;
1677 			}
1678 
1679 			packl -= dskip;
1680 			packp += dskip;
1681 			dskip = 0;
1682 
1683 			if (packl > 0xfffc) {
1684 				err = EIO;
1685 				goto errout;
1686 			}
1687 
1688 			if (j == 0)
1689 				pb = &dmap->d_dma->d_mcr.mcr_opktbuf;
1690 			else
1691 				pb = &dmap->d_dma->d_dbuf[j - 1];
1692 
1693 			pb->pb_addr = htole32(packp);
1694 
1695 			if (dtheend) {
1696 				if (packl > dtheend) {
1697 					pb->pb_len = htole32(dtheend);
1698 					dtheend = 0;
1699 				} else {
1700 					pb->pb_len = htole32(packl);
1701 					dtheend -= packl;
1702 				}
1703 			} else
1704 				pb->pb_len = htole32(packl);
1705 
1706 			if ((i + 1) == q->q_dst_map->dm_nsegs) {
1707 				if (maccrd)
1708 					pb->pb_next = htole32(dmap->d_alloc.dma_paddr +
1709 					    offsetof(struct ubsec_dmachunk, d_macbuf[0]));
1710 				else
1711 					pb->pb_next = 0;
1712 			} else
1713 				pb->pb_next = htole32(dmap->d_alloc.dma_paddr +
1714 				    offsetof(struct ubsec_dmachunk, d_dbuf[j]));
1715 			j++;
1716 		}
1717 	}
1718 
1719 	dmap->d_dma->d_mcr.mcr_cmdctxp = htole32(dmap->d_alloc.dma_paddr +
1720 	    offsetof(struct ubsec_dmachunk, d_ctx));
1721 
1722 	if (enccrd && enccrd->crd_alg == CRYPTO_AES_CBC) {
1723 		struct ubsec_pktctx_aes128	*aes128;
1724 		struct ubsec_pktctx_aes192	*aes192;
1725 		struct ubsec_pktctx_aes256	*aes256;
1726 		struct ubsec_pktctx_hdr		*ph;
1727 		u_int8_t			*ctx;
1728 
1729 		ctx = (u_int8_t *)(dmap->d_alloc.dma_vaddr) +
1730 		    offsetof(struct ubsec_dmachunk, d_ctx);
1731 
1732 		ph = (struct ubsec_pktctx_hdr *)ctx;
1733 		ph->ph_type = htole16(UBS_PKTCTX_TYPE_IPSEC_AES);
1734 		ph->ph_flags = flags;
1735 		ph->ph_offset = htole16(coffset >> 2);
1736 
1737 		switch (enccrd->crd_klen) {
1738 		case 128:
1739 			aes128 = (struct ubsec_pktctx_aes128 *)ctx;
1740  			ph->ph_len = htole16(sizeof(*aes128));
1741 			ph->ph_flags |= htole16(UBS_PKTCTX_KEYSIZE_128);
1742 			for (i = 0; i < 4; i++)
1743 				aes128->pc_aeskey[i] = key.ses_key[i];
1744 			for (i = 0; i < 5; i++)
1745 				aes128->pc_hminner[i] = key.ses_hminner[i];
1746 			for (i = 0; i < 5; i++)
1747 				aes128->pc_hmouter[i] = key.ses_hmouter[i];
1748 			for (i = 0; i < 4; i++)
1749 				aes128->pc_iv[i] = key.ses_iv[i];
1750 			break;
1751 		case 192:
1752 			aes192 = (struct ubsec_pktctx_aes192 *)ctx;
1753 			ph->ph_len = htole16(sizeof(*aes192));
1754 			ph->ph_flags |= htole16(UBS_PKTCTX_KEYSIZE_192);
1755 			for (i = 0; i < 6; i++)
1756 				aes192->pc_aeskey[i] = key.ses_key[i];
1757 			for (i = 0; i < 5; i++)
1758 				aes192->pc_hminner[i] = key.ses_hminner[i];
1759 			for (i = 0; i < 5; i++)
1760 				aes192->pc_hmouter[i] = key.ses_hmouter[i];
1761 			for (i = 0; i < 4; i++)
1762 				aes192->pc_iv[i] = key.ses_iv[i];
1763 			break;
1764 		case 256:
1765 			aes256 = (struct ubsec_pktctx_aes256 *)ctx;
1766 			ph->ph_len = htole16(sizeof(*aes256));
1767 			ph->ph_flags |= htole16(UBS_PKTCTX_KEYSIZE_256);
1768 			for (i = 0; i < 8; i++)
1769 				aes256->pc_aeskey[i] = key.ses_key[i];
1770 			for (i = 0; i < 5; i++)
1771 				aes256->pc_hminner[i] = key.ses_hminner[i];
1772 			for (i = 0; i < 5; i++)
1773 				aes256->pc_hmouter[i] = key.ses_hmouter[i];
1774 			for (i = 0; i < 4; i++)
1775 				aes256->pc_iv[i] = key.ses_iv[i];
1776 			break;
1777 		}
1778 	} else if (sc->sc_flags & UBS_FLAGS_LONGCTX) {
1779 		struct ubsec_pktctx_3des	*ctx;
1780 		struct ubsec_pktctx_hdr		*ph;
1781 
1782 		ctx = (struct ubsec_pktctx_3des *)
1783 		    ((u_int8_t *)(dmap->d_alloc.dma_vaddr) +
1784 		    offsetof(struct ubsec_dmachunk, d_ctx));
1785 
1786 		ph = (struct ubsec_pktctx_hdr *)ctx;
1787 		ph->ph_len = htole16(sizeof(*ctx));
1788 		ph->ph_type = htole16(UBS_PKTCTX_TYPE_IPSEC_3DES);
1789 		ph->ph_flags = flags;
1790 		ph->ph_offset = htole16(coffset >> 2);
1791 
1792 		for (i = 0; i < 6; i++)
1793 			ctx->pc_deskey[i] = key.ses_key[i];
1794 		for (i = 0; i < 5; i++)
1795 			ctx->pc_hminner[i] = key.ses_hminner[i];
1796 		for (i = 0; i < 5; i++)
1797 			ctx->pc_hmouter[i] = key.ses_hmouter[i];
1798 		for (i = 0; i < 2; i++)
1799 			ctx->pc_iv[i] = key.ses_iv[i];
1800 	} else {
1801 		struct ubsec_pktctx *ctx = (struct ubsec_pktctx *)
1802 		    ((u_int8_t *)dmap->d_alloc.dma_vaddr +
1803 		    offsetof(struct ubsec_dmachunk, d_ctx));
1804 
1805 		ctx->pc_flags = flags;
1806 		ctx->pc_offset = htole16(coffset >> 2);
1807 		for (i = 0; i < 6; i++)
1808 			ctx->pc_deskey[i] = key.ses_key[i];
1809 		for (i = 0; i < 5; i++)
1810 			ctx->pc_hminner[i] = key.ses_hminner[i];
1811 		for (i = 0; i < 5; i++)
1812 			ctx->pc_hmouter[i] = key.ses_hmouter[i];
1813 		for (i = 0; i < 2; i++)
1814 			ctx->pc_iv[i] = key.ses_iv[i];
1815 	}
1816 
1817 	mutex_spin_enter(&sc->sc_mtx);
1818 	SIMPLEQ_INSERT_TAIL(&sc->sc_queue, q, q_next);
1819 	sc->sc_nqueue++;
1820 	ubsecstats.hst_ipackets++;
1821 	ubsecstats.hst_ibytes += dmap->d_alloc.dma_map->dm_mapsize;
1822 	if ((hint & CRYPTO_HINT_MORE) == 0 || sc->sc_nqueue >= ubsec_maxbatch)
1823 		ubsec_feed(sc);
1824 	mutex_spin_exit(&sc->sc_mtx);
1825 	return (0);
1826 
1827 errout:
1828 	if (q != NULL) {
1829 		if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m))
1830 			m_freem(q->q_dst_m);
1831 
1832 		if (q->q_dst_map != NULL && q->q_dst_map != q->q_src_map) {
1833 			bus_dmamap_unload(sc->sc_dmat, q->q_dst_map);
1834 		}
1835 		if (q->q_src_map != NULL) {
1836 			bus_dmamap_unload(sc->sc_dmat, q->q_src_map);
1837 		}
1838 
1839 		mutex_spin_enter(&sc->sc_mtx);
1840 		SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
1841 		mutex_spin_exit(&sc->sc_mtx);
1842 	}
1843 #if 0 /* jonathan says: this openbsd code seems to be subsumed elsewhere */
1844 	if (err == EINVAL)
1845 		ubsecstats.hst_invalid++;
1846 	else
1847 		ubsecstats.hst_nomem++;
1848 #endif
1849 	if (err != ERESTART) {
1850 		crp->crp_etype = err;
1851 		crypto_done(crp);
1852 	} else {
1853 		sc->sc_needwakeup |= CRYPTO_SYMQ;
1854 	}
1855 	return (err);
1856 }
1857 
1858 static void
1859 ubsec_callback(struct ubsec_softc *sc, struct ubsec_q *q)
1860 {
1861 	struct cryptop *crp = (struct cryptop *)q->q_crp;
1862 	struct cryptodesc *crd;
1863 	struct ubsec_dma *dmap = q->q_dma;
1864 
1865 	ubsecstats.hst_opackets++;
1866 	ubsecstats.hst_obytes += dmap->d_alloc.dma_size;
1867 
1868 	bus_dmamap_sync(sc->sc_dmat, dmap->d_alloc.dma_map, 0,
1869 	    dmap->d_alloc.dma_map->dm_mapsize,
1870 	    BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
1871 	if (q->q_dst_map != NULL && q->q_dst_map != q->q_src_map) {
1872 		bus_dmamap_sync(sc->sc_dmat, q->q_dst_map,
1873 		    0, q->q_dst_map->dm_mapsize, BUS_DMASYNC_POSTREAD);
1874 		bus_dmamap_unload(sc->sc_dmat, q->q_dst_map);
1875 	}
1876 	bus_dmamap_sync(sc->sc_dmat, q->q_src_map,
1877 	    0, q->q_src_map->dm_mapsize, BUS_DMASYNC_POSTWRITE);
1878 	bus_dmamap_unload(sc->sc_dmat, q->q_src_map);
1879 
1880 	if ((crp->crp_flags & CRYPTO_F_IMBUF) && (q->q_src_m != q->q_dst_m)) {
1881 		m_freem(q->q_src_m);
1882 		crp->crp_buf = (void *)q->q_dst_m;
1883 	}
1884 
1885 	/* copy out IV for future use */
1886 	if (q->q_flags & UBSEC_QFLAGS_COPYOUTIV) {
1887 		for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
1888 			if (crd->crd_alg != CRYPTO_DES_CBC &&
1889 			    crd->crd_alg != CRYPTO_3DES_CBC &&
1890 			    crd->crd_alg != CRYPTO_AES_CBC)
1891 				continue;
1892 			if (crp->crp_flags & CRYPTO_F_IMBUF)
1893 				m_copydata((struct mbuf *)crp->crp_buf,
1894 				    crd->crd_skip + crd->crd_len - 8, 8,
1895 				    (void *)sc->sc_sessions[q->q_sesn].ses_iv);
1896 			else if (crp->crp_flags & CRYPTO_F_IOV) {
1897 				cuio_copydata((struct uio *)crp->crp_buf,
1898 				    crd->crd_skip + crd->crd_len - 8, 8,
1899 				    (void *)sc->sc_sessions[q->q_sesn].ses_iv);
1900 			}
1901 			break;
1902 		}
1903 	}
1904 
1905 	for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
1906 		if (crd->crd_alg != CRYPTO_MD5_HMAC_96 &&
1907 		    crd->crd_alg != CRYPTO_SHA1_HMAC_96)
1908 			continue;
1909 		if (crp->crp_flags & CRYPTO_F_IMBUF)
1910 			m_copyback((struct mbuf *)crp->crp_buf,
1911 			    crd->crd_inject, 12,
1912 			    (void *)dmap->d_dma->d_macbuf);
1913 		else if (crp->crp_flags & CRYPTO_F_IOV && crp->crp_mac)
1914 			bcopy((void *)dmap->d_dma->d_macbuf,
1915 			    crp->crp_mac, 12);
1916 		break;
1917 	}
1918 	SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
1919 	crypto_done(crp);
1920 }
1921 
1922 static void
1923 ubsec_mcopy(struct mbuf *srcm, struct mbuf *dstm, int hoffset, int toffset)
1924 {
1925 	int i, j, dlen, slen;
1926 	char *dptr, *sptr;
1927 
1928 	j = 0;
1929 	sptr = srcm->m_data;
1930 	slen = srcm->m_len;
1931 	dptr = dstm->m_data;
1932 	dlen = dstm->m_len;
1933 
1934 	while (1) {
1935 		for (i = 0; i < uimin(slen, dlen); i++) {
1936 			if (j < hoffset || j >= toffset)
1937 				*dptr++ = *sptr++;
1938 			slen--;
1939 			dlen--;
1940 			j++;
1941 		}
1942 		if (slen == 0) {
1943 			srcm = srcm->m_next;
1944 			if (srcm == NULL)
1945 				return;
1946 			sptr = srcm->m_data;
1947 			slen = srcm->m_len;
1948 		}
1949 		if (dlen == 0) {
1950 			dstm = dstm->m_next;
1951 			if (dstm == NULL)
1952 				return;
1953 			dptr = dstm->m_data;
1954 			dlen = dstm->m_len;
1955 		}
1956 	}
1957 }
1958 
1959 /*
1960  * feed the key generator, must be called at splnet() or higher.
1961  */
1962 static void
1963 ubsec_feed2(struct ubsec_softc *sc)
1964 {
1965 	struct ubsec_q2 *q;
1966 
1967 	while (!SIMPLEQ_EMPTY(&sc->sc_queue2)) {
1968 		if (READ_REG(sc, BS_STAT) & BS_STAT_MCR2_FULL)
1969 			break;
1970 		q = SIMPLEQ_FIRST(&sc->sc_queue2);
1971 
1972 		bus_dmamap_sync(sc->sc_dmat, q->q_mcr.dma_map, 0,
1973 		    q->q_mcr.dma_map->dm_mapsize,
1974 		    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1975 		bus_dmamap_sync(sc->sc_dmat, q->q_ctx.dma_map, 0,
1976 		    q->q_ctx.dma_map->dm_mapsize,
1977 		    BUS_DMASYNC_PREWRITE);
1978 
1979 		WRITE_REG(sc, BS_MCR2, q->q_mcr.dma_paddr);
1980 		q = SIMPLEQ_FIRST(&sc->sc_queue2);
1981 		SIMPLEQ_REMOVE_HEAD(&sc->sc_queue2, /*q,*/ q_next);
1982 		--sc->sc_nqueue2;
1983 		SIMPLEQ_INSERT_TAIL(&sc->sc_qchip2, q, q_next);
1984 	}
1985 }
1986 
1987 /*
1988  * feed the RNG (used instead of ubsec_feed2() on 5827+ devices)
1989  */
1990 void
1991 ubsec_feed4(struct ubsec_softc *sc)
1992 {
1993 	struct ubsec_q2 *q;
1994 
1995 	while (!SIMPLEQ_EMPTY(&sc->sc_queue4)) {
1996 		if (READ_REG(sc, BS_STAT) & BS_STAT_MCR4_FULL)
1997 			break;
1998 		q = SIMPLEQ_FIRST(&sc->sc_queue4);
1999 
2000 		bus_dmamap_sync(sc->sc_dmat, q->q_mcr.dma_map, 0,
2001 		    q->q_mcr.dma_map->dm_mapsize,
2002 		    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2003 		bus_dmamap_sync(sc->sc_dmat, q->q_ctx.dma_map, 0,
2004 		    q->q_ctx.dma_map->dm_mapsize,
2005 		    BUS_DMASYNC_PREWRITE);
2006 
2007 		WRITE_REG(sc, BS_MCR4, q->q_mcr.dma_paddr);
2008 		SIMPLEQ_REMOVE_HEAD(&sc->sc_queue4, q_next);
2009 		--sc->sc_nqueue4;
2010 		SIMPLEQ_INSERT_TAIL(&sc->sc_qchip4, q, q_next);
2011 	}
2012 }
2013 
2014 /*
2015  * Callback for handling random numbers
2016  */
2017 static void
2018 ubsec_callback2(struct ubsec_softc *sc, struct ubsec_q2 *q)
2019 {
2020 	struct cryptkop *krp;
2021 	struct ubsec_ctx_keyop *ctx;
2022 
2023 	ctx = (struct ubsec_ctx_keyop *)q->q_ctx.dma_vaddr;
2024 	bus_dmamap_sync(sc->sc_dmat, q->q_ctx.dma_map, 0,
2025 	    q->q_ctx.dma_map->dm_mapsize, BUS_DMASYNC_POSTWRITE);
2026 
2027 	switch (q->q_type) {
2028 #ifndef UBSEC_NO_RNG
2029 	case UBS_CTXOP_RNGSHA1:
2030 	case UBS_CTXOP_RNGBYPASS: {
2031 		struct ubsec_q2_rng *rng = (struct ubsec_q2_rng *)q;
2032 		u_int32_t *p;
2033 		int i;
2034 
2035 		bus_dmamap_sync(sc->sc_dmat, rng->rng_buf.dma_map, 0,
2036 		    rng->rng_buf.dma_map->dm_mapsize, BUS_DMASYNC_POSTREAD);
2037 		p = (u_int32_t *)rng->rng_buf.dma_vaddr;
2038 #ifndef __NetBSD__
2039 		for (i = 0; i < UBSEC_RNG_BUFSIZ; p++, i++)
2040 			add_true_randomness(letoh32(*p));
2041 #else
2042 		i = UBSEC_RNG_BUFSIZ * sizeof(u_int32_t);
2043 		rnd_add_data(&sc->sc_rnd_source, (char *)p, i, i * NBBY);
2044 		sc->sc_rng_need -= i;
2045 #endif
2046 		rng->rng_used = 0;
2047 #ifdef __OpenBSD__
2048 		timeout_add(&sc->sc_rngto, sc->sc_rnghz);
2049 #else
2050 		if (sc->sc_rng_need > 0) {
2051 			callout_schedule(&sc->sc_rngto, sc->sc_rnghz);
2052 		}
2053 #endif
2054 		break;
2055 	}
2056 #endif
2057 	case UBS_CTXOP_MODEXP: {
2058 		struct ubsec_q2_modexp *me = (struct ubsec_q2_modexp *)q;
2059 		u_int rlen, clen;
2060 
2061 		krp = me->me_krp;
2062 		rlen = (me->me_modbits + 7) / 8;
2063 		clen = (krp->krp_param[krp->krp_iparams].crp_nbits + 7) / 8;
2064 
2065 		bus_dmamap_sync(sc->sc_dmat, me->me_M.dma_map,
2066 		    0, me->me_M.dma_map->dm_mapsize, BUS_DMASYNC_POSTWRITE);
2067 		bus_dmamap_sync(sc->sc_dmat, me->me_E.dma_map,
2068 		    0, me->me_E.dma_map->dm_mapsize, BUS_DMASYNC_POSTWRITE);
2069 		bus_dmamap_sync(sc->sc_dmat, me->me_C.dma_map,
2070 		    0, me->me_C.dma_map->dm_mapsize, BUS_DMASYNC_POSTREAD);
2071 		bus_dmamap_sync(sc->sc_dmat, me->me_epb.dma_map,
2072 		    0, me->me_epb.dma_map->dm_mapsize, BUS_DMASYNC_POSTWRITE);
2073 
2074 		if (clen < rlen)
2075 			krp->krp_status = E2BIG;
2076 		else {
2077 			if (sc->sc_flags & UBS_FLAGS_HWNORM) {
2078 				memset(krp->krp_param[krp->krp_iparams].crp_p, 0,
2079 				    (krp->krp_param[krp->krp_iparams].crp_nbits
2080 					+ 7) / 8);
2081 				bcopy(me->me_C.dma_vaddr,
2082 				    krp->krp_param[krp->krp_iparams].crp_p,
2083 				    (me->me_modbits + 7) / 8);
2084 			} else
2085 				ubsec_kshift_l(me->me_shiftbits,
2086 				    me->me_C.dma_vaddr, me->me_normbits,
2087 				    krp->krp_param[krp->krp_iparams].crp_p,
2088 				    krp->krp_param[krp->krp_iparams].crp_nbits);
2089 		}
2090 
2091 		crypto_kdone(krp);
2092 
2093 		/* bzero all potentially sensitive data */
2094 		memset(me->me_E.dma_vaddr, 0, me->me_E.dma_size);
2095 		memset(me->me_M.dma_vaddr, 0, me->me_M.dma_size);
2096 		memset(me->me_C.dma_vaddr, 0, me->me_C.dma_size);
2097 		memset(me->me_q.q_ctx.dma_vaddr, 0, me->me_q.q_ctx.dma_size);
2098 
2099 		/* Can't free here, so put us on the free list. */
2100 		SIMPLEQ_INSERT_TAIL(&sc->sc_q2free, &me->me_q, q_next);
2101 		break;
2102 	}
2103 	case UBS_CTXOP_RSAPRIV: {
2104 		struct ubsec_q2_rsapriv *rp = (struct ubsec_q2_rsapriv *)q;
2105 		u_int len;
2106 
2107 		krp = rp->rpr_krp;
2108 		bus_dmamap_sync(sc->sc_dmat, rp->rpr_msgin.dma_map, 0,
2109 		    rp->rpr_msgin.dma_map->dm_mapsize, BUS_DMASYNC_POSTWRITE);
2110 		bus_dmamap_sync(sc->sc_dmat, rp->rpr_msgout.dma_map, 0,
2111 		    rp->rpr_msgout.dma_map->dm_mapsize, BUS_DMASYNC_POSTREAD);
2112 
2113 		len = (krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT].crp_nbits + 7)
2114 		    / 8;
2115 		bcopy(rp->rpr_msgout.dma_vaddr,
2116 		    krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT].crp_p, len);
2117 
2118 		crypto_kdone(krp);
2119 
2120 		memset(rp->rpr_msgin.dma_vaddr, 0, rp->rpr_msgin.dma_size);
2121 		memset(rp->rpr_msgout.dma_vaddr, 0, rp->rpr_msgout.dma_size);
2122 		memset(rp->rpr_q.q_ctx.dma_vaddr, 0, rp->rpr_q.q_ctx.dma_size);
2123 
2124 		/* Can't free here, so put us on the free list. */
2125 		SIMPLEQ_INSERT_TAIL(&sc->sc_q2free, &rp->rpr_q, q_next);
2126 		break;
2127 	}
2128 	default:
2129 		printf("%s: unknown ctx op: %x\n", device_xname(sc->sc_dev),
2130 		    letoh16(ctx->ctx_op));
2131 		break;
2132 	}
2133 }
2134 
2135 #ifndef UBSEC_NO_RNG
2136 
2137 static void
2138 ubsec_rng_get(size_t bytes, void *vsc)
2139 {
2140 	struct ubsec_softc *sc = vsc;
2141 
2142 	mutex_spin_enter(&sc->sc_mtx);
2143 	sc->sc_rng_need = bytes;
2144 	ubsec_rng_locked(sc);
2145 	mutex_spin_exit(&sc->sc_mtx);
2146 
2147 }
2148 
2149 static void
2150 ubsec_rng(void *vsc)
2151 {
2152 	struct ubsec_softc *sc = vsc;
2153 	mutex_spin_enter(&sc->sc_mtx);
2154 	ubsec_rng_locked(sc);
2155 	mutex_spin_exit(&sc->sc_mtx);
2156 }
2157 
2158 static void
2159 ubsec_rng_locked(void *vsc)
2160 {
2161 	struct ubsec_softc *sc = vsc;
2162 	struct ubsec_q2_rng *rng = &sc->sc_rng;
2163 	struct ubsec_mcr *mcr;
2164 	struct ubsec_ctx_rngbypass *ctx;
2165 	int *nqueue;
2166 
2167 	/* Caller is responsible to lock and release sc_mtx. */
2168 	KASSERT(mutex_owned(&sc->sc_mtx));
2169 
2170 	if (rng->rng_used) {
2171 		return;
2172 	}
2173 
2174 	if (sc->sc_rng_need < 1) {
2175 		callout_stop(&sc->sc_rngto);
2176 		return;
2177 	}
2178 
2179 	if (sc->sc_flags & UBS_FLAGS_RNG4)
2180 		nqueue = &sc->sc_nqueue4;
2181 	else
2182 		nqueue = &sc->sc_nqueue2;
2183 
2184 	(*nqueue)++;
2185 	if (*nqueue >= UBS_MAX_NQUEUE)
2186  		goto out;
2187 
2188 	mcr = (struct ubsec_mcr *)rng->rng_q.q_mcr.dma_vaddr;
2189 	ctx = (struct ubsec_ctx_rngbypass *)rng->rng_q.q_ctx.dma_vaddr;
2190 
2191 	mcr->mcr_pkts = htole16(1);
2192 	mcr->mcr_flags = 0;
2193 	mcr->mcr_cmdctxp = htole32(rng->rng_q.q_ctx.dma_paddr);
2194 	mcr->mcr_ipktbuf.pb_addr = mcr->mcr_ipktbuf.pb_next = 0;
2195 	mcr->mcr_ipktbuf.pb_len = 0;
2196 	mcr->mcr_reserved = mcr->mcr_pktlen = 0;
2197 	mcr->mcr_opktbuf.pb_addr = htole32(rng->rng_buf.dma_paddr);
2198 	mcr->mcr_opktbuf.pb_len = htole32(((sizeof(u_int32_t) * UBSEC_RNG_BUFSIZ)) &
2199 	    UBS_PKTBUF_LEN);
2200 	mcr->mcr_opktbuf.pb_next = 0;
2201 
2202 	ctx->rbp_len = htole16(sizeof(struct ubsec_ctx_rngbypass));
2203 	ctx->rbp_op = htole16(UBS_CTXOP_RNGSHA1);
2204 	rng->rng_q.q_type = UBS_CTXOP_RNGSHA1;
2205 
2206 	bus_dmamap_sync(sc->sc_dmat, rng->rng_buf.dma_map, 0,
2207 	    rng->rng_buf.dma_map->dm_mapsize, BUS_DMASYNC_PREREAD);
2208 
2209 	if (sc->sc_flags & UBS_FLAGS_RNG4) {
2210 		SIMPLEQ_INSERT_TAIL(&sc->sc_queue4, &rng->rng_q, q_next);
2211 		ubsec_feed4(sc);
2212 	} else {
2213 		SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &rng->rng_q, q_next);
2214 		ubsec_feed2(sc);
2215 	}
2216 	rng->rng_used = 1;
2217 	ubsecstats.hst_rng++;
2218 
2219 	return;
2220 
2221 out:
2222 	/*
2223 	 * Something weird happened, generate our own call back.
2224 	 */
2225 	(*nqueue)--;
2226 #ifdef __OpenBSD__
2227 	timeout_add(&sc->sc_rngto, sc->sc_rnghz);
2228 #else
2229 	callout_schedule(&sc->sc_rngto, sc->sc_rnghz);
2230 #endif
2231 }
2232 #endif /* UBSEC_NO_RNG */
2233 
2234 static int
2235 ubsec_dma_malloc(struct ubsec_softc *sc, bus_size_t size,
2236 		 struct ubsec_dma_alloc *dma,int mapflags)
2237 {
2238 	int r;
2239 
2240 	if ((r = bus_dmamem_alloc(sc->sc_dmat, size, PAGE_SIZE, 0,
2241 	    &dma->dma_seg, 1, &dma->dma_nseg, BUS_DMA_NOWAIT)) != 0)
2242 		goto fail_0;
2243 
2244 	if ((r = bus_dmamem_map(sc->sc_dmat, &dma->dma_seg, dma->dma_nseg,
2245 	    size, &dma->dma_vaddr, mapflags | BUS_DMA_NOWAIT)) != 0)
2246 		goto fail_1;
2247 
2248 	if ((r = bus_dmamap_create(sc->sc_dmat, size, 1, size, 0,
2249 	    BUS_DMA_NOWAIT, &dma->dma_map)) != 0)
2250 		goto fail_2;
2251 
2252 	if ((r = bus_dmamap_load(sc->sc_dmat, dma->dma_map, dma->dma_vaddr,
2253 	    size, NULL, BUS_DMA_NOWAIT)) != 0)
2254 		goto fail_3;
2255 
2256 	dma->dma_paddr = dma->dma_map->dm_segs[0].ds_addr;
2257 	dma->dma_size = size;
2258 	return (0);
2259 
2260 fail_3:
2261 	bus_dmamap_destroy(sc->sc_dmat, dma->dma_map);
2262 fail_2:
2263 	bus_dmamem_unmap(sc->sc_dmat, dma->dma_vaddr, size);
2264 fail_1:
2265 	bus_dmamem_free(sc->sc_dmat, &dma->dma_seg, dma->dma_nseg);
2266 fail_0:
2267 	dma->dma_map = NULL;
2268 	return (r);
2269 }
2270 
2271 static void
2272 ubsec_dma_free(struct ubsec_softc *sc, struct ubsec_dma_alloc *dma)
2273 {
2274 	bus_dmamap_unload(sc->sc_dmat, dma->dma_map);
2275 	bus_dmamem_unmap(sc->sc_dmat, dma->dma_vaddr, dma->dma_size);
2276 	bus_dmamem_free(sc->sc_dmat, &dma->dma_seg, dma->dma_nseg);
2277 	bus_dmamap_destroy(sc->sc_dmat, dma->dma_map);
2278 }
2279 
2280 /*
2281  * Resets the board.  Values in the regesters are left as is
2282  * from the reset (i.e. initial values are assigned elsewhere).
2283  */
2284 static void
2285 ubsec_reset_board(struct ubsec_softc *sc)
2286 {
2287 	volatile u_int32_t ctrl;
2288 
2289 	ctrl = READ_REG(sc, BS_CTRL);
2290 	ctrl |= BS_CTRL_RESET;
2291 	WRITE_REG(sc, BS_CTRL, ctrl);
2292 
2293 	/*
2294 	* Wait aprox. 30 PCI clocks = 900 ns = 0.9 us
2295 	*/
2296 	DELAY(10);
2297 
2298 	/* Enable RNG and interrupts on newer devices */
2299 	if (sc->sc_flags & UBS_FLAGS_MULTIMCR) {
2300 #ifndef UBSEC_NO_RNG
2301 		WRITE_REG(sc, BS_CFG, BS_CFG_RNG);
2302 #endif
2303 		WRITE_REG(sc, BS_INT, BS_INT_DMAINT);
2304 	}
2305 }
2306 
2307 /*
2308  * Init Broadcom registers
2309  */
2310 static void
2311 ubsec_init_board(struct ubsec_softc *sc)
2312 {
2313 	u_int32_t ctrl;
2314 
2315 	ctrl = READ_REG(sc, BS_CTRL);
2316 	ctrl &= ~(BS_CTRL_BE32 | BS_CTRL_BE64);
2317 	ctrl |= BS_CTRL_LITTLE_ENDIAN | BS_CTRL_MCR1INT;
2318 
2319 	/*
2320 	 * XXX: Sam Leffler's code has (UBS_FLAGS_KEY|UBS_FLAGS_RNG)).
2321 	 * anyone got hw docs?
2322 	 */
2323 	if (sc->sc_flags & UBS_FLAGS_KEY)
2324 		ctrl |= BS_CTRL_MCR2INT;
2325 	else
2326 		ctrl &= ~BS_CTRL_MCR2INT;
2327 
2328 	if (sc->sc_flags & UBS_FLAGS_HWNORM)
2329 		ctrl &= ~BS_CTRL_SWNORM;
2330 
2331 	if (sc->sc_flags & UBS_FLAGS_MULTIMCR) {
2332 		ctrl |= BS_CTRL_BSIZE240;
2333 		ctrl &= ~BS_CTRL_MCR3INT; /* MCR3 is reserved for SSL */
2334 
2335 		if (sc->sc_flags & UBS_FLAGS_RNG4)
2336 			ctrl |= BS_CTRL_MCR4INT;
2337 		else
2338 			ctrl &= ~BS_CTRL_MCR4INT;
2339 	}
2340 
2341 	WRITE_REG(sc, BS_CTRL, ctrl);
2342 }
2343 
2344 /*
2345  * Init Broadcom PCI registers
2346  */
2347 static void
2348 ubsec_init_pciregs(struct pci_attach_args *pa)
2349 {
2350 	pci_chipset_tag_t pc = pa->pa_pc;
2351 	u_int32_t misc;
2352 
2353 	/*
2354 	 * This will set the cache line size to 1, this will
2355 	 * force the BCM58xx chip just to do burst read/writes.
2356 	 * Cache line read/writes are to slow
2357 	 */
2358 	misc = pci_conf_read(pc, pa->pa_tag, PCI_BHLC_REG);
2359 	misc = (misc & ~(PCI_CACHELINE_MASK << PCI_CACHELINE_SHIFT))
2360 	    | ((UBS_DEF_CACHELINE & 0xff) << PCI_CACHELINE_SHIFT);
2361 	pci_conf_write(pc, pa->pa_tag, PCI_BHLC_REG, misc);
2362 }
2363 
2364 /*
2365  * Clean up after a chip crash.
2366  * It is assumed that the caller in splnet()
2367  */
2368 static void
2369 ubsec_cleanchip(struct ubsec_softc *sc)
2370 {
2371 	struct ubsec_q *q;
2372 
2373 	while (!SIMPLEQ_EMPTY(&sc->sc_qchip)) {
2374 		q = SIMPLEQ_FIRST(&sc->sc_qchip);
2375 		SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, /*q,*/ q_next);
2376 		ubsec_free_q(sc, q);
2377 	}
2378 	sc->sc_nqchip = 0;
2379 }
2380 
2381 /*
2382  * free a ubsec_q
2383  * It is assumed that the caller is within splnet()
2384  */
2385 static int
2386 ubsec_free_q(struct ubsec_softc *sc, struct ubsec_q *q)
2387 {
2388 	struct ubsec_q *q2;
2389 	struct cryptop *crp;
2390 	int npkts;
2391 	int i;
2392 
2393 	npkts = q->q_nstacked_mcrs;
2394 
2395 	for (i = 0; i < npkts; i++) {
2396 		if(q->q_stacked_mcr[i]) {
2397 			q2 = q->q_stacked_mcr[i];
2398 
2399 			if ((q2->q_dst_m != NULL)
2400 			    && (q2->q_src_m != q2->q_dst_m))
2401 				m_freem(q2->q_dst_m);
2402 
2403 			crp = (struct cryptop *)q2->q_crp;
2404 
2405 			SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q2, q_next);
2406 
2407 			crp->crp_etype = EFAULT;
2408 			crypto_done(crp);
2409 		} else {
2410 			break;
2411 		}
2412 	}
2413 
2414 	/*
2415 	 * Free header MCR
2416 	 */
2417 	if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m))
2418 		m_freem(q->q_dst_m);
2419 
2420 	crp = (struct cryptop *)q->q_crp;
2421 
2422 	SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
2423 
2424 	crp->crp_etype = EFAULT;
2425 	crypto_done(crp);
2426 	return(0);
2427 }
2428 
2429 /*
2430  * Routine to reset the chip and clean up.
2431  * It is assumed that the caller is in splnet()
2432  */
2433 static void
2434 ubsec_totalreset(struct ubsec_softc *sc)
2435 {
2436 	ubsec_reset_board(sc);
2437 	ubsec_init_board(sc);
2438 	ubsec_cleanchip(sc);
2439 }
2440 
2441 static int
2442 ubsec_dmamap_aligned(bus_dmamap_t map)
2443 {
2444 	int i;
2445 
2446 	for (i = 0; i < map->dm_nsegs; i++) {
2447 		if (map->dm_segs[i].ds_addr & 3)
2448 			return (0);
2449 		if ((i != (map->dm_nsegs - 1)) &&
2450 		    (map->dm_segs[i].ds_len & 3))
2451 			return (0);
2452 	}
2453 	return (1);
2454 }
2455 
2456 #ifdef __OpenBSD__
2457 struct ubsec_softc *
2458 ubsec_kfind(struct cryptkop *krp)
2459 {
2460 	struct ubsec_softc *sc;
2461 	int i;
2462 
2463 	for (i = 0; i < ubsec_cd.cd_ndevs; i++) {
2464 		sc = ubsec_cd.cd_devs[i];
2465 		if (sc == NULL)
2466 			continue;
2467 		if (sc->sc_cid == krp->krp_hid)
2468 			return (sc);
2469 	}
2470 	return (NULL);
2471 }
2472 #endif
2473 
2474 static void
2475 ubsec_kfree(struct ubsec_softc *sc, struct ubsec_q2 *q)
2476 {
2477 	switch (q->q_type) {
2478 	case UBS_CTXOP_MODEXP: {
2479 		struct ubsec_q2_modexp *me = (struct ubsec_q2_modexp *)q;
2480 
2481 		ubsec_dma_free(sc, &me->me_q.q_mcr);
2482 		ubsec_dma_free(sc, &me->me_q.q_ctx);
2483 		ubsec_dma_free(sc, &me->me_M);
2484 		ubsec_dma_free(sc, &me->me_E);
2485 		ubsec_dma_free(sc, &me->me_C);
2486 		ubsec_dma_free(sc, &me->me_epb);
2487 		free(me, M_DEVBUF);
2488 		break;
2489 	}
2490 	case UBS_CTXOP_RSAPRIV: {
2491 		struct ubsec_q2_rsapriv *rp = (struct ubsec_q2_rsapriv *)q;
2492 
2493 		ubsec_dma_free(sc, &rp->rpr_q.q_mcr);
2494 		ubsec_dma_free(sc, &rp->rpr_q.q_ctx);
2495 		ubsec_dma_free(sc, &rp->rpr_msgin);
2496 		ubsec_dma_free(sc, &rp->rpr_msgout);
2497 		free(rp, M_DEVBUF);
2498 		break;
2499 	}
2500 	default:
2501 		printf("%s: invalid kfree 0x%x\n", device_xname(sc->sc_dev),
2502 		    q->q_type);
2503 		break;
2504 	}
2505 }
2506 
2507 static int
2508 ubsec_kprocess(void *arg, struct cryptkop *krp, int hint)
2509 {
2510 	struct ubsec_softc *sc;
2511 	int r;
2512 
2513 	if (krp == NULL || krp->krp_callback == NULL)
2514 		return (EINVAL);
2515 #ifdef __OpenBSD__
2516 	if ((sc = ubsec_kfind(krp)) == NULL)
2517 		return (EINVAL);
2518 #else
2519 	sc = arg;
2520 	KASSERT(sc != NULL /*, ("ubsec_kprocess: null softc")*/);
2521 #endif
2522 
2523 	while (!SIMPLEQ_EMPTY(&sc->sc_q2free)) {
2524 		struct ubsec_q2 *q;
2525 
2526 		q = SIMPLEQ_FIRST(&sc->sc_q2free);
2527 		SIMPLEQ_REMOVE_HEAD(&sc->sc_q2free, /*q,*/ q_next);
2528 		ubsec_kfree(sc, q);
2529 	}
2530 
2531 	switch (krp->krp_op) {
2532 	case CRK_MOD_EXP:
2533 		if (sc->sc_flags & UBS_FLAGS_HWNORM)
2534 			r = ubsec_kprocess_modexp_hw(sc, krp, hint);
2535 		else
2536 			r = ubsec_kprocess_modexp_sw(sc, krp, hint);
2537 		break;
2538 	case CRK_MOD_EXP_CRT:
2539 		r = ubsec_kprocess_rsapriv(sc, krp, hint);
2540 		break;
2541 	default:
2542 		printf("%s: kprocess: invalid op 0x%x\n",
2543 		    device_xname(sc->sc_dev), krp->krp_op);
2544 		krp->krp_status = EOPNOTSUPP;
2545 		crypto_kdone(krp);
2546 		r = 0;
2547 	}
2548 	return (r);
2549 }
2550 
2551 /*
2552  * Start computation of cr[C] = (cr[M] ^ cr[E]) mod cr[N] (sw normalization)
2553  */
2554 static int
2555 ubsec_kprocess_modexp_sw(struct ubsec_softc *sc, struct cryptkop *krp,
2556 			 int hint)
2557 {
2558 	struct ubsec_q2_modexp *me;
2559 	struct ubsec_mcr *mcr;
2560 	struct ubsec_ctx_modexp *ctx;
2561 	struct ubsec_pktbuf *epb;
2562 	int err = 0;
2563 	u_int nbits, normbits, mbits, shiftbits, ebits;
2564 
2565 	me = (struct ubsec_q2_modexp *)malloc(sizeof *me, M_DEVBUF, M_NOWAIT);
2566 	if (me == NULL) {
2567 		err = ENOMEM;
2568 		goto errout;
2569 	}
2570 	memset(me, 0, sizeof *me);
2571 	me->me_krp = krp;
2572 	me->me_q.q_type = UBS_CTXOP_MODEXP;
2573 
2574 	nbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_N]);
2575 	if (nbits <= 512)
2576 		normbits = 512;
2577 	else if (nbits <= 768)
2578 		normbits = 768;
2579 	else if (nbits <= 1024)
2580 		normbits = 1024;
2581 	else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 1536)
2582 		normbits = 1536;
2583 	else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 2048)
2584 		normbits = 2048;
2585 	else {
2586 		err = E2BIG;
2587 		goto errout;
2588 	}
2589 
2590 	shiftbits = normbits - nbits;
2591 
2592 	me->me_modbits = nbits;
2593 	me->me_shiftbits = shiftbits;
2594 	me->me_normbits = normbits;
2595 
2596 	/* Sanity check: result bits must be >= true modulus bits. */
2597 	if (krp->krp_param[krp->krp_iparams].crp_nbits < nbits) {
2598 		err = ERANGE;
2599 		goto errout;
2600 	}
2601 
2602 	if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr),
2603 	    &me->me_q.q_mcr, 0)) {
2604 		err = ENOMEM;
2605 		goto errout;
2606 	}
2607 	mcr = (struct ubsec_mcr *)me->me_q.q_mcr.dma_vaddr;
2608 
2609 	if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_modexp),
2610 	    &me->me_q.q_ctx, 0)) {
2611 		err = ENOMEM;
2612 		goto errout;
2613 	}
2614 
2615 	mbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_M]);
2616 	if (mbits > nbits) {
2617 		err = E2BIG;
2618 		goto errout;
2619 	}
2620 	if (ubsec_dma_malloc(sc, normbits / 8, &me->me_M, 0)) {
2621 		err = ENOMEM;
2622 		goto errout;
2623 	}
2624 	ubsec_kshift_r(shiftbits,
2625 	    krp->krp_param[UBS_MODEXP_PAR_M].crp_p, mbits,
2626 	    me->me_M.dma_vaddr, normbits);
2627 
2628 	if (ubsec_dma_malloc(sc, normbits / 8, &me->me_C, 0)) {
2629 		err = ENOMEM;
2630 		goto errout;
2631 	}
2632 	memset(me->me_C.dma_vaddr, 0, me->me_C.dma_size);
2633 
2634 	ebits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_E]);
2635 	if (ebits > nbits) {
2636 		err = E2BIG;
2637 		goto errout;
2638 	}
2639 	if (ubsec_dma_malloc(sc, normbits / 8, &me->me_E, 0)) {
2640 		err = ENOMEM;
2641 		goto errout;
2642 	}
2643 	ubsec_kshift_r(shiftbits,
2644 	    krp->krp_param[UBS_MODEXP_PAR_E].crp_p, ebits,
2645 	    me->me_E.dma_vaddr, normbits);
2646 
2647 	if (ubsec_dma_malloc(sc, sizeof(struct ubsec_pktbuf),
2648 	    &me->me_epb, 0)) {
2649 		err = ENOMEM;
2650 		goto errout;
2651 	}
2652 	epb = (struct ubsec_pktbuf *)me->me_epb.dma_vaddr;
2653 	epb->pb_addr = htole32(me->me_E.dma_paddr);
2654 	epb->pb_next = 0;
2655 	epb->pb_len = htole32(normbits / 8);
2656 
2657 #ifdef UBSEC_DEBUG
2658 	if (ubsec_debug) {
2659 		printf("Epb ");
2660 		ubsec_dump_pb(epb);
2661 	}
2662 #endif
2663 
2664 	mcr->mcr_pkts = htole16(1);
2665 	mcr->mcr_flags = 0;
2666 	mcr->mcr_cmdctxp = htole32(me->me_q.q_ctx.dma_paddr);
2667 	mcr->mcr_reserved = 0;
2668 	mcr->mcr_pktlen = 0;
2669 
2670 	mcr->mcr_ipktbuf.pb_addr = htole32(me->me_M.dma_paddr);
2671 	mcr->mcr_ipktbuf.pb_len = htole32(normbits / 8);
2672 	mcr->mcr_ipktbuf.pb_next = htole32(me->me_epb.dma_paddr);
2673 
2674 	mcr->mcr_opktbuf.pb_addr = htole32(me->me_C.dma_paddr);
2675 	mcr->mcr_opktbuf.pb_next = 0;
2676 	mcr->mcr_opktbuf.pb_len = htole32(normbits / 8);
2677 
2678 #ifdef DIAGNOSTIC
2679 	/* Misaligned output buffer will hang the chip. */
2680 	if ((letoh32(mcr->mcr_opktbuf.pb_addr) & 3) != 0)
2681 		panic("%s: modexp invalid addr 0x%x", device_xname(sc->sc_dev),
2682 		    letoh32(mcr->mcr_opktbuf.pb_addr));
2683 	if ((letoh32(mcr->mcr_opktbuf.pb_len) & 3) != 0)
2684 		panic("%s: modexp invalid len 0x%x",  device_xname(sc->sc_dev),
2685 		    letoh32(mcr->mcr_opktbuf.pb_len));
2686 #endif
2687 
2688 	ctx = (struct ubsec_ctx_modexp *)me->me_q.q_ctx.dma_vaddr;
2689 	memset(ctx, 0, sizeof(*ctx));
2690 	ubsec_kshift_r(shiftbits,
2691 	    krp->krp_param[UBS_MODEXP_PAR_N].crp_p, nbits,
2692 	    ctx->me_N, normbits);
2693 	ctx->me_len = htole16((normbits / 8) + (4 * sizeof(u_int16_t)));
2694 	ctx->me_op = htole16(UBS_CTXOP_MODEXP);
2695 	ctx->me_E_len = htole16(nbits);
2696 	ctx->me_N_len = htole16(nbits);
2697 
2698 #ifdef UBSEC_DEBUG
2699 	if (ubsec_debug) {
2700 		ubsec_dump_mcr(mcr);
2701 		ubsec_dump_ctx2((struct ubsec_ctx_keyop *)ctx);
2702 	}
2703 #endif
2704 
2705 	/*
2706 	 * ubsec_feed2 will sync mcr and ctx, we just need to sync
2707 	 * everything else.
2708 	 */
2709 	bus_dmamap_sync(sc->sc_dmat, me->me_M.dma_map,
2710 	    0, me->me_M.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE);
2711 	bus_dmamap_sync(sc->sc_dmat, me->me_E.dma_map,
2712 	    0, me->me_E.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE);
2713 	bus_dmamap_sync(sc->sc_dmat, me->me_C.dma_map,
2714 	    0, me->me_C.dma_map->dm_mapsize, BUS_DMASYNC_PREREAD);
2715 	bus_dmamap_sync(sc->sc_dmat, me->me_epb.dma_map,
2716 	    0, me->me_epb.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE);
2717 
2718 	/* Enqueue and we're done... */
2719 	mutex_spin_enter(&sc->sc_mtx);
2720 	SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &me->me_q, q_next);
2721 	ubsec_feed2(sc);
2722 	ubsecstats.hst_modexp++;
2723 	mutex_spin_exit(&sc->sc_mtx);
2724 
2725 	return (0);
2726 
2727 errout:
2728 	if (me != NULL) {
2729 		if (me->me_q.q_mcr.dma_map != NULL)
2730 			ubsec_dma_free(sc, &me->me_q.q_mcr);
2731 		if (me->me_q.q_ctx.dma_map != NULL) {
2732 			memset(me->me_q.q_ctx.dma_vaddr, 0,
2733 			    me->me_q.q_ctx.dma_size);
2734 			ubsec_dma_free(sc, &me->me_q.q_ctx);
2735 		}
2736 		if (me->me_M.dma_map != NULL) {
2737 			memset(me->me_M.dma_vaddr, 0, me->me_M.dma_size);
2738 			ubsec_dma_free(sc, &me->me_M);
2739 		}
2740 		if (me->me_E.dma_map != NULL) {
2741 			memset(me->me_E.dma_vaddr, 0, me->me_E.dma_size);
2742 			ubsec_dma_free(sc, &me->me_E);
2743 		}
2744 		if (me->me_C.dma_map != NULL) {
2745 			memset(me->me_C.dma_vaddr, 0, me->me_C.dma_size);
2746 			ubsec_dma_free(sc, &me->me_C);
2747 		}
2748 		if (me->me_epb.dma_map != NULL)
2749 			ubsec_dma_free(sc, &me->me_epb);
2750 		free(me, M_DEVBUF);
2751 	}
2752 	krp->krp_status = err;
2753 	crypto_kdone(krp);
2754 	return (0);
2755 }
2756 
2757 /*
2758  * Start computation of cr[C] = (cr[M] ^ cr[E]) mod cr[N] (hw normalization)
2759  */
2760 static int
2761 ubsec_kprocess_modexp_hw(struct ubsec_softc *sc, struct cryptkop *krp,
2762 			 int hint)
2763 {
2764 	struct ubsec_q2_modexp *me;
2765 	struct ubsec_mcr *mcr;
2766 	struct ubsec_ctx_modexp *ctx;
2767 	struct ubsec_pktbuf *epb;
2768 	int err = 0;
2769 	u_int nbits, normbits, mbits, shiftbits, ebits;
2770 
2771 	me = (struct ubsec_q2_modexp *)malloc(sizeof *me, M_DEVBUF, M_NOWAIT);
2772 	if (me == NULL) {
2773 		err = ENOMEM;
2774 		goto errout;
2775 	}
2776 	memset(me, 0, sizeof *me);
2777 	me->me_krp = krp;
2778 	me->me_q.q_type = UBS_CTXOP_MODEXP;
2779 
2780 	nbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_N]);
2781 	if (nbits <= 512)
2782 		normbits = 512;
2783 	else if (nbits <= 768)
2784 		normbits = 768;
2785 	else if (nbits <= 1024)
2786 		normbits = 1024;
2787 	else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 1536)
2788 		normbits = 1536;
2789 	else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 2048)
2790 		normbits = 2048;
2791 	else {
2792 		err = E2BIG;
2793 		goto errout;
2794 	}
2795 
2796 	shiftbits = normbits - nbits;
2797 
2798 	/* XXX ??? */
2799 	me->me_modbits = nbits;
2800 	me->me_shiftbits = shiftbits;
2801 	me->me_normbits = normbits;
2802 
2803 	/* Sanity check: result bits must be >= true modulus bits. */
2804 	if (krp->krp_param[krp->krp_iparams].crp_nbits < nbits) {
2805 		err = ERANGE;
2806 		goto errout;
2807 	}
2808 
2809 	if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr),
2810 	    &me->me_q.q_mcr, 0)) {
2811 		err = ENOMEM;
2812 		goto errout;
2813 	}
2814 	mcr = (struct ubsec_mcr *)me->me_q.q_mcr.dma_vaddr;
2815 
2816 	if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_modexp),
2817 	    &me->me_q.q_ctx, 0)) {
2818 		err = ENOMEM;
2819 		goto errout;
2820 	}
2821 
2822 	mbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_M]);
2823 	if (mbits > nbits) {
2824 		err = E2BIG;
2825 		goto errout;
2826 	}
2827 	if (ubsec_dma_malloc(sc, normbits / 8, &me->me_M, 0)) {
2828 		err = ENOMEM;
2829 		goto errout;
2830 	}
2831 	memset(me->me_M.dma_vaddr, 0, normbits / 8);
2832 	bcopy(krp->krp_param[UBS_MODEXP_PAR_M].crp_p,
2833 	    me->me_M.dma_vaddr, (mbits + 7) / 8);
2834 
2835 	if (ubsec_dma_malloc(sc, normbits / 8, &me->me_C, 0)) {
2836 		err = ENOMEM;
2837 		goto errout;
2838 	}
2839 	memset(me->me_C.dma_vaddr, 0, me->me_C.dma_size);
2840 
2841 	ebits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_E]);
2842 	if (ebits > nbits) {
2843 		err = E2BIG;
2844 		goto errout;
2845 	}
2846 	if (ubsec_dma_malloc(sc, normbits / 8, &me->me_E, 0)) {
2847 		err = ENOMEM;
2848 		goto errout;
2849 	}
2850 	memset(me->me_E.dma_vaddr, 0, normbits / 8);
2851 	bcopy(krp->krp_param[UBS_MODEXP_PAR_E].crp_p,
2852 	    me->me_E.dma_vaddr, (ebits + 7) / 8);
2853 
2854 	if (ubsec_dma_malloc(sc, sizeof(struct ubsec_pktbuf),
2855 	    &me->me_epb, 0)) {
2856 		err = ENOMEM;
2857 		goto errout;
2858 	}
2859 	epb = (struct ubsec_pktbuf *)me->me_epb.dma_vaddr;
2860 	epb->pb_addr = htole32(me->me_E.dma_paddr);
2861 	epb->pb_next = 0;
2862 	epb->pb_len = htole32((ebits + 7) / 8);
2863 
2864 #ifdef UBSEC_DEBUG
2865 	if (ubsec_debug) {
2866 		printf("Epb ");
2867 		ubsec_dump_pb(epb);
2868 	}
2869 #endif
2870 
2871 	mcr->mcr_pkts = htole16(1);
2872 	mcr->mcr_flags = 0;
2873 	mcr->mcr_cmdctxp = htole32(me->me_q.q_ctx.dma_paddr);
2874 	mcr->mcr_reserved = 0;
2875 	mcr->mcr_pktlen = 0;
2876 
2877 	mcr->mcr_ipktbuf.pb_addr = htole32(me->me_M.dma_paddr);
2878 	mcr->mcr_ipktbuf.pb_len = htole32(normbits / 8);
2879 	mcr->mcr_ipktbuf.pb_next = htole32(me->me_epb.dma_paddr);
2880 
2881 	mcr->mcr_opktbuf.pb_addr = htole32(me->me_C.dma_paddr);
2882 	mcr->mcr_opktbuf.pb_next = 0;
2883 	mcr->mcr_opktbuf.pb_len = htole32(normbits / 8);
2884 
2885 #ifdef DIAGNOSTIC
2886 	/* Misaligned output buffer will hang the chip. */
2887 	if ((letoh32(mcr->mcr_opktbuf.pb_addr) & 3) != 0)
2888 		panic("%s: modexp invalid addr 0x%x", device_xname(sc->sc_dev),
2889 		    letoh32(mcr->mcr_opktbuf.pb_addr));
2890 	if ((letoh32(mcr->mcr_opktbuf.pb_len) & 3) != 0)
2891 		panic("%s: modexp invalid len 0x%x", device_xname(sc->sc_dev),
2892 		    letoh32(mcr->mcr_opktbuf.pb_len));
2893 #endif
2894 
2895 	ctx = (struct ubsec_ctx_modexp *)me->me_q.q_ctx.dma_vaddr;
2896 	memset(ctx, 0, sizeof(*ctx));
2897 	memcpy(ctx->me_N, krp->krp_param[UBS_MODEXP_PAR_N].crp_p,
2898 	    (nbits + 7) / 8);
2899 	ctx->me_len = htole16((normbits / 8) + (4 * sizeof(u_int16_t)));
2900 	ctx->me_op = htole16(UBS_CTXOP_MODEXP);
2901 	ctx->me_E_len = htole16(ebits);
2902 	ctx->me_N_len = htole16(nbits);
2903 
2904 #ifdef UBSEC_DEBUG
2905 	if (ubsec_debug) {
2906 		ubsec_dump_mcr(mcr);
2907 		ubsec_dump_ctx2((struct ubsec_ctx_keyop *)ctx);
2908 	}
2909 #endif
2910 
2911 	/*
2912 	 * ubsec_feed2 will sync mcr and ctx, we just need to sync
2913 	 * everything else.
2914 	 */
2915 	bus_dmamap_sync(sc->sc_dmat, me->me_M.dma_map,
2916 	    0, me->me_M.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE);
2917 	bus_dmamap_sync(sc->sc_dmat, me->me_E.dma_map,
2918 	    0, me->me_E.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE);
2919 	bus_dmamap_sync(sc->sc_dmat, me->me_C.dma_map,
2920 	    0, me->me_C.dma_map->dm_mapsize, BUS_DMASYNC_PREREAD);
2921 	bus_dmamap_sync(sc->sc_dmat, me->me_epb.dma_map,
2922 	    0, me->me_epb.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE);
2923 
2924 	/* Enqueue and we're done... */
2925 	mutex_spin_enter(&sc->sc_mtx);
2926 	SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &me->me_q, q_next);
2927 	ubsec_feed2(sc);
2928 	mutex_spin_exit(&sc->sc_mtx);
2929 
2930 	return (0);
2931 
2932 errout:
2933 	if (me != NULL) {
2934 		if (me->me_q.q_mcr.dma_map != NULL)
2935 			ubsec_dma_free(sc, &me->me_q.q_mcr);
2936 		if (me->me_q.q_ctx.dma_map != NULL) {
2937 			memset(me->me_q.q_ctx.dma_vaddr, 0,
2938 			    me->me_q.q_ctx.dma_size);
2939 			ubsec_dma_free(sc, &me->me_q.q_ctx);
2940 		}
2941 		if (me->me_M.dma_map != NULL) {
2942 			memset(me->me_M.dma_vaddr, 0, me->me_M.dma_size);
2943 			ubsec_dma_free(sc, &me->me_M);
2944 		}
2945 		if (me->me_E.dma_map != NULL) {
2946 			memset(me->me_E.dma_vaddr, 0, me->me_E.dma_size);
2947 			ubsec_dma_free(sc, &me->me_E);
2948 		}
2949 		if (me->me_C.dma_map != NULL) {
2950 			memset(me->me_C.dma_vaddr, 0, me->me_C.dma_size);
2951 			ubsec_dma_free(sc, &me->me_C);
2952 		}
2953 		if (me->me_epb.dma_map != NULL)
2954 			ubsec_dma_free(sc, &me->me_epb);
2955 		free(me, M_DEVBUF);
2956 	}
2957 	krp->krp_status = err;
2958 	crypto_kdone(krp);
2959 	return (0);
2960 }
2961 
2962 static int
2963 ubsec_kprocess_rsapriv(struct ubsec_softc *sc, struct cryptkop *krp,
2964 		       int hint)
2965 {
2966 	struct ubsec_q2_rsapriv *rp = NULL;
2967 	struct ubsec_mcr *mcr;
2968 	struct ubsec_ctx_rsapriv *ctx;
2969 	int err = 0;
2970 	u_int padlen, msglen;
2971 
2972 	msglen = ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_P]);
2973 	padlen = ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_Q]);
2974 	if (msglen > padlen)
2975 		padlen = msglen;
2976 
2977 	if (padlen <= 256)
2978 		padlen = 256;
2979 	else if (padlen <= 384)
2980 		padlen = 384;
2981 	else if (padlen <= 512)
2982 		padlen = 512;
2983 	else if (sc->sc_flags & UBS_FLAGS_BIGKEY && padlen <= 768)
2984 		padlen = 768;
2985 	else if (sc->sc_flags & UBS_FLAGS_BIGKEY && padlen <= 1024)
2986 		padlen = 1024;
2987 	else {
2988 		err = E2BIG;
2989 		goto errout;
2990 	}
2991 
2992 	if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_DP]) > padlen) {
2993 		err = E2BIG;
2994 		goto errout;
2995 	}
2996 
2997 	if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_DQ]) > padlen) {
2998 		err = E2BIG;
2999 		goto errout;
3000 	}
3001 
3002 	if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_PINV]) > padlen) {
3003 		err = E2BIG;
3004 		goto errout;
3005 	}
3006 
3007 	rp = malloc(sizeof *rp, M_DEVBUF, M_NOWAIT|M_ZERO);
3008 	if (rp == NULL)
3009 		return (ENOMEM);
3010 	rp->rpr_krp = krp;
3011 	rp->rpr_q.q_type = UBS_CTXOP_RSAPRIV;
3012 
3013 	if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr),
3014 	    &rp->rpr_q.q_mcr, 0)) {
3015 		err = ENOMEM;
3016 		goto errout;
3017 	}
3018 	mcr = (struct ubsec_mcr *)rp->rpr_q.q_mcr.dma_vaddr;
3019 
3020 	if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_rsapriv),
3021 	    &rp->rpr_q.q_ctx, 0)) {
3022 		err = ENOMEM;
3023 		goto errout;
3024 	}
3025 	ctx = (struct ubsec_ctx_rsapriv *)rp->rpr_q.q_ctx.dma_vaddr;
3026 	memset(ctx, 0, sizeof *ctx);
3027 
3028 	/* Copy in p */
3029 	bcopy(krp->krp_param[UBS_RSAPRIV_PAR_P].crp_p,
3030 	    &ctx->rpr_buf[0 * (padlen / 8)],
3031 	    (krp->krp_param[UBS_RSAPRIV_PAR_P].crp_nbits + 7) / 8);
3032 
3033 	/* Copy in q */
3034 	bcopy(krp->krp_param[UBS_RSAPRIV_PAR_Q].crp_p,
3035 	    &ctx->rpr_buf[1 * (padlen / 8)],
3036 	    (krp->krp_param[UBS_RSAPRIV_PAR_Q].crp_nbits + 7) / 8);
3037 
3038 	/* Copy in dp */
3039 	bcopy(krp->krp_param[UBS_RSAPRIV_PAR_DP].crp_p,
3040 	    &ctx->rpr_buf[2 * (padlen / 8)],
3041 	    (krp->krp_param[UBS_RSAPRIV_PAR_DP].crp_nbits + 7) / 8);
3042 
3043 	/* Copy in dq */
3044 	bcopy(krp->krp_param[UBS_RSAPRIV_PAR_DQ].crp_p,
3045 	    &ctx->rpr_buf[3 * (padlen / 8)],
3046 	    (krp->krp_param[UBS_RSAPRIV_PAR_DQ].crp_nbits + 7) / 8);
3047 
3048 	/* Copy in pinv */
3049 	bcopy(krp->krp_param[UBS_RSAPRIV_PAR_PINV].crp_p,
3050 	    &ctx->rpr_buf[4 * (padlen / 8)],
3051 	    (krp->krp_param[UBS_RSAPRIV_PAR_PINV].crp_nbits + 7) / 8);
3052 
3053 	msglen = padlen * 2;
3054 
3055 	/* Copy in input message (aligned buffer/length). */
3056 	if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_MSGIN]) > msglen) {
3057 		/* Is this likely? */
3058 		err = E2BIG;
3059 		goto errout;
3060 	}
3061 	if (ubsec_dma_malloc(sc, (msglen + 7) / 8, &rp->rpr_msgin, 0)) {
3062 		err = ENOMEM;
3063 		goto errout;
3064 	}
3065 	memset(rp->rpr_msgin.dma_vaddr, 0, (msglen + 7) / 8);
3066 	bcopy(krp->krp_param[UBS_RSAPRIV_PAR_MSGIN].crp_p,
3067 	    rp->rpr_msgin.dma_vaddr,
3068 	    (krp->krp_param[UBS_RSAPRIV_PAR_MSGIN].crp_nbits + 7) / 8);
3069 
3070 	/* Prepare space for output message (aligned buffer/length). */
3071 	if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT]) < msglen) {
3072 		/* Is this likely? */
3073 		err = E2BIG;
3074 		goto errout;
3075 	}
3076 	if (ubsec_dma_malloc(sc, (msglen + 7) / 8, &rp->rpr_msgout, 0)) {
3077 		err = ENOMEM;
3078 		goto errout;
3079 	}
3080 	memset(rp->rpr_msgout.dma_vaddr, 0, (msglen + 7) / 8);
3081 
3082 	mcr->mcr_pkts = htole16(1);
3083 	mcr->mcr_flags = 0;
3084 	mcr->mcr_cmdctxp = htole32(rp->rpr_q.q_ctx.dma_paddr);
3085 	mcr->mcr_ipktbuf.pb_addr = htole32(rp->rpr_msgin.dma_paddr);
3086 	mcr->mcr_ipktbuf.pb_next = 0;
3087 	mcr->mcr_ipktbuf.pb_len = htole32(rp->rpr_msgin.dma_size);
3088 	mcr->mcr_reserved = 0;
3089 	mcr->mcr_pktlen = htole16(msglen);
3090 	mcr->mcr_opktbuf.pb_addr = htole32(rp->rpr_msgout.dma_paddr);
3091 	mcr->mcr_opktbuf.pb_next = 0;
3092 	mcr->mcr_opktbuf.pb_len = htole32(rp->rpr_msgout.dma_size);
3093 
3094 #ifdef DIAGNOSTIC
3095 	if (rp->rpr_msgin.dma_paddr & 3 || rp->rpr_msgin.dma_size & 3) {
3096 		panic("%s: rsapriv: invalid msgin 0x%lx(0x%lx)",
3097 		    device_xname(sc->sc_dev), (u_long) rp->rpr_msgin.dma_paddr,
3098 		    (u_long) rp->rpr_msgin.dma_size);
3099 	}
3100 	if (rp->rpr_msgout.dma_paddr & 3 || rp->rpr_msgout.dma_size & 3) {
3101 		panic("%s: rsapriv: invalid msgout 0x%lx(0x%lx)",
3102 		    device_xname(sc->sc_dev), (u_long) rp->rpr_msgout.dma_paddr,
3103 		    (u_long) rp->rpr_msgout.dma_size);
3104 	}
3105 #endif
3106 
3107 	ctx->rpr_len = (sizeof(u_int16_t) * 4) + (5 * (padlen / 8));
3108 	ctx->rpr_op = htole16(UBS_CTXOP_RSAPRIV);
3109 	ctx->rpr_q_len = htole16(padlen);
3110 	ctx->rpr_p_len = htole16(padlen);
3111 
3112 	/*
3113 	 * ubsec_feed2 will sync mcr and ctx, we just need to sync
3114 	 * everything else.
3115 	 */
3116 	bus_dmamap_sync(sc->sc_dmat, rp->rpr_msgin.dma_map,
3117 	    0, rp->rpr_msgin.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE);
3118 	bus_dmamap_sync(sc->sc_dmat, rp->rpr_msgout.dma_map,
3119 	    0, rp->rpr_msgout.dma_map->dm_mapsize, BUS_DMASYNC_PREREAD);
3120 
3121 	/* Enqueue and we're done... */
3122 	mutex_spin_enter(&sc->sc_mtx);
3123 	SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &rp->rpr_q, q_next);
3124 	ubsec_feed2(sc);
3125 	ubsecstats.hst_modexpcrt++;
3126 	mutex_spin_exit(&sc->sc_mtx);
3127 	return (0);
3128 
3129 errout:
3130 	if (rp != NULL) {
3131 		if (rp->rpr_q.q_mcr.dma_map != NULL)
3132 			ubsec_dma_free(sc, &rp->rpr_q.q_mcr);
3133 		if (rp->rpr_msgin.dma_map != NULL) {
3134 			memset(rp->rpr_msgin.dma_vaddr, 0,
3135 			    rp->rpr_msgin.dma_size);
3136 			ubsec_dma_free(sc, &rp->rpr_msgin);
3137 		}
3138 		if (rp->rpr_msgout.dma_map != NULL) {
3139 			memset(rp->rpr_msgout.dma_vaddr, 0,
3140 			    rp->rpr_msgout.dma_size);
3141 			ubsec_dma_free(sc, &rp->rpr_msgout);
3142 		}
3143 		free(rp, M_DEVBUF);
3144 	}
3145 	krp->krp_status = err;
3146 	crypto_kdone(krp);
3147 	return (0);
3148 }
3149 
3150 #ifdef UBSEC_DEBUG
3151 static void
3152 ubsec_dump_pb(volatile struct ubsec_pktbuf *pb)
3153 {
3154 	printf("addr 0x%x (0x%x) next 0x%x\n",
3155 	    pb->pb_addr, pb->pb_len, pb->pb_next);
3156 }
3157 
3158 static void
3159 ubsec_dump_ctx2(volatile struct ubsec_ctx_keyop *c)
3160 {
3161 	printf("CTX (0x%x):\n", c->ctx_len);
3162 	switch (letoh16(c->ctx_op)) {
3163 	case UBS_CTXOP_RNGBYPASS:
3164 	case UBS_CTXOP_RNGSHA1:
3165 		break;
3166 	case UBS_CTXOP_MODEXP:
3167 	{
3168 		struct ubsec_ctx_modexp *cx = (void *)c;
3169 		int i, len;
3170 
3171 		printf(" Elen %u, Nlen %u\n",
3172 		    letoh16(cx->me_E_len), letoh16(cx->me_N_len));
3173 		len = (cx->me_N_len + 7)/8;
3174 		for (i = 0; i < len; i++)
3175 			printf("%s%02x", (i == 0) ? " N: " : ":", cx->me_N[i]);
3176 		printf("\n");
3177 		break;
3178 	}
3179 	default:
3180 		printf("unknown context: %x\n", c->ctx_op);
3181 	}
3182 	printf("END CTX\n");
3183 }
3184 
3185 static void
3186 ubsec_dump_mcr(struct ubsec_mcr *mcr)
3187 {
3188 	volatile struct ubsec_mcr_add *ma;
3189 	int i;
3190 
3191 	printf("MCR:\n");
3192 	printf(" pkts: %u, flags 0x%x\n",
3193 	    letoh16(mcr->mcr_pkts), letoh16(mcr->mcr_flags));
3194 	ma = (volatile struct ubsec_mcr_add *)&mcr->mcr_cmdctxp;
3195 	for (i = 0; i < letoh16(mcr->mcr_pkts); i++) {
3196 		printf(" %d: ctx 0x%x len 0x%x rsvd 0x%x\n", i,
3197 		    letoh32(ma->mcr_cmdctxp), letoh16(ma->mcr_pktlen),
3198 		    letoh16(ma->mcr_reserved));
3199 		printf(" %d: ipkt ", i);
3200 		ubsec_dump_pb(&ma->mcr_ipktbuf);
3201 		printf(" %d: opkt ", i);
3202 		ubsec_dump_pb(&ma->mcr_opktbuf);
3203 		ma++;
3204 	}
3205 	printf("END MCR\n");
3206 }
3207 #endif /* UBSEC_DEBUG */
3208 
3209 /*
3210  * Return the number of significant bits of a big number.
3211  */
3212 static int
3213 ubsec_ksigbits(struct crparam *cr)
3214 {
3215 	u_int plen = (cr->crp_nbits + 7) / 8;
3216 	int i, sig = plen * 8;
3217 	u_int8_t c, *p = cr->crp_p;
3218 
3219 	for (i = plen - 1; i >= 0; i--) {
3220 		c = p[i];
3221 		if (c != 0) {
3222 			while ((c & 0x80) == 0) {
3223 				sig--;
3224 				c <<= 1;
3225 			}
3226 			break;
3227 		}
3228 		sig -= 8;
3229 	}
3230 	return (sig);
3231 }
3232 
3233 static void
3234 ubsec_kshift_r(u_int shiftbits, u_int8_t *src, u_int srcbits,
3235     u_int8_t *dst, u_int dstbits)
3236 {
3237 	u_int slen, dlen;
3238 	int i, si, di, n;
3239 
3240 	slen = (srcbits + 7) / 8;
3241 	dlen = (dstbits + 7) / 8;
3242 
3243 	for (i = 0; i < slen; i++)
3244 		dst[i] = src[i];
3245 	for (i = 0; i < dlen - slen; i++)
3246 		dst[slen + i] = 0;
3247 
3248 	n = shiftbits / 8;
3249 	if (n != 0) {
3250 		si = dlen - n - 1;
3251 		di = dlen - 1;
3252 		while (si >= 0)
3253 			dst[di--] = dst[si--];
3254 		while (di >= 0)
3255 			dst[di--] = 0;
3256 	}
3257 
3258 	n = shiftbits % 8;
3259 	if (n != 0) {
3260 		for (i = dlen - 1; i > 0; i--)
3261 			dst[i] = (dst[i] << n) |
3262 			    (dst[i - 1] >> (8 - n));
3263 		dst[0] = dst[0] << n;
3264 	}
3265 }
3266 
3267 static void
3268 ubsec_kshift_l(u_int shiftbits, u_int8_t *src, u_int srcbits,
3269     u_int8_t *dst, u_int dstbits)
3270 {
3271 	int slen, dlen, i, n;
3272 
3273 	slen = (srcbits + 7) / 8;
3274 	dlen = (dstbits + 7) / 8;
3275 
3276 	n = shiftbits / 8;
3277 	for (i = 0; i < slen; i++)
3278 		dst[i] = src[i + n];
3279 	for (i = 0; i < dlen - slen; i++)
3280 		dst[slen + i] = 0;
3281 
3282 	n = shiftbits % 8;
3283 	if (n != 0) {
3284 		for (i = 0; i < (dlen - 1); i++)
3285 			dst[i] = (dst[i] >> n) | (dst[i + 1] << (8 - n));
3286 		dst[dlen - 1] = dst[dlen - 1] >> n;
3287 	}
3288 }
3289