xref: /netbsd-src/sys/dev/pci/hifn7751.c (revision b7b7574d3bf8eeb51a1fa3977b59142ec6434a55)
1 /*	$NetBSD: hifn7751.c,v 1.55 2014/06/03 13:53:28 msaitoh Exp $	*/
2 /*	$FreeBSD: hifn7751.c,v 1.5.2.7 2003/10/08 23:52:00 sam Exp $ */
3 /*	$OpenBSD: hifn7751.c,v 1.140 2003/08/01 17:55:54 deraadt Exp $	*/
4 
5 /*
6  * Invertex AEON / Hifn 7751 driver
7  * Copyright (c) 1999 Invertex Inc. All rights reserved.
8  * Copyright (c) 1999 Theo de Raadt
9  * Copyright (c) 2000-2001 Network Security Technologies, Inc.
10  *			http://www.netsec.net
11  * Copyright (c) 2003 Hifn Inc.
12  *
13  * This driver is based on a previous driver by Invertex, for which they
14  * requested:  Please send any comments, feedback, bug-fixes, or feature
15  * requests to software@invertex.com.
16  *
17  * Redistribution and use in source and binary forms, with or without
18  * modification, are permitted provided that the following conditions
19  * are met:
20  *
21  * 1. Redistributions of source code must retain the above copyright
22  *   notice, this list of conditions and the following disclaimer.
23  * 2. Redistributions in binary form must reproduce the above copyright
24  *   notice, this list of conditions and the following disclaimer in the
25  *   documentation and/or other materials provided with the distribution.
26  * 3. The name of the author may not be used to endorse or promote products
27  *   derived from this software without specific prior written permission.
28  *
29  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
30  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
31  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
32  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
33  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
34  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
35  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
36  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
37  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
38  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
39  *
40  * Effort sponsored in part by the Defense Advanced Research Projects
41  * Agency (DARPA) and Air Force Research Laboratory, Air Force
42  * Materiel Command, USAF, under agreement number F30602-01-2-0537.
43  *
44  */
45 
46 /*
47  * Driver for various  Hifn pre-HIPP encryption processors.
48  */
49 
50 #include <sys/cdefs.h>
51 __KERNEL_RCSID(0, "$NetBSD: hifn7751.c,v 1.55 2014/06/03 13:53:28 msaitoh Exp $");
52 
53 #include <sys/param.h>
54 #include <sys/systm.h>
55 #include <sys/mutex.h>
56 #include <sys/proc.h>
57 #include <sys/errno.h>
58 #include <sys/malloc.h>
59 #include <sys/kernel.h>
60 #include <sys/mbuf.h>
61 #include <sys/device.h>
62 #include <sys/module.h>
63 
64 #ifdef __OpenBSD__
65 #include <crypto/crypto.h>
66 #include <dev/rndvar.h>
67 #else
68 #include <opencrypto/cryptodev.h>
69 #include <sys/cprng.h>
70 #include <sys/rnd.h>
71 #include <sys/sha1.h>
72 #endif
73 
74 #include <dev/pci/pcireg.h>
75 #include <dev/pci/pcivar.h>
76 #include <dev/pci/pcidevs.h>
77 
78 #include <dev/pci/hifn7751reg.h>
79 #include <dev/pci/hifn7751var.h>
80 
81 #undef HIFN_DEBUG
82 
83 #ifdef __NetBSD__
84 #define M_DUP_PKTHDR M_COPY_PKTHDR	/* XXX */
85 #endif
86 
87 #ifdef HIFN_DEBUG
88 extern int hifn_debug;		/* patchable */
89 int hifn_debug = 1;
90 #endif
91 
92 #ifdef __OpenBSD__
93 #define HAVE_CRYPTO_LZS		/* OpenBSD OCF supports CRYPTO_COMP_LZS */
94 #endif
95 
96 /*
97  * Prototypes and count for the pci_device structure
98  */
99 #ifdef __OpenBSD__
100 static int hifn_probe((struct device *, void *, void *);
101 #else
102 static int hifn_probe(device_t, cfdata_t, void *);
103 #endif
104 static void hifn_attach(device_t, device_t, void *);
105 #ifdef __NetBSD__
106 static int hifn_detach(device_t, int);
107 
108 CFATTACH_DECL_NEW(hifn, sizeof(struct hifn_softc),
109     hifn_probe, hifn_attach, hifn_detach, NULL);
110 #else
111 CFATTACH_DECL_NEW(hifn, sizeof(struct hifn_softc),
112     hifn_probe, hifn_attach, NULL, NULL);
113 #endif
114 
115 #ifdef __OpenBSD__
116 struct cfdriver hifn_cd = {
117 	0, "hifn", DV_DULL
118 };
119 #endif
120 
121 static void	hifn_reset_board(struct hifn_softc *, int);
122 static void	hifn_reset_puc(struct hifn_softc *);
123 static void	hifn_puc_wait(struct hifn_softc *);
124 static const char *hifn_enable_crypto(struct hifn_softc *, pcireg_t);
125 static void	hifn_set_retry(struct hifn_softc *);
126 static void	hifn_init_dma(struct hifn_softc *);
127 static void	hifn_init_pci_registers(struct hifn_softc *);
128 static int	hifn_sramsize(struct hifn_softc *);
129 static int	hifn_dramsize(struct hifn_softc *);
130 static int	hifn_ramtype(struct hifn_softc *);
131 static void	hifn_sessions(struct hifn_softc *);
132 static int	hifn_intr(void *);
133 static u_int	hifn_write_command(struct hifn_command *, u_int8_t *);
134 static u_int32_t hifn_next_signature(u_int32_t a, u_int cnt);
135 static int	hifn_newsession(void*, u_int32_t *, struct cryptoini *);
136 static int	hifn_freesession(void*, u_int64_t);
137 static int	hifn_process(void*, struct cryptop *, int);
138 static void	hifn_callback(struct hifn_softc *, struct hifn_command *,
139 			      u_int8_t *);
140 static int	hifn_crypto(struct hifn_softc *, struct hifn_command *,
141 			    struct cryptop*, int);
142 static int	hifn_readramaddr(struct hifn_softc *, int, u_int8_t *);
143 static int	hifn_writeramaddr(struct hifn_softc *, int, u_int8_t *);
144 static int	hifn_dmamap_aligned(bus_dmamap_t);
145 static int	hifn_dmamap_load_src(struct hifn_softc *,
146 				     struct hifn_command *);
147 static int	hifn_dmamap_load_dst(struct hifn_softc *,
148 				     struct hifn_command *);
149 static int	hifn_init_pubrng(struct hifn_softc *);
150 static void	hifn_rng(void *);
151 static void	hifn_rng_locked(void *);
152 static void	hifn_tick(void *);
153 static void	hifn_abort(struct hifn_softc *);
154 static void	hifn_alloc_slot(struct hifn_softc *, int *, int *, int *,
155 				int *);
156 static void	hifn_write_4(struct hifn_softc *, int, bus_size_t, u_int32_t);
157 static u_int32_t hifn_read_4(struct hifn_softc *, int, bus_size_t);
158 #ifdef	HAVE_CRYPTO_LZS
159 static int	hifn_compression(struct hifn_softc *, struct cryptop *,
160 				 struct hifn_command *);
161 static struct mbuf *hifn_mkmbuf_chain(int, struct mbuf *);
162 static int	hifn_compress_enter(struct hifn_softc *, struct hifn_command *);
163 static void	hifn_callback_comp(struct hifn_softc *, struct hifn_command *,
164 				   u_int8_t *);
165 #endif	/* HAVE_CRYPTO_LZS */
166 
167 struct hifn_stats hifnstats;
168 
169 static const struct hifn_product {
170 	pci_vendor_id_t		hifn_vendor;
171 	pci_product_id_t	hifn_product;
172 	int			hifn_flags;
173 	const char		*hifn_name;
174 } hifn_products[] = {
175 	{ PCI_VENDOR_INVERTEX,	PCI_PRODUCT_INVERTEX_AEON,
176 	  0,
177 	  "Invertex AEON",
178 	},
179 
180 	{ PCI_VENDOR_HIFN,	PCI_PRODUCT_HIFN_7751,
181 	  0,
182 	  "Hifn 7751",
183 	},
184 	{ PCI_VENDOR_NETSEC,	PCI_PRODUCT_NETSEC_7751,
185 	  0,
186 	  "Hifn 7751 (NetSec)"
187 	},
188 
189 	{ PCI_VENDOR_HIFN,	PCI_PRODUCT_HIFN_7811,
190 	  HIFN_IS_7811 | HIFN_HAS_RNG | HIFN_HAS_LEDS | HIFN_NO_BURSTWRITE,
191 	  "Hifn 7811",
192 	},
193 
194 	{ PCI_VENDOR_HIFN,	PCI_PRODUCT_HIFN_7951,
195 	  HIFN_HAS_RNG | HIFN_HAS_PUBLIC,
196 	  "Hifn 7951",
197 	},
198 
199 	{ PCI_VENDOR_HIFN,	PCI_PRODUCT_HIFN_7955,
200 	  HIFN_HAS_RNG | HIFN_HAS_PUBLIC | HIFN_IS_7956 | HIFN_HAS_AES,
201 	  "Hifn 7955",
202 	},
203 
204 	{ PCI_VENDOR_HIFN,	PCI_PRODUCT_HIFN_7956,
205 	  HIFN_HAS_RNG | HIFN_HAS_PUBLIC | HIFN_IS_7956 | HIFN_HAS_AES,
206 	  "Hifn 7956",
207 	},
208 
209 
210 	{ 0,			0,
211 	  0,
212 	  NULL
213 	}
214 };
215 
216 static const struct hifn_product *
217 hifn_lookup(const struct pci_attach_args *pa)
218 {
219 	const struct hifn_product *hp;
220 
221 	for (hp = hifn_products; hp->hifn_name != NULL; hp++) {
222 		if (PCI_VENDOR(pa->pa_id) == hp->hifn_vendor &&
223 		    PCI_PRODUCT(pa->pa_id) == hp->hifn_product)
224 			return (hp);
225 	}
226 	return (NULL);
227 }
228 
229 static int
230 hifn_probe(device_t parent, cfdata_t match, void *aux)
231 {
232 	struct pci_attach_args *pa = aux;
233 
234 	if (hifn_lookup(pa) != NULL)
235 		return 1;
236 
237 	return 0;
238 }
239 
240 static void
241 hifn_attach(device_t parent, device_t self, void *aux)
242 {
243 	struct hifn_softc *sc = device_private(self);
244 	struct pci_attach_args *pa = aux;
245 	const struct hifn_product *hp;
246 	pci_chipset_tag_t pc = pa->pa_pc;
247 	pci_intr_handle_t ih;
248 	const char *intrstr = NULL;
249 	const char *hifncap;
250 	char rbase;
251 #ifdef __NetBSD__
252 #define iosize0 sc->sc_iosz0
253 #define iosize1 sc->sc_iosz1
254 #else
255 	bus_size_t iosize0, iosize1;
256 #endif
257 	u_int32_t cmd;
258 	u_int16_t ena;
259 	bus_dma_segment_t seg;
260 	bus_dmamap_t dmamap;
261 	int rseg;
262 	void *kva;
263 	char intrbuf[PCI_INTRSTR_LEN];
264 
265 	hp = hifn_lookup(pa);
266 	if (hp == NULL) {
267 		printf("\n");
268 		panic("hifn_attach: impossible");
269 	}
270 
271 	pci_aprint_devinfo_fancy(pa, "Crypto processor", hp->hifn_name, 1);
272 
273 	sc->sc_dv = self;
274 	sc->sc_pci_pc = pa->pa_pc;
275 	sc->sc_pci_tag = pa->pa_tag;
276 
277 	sc->sc_flags = hp->hifn_flags;
278 
279 	cmd = pci_conf_read(pc, pa->pa_tag, PCI_COMMAND_STATUS_REG);
280 	cmd |= PCI_COMMAND_MASTER_ENABLE;
281 	pci_conf_write(pc, pa->pa_tag, PCI_COMMAND_STATUS_REG, cmd);
282 
283 	if (pci_mapreg_map(pa, HIFN_BAR0, PCI_MAPREG_TYPE_MEM, 0,
284 	    &sc->sc_st0, &sc->sc_sh0, NULL, &iosize0)) {
285 		aprint_error_dev(sc->sc_dv, "can't map mem space %d\n", 0);
286 		return;
287 	}
288 
289 	if (pci_mapreg_map(pa, HIFN_BAR1, PCI_MAPREG_TYPE_MEM, 0,
290 	    &sc->sc_st1, &sc->sc_sh1, NULL, &iosize1)) {
291 		aprint_error_dev(sc->sc_dv, "can't find mem space %d\n", 1);
292 		goto fail_io0;
293 	}
294 
295 	hifn_set_retry(sc);
296 
297 	if (sc->sc_flags & HIFN_NO_BURSTWRITE) {
298 		sc->sc_waw_lastgroup = -1;
299 		sc->sc_waw_lastreg = 1;
300 	}
301 
302 	sc->sc_dmat = pa->pa_dmat;
303 	if (bus_dmamem_alloc(sc->sc_dmat, sizeof(*sc->sc_dma), PAGE_SIZE, 0,
304 	    &seg, 1, &rseg, BUS_DMA_NOWAIT)) {
305 		aprint_error_dev(sc->sc_dv, "can't alloc DMA buffer\n");
306 		goto fail_io1;
307         }
308 	if (bus_dmamem_map(sc->sc_dmat, &seg, rseg, sizeof(*sc->sc_dma), &kva,
309 	    BUS_DMA_NOWAIT)) {
310 		aprint_error_dev(sc->sc_dv, "can't map DMA buffers (%lu bytes)\n",
311 		    (u_long)sizeof(*sc->sc_dma));
312 		bus_dmamem_free(sc->sc_dmat, &seg, rseg);
313 		goto fail_io1;
314 	}
315 	if (bus_dmamap_create(sc->sc_dmat, sizeof(*sc->sc_dma), 1,
316 	    sizeof(*sc->sc_dma), 0, BUS_DMA_NOWAIT, &dmamap)) {
317 		aprint_error_dev(sc->sc_dv, "can't create DMA map\n");
318 		bus_dmamem_unmap(sc->sc_dmat, kva, sizeof(*sc->sc_dma));
319 		bus_dmamem_free(sc->sc_dmat, &seg, rseg);
320 		goto fail_io1;
321 	}
322 	if (bus_dmamap_load(sc->sc_dmat, dmamap, kva, sizeof(*sc->sc_dma),
323 	    NULL, BUS_DMA_NOWAIT)) {
324 		aprint_error_dev(sc->sc_dv, "can't load DMA map\n");
325 		bus_dmamap_destroy(sc->sc_dmat, dmamap);
326 		bus_dmamem_unmap(sc->sc_dmat, kva, sizeof(*sc->sc_dma));
327 		bus_dmamem_free(sc->sc_dmat, &seg, rseg);
328 		goto fail_io1;
329 	}
330 	sc->sc_dmamap = dmamap;
331 	sc->sc_dma = (struct hifn_dma *)kva;
332 	memset(sc->sc_dma, 0, sizeof(*sc->sc_dma));
333 
334 	hifn_reset_board(sc, 0);
335 
336 	if ((hifncap = hifn_enable_crypto(sc, pa->pa_id)) == NULL) {
337 		aprint_error_dev(sc->sc_dv, "crypto enabling failed\n");
338 		goto fail_mem;
339 	}
340 	hifn_reset_puc(sc);
341 
342 	hifn_init_dma(sc);
343 	hifn_init_pci_registers(sc);
344 
345 	/* XXX can't dynamically determine ram type for 795x; force dram */
346 	if (sc->sc_flags & HIFN_IS_7956)
347 		sc->sc_drammodel = 1;
348 	else if (hifn_ramtype(sc))
349 		goto fail_mem;
350 
351 	if (sc->sc_drammodel == 0)
352 		hifn_sramsize(sc);
353 	else
354 		hifn_dramsize(sc);
355 
356 	/*
357 	 * Workaround for NetSec 7751 rev A: half ram size because two
358 	 * of the address lines were left floating
359 	 */
360 	if (PCI_VENDOR(pa->pa_id) == PCI_VENDOR_NETSEC &&
361 	    PCI_PRODUCT(pa->pa_id) == PCI_PRODUCT_NETSEC_7751 &&
362 	    PCI_REVISION(pa->pa_class) == 0x61)
363 		sc->sc_ramsize >>= 1;
364 
365 	if (pci_intr_map(pa, &ih)) {
366 		aprint_error_dev(sc->sc_dv, "couldn't map interrupt\n");
367 		goto fail_mem;
368 	}
369 	intrstr = pci_intr_string(pc, ih, intrbuf, sizeof(intrbuf));
370 #ifdef	__OpenBSD__
371 	sc->sc_ih = pci_intr_establish(pc, ih, IPL_NET, hifn_intr, sc,
372 	    device_xname(self));
373 #else
374 	sc->sc_ih = pci_intr_establish(pc, ih, IPL_NET, hifn_intr, sc);
375 #endif
376 	if (sc->sc_ih == NULL) {
377 		aprint_error_dev(sc->sc_dv, "couldn't establish interrupt\n");
378 		if (intrstr != NULL)
379 			aprint_error(" at %s", intrstr);
380 		aprint_error("\n");
381 		goto fail_mem;
382 	}
383 
384 	hifn_sessions(sc);
385 
386 	rseg = sc->sc_ramsize / 1024;
387 	rbase = 'K';
388 	if (sc->sc_ramsize >= (1024 * 1024)) {
389 		rbase = 'M';
390 		rseg /= 1024;
391 	}
392 	aprint_normal_dev(sc->sc_dv, "%s, %d%cB %cRAM, interrupting at %s\n",
393 	    hifncap, rseg, rbase,
394 	    sc->sc_drammodel ? 'D' : 'S', intrstr);
395 
396 	sc->sc_cid = crypto_get_driverid(0);
397 	if (sc->sc_cid < 0) {
398 		aprint_error_dev(sc->sc_dv, "couldn't get crypto driver id\n");
399 		goto fail_intr;
400 	}
401 
402 	WRITE_REG_0(sc, HIFN_0_PUCNFG,
403 	    READ_REG_0(sc, HIFN_0_PUCNFG) | HIFN_PUCNFG_CHIPID);
404 	ena = READ_REG_0(sc, HIFN_0_PUSTAT) & HIFN_PUSTAT_CHIPENA;
405 
406 	switch (ena) {
407 	case HIFN_PUSTAT_ENA_2:
408 		crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0,
409 		    hifn_newsession, hifn_freesession, hifn_process, sc);
410 		crypto_register(sc->sc_cid, CRYPTO_ARC4, 0, 0,
411 		    hifn_newsession, hifn_freesession, hifn_process, sc);
412 		if (sc->sc_flags & HIFN_HAS_AES)
413 			crypto_register(sc->sc_cid, CRYPTO_AES_CBC,  0, 0,
414 				hifn_newsession, hifn_freesession,
415 				hifn_process, sc);
416 		/*FALLTHROUGH*/
417 	case HIFN_PUSTAT_ENA_1:
418 		crypto_register(sc->sc_cid, CRYPTO_MD5, 0, 0,
419 		    hifn_newsession, hifn_freesession, hifn_process, sc);
420 		crypto_register(sc->sc_cid, CRYPTO_SHA1, 0, 0,
421 		    hifn_newsession, hifn_freesession, hifn_process, sc);
422 		crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC_96, 0, 0,
423 		    hifn_newsession, hifn_freesession, hifn_process, sc);
424 		crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC_96, 0, 0,
425 		    hifn_newsession, hifn_freesession, hifn_process, sc);
426 		crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0,
427 		    hifn_newsession, hifn_freesession, hifn_process, sc);
428 		break;
429 	}
430 
431 	bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap, 0,
432 	    sc->sc_dmamap->dm_mapsize,
433 	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
434 
435 	if (sc->sc_flags & (HIFN_HAS_PUBLIC | HIFN_HAS_RNG)) {
436 		hifn_init_pubrng(sc);
437 		sc->sc_rng_need = RND_POOLBITS / NBBY;
438 	}
439 
440 	mutex_init(&sc->sc_mtx, MUTEX_DEFAULT, IPL_VM);
441 
442 #ifdef	__OpenBSD__
443 	timeout_set(&sc->sc_tickto, hifn_tick, sc);
444 	timeout_add(&sc->sc_tickto, hz);
445 #else
446 	callout_init(&sc->sc_tickto, CALLOUT_MPSAFE);
447 	callout_reset(&sc->sc_tickto, hz, hifn_tick, sc);
448 #endif
449 	return;
450 
451 fail_intr:
452 	pci_intr_disestablish(pc, sc->sc_ih);
453 fail_mem:
454 	bus_dmamap_unload(sc->sc_dmat, dmamap);
455 	bus_dmamap_destroy(sc->sc_dmat, dmamap);
456 	bus_dmamem_unmap(sc->sc_dmat, kva, sizeof(*sc->sc_dma));
457 	bus_dmamem_free(sc->sc_dmat, &seg, rseg);
458 
459 	/* Turn off DMA polling */
460 	WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
461 	    HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
462 
463 fail_io1:
464 	bus_space_unmap(sc->sc_st1, sc->sc_sh1, iosize1);
465 fail_io0:
466 	bus_space_unmap(sc->sc_st0, sc->sc_sh0, iosize0);
467 }
468 
469 #ifdef __NetBSD__
470 static int
471 hifn_detach(device_t self, int flags)
472 {
473 	struct hifn_softc *sc = device_private(self);
474 
475 	hifn_abort(sc);
476 
477 	hifn_reset_board(sc, 1);
478 
479 	pci_intr_disestablish(sc->sc_pci_pc, sc->sc_ih);
480 
481 	crypto_unregister_all(sc->sc_cid);
482 
483 	rnd_detach_source(&sc->sc_rnd_source);
484 
485 	mutex_enter(&sc->sc_mtx);
486 	callout_halt(&sc->sc_tickto, NULL);
487 	if (sc->sc_flags & (HIFN_HAS_PUBLIC | HIFN_HAS_RNG))
488 		callout_halt(&sc->sc_rngto, NULL);
489 	mutex_exit(&sc->sc_mtx);
490 
491 	bus_space_unmap(sc->sc_st1, sc->sc_sh1, sc->sc_iosz1);
492 	bus_space_unmap(sc->sc_st0, sc->sc_sh0, sc->sc_iosz0);
493 
494 	/*
495 	 * XXX It's not clear if any additional buffers have been
496 	 * XXX allocated and require free()ing
497 	 */
498 
499 	return 0;
500 }
501 
502 MODULE(MODULE_CLASS_DRIVER, hifn, "pci,opencrypto");
503 
504 #ifdef _MODULE
505 #include "ioconf.c"
506 #endif
507 
508 static int
509 hifn_modcmd(modcmd_t cmd, void *data)
510 {
511 	int error = 0;
512 
513 	switch(cmd) {
514 	case MODULE_CMD_INIT:
515 #ifdef _MODULE
516 		error = config_init_component(cfdriver_ioconf_hifn,
517 		    cfattach_ioconf_hifn, cfdata_ioconf_hifn);
518 #endif
519 		return error;
520 	case MODULE_CMD_FINI:
521 #ifdef _MODULE
522 		error = config_fini_component(cfdriver_ioconf_hifn,
523 		    cfattach_ioconf_hifn, cfdata_ioconf_hifn);
524 #endif
525 		return error;
526 	default:
527 		return ENOTTY;
528 	}
529 }
530 
531 #endif /* ifdef __NetBSD__ */
532 
533 static void
534 hifn_rng_get(size_t bytes, void *priv)
535 {
536 	struct hifn_softc *sc = priv;
537 
538 	mutex_enter(&sc->sc_mtx);
539 	sc->sc_rng_need = bytes;
540 
541 	hifn_rng_locked(sc);
542 	mutex_exit(&sc->sc_mtx);
543 }
544 
545 static int
546 hifn_init_pubrng(struct hifn_softc *sc)
547 {
548 	u_int32_t r;
549 	int i;
550 
551 	if ((sc->sc_flags & HIFN_IS_7811) == 0) {
552 		/* Reset 7951 public key/rng engine */
553 		WRITE_REG_1(sc, HIFN_1_PUB_RESET,
554 		    READ_REG_1(sc, HIFN_1_PUB_RESET) | HIFN_PUBRST_RESET);
555 
556 		for (i = 0; i < 100; i++) {
557 			DELAY(1000);
558 			if ((READ_REG_1(sc, HIFN_1_PUB_RESET) &
559 			    HIFN_PUBRST_RESET) == 0)
560 				break;
561 		}
562 
563 		if (i == 100) {
564 			printf("%s: public key init failed\n",
565 			    device_xname(sc->sc_dv));
566 			return (1);
567 		}
568 	}
569 
570 	/* Enable the rng, if available */
571 	if (sc->sc_flags & HIFN_HAS_RNG) {
572 		if (sc->sc_flags & HIFN_IS_7811) {
573 			r = READ_REG_1(sc, HIFN_1_7811_RNGENA);
574 			if (r & HIFN_7811_RNGENA_ENA) {
575 				r &= ~HIFN_7811_RNGENA_ENA;
576 				WRITE_REG_1(sc, HIFN_1_7811_RNGENA, r);
577 			}
578 			WRITE_REG_1(sc, HIFN_1_7811_RNGCFG,
579 			    HIFN_7811_RNGCFG_DEFL);
580 			r |= HIFN_7811_RNGENA_ENA;
581 			WRITE_REG_1(sc, HIFN_1_7811_RNGENA, r);
582 		} else
583 			WRITE_REG_1(sc, HIFN_1_RNG_CONFIG,
584 			    READ_REG_1(sc, HIFN_1_RNG_CONFIG) |
585 			    HIFN_RNGCFG_ENA);
586 
587 		/*
588 		 * The Hifn RNG documentation states that at their
589 		 * recommended "conservative" RNG config values,
590 		 * the RNG must warm up for 0.4s before providing
591 		 * data that meet their worst-case estimate of 0.06
592 		 * bits of random data per output register bit.
593 		 */
594 		DELAY(4000);
595 
596 #ifdef __NetBSD__
597 		rndsource_setcb(&sc->sc_rnd_source, hifn_rng_get, sc);
598 		/*
599 		 * XXX Careful!  The use of RND_FLAG_NO_ESTIMATE
600 		 * XXX here is unobvious: we later feed raw bits
601 		 * XXX into the "entropy pool" with rnd_add_data,
602 		 * XXX explicitly supplying an entropy estimate.
603 		 * XXX In this context, NO_ESTIMATE serves only
604 		 * XXX to prevent rnd_add_data from trying to
605 		 * XXX use the *time at which we added the data*
606 		 * XXX as entropy, which is not a good idea since
607 		 * XXX we add data periodically from a callout.
608 		 */
609 		rnd_attach_source(&sc->sc_rnd_source, device_xname(sc->sc_dv),
610 				  RND_TYPE_RNG,
611 				  RND_FLAG_NO_ESTIMATE|RND_FLAG_HASCB);
612 #endif
613 
614 		if (hz >= 100)
615 			sc->sc_rnghz = hz / 100;
616 		else
617 			sc->sc_rnghz = 1;
618 #ifdef	__OpenBSD__
619 		timeout_set(&sc->sc_rngto, hifn_rng, sc);
620 #else	/* !__OpenBSD__ */
621 		callout_init(&sc->sc_rngto, CALLOUT_MPSAFE);
622 #endif	/* !__OpenBSD__ */
623 	}
624 
625 	/* Enable public key engine, if available */
626 	if (sc->sc_flags & HIFN_HAS_PUBLIC) {
627 		WRITE_REG_1(sc, HIFN_1_PUB_IEN, HIFN_PUBIEN_DONE);
628 		sc->sc_dmaier |= HIFN_DMAIER_PUBDONE;
629 		WRITE_REG_1(sc, HIFN_1_DMA_IER, sc->sc_dmaier);
630 	}
631 
632 	/* Call directly into the RNG once to prime the pool. */
633 	hifn_rng(sc);   /* Sets callout/timeout at end */
634 
635 	return (0);
636 }
637 
638 static void
639 hifn_rng_locked(void *vsc)
640 {
641 	struct hifn_softc *sc = vsc;
642 #ifdef __NetBSD__
643 	uint32_t num[64];
644 #else
645 	uint32_t num[2];
646 #endif
647 	uint32_t sts;
648 	int i;
649 	size_t got, gotent;
650 
651 	if (sc->sc_rng_need < 1) {
652 		callout_stop(&sc->sc_rngto);
653 		return;
654 	}
655 
656 	if (sc->sc_flags & HIFN_IS_7811) {
657 		for (i = 0; i < 5; i++) {	/* XXX why 5? */
658 			sts = READ_REG_1(sc, HIFN_1_7811_RNGSTS);
659 			if (sts & HIFN_7811_RNGSTS_UFL) {
660 				printf("%s: RNG underflow: disabling\n",
661 				    device_xname(sc->sc_dv));
662 				return;
663 			}
664 			if ((sts & HIFN_7811_RNGSTS_RDY) == 0)
665 				break;
666 
667 			/*
668 			 * There are at least two words in the RNG FIFO
669 			 * at this point.
670 			 */
671 			num[0] = READ_REG_1(sc, HIFN_1_7811_RNGDAT);
672 			num[1] = READ_REG_1(sc, HIFN_1_7811_RNGDAT);
673 			got = 2 * sizeof(num[0]);
674 			gotent = (got * NBBY) / HIFN_RNG_BITSPER;
675 
676 #ifdef __NetBSD__
677 			rnd_add_data(&sc->sc_rnd_source, num, got, gotent);
678 			sc->sc_rng_need -= gotent;
679 #else
680 			/*
681 			 * XXX This is a really bad idea.
682 			 * XXX Hifn estimate as little as 0.06
683 			 * XXX actual bits of entropy per output
684 			 * XXX register bit.  How can we tell the
685 			 * XXX kernel RNG subsystem we're handing
686 			 * XXX it 64 "true" random bits, for any
687 			 * XXX sane value of "true"?
688 			 * XXX
689 			 * XXX The right thing to do here, if we
690 			 * XXX cannot supply an estimate ourselves,
691 			 * XXX would be to hash the bits locally.
692 			 */
693 			add_true_randomness(num[0]);
694 			add_true_randomness(num[1]);
695 #endif
696 
697 		}
698 	} else {
699 		int nwords = 0;
700 
701 		if (sc->sc_rng_need) {
702 			nwords = (sc->sc_rng_need * NBBY) / HIFN_RNG_BITSPER;
703 			nwords = MIN(__arraycount(num), nwords);
704 		}
705 
706 		if (nwords < 2) {
707 			nwords = 2;
708 		}
709 
710 		/*
711 		 * We must be *extremely* careful here.  The Hifn
712 		 * 795x differ from the published 6500 RNG design
713 		 * in more ways than the obvious lack of the output
714 		 * FIFO and LFSR control registers.  In fact, there
715 		 * is only one LFSR, instead of the 6500's two, and
716 		 * it's 32 bits, not 31.
717 		 *
718 		 * Further, a block diagram obtained from Hifn shows
719 		 * a very curious latching of this register: the LFSR
720 		 * rotates at a frequency of RNG_Clk / 8, but the
721 		 * RNG_Data register is latched at a frequency of
722 		 * RNG_Clk, which means that it is possible for
723 		 * consecutive reads of the RNG_Data register to read
724 		 * identical state from the LFSR.  The simplest
725 		 * workaround seems to be to read eight samples from
726 		 * the register for each one that we use.  Since each
727 		 * read must require at least one PCI cycle, and
728 		 * RNG_Clk is at least PCI_Clk, this is safe.
729 		 */
730 		for(i = 0 ; i < nwords * 8; i++)
731 		{
732 			volatile u_int32_t regtmp;
733 			regtmp = READ_REG_1(sc, HIFN_1_RNG_DATA);
734 			num[i / 8] = regtmp;
735 		}
736 
737 		got = nwords * sizeof(num[0]);
738 		gotent = (got * NBBY) / HIFN_RNG_BITSPER;
739 #ifdef __NetBSD__
740 		rnd_add_data(&sc->sc_rnd_source, num, got, gotent);
741 		sc->sc_rng_need -= gotent;
742 #else
743 		/* XXX a bad idea; see 7811 block above */
744 		add_true_randomness(num[0]);
745 #endif
746 	}
747 
748 #ifdef	__OpenBSD__
749 	timeout_add(&sc->sc_rngto, sc->sc_rnghz);
750 #else
751 	if (sc->sc_rng_need > 0) {
752 		callout_reset(&sc->sc_rngto, sc->sc_rnghz, hifn_rng, sc);
753 	}
754 #endif
755 }
756 
757 static void
758 hifn_rng(void *vsc)
759 {
760 	struct hifn_softc *sc = vsc;
761 
762 	mutex_spin_enter(&sc->sc_mtx);
763 	hifn_rng_locked(vsc);
764 	mutex_spin_exit(&sc->sc_mtx);
765 }
766 
767 static void
768 hifn_puc_wait(struct hifn_softc *sc)
769 {
770 	int i;
771 
772 	for (i = 5000; i > 0; i--) {
773 		DELAY(1);
774 		if (!(READ_REG_0(sc, HIFN_0_PUCTRL) & HIFN_PUCTRL_RESET))
775 			break;
776 	}
777 	if (!i)
778 		printf("%s: proc unit did not reset\n", device_xname(sc->sc_dv));
779 }
780 
781 /*
782  * Reset the processing unit.
783  */
784 static void
785 hifn_reset_puc(struct hifn_softc *sc)
786 {
787 	/* Reset processing unit */
788 	WRITE_REG_0(sc, HIFN_0_PUCTRL, HIFN_PUCTRL_DMAENA);
789 	hifn_puc_wait(sc);
790 }
791 
792 static void
793 hifn_set_retry(struct hifn_softc *sc)
794 {
795 	u_int32_t r;
796 
797 	r = pci_conf_read(sc->sc_pci_pc, sc->sc_pci_tag, HIFN_TRDY_TIMEOUT);
798 	r &= 0xffff0000;
799 	pci_conf_write(sc->sc_pci_pc, sc->sc_pci_tag, HIFN_TRDY_TIMEOUT, r);
800 }
801 
802 /*
803  * Resets the board.  Values in the regesters are left as is
804  * from the reset (i.e. initial values are assigned elsewhere).
805  */
806 static void
807 hifn_reset_board(struct hifn_softc *sc, int full)
808 {
809 	u_int32_t reg;
810 
811 	/*
812 	 * Set polling in the DMA configuration register to zero.  0x7 avoids
813 	 * resetting the board and zeros out the other fields.
814 	 */
815 	WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
816 	    HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
817 
818 	/*
819 	 * Now that polling has been disabled, we have to wait 1 ms
820 	 * before resetting the board.
821 	 */
822 	DELAY(1000);
823 
824 	/* Reset the DMA unit */
825 	if (full) {
826 		WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MODE);
827 		DELAY(1000);
828 	} else {
829 		WRITE_REG_1(sc, HIFN_1_DMA_CNFG,
830 		    HIFN_DMACNFG_MODE | HIFN_DMACNFG_MSTRESET);
831 		hifn_reset_puc(sc);
832 	}
833 
834 	memset(sc->sc_dma, 0, sizeof(*sc->sc_dma));
835 
836 	/* Bring dma unit out of reset */
837 	WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
838 	    HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
839 
840 	hifn_puc_wait(sc);
841 
842 	hifn_set_retry(sc);
843 
844 	if (sc->sc_flags & HIFN_IS_7811) {
845 		for (reg = 0; reg < 1000; reg++) {
846 			if (READ_REG_1(sc, HIFN_1_7811_MIPSRST) &
847 			    HIFN_MIPSRST_CRAMINIT)
848 				break;
849 			DELAY(1000);
850 		}
851 		if (reg == 1000)
852 			printf(": cram init timeout\n");
853 	}
854 }
855 
856 static u_int32_t
857 hifn_next_signature(u_int32_t a, u_int cnt)
858 {
859 	int i;
860 	u_int32_t v;
861 
862 	for (i = 0; i < cnt; i++) {
863 
864 		/* get the parity */
865 		v = a & 0x80080125;
866 		v ^= v >> 16;
867 		v ^= v >> 8;
868 		v ^= v >> 4;
869 		v ^= v >> 2;
870 		v ^= v >> 1;
871 
872 		a = (v & 1) ^ (a << 1);
873 	}
874 
875 	return a;
876 }
877 
878 static struct pci2id {
879 	u_short		pci_vendor;
880 	u_short		pci_prod;
881 	char		card_id[13];
882 } const pci2id[] = {
883 	{
884 		PCI_VENDOR_HIFN,
885 		PCI_PRODUCT_HIFN_7951,
886 		{ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
887 		  0x00, 0x00, 0x00, 0x00, 0x00 }
888 	}, {
889 		PCI_VENDOR_HIFN,
890 		PCI_PRODUCT_HIFN_7955,
891 		{ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
892 		  0x00, 0x00, 0x00, 0x00, 0x00 }
893 	}, {
894 		PCI_VENDOR_HIFN,
895 		PCI_PRODUCT_HIFN_7956,
896 		{ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
897 		  0x00, 0x00, 0x00, 0x00, 0x00 }
898 	}, {
899 		PCI_VENDOR_NETSEC,
900 		PCI_PRODUCT_NETSEC_7751,
901 		{ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
902 		  0x00, 0x00, 0x00, 0x00, 0x00 }
903 	}, {
904 		PCI_VENDOR_INVERTEX,
905 		PCI_PRODUCT_INVERTEX_AEON,
906 		{ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
907 		  0x00, 0x00, 0x00, 0x00, 0x00 }
908 	}, {
909 		PCI_VENDOR_HIFN,
910 		PCI_PRODUCT_HIFN_7811,
911 		{ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
912 		  0x00, 0x00, 0x00, 0x00, 0x00 }
913 	}, {
914 		/*
915 		 * Other vendors share this PCI ID as well, such as
916 		 * http://www.powercrypt.com, and obviously they also
917 		 * use the same key.
918 		 */
919 		PCI_VENDOR_HIFN,
920 		PCI_PRODUCT_HIFN_7751,
921 		{ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
922 		  0x00, 0x00, 0x00, 0x00, 0x00 }
923 	},
924 };
925 
926 /*
927  * Checks to see if crypto is already enabled.  If crypto isn't enable,
928  * "hifn_enable_crypto" is called to enable it.  The check is important,
929  * as enabling crypto twice will lock the board.
930  */
931 static const char *
932 hifn_enable_crypto(struct hifn_softc *sc, pcireg_t pciid)
933 {
934 	u_int32_t dmacfg, ramcfg, encl, addr, i;
935 	const char *offtbl = NULL;
936 
937 	for (i = 0; i < sizeof(pci2id)/sizeof(pci2id[0]); i++) {
938 		if (pci2id[i].pci_vendor == PCI_VENDOR(pciid) &&
939 		    pci2id[i].pci_prod == PCI_PRODUCT(pciid)) {
940 			offtbl = pci2id[i].card_id;
941 			break;
942 		}
943 	}
944 
945 	if (offtbl == NULL) {
946 #ifdef HIFN_DEBUG
947 		aprint_debug_dev(sc->sc_dv, "Unknown card!\n");
948 #endif
949 		return (NULL);
950 	}
951 
952 	ramcfg = READ_REG_0(sc, HIFN_0_PUCNFG);
953 	dmacfg = READ_REG_1(sc, HIFN_1_DMA_CNFG);
954 
955 	/*
956 	 * The RAM config register's encrypt level bit needs to be set before
957 	 * every read performed on the encryption level register.
958 	 */
959 	WRITE_REG_0(sc, HIFN_0_PUCNFG, ramcfg | HIFN_PUCNFG_CHIPID);
960 
961 	encl = READ_REG_0(sc, HIFN_0_PUSTAT) & HIFN_PUSTAT_CHIPENA;
962 
963 	/*
964 	 * Make sure we don't re-unlock.  Two unlocks kills chip until the
965 	 * next reboot.
966 	 */
967 	if (encl == HIFN_PUSTAT_ENA_1 || encl == HIFN_PUSTAT_ENA_2) {
968 #ifdef HIFN_DEBUG
969 		aprint_debug_dev(sc->sc_dv, "Strong Crypto already enabled!\n");
970 #endif
971 		goto report;
972 	}
973 
974 	if (encl != 0 && encl != HIFN_PUSTAT_ENA_0) {
975 #ifdef HIFN_DEBUG
976 		aprint_debug_dev(sc->sc_dv, "Unknown encryption level\n");
977 #endif
978 		return (NULL);
979 	}
980 
981 	WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_UNLOCK |
982 	    HIFN_DMACNFG_MSTRESET | HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
983 	DELAY(1000);
984 	addr = READ_REG_1(sc, HIFN_1_UNLOCK_SECRET1);
985 	DELAY(1000);
986 	WRITE_REG_1(sc, HIFN_1_UNLOCK_SECRET2, 0);
987 	DELAY(1000);
988 
989 	for (i = 0; i <= 12; i++) {
990 		addr = hifn_next_signature(addr, offtbl[i] + 0x101);
991 		WRITE_REG_1(sc, HIFN_1_UNLOCK_SECRET2, addr);
992 
993 		DELAY(1000);
994 	}
995 
996 	WRITE_REG_0(sc, HIFN_0_PUCNFG, ramcfg | HIFN_PUCNFG_CHIPID);
997 	encl = READ_REG_0(sc, HIFN_0_PUSTAT) & HIFN_PUSTAT_CHIPENA;
998 
999 #ifdef HIFN_DEBUG
1000 	if (encl != HIFN_PUSTAT_ENA_1 && encl != HIFN_PUSTAT_ENA_2)
1001 		aprint_debug("Encryption engine is permanently locked until next system reset.");
1002 	else
1003 		aprint_debug("Encryption engine enabled successfully!");
1004 #endif
1005 
1006 report:
1007 	WRITE_REG_0(sc, HIFN_0_PUCNFG, ramcfg);
1008 	WRITE_REG_1(sc, HIFN_1_DMA_CNFG, dmacfg);
1009 
1010 	switch (encl) {
1011 	case HIFN_PUSTAT_ENA_0:
1012 		return ("LZS-only (no encr/auth)");
1013 
1014 	case HIFN_PUSTAT_ENA_1:
1015 		return ("DES");
1016 
1017 	case HIFN_PUSTAT_ENA_2:
1018 		if (sc->sc_flags & HIFN_HAS_AES)
1019 		    return ("3DES/AES");
1020 		else
1021 		    return ("3DES");
1022 
1023 	default:
1024 		return ("disabled");
1025 	}
1026 	/* NOTREACHED */
1027 }
1028 
1029 /*
1030  * Give initial values to the registers listed in the "Register Space"
1031  * section of the HIFN Software Development reference manual.
1032  */
1033 static void
1034 hifn_init_pci_registers(struct hifn_softc *sc)
1035 {
1036 	/* write fixed values needed by the Initialization registers */
1037 	WRITE_REG_0(sc, HIFN_0_PUCTRL, HIFN_PUCTRL_DMAENA);
1038 	WRITE_REG_0(sc, HIFN_0_FIFOCNFG, HIFN_FIFOCNFG_THRESHOLD);
1039 	WRITE_REG_0(sc, HIFN_0_PUIER, HIFN_PUIER_DSTOVER);
1040 
1041 	/* write all 4 ring address registers */
1042 	WRITE_REG_1(sc, HIFN_1_DMA_CRAR, sc->sc_dmamap->dm_segs[0].ds_addr +
1043 	    offsetof(struct hifn_dma, cmdr[0]));
1044 	WRITE_REG_1(sc, HIFN_1_DMA_SRAR, sc->sc_dmamap->dm_segs[0].ds_addr +
1045 	    offsetof(struct hifn_dma, srcr[0]));
1046 	WRITE_REG_1(sc, HIFN_1_DMA_DRAR, sc->sc_dmamap->dm_segs[0].ds_addr +
1047 	    offsetof(struct hifn_dma, dstr[0]));
1048 	WRITE_REG_1(sc, HIFN_1_DMA_RRAR, sc->sc_dmamap->dm_segs[0].ds_addr +
1049 	    offsetof(struct hifn_dma, resr[0]));
1050 
1051 	DELAY(2000);
1052 
1053 	/* write status register */
1054 	WRITE_REG_1(sc, HIFN_1_DMA_CSR,
1055 	    HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS |
1056 	    HIFN_DMACSR_S_CTRL_DIS | HIFN_DMACSR_C_CTRL_DIS |
1057 	    HIFN_DMACSR_D_ABORT | HIFN_DMACSR_D_DONE | HIFN_DMACSR_D_LAST |
1058 	    HIFN_DMACSR_D_WAIT | HIFN_DMACSR_D_OVER |
1059 	    HIFN_DMACSR_R_ABORT | HIFN_DMACSR_R_DONE | HIFN_DMACSR_R_LAST |
1060 	    HIFN_DMACSR_R_WAIT | HIFN_DMACSR_R_OVER |
1061 	    HIFN_DMACSR_S_ABORT | HIFN_DMACSR_S_DONE | HIFN_DMACSR_S_LAST |
1062 	    HIFN_DMACSR_S_WAIT |
1063 	    HIFN_DMACSR_C_ABORT | HIFN_DMACSR_C_DONE | HIFN_DMACSR_C_LAST |
1064 	    HIFN_DMACSR_C_WAIT |
1065 	    HIFN_DMACSR_ENGINE |
1066 	    ((sc->sc_flags & HIFN_HAS_PUBLIC) ?
1067 		HIFN_DMACSR_PUBDONE : 0) |
1068 	    ((sc->sc_flags & HIFN_IS_7811) ?
1069 		HIFN_DMACSR_ILLW | HIFN_DMACSR_ILLR : 0));
1070 
1071 	sc->sc_d_busy = sc->sc_r_busy = sc->sc_s_busy = sc->sc_c_busy = 0;
1072 	sc->sc_dmaier |= HIFN_DMAIER_R_DONE | HIFN_DMAIER_C_ABORT |
1073 	    HIFN_DMAIER_D_OVER | HIFN_DMAIER_R_OVER |
1074 	    HIFN_DMAIER_S_ABORT | HIFN_DMAIER_D_ABORT | HIFN_DMAIER_R_ABORT |
1075 	    HIFN_DMAIER_ENGINE |
1076 	    ((sc->sc_flags & HIFN_IS_7811) ?
1077 		HIFN_DMAIER_ILLW | HIFN_DMAIER_ILLR : 0);
1078 	sc->sc_dmaier &= ~HIFN_DMAIER_C_WAIT;
1079 	WRITE_REG_1(sc, HIFN_1_DMA_IER, sc->sc_dmaier);
1080 	CLR_LED(sc, HIFN_MIPSRST_LED0 | HIFN_MIPSRST_LED1 | HIFN_MIPSRST_LED2);
1081 
1082 	if (sc->sc_flags & HIFN_IS_7956) {
1083 		WRITE_REG_0(sc, HIFN_0_PUCNFG, HIFN_PUCNFG_COMPSING |
1084 		    HIFN_PUCNFG_TCALLPHASES |
1085 		    HIFN_PUCNFG_TCDRVTOTEM | HIFN_PUCNFG_BUS32);
1086 		WRITE_REG_1(sc, HIFN_1_PLL, HIFN_PLL_7956);
1087 	} else {
1088 		WRITE_REG_0(sc, HIFN_0_PUCNFG, HIFN_PUCNFG_COMPSING |
1089 		    HIFN_PUCNFG_DRFR_128 | HIFN_PUCNFG_TCALLPHASES |
1090 		    HIFN_PUCNFG_TCDRVTOTEM | HIFN_PUCNFG_BUS32 |
1091 		    (sc->sc_drammodel ? HIFN_PUCNFG_DRAM : HIFN_PUCNFG_SRAM));
1092 	}
1093 
1094 	WRITE_REG_0(sc, HIFN_0_PUISR, HIFN_PUISR_DSTOVER);
1095 	WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
1096 	    HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE | HIFN_DMACNFG_LAST |
1097 	    ((HIFN_POLL_FREQUENCY << 16 ) & HIFN_DMACNFG_POLLFREQ) |
1098 	    ((HIFN_POLL_SCALAR << 8) & HIFN_DMACNFG_POLLINVAL));
1099 }
1100 
1101 /*
1102  * The maximum number of sessions supported by the card
1103  * is dependent on the amount of context ram, which
1104  * encryption algorithms are enabled, and how compression
1105  * is configured.  This should be configured before this
1106  * routine is called.
1107  */
1108 static void
1109 hifn_sessions(struct hifn_softc *sc)
1110 {
1111 	u_int32_t pucnfg;
1112 	int ctxsize;
1113 
1114 	pucnfg = READ_REG_0(sc, HIFN_0_PUCNFG);
1115 
1116 	if (pucnfg & HIFN_PUCNFG_COMPSING) {
1117 		if (pucnfg & HIFN_PUCNFG_ENCCNFG)
1118 			ctxsize = 128;
1119 		else
1120 			ctxsize = 512;
1121 		/*
1122 		 * 7955/7956 has internal context memory of 32K
1123 		 */
1124 		if (sc->sc_flags & HIFN_IS_7956)
1125 			sc->sc_maxses = 32768 / ctxsize;
1126 		else
1127 			sc->sc_maxses = 1 +
1128 			    ((sc->sc_ramsize - 32768) / ctxsize);
1129 	}
1130 	else
1131 		sc->sc_maxses = sc->sc_ramsize / 16384;
1132 
1133 	if (sc->sc_maxses > 2048)
1134 		sc->sc_maxses = 2048;
1135 }
1136 
1137 /*
1138  * Determine ram type (sram or dram).  Board should be just out of a reset
1139  * state when this is called.
1140  */
1141 static int
1142 hifn_ramtype(struct hifn_softc *sc)
1143 {
1144 	u_int8_t data[8], dataexpect[8];
1145 	int i;
1146 
1147 	for (i = 0; i < sizeof(data); i++)
1148 		data[i] = dataexpect[i] = 0x55;
1149 	if (hifn_writeramaddr(sc, 0, data))
1150 		return (-1);
1151 	if (hifn_readramaddr(sc, 0, data))
1152 		return (-1);
1153 	if (memcmp(data, dataexpect, sizeof(data)) != 0) {
1154 		sc->sc_drammodel = 1;
1155 		return (0);
1156 	}
1157 
1158 	for (i = 0; i < sizeof(data); i++)
1159 		data[i] = dataexpect[i] = 0xaa;
1160 	if (hifn_writeramaddr(sc, 0, data))
1161 		return (-1);
1162 	if (hifn_readramaddr(sc, 0, data))
1163 		return (-1);
1164 	if (memcmp(data, dataexpect, sizeof(data)) != 0) {
1165 		sc->sc_drammodel = 1;
1166 		return (0);
1167 	}
1168 
1169 	return (0);
1170 }
1171 
1172 #define	HIFN_SRAM_MAX		(32 << 20)
1173 #define	HIFN_SRAM_STEP_SIZE	16384
1174 #define	HIFN_SRAM_GRANULARITY	(HIFN_SRAM_MAX / HIFN_SRAM_STEP_SIZE)
1175 
1176 static int
1177 hifn_sramsize(struct hifn_softc *sc)
1178 {
1179 	u_int32_t a;
1180 	u_int8_t data[8];
1181 	u_int8_t dataexpect[sizeof(data)];
1182 	int32_t i;
1183 
1184 	for (i = 0; i < sizeof(data); i++)
1185 		data[i] = dataexpect[i] = i ^ 0x5a;
1186 
1187 	for (i = HIFN_SRAM_GRANULARITY - 1; i >= 0; i--) {
1188 		a = i * HIFN_SRAM_STEP_SIZE;
1189 		memcpy(data, &i, sizeof(i));
1190 		hifn_writeramaddr(sc, a, data);
1191 	}
1192 
1193 	for (i = 0; i < HIFN_SRAM_GRANULARITY; i++) {
1194 		a = i * HIFN_SRAM_STEP_SIZE;
1195 		memcpy(dataexpect, &i, sizeof(i));
1196 		if (hifn_readramaddr(sc, a, data) < 0)
1197 			return (0);
1198 		if (memcmp(data, dataexpect, sizeof(data)) != 0)
1199 			return (0);
1200 		sc->sc_ramsize = a + HIFN_SRAM_STEP_SIZE;
1201 	}
1202 
1203 	return (0);
1204 }
1205 
1206 /*
1207  * XXX For dram boards, one should really try all of the
1208  * HIFN_PUCNFG_DSZ_*'s.  This just assumes that PUCNFG
1209  * is already set up correctly.
1210  */
1211 static int
1212 hifn_dramsize(struct hifn_softc *sc)
1213 {
1214 	u_int32_t cnfg;
1215 
1216 	if (sc->sc_flags & HIFN_IS_7956) {
1217 		/*
1218 		 * 7955/7956 have a fixed internal ram of only 32K.
1219 		 */
1220 		sc->sc_ramsize = 32768;
1221 	} else {
1222 		cnfg = READ_REG_0(sc, HIFN_0_PUCNFG) &
1223 		    HIFN_PUCNFG_DRAMMASK;
1224 		sc->sc_ramsize = 1 << ((cnfg >> 13) + 18);
1225 	}
1226 	return (0);
1227 }
1228 
1229 static void
1230 hifn_alloc_slot(struct hifn_softc *sc, int *cmdp, int *srcp, int *dstp,
1231     int *resp)
1232 {
1233 	struct hifn_dma *dma = sc->sc_dma;
1234 
1235 	if (dma->cmdi == HIFN_D_CMD_RSIZE) {
1236 		dma->cmdi = 0;
1237 		dma->cmdr[HIFN_D_CMD_RSIZE].l = htole32(HIFN_D_VALID |
1238 		    HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
1239 		HIFN_CMDR_SYNC(sc, HIFN_D_CMD_RSIZE,
1240 		    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
1241 	}
1242 	*cmdp = dma->cmdi++;
1243 	dma->cmdk = dma->cmdi;
1244 
1245 	if (dma->srci == HIFN_D_SRC_RSIZE) {
1246 		dma->srci = 0;
1247 		dma->srcr[HIFN_D_SRC_RSIZE].l = htole32(HIFN_D_VALID |
1248 		    HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
1249 		HIFN_SRCR_SYNC(sc, HIFN_D_SRC_RSIZE,
1250 		    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
1251 	}
1252 	*srcp = dma->srci++;
1253 	dma->srck = dma->srci;
1254 
1255 	if (dma->dsti == HIFN_D_DST_RSIZE) {
1256 		dma->dsti = 0;
1257 		dma->dstr[HIFN_D_DST_RSIZE].l = htole32(HIFN_D_VALID |
1258 		    HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
1259 		HIFN_DSTR_SYNC(sc, HIFN_D_DST_RSIZE,
1260 		    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
1261 	}
1262 	*dstp = dma->dsti++;
1263 	dma->dstk = dma->dsti;
1264 
1265 	if (dma->resi == HIFN_D_RES_RSIZE) {
1266 		dma->resi = 0;
1267 		dma->resr[HIFN_D_RES_RSIZE].l = htole32(HIFN_D_VALID |
1268 		    HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
1269 		HIFN_RESR_SYNC(sc, HIFN_D_RES_RSIZE,
1270 		    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
1271 	}
1272 	*resp = dma->resi++;
1273 	dma->resk = dma->resi;
1274 }
1275 
1276 static int
1277 hifn_writeramaddr(struct hifn_softc *sc, int addr, u_int8_t *data)
1278 {
1279 	struct hifn_dma *dma = sc->sc_dma;
1280 	struct hifn_base_command wc;
1281 	const u_int32_t masks = HIFN_D_VALID | HIFN_D_LAST | HIFN_D_MASKDONEIRQ;
1282 	int r, cmdi, resi, srci, dsti;
1283 
1284 	wc.masks = htole16(3 << 13);
1285 	wc.session_num = htole16(addr >> 14);
1286 	wc.total_source_count = htole16(8);
1287 	wc.total_dest_count = htole16(addr & 0x3fff);
1288 
1289 	hifn_alloc_slot(sc, &cmdi, &srci, &dsti, &resi);
1290 
1291 	WRITE_REG_1(sc, HIFN_1_DMA_CSR,
1292 	    HIFN_DMACSR_C_CTRL_ENA | HIFN_DMACSR_S_CTRL_ENA |
1293 	    HIFN_DMACSR_D_CTRL_ENA | HIFN_DMACSR_R_CTRL_ENA);
1294 
1295 	/* build write command */
1296 	memset(dma->command_bufs[cmdi], 0, HIFN_MAX_COMMAND);
1297 	*(struct hifn_base_command *)dma->command_bufs[cmdi] = wc;
1298 	memcpy(&dma->test_src, data, sizeof(dma->test_src));
1299 
1300 	dma->srcr[srci].p = htole32(sc->sc_dmamap->dm_segs[0].ds_addr
1301 	    + offsetof(struct hifn_dma, test_src));
1302 	dma->dstr[dsti].p = htole32(sc->sc_dmamap->dm_segs[0].ds_addr
1303 	    + offsetof(struct hifn_dma, test_dst));
1304 
1305 	dma->cmdr[cmdi].l = htole32(16 | masks);
1306 	dma->srcr[srci].l = htole32(8 | masks);
1307 	dma->dstr[dsti].l = htole32(4 | masks);
1308 	dma->resr[resi].l = htole32(4 | masks);
1309 
1310 	bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
1311 	    0, sc->sc_dmamap->dm_mapsize,
1312 	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1313 
1314 	for (r = 10000; r >= 0; r--) {
1315 		DELAY(10);
1316 		bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
1317 		    0, sc->sc_dmamap->dm_mapsize,
1318 		    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
1319 		if ((dma->resr[resi].l & htole32(HIFN_D_VALID)) == 0)
1320 			break;
1321 		bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
1322 		    0, sc->sc_dmamap->dm_mapsize,
1323 		    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1324 	}
1325 	if (r == 0) {
1326 		printf("%s: writeramaddr -- "
1327 		    "result[%d](addr %d) still valid\n",
1328 		    device_xname(sc->sc_dv), resi, addr);
1329 		r = -1;
1330 		return (-1);
1331 	} else
1332 		r = 0;
1333 
1334 	WRITE_REG_1(sc, HIFN_1_DMA_CSR,
1335 	    HIFN_DMACSR_C_CTRL_DIS | HIFN_DMACSR_S_CTRL_DIS |
1336 	    HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS);
1337 
1338 	return (r);
1339 }
1340 
1341 static int
1342 hifn_readramaddr(struct hifn_softc *sc, int addr, u_int8_t *data)
1343 {
1344 	struct hifn_dma *dma = sc->sc_dma;
1345 	struct hifn_base_command rc;
1346 	const u_int32_t masks = HIFN_D_VALID | HIFN_D_LAST | HIFN_D_MASKDONEIRQ;
1347 	int r, cmdi, srci, dsti, resi;
1348 
1349 	rc.masks = htole16(2 << 13);
1350 	rc.session_num = htole16(addr >> 14);
1351 	rc.total_source_count = htole16(addr & 0x3fff);
1352 	rc.total_dest_count = htole16(8);
1353 
1354 	hifn_alloc_slot(sc, &cmdi, &srci, &dsti, &resi);
1355 
1356 	WRITE_REG_1(sc, HIFN_1_DMA_CSR,
1357 	    HIFN_DMACSR_C_CTRL_ENA | HIFN_DMACSR_S_CTRL_ENA |
1358 	    HIFN_DMACSR_D_CTRL_ENA | HIFN_DMACSR_R_CTRL_ENA);
1359 
1360 	memset(dma->command_bufs[cmdi], 0, HIFN_MAX_COMMAND);
1361 	*(struct hifn_base_command *)dma->command_bufs[cmdi] = rc;
1362 
1363 	dma->srcr[srci].p = htole32(sc->sc_dmamap->dm_segs[0].ds_addr +
1364 	    offsetof(struct hifn_dma, test_src));
1365 	dma->test_src = 0;
1366 	dma->dstr[dsti].p =  htole32(sc->sc_dmamap->dm_segs[0].ds_addr +
1367 	    offsetof(struct hifn_dma, test_dst));
1368 	dma->test_dst = 0;
1369 	dma->cmdr[cmdi].l = htole32(8 | masks);
1370 	dma->srcr[srci].l = htole32(8 | masks);
1371 	dma->dstr[dsti].l = htole32(8 | masks);
1372 	dma->resr[resi].l = htole32(HIFN_MAX_RESULT | masks);
1373 
1374 	bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
1375 	    0, sc->sc_dmamap->dm_mapsize,
1376 	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1377 
1378 	for (r = 10000; r >= 0; r--) {
1379 		DELAY(10);
1380 		bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
1381 		    0, sc->sc_dmamap->dm_mapsize,
1382 		    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
1383 		if ((dma->resr[resi].l & htole32(HIFN_D_VALID)) == 0)
1384 			break;
1385 		bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
1386 		    0, sc->sc_dmamap->dm_mapsize,
1387 		    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1388 	}
1389 	if (r == 0) {
1390 		printf("%s: readramaddr -- "
1391 		    "result[%d](addr %d) still valid\n",
1392 		    device_xname(sc->sc_dv), resi, addr);
1393 		r = -1;
1394 	} else {
1395 		r = 0;
1396 		memcpy(data, &dma->test_dst, sizeof(dma->test_dst));
1397 	}
1398 
1399 	WRITE_REG_1(sc, HIFN_1_DMA_CSR,
1400 	    HIFN_DMACSR_C_CTRL_DIS | HIFN_DMACSR_S_CTRL_DIS |
1401 	    HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS);
1402 
1403 	return (r);
1404 }
1405 
1406 /*
1407  * Initialize the descriptor rings.
1408  */
1409 static void
1410 hifn_init_dma(struct hifn_softc *sc)
1411 {
1412 	struct hifn_dma *dma = sc->sc_dma;
1413 	int i;
1414 
1415 	hifn_set_retry(sc);
1416 
1417 	/* initialize static pointer values */
1418 	for (i = 0; i < HIFN_D_CMD_RSIZE; i++)
1419 		dma->cmdr[i].p = htole32(sc->sc_dmamap->dm_segs[0].ds_addr +
1420 		    offsetof(struct hifn_dma, command_bufs[i][0]));
1421 	for (i = 0; i < HIFN_D_RES_RSIZE; i++)
1422 		dma->resr[i].p = htole32(sc->sc_dmamap->dm_segs[0].ds_addr +
1423 		    offsetof(struct hifn_dma, result_bufs[i][0]));
1424 
1425 	dma->cmdr[HIFN_D_CMD_RSIZE].p =
1426 	    htole32(sc->sc_dmamap->dm_segs[0].ds_addr +
1427 		offsetof(struct hifn_dma, cmdr[0]));
1428 	dma->srcr[HIFN_D_SRC_RSIZE].p =
1429 	    htole32(sc->sc_dmamap->dm_segs[0].ds_addr +
1430 		offsetof(struct hifn_dma, srcr[0]));
1431 	dma->dstr[HIFN_D_DST_RSIZE].p =
1432 	    htole32(sc->sc_dmamap->dm_segs[0].ds_addr +
1433 		offsetof(struct hifn_dma, dstr[0]));
1434 	dma->resr[HIFN_D_RES_RSIZE].p =
1435 	    htole32(sc->sc_dmamap->dm_segs[0].ds_addr +
1436 		offsetof(struct hifn_dma, resr[0]));
1437 
1438 	dma->cmdu = dma->srcu = dma->dstu = dma->resu = 0;
1439 	dma->cmdi = dma->srci = dma->dsti = dma->resi = 0;
1440 	dma->cmdk = dma->srck = dma->dstk = dma->resk = 0;
1441 }
1442 
1443 /*
1444  * Writes out the raw command buffer space.  Returns the
1445  * command buffer size.
1446  */
1447 static u_int
1448 hifn_write_command(struct hifn_command *cmd, u_int8_t *buf)
1449 {
1450 	u_int8_t *buf_pos;
1451 	struct hifn_base_command *base_cmd;
1452 	struct hifn_mac_command *mac_cmd;
1453 	struct hifn_crypt_command *cry_cmd;
1454 	struct hifn_comp_command *comp_cmd;
1455 	int using_mac, using_crypt, using_comp, len, ivlen;
1456 	u_int32_t dlen, slen;
1457 
1458 	buf_pos = buf;
1459 	using_mac = cmd->base_masks & HIFN_BASE_CMD_MAC;
1460 	using_crypt = cmd->base_masks & HIFN_BASE_CMD_CRYPT;
1461 	using_comp = cmd->base_masks & HIFN_BASE_CMD_COMP;
1462 
1463 	base_cmd = (struct hifn_base_command *)buf_pos;
1464 	base_cmd->masks = htole16(cmd->base_masks);
1465 	slen = cmd->src_map->dm_mapsize;
1466 	if (cmd->sloplen)
1467 		dlen = cmd->dst_map->dm_mapsize - cmd->sloplen +
1468 		    sizeof(u_int32_t);
1469 	else
1470 		dlen = cmd->dst_map->dm_mapsize;
1471 	base_cmd->total_source_count = htole16(slen & HIFN_BASE_CMD_LENMASK_LO);
1472 	base_cmd->total_dest_count = htole16(dlen & HIFN_BASE_CMD_LENMASK_LO);
1473 	dlen >>= 16;
1474 	slen >>= 16;
1475 	base_cmd->session_num = htole16(cmd->session_num |
1476 	    ((slen << HIFN_BASE_CMD_SRCLEN_S) & HIFN_BASE_CMD_SRCLEN_M) |
1477 	    ((dlen << HIFN_BASE_CMD_DSTLEN_S) & HIFN_BASE_CMD_DSTLEN_M));
1478 	buf_pos += sizeof(struct hifn_base_command);
1479 
1480 	if (using_comp) {
1481 		comp_cmd = (struct hifn_comp_command *)buf_pos;
1482 		dlen = cmd->compcrd->crd_len;
1483 		comp_cmd->source_count = htole16(dlen & 0xffff);
1484 		dlen >>= 16;
1485 		comp_cmd->masks = htole16(cmd->comp_masks |
1486 		    ((dlen << HIFN_COMP_CMD_SRCLEN_S) & HIFN_COMP_CMD_SRCLEN_M));
1487 		comp_cmd->header_skip = htole16(cmd->compcrd->crd_skip);
1488 		comp_cmd->reserved = 0;
1489 		buf_pos += sizeof(struct hifn_comp_command);
1490 	}
1491 
1492 	if (using_mac) {
1493 		mac_cmd = (struct hifn_mac_command *)buf_pos;
1494 		dlen = cmd->maccrd->crd_len;
1495 		mac_cmd->source_count = htole16(dlen & 0xffff);
1496 		dlen >>= 16;
1497 		mac_cmd->masks = htole16(cmd->mac_masks |
1498 		    ((dlen << HIFN_MAC_CMD_SRCLEN_S) & HIFN_MAC_CMD_SRCLEN_M));
1499 		mac_cmd->header_skip = htole16(cmd->maccrd->crd_skip);
1500 		mac_cmd->reserved = 0;
1501 		buf_pos += sizeof(struct hifn_mac_command);
1502 	}
1503 
1504 	if (using_crypt) {
1505 		cry_cmd = (struct hifn_crypt_command *)buf_pos;
1506 		dlen = cmd->enccrd->crd_len;
1507 		cry_cmd->source_count = htole16(dlen & 0xffff);
1508 		dlen >>= 16;
1509 		cry_cmd->masks = htole16(cmd->cry_masks |
1510 		    ((dlen << HIFN_CRYPT_CMD_SRCLEN_S) & HIFN_CRYPT_CMD_SRCLEN_M));
1511 		cry_cmd->header_skip = htole16(cmd->enccrd->crd_skip);
1512 		cry_cmd->reserved = 0;
1513 		buf_pos += sizeof(struct hifn_crypt_command);
1514 	}
1515 
1516 	if (using_mac && cmd->mac_masks & HIFN_MAC_CMD_NEW_KEY) {
1517 		memcpy(buf_pos, cmd->mac, HIFN_MAC_KEY_LENGTH);
1518 		buf_pos += HIFN_MAC_KEY_LENGTH;
1519 	}
1520 
1521 	if (using_crypt && cmd->cry_masks & HIFN_CRYPT_CMD_NEW_KEY) {
1522 		switch (cmd->cry_masks & HIFN_CRYPT_CMD_ALG_MASK) {
1523 		case HIFN_CRYPT_CMD_ALG_3DES:
1524 			memcpy(buf_pos, cmd->ck, HIFN_3DES_KEY_LENGTH);
1525 			buf_pos += HIFN_3DES_KEY_LENGTH;
1526 			break;
1527 		case HIFN_CRYPT_CMD_ALG_DES:
1528 			memcpy(buf_pos, cmd->ck, HIFN_DES_KEY_LENGTH);
1529 			buf_pos += HIFN_DES_KEY_LENGTH;
1530 			break;
1531 		case HIFN_CRYPT_CMD_ALG_RC4:
1532 			len = 256;
1533 			do {
1534 				int clen;
1535 
1536 				clen = MIN(cmd->cklen, len);
1537 				memcpy(buf_pos, cmd->ck, clen);
1538 				len -= clen;
1539 				buf_pos += clen;
1540 			} while (len > 0);
1541 			memset(buf_pos, 0, 4);
1542 			buf_pos += 4;
1543 			break;
1544 		case HIFN_CRYPT_CMD_ALG_AES:
1545 			/*
1546 			 * AES keys are variable 128, 192 and
1547 			 * 256 bits (16, 24 and 32 bytes).
1548 			 */
1549 			memcpy(buf_pos, cmd->ck, cmd->cklen);
1550 			buf_pos += cmd->cklen;
1551 			break;
1552 		}
1553 	}
1554 
1555 	if (using_crypt && cmd->cry_masks & HIFN_CRYPT_CMD_NEW_IV) {
1556 		switch (cmd->cry_masks & HIFN_CRYPT_CMD_ALG_MASK) {
1557 		case HIFN_CRYPT_CMD_ALG_AES:
1558 			ivlen = HIFN_AES_IV_LENGTH;
1559 			break;
1560 		default:
1561 			ivlen = HIFN_IV_LENGTH;
1562 			break;
1563 		}
1564 		memcpy(buf_pos, cmd->iv, ivlen);
1565 		buf_pos += ivlen;
1566 	}
1567 
1568 	if ((cmd->base_masks & (HIFN_BASE_CMD_MAC | HIFN_BASE_CMD_CRYPT |
1569 	    HIFN_BASE_CMD_COMP)) == 0) {
1570 		memset(buf_pos, 0, 8);
1571 		buf_pos += 8;
1572 	}
1573 
1574 	return (buf_pos - buf);
1575 }
1576 
1577 static int
1578 hifn_dmamap_aligned(bus_dmamap_t map)
1579 {
1580 	int i;
1581 
1582 	for (i = 0; i < map->dm_nsegs; i++) {
1583 		if (map->dm_segs[i].ds_addr & 3)
1584 			return (0);
1585 		if ((i != (map->dm_nsegs - 1)) &&
1586 		    (map->dm_segs[i].ds_len & 3))
1587 			return (0);
1588 	}
1589 	return (1);
1590 }
1591 
1592 static int
1593 hifn_dmamap_load_dst(struct hifn_softc *sc, struct hifn_command *cmd)
1594 {
1595 	struct hifn_dma *dma = sc->sc_dma;
1596 	bus_dmamap_t map = cmd->dst_map;
1597 	u_int32_t p, l;
1598 	int idx, used = 0, i;
1599 
1600 	idx = dma->dsti;
1601 	for (i = 0; i < map->dm_nsegs - 1; i++) {
1602 		dma->dstr[idx].p = htole32(map->dm_segs[i].ds_addr);
1603 		dma->dstr[idx].l = htole32(HIFN_D_VALID |
1604 		    HIFN_D_MASKDONEIRQ | map->dm_segs[i].ds_len);
1605 		HIFN_DSTR_SYNC(sc, idx,
1606 		    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1607 		used++;
1608 
1609 		if (++idx == HIFN_D_DST_RSIZE) {
1610 			dma->dstr[idx].l = htole32(HIFN_D_VALID |
1611 			    HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
1612 			HIFN_DSTR_SYNC(sc, idx,
1613 			    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1614 			idx = 0;
1615 		}
1616 	}
1617 
1618 	if (cmd->sloplen == 0) {
1619 		p = map->dm_segs[i].ds_addr;
1620 		l = HIFN_D_VALID | HIFN_D_MASKDONEIRQ | HIFN_D_LAST |
1621 		    map->dm_segs[i].ds_len;
1622 	} else {
1623 		p = sc->sc_dmamap->dm_segs[0].ds_addr +
1624 		    offsetof(struct hifn_dma, slop[cmd->slopidx]);
1625 		l = HIFN_D_VALID | HIFN_D_MASKDONEIRQ | HIFN_D_LAST |
1626 		    sizeof(u_int32_t);
1627 
1628 		if ((map->dm_segs[i].ds_len - cmd->sloplen) != 0) {
1629 			dma->dstr[idx].p = htole32(map->dm_segs[i].ds_addr);
1630 			dma->dstr[idx].l = htole32(HIFN_D_VALID |
1631 			    HIFN_D_MASKDONEIRQ |
1632 			    (map->dm_segs[i].ds_len - cmd->sloplen));
1633 			HIFN_DSTR_SYNC(sc, idx,
1634 			    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1635 			used++;
1636 
1637 			if (++idx == HIFN_D_DST_RSIZE) {
1638 				dma->dstr[idx].l = htole32(HIFN_D_VALID |
1639 				    HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
1640 				HIFN_DSTR_SYNC(sc, idx,
1641 				    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1642 				idx = 0;
1643 			}
1644 		}
1645 	}
1646 	dma->dstr[idx].p = htole32(p);
1647 	dma->dstr[idx].l = htole32(l);
1648 	HIFN_DSTR_SYNC(sc, idx, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1649 	used++;
1650 
1651 	if (++idx == HIFN_D_DST_RSIZE) {
1652 		dma->dstr[idx].l = htole32(HIFN_D_VALID | HIFN_D_JUMP |
1653 		    HIFN_D_MASKDONEIRQ);
1654 		HIFN_DSTR_SYNC(sc, idx,
1655 		    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1656 		idx = 0;
1657 	}
1658 
1659 	dma->dsti = idx;
1660 	dma->dstu += used;
1661 	return (idx);
1662 }
1663 
1664 static int
1665 hifn_dmamap_load_src(struct hifn_softc *sc, struct hifn_command *cmd)
1666 {
1667 	struct hifn_dma *dma = sc->sc_dma;
1668 	bus_dmamap_t map = cmd->src_map;
1669 	int idx, i;
1670 	u_int32_t last = 0;
1671 
1672 	idx = dma->srci;
1673 	for (i = 0; i < map->dm_nsegs; i++) {
1674 		if (i == map->dm_nsegs - 1)
1675 			last = HIFN_D_LAST;
1676 
1677 		dma->srcr[idx].p = htole32(map->dm_segs[i].ds_addr);
1678 		dma->srcr[idx].l = htole32(map->dm_segs[i].ds_len |
1679 		    HIFN_D_VALID | HIFN_D_MASKDONEIRQ | last);
1680 		HIFN_SRCR_SYNC(sc, idx,
1681 		    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
1682 
1683 		if (++idx == HIFN_D_SRC_RSIZE) {
1684 			dma->srcr[idx].l = htole32(HIFN_D_VALID |
1685 			    HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
1686 			HIFN_SRCR_SYNC(sc, HIFN_D_SRC_RSIZE,
1687 			    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
1688 			idx = 0;
1689 		}
1690 	}
1691 	dma->srci = idx;
1692 	dma->srcu += map->dm_nsegs;
1693 	return (idx);
1694 }
1695 
1696 static int
1697 hifn_crypto(struct hifn_softc *sc, struct hifn_command *cmd,
1698     struct cryptop *crp, int hint)
1699 {
1700 	struct	hifn_dma *dma = sc->sc_dma;
1701 	u_int32_t cmdlen;
1702 	int	cmdi, resi, err = 0;
1703 
1704 	if (bus_dmamap_create(sc->sc_dmat, HIFN_MAX_DMALEN, MAX_SCATTER,
1705 	    HIFN_MAX_SEGLEN, 0, BUS_DMA_NOWAIT, &cmd->src_map))
1706 		return (ENOMEM);
1707 
1708 	if (crp->crp_flags & CRYPTO_F_IMBUF) {
1709 		if (bus_dmamap_load_mbuf(sc->sc_dmat, cmd->src_map,
1710 		    cmd->srcu.src_m, BUS_DMA_NOWAIT)) {
1711 			err = ENOMEM;
1712 			goto err_srcmap1;
1713 		}
1714 	} else if (crp->crp_flags & CRYPTO_F_IOV) {
1715 		if (bus_dmamap_load_uio(sc->sc_dmat, cmd->src_map,
1716 		    cmd->srcu.src_io, BUS_DMA_NOWAIT)) {
1717 			err = ENOMEM;
1718 			goto err_srcmap1;
1719 		}
1720 	} else {
1721 		err = EINVAL;
1722 		goto err_srcmap1;
1723 	}
1724 
1725 	if (hifn_dmamap_aligned(cmd->src_map)) {
1726 		cmd->sloplen = cmd->src_map->dm_mapsize & 3;
1727 		if (crp->crp_flags & CRYPTO_F_IOV)
1728 			cmd->dstu.dst_io = cmd->srcu.src_io;
1729 		else if (crp->crp_flags & CRYPTO_F_IMBUF)
1730 			cmd->dstu.dst_m = cmd->srcu.src_m;
1731 		cmd->dst_map = cmd->src_map;
1732 	} else {
1733 		if (crp->crp_flags & CRYPTO_F_IOV) {
1734 			err = EINVAL;
1735 			goto err_srcmap;
1736 		} else if (crp->crp_flags & CRYPTO_F_IMBUF) {
1737 			int totlen, len;
1738 			struct mbuf *m, *m0, *mlast;
1739 
1740 			totlen = cmd->src_map->dm_mapsize;
1741 			if (cmd->srcu.src_m->m_flags & M_PKTHDR) {
1742 				len = MHLEN;
1743 				MGETHDR(m0, M_DONTWAIT, MT_DATA);
1744 			} else {
1745 				len = MLEN;
1746 				MGET(m0, M_DONTWAIT, MT_DATA);
1747 			}
1748 			if (m0 == NULL) {
1749 				err = ENOMEM;
1750 				goto err_srcmap;
1751 			}
1752 			if (len == MHLEN)
1753 				M_DUP_PKTHDR(m0, cmd->srcu.src_m);
1754 			if (totlen >= MINCLSIZE) {
1755 				MCLGET(m0, M_DONTWAIT);
1756 				if (m0->m_flags & M_EXT)
1757 					len = MCLBYTES;
1758 			}
1759 			totlen -= len;
1760 			m0->m_pkthdr.len = m0->m_len = len;
1761 			mlast = m0;
1762 
1763 			while (totlen > 0) {
1764 				MGET(m, M_DONTWAIT, MT_DATA);
1765 				if (m == NULL) {
1766 					err = ENOMEM;
1767 					m_freem(m0);
1768 					goto err_srcmap;
1769 				}
1770 				len = MLEN;
1771 				if (totlen >= MINCLSIZE) {
1772 					MCLGET(m, M_DONTWAIT);
1773 					if (m->m_flags & M_EXT)
1774 						len = MCLBYTES;
1775 				}
1776 
1777 				m->m_len = len;
1778 				if (m0->m_flags & M_PKTHDR)
1779 					m0->m_pkthdr.len += len;
1780 				totlen -= len;
1781 
1782 				mlast->m_next = m;
1783 				mlast = m;
1784 			}
1785 			cmd->dstu.dst_m = m0;
1786 		}
1787 	}
1788 
1789 	if (cmd->dst_map == NULL) {
1790 		if (bus_dmamap_create(sc->sc_dmat,
1791 		    HIFN_MAX_SEGLEN * MAX_SCATTER, MAX_SCATTER,
1792 		    HIFN_MAX_SEGLEN, 0, BUS_DMA_NOWAIT, &cmd->dst_map)) {
1793 			err = ENOMEM;
1794 			goto err_srcmap;
1795 		}
1796 		if (crp->crp_flags & CRYPTO_F_IMBUF) {
1797 			if (bus_dmamap_load_mbuf(sc->sc_dmat, cmd->dst_map,
1798 			    cmd->dstu.dst_m, BUS_DMA_NOWAIT)) {
1799 				err = ENOMEM;
1800 				goto err_dstmap1;
1801 			}
1802 		} else if (crp->crp_flags & CRYPTO_F_IOV) {
1803 			if (bus_dmamap_load_uio(sc->sc_dmat, cmd->dst_map,
1804 			    cmd->dstu.dst_io, BUS_DMA_NOWAIT)) {
1805 				err = ENOMEM;
1806 				goto err_dstmap1;
1807 			}
1808 		}
1809 	}
1810 
1811 #ifdef HIFN_DEBUG
1812 	if (hifn_debug)
1813 		printf("%s: Entering cmd: stat %8x ien %8x u %d/%d/%d/%d n %d/%d\n",
1814 		    device_xname(sc->sc_dv),
1815 		    READ_REG_1(sc, HIFN_1_DMA_CSR),
1816 		    READ_REG_1(sc, HIFN_1_DMA_IER),
1817 		    dma->cmdu, dma->srcu, dma->dstu, dma->resu,
1818 		    cmd->src_map->dm_nsegs, cmd->dst_map->dm_nsegs);
1819 #endif
1820 
1821 	if (cmd->src_map == cmd->dst_map)
1822 		bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
1823 		    0, cmd->src_map->dm_mapsize,
1824 		    BUS_DMASYNC_PREWRITE|BUS_DMASYNC_PREREAD);
1825 	else {
1826 		bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
1827 		    0, cmd->src_map->dm_mapsize, BUS_DMASYNC_PREWRITE);
1828 		bus_dmamap_sync(sc->sc_dmat, cmd->dst_map,
1829 		    0, cmd->dst_map->dm_mapsize, BUS_DMASYNC_PREREAD);
1830 	}
1831 
1832 	/*
1833 	 * need 1 cmd, and 1 res
1834 	 * need N src, and N dst
1835 	 */
1836 	if ((dma->cmdu + 1) > HIFN_D_CMD_RSIZE ||
1837 	    (dma->resu + 1) > HIFN_D_RES_RSIZE) {
1838 		err = ENOMEM;
1839 		goto err_dstmap;
1840 	}
1841 	if ((dma->srcu + cmd->src_map->dm_nsegs) > HIFN_D_SRC_RSIZE ||
1842 	    (dma->dstu + cmd->dst_map->dm_nsegs + 1) > HIFN_D_DST_RSIZE) {
1843 		err = ENOMEM;
1844 		goto err_dstmap;
1845 	}
1846 
1847 	if (dma->cmdi == HIFN_D_CMD_RSIZE) {
1848 		dma->cmdi = 0;
1849 		dma->cmdr[HIFN_D_CMD_RSIZE].l = htole32(HIFN_D_VALID |
1850 		    HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
1851 		HIFN_CMDR_SYNC(sc, HIFN_D_CMD_RSIZE,
1852 		    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
1853 	}
1854 	cmdi = dma->cmdi++;
1855 	cmdlen = hifn_write_command(cmd, dma->command_bufs[cmdi]);
1856 	HIFN_CMD_SYNC(sc, cmdi, BUS_DMASYNC_PREWRITE);
1857 
1858 	/* .p for command/result already set */
1859 	dma->cmdr[cmdi].l = htole32(cmdlen | HIFN_D_VALID | HIFN_D_LAST |
1860 	    HIFN_D_MASKDONEIRQ);
1861 	HIFN_CMDR_SYNC(sc, cmdi,
1862 	    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
1863 	dma->cmdu++;
1864 	if (sc->sc_c_busy == 0) {
1865 		WRITE_REG_1(sc, HIFN_1_DMA_CSR, HIFN_DMACSR_C_CTRL_ENA);
1866 		sc->sc_c_busy = 1;
1867 		SET_LED(sc, HIFN_MIPSRST_LED0);
1868 	}
1869 
1870 	/*
1871 	 * We don't worry about missing an interrupt (which a "command wait"
1872 	 * interrupt salvages us from), unless there is more than one command
1873 	 * in the queue.
1874 	 *
1875 	 * XXX We do seem to miss some interrupts.  So we always enable
1876 	 * XXX command wait.  From OpenBSD revision 1.149.
1877 	 *
1878 	 */
1879 #if 0
1880 	if (dma->cmdu > 1) {
1881 #endif
1882 		sc->sc_dmaier |= HIFN_DMAIER_C_WAIT;
1883 		WRITE_REG_1(sc, HIFN_1_DMA_IER, sc->sc_dmaier);
1884 #if 0
1885 	}
1886 #endif
1887 
1888 	hifnstats.hst_ipackets++;
1889 	hifnstats.hst_ibytes += cmd->src_map->dm_mapsize;
1890 
1891 	hifn_dmamap_load_src(sc, cmd);
1892 	if (sc->sc_s_busy == 0) {
1893 		WRITE_REG_1(sc, HIFN_1_DMA_CSR, HIFN_DMACSR_S_CTRL_ENA);
1894 		sc->sc_s_busy = 1;
1895 		SET_LED(sc, HIFN_MIPSRST_LED1);
1896 	}
1897 
1898 	/*
1899 	 * Unlike other descriptors, we don't mask done interrupt from
1900 	 * result descriptor.
1901 	 */
1902 #ifdef HIFN_DEBUG
1903 	if (hifn_debug)
1904 		printf("load res\n");
1905 #endif
1906 	if (dma->resi == HIFN_D_RES_RSIZE) {
1907 		dma->resi = 0;
1908 		dma->resr[HIFN_D_RES_RSIZE].l = htole32(HIFN_D_VALID |
1909 		    HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
1910 		HIFN_RESR_SYNC(sc, HIFN_D_RES_RSIZE,
1911 		    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1912 	}
1913 	resi = dma->resi++;
1914 	dma->hifn_commands[resi] = cmd;
1915 	HIFN_RES_SYNC(sc, resi, BUS_DMASYNC_PREREAD);
1916 	dma->resr[resi].l = htole32(HIFN_MAX_RESULT |
1917 	    HIFN_D_VALID | HIFN_D_LAST);
1918 	HIFN_RESR_SYNC(sc, resi,
1919 	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1920 	dma->resu++;
1921 	if (sc->sc_r_busy == 0) {
1922 		WRITE_REG_1(sc, HIFN_1_DMA_CSR, HIFN_DMACSR_R_CTRL_ENA);
1923 		sc->sc_r_busy = 1;
1924 		SET_LED(sc, HIFN_MIPSRST_LED2);
1925 	}
1926 
1927 	if (cmd->sloplen)
1928 		cmd->slopidx = resi;
1929 
1930 	hifn_dmamap_load_dst(sc, cmd);
1931 
1932 	if (sc->sc_d_busy == 0) {
1933 		WRITE_REG_1(sc, HIFN_1_DMA_CSR, HIFN_DMACSR_D_CTRL_ENA);
1934 		sc->sc_d_busy = 1;
1935 	}
1936 
1937 #ifdef HIFN_DEBUG
1938 	if (hifn_debug)
1939 		printf("%s: command: stat %8x ier %8x\n",
1940 		    device_xname(sc->sc_dv),
1941 		    READ_REG_1(sc, HIFN_1_DMA_CSR), READ_REG_1(sc, HIFN_1_DMA_IER));
1942 #endif
1943 
1944 	sc->sc_active = 5;
1945 	return (err);		/* success */
1946 
1947 err_dstmap:
1948 	if (cmd->src_map != cmd->dst_map)
1949 		bus_dmamap_unload(sc->sc_dmat, cmd->dst_map);
1950 err_dstmap1:
1951 	if (cmd->src_map != cmd->dst_map)
1952 		bus_dmamap_destroy(sc->sc_dmat, cmd->dst_map);
1953 err_srcmap:
1954 	if (crp->crp_flags & CRYPTO_F_IMBUF &&
1955 	    cmd->srcu.src_m != cmd->dstu.dst_m)
1956 		m_freem(cmd->dstu.dst_m);
1957 	bus_dmamap_unload(sc->sc_dmat, cmd->src_map);
1958 err_srcmap1:
1959 	bus_dmamap_destroy(sc->sc_dmat, cmd->src_map);
1960 	return (err);
1961 }
1962 
1963 static void
1964 hifn_tick(void *vsc)
1965 {
1966 	struct hifn_softc *sc = vsc;
1967 
1968 	mutex_spin_enter(&sc->sc_mtx);
1969 	if (sc->sc_active == 0) {
1970 		struct hifn_dma *dma = sc->sc_dma;
1971 		u_int32_t r = 0;
1972 
1973 		if (dma->cmdu == 0 && sc->sc_c_busy) {
1974 			sc->sc_c_busy = 0;
1975 			r |= HIFN_DMACSR_C_CTRL_DIS;
1976 			CLR_LED(sc, HIFN_MIPSRST_LED0);
1977 		}
1978 		if (dma->srcu == 0 && sc->sc_s_busy) {
1979 			sc->sc_s_busy = 0;
1980 			r |= HIFN_DMACSR_S_CTRL_DIS;
1981 			CLR_LED(sc, HIFN_MIPSRST_LED1);
1982 		}
1983 		if (dma->dstu == 0 && sc->sc_d_busy) {
1984 			sc->sc_d_busy = 0;
1985 			r |= HIFN_DMACSR_D_CTRL_DIS;
1986 		}
1987 		if (dma->resu == 0 && sc->sc_r_busy) {
1988 			sc->sc_r_busy = 0;
1989 			r |= HIFN_DMACSR_R_CTRL_DIS;
1990 			CLR_LED(sc, HIFN_MIPSRST_LED2);
1991 		}
1992 		if (r)
1993 			WRITE_REG_1(sc, HIFN_1_DMA_CSR, r);
1994 	}
1995 	else
1996 		sc->sc_active--;
1997 #ifdef	__OpenBSD__
1998 	timeout_add(&sc->sc_tickto, hz);
1999 #else
2000 	callout_reset(&sc->sc_tickto, hz, hifn_tick, sc);
2001 #endif
2002 	mutex_spin_exit(&sc->sc_mtx);
2003 }
2004 
2005 static int
2006 hifn_intr(void *arg)
2007 {
2008 	struct hifn_softc *sc = arg;
2009 	struct hifn_dma *dma = sc->sc_dma;
2010 	u_int32_t dmacsr, restart;
2011 	int i, u;
2012 
2013 	dmacsr = READ_REG_1(sc, HIFN_1_DMA_CSR);
2014 
2015 #ifdef HIFN_DEBUG
2016 	if (hifn_debug)
2017 		printf("%s: irq: stat %08x ien %08x u %d/%d/%d/%d\n",
2018 		       device_xname(sc->sc_dv),
2019 		       dmacsr, READ_REG_1(sc, HIFN_1_DMA_IER),
2020 		       dma->cmdu, dma->srcu, dma->dstu, dma->resu);
2021 #endif
2022 
2023 	mutex_spin_enter(&sc->sc_mtx);
2024 
2025 	/* Nothing in the DMA unit interrupted */
2026 	if ((dmacsr & sc->sc_dmaier) == 0) {
2027 		mutex_spin_exit(&sc->sc_mtx);
2028 		return (0);
2029 	}
2030 
2031 	WRITE_REG_1(sc, HIFN_1_DMA_CSR, dmacsr & sc->sc_dmaier);
2032 
2033 	if (dmacsr & HIFN_DMACSR_ENGINE)
2034 		WRITE_REG_0(sc, HIFN_0_PUISR, READ_REG_0(sc, HIFN_0_PUISR));
2035 
2036 	if ((sc->sc_flags & HIFN_HAS_PUBLIC) &&
2037 	    (dmacsr & HIFN_DMACSR_PUBDONE))
2038 		WRITE_REG_1(sc, HIFN_1_PUB_STATUS,
2039 		    READ_REG_1(sc, HIFN_1_PUB_STATUS) | HIFN_PUBSTS_DONE);
2040 
2041 	restart = dmacsr & (HIFN_DMACSR_R_OVER | HIFN_DMACSR_D_OVER);
2042 	if (restart)
2043 		printf("%s: overrun %x\n", device_xname(sc->sc_dv), dmacsr);
2044 
2045 	if (sc->sc_flags & HIFN_IS_7811) {
2046 		if (dmacsr & HIFN_DMACSR_ILLR)
2047 			printf("%s: illegal read\n", device_xname(sc->sc_dv));
2048 		if (dmacsr & HIFN_DMACSR_ILLW)
2049 			printf("%s: illegal write\n", device_xname(sc->sc_dv));
2050 	}
2051 
2052 	restart = dmacsr & (HIFN_DMACSR_C_ABORT | HIFN_DMACSR_S_ABORT |
2053 	    HIFN_DMACSR_D_ABORT | HIFN_DMACSR_R_ABORT);
2054 	if (restart) {
2055 		printf("%s: abort, resetting.\n", device_xname(sc->sc_dv));
2056 		hifnstats.hst_abort++;
2057 		hifn_abort(sc);
2058 		goto out;
2059 	}
2060 
2061 	if ((dmacsr & HIFN_DMACSR_C_WAIT) && (dma->resu == 0)) {
2062 		/*
2063 		 * If no slots to process and we receive a "waiting on
2064 		 * command" interrupt, we disable the "waiting on command"
2065 		 * (by clearing it).
2066 		 */
2067 		sc->sc_dmaier &= ~HIFN_DMAIER_C_WAIT;
2068 		WRITE_REG_1(sc, HIFN_1_DMA_IER, sc->sc_dmaier);
2069 	}
2070 
2071 	/* clear the rings */
2072 	i = dma->resk;
2073 	while (dma->resu != 0) {
2074 		HIFN_RESR_SYNC(sc, i,
2075 		    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2076 		if (dma->resr[i].l & htole32(HIFN_D_VALID)) {
2077 			HIFN_RESR_SYNC(sc, i,
2078 			    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2079 			break;
2080 		}
2081 
2082 		if (i != HIFN_D_RES_RSIZE) {
2083 			struct hifn_command *cmd;
2084 
2085 			HIFN_RES_SYNC(sc, i, BUS_DMASYNC_POSTREAD);
2086 			cmd = dma->hifn_commands[i];
2087 			KASSERT(cmd != NULL
2088 				/*("hifn_intr: null command slot %u", i)*/);
2089 			dma->hifn_commands[i] = NULL;
2090 
2091 			hifn_callback(sc, cmd, dma->result_bufs[i]);
2092 			hifnstats.hst_opackets++;
2093 		}
2094 
2095 		if (++i == (HIFN_D_RES_RSIZE + 1))
2096 			i = 0;
2097 		else
2098 			dma->resu--;
2099 	}
2100 	dma->resk = i;
2101 
2102 	i = dma->srck; u = dma->srcu;
2103 	while (u != 0) {
2104 		HIFN_SRCR_SYNC(sc, i,
2105 		    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2106 		if (dma->srcr[i].l & htole32(HIFN_D_VALID)) {
2107 			HIFN_SRCR_SYNC(sc, i,
2108 			    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2109 			break;
2110 		}
2111 		if (++i == (HIFN_D_SRC_RSIZE + 1))
2112 			i = 0;
2113 		else
2114 			u--;
2115 	}
2116 	dma->srck = i; dma->srcu = u;
2117 
2118 	i = dma->cmdk; u = dma->cmdu;
2119 	while (u != 0) {
2120 		HIFN_CMDR_SYNC(sc, i,
2121 		    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2122 		if (dma->cmdr[i].l & htole32(HIFN_D_VALID)) {
2123 			HIFN_CMDR_SYNC(sc, i,
2124 			    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2125 			break;
2126 		}
2127 		if (i != HIFN_D_CMD_RSIZE) {
2128 			u--;
2129 			HIFN_CMD_SYNC(sc, i, BUS_DMASYNC_POSTWRITE);
2130 		}
2131 		if (++i == (HIFN_D_CMD_RSIZE + 1))
2132 			i = 0;
2133 	}
2134 	dma->cmdk = i; dma->cmdu = u;
2135 
2136 out:
2137 	mutex_spin_exit(&sc->sc_mtx);
2138 	return (1);
2139 }
2140 
2141 /*
2142  * Allocate a new 'session' and return an encoded session id.  'sidp'
2143  * contains our registration id, and should contain an encoded session
2144  * id on successful allocation.
2145  */
2146 static int
2147 hifn_newsession(void *arg, u_int32_t *sidp, struct cryptoini *cri)
2148 {
2149 	struct cryptoini *c;
2150 	struct hifn_softc *sc = arg;
2151 	int i, mac = 0, cry = 0, comp = 0, retval = EINVAL;
2152 
2153 	KASSERT(sc != NULL /*, ("hifn_newsession: null softc")*/);
2154 	if (sidp == NULL || cri == NULL || sc == NULL)
2155 		return retval;
2156 
2157 	mutex_spin_enter(&sc->sc_mtx);
2158 
2159 	for (i = 0; i < sc->sc_maxses; i++)
2160 		if (sc->sc_sessions[i].hs_state == HS_STATE_FREE)
2161 			break;
2162 	if (i == sc->sc_maxses) {
2163 		retval = ENOMEM;
2164 		goto out;
2165 	}
2166 
2167 	for (c = cri; c != NULL; c = c->cri_next) {
2168 		switch (c->cri_alg) {
2169 		case CRYPTO_MD5:
2170 		case CRYPTO_SHA1:
2171 		case CRYPTO_MD5_HMAC_96:
2172 		case CRYPTO_SHA1_HMAC_96:
2173 			if (mac) {
2174 				goto out;
2175 			}
2176 			mac = 1;
2177 			break;
2178 		case CRYPTO_DES_CBC:
2179 		case CRYPTO_3DES_CBC:
2180 		case CRYPTO_AES_CBC:
2181 			/* Note that this is an initialization
2182 			   vector, not a cipher key; any function
2183 			   giving sufficient Hamming distance
2184 			   between outputs is fine.  Use of RC4
2185 			   to generate IVs has been FIPS140-2
2186 			   certified by several labs. */
2187 #ifdef __NetBSD__
2188 			cprng_fast(sc->sc_sessions[i].hs_iv,
2189 			    c->cri_alg == CRYPTO_AES_CBC ?
2190 				HIFN_AES_IV_LENGTH : HIFN_IV_LENGTH);
2191 #else	/* FreeBSD and OpenBSD have get_random_bytes */
2192 			/* XXX this may read fewer, does it matter? */
2193  			get_random_bytes(sc->sc_sessions[i].hs_iv,
2194 				c->cri_alg == CRYPTO_AES_CBC ?
2195 					HIFN_AES_IV_LENGTH : HIFN_IV_LENGTH);
2196 #endif
2197 			/*FALLTHROUGH*/
2198 		case CRYPTO_ARC4:
2199 			if (cry) {
2200 				goto out;
2201 			}
2202 			cry = 1;
2203 			break;
2204 #ifdef HAVE_CRYPTO_LZS
2205 		case CRYPTO_LZS_COMP:
2206 			if (comp) {
2207 				goto out;
2208 			}
2209 			comp = 1;
2210 			break;
2211 #endif
2212 		default:
2213 			goto out;
2214 		}
2215 	}
2216 	if (mac == 0 && cry == 0 && comp == 0) {
2217 		goto out;
2218 	}
2219 
2220 	/*
2221 	 * XXX only want to support compression without chaining to
2222 	 * MAC/crypt engine right now
2223 	 */
2224 	if ((comp && mac) || (comp && cry)) {
2225 		goto out;
2226 	}
2227 
2228 	*sidp = HIFN_SID(device_unit(sc->sc_dv), i);
2229 	sc->sc_sessions[i].hs_state = HS_STATE_USED;
2230 
2231 	retval = 0;
2232 out:
2233 	mutex_spin_exit(&sc->sc_mtx);
2234 	return retval;
2235 }
2236 
2237 /*
2238  * Deallocate a session.
2239  * XXX this routine should run a zero'd mac/encrypt key into context ram.
2240  * XXX to blow away any keys already stored there.
2241  */
2242 static int
2243 hifn_freesession(void *arg, u_int64_t tid)
2244 {
2245 	struct hifn_softc *sc = arg;
2246 	int session;
2247 	u_int32_t sid = ((u_int32_t) tid) & 0xffffffff;
2248 
2249 	KASSERT(sc != NULL /*, ("hifn_freesession: null softc")*/);
2250 	if (sc == NULL)
2251 		return (EINVAL);
2252 
2253 	mutex_spin_enter(&sc->sc_mtx);
2254 	session = HIFN_SESSION(sid);
2255 	if (session >= sc->sc_maxses) {
2256 		mutex_spin_exit(&sc->sc_mtx);
2257 		return (EINVAL);
2258 	}
2259 
2260 	memset(&sc->sc_sessions[session], 0, sizeof(sc->sc_sessions[session]));
2261 	mutex_spin_exit(&sc->sc_mtx);
2262 	return (0);
2263 }
2264 
2265 static int
2266 hifn_process(void *arg, struct cryptop *crp, int hint)
2267 {
2268 	struct hifn_softc *sc = arg;
2269 	struct hifn_command *cmd = NULL;
2270 	int session, err, ivlen;
2271 	struct cryptodesc *crd1, *crd2, *maccrd, *enccrd;
2272 
2273 	if (crp == NULL || crp->crp_callback == NULL) {
2274 		hifnstats.hst_invalid++;
2275 		return (EINVAL);
2276 	}
2277 
2278 	mutex_spin_enter(&sc->sc_mtx);
2279 	session = HIFN_SESSION(crp->crp_sid);
2280 
2281 	if (sc == NULL || session >= sc->sc_maxses) {
2282 		err = EINVAL;
2283 		goto errout;
2284 	}
2285 
2286 	cmd = (struct hifn_command *)malloc(sizeof(struct hifn_command),
2287 	    M_DEVBUF, M_NOWAIT|M_ZERO);
2288 	if (cmd == NULL) {
2289 		hifnstats.hst_nomem++;
2290 		err = ENOMEM;
2291 		goto errout;
2292 	}
2293 
2294 	if (crp->crp_flags & CRYPTO_F_IMBUF) {
2295 		cmd->srcu.src_m = (struct mbuf *)crp->crp_buf;
2296 		cmd->dstu.dst_m = (struct mbuf *)crp->crp_buf;
2297 	} else if (crp->crp_flags & CRYPTO_F_IOV) {
2298 		cmd->srcu.src_io = (struct uio *)crp->crp_buf;
2299 		cmd->dstu.dst_io = (struct uio *)crp->crp_buf;
2300 	} else {
2301 		err = EINVAL;
2302 		goto errout;	/* XXX we don't handle contiguous buffers! */
2303 	}
2304 
2305 	crd1 = crp->crp_desc;
2306 	if (crd1 == NULL) {
2307 		err = EINVAL;
2308 		goto errout;
2309 	}
2310 	crd2 = crd1->crd_next;
2311 
2312 	if (crd2 == NULL) {
2313 		if (crd1->crd_alg == CRYPTO_MD5_HMAC_96 ||
2314 		    crd1->crd_alg == CRYPTO_SHA1_HMAC_96 ||
2315 		    crd1->crd_alg == CRYPTO_SHA1 ||
2316 		    crd1->crd_alg == CRYPTO_MD5) {
2317 			maccrd = crd1;
2318 			enccrd = NULL;
2319 		} else if (crd1->crd_alg == CRYPTO_DES_CBC ||
2320 			   crd1->crd_alg == CRYPTO_3DES_CBC ||
2321 			   crd1->crd_alg == CRYPTO_AES_CBC ||
2322 			   crd1->crd_alg == CRYPTO_ARC4) {
2323 			if ((crd1->crd_flags & CRD_F_ENCRYPT) == 0)
2324 				cmd->base_masks |= HIFN_BASE_CMD_DECODE;
2325 			maccrd = NULL;
2326 			enccrd = crd1;
2327 #ifdef	HAVE_CRYPTO_LZS
2328 		} else if (crd1->crd_alg == CRYPTO_LZS_COMP) {
2329 		  return (hifn_compression(sc, crp, cmd));
2330 #endif
2331 		} else {
2332 			err = EINVAL;
2333 			goto errout;
2334 		}
2335 	} else {
2336 		if ((crd1->crd_alg == CRYPTO_MD5_HMAC_96 ||
2337 		     crd1->crd_alg == CRYPTO_SHA1_HMAC_96 ||
2338 		     crd1->crd_alg == CRYPTO_MD5 ||
2339 		     crd1->crd_alg == CRYPTO_SHA1) &&
2340 		    (crd2->crd_alg == CRYPTO_DES_CBC ||
2341 		     crd2->crd_alg == CRYPTO_3DES_CBC ||
2342 		     crd2->crd_alg == CRYPTO_AES_CBC ||
2343 		     crd2->crd_alg == CRYPTO_ARC4) &&
2344 		    ((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) {
2345 			cmd->base_masks = HIFN_BASE_CMD_DECODE;
2346 			maccrd = crd1;
2347 			enccrd = crd2;
2348 		} else if ((crd1->crd_alg == CRYPTO_DES_CBC ||
2349 			    crd1->crd_alg == CRYPTO_ARC4 ||
2350 			    crd1->crd_alg == CRYPTO_3DES_CBC ||
2351 			    crd1->crd_alg == CRYPTO_AES_CBC) &&
2352 			   (crd2->crd_alg == CRYPTO_MD5_HMAC_96 ||
2353 			    crd2->crd_alg == CRYPTO_SHA1_HMAC_96 ||
2354 			    crd2->crd_alg == CRYPTO_MD5 ||
2355 			    crd2->crd_alg == CRYPTO_SHA1) &&
2356 			   (crd1->crd_flags & CRD_F_ENCRYPT)) {
2357 			enccrd = crd1;
2358 			maccrd = crd2;
2359 		} else {
2360 			/*
2361 			 * We cannot order the 7751 as requested
2362 			 */
2363 			err = EINVAL;
2364 			goto errout;
2365 		}
2366 	}
2367 
2368 	if (enccrd) {
2369 		cmd->enccrd = enccrd;
2370 		cmd->base_masks |= HIFN_BASE_CMD_CRYPT;
2371 		switch (enccrd->crd_alg) {
2372 		case CRYPTO_ARC4:
2373 			cmd->cry_masks |= HIFN_CRYPT_CMD_ALG_RC4;
2374 			if ((enccrd->crd_flags & CRD_F_ENCRYPT)
2375 			    != sc->sc_sessions[session].hs_prev_op)
2376 				sc->sc_sessions[session].hs_state =
2377 				    HS_STATE_USED;
2378 			break;
2379 		case CRYPTO_DES_CBC:
2380 			cmd->cry_masks |= HIFN_CRYPT_CMD_ALG_DES |
2381 			    HIFN_CRYPT_CMD_MODE_CBC |
2382 			    HIFN_CRYPT_CMD_NEW_IV;
2383 			break;
2384 		case CRYPTO_3DES_CBC:
2385 			cmd->cry_masks |= HIFN_CRYPT_CMD_ALG_3DES |
2386 			    HIFN_CRYPT_CMD_MODE_CBC |
2387 			    HIFN_CRYPT_CMD_NEW_IV;
2388 			break;
2389 		case CRYPTO_AES_CBC:
2390 			cmd->cry_masks |= HIFN_CRYPT_CMD_ALG_AES |
2391 			    HIFN_CRYPT_CMD_MODE_CBC |
2392 			    HIFN_CRYPT_CMD_NEW_IV;
2393 			break;
2394 		default:
2395 			err = EINVAL;
2396 			goto errout;
2397 		}
2398 		if (enccrd->crd_alg != CRYPTO_ARC4) {
2399 			ivlen = ((enccrd->crd_alg == CRYPTO_AES_CBC) ?
2400 				HIFN_AES_IV_LENGTH : HIFN_IV_LENGTH);
2401 			if (enccrd->crd_flags & CRD_F_ENCRYPT) {
2402 				if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
2403 					memcpy(cmd->iv, enccrd->crd_iv, ivlen);
2404 				else
2405 					bcopy(sc->sc_sessions[session].hs_iv,
2406 					    cmd->iv, ivlen);
2407 
2408 				if ((enccrd->crd_flags & CRD_F_IV_PRESENT)
2409 				    == 0) {
2410 					if (crp->crp_flags & CRYPTO_F_IMBUF)
2411 						m_copyback(cmd->srcu.src_m,
2412 						    enccrd->crd_inject,
2413 						    ivlen, cmd->iv);
2414 					else if (crp->crp_flags & CRYPTO_F_IOV)
2415 						cuio_copyback(cmd->srcu.src_io,
2416 						    enccrd->crd_inject,
2417 						    ivlen, cmd->iv);
2418 				}
2419 			} else {
2420 				if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
2421 					memcpy(cmd->iv, enccrd->crd_iv, ivlen);
2422 				else if (crp->crp_flags & CRYPTO_F_IMBUF)
2423 					m_copydata(cmd->srcu.src_m,
2424 					    enccrd->crd_inject, ivlen, cmd->iv);
2425 				else if (crp->crp_flags & CRYPTO_F_IOV)
2426 					cuio_copydata(cmd->srcu.src_io,
2427 					    enccrd->crd_inject, ivlen, cmd->iv);
2428 			}
2429 		}
2430 
2431 		cmd->ck = enccrd->crd_key;
2432 		cmd->cklen = enccrd->crd_klen >> 3;
2433 
2434 		/*
2435 		 * Need to specify the size for the AES key in the masks.
2436 		 */
2437 		if ((cmd->cry_masks & HIFN_CRYPT_CMD_ALG_MASK) ==
2438 		    HIFN_CRYPT_CMD_ALG_AES) {
2439 			switch (cmd->cklen) {
2440 			case 16:
2441 				cmd->cry_masks |= HIFN_CRYPT_CMD_KSZ_128;
2442 				break;
2443 			case 24:
2444 				cmd->cry_masks |= HIFN_CRYPT_CMD_KSZ_192;
2445 				break;
2446 			case 32:
2447 				cmd->cry_masks |= HIFN_CRYPT_CMD_KSZ_256;
2448 				break;
2449 			default:
2450 				err = EINVAL;
2451 				goto errout;
2452 			}
2453 		}
2454 
2455 		if (sc->sc_sessions[session].hs_state == HS_STATE_USED)
2456 			cmd->cry_masks |= HIFN_CRYPT_CMD_NEW_KEY;
2457 	}
2458 
2459 	if (maccrd) {
2460 		cmd->maccrd = maccrd;
2461 		cmd->base_masks |= HIFN_BASE_CMD_MAC;
2462 
2463 		switch (maccrd->crd_alg) {
2464 		case CRYPTO_MD5:
2465 			cmd->mac_masks |= HIFN_MAC_CMD_ALG_MD5 |
2466 			    HIFN_MAC_CMD_RESULT | HIFN_MAC_CMD_MODE_HASH |
2467 			    HIFN_MAC_CMD_POS_IPSEC;
2468 			break;
2469 		case CRYPTO_MD5_HMAC_96:
2470 			cmd->mac_masks |= HIFN_MAC_CMD_ALG_MD5 |
2471 			    HIFN_MAC_CMD_RESULT | HIFN_MAC_CMD_MODE_HMAC |
2472 			    HIFN_MAC_CMD_POS_IPSEC | HIFN_MAC_CMD_TRUNC;
2473 			break;
2474 		case CRYPTO_SHA1:
2475 			cmd->mac_masks |= HIFN_MAC_CMD_ALG_SHA1 |
2476 			    HIFN_MAC_CMD_RESULT | HIFN_MAC_CMD_MODE_HASH |
2477 			    HIFN_MAC_CMD_POS_IPSEC;
2478 			break;
2479 		case CRYPTO_SHA1_HMAC_96:
2480 			cmd->mac_masks |= HIFN_MAC_CMD_ALG_SHA1 |
2481 			    HIFN_MAC_CMD_RESULT | HIFN_MAC_CMD_MODE_HMAC |
2482 			    HIFN_MAC_CMD_POS_IPSEC | HIFN_MAC_CMD_TRUNC;
2483 			break;
2484 		}
2485 
2486 		if ((maccrd->crd_alg == CRYPTO_SHA1_HMAC_96 ||
2487 		     maccrd->crd_alg == CRYPTO_MD5_HMAC_96) &&
2488 		    sc->sc_sessions[session].hs_state == HS_STATE_USED) {
2489 			cmd->mac_masks |= HIFN_MAC_CMD_NEW_KEY;
2490 			memcpy(cmd->mac, maccrd->crd_key, maccrd->crd_klen >> 3);
2491 			memset(cmd->mac + (maccrd->crd_klen >> 3), 0,
2492 			    HIFN_MAC_KEY_LENGTH - (maccrd->crd_klen >> 3));
2493 		}
2494 	}
2495 
2496 	cmd->crp = crp;
2497 	cmd->session_num = session;
2498 	cmd->softc = sc;
2499 
2500 	err = hifn_crypto(sc, cmd, crp, hint);
2501 	if (err == 0) {
2502 		if (enccrd)
2503 			sc->sc_sessions[session].hs_prev_op =
2504 				enccrd->crd_flags & CRD_F_ENCRYPT;
2505 		if (sc->sc_sessions[session].hs_state == HS_STATE_USED)
2506 			sc->sc_sessions[session].hs_state = HS_STATE_KEY;
2507 		mutex_spin_exit(&sc->sc_mtx);
2508 		return 0;
2509 	} else if (err == ERESTART) {
2510 		/*
2511 		 * There weren't enough resources to dispatch the request
2512 		 * to the part.  Notify the caller so they'll requeue this
2513 		 * request and resubmit it again soon.
2514 		 */
2515 #ifdef HIFN_DEBUG
2516 		if (hifn_debug)
2517 			printf("%s: requeue request\n", device_xname(sc->sc_dv));
2518 #endif
2519 		free(cmd, M_DEVBUF);
2520 		sc->sc_needwakeup |= CRYPTO_SYMQ;
2521 		mutex_spin_exit(&sc->sc_mtx);
2522 		return (err);
2523 	}
2524 
2525 errout:
2526 	if (cmd != NULL)
2527 		free(cmd, M_DEVBUF);
2528 	if (err == EINVAL)
2529 		hifnstats.hst_invalid++;
2530 	else
2531 		hifnstats.hst_nomem++;
2532 	crp->crp_etype = err;
2533 	mutex_spin_exit(&sc->sc_mtx);
2534 	crypto_done(crp);
2535 	return (0);
2536 }
2537 
2538 static void
2539 hifn_abort(struct hifn_softc *sc)
2540 {
2541 	struct hifn_dma *dma = sc->sc_dma;
2542 	struct hifn_command *cmd;
2543 	struct cryptop *crp;
2544 	int i, u;
2545 
2546 	i = dma->resk; u = dma->resu;
2547 	while (u != 0) {
2548 		cmd = dma->hifn_commands[i];
2549 		KASSERT(cmd != NULL /*, ("hifn_abort: null cmd slot %u", i)*/);
2550 		dma->hifn_commands[i] = NULL;
2551 		crp = cmd->crp;
2552 
2553 		if ((dma->resr[i].l & htole32(HIFN_D_VALID)) == 0) {
2554 			/* Salvage what we can. */
2555 			hifnstats.hst_opackets++;
2556 			hifn_callback(sc, cmd, dma->result_bufs[i]);
2557 		} else {
2558 			if (cmd->src_map == cmd->dst_map) {
2559 				bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
2560 						0, cmd->src_map->dm_mapsize,
2561 				    BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
2562 			} else {
2563 				bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
2564 				    0, cmd->src_map->dm_mapsize,
2565 				    BUS_DMASYNC_POSTWRITE);
2566 				bus_dmamap_sync(sc->sc_dmat, cmd->dst_map,
2567 				    0, cmd->dst_map->dm_mapsize,
2568 				    BUS_DMASYNC_POSTREAD);
2569 			}
2570 
2571 			if (cmd->srcu.src_m != cmd->dstu.dst_m) {
2572 				m_freem(cmd->srcu.src_m);
2573 				crp->crp_buf = (void *)cmd->dstu.dst_m;
2574 			}
2575 
2576 			/* non-shared buffers cannot be restarted */
2577 			if (cmd->src_map != cmd->dst_map) {
2578 				/*
2579 				 * XXX should be EAGAIN, delayed until
2580 				 * after the reset.
2581 				 */
2582 				crp->crp_etype = ENOMEM;
2583 				bus_dmamap_unload(sc->sc_dmat, cmd->dst_map);
2584 				bus_dmamap_destroy(sc->sc_dmat, cmd->dst_map);
2585 			} else
2586 				crp->crp_etype = ENOMEM;
2587 
2588 			bus_dmamap_unload(sc->sc_dmat, cmd->src_map);
2589 			bus_dmamap_destroy(sc->sc_dmat, cmd->src_map);
2590 
2591 			free(cmd, M_DEVBUF);
2592 			if (crp->crp_etype != EAGAIN)
2593 				crypto_done(crp);
2594 		}
2595 
2596 		if (++i == HIFN_D_RES_RSIZE)
2597 			i = 0;
2598 		u--;
2599 	}
2600 	dma->resk = i; dma->resu = u;
2601 
2602 	/* Force upload of key next time */
2603 	for (i = 0; i < sc->sc_maxses; i++)
2604 		if (sc->sc_sessions[i].hs_state == HS_STATE_KEY)
2605 			sc->sc_sessions[i].hs_state = HS_STATE_USED;
2606 
2607 	hifn_reset_board(sc, 1);
2608 	hifn_init_dma(sc);
2609 	hifn_init_pci_registers(sc);
2610 }
2611 
2612 static void
2613 hifn_callback(struct hifn_softc *sc, struct hifn_command *cmd, u_int8_t *resbuf)
2614 {
2615 	struct hifn_dma *dma = sc->sc_dma;
2616 	struct cryptop *crp = cmd->crp;
2617 	struct cryptodesc *crd;
2618 	struct mbuf *m;
2619 	int totlen, i, u, ivlen;
2620 
2621 	if (cmd->src_map == cmd->dst_map)
2622 		bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
2623 		    0, cmd->src_map->dm_mapsize,
2624 		    BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
2625 	else {
2626 		bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
2627 		    0, cmd->src_map->dm_mapsize, BUS_DMASYNC_POSTWRITE);
2628 		bus_dmamap_sync(sc->sc_dmat, cmd->dst_map,
2629 		    0, cmd->dst_map->dm_mapsize, BUS_DMASYNC_POSTREAD);
2630 	}
2631 
2632 	if (crp->crp_flags & CRYPTO_F_IMBUF) {
2633 		if (cmd->srcu.src_m != cmd->dstu.dst_m) {
2634 			crp->crp_buf = (void *)cmd->dstu.dst_m;
2635 			totlen = cmd->src_map->dm_mapsize;
2636 			for (m = cmd->dstu.dst_m; m != NULL; m = m->m_next) {
2637 				if (totlen < m->m_len) {
2638 					m->m_len = totlen;
2639 					totlen = 0;
2640 				} else
2641 					totlen -= m->m_len;
2642 			}
2643 			cmd->dstu.dst_m->m_pkthdr.len =
2644 			    cmd->srcu.src_m->m_pkthdr.len;
2645 			m_freem(cmd->srcu.src_m);
2646 		}
2647 	}
2648 
2649 	if (cmd->sloplen != 0) {
2650 		if (crp->crp_flags & CRYPTO_F_IMBUF)
2651 			m_copyback((struct mbuf *)crp->crp_buf,
2652 			    cmd->src_map->dm_mapsize - cmd->sloplen,
2653 			    cmd->sloplen, (void *)&dma->slop[cmd->slopidx]);
2654 		else if (crp->crp_flags & CRYPTO_F_IOV)
2655 			cuio_copyback((struct uio *)crp->crp_buf,
2656 			    cmd->src_map->dm_mapsize - cmd->sloplen,
2657 			    cmd->sloplen, (void *)&dma->slop[cmd->slopidx]);
2658 	}
2659 
2660 	i = dma->dstk; u = dma->dstu;
2661 	while (u != 0) {
2662 		bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
2663 		    offsetof(struct hifn_dma, dstr[i]), sizeof(struct hifn_desc),
2664 		    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2665 		if (dma->dstr[i].l & htole32(HIFN_D_VALID)) {
2666 			bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
2667 			    offsetof(struct hifn_dma, dstr[i]),
2668 			    sizeof(struct hifn_desc),
2669 			    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2670 			break;
2671 		}
2672 		if (++i == (HIFN_D_DST_RSIZE + 1))
2673 			i = 0;
2674 		else
2675 			u--;
2676 	}
2677 	dma->dstk = i; dma->dstu = u;
2678 
2679 	hifnstats.hst_obytes += cmd->dst_map->dm_mapsize;
2680 
2681 	if ((cmd->base_masks & (HIFN_BASE_CMD_CRYPT | HIFN_BASE_CMD_DECODE)) ==
2682 	    HIFN_BASE_CMD_CRYPT) {
2683 		for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
2684 			if (crd->crd_alg != CRYPTO_DES_CBC &&
2685 			    crd->crd_alg != CRYPTO_3DES_CBC &&
2686 			    crd->crd_alg != CRYPTO_AES_CBC)
2687 				continue;
2688 			ivlen = ((crd->crd_alg == CRYPTO_AES_CBC) ?
2689 				HIFN_AES_IV_LENGTH : HIFN_IV_LENGTH);
2690 			if (crp->crp_flags & CRYPTO_F_IMBUF)
2691 				m_copydata((struct mbuf *)crp->crp_buf,
2692 				    crd->crd_skip + crd->crd_len - ivlen,
2693 				    ivlen,
2694 				    cmd->softc->sc_sessions[cmd->session_num].hs_iv);
2695 			else if (crp->crp_flags & CRYPTO_F_IOV) {
2696 				cuio_copydata((struct uio *)crp->crp_buf,
2697 				    crd->crd_skip + crd->crd_len - ivlen,
2698 				    ivlen,
2699 				    cmd->softc->sc_sessions[cmd->session_num].hs_iv);
2700 			}
2701 			/* XXX We do not handle contig data */
2702 			break;
2703 		}
2704 	}
2705 
2706 	if (cmd->base_masks & HIFN_BASE_CMD_MAC) {
2707 		u_int8_t *macbuf;
2708 
2709 		macbuf = resbuf + sizeof(struct hifn_base_result);
2710 		if (cmd->base_masks & HIFN_BASE_CMD_COMP)
2711 			macbuf += sizeof(struct hifn_comp_result);
2712 		macbuf += sizeof(struct hifn_mac_result);
2713 
2714 		for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
2715 			int len;
2716 
2717 			if (crd->crd_alg == CRYPTO_MD5)
2718 				len = 16;
2719 			else if (crd->crd_alg == CRYPTO_SHA1)
2720 				len = 20;
2721 			else if (crd->crd_alg == CRYPTO_MD5_HMAC_96 ||
2722 			    crd->crd_alg == CRYPTO_SHA1_HMAC_96)
2723 				len = 12;
2724 			else
2725 				continue;
2726 
2727 			if (crp->crp_flags & CRYPTO_F_IMBUF)
2728 				m_copyback((struct mbuf *)crp->crp_buf,
2729 				    crd->crd_inject, len, macbuf);
2730 			else if ((crp->crp_flags & CRYPTO_F_IOV) && crp->crp_mac)
2731 				memcpy(crp->crp_mac, (void *)macbuf, len);
2732 			break;
2733 		}
2734 	}
2735 
2736 	if (cmd->src_map != cmd->dst_map) {
2737 		bus_dmamap_unload(sc->sc_dmat, cmd->dst_map);
2738 		bus_dmamap_destroy(sc->sc_dmat, cmd->dst_map);
2739 	}
2740 	bus_dmamap_unload(sc->sc_dmat, cmd->src_map);
2741 	bus_dmamap_destroy(sc->sc_dmat, cmd->src_map);
2742 	free(cmd, M_DEVBUF);
2743 	crypto_done(crp);
2744 }
2745 
2746 #ifdef HAVE_CRYPTO_LZS
2747 
2748 static int
2749 hifn_compression(struct hifn_softc *sc, struct cryptop *crp,
2750     struct hifn_command *cmd)
2751 {
2752 	struct cryptodesc *crd = crp->crp_desc;
2753 	int s, err = 0;
2754 
2755 	cmd->compcrd = crd;
2756 	cmd->base_masks |= HIFN_BASE_CMD_COMP;
2757 
2758 	if ((crp->crp_flags & CRYPTO_F_IMBUF) == 0) {
2759 		/*
2760 		 * XXX can only handle mbufs right now since we can
2761 		 * XXX dynamically resize them.
2762 		 */
2763 		err = EINVAL;
2764 		return (ENOMEM);
2765 	}
2766 
2767 	if ((crd->crd_flags & CRD_F_COMP) == 0)
2768 		cmd->base_masks |= HIFN_BASE_CMD_DECODE;
2769 	if (crd->crd_alg == CRYPTO_LZS_COMP)
2770 		cmd->comp_masks |= HIFN_COMP_CMD_ALG_LZS |
2771 		    HIFN_COMP_CMD_CLEARHIST;
2772 
2773 	if (bus_dmamap_create(sc->sc_dmat, HIFN_MAX_DMALEN, MAX_SCATTER,
2774 	    HIFN_MAX_SEGLEN, 0, BUS_DMA_NOWAIT, &cmd->src_map)) {
2775 		err = ENOMEM;
2776 		goto fail;
2777 	}
2778 
2779 	if (bus_dmamap_create(sc->sc_dmat, HIFN_MAX_DMALEN, MAX_SCATTER,
2780 	    HIFN_MAX_SEGLEN, 0, BUS_DMA_NOWAIT, &cmd->dst_map)) {
2781 		err = ENOMEM;
2782 		goto fail;
2783 	}
2784 
2785 	if (crp->crp_flags & CRYPTO_F_IMBUF) {
2786 		int len;
2787 
2788 		if (bus_dmamap_load_mbuf(sc->sc_dmat, cmd->src_map,
2789 		    cmd->srcu.src_m, BUS_DMA_NOWAIT)) {
2790 			err = ENOMEM;
2791 			goto fail;
2792 		}
2793 
2794 		len = cmd->src_map->dm_mapsize / MCLBYTES;
2795 		if ((cmd->src_map->dm_mapsize % MCLBYTES) != 0)
2796 			len++;
2797 		len *= MCLBYTES;
2798 
2799 		if ((crd->crd_flags & CRD_F_COMP) == 0)
2800 			len *= 4;
2801 
2802 		if (len > HIFN_MAX_DMALEN)
2803 			len = HIFN_MAX_DMALEN;
2804 
2805 		cmd->dstu.dst_m = hifn_mkmbuf_chain(len, cmd->srcu.src_m);
2806 		if (cmd->dstu.dst_m == NULL) {
2807 			err = ENOMEM;
2808 			goto fail;
2809 		}
2810 
2811 		if (bus_dmamap_load_mbuf(sc->sc_dmat, cmd->dst_map,
2812 		    cmd->dstu.dst_m, BUS_DMA_NOWAIT)) {
2813 			err = ENOMEM;
2814 			goto fail;
2815 		}
2816 	} else if (crp->crp_flags & CRYPTO_F_IOV) {
2817 		if (bus_dmamap_load_uio(sc->sc_dmat, cmd->src_map,
2818 		    cmd->srcu.src_io, BUS_DMA_NOWAIT)) {
2819 			err = ENOMEM;
2820 			goto fail;
2821 		}
2822 		if (bus_dmamap_load_uio(sc->sc_dmat, cmd->dst_map,
2823 		    cmd->dstu.dst_io, BUS_DMA_NOWAIT)) {
2824 			err = ENOMEM;
2825 			goto fail;
2826 		}
2827 	}
2828 
2829 	if (cmd->src_map == cmd->dst_map)
2830 		bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
2831 		    0, cmd->src_map->dm_mapsize,
2832 		    BUS_DMASYNC_PREWRITE|BUS_DMASYNC_PREREAD);
2833 	else {
2834 		bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
2835 		    0, cmd->src_map->dm_mapsize, BUS_DMASYNC_PREWRITE);
2836 		bus_dmamap_sync(sc->sc_dmat, cmd->dst_map,
2837 		    0, cmd->dst_map->dm_mapsize, BUS_DMASYNC_PREREAD);
2838 	}
2839 
2840 	cmd->crp = crp;
2841 	/*
2842 	 * Always use session 0.  The modes of compression we use are
2843 	 * stateless and there is always at least one compression
2844 	 * context, zero.
2845 	 */
2846 	cmd->session_num = 0;
2847 	cmd->softc = sc;
2848 
2849 	err = hifn_compress_enter(sc, cmd);
2850 
2851 	if (err != 0)
2852 		goto fail;
2853 	return (0);
2854 
2855 fail:
2856 	if (cmd->dst_map != NULL) {
2857 		if (cmd->dst_map->dm_nsegs > 0)
2858 			bus_dmamap_unload(sc->sc_dmat, cmd->dst_map);
2859 		bus_dmamap_destroy(sc->sc_dmat, cmd->dst_map);
2860 	}
2861 	if (cmd->src_map != NULL) {
2862 		if (cmd->src_map->dm_nsegs > 0)
2863 			bus_dmamap_unload(sc->sc_dmat, cmd->src_map);
2864 		bus_dmamap_destroy(sc->sc_dmat, cmd->src_map);
2865 	}
2866 	free(cmd, M_DEVBUF);
2867 	if (err == EINVAL)
2868 		hifnstats.hst_invalid++;
2869 	else
2870 		hifnstats.hst_nomem++;
2871 	crp->crp_etype = err;
2872 	crypto_done(crp);
2873 	return (0);
2874 }
2875 
2876 static int
2877 hifn_compress_enter(struct hifn_softc *sc, struct hifn_command *cmd)
2878 {
2879 	struct hifn_dma *dma = sc->sc_dma;
2880 	int cmdi, resi;
2881 	u_int32_t cmdlen;
2882 
2883 	if ((dma->cmdu + 1) > HIFN_D_CMD_RSIZE ||
2884 	    (dma->resu + 1) > HIFN_D_CMD_RSIZE)
2885 		return (ENOMEM);
2886 
2887 	if ((dma->srcu + cmd->src_map->dm_nsegs) > HIFN_D_SRC_RSIZE ||
2888 	    (dma->dstu + cmd->dst_map->dm_nsegs) > HIFN_D_DST_RSIZE)
2889 		return (ENOMEM);
2890 
2891 	if (dma->cmdi == HIFN_D_CMD_RSIZE) {
2892 		dma->cmdi = 0;
2893 		dma->cmdr[HIFN_D_CMD_RSIZE].l = htole32(HIFN_D_VALID |
2894 		    HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
2895 		HIFN_CMDR_SYNC(sc, HIFN_D_CMD_RSIZE,
2896 		    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
2897 	}
2898 	cmdi = dma->cmdi++;
2899 	cmdlen = hifn_write_command(cmd, dma->command_bufs[cmdi]);
2900 	HIFN_CMD_SYNC(sc, cmdi, BUS_DMASYNC_PREWRITE);
2901 
2902 	/* .p for command/result already set */
2903 	dma->cmdr[cmdi].l = htole32(cmdlen | HIFN_D_VALID | HIFN_D_LAST |
2904 	    HIFN_D_MASKDONEIRQ);
2905 	HIFN_CMDR_SYNC(sc, cmdi,
2906 	    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
2907 	dma->cmdu++;
2908 	if (sc->sc_c_busy == 0) {
2909 		WRITE_REG_1(sc, HIFN_1_DMA_CSR, HIFN_DMACSR_C_CTRL_ENA);
2910 		sc->sc_c_busy = 1;
2911 		SET_LED(sc, HIFN_MIPSRST_LED0);
2912 	}
2913 
2914 	/*
2915 	 * We don't worry about missing an interrupt (which a "command wait"
2916 	 * interrupt salvages us from), unless there is more than one command
2917 	 * in the queue.
2918 	 */
2919 	if (dma->cmdu > 1) {
2920 		sc->sc_dmaier |= HIFN_DMAIER_C_WAIT;
2921 		WRITE_REG_1(sc, HIFN_1_DMA_IER, sc->sc_dmaier);
2922 	}
2923 
2924 	hifnstats.hst_ipackets++;
2925 	hifnstats.hst_ibytes += cmd->src_map->dm_mapsize;
2926 
2927 	hifn_dmamap_load_src(sc, cmd);
2928 	if (sc->sc_s_busy == 0) {
2929 		WRITE_REG_1(sc, HIFN_1_DMA_CSR, HIFN_DMACSR_S_CTRL_ENA);
2930 		sc->sc_s_busy = 1;
2931 		SET_LED(sc, HIFN_MIPSRST_LED1);
2932 	}
2933 
2934 	/*
2935 	 * Unlike other descriptors, we don't mask done interrupt from
2936 	 * result descriptor.
2937 	 */
2938 	if (dma->resi == HIFN_D_RES_RSIZE) {
2939 		dma->resi = 0;
2940 		dma->resr[HIFN_D_RES_RSIZE].l = htole32(HIFN_D_VALID |
2941 		    HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
2942 		HIFN_RESR_SYNC(sc, HIFN_D_RES_RSIZE,
2943 		    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2944 	}
2945 	resi = dma->resi++;
2946 	dma->hifn_commands[resi] = cmd;
2947 	HIFN_RES_SYNC(sc, resi, BUS_DMASYNC_PREREAD);
2948 	dma->resr[resi].l = htole32(HIFN_MAX_RESULT |
2949 	    HIFN_D_VALID | HIFN_D_LAST);
2950 	HIFN_RESR_SYNC(sc, resi,
2951 	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2952 	dma->resu++;
2953 	if (sc->sc_r_busy == 0) {
2954 		WRITE_REG_1(sc, HIFN_1_DMA_CSR, HIFN_DMACSR_R_CTRL_ENA);
2955 		sc->sc_r_busy = 1;
2956 		SET_LED(sc, HIFN_MIPSRST_LED2);
2957 	}
2958 
2959 	if (cmd->sloplen)
2960 		cmd->slopidx = resi;
2961 
2962 	hifn_dmamap_load_dst(sc, cmd);
2963 
2964 	if (sc->sc_d_busy == 0) {
2965 		WRITE_REG_1(sc, HIFN_1_DMA_CSR, HIFN_DMACSR_D_CTRL_ENA);
2966 		sc->sc_d_busy = 1;
2967 	}
2968 	sc->sc_active = 5;
2969 	cmd->cmd_callback = hifn_callback_comp;
2970 	return (0);
2971 }
2972 
2973 static void
2974 hifn_callback_comp(struct hifn_softc *sc, struct hifn_command *cmd,
2975     u_int8_t *resbuf)
2976 {
2977 	struct hifn_base_result baseres;
2978 	struct cryptop *crp = cmd->crp;
2979 	struct hifn_dma *dma = sc->sc_dma;
2980 	struct mbuf *m;
2981 	int err = 0, i, u;
2982 	u_int32_t olen;
2983 	bus_size_t dstsize;
2984 
2985 	bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
2986 	    0, cmd->src_map->dm_mapsize, BUS_DMASYNC_POSTWRITE);
2987 	bus_dmamap_sync(sc->sc_dmat, cmd->dst_map,
2988 	    0, cmd->dst_map->dm_mapsize, BUS_DMASYNC_POSTREAD);
2989 
2990 	dstsize = cmd->dst_map->dm_mapsize;
2991 	bus_dmamap_unload(sc->sc_dmat, cmd->dst_map);
2992 
2993 	memcpy(&baseres, resbuf, sizeof(struct hifn_base_result));
2994 
2995 	i = dma->dstk; u = dma->dstu;
2996 	while (u != 0) {
2997 		bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
2998 		    offsetof(struct hifn_dma, dstr[i]), sizeof(struct hifn_desc),
2999 		    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
3000 		if (dma->dstr[i].l & htole32(HIFN_D_VALID)) {
3001 			bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
3002 			    offsetof(struct hifn_dma, dstr[i]),
3003 			    sizeof(struct hifn_desc),
3004 			    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3005 			break;
3006 		}
3007 		if (++i == (HIFN_D_DST_RSIZE + 1))
3008 			i = 0;
3009 		else
3010 			u--;
3011 	}
3012 	dma->dstk = i; dma->dstu = u;
3013 
3014 	if (baseres.flags & htole16(HIFN_BASE_RES_DSTOVERRUN)) {
3015 		bus_size_t xlen;
3016 
3017 		xlen = dstsize;
3018 
3019 		m_freem(cmd->dstu.dst_m);
3020 
3021 		if (xlen == HIFN_MAX_DMALEN) {
3022 			/* We've done all we can. */
3023 			err = E2BIG;
3024 			goto out;
3025 		}
3026 
3027 		xlen += MCLBYTES;
3028 
3029 		if (xlen > HIFN_MAX_DMALEN)
3030 			xlen = HIFN_MAX_DMALEN;
3031 
3032 		cmd->dstu.dst_m = hifn_mkmbuf_chain(xlen,
3033 		    cmd->srcu.src_m);
3034 		if (cmd->dstu.dst_m == NULL) {
3035 			err = ENOMEM;
3036 			goto out;
3037 		}
3038 		if (bus_dmamap_load_mbuf(sc->sc_dmat, cmd->dst_map,
3039 		    cmd->dstu.dst_m, BUS_DMA_NOWAIT)) {
3040 			err = ENOMEM;
3041 			goto out;
3042 		}
3043 
3044 		bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
3045 		    0, cmd->src_map->dm_mapsize, BUS_DMASYNC_PREWRITE);
3046 		bus_dmamap_sync(sc->sc_dmat, cmd->dst_map,
3047 		    0, cmd->dst_map->dm_mapsize, BUS_DMASYNC_PREREAD);
3048 
3049 		err = hifn_compress_enter(sc, cmd);
3050 		if (err != 0)
3051 			goto out;
3052 		return;
3053 	}
3054 
3055 	olen = dstsize - (letoh16(baseres.dst_cnt) |
3056 	    (((letoh16(baseres.session) & HIFN_BASE_RES_DSTLEN_M) >>
3057 	    HIFN_BASE_RES_DSTLEN_S) << 16));
3058 
3059 	crp->crp_olen = olen - cmd->compcrd->crd_skip;
3060 
3061 	bus_dmamap_unload(sc->sc_dmat, cmd->src_map);
3062 	bus_dmamap_destroy(sc->sc_dmat, cmd->src_map);
3063 	bus_dmamap_destroy(sc->sc_dmat, cmd->dst_map);
3064 
3065 	m = cmd->dstu.dst_m;
3066 	if (m->m_flags & M_PKTHDR)
3067 		m->m_pkthdr.len = olen;
3068 	crp->crp_buf = (void *)m;
3069 	for (; m != NULL; m = m->m_next) {
3070 		if (olen >= m->m_len)
3071 			olen -= m->m_len;
3072 		else {
3073 			m->m_len = olen;
3074 			olen = 0;
3075 		}
3076 	}
3077 
3078 	m_freem(cmd->srcu.src_m);
3079 	free(cmd, M_DEVBUF);
3080 	crp->crp_etype = 0;
3081 	crypto_done(crp);
3082 	return;
3083 
3084 out:
3085 	if (cmd->dst_map != NULL) {
3086 		if (cmd->src_map->dm_nsegs != 0)
3087 			bus_dmamap_unload(sc->sc_dmat, cmd->src_map);
3088 		bus_dmamap_destroy(sc->sc_dmat, cmd->dst_map);
3089 	}
3090 	if (cmd->src_map != NULL) {
3091 		if (cmd->src_map->dm_nsegs != 0)
3092 			bus_dmamap_unload(sc->sc_dmat, cmd->src_map);
3093 		bus_dmamap_destroy(sc->sc_dmat, cmd->src_map);
3094 	}
3095 	if (cmd->dstu.dst_m != NULL)
3096 		m_freem(cmd->dstu.dst_m);
3097 	free(cmd, M_DEVBUF);
3098 	crp->crp_etype = err;
3099 	crypto_done(crp);
3100 }
3101 
3102 static struct mbuf *
3103 hifn_mkmbuf_chain(int totlen, struct mbuf *mtemplate)
3104 {
3105 	int len;
3106 	struct mbuf *m, *m0, *mlast;
3107 
3108 	if (mtemplate->m_flags & M_PKTHDR) {
3109 		len = MHLEN;
3110 		MGETHDR(m0, M_DONTWAIT, MT_DATA);
3111 	} else {
3112 		len = MLEN;
3113 		MGET(m0, M_DONTWAIT, MT_DATA);
3114 	}
3115 	if (m0 == NULL)
3116 		return (NULL);
3117 	if (len == MHLEN)
3118 		M_DUP_PKTHDR(m0, mtemplate);
3119 	MCLGET(m0, M_DONTWAIT);
3120 	if (!(m0->m_flags & M_EXT))
3121 		m_freem(m0);
3122 	len = MCLBYTES;
3123 
3124 	totlen -= len;
3125 	m0->m_pkthdr.len = m0->m_len = len;
3126 	mlast = m0;
3127 
3128 	while (totlen > 0) {
3129 		MGET(m, M_DONTWAIT, MT_DATA);
3130 		if (m == NULL) {
3131 			m_freem(m0);
3132 			return (NULL);
3133 		}
3134 		MCLGET(m, M_DONTWAIT);
3135 		if (!(m->m_flags & M_EXT)) {
3136 			m_freem(m0);
3137 			return (NULL);
3138 		}
3139 		len = MCLBYTES;
3140 		m->m_len = len;
3141 		if (m0->m_flags & M_PKTHDR)
3142 			m0->m_pkthdr.len += len;
3143 		totlen -= len;
3144 
3145 		mlast->m_next = m;
3146 		mlast = m;
3147 	}
3148 
3149 	return (m0);
3150 }
3151 #endif	/* HAVE_CRYPTO_LZS */
3152 
3153 static void
3154 hifn_write_4(struct hifn_softc *sc, int reggrp, bus_size_t reg, u_int32_t val)
3155 {
3156 	/*
3157 	 * 7811 PB3 rev/2 parts lock-up on burst writes to Group 0
3158 	 * and Group 1 registers; avoid conditions that could create
3159 	 * burst writes by doing a read in between the writes.
3160 	 */
3161 	if (sc->sc_flags & HIFN_NO_BURSTWRITE) {
3162 		if (sc->sc_waw_lastgroup == reggrp &&
3163 		    sc->sc_waw_lastreg == reg - 4) {
3164 			bus_space_read_4(sc->sc_st1, sc->sc_sh1, HIFN_1_REVID);
3165 		}
3166 		sc->sc_waw_lastgroup = reggrp;
3167 		sc->sc_waw_lastreg = reg;
3168 	}
3169 	if (reggrp == 0)
3170 		bus_space_write_4(sc->sc_st0, sc->sc_sh0, reg, val);
3171 	else
3172 		bus_space_write_4(sc->sc_st1, sc->sc_sh1, reg, val);
3173 
3174 }
3175 
3176 static u_int32_t
3177 hifn_read_4(struct hifn_softc *sc, int reggrp, bus_size_t reg)
3178 {
3179 	if (sc->sc_flags & HIFN_NO_BURSTWRITE) {
3180 		sc->sc_waw_lastgroup = -1;
3181 		sc->sc_waw_lastreg = 1;
3182 	}
3183 	if (reggrp == 0)
3184 		return (bus_space_read_4(sc->sc_st0, sc->sc_sh0, reg));
3185 	return (bus_space_read_4(sc->sc_st1, sc->sc_sh1, reg));
3186 }
3187