xref: /netbsd-src/sys/dev/marvell/mvxpsec.c (revision 100a3398b8d3c64e571cff36b46c23431b410e09)
1*100a3398Sandvar /*	$NetBSD: mvxpsec.c,v 1.20 2024/02/09 22:08:35 andvar Exp $	*/
21a8031e1Shsuenaga /*
31a8031e1Shsuenaga  * Copyright (c) 2015 Internet Initiative Japan Inc.
41a8031e1Shsuenaga  * All rights reserved.
51a8031e1Shsuenaga  *
61a8031e1Shsuenaga  * Redistribution and use in source and binary forms, with or without
71a8031e1Shsuenaga  * modification, are permitted provided that the following conditions
81a8031e1Shsuenaga  * are met:
91a8031e1Shsuenaga  * 1. Redistributions of source code must retain the above copyright
101a8031e1Shsuenaga  *    notice, this list of conditions and the following disclaimer.
111a8031e1Shsuenaga  * 2. Redistributions in binary form must reproduce the above copyright
121a8031e1Shsuenaga  *    notice, this list of conditions and the following disclaimer in the
131a8031e1Shsuenaga  *    documentation and/or other materials provided with the distribution.
141a8031e1Shsuenaga  *
151a8031e1Shsuenaga  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
161a8031e1Shsuenaga  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
171a8031e1Shsuenaga  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
181a8031e1Shsuenaga  * DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
191a8031e1Shsuenaga  * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
201a8031e1Shsuenaga  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
211a8031e1Shsuenaga  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
221a8031e1Shsuenaga  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
231a8031e1Shsuenaga  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
241a8031e1Shsuenaga  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
251a8031e1Shsuenaga  * POSSIBILITY OF SUCH DAMAGE.
261a8031e1Shsuenaga  */
272499f515Sriastradh 
282499f515Sriastradh #ifdef _KERNEL_OPT
292499f515Sriastradh #include "opt_ipsec.h"
302499f515Sriastradh #endif
312499f515Sriastradh 
321a8031e1Shsuenaga /*
331a8031e1Shsuenaga  * Cryptographic Engine and Security Accelerator(MVXPSEC)
341a8031e1Shsuenaga  */
351a8031e1Shsuenaga #include <sys/cdefs.h>
361a8031e1Shsuenaga #include <sys/param.h>
371a8031e1Shsuenaga #include <sys/types.h>
381a8031e1Shsuenaga #include <sys/kernel.h>
391a8031e1Shsuenaga #include <sys/queue.h>
401a8031e1Shsuenaga #include <sys/conf.h>
411a8031e1Shsuenaga #include <sys/proc.h>
421a8031e1Shsuenaga #include <sys/bus.h>
431a8031e1Shsuenaga #include <sys/evcnt.h>
441a8031e1Shsuenaga #include <sys/device.h>
451a8031e1Shsuenaga #include <sys/endian.h>
461a8031e1Shsuenaga #include <sys/errno.h>
471a8031e1Shsuenaga #include <sys/kmem.h>
481a8031e1Shsuenaga #include <sys/mbuf.h>
491a8031e1Shsuenaga #include <sys/callout.h>
501a8031e1Shsuenaga #include <sys/pool.h>
511a8031e1Shsuenaga #include <sys/cprng.h>
521a8031e1Shsuenaga #include <sys/syslog.h>
531a8031e1Shsuenaga #include <sys/mutex.h>
541a8031e1Shsuenaga #include <sys/kthread.h>
551a8031e1Shsuenaga #include <sys/atomic.h>
561a8031e1Shsuenaga #include <sys/sha1.h>
571a8031e1Shsuenaga #include <sys/md5.h>
581a8031e1Shsuenaga 
591a8031e1Shsuenaga #include <uvm/uvm_extern.h>
601a8031e1Shsuenaga 
611a8031e1Shsuenaga #include <opencrypto/cryptodev.h>
621a8031e1Shsuenaga #include <opencrypto/xform.h>
631a8031e1Shsuenaga 
641a8031e1Shsuenaga #include <net/net_stats.h>
651a8031e1Shsuenaga 
661a8031e1Shsuenaga #include <netinet/in_systm.h>
671a8031e1Shsuenaga #include <netinet/in.h>
681a8031e1Shsuenaga #include <netinet/ip.h>
691a8031e1Shsuenaga #include <netinet/ip6.h>
701a8031e1Shsuenaga 
712499f515Sriastradh #if NIPSEC > 0
721a8031e1Shsuenaga #include <netipsec/esp_var.h>
732499f515Sriastradh #endif
741a8031e1Shsuenaga 
751a8031e1Shsuenaga #include <arm/cpufunc.h>
761a8031e1Shsuenaga #include <arm/marvell/mvsocvar.h>
771a8031e1Shsuenaga #include <arm/marvell/armadaxpreg.h>
781a8031e1Shsuenaga #include <dev/marvell/marvellreg.h>
791a8031e1Shsuenaga #include <dev/marvell/marvellvar.h>
801a8031e1Shsuenaga #include <dev/marvell/mvxpsecreg.h>
811a8031e1Shsuenaga #include <dev/marvell/mvxpsecvar.h>
821a8031e1Shsuenaga 
831a8031e1Shsuenaga #ifdef DEBUG
841a8031e1Shsuenaga #define STATIC __attribute__ ((noinline)) extern
851a8031e1Shsuenaga #define _STATIC __attribute__ ((noinline)) extern
861a8031e1Shsuenaga #define INLINE __attribute__ ((noinline)) extern
871a8031e1Shsuenaga #define _INLINE __attribute__ ((noinline)) extern
881a8031e1Shsuenaga #else
891a8031e1Shsuenaga #define STATIC static
901a8031e1Shsuenaga #define _STATIC __attribute__ ((unused)) static
911a8031e1Shsuenaga #define INLINE static inline
921a8031e1Shsuenaga #define _INLINE __attribute__ ((unused)) static inline
931a8031e1Shsuenaga #endif
941a8031e1Shsuenaga 
951a8031e1Shsuenaga /*
961a8031e1Shsuenaga  * IRQ and SRAM spaces for each of unit
971a8031e1Shsuenaga  * XXX: move to attach_args
981a8031e1Shsuenaga  */
991a8031e1Shsuenaga struct {
1001a8031e1Shsuenaga 	int		err_int;
1011a8031e1Shsuenaga } mvxpsec_config[] = {
1021a8031e1Shsuenaga 	{ .err_int = ARMADAXP_IRQ_CESA0_ERR, }, /* unit 0 */
1031a8031e1Shsuenaga 	{ .err_int = ARMADAXP_IRQ_CESA1_ERR, }, /* unit 1 */
1041a8031e1Shsuenaga };
1051a8031e1Shsuenaga #define MVXPSEC_ERR_INT(sc) \
1061a8031e1Shsuenaga     mvxpsec_config[device_unit((sc)->sc_dev)].err_int
1071a8031e1Shsuenaga 
1081a8031e1Shsuenaga /*
1091a8031e1Shsuenaga  * AES
1101a8031e1Shsuenaga  */
1111a8031e1Shsuenaga #define MAXBC				(128/32)
1121a8031e1Shsuenaga #define MAXKC				(256/32)
1131a8031e1Shsuenaga #define MAXROUNDS			14
1141a8031e1Shsuenaga STATIC int mv_aes_ksched(uint8_t[4][MAXKC], int,
1151a8031e1Shsuenaga     uint8_t[MAXROUNDS+1][4][MAXBC]);
1161a8031e1Shsuenaga STATIC int mv_aes_deckey(uint8_t *, uint8_t *, int);
1171a8031e1Shsuenaga 
1181a8031e1Shsuenaga /*
1191a8031e1Shsuenaga  * device driver autoconf interface
1201a8031e1Shsuenaga  */
1211a8031e1Shsuenaga STATIC int mvxpsec_match(device_t, cfdata_t, void *);
1221a8031e1Shsuenaga STATIC void mvxpsec_attach(device_t, device_t, void *);
1231a8031e1Shsuenaga STATIC void mvxpsec_evcnt_attach(struct mvxpsec_softc *);
1241a8031e1Shsuenaga 
1251a8031e1Shsuenaga /*
1261a8031e1Shsuenaga  * register setup
1271a8031e1Shsuenaga  */
1281a8031e1Shsuenaga STATIC int mvxpsec_wininit(struct mvxpsec_softc *, enum marvell_tags *);
1291a8031e1Shsuenaga 
1301a8031e1Shsuenaga /*
1311a8031e1Shsuenaga  * timer(callout) interface
1321a8031e1Shsuenaga  *
1331a8031e1Shsuenaga  * XXX: callout is not MP safe...
1341a8031e1Shsuenaga  */
1351a8031e1Shsuenaga STATIC void mvxpsec_timer(void *);
1361a8031e1Shsuenaga 
1371a8031e1Shsuenaga /*
1381a8031e1Shsuenaga  * interrupt interface
1391a8031e1Shsuenaga  */
1401a8031e1Shsuenaga STATIC int mvxpsec_intr(void *);
1411a8031e1Shsuenaga INLINE void mvxpsec_intr_cleanup(struct mvxpsec_softc *);
1421a8031e1Shsuenaga STATIC int mvxpsec_eintr(void *);
1431a8031e1Shsuenaga STATIC uint32_t mvxpsec_intr_ack(struct mvxpsec_softc *);
1441a8031e1Shsuenaga STATIC uint32_t mvxpsec_eintr_ack(struct mvxpsec_softc *);
1451a8031e1Shsuenaga INLINE void mvxpsec_intr_cnt(struct mvxpsec_softc *, int);
1461a8031e1Shsuenaga 
1471a8031e1Shsuenaga /*
1481a8031e1Shsuenaga  * memory allocators and VM management
1491a8031e1Shsuenaga  */
1501a8031e1Shsuenaga STATIC struct mvxpsec_devmem *mvxpsec_alloc_devmem(struct mvxpsec_softc *,
1511a8031e1Shsuenaga     paddr_t, int);
1521a8031e1Shsuenaga STATIC int mvxpsec_init_sram(struct mvxpsec_softc *);
1531a8031e1Shsuenaga 
1541a8031e1Shsuenaga /*
1551a8031e1Shsuenaga  * Low-level DMA interface
1561a8031e1Shsuenaga  */
1571a8031e1Shsuenaga STATIC int mvxpsec_init_dma(struct mvxpsec_softc *,
1581a8031e1Shsuenaga     struct marvell_attach_args *);
1591a8031e1Shsuenaga INLINE int mvxpsec_dma_wait(struct mvxpsec_softc *);
1601a8031e1Shsuenaga INLINE int mvxpsec_acc_wait(struct mvxpsec_softc *);
1611a8031e1Shsuenaga INLINE struct mvxpsec_descriptor_handle *mvxpsec_dma_getdesc(struct mvxpsec_softc *);
1621a8031e1Shsuenaga _INLINE void mvxpsec_dma_putdesc(struct mvxpsec_softc *, struct mvxpsec_descriptor_handle *);
1631a8031e1Shsuenaga INLINE void mvxpsec_dma_setup(struct mvxpsec_descriptor_handle *,
1641a8031e1Shsuenaga     uint32_t, uint32_t, uint32_t);
1651a8031e1Shsuenaga INLINE void mvxpsec_dma_cat(struct mvxpsec_softc *,
1661a8031e1Shsuenaga     struct mvxpsec_descriptor_handle *, struct mvxpsec_descriptor_handle *);
1671a8031e1Shsuenaga 
1681a8031e1Shsuenaga /*
1691a8031e1Shsuenaga  * High-level DMA interface
1701a8031e1Shsuenaga  */
1711a8031e1Shsuenaga INLINE int mvxpsec_dma_copy0(struct mvxpsec_softc *,
1721a8031e1Shsuenaga     mvxpsec_dma_ring *, uint32_t, uint32_t, uint32_t);
1731a8031e1Shsuenaga INLINE int mvxpsec_dma_copy(struct mvxpsec_softc *,
1741a8031e1Shsuenaga     mvxpsec_dma_ring *, uint32_t, uint32_t, uint32_t);
1751a8031e1Shsuenaga INLINE int mvxpsec_dma_acc_activate(struct mvxpsec_softc *,
1761a8031e1Shsuenaga     mvxpsec_dma_ring *);
1771a8031e1Shsuenaga INLINE void mvxpsec_dma_finalize(struct mvxpsec_softc *,
1781a8031e1Shsuenaga     mvxpsec_dma_ring *);
1791a8031e1Shsuenaga INLINE void mvxpsec_dma_free(struct mvxpsec_softc *,
1801a8031e1Shsuenaga     mvxpsec_dma_ring *);
1811a8031e1Shsuenaga INLINE int mvxpsec_dma_copy_packet(struct mvxpsec_softc *, struct mvxpsec_packet *);
1821a8031e1Shsuenaga INLINE int mvxpsec_dma_sync_packet(struct mvxpsec_softc *, struct mvxpsec_packet *);
1831a8031e1Shsuenaga 
1841a8031e1Shsuenaga /*
1851a8031e1Shsuenaga  * Session management interface (OpenCrypto)
1861a8031e1Shsuenaga  */
1871a8031e1Shsuenaga #define MVXPSEC_SESSION(sid)	((sid) & 0x0fffffff)
1881a8031e1Shsuenaga #define MVXPSEC_SID(crd, sesn)	(((crd) << 28) | ((sesn) & 0x0fffffff))
1891a8031e1Shsuenaga /* pool management */
1901a8031e1Shsuenaga STATIC int mvxpsec_session_ctor(void *, void *, int);
1911a8031e1Shsuenaga STATIC void mvxpsec_session_dtor(void *, void *);
1921a8031e1Shsuenaga STATIC int mvxpsec_packet_ctor(void *, void *, int);
1931a8031e1Shsuenaga STATIC void mvxpsec_packet_dtor(void *, void *);
1941a8031e1Shsuenaga 
1951a8031e1Shsuenaga /* session management */
1961a8031e1Shsuenaga STATIC struct mvxpsec_session *mvxpsec_session_alloc(struct mvxpsec_softc *);
1971a8031e1Shsuenaga STATIC void mvxpsec_session_dealloc(struct mvxpsec_session *);
1981a8031e1Shsuenaga INLINE struct mvxpsec_session *mvxpsec_session_lookup(struct mvxpsec_softc *, int);
1991a8031e1Shsuenaga INLINE int mvxpsec_session_ref(struct mvxpsec_session *);
2001a8031e1Shsuenaga INLINE void mvxpsec_session_unref(struct mvxpsec_session *);
2011a8031e1Shsuenaga 
2021a8031e1Shsuenaga /* packet management */
2031a8031e1Shsuenaga STATIC struct mvxpsec_packet *mvxpsec_packet_alloc(struct mvxpsec_session *);
2041a8031e1Shsuenaga INLINE void mvxpsec_packet_enqueue(struct mvxpsec_packet *);
2051a8031e1Shsuenaga STATIC void mvxpsec_packet_dealloc(struct mvxpsec_packet *);
2061a8031e1Shsuenaga STATIC int mvxpsec_done_packet(struct mvxpsec_packet *);
2071a8031e1Shsuenaga 
2081a8031e1Shsuenaga /* session header manegement */
2091a8031e1Shsuenaga STATIC int mvxpsec_header_finalize(struct mvxpsec_packet *);
2101a8031e1Shsuenaga 
2111a8031e1Shsuenaga /* packet queue management */
2121a8031e1Shsuenaga INLINE void mvxpsec_drop(struct mvxpsec_softc *, struct cryptop *, struct mvxpsec_packet *, int);
2131a8031e1Shsuenaga STATIC int mvxpsec_dispatch_queue(struct mvxpsec_softc *);
2141a8031e1Shsuenaga 
21588af2332Smsaitoh /* opencrypto operation */
2161a8031e1Shsuenaga INLINE int mvxpsec_parse_crd(struct mvxpsec_packet *, struct cryptodesc *);
2171a8031e1Shsuenaga INLINE int mvxpsec_parse_crp(struct mvxpsec_packet *);
2181a8031e1Shsuenaga 
2191a8031e1Shsuenaga /* payload data management */
2201a8031e1Shsuenaga INLINE int mvxpsec_packet_setcrp(struct mvxpsec_packet *, struct cryptop *);
2211a8031e1Shsuenaga STATIC int mvxpsec_packet_setdata(struct mvxpsec_packet *, void *, uint32_t);
2221a8031e1Shsuenaga STATIC int mvxpsec_packet_setmbuf(struct mvxpsec_packet *, struct mbuf *);
2231a8031e1Shsuenaga STATIC int mvxpsec_packet_setuio(struct mvxpsec_packet *, struct uio *);
2241a8031e1Shsuenaga STATIC int mvxpsec_packet_rdata(struct mvxpsec_packet *, int, int, void *);
2251a8031e1Shsuenaga _STATIC int mvxpsec_packet_wdata(struct mvxpsec_packet *, int, int, void *);
2261a8031e1Shsuenaga STATIC int mvxpsec_packet_write_iv(struct mvxpsec_packet *, void *, int);
2271a8031e1Shsuenaga STATIC int mvxpsec_packet_copy_iv(struct mvxpsec_packet *, int, int);
2281a8031e1Shsuenaga 
2291a8031e1Shsuenaga /* key pre-computation */
2301a8031e1Shsuenaga STATIC int mvxpsec_key_precomp(int, void *, int, void *, void *);
2311a8031e1Shsuenaga STATIC int mvxpsec_hmac_precomp(int, void *, int, void *, void *);
2321a8031e1Shsuenaga 
2331a8031e1Shsuenaga /* crypto operation management */
2341a8031e1Shsuenaga INLINE void mvxpsec_packet_reset_op(struct mvxpsec_packet *);
2351a8031e1Shsuenaga INLINE void mvxpsec_packet_update_op_order(struct mvxpsec_packet *, int);
2361a8031e1Shsuenaga 
2371a8031e1Shsuenaga /*
2381a8031e1Shsuenaga  * parameter converters
2391a8031e1Shsuenaga  */
2401a8031e1Shsuenaga INLINE uint32_t mvxpsec_alg2acc(uint32_t alg);
2411a8031e1Shsuenaga INLINE uint32_t mvxpsec_aesklen(int klen);
2421a8031e1Shsuenaga 
2431a8031e1Shsuenaga /*
2441a8031e1Shsuenaga  * string formatters
2451a8031e1Shsuenaga  */
2461a8031e1Shsuenaga _STATIC const char *s_ctrlreg(uint32_t);
2471a8031e1Shsuenaga _STATIC const char *s_winreg(uint32_t);
2481a8031e1Shsuenaga _STATIC const char *s_errreg(uint32_t);
2491a8031e1Shsuenaga _STATIC const char *s_xpsecintr(uint32_t);
2501a8031e1Shsuenaga _STATIC const char *s_ctlalg(uint32_t);
2511a8031e1Shsuenaga _STATIC const char *s_xpsec_op(uint32_t);
2521a8031e1Shsuenaga _STATIC const char *s_xpsec_enc(uint32_t);
2531a8031e1Shsuenaga _STATIC const char *s_xpsec_mac(uint32_t);
2541a8031e1Shsuenaga _STATIC const char *s_xpsec_frag(uint32_t);
2551a8031e1Shsuenaga 
2561a8031e1Shsuenaga /*
2571a8031e1Shsuenaga  * debugging supports
2581a8031e1Shsuenaga  */
2591a8031e1Shsuenaga #ifdef MVXPSEC_DEBUG
2601a8031e1Shsuenaga _STATIC void mvxpsec_dump_dmaq(struct mvxpsec_descriptor_handle *);
2611a8031e1Shsuenaga _STATIC void mvxpsec_dump_reg(struct mvxpsec_softc *);
2621a8031e1Shsuenaga _STATIC void mvxpsec_dump_sram(const char *, struct mvxpsec_softc *, size_t);
2631a8031e1Shsuenaga _STATIC void mvxpsec_dump_data(const char *, void *, size_t);
2641a8031e1Shsuenaga 
2651a8031e1Shsuenaga _STATIC void mvxpsec_dump_packet(const char *, struct mvxpsec_packet *);
2661a8031e1Shsuenaga _STATIC void mvxpsec_dump_packet_data(const char *, struct mvxpsec_packet *);
2671a8031e1Shsuenaga _STATIC void mvxpsec_dump_packet_desc(const char *, struct mvxpsec_packet *);
2681a8031e1Shsuenaga 
2691a8031e1Shsuenaga _STATIC void mvxpsec_dump_acc_config(const char *, uint32_t);
2701a8031e1Shsuenaga _STATIC void mvxpsec_dump_acc_encdata(const char *, uint32_t, uint32_t);
2711a8031e1Shsuenaga _STATIC void mvxpsec_dump_acc_enclen(const char *, uint32_t);
2721a8031e1Shsuenaga _STATIC void mvxpsec_dump_acc_enckey(const char *, uint32_t);
2731a8031e1Shsuenaga _STATIC void mvxpsec_dump_acc_enciv(const char *, uint32_t);
2741a8031e1Shsuenaga _STATIC void mvxpsec_dump_acc_macsrc(const char *, uint32_t);
2751a8031e1Shsuenaga _STATIC void mvxpsec_dump_acc_macdst(const char *, uint32_t);
2761a8031e1Shsuenaga _STATIC void mvxpsec_dump_acc_maciv(const char *, uint32_t);
2771a8031e1Shsuenaga #endif
2781a8031e1Shsuenaga 
2791a8031e1Shsuenaga /*
2801a8031e1Shsuenaga  * global configurations, params, work spaces, ...
2811a8031e1Shsuenaga  *
2821a8031e1Shsuenaga  * XXX: use sysctl for global configurations
2831a8031e1Shsuenaga  */
2841a8031e1Shsuenaga /* waiting for device */
2851a8031e1Shsuenaga static int mvxpsec_wait_interval = 10;		/* usec */
2861a8031e1Shsuenaga static int mvxpsec_wait_retry = 100;		/* times = wait for 1 [msec] */
2871a8031e1Shsuenaga #ifdef MVXPSEC_DEBUG
2881a8031e1Shsuenaga static uint32_t mvxpsec_debug = MVXPSEC_DEBUG;	/* debug level */
2891a8031e1Shsuenaga #endif
2901a8031e1Shsuenaga 
2911a8031e1Shsuenaga /*
2921a8031e1Shsuenaga  * Register accessors
2931a8031e1Shsuenaga  */
2941a8031e1Shsuenaga #define MVXPSEC_WRITE(sc, off, val) \
2951a8031e1Shsuenaga 	bus_space_write_4((sc)->sc_iot, (sc)->sc_ioh, (off), (val))
2961a8031e1Shsuenaga #define MVXPSEC_READ(sc, off) \
2971a8031e1Shsuenaga 	bus_space_read_4((sc)->sc_iot, (sc)->sc_ioh, (off))
2981a8031e1Shsuenaga 
2991a8031e1Shsuenaga /*
3001a8031e1Shsuenaga  * device driver autoconf interface
3011a8031e1Shsuenaga  */
3021a8031e1Shsuenaga CFATTACH_DECL2_NEW(mvxpsec_mbus, sizeof(struct mvxpsec_softc),
3031a8031e1Shsuenaga     mvxpsec_match, mvxpsec_attach, NULL, NULL, NULL, NULL);
3041a8031e1Shsuenaga 
3051a8031e1Shsuenaga STATIC int
mvxpsec_match(device_t dev,cfdata_t match,void * aux)3061a8031e1Shsuenaga mvxpsec_match(device_t dev, cfdata_t match, void *aux)
3071a8031e1Shsuenaga {
3081a8031e1Shsuenaga 	struct marvell_attach_args *mva = aux;
3091a8031e1Shsuenaga 	uint32_t tag;
3101a8031e1Shsuenaga 	int window;
3111a8031e1Shsuenaga 
3121a8031e1Shsuenaga 	if (strcmp(mva->mva_name, match->cf_name) != 0)
3131a8031e1Shsuenaga 		return 0;
3141a8031e1Shsuenaga 	if (mva->mva_offset == MVA_OFFSET_DEFAULT)
3151a8031e1Shsuenaga 		return 0;
3161a8031e1Shsuenaga 
3171a8031e1Shsuenaga 	switch (mva->mva_unit) {
3181a8031e1Shsuenaga 	case 0:
3191a8031e1Shsuenaga 		tag = ARMADAXP_TAG_CRYPT0;
3201a8031e1Shsuenaga 		break;
3211a8031e1Shsuenaga 	case 1:
3221a8031e1Shsuenaga 		tag = ARMADAXP_TAG_CRYPT1;
3231a8031e1Shsuenaga 		break;
3241a8031e1Shsuenaga 	default:
3251a8031e1Shsuenaga 		aprint_error_dev(dev,
3261a8031e1Shsuenaga 		    "unit %d is not supported\n", mva->mva_unit);
3271a8031e1Shsuenaga 		return 0;
3281a8031e1Shsuenaga 	}
3291a8031e1Shsuenaga 
3301a8031e1Shsuenaga 	window = mvsoc_target(tag, NULL, NULL, NULL, NULL);
3311a8031e1Shsuenaga 	if (window >= nwindow) {
3321a8031e1Shsuenaga 		aprint_error_dev(dev,
3331a8031e1Shsuenaga 		    "Security Accelerator SRAM is not configured.\n");
3341a8031e1Shsuenaga 		return 0;
3351a8031e1Shsuenaga 	}
3361a8031e1Shsuenaga 
3371a8031e1Shsuenaga 	return 1;
3381a8031e1Shsuenaga }
3391a8031e1Shsuenaga 
3401a8031e1Shsuenaga STATIC void
mvxpsec_attach(device_t parent,device_t self,void * aux)3411a8031e1Shsuenaga mvxpsec_attach(device_t parent, device_t self, void *aux)
3421a8031e1Shsuenaga {
3431a8031e1Shsuenaga 	struct marvell_attach_args *mva = aux;
3441a8031e1Shsuenaga 	struct mvxpsec_softc *sc = device_private(self);
3451a8031e1Shsuenaga 	int v;
3461a8031e1Shsuenaga 	int i;
3471a8031e1Shsuenaga 
3481a8031e1Shsuenaga 	sc->sc_dev = self;
3491a8031e1Shsuenaga 
3501a8031e1Shsuenaga 	aprint_normal(": Marvell Crypto Engines and Security Accelerator\n");
3511a8031e1Shsuenaga 	aprint_naive("\n");
3521a8031e1Shsuenaga #ifdef MVXPSEC_MULTI_PACKET
3531a8031e1Shsuenaga 	aprint_normal_dev(sc->sc_dev, "multi-packet chained mode enabled.\n");
3541a8031e1Shsuenaga #else
3551a8031e1Shsuenaga 	aprint_normal_dev(sc->sc_dev, "multi-packet chained mode disabled.\n");
3561a8031e1Shsuenaga #endif
3571a8031e1Shsuenaga 	aprint_normal_dev(sc->sc_dev,
3581a8031e1Shsuenaga 	    "Max %d sessions.\n", MVXPSEC_MAX_SESSIONS);
3591a8031e1Shsuenaga 
3601a8031e1Shsuenaga 	/* mutex */
3611a8031e1Shsuenaga 	mutex_init(&sc->sc_session_mtx, MUTEX_DEFAULT, IPL_NET);
3621a8031e1Shsuenaga 	mutex_init(&sc->sc_dma_mtx, MUTEX_DEFAULT, IPL_NET);
3631a8031e1Shsuenaga 	mutex_init(&sc->sc_queue_mtx, MUTEX_DEFAULT, IPL_NET);
3641a8031e1Shsuenaga 
3651a8031e1Shsuenaga 	/* Packet queue */
3661a8031e1Shsuenaga 	SIMPLEQ_INIT(&sc->sc_wait_queue);
3671a8031e1Shsuenaga 	SIMPLEQ_INIT(&sc->sc_run_queue);
3681a8031e1Shsuenaga 	SLIST_INIT(&sc->sc_free_list);
3691a8031e1Shsuenaga 	sc->sc_wait_qlen = 0;
3701a8031e1Shsuenaga #ifdef MVXPSEC_MULTI_PACKET
3711a8031e1Shsuenaga 	sc->sc_wait_qlimit = 16;
3721a8031e1Shsuenaga #else
3731a8031e1Shsuenaga 	sc->sc_wait_qlimit = 0;
3741a8031e1Shsuenaga #endif
3751a8031e1Shsuenaga 	sc->sc_free_qlen = 0;
3761a8031e1Shsuenaga 
3771a8031e1Shsuenaga 	/* Timer */
3781a8031e1Shsuenaga 	callout_init(&sc->sc_timeout, 0); /* XXX: use CALLOUT_MPSAFE */
3791a8031e1Shsuenaga 	callout_setfunc(&sc->sc_timeout, mvxpsec_timer, sc);
3801a8031e1Shsuenaga 
3811a8031e1Shsuenaga 	/* I/O */
3821a8031e1Shsuenaga 	sc->sc_iot = mva->mva_iot;
3831a8031e1Shsuenaga 	if (bus_space_subregion(mva->mva_iot, mva->mva_ioh,
3841a8031e1Shsuenaga 	    mva->mva_offset, mva->mva_size, &sc->sc_ioh)) {
3851a8031e1Shsuenaga 		aprint_error_dev(self, "Cannot map registers\n");
3861a8031e1Shsuenaga 		return;
3871a8031e1Shsuenaga 	}
3881a8031e1Shsuenaga 
3891a8031e1Shsuenaga 	/* DMA */
3901a8031e1Shsuenaga 	sc->sc_dmat = mva->mva_dmat;
3911a8031e1Shsuenaga 	if (mvxpsec_init_dma(sc, mva) < 0)
3921a8031e1Shsuenaga 		return;
3931a8031e1Shsuenaga 
3941a8031e1Shsuenaga 	/* SRAM */
3951a8031e1Shsuenaga 	if (mvxpsec_init_sram(sc) < 0)
3961a8031e1Shsuenaga 		return;
3971a8031e1Shsuenaga 
3981a8031e1Shsuenaga 	/* Registers */
3991a8031e1Shsuenaga 	mvxpsec_wininit(sc, mva->mva_tags);
4001a8031e1Shsuenaga 
4011a8031e1Shsuenaga 	/* INTR */
4021a8031e1Shsuenaga 	MVXPSEC_WRITE(sc, MVXPSEC_INT_MASK, MVXPSEC_DEFAULT_INT);
4031a8031e1Shsuenaga 	MVXPSEC_WRITE(sc, MV_TDMA_ERR_MASK, MVXPSEC_DEFAULT_ERR);
4041a8031e1Shsuenaga 	sc->sc_done_ih =
4051a8031e1Shsuenaga 	    marvell_intr_establish(mva->mva_irq, IPL_NET, mvxpsec_intr, sc);
406de0a2812Sandvar 	/* XXX: should pass error IRQ using mva */
4071a8031e1Shsuenaga 	sc->sc_error_ih = marvell_intr_establish(MVXPSEC_ERR_INT(sc),
4081a8031e1Shsuenaga 	    IPL_NET, mvxpsec_eintr, sc);
4091a8031e1Shsuenaga 	aprint_normal_dev(self,
4101a8031e1Shsuenaga 	    "Error Reporting IRQ %d\n", MVXPSEC_ERR_INT(sc));
4111a8031e1Shsuenaga 
4121a8031e1Shsuenaga 	/* Initialize TDMA (It's enabled here, but waiting for SA) */
4131a8031e1Shsuenaga 	if (mvxpsec_dma_wait(sc) < 0)
4141a8031e1Shsuenaga 		panic("%s: DMA DEVICE not responding\n", __func__);
4151a8031e1Shsuenaga 	MVXPSEC_WRITE(sc, MV_TDMA_CNT, 0);
4161a8031e1Shsuenaga 	MVXPSEC_WRITE(sc, MV_TDMA_SRC, 0);
4171a8031e1Shsuenaga 	MVXPSEC_WRITE(sc, MV_TDMA_DST, 0);
4181a8031e1Shsuenaga 	MVXPSEC_WRITE(sc, MV_TDMA_NXT, 0);
4191a8031e1Shsuenaga 	MVXPSEC_WRITE(sc, MV_TDMA_CUR, 0);
4201a8031e1Shsuenaga 	v  = MVXPSEC_READ(sc, MV_TDMA_CONTROL);
4211a8031e1Shsuenaga 	v |= MV_TDMA_CONTROL_ENABLE;
4221a8031e1Shsuenaga 	MVXPSEC_WRITE(sc, MV_TDMA_CONTROL, v);
4231a8031e1Shsuenaga 
4241a8031e1Shsuenaga 	/* Initialize SA */
4251a8031e1Shsuenaga 	if (mvxpsec_acc_wait(sc) < 0)
4261a8031e1Shsuenaga 		panic("%s: MVXPSEC not responding\n", __func__);
4271a8031e1Shsuenaga 	v  = MVXPSEC_READ(sc, MV_ACC_CONFIG);
4281a8031e1Shsuenaga 	v &= ~MV_ACC_CONFIG_STOP_ON_ERR;
4291a8031e1Shsuenaga 	v |= MV_ACC_CONFIG_MULT_PKT;
4301a8031e1Shsuenaga 	v |= MV_ACC_CONFIG_WAIT_TDMA;
4311a8031e1Shsuenaga 	v |= MV_ACC_CONFIG_ACT_TDMA;
4321a8031e1Shsuenaga 	MVXPSEC_WRITE(sc, MV_ACC_CONFIG, v);
4331a8031e1Shsuenaga 	MVXPSEC_WRITE(sc, MV_ACC_DESC, 0);
4341a8031e1Shsuenaga 	MVXPSEC_WRITE(sc, MV_ACC_COMMAND, MV_ACC_COMMAND_STOP);
4351a8031e1Shsuenaga 
4361a8031e1Shsuenaga 	/* Session */
4371a8031e1Shsuenaga 	sc->sc_session_pool =
4381a8031e1Shsuenaga 	    pool_cache_init(sizeof(struct mvxpsec_session), 0, 0, 0,
4391a8031e1Shsuenaga 	    "mvxpsecpl", NULL, IPL_NET,
4401a8031e1Shsuenaga 	    mvxpsec_session_ctor, mvxpsec_session_dtor, sc);
4411a8031e1Shsuenaga 	pool_cache_sethiwat(sc->sc_session_pool, MVXPSEC_MAX_SESSIONS);
4421a8031e1Shsuenaga 	pool_cache_setlowat(sc->sc_session_pool, MVXPSEC_MAX_SESSIONS / 2);
4431a8031e1Shsuenaga 	sc->sc_last_session = NULL;
4441a8031e1Shsuenaga 
445ab9bc2adSandvar 	/* Packet */
4461a8031e1Shsuenaga 	sc->sc_packet_pool =
4471a8031e1Shsuenaga 	    pool_cache_init(sizeof(struct mvxpsec_session), 0, 0, 0,
4481a8031e1Shsuenaga 	    "mvxpsec_pktpl", NULL, IPL_NET,
4491a8031e1Shsuenaga 	    mvxpsec_packet_ctor, mvxpsec_packet_dtor, sc);
4501a8031e1Shsuenaga 	pool_cache_sethiwat(sc->sc_packet_pool, MVXPSEC_MAX_SESSIONS);
4511a8031e1Shsuenaga 	pool_cache_setlowat(sc->sc_packet_pool, MVXPSEC_MAX_SESSIONS / 2);
4521a8031e1Shsuenaga 
4531a8031e1Shsuenaga 	/* Register to EVCNT framework */
4541a8031e1Shsuenaga 	mvxpsec_evcnt_attach(sc);
4551a8031e1Shsuenaga 
4561a8031e1Shsuenaga 	/* Register to Opencrypto */
4571a8031e1Shsuenaga 	for (i = 0; i < MVXPSEC_MAX_SESSIONS; i++) {
4581a8031e1Shsuenaga 		sc->sc_sessions[i] = NULL;
4591a8031e1Shsuenaga 	}
4601a8031e1Shsuenaga 	if (mvxpsec_register(sc))
4611a8031e1Shsuenaga 		panic("cannot initialize OpenCrypto module.\n");
4621a8031e1Shsuenaga 
4631a8031e1Shsuenaga 	return;
4641a8031e1Shsuenaga }
4651a8031e1Shsuenaga 
4661a8031e1Shsuenaga STATIC void
mvxpsec_evcnt_attach(struct mvxpsec_softc * sc)4671a8031e1Shsuenaga mvxpsec_evcnt_attach(struct mvxpsec_softc *sc)
4681a8031e1Shsuenaga {
4691a8031e1Shsuenaga 	struct mvxpsec_evcnt *sc_ev = &sc->sc_ev;
4701a8031e1Shsuenaga 
4711a8031e1Shsuenaga 	evcnt_attach_dynamic(&sc_ev->intr_all, EVCNT_TYPE_INTR,
4721a8031e1Shsuenaga 	    NULL, device_xname(sc->sc_dev), "Main Intr.");
4731a8031e1Shsuenaga 	evcnt_attach_dynamic(&sc_ev->intr_auth, EVCNT_TYPE_INTR,
4741a8031e1Shsuenaga 	    NULL, device_xname(sc->sc_dev), "Auth Intr.");
4751a8031e1Shsuenaga 	evcnt_attach_dynamic(&sc_ev->intr_des, EVCNT_TYPE_INTR,
4761a8031e1Shsuenaga 	    NULL, device_xname(sc->sc_dev), "DES Intr.");
4771a8031e1Shsuenaga 	evcnt_attach_dynamic(&sc_ev->intr_aes_enc, EVCNT_TYPE_INTR,
4781a8031e1Shsuenaga 	    NULL, device_xname(sc->sc_dev), "AES-Encrypt Intr.");
4791a8031e1Shsuenaga 	evcnt_attach_dynamic(&sc_ev->intr_aes_dec, EVCNT_TYPE_INTR,
4801a8031e1Shsuenaga 	    NULL, device_xname(sc->sc_dev), "AES-Decrypt Intr.");
4811a8031e1Shsuenaga 	evcnt_attach_dynamic(&sc_ev->intr_enc, EVCNT_TYPE_INTR,
4821a8031e1Shsuenaga 	    NULL, device_xname(sc->sc_dev), "Crypto Intr.");
4831a8031e1Shsuenaga 	evcnt_attach_dynamic(&sc_ev->intr_sa, EVCNT_TYPE_INTR,
4841a8031e1Shsuenaga 	    NULL, device_xname(sc->sc_dev), "SA Intr.");
4851a8031e1Shsuenaga 	evcnt_attach_dynamic(&sc_ev->intr_acctdma, EVCNT_TYPE_INTR,
4861a8031e1Shsuenaga 	    NULL, device_xname(sc->sc_dev), "AccTDMA Intr.");
4871a8031e1Shsuenaga 	evcnt_attach_dynamic(&sc_ev->intr_comp, EVCNT_TYPE_INTR,
4881a8031e1Shsuenaga 	    NULL, device_xname(sc->sc_dev), "TDMA-Complete Intr.");
4891a8031e1Shsuenaga 	evcnt_attach_dynamic(&sc_ev->intr_own, EVCNT_TYPE_INTR,
4901a8031e1Shsuenaga 	    NULL, device_xname(sc->sc_dev), "TDMA-Ownership Intr.");
4911a8031e1Shsuenaga 	evcnt_attach_dynamic(&sc_ev->intr_acctdma_cont, EVCNT_TYPE_INTR,
4921a8031e1Shsuenaga 	    NULL, device_xname(sc->sc_dev), "AccTDMA-Continue Intr.");
4931a8031e1Shsuenaga 
4941a8031e1Shsuenaga 	evcnt_attach_dynamic(&sc_ev->session_new, EVCNT_TYPE_MISC,
4951a8031e1Shsuenaga 	    NULL, device_xname(sc->sc_dev), "New-Session");
4961a8031e1Shsuenaga 	evcnt_attach_dynamic(&sc_ev->session_free, EVCNT_TYPE_MISC,
4971a8031e1Shsuenaga 	    NULL, device_xname(sc->sc_dev), "Free-Session");
4981a8031e1Shsuenaga 
4991a8031e1Shsuenaga 	evcnt_attach_dynamic(&sc_ev->packet_ok, EVCNT_TYPE_MISC,
5001a8031e1Shsuenaga 	    NULL, device_xname(sc->sc_dev), "Packet-OK");
5011a8031e1Shsuenaga 	evcnt_attach_dynamic(&sc_ev->packet_err, EVCNT_TYPE_MISC,
5021a8031e1Shsuenaga 	    NULL, device_xname(sc->sc_dev), "Packet-ERR");
5031a8031e1Shsuenaga 
5041a8031e1Shsuenaga 	evcnt_attach_dynamic(&sc_ev->dispatch_packets, EVCNT_TYPE_MISC,
5051a8031e1Shsuenaga 	    NULL, device_xname(sc->sc_dev), "Packet-Dispatch");
5061a8031e1Shsuenaga 	evcnt_attach_dynamic(&sc_ev->dispatch_queue, EVCNT_TYPE_MISC,
5071a8031e1Shsuenaga 	    NULL, device_xname(sc->sc_dev), "Queue-Dispatch");
5081a8031e1Shsuenaga 	evcnt_attach_dynamic(&sc_ev->queue_full, EVCNT_TYPE_MISC,
5091a8031e1Shsuenaga 	    NULL, device_xname(sc->sc_dev), "Queue-Full");
5101a8031e1Shsuenaga 	evcnt_attach_dynamic(&sc_ev->max_dispatch, EVCNT_TYPE_MISC,
5111a8031e1Shsuenaga 	    NULL, device_xname(sc->sc_dev), "Max-Dispatch");
5121a8031e1Shsuenaga 	evcnt_attach_dynamic(&sc_ev->max_done, EVCNT_TYPE_MISC,
5131a8031e1Shsuenaga 	    NULL, device_xname(sc->sc_dev), "Max-Done");
5141a8031e1Shsuenaga }
5151a8031e1Shsuenaga 
5161a8031e1Shsuenaga /*
5171a8031e1Shsuenaga  * Register setup
5181a8031e1Shsuenaga  */
mvxpsec_wininit(struct mvxpsec_softc * sc,enum marvell_tags * tags)5191a8031e1Shsuenaga STATIC int mvxpsec_wininit(struct mvxpsec_softc *sc, enum marvell_tags *tags)
5201a8031e1Shsuenaga {
5211a8031e1Shsuenaga 	device_t pdev = device_parent(sc->sc_dev);
5221a8031e1Shsuenaga 	uint64_t base;
5231a8031e1Shsuenaga 	uint32_t size, reg;
5241a8031e1Shsuenaga 	int window, target, attr, rv, i;
5251a8031e1Shsuenaga 
5261a8031e1Shsuenaga 	/* disable all window */
5271a8031e1Shsuenaga 	for (window = 0; window < MV_TDMA_NWINDOW; window++)
5281a8031e1Shsuenaga 	{
5291a8031e1Shsuenaga 		MVXPSEC_WRITE(sc, MV_TDMA_BAR(window), 0);
5301a8031e1Shsuenaga 		MVXPSEC_WRITE(sc, MV_TDMA_ATTR(window), 0);
5311a8031e1Shsuenaga 	}
5321a8031e1Shsuenaga 
5331a8031e1Shsuenaga 	for (window = 0, i = 0;
5341a8031e1Shsuenaga 	    tags[i] != MARVELL_TAG_UNDEFINED && window < MV_TDMA_NWINDOW; i++) {
5351a8031e1Shsuenaga 		rv = marvell_winparams_by_tag(pdev, tags[i],
5361a8031e1Shsuenaga 		    &target, &attr, &base, &size);
5371a8031e1Shsuenaga 		if (rv != 0 || size == 0)
5381a8031e1Shsuenaga 			continue;
5391a8031e1Shsuenaga 
5401a8031e1Shsuenaga 		if (base > 0xffffffffULL) {
5411a8031e1Shsuenaga 			aprint_error_dev(sc->sc_dev,
5421a8031e1Shsuenaga 			    "can't remap window %d\n", window);
5431a8031e1Shsuenaga 			continue;
5441a8031e1Shsuenaga 		}
5451a8031e1Shsuenaga 
5461a8031e1Shsuenaga 		reg  = MV_TDMA_BAR_BASE(base);
5471a8031e1Shsuenaga 		MVXPSEC_WRITE(sc, MV_TDMA_BAR(window), reg);
5481a8031e1Shsuenaga 
5491a8031e1Shsuenaga 		reg  = MV_TDMA_ATTR_TARGET(target);
5501a8031e1Shsuenaga 		reg |= MV_TDMA_ATTR_ATTR(attr);
5511a8031e1Shsuenaga 		reg |= MV_TDMA_ATTR_SIZE(size);
5521a8031e1Shsuenaga 		reg |= MV_TDMA_ATTR_ENABLE;
5531a8031e1Shsuenaga 		MVXPSEC_WRITE(sc, MV_TDMA_ATTR(window), reg);
5541a8031e1Shsuenaga 
5551a8031e1Shsuenaga 		window++;
5561a8031e1Shsuenaga 	}
5571a8031e1Shsuenaga 
5581a8031e1Shsuenaga 	return 0;
5591a8031e1Shsuenaga }
5601a8031e1Shsuenaga 
5611a8031e1Shsuenaga /*
5621a8031e1Shsuenaga  * Timer handling
5631a8031e1Shsuenaga  */
5641a8031e1Shsuenaga STATIC void
mvxpsec_timer(void * aux)5651a8031e1Shsuenaga mvxpsec_timer(void *aux)
5661a8031e1Shsuenaga {
5671a8031e1Shsuenaga 	struct mvxpsec_softc *sc = aux;
5681a8031e1Shsuenaga 	struct mvxpsec_packet *mv_p;
5691a8031e1Shsuenaga 	uint32_t reg;
5701a8031e1Shsuenaga 	int ndone;
5711a8031e1Shsuenaga 	int refill;
5721a8031e1Shsuenaga 	int s;
5731a8031e1Shsuenaga 
5741a8031e1Shsuenaga 	/* IPL_SOFTCLOCK */
5751a8031e1Shsuenaga 
5761a8031e1Shsuenaga 	log(LOG_ERR, "%s: device timeout.\n", __func__);
5771a8031e1Shsuenaga #ifdef MVXPSEC_DEBUG
5781a8031e1Shsuenaga 	mvxpsec_dump_reg(sc);
5791a8031e1Shsuenaga #endif
5801a8031e1Shsuenaga 
5811a8031e1Shsuenaga 	s = splnet();
5821a8031e1Shsuenaga 	/* stop security accelerator */
5831a8031e1Shsuenaga 	MVXPSEC_WRITE(sc, MV_ACC_COMMAND, MV_ACC_COMMAND_STOP);
5841a8031e1Shsuenaga 
5851a8031e1Shsuenaga 	/* stop TDMA */
5861a8031e1Shsuenaga 	MVXPSEC_WRITE(sc, MV_TDMA_CONTROL, 0);
5871a8031e1Shsuenaga 
5881a8031e1Shsuenaga 	/* cleanup packet queue */
5891a8031e1Shsuenaga 	mutex_enter(&sc->sc_queue_mtx);
5901a8031e1Shsuenaga 	ndone = 0;
5911a8031e1Shsuenaga 	while ( (mv_p = SIMPLEQ_FIRST(&sc->sc_run_queue)) != NULL) {
5921a8031e1Shsuenaga 		SIMPLEQ_REMOVE_HEAD(&sc->sc_run_queue, queue);
5931a8031e1Shsuenaga 
5941a8031e1Shsuenaga 		mv_p->crp->crp_etype = EINVAL;
5951a8031e1Shsuenaga 		mvxpsec_done_packet(mv_p);
5961a8031e1Shsuenaga 		ndone++;
5971a8031e1Shsuenaga 	}
5981a8031e1Shsuenaga 	MVXPSEC_EVCNT_MAX(sc, max_done, ndone);
5991a8031e1Shsuenaga 	sc->sc_flags &= ~HW_RUNNING;
6001a8031e1Shsuenaga 	refill = (sc->sc_wait_qlen > 0) ? 1 : 0;
6011a8031e1Shsuenaga 	mutex_exit(&sc->sc_queue_mtx);
6021a8031e1Shsuenaga 
6031a8031e1Shsuenaga 	/* reenable TDMA */
6041a8031e1Shsuenaga 	if (mvxpsec_dma_wait(sc) < 0)
6051a8031e1Shsuenaga 		panic("%s: failed to reset DMA DEVICE. give up.", __func__);
6061a8031e1Shsuenaga 	MVXPSEC_WRITE(sc, MV_TDMA_CNT, 0);
6071a8031e1Shsuenaga 	MVXPSEC_WRITE(sc, MV_TDMA_SRC, 0);
6081a8031e1Shsuenaga 	MVXPSEC_WRITE(sc, MV_TDMA_DST, 0);
6091a8031e1Shsuenaga 	MVXPSEC_WRITE(sc, MV_TDMA_CUR, 0);
6101a8031e1Shsuenaga 	MVXPSEC_WRITE(sc, MV_TDMA_NXT, 0);
6111a8031e1Shsuenaga 	reg  = MV_TDMA_DEFAULT_CONTROL;
6121a8031e1Shsuenaga 	reg |= MV_TDMA_CONTROL_ENABLE;
6131a8031e1Shsuenaga 	MVXPSEC_WRITE(sc, MV_TDMA_CONTROL, reg);
6141a8031e1Shsuenaga 
6151a8031e1Shsuenaga 	if (mvxpsec_acc_wait(sc) < 0)
6161a8031e1Shsuenaga 		panic("%s: failed to reset MVXPSEC. give up.", __func__);
6171a8031e1Shsuenaga 	reg  = MV_ACC_CONFIG_MULT_PKT;
6181a8031e1Shsuenaga 	reg |= MV_ACC_CONFIG_WAIT_TDMA;
6191a8031e1Shsuenaga 	reg |= MV_ACC_CONFIG_ACT_TDMA;
6201a8031e1Shsuenaga 	MVXPSEC_WRITE(sc, MV_ACC_CONFIG, reg);
6211a8031e1Shsuenaga 	MVXPSEC_WRITE(sc, MV_ACC_DESC, 0);
6221a8031e1Shsuenaga 
6231a8031e1Shsuenaga 	if (refill) {
6241a8031e1Shsuenaga 		mutex_enter(&sc->sc_queue_mtx);
6251a8031e1Shsuenaga 		mvxpsec_dispatch_queue(sc);
6261a8031e1Shsuenaga 		mutex_exit(&sc->sc_queue_mtx);
6271a8031e1Shsuenaga 	}
6281a8031e1Shsuenaga 
6291a8031e1Shsuenaga 	crypto_unblock(sc->sc_cid, CRYPTO_SYMQ|CRYPTO_ASYMQ);
6301a8031e1Shsuenaga 	splx(s);
6311a8031e1Shsuenaga }
6321a8031e1Shsuenaga 
6331a8031e1Shsuenaga /*
6341a8031e1Shsuenaga  * DMA handling
6351a8031e1Shsuenaga  */
6361a8031e1Shsuenaga 
6371a8031e1Shsuenaga /*
6381a8031e1Shsuenaga  * Allocate kernel devmem and DMA safe memory with bus_dma API
6391a8031e1Shsuenaga  * used for DMA descriptors.
6401a8031e1Shsuenaga  *
6411a8031e1Shsuenaga  * if phys != 0, assume phys is a DMA safe memory and bypass
6421a8031e1Shsuenaga  * allocator.
6431a8031e1Shsuenaga  */
6441a8031e1Shsuenaga STATIC struct mvxpsec_devmem *
mvxpsec_alloc_devmem(struct mvxpsec_softc * sc,paddr_t phys,int size)6451a8031e1Shsuenaga mvxpsec_alloc_devmem(struct mvxpsec_softc *sc, paddr_t phys, int size)
6461a8031e1Shsuenaga {
6471a8031e1Shsuenaga 	struct mvxpsec_devmem *devmem;
6481a8031e1Shsuenaga 	bus_dma_segment_t seg;
6491a8031e1Shsuenaga 	int rseg;
6501a8031e1Shsuenaga 	int err;
6511a8031e1Shsuenaga 
6521a8031e1Shsuenaga 	if (sc == NULL)
6531a8031e1Shsuenaga 		return NULL;
6541a8031e1Shsuenaga 
6550a827a3fSchs 	devmem = kmem_alloc(sizeof(*devmem), KM_SLEEP);
6561a8031e1Shsuenaga 	devmem->size = size;
6571a8031e1Shsuenaga 
6581a8031e1Shsuenaga 	if (phys) {
6591a8031e1Shsuenaga 		seg.ds_addr = phys;
6601a8031e1Shsuenaga 		seg.ds_len = devmem->size;
6611a8031e1Shsuenaga 		rseg = 1;
6621a8031e1Shsuenaga 		err = 0;
6631a8031e1Shsuenaga 	}
6641a8031e1Shsuenaga 	else {
6651a8031e1Shsuenaga 		err = bus_dmamem_alloc(sc->sc_dmat,
6661a8031e1Shsuenaga 		    devmem->size, PAGE_SIZE, 0,
6671a8031e1Shsuenaga 		    &seg, MVXPSEC_DMA_MAX_SEGS, &rseg, BUS_DMA_NOWAIT);
6681a8031e1Shsuenaga 	}
6691a8031e1Shsuenaga 	if (err) {
6701a8031e1Shsuenaga 		aprint_error_dev(sc->sc_dev, "can't alloc DMA buffer\n");
6711a8031e1Shsuenaga 		goto fail_kmem_free;
6721a8031e1Shsuenaga 	}
6731a8031e1Shsuenaga 
6741a8031e1Shsuenaga 	err = bus_dmamem_map(sc->sc_dmat, &seg, rseg,
6751a8031e1Shsuenaga 	     devmem->size, &devmem->kva, BUS_DMA_NOWAIT);
6761a8031e1Shsuenaga 	if (err) {
6771a8031e1Shsuenaga 		aprint_error_dev(sc->sc_dev, "can't map DMA buffer\n");
6781a8031e1Shsuenaga 		goto fail_dmamem_free;
6791a8031e1Shsuenaga 	}
6801a8031e1Shsuenaga 
6811a8031e1Shsuenaga 	err = bus_dmamap_create(sc->sc_dmat,
6821a8031e1Shsuenaga 	    size, 1, size, 0, BUS_DMA_NOWAIT, &devmem->map);
6831a8031e1Shsuenaga 	if (err) {
6841a8031e1Shsuenaga 		aprint_error_dev(sc->sc_dev, "can't create DMA map\n");
6851a8031e1Shsuenaga 		goto fail_unmap;
6861a8031e1Shsuenaga 	}
6871a8031e1Shsuenaga 
6881a8031e1Shsuenaga 	err = bus_dmamap_load(sc->sc_dmat,
6891a8031e1Shsuenaga 	    devmem->map, devmem->kva, devmem->size, NULL,
6901a8031e1Shsuenaga 	    BUS_DMA_NOWAIT);
6911a8031e1Shsuenaga 	if (err) {
6921a8031e1Shsuenaga 		aprint_error_dev(sc->sc_dev,
6931a8031e1Shsuenaga 		   "can't load DMA buffer VA:%p PA:0x%08x\n",
6941a8031e1Shsuenaga 		    devmem->kva, (int)seg.ds_addr);
6951a8031e1Shsuenaga 		goto fail_destroy;
6961a8031e1Shsuenaga 	}
6971a8031e1Shsuenaga 
6981a8031e1Shsuenaga 	return devmem;
6991a8031e1Shsuenaga 
7001a8031e1Shsuenaga fail_destroy:
7011a8031e1Shsuenaga 	bus_dmamap_destroy(sc->sc_dmat, devmem->map);
7021a8031e1Shsuenaga fail_unmap:
7031a8031e1Shsuenaga 	bus_dmamem_unmap(sc->sc_dmat, devmem->kva, devmem->size);
7041a8031e1Shsuenaga fail_dmamem_free:
7051a8031e1Shsuenaga 	bus_dmamem_free(sc->sc_dmat, &seg, rseg);
7061a8031e1Shsuenaga fail_kmem_free:
7071a8031e1Shsuenaga 	kmem_free(devmem, sizeof(*devmem));
7081a8031e1Shsuenaga 
7091a8031e1Shsuenaga 	return NULL;
7101a8031e1Shsuenaga }
7111a8031e1Shsuenaga 
7121a8031e1Shsuenaga /*
7131a8031e1Shsuenaga  * Get DMA Descriptor from (DMA safe) descriptor pool.
7141a8031e1Shsuenaga  */
7151a8031e1Shsuenaga INLINE struct mvxpsec_descriptor_handle *
mvxpsec_dma_getdesc(struct mvxpsec_softc * sc)7161a8031e1Shsuenaga mvxpsec_dma_getdesc(struct mvxpsec_softc *sc)
7171a8031e1Shsuenaga {
7181a8031e1Shsuenaga 	struct mvxpsec_descriptor_handle *entry;
7191a8031e1Shsuenaga 
7201a8031e1Shsuenaga 	/* must called with sc->sc_dma_mtx held */
7211a8031e1Shsuenaga 	KASSERT(mutex_owned(&sc->sc_dma_mtx));
7221a8031e1Shsuenaga 
7231a8031e1Shsuenaga 	if (sc->sc_desc_ring_prod == sc->sc_desc_ring_cons)
7241a8031e1Shsuenaga 		return NULL;
7251a8031e1Shsuenaga 
7261a8031e1Shsuenaga 	entry = &sc->sc_desc_ring[sc->sc_desc_ring_prod];
7271a8031e1Shsuenaga 	sc->sc_desc_ring_prod++;
7281a8031e1Shsuenaga 	if (sc->sc_desc_ring_prod >= sc->sc_desc_ring_size)
7291a8031e1Shsuenaga 		sc->sc_desc_ring_prod -= sc->sc_desc_ring_size;
7301a8031e1Shsuenaga 
7311a8031e1Shsuenaga 	return entry;
7321a8031e1Shsuenaga }
7331a8031e1Shsuenaga 
7341a8031e1Shsuenaga /*
7351a8031e1Shsuenaga  * Put DMA Descriptor to descriptor pool.
7361a8031e1Shsuenaga  */
7371a8031e1Shsuenaga _INLINE void
mvxpsec_dma_putdesc(struct mvxpsec_softc * sc,struct mvxpsec_descriptor_handle * dh)7381a8031e1Shsuenaga mvxpsec_dma_putdesc(struct mvxpsec_softc *sc,
7391a8031e1Shsuenaga     struct mvxpsec_descriptor_handle *dh)
7401a8031e1Shsuenaga {
7411a8031e1Shsuenaga 	/* must called with sc->sc_dma_mtx held */
7421a8031e1Shsuenaga 	KASSERT(mutex_owned(&sc->sc_dma_mtx));
7431a8031e1Shsuenaga 
7441a8031e1Shsuenaga 	sc->sc_desc_ring_cons++;
7451a8031e1Shsuenaga 	if (sc->sc_desc_ring_cons >= sc->sc_desc_ring_size)
7461a8031e1Shsuenaga 		sc->sc_desc_ring_cons -= sc->sc_desc_ring_size;
7471a8031e1Shsuenaga 
7481a8031e1Shsuenaga 	return;
7491a8031e1Shsuenaga }
7501a8031e1Shsuenaga 
7511a8031e1Shsuenaga /*
7521a8031e1Shsuenaga  * Setup DMA Descriptor
7531a8031e1Shsuenaga  * copy from 'src' to 'dst' by 'size' bytes.
7541a8031e1Shsuenaga  * 'src' or 'dst' must be SRAM address.
7551a8031e1Shsuenaga  */
7561a8031e1Shsuenaga INLINE void
mvxpsec_dma_setup(struct mvxpsec_descriptor_handle * dh,uint32_t dst,uint32_t src,uint32_t size)7571a8031e1Shsuenaga mvxpsec_dma_setup(struct mvxpsec_descriptor_handle *dh,
7581a8031e1Shsuenaga     uint32_t dst, uint32_t src, uint32_t size)
7591a8031e1Shsuenaga {
7601a8031e1Shsuenaga 	struct mvxpsec_descriptor *desc;
7611a8031e1Shsuenaga 
7621a8031e1Shsuenaga 	desc = (struct mvxpsec_descriptor *)dh->_desc;
7631a8031e1Shsuenaga 
7641a8031e1Shsuenaga 	desc->tdma_dst = dst;
7651a8031e1Shsuenaga 	desc->tdma_src = src;
7661a8031e1Shsuenaga 	desc->tdma_word0 = size;
7671a8031e1Shsuenaga 	if (size != 0)
7681a8031e1Shsuenaga 		desc->tdma_word0 |= MV_TDMA_CNT_OWN;
7691a8031e1Shsuenaga 	/* size == 0 is owned by ACC, not TDMA */
7701a8031e1Shsuenaga 
7711a8031e1Shsuenaga #ifdef MVXPSEC_DEBUG
7721a8031e1Shsuenaga 	mvxpsec_dump_dmaq(dh);
7731a8031e1Shsuenaga #endif
7741a8031e1Shsuenaga }
7751a8031e1Shsuenaga 
7761a8031e1Shsuenaga /*
7771a8031e1Shsuenaga  * Concat 2 DMA
7781a8031e1Shsuenaga  */
7791a8031e1Shsuenaga INLINE void
mvxpsec_dma_cat(struct mvxpsec_softc * sc,struct mvxpsec_descriptor_handle * dh1,struct mvxpsec_descriptor_handle * dh2)7801a8031e1Shsuenaga mvxpsec_dma_cat(struct mvxpsec_softc *sc,
7811a8031e1Shsuenaga     struct mvxpsec_descriptor_handle *dh1,
7821a8031e1Shsuenaga     struct mvxpsec_descriptor_handle *dh2)
7831a8031e1Shsuenaga {
7841a8031e1Shsuenaga 	((struct mvxpsec_descriptor*)dh1->_desc)->tdma_nxt = dh2->phys_addr;
7851a8031e1Shsuenaga 	MVXPSEC_SYNC_DESC(sc, dh1, BUS_DMASYNC_PREWRITE);
7861a8031e1Shsuenaga }
7871a8031e1Shsuenaga 
7881a8031e1Shsuenaga /*
7891a8031e1Shsuenaga  * Schedule DMA Copy
7901a8031e1Shsuenaga  */
7911a8031e1Shsuenaga INLINE int
mvxpsec_dma_copy0(struct mvxpsec_softc * sc,mvxpsec_dma_ring * r,uint32_t dst,uint32_t src,uint32_t size)7921a8031e1Shsuenaga mvxpsec_dma_copy0(struct mvxpsec_softc *sc, mvxpsec_dma_ring *r,
7931a8031e1Shsuenaga     uint32_t dst, uint32_t src, uint32_t size)
7941a8031e1Shsuenaga {
7951a8031e1Shsuenaga 	struct mvxpsec_descriptor_handle *dh;
7961a8031e1Shsuenaga 
7971a8031e1Shsuenaga 	dh = mvxpsec_dma_getdesc(sc);
7981a8031e1Shsuenaga 	if (dh == NULL) {
7991a8031e1Shsuenaga 		log(LOG_ERR, "%s: descriptor full\n", __func__);
8001a8031e1Shsuenaga 		return -1;
8011a8031e1Shsuenaga 	}
8021a8031e1Shsuenaga 
8031a8031e1Shsuenaga 	mvxpsec_dma_setup(dh, dst, src, size);
8041a8031e1Shsuenaga 	if (r->dma_head == NULL) {
8051a8031e1Shsuenaga 		r->dma_head = dh;
8061a8031e1Shsuenaga 		r->dma_last = dh;
8071a8031e1Shsuenaga 		r->dma_size = 1;
8081a8031e1Shsuenaga 	}
8091a8031e1Shsuenaga 	else {
8101a8031e1Shsuenaga 		mvxpsec_dma_cat(sc, r->dma_last, dh);
8111a8031e1Shsuenaga 		r->dma_last = dh;
8121a8031e1Shsuenaga 		r->dma_size++;
8131a8031e1Shsuenaga 	}
8141a8031e1Shsuenaga 
8151a8031e1Shsuenaga 	return 0;
8161a8031e1Shsuenaga }
8171a8031e1Shsuenaga 
8181a8031e1Shsuenaga INLINE int
mvxpsec_dma_copy(struct mvxpsec_softc * sc,mvxpsec_dma_ring * r,uint32_t dst,uint32_t src,uint32_t size)8191a8031e1Shsuenaga mvxpsec_dma_copy(struct mvxpsec_softc *sc, mvxpsec_dma_ring *r,
8201a8031e1Shsuenaga     uint32_t dst, uint32_t src, uint32_t size)
8211a8031e1Shsuenaga {
8221a8031e1Shsuenaga 	if (size == 0) /* 0 is very special descriptor */
8231a8031e1Shsuenaga 		return 0;
8241a8031e1Shsuenaga 
8251a8031e1Shsuenaga 	return mvxpsec_dma_copy0(sc, r, dst, src, size);
8261a8031e1Shsuenaga }
8271a8031e1Shsuenaga 
8281a8031e1Shsuenaga /*
8291a8031e1Shsuenaga  * Schedule ACC Activate
8301a8031e1Shsuenaga  */
8311a8031e1Shsuenaga INLINE int
mvxpsec_dma_acc_activate(struct mvxpsec_softc * sc,mvxpsec_dma_ring * r)8321a8031e1Shsuenaga mvxpsec_dma_acc_activate(struct mvxpsec_softc *sc, mvxpsec_dma_ring *r)
8331a8031e1Shsuenaga {
8341a8031e1Shsuenaga 	return mvxpsec_dma_copy0(sc, r, 0, 0, 0);
8351a8031e1Shsuenaga }
8361a8031e1Shsuenaga 
8371a8031e1Shsuenaga /*
8381a8031e1Shsuenaga  * Finalize DMA setup
8391a8031e1Shsuenaga  */
8401a8031e1Shsuenaga INLINE void
mvxpsec_dma_finalize(struct mvxpsec_softc * sc,mvxpsec_dma_ring * r)8411a8031e1Shsuenaga mvxpsec_dma_finalize(struct mvxpsec_softc *sc, mvxpsec_dma_ring *r)
8421a8031e1Shsuenaga {
8431a8031e1Shsuenaga 	struct mvxpsec_descriptor_handle *dh;
8441a8031e1Shsuenaga 
8451a8031e1Shsuenaga 	dh = r->dma_last;
8461a8031e1Shsuenaga 	((struct mvxpsec_descriptor*)dh->_desc)->tdma_nxt = 0;
8471a8031e1Shsuenaga 	MVXPSEC_SYNC_DESC(sc, dh, BUS_DMASYNC_PREWRITE);
8481a8031e1Shsuenaga }
8491a8031e1Shsuenaga 
8501a8031e1Shsuenaga /*
8511a8031e1Shsuenaga  * Free entire DMA ring
8521a8031e1Shsuenaga  */
8531a8031e1Shsuenaga INLINE void
mvxpsec_dma_free(struct mvxpsec_softc * sc,mvxpsec_dma_ring * r)8541a8031e1Shsuenaga mvxpsec_dma_free(struct mvxpsec_softc *sc, mvxpsec_dma_ring *r)
8551a8031e1Shsuenaga {
8561a8031e1Shsuenaga 	sc->sc_desc_ring_cons += r->dma_size;
8571a8031e1Shsuenaga 	if (sc->sc_desc_ring_cons >= sc->sc_desc_ring_size)
8581a8031e1Shsuenaga 		sc->sc_desc_ring_cons -= sc->sc_desc_ring_size;
8591a8031e1Shsuenaga 	r->dma_head = NULL;
8601a8031e1Shsuenaga 	r->dma_last = NULL;
8611a8031e1Shsuenaga 	r->dma_size = 0;
8621a8031e1Shsuenaga }
8631a8031e1Shsuenaga 
8641a8031e1Shsuenaga /*
8651a8031e1Shsuenaga  * create DMA descriptor chain for the packet
8661a8031e1Shsuenaga  */
8671a8031e1Shsuenaga INLINE int
mvxpsec_dma_copy_packet(struct mvxpsec_softc * sc,struct mvxpsec_packet * mv_p)8681a8031e1Shsuenaga mvxpsec_dma_copy_packet(struct mvxpsec_softc *sc, struct mvxpsec_packet *mv_p)
8691a8031e1Shsuenaga {
8701a8031e1Shsuenaga 	struct mvxpsec_session *mv_s = mv_p->mv_s;
8711a8031e1Shsuenaga 	uint32_t src, dst, len;
8721a8031e1Shsuenaga 	uint32_t pkt_off, pkt_off_r;
8731a8031e1Shsuenaga 	int err;
8741a8031e1Shsuenaga 	int i;
8751a8031e1Shsuenaga 
8761a8031e1Shsuenaga 	/* must called with sc->sc_dma_mtx held */
8771a8031e1Shsuenaga 	KASSERT(mutex_owned(&sc->sc_dma_mtx));
8781a8031e1Shsuenaga 
8791a8031e1Shsuenaga 	/*
8801a8031e1Shsuenaga 	 * set offset for mem->device copy
8811a8031e1Shsuenaga 	 *
8821a8031e1Shsuenaga 	 * typical packet image:
8831a8031e1Shsuenaga 	 *
8841a8031e1Shsuenaga 	 *   enc_ivoff
8851a8031e1Shsuenaga 	 *   mac_off
8861a8031e1Shsuenaga 	 *   |
8871a8031e1Shsuenaga 	 *   |    enc_off
8881a8031e1Shsuenaga 	 *   |    |
8891a8031e1Shsuenaga 	 *   v    v
8901a8031e1Shsuenaga 	 *   +----+--------...
8911a8031e1Shsuenaga 	 *   |IV  |DATA
8921a8031e1Shsuenaga 	 *   +----+--------...
8931a8031e1Shsuenaga 	 */
8941a8031e1Shsuenaga 	pkt_off = 0;
8951a8031e1Shsuenaga 	if (mv_p->mac_off > 0)
8961a8031e1Shsuenaga 		pkt_off = mv_p->mac_off;
8971a8031e1Shsuenaga 	if ((mv_p->flags & CRP_EXT_IV) == 0 && pkt_off > mv_p->enc_ivoff)
8981a8031e1Shsuenaga 		pkt_off = mv_p->enc_ivoff;
8991a8031e1Shsuenaga 	if (mv_p->enc_off > 0 && pkt_off > mv_p->enc_off)
9001a8031e1Shsuenaga 		pkt_off = mv_p->enc_off;
9011a8031e1Shsuenaga 	pkt_off_r = pkt_off;
9021a8031e1Shsuenaga 
9031a8031e1Shsuenaga 	/* make DMA descriptors to copy packet header: DRAM -> SRAM */
9041a8031e1Shsuenaga 	dst = (uint32_t)MVXPSEC_SRAM_PKT_HDR_PA(sc);
9051a8031e1Shsuenaga 	src = (uint32_t)mv_p->pkt_header_map->dm_segs[0].ds_addr;
9061a8031e1Shsuenaga 	len = sizeof(mv_p->pkt_header);
9071a8031e1Shsuenaga 	err = mvxpsec_dma_copy(sc, &mv_p->dma_ring, dst, src, len);
9081a8031e1Shsuenaga 	if (__predict_false(err))
9091a8031e1Shsuenaga 		return err;
9101a8031e1Shsuenaga 
9111a8031e1Shsuenaga 	/*
9121a8031e1Shsuenaga 	 * make DMA descriptors to copy session header: DRAM -> SRAM
9131a8031e1Shsuenaga 	 * we can reuse session header on SRAM if session is not changed.
9141a8031e1Shsuenaga 	 */
9151a8031e1Shsuenaga 	if (sc->sc_last_session != mv_s) {
9161a8031e1Shsuenaga 		dst = (uint32_t)MVXPSEC_SRAM_SESS_HDR_PA(sc);
9171a8031e1Shsuenaga 		src = (uint32_t)mv_s->session_header_map->dm_segs[0].ds_addr;
9181a8031e1Shsuenaga 		len = sizeof(mv_s->session_header);
9191a8031e1Shsuenaga 		err = mvxpsec_dma_copy(sc, &mv_p->dma_ring, dst, src, len);
9201a8031e1Shsuenaga 		if (__predict_false(err))
9211a8031e1Shsuenaga 			return err;
9221a8031e1Shsuenaga 		sc->sc_last_session = mv_s;
9231a8031e1Shsuenaga 	}
9241a8031e1Shsuenaga 
9251a8031e1Shsuenaga 	/* make DMA descriptor to copy payload data: DRAM -> SRAM */
9261a8031e1Shsuenaga 	dst = MVXPSEC_SRAM_PAYLOAD_PA(sc, 0);
9271a8031e1Shsuenaga 	for (i = 0; i < mv_p->data_map->dm_nsegs; i++) {
9281a8031e1Shsuenaga 		src = mv_p->data_map->dm_segs[i].ds_addr;
9291a8031e1Shsuenaga 		len = mv_p->data_map->dm_segs[i].ds_len;
9301a8031e1Shsuenaga 		if (pkt_off) {
9311a8031e1Shsuenaga 			if (len <= pkt_off) {
9321a8031e1Shsuenaga 				/* ignore the segment */
9331a8031e1Shsuenaga 				dst += len;
9341a8031e1Shsuenaga 				pkt_off -= len;
9351a8031e1Shsuenaga 				continue;
9361a8031e1Shsuenaga 			}
9371a8031e1Shsuenaga 			/* copy from the middle of the segment */
9381a8031e1Shsuenaga 			dst += pkt_off;
9391a8031e1Shsuenaga 			src += pkt_off;
9401a8031e1Shsuenaga 			len -= pkt_off;
9411a8031e1Shsuenaga 			pkt_off = 0;
9421a8031e1Shsuenaga 		}
9431a8031e1Shsuenaga 		err = mvxpsec_dma_copy(sc, &mv_p->dma_ring, dst, src, len);
9441a8031e1Shsuenaga 		if (__predict_false(err))
9451a8031e1Shsuenaga 			return err;
9461a8031e1Shsuenaga 		dst += len;
9471a8031e1Shsuenaga 	}
9481a8031e1Shsuenaga 
9491a8031e1Shsuenaga 	/* make special descriptor to activate security accelerator */
9501a8031e1Shsuenaga 	err = mvxpsec_dma_acc_activate(sc, &mv_p->dma_ring);
9511a8031e1Shsuenaga 	if (__predict_false(err))
9521a8031e1Shsuenaga 		return err;
9531a8031e1Shsuenaga 
9541a8031e1Shsuenaga 	/* make DMA descriptors to copy payload: SRAM -> DRAM */
9551a8031e1Shsuenaga 	src = (uint32_t)MVXPSEC_SRAM_PAYLOAD_PA(sc, 0);
9561a8031e1Shsuenaga 	for (i = 0; i < mv_p->data_map->dm_nsegs; i++) {
9571a8031e1Shsuenaga 		dst = (uint32_t)mv_p->data_map->dm_segs[i].ds_addr;
9581a8031e1Shsuenaga 		len = (uint32_t)mv_p->data_map->dm_segs[i].ds_len;
9591a8031e1Shsuenaga 		if (pkt_off_r) {
9601a8031e1Shsuenaga 			if (len <= pkt_off_r) {
9611a8031e1Shsuenaga 				/* ignore the segment */
9621a8031e1Shsuenaga 				src += len;
9631a8031e1Shsuenaga 				pkt_off_r -= len;
9641a8031e1Shsuenaga 				continue;
9651a8031e1Shsuenaga 			}
9661a8031e1Shsuenaga 			/* copy from the middle of the segment */
9671a8031e1Shsuenaga 			src += pkt_off_r;
9681a8031e1Shsuenaga 			dst += pkt_off_r;
9691a8031e1Shsuenaga 			len -= pkt_off_r;
9701a8031e1Shsuenaga 			pkt_off_r = 0;
9711a8031e1Shsuenaga 		}
9721a8031e1Shsuenaga 		err = mvxpsec_dma_copy(sc, &mv_p->dma_ring, dst, src, len);
9731a8031e1Shsuenaga 		if (__predict_false(err))
9741a8031e1Shsuenaga 			return err;
9751a8031e1Shsuenaga 		src += len;
9761a8031e1Shsuenaga 	}
9771a8031e1Shsuenaga 	KASSERT(pkt_off == 0);
9781a8031e1Shsuenaga 	KASSERT(pkt_off_r == 0);
9791a8031e1Shsuenaga 
9801a8031e1Shsuenaga 	/*
9811a8031e1Shsuenaga 	 * make DMA descriptors to copy packet header: SRAM->DRAM
9821a8031e1Shsuenaga 	 * if IV is present in the payload, no need to copy.
9831a8031e1Shsuenaga 	 */
9841a8031e1Shsuenaga 	if (mv_p->flags & CRP_EXT_IV) {
9851a8031e1Shsuenaga 		dst = (uint32_t)mv_p->pkt_header_map->dm_segs[0].ds_addr;
9861a8031e1Shsuenaga 		src = (uint32_t)MVXPSEC_SRAM_PKT_HDR_PA(sc);
9871a8031e1Shsuenaga 		len = sizeof(mv_p->pkt_header);
9881a8031e1Shsuenaga 		err = mvxpsec_dma_copy(sc, &mv_p->dma_ring, dst, src, len);
9891a8031e1Shsuenaga 		if (__predict_false(err))
9901a8031e1Shsuenaga 			return err;
9911a8031e1Shsuenaga 	}
9921a8031e1Shsuenaga 
9931a8031e1Shsuenaga 	return 0;
9941a8031e1Shsuenaga }
9951a8031e1Shsuenaga 
9961a8031e1Shsuenaga INLINE int
mvxpsec_dma_sync_packet(struct mvxpsec_softc * sc,struct mvxpsec_packet * mv_p)9971a8031e1Shsuenaga mvxpsec_dma_sync_packet(struct mvxpsec_softc *sc, struct mvxpsec_packet *mv_p)
9981a8031e1Shsuenaga {
9991a8031e1Shsuenaga 	/* sync packet header */
10001a8031e1Shsuenaga 	bus_dmamap_sync(sc->sc_dmat,
10011a8031e1Shsuenaga 	    mv_p->pkt_header_map, 0, sizeof(mv_p->pkt_header),
10021a8031e1Shsuenaga 	    BUS_DMASYNC_PREWRITE);
10031a8031e1Shsuenaga 
10041a8031e1Shsuenaga #ifdef MVXPSEC_DEBUG
10051a8031e1Shsuenaga 	/* sync session header */
10061a8031e1Shsuenaga 	if (mvxpsec_debug != 0) {
10071a8031e1Shsuenaga 		struct mvxpsec_session *mv_s = mv_p->mv_s;
10081a8031e1Shsuenaga 
10091a8031e1Shsuenaga 		/* only debug code touch the session header after newsession */
10101a8031e1Shsuenaga 		bus_dmamap_sync(sc->sc_dmat,
10111a8031e1Shsuenaga 		    mv_s->session_header_map,
10121a8031e1Shsuenaga 		    0, sizeof(mv_s->session_header),
10131a8031e1Shsuenaga 		    BUS_DMASYNC_PREWRITE);
10141a8031e1Shsuenaga 	}
10151a8031e1Shsuenaga #endif
10161a8031e1Shsuenaga 
10171a8031e1Shsuenaga 	/* sync packet buffer */
10181a8031e1Shsuenaga 	bus_dmamap_sync(sc->sc_dmat,
10191a8031e1Shsuenaga 	    mv_p->data_map, 0, mv_p->data_len,
10201a8031e1Shsuenaga 	    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
10211a8031e1Shsuenaga 
10221a8031e1Shsuenaga 	return 0;
10231a8031e1Shsuenaga }
10241a8031e1Shsuenaga 
10251a8031e1Shsuenaga /*
10261a8031e1Shsuenaga  * Initialize MVXPSEC Internal SRAM
10271a8031e1Shsuenaga  *
1028ab9bc2adSandvar  * - must be called after DMA initialization.
10291a8031e1Shsuenaga  * - make VM mapping for SRAM area on MBus.
10301a8031e1Shsuenaga  */
10311a8031e1Shsuenaga STATIC int
mvxpsec_init_sram(struct mvxpsec_softc * sc)10321a8031e1Shsuenaga mvxpsec_init_sram(struct mvxpsec_softc *sc)
10331a8031e1Shsuenaga {
10341a8031e1Shsuenaga 	uint32_t tag, target, attr, base, size;
10351a8031e1Shsuenaga 	vaddr_t va;
10361a8031e1Shsuenaga 	int window;
10371a8031e1Shsuenaga 
10383aa5a3aeSriastradh 	switch (device_unit(sc->sc_dev)) {
10391a8031e1Shsuenaga 	case 0:
10401a8031e1Shsuenaga 		tag = ARMADAXP_TAG_CRYPT0;
10411a8031e1Shsuenaga 		break;
10421a8031e1Shsuenaga 	case 1:
10431a8031e1Shsuenaga 		tag = ARMADAXP_TAG_CRYPT1;
10441a8031e1Shsuenaga 		break;
10451a8031e1Shsuenaga 	default:
10461a8031e1Shsuenaga 		aprint_error_dev(sc->sc_dev, "no internal SRAM mapping\n");
10471a8031e1Shsuenaga 		return -1;
10481a8031e1Shsuenaga 	}
10491a8031e1Shsuenaga 
10501a8031e1Shsuenaga 	window = mvsoc_target(tag, &target, &attr, &base, &size);
10511a8031e1Shsuenaga 	if (window >= nwindow) {
10521a8031e1Shsuenaga 		aprint_error_dev(sc->sc_dev, "no internal SRAM mapping\n");
10531a8031e1Shsuenaga 		return -1;
10541a8031e1Shsuenaga 	}
10551a8031e1Shsuenaga 
10561a8031e1Shsuenaga 	if (sizeof(struct mvxpsec_crypt_sram) > size) {
10571a8031e1Shsuenaga 		aprint_error_dev(sc->sc_dev,
10581a8031e1Shsuenaga 		    "SRAM Data Structure Excceeds SRAM window size.\n");
10591a8031e1Shsuenaga 		return -1;
10601a8031e1Shsuenaga 	}
10611a8031e1Shsuenaga 
10621a8031e1Shsuenaga 	aprint_normal_dev(sc->sc_dev,
10631a8031e1Shsuenaga 	    "internal SRAM window at 0x%08x-0x%08x",
10641a8031e1Shsuenaga 	    base, base + size - 1);
10651a8031e1Shsuenaga 	sc->sc_sram_pa = base;
10661a8031e1Shsuenaga 
10671a8031e1Shsuenaga 	/* get vmspace to read/write device internal SRAM */
10681a8031e1Shsuenaga 	va = uvm_km_alloc(kernel_map, PAGE_SIZE, PAGE_SIZE,
10691a8031e1Shsuenaga 			UVM_KMF_VAONLY | UVM_KMF_NOWAIT);
10701a8031e1Shsuenaga 	if (va == 0) {
10711a8031e1Shsuenaga 		aprint_error_dev(sc->sc_dev, "cannot map SRAM window\n");
10721a8031e1Shsuenaga 		sc->sc_sram_va = NULL;
10731a8031e1Shsuenaga 		aprint_normal("\n");
10741a8031e1Shsuenaga 		return 0;
10751a8031e1Shsuenaga 	}
10761a8031e1Shsuenaga 	/* XXX: not working. PMAP_NOCACHE is not affected? */
10771a8031e1Shsuenaga 	pmap_kenter_pa(va, base, VM_PROT_READ|VM_PROT_WRITE, PMAP_NOCACHE);
10781a8031e1Shsuenaga 	pmap_update(pmap_kernel());
10791a8031e1Shsuenaga 	sc->sc_sram_va = (void *)va;
10801a8031e1Shsuenaga 	aprint_normal(" va %p\n", sc->sc_sram_va);
10811a8031e1Shsuenaga 	memset(sc->sc_sram_va, 0xff, MV_ACC_SRAM_SIZE);
10821a8031e1Shsuenaga 
10831a8031e1Shsuenaga 	return 0;
10841a8031e1Shsuenaga }
10851a8031e1Shsuenaga 
10861a8031e1Shsuenaga /*
10871a8031e1Shsuenaga  * Initialize TDMA engine.
10881a8031e1Shsuenaga  */
10891a8031e1Shsuenaga STATIC int
mvxpsec_init_dma(struct mvxpsec_softc * sc,struct marvell_attach_args * mva)10901a8031e1Shsuenaga mvxpsec_init_dma(struct mvxpsec_softc *sc, struct marvell_attach_args *mva)
10911a8031e1Shsuenaga {
10921a8031e1Shsuenaga 	struct mvxpsec_descriptor_handle *dh;
10931a8031e1Shsuenaga 	uint8_t *va;
10941a8031e1Shsuenaga 	paddr_t pa;
10951a8031e1Shsuenaga 	off_t va_off, pa_off;
10961a8031e1Shsuenaga 	int i, n, seg, ndh;
10971a8031e1Shsuenaga 
10981a8031e1Shsuenaga 	/* Init Deviced's control parameters (disabled yet) */
10991a8031e1Shsuenaga 	MVXPSEC_WRITE(sc, MV_TDMA_CONTROL, MV_TDMA_DEFAULT_CONTROL);
11001a8031e1Shsuenaga 
11011a8031e1Shsuenaga 	/* Init Software DMA Handlers */
11021a8031e1Shsuenaga 	sc->sc_devmem_desc =
11031a8031e1Shsuenaga 	    mvxpsec_alloc_devmem(sc, 0, PAGE_SIZE * MVXPSEC_DMA_DESC_PAGES);
11041a8031e1Shsuenaga 	ndh = (PAGE_SIZE / sizeof(struct mvxpsec_descriptor))
11051a8031e1Shsuenaga 	    * MVXPSEC_DMA_DESC_PAGES;
11061a8031e1Shsuenaga 	sc->sc_desc_ring =
11071a8031e1Shsuenaga 	    kmem_alloc(sizeof(struct mvxpsec_descriptor_handle) * ndh,
11080a827a3fSchs 	        KM_SLEEP);
11091a8031e1Shsuenaga 	aprint_normal_dev(sc->sc_dev, "%d DMA handles in %zu bytes array\n",
11101a8031e1Shsuenaga 	    ndh, sizeof(struct mvxpsec_descriptor_handle) * ndh);
11111a8031e1Shsuenaga 
11121a8031e1Shsuenaga 	ndh = 0;
11131a8031e1Shsuenaga 	for (seg = 0; seg < devmem_nseg(sc->sc_devmem_desc); seg++) {
11141a8031e1Shsuenaga 		va = devmem_va(sc->sc_devmem_desc);
11151a8031e1Shsuenaga 		pa = devmem_pa(sc->sc_devmem_desc, seg);
11161a8031e1Shsuenaga 		n = devmem_palen(sc->sc_devmem_desc, seg) /
11171a8031e1Shsuenaga 		       	sizeof(struct mvxpsec_descriptor);
11181a8031e1Shsuenaga 		va_off = (PAGE_SIZE * seg);
11191a8031e1Shsuenaga 		pa_off = 0;
11201a8031e1Shsuenaga 		for (i = 0; i < n; i++) {
11211a8031e1Shsuenaga 			dh = &sc->sc_desc_ring[ndh];
11221a8031e1Shsuenaga 			dh->map = devmem_map(sc->sc_devmem_desc);
11231a8031e1Shsuenaga 			dh->off = va_off + pa_off;
11241a8031e1Shsuenaga 			dh->_desc = (void *)(va + va_off + pa_off);
11251a8031e1Shsuenaga 			dh->phys_addr = pa + pa_off;
11261a8031e1Shsuenaga 			pa_off += sizeof(struct mvxpsec_descriptor);
11271a8031e1Shsuenaga 			ndh++;
11281a8031e1Shsuenaga 		}
11291a8031e1Shsuenaga 	}
11301a8031e1Shsuenaga 	sc->sc_desc_ring_size = ndh;
11311a8031e1Shsuenaga 	sc->sc_desc_ring_prod = 0;
11321a8031e1Shsuenaga 	sc->sc_desc_ring_cons = sc->sc_desc_ring_size - 1;
11331a8031e1Shsuenaga 
11341a8031e1Shsuenaga 	return 0;
11351a8031e1Shsuenaga }
11361a8031e1Shsuenaga 
11371a8031e1Shsuenaga /*
11381a8031e1Shsuenaga  * Wait for TDMA controller become idle
11391a8031e1Shsuenaga  */
11401a8031e1Shsuenaga INLINE int
mvxpsec_dma_wait(struct mvxpsec_softc * sc)11411a8031e1Shsuenaga mvxpsec_dma_wait(struct mvxpsec_softc *sc)
11421a8031e1Shsuenaga {
11431a8031e1Shsuenaga 	int retry = 0;
11441a8031e1Shsuenaga 
11451a8031e1Shsuenaga 	while (MVXPSEC_READ(sc, MV_TDMA_CONTROL) & MV_TDMA_CONTROL_ACT) {
11461a8031e1Shsuenaga 		delay(mvxpsec_wait_interval);
11471a8031e1Shsuenaga 		if (retry++ >= mvxpsec_wait_retry)
11481a8031e1Shsuenaga 			return -1;
11491a8031e1Shsuenaga 	}
11501a8031e1Shsuenaga 	return 0;
11511a8031e1Shsuenaga }
11521a8031e1Shsuenaga 
11531a8031e1Shsuenaga /*
11541a8031e1Shsuenaga  * Wait for Security Accelerator become idle
11551a8031e1Shsuenaga  */
11561a8031e1Shsuenaga INLINE int
mvxpsec_acc_wait(struct mvxpsec_softc * sc)11571a8031e1Shsuenaga mvxpsec_acc_wait(struct mvxpsec_softc *sc)
11581a8031e1Shsuenaga {
11591a8031e1Shsuenaga 	int retry = 0;
11601a8031e1Shsuenaga 
11611a8031e1Shsuenaga 	while (MVXPSEC_READ(sc, MV_ACC_COMMAND) & MV_ACC_COMMAND_ACT) {
11621a8031e1Shsuenaga 		delay(mvxpsec_wait_interval);
11631a8031e1Shsuenaga 		if (++retry >= mvxpsec_wait_retry)
11641a8031e1Shsuenaga 			return -1;
11651a8031e1Shsuenaga 	}
11661a8031e1Shsuenaga 	return 0;
11671a8031e1Shsuenaga }
11681a8031e1Shsuenaga 
11691a8031e1Shsuenaga /*
11701a8031e1Shsuenaga  * Entry of interrupt handler
11711a8031e1Shsuenaga  *
11721a8031e1Shsuenaga  * register this to kernel via marvell_intr_establish()
11731a8031e1Shsuenaga  */
11741a8031e1Shsuenaga int
mvxpsec_intr(void * arg)11751a8031e1Shsuenaga mvxpsec_intr(void *arg)
11761a8031e1Shsuenaga {
11771a8031e1Shsuenaga 	struct mvxpsec_softc *sc = arg;
11781a8031e1Shsuenaga 	uint32_t v;
11791a8031e1Shsuenaga 
11801a8031e1Shsuenaga 	/* IPL_NET */
11811a8031e1Shsuenaga 	while ((v = mvxpsec_intr_ack(sc)) != 0) {
11821a8031e1Shsuenaga 		mvxpsec_intr_cnt(sc, v);
11831a8031e1Shsuenaga 		MVXPSEC_PRINTF(MVXPSEC_DEBUG_INTR, "MVXPSEC Intr 0x%08x\n", v);
11841a8031e1Shsuenaga 		MVXPSEC_PRINTF(MVXPSEC_DEBUG_INTR, "%s\n", s_xpsecintr(v));
11851a8031e1Shsuenaga #ifdef MVXPSEC_DEBUG
11861a8031e1Shsuenaga 		mvxpsec_dump_reg(sc);
11871a8031e1Shsuenaga #endif
11881a8031e1Shsuenaga 
11891a8031e1Shsuenaga 		/* call high-level handlers */
11901a8031e1Shsuenaga 		if (v & MVXPSEC_INT_ACCTDMA)
11911a8031e1Shsuenaga 			mvxpsec_done(sc);
11921a8031e1Shsuenaga 	}
11931a8031e1Shsuenaga 
11941a8031e1Shsuenaga 	return 0;
11951a8031e1Shsuenaga }
11961a8031e1Shsuenaga 
11971a8031e1Shsuenaga INLINE void
mvxpsec_intr_cleanup(struct mvxpsec_softc * sc)11981a8031e1Shsuenaga mvxpsec_intr_cleanup(struct mvxpsec_softc *sc)
11991a8031e1Shsuenaga {
12001a8031e1Shsuenaga 	struct mvxpsec_packet *mv_p;
12011a8031e1Shsuenaga 
12021a8031e1Shsuenaga 	/* must called with sc->sc_dma_mtx held */
12031a8031e1Shsuenaga 	KASSERT(mutex_owned(&sc->sc_dma_mtx));
12041a8031e1Shsuenaga 
12051a8031e1Shsuenaga 	/*
12061a8031e1Shsuenaga 	 * there is only one intr for run_queue.
12071a8031e1Shsuenaga 	 * no one touch sc_run_queue.
12081a8031e1Shsuenaga 	 */
12091a8031e1Shsuenaga 	SIMPLEQ_FOREACH(mv_p, &sc->sc_run_queue, queue)
12101a8031e1Shsuenaga 		mvxpsec_dma_free(sc, &mv_p->dma_ring);
12111a8031e1Shsuenaga }
12121a8031e1Shsuenaga 
12131a8031e1Shsuenaga /*
12141a8031e1Shsuenaga  * Acknowledge to interrupt
12151a8031e1Shsuenaga  *
12161a8031e1Shsuenaga  * read cause bits, clear it, and return it.
12171a8031e1Shsuenaga  * NOTE: multiple cause bits may be returned at once.
12181a8031e1Shsuenaga  */
12191a8031e1Shsuenaga STATIC uint32_t
mvxpsec_intr_ack(struct mvxpsec_softc * sc)12201a8031e1Shsuenaga mvxpsec_intr_ack(struct mvxpsec_softc *sc)
12211a8031e1Shsuenaga {
12221a8031e1Shsuenaga 	uint32_t reg;
12231a8031e1Shsuenaga 
12241a8031e1Shsuenaga 	reg  = MVXPSEC_READ(sc, MVXPSEC_INT_CAUSE);
12251a8031e1Shsuenaga 	reg &= MVXPSEC_DEFAULT_INT;
12261a8031e1Shsuenaga 	MVXPSEC_WRITE(sc, MVXPSEC_INT_CAUSE, ~reg);
12271a8031e1Shsuenaga 	MVXPSEC_PRINTF(MVXPSEC_DEBUG_INTR, "Int: %s\n", s_xpsecintr(reg));
12281a8031e1Shsuenaga 
12291a8031e1Shsuenaga 	return reg;
12301a8031e1Shsuenaga }
12311a8031e1Shsuenaga 
12321a8031e1Shsuenaga /*
12331a8031e1Shsuenaga  * Entry of TDMA error interrupt handler
12341a8031e1Shsuenaga  *
12351a8031e1Shsuenaga  * register this to kernel via marvell_intr_establish()
12361a8031e1Shsuenaga  */
12371a8031e1Shsuenaga int
mvxpsec_eintr(void * arg)12381a8031e1Shsuenaga mvxpsec_eintr(void *arg)
12391a8031e1Shsuenaga {
12401a8031e1Shsuenaga 	struct mvxpsec_softc *sc = arg;
12411a8031e1Shsuenaga 	uint32_t err;
12421a8031e1Shsuenaga 
12431a8031e1Shsuenaga 	/* IPL_NET */
12441a8031e1Shsuenaga again:
12451a8031e1Shsuenaga 	err = mvxpsec_eintr_ack(sc);
12461a8031e1Shsuenaga 	if (err == 0)
12471a8031e1Shsuenaga 		goto done;
12481a8031e1Shsuenaga 
12491a8031e1Shsuenaga 	log(LOG_ERR, "%s: DMA Error Interrupt: %s\n", __func__,
12501a8031e1Shsuenaga 	    s_errreg(err));
12511a8031e1Shsuenaga #ifdef MVXPSEC_DEBUG
12521a8031e1Shsuenaga 	mvxpsec_dump_reg(sc);
12531a8031e1Shsuenaga #endif
12541a8031e1Shsuenaga 
12551a8031e1Shsuenaga 	goto again;
12561a8031e1Shsuenaga done:
12571a8031e1Shsuenaga 	return 0;
12581a8031e1Shsuenaga }
12591a8031e1Shsuenaga 
12601a8031e1Shsuenaga /*
12611a8031e1Shsuenaga  * Acknowledge to TDMA error interrupt
12621a8031e1Shsuenaga  *
12631a8031e1Shsuenaga  * read cause bits, clear it, and return it.
12641a8031e1Shsuenaga  * NOTE: multiple cause bits may be returned at once.
12651a8031e1Shsuenaga  */
12661a8031e1Shsuenaga STATIC uint32_t
mvxpsec_eintr_ack(struct mvxpsec_softc * sc)12671a8031e1Shsuenaga mvxpsec_eintr_ack(struct mvxpsec_softc *sc)
12681a8031e1Shsuenaga {
12691a8031e1Shsuenaga 	uint32_t reg;
12701a8031e1Shsuenaga 
12711a8031e1Shsuenaga 	reg  = MVXPSEC_READ(sc, MV_TDMA_ERR_CAUSE);
12721a8031e1Shsuenaga 	reg &= MVXPSEC_DEFAULT_ERR;
12731a8031e1Shsuenaga 	MVXPSEC_WRITE(sc, MV_TDMA_ERR_CAUSE, ~reg);
12741a8031e1Shsuenaga 	MVXPSEC_PRINTF(MVXPSEC_DEBUG_INTR, "Int: %s\n", s_xpsecintr(reg));
12751a8031e1Shsuenaga 
12761a8031e1Shsuenaga 	return reg;
12771a8031e1Shsuenaga }
12781a8031e1Shsuenaga 
12791a8031e1Shsuenaga /*
12801a8031e1Shsuenaga  * Interrupt statistics
12811a8031e1Shsuenaga  *
1282cbf5c65aSandvar  * this is NOT a statistics of how many times the events 'occurred'.
12831a8031e1Shsuenaga  * this ONLY means how many times the events 'handled'.
12841a8031e1Shsuenaga  */
12851a8031e1Shsuenaga INLINE void
mvxpsec_intr_cnt(struct mvxpsec_softc * sc,int cause)12861a8031e1Shsuenaga mvxpsec_intr_cnt(struct mvxpsec_softc *sc, int cause)
12871a8031e1Shsuenaga {
12881a8031e1Shsuenaga 	MVXPSEC_EVCNT_INCR(sc, intr_all);
12891a8031e1Shsuenaga 	if (cause & MVXPSEC_INT_AUTH)
12901a8031e1Shsuenaga 		MVXPSEC_EVCNT_INCR(sc, intr_auth);
12911a8031e1Shsuenaga 	if (cause & MVXPSEC_INT_DES)
12921a8031e1Shsuenaga 		MVXPSEC_EVCNT_INCR(sc, intr_des);
12931a8031e1Shsuenaga 	if (cause & MVXPSEC_INT_AES_ENC)
12941a8031e1Shsuenaga 		MVXPSEC_EVCNT_INCR(sc, intr_aes_enc);
12951a8031e1Shsuenaga 	if (cause & MVXPSEC_INT_AES_DEC)
12961a8031e1Shsuenaga 		MVXPSEC_EVCNT_INCR(sc, intr_aes_dec);
12971a8031e1Shsuenaga 	if (cause & MVXPSEC_INT_ENC)
12981a8031e1Shsuenaga 		MVXPSEC_EVCNT_INCR(sc, intr_enc);
12991a8031e1Shsuenaga 	if (cause & MVXPSEC_INT_SA)
13001a8031e1Shsuenaga 		MVXPSEC_EVCNT_INCR(sc, intr_sa);
13011a8031e1Shsuenaga 	if (cause & MVXPSEC_INT_ACCTDMA)
13021a8031e1Shsuenaga 		MVXPSEC_EVCNT_INCR(sc, intr_acctdma);
13031a8031e1Shsuenaga 	if (cause & MVXPSEC_INT_TDMA_COMP)
13041a8031e1Shsuenaga 		MVXPSEC_EVCNT_INCR(sc, intr_comp);
13051a8031e1Shsuenaga 	if (cause & MVXPSEC_INT_TDMA_OWN)
13061a8031e1Shsuenaga 		MVXPSEC_EVCNT_INCR(sc, intr_own);
13071a8031e1Shsuenaga 	if (cause & MVXPSEC_INT_ACCTDMA_CONT)
13081a8031e1Shsuenaga 		MVXPSEC_EVCNT_INCR(sc, intr_acctdma_cont);
13091a8031e1Shsuenaga }
13101a8031e1Shsuenaga 
13111a8031e1Shsuenaga /*
13121a8031e1Shsuenaga  * Setup MVXPSEC header structure.
13131a8031e1Shsuenaga  *
13141a8031e1Shsuenaga  * the header contains descriptor of security accelerator,
1315ab9bc2adSandvar  * key material of ciphers, iv of ciphers and macs, ...
13161a8031e1Shsuenaga  *
1317a0403cdeSmsaitoh  * the header is transferred to MVXPSEC Internal SRAM by TDMA,
13181a8031e1Shsuenaga  * and parsed by MVXPSEC H/W.
13191a8031e1Shsuenaga  */
13201a8031e1Shsuenaga STATIC int
mvxpsec_header_finalize(struct mvxpsec_packet * mv_p)13211a8031e1Shsuenaga mvxpsec_header_finalize(struct mvxpsec_packet *mv_p)
13221a8031e1Shsuenaga {
13231a8031e1Shsuenaga 	struct mvxpsec_acc_descriptor *desc = &mv_p->pkt_header.desc;
13241a8031e1Shsuenaga 	int enc_start, enc_len, iv_offset;
13251a8031e1Shsuenaga 	int mac_start, mac_len, mac_offset;
13261a8031e1Shsuenaga 
13271a8031e1Shsuenaga 	/* offset -> device address */
13281a8031e1Shsuenaga 	enc_start = MVXPSEC_SRAM_PAYLOAD_DA(mv_p->enc_off);
13291a8031e1Shsuenaga 	enc_len = mv_p->enc_len;
13301a8031e1Shsuenaga 	if (mv_p->flags & CRP_EXT_IV)
13311a8031e1Shsuenaga 		iv_offset = mv_p->enc_ivoff;
13321a8031e1Shsuenaga 	else
13331a8031e1Shsuenaga 		iv_offset = MVXPSEC_SRAM_PAYLOAD_DA(mv_p->enc_ivoff);
13341a8031e1Shsuenaga 	mac_start = MVXPSEC_SRAM_PAYLOAD_DA(mv_p->mac_off);
13351a8031e1Shsuenaga 	mac_len = mv_p->mac_len;
13361a8031e1Shsuenaga 	mac_offset = MVXPSEC_SRAM_PAYLOAD_DA(mv_p->mac_dst);
13371a8031e1Shsuenaga 
13381a8031e1Shsuenaga 	MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO,
13391a8031e1Shsuenaga 	    "PAYLOAD at 0x%08x\n", (int)MVXPSEC_SRAM_PAYLOAD_OFF);
13401a8031e1Shsuenaga 	MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO,
13411a8031e1Shsuenaga 	    "ENC from 0x%08x\n", enc_start);
13421a8031e1Shsuenaga 	MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO,
13431a8031e1Shsuenaga 	    "MAC from 0x%08x\n", mac_start);
13441a8031e1Shsuenaga 	MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO,
13451a8031e1Shsuenaga 	    "MAC to 0x%08x\n", mac_offset);
13461a8031e1Shsuenaga 	MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO,
13471a8031e1Shsuenaga 	    "ENC IV at 0x%08x\n", iv_offset);
13481a8031e1Shsuenaga 
13491a8031e1Shsuenaga 	/* setup device addresses in Security Accelerator Descriptors */
13501a8031e1Shsuenaga 	desc->acc_encdata = MV_ACC_DESC_ENC_DATA(enc_start, enc_start);
13511a8031e1Shsuenaga 	desc->acc_enclen = MV_ACC_DESC_ENC_LEN(enc_len);
13521a8031e1Shsuenaga 	if (desc->acc_config & MV_ACC_CRYPTO_DECRYPT)
13531a8031e1Shsuenaga 		desc->acc_enckey =
13541a8031e1Shsuenaga 		    MV_ACC_DESC_ENC_KEY(MVXPSEC_SRAM_KEY_D_DA);
13551a8031e1Shsuenaga 	else
13561a8031e1Shsuenaga 		desc->acc_enckey =
13571a8031e1Shsuenaga 		    MV_ACC_DESC_ENC_KEY(MVXPSEC_SRAM_KEY_DA);
13581a8031e1Shsuenaga 	desc->acc_enciv =
13591a8031e1Shsuenaga 	    MV_ACC_DESC_ENC_IV(MVXPSEC_SRAM_IV_WORK_DA, iv_offset);
13601a8031e1Shsuenaga 
13611a8031e1Shsuenaga 	desc->acc_macsrc = MV_ACC_DESC_MAC_SRC(mac_start, mac_len);
13621a8031e1Shsuenaga 	desc->acc_macdst = MV_ACC_DESC_MAC_DST(mac_offset, mac_len);
13631a8031e1Shsuenaga 	desc->acc_maciv =
13641a8031e1Shsuenaga 	    MV_ACC_DESC_MAC_IV(MVXPSEC_SRAM_MIV_IN_DA,
13651a8031e1Shsuenaga 	        MVXPSEC_SRAM_MIV_OUT_DA);
13661a8031e1Shsuenaga 
13671a8031e1Shsuenaga 	return 0;
13681a8031e1Shsuenaga }
13691a8031e1Shsuenaga 
13701a8031e1Shsuenaga /*
13711a8031e1Shsuenaga  * constractor of session structure.
13721a8031e1Shsuenaga  *
13731a8031e1Shsuenaga  * this constrator will be called by pool_cache framework.
13741a8031e1Shsuenaga  */
13751a8031e1Shsuenaga STATIC int
mvxpsec_session_ctor(void * arg,void * obj,int flags)13761a8031e1Shsuenaga mvxpsec_session_ctor(void *arg, void *obj, int flags)
13771a8031e1Shsuenaga {
13781a8031e1Shsuenaga 	struct mvxpsec_softc *sc = arg;
13791a8031e1Shsuenaga 	struct mvxpsec_session *mv_s = obj;
13801a8031e1Shsuenaga 
13811a8031e1Shsuenaga 	/* pool is owned by softc */
13821a8031e1Shsuenaga 	mv_s->sc = sc;
13831a8031e1Shsuenaga 
13841a8031e1Shsuenaga 	/* Create and load DMA map for session header */
13851a8031e1Shsuenaga 	mv_s->session_header_map = 0;
13861a8031e1Shsuenaga 	if (bus_dmamap_create(sc->sc_dmat,
13871a8031e1Shsuenaga 	    sizeof(mv_s->session_header), 1,
13881a8031e1Shsuenaga 	    sizeof(mv_s->session_header), 0,
13891a8031e1Shsuenaga 	    BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW,
13901a8031e1Shsuenaga 	    &mv_s->session_header_map)) {
13911a8031e1Shsuenaga 		log(LOG_ERR, "%s: cannot create DMA map\n", __func__);
13921a8031e1Shsuenaga 		goto fail;
13931a8031e1Shsuenaga 	}
13941a8031e1Shsuenaga 	if (bus_dmamap_load(sc->sc_dmat, mv_s->session_header_map,
13951a8031e1Shsuenaga 	    &mv_s->session_header, sizeof(mv_s->session_header),
13961a8031e1Shsuenaga 	    NULL, BUS_DMA_NOWAIT)) {
13971a8031e1Shsuenaga 		log(LOG_ERR, "%s: cannot load header\n", __func__);
13981a8031e1Shsuenaga 		goto fail;
13991a8031e1Shsuenaga 	}
14001a8031e1Shsuenaga 
14011a8031e1Shsuenaga 	return 0;
14021a8031e1Shsuenaga fail:
14031a8031e1Shsuenaga 	if (mv_s->session_header_map)
14041a8031e1Shsuenaga 		bus_dmamap_destroy(sc->sc_dmat, mv_s->session_header_map);
14051a8031e1Shsuenaga 	return ENOMEM;
14061a8031e1Shsuenaga }
14071a8031e1Shsuenaga 
14081a8031e1Shsuenaga /*
14091a8031e1Shsuenaga  * destractor of session structure.
14101a8031e1Shsuenaga  *
14111a8031e1Shsuenaga  * this destrator will be called by pool_cache framework.
14121a8031e1Shsuenaga  */
14131a8031e1Shsuenaga STATIC void
mvxpsec_session_dtor(void * arg,void * obj)14141a8031e1Shsuenaga mvxpsec_session_dtor(void *arg, void *obj)
14151a8031e1Shsuenaga {
14161a8031e1Shsuenaga 	struct mvxpsec_softc *sc = arg;
14171a8031e1Shsuenaga 	struct mvxpsec_session *mv_s = obj;
14181a8031e1Shsuenaga 
14191a8031e1Shsuenaga 	if (mv_s->sc != sc)
14201a8031e1Shsuenaga 		panic("inconsitent context\n");
14211a8031e1Shsuenaga 
14221a8031e1Shsuenaga 	bus_dmamap_destroy(sc->sc_dmat, mv_s->session_header_map);
14231a8031e1Shsuenaga }
14241a8031e1Shsuenaga 
14251a8031e1Shsuenaga /*
14261a8031e1Shsuenaga  * constructor of packet structure.
14271a8031e1Shsuenaga  */
14281a8031e1Shsuenaga STATIC int
mvxpsec_packet_ctor(void * arg,void * obj,int flags)14291a8031e1Shsuenaga mvxpsec_packet_ctor(void *arg, void *obj, int flags)
14301a8031e1Shsuenaga {
14311a8031e1Shsuenaga 	struct mvxpsec_softc *sc = arg;
14321a8031e1Shsuenaga 	struct mvxpsec_packet *mv_p = obj;
14331a8031e1Shsuenaga 
14341a8031e1Shsuenaga 	mv_p->dma_ring.dma_head = NULL;
14351a8031e1Shsuenaga 	mv_p->dma_ring.dma_last = NULL;
14361a8031e1Shsuenaga 	mv_p->dma_ring.dma_size = 0;
14371a8031e1Shsuenaga 
14381a8031e1Shsuenaga 	/* Create and load DMA map for packet header */
14391a8031e1Shsuenaga 	mv_p->pkt_header_map = 0;
14401a8031e1Shsuenaga 	if (bus_dmamap_create(sc->sc_dmat,
14411a8031e1Shsuenaga 	    sizeof(mv_p->pkt_header), 1, sizeof(mv_p->pkt_header), 0,
14421a8031e1Shsuenaga 	    BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW,
14431a8031e1Shsuenaga 	    &mv_p->pkt_header_map)) {
14441a8031e1Shsuenaga 		log(LOG_ERR, "%s: cannot create DMA map\n", __func__);
14451a8031e1Shsuenaga 		goto fail;
14461a8031e1Shsuenaga 	}
14471a8031e1Shsuenaga 	if (bus_dmamap_load(sc->sc_dmat, mv_p->pkt_header_map,
14481a8031e1Shsuenaga 	    &mv_p->pkt_header, sizeof(mv_p->pkt_header),
14491a8031e1Shsuenaga 	    NULL, BUS_DMA_NOWAIT)) {
14501a8031e1Shsuenaga 		log(LOG_ERR, "%s: cannot load header\n", __func__);
14511a8031e1Shsuenaga 		goto fail;
14521a8031e1Shsuenaga 	}
14531a8031e1Shsuenaga 
14541a8031e1Shsuenaga 	/* Create DMA map for session data. */
14551a8031e1Shsuenaga 	mv_p->data_map = 0;
14561a8031e1Shsuenaga 	if (bus_dmamap_create(sc->sc_dmat,
14571a8031e1Shsuenaga 	    MVXPSEC_DMA_MAX_SIZE, MVXPSEC_DMA_MAX_SEGS, MVXPSEC_DMA_MAX_SIZE,
14581a8031e1Shsuenaga 	    0, BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW, &mv_p->data_map)) {
14591a8031e1Shsuenaga 		log(LOG_ERR, "%s: cannot create DMA map\n", __func__);
14601a8031e1Shsuenaga 		goto fail;
14611a8031e1Shsuenaga 	}
14621a8031e1Shsuenaga 
14631a8031e1Shsuenaga 	return 0;
14641a8031e1Shsuenaga fail:
14651a8031e1Shsuenaga 	if (mv_p->pkt_header_map)
14661a8031e1Shsuenaga 		bus_dmamap_destroy(sc->sc_dmat, mv_p->pkt_header_map);
14671a8031e1Shsuenaga 	if (mv_p->data_map)
14681a8031e1Shsuenaga 		bus_dmamap_destroy(sc->sc_dmat, mv_p->data_map);
14691a8031e1Shsuenaga 	return ENOMEM;
14701a8031e1Shsuenaga }
14711a8031e1Shsuenaga 
14721a8031e1Shsuenaga /*
14731a8031e1Shsuenaga  * destractor of packet structure.
14741a8031e1Shsuenaga  */
14751a8031e1Shsuenaga STATIC void
mvxpsec_packet_dtor(void * arg,void * obj)14761a8031e1Shsuenaga mvxpsec_packet_dtor(void *arg, void *obj)
14771a8031e1Shsuenaga {
14781a8031e1Shsuenaga 	struct mvxpsec_softc *sc = arg;
14791a8031e1Shsuenaga 	struct mvxpsec_packet *mv_p = obj;
14801a8031e1Shsuenaga 
14811a8031e1Shsuenaga 	mutex_enter(&sc->sc_dma_mtx);
14821a8031e1Shsuenaga 	mvxpsec_dma_free(sc, &mv_p->dma_ring);
14831a8031e1Shsuenaga 	mutex_exit(&sc->sc_dma_mtx);
14841a8031e1Shsuenaga 	bus_dmamap_destroy(sc->sc_dmat, mv_p->pkt_header_map);
14851a8031e1Shsuenaga 	bus_dmamap_destroy(sc->sc_dmat, mv_p->data_map);
14861a8031e1Shsuenaga }
14871a8031e1Shsuenaga 
14881a8031e1Shsuenaga /*
1489c74fd0bbSandvar  * allocate new session structure.
14901a8031e1Shsuenaga  */
14911a8031e1Shsuenaga STATIC struct mvxpsec_session *
mvxpsec_session_alloc(struct mvxpsec_softc * sc)14921a8031e1Shsuenaga mvxpsec_session_alloc(struct mvxpsec_softc *sc)
14931a8031e1Shsuenaga {
14941a8031e1Shsuenaga 	struct mvxpsec_session *mv_s;
14951a8031e1Shsuenaga 
1496127c64cbSchristos 	mv_s = pool_cache_get(sc->sc_session_pool, PR_NOWAIT);
14971a8031e1Shsuenaga 	if (mv_s == NULL) {
14981a8031e1Shsuenaga 		log(LOG_ERR, "%s: cannot allocate memory\n", __func__);
14991a8031e1Shsuenaga 		return NULL;
15001a8031e1Shsuenaga 	}
1501afda606fSskrll 	mv_s->refs = 1; /* 0 means session is already invalid */
15021a8031e1Shsuenaga 	mv_s->sflags = 0;
15031a8031e1Shsuenaga 
15041a8031e1Shsuenaga 	return mv_s;
15051a8031e1Shsuenaga }
15061a8031e1Shsuenaga 
15071a8031e1Shsuenaga /*
15081a8031e1Shsuenaga  * deallocate session structure.
15091a8031e1Shsuenaga  */
15101a8031e1Shsuenaga STATIC void
mvxpsec_session_dealloc(struct mvxpsec_session * mv_s)15111a8031e1Shsuenaga mvxpsec_session_dealloc(struct mvxpsec_session *mv_s)
15121a8031e1Shsuenaga {
15131a8031e1Shsuenaga 	struct mvxpsec_softc *sc = mv_s->sc;
15141a8031e1Shsuenaga 
15151a8031e1Shsuenaga 	mv_s->sflags |= DELETED;
15161a8031e1Shsuenaga 	mvxpsec_session_unref(mv_s);
15171a8031e1Shsuenaga 	crypto_unblock(sc->sc_cid, CRYPTO_SYMQ|CRYPTO_ASYMQ);
15181a8031e1Shsuenaga 
15191a8031e1Shsuenaga 	return;
15201a8031e1Shsuenaga }
15211a8031e1Shsuenaga 
15221a8031e1Shsuenaga STATIC int
mvxpsec_session_ref(struct mvxpsec_session * mv_s)15231a8031e1Shsuenaga mvxpsec_session_ref(struct mvxpsec_session *mv_s)
15241a8031e1Shsuenaga {
15251a8031e1Shsuenaga 	uint32_t refs;
15261a8031e1Shsuenaga 
15271a8031e1Shsuenaga 	if (mv_s->sflags & DELETED) {
15281a8031e1Shsuenaga 		log(LOG_ERR,
15291a8031e1Shsuenaga 		    "%s: session is already deleted.\n", __func__);
15301a8031e1Shsuenaga 		return -1;
15311a8031e1Shsuenaga 	}
15321a8031e1Shsuenaga 
15331a8031e1Shsuenaga 	refs = atomic_inc_32_nv(&mv_s->refs);
15341a8031e1Shsuenaga 	if (refs == 1) {
15351a8031e1Shsuenaga 		/*
15361a8031e1Shsuenaga 		 * a session with refs == 0 is
15371a8031e1Shsuenaga 		 * already invalidated. revert it.
15381a8031e1Shsuenaga 		 * XXX: use CAS ?
15391a8031e1Shsuenaga 		 */
15401a8031e1Shsuenaga 		atomic_dec_32(&mv_s->refs);
15411a8031e1Shsuenaga 		log(LOG_ERR,
15421a8031e1Shsuenaga 		    "%s: session is already invalidated.\n", __func__);
15431a8031e1Shsuenaga 		return -1;
15441a8031e1Shsuenaga 	}
15451a8031e1Shsuenaga 
15461a8031e1Shsuenaga 	return 0;
15471a8031e1Shsuenaga }
15481a8031e1Shsuenaga 
15491a8031e1Shsuenaga STATIC void
mvxpsec_session_unref(struct mvxpsec_session * mv_s)15501a8031e1Shsuenaga mvxpsec_session_unref(struct mvxpsec_session *mv_s)
15511a8031e1Shsuenaga {
15521a8031e1Shsuenaga 	uint32_t refs;
15531a8031e1Shsuenaga 
1554ef3476fbSriastradh 	membar_release();
15551a8031e1Shsuenaga 	refs = atomic_dec_32_nv(&mv_s->refs);
1556122a3e8aSriastradh 	if (refs == 0) {
1557ef3476fbSriastradh 		membar_acquire();
15581a8031e1Shsuenaga 		pool_cache_put(mv_s->sc->sc_session_pool, mv_s);
15591a8031e1Shsuenaga 	}
1560122a3e8aSriastradh }
15611a8031e1Shsuenaga 
15621a8031e1Shsuenaga /*
15631a8031e1Shsuenaga  * look for session is exist or not
15641a8031e1Shsuenaga  */
15651a8031e1Shsuenaga INLINE struct mvxpsec_session *
mvxpsec_session_lookup(struct mvxpsec_softc * sc,int sid)15661a8031e1Shsuenaga mvxpsec_session_lookup(struct mvxpsec_softc *sc, int sid)
15671a8031e1Shsuenaga {
15681a8031e1Shsuenaga 	struct mvxpsec_session *mv_s;
15691a8031e1Shsuenaga 	int session;
15701a8031e1Shsuenaga 
15711a8031e1Shsuenaga 	/* must called sc->sc_session_mtx held */
15721a8031e1Shsuenaga 	KASSERT(mutex_owned(&sc->sc_session_mtx));
15731a8031e1Shsuenaga 
15741a8031e1Shsuenaga 	session = MVXPSEC_SESSION(sid);
15751a8031e1Shsuenaga 	if (__predict_false(session > MVXPSEC_MAX_SESSIONS)) {
15761a8031e1Shsuenaga 		log(LOG_ERR, "%s: session number too large %d\n",
15771a8031e1Shsuenaga 		    __func__, session);
15781a8031e1Shsuenaga 		return NULL;
15791a8031e1Shsuenaga 	}
15801a8031e1Shsuenaga 	if (__predict_false( (mv_s = sc->sc_sessions[session]) == NULL)) {
15811a8031e1Shsuenaga 		log(LOG_ERR, "%s: invalid session %d\n",
15821a8031e1Shsuenaga 		    __func__, session);
15831a8031e1Shsuenaga 		return NULL;
15841a8031e1Shsuenaga 	}
15851a8031e1Shsuenaga 
15861a8031e1Shsuenaga 	KASSERT(mv_s->sid == session);
15871a8031e1Shsuenaga 
15881a8031e1Shsuenaga 	return mv_s;
15891a8031e1Shsuenaga }
15901a8031e1Shsuenaga 
15911a8031e1Shsuenaga /*
15921a8031e1Shsuenaga  * allocation new packet structure.
15931a8031e1Shsuenaga  */
15941a8031e1Shsuenaga STATIC struct mvxpsec_packet *
mvxpsec_packet_alloc(struct mvxpsec_session * mv_s)15951a8031e1Shsuenaga mvxpsec_packet_alloc(struct mvxpsec_session *mv_s)
15961a8031e1Shsuenaga {
15971a8031e1Shsuenaga 	struct mvxpsec_softc *sc = mv_s->sc;
15981a8031e1Shsuenaga 	struct mvxpsec_packet *mv_p;
15991a8031e1Shsuenaga 
16001a8031e1Shsuenaga 	/* must be called mv_queue_mtx held. */
16011a8031e1Shsuenaga 	KASSERT(mutex_owned(&sc->sc_queue_mtx));
16021a8031e1Shsuenaga 	/* must be called mv_session_mtx held. */
16031a8031e1Shsuenaga 	KASSERT(mutex_owned(&sc->sc_session_mtx));
16041a8031e1Shsuenaga 
16051a8031e1Shsuenaga 	if (mvxpsec_session_ref(mv_s) < 0) {
16061a8031e1Shsuenaga 		log(LOG_ERR, "%s: invalid session.\n", __func__);
16071a8031e1Shsuenaga 		return NULL;
16081a8031e1Shsuenaga 	}
16091a8031e1Shsuenaga 
16101a8031e1Shsuenaga 	if ( (mv_p = SLIST_FIRST(&sc->sc_free_list)) != NULL) {
16111a8031e1Shsuenaga 		SLIST_REMOVE_HEAD(&sc->sc_free_list, free_list);
16121a8031e1Shsuenaga 		sc->sc_free_qlen--;
16131a8031e1Shsuenaga 	}
16141a8031e1Shsuenaga 	else {
1615127c64cbSchristos 		mv_p = pool_cache_get(sc->sc_packet_pool, PR_NOWAIT);
16161a8031e1Shsuenaga 		if (mv_p == NULL) {
16171a8031e1Shsuenaga 			log(LOG_ERR, "%s: cannot allocate memory\n",
16181a8031e1Shsuenaga 			    __func__);
16191a8031e1Shsuenaga 			mvxpsec_session_unref(mv_s);
16201a8031e1Shsuenaga 			return NULL;
16211a8031e1Shsuenaga 		}
16221a8031e1Shsuenaga 	}
16231a8031e1Shsuenaga 	mv_p->mv_s = mv_s;
16241a8031e1Shsuenaga 	mv_p->flags = 0;
16251a8031e1Shsuenaga 	mv_p->data_ptr = NULL;
16261a8031e1Shsuenaga 
16271a8031e1Shsuenaga 	return mv_p;
16281a8031e1Shsuenaga }
16291a8031e1Shsuenaga 
16301a8031e1Shsuenaga /*
16311a8031e1Shsuenaga  * free packet structure.
16321a8031e1Shsuenaga  */
16331a8031e1Shsuenaga STATIC void
mvxpsec_packet_dealloc(struct mvxpsec_packet * mv_p)16341a8031e1Shsuenaga mvxpsec_packet_dealloc(struct mvxpsec_packet *mv_p)
16351a8031e1Shsuenaga {
16361a8031e1Shsuenaga 	struct mvxpsec_session *mv_s = mv_p->mv_s;
16371a8031e1Shsuenaga 	struct mvxpsec_softc *sc = mv_s->sc;
16381a8031e1Shsuenaga 
16391a8031e1Shsuenaga 	/* must called with sc->sc_queue_mtx held */
16401a8031e1Shsuenaga 	KASSERT(mutex_owned(&sc->sc_queue_mtx));
16411a8031e1Shsuenaga 
16421a8031e1Shsuenaga 	if (mv_p->dma_ring.dma_size != 0) {
16431a8031e1Shsuenaga 		sc->sc_desc_ring_cons += mv_p->dma_ring.dma_size;
16441a8031e1Shsuenaga 	}
16451a8031e1Shsuenaga 	mv_p->dma_ring.dma_head = NULL;
16461a8031e1Shsuenaga 	mv_p->dma_ring.dma_last = NULL;
16471a8031e1Shsuenaga 	mv_p->dma_ring.dma_size = 0;
16481a8031e1Shsuenaga 
16491a8031e1Shsuenaga 	if (mv_p->data_map) {
16501a8031e1Shsuenaga 		if (mv_p->flags & RDY_DATA) {
16511a8031e1Shsuenaga 			bus_dmamap_unload(sc->sc_dmat, mv_p->data_map);
16521a8031e1Shsuenaga 			mv_p->flags &= ~RDY_DATA;
16531a8031e1Shsuenaga 		}
16541a8031e1Shsuenaga 	}
16551a8031e1Shsuenaga 
16561a8031e1Shsuenaga 	if (sc->sc_free_qlen > sc->sc_wait_qlimit)
16571a8031e1Shsuenaga 		pool_cache_put(sc->sc_packet_pool, mv_p);
16581a8031e1Shsuenaga 	else {
16591a8031e1Shsuenaga 		SLIST_INSERT_HEAD(&sc->sc_free_list, mv_p, free_list);
16601a8031e1Shsuenaga 		sc->sc_free_qlen++;
16611a8031e1Shsuenaga 	}
16621a8031e1Shsuenaga 	mvxpsec_session_unref(mv_s);
16631a8031e1Shsuenaga }
16641a8031e1Shsuenaga 
16651a8031e1Shsuenaga INLINE void
mvxpsec_packet_enqueue(struct mvxpsec_packet * mv_p)16661a8031e1Shsuenaga mvxpsec_packet_enqueue(struct mvxpsec_packet *mv_p)
16671a8031e1Shsuenaga {
16681a8031e1Shsuenaga 	struct mvxpsec_softc *sc = mv_p->mv_s->sc;
16691a8031e1Shsuenaga 	struct mvxpsec_packet *last_packet;
16701a8031e1Shsuenaga 	struct mvxpsec_descriptor_handle *cur_dma, *prev_dma;
16711a8031e1Shsuenaga 
16721a8031e1Shsuenaga 	/* must called with sc->sc_queue_mtx held */
16731a8031e1Shsuenaga 	KASSERT(mutex_owned(&sc->sc_queue_mtx));
16741a8031e1Shsuenaga 
16751a8031e1Shsuenaga 	if (sc->sc_wait_qlen == 0) {
16761a8031e1Shsuenaga 		SIMPLEQ_INSERT_TAIL(&sc->sc_wait_queue, mv_p, queue);
16771a8031e1Shsuenaga 		sc->sc_wait_qlen++;
16781a8031e1Shsuenaga 		mv_p->flags |= SETUP_DONE;
16791a8031e1Shsuenaga 		return;
16801a8031e1Shsuenaga 	}
16811a8031e1Shsuenaga 
16821a8031e1Shsuenaga 	last_packet = SIMPLEQ_LAST(&sc->sc_wait_queue, mvxpsec_packet, queue);
16831a8031e1Shsuenaga 	SIMPLEQ_INSERT_TAIL(&sc->sc_wait_queue, mv_p, queue);
16841a8031e1Shsuenaga 	sc->sc_wait_qlen++;
16851a8031e1Shsuenaga 
16861a8031e1Shsuenaga 	/* chain the DMA */
16871a8031e1Shsuenaga 	cur_dma = mv_p->dma_ring.dma_head;
16881a8031e1Shsuenaga 	prev_dma = last_packet->dma_ring.dma_last;
16891a8031e1Shsuenaga 	mvxpsec_dma_cat(sc, prev_dma, cur_dma);
16901a8031e1Shsuenaga 	mv_p->flags |= SETUP_DONE;
16911a8031e1Shsuenaga }
16921a8031e1Shsuenaga 
16931a8031e1Shsuenaga /*
16941a8031e1Shsuenaga  * called by interrupt handler
16951a8031e1Shsuenaga  */
16961a8031e1Shsuenaga STATIC int
mvxpsec_done_packet(struct mvxpsec_packet * mv_p)16971a8031e1Shsuenaga mvxpsec_done_packet(struct mvxpsec_packet *mv_p)
16981a8031e1Shsuenaga {
16991a8031e1Shsuenaga 	struct mvxpsec_session *mv_s = mv_p->mv_s;
17001a8031e1Shsuenaga 	struct mvxpsec_softc *sc = mv_s->sc;
17011a8031e1Shsuenaga 
17021a8031e1Shsuenaga 	KASSERT((mv_p->flags & RDY_DATA));
17031a8031e1Shsuenaga 	KASSERT((mv_p->flags & SETUP_DONE));
17041a8031e1Shsuenaga 
17051a8031e1Shsuenaga 	/* unload data */
17061a8031e1Shsuenaga 	bus_dmamap_sync(sc->sc_dmat, mv_p->data_map,
17071a8031e1Shsuenaga 	    0, mv_p->data_len,
17081a8031e1Shsuenaga 	    BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
17091a8031e1Shsuenaga 	bus_dmamap_unload(sc->sc_dmat, mv_p->data_map);
17101a8031e1Shsuenaga 	mv_p->flags &= ~RDY_DATA;
17111a8031e1Shsuenaga 
17121a8031e1Shsuenaga #ifdef MVXPSEC_DEBUG
17131a8031e1Shsuenaga 	if (mvxpsec_debug != 0) {
17141a8031e1Shsuenaga 		int s;
17151a8031e1Shsuenaga 
17161a8031e1Shsuenaga 		bus_dmamap_sync(sc->sc_dmat, mv_p->pkt_header_map,
17171a8031e1Shsuenaga 		    0, sizeof(mv_p->pkt_header),
17181a8031e1Shsuenaga 		    BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
17191a8031e1Shsuenaga 		bus_dmamap_sync(sc->sc_dmat, mv_s->session_header_map,
17201a8031e1Shsuenaga 		    0, sizeof(mv_s->session_header),
17211a8031e1Shsuenaga 		    BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
17221a8031e1Shsuenaga 
17231a8031e1Shsuenaga 		if (mvxpsec_debug & MVXPSEC_DEBUG_OPENCRYPTO) {
17241a8031e1Shsuenaga 			char buf[1500];
17251a8031e1Shsuenaga 			struct mbuf *m;
17261a8031e1Shsuenaga 			struct uio *uio;
17271a8031e1Shsuenaga 			size_t len;
17281a8031e1Shsuenaga 
17291a8031e1Shsuenaga 			switch (mv_p->data_type) {
17301a8031e1Shsuenaga 			case MVXPSEC_DATA_MBUF:
17311a8031e1Shsuenaga 				m = mv_p->data_mbuf;
17321a8031e1Shsuenaga 				len = m->m_pkthdr.len;
17331a8031e1Shsuenaga 				if (len > sizeof(buf))
17341a8031e1Shsuenaga 					len = sizeof(buf);
17351a8031e1Shsuenaga 				m_copydata(m, 0, len, buf);
17361a8031e1Shsuenaga 				break;
17371a8031e1Shsuenaga 			case MVXPSEC_DATA_UIO:
17381a8031e1Shsuenaga 				uio = mv_p->data_uio;
17391a8031e1Shsuenaga 				len = uio->uio_resid;
17401a8031e1Shsuenaga 				if (len > sizeof(buf))
17411a8031e1Shsuenaga 					len = sizeof(buf);
17421a8031e1Shsuenaga 				cuio_copydata(uio, 0, len, buf);
17431a8031e1Shsuenaga 				break;
17441a8031e1Shsuenaga 			default:
17451a8031e1Shsuenaga 				len = 0;
17461a8031e1Shsuenaga 			}
17471a8031e1Shsuenaga 			if (len > 0)
17481a8031e1Shsuenaga 				mvxpsec_dump_data(__func__, buf, len);
17491a8031e1Shsuenaga 		}
17501a8031e1Shsuenaga 
17511a8031e1Shsuenaga 		if (mvxpsec_debug & MVXPSEC_DEBUG_PAYLOAD) {
17521a8031e1Shsuenaga 			MVXPSEC_PRINTF(MVXPSEC_DEBUG_PAYLOAD,
17531a8031e1Shsuenaga 			    "%s: session_descriptor:\n", __func__);
17541a8031e1Shsuenaga 			mvxpsec_dump_packet_desc(__func__, mv_p);
17551a8031e1Shsuenaga 			MVXPSEC_PRINTF(MVXPSEC_DEBUG_PAYLOAD,
17561a8031e1Shsuenaga 			    "%s: session_data:\n", __func__);
17571a8031e1Shsuenaga 			mvxpsec_dump_packet_data(__func__, mv_p);
17581a8031e1Shsuenaga 		}
17591a8031e1Shsuenaga 
17601a8031e1Shsuenaga 		if (mvxpsec_debug & MVXPSEC_DEBUG_SRAM) {
17611a8031e1Shsuenaga 			MVXPSEC_PRINTF(MVXPSEC_DEBUG_SRAM,
17621a8031e1Shsuenaga 			    "%s: SRAM\n", __func__);
17631a8031e1Shsuenaga 			mvxpsec_dump_sram(__func__, sc, 2000);
17641a8031e1Shsuenaga 		}
17651a8031e1Shsuenaga 
17661a8031e1Shsuenaga 		s = MVXPSEC_READ(sc, MV_ACC_STATUS);
17671a8031e1Shsuenaga 		if (s & MV_ACC_STATUS_MAC_ERR) {
17681a8031e1Shsuenaga 			MVXPSEC_PRINTF(MVXPSEC_DEBUG_INTR,
17691a8031e1Shsuenaga 			    "%s: Message Authentication Failed.\n", __func__);
17701a8031e1Shsuenaga 		}
17711a8031e1Shsuenaga 	}
17721a8031e1Shsuenaga #endif
17731a8031e1Shsuenaga 
17741a8031e1Shsuenaga 	/* copy back IV */
17751a8031e1Shsuenaga 	if (mv_p->flags & CRP_EXT_IV) {
17761a8031e1Shsuenaga 		memcpy(mv_p->ext_iv,
17771a8031e1Shsuenaga 		    &mv_p->pkt_header.crp_iv_ext, mv_p->ext_ivlen);
17781a8031e1Shsuenaga 		mv_p->ext_iv = NULL;
17791a8031e1Shsuenaga 		mv_p->ext_ivlen = 0;
17801a8031e1Shsuenaga 	}
17811a8031e1Shsuenaga 
17821a8031e1Shsuenaga 	/* notify opencrypto */
17831a8031e1Shsuenaga 	mv_p->crp->crp_etype = 0;
17841a8031e1Shsuenaga 	crypto_done(mv_p->crp);
17851a8031e1Shsuenaga 	mv_p->crp = NULL;
17861a8031e1Shsuenaga 
17871a8031e1Shsuenaga 	/* unblock driver */
17881a8031e1Shsuenaga 	mvxpsec_packet_dealloc(mv_p);
17891a8031e1Shsuenaga 	crypto_unblock(sc->sc_cid, CRYPTO_SYMQ|CRYPTO_ASYMQ);
17901a8031e1Shsuenaga 
17911a8031e1Shsuenaga 	MVXPSEC_EVCNT_INCR(sc, packet_ok);
17921a8031e1Shsuenaga 
17931a8031e1Shsuenaga 	return 0;
17941a8031e1Shsuenaga }
17951a8031e1Shsuenaga 
17961a8031e1Shsuenaga 
17971a8031e1Shsuenaga /*
17981a8031e1Shsuenaga  * Opencrypto API registration
17991a8031e1Shsuenaga  */
18001a8031e1Shsuenaga int
mvxpsec_register(struct mvxpsec_softc * sc)18011a8031e1Shsuenaga mvxpsec_register(struct mvxpsec_softc *sc)
18021a8031e1Shsuenaga {
18031a8031e1Shsuenaga 	int oplen = SRAM_PAYLOAD_SIZE;
18041a8031e1Shsuenaga 	int flags = 0;
18051a8031e1Shsuenaga 	int err;
18061a8031e1Shsuenaga 
18071a8031e1Shsuenaga 	sc->sc_nsessions = 0;
18081a8031e1Shsuenaga 	sc->sc_cid = crypto_get_driverid(0);
18091a8031e1Shsuenaga 	if (sc->sc_cid < 0) {
18101a8031e1Shsuenaga 		log(LOG_ERR,
18111a8031e1Shsuenaga 		    "%s: crypto_get_driverid() failed.\n", __func__);
18121a8031e1Shsuenaga 		err = EINVAL;
18131a8031e1Shsuenaga 		goto done;
18141a8031e1Shsuenaga 	}
18151a8031e1Shsuenaga 
18161a8031e1Shsuenaga 	/* Ciphers */
18171a8031e1Shsuenaga 	err = crypto_register(sc->sc_cid, CRYPTO_DES_CBC, oplen, flags,
18181a8031e1Shsuenaga 	    mvxpsec_newsession, mvxpsec_freesession, mvxpsec_dispatch, sc);
18191a8031e1Shsuenaga 	if (err)
18201a8031e1Shsuenaga 		goto done;
18211a8031e1Shsuenaga 
18221a8031e1Shsuenaga 	err = crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, oplen, flags,
18231a8031e1Shsuenaga 	    mvxpsec_newsession, mvxpsec_freesession, mvxpsec_dispatch, sc);
18241a8031e1Shsuenaga 	if (err)
18251a8031e1Shsuenaga 		goto done;
18261a8031e1Shsuenaga 
18271a8031e1Shsuenaga 	err = crypto_register(sc->sc_cid, CRYPTO_AES_CBC, oplen, flags,
18281a8031e1Shsuenaga 	    mvxpsec_newsession, mvxpsec_freesession, mvxpsec_dispatch, sc);
18291a8031e1Shsuenaga 	if (err)
18301a8031e1Shsuenaga 		goto done;
18311a8031e1Shsuenaga 
18321a8031e1Shsuenaga 	/* MACs */
18331a8031e1Shsuenaga 	err = crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC_96,
18341a8031e1Shsuenaga 	    oplen, flags,
18351a8031e1Shsuenaga 	    mvxpsec_newsession, mvxpsec_freesession, mvxpsec_dispatch, sc);
18361a8031e1Shsuenaga 	if (err)
18371a8031e1Shsuenaga 		goto done;
18381a8031e1Shsuenaga 
18391a8031e1Shsuenaga 	err = crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC_96,
18401a8031e1Shsuenaga 	    oplen, flags,
18411a8031e1Shsuenaga 	    mvxpsec_newsession, mvxpsec_freesession, mvxpsec_dispatch, sc);
18421a8031e1Shsuenaga 	if (err)
18431a8031e1Shsuenaga 		goto done;
18441a8031e1Shsuenaga 
18451a8031e1Shsuenaga #ifdef DEBUG
18461a8031e1Shsuenaga 	log(LOG_DEBUG,
18471a8031e1Shsuenaga 	    "%s: registered to opencrypto(max data = %d bytes)\n",
18481a8031e1Shsuenaga 	    device_xname(sc->sc_dev), oplen);
18491a8031e1Shsuenaga #endif
18501a8031e1Shsuenaga 
18511a8031e1Shsuenaga 	err = 0;
18521a8031e1Shsuenaga done:
18531a8031e1Shsuenaga 	return err;
18541a8031e1Shsuenaga }
18551a8031e1Shsuenaga 
18561a8031e1Shsuenaga /*
18571a8031e1Shsuenaga  * Create new opencrypto session
18581a8031e1Shsuenaga  *
18591a8031e1Shsuenaga  *   - register cipher key, mac key.
18601a8031e1Shsuenaga  *   - initialize mac internal state.
18611a8031e1Shsuenaga  */
18621a8031e1Shsuenaga int
mvxpsec_newsession(void * arg,uint32_t * sidp,struct cryptoini * cri)18631a8031e1Shsuenaga mvxpsec_newsession(void *arg, uint32_t *sidp, struct cryptoini *cri)
18641a8031e1Shsuenaga {
18651a8031e1Shsuenaga 	struct mvxpsec_softc *sc = arg;
18661a8031e1Shsuenaga 	struct mvxpsec_session *mv_s = NULL;
18671a8031e1Shsuenaga 	struct cryptoini *c;
18681a8031e1Shsuenaga 	static int hint = 0;
18691a8031e1Shsuenaga 	int session = -1;
18701a8031e1Shsuenaga 	int sid;
18711a8031e1Shsuenaga 	int err;
18721a8031e1Shsuenaga 	int i;
18731a8031e1Shsuenaga 
18741a8031e1Shsuenaga 	/* allocate driver session context */
18751a8031e1Shsuenaga 	mv_s = mvxpsec_session_alloc(sc);
18761a8031e1Shsuenaga 	if (mv_s == NULL)
18771a8031e1Shsuenaga 		return ENOMEM;
18781a8031e1Shsuenaga 
18791a8031e1Shsuenaga 	/*
18801a8031e1Shsuenaga 	 * lookup opencrypto session table
18811a8031e1Shsuenaga 	 *
18821a8031e1Shsuenaga 	 * we have sc_session_mtx after here.
18831a8031e1Shsuenaga 	 */
18841a8031e1Shsuenaga 	mutex_enter(&sc->sc_session_mtx);
18851a8031e1Shsuenaga 	if (sc->sc_nsessions >= MVXPSEC_MAX_SESSIONS) {
18861a8031e1Shsuenaga 		mutex_exit(&sc->sc_session_mtx);
18871a8031e1Shsuenaga 		log(LOG_ERR, "%s: too many IPsec SA(max %d)\n",
18881a8031e1Shsuenaga 				__func__, MVXPSEC_MAX_SESSIONS);
18891a8031e1Shsuenaga 		mvxpsec_session_dealloc(mv_s);
18901a8031e1Shsuenaga 		return ENOMEM;
18911a8031e1Shsuenaga 	}
18921a8031e1Shsuenaga 	for (i = hint; i < MVXPSEC_MAX_SESSIONS; i++) {
18931a8031e1Shsuenaga 		if (sc->sc_sessions[i])
18941a8031e1Shsuenaga 			continue;
18951a8031e1Shsuenaga 		session = i;
18961a8031e1Shsuenaga 		hint = session + 1;
18971a8031e1Shsuenaga 	       	break;
18981a8031e1Shsuenaga 	}
18991a8031e1Shsuenaga 	if (session < 0) {
19001a8031e1Shsuenaga 		for (i = 0; i < hint; i++) {
19011a8031e1Shsuenaga 			if (sc->sc_sessions[i])
19021a8031e1Shsuenaga 				continue;
19031a8031e1Shsuenaga 			session = i;
19041a8031e1Shsuenaga 			hint = session + 1;
19051a8031e1Shsuenaga 			break;
19061a8031e1Shsuenaga 		}
19071a8031e1Shsuenaga 		if (session < 0) {
19081a8031e1Shsuenaga 			mutex_exit(&sc->sc_session_mtx);
19091a8031e1Shsuenaga 			/* session full */
19101a8031e1Shsuenaga 			log(LOG_ERR, "%s: too many IPsec SA(max %d)\n",
19111a8031e1Shsuenaga 				__func__, MVXPSEC_MAX_SESSIONS);
19121a8031e1Shsuenaga 			mvxpsec_session_dealloc(mv_s);
19131a8031e1Shsuenaga 			hint = 0;
19141a8031e1Shsuenaga 			return ENOMEM;
19151a8031e1Shsuenaga 		}
19161a8031e1Shsuenaga 	}
19171a8031e1Shsuenaga 	if (hint >= MVXPSEC_MAX_SESSIONS)
19181a8031e1Shsuenaga 		hint = 0;
19191a8031e1Shsuenaga 	sc->sc_nsessions++;
19201a8031e1Shsuenaga 	sc->sc_sessions[session] = mv_s;
19211a8031e1Shsuenaga #ifdef DEBUG
19221a8031e1Shsuenaga 	log(LOG_DEBUG, "%s: new session %d allocated\n", __func__, session);
19231a8031e1Shsuenaga #endif
19241a8031e1Shsuenaga 
19251a8031e1Shsuenaga 	sid = MVXPSEC_SID(device_unit(sc->sc_dev), session);
19261a8031e1Shsuenaga 	mv_s->sid = sid;
19271a8031e1Shsuenaga 
19281a8031e1Shsuenaga 	/* setup the session key ... */
19291a8031e1Shsuenaga 	for (c = cri; c; c = c->cri_next) {
19301a8031e1Shsuenaga 		switch (c->cri_alg) {
19311a8031e1Shsuenaga 		case CRYPTO_DES_CBC:
19321a8031e1Shsuenaga 		case CRYPTO_3DES_CBC:
19331a8031e1Shsuenaga 		case CRYPTO_AES_CBC:
19341a8031e1Shsuenaga 			/* key */
19351a8031e1Shsuenaga 			if (mvxpsec_key_precomp(c->cri_alg,
19361a8031e1Shsuenaga 			    c->cri_key, c->cri_klen,
19371a8031e1Shsuenaga 			    &mv_s->session_header.crp_key,
19381a8031e1Shsuenaga 			    &mv_s->session_header.crp_key_d)) {
19391a8031e1Shsuenaga 				log(LOG_ERR,
19401a8031e1Shsuenaga 				    "%s: Invalid HMAC key for %s.\n",
19411a8031e1Shsuenaga 				    __func__, s_ctlalg(c->cri_alg));
19421a8031e1Shsuenaga 				err = EINVAL;
19431a8031e1Shsuenaga 				goto fail;
19441a8031e1Shsuenaga 			}
19451a8031e1Shsuenaga 			if (mv_s->sflags & RDY_CRP_KEY) {
19461a8031e1Shsuenaga 				log(LOG_WARNING,
19471a8031e1Shsuenaga 				    "%s: overwrite cipher: %s->%s.\n",
19481a8031e1Shsuenaga 				    __func__,
19491a8031e1Shsuenaga 				    s_ctlalg(mv_s->cipher_alg),
19501a8031e1Shsuenaga 				    s_ctlalg(c->cri_alg));
19511a8031e1Shsuenaga 			}
19521a8031e1Shsuenaga 			mv_s->sflags |= RDY_CRP_KEY;
19531a8031e1Shsuenaga 			mv_s->enc_klen = c->cri_klen;
19541a8031e1Shsuenaga 			mv_s->cipher_alg = c->cri_alg;
19551a8031e1Shsuenaga 			/* create per session IV (compatible with KAME IPsec) */
19561a8031e1Shsuenaga 			cprng_fast(&mv_s->session_iv, sizeof(mv_s->session_iv));
19571a8031e1Shsuenaga 			mv_s->sflags |= RDY_CRP_IV;
19581a8031e1Shsuenaga 			break;
19591a8031e1Shsuenaga 		case CRYPTO_SHA1_HMAC_96:
19601a8031e1Shsuenaga 		case CRYPTO_MD5_HMAC_96:
19611a8031e1Shsuenaga 			/* key */
19621a8031e1Shsuenaga 			if (mvxpsec_hmac_precomp(c->cri_alg,
19631a8031e1Shsuenaga 			    c->cri_key, c->cri_klen,
19641a8031e1Shsuenaga 			    (uint32_t *)&mv_s->session_header.miv_in,
19651a8031e1Shsuenaga 			    (uint32_t *)&mv_s->session_header.miv_out)) {
19661a8031e1Shsuenaga 				log(LOG_ERR,
19671a8031e1Shsuenaga 				    "%s: Invalid MAC key\n", __func__);
19681a8031e1Shsuenaga 				err = EINVAL;
19691a8031e1Shsuenaga 				goto fail;
19701a8031e1Shsuenaga 			}
19711a8031e1Shsuenaga 			if (mv_s->sflags & RDY_MAC_KEY ||
19721a8031e1Shsuenaga 			    mv_s->sflags & RDY_MAC_IV) {
19731a8031e1Shsuenaga 				log(LOG_ERR,
19741a8031e1Shsuenaga 				    "%s: overwrite HMAC: %s->%s.\n",
19751a8031e1Shsuenaga 				    __func__, s_ctlalg(mv_s->hmac_alg),
19761a8031e1Shsuenaga 				    s_ctlalg(c->cri_alg));
19771a8031e1Shsuenaga 			}
19781a8031e1Shsuenaga 			mv_s->sflags |= RDY_MAC_KEY;
19791a8031e1Shsuenaga 			mv_s->sflags |= RDY_MAC_IV;
19801a8031e1Shsuenaga 
19811a8031e1Shsuenaga 			mv_s->mac_klen = c->cri_klen;
19821a8031e1Shsuenaga 			mv_s->hmac_alg = c->cri_alg;
19831a8031e1Shsuenaga 			break;
19841a8031e1Shsuenaga 		default:
19851a8031e1Shsuenaga 			log(LOG_ERR, "%s: Unknown algorithm %d\n",
19861a8031e1Shsuenaga 			    __func__, c->cri_alg);
19871a8031e1Shsuenaga 			err = EINVAL;
19881a8031e1Shsuenaga 			goto fail;
19891a8031e1Shsuenaga 		}
19901a8031e1Shsuenaga 	}
19911a8031e1Shsuenaga 	MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO,
19921a8031e1Shsuenaga 	    "H/W Crypto session (id:%u) added.\n", session);
19931a8031e1Shsuenaga 
19941a8031e1Shsuenaga 	*sidp = sid;
19951a8031e1Shsuenaga 	MVXPSEC_EVCNT_INCR(sc, session_new);
19961a8031e1Shsuenaga 	mutex_exit(&sc->sc_session_mtx);
19971a8031e1Shsuenaga 
19981a8031e1Shsuenaga 	/* sync session header(it's never touched after here) */
19991a8031e1Shsuenaga 	bus_dmamap_sync(sc->sc_dmat,
20001a8031e1Shsuenaga 	    mv_s->session_header_map,
20011a8031e1Shsuenaga 	    0, sizeof(mv_s->session_header),
20021a8031e1Shsuenaga 	    BUS_DMASYNC_PREWRITE);
20031a8031e1Shsuenaga 
20041a8031e1Shsuenaga 	return 0;
20051a8031e1Shsuenaga 
20061a8031e1Shsuenaga fail:
20071a8031e1Shsuenaga 	sc->sc_nsessions--;
20081a8031e1Shsuenaga 	sc->sc_sessions[session] = NULL;
20091a8031e1Shsuenaga 	hint = session;
20101a8031e1Shsuenaga 	if (mv_s)
20111a8031e1Shsuenaga 		mvxpsec_session_dealloc(mv_s);
20121a8031e1Shsuenaga 	log(LOG_WARNING,
2013176f1e87Sandvar 	    "%s: Failed to add H/W crypto session (id:%u): err=%d\n",
20141a8031e1Shsuenaga 	   __func__, session, err);
20151a8031e1Shsuenaga 
20161a8031e1Shsuenaga 	mutex_exit(&sc->sc_session_mtx);
20171a8031e1Shsuenaga 	return err;
20181a8031e1Shsuenaga }
20191a8031e1Shsuenaga 
20201a8031e1Shsuenaga /*
20211a8031e1Shsuenaga  * remove opencrypto session
20221a8031e1Shsuenaga  */
20238449a122Sriastradh void
mvxpsec_freesession(void * arg,uint64_t tid)20241a8031e1Shsuenaga mvxpsec_freesession(void *arg, uint64_t tid)
20251a8031e1Shsuenaga {
20261a8031e1Shsuenaga 	struct mvxpsec_softc *sc = arg;
20271a8031e1Shsuenaga 	struct mvxpsec_session *mv_s;
20281a8031e1Shsuenaga 	int session;
20291a8031e1Shsuenaga 	uint32_t sid = ((uint32_t)tid) & 0xffffffff;
20301a8031e1Shsuenaga 
20311a8031e1Shsuenaga 	session = MVXPSEC_SESSION(sid);
2032f32a8269Sriastradh 	KASSERTMSG(session >= 0, "session=%d", session);
2033f32a8269Sriastradh 	KASSERTMSG(session < MVXPSEC_MAX_SESSIONS, "session=%d max=%d",
2034f32a8269Sriastradh 	    session, MVXPSEC_MAX_SESSIONS);
20351a8031e1Shsuenaga 
20361a8031e1Shsuenaga 	mutex_enter(&sc->sc_session_mtx);
2037f32a8269Sriastradh 	mv_s = sc->sc_sessions[session];
2038f32a8269Sriastradh 	KASSERT(mv_s != NULL);
20391a8031e1Shsuenaga 	MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO,
20401a8031e1Shsuenaga 	    "%s: inactivate session %d\n", __func__, session);
20411a8031e1Shsuenaga 
20421a8031e1Shsuenaga 	/* inactivate mvxpsec session */
20431a8031e1Shsuenaga 	sc->sc_sessions[session] = NULL;
20441a8031e1Shsuenaga 	sc->sc_nsessions--;
20451a8031e1Shsuenaga 	sc->sc_last_session = NULL;
20461a8031e1Shsuenaga 	mutex_exit(&sc->sc_session_mtx);
20471a8031e1Shsuenaga 
20481a8031e1Shsuenaga 	KASSERT(sc->sc_nsessions >= 0);
20491a8031e1Shsuenaga 	KASSERT(mv_s->sid == sid);
20501a8031e1Shsuenaga 
20511a8031e1Shsuenaga 	mvxpsec_session_dealloc(mv_s);
20521a8031e1Shsuenaga 	MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO,
20531a8031e1Shsuenaga 	    "H/W Crypto session (id: %d) deleted.\n", session);
20541a8031e1Shsuenaga 
20551a8031e1Shsuenaga 	/* force unblock opencrypto */
20561a8031e1Shsuenaga 	crypto_unblock(sc->sc_cid, CRYPTO_SYMQ|CRYPTO_ASYMQ);
20571a8031e1Shsuenaga 
20581a8031e1Shsuenaga 	MVXPSEC_EVCNT_INCR(sc, session_free);
20591a8031e1Shsuenaga }
20601a8031e1Shsuenaga 
20611a8031e1Shsuenaga /*
20621a8031e1Shsuenaga  * process data with existing session
20631a8031e1Shsuenaga  */
20641a8031e1Shsuenaga int
mvxpsec_dispatch(void * arg,struct cryptop * crp,int hint)20651a8031e1Shsuenaga mvxpsec_dispatch(void *arg, struct cryptop *crp, int hint)
20661a8031e1Shsuenaga {
20671a8031e1Shsuenaga 	struct mvxpsec_softc *sc = arg;
20681a8031e1Shsuenaga 	struct mvxpsec_session *mv_s;
20691a8031e1Shsuenaga 	struct mvxpsec_packet *mv_p;
20701a8031e1Shsuenaga 	int q_full;
20711a8031e1Shsuenaga 	int running;
20721a8031e1Shsuenaga 	int err;
20731a8031e1Shsuenaga 
20741a8031e1Shsuenaga 	mutex_enter(&sc->sc_queue_mtx);
20751a8031e1Shsuenaga 
20761a8031e1Shsuenaga 	/*
20771a8031e1Shsuenaga 	 * lookup session
20781a8031e1Shsuenaga 	 */
20791a8031e1Shsuenaga 	mutex_enter(&sc->sc_session_mtx);
20801a8031e1Shsuenaga 	mv_s = mvxpsec_session_lookup(sc, crp->crp_sid);
20811a8031e1Shsuenaga 	if (__predict_false(mv_s == NULL)) {
20821a8031e1Shsuenaga 		err = EINVAL;
20831a8031e1Shsuenaga 		mv_p = NULL;
20841a8031e1Shsuenaga 		mutex_exit(&sc->sc_session_mtx);
20851a8031e1Shsuenaga 		goto fail;
20861a8031e1Shsuenaga 	}
20871a8031e1Shsuenaga 	mv_p = mvxpsec_packet_alloc(mv_s);
20881a8031e1Shsuenaga 	if (__predict_false(mv_p == NULL)) {
20891a8031e1Shsuenaga 		mutex_exit(&sc->sc_session_mtx);
20901a8031e1Shsuenaga 		mutex_exit(&sc->sc_queue_mtx);
20911a8031e1Shsuenaga 		return ERESTART; /* => queued in opencrypto layer */
20921a8031e1Shsuenaga 	}
20931a8031e1Shsuenaga 	mutex_exit(&sc->sc_session_mtx);
20941a8031e1Shsuenaga 
20951a8031e1Shsuenaga 	/*
20961a8031e1Shsuenaga 	 * check queue status
20971a8031e1Shsuenaga 	 */
20981a8031e1Shsuenaga #ifdef MVXPSEC_MULTI_PACKET
20991a8031e1Shsuenaga 	q_full = (sc->sc_wait_qlen >= sc->sc_wait_qlimit) ? 1 : 0;
21001a8031e1Shsuenaga #else
21011a8031e1Shsuenaga 	q_full = (sc->sc_wait_qlen != 0) ? 1 : 0;
21021a8031e1Shsuenaga #endif
21031a8031e1Shsuenaga 	running = (sc->sc_flags & HW_RUNNING) ?  1: 0;
21041a8031e1Shsuenaga 	if (q_full) {
21051a8031e1Shsuenaga 		/* input queue is full. */
21061a8031e1Shsuenaga 		if (!running && sc->sc_wait_qlen > 0)
21071a8031e1Shsuenaga 			mvxpsec_dispatch_queue(sc);
21081a8031e1Shsuenaga 		MVXPSEC_EVCNT_INCR(sc, queue_full);
21091a8031e1Shsuenaga 		mvxpsec_packet_dealloc(mv_p);
21101a8031e1Shsuenaga 		mutex_exit(&sc->sc_queue_mtx);
21111a8031e1Shsuenaga 		return ERESTART; /* => queued in opencrypto layer */
21121a8031e1Shsuenaga 	}
21131a8031e1Shsuenaga 
21141a8031e1Shsuenaga 	/*
21151a8031e1Shsuenaga 	 * Load and setup packet data
21161a8031e1Shsuenaga 	 */
21171a8031e1Shsuenaga 	err = mvxpsec_packet_setcrp(mv_p, crp);
21181a8031e1Shsuenaga 	if (__predict_false(err))
21191a8031e1Shsuenaga 		goto fail;
21201a8031e1Shsuenaga 
21211a8031e1Shsuenaga 	/*
21221a8031e1Shsuenaga 	 * Setup DMA descriptor chains
21231a8031e1Shsuenaga 	 */
21241a8031e1Shsuenaga 	mutex_enter(&sc->sc_dma_mtx);
21251a8031e1Shsuenaga 	err = mvxpsec_dma_copy_packet(sc, mv_p);
21261a8031e1Shsuenaga 	mutex_exit(&sc->sc_dma_mtx);
21271a8031e1Shsuenaga 	if (__predict_false(err))
21281a8031e1Shsuenaga 		goto fail;
21291a8031e1Shsuenaga 
21301a8031e1Shsuenaga #ifdef MVXPSEC_DEBUG
21311a8031e1Shsuenaga 	mvxpsec_dump_packet(__func__, mv_p);
21321a8031e1Shsuenaga #endif
21331a8031e1Shsuenaga 
21341a8031e1Shsuenaga 	/*
21351a8031e1Shsuenaga 	 * Sync/inval the data cache
21361a8031e1Shsuenaga 	 */
21371a8031e1Shsuenaga 	err = mvxpsec_dma_sync_packet(sc, mv_p);
21381a8031e1Shsuenaga 	if (__predict_false(err))
21391a8031e1Shsuenaga 		goto fail;
21401a8031e1Shsuenaga 
21411a8031e1Shsuenaga 	/*
21421a8031e1Shsuenaga 	 * Enqueue the packet
21431a8031e1Shsuenaga 	 */
21441a8031e1Shsuenaga 	MVXPSEC_EVCNT_INCR(sc, dispatch_packets);
21451a8031e1Shsuenaga #ifdef MVXPSEC_MULTI_PACKET
21461a8031e1Shsuenaga 	mvxpsec_packet_enqueue(mv_p);
21471a8031e1Shsuenaga 	if (!running)
21481a8031e1Shsuenaga 		mvxpsec_dispatch_queue(sc);
21491a8031e1Shsuenaga #else
21501a8031e1Shsuenaga 	SIMPLEQ_INSERT_TAIL(&sc->sc_wait_queue, mv_p, queue);
21511a8031e1Shsuenaga 	sc->sc_wait_qlen++;
21521a8031e1Shsuenaga 	mv_p->flags |= SETUP_DONE;
21531a8031e1Shsuenaga 	if (!running)
21541a8031e1Shsuenaga 		mvxpsec_dispatch_queue(sc);
21551a8031e1Shsuenaga #endif
21561a8031e1Shsuenaga 	mutex_exit(&sc->sc_queue_mtx);
21571a8031e1Shsuenaga 	return 0;
21581a8031e1Shsuenaga 
21591a8031e1Shsuenaga fail:
21601a8031e1Shsuenaga 	/* Drop the incoming packet */
21611a8031e1Shsuenaga 	mvxpsec_drop(sc, crp, mv_p, err);
21621a8031e1Shsuenaga 	mutex_exit(&sc->sc_queue_mtx);
21631a8031e1Shsuenaga 	return 0;
21641a8031e1Shsuenaga }
21651a8031e1Shsuenaga 
21661a8031e1Shsuenaga /*
21671a8031e1Shsuenaga  * back the packet to the IP stack
21681a8031e1Shsuenaga  */
21691a8031e1Shsuenaga void
mvxpsec_done(void * arg)21701a8031e1Shsuenaga mvxpsec_done(void *arg)
21711a8031e1Shsuenaga {
21721a8031e1Shsuenaga 	struct mvxpsec_softc *sc = arg;
21731a8031e1Shsuenaga 	struct mvxpsec_packet *mv_p;
21741a8031e1Shsuenaga 	mvxpsec_queue_t ret_queue;
21751a8031e1Shsuenaga 	int ndone;
21761a8031e1Shsuenaga 
21771a8031e1Shsuenaga 	mutex_enter(&sc->sc_queue_mtx);
21781a8031e1Shsuenaga 
21791a8031e1Shsuenaga 	/* stop wdog timer */
21801a8031e1Shsuenaga 	callout_stop(&sc->sc_timeout);
21811a8031e1Shsuenaga 
21821a8031e1Shsuenaga 	/* refill MVXPSEC */
21831a8031e1Shsuenaga 	ret_queue = sc->sc_run_queue;
21841a8031e1Shsuenaga 	SIMPLEQ_INIT(&sc->sc_run_queue);
21851a8031e1Shsuenaga 	sc->sc_flags &= ~HW_RUNNING;
21861a8031e1Shsuenaga 	if (sc->sc_wait_qlen > 0)
21871a8031e1Shsuenaga 		mvxpsec_dispatch_queue(sc);
21881a8031e1Shsuenaga 
21891a8031e1Shsuenaga 	ndone = 0;
21901a8031e1Shsuenaga 	while ( (mv_p = SIMPLEQ_FIRST(&ret_queue)) != NULL) {
21911a8031e1Shsuenaga 		SIMPLEQ_REMOVE_HEAD(&ret_queue, queue);
21921a8031e1Shsuenaga 		mvxpsec_dma_free(sc, &mv_p->dma_ring);
21931a8031e1Shsuenaga 		mvxpsec_done_packet(mv_p);
21941a8031e1Shsuenaga 		ndone++;
21951a8031e1Shsuenaga 	}
21961a8031e1Shsuenaga 	MVXPSEC_EVCNT_MAX(sc, max_done, ndone);
21971a8031e1Shsuenaga 
21981a8031e1Shsuenaga 	mutex_exit(&sc->sc_queue_mtx);
21991a8031e1Shsuenaga }
22001a8031e1Shsuenaga 
22011a8031e1Shsuenaga /*
22021a8031e1Shsuenaga  * drop the packet
22031a8031e1Shsuenaga  */
22041a8031e1Shsuenaga INLINE void
mvxpsec_drop(struct mvxpsec_softc * sc,struct cryptop * crp,struct mvxpsec_packet * mv_p,int err)22051a8031e1Shsuenaga mvxpsec_drop(struct mvxpsec_softc *sc, struct cryptop *crp,
22061a8031e1Shsuenaga     struct mvxpsec_packet *mv_p, int err)
22071a8031e1Shsuenaga {
22081a8031e1Shsuenaga 	/* must called with sc->sc_queue_mtx held */
22091a8031e1Shsuenaga 	KASSERT(mutex_owned(&sc->sc_queue_mtx));
22101a8031e1Shsuenaga 
22111a8031e1Shsuenaga 	if (mv_p)
22121a8031e1Shsuenaga 		mvxpsec_packet_dealloc(mv_p);
22131a8031e1Shsuenaga 	if (err < 0)
22141a8031e1Shsuenaga 		err = EINVAL;
22151a8031e1Shsuenaga 	crp->crp_etype = err;
22161a8031e1Shsuenaga 	crypto_done(crp);
22171a8031e1Shsuenaga 	MVXPSEC_EVCNT_INCR(sc, packet_err);
22181a8031e1Shsuenaga 
22191a8031e1Shsuenaga 	/* dispatch other packets in queue */
22201a8031e1Shsuenaga 	if (sc->sc_wait_qlen > 0 &&
22211a8031e1Shsuenaga 	    !(sc->sc_flags & HW_RUNNING))
22221a8031e1Shsuenaga 		mvxpsec_dispatch_queue(sc);
22231a8031e1Shsuenaga 
22241a8031e1Shsuenaga 	/* unblock driver for dropped packet */
22251a8031e1Shsuenaga 	crypto_unblock(sc->sc_cid, CRYPTO_SYMQ|CRYPTO_ASYMQ);
22261a8031e1Shsuenaga }
22271a8031e1Shsuenaga 
22281a8031e1Shsuenaga /* move wait queue entry to run queue */
22291a8031e1Shsuenaga STATIC int
mvxpsec_dispatch_queue(struct mvxpsec_softc * sc)22301a8031e1Shsuenaga mvxpsec_dispatch_queue(struct mvxpsec_softc *sc)
22311a8031e1Shsuenaga {
22321a8031e1Shsuenaga 	struct mvxpsec_packet *mv_p;
22331a8031e1Shsuenaga 	paddr_t head;
22341a8031e1Shsuenaga 	int ndispatch = 0;
22351a8031e1Shsuenaga 
22361a8031e1Shsuenaga 	/* must called with sc->sc_queue_mtx held */
22371a8031e1Shsuenaga 	KASSERT(mutex_owned(&sc->sc_queue_mtx));
22381a8031e1Shsuenaga 
22391a8031e1Shsuenaga 	/* check there is any task */
22401a8031e1Shsuenaga 	if (__predict_false(sc->sc_flags & HW_RUNNING)) {
22411a8031e1Shsuenaga 		log(LOG_WARNING,
22421a8031e1Shsuenaga 		    "%s: another packet already exist.\n", __func__);
22431a8031e1Shsuenaga 		return 0;
22441a8031e1Shsuenaga 	}
22451a8031e1Shsuenaga 	if (__predict_false(SIMPLEQ_EMPTY(&sc->sc_wait_queue))) {
22461a8031e1Shsuenaga 		log(LOG_WARNING,
22471a8031e1Shsuenaga 		    "%s: no waiting packet yet(qlen=%d).\n",
22481a8031e1Shsuenaga 		    __func__, sc->sc_wait_qlen);
22491a8031e1Shsuenaga 		return 0;
22501a8031e1Shsuenaga 	}
22511a8031e1Shsuenaga 
22521a8031e1Shsuenaga 	/* move queue */
22531a8031e1Shsuenaga 	sc->sc_run_queue = sc->sc_wait_queue;
22541a8031e1Shsuenaga 	sc->sc_flags |= HW_RUNNING; /* dropped by intr or timeout */
22551a8031e1Shsuenaga 	SIMPLEQ_INIT(&sc->sc_wait_queue);
22561a8031e1Shsuenaga 	ndispatch = sc->sc_wait_qlen;
22571a8031e1Shsuenaga 	sc->sc_wait_qlen = 0;
22581a8031e1Shsuenaga 
22591a8031e1Shsuenaga 	/* get 1st DMA descriptor */
22601a8031e1Shsuenaga 	mv_p = SIMPLEQ_FIRST(&sc->sc_run_queue);
22611a8031e1Shsuenaga 	head = mv_p->dma_ring.dma_head->phys_addr;
22621a8031e1Shsuenaga 
22631a8031e1Shsuenaga 	/* terminate last DMA descriptor */
22641a8031e1Shsuenaga 	mv_p = SIMPLEQ_LAST(&sc->sc_run_queue, mvxpsec_packet, queue);
22651a8031e1Shsuenaga 	mvxpsec_dma_finalize(sc, &mv_p->dma_ring);
22661a8031e1Shsuenaga 
22671a8031e1Shsuenaga 	/* configure TDMA */
22681a8031e1Shsuenaga 	if (mvxpsec_dma_wait(sc) < 0) {
22691a8031e1Shsuenaga 		log(LOG_ERR, "%s: DMA DEVICE not responding", __func__);
22701a8031e1Shsuenaga 		callout_schedule(&sc->sc_timeout, hz);
22711a8031e1Shsuenaga 		return 0;
22721a8031e1Shsuenaga 	}
22731a8031e1Shsuenaga 	MVXPSEC_WRITE(sc, MV_TDMA_NXT, head);
22741a8031e1Shsuenaga 
22751a8031e1Shsuenaga 	/* trigger ACC */
22761a8031e1Shsuenaga 	if (mvxpsec_acc_wait(sc) < 0) {
22771a8031e1Shsuenaga 		log(LOG_ERR, "%s: MVXPSEC not responding", __func__);
22781a8031e1Shsuenaga 		callout_schedule(&sc->sc_timeout, hz);
22791a8031e1Shsuenaga 		return 0;
22801a8031e1Shsuenaga 	}
22811a8031e1Shsuenaga 	MVXPSEC_WRITE(sc, MV_ACC_COMMAND, MV_ACC_COMMAND_ACT);
22821a8031e1Shsuenaga 
22831a8031e1Shsuenaga 	MVXPSEC_EVCNT_MAX(sc, max_dispatch, ndispatch);
22841a8031e1Shsuenaga 	MVXPSEC_EVCNT_INCR(sc, dispatch_queue);
22851a8031e1Shsuenaga 	callout_schedule(&sc->sc_timeout, hz);
22861a8031e1Shsuenaga 	return 0;
22871a8031e1Shsuenaga }
22881a8031e1Shsuenaga 
22891a8031e1Shsuenaga /*
22901a8031e1Shsuenaga  * process opencrypto operations(cryptop) for packets.
22911a8031e1Shsuenaga  */
22921a8031e1Shsuenaga INLINE int
mvxpsec_parse_crd(struct mvxpsec_packet * mv_p,struct cryptodesc * crd)22931a8031e1Shsuenaga mvxpsec_parse_crd(struct mvxpsec_packet *mv_p, struct cryptodesc *crd)
22941a8031e1Shsuenaga {
22951a8031e1Shsuenaga 	int ivlen;
22961a8031e1Shsuenaga 
22971a8031e1Shsuenaga 	KASSERT(mv_p->flags & RDY_DATA);
22981a8031e1Shsuenaga 
22991a8031e1Shsuenaga 	/* MAC & Ciphers: set data location and operation */
23001a8031e1Shsuenaga 	switch (crd->crd_alg) {
23011a8031e1Shsuenaga 	case CRYPTO_SHA1_HMAC_96:
23021a8031e1Shsuenaga 		mv_p->pkt_header.desc.acc_config |= MV_ACC_CRYPTO_MAC_96;
23031a8031e1Shsuenaga 		/* fall through */
23041a8031e1Shsuenaga 	case CRYPTO_SHA1_HMAC:
23051a8031e1Shsuenaga 		mv_p->mac_dst = crd->crd_inject;
23061a8031e1Shsuenaga 		mv_p->mac_off = crd->crd_skip;
23071a8031e1Shsuenaga 		mv_p->mac_len = crd->crd_len;
23081a8031e1Shsuenaga 		MV_ACC_CRYPTO_MAC_SET(mv_p->pkt_header.desc.acc_config,
23091a8031e1Shsuenaga 		    MV_ACC_CRYPTO_MAC_HMAC_SHA1);
23101a8031e1Shsuenaga 		mvxpsec_packet_update_op_order(mv_p, MV_ACC_CRYPTO_OP_MAC);
23111a8031e1Shsuenaga 		/* No more setup for MAC */
23121a8031e1Shsuenaga 		return 0;
23131a8031e1Shsuenaga 	case CRYPTO_MD5_HMAC_96:
23141a8031e1Shsuenaga 		mv_p->pkt_header.desc.acc_config |= MV_ACC_CRYPTO_MAC_96;
23151a8031e1Shsuenaga 		/* fall through */
23161a8031e1Shsuenaga 	case CRYPTO_MD5_HMAC:
23171a8031e1Shsuenaga 		mv_p->mac_dst = crd->crd_inject;
23181a8031e1Shsuenaga 		mv_p->mac_off = crd->crd_skip;
23191a8031e1Shsuenaga 		mv_p->mac_len = crd->crd_len;
23201a8031e1Shsuenaga 		MV_ACC_CRYPTO_MAC_SET(mv_p->pkt_header.desc.acc_config,
23211a8031e1Shsuenaga 		    MV_ACC_CRYPTO_MAC_HMAC_MD5);
23221a8031e1Shsuenaga 		mvxpsec_packet_update_op_order(mv_p, MV_ACC_CRYPTO_OP_MAC);
23231a8031e1Shsuenaga 		/* No more setup for MAC */
23241a8031e1Shsuenaga 		return 0;
23251a8031e1Shsuenaga 	case CRYPTO_DES_CBC:
23261a8031e1Shsuenaga 		mv_p->enc_ivoff = crd->crd_inject;
23271a8031e1Shsuenaga 		mv_p->enc_off = crd->crd_skip;
23281a8031e1Shsuenaga 		mv_p->enc_len = crd->crd_len;
23291a8031e1Shsuenaga 		ivlen = 8;
23301a8031e1Shsuenaga 		MV_ACC_CRYPTO_ENC_SET(mv_p->pkt_header.desc.acc_config,
23311a8031e1Shsuenaga 		    MV_ACC_CRYPTO_ENC_DES);
23321a8031e1Shsuenaga 		mv_p->pkt_header.desc.acc_config |= MV_ACC_CRYPTO_CBC;
23331a8031e1Shsuenaga 		mvxpsec_packet_update_op_order(mv_p, MV_ACC_CRYPTO_OP_ENC);
23341a8031e1Shsuenaga 		break;
23351a8031e1Shsuenaga 	case CRYPTO_3DES_CBC:
23361a8031e1Shsuenaga 		mv_p->enc_ivoff = crd->crd_inject;
23371a8031e1Shsuenaga 		mv_p->enc_off = crd->crd_skip;
23381a8031e1Shsuenaga 		mv_p->enc_len = crd->crd_len;
23391a8031e1Shsuenaga 		ivlen = 8;
23401a8031e1Shsuenaga 		MV_ACC_CRYPTO_ENC_SET(mv_p->pkt_header.desc.acc_config,
23411a8031e1Shsuenaga 		    MV_ACC_CRYPTO_ENC_3DES);
23421a8031e1Shsuenaga 		mv_p->pkt_header.desc.acc_config |= MV_ACC_CRYPTO_CBC;
23431a8031e1Shsuenaga 		mv_p->pkt_header.desc.acc_config |= MV_ACC_CRYPTO_3DES_EDE;
23441a8031e1Shsuenaga 		mvxpsec_packet_update_op_order(mv_p, MV_ACC_CRYPTO_OP_ENC);
23451a8031e1Shsuenaga 		break;
23461a8031e1Shsuenaga 	case CRYPTO_AES_CBC:
23471a8031e1Shsuenaga 		mv_p->enc_ivoff = crd->crd_inject;
23481a8031e1Shsuenaga 		mv_p->enc_off = crd->crd_skip;
23491a8031e1Shsuenaga 		mv_p->enc_len = crd->crd_len;
23501a8031e1Shsuenaga 		ivlen = 16;
23511a8031e1Shsuenaga 		MV_ACC_CRYPTO_ENC_SET(mv_p->pkt_header.desc.acc_config,
23521a8031e1Shsuenaga 		    MV_ACC_CRYPTO_ENC_AES);
23531a8031e1Shsuenaga 		MV_ACC_CRYPTO_AES_KLEN_SET(
23541a8031e1Shsuenaga 		    mv_p->pkt_header.desc.acc_config,
23551a8031e1Shsuenaga 		   mvxpsec_aesklen(mv_p->mv_s->enc_klen));
23561a8031e1Shsuenaga 		mv_p->pkt_header.desc.acc_config |= MV_ACC_CRYPTO_CBC;
23571a8031e1Shsuenaga 		mvxpsec_packet_update_op_order(mv_p, MV_ACC_CRYPTO_OP_ENC);
23581a8031e1Shsuenaga 		break;
23591a8031e1Shsuenaga 	default:
23601a8031e1Shsuenaga 		log(LOG_ERR, "%s: Unknown algorithm %d\n",
23611a8031e1Shsuenaga 		    __func__, crd->crd_alg);
23621a8031e1Shsuenaga 		return EINVAL;
23631a8031e1Shsuenaga 	}
23641a8031e1Shsuenaga 
23651a8031e1Shsuenaga 	/* Operations only for Cipher, not MAC */
23661a8031e1Shsuenaga 	if (crd->crd_flags & CRD_F_ENCRYPT) {
23671a8031e1Shsuenaga 		/* Ciphers: Originate IV for Encryption.*/
23681a8031e1Shsuenaga 		mv_p->pkt_header.desc.acc_config &= ~MV_ACC_CRYPTO_DECRYPT;
23691a8031e1Shsuenaga 		mv_p->flags |= DIR_ENCRYPT;
23701a8031e1Shsuenaga 
23711a8031e1Shsuenaga 		if (crd->crd_flags & CRD_F_IV_EXPLICIT) {
23721a8031e1Shsuenaga 			MVXPSEC_PRINTF(MVXPSEC_DEBUG_ENC_IV, "EXPLICIT IV\n");
23731a8031e1Shsuenaga 			mv_p->flags |= CRP_EXT_IV;
23741a8031e1Shsuenaga 			mvxpsec_packet_write_iv(mv_p, crd->crd_iv, ivlen);
23751a8031e1Shsuenaga 			mv_p->enc_ivoff = MVXPSEC_SRAM_IV_EXT_OFF;
23761a8031e1Shsuenaga 		}
23771a8031e1Shsuenaga 		else if (crd->crd_flags & CRD_F_IV_PRESENT) {
23781a8031e1Shsuenaga 			MVXPSEC_PRINTF(MVXPSEC_DEBUG_ENC_IV, "IV is present\n");
23791a8031e1Shsuenaga 			mvxpsec_packet_copy_iv(mv_p, crd->crd_inject, ivlen);
23801a8031e1Shsuenaga 		}
23811a8031e1Shsuenaga 		else {
23821a8031e1Shsuenaga 			MVXPSEC_PRINTF(MVXPSEC_DEBUG_ENC_IV, "Create New IV\n");
23831a8031e1Shsuenaga 			mvxpsec_packet_write_iv(mv_p, NULL, ivlen);
23841a8031e1Shsuenaga 		}
23851a8031e1Shsuenaga 	}
23861a8031e1Shsuenaga 	else {
23871a8031e1Shsuenaga 		/* Ciphers: IV is loadded from crd_inject when it's present */
23881a8031e1Shsuenaga 		mv_p->pkt_header.desc.acc_config |= MV_ACC_CRYPTO_DECRYPT;
23891a8031e1Shsuenaga 		mv_p->flags |= DIR_DECRYPT;
23901a8031e1Shsuenaga 
23911a8031e1Shsuenaga 		if (crd->crd_flags & CRD_F_IV_EXPLICIT) {
23921a8031e1Shsuenaga #ifdef MVXPSEC_DEBUG
23931a8031e1Shsuenaga 			if (mvxpsec_debug & MVXPSEC_DEBUG_ENC_IV) {
23941a8031e1Shsuenaga 				MVXPSEC_PRINTF(MVXPSEC_DEBUG_ENC_IV,
23951a8031e1Shsuenaga 				    "EXPLICIT IV(Decrypt)\n");
23961a8031e1Shsuenaga 				mvxpsec_dump_data(__func__, crd->crd_iv, ivlen);
23971a8031e1Shsuenaga 			}
23981a8031e1Shsuenaga #endif
23991a8031e1Shsuenaga 			mv_p->flags |= CRP_EXT_IV;
24001a8031e1Shsuenaga 			mvxpsec_packet_write_iv(mv_p, crd->crd_iv, ivlen);
24011a8031e1Shsuenaga 			mv_p->enc_ivoff = MVXPSEC_SRAM_IV_EXT_OFF;
24021a8031e1Shsuenaga 		}
24031a8031e1Shsuenaga 	}
24041a8031e1Shsuenaga 
24051a8031e1Shsuenaga 	KASSERT(!((mv_p->flags & DIR_ENCRYPT) && (mv_p->flags & DIR_DECRYPT)));
24061a8031e1Shsuenaga 
24071a8031e1Shsuenaga 	return 0;
24081a8031e1Shsuenaga }
24091a8031e1Shsuenaga 
24101a8031e1Shsuenaga INLINE int
mvxpsec_parse_crp(struct mvxpsec_packet * mv_p)24111a8031e1Shsuenaga mvxpsec_parse_crp(struct mvxpsec_packet *mv_p)
24121a8031e1Shsuenaga {
24131a8031e1Shsuenaga 	struct cryptop *crp = mv_p->crp;
24141a8031e1Shsuenaga 	struct cryptodesc *crd;
24151a8031e1Shsuenaga 	int err;
24161a8031e1Shsuenaga 
24171a8031e1Shsuenaga 	KASSERT(crp);
24181a8031e1Shsuenaga 
24191a8031e1Shsuenaga 	mvxpsec_packet_reset_op(mv_p);
24201a8031e1Shsuenaga 
24211a8031e1Shsuenaga 	for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
24221a8031e1Shsuenaga 		err = mvxpsec_parse_crd(mv_p, crd);
24231a8031e1Shsuenaga 		if (err)
24241a8031e1Shsuenaga 			return err;
24251a8031e1Shsuenaga 	}
24261a8031e1Shsuenaga 
24271a8031e1Shsuenaga 	return 0;
24281a8031e1Shsuenaga }
24291a8031e1Shsuenaga 
24301a8031e1Shsuenaga INLINE int
mvxpsec_packet_setcrp(struct mvxpsec_packet * mv_p,struct cryptop * crp)24311a8031e1Shsuenaga mvxpsec_packet_setcrp(struct mvxpsec_packet *mv_p, struct cryptop *crp)
24321a8031e1Shsuenaga {
24331a8031e1Shsuenaga 	int err = EINVAL;
24341a8031e1Shsuenaga 
2435*100a3398Sandvar 	/* register crp to the MVXPSEC packet */
24361a8031e1Shsuenaga 	if (crp->crp_flags & CRYPTO_F_IMBUF) {
24371a8031e1Shsuenaga 		err = mvxpsec_packet_setmbuf(mv_p,
24381a8031e1Shsuenaga 		    (struct mbuf *)crp->crp_buf);
24391a8031e1Shsuenaga 		mv_p->crp = crp;
24401a8031e1Shsuenaga 	}
24411a8031e1Shsuenaga 	else if (crp->crp_flags & CRYPTO_F_IOV) {
24421a8031e1Shsuenaga 		err = mvxpsec_packet_setuio(mv_p,
24431a8031e1Shsuenaga 		    (struct uio *)crp->crp_buf);
24441a8031e1Shsuenaga 		mv_p->crp = crp;
24451a8031e1Shsuenaga 	}
24461a8031e1Shsuenaga 	else {
24471a8031e1Shsuenaga 		err = mvxpsec_packet_setdata(mv_p,
24481a8031e1Shsuenaga 		    (struct mbuf *)crp->crp_buf, crp->crp_ilen);
24491a8031e1Shsuenaga 		mv_p->crp = crp;
24501a8031e1Shsuenaga 	}
24511a8031e1Shsuenaga 	if (__predict_false(err))
24521a8031e1Shsuenaga 		return err;
24531a8031e1Shsuenaga 
24541a8031e1Shsuenaga 	/* parse crp and setup MVXPSEC registers/descriptors */
24551a8031e1Shsuenaga 	err = mvxpsec_parse_crp(mv_p);
24561a8031e1Shsuenaga 	if (__predict_false(err))
24571a8031e1Shsuenaga 		return err;
24581a8031e1Shsuenaga 
24591a8031e1Shsuenaga 	/* fixup data offset to fit MVXPSEC internal SRAM */
24601a8031e1Shsuenaga 	err = mvxpsec_header_finalize(mv_p);
24611a8031e1Shsuenaga 	if (__predict_false(err))
24621a8031e1Shsuenaga 		return err;
24631a8031e1Shsuenaga 
24641a8031e1Shsuenaga 	return 0;
24651a8031e1Shsuenaga }
24661a8031e1Shsuenaga 
24671a8031e1Shsuenaga /*
24681a8031e1Shsuenaga  * load data for encrypt/decrypt/authentication
24691a8031e1Shsuenaga  *
24701a8031e1Shsuenaga  * data is raw kernel memory area.
24711a8031e1Shsuenaga  */
24721a8031e1Shsuenaga STATIC int
mvxpsec_packet_setdata(struct mvxpsec_packet * mv_p,void * data,uint32_t data_len)24731a8031e1Shsuenaga mvxpsec_packet_setdata(struct mvxpsec_packet *mv_p,
24741a8031e1Shsuenaga     void *data, uint32_t data_len)
24751a8031e1Shsuenaga {
24761a8031e1Shsuenaga 	struct mvxpsec_session *mv_s = mv_p->mv_s;
24771a8031e1Shsuenaga 	struct mvxpsec_softc *sc = mv_s->sc;
24781a8031e1Shsuenaga 
24791a8031e1Shsuenaga 	if (bus_dmamap_load(sc->sc_dmat, mv_p->data_map, data, data_len,
24801a8031e1Shsuenaga 	    NULL, BUS_DMA_NOWAIT)) {
24811a8031e1Shsuenaga 		log(LOG_ERR, "%s: cannot load data\n", __func__);
24821a8031e1Shsuenaga 		return -1;
24831a8031e1Shsuenaga 	}
24841a8031e1Shsuenaga 	mv_p->data_type = MVXPSEC_DATA_RAW;
24851a8031e1Shsuenaga 	mv_p->data_raw = data;
24861a8031e1Shsuenaga 	mv_p->data_len = data_len;
24871a8031e1Shsuenaga 	mv_p->flags |= RDY_DATA;
24881a8031e1Shsuenaga 
24891a8031e1Shsuenaga 	return 0;
24901a8031e1Shsuenaga }
24911a8031e1Shsuenaga 
24921a8031e1Shsuenaga /*
24931a8031e1Shsuenaga  * load data for encrypt/decrypt/authentication
24941a8031e1Shsuenaga  *
24951a8031e1Shsuenaga  * data is mbuf based network data.
24961a8031e1Shsuenaga  */
24971a8031e1Shsuenaga STATIC int
mvxpsec_packet_setmbuf(struct mvxpsec_packet * mv_p,struct mbuf * m)24981a8031e1Shsuenaga mvxpsec_packet_setmbuf(struct mvxpsec_packet *mv_p, struct mbuf *m)
24991a8031e1Shsuenaga {
25001a8031e1Shsuenaga 	struct mvxpsec_session *mv_s = mv_p->mv_s;
25011a8031e1Shsuenaga 	struct mvxpsec_softc *sc = mv_s->sc;
25021a8031e1Shsuenaga 	size_t pktlen = 0;
25031a8031e1Shsuenaga 
25041a8031e1Shsuenaga 	if (__predict_true(m->m_flags & M_PKTHDR))
25051a8031e1Shsuenaga 		pktlen = m->m_pkthdr.len;
25061a8031e1Shsuenaga 	else {
25071a8031e1Shsuenaga 		struct mbuf *mp = m;
25081a8031e1Shsuenaga 
25091a8031e1Shsuenaga 		while (mp != NULL) {
25101a8031e1Shsuenaga 			pktlen += m->m_len;
25111a8031e1Shsuenaga 			mp = mp->m_next;
25121a8031e1Shsuenaga 		}
25131a8031e1Shsuenaga 	}
25141a8031e1Shsuenaga 	if (pktlen > SRAM_PAYLOAD_SIZE) {
25152499f515Sriastradh #if NIPSEC > 0
25161a8031e1Shsuenaga 		extern   percpu_t *espstat_percpu;
25171a8031e1Shsuenaga 	       	/* XXX:
25181a8031e1Shsuenaga 		 * layer violation. opencrypto knows our max packet size
25191a8031e1Shsuenaga 		 * from crypto_register(9) API.
25201a8031e1Shsuenaga 		 */
25211a8031e1Shsuenaga 
25221a8031e1Shsuenaga 		_NET_STATINC(espstat_percpu, ESP_STAT_TOOBIG);
25232499f515Sriastradh #endif
25241a8031e1Shsuenaga 		log(LOG_ERR,
25251a8031e1Shsuenaga 		    "%s: ESP Packet too large: %zu [oct.] > %zu [oct.]\n",
25261a8031e1Shsuenaga 		    device_xname(sc->sc_dev),
25271a8031e1Shsuenaga 		    (size_t)pktlen, SRAM_PAYLOAD_SIZE);
25281a8031e1Shsuenaga 		mv_p->data_type = MVXPSEC_DATA_NONE;
25291a8031e1Shsuenaga 		mv_p->data_mbuf = NULL;
25301a8031e1Shsuenaga 		return -1;
25311a8031e1Shsuenaga 	}
25321a8031e1Shsuenaga 
25331a8031e1Shsuenaga 	if (bus_dmamap_load_mbuf(sc->sc_dmat, mv_p->data_map, m,
25341a8031e1Shsuenaga 	    BUS_DMA_NOWAIT)) {
25351a8031e1Shsuenaga 		mv_p->data_type = MVXPSEC_DATA_NONE;
25361a8031e1Shsuenaga 		mv_p->data_mbuf = NULL;
25371a8031e1Shsuenaga 		log(LOG_ERR, "%s: cannot load mbuf\n", __func__);
25381a8031e1Shsuenaga 		return -1;
25391a8031e1Shsuenaga 	}
25401a8031e1Shsuenaga 
25411a8031e1Shsuenaga 	/* set payload buffer */
25421a8031e1Shsuenaga 	mv_p->data_type = MVXPSEC_DATA_MBUF;
25431a8031e1Shsuenaga 	mv_p->data_mbuf = m;
25441a8031e1Shsuenaga 	if (m->m_flags & M_PKTHDR) {
25451a8031e1Shsuenaga 		mv_p->data_len = m->m_pkthdr.len;
25461a8031e1Shsuenaga 	}
25471a8031e1Shsuenaga 	else {
25481a8031e1Shsuenaga 		mv_p->data_len = 0;
25491a8031e1Shsuenaga 		while (m) {
25501a8031e1Shsuenaga 			mv_p->data_len += m->m_len;
25511a8031e1Shsuenaga 			m = m->m_next;
25521a8031e1Shsuenaga 		}
25531a8031e1Shsuenaga 	}
25541a8031e1Shsuenaga 	mv_p->flags |= RDY_DATA;
25551a8031e1Shsuenaga 
25561a8031e1Shsuenaga 	return 0;
25571a8031e1Shsuenaga }
25581a8031e1Shsuenaga 
25591a8031e1Shsuenaga STATIC int
mvxpsec_packet_setuio(struct mvxpsec_packet * mv_p,struct uio * uio)25601a8031e1Shsuenaga mvxpsec_packet_setuio(struct mvxpsec_packet *mv_p, struct uio *uio)
25611a8031e1Shsuenaga {
25621a8031e1Shsuenaga 	struct mvxpsec_session *mv_s = mv_p->mv_s;
25631a8031e1Shsuenaga 	struct mvxpsec_softc *sc = mv_s->sc;
25641a8031e1Shsuenaga 
25651a8031e1Shsuenaga 	if (uio->uio_resid > SRAM_PAYLOAD_SIZE) {
25662499f515Sriastradh #if NIPSEC > 0
25671a8031e1Shsuenaga 		extern   percpu_t *espstat_percpu;
25681a8031e1Shsuenaga 	       	/* XXX:
25691a8031e1Shsuenaga 		 * layer violation. opencrypto knows our max packet size
25701a8031e1Shsuenaga 		 * from crypto_register(9) API.
25711a8031e1Shsuenaga 		 */
25721a8031e1Shsuenaga 
25731a8031e1Shsuenaga 		_NET_STATINC(espstat_percpu, ESP_STAT_TOOBIG);
25742499f515Sriastradh #endif
25751a8031e1Shsuenaga 		log(LOG_ERR,
25761a8031e1Shsuenaga 		    "%s: uio request too large: %zu [oct.] > %zu [oct.]\n",
25771a8031e1Shsuenaga 		    device_xname(sc->sc_dev),
25781a8031e1Shsuenaga 		    uio->uio_resid, SRAM_PAYLOAD_SIZE);
25791a8031e1Shsuenaga 		mv_p->data_type = MVXPSEC_DATA_NONE;
25801a8031e1Shsuenaga 		mv_p->data_mbuf = NULL;
25811a8031e1Shsuenaga 		return -1;
25821a8031e1Shsuenaga 	}
25831a8031e1Shsuenaga 
25841a8031e1Shsuenaga 	if (bus_dmamap_load_uio(sc->sc_dmat, mv_p->data_map, uio,
25851a8031e1Shsuenaga 	    BUS_DMA_NOWAIT)) {
25861a8031e1Shsuenaga 		mv_p->data_type = MVXPSEC_DATA_NONE;
25871a8031e1Shsuenaga 		mv_p->data_mbuf = NULL;
25881a8031e1Shsuenaga 		log(LOG_ERR, "%s: cannot load uio buf\n", __func__);
25891a8031e1Shsuenaga 		return -1;
25901a8031e1Shsuenaga 	}
25911a8031e1Shsuenaga 
25921a8031e1Shsuenaga 	/* set payload buffer */
25931a8031e1Shsuenaga 	mv_p->data_type = MVXPSEC_DATA_UIO;
25941a8031e1Shsuenaga 	mv_p->data_uio = uio;
25951a8031e1Shsuenaga 	mv_p->data_len = uio->uio_resid;
25961a8031e1Shsuenaga 	mv_p->flags |= RDY_DATA;
25971a8031e1Shsuenaga 
25981a8031e1Shsuenaga 	return 0;
25991a8031e1Shsuenaga }
26001a8031e1Shsuenaga 
26011a8031e1Shsuenaga STATIC int
mvxpsec_packet_rdata(struct mvxpsec_packet * mv_p,int off,int len,void * cp)26021a8031e1Shsuenaga mvxpsec_packet_rdata(struct mvxpsec_packet *mv_p,
26031a8031e1Shsuenaga     int off, int len, void *cp)
26041a8031e1Shsuenaga {
26051a8031e1Shsuenaga 	uint8_t *p;
26061a8031e1Shsuenaga 
26071a8031e1Shsuenaga 	if (mv_p->data_type == MVXPSEC_DATA_RAW) {
26081a8031e1Shsuenaga 		p = (uint8_t *)mv_p->data_raw + off;
26091a8031e1Shsuenaga 		memcpy(cp, p, len);
26101a8031e1Shsuenaga 	}
26111a8031e1Shsuenaga 	else if (mv_p->data_type == MVXPSEC_DATA_MBUF) {
26121a8031e1Shsuenaga 		m_copydata(mv_p->data_mbuf, off, len, cp);
26131a8031e1Shsuenaga 	}
26141a8031e1Shsuenaga 	else if (mv_p->data_type == MVXPSEC_DATA_UIO) {
26151a8031e1Shsuenaga 		cuio_copydata(mv_p->data_uio, off, len, cp);
26161a8031e1Shsuenaga 	}
26171a8031e1Shsuenaga 	else
26181a8031e1Shsuenaga 		return -1;
26191a8031e1Shsuenaga 
26201a8031e1Shsuenaga 	return 0;
26211a8031e1Shsuenaga }
26221a8031e1Shsuenaga 
26231a8031e1Shsuenaga STATIC int
mvxpsec_packet_wdata(struct mvxpsec_packet * mv_p,int off,int len,void * cp)26241a8031e1Shsuenaga mvxpsec_packet_wdata(struct mvxpsec_packet *mv_p,
26251a8031e1Shsuenaga     int off, int len, void *cp)
26261a8031e1Shsuenaga {
26271a8031e1Shsuenaga 	uint8_t *p;
26281a8031e1Shsuenaga 
26291a8031e1Shsuenaga 	if (mv_p->data_type == MVXPSEC_DATA_RAW) {
26301a8031e1Shsuenaga 		p = (uint8_t *)mv_p->data_raw + off;
26311a8031e1Shsuenaga 		memcpy(p, cp, len);
26321a8031e1Shsuenaga 	}
26331a8031e1Shsuenaga 	else if (mv_p->data_type == MVXPSEC_DATA_MBUF) {
26341a8031e1Shsuenaga 		m_copyback(mv_p->data_mbuf, off, len, cp);
26351a8031e1Shsuenaga 	}
26361a8031e1Shsuenaga 	else if (mv_p->data_type == MVXPSEC_DATA_UIO) {
26371a8031e1Shsuenaga 		cuio_copyback(mv_p->data_uio, off, len, cp);
26381a8031e1Shsuenaga 	}
26391a8031e1Shsuenaga 	else
26401a8031e1Shsuenaga 		return -1;
26411a8031e1Shsuenaga 
26421a8031e1Shsuenaga 	return 0;
26431a8031e1Shsuenaga }
26441a8031e1Shsuenaga 
26451a8031e1Shsuenaga /*
26461a8031e1Shsuenaga  * Set initial vector of cipher to the session.
26471a8031e1Shsuenaga  */
26481a8031e1Shsuenaga STATIC int
mvxpsec_packet_write_iv(struct mvxpsec_packet * mv_p,void * iv,int ivlen)26491a8031e1Shsuenaga mvxpsec_packet_write_iv(struct mvxpsec_packet *mv_p, void *iv, int ivlen)
26501a8031e1Shsuenaga {
26511a8031e1Shsuenaga 	uint8_t ivbuf[16];
26521a8031e1Shsuenaga 
26531a8031e1Shsuenaga 	KASSERT(ivlen == 8 || ivlen == 16);
26541a8031e1Shsuenaga 
26551a8031e1Shsuenaga 	if (iv == NULL) {
26561a8031e1Shsuenaga 	       	if (mv_p->mv_s->sflags & RDY_CRP_IV) {
26571a8031e1Shsuenaga 			/* use per session IV (compatible with KAME IPsec) */
26581a8031e1Shsuenaga 			mv_p->pkt_header.crp_iv_work = mv_p->mv_s->session_iv;
26591a8031e1Shsuenaga 			mv_p->flags |= RDY_CRP_IV;
26601a8031e1Shsuenaga 			return 0;
26611a8031e1Shsuenaga 		}
26621a8031e1Shsuenaga 		cprng_fast(ivbuf, ivlen);
26631a8031e1Shsuenaga 		iv = ivbuf;
26641a8031e1Shsuenaga 	}
26651a8031e1Shsuenaga 	memcpy(&mv_p->pkt_header.crp_iv_work, iv, ivlen);
26661a8031e1Shsuenaga 	if (mv_p->flags & CRP_EXT_IV) {
26671a8031e1Shsuenaga 		memcpy(&mv_p->pkt_header.crp_iv_ext, iv, ivlen);
26681a8031e1Shsuenaga 		mv_p->ext_iv = iv;
26691a8031e1Shsuenaga 		mv_p->ext_ivlen = ivlen;
26701a8031e1Shsuenaga 	}
26711a8031e1Shsuenaga 	mv_p->flags |= RDY_CRP_IV;
26721a8031e1Shsuenaga 
26731a8031e1Shsuenaga 	return 0;
26741a8031e1Shsuenaga }
26751a8031e1Shsuenaga 
26761a8031e1Shsuenaga STATIC int
mvxpsec_packet_copy_iv(struct mvxpsec_packet * mv_p,int off,int ivlen)26771a8031e1Shsuenaga mvxpsec_packet_copy_iv(struct mvxpsec_packet *mv_p, int off, int ivlen)
26781a8031e1Shsuenaga {
26791a8031e1Shsuenaga 	mvxpsec_packet_rdata(mv_p, off, ivlen,
26801a8031e1Shsuenaga 	    &mv_p->pkt_header.crp_iv_work);
26811a8031e1Shsuenaga 	mv_p->flags |= RDY_CRP_IV;
26821a8031e1Shsuenaga 
26831a8031e1Shsuenaga 	return 0;
26841a8031e1Shsuenaga }
26851a8031e1Shsuenaga 
26861a8031e1Shsuenaga /*
26871a8031e1Shsuenaga  * set a encryption or decryption key to the session
26881a8031e1Shsuenaga  *
26891a8031e1Shsuenaga  * Input key material is big endian.
26901a8031e1Shsuenaga  */
26911a8031e1Shsuenaga STATIC int
mvxpsec_key_precomp(int alg,void * keymat,int kbitlen,void * key_encrypt,void * key_decrypt)26921a8031e1Shsuenaga mvxpsec_key_precomp(int alg, void *keymat, int kbitlen,
26931a8031e1Shsuenaga     void *key_encrypt, void *key_decrypt)
26941a8031e1Shsuenaga {
26951a8031e1Shsuenaga 	uint32_t *kp = keymat;
26961a8031e1Shsuenaga 	uint32_t *ekp = key_encrypt;
26971a8031e1Shsuenaga 	uint32_t *dkp = key_decrypt;
26981a8031e1Shsuenaga 	int i;
26991a8031e1Shsuenaga 
27001a8031e1Shsuenaga 	switch (alg) {
27011a8031e1Shsuenaga 	case CRYPTO_DES_CBC:
27021a8031e1Shsuenaga 		if (kbitlen < 64 || (kbitlen % 8) != 0) {
27031a8031e1Shsuenaga 			log(LOG_WARNING,
27041a8031e1Shsuenaga 			    "mvxpsec: invalid DES keylen %d\n", kbitlen);
27051a8031e1Shsuenaga 			return EINVAL;
27061a8031e1Shsuenaga 		}
27071a8031e1Shsuenaga 		for (i = 0; i < 2; i++)
27081a8031e1Shsuenaga 			dkp[i] = ekp[i] = kp[i];
27091a8031e1Shsuenaga 		for (; i < 8; i++)
27101a8031e1Shsuenaga 			dkp[i] = ekp[i] = 0;
27111a8031e1Shsuenaga 		break;
27121a8031e1Shsuenaga 	case CRYPTO_3DES_CBC:
27131a8031e1Shsuenaga 		if (kbitlen < 192 || (kbitlen % 8) != 0) {
27141a8031e1Shsuenaga 			log(LOG_WARNING,
27151a8031e1Shsuenaga 			    "mvxpsec: invalid 3DES keylen %d\n", kbitlen);
27161a8031e1Shsuenaga 			return EINVAL;
27171a8031e1Shsuenaga 		}
27181a8031e1Shsuenaga 		for (i = 0; i < 8; i++)
27191a8031e1Shsuenaga 			dkp[i] = ekp[i] = kp[i];
27201a8031e1Shsuenaga 		break;
27211a8031e1Shsuenaga 	case CRYPTO_AES_CBC:
27221a8031e1Shsuenaga 		if (kbitlen < 128) {
27231a8031e1Shsuenaga 			log(LOG_WARNING,
27241a8031e1Shsuenaga 			    "mvxpsec: invalid AES keylen %d\n", kbitlen);
27251a8031e1Shsuenaga 			return EINVAL;
27261a8031e1Shsuenaga 		}
27271a8031e1Shsuenaga 		else if (kbitlen < 192) {
27281a8031e1Shsuenaga 			/* AES-128 */
27291a8031e1Shsuenaga 			for (i = 0; i < 4; i++)
27301a8031e1Shsuenaga 				ekp[i] = kp[i];
27311a8031e1Shsuenaga 			for (; i < 8; i++)
27321a8031e1Shsuenaga 				ekp[i] = 0;
27331a8031e1Shsuenaga 		}
27341a8031e1Shsuenaga 	       	else if (kbitlen < 256) {
27351a8031e1Shsuenaga 			/* AES-192 */
27361a8031e1Shsuenaga 			for (i = 0; i < 6; i++)
27371a8031e1Shsuenaga 				ekp[i] = kp[i];
27381a8031e1Shsuenaga 			for (; i < 8; i++)
27391a8031e1Shsuenaga 				ekp[i] = 0;
27401a8031e1Shsuenaga 		}
27411a8031e1Shsuenaga 		else  {
27421a8031e1Shsuenaga 			/* AES-256 */
27431a8031e1Shsuenaga 			for (i = 0; i < 8; i++)
27441a8031e1Shsuenaga 				ekp[i] = kp[i];
27451a8031e1Shsuenaga 		}
27461a8031e1Shsuenaga 		/* make decryption key */
27471a8031e1Shsuenaga 		mv_aes_deckey((uint8_t *)dkp, (uint8_t *)ekp, kbitlen);
27481a8031e1Shsuenaga 		break;
27491a8031e1Shsuenaga 	default:
27501a8031e1Shsuenaga 		for (i = 0; i < 8; i++)
27511a8031e1Shsuenaga 			ekp[0] = dkp[0] = 0;
27521a8031e1Shsuenaga 		break;
27531a8031e1Shsuenaga 	}
27541a8031e1Shsuenaga 
27551a8031e1Shsuenaga #ifdef MVXPSEC_DEBUG
27561a8031e1Shsuenaga 	if (mvxpsec_debug & MVXPSEC_DEBUG_OPENCRYPTO) {
27571a8031e1Shsuenaga 		MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO,
27581a8031e1Shsuenaga 		    "%s: keyregistered\n", __func__);
27591a8031e1Shsuenaga 		mvxpsec_dump_data(__func__, ekp, 32);
27601a8031e1Shsuenaga 	}
27611a8031e1Shsuenaga #endif
27621a8031e1Shsuenaga 
27631a8031e1Shsuenaga 	return 0;
27641a8031e1Shsuenaga }
27651a8031e1Shsuenaga 
27661a8031e1Shsuenaga /*
27671a8031e1Shsuenaga  * set MAC key to the session
27681a8031e1Shsuenaga  *
27691a8031e1Shsuenaga  * MAC engine has no register for key itself, but the engine has
27701a8031e1Shsuenaga  * inner and outer IV register. software must compute IV before
27711a8031e1Shsuenaga  * enable the engine.
27721a8031e1Shsuenaga  *
27731a8031e1Shsuenaga  * IV is a hash of ipad/opad. these are defined by FIPS-198a
27741a8031e1Shsuenaga  * standard.
27751a8031e1Shsuenaga  */
27761a8031e1Shsuenaga STATIC int
mvxpsec_hmac_precomp(int alg,void * key,int kbitlen,void * iv_inner,void * iv_outer)27771a8031e1Shsuenaga mvxpsec_hmac_precomp(int alg, void *key, int kbitlen,
27781a8031e1Shsuenaga     void *iv_inner, void *iv_outer)
27791a8031e1Shsuenaga {
27801a8031e1Shsuenaga 	SHA1_CTX sha1;
27811a8031e1Shsuenaga 	MD5_CTX md5;
27821a8031e1Shsuenaga 	uint8_t *key8 = key;
27831a8031e1Shsuenaga 	uint8_t kbuf[64];
27841a8031e1Shsuenaga 	uint8_t ipad[64];
27851a8031e1Shsuenaga 	uint8_t opad[64];
27861a8031e1Shsuenaga 	uint32_t *iv_in = iv_inner;
27871a8031e1Shsuenaga 	uint32_t *iv_out = iv_outer;
27881a8031e1Shsuenaga 	int kbytelen;
27891a8031e1Shsuenaga 	int i;
27901a8031e1Shsuenaga #define HMAC_IPAD 0x36
27911a8031e1Shsuenaga #define HMAC_OPAD 0x5c
27921a8031e1Shsuenaga 
27931a8031e1Shsuenaga 	kbytelen = kbitlen / 8;
27941a8031e1Shsuenaga 	KASSERT(kbitlen == kbytelen * 8);
27951a8031e1Shsuenaga 	if (kbytelen > 64) {
27961a8031e1Shsuenaga 		SHA1Init(&sha1);
27971a8031e1Shsuenaga 		SHA1Update(&sha1, key, kbytelen);
27981a8031e1Shsuenaga 		SHA1Final(kbuf, &sha1);
27991a8031e1Shsuenaga 		key8 = kbuf;
28001a8031e1Shsuenaga 		kbytelen = 64;
28011a8031e1Shsuenaga 	}
28021a8031e1Shsuenaga 
28031a8031e1Shsuenaga 	/* make initial 64 oct. string */
28041a8031e1Shsuenaga 	switch (alg) {
28051a8031e1Shsuenaga 	case CRYPTO_SHA1_HMAC_96:
28061a8031e1Shsuenaga 	case CRYPTO_SHA1_HMAC:
28071a8031e1Shsuenaga 	case CRYPTO_MD5_HMAC_96:
28081a8031e1Shsuenaga 	case CRYPTO_MD5_HMAC:
28091a8031e1Shsuenaga 		for (i = 0; i < kbytelen; i++) {
28101a8031e1Shsuenaga 			ipad[i] = (key8[i] ^ HMAC_IPAD);
28111a8031e1Shsuenaga 			opad[i] = (key8[i] ^ HMAC_OPAD);
28121a8031e1Shsuenaga 		}
28131a8031e1Shsuenaga 		for (; i < 64; i++) {
28141a8031e1Shsuenaga 			ipad[i] = HMAC_IPAD;
28151a8031e1Shsuenaga 			opad[i] = HMAC_OPAD;
28161a8031e1Shsuenaga 		}
28171a8031e1Shsuenaga 		break;
28181a8031e1Shsuenaga 	default:
28191a8031e1Shsuenaga 		break;
28201a8031e1Shsuenaga 	}
28211a8031e1Shsuenaga #ifdef MVXPSEC_DEBUG
28221a8031e1Shsuenaga 	if (mvxpsec_debug & MVXPSEC_DEBUG_OPENCRYPTO) {
28231a8031e1Shsuenaga 		MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO,
28241a8031e1Shsuenaga 		    "%s: HMAC-KEY Pre-comp:\n", __func__);
28251a8031e1Shsuenaga 		mvxpsec_dump_data(__func__, key, 64);
28261a8031e1Shsuenaga 		MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO,
28271a8031e1Shsuenaga 		    "%s: ipad:\n", __func__);
28281a8031e1Shsuenaga 		mvxpsec_dump_data(__func__, ipad, sizeof(ipad));
28291a8031e1Shsuenaga 		MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO,
28301a8031e1Shsuenaga 		    "%s: opad:\n", __func__);
28311a8031e1Shsuenaga 		mvxpsec_dump_data(__func__, opad, sizeof(opad));
28321a8031e1Shsuenaga 	}
28331a8031e1Shsuenaga #endif
28341a8031e1Shsuenaga 
28351a8031e1Shsuenaga 	/* make iv from string */
28361a8031e1Shsuenaga 	switch (alg) {
28371a8031e1Shsuenaga 	case CRYPTO_SHA1_HMAC_96:
28381a8031e1Shsuenaga 	case CRYPTO_SHA1_HMAC:
28391a8031e1Shsuenaga 		MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO,
28401a8031e1Shsuenaga 		    "%s: Generate iv_in(SHA1)\n", __func__);
28411a8031e1Shsuenaga 		SHA1Init(&sha1);
28421a8031e1Shsuenaga 		SHA1Update(&sha1, ipad, 64);
28431a8031e1Shsuenaga 		/* XXX: private state... (LE) */
28441a8031e1Shsuenaga 		iv_in[0] = htobe32(sha1.state[0]);
28451a8031e1Shsuenaga 		iv_in[1] = htobe32(sha1.state[1]);
28461a8031e1Shsuenaga 		iv_in[2] = htobe32(sha1.state[2]);
28471a8031e1Shsuenaga 		iv_in[3] = htobe32(sha1.state[3]);
28481a8031e1Shsuenaga 		iv_in[4] = htobe32(sha1.state[4]);
28491a8031e1Shsuenaga 
28501a8031e1Shsuenaga 		MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO,
28511a8031e1Shsuenaga 		    "%s: Generate iv_out(SHA1)\n", __func__);
28521a8031e1Shsuenaga 		SHA1Init(&sha1);
28531a8031e1Shsuenaga 		SHA1Update(&sha1, opad, 64);
28541a8031e1Shsuenaga 		/* XXX: private state... (LE) */
28551a8031e1Shsuenaga 		iv_out[0] = htobe32(sha1.state[0]);
28561a8031e1Shsuenaga 		iv_out[1] = htobe32(sha1.state[1]);
28571a8031e1Shsuenaga 		iv_out[2] = htobe32(sha1.state[2]);
28581a8031e1Shsuenaga 		iv_out[3] = htobe32(sha1.state[3]);
28591a8031e1Shsuenaga 		iv_out[4] = htobe32(sha1.state[4]);
28601a8031e1Shsuenaga 		break;
28611a8031e1Shsuenaga 	case CRYPTO_MD5_HMAC_96:
28621a8031e1Shsuenaga 	case CRYPTO_MD5_HMAC:
28631a8031e1Shsuenaga 		MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO,
28641a8031e1Shsuenaga 		    "%s: Generate iv_in(MD5)\n", __func__);
28651a8031e1Shsuenaga 		MD5Init(&md5);
28661a8031e1Shsuenaga 		MD5Update(&md5, ipad, sizeof(ipad));
28671a8031e1Shsuenaga 		/* XXX: private state... (LE) */
28681a8031e1Shsuenaga 		iv_in[0] = htobe32(md5.state[0]);
28691a8031e1Shsuenaga 		iv_in[1] = htobe32(md5.state[1]);
28701a8031e1Shsuenaga 		iv_in[2] = htobe32(md5.state[2]);
28711a8031e1Shsuenaga 		iv_in[3] = htobe32(md5.state[3]);
28721a8031e1Shsuenaga 		iv_in[4] = 0;
28731a8031e1Shsuenaga 
28741a8031e1Shsuenaga 		MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO,
28751a8031e1Shsuenaga 		    "%s: Generate iv_out(MD5)\n", __func__);
28761a8031e1Shsuenaga 		MD5Init(&md5);
28771a8031e1Shsuenaga 		MD5Update(&md5, opad, sizeof(opad));
28781a8031e1Shsuenaga 		/* XXX: private state... (LE) */
28791a8031e1Shsuenaga 		iv_out[0] = htobe32(md5.state[0]);
28801a8031e1Shsuenaga 		iv_out[1] = htobe32(md5.state[1]);
28811a8031e1Shsuenaga 		iv_out[2] = htobe32(md5.state[2]);
28821a8031e1Shsuenaga 		iv_out[3] = htobe32(md5.state[3]);
28831a8031e1Shsuenaga 		iv_out[4] = 0;
28841a8031e1Shsuenaga 		break;
28851a8031e1Shsuenaga 	default:
28861a8031e1Shsuenaga 		break;
28871a8031e1Shsuenaga 	}
28881a8031e1Shsuenaga 
28891a8031e1Shsuenaga #ifdef MVXPSEC_DEBUG
28901a8031e1Shsuenaga 	if (mvxpsec_debug & MVXPSEC_DEBUG_HASH_IV) {
28911a8031e1Shsuenaga 		MVXPSEC_PRINTF(MVXPSEC_DEBUG_HASH_IV,
28921a8031e1Shsuenaga 		    "%s: HMAC IV-IN\n", __func__);
28931a8031e1Shsuenaga 		mvxpsec_dump_data(__func__, (uint8_t *)iv_in, 20);
28941a8031e1Shsuenaga 		MVXPSEC_PRINTF(MVXPSEC_DEBUG_HASH_IV,
28951a8031e1Shsuenaga 		    "%s: HMAC IV-OUT\n", __func__);
28961a8031e1Shsuenaga 		mvxpsec_dump_data(__func__, (uint8_t *)iv_out, 20);
28971a8031e1Shsuenaga 	}
28981a8031e1Shsuenaga #endif
28991a8031e1Shsuenaga 
29001a8031e1Shsuenaga 	return 0;
29011a8031e1Shsuenaga #undef HMAC_IPAD
29021a8031e1Shsuenaga #undef HMAC_OPAD
29031a8031e1Shsuenaga }
29041a8031e1Shsuenaga 
29051a8031e1Shsuenaga /*
29061a8031e1Shsuenaga  * AES Support routine
29071a8031e1Shsuenaga  */
29081a8031e1Shsuenaga static uint8_t AES_SBOX[256] = {
29091a8031e1Shsuenaga 	 99, 124, 119, 123, 242, 107, 111, 197,  48,   1, 103,  43, 254, 215,
29101a8031e1Shsuenaga        	171, 118, 202, 130, 201, 125, 250,  89,  71, 240, 173, 212, 162, 175,
29111a8031e1Shsuenaga        	156, 164, 114, 192, 183, 253, 147,  38,  54,  63, 247, 204,  52, 165,
29121a8031e1Shsuenaga        	229, 241, 113, 216,  49,  21,   4, 199,  35, 195,  24, 150,   5, 154,
29131a8031e1Shsuenaga        	  7,  18, 128, 226, 235,  39, 178, 117,   9, 131,  44,  26,  27, 110,
29141a8031e1Shsuenaga 	 90, 160,  82,  59, 214, 179,  41, 227,  47, 132,  83, 209,   0, 237,
29151a8031e1Shsuenaga        	 32, 252, 177,  91, 106, 203, 190,  57,  74,  76,  88, 207, 208, 239,
29161a8031e1Shsuenaga 	170, 251,  67,  77,  51, 133,  69, 249,   2, 127,  80,  60, 159, 168,
29171a8031e1Shsuenaga 	 81, 163,  64, 143, 146, 157,  56, 245, 188, 182, 218,  33,  16, 255,
29181a8031e1Shsuenaga 	243, 210, 205,  12,  19, 236,  95, 151,  68,  23, 196, 167, 126,  61,
29191a8031e1Shsuenaga        	100,  93,  25, 115,  96, 129,  79, 220,  34,  42, 144, 136,  70, 238,
29201a8031e1Shsuenaga        	184,  20, 222,  94,  11, 219, 224,  50,  58,  10,  73,   6,  36,  92,
29211a8031e1Shsuenaga        	194, 211, 172,  98, 145, 149, 228, 121, 231, 200,  55, 109, 141, 213,
29221a8031e1Shsuenaga       	 78, 169, 108,  86, 244, 234, 101, 122, 174,   8, 186, 120,  37,  46,
29231a8031e1Shsuenaga        	 28, 166, 180, 198, 232, 221, 116,  31,  75, 189, 139, 138, 112,  62,
29241a8031e1Shsuenaga 	181, 102,  72,   3, 246,  14,  97,  53,  87, 185, 134, 193,  29, 158,
29251a8031e1Shsuenaga        	225, 248, 152,  17, 105, 217, 142, 148, 155,  30, 135, 233, 206,  85,
29261a8031e1Shsuenaga       	 40, 223, 140, 161, 137,  13, 191, 230,  66, 104,  65, 153,  45,  15,
29271a8031e1Shsuenaga 	176,  84, 187,  22
29281a8031e1Shsuenaga };
29291a8031e1Shsuenaga 
29301a8031e1Shsuenaga static uint32_t AES_RCON[30] = {
29311a8031e1Shsuenaga 	0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36, 0x6c, 0xd8,
29321a8031e1Shsuenaga        	0xab, 0x4d, 0x9a, 0x2f, 0x5e, 0xbc, 0x63, 0xc6, 0x97, 0x35, 0x6a, 0xd4,
29331a8031e1Shsuenaga        	0xb3, 0x7d, 0xfa, 0xef, 0xc5, 0x91
29341a8031e1Shsuenaga };
29351a8031e1Shsuenaga 
29361a8031e1Shsuenaga STATIC int
mv_aes_ksched(uint8_t k[4][MAXKC],int keyBits,uint8_t W[MAXROUNDS+1][4][MAXBC])29371a8031e1Shsuenaga mv_aes_ksched(uint8_t k[4][MAXKC], int keyBits,
29381a8031e1Shsuenaga     uint8_t W[MAXROUNDS+1][4][MAXBC])
29391a8031e1Shsuenaga {
29401a8031e1Shsuenaga 	int KC, BC, ROUNDS;
29411a8031e1Shsuenaga 	int i, j, t, rconpointer = 0;
29421a8031e1Shsuenaga 	uint8_t tk[4][MAXKC];
29431a8031e1Shsuenaga 
29441a8031e1Shsuenaga 	switch (keyBits) {
29451a8031e1Shsuenaga 	case 128:
29461a8031e1Shsuenaga 		ROUNDS = 10;
29471a8031e1Shsuenaga 		KC = 4;
29481a8031e1Shsuenaga 		break;
29491a8031e1Shsuenaga 	case 192:
29501a8031e1Shsuenaga 		ROUNDS = 12;
29511a8031e1Shsuenaga 		KC = 6;
29521a8031e1Shsuenaga 	       	break;
29531a8031e1Shsuenaga 	case 256:
29541a8031e1Shsuenaga 		ROUNDS = 14;
29551a8031e1Shsuenaga 	       	KC = 8;
29561a8031e1Shsuenaga 	       	break;
29571a8031e1Shsuenaga 	default:
29581a8031e1Shsuenaga 	       	return (-1);
29591a8031e1Shsuenaga 	}
29601a8031e1Shsuenaga 	BC = 4; /* 128 bits */
29611a8031e1Shsuenaga 
29621a8031e1Shsuenaga 	for(j = 0; j < KC; j++)
29631a8031e1Shsuenaga 		for(i = 0; i < 4; i++)
29641a8031e1Shsuenaga 			tk[i][j] = k[i][j];
29651a8031e1Shsuenaga 	t = 0;
29661a8031e1Shsuenaga 
29671a8031e1Shsuenaga 	/* copy values into round key array */
29681a8031e1Shsuenaga 	for(j = 0; (j < KC) && (t < (ROUNDS+1)*BC); j++, t++)
29691a8031e1Shsuenaga 		for(i = 0; i < 4; i++) W[t / BC][i][t % BC] = tk[i][j];
29701a8031e1Shsuenaga 
29711a8031e1Shsuenaga 	while (t < (ROUNDS+1)*BC) { /* while not enough round key material calculated */
29721a8031e1Shsuenaga 		/* calculate new values */
29731a8031e1Shsuenaga 		for(i = 0; i < 4; i++)
29741a8031e1Shsuenaga 			tk[i][0] ^= AES_SBOX[tk[(i+1)%4][KC-1]];
29751a8031e1Shsuenaga 		tk[0][0] ^= AES_RCON[rconpointer++];
29761a8031e1Shsuenaga 
29771a8031e1Shsuenaga 		if (KC != 8)
29781a8031e1Shsuenaga 			for(j = 1; j < KC; j++)
29791a8031e1Shsuenaga 				for(i = 0; i < 4; i++)
29801a8031e1Shsuenaga 				       	tk[i][j] ^= tk[i][j-1];
29811a8031e1Shsuenaga 		else {
29821a8031e1Shsuenaga 			for(j = 1; j < KC/2; j++)
29831a8031e1Shsuenaga 				for(i = 0; i < 4; i++)
29841a8031e1Shsuenaga 				       	tk[i][j] ^= tk[i][j-1];
29851a8031e1Shsuenaga 			for(i = 0; i < 4; i++)
29861a8031e1Shsuenaga 			       	tk[i][KC/2] ^= AES_SBOX[tk[i][KC/2 - 1]];
29871a8031e1Shsuenaga 			for(j = KC/2 + 1; j < KC; j++)
29881a8031e1Shsuenaga 				for(i = 0; i < 4; i++)
29891a8031e1Shsuenaga 				       	tk[i][j] ^= tk[i][j-1];
29901a8031e1Shsuenaga 	}
29911a8031e1Shsuenaga 	/* copy values into round key array */
29921a8031e1Shsuenaga 	for(j = 0; (j < KC) && (t < (ROUNDS+1)*BC); j++, t++)
29931a8031e1Shsuenaga 		for(i = 0; i < 4; i++) W[t / BC][i][t % BC] = tk[i][j];
29941a8031e1Shsuenaga 	}
29951a8031e1Shsuenaga 
29961a8031e1Shsuenaga 	return 0;
29971a8031e1Shsuenaga }
29981a8031e1Shsuenaga 
29991a8031e1Shsuenaga STATIC int
mv_aes_deckey(uint8_t * expandedKey,uint8_t * keyMaterial,int keyLen)30001a8031e1Shsuenaga mv_aes_deckey(uint8_t *expandedKey, uint8_t *keyMaterial, int keyLen)
30011a8031e1Shsuenaga {
30021a8031e1Shsuenaga 	uint8_t   W[MAXROUNDS+1][4][MAXBC];
30031a8031e1Shsuenaga 	uint8_t   k[4][MAXKC];
30041a8031e1Shsuenaga 	uint8_t   j;
30051a8031e1Shsuenaga 	int     i, rounds, KC;
30061a8031e1Shsuenaga 
30071a8031e1Shsuenaga 	if (expandedKey == NULL)
30081a8031e1Shsuenaga 		return -1;
30091a8031e1Shsuenaga 
30101a8031e1Shsuenaga 	if (!((keyLen == 128) || (keyLen == 192) || (keyLen == 256)))
30111a8031e1Shsuenaga 		return -1;
30121a8031e1Shsuenaga 
30131a8031e1Shsuenaga 	if (keyMaterial == NULL)
30141a8031e1Shsuenaga 		return -1;
30151a8031e1Shsuenaga 
30161a8031e1Shsuenaga 	/* initialize key schedule: */
30171a8031e1Shsuenaga 	for (i=0; i<keyLen/8; i++) {
30181a8031e1Shsuenaga 		j = keyMaterial[i];
30191a8031e1Shsuenaga 		k[i % 4][i / 4] = j;
30201a8031e1Shsuenaga 	}
30211a8031e1Shsuenaga 
30221a8031e1Shsuenaga 	mv_aes_ksched(k, keyLen, W);
30231a8031e1Shsuenaga 	switch (keyLen) {
30241a8031e1Shsuenaga 	case 128:
30251a8031e1Shsuenaga 		rounds = 10;
30261a8031e1Shsuenaga 		KC = 4;
30271a8031e1Shsuenaga 		break;
30281a8031e1Shsuenaga 	case 192:
30291a8031e1Shsuenaga 		rounds = 12;
30301a8031e1Shsuenaga 		KC = 6;
30311a8031e1Shsuenaga 		break;
30321a8031e1Shsuenaga 	case 256:
30331a8031e1Shsuenaga 		rounds = 14;
30341a8031e1Shsuenaga 		KC = 8;
30351a8031e1Shsuenaga 		break;
30361a8031e1Shsuenaga 	default:
30371a8031e1Shsuenaga 		return -1;
30381a8031e1Shsuenaga 	}
30391a8031e1Shsuenaga 
30401a8031e1Shsuenaga 	for(i=0; i<MAXBC; i++)
30411a8031e1Shsuenaga 		for(j=0; j<4; j++)
30421a8031e1Shsuenaga 			expandedKey[i*4+j] = W[rounds][j][i];
30431a8031e1Shsuenaga 	for(; i<KC; i++)
30441a8031e1Shsuenaga 		for(j=0; j<4; j++)
30451a8031e1Shsuenaga 			expandedKey[i*4+j] = W[rounds-1][j][i+MAXBC-KC];
30461a8031e1Shsuenaga 
30471a8031e1Shsuenaga 	return 0;
30481a8031e1Shsuenaga }
30491a8031e1Shsuenaga 
30501a8031e1Shsuenaga /*
30511a8031e1Shsuenaga  * Clear cipher/mac operation state
30521a8031e1Shsuenaga  */
30531a8031e1Shsuenaga INLINE void
mvxpsec_packet_reset_op(struct mvxpsec_packet * mv_p)30541a8031e1Shsuenaga mvxpsec_packet_reset_op(struct mvxpsec_packet *mv_p)
30551a8031e1Shsuenaga {
30561a8031e1Shsuenaga 	mv_p->pkt_header.desc.acc_config = 0;
30571a8031e1Shsuenaga 	mv_p->enc_off = mv_p->enc_ivoff = mv_p->enc_len = 0;
30581a8031e1Shsuenaga 	mv_p->mac_off = mv_p->mac_dst = mv_p->mac_len = 0;
30591a8031e1Shsuenaga }
30601a8031e1Shsuenaga 
30611a8031e1Shsuenaga /*
30621a8031e1Shsuenaga  * update MVXPSEC operation order
30631a8031e1Shsuenaga  */
30641a8031e1Shsuenaga INLINE void
mvxpsec_packet_update_op_order(struct mvxpsec_packet * mv_p,int op)30651a8031e1Shsuenaga mvxpsec_packet_update_op_order(struct mvxpsec_packet *mv_p, int op)
30661a8031e1Shsuenaga {
30671a8031e1Shsuenaga 	struct mvxpsec_acc_descriptor *acc_desc = &mv_p->pkt_header.desc;
30681a8031e1Shsuenaga 	uint32_t cur_op = acc_desc->acc_config & MV_ACC_CRYPTO_OP_MASK;
30691a8031e1Shsuenaga 
30701a8031e1Shsuenaga 	KASSERT(op == MV_ACC_CRYPTO_OP_MAC || op == MV_ACC_CRYPTO_OP_ENC);
30711a8031e1Shsuenaga 	KASSERT((op & MV_ACC_CRYPTO_OP_MASK) == op);
30721a8031e1Shsuenaga 
30731a8031e1Shsuenaga 	if (cur_op == 0)
30741a8031e1Shsuenaga 		acc_desc->acc_config |= op;
30751a8031e1Shsuenaga 	else if (cur_op == MV_ACC_CRYPTO_OP_MAC && op == MV_ACC_CRYPTO_OP_ENC) {
30761a8031e1Shsuenaga 		acc_desc->acc_config &= ~MV_ACC_CRYPTO_OP_MASK;
30771a8031e1Shsuenaga 		acc_desc->acc_config |= MV_ACC_CRYPTO_OP_MACENC;
30781a8031e1Shsuenaga 		/* MAC then ENC (= decryption) */
30791a8031e1Shsuenaga 	}
30801a8031e1Shsuenaga 	else if (cur_op == MV_ACC_CRYPTO_OP_ENC && op == MV_ACC_CRYPTO_OP_MAC) {
30811a8031e1Shsuenaga 		acc_desc->acc_config &= ~MV_ACC_CRYPTO_OP_MASK;
30821a8031e1Shsuenaga 		acc_desc->acc_config |= MV_ACC_CRYPTO_OP_ENCMAC;
30831a8031e1Shsuenaga 		/* ENC then MAC (= encryption) */
30841a8031e1Shsuenaga 	}
30851a8031e1Shsuenaga 	else {
30861a8031e1Shsuenaga 		log(LOG_ERR, "%s: multiple %s algorithm is not supported.\n",
30871a8031e1Shsuenaga 		    __func__,
30881a8031e1Shsuenaga 		    (op == MV_ACC_CRYPTO_OP_ENC) ?  "encryption" : "authentication");
30891a8031e1Shsuenaga 	}
30901a8031e1Shsuenaga }
30911a8031e1Shsuenaga 
30921a8031e1Shsuenaga /*
30931a8031e1Shsuenaga  * Parameter Conversions
30941a8031e1Shsuenaga  */
30951a8031e1Shsuenaga INLINE uint32_t
mvxpsec_alg2acc(uint32_t alg)30961a8031e1Shsuenaga mvxpsec_alg2acc(uint32_t alg)
30971a8031e1Shsuenaga {
30981a8031e1Shsuenaga 	uint32_t reg;
30991a8031e1Shsuenaga 
31001a8031e1Shsuenaga 	switch (alg) {
31011a8031e1Shsuenaga 	case CRYPTO_DES_CBC:
31021a8031e1Shsuenaga 		reg = MV_ACC_CRYPTO_ENC_DES;
31031a8031e1Shsuenaga 		reg |= MV_ACC_CRYPTO_CBC;
31041a8031e1Shsuenaga 		break;
31051a8031e1Shsuenaga 	case CRYPTO_3DES_CBC:
31061a8031e1Shsuenaga 		reg = MV_ACC_CRYPTO_ENC_3DES;
31071a8031e1Shsuenaga 		reg |= MV_ACC_CRYPTO_3DES_EDE;
31081a8031e1Shsuenaga 		reg |= MV_ACC_CRYPTO_CBC;
31091a8031e1Shsuenaga 		break;
31101a8031e1Shsuenaga 	case CRYPTO_AES_CBC:
31111a8031e1Shsuenaga 		reg = MV_ACC_CRYPTO_ENC_AES;
31121a8031e1Shsuenaga 		reg |= MV_ACC_CRYPTO_CBC;
31131a8031e1Shsuenaga 		break;
31141a8031e1Shsuenaga 	case CRYPTO_SHA1_HMAC_96:
31151a8031e1Shsuenaga 		reg = MV_ACC_CRYPTO_MAC_HMAC_SHA1;
31161a8031e1Shsuenaga 		reg |= MV_ACC_CRYPTO_MAC_96;
31171a8031e1Shsuenaga 		break;
31181a8031e1Shsuenaga 	case CRYPTO_MD5_HMAC_96:
31191a8031e1Shsuenaga 		reg = MV_ACC_CRYPTO_MAC_HMAC_MD5;
31201a8031e1Shsuenaga 		reg |= MV_ACC_CRYPTO_MAC_96;
31211a8031e1Shsuenaga 		break;
31221a8031e1Shsuenaga 	default:
31231a8031e1Shsuenaga 		reg = 0;
31241a8031e1Shsuenaga 		break;
31251a8031e1Shsuenaga 	}
31261a8031e1Shsuenaga 
31271a8031e1Shsuenaga 	return reg;
31281a8031e1Shsuenaga }
31291a8031e1Shsuenaga 
31301a8031e1Shsuenaga INLINE uint32_t
mvxpsec_aesklen(int klen)31311a8031e1Shsuenaga mvxpsec_aesklen(int klen)
31321a8031e1Shsuenaga {
31331a8031e1Shsuenaga 	if (klen < 128)
31341a8031e1Shsuenaga 		return 0;
31351a8031e1Shsuenaga 	else if (klen < 192)
31361a8031e1Shsuenaga 		return MV_ACC_CRYPTO_AES_KLEN_128;
31371a8031e1Shsuenaga 	else if (klen < 256)
31381a8031e1Shsuenaga 		return MV_ACC_CRYPTO_AES_KLEN_192;
31391a8031e1Shsuenaga 	else
31401a8031e1Shsuenaga 		return MV_ACC_CRYPTO_AES_KLEN_256;
31411a8031e1Shsuenaga 
31421a8031e1Shsuenaga 	return 0;
31431a8031e1Shsuenaga }
31441a8031e1Shsuenaga 
31451a8031e1Shsuenaga /*
31461a8031e1Shsuenaga  * String Conversions
31471a8031e1Shsuenaga  */
31481a8031e1Shsuenaga STATIC const char *
s_errreg(uint32_t v)31491a8031e1Shsuenaga s_errreg(uint32_t v)
31501a8031e1Shsuenaga {
31511a8031e1Shsuenaga 	static char buf[80];
31521a8031e1Shsuenaga 
31531a8031e1Shsuenaga 	snprintf(buf, sizeof(buf),
31541a8031e1Shsuenaga 	    "%sMiss %sDoubleHit %sBothHit %sDataError",
31551a8031e1Shsuenaga 	    (v & MV_TDMA_ERRC_MISS) ? "+" : "-",
31561a8031e1Shsuenaga 	    (v & MV_TDMA_ERRC_DHIT) ? "+" : "-",
31571a8031e1Shsuenaga 	    (v & MV_TDMA_ERRC_BHIT) ? "+" : "-",
31581a8031e1Shsuenaga 	    (v & MV_TDMA_ERRC_DERR) ? "+" : "-");
31591a8031e1Shsuenaga 
31601a8031e1Shsuenaga 	return (const char *)buf;
31611a8031e1Shsuenaga }
31621a8031e1Shsuenaga 
31631a8031e1Shsuenaga STATIC const char *
s_winreg(uint32_t v)31641a8031e1Shsuenaga s_winreg(uint32_t v)
31651a8031e1Shsuenaga {
31661a8031e1Shsuenaga 	static char buf[80];
31671a8031e1Shsuenaga 
31681a8031e1Shsuenaga 	snprintf(buf, sizeof(buf),
31691a8031e1Shsuenaga 	    "%s TGT 0x%x ATTR 0x%02x size %u(0x%04x)[64KB]",
31701a8031e1Shsuenaga 	    (v & MV_TDMA_ATTR_ENABLE) ? "EN" : "DIS",
31711a8031e1Shsuenaga 	    MV_TDMA_ATTR_GET_TARGET(v), MV_TDMA_ATTR_GET_ATTR(v),
31721a8031e1Shsuenaga 	    MV_TDMA_ATTR_GET_SIZE(v), MV_TDMA_ATTR_GET_SIZE(v));
31731a8031e1Shsuenaga 
31741a8031e1Shsuenaga 	return (const char *)buf;
31751a8031e1Shsuenaga }
31761a8031e1Shsuenaga 
31771a8031e1Shsuenaga STATIC const char *
s_ctrlreg(uint32_t reg)31781a8031e1Shsuenaga s_ctrlreg(uint32_t reg)
31791a8031e1Shsuenaga {
31801a8031e1Shsuenaga 	static char buf[80];
31811a8031e1Shsuenaga 
31821a8031e1Shsuenaga 	snprintf(buf, sizeof(buf),
31831a8031e1Shsuenaga 	    "%s: %sFETCH DBURST-%u SBURST-%u %sOUTS %sCHAIN %sBSWAP %sACT",
31841a8031e1Shsuenaga 	    (reg & MV_TDMA_CONTROL_ENABLE) ? "ENABLE" : "DISABLE",
31851a8031e1Shsuenaga 	    (reg & MV_TDMA_CONTROL_FETCH) ? "+" : "-",
31861a8031e1Shsuenaga 	    MV_TDMA_CONTROL_GET_DST_BURST(reg),
31871a8031e1Shsuenaga 	    MV_TDMA_CONTROL_GET_SRC_BURST(reg),
31881a8031e1Shsuenaga 	    (reg & MV_TDMA_CONTROL_OUTS_EN) ? "+" : "-",
31891a8031e1Shsuenaga 	    (reg & MV_TDMA_CONTROL_CHAIN_DIS) ? "-" : "+",
31901a8031e1Shsuenaga 	    (reg & MV_TDMA_CONTROL_BSWAP_DIS) ? "-" : "+",
31911a8031e1Shsuenaga 	    (reg & MV_TDMA_CONTROL_ACT) ? "+" : "-");
31921a8031e1Shsuenaga 
31931a8031e1Shsuenaga 	return (const char *)buf;
31941a8031e1Shsuenaga }
31951a8031e1Shsuenaga 
31961a8031e1Shsuenaga _STATIC const char *
s_xpsecintr(uint32_t v)31971a8031e1Shsuenaga s_xpsecintr(uint32_t v)
31981a8031e1Shsuenaga {
31991a8031e1Shsuenaga 	static char buf[160];
32001a8031e1Shsuenaga 
32011a8031e1Shsuenaga 	snprintf(buf, sizeof(buf),
32021a8031e1Shsuenaga 	    "%sAuth %sDES %sAES-ENC %sAES-DEC %sENC %sSA %sAccAndTDMA "
32031a8031e1Shsuenaga 	    "%sTDMAComp %sTDMAOwn %sAccAndTDMA_Cont",
32041a8031e1Shsuenaga 	    (v & MVXPSEC_INT_AUTH) ? "+" : "-",
32051a8031e1Shsuenaga 	    (v & MVXPSEC_INT_DES) ? "+" : "-",
32061a8031e1Shsuenaga 	    (v & MVXPSEC_INT_AES_ENC) ? "+" : "-",
32071a8031e1Shsuenaga 	    (v & MVXPSEC_INT_AES_DEC) ? "+" : "-",
32081a8031e1Shsuenaga 	    (v & MVXPSEC_INT_ENC) ? "+" : "-",
32091a8031e1Shsuenaga 	    (v & MVXPSEC_INT_SA) ? "+" : "-",
32101a8031e1Shsuenaga 	    (v & MVXPSEC_INT_ACCTDMA) ? "+" : "-",
32111a8031e1Shsuenaga 	    (v & MVXPSEC_INT_TDMA_COMP) ? "+" : "-",
32121a8031e1Shsuenaga 	    (v & MVXPSEC_INT_TDMA_OWN) ? "+" : "-",
32131a8031e1Shsuenaga 	    (v & MVXPSEC_INT_ACCTDMA_CONT) ? "+" : "-");
32141a8031e1Shsuenaga 
32151a8031e1Shsuenaga 	return (const char *)buf;
32161a8031e1Shsuenaga }
32171a8031e1Shsuenaga 
32181a8031e1Shsuenaga STATIC const char *
s_ctlalg(uint32_t alg)32191a8031e1Shsuenaga s_ctlalg(uint32_t alg)
32201a8031e1Shsuenaga {
32211a8031e1Shsuenaga 	switch (alg) {
32221a8031e1Shsuenaga 	case CRYPTO_SHA1_HMAC_96:
32231a8031e1Shsuenaga 		return "HMAC-SHA1-96";
32241a8031e1Shsuenaga 	case CRYPTO_SHA1_HMAC:
32251a8031e1Shsuenaga 		return "HMAC-SHA1";
32261a8031e1Shsuenaga 	case CRYPTO_SHA1:
32271a8031e1Shsuenaga 		return "SHA1";
32281a8031e1Shsuenaga 	case CRYPTO_MD5_HMAC_96:
32291a8031e1Shsuenaga 		return "HMAC-MD5-96";
32301a8031e1Shsuenaga 	case CRYPTO_MD5_HMAC:
32311a8031e1Shsuenaga 		return "HMAC-MD5";
32321a8031e1Shsuenaga 	case CRYPTO_MD5:
32331a8031e1Shsuenaga 		return "MD5";
32341a8031e1Shsuenaga 	case CRYPTO_DES_CBC:
32351a8031e1Shsuenaga 		return "DES-CBC";
32361a8031e1Shsuenaga 	case CRYPTO_3DES_CBC:
32371a8031e1Shsuenaga 		return "3DES-CBC";
32381a8031e1Shsuenaga 	case CRYPTO_AES_CBC:
32391a8031e1Shsuenaga 		return "AES-CBC";
32401a8031e1Shsuenaga 	default:
32411a8031e1Shsuenaga 		break;
32421a8031e1Shsuenaga 	}
32431a8031e1Shsuenaga 
32441a8031e1Shsuenaga 	return "Unknown";
32451a8031e1Shsuenaga }
32461a8031e1Shsuenaga 
32471a8031e1Shsuenaga STATIC const char *
s_xpsec_op(uint32_t reg)32481a8031e1Shsuenaga s_xpsec_op(uint32_t reg)
32491a8031e1Shsuenaga {
32501a8031e1Shsuenaga 	reg &= MV_ACC_CRYPTO_OP_MASK;
32511a8031e1Shsuenaga 	switch (reg) {
32521a8031e1Shsuenaga 	case MV_ACC_CRYPTO_OP_ENC:
32531a8031e1Shsuenaga 		return "ENC";
32541a8031e1Shsuenaga 	case MV_ACC_CRYPTO_OP_MAC:
32551a8031e1Shsuenaga 		return "MAC";
32561a8031e1Shsuenaga 	case MV_ACC_CRYPTO_OP_ENCMAC:
32571a8031e1Shsuenaga 		return "ENC-MAC";
32581a8031e1Shsuenaga 	case MV_ACC_CRYPTO_OP_MACENC:
32591a8031e1Shsuenaga 		return "MAC-ENC";
32601a8031e1Shsuenaga 	default:
32611a8031e1Shsuenaga 		break;
32621a8031e1Shsuenaga 	}
32631a8031e1Shsuenaga 
32641a8031e1Shsuenaga 	return "Unknown";
32651a8031e1Shsuenaga }
32661a8031e1Shsuenaga 
32671a8031e1Shsuenaga STATIC const char *
s_xpsec_enc(uint32_t alg)32681a8031e1Shsuenaga s_xpsec_enc(uint32_t alg)
32691a8031e1Shsuenaga {
32701a8031e1Shsuenaga 	alg <<= MV_ACC_CRYPTO_ENC_SHIFT;
32711a8031e1Shsuenaga 	switch (alg) {
32721a8031e1Shsuenaga 	case MV_ACC_CRYPTO_ENC_DES:
32731a8031e1Shsuenaga 		return "DES";
32741a8031e1Shsuenaga 	case MV_ACC_CRYPTO_ENC_3DES:
32751a8031e1Shsuenaga 		return "3DES";
32761a8031e1Shsuenaga 	case MV_ACC_CRYPTO_ENC_AES:
32771a8031e1Shsuenaga 		return "AES";
32781a8031e1Shsuenaga 	default:
32791a8031e1Shsuenaga 		break;
32801a8031e1Shsuenaga 	}
32811a8031e1Shsuenaga 
32821a8031e1Shsuenaga 	return "Unknown";
32831a8031e1Shsuenaga }
32841a8031e1Shsuenaga 
32851a8031e1Shsuenaga STATIC const char *
s_xpsec_mac(uint32_t alg)32861a8031e1Shsuenaga s_xpsec_mac(uint32_t alg)
32871a8031e1Shsuenaga {
32881a8031e1Shsuenaga 	alg <<= MV_ACC_CRYPTO_MAC_SHIFT;
32891a8031e1Shsuenaga 	switch (alg) {
32901a8031e1Shsuenaga 	case MV_ACC_CRYPTO_MAC_NONE:
32911a8031e1Shsuenaga 		return "Disabled";
32921a8031e1Shsuenaga 	case MV_ACC_CRYPTO_MAC_MD5:
32931a8031e1Shsuenaga 		return "MD5";
32941a8031e1Shsuenaga 	case MV_ACC_CRYPTO_MAC_SHA1:
32951a8031e1Shsuenaga 		return "SHA1";
32961a8031e1Shsuenaga 	case MV_ACC_CRYPTO_MAC_HMAC_MD5:
32971a8031e1Shsuenaga 		return "HMAC-MD5";
32981a8031e1Shsuenaga 	case MV_ACC_CRYPTO_MAC_HMAC_SHA1:
32991a8031e1Shsuenaga 		return "HMAC-SHA1";
33001a8031e1Shsuenaga 	default:
33011a8031e1Shsuenaga 		break;
33021a8031e1Shsuenaga 	}
33031a8031e1Shsuenaga 
33041a8031e1Shsuenaga 	return "Unknown";
33051a8031e1Shsuenaga }
33061a8031e1Shsuenaga 
33071a8031e1Shsuenaga STATIC const char *
s_xpsec_frag(uint32_t frag)33081a8031e1Shsuenaga s_xpsec_frag(uint32_t frag)
33091a8031e1Shsuenaga {
33101a8031e1Shsuenaga 	frag <<= MV_ACC_CRYPTO_FRAG_SHIFT;
33111a8031e1Shsuenaga 	switch (frag) {
33121a8031e1Shsuenaga 	case MV_ACC_CRYPTO_NOFRAG:
33131a8031e1Shsuenaga 		return "NoFragment";
33141a8031e1Shsuenaga 	case MV_ACC_CRYPTO_FRAG_FIRST:
33151a8031e1Shsuenaga 		return "FirstFragment";
33161a8031e1Shsuenaga 	case MV_ACC_CRYPTO_FRAG_MID:
33171a8031e1Shsuenaga 		return "MiddleFragment";
33181a8031e1Shsuenaga 	case MV_ACC_CRYPTO_FRAG_LAST:
33191a8031e1Shsuenaga 		return "LastFragment";
33201a8031e1Shsuenaga 	default:
33211a8031e1Shsuenaga 		break;
33221a8031e1Shsuenaga 	}
33231a8031e1Shsuenaga 
33241a8031e1Shsuenaga 	return "Unknown";
33251a8031e1Shsuenaga }
33261a8031e1Shsuenaga 
33271a8031e1Shsuenaga #ifdef MVXPSEC_DEBUG
33281a8031e1Shsuenaga void
mvxpsec_dump_reg(struct mvxpsec_softc * sc)33291a8031e1Shsuenaga mvxpsec_dump_reg(struct mvxpsec_softc *sc)
33301a8031e1Shsuenaga {
33311a8031e1Shsuenaga 	uint32_t reg;
33321a8031e1Shsuenaga 	int i;
33331a8031e1Shsuenaga 
33341a8031e1Shsuenaga 	if ((mvxpsec_debug & MVXPSEC_DEBUG_DESC) == 0)
33351a8031e1Shsuenaga 		return;
33361a8031e1Shsuenaga 
33371a8031e1Shsuenaga 	printf("--- Interrupt Registers ---\n");
33381a8031e1Shsuenaga 	reg = MVXPSEC_READ(sc, MVXPSEC_INT_CAUSE);
33391a8031e1Shsuenaga 	printf("MVXPSEC INT CAUSE: 0x%08x\n", reg);
33401a8031e1Shsuenaga 	printf("MVXPSEC INT CAUSE: %s\n", s_xpsecintr(reg));
33411a8031e1Shsuenaga 	reg = MVXPSEC_READ(sc, MVXPSEC_INT_MASK);
33421a8031e1Shsuenaga 	printf("MVXPSEC INT MASK: 0x%08x\n", reg);
33431a8031e1Shsuenaga 	printf("MVXPSEC INT MASKE: %s\n", s_xpsecintr(reg));
33441a8031e1Shsuenaga 
33451a8031e1Shsuenaga 	printf("--- DMA Configuration Registers ---\n");
33461a8031e1Shsuenaga 	for (i = 0; i < MV_TDMA_NWINDOW; i++) {
33471a8031e1Shsuenaga 		reg = MVXPSEC_READ(sc, MV_TDMA_BAR(i));
33481a8031e1Shsuenaga 		printf("TDMA BAR%d: 0x%08x\n", i, reg);
33491a8031e1Shsuenaga 		reg = MVXPSEC_READ(sc, MV_TDMA_ATTR(i));
33501a8031e1Shsuenaga 		printf("TDMA ATTR%d: 0x%08x\n", i, reg);
33511a8031e1Shsuenaga 		printf("  -> %s\n", s_winreg(reg));
33521a8031e1Shsuenaga 	}
33531a8031e1Shsuenaga 
33541a8031e1Shsuenaga 	printf("--- DMA Control Registers ---\n");
33551a8031e1Shsuenaga 
33561a8031e1Shsuenaga 	reg = MVXPSEC_READ(sc, MV_TDMA_CONTROL);
33571a8031e1Shsuenaga 	printf("TDMA CONTROL: 0x%08x\n", reg);
33581a8031e1Shsuenaga 	printf("  -> %s\n", s_ctrlreg(reg));
33591a8031e1Shsuenaga 
33601a8031e1Shsuenaga 	printf("--- DMA Current Command Descriptors ---\n");
33611a8031e1Shsuenaga 
33621a8031e1Shsuenaga 	reg = MVXPSEC_READ(sc, MV_TDMA_ERR_CAUSE);
33631a8031e1Shsuenaga 	printf("TDMA ERR CAUSE: 0x%08x\n", reg);
33641a8031e1Shsuenaga 
33651a8031e1Shsuenaga 	reg = MVXPSEC_READ(sc, MV_TDMA_ERR_MASK);
33661a8031e1Shsuenaga 	printf("TDMA ERR MASK: 0x%08x\n", reg);
33671a8031e1Shsuenaga 
33681a8031e1Shsuenaga 	reg = MVXPSEC_READ(sc, MV_TDMA_CNT);
33691a8031e1Shsuenaga 	printf("TDMA DATA OWNER: %s\n",
33701a8031e1Shsuenaga 	    (reg & MV_TDMA_CNT_OWN) ? "DMAC" : "CPU");
33711a8031e1Shsuenaga 	printf("TDMA DATA COUNT: %d(0x%x)\n",
33721a8031e1Shsuenaga 	    (reg & ~MV_TDMA_CNT_OWN), (reg & ~MV_TDMA_CNT_OWN));
33731a8031e1Shsuenaga 
33741a8031e1Shsuenaga 	reg = MVXPSEC_READ(sc, MV_TDMA_SRC);
33751a8031e1Shsuenaga 	printf("TDMA DATA SRC: 0x%08x\n", reg);
33761a8031e1Shsuenaga 
33771a8031e1Shsuenaga 	reg = MVXPSEC_READ(sc, MV_TDMA_DST);
33781a8031e1Shsuenaga 	printf("TDMA DATA DST: 0x%08x\n", reg);
33791a8031e1Shsuenaga 
33801a8031e1Shsuenaga 	reg = MVXPSEC_READ(sc, MV_TDMA_NXT);
33811a8031e1Shsuenaga 	printf("TDMA DATA NXT: 0x%08x\n", reg);
33821a8031e1Shsuenaga 
33831a8031e1Shsuenaga 	reg = MVXPSEC_READ(sc, MV_TDMA_CUR);
33841a8031e1Shsuenaga 	printf("TDMA DATA CUR: 0x%08x\n", reg);
33851a8031e1Shsuenaga 
33861a8031e1Shsuenaga 	printf("--- ACC Command Register ---\n");
33871a8031e1Shsuenaga 	reg = MVXPSEC_READ(sc, MV_ACC_COMMAND);
33881a8031e1Shsuenaga 	printf("ACC COMMAND: 0x%08x\n", reg);
33891a8031e1Shsuenaga 	printf("ACC: %sACT %sSTOP\n",
33901a8031e1Shsuenaga 	    (reg & MV_ACC_COMMAND_ACT) ? "+" : "-",
33911a8031e1Shsuenaga 	    (reg & MV_ACC_COMMAND_STOP) ? "+" : "-");
33921a8031e1Shsuenaga 
33931a8031e1Shsuenaga 	reg = MVXPSEC_READ(sc, MV_ACC_CONFIG);
33941a8031e1Shsuenaga 	printf("ACC CONFIG: 0x%08x\n", reg);
33951a8031e1Shsuenaga 	reg = MVXPSEC_READ(sc, MV_ACC_DESC);
33961a8031e1Shsuenaga 	printf("ACC DESC: 0x%08x\n", reg);
33971a8031e1Shsuenaga 
33981a8031e1Shsuenaga 	printf("--- DES Key Register ---\n");
33991a8031e1Shsuenaga 	reg = MVXPSEC_READ(sc, MV_CE_DES_KEY0L);
34001a8031e1Shsuenaga 	printf("DES KEY0  Low: 0x%08x\n", reg);
34011a8031e1Shsuenaga 	reg = MVXPSEC_READ(sc, MV_CE_DES_KEY0H);
34021a8031e1Shsuenaga 	printf("DES KEY0 High: 0x%08x\n", reg);
34031a8031e1Shsuenaga 	reg = MVXPSEC_READ(sc, MV_CE_DES_KEY1L);
34041a8031e1Shsuenaga 	printf("DES KEY1  Low: 0x%08x\n", reg);
34051a8031e1Shsuenaga 	reg = MVXPSEC_READ(sc, MV_CE_DES_KEY1H);
34061a8031e1Shsuenaga 	printf("DES KEY1 High: 0x%08x\n", reg);
34071a8031e1Shsuenaga 	reg = MVXPSEC_READ(sc, MV_CE_DES_KEY2L);
34081a8031e1Shsuenaga 	printf("DES KEY2  Low: 0x%08x\n", reg);
34091a8031e1Shsuenaga 	reg = MVXPSEC_READ(sc, MV_CE_DES_KEY2H);
34101a8031e1Shsuenaga 	printf("DES KEY2 High: 0x%08x\n", reg);
34111a8031e1Shsuenaga 
34121a8031e1Shsuenaga 	printf("--- AES Key Register ---\n");
34131a8031e1Shsuenaga 	for (i = 0; i < 8; i++) {
34141a8031e1Shsuenaga 		reg = MVXPSEC_READ(sc, MV_CE_AES_EKEY(i));
34151a8031e1Shsuenaga 		printf("AES ENC KEY COL%d: %08x\n", i, reg);
34161a8031e1Shsuenaga 	}
34171a8031e1Shsuenaga 	for (i = 0; i < 8; i++) {
34181a8031e1Shsuenaga 		reg = MVXPSEC_READ(sc, MV_CE_AES_DKEY(i));
34191a8031e1Shsuenaga 		printf("AES DEC KEY COL%d: %08x\n", i, reg);
34201a8031e1Shsuenaga 	}
34211a8031e1Shsuenaga 
34221a8031e1Shsuenaga 	return;
34231a8031e1Shsuenaga }
34241a8031e1Shsuenaga 
34251a8031e1Shsuenaga STATIC void
mvxpsec_dump_sram(const char * name,struct mvxpsec_softc * sc,size_t len)34261a8031e1Shsuenaga mvxpsec_dump_sram(const char *name, struct mvxpsec_softc *sc, size_t len)
34271a8031e1Shsuenaga {
34281a8031e1Shsuenaga 	uint32_t reg;
34291a8031e1Shsuenaga 
34301a8031e1Shsuenaga 	if (sc->sc_sram_va == NULL)
34311a8031e1Shsuenaga 		return;
34321a8031e1Shsuenaga 
34331a8031e1Shsuenaga 	if (len == 0) {
34341a8031e1Shsuenaga 		printf("\n%s NO DATA(len=0)\n", name);
34351a8031e1Shsuenaga 		return;
34361a8031e1Shsuenaga 	}
34371a8031e1Shsuenaga 	else if (len > MV_ACC_SRAM_SIZE)
34381a8031e1Shsuenaga 		len = MV_ACC_SRAM_SIZE;
34391a8031e1Shsuenaga 
34401a8031e1Shsuenaga 	mutex_enter(&sc->sc_dma_mtx);
34411a8031e1Shsuenaga 	reg = MVXPSEC_READ(sc, MV_TDMA_CONTROL);
34421a8031e1Shsuenaga 	if (reg & MV_TDMA_CONTROL_ACT) {
34431a8031e1Shsuenaga 		printf("TDMA is active, cannot access SRAM\n");
34441a8031e1Shsuenaga 		mutex_exit(&sc->sc_dma_mtx);
34451a8031e1Shsuenaga 		return;
34461a8031e1Shsuenaga 	}
34471a8031e1Shsuenaga 	reg = MVXPSEC_READ(sc, MV_ACC_COMMAND);
34481a8031e1Shsuenaga 	if (reg & MV_ACC_COMMAND_ACT) {
34491a8031e1Shsuenaga 		printf("SA is active, cannot access SRAM\n");
34501a8031e1Shsuenaga 		mutex_exit(&sc->sc_dma_mtx);
34511a8031e1Shsuenaga 		return;
34521a8031e1Shsuenaga 	}
34531a8031e1Shsuenaga 
34541a8031e1Shsuenaga 	printf("%s: dump SRAM, %zu bytes\n", name, len);
34551a8031e1Shsuenaga 	mvxpsec_dump_data(name, sc->sc_sram_va, len);
34561a8031e1Shsuenaga 	mutex_exit(&sc->sc_dma_mtx);
34571a8031e1Shsuenaga 	return;
34581a8031e1Shsuenaga }
34591a8031e1Shsuenaga 
34601a8031e1Shsuenaga 
34611a8031e1Shsuenaga _STATIC void
mvxpsec_dump_dmaq(struct mvxpsec_descriptor_handle * dh)34621a8031e1Shsuenaga mvxpsec_dump_dmaq(struct mvxpsec_descriptor_handle *dh)
34631a8031e1Shsuenaga {
34641a8031e1Shsuenaga 	struct mvxpsec_descriptor *d =
34651a8031e1Shsuenaga            (struct mvxpsec_descriptor *)dh->_desc;
34661a8031e1Shsuenaga 
34671a8031e1Shsuenaga 	printf("--- DMA Command Descriptor ---\n");
34681a8031e1Shsuenaga 	printf("DESC: VA=%p PA=0x%08x\n",
34691a8031e1Shsuenaga 	    d, (uint32_t)dh->phys_addr);
34701a8031e1Shsuenaga 	printf("DESC: WORD0 = 0x%08x\n", d->tdma_word0);
34711a8031e1Shsuenaga 	printf("DESC: SRC = 0x%08x\n", d->tdma_src);
34721a8031e1Shsuenaga 	printf("DESC: DST = 0x%08x\n", d->tdma_dst);
34731a8031e1Shsuenaga 	printf("DESC: NXT = 0x%08x\n", d->tdma_nxt);
34741a8031e1Shsuenaga 
34751a8031e1Shsuenaga 	return;
34761a8031e1Shsuenaga }
34771a8031e1Shsuenaga 
34781a8031e1Shsuenaga STATIC void
mvxpsec_dump_data(const char * name,void * p,size_t len)34791a8031e1Shsuenaga mvxpsec_dump_data(const char *name, void *p, size_t len)
34801a8031e1Shsuenaga {
34811a8031e1Shsuenaga 	uint8_t *data = p;
34821a8031e1Shsuenaga 	off_t off;
34831a8031e1Shsuenaga 
34841a8031e1Shsuenaga 	printf("%s: dump %p, %zu bytes", name, p, len);
34851a8031e1Shsuenaga 	if (p == NULL || len == 0) {
34861a8031e1Shsuenaga 		printf("\n%s: NO DATA\n", name);
34871a8031e1Shsuenaga 		return;
34881a8031e1Shsuenaga 	}
34891a8031e1Shsuenaga 	for (off = 0; off < len; off++) {
34901a8031e1Shsuenaga 		if ((off % 16) == 0) {
34911a8031e1Shsuenaga 			printf("\n%s: 0x%08x:", name, (uint32_t)off);
34921a8031e1Shsuenaga 		}
34931a8031e1Shsuenaga 		if ((off % 4) == 0) {
34941a8031e1Shsuenaga 			printf(" ");
34951a8031e1Shsuenaga 		}
34961a8031e1Shsuenaga 		printf("%02x", data[off]);
34971a8031e1Shsuenaga 	}
34981a8031e1Shsuenaga 	printf("\n");
34991a8031e1Shsuenaga 
35001a8031e1Shsuenaga 	return;
35011a8031e1Shsuenaga }
35021a8031e1Shsuenaga 
35031a8031e1Shsuenaga _STATIC void
mvxpsec_dump_packet(const char * name,struct mvxpsec_packet * mv_p)35041a8031e1Shsuenaga mvxpsec_dump_packet(const char *name, struct mvxpsec_packet *mv_p)
35051a8031e1Shsuenaga {
35061a8031e1Shsuenaga 	struct mvxpsec_softc *sc = mv_p->mv_s->sc;
35071a8031e1Shsuenaga 
35081a8031e1Shsuenaga 	printf("%s: packet_data:\n", name);
35091a8031e1Shsuenaga 	mvxpsec_dump_packet_data(name, mv_p);
35101a8031e1Shsuenaga 
35111a8031e1Shsuenaga 	printf("%s: SRAM:\n", name);
35121a8031e1Shsuenaga 	mvxpsec_dump_sram(name, sc, 2000);
35131a8031e1Shsuenaga 
35141a8031e1Shsuenaga 	printf("%s: packet_descriptor:\n", name);
35151a8031e1Shsuenaga 	mvxpsec_dump_packet_desc(name, mv_p);
35161a8031e1Shsuenaga }
35171a8031e1Shsuenaga 
35181a8031e1Shsuenaga _STATIC void
mvxpsec_dump_packet_data(const char * name,struct mvxpsec_packet * mv_p)35191a8031e1Shsuenaga mvxpsec_dump_packet_data(const char *name, struct mvxpsec_packet *mv_p)
35201a8031e1Shsuenaga {
35211a8031e1Shsuenaga 	static char buf[1500];
35221a8031e1Shsuenaga 	int len;
35231a8031e1Shsuenaga 
35241a8031e1Shsuenaga 	if (mv_p->data_type == MVXPSEC_DATA_MBUF) {
35251a8031e1Shsuenaga 		struct mbuf *m;
35261a8031e1Shsuenaga 
35271a8031e1Shsuenaga 		m = mv_p->data.mbuf;
35281a8031e1Shsuenaga 		len = m->m_pkthdr.len;
35291a8031e1Shsuenaga 		if (len > sizeof(buf))
35301a8031e1Shsuenaga 			len = sizeof(buf);
35311a8031e1Shsuenaga 		m_copydata(m, 0, len, buf);
35321a8031e1Shsuenaga 	}
35331a8031e1Shsuenaga 	else if (mv_p->data_type == MVXPSEC_DATA_UIO) {
35341a8031e1Shsuenaga 		struct uio *uio;
35351a8031e1Shsuenaga 
35361a8031e1Shsuenaga 		uio = mv_p->data.uio;
35371a8031e1Shsuenaga 		len = uio->uio_resid;
35381a8031e1Shsuenaga 		if (len > sizeof(buf))
35391a8031e1Shsuenaga 			len = sizeof(buf);
35401a8031e1Shsuenaga 		cuio_copydata(uio, 0, len, buf);
35411a8031e1Shsuenaga 	}
35421a8031e1Shsuenaga 	else if (mv_p->data_type == MVXPSEC_DATA_RAW) {
35431a8031e1Shsuenaga 		len = mv_p->data_len;
35441a8031e1Shsuenaga 		if (len > sizeof(buf))
35451a8031e1Shsuenaga 			len = sizeof(buf);
35461a8031e1Shsuenaga 		memcpy(buf, mv_p->data.raw, len);
35471a8031e1Shsuenaga 	}
35481a8031e1Shsuenaga 	else
35491a8031e1Shsuenaga 		return;
35501a8031e1Shsuenaga 	mvxpsec_dump_data(name, buf, len);
35511a8031e1Shsuenaga 
35521a8031e1Shsuenaga 	return;
35531a8031e1Shsuenaga }
35541a8031e1Shsuenaga 
35551a8031e1Shsuenaga _STATIC void
mvxpsec_dump_packet_desc(const char * name,struct mvxpsec_packet * mv_p)35561a8031e1Shsuenaga mvxpsec_dump_packet_desc(const char *name, struct mvxpsec_packet *mv_p)
35571a8031e1Shsuenaga {
35581a8031e1Shsuenaga 	uint32_t *words;
35591a8031e1Shsuenaga 
35601a8031e1Shsuenaga 	if (mv_p == NULL)
35611a8031e1Shsuenaga 		return;
35621a8031e1Shsuenaga 
35631a8031e1Shsuenaga 	words = &mv_p->pkt_header.desc.acc_desc_dword0;
35641a8031e1Shsuenaga 	mvxpsec_dump_acc_config(name, words[0]);
35651a8031e1Shsuenaga 	mvxpsec_dump_acc_encdata(name, words[1], words[2]);
35661a8031e1Shsuenaga 	mvxpsec_dump_acc_enclen(name, words[2]);
35671a8031e1Shsuenaga 	mvxpsec_dump_acc_enckey(name, words[3]);
35681a8031e1Shsuenaga 	mvxpsec_dump_acc_enciv(name, words[4]);
35691a8031e1Shsuenaga 	mvxpsec_dump_acc_macsrc(name, words[5]);
35701a8031e1Shsuenaga 	mvxpsec_dump_acc_macdst(name, words[6]);
35711a8031e1Shsuenaga 	mvxpsec_dump_acc_maciv(name, words[7]);
35721a8031e1Shsuenaga 
35731a8031e1Shsuenaga 	return;
35741a8031e1Shsuenaga }
35751a8031e1Shsuenaga 
35761a8031e1Shsuenaga _STATIC void
mvxpsec_dump_acc_config(const char * name,uint32_t w)35771a8031e1Shsuenaga mvxpsec_dump_acc_config(const char *name, uint32_t w)
35781a8031e1Shsuenaga {
35791a8031e1Shsuenaga 	/* SA: Dword 0 */
35801a8031e1Shsuenaga 	printf("%s: Dword0=0x%08x\n", name, w);
35811a8031e1Shsuenaga 	printf("%s:   OP = %s\n", name,
35821a8031e1Shsuenaga 	    s_xpsec_op(MV_ACC_CRYPTO_OP(w)));
35831a8031e1Shsuenaga 	printf("%s:   MAC = %s\n", name,
35841a8031e1Shsuenaga 	    s_xpsec_mac(MV_ACC_CRYPTO_MAC(w)));
35851a8031e1Shsuenaga 	printf("%s:   MAC_LEN = %s\n", name,
35861a8031e1Shsuenaga 	    w & MV_ACC_CRYPTO_MAC_96 ? "96-bit" : "full-bit");
35871a8031e1Shsuenaga 	printf("%s:   ENC = %s\n", name,
35881a8031e1Shsuenaga 	    s_xpsec_enc(MV_ACC_CRYPTO_ENC(w)));
35891a8031e1Shsuenaga 	printf("%s:   DIR = %s\n", name,
35901a8031e1Shsuenaga 	    w & MV_ACC_CRYPTO_DECRYPT ? "decryption" : "encryption");
35911a8031e1Shsuenaga 	printf("%s:   CHAIN = %s\n", name,
35921a8031e1Shsuenaga 	    w & MV_ACC_CRYPTO_CBC ? "CBC" : "ECB");
35931a8031e1Shsuenaga 	printf("%s:   3DES = %s\n", name,
35941a8031e1Shsuenaga 	    w & MV_ACC_CRYPTO_3DES_EDE ? "EDE" : "EEE");
35951a8031e1Shsuenaga 	printf("%s:   FRAGMENT = %s\n", name,
35961a8031e1Shsuenaga 	    s_xpsec_frag(MV_ACC_CRYPTO_FRAG(w)));
35971a8031e1Shsuenaga 	return;
35981a8031e1Shsuenaga }
35991a8031e1Shsuenaga 
36001a8031e1Shsuenaga STATIC void
mvxpsec_dump_acc_encdata(const char * name,uint32_t w,uint32_t w2)36011a8031e1Shsuenaga mvxpsec_dump_acc_encdata(const char *name, uint32_t w, uint32_t w2)
36021a8031e1Shsuenaga {
36031a8031e1Shsuenaga 	/* SA: Dword 1 */
36041a8031e1Shsuenaga 	printf("%s: Dword1=0x%08x\n", name, w);
36051a8031e1Shsuenaga 	printf("%s:   ENC SRC = 0x%x\n", name, MV_ACC_DESC_GET_VAL_1(w));
36061a8031e1Shsuenaga 	printf("%s:   ENC DST = 0x%x\n", name, MV_ACC_DESC_GET_VAL_2(w));
36071a8031e1Shsuenaga 	printf("%s:   ENC RANGE = 0x%x - 0x%x\n", name,
36081a8031e1Shsuenaga 	    MV_ACC_DESC_GET_VAL_1(w),
36091a8031e1Shsuenaga 	    MV_ACC_DESC_GET_VAL_1(w) + MV_ACC_DESC_GET_VAL_1(w2) - 1);
36101a8031e1Shsuenaga 	return;
36111a8031e1Shsuenaga }
36121a8031e1Shsuenaga 
36131a8031e1Shsuenaga STATIC void
mvxpsec_dump_acc_enclen(const char * name,uint32_t w)36141a8031e1Shsuenaga mvxpsec_dump_acc_enclen(const char *name, uint32_t w)
36151a8031e1Shsuenaga {
36161a8031e1Shsuenaga 	/* SA: Dword 2 */
36171a8031e1Shsuenaga 	printf("%s: Dword2=0x%08x\n", name, w);
36181a8031e1Shsuenaga 	printf("%s:   ENC LEN = %d\n", name,
36191a8031e1Shsuenaga 	    MV_ACC_DESC_GET_VAL_1(w));
36201a8031e1Shsuenaga 	return;
36211a8031e1Shsuenaga }
36221a8031e1Shsuenaga 
36231a8031e1Shsuenaga STATIC void
mvxpsec_dump_acc_enckey(const char * name,uint32_t w)36241a8031e1Shsuenaga mvxpsec_dump_acc_enckey(const char *name, uint32_t w)
36251a8031e1Shsuenaga {
36261a8031e1Shsuenaga 	/* SA: Dword 3 */
36271a8031e1Shsuenaga 	printf("%s: Dword3=0x%08x\n", name, w);
36281a8031e1Shsuenaga 	printf("%s:   EKEY = 0x%x\n", name,
36291a8031e1Shsuenaga 	    MV_ACC_DESC_GET_VAL_1(w));
36301a8031e1Shsuenaga 	return;
36311a8031e1Shsuenaga }
36321a8031e1Shsuenaga 
36331a8031e1Shsuenaga STATIC void
mvxpsec_dump_acc_enciv(const char * name,uint32_t w)36341a8031e1Shsuenaga mvxpsec_dump_acc_enciv(const char *name, uint32_t w)
36351a8031e1Shsuenaga {
36361a8031e1Shsuenaga 	/* SA: Dword 4 */
36371a8031e1Shsuenaga 	printf("%s: Dword4=0x%08x\n", name, w);
36381a8031e1Shsuenaga 	printf("%s:   EIV = 0x%x\n", name, MV_ACC_DESC_GET_VAL_1(w));
36391a8031e1Shsuenaga 	printf("%s:   EIV_BUF = 0x%x\n", name, MV_ACC_DESC_GET_VAL_2(w));
36401a8031e1Shsuenaga 	return;
36411a8031e1Shsuenaga }
36421a8031e1Shsuenaga 
36431a8031e1Shsuenaga STATIC void
mvxpsec_dump_acc_macsrc(const char * name,uint32_t w)36441a8031e1Shsuenaga mvxpsec_dump_acc_macsrc(const char *name, uint32_t w)
36451a8031e1Shsuenaga {
36461a8031e1Shsuenaga 	/* SA: Dword 5 */
36471a8031e1Shsuenaga 	printf("%s: Dword5=0x%08x\n", name, w);
36481a8031e1Shsuenaga 	printf("%s:   MAC_SRC = 0x%x\n", name,
36491a8031e1Shsuenaga 	    MV_ACC_DESC_GET_VAL_1(w));
36501a8031e1Shsuenaga 	printf("%s:   MAC_TOTAL_LEN = %d\n", name,
36511a8031e1Shsuenaga 	    MV_ACC_DESC_GET_VAL_3(w));
36521a8031e1Shsuenaga 	printf("%s:   MAC_RANGE = 0x%0x - 0x%0x\n", name,
36531a8031e1Shsuenaga 	    MV_ACC_DESC_GET_VAL_1(w),
36541a8031e1Shsuenaga 	    MV_ACC_DESC_GET_VAL_1(w) + MV_ACC_DESC_GET_VAL_3(w) - 1);
36551a8031e1Shsuenaga 	return;
36561a8031e1Shsuenaga }
36571a8031e1Shsuenaga 
36581a8031e1Shsuenaga STATIC void
mvxpsec_dump_acc_macdst(const char * name,uint32_t w)36591a8031e1Shsuenaga mvxpsec_dump_acc_macdst(const char *name, uint32_t w)
36601a8031e1Shsuenaga {
36611a8031e1Shsuenaga 	/* SA: Dword 6 */
36621a8031e1Shsuenaga 	printf("%s: Dword6=0x%08x\n", name, w);
36631a8031e1Shsuenaga 	printf("%s:   MAC_DST = 0x%x\n", name, MV_ACC_DESC_GET_VAL_1(w));
36641a8031e1Shsuenaga 	printf("%s:   MAC_BLOCK_LEN = %d\n", name,
36651a8031e1Shsuenaga 	    MV_ACC_DESC_GET_VAL_2(w));
36661a8031e1Shsuenaga 	return;
36671a8031e1Shsuenaga }
36681a8031e1Shsuenaga 
36691a8031e1Shsuenaga STATIC void
mvxpsec_dump_acc_maciv(const char * name,uint32_t w)36701a8031e1Shsuenaga mvxpsec_dump_acc_maciv(const char *name, uint32_t w)
36711a8031e1Shsuenaga {
36721a8031e1Shsuenaga 	/* SA: Dword 7 */
36731a8031e1Shsuenaga 	printf("%s: Dword7=0x%08x\n", name, w);
36741a8031e1Shsuenaga 	printf("%s:   MAC_INNER_IV = 0x%x\n", name,
36751a8031e1Shsuenaga 	    MV_ACC_DESC_GET_VAL_1(w));
36761a8031e1Shsuenaga 	printf("%s:   MAC_OUTER_IV = 0x%x\n", name,
36771a8031e1Shsuenaga 	    MV_ACC_DESC_GET_VAL_2(w));
36781a8031e1Shsuenaga 	return;
36791a8031e1Shsuenaga }
36801a8031e1Shsuenaga #endif
3681