1*100a3398Sandvar /* $NetBSD: mvxpsec.c,v 1.20 2024/02/09 22:08:35 andvar Exp $ */
21a8031e1Shsuenaga /*
31a8031e1Shsuenaga * Copyright (c) 2015 Internet Initiative Japan Inc.
41a8031e1Shsuenaga * All rights reserved.
51a8031e1Shsuenaga *
61a8031e1Shsuenaga * Redistribution and use in source and binary forms, with or without
71a8031e1Shsuenaga * modification, are permitted provided that the following conditions
81a8031e1Shsuenaga * are met:
91a8031e1Shsuenaga * 1. Redistributions of source code must retain the above copyright
101a8031e1Shsuenaga * notice, this list of conditions and the following disclaimer.
111a8031e1Shsuenaga * 2. Redistributions in binary form must reproduce the above copyright
121a8031e1Shsuenaga * notice, this list of conditions and the following disclaimer in the
131a8031e1Shsuenaga * documentation and/or other materials provided with the distribution.
141a8031e1Shsuenaga *
151a8031e1Shsuenaga * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
161a8031e1Shsuenaga * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
171a8031e1Shsuenaga * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
181a8031e1Shsuenaga * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
191a8031e1Shsuenaga * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
201a8031e1Shsuenaga * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
211a8031e1Shsuenaga * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
221a8031e1Shsuenaga * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
231a8031e1Shsuenaga * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
241a8031e1Shsuenaga * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
251a8031e1Shsuenaga * POSSIBILITY OF SUCH DAMAGE.
261a8031e1Shsuenaga */
272499f515Sriastradh
282499f515Sriastradh #ifdef _KERNEL_OPT
292499f515Sriastradh #include "opt_ipsec.h"
302499f515Sriastradh #endif
312499f515Sriastradh
321a8031e1Shsuenaga /*
331a8031e1Shsuenaga * Cryptographic Engine and Security Accelerator(MVXPSEC)
341a8031e1Shsuenaga */
351a8031e1Shsuenaga #include <sys/cdefs.h>
361a8031e1Shsuenaga #include <sys/param.h>
371a8031e1Shsuenaga #include <sys/types.h>
381a8031e1Shsuenaga #include <sys/kernel.h>
391a8031e1Shsuenaga #include <sys/queue.h>
401a8031e1Shsuenaga #include <sys/conf.h>
411a8031e1Shsuenaga #include <sys/proc.h>
421a8031e1Shsuenaga #include <sys/bus.h>
431a8031e1Shsuenaga #include <sys/evcnt.h>
441a8031e1Shsuenaga #include <sys/device.h>
451a8031e1Shsuenaga #include <sys/endian.h>
461a8031e1Shsuenaga #include <sys/errno.h>
471a8031e1Shsuenaga #include <sys/kmem.h>
481a8031e1Shsuenaga #include <sys/mbuf.h>
491a8031e1Shsuenaga #include <sys/callout.h>
501a8031e1Shsuenaga #include <sys/pool.h>
511a8031e1Shsuenaga #include <sys/cprng.h>
521a8031e1Shsuenaga #include <sys/syslog.h>
531a8031e1Shsuenaga #include <sys/mutex.h>
541a8031e1Shsuenaga #include <sys/kthread.h>
551a8031e1Shsuenaga #include <sys/atomic.h>
561a8031e1Shsuenaga #include <sys/sha1.h>
571a8031e1Shsuenaga #include <sys/md5.h>
581a8031e1Shsuenaga
591a8031e1Shsuenaga #include <uvm/uvm_extern.h>
601a8031e1Shsuenaga
611a8031e1Shsuenaga #include <opencrypto/cryptodev.h>
621a8031e1Shsuenaga #include <opencrypto/xform.h>
631a8031e1Shsuenaga
641a8031e1Shsuenaga #include <net/net_stats.h>
651a8031e1Shsuenaga
661a8031e1Shsuenaga #include <netinet/in_systm.h>
671a8031e1Shsuenaga #include <netinet/in.h>
681a8031e1Shsuenaga #include <netinet/ip.h>
691a8031e1Shsuenaga #include <netinet/ip6.h>
701a8031e1Shsuenaga
712499f515Sriastradh #if NIPSEC > 0
721a8031e1Shsuenaga #include <netipsec/esp_var.h>
732499f515Sriastradh #endif
741a8031e1Shsuenaga
751a8031e1Shsuenaga #include <arm/cpufunc.h>
761a8031e1Shsuenaga #include <arm/marvell/mvsocvar.h>
771a8031e1Shsuenaga #include <arm/marvell/armadaxpreg.h>
781a8031e1Shsuenaga #include <dev/marvell/marvellreg.h>
791a8031e1Shsuenaga #include <dev/marvell/marvellvar.h>
801a8031e1Shsuenaga #include <dev/marvell/mvxpsecreg.h>
811a8031e1Shsuenaga #include <dev/marvell/mvxpsecvar.h>
821a8031e1Shsuenaga
831a8031e1Shsuenaga #ifdef DEBUG
841a8031e1Shsuenaga #define STATIC __attribute__ ((noinline)) extern
851a8031e1Shsuenaga #define _STATIC __attribute__ ((noinline)) extern
861a8031e1Shsuenaga #define INLINE __attribute__ ((noinline)) extern
871a8031e1Shsuenaga #define _INLINE __attribute__ ((noinline)) extern
881a8031e1Shsuenaga #else
891a8031e1Shsuenaga #define STATIC static
901a8031e1Shsuenaga #define _STATIC __attribute__ ((unused)) static
911a8031e1Shsuenaga #define INLINE static inline
921a8031e1Shsuenaga #define _INLINE __attribute__ ((unused)) static inline
931a8031e1Shsuenaga #endif
941a8031e1Shsuenaga
951a8031e1Shsuenaga /*
961a8031e1Shsuenaga * IRQ and SRAM spaces for each of unit
971a8031e1Shsuenaga * XXX: move to attach_args
981a8031e1Shsuenaga */
991a8031e1Shsuenaga struct {
1001a8031e1Shsuenaga int err_int;
1011a8031e1Shsuenaga } mvxpsec_config[] = {
1021a8031e1Shsuenaga { .err_int = ARMADAXP_IRQ_CESA0_ERR, }, /* unit 0 */
1031a8031e1Shsuenaga { .err_int = ARMADAXP_IRQ_CESA1_ERR, }, /* unit 1 */
1041a8031e1Shsuenaga };
1051a8031e1Shsuenaga #define MVXPSEC_ERR_INT(sc) \
1061a8031e1Shsuenaga mvxpsec_config[device_unit((sc)->sc_dev)].err_int
1071a8031e1Shsuenaga
1081a8031e1Shsuenaga /*
1091a8031e1Shsuenaga * AES
1101a8031e1Shsuenaga */
1111a8031e1Shsuenaga #define MAXBC (128/32)
1121a8031e1Shsuenaga #define MAXKC (256/32)
1131a8031e1Shsuenaga #define MAXROUNDS 14
1141a8031e1Shsuenaga STATIC int mv_aes_ksched(uint8_t[4][MAXKC], int,
1151a8031e1Shsuenaga uint8_t[MAXROUNDS+1][4][MAXBC]);
1161a8031e1Shsuenaga STATIC int mv_aes_deckey(uint8_t *, uint8_t *, int);
1171a8031e1Shsuenaga
1181a8031e1Shsuenaga /*
1191a8031e1Shsuenaga * device driver autoconf interface
1201a8031e1Shsuenaga */
1211a8031e1Shsuenaga STATIC int mvxpsec_match(device_t, cfdata_t, void *);
1221a8031e1Shsuenaga STATIC void mvxpsec_attach(device_t, device_t, void *);
1231a8031e1Shsuenaga STATIC void mvxpsec_evcnt_attach(struct mvxpsec_softc *);
1241a8031e1Shsuenaga
1251a8031e1Shsuenaga /*
1261a8031e1Shsuenaga * register setup
1271a8031e1Shsuenaga */
1281a8031e1Shsuenaga STATIC int mvxpsec_wininit(struct mvxpsec_softc *, enum marvell_tags *);
1291a8031e1Shsuenaga
1301a8031e1Shsuenaga /*
1311a8031e1Shsuenaga * timer(callout) interface
1321a8031e1Shsuenaga *
1331a8031e1Shsuenaga * XXX: callout is not MP safe...
1341a8031e1Shsuenaga */
1351a8031e1Shsuenaga STATIC void mvxpsec_timer(void *);
1361a8031e1Shsuenaga
1371a8031e1Shsuenaga /*
1381a8031e1Shsuenaga * interrupt interface
1391a8031e1Shsuenaga */
1401a8031e1Shsuenaga STATIC int mvxpsec_intr(void *);
1411a8031e1Shsuenaga INLINE void mvxpsec_intr_cleanup(struct mvxpsec_softc *);
1421a8031e1Shsuenaga STATIC int mvxpsec_eintr(void *);
1431a8031e1Shsuenaga STATIC uint32_t mvxpsec_intr_ack(struct mvxpsec_softc *);
1441a8031e1Shsuenaga STATIC uint32_t mvxpsec_eintr_ack(struct mvxpsec_softc *);
1451a8031e1Shsuenaga INLINE void mvxpsec_intr_cnt(struct mvxpsec_softc *, int);
1461a8031e1Shsuenaga
1471a8031e1Shsuenaga /*
1481a8031e1Shsuenaga * memory allocators and VM management
1491a8031e1Shsuenaga */
1501a8031e1Shsuenaga STATIC struct mvxpsec_devmem *mvxpsec_alloc_devmem(struct mvxpsec_softc *,
1511a8031e1Shsuenaga paddr_t, int);
1521a8031e1Shsuenaga STATIC int mvxpsec_init_sram(struct mvxpsec_softc *);
1531a8031e1Shsuenaga
1541a8031e1Shsuenaga /*
1551a8031e1Shsuenaga * Low-level DMA interface
1561a8031e1Shsuenaga */
1571a8031e1Shsuenaga STATIC int mvxpsec_init_dma(struct mvxpsec_softc *,
1581a8031e1Shsuenaga struct marvell_attach_args *);
1591a8031e1Shsuenaga INLINE int mvxpsec_dma_wait(struct mvxpsec_softc *);
1601a8031e1Shsuenaga INLINE int mvxpsec_acc_wait(struct mvxpsec_softc *);
1611a8031e1Shsuenaga INLINE struct mvxpsec_descriptor_handle *mvxpsec_dma_getdesc(struct mvxpsec_softc *);
1621a8031e1Shsuenaga _INLINE void mvxpsec_dma_putdesc(struct mvxpsec_softc *, struct mvxpsec_descriptor_handle *);
1631a8031e1Shsuenaga INLINE void mvxpsec_dma_setup(struct mvxpsec_descriptor_handle *,
1641a8031e1Shsuenaga uint32_t, uint32_t, uint32_t);
1651a8031e1Shsuenaga INLINE void mvxpsec_dma_cat(struct mvxpsec_softc *,
1661a8031e1Shsuenaga struct mvxpsec_descriptor_handle *, struct mvxpsec_descriptor_handle *);
1671a8031e1Shsuenaga
1681a8031e1Shsuenaga /*
1691a8031e1Shsuenaga * High-level DMA interface
1701a8031e1Shsuenaga */
1711a8031e1Shsuenaga INLINE int mvxpsec_dma_copy0(struct mvxpsec_softc *,
1721a8031e1Shsuenaga mvxpsec_dma_ring *, uint32_t, uint32_t, uint32_t);
1731a8031e1Shsuenaga INLINE int mvxpsec_dma_copy(struct mvxpsec_softc *,
1741a8031e1Shsuenaga mvxpsec_dma_ring *, uint32_t, uint32_t, uint32_t);
1751a8031e1Shsuenaga INLINE int mvxpsec_dma_acc_activate(struct mvxpsec_softc *,
1761a8031e1Shsuenaga mvxpsec_dma_ring *);
1771a8031e1Shsuenaga INLINE void mvxpsec_dma_finalize(struct mvxpsec_softc *,
1781a8031e1Shsuenaga mvxpsec_dma_ring *);
1791a8031e1Shsuenaga INLINE void mvxpsec_dma_free(struct mvxpsec_softc *,
1801a8031e1Shsuenaga mvxpsec_dma_ring *);
1811a8031e1Shsuenaga INLINE int mvxpsec_dma_copy_packet(struct mvxpsec_softc *, struct mvxpsec_packet *);
1821a8031e1Shsuenaga INLINE int mvxpsec_dma_sync_packet(struct mvxpsec_softc *, struct mvxpsec_packet *);
1831a8031e1Shsuenaga
1841a8031e1Shsuenaga /*
1851a8031e1Shsuenaga * Session management interface (OpenCrypto)
1861a8031e1Shsuenaga */
1871a8031e1Shsuenaga #define MVXPSEC_SESSION(sid) ((sid) & 0x0fffffff)
1881a8031e1Shsuenaga #define MVXPSEC_SID(crd, sesn) (((crd) << 28) | ((sesn) & 0x0fffffff))
1891a8031e1Shsuenaga /* pool management */
1901a8031e1Shsuenaga STATIC int mvxpsec_session_ctor(void *, void *, int);
1911a8031e1Shsuenaga STATIC void mvxpsec_session_dtor(void *, void *);
1921a8031e1Shsuenaga STATIC int mvxpsec_packet_ctor(void *, void *, int);
1931a8031e1Shsuenaga STATIC void mvxpsec_packet_dtor(void *, void *);
1941a8031e1Shsuenaga
1951a8031e1Shsuenaga /* session management */
1961a8031e1Shsuenaga STATIC struct mvxpsec_session *mvxpsec_session_alloc(struct mvxpsec_softc *);
1971a8031e1Shsuenaga STATIC void mvxpsec_session_dealloc(struct mvxpsec_session *);
1981a8031e1Shsuenaga INLINE struct mvxpsec_session *mvxpsec_session_lookup(struct mvxpsec_softc *, int);
1991a8031e1Shsuenaga INLINE int mvxpsec_session_ref(struct mvxpsec_session *);
2001a8031e1Shsuenaga INLINE void mvxpsec_session_unref(struct mvxpsec_session *);
2011a8031e1Shsuenaga
2021a8031e1Shsuenaga /* packet management */
2031a8031e1Shsuenaga STATIC struct mvxpsec_packet *mvxpsec_packet_alloc(struct mvxpsec_session *);
2041a8031e1Shsuenaga INLINE void mvxpsec_packet_enqueue(struct mvxpsec_packet *);
2051a8031e1Shsuenaga STATIC void mvxpsec_packet_dealloc(struct mvxpsec_packet *);
2061a8031e1Shsuenaga STATIC int mvxpsec_done_packet(struct mvxpsec_packet *);
2071a8031e1Shsuenaga
2081a8031e1Shsuenaga /* session header manegement */
2091a8031e1Shsuenaga STATIC int mvxpsec_header_finalize(struct mvxpsec_packet *);
2101a8031e1Shsuenaga
2111a8031e1Shsuenaga /* packet queue management */
2121a8031e1Shsuenaga INLINE void mvxpsec_drop(struct mvxpsec_softc *, struct cryptop *, struct mvxpsec_packet *, int);
2131a8031e1Shsuenaga STATIC int mvxpsec_dispatch_queue(struct mvxpsec_softc *);
2141a8031e1Shsuenaga
21588af2332Smsaitoh /* opencrypto operation */
2161a8031e1Shsuenaga INLINE int mvxpsec_parse_crd(struct mvxpsec_packet *, struct cryptodesc *);
2171a8031e1Shsuenaga INLINE int mvxpsec_parse_crp(struct mvxpsec_packet *);
2181a8031e1Shsuenaga
2191a8031e1Shsuenaga /* payload data management */
2201a8031e1Shsuenaga INLINE int mvxpsec_packet_setcrp(struct mvxpsec_packet *, struct cryptop *);
2211a8031e1Shsuenaga STATIC int mvxpsec_packet_setdata(struct mvxpsec_packet *, void *, uint32_t);
2221a8031e1Shsuenaga STATIC int mvxpsec_packet_setmbuf(struct mvxpsec_packet *, struct mbuf *);
2231a8031e1Shsuenaga STATIC int mvxpsec_packet_setuio(struct mvxpsec_packet *, struct uio *);
2241a8031e1Shsuenaga STATIC int mvxpsec_packet_rdata(struct mvxpsec_packet *, int, int, void *);
2251a8031e1Shsuenaga _STATIC int mvxpsec_packet_wdata(struct mvxpsec_packet *, int, int, void *);
2261a8031e1Shsuenaga STATIC int mvxpsec_packet_write_iv(struct mvxpsec_packet *, void *, int);
2271a8031e1Shsuenaga STATIC int mvxpsec_packet_copy_iv(struct mvxpsec_packet *, int, int);
2281a8031e1Shsuenaga
2291a8031e1Shsuenaga /* key pre-computation */
2301a8031e1Shsuenaga STATIC int mvxpsec_key_precomp(int, void *, int, void *, void *);
2311a8031e1Shsuenaga STATIC int mvxpsec_hmac_precomp(int, void *, int, void *, void *);
2321a8031e1Shsuenaga
2331a8031e1Shsuenaga /* crypto operation management */
2341a8031e1Shsuenaga INLINE void mvxpsec_packet_reset_op(struct mvxpsec_packet *);
2351a8031e1Shsuenaga INLINE void mvxpsec_packet_update_op_order(struct mvxpsec_packet *, int);
2361a8031e1Shsuenaga
2371a8031e1Shsuenaga /*
2381a8031e1Shsuenaga * parameter converters
2391a8031e1Shsuenaga */
2401a8031e1Shsuenaga INLINE uint32_t mvxpsec_alg2acc(uint32_t alg);
2411a8031e1Shsuenaga INLINE uint32_t mvxpsec_aesklen(int klen);
2421a8031e1Shsuenaga
2431a8031e1Shsuenaga /*
2441a8031e1Shsuenaga * string formatters
2451a8031e1Shsuenaga */
2461a8031e1Shsuenaga _STATIC const char *s_ctrlreg(uint32_t);
2471a8031e1Shsuenaga _STATIC const char *s_winreg(uint32_t);
2481a8031e1Shsuenaga _STATIC const char *s_errreg(uint32_t);
2491a8031e1Shsuenaga _STATIC const char *s_xpsecintr(uint32_t);
2501a8031e1Shsuenaga _STATIC const char *s_ctlalg(uint32_t);
2511a8031e1Shsuenaga _STATIC const char *s_xpsec_op(uint32_t);
2521a8031e1Shsuenaga _STATIC const char *s_xpsec_enc(uint32_t);
2531a8031e1Shsuenaga _STATIC const char *s_xpsec_mac(uint32_t);
2541a8031e1Shsuenaga _STATIC const char *s_xpsec_frag(uint32_t);
2551a8031e1Shsuenaga
2561a8031e1Shsuenaga /*
2571a8031e1Shsuenaga * debugging supports
2581a8031e1Shsuenaga */
2591a8031e1Shsuenaga #ifdef MVXPSEC_DEBUG
2601a8031e1Shsuenaga _STATIC void mvxpsec_dump_dmaq(struct mvxpsec_descriptor_handle *);
2611a8031e1Shsuenaga _STATIC void mvxpsec_dump_reg(struct mvxpsec_softc *);
2621a8031e1Shsuenaga _STATIC void mvxpsec_dump_sram(const char *, struct mvxpsec_softc *, size_t);
2631a8031e1Shsuenaga _STATIC void mvxpsec_dump_data(const char *, void *, size_t);
2641a8031e1Shsuenaga
2651a8031e1Shsuenaga _STATIC void mvxpsec_dump_packet(const char *, struct mvxpsec_packet *);
2661a8031e1Shsuenaga _STATIC void mvxpsec_dump_packet_data(const char *, struct mvxpsec_packet *);
2671a8031e1Shsuenaga _STATIC void mvxpsec_dump_packet_desc(const char *, struct mvxpsec_packet *);
2681a8031e1Shsuenaga
2691a8031e1Shsuenaga _STATIC void mvxpsec_dump_acc_config(const char *, uint32_t);
2701a8031e1Shsuenaga _STATIC void mvxpsec_dump_acc_encdata(const char *, uint32_t, uint32_t);
2711a8031e1Shsuenaga _STATIC void mvxpsec_dump_acc_enclen(const char *, uint32_t);
2721a8031e1Shsuenaga _STATIC void mvxpsec_dump_acc_enckey(const char *, uint32_t);
2731a8031e1Shsuenaga _STATIC void mvxpsec_dump_acc_enciv(const char *, uint32_t);
2741a8031e1Shsuenaga _STATIC void mvxpsec_dump_acc_macsrc(const char *, uint32_t);
2751a8031e1Shsuenaga _STATIC void mvxpsec_dump_acc_macdst(const char *, uint32_t);
2761a8031e1Shsuenaga _STATIC void mvxpsec_dump_acc_maciv(const char *, uint32_t);
2771a8031e1Shsuenaga #endif
2781a8031e1Shsuenaga
2791a8031e1Shsuenaga /*
2801a8031e1Shsuenaga * global configurations, params, work spaces, ...
2811a8031e1Shsuenaga *
2821a8031e1Shsuenaga * XXX: use sysctl for global configurations
2831a8031e1Shsuenaga */
2841a8031e1Shsuenaga /* waiting for device */
2851a8031e1Shsuenaga static int mvxpsec_wait_interval = 10; /* usec */
2861a8031e1Shsuenaga static int mvxpsec_wait_retry = 100; /* times = wait for 1 [msec] */
2871a8031e1Shsuenaga #ifdef MVXPSEC_DEBUG
2881a8031e1Shsuenaga static uint32_t mvxpsec_debug = MVXPSEC_DEBUG; /* debug level */
2891a8031e1Shsuenaga #endif
2901a8031e1Shsuenaga
2911a8031e1Shsuenaga /*
2921a8031e1Shsuenaga * Register accessors
2931a8031e1Shsuenaga */
2941a8031e1Shsuenaga #define MVXPSEC_WRITE(sc, off, val) \
2951a8031e1Shsuenaga bus_space_write_4((sc)->sc_iot, (sc)->sc_ioh, (off), (val))
2961a8031e1Shsuenaga #define MVXPSEC_READ(sc, off) \
2971a8031e1Shsuenaga bus_space_read_4((sc)->sc_iot, (sc)->sc_ioh, (off))
2981a8031e1Shsuenaga
2991a8031e1Shsuenaga /*
3001a8031e1Shsuenaga * device driver autoconf interface
3011a8031e1Shsuenaga */
3021a8031e1Shsuenaga CFATTACH_DECL2_NEW(mvxpsec_mbus, sizeof(struct mvxpsec_softc),
3031a8031e1Shsuenaga mvxpsec_match, mvxpsec_attach, NULL, NULL, NULL, NULL);
3041a8031e1Shsuenaga
3051a8031e1Shsuenaga STATIC int
mvxpsec_match(device_t dev,cfdata_t match,void * aux)3061a8031e1Shsuenaga mvxpsec_match(device_t dev, cfdata_t match, void *aux)
3071a8031e1Shsuenaga {
3081a8031e1Shsuenaga struct marvell_attach_args *mva = aux;
3091a8031e1Shsuenaga uint32_t tag;
3101a8031e1Shsuenaga int window;
3111a8031e1Shsuenaga
3121a8031e1Shsuenaga if (strcmp(mva->mva_name, match->cf_name) != 0)
3131a8031e1Shsuenaga return 0;
3141a8031e1Shsuenaga if (mva->mva_offset == MVA_OFFSET_DEFAULT)
3151a8031e1Shsuenaga return 0;
3161a8031e1Shsuenaga
3171a8031e1Shsuenaga switch (mva->mva_unit) {
3181a8031e1Shsuenaga case 0:
3191a8031e1Shsuenaga tag = ARMADAXP_TAG_CRYPT0;
3201a8031e1Shsuenaga break;
3211a8031e1Shsuenaga case 1:
3221a8031e1Shsuenaga tag = ARMADAXP_TAG_CRYPT1;
3231a8031e1Shsuenaga break;
3241a8031e1Shsuenaga default:
3251a8031e1Shsuenaga aprint_error_dev(dev,
3261a8031e1Shsuenaga "unit %d is not supported\n", mva->mva_unit);
3271a8031e1Shsuenaga return 0;
3281a8031e1Shsuenaga }
3291a8031e1Shsuenaga
3301a8031e1Shsuenaga window = mvsoc_target(tag, NULL, NULL, NULL, NULL);
3311a8031e1Shsuenaga if (window >= nwindow) {
3321a8031e1Shsuenaga aprint_error_dev(dev,
3331a8031e1Shsuenaga "Security Accelerator SRAM is not configured.\n");
3341a8031e1Shsuenaga return 0;
3351a8031e1Shsuenaga }
3361a8031e1Shsuenaga
3371a8031e1Shsuenaga return 1;
3381a8031e1Shsuenaga }
3391a8031e1Shsuenaga
3401a8031e1Shsuenaga STATIC void
mvxpsec_attach(device_t parent,device_t self,void * aux)3411a8031e1Shsuenaga mvxpsec_attach(device_t parent, device_t self, void *aux)
3421a8031e1Shsuenaga {
3431a8031e1Shsuenaga struct marvell_attach_args *mva = aux;
3441a8031e1Shsuenaga struct mvxpsec_softc *sc = device_private(self);
3451a8031e1Shsuenaga int v;
3461a8031e1Shsuenaga int i;
3471a8031e1Shsuenaga
3481a8031e1Shsuenaga sc->sc_dev = self;
3491a8031e1Shsuenaga
3501a8031e1Shsuenaga aprint_normal(": Marvell Crypto Engines and Security Accelerator\n");
3511a8031e1Shsuenaga aprint_naive("\n");
3521a8031e1Shsuenaga #ifdef MVXPSEC_MULTI_PACKET
3531a8031e1Shsuenaga aprint_normal_dev(sc->sc_dev, "multi-packet chained mode enabled.\n");
3541a8031e1Shsuenaga #else
3551a8031e1Shsuenaga aprint_normal_dev(sc->sc_dev, "multi-packet chained mode disabled.\n");
3561a8031e1Shsuenaga #endif
3571a8031e1Shsuenaga aprint_normal_dev(sc->sc_dev,
3581a8031e1Shsuenaga "Max %d sessions.\n", MVXPSEC_MAX_SESSIONS);
3591a8031e1Shsuenaga
3601a8031e1Shsuenaga /* mutex */
3611a8031e1Shsuenaga mutex_init(&sc->sc_session_mtx, MUTEX_DEFAULT, IPL_NET);
3621a8031e1Shsuenaga mutex_init(&sc->sc_dma_mtx, MUTEX_DEFAULT, IPL_NET);
3631a8031e1Shsuenaga mutex_init(&sc->sc_queue_mtx, MUTEX_DEFAULT, IPL_NET);
3641a8031e1Shsuenaga
3651a8031e1Shsuenaga /* Packet queue */
3661a8031e1Shsuenaga SIMPLEQ_INIT(&sc->sc_wait_queue);
3671a8031e1Shsuenaga SIMPLEQ_INIT(&sc->sc_run_queue);
3681a8031e1Shsuenaga SLIST_INIT(&sc->sc_free_list);
3691a8031e1Shsuenaga sc->sc_wait_qlen = 0;
3701a8031e1Shsuenaga #ifdef MVXPSEC_MULTI_PACKET
3711a8031e1Shsuenaga sc->sc_wait_qlimit = 16;
3721a8031e1Shsuenaga #else
3731a8031e1Shsuenaga sc->sc_wait_qlimit = 0;
3741a8031e1Shsuenaga #endif
3751a8031e1Shsuenaga sc->sc_free_qlen = 0;
3761a8031e1Shsuenaga
3771a8031e1Shsuenaga /* Timer */
3781a8031e1Shsuenaga callout_init(&sc->sc_timeout, 0); /* XXX: use CALLOUT_MPSAFE */
3791a8031e1Shsuenaga callout_setfunc(&sc->sc_timeout, mvxpsec_timer, sc);
3801a8031e1Shsuenaga
3811a8031e1Shsuenaga /* I/O */
3821a8031e1Shsuenaga sc->sc_iot = mva->mva_iot;
3831a8031e1Shsuenaga if (bus_space_subregion(mva->mva_iot, mva->mva_ioh,
3841a8031e1Shsuenaga mva->mva_offset, mva->mva_size, &sc->sc_ioh)) {
3851a8031e1Shsuenaga aprint_error_dev(self, "Cannot map registers\n");
3861a8031e1Shsuenaga return;
3871a8031e1Shsuenaga }
3881a8031e1Shsuenaga
3891a8031e1Shsuenaga /* DMA */
3901a8031e1Shsuenaga sc->sc_dmat = mva->mva_dmat;
3911a8031e1Shsuenaga if (mvxpsec_init_dma(sc, mva) < 0)
3921a8031e1Shsuenaga return;
3931a8031e1Shsuenaga
3941a8031e1Shsuenaga /* SRAM */
3951a8031e1Shsuenaga if (mvxpsec_init_sram(sc) < 0)
3961a8031e1Shsuenaga return;
3971a8031e1Shsuenaga
3981a8031e1Shsuenaga /* Registers */
3991a8031e1Shsuenaga mvxpsec_wininit(sc, mva->mva_tags);
4001a8031e1Shsuenaga
4011a8031e1Shsuenaga /* INTR */
4021a8031e1Shsuenaga MVXPSEC_WRITE(sc, MVXPSEC_INT_MASK, MVXPSEC_DEFAULT_INT);
4031a8031e1Shsuenaga MVXPSEC_WRITE(sc, MV_TDMA_ERR_MASK, MVXPSEC_DEFAULT_ERR);
4041a8031e1Shsuenaga sc->sc_done_ih =
4051a8031e1Shsuenaga marvell_intr_establish(mva->mva_irq, IPL_NET, mvxpsec_intr, sc);
406de0a2812Sandvar /* XXX: should pass error IRQ using mva */
4071a8031e1Shsuenaga sc->sc_error_ih = marvell_intr_establish(MVXPSEC_ERR_INT(sc),
4081a8031e1Shsuenaga IPL_NET, mvxpsec_eintr, sc);
4091a8031e1Shsuenaga aprint_normal_dev(self,
4101a8031e1Shsuenaga "Error Reporting IRQ %d\n", MVXPSEC_ERR_INT(sc));
4111a8031e1Shsuenaga
4121a8031e1Shsuenaga /* Initialize TDMA (It's enabled here, but waiting for SA) */
4131a8031e1Shsuenaga if (mvxpsec_dma_wait(sc) < 0)
4141a8031e1Shsuenaga panic("%s: DMA DEVICE not responding\n", __func__);
4151a8031e1Shsuenaga MVXPSEC_WRITE(sc, MV_TDMA_CNT, 0);
4161a8031e1Shsuenaga MVXPSEC_WRITE(sc, MV_TDMA_SRC, 0);
4171a8031e1Shsuenaga MVXPSEC_WRITE(sc, MV_TDMA_DST, 0);
4181a8031e1Shsuenaga MVXPSEC_WRITE(sc, MV_TDMA_NXT, 0);
4191a8031e1Shsuenaga MVXPSEC_WRITE(sc, MV_TDMA_CUR, 0);
4201a8031e1Shsuenaga v = MVXPSEC_READ(sc, MV_TDMA_CONTROL);
4211a8031e1Shsuenaga v |= MV_TDMA_CONTROL_ENABLE;
4221a8031e1Shsuenaga MVXPSEC_WRITE(sc, MV_TDMA_CONTROL, v);
4231a8031e1Shsuenaga
4241a8031e1Shsuenaga /* Initialize SA */
4251a8031e1Shsuenaga if (mvxpsec_acc_wait(sc) < 0)
4261a8031e1Shsuenaga panic("%s: MVXPSEC not responding\n", __func__);
4271a8031e1Shsuenaga v = MVXPSEC_READ(sc, MV_ACC_CONFIG);
4281a8031e1Shsuenaga v &= ~MV_ACC_CONFIG_STOP_ON_ERR;
4291a8031e1Shsuenaga v |= MV_ACC_CONFIG_MULT_PKT;
4301a8031e1Shsuenaga v |= MV_ACC_CONFIG_WAIT_TDMA;
4311a8031e1Shsuenaga v |= MV_ACC_CONFIG_ACT_TDMA;
4321a8031e1Shsuenaga MVXPSEC_WRITE(sc, MV_ACC_CONFIG, v);
4331a8031e1Shsuenaga MVXPSEC_WRITE(sc, MV_ACC_DESC, 0);
4341a8031e1Shsuenaga MVXPSEC_WRITE(sc, MV_ACC_COMMAND, MV_ACC_COMMAND_STOP);
4351a8031e1Shsuenaga
4361a8031e1Shsuenaga /* Session */
4371a8031e1Shsuenaga sc->sc_session_pool =
4381a8031e1Shsuenaga pool_cache_init(sizeof(struct mvxpsec_session), 0, 0, 0,
4391a8031e1Shsuenaga "mvxpsecpl", NULL, IPL_NET,
4401a8031e1Shsuenaga mvxpsec_session_ctor, mvxpsec_session_dtor, sc);
4411a8031e1Shsuenaga pool_cache_sethiwat(sc->sc_session_pool, MVXPSEC_MAX_SESSIONS);
4421a8031e1Shsuenaga pool_cache_setlowat(sc->sc_session_pool, MVXPSEC_MAX_SESSIONS / 2);
4431a8031e1Shsuenaga sc->sc_last_session = NULL;
4441a8031e1Shsuenaga
445ab9bc2adSandvar /* Packet */
4461a8031e1Shsuenaga sc->sc_packet_pool =
4471a8031e1Shsuenaga pool_cache_init(sizeof(struct mvxpsec_session), 0, 0, 0,
4481a8031e1Shsuenaga "mvxpsec_pktpl", NULL, IPL_NET,
4491a8031e1Shsuenaga mvxpsec_packet_ctor, mvxpsec_packet_dtor, sc);
4501a8031e1Shsuenaga pool_cache_sethiwat(sc->sc_packet_pool, MVXPSEC_MAX_SESSIONS);
4511a8031e1Shsuenaga pool_cache_setlowat(sc->sc_packet_pool, MVXPSEC_MAX_SESSIONS / 2);
4521a8031e1Shsuenaga
4531a8031e1Shsuenaga /* Register to EVCNT framework */
4541a8031e1Shsuenaga mvxpsec_evcnt_attach(sc);
4551a8031e1Shsuenaga
4561a8031e1Shsuenaga /* Register to Opencrypto */
4571a8031e1Shsuenaga for (i = 0; i < MVXPSEC_MAX_SESSIONS; i++) {
4581a8031e1Shsuenaga sc->sc_sessions[i] = NULL;
4591a8031e1Shsuenaga }
4601a8031e1Shsuenaga if (mvxpsec_register(sc))
4611a8031e1Shsuenaga panic("cannot initialize OpenCrypto module.\n");
4621a8031e1Shsuenaga
4631a8031e1Shsuenaga return;
4641a8031e1Shsuenaga }
4651a8031e1Shsuenaga
4661a8031e1Shsuenaga STATIC void
mvxpsec_evcnt_attach(struct mvxpsec_softc * sc)4671a8031e1Shsuenaga mvxpsec_evcnt_attach(struct mvxpsec_softc *sc)
4681a8031e1Shsuenaga {
4691a8031e1Shsuenaga struct mvxpsec_evcnt *sc_ev = &sc->sc_ev;
4701a8031e1Shsuenaga
4711a8031e1Shsuenaga evcnt_attach_dynamic(&sc_ev->intr_all, EVCNT_TYPE_INTR,
4721a8031e1Shsuenaga NULL, device_xname(sc->sc_dev), "Main Intr.");
4731a8031e1Shsuenaga evcnt_attach_dynamic(&sc_ev->intr_auth, EVCNT_TYPE_INTR,
4741a8031e1Shsuenaga NULL, device_xname(sc->sc_dev), "Auth Intr.");
4751a8031e1Shsuenaga evcnt_attach_dynamic(&sc_ev->intr_des, EVCNT_TYPE_INTR,
4761a8031e1Shsuenaga NULL, device_xname(sc->sc_dev), "DES Intr.");
4771a8031e1Shsuenaga evcnt_attach_dynamic(&sc_ev->intr_aes_enc, EVCNT_TYPE_INTR,
4781a8031e1Shsuenaga NULL, device_xname(sc->sc_dev), "AES-Encrypt Intr.");
4791a8031e1Shsuenaga evcnt_attach_dynamic(&sc_ev->intr_aes_dec, EVCNT_TYPE_INTR,
4801a8031e1Shsuenaga NULL, device_xname(sc->sc_dev), "AES-Decrypt Intr.");
4811a8031e1Shsuenaga evcnt_attach_dynamic(&sc_ev->intr_enc, EVCNT_TYPE_INTR,
4821a8031e1Shsuenaga NULL, device_xname(sc->sc_dev), "Crypto Intr.");
4831a8031e1Shsuenaga evcnt_attach_dynamic(&sc_ev->intr_sa, EVCNT_TYPE_INTR,
4841a8031e1Shsuenaga NULL, device_xname(sc->sc_dev), "SA Intr.");
4851a8031e1Shsuenaga evcnt_attach_dynamic(&sc_ev->intr_acctdma, EVCNT_TYPE_INTR,
4861a8031e1Shsuenaga NULL, device_xname(sc->sc_dev), "AccTDMA Intr.");
4871a8031e1Shsuenaga evcnt_attach_dynamic(&sc_ev->intr_comp, EVCNT_TYPE_INTR,
4881a8031e1Shsuenaga NULL, device_xname(sc->sc_dev), "TDMA-Complete Intr.");
4891a8031e1Shsuenaga evcnt_attach_dynamic(&sc_ev->intr_own, EVCNT_TYPE_INTR,
4901a8031e1Shsuenaga NULL, device_xname(sc->sc_dev), "TDMA-Ownership Intr.");
4911a8031e1Shsuenaga evcnt_attach_dynamic(&sc_ev->intr_acctdma_cont, EVCNT_TYPE_INTR,
4921a8031e1Shsuenaga NULL, device_xname(sc->sc_dev), "AccTDMA-Continue Intr.");
4931a8031e1Shsuenaga
4941a8031e1Shsuenaga evcnt_attach_dynamic(&sc_ev->session_new, EVCNT_TYPE_MISC,
4951a8031e1Shsuenaga NULL, device_xname(sc->sc_dev), "New-Session");
4961a8031e1Shsuenaga evcnt_attach_dynamic(&sc_ev->session_free, EVCNT_TYPE_MISC,
4971a8031e1Shsuenaga NULL, device_xname(sc->sc_dev), "Free-Session");
4981a8031e1Shsuenaga
4991a8031e1Shsuenaga evcnt_attach_dynamic(&sc_ev->packet_ok, EVCNT_TYPE_MISC,
5001a8031e1Shsuenaga NULL, device_xname(sc->sc_dev), "Packet-OK");
5011a8031e1Shsuenaga evcnt_attach_dynamic(&sc_ev->packet_err, EVCNT_TYPE_MISC,
5021a8031e1Shsuenaga NULL, device_xname(sc->sc_dev), "Packet-ERR");
5031a8031e1Shsuenaga
5041a8031e1Shsuenaga evcnt_attach_dynamic(&sc_ev->dispatch_packets, EVCNT_TYPE_MISC,
5051a8031e1Shsuenaga NULL, device_xname(sc->sc_dev), "Packet-Dispatch");
5061a8031e1Shsuenaga evcnt_attach_dynamic(&sc_ev->dispatch_queue, EVCNT_TYPE_MISC,
5071a8031e1Shsuenaga NULL, device_xname(sc->sc_dev), "Queue-Dispatch");
5081a8031e1Shsuenaga evcnt_attach_dynamic(&sc_ev->queue_full, EVCNT_TYPE_MISC,
5091a8031e1Shsuenaga NULL, device_xname(sc->sc_dev), "Queue-Full");
5101a8031e1Shsuenaga evcnt_attach_dynamic(&sc_ev->max_dispatch, EVCNT_TYPE_MISC,
5111a8031e1Shsuenaga NULL, device_xname(sc->sc_dev), "Max-Dispatch");
5121a8031e1Shsuenaga evcnt_attach_dynamic(&sc_ev->max_done, EVCNT_TYPE_MISC,
5131a8031e1Shsuenaga NULL, device_xname(sc->sc_dev), "Max-Done");
5141a8031e1Shsuenaga }
5151a8031e1Shsuenaga
5161a8031e1Shsuenaga /*
5171a8031e1Shsuenaga * Register setup
5181a8031e1Shsuenaga */
mvxpsec_wininit(struct mvxpsec_softc * sc,enum marvell_tags * tags)5191a8031e1Shsuenaga STATIC int mvxpsec_wininit(struct mvxpsec_softc *sc, enum marvell_tags *tags)
5201a8031e1Shsuenaga {
5211a8031e1Shsuenaga device_t pdev = device_parent(sc->sc_dev);
5221a8031e1Shsuenaga uint64_t base;
5231a8031e1Shsuenaga uint32_t size, reg;
5241a8031e1Shsuenaga int window, target, attr, rv, i;
5251a8031e1Shsuenaga
5261a8031e1Shsuenaga /* disable all window */
5271a8031e1Shsuenaga for (window = 0; window < MV_TDMA_NWINDOW; window++)
5281a8031e1Shsuenaga {
5291a8031e1Shsuenaga MVXPSEC_WRITE(sc, MV_TDMA_BAR(window), 0);
5301a8031e1Shsuenaga MVXPSEC_WRITE(sc, MV_TDMA_ATTR(window), 0);
5311a8031e1Shsuenaga }
5321a8031e1Shsuenaga
5331a8031e1Shsuenaga for (window = 0, i = 0;
5341a8031e1Shsuenaga tags[i] != MARVELL_TAG_UNDEFINED && window < MV_TDMA_NWINDOW; i++) {
5351a8031e1Shsuenaga rv = marvell_winparams_by_tag(pdev, tags[i],
5361a8031e1Shsuenaga &target, &attr, &base, &size);
5371a8031e1Shsuenaga if (rv != 0 || size == 0)
5381a8031e1Shsuenaga continue;
5391a8031e1Shsuenaga
5401a8031e1Shsuenaga if (base > 0xffffffffULL) {
5411a8031e1Shsuenaga aprint_error_dev(sc->sc_dev,
5421a8031e1Shsuenaga "can't remap window %d\n", window);
5431a8031e1Shsuenaga continue;
5441a8031e1Shsuenaga }
5451a8031e1Shsuenaga
5461a8031e1Shsuenaga reg = MV_TDMA_BAR_BASE(base);
5471a8031e1Shsuenaga MVXPSEC_WRITE(sc, MV_TDMA_BAR(window), reg);
5481a8031e1Shsuenaga
5491a8031e1Shsuenaga reg = MV_TDMA_ATTR_TARGET(target);
5501a8031e1Shsuenaga reg |= MV_TDMA_ATTR_ATTR(attr);
5511a8031e1Shsuenaga reg |= MV_TDMA_ATTR_SIZE(size);
5521a8031e1Shsuenaga reg |= MV_TDMA_ATTR_ENABLE;
5531a8031e1Shsuenaga MVXPSEC_WRITE(sc, MV_TDMA_ATTR(window), reg);
5541a8031e1Shsuenaga
5551a8031e1Shsuenaga window++;
5561a8031e1Shsuenaga }
5571a8031e1Shsuenaga
5581a8031e1Shsuenaga return 0;
5591a8031e1Shsuenaga }
5601a8031e1Shsuenaga
5611a8031e1Shsuenaga /*
5621a8031e1Shsuenaga * Timer handling
5631a8031e1Shsuenaga */
5641a8031e1Shsuenaga STATIC void
mvxpsec_timer(void * aux)5651a8031e1Shsuenaga mvxpsec_timer(void *aux)
5661a8031e1Shsuenaga {
5671a8031e1Shsuenaga struct mvxpsec_softc *sc = aux;
5681a8031e1Shsuenaga struct mvxpsec_packet *mv_p;
5691a8031e1Shsuenaga uint32_t reg;
5701a8031e1Shsuenaga int ndone;
5711a8031e1Shsuenaga int refill;
5721a8031e1Shsuenaga int s;
5731a8031e1Shsuenaga
5741a8031e1Shsuenaga /* IPL_SOFTCLOCK */
5751a8031e1Shsuenaga
5761a8031e1Shsuenaga log(LOG_ERR, "%s: device timeout.\n", __func__);
5771a8031e1Shsuenaga #ifdef MVXPSEC_DEBUG
5781a8031e1Shsuenaga mvxpsec_dump_reg(sc);
5791a8031e1Shsuenaga #endif
5801a8031e1Shsuenaga
5811a8031e1Shsuenaga s = splnet();
5821a8031e1Shsuenaga /* stop security accelerator */
5831a8031e1Shsuenaga MVXPSEC_WRITE(sc, MV_ACC_COMMAND, MV_ACC_COMMAND_STOP);
5841a8031e1Shsuenaga
5851a8031e1Shsuenaga /* stop TDMA */
5861a8031e1Shsuenaga MVXPSEC_WRITE(sc, MV_TDMA_CONTROL, 0);
5871a8031e1Shsuenaga
5881a8031e1Shsuenaga /* cleanup packet queue */
5891a8031e1Shsuenaga mutex_enter(&sc->sc_queue_mtx);
5901a8031e1Shsuenaga ndone = 0;
5911a8031e1Shsuenaga while ( (mv_p = SIMPLEQ_FIRST(&sc->sc_run_queue)) != NULL) {
5921a8031e1Shsuenaga SIMPLEQ_REMOVE_HEAD(&sc->sc_run_queue, queue);
5931a8031e1Shsuenaga
5941a8031e1Shsuenaga mv_p->crp->crp_etype = EINVAL;
5951a8031e1Shsuenaga mvxpsec_done_packet(mv_p);
5961a8031e1Shsuenaga ndone++;
5971a8031e1Shsuenaga }
5981a8031e1Shsuenaga MVXPSEC_EVCNT_MAX(sc, max_done, ndone);
5991a8031e1Shsuenaga sc->sc_flags &= ~HW_RUNNING;
6001a8031e1Shsuenaga refill = (sc->sc_wait_qlen > 0) ? 1 : 0;
6011a8031e1Shsuenaga mutex_exit(&sc->sc_queue_mtx);
6021a8031e1Shsuenaga
6031a8031e1Shsuenaga /* reenable TDMA */
6041a8031e1Shsuenaga if (mvxpsec_dma_wait(sc) < 0)
6051a8031e1Shsuenaga panic("%s: failed to reset DMA DEVICE. give up.", __func__);
6061a8031e1Shsuenaga MVXPSEC_WRITE(sc, MV_TDMA_CNT, 0);
6071a8031e1Shsuenaga MVXPSEC_WRITE(sc, MV_TDMA_SRC, 0);
6081a8031e1Shsuenaga MVXPSEC_WRITE(sc, MV_TDMA_DST, 0);
6091a8031e1Shsuenaga MVXPSEC_WRITE(sc, MV_TDMA_CUR, 0);
6101a8031e1Shsuenaga MVXPSEC_WRITE(sc, MV_TDMA_NXT, 0);
6111a8031e1Shsuenaga reg = MV_TDMA_DEFAULT_CONTROL;
6121a8031e1Shsuenaga reg |= MV_TDMA_CONTROL_ENABLE;
6131a8031e1Shsuenaga MVXPSEC_WRITE(sc, MV_TDMA_CONTROL, reg);
6141a8031e1Shsuenaga
6151a8031e1Shsuenaga if (mvxpsec_acc_wait(sc) < 0)
6161a8031e1Shsuenaga panic("%s: failed to reset MVXPSEC. give up.", __func__);
6171a8031e1Shsuenaga reg = MV_ACC_CONFIG_MULT_PKT;
6181a8031e1Shsuenaga reg |= MV_ACC_CONFIG_WAIT_TDMA;
6191a8031e1Shsuenaga reg |= MV_ACC_CONFIG_ACT_TDMA;
6201a8031e1Shsuenaga MVXPSEC_WRITE(sc, MV_ACC_CONFIG, reg);
6211a8031e1Shsuenaga MVXPSEC_WRITE(sc, MV_ACC_DESC, 0);
6221a8031e1Shsuenaga
6231a8031e1Shsuenaga if (refill) {
6241a8031e1Shsuenaga mutex_enter(&sc->sc_queue_mtx);
6251a8031e1Shsuenaga mvxpsec_dispatch_queue(sc);
6261a8031e1Shsuenaga mutex_exit(&sc->sc_queue_mtx);
6271a8031e1Shsuenaga }
6281a8031e1Shsuenaga
6291a8031e1Shsuenaga crypto_unblock(sc->sc_cid, CRYPTO_SYMQ|CRYPTO_ASYMQ);
6301a8031e1Shsuenaga splx(s);
6311a8031e1Shsuenaga }
6321a8031e1Shsuenaga
6331a8031e1Shsuenaga /*
6341a8031e1Shsuenaga * DMA handling
6351a8031e1Shsuenaga */
6361a8031e1Shsuenaga
6371a8031e1Shsuenaga /*
6381a8031e1Shsuenaga * Allocate kernel devmem and DMA safe memory with bus_dma API
6391a8031e1Shsuenaga * used for DMA descriptors.
6401a8031e1Shsuenaga *
6411a8031e1Shsuenaga * if phys != 0, assume phys is a DMA safe memory and bypass
6421a8031e1Shsuenaga * allocator.
6431a8031e1Shsuenaga */
6441a8031e1Shsuenaga STATIC struct mvxpsec_devmem *
mvxpsec_alloc_devmem(struct mvxpsec_softc * sc,paddr_t phys,int size)6451a8031e1Shsuenaga mvxpsec_alloc_devmem(struct mvxpsec_softc *sc, paddr_t phys, int size)
6461a8031e1Shsuenaga {
6471a8031e1Shsuenaga struct mvxpsec_devmem *devmem;
6481a8031e1Shsuenaga bus_dma_segment_t seg;
6491a8031e1Shsuenaga int rseg;
6501a8031e1Shsuenaga int err;
6511a8031e1Shsuenaga
6521a8031e1Shsuenaga if (sc == NULL)
6531a8031e1Shsuenaga return NULL;
6541a8031e1Shsuenaga
6550a827a3fSchs devmem = kmem_alloc(sizeof(*devmem), KM_SLEEP);
6561a8031e1Shsuenaga devmem->size = size;
6571a8031e1Shsuenaga
6581a8031e1Shsuenaga if (phys) {
6591a8031e1Shsuenaga seg.ds_addr = phys;
6601a8031e1Shsuenaga seg.ds_len = devmem->size;
6611a8031e1Shsuenaga rseg = 1;
6621a8031e1Shsuenaga err = 0;
6631a8031e1Shsuenaga }
6641a8031e1Shsuenaga else {
6651a8031e1Shsuenaga err = bus_dmamem_alloc(sc->sc_dmat,
6661a8031e1Shsuenaga devmem->size, PAGE_SIZE, 0,
6671a8031e1Shsuenaga &seg, MVXPSEC_DMA_MAX_SEGS, &rseg, BUS_DMA_NOWAIT);
6681a8031e1Shsuenaga }
6691a8031e1Shsuenaga if (err) {
6701a8031e1Shsuenaga aprint_error_dev(sc->sc_dev, "can't alloc DMA buffer\n");
6711a8031e1Shsuenaga goto fail_kmem_free;
6721a8031e1Shsuenaga }
6731a8031e1Shsuenaga
6741a8031e1Shsuenaga err = bus_dmamem_map(sc->sc_dmat, &seg, rseg,
6751a8031e1Shsuenaga devmem->size, &devmem->kva, BUS_DMA_NOWAIT);
6761a8031e1Shsuenaga if (err) {
6771a8031e1Shsuenaga aprint_error_dev(sc->sc_dev, "can't map DMA buffer\n");
6781a8031e1Shsuenaga goto fail_dmamem_free;
6791a8031e1Shsuenaga }
6801a8031e1Shsuenaga
6811a8031e1Shsuenaga err = bus_dmamap_create(sc->sc_dmat,
6821a8031e1Shsuenaga size, 1, size, 0, BUS_DMA_NOWAIT, &devmem->map);
6831a8031e1Shsuenaga if (err) {
6841a8031e1Shsuenaga aprint_error_dev(sc->sc_dev, "can't create DMA map\n");
6851a8031e1Shsuenaga goto fail_unmap;
6861a8031e1Shsuenaga }
6871a8031e1Shsuenaga
6881a8031e1Shsuenaga err = bus_dmamap_load(sc->sc_dmat,
6891a8031e1Shsuenaga devmem->map, devmem->kva, devmem->size, NULL,
6901a8031e1Shsuenaga BUS_DMA_NOWAIT);
6911a8031e1Shsuenaga if (err) {
6921a8031e1Shsuenaga aprint_error_dev(sc->sc_dev,
6931a8031e1Shsuenaga "can't load DMA buffer VA:%p PA:0x%08x\n",
6941a8031e1Shsuenaga devmem->kva, (int)seg.ds_addr);
6951a8031e1Shsuenaga goto fail_destroy;
6961a8031e1Shsuenaga }
6971a8031e1Shsuenaga
6981a8031e1Shsuenaga return devmem;
6991a8031e1Shsuenaga
7001a8031e1Shsuenaga fail_destroy:
7011a8031e1Shsuenaga bus_dmamap_destroy(sc->sc_dmat, devmem->map);
7021a8031e1Shsuenaga fail_unmap:
7031a8031e1Shsuenaga bus_dmamem_unmap(sc->sc_dmat, devmem->kva, devmem->size);
7041a8031e1Shsuenaga fail_dmamem_free:
7051a8031e1Shsuenaga bus_dmamem_free(sc->sc_dmat, &seg, rseg);
7061a8031e1Shsuenaga fail_kmem_free:
7071a8031e1Shsuenaga kmem_free(devmem, sizeof(*devmem));
7081a8031e1Shsuenaga
7091a8031e1Shsuenaga return NULL;
7101a8031e1Shsuenaga }
7111a8031e1Shsuenaga
7121a8031e1Shsuenaga /*
7131a8031e1Shsuenaga * Get DMA Descriptor from (DMA safe) descriptor pool.
7141a8031e1Shsuenaga */
7151a8031e1Shsuenaga INLINE struct mvxpsec_descriptor_handle *
mvxpsec_dma_getdesc(struct mvxpsec_softc * sc)7161a8031e1Shsuenaga mvxpsec_dma_getdesc(struct mvxpsec_softc *sc)
7171a8031e1Shsuenaga {
7181a8031e1Shsuenaga struct mvxpsec_descriptor_handle *entry;
7191a8031e1Shsuenaga
7201a8031e1Shsuenaga /* must called with sc->sc_dma_mtx held */
7211a8031e1Shsuenaga KASSERT(mutex_owned(&sc->sc_dma_mtx));
7221a8031e1Shsuenaga
7231a8031e1Shsuenaga if (sc->sc_desc_ring_prod == sc->sc_desc_ring_cons)
7241a8031e1Shsuenaga return NULL;
7251a8031e1Shsuenaga
7261a8031e1Shsuenaga entry = &sc->sc_desc_ring[sc->sc_desc_ring_prod];
7271a8031e1Shsuenaga sc->sc_desc_ring_prod++;
7281a8031e1Shsuenaga if (sc->sc_desc_ring_prod >= sc->sc_desc_ring_size)
7291a8031e1Shsuenaga sc->sc_desc_ring_prod -= sc->sc_desc_ring_size;
7301a8031e1Shsuenaga
7311a8031e1Shsuenaga return entry;
7321a8031e1Shsuenaga }
7331a8031e1Shsuenaga
7341a8031e1Shsuenaga /*
7351a8031e1Shsuenaga * Put DMA Descriptor to descriptor pool.
7361a8031e1Shsuenaga */
7371a8031e1Shsuenaga _INLINE void
mvxpsec_dma_putdesc(struct mvxpsec_softc * sc,struct mvxpsec_descriptor_handle * dh)7381a8031e1Shsuenaga mvxpsec_dma_putdesc(struct mvxpsec_softc *sc,
7391a8031e1Shsuenaga struct mvxpsec_descriptor_handle *dh)
7401a8031e1Shsuenaga {
7411a8031e1Shsuenaga /* must called with sc->sc_dma_mtx held */
7421a8031e1Shsuenaga KASSERT(mutex_owned(&sc->sc_dma_mtx));
7431a8031e1Shsuenaga
7441a8031e1Shsuenaga sc->sc_desc_ring_cons++;
7451a8031e1Shsuenaga if (sc->sc_desc_ring_cons >= sc->sc_desc_ring_size)
7461a8031e1Shsuenaga sc->sc_desc_ring_cons -= sc->sc_desc_ring_size;
7471a8031e1Shsuenaga
7481a8031e1Shsuenaga return;
7491a8031e1Shsuenaga }
7501a8031e1Shsuenaga
7511a8031e1Shsuenaga /*
7521a8031e1Shsuenaga * Setup DMA Descriptor
7531a8031e1Shsuenaga * copy from 'src' to 'dst' by 'size' bytes.
7541a8031e1Shsuenaga * 'src' or 'dst' must be SRAM address.
7551a8031e1Shsuenaga */
7561a8031e1Shsuenaga INLINE void
mvxpsec_dma_setup(struct mvxpsec_descriptor_handle * dh,uint32_t dst,uint32_t src,uint32_t size)7571a8031e1Shsuenaga mvxpsec_dma_setup(struct mvxpsec_descriptor_handle *dh,
7581a8031e1Shsuenaga uint32_t dst, uint32_t src, uint32_t size)
7591a8031e1Shsuenaga {
7601a8031e1Shsuenaga struct mvxpsec_descriptor *desc;
7611a8031e1Shsuenaga
7621a8031e1Shsuenaga desc = (struct mvxpsec_descriptor *)dh->_desc;
7631a8031e1Shsuenaga
7641a8031e1Shsuenaga desc->tdma_dst = dst;
7651a8031e1Shsuenaga desc->tdma_src = src;
7661a8031e1Shsuenaga desc->tdma_word0 = size;
7671a8031e1Shsuenaga if (size != 0)
7681a8031e1Shsuenaga desc->tdma_word0 |= MV_TDMA_CNT_OWN;
7691a8031e1Shsuenaga /* size == 0 is owned by ACC, not TDMA */
7701a8031e1Shsuenaga
7711a8031e1Shsuenaga #ifdef MVXPSEC_DEBUG
7721a8031e1Shsuenaga mvxpsec_dump_dmaq(dh);
7731a8031e1Shsuenaga #endif
7741a8031e1Shsuenaga }
7751a8031e1Shsuenaga
7761a8031e1Shsuenaga /*
7771a8031e1Shsuenaga * Concat 2 DMA
7781a8031e1Shsuenaga */
7791a8031e1Shsuenaga INLINE void
mvxpsec_dma_cat(struct mvxpsec_softc * sc,struct mvxpsec_descriptor_handle * dh1,struct mvxpsec_descriptor_handle * dh2)7801a8031e1Shsuenaga mvxpsec_dma_cat(struct mvxpsec_softc *sc,
7811a8031e1Shsuenaga struct mvxpsec_descriptor_handle *dh1,
7821a8031e1Shsuenaga struct mvxpsec_descriptor_handle *dh2)
7831a8031e1Shsuenaga {
7841a8031e1Shsuenaga ((struct mvxpsec_descriptor*)dh1->_desc)->tdma_nxt = dh2->phys_addr;
7851a8031e1Shsuenaga MVXPSEC_SYNC_DESC(sc, dh1, BUS_DMASYNC_PREWRITE);
7861a8031e1Shsuenaga }
7871a8031e1Shsuenaga
7881a8031e1Shsuenaga /*
7891a8031e1Shsuenaga * Schedule DMA Copy
7901a8031e1Shsuenaga */
7911a8031e1Shsuenaga INLINE int
mvxpsec_dma_copy0(struct mvxpsec_softc * sc,mvxpsec_dma_ring * r,uint32_t dst,uint32_t src,uint32_t size)7921a8031e1Shsuenaga mvxpsec_dma_copy0(struct mvxpsec_softc *sc, mvxpsec_dma_ring *r,
7931a8031e1Shsuenaga uint32_t dst, uint32_t src, uint32_t size)
7941a8031e1Shsuenaga {
7951a8031e1Shsuenaga struct mvxpsec_descriptor_handle *dh;
7961a8031e1Shsuenaga
7971a8031e1Shsuenaga dh = mvxpsec_dma_getdesc(sc);
7981a8031e1Shsuenaga if (dh == NULL) {
7991a8031e1Shsuenaga log(LOG_ERR, "%s: descriptor full\n", __func__);
8001a8031e1Shsuenaga return -1;
8011a8031e1Shsuenaga }
8021a8031e1Shsuenaga
8031a8031e1Shsuenaga mvxpsec_dma_setup(dh, dst, src, size);
8041a8031e1Shsuenaga if (r->dma_head == NULL) {
8051a8031e1Shsuenaga r->dma_head = dh;
8061a8031e1Shsuenaga r->dma_last = dh;
8071a8031e1Shsuenaga r->dma_size = 1;
8081a8031e1Shsuenaga }
8091a8031e1Shsuenaga else {
8101a8031e1Shsuenaga mvxpsec_dma_cat(sc, r->dma_last, dh);
8111a8031e1Shsuenaga r->dma_last = dh;
8121a8031e1Shsuenaga r->dma_size++;
8131a8031e1Shsuenaga }
8141a8031e1Shsuenaga
8151a8031e1Shsuenaga return 0;
8161a8031e1Shsuenaga }
8171a8031e1Shsuenaga
8181a8031e1Shsuenaga INLINE int
mvxpsec_dma_copy(struct mvxpsec_softc * sc,mvxpsec_dma_ring * r,uint32_t dst,uint32_t src,uint32_t size)8191a8031e1Shsuenaga mvxpsec_dma_copy(struct mvxpsec_softc *sc, mvxpsec_dma_ring *r,
8201a8031e1Shsuenaga uint32_t dst, uint32_t src, uint32_t size)
8211a8031e1Shsuenaga {
8221a8031e1Shsuenaga if (size == 0) /* 0 is very special descriptor */
8231a8031e1Shsuenaga return 0;
8241a8031e1Shsuenaga
8251a8031e1Shsuenaga return mvxpsec_dma_copy0(sc, r, dst, src, size);
8261a8031e1Shsuenaga }
8271a8031e1Shsuenaga
8281a8031e1Shsuenaga /*
8291a8031e1Shsuenaga * Schedule ACC Activate
8301a8031e1Shsuenaga */
8311a8031e1Shsuenaga INLINE int
mvxpsec_dma_acc_activate(struct mvxpsec_softc * sc,mvxpsec_dma_ring * r)8321a8031e1Shsuenaga mvxpsec_dma_acc_activate(struct mvxpsec_softc *sc, mvxpsec_dma_ring *r)
8331a8031e1Shsuenaga {
8341a8031e1Shsuenaga return mvxpsec_dma_copy0(sc, r, 0, 0, 0);
8351a8031e1Shsuenaga }
8361a8031e1Shsuenaga
8371a8031e1Shsuenaga /*
8381a8031e1Shsuenaga * Finalize DMA setup
8391a8031e1Shsuenaga */
8401a8031e1Shsuenaga INLINE void
mvxpsec_dma_finalize(struct mvxpsec_softc * sc,mvxpsec_dma_ring * r)8411a8031e1Shsuenaga mvxpsec_dma_finalize(struct mvxpsec_softc *sc, mvxpsec_dma_ring *r)
8421a8031e1Shsuenaga {
8431a8031e1Shsuenaga struct mvxpsec_descriptor_handle *dh;
8441a8031e1Shsuenaga
8451a8031e1Shsuenaga dh = r->dma_last;
8461a8031e1Shsuenaga ((struct mvxpsec_descriptor*)dh->_desc)->tdma_nxt = 0;
8471a8031e1Shsuenaga MVXPSEC_SYNC_DESC(sc, dh, BUS_DMASYNC_PREWRITE);
8481a8031e1Shsuenaga }
8491a8031e1Shsuenaga
8501a8031e1Shsuenaga /*
8511a8031e1Shsuenaga * Free entire DMA ring
8521a8031e1Shsuenaga */
8531a8031e1Shsuenaga INLINE void
mvxpsec_dma_free(struct mvxpsec_softc * sc,mvxpsec_dma_ring * r)8541a8031e1Shsuenaga mvxpsec_dma_free(struct mvxpsec_softc *sc, mvxpsec_dma_ring *r)
8551a8031e1Shsuenaga {
8561a8031e1Shsuenaga sc->sc_desc_ring_cons += r->dma_size;
8571a8031e1Shsuenaga if (sc->sc_desc_ring_cons >= sc->sc_desc_ring_size)
8581a8031e1Shsuenaga sc->sc_desc_ring_cons -= sc->sc_desc_ring_size;
8591a8031e1Shsuenaga r->dma_head = NULL;
8601a8031e1Shsuenaga r->dma_last = NULL;
8611a8031e1Shsuenaga r->dma_size = 0;
8621a8031e1Shsuenaga }
8631a8031e1Shsuenaga
8641a8031e1Shsuenaga /*
8651a8031e1Shsuenaga * create DMA descriptor chain for the packet
8661a8031e1Shsuenaga */
8671a8031e1Shsuenaga INLINE int
mvxpsec_dma_copy_packet(struct mvxpsec_softc * sc,struct mvxpsec_packet * mv_p)8681a8031e1Shsuenaga mvxpsec_dma_copy_packet(struct mvxpsec_softc *sc, struct mvxpsec_packet *mv_p)
8691a8031e1Shsuenaga {
8701a8031e1Shsuenaga struct mvxpsec_session *mv_s = mv_p->mv_s;
8711a8031e1Shsuenaga uint32_t src, dst, len;
8721a8031e1Shsuenaga uint32_t pkt_off, pkt_off_r;
8731a8031e1Shsuenaga int err;
8741a8031e1Shsuenaga int i;
8751a8031e1Shsuenaga
8761a8031e1Shsuenaga /* must called with sc->sc_dma_mtx held */
8771a8031e1Shsuenaga KASSERT(mutex_owned(&sc->sc_dma_mtx));
8781a8031e1Shsuenaga
8791a8031e1Shsuenaga /*
8801a8031e1Shsuenaga * set offset for mem->device copy
8811a8031e1Shsuenaga *
8821a8031e1Shsuenaga * typical packet image:
8831a8031e1Shsuenaga *
8841a8031e1Shsuenaga * enc_ivoff
8851a8031e1Shsuenaga * mac_off
8861a8031e1Shsuenaga * |
8871a8031e1Shsuenaga * | enc_off
8881a8031e1Shsuenaga * | |
8891a8031e1Shsuenaga * v v
8901a8031e1Shsuenaga * +----+--------...
8911a8031e1Shsuenaga * |IV |DATA
8921a8031e1Shsuenaga * +----+--------...
8931a8031e1Shsuenaga */
8941a8031e1Shsuenaga pkt_off = 0;
8951a8031e1Shsuenaga if (mv_p->mac_off > 0)
8961a8031e1Shsuenaga pkt_off = mv_p->mac_off;
8971a8031e1Shsuenaga if ((mv_p->flags & CRP_EXT_IV) == 0 && pkt_off > mv_p->enc_ivoff)
8981a8031e1Shsuenaga pkt_off = mv_p->enc_ivoff;
8991a8031e1Shsuenaga if (mv_p->enc_off > 0 && pkt_off > mv_p->enc_off)
9001a8031e1Shsuenaga pkt_off = mv_p->enc_off;
9011a8031e1Shsuenaga pkt_off_r = pkt_off;
9021a8031e1Shsuenaga
9031a8031e1Shsuenaga /* make DMA descriptors to copy packet header: DRAM -> SRAM */
9041a8031e1Shsuenaga dst = (uint32_t)MVXPSEC_SRAM_PKT_HDR_PA(sc);
9051a8031e1Shsuenaga src = (uint32_t)mv_p->pkt_header_map->dm_segs[0].ds_addr;
9061a8031e1Shsuenaga len = sizeof(mv_p->pkt_header);
9071a8031e1Shsuenaga err = mvxpsec_dma_copy(sc, &mv_p->dma_ring, dst, src, len);
9081a8031e1Shsuenaga if (__predict_false(err))
9091a8031e1Shsuenaga return err;
9101a8031e1Shsuenaga
9111a8031e1Shsuenaga /*
9121a8031e1Shsuenaga * make DMA descriptors to copy session header: DRAM -> SRAM
9131a8031e1Shsuenaga * we can reuse session header on SRAM if session is not changed.
9141a8031e1Shsuenaga */
9151a8031e1Shsuenaga if (sc->sc_last_session != mv_s) {
9161a8031e1Shsuenaga dst = (uint32_t)MVXPSEC_SRAM_SESS_HDR_PA(sc);
9171a8031e1Shsuenaga src = (uint32_t)mv_s->session_header_map->dm_segs[0].ds_addr;
9181a8031e1Shsuenaga len = sizeof(mv_s->session_header);
9191a8031e1Shsuenaga err = mvxpsec_dma_copy(sc, &mv_p->dma_ring, dst, src, len);
9201a8031e1Shsuenaga if (__predict_false(err))
9211a8031e1Shsuenaga return err;
9221a8031e1Shsuenaga sc->sc_last_session = mv_s;
9231a8031e1Shsuenaga }
9241a8031e1Shsuenaga
9251a8031e1Shsuenaga /* make DMA descriptor to copy payload data: DRAM -> SRAM */
9261a8031e1Shsuenaga dst = MVXPSEC_SRAM_PAYLOAD_PA(sc, 0);
9271a8031e1Shsuenaga for (i = 0; i < mv_p->data_map->dm_nsegs; i++) {
9281a8031e1Shsuenaga src = mv_p->data_map->dm_segs[i].ds_addr;
9291a8031e1Shsuenaga len = mv_p->data_map->dm_segs[i].ds_len;
9301a8031e1Shsuenaga if (pkt_off) {
9311a8031e1Shsuenaga if (len <= pkt_off) {
9321a8031e1Shsuenaga /* ignore the segment */
9331a8031e1Shsuenaga dst += len;
9341a8031e1Shsuenaga pkt_off -= len;
9351a8031e1Shsuenaga continue;
9361a8031e1Shsuenaga }
9371a8031e1Shsuenaga /* copy from the middle of the segment */
9381a8031e1Shsuenaga dst += pkt_off;
9391a8031e1Shsuenaga src += pkt_off;
9401a8031e1Shsuenaga len -= pkt_off;
9411a8031e1Shsuenaga pkt_off = 0;
9421a8031e1Shsuenaga }
9431a8031e1Shsuenaga err = mvxpsec_dma_copy(sc, &mv_p->dma_ring, dst, src, len);
9441a8031e1Shsuenaga if (__predict_false(err))
9451a8031e1Shsuenaga return err;
9461a8031e1Shsuenaga dst += len;
9471a8031e1Shsuenaga }
9481a8031e1Shsuenaga
9491a8031e1Shsuenaga /* make special descriptor to activate security accelerator */
9501a8031e1Shsuenaga err = mvxpsec_dma_acc_activate(sc, &mv_p->dma_ring);
9511a8031e1Shsuenaga if (__predict_false(err))
9521a8031e1Shsuenaga return err;
9531a8031e1Shsuenaga
9541a8031e1Shsuenaga /* make DMA descriptors to copy payload: SRAM -> DRAM */
9551a8031e1Shsuenaga src = (uint32_t)MVXPSEC_SRAM_PAYLOAD_PA(sc, 0);
9561a8031e1Shsuenaga for (i = 0; i < mv_p->data_map->dm_nsegs; i++) {
9571a8031e1Shsuenaga dst = (uint32_t)mv_p->data_map->dm_segs[i].ds_addr;
9581a8031e1Shsuenaga len = (uint32_t)mv_p->data_map->dm_segs[i].ds_len;
9591a8031e1Shsuenaga if (pkt_off_r) {
9601a8031e1Shsuenaga if (len <= pkt_off_r) {
9611a8031e1Shsuenaga /* ignore the segment */
9621a8031e1Shsuenaga src += len;
9631a8031e1Shsuenaga pkt_off_r -= len;
9641a8031e1Shsuenaga continue;
9651a8031e1Shsuenaga }
9661a8031e1Shsuenaga /* copy from the middle of the segment */
9671a8031e1Shsuenaga src += pkt_off_r;
9681a8031e1Shsuenaga dst += pkt_off_r;
9691a8031e1Shsuenaga len -= pkt_off_r;
9701a8031e1Shsuenaga pkt_off_r = 0;
9711a8031e1Shsuenaga }
9721a8031e1Shsuenaga err = mvxpsec_dma_copy(sc, &mv_p->dma_ring, dst, src, len);
9731a8031e1Shsuenaga if (__predict_false(err))
9741a8031e1Shsuenaga return err;
9751a8031e1Shsuenaga src += len;
9761a8031e1Shsuenaga }
9771a8031e1Shsuenaga KASSERT(pkt_off == 0);
9781a8031e1Shsuenaga KASSERT(pkt_off_r == 0);
9791a8031e1Shsuenaga
9801a8031e1Shsuenaga /*
9811a8031e1Shsuenaga * make DMA descriptors to copy packet header: SRAM->DRAM
9821a8031e1Shsuenaga * if IV is present in the payload, no need to copy.
9831a8031e1Shsuenaga */
9841a8031e1Shsuenaga if (mv_p->flags & CRP_EXT_IV) {
9851a8031e1Shsuenaga dst = (uint32_t)mv_p->pkt_header_map->dm_segs[0].ds_addr;
9861a8031e1Shsuenaga src = (uint32_t)MVXPSEC_SRAM_PKT_HDR_PA(sc);
9871a8031e1Shsuenaga len = sizeof(mv_p->pkt_header);
9881a8031e1Shsuenaga err = mvxpsec_dma_copy(sc, &mv_p->dma_ring, dst, src, len);
9891a8031e1Shsuenaga if (__predict_false(err))
9901a8031e1Shsuenaga return err;
9911a8031e1Shsuenaga }
9921a8031e1Shsuenaga
9931a8031e1Shsuenaga return 0;
9941a8031e1Shsuenaga }
9951a8031e1Shsuenaga
9961a8031e1Shsuenaga INLINE int
mvxpsec_dma_sync_packet(struct mvxpsec_softc * sc,struct mvxpsec_packet * mv_p)9971a8031e1Shsuenaga mvxpsec_dma_sync_packet(struct mvxpsec_softc *sc, struct mvxpsec_packet *mv_p)
9981a8031e1Shsuenaga {
9991a8031e1Shsuenaga /* sync packet header */
10001a8031e1Shsuenaga bus_dmamap_sync(sc->sc_dmat,
10011a8031e1Shsuenaga mv_p->pkt_header_map, 0, sizeof(mv_p->pkt_header),
10021a8031e1Shsuenaga BUS_DMASYNC_PREWRITE);
10031a8031e1Shsuenaga
10041a8031e1Shsuenaga #ifdef MVXPSEC_DEBUG
10051a8031e1Shsuenaga /* sync session header */
10061a8031e1Shsuenaga if (mvxpsec_debug != 0) {
10071a8031e1Shsuenaga struct mvxpsec_session *mv_s = mv_p->mv_s;
10081a8031e1Shsuenaga
10091a8031e1Shsuenaga /* only debug code touch the session header after newsession */
10101a8031e1Shsuenaga bus_dmamap_sync(sc->sc_dmat,
10111a8031e1Shsuenaga mv_s->session_header_map,
10121a8031e1Shsuenaga 0, sizeof(mv_s->session_header),
10131a8031e1Shsuenaga BUS_DMASYNC_PREWRITE);
10141a8031e1Shsuenaga }
10151a8031e1Shsuenaga #endif
10161a8031e1Shsuenaga
10171a8031e1Shsuenaga /* sync packet buffer */
10181a8031e1Shsuenaga bus_dmamap_sync(sc->sc_dmat,
10191a8031e1Shsuenaga mv_p->data_map, 0, mv_p->data_len,
10201a8031e1Shsuenaga BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
10211a8031e1Shsuenaga
10221a8031e1Shsuenaga return 0;
10231a8031e1Shsuenaga }
10241a8031e1Shsuenaga
10251a8031e1Shsuenaga /*
10261a8031e1Shsuenaga * Initialize MVXPSEC Internal SRAM
10271a8031e1Shsuenaga *
1028ab9bc2adSandvar * - must be called after DMA initialization.
10291a8031e1Shsuenaga * - make VM mapping for SRAM area on MBus.
10301a8031e1Shsuenaga */
10311a8031e1Shsuenaga STATIC int
mvxpsec_init_sram(struct mvxpsec_softc * sc)10321a8031e1Shsuenaga mvxpsec_init_sram(struct mvxpsec_softc *sc)
10331a8031e1Shsuenaga {
10341a8031e1Shsuenaga uint32_t tag, target, attr, base, size;
10351a8031e1Shsuenaga vaddr_t va;
10361a8031e1Shsuenaga int window;
10371a8031e1Shsuenaga
10383aa5a3aeSriastradh switch (device_unit(sc->sc_dev)) {
10391a8031e1Shsuenaga case 0:
10401a8031e1Shsuenaga tag = ARMADAXP_TAG_CRYPT0;
10411a8031e1Shsuenaga break;
10421a8031e1Shsuenaga case 1:
10431a8031e1Shsuenaga tag = ARMADAXP_TAG_CRYPT1;
10441a8031e1Shsuenaga break;
10451a8031e1Shsuenaga default:
10461a8031e1Shsuenaga aprint_error_dev(sc->sc_dev, "no internal SRAM mapping\n");
10471a8031e1Shsuenaga return -1;
10481a8031e1Shsuenaga }
10491a8031e1Shsuenaga
10501a8031e1Shsuenaga window = mvsoc_target(tag, &target, &attr, &base, &size);
10511a8031e1Shsuenaga if (window >= nwindow) {
10521a8031e1Shsuenaga aprint_error_dev(sc->sc_dev, "no internal SRAM mapping\n");
10531a8031e1Shsuenaga return -1;
10541a8031e1Shsuenaga }
10551a8031e1Shsuenaga
10561a8031e1Shsuenaga if (sizeof(struct mvxpsec_crypt_sram) > size) {
10571a8031e1Shsuenaga aprint_error_dev(sc->sc_dev,
10581a8031e1Shsuenaga "SRAM Data Structure Excceeds SRAM window size.\n");
10591a8031e1Shsuenaga return -1;
10601a8031e1Shsuenaga }
10611a8031e1Shsuenaga
10621a8031e1Shsuenaga aprint_normal_dev(sc->sc_dev,
10631a8031e1Shsuenaga "internal SRAM window at 0x%08x-0x%08x",
10641a8031e1Shsuenaga base, base + size - 1);
10651a8031e1Shsuenaga sc->sc_sram_pa = base;
10661a8031e1Shsuenaga
10671a8031e1Shsuenaga /* get vmspace to read/write device internal SRAM */
10681a8031e1Shsuenaga va = uvm_km_alloc(kernel_map, PAGE_SIZE, PAGE_SIZE,
10691a8031e1Shsuenaga UVM_KMF_VAONLY | UVM_KMF_NOWAIT);
10701a8031e1Shsuenaga if (va == 0) {
10711a8031e1Shsuenaga aprint_error_dev(sc->sc_dev, "cannot map SRAM window\n");
10721a8031e1Shsuenaga sc->sc_sram_va = NULL;
10731a8031e1Shsuenaga aprint_normal("\n");
10741a8031e1Shsuenaga return 0;
10751a8031e1Shsuenaga }
10761a8031e1Shsuenaga /* XXX: not working. PMAP_NOCACHE is not affected? */
10771a8031e1Shsuenaga pmap_kenter_pa(va, base, VM_PROT_READ|VM_PROT_WRITE, PMAP_NOCACHE);
10781a8031e1Shsuenaga pmap_update(pmap_kernel());
10791a8031e1Shsuenaga sc->sc_sram_va = (void *)va;
10801a8031e1Shsuenaga aprint_normal(" va %p\n", sc->sc_sram_va);
10811a8031e1Shsuenaga memset(sc->sc_sram_va, 0xff, MV_ACC_SRAM_SIZE);
10821a8031e1Shsuenaga
10831a8031e1Shsuenaga return 0;
10841a8031e1Shsuenaga }
10851a8031e1Shsuenaga
10861a8031e1Shsuenaga /*
10871a8031e1Shsuenaga * Initialize TDMA engine.
10881a8031e1Shsuenaga */
10891a8031e1Shsuenaga STATIC int
mvxpsec_init_dma(struct mvxpsec_softc * sc,struct marvell_attach_args * mva)10901a8031e1Shsuenaga mvxpsec_init_dma(struct mvxpsec_softc *sc, struct marvell_attach_args *mva)
10911a8031e1Shsuenaga {
10921a8031e1Shsuenaga struct mvxpsec_descriptor_handle *dh;
10931a8031e1Shsuenaga uint8_t *va;
10941a8031e1Shsuenaga paddr_t pa;
10951a8031e1Shsuenaga off_t va_off, pa_off;
10961a8031e1Shsuenaga int i, n, seg, ndh;
10971a8031e1Shsuenaga
10981a8031e1Shsuenaga /* Init Deviced's control parameters (disabled yet) */
10991a8031e1Shsuenaga MVXPSEC_WRITE(sc, MV_TDMA_CONTROL, MV_TDMA_DEFAULT_CONTROL);
11001a8031e1Shsuenaga
11011a8031e1Shsuenaga /* Init Software DMA Handlers */
11021a8031e1Shsuenaga sc->sc_devmem_desc =
11031a8031e1Shsuenaga mvxpsec_alloc_devmem(sc, 0, PAGE_SIZE * MVXPSEC_DMA_DESC_PAGES);
11041a8031e1Shsuenaga ndh = (PAGE_SIZE / sizeof(struct mvxpsec_descriptor))
11051a8031e1Shsuenaga * MVXPSEC_DMA_DESC_PAGES;
11061a8031e1Shsuenaga sc->sc_desc_ring =
11071a8031e1Shsuenaga kmem_alloc(sizeof(struct mvxpsec_descriptor_handle) * ndh,
11080a827a3fSchs KM_SLEEP);
11091a8031e1Shsuenaga aprint_normal_dev(sc->sc_dev, "%d DMA handles in %zu bytes array\n",
11101a8031e1Shsuenaga ndh, sizeof(struct mvxpsec_descriptor_handle) * ndh);
11111a8031e1Shsuenaga
11121a8031e1Shsuenaga ndh = 0;
11131a8031e1Shsuenaga for (seg = 0; seg < devmem_nseg(sc->sc_devmem_desc); seg++) {
11141a8031e1Shsuenaga va = devmem_va(sc->sc_devmem_desc);
11151a8031e1Shsuenaga pa = devmem_pa(sc->sc_devmem_desc, seg);
11161a8031e1Shsuenaga n = devmem_palen(sc->sc_devmem_desc, seg) /
11171a8031e1Shsuenaga sizeof(struct mvxpsec_descriptor);
11181a8031e1Shsuenaga va_off = (PAGE_SIZE * seg);
11191a8031e1Shsuenaga pa_off = 0;
11201a8031e1Shsuenaga for (i = 0; i < n; i++) {
11211a8031e1Shsuenaga dh = &sc->sc_desc_ring[ndh];
11221a8031e1Shsuenaga dh->map = devmem_map(sc->sc_devmem_desc);
11231a8031e1Shsuenaga dh->off = va_off + pa_off;
11241a8031e1Shsuenaga dh->_desc = (void *)(va + va_off + pa_off);
11251a8031e1Shsuenaga dh->phys_addr = pa + pa_off;
11261a8031e1Shsuenaga pa_off += sizeof(struct mvxpsec_descriptor);
11271a8031e1Shsuenaga ndh++;
11281a8031e1Shsuenaga }
11291a8031e1Shsuenaga }
11301a8031e1Shsuenaga sc->sc_desc_ring_size = ndh;
11311a8031e1Shsuenaga sc->sc_desc_ring_prod = 0;
11321a8031e1Shsuenaga sc->sc_desc_ring_cons = sc->sc_desc_ring_size - 1;
11331a8031e1Shsuenaga
11341a8031e1Shsuenaga return 0;
11351a8031e1Shsuenaga }
11361a8031e1Shsuenaga
11371a8031e1Shsuenaga /*
11381a8031e1Shsuenaga * Wait for TDMA controller become idle
11391a8031e1Shsuenaga */
11401a8031e1Shsuenaga INLINE int
mvxpsec_dma_wait(struct mvxpsec_softc * sc)11411a8031e1Shsuenaga mvxpsec_dma_wait(struct mvxpsec_softc *sc)
11421a8031e1Shsuenaga {
11431a8031e1Shsuenaga int retry = 0;
11441a8031e1Shsuenaga
11451a8031e1Shsuenaga while (MVXPSEC_READ(sc, MV_TDMA_CONTROL) & MV_TDMA_CONTROL_ACT) {
11461a8031e1Shsuenaga delay(mvxpsec_wait_interval);
11471a8031e1Shsuenaga if (retry++ >= mvxpsec_wait_retry)
11481a8031e1Shsuenaga return -1;
11491a8031e1Shsuenaga }
11501a8031e1Shsuenaga return 0;
11511a8031e1Shsuenaga }
11521a8031e1Shsuenaga
11531a8031e1Shsuenaga /*
11541a8031e1Shsuenaga * Wait for Security Accelerator become idle
11551a8031e1Shsuenaga */
11561a8031e1Shsuenaga INLINE int
mvxpsec_acc_wait(struct mvxpsec_softc * sc)11571a8031e1Shsuenaga mvxpsec_acc_wait(struct mvxpsec_softc *sc)
11581a8031e1Shsuenaga {
11591a8031e1Shsuenaga int retry = 0;
11601a8031e1Shsuenaga
11611a8031e1Shsuenaga while (MVXPSEC_READ(sc, MV_ACC_COMMAND) & MV_ACC_COMMAND_ACT) {
11621a8031e1Shsuenaga delay(mvxpsec_wait_interval);
11631a8031e1Shsuenaga if (++retry >= mvxpsec_wait_retry)
11641a8031e1Shsuenaga return -1;
11651a8031e1Shsuenaga }
11661a8031e1Shsuenaga return 0;
11671a8031e1Shsuenaga }
11681a8031e1Shsuenaga
11691a8031e1Shsuenaga /*
11701a8031e1Shsuenaga * Entry of interrupt handler
11711a8031e1Shsuenaga *
11721a8031e1Shsuenaga * register this to kernel via marvell_intr_establish()
11731a8031e1Shsuenaga */
11741a8031e1Shsuenaga int
mvxpsec_intr(void * arg)11751a8031e1Shsuenaga mvxpsec_intr(void *arg)
11761a8031e1Shsuenaga {
11771a8031e1Shsuenaga struct mvxpsec_softc *sc = arg;
11781a8031e1Shsuenaga uint32_t v;
11791a8031e1Shsuenaga
11801a8031e1Shsuenaga /* IPL_NET */
11811a8031e1Shsuenaga while ((v = mvxpsec_intr_ack(sc)) != 0) {
11821a8031e1Shsuenaga mvxpsec_intr_cnt(sc, v);
11831a8031e1Shsuenaga MVXPSEC_PRINTF(MVXPSEC_DEBUG_INTR, "MVXPSEC Intr 0x%08x\n", v);
11841a8031e1Shsuenaga MVXPSEC_PRINTF(MVXPSEC_DEBUG_INTR, "%s\n", s_xpsecintr(v));
11851a8031e1Shsuenaga #ifdef MVXPSEC_DEBUG
11861a8031e1Shsuenaga mvxpsec_dump_reg(sc);
11871a8031e1Shsuenaga #endif
11881a8031e1Shsuenaga
11891a8031e1Shsuenaga /* call high-level handlers */
11901a8031e1Shsuenaga if (v & MVXPSEC_INT_ACCTDMA)
11911a8031e1Shsuenaga mvxpsec_done(sc);
11921a8031e1Shsuenaga }
11931a8031e1Shsuenaga
11941a8031e1Shsuenaga return 0;
11951a8031e1Shsuenaga }
11961a8031e1Shsuenaga
11971a8031e1Shsuenaga INLINE void
mvxpsec_intr_cleanup(struct mvxpsec_softc * sc)11981a8031e1Shsuenaga mvxpsec_intr_cleanup(struct mvxpsec_softc *sc)
11991a8031e1Shsuenaga {
12001a8031e1Shsuenaga struct mvxpsec_packet *mv_p;
12011a8031e1Shsuenaga
12021a8031e1Shsuenaga /* must called with sc->sc_dma_mtx held */
12031a8031e1Shsuenaga KASSERT(mutex_owned(&sc->sc_dma_mtx));
12041a8031e1Shsuenaga
12051a8031e1Shsuenaga /*
12061a8031e1Shsuenaga * there is only one intr for run_queue.
12071a8031e1Shsuenaga * no one touch sc_run_queue.
12081a8031e1Shsuenaga */
12091a8031e1Shsuenaga SIMPLEQ_FOREACH(mv_p, &sc->sc_run_queue, queue)
12101a8031e1Shsuenaga mvxpsec_dma_free(sc, &mv_p->dma_ring);
12111a8031e1Shsuenaga }
12121a8031e1Shsuenaga
12131a8031e1Shsuenaga /*
12141a8031e1Shsuenaga * Acknowledge to interrupt
12151a8031e1Shsuenaga *
12161a8031e1Shsuenaga * read cause bits, clear it, and return it.
12171a8031e1Shsuenaga * NOTE: multiple cause bits may be returned at once.
12181a8031e1Shsuenaga */
12191a8031e1Shsuenaga STATIC uint32_t
mvxpsec_intr_ack(struct mvxpsec_softc * sc)12201a8031e1Shsuenaga mvxpsec_intr_ack(struct mvxpsec_softc *sc)
12211a8031e1Shsuenaga {
12221a8031e1Shsuenaga uint32_t reg;
12231a8031e1Shsuenaga
12241a8031e1Shsuenaga reg = MVXPSEC_READ(sc, MVXPSEC_INT_CAUSE);
12251a8031e1Shsuenaga reg &= MVXPSEC_DEFAULT_INT;
12261a8031e1Shsuenaga MVXPSEC_WRITE(sc, MVXPSEC_INT_CAUSE, ~reg);
12271a8031e1Shsuenaga MVXPSEC_PRINTF(MVXPSEC_DEBUG_INTR, "Int: %s\n", s_xpsecintr(reg));
12281a8031e1Shsuenaga
12291a8031e1Shsuenaga return reg;
12301a8031e1Shsuenaga }
12311a8031e1Shsuenaga
12321a8031e1Shsuenaga /*
12331a8031e1Shsuenaga * Entry of TDMA error interrupt handler
12341a8031e1Shsuenaga *
12351a8031e1Shsuenaga * register this to kernel via marvell_intr_establish()
12361a8031e1Shsuenaga */
12371a8031e1Shsuenaga int
mvxpsec_eintr(void * arg)12381a8031e1Shsuenaga mvxpsec_eintr(void *arg)
12391a8031e1Shsuenaga {
12401a8031e1Shsuenaga struct mvxpsec_softc *sc = arg;
12411a8031e1Shsuenaga uint32_t err;
12421a8031e1Shsuenaga
12431a8031e1Shsuenaga /* IPL_NET */
12441a8031e1Shsuenaga again:
12451a8031e1Shsuenaga err = mvxpsec_eintr_ack(sc);
12461a8031e1Shsuenaga if (err == 0)
12471a8031e1Shsuenaga goto done;
12481a8031e1Shsuenaga
12491a8031e1Shsuenaga log(LOG_ERR, "%s: DMA Error Interrupt: %s\n", __func__,
12501a8031e1Shsuenaga s_errreg(err));
12511a8031e1Shsuenaga #ifdef MVXPSEC_DEBUG
12521a8031e1Shsuenaga mvxpsec_dump_reg(sc);
12531a8031e1Shsuenaga #endif
12541a8031e1Shsuenaga
12551a8031e1Shsuenaga goto again;
12561a8031e1Shsuenaga done:
12571a8031e1Shsuenaga return 0;
12581a8031e1Shsuenaga }
12591a8031e1Shsuenaga
12601a8031e1Shsuenaga /*
12611a8031e1Shsuenaga * Acknowledge to TDMA error interrupt
12621a8031e1Shsuenaga *
12631a8031e1Shsuenaga * read cause bits, clear it, and return it.
12641a8031e1Shsuenaga * NOTE: multiple cause bits may be returned at once.
12651a8031e1Shsuenaga */
12661a8031e1Shsuenaga STATIC uint32_t
mvxpsec_eintr_ack(struct mvxpsec_softc * sc)12671a8031e1Shsuenaga mvxpsec_eintr_ack(struct mvxpsec_softc *sc)
12681a8031e1Shsuenaga {
12691a8031e1Shsuenaga uint32_t reg;
12701a8031e1Shsuenaga
12711a8031e1Shsuenaga reg = MVXPSEC_READ(sc, MV_TDMA_ERR_CAUSE);
12721a8031e1Shsuenaga reg &= MVXPSEC_DEFAULT_ERR;
12731a8031e1Shsuenaga MVXPSEC_WRITE(sc, MV_TDMA_ERR_CAUSE, ~reg);
12741a8031e1Shsuenaga MVXPSEC_PRINTF(MVXPSEC_DEBUG_INTR, "Int: %s\n", s_xpsecintr(reg));
12751a8031e1Shsuenaga
12761a8031e1Shsuenaga return reg;
12771a8031e1Shsuenaga }
12781a8031e1Shsuenaga
12791a8031e1Shsuenaga /*
12801a8031e1Shsuenaga * Interrupt statistics
12811a8031e1Shsuenaga *
1282cbf5c65aSandvar * this is NOT a statistics of how many times the events 'occurred'.
12831a8031e1Shsuenaga * this ONLY means how many times the events 'handled'.
12841a8031e1Shsuenaga */
12851a8031e1Shsuenaga INLINE void
mvxpsec_intr_cnt(struct mvxpsec_softc * sc,int cause)12861a8031e1Shsuenaga mvxpsec_intr_cnt(struct mvxpsec_softc *sc, int cause)
12871a8031e1Shsuenaga {
12881a8031e1Shsuenaga MVXPSEC_EVCNT_INCR(sc, intr_all);
12891a8031e1Shsuenaga if (cause & MVXPSEC_INT_AUTH)
12901a8031e1Shsuenaga MVXPSEC_EVCNT_INCR(sc, intr_auth);
12911a8031e1Shsuenaga if (cause & MVXPSEC_INT_DES)
12921a8031e1Shsuenaga MVXPSEC_EVCNT_INCR(sc, intr_des);
12931a8031e1Shsuenaga if (cause & MVXPSEC_INT_AES_ENC)
12941a8031e1Shsuenaga MVXPSEC_EVCNT_INCR(sc, intr_aes_enc);
12951a8031e1Shsuenaga if (cause & MVXPSEC_INT_AES_DEC)
12961a8031e1Shsuenaga MVXPSEC_EVCNT_INCR(sc, intr_aes_dec);
12971a8031e1Shsuenaga if (cause & MVXPSEC_INT_ENC)
12981a8031e1Shsuenaga MVXPSEC_EVCNT_INCR(sc, intr_enc);
12991a8031e1Shsuenaga if (cause & MVXPSEC_INT_SA)
13001a8031e1Shsuenaga MVXPSEC_EVCNT_INCR(sc, intr_sa);
13011a8031e1Shsuenaga if (cause & MVXPSEC_INT_ACCTDMA)
13021a8031e1Shsuenaga MVXPSEC_EVCNT_INCR(sc, intr_acctdma);
13031a8031e1Shsuenaga if (cause & MVXPSEC_INT_TDMA_COMP)
13041a8031e1Shsuenaga MVXPSEC_EVCNT_INCR(sc, intr_comp);
13051a8031e1Shsuenaga if (cause & MVXPSEC_INT_TDMA_OWN)
13061a8031e1Shsuenaga MVXPSEC_EVCNT_INCR(sc, intr_own);
13071a8031e1Shsuenaga if (cause & MVXPSEC_INT_ACCTDMA_CONT)
13081a8031e1Shsuenaga MVXPSEC_EVCNT_INCR(sc, intr_acctdma_cont);
13091a8031e1Shsuenaga }
13101a8031e1Shsuenaga
13111a8031e1Shsuenaga /*
13121a8031e1Shsuenaga * Setup MVXPSEC header structure.
13131a8031e1Shsuenaga *
13141a8031e1Shsuenaga * the header contains descriptor of security accelerator,
1315ab9bc2adSandvar * key material of ciphers, iv of ciphers and macs, ...
13161a8031e1Shsuenaga *
1317a0403cdeSmsaitoh * the header is transferred to MVXPSEC Internal SRAM by TDMA,
13181a8031e1Shsuenaga * and parsed by MVXPSEC H/W.
13191a8031e1Shsuenaga */
13201a8031e1Shsuenaga STATIC int
mvxpsec_header_finalize(struct mvxpsec_packet * mv_p)13211a8031e1Shsuenaga mvxpsec_header_finalize(struct mvxpsec_packet *mv_p)
13221a8031e1Shsuenaga {
13231a8031e1Shsuenaga struct mvxpsec_acc_descriptor *desc = &mv_p->pkt_header.desc;
13241a8031e1Shsuenaga int enc_start, enc_len, iv_offset;
13251a8031e1Shsuenaga int mac_start, mac_len, mac_offset;
13261a8031e1Shsuenaga
13271a8031e1Shsuenaga /* offset -> device address */
13281a8031e1Shsuenaga enc_start = MVXPSEC_SRAM_PAYLOAD_DA(mv_p->enc_off);
13291a8031e1Shsuenaga enc_len = mv_p->enc_len;
13301a8031e1Shsuenaga if (mv_p->flags & CRP_EXT_IV)
13311a8031e1Shsuenaga iv_offset = mv_p->enc_ivoff;
13321a8031e1Shsuenaga else
13331a8031e1Shsuenaga iv_offset = MVXPSEC_SRAM_PAYLOAD_DA(mv_p->enc_ivoff);
13341a8031e1Shsuenaga mac_start = MVXPSEC_SRAM_PAYLOAD_DA(mv_p->mac_off);
13351a8031e1Shsuenaga mac_len = mv_p->mac_len;
13361a8031e1Shsuenaga mac_offset = MVXPSEC_SRAM_PAYLOAD_DA(mv_p->mac_dst);
13371a8031e1Shsuenaga
13381a8031e1Shsuenaga MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO,
13391a8031e1Shsuenaga "PAYLOAD at 0x%08x\n", (int)MVXPSEC_SRAM_PAYLOAD_OFF);
13401a8031e1Shsuenaga MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO,
13411a8031e1Shsuenaga "ENC from 0x%08x\n", enc_start);
13421a8031e1Shsuenaga MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO,
13431a8031e1Shsuenaga "MAC from 0x%08x\n", mac_start);
13441a8031e1Shsuenaga MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO,
13451a8031e1Shsuenaga "MAC to 0x%08x\n", mac_offset);
13461a8031e1Shsuenaga MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO,
13471a8031e1Shsuenaga "ENC IV at 0x%08x\n", iv_offset);
13481a8031e1Shsuenaga
13491a8031e1Shsuenaga /* setup device addresses in Security Accelerator Descriptors */
13501a8031e1Shsuenaga desc->acc_encdata = MV_ACC_DESC_ENC_DATA(enc_start, enc_start);
13511a8031e1Shsuenaga desc->acc_enclen = MV_ACC_DESC_ENC_LEN(enc_len);
13521a8031e1Shsuenaga if (desc->acc_config & MV_ACC_CRYPTO_DECRYPT)
13531a8031e1Shsuenaga desc->acc_enckey =
13541a8031e1Shsuenaga MV_ACC_DESC_ENC_KEY(MVXPSEC_SRAM_KEY_D_DA);
13551a8031e1Shsuenaga else
13561a8031e1Shsuenaga desc->acc_enckey =
13571a8031e1Shsuenaga MV_ACC_DESC_ENC_KEY(MVXPSEC_SRAM_KEY_DA);
13581a8031e1Shsuenaga desc->acc_enciv =
13591a8031e1Shsuenaga MV_ACC_DESC_ENC_IV(MVXPSEC_SRAM_IV_WORK_DA, iv_offset);
13601a8031e1Shsuenaga
13611a8031e1Shsuenaga desc->acc_macsrc = MV_ACC_DESC_MAC_SRC(mac_start, mac_len);
13621a8031e1Shsuenaga desc->acc_macdst = MV_ACC_DESC_MAC_DST(mac_offset, mac_len);
13631a8031e1Shsuenaga desc->acc_maciv =
13641a8031e1Shsuenaga MV_ACC_DESC_MAC_IV(MVXPSEC_SRAM_MIV_IN_DA,
13651a8031e1Shsuenaga MVXPSEC_SRAM_MIV_OUT_DA);
13661a8031e1Shsuenaga
13671a8031e1Shsuenaga return 0;
13681a8031e1Shsuenaga }
13691a8031e1Shsuenaga
13701a8031e1Shsuenaga /*
13711a8031e1Shsuenaga * constractor of session structure.
13721a8031e1Shsuenaga *
13731a8031e1Shsuenaga * this constrator will be called by pool_cache framework.
13741a8031e1Shsuenaga */
13751a8031e1Shsuenaga STATIC int
mvxpsec_session_ctor(void * arg,void * obj,int flags)13761a8031e1Shsuenaga mvxpsec_session_ctor(void *arg, void *obj, int flags)
13771a8031e1Shsuenaga {
13781a8031e1Shsuenaga struct mvxpsec_softc *sc = arg;
13791a8031e1Shsuenaga struct mvxpsec_session *mv_s = obj;
13801a8031e1Shsuenaga
13811a8031e1Shsuenaga /* pool is owned by softc */
13821a8031e1Shsuenaga mv_s->sc = sc;
13831a8031e1Shsuenaga
13841a8031e1Shsuenaga /* Create and load DMA map for session header */
13851a8031e1Shsuenaga mv_s->session_header_map = 0;
13861a8031e1Shsuenaga if (bus_dmamap_create(sc->sc_dmat,
13871a8031e1Shsuenaga sizeof(mv_s->session_header), 1,
13881a8031e1Shsuenaga sizeof(mv_s->session_header), 0,
13891a8031e1Shsuenaga BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW,
13901a8031e1Shsuenaga &mv_s->session_header_map)) {
13911a8031e1Shsuenaga log(LOG_ERR, "%s: cannot create DMA map\n", __func__);
13921a8031e1Shsuenaga goto fail;
13931a8031e1Shsuenaga }
13941a8031e1Shsuenaga if (bus_dmamap_load(sc->sc_dmat, mv_s->session_header_map,
13951a8031e1Shsuenaga &mv_s->session_header, sizeof(mv_s->session_header),
13961a8031e1Shsuenaga NULL, BUS_DMA_NOWAIT)) {
13971a8031e1Shsuenaga log(LOG_ERR, "%s: cannot load header\n", __func__);
13981a8031e1Shsuenaga goto fail;
13991a8031e1Shsuenaga }
14001a8031e1Shsuenaga
14011a8031e1Shsuenaga return 0;
14021a8031e1Shsuenaga fail:
14031a8031e1Shsuenaga if (mv_s->session_header_map)
14041a8031e1Shsuenaga bus_dmamap_destroy(sc->sc_dmat, mv_s->session_header_map);
14051a8031e1Shsuenaga return ENOMEM;
14061a8031e1Shsuenaga }
14071a8031e1Shsuenaga
14081a8031e1Shsuenaga /*
14091a8031e1Shsuenaga * destractor of session structure.
14101a8031e1Shsuenaga *
14111a8031e1Shsuenaga * this destrator will be called by pool_cache framework.
14121a8031e1Shsuenaga */
14131a8031e1Shsuenaga STATIC void
mvxpsec_session_dtor(void * arg,void * obj)14141a8031e1Shsuenaga mvxpsec_session_dtor(void *arg, void *obj)
14151a8031e1Shsuenaga {
14161a8031e1Shsuenaga struct mvxpsec_softc *sc = arg;
14171a8031e1Shsuenaga struct mvxpsec_session *mv_s = obj;
14181a8031e1Shsuenaga
14191a8031e1Shsuenaga if (mv_s->sc != sc)
14201a8031e1Shsuenaga panic("inconsitent context\n");
14211a8031e1Shsuenaga
14221a8031e1Shsuenaga bus_dmamap_destroy(sc->sc_dmat, mv_s->session_header_map);
14231a8031e1Shsuenaga }
14241a8031e1Shsuenaga
14251a8031e1Shsuenaga /*
14261a8031e1Shsuenaga * constructor of packet structure.
14271a8031e1Shsuenaga */
14281a8031e1Shsuenaga STATIC int
mvxpsec_packet_ctor(void * arg,void * obj,int flags)14291a8031e1Shsuenaga mvxpsec_packet_ctor(void *arg, void *obj, int flags)
14301a8031e1Shsuenaga {
14311a8031e1Shsuenaga struct mvxpsec_softc *sc = arg;
14321a8031e1Shsuenaga struct mvxpsec_packet *mv_p = obj;
14331a8031e1Shsuenaga
14341a8031e1Shsuenaga mv_p->dma_ring.dma_head = NULL;
14351a8031e1Shsuenaga mv_p->dma_ring.dma_last = NULL;
14361a8031e1Shsuenaga mv_p->dma_ring.dma_size = 0;
14371a8031e1Shsuenaga
14381a8031e1Shsuenaga /* Create and load DMA map for packet header */
14391a8031e1Shsuenaga mv_p->pkt_header_map = 0;
14401a8031e1Shsuenaga if (bus_dmamap_create(sc->sc_dmat,
14411a8031e1Shsuenaga sizeof(mv_p->pkt_header), 1, sizeof(mv_p->pkt_header), 0,
14421a8031e1Shsuenaga BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW,
14431a8031e1Shsuenaga &mv_p->pkt_header_map)) {
14441a8031e1Shsuenaga log(LOG_ERR, "%s: cannot create DMA map\n", __func__);
14451a8031e1Shsuenaga goto fail;
14461a8031e1Shsuenaga }
14471a8031e1Shsuenaga if (bus_dmamap_load(sc->sc_dmat, mv_p->pkt_header_map,
14481a8031e1Shsuenaga &mv_p->pkt_header, sizeof(mv_p->pkt_header),
14491a8031e1Shsuenaga NULL, BUS_DMA_NOWAIT)) {
14501a8031e1Shsuenaga log(LOG_ERR, "%s: cannot load header\n", __func__);
14511a8031e1Shsuenaga goto fail;
14521a8031e1Shsuenaga }
14531a8031e1Shsuenaga
14541a8031e1Shsuenaga /* Create DMA map for session data. */
14551a8031e1Shsuenaga mv_p->data_map = 0;
14561a8031e1Shsuenaga if (bus_dmamap_create(sc->sc_dmat,
14571a8031e1Shsuenaga MVXPSEC_DMA_MAX_SIZE, MVXPSEC_DMA_MAX_SEGS, MVXPSEC_DMA_MAX_SIZE,
14581a8031e1Shsuenaga 0, BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW, &mv_p->data_map)) {
14591a8031e1Shsuenaga log(LOG_ERR, "%s: cannot create DMA map\n", __func__);
14601a8031e1Shsuenaga goto fail;
14611a8031e1Shsuenaga }
14621a8031e1Shsuenaga
14631a8031e1Shsuenaga return 0;
14641a8031e1Shsuenaga fail:
14651a8031e1Shsuenaga if (mv_p->pkt_header_map)
14661a8031e1Shsuenaga bus_dmamap_destroy(sc->sc_dmat, mv_p->pkt_header_map);
14671a8031e1Shsuenaga if (mv_p->data_map)
14681a8031e1Shsuenaga bus_dmamap_destroy(sc->sc_dmat, mv_p->data_map);
14691a8031e1Shsuenaga return ENOMEM;
14701a8031e1Shsuenaga }
14711a8031e1Shsuenaga
14721a8031e1Shsuenaga /*
14731a8031e1Shsuenaga * destractor of packet structure.
14741a8031e1Shsuenaga */
14751a8031e1Shsuenaga STATIC void
mvxpsec_packet_dtor(void * arg,void * obj)14761a8031e1Shsuenaga mvxpsec_packet_dtor(void *arg, void *obj)
14771a8031e1Shsuenaga {
14781a8031e1Shsuenaga struct mvxpsec_softc *sc = arg;
14791a8031e1Shsuenaga struct mvxpsec_packet *mv_p = obj;
14801a8031e1Shsuenaga
14811a8031e1Shsuenaga mutex_enter(&sc->sc_dma_mtx);
14821a8031e1Shsuenaga mvxpsec_dma_free(sc, &mv_p->dma_ring);
14831a8031e1Shsuenaga mutex_exit(&sc->sc_dma_mtx);
14841a8031e1Shsuenaga bus_dmamap_destroy(sc->sc_dmat, mv_p->pkt_header_map);
14851a8031e1Shsuenaga bus_dmamap_destroy(sc->sc_dmat, mv_p->data_map);
14861a8031e1Shsuenaga }
14871a8031e1Shsuenaga
14881a8031e1Shsuenaga /*
1489c74fd0bbSandvar * allocate new session structure.
14901a8031e1Shsuenaga */
14911a8031e1Shsuenaga STATIC struct mvxpsec_session *
mvxpsec_session_alloc(struct mvxpsec_softc * sc)14921a8031e1Shsuenaga mvxpsec_session_alloc(struct mvxpsec_softc *sc)
14931a8031e1Shsuenaga {
14941a8031e1Shsuenaga struct mvxpsec_session *mv_s;
14951a8031e1Shsuenaga
1496127c64cbSchristos mv_s = pool_cache_get(sc->sc_session_pool, PR_NOWAIT);
14971a8031e1Shsuenaga if (mv_s == NULL) {
14981a8031e1Shsuenaga log(LOG_ERR, "%s: cannot allocate memory\n", __func__);
14991a8031e1Shsuenaga return NULL;
15001a8031e1Shsuenaga }
1501afda606fSskrll mv_s->refs = 1; /* 0 means session is already invalid */
15021a8031e1Shsuenaga mv_s->sflags = 0;
15031a8031e1Shsuenaga
15041a8031e1Shsuenaga return mv_s;
15051a8031e1Shsuenaga }
15061a8031e1Shsuenaga
15071a8031e1Shsuenaga /*
15081a8031e1Shsuenaga * deallocate session structure.
15091a8031e1Shsuenaga */
15101a8031e1Shsuenaga STATIC void
mvxpsec_session_dealloc(struct mvxpsec_session * mv_s)15111a8031e1Shsuenaga mvxpsec_session_dealloc(struct mvxpsec_session *mv_s)
15121a8031e1Shsuenaga {
15131a8031e1Shsuenaga struct mvxpsec_softc *sc = mv_s->sc;
15141a8031e1Shsuenaga
15151a8031e1Shsuenaga mv_s->sflags |= DELETED;
15161a8031e1Shsuenaga mvxpsec_session_unref(mv_s);
15171a8031e1Shsuenaga crypto_unblock(sc->sc_cid, CRYPTO_SYMQ|CRYPTO_ASYMQ);
15181a8031e1Shsuenaga
15191a8031e1Shsuenaga return;
15201a8031e1Shsuenaga }
15211a8031e1Shsuenaga
15221a8031e1Shsuenaga STATIC int
mvxpsec_session_ref(struct mvxpsec_session * mv_s)15231a8031e1Shsuenaga mvxpsec_session_ref(struct mvxpsec_session *mv_s)
15241a8031e1Shsuenaga {
15251a8031e1Shsuenaga uint32_t refs;
15261a8031e1Shsuenaga
15271a8031e1Shsuenaga if (mv_s->sflags & DELETED) {
15281a8031e1Shsuenaga log(LOG_ERR,
15291a8031e1Shsuenaga "%s: session is already deleted.\n", __func__);
15301a8031e1Shsuenaga return -1;
15311a8031e1Shsuenaga }
15321a8031e1Shsuenaga
15331a8031e1Shsuenaga refs = atomic_inc_32_nv(&mv_s->refs);
15341a8031e1Shsuenaga if (refs == 1) {
15351a8031e1Shsuenaga /*
15361a8031e1Shsuenaga * a session with refs == 0 is
15371a8031e1Shsuenaga * already invalidated. revert it.
15381a8031e1Shsuenaga * XXX: use CAS ?
15391a8031e1Shsuenaga */
15401a8031e1Shsuenaga atomic_dec_32(&mv_s->refs);
15411a8031e1Shsuenaga log(LOG_ERR,
15421a8031e1Shsuenaga "%s: session is already invalidated.\n", __func__);
15431a8031e1Shsuenaga return -1;
15441a8031e1Shsuenaga }
15451a8031e1Shsuenaga
15461a8031e1Shsuenaga return 0;
15471a8031e1Shsuenaga }
15481a8031e1Shsuenaga
15491a8031e1Shsuenaga STATIC void
mvxpsec_session_unref(struct mvxpsec_session * mv_s)15501a8031e1Shsuenaga mvxpsec_session_unref(struct mvxpsec_session *mv_s)
15511a8031e1Shsuenaga {
15521a8031e1Shsuenaga uint32_t refs;
15531a8031e1Shsuenaga
1554ef3476fbSriastradh membar_release();
15551a8031e1Shsuenaga refs = atomic_dec_32_nv(&mv_s->refs);
1556122a3e8aSriastradh if (refs == 0) {
1557ef3476fbSriastradh membar_acquire();
15581a8031e1Shsuenaga pool_cache_put(mv_s->sc->sc_session_pool, mv_s);
15591a8031e1Shsuenaga }
1560122a3e8aSriastradh }
15611a8031e1Shsuenaga
15621a8031e1Shsuenaga /*
15631a8031e1Shsuenaga * look for session is exist or not
15641a8031e1Shsuenaga */
15651a8031e1Shsuenaga INLINE struct mvxpsec_session *
mvxpsec_session_lookup(struct mvxpsec_softc * sc,int sid)15661a8031e1Shsuenaga mvxpsec_session_lookup(struct mvxpsec_softc *sc, int sid)
15671a8031e1Shsuenaga {
15681a8031e1Shsuenaga struct mvxpsec_session *mv_s;
15691a8031e1Shsuenaga int session;
15701a8031e1Shsuenaga
15711a8031e1Shsuenaga /* must called sc->sc_session_mtx held */
15721a8031e1Shsuenaga KASSERT(mutex_owned(&sc->sc_session_mtx));
15731a8031e1Shsuenaga
15741a8031e1Shsuenaga session = MVXPSEC_SESSION(sid);
15751a8031e1Shsuenaga if (__predict_false(session > MVXPSEC_MAX_SESSIONS)) {
15761a8031e1Shsuenaga log(LOG_ERR, "%s: session number too large %d\n",
15771a8031e1Shsuenaga __func__, session);
15781a8031e1Shsuenaga return NULL;
15791a8031e1Shsuenaga }
15801a8031e1Shsuenaga if (__predict_false( (mv_s = sc->sc_sessions[session]) == NULL)) {
15811a8031e1Shsuenaga log(LOG_ERR, "%s: invalid session %d\n",
15821a8031e1Shsuenaga __func__, session);
15831a8031e1Shsuenaga return NULL;
15841a8031e1Shsuenaga }
15851a8031e1Shsuenaga
15861a8031e1Shsuenaga KASSERT(mv_s->sid == session);
15871a8031e1Shsuenaga
15881a8031e1Shsuenaga return mv_s;
15891a8031e1Shsuenaga }
15901a8031e1Shsuenaga
15911a8031e1Shsuenaga /*
15921a8031e1Shsuenaga * allocation new packet structure.
15931a8031e1Shsuenaga */
15941a8031e1Shsuenaga STATIC struct mvxpsec_packet *
mvxpsec_packet_alloc(struct mvxpsec_session * mv_s)15951a8031e1Shsuenaga mvxpsec_packet_alloc(struct mvxpsec_session *mv_s)
15961a8031e1Shsuenaga {
15971a8031e1Shsuenaga struct mvxpsec_softc *sc = mv_s->sc;
15981a8031e1Shsuenaga struct mvxpsec_packet *mv_p;
15991a8031e1Shsuenaga
16001a8031e1Shsuenaga /* must be called mv_queue_mtx held. */
16011a8031e1Shsuenaga KASSERT(mutex_owned(&sc->sc_queue_mtx));
16021a8031e1Shsuenaga /* must be called mv_session_mtx held. */
16031a8031e1Shsuenaga KASSERT(mutex_owned(&sc->sc_session_mtx));
16041a8031e1Shsuenaga
16051a8031e1Shsuenaga if (mvxpsec_session_ref(mv_s) < 0) {
16061a8031e1Shsuenaga log(LOG_ERR, "%s: invalid session.\n", __func__);
16071a8031e1Shsuenaga return NULL;
16081a8031e1Shsuenaga }
16091a8031e1Shsuenaga
16101a8031e1Shsuenaga if ( (mv_p = SLIST_FIRST(&sc->sc_free_list)) != NULL) {
16111a8031e1Shsuenaga SLIST_REMOVE_HEAD(&sc->sc_free_list, free_list);
16121a8031e1Shsuenaga sc->sc_free_qlen--;
16131a8031e1Shsuenaga }
16141a8031e1Shsuenaga else {
1615127c64cbSchristos mv_p = pool_cache_get(sc->sc_packet_pool, PR_NOWAIT);
16161a8031e1Shsuenaga if (mv_p == NULL) {
16171a8031e1Shsuenaga log(LOG_ERR, "%s: cannot allocate memory\n",
16181a8031e1Shsuenaga __func__);
16191a8031e1Shsuenaga mvxpsec_session_unref(mv_s);
16201a8031e1Shsuenaga return NULL;
16211a8031e1Shsuenaga }
16221a8031e1Shsuenaga }
16231a8031e1Shsuenaga mv_p->mv_s = mv_s;
16241a8031e1Shsuenaga mv_p->flags = 0;
16251a8031e1Shsuenaga mv_p->data_ptr = NULL;
16261a8031e1Shsuenaga
16271a8031e1Shsuenaga return mv_p;
16281a8031e1Shsuenaga }
16291a8031e1Shsuenaga
16301a8031e1Shsuenaga /*
16311a8031e1Shsuenaga * free packet structure.
16321a8031e1Shsuenaga */
16331a8031e1Shsuenaga STATIC void
mvxpsec_packet_dealloc(struct mvxpsec_packet * mv_p)16341a8031e1Shsuenaga mvxpsec_packet_dealloc(struct mvxpsec_packet *mv_p)
16351a8031e1Shsuenaga {
16361a8031e1Shsuenaga struct mvxpsec_session *mv_s = mv_p->mv_s;
16371a8031e1Shsuenaga struct mvxpsec_softc *sc = mv_s->sc;
16381a8031e1Shsuenaga
16391a8031e1Shsuenaga /* must called with sc->sc_queue_mtx held */
16401a8031e1Shsuenaga KASSERT(mutex_owned(&sc->sc_queue_mtx));
16411a8031e1Shsuenaga
16421a8031e1Shsuenaga if (mv_p->dma_ring.dma_size != 0) {
16431a8031e1Shsuenaga sc->sc_desc_ring_cons += mv_p->dma_ring.dma_size;
16441a8031e1Shsuenaga }
16451a8031e1Shsuenaga mv_p->dma_ring.dma_head = NULL;
16461a8031e1Shsuenaga mv_p->dma_ring.dma_last = NULL;
16471a8031e1Shsuenaga mv_p->dma_ring.dma_size = 0;
16481a8031e1Shsuenaga
16491a8031e1Shsuenaga if (mv_p->data_map) {
16501a8031e1Shsuenaga if (mv_p->flags & RDY_DATA) {
16511a8031e1Shsuenaga bus_dmamap_unload(sc->sc_dmat, mv_p->data_map);
16521a8031e1Shsuenaga mv_p->flags &= ~RDY_DATA;
16531a8031e1Shsuenaga }
16541a8031e1Shsuenaga }
16551a8031e1Shsuenaga
16561a8031e1Shsuenaga if (sc->sc_free_qlen > sc->sc_wait_qlimit)
16571a8031e1Shsuenaga pool_cache_put(sc->sc_packet_pool, mv_p);
16581a8031e1Shsuenaga else {
16591a8031e1Shsuenaga SLIST_INSERT_HEAD(&sc->sc_free_list, mv_p, free_list);
16601a8031e1Shsuenaga sc->sc_free_qlen++;
16611a8031e1Shsuenaga }
16621a8031e1Shsuenaga mvxpsec_session_unref(mv_s);
16631a8031e1Shsuenaga }
16641a8031e1Shsuenaga
16651a8031e1Shsuenaga INLINE void
mvxpsec_packet_enqueue(struct mvxpsec_packet * mv_p)16661a8031e1Shsuenaga mvxpsec_packet_enqueue(struct mvxpsec_packet *mv_p)
16671a8031e1Shsuenaga {
16681a8031e1Shsuenaga struct mvxpsec_softc *sc = mv_p->mv_s->sc;
16691a8031e1Shsuenaga struct mvxpsec_packet *last_packet;
16701a8031e1Shsuenaga struct mvxpsec_descriptor_handle *cur_dma, *prev_dma;
16711a8031e1Shsuenaga
16721a8031e1Shsuenaga /* must called with sc->sc_queue_mtx held */
16731a8031e1Shsuenaga KASSERT(mutex_owned(&sc->sc_queue_mtx));
16741a8031e1Shsuenaga
16751a8031e1Shsuenaga if (sc->sc_wait_qlen == 0) {
16761a8031e1Shsuenaga SIMPLEQ_INSERT_TAIL(&sc->sc_wait_queue, mv_p, queue);
16771a8031e1Shsuenaga sc->sc_wait_qlen++;
16781a8031e1Shsuenaga mv_p->flags |= SETUP_DONE;
16791a8031e1Shsuenaga return;
16801a8031e1Shsuenaga }
16811a8031e1Shsuenaga
16821a8031e1Shsuenaga last_packet = SIMPLEQ_LAST(&sc->sc_wait_queue, mvxpsec_packet, queue);
16831a8031e1Shsuenaga SIMPLEQ_INSERT_TAIL(&sc->sc_wait_queue, mv_p, queue);
16841a8031e1Shsuenaga sc->sc_wait_qlen++;
16851a8031e1Shsuenaga
16861a8031e1Shsuenaga /* chain the DMA */
16871a8031e1Shsuenaga cur_dma = mv_p->dma_ring.dma_head;
16881a8031e1Shsuenaga prev_dma = last_packet->dma_ring.dma_last;
16891a8031e1Shsuenaga mvxpsec_dma_cat(sc, prev_dma, cur_dma);
16901a8031e1Shsuenaga mv_p->flags |= SETUP_DONE;
16911a8031e1Shsuenaga }
16921a8031e1Shsuenaga
16931a8031e1Shsuenaga /*
16941a8031e1Shsuenaga * called by interrupt handler
16951a8031e1Shsuenaga */
16961a8031e1Shsuenaga STATIC int
mvxpsec_done_packet(struct mvxpsec_packet * mv_p)16971a8031e1Shsuenaga mvxpsec_done_packet(struct mvxpsec_packet *mv_p)
16981a8031e1Shsuenaga {
16991a8031e1Shsuenaga struct mvxpsec_session *mv_s = mv_p->mv_s;
17001a8031e1Shsuenaga struct mvxpsec_softc *sc = mv_s->sc;
17011a8031e1Shsuenaga
17021a8031e1Shsuenaga KASSERT((mv_p->flags & RDY_DATA));
17031a8031e1Shsuenaga KASSERT((mv_p->flags & SETUP_DONE));
17041a8031e1Shsuenaga
17051a8031e1Shsuenaga /* unload data */
17061a8031e1Shsuenaga bus_dmamap_sync(sc->sc_dmat, mv_p->data_map,
17071a8031e1Shsuenaga 0, mv_p->data_len,
17081a8031e1Shsuenaga BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
17091a8031e1Shsuenaga bus_dmamap_unload(sc->sc_dmat, mv_p->data_map);
17101a8031e1Shsuenaga mv_p->flags &= ~RDY_DATA;
17111a8031e1Shsuenaga
17121a8031e1Shsuenaga #ifdef MVXPSEC_DEBUG
17131a8031e1Shsuenaga if (mvxpsec_debug != 0) {
17141a8031e1Shsuenaga int s;
17151a8031e1Shsuenaga
17161a8031e1Shsuenaga bus_dmamap_sync(sc->sc_dmat, mv_p->pkt_header_map,
17171a8031e1Shsuenaga 0, sizeof(mv_p->pkt_header),
17181a8031e1Shsuenaga BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
17191a8031e1Shsuenaga bus_dmamap_sync(sc->sc_dmat, mv_s->session_header_map,
17201a8031e1Shsuenaga 0, sizeof(mv_s->session_header),
17211a8031e1Shsuenaga BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
17221a8031e1Shsuenaga
17231a8031e1Shsuenaga if (mvxpsec_debug & MVXPSEC_DEBUG_OPENCRYPTO) {
17241a8031e1Shsuenaga char buf[1500];
17251a8031e1Shsuenaga struct mbuf *m;
17261a8031e1Shsuenaga struct uio *uio;
17271a8031e1Shsuenaga size_t len;
17281a8031e1Shsuenaga
17291a8031e1Shsuenaga switch (mv_p->data_type) {
17301a8031e1Shsuenaga case MVXPSEC_DATA_MBUF:
17311a8031e1Shsuenaga m = mv_p->data_mbuf;
17321a8031e1Shsuenaga len = m->m_pkthdr.len;
17331a8031e1Shsuenaga if (len > sizeof(buf))
17341a8031e1Shsuenaga len = sizeof(buf);
17351a8031e1Shsuenaga m_copydata(m, 0, len, buf);
17361a8031e1Shsuenaga break;
17371a8031e1Shsuenaga case MVXPSEC_DATA_UIO:
17381a8031e1Shsuenaga uio = mv_p->data_uio;
17391a8031e1Shsuenaga len = uio->uio_resid;
17401a8031e1Shsuenaga if (len > sizeof(buf))
17411a8031e1Shsuenaga len = sizeof(buf);
17421a8031e1Shsuenaga cuio_copydata(uio, 0, len, buf);
17431a8031e1Shsuenaga break;
17441a8031e1Shsuenaga default:
17451a8031e1Shsuenaga len = 0;
17461a8031e1Shsuenaga }
17471a8031e1Shsuenaga if (len > 0)
17481a8031e1Shsuenaga mvxpsec_dump_data(__func__, buf, len);
17491a8031e1Shsuenaga }
17501a8031e1Shsuenaga
17511a8031e1Shsuenaga if (mvxpsec_debug & MVXPSEC_DEBUG_PAYLOAD) {
17521a8031e1Shsuenaga MVXPSEC_PRINTF(MVXPSEC_DEBUG_PAYLOAD,
17531a8031e1Shsuenaga "%s: session_descriptor:\n", __func__);
17541a8031e1Shsuenaga mvxpsec_dump_packet_desc(__func__, mv_p);
17551a8031e1Shsuenaga MVXPSEC_PRINTF(MVXPSEC_DEBUG_PAYLOAD,
17561a8031e1Shsuenaga "%s: session_data:\n", __func__);
17571a8031e1Shsuenaga mvxpsec_dump_packet_data(__func__, mv_p);
17581a8031e1Shsuenaga }
17591a8031e1Shsuenaga
17601a8031e1Shsuenaga if (mvxpsec_debug & MVXPSEC_DEBUG_SRAM) {
17611a8031e1Shsuenaga MVXPSEC_PRINTF(MVXPSEC_DEBUG_SRAM,
17621a8031e1Shsuenaga "%s: SRAM\n", __func__);
17631a8031e1Shsuenaga mvxpsec_dump_sram(__func__, sc, 2000);
17641a8031e1Shsuenaga }
17651a8031e1Shsuenaga
17661a8031e1Shsuenaga s = MVXPSEC_READ(sc, MV_ACC_STATUS);
17671a8031e1Shsuenaga if (s & MV_ACC_STATUS_MAC_ERR) {
17681a8031e1Shsuenaga MVXPSEC_PRINTF(MVXPSEC_DEBUG_INTR,
17691a8031e1Shsuenaga "%s: Message Authentication Failed.\n", __func__);
17701a8031e1Shsuenaga }
17711a8031e1Shsuenaga }
17721a8031e1Shsuenaga #endif
17731a8031e1Shsuenaga
17741a8031e1Shsuenaga /* copy back IV */
17751a8031e1Shsuenaga if (mv_p->flags & CRP_EXT_IV) {
17761a8031e1Shsuenaga memcpy(mv_p->ext_iv,
17771a8031e1Shsuenaga &mv_p->pkt_header.crp_iv_ext, mv_p->ext_ivlen);
17781a8031e1Shsuenaga mv_p->ext_iv = NULL;
17791a8031e1Shsuenaga mv_p->ext_ivlen = 0;
17801a8031e1Shsuenaga }
17811a8031e1Shsuenaga
17821a8031e1Shsuenaga /* notify opencrypto */
17831a8031e1Shsuenaga mv_p->crp->crp_etype = 0;
17841a8031e1Shsuenaga crypto_done(mv_p->crp);
17851a8031e1Shsuenaga mv_p->crp = NULL;
17861a8031e1Shsuenaga
17871a8031e1Shsuenaga /* unblock driver */
17881a8031e1Shsuenaga mvxpsec_packet_dealloc(mv_p);
17891a8031e1Shsuenaga crypto_unblock(sc->sc_cid, CRYPTO_SYMQ|CRYPTO_ASYMQ);
17901a8031e1Shsuenaga
17911a8031e1Shsuenaga MVXPSEC_EVCNT_INCR(sc, packet_ok);
17921a8031e1Shsuenaga
17931a8031e1Shsuenaga return 0;
17941a8031e1Shsuenaga }
17951a8031e1Shsuenaga
17961a8031e1Shsuenaga
17971a8031e1Shsuenaga /*
17981a8031e1Shsuenaga * Opencrypto API registration
17991a8031e1Shsuenaga */
18001a8031e1Shsuenaga int
mvxpsec_register(struct mvxpsec_softc * sc)18011a8031e1Shsuenaga mvxpsec_register(struct mvxpsec_softc *sc)
18021a8031e1Shsuenaga {
18031a8031e1Shsuenaga int oplen = SRAM_PAYLOAD_SIZE;
18041a8031e1Shsuenaga int flags = 0;
18051a8031e1Shsuenaga int err;
18061a8031e1Shsuenaga
18071a8031e1Shsuenaga sc->sc_nsessions = 0;
18081a8031e1Shsuenaga sc->sc_cid = crypto_get_driverid(0);
18091a8031e1Shsuenaga if (sc->sc_cid < 0) {
18101a8031e1Shsuenaga log(LOG_ERR,
18111a8031e1Shsuenaga "%s: crypto_get_driverid() failed.\n", __func__);
18121a8031e1Shsuenaga err = EINVAL;
18131a8031e1Shsuenaga goto done;
18141a8031e1Shsuenaga }
18151a8031e1Shsuenaga
18161a8031e1Shsuenaga /* Ciphers */
18171a8031e1Shsuenaga err = crypto_register(sc->sc_cid, CRYPTO_DES_CBC, oplen, flags,
18181a8031e1Shsuenaga mvxpsec_newsession, mvxpsec_freesession, mvxpsec_dispatch, sc);
18191a8031e1Shsuenaga if (err)
18201a8031e1Shsuenaga goto done;
18211a8031e1Shsuenaga
18221a8031e1Shsuenaga err = crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, oplen, flags,
18231a8031e1Shsuenaga mvxpsec_newsession, mvxpsec_freesession, mvxpsec_dispatch, sc);
18241a8031e1Shsuenaga if (err)
18251a8031e1Shsuenaga goto done;
18261a8031e1Shsuenaga
18271a8031e1Shsuenaga err = crypto_register(sc->sc_cid, CRYPTO_AES_CBC, oplen, flags,
18281a8031e1Shsuenaga mvxpsec_newsession, mvxpsec_freesession, mvxpsec_dispatch, sc);
18291a8031e1Shsuenaga if (err)
18301a8031e1Shsuenaga goto done;
18311a8031e1Shsuenaga
18321a8031e1Shsuenaga /* MACs */
18331a8031e1Shsuenaga err = crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC_96,
18341a8031e1Shsuenaga oplen, flags,
18351a8031e1Shsuenaga mvxpsec_newsession, mvxpsec_freesession, mvxpsec_dispatch, sc);
18361a8031e1Shsuenaga if (err)
18371a8031e1Shsuenaga goto done;
18381a8031e1Shsuenaga
18391a8031e1Shsuenaga err = crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC_96,
18401a8031e1Shsuenaga oplen, flags,
18411a8031e1Shsuenaga mvxpsec_newsession, mvxpsec_freesession, mvxpsec_dispatch, sc);
18421a8031e1Shsuenaga if (err)
18431a8031e1Shsuenaga goto done;
18441a8031e1Shsuenaga
18451a8031e1Shsuenaga #ifdef DEBUG
18461a8031e1Shsuenaga log(LOG_DEBUG,
18471a8031e1Shsuenaga "%s: registered to opencrypto(max data = %d bytes)\n",
18481a8031e1Shsuenaga device_xname(sc->sc_dev), oplen);
18491a8031e1Shsuenaga #endif
18501a8031e1Shsuenaga
18511a8031e1Shsuenaga err = 0;
18521a8031e1Shsuenaga done:
18531a8031e1Shsuenaga return err;
18541a8031e1Shsuenaga }
18551a8031e1Shsuenaga
18561a8031e1Shsuenaga /*
18571a8031e1Shsuenaga * Create new opencrypto session
18581a8031e1Shsuenaga *
18591a8031e1Shsuenaga * - register cipher key, mac key.
18601a8031e1Shsuenaga * - initialize mac internal state.
18611a8031e1Shsuenaga */
18621a8031e1Shsuenaga int
mvxpsec_newsession(void * arg,uint32_t * sidp,struct cryptoini * cri)18631a8031e1Shsuenaga mvxpsec_newsession(void *arg, uint32_t *sidp, struct cryptoini *cri)
18641a8031e1Shsuenaga {
18651a8031e1Shsuenaga struct mvxpsec_softc *sc = arg;
18661a8031e1Shsuenaga struct mvxpsec_session *mv_s = NULL;
18671a8031e1Shsuenaga struct cryptoini *c;
18681a8031e1Shsuenaga static int hint = 0;
18691a8031e1Shsuenaga int session = -1;
18701a8031e1Shsuenaga int sid;
18711a8031e1Shsuenaga int err;
18721a8031e1Shsuenaga int i;
18731a8031e1Shsuenaga
18741a8031e1Shsuenaga /* allocate driver session context */
18751a8031e1Shsuenaga mv_s = mvxpsec_session_alloc(sc);
18761a8031e1Shsuenaga if (mv_s == NULL)
18771a8031e1Shsuenaga return ENOMEM;
18781a8031e1Shsuenaga
18791a8031e1Shsuenaga /*
18801a8031e1Shsuenaga * lookup opencrypto session table
18811a8031e1Shsuenaga *
18821a8031e1Shsuenaga * we have sc_session_mtx after here.
18831a8031e1Shsuenaga */
18841a8031e1Shsuenaga mutex_enter(&sc->sc_session_mtx);
18851a8031e1Shsuenaga if (sc->sc_nsessions >= MVXPSEC_MAX_SESSIONS) {
18861a8031e1Shsuenaga mutex_exit(&sc->sc_session_mtx);
18871a8031e1Shsuenaga log(LOG_ERR, "%s: too many IPsec SA(max %d)\n",
18881a8031e1Shsuenaga __func__, MVXPSEC_MAX_SESSIONS);
18891a8031e1Shsuenaga mvxpsec_session_dealloc(mv_s);
18901a8031e1Shsuenaga return ENOMEM;
18911a8031e1Shsuenaga }
18921a8031e1Shsuenaga for (i = hint; i < MVXPSEC_MAX_SESSIONS; i++) {
18931a8031e1Shsuenaga if (sc->sc_sessions[i])
18941a8031e1Shsuenaga continue;
18951a8031e1Shsuenaga session = i;
18961a8031e1Shsuenaga hint = session + 1;
18971a8031e1Shsuenaga break;
18981a8031e1Shsuenaga }
18991a8031e1Shsuenaga if (session < 0) {
19001a8031e1Shsuenaga for (i = 0; i < hint; i++) {
19011a8031e1Shsuenaga if (sc->sc_sessions[i])
19021a8031e1Shsuenaga continue;
19031a8031e1Shsuenaga session = i;
19041a8031e1Shsuenaga hint = session + 1;
19051a8031e1Shsuenaga break;
19061a8031e1Shsuenaga }
19071a8031e1Shsuenaga if (session < 0) {
19081a8031e1Shsuenaga mutex_exit(&sc->sc_session_mtx);
19091a8031e1Shsuenaga /* session full */
19101a8031e1Shsuenaga log(LOG_ERR, "%s: too many IPsec SA(max %d)\n",
19111a8031e1Shsuenaga __func__, MVXPSEC_MAX_SESSIONS);
19121a8031e1Shsuenaga mvxpsec_session_dealloc(mv_s);
19131a8031e1Shsuenaga hint = 0;
19141a8031e1Shsuenaga return ENOMEM;
19151a8031e1Shsuenaga }
19161a8031e1Shsuenaga }
19171a8031e1Shsuenaga if (hint >= MVXPSEC_MAX_SESSIONS)
19181a8031e1Shsuenaga hint = 0;
19191a8031e1Shsuenaga sc->sc_nsessions++;
19201a8031e1Shsuenaga sc->sc_sessions[session] = mv_s;
19211a8031e1Shsuenaga #ifdef DEBUG
19221a8031e1Shsuenaga log(LOG_DEBUG, "%s: new session %d allocated\n", __func__, session);
19231a8031e1Shsuenaga #endif
19241a8031e1Shsuenaga
19251a8031e1Shsuenaga sid = MVXPSEC_SID(device_unit(sc->sc_dev), session);
19261a8031e1Shsuenaga mv_s->sid = sid;
19271a8031e1Shsuenaga
19281a8031e1Shsuenaga /* setup the session key ... */
19291a8031e1Shsuenaga for (c = cri; c; c = c->cri_next) {
19301a8031e1Shsuenaga switch (c->cri_alg) {
19311a8031e1Shsuenaga case CRYPTO_DES_CBC:
19321a8031e1Shsuenaga case CRYPTO_3DES_CBC:
19331a8031e1Shsuenaga case CRYPTO_AES_CBC:
19341a8031e1Shsuenaga /* key */
19351a8031e1Shsuenaga if (mvxpsec_key_precomp(c->cri_alg,
19361a8031e1Shsuenaga c->cri_key, c->cri_klen,
19371a8031e1Shsuenaga &mv_s->session_header.crp_key,
19381a8031e1Shsuenaga &mv_s->session_header.crp_key_d)) {
19391a8031e1Shsuenaga log(LOG_ERR,
19401a8031e1Shsuenaga "%s: Invalid HMAC key for %s.\n",
19411a8031e1Shsuenaga __func__, s_ctlalg(c->cri_alg));
19421a8031e1Shsuenaga err = EINVAL;
19431a8031e1Shsuenaga goto fail;
19441a8031e1Shsuenaga }
19451a8031e1Shsuenaga if (mv_s->sflags & RDY_CRP_KEY) {
19461a8031e1Shsuenaga log(LOG_WARNING,
19471a8031e1Shsuenaga "%s: overwrite cipher: %s->%s.\n",
19481a8031e1Shsuenaga __func__,
19491a8031e1Shsuenaga s_ctlalg(mv_s->cipher_alg),
19501a8031e1Shsuenaga s_ctlalg(c->cri_alg));
19511a8031e1Shsuenaga }
19521a8031e1Shsuenaga mv_s->sflags |= RDY_CRP_KEY;
19531a8031e1Shsuenaga mv_s->enc_klen = c->cri_klen;
19541a8031e1Shsuenaga mv_s->cipher_alg = c->cri_alg;
19551a8031e1Shsuenaga /* create per session IV (compatible with KAME IPsec) */
19561a8031e1Shsuenaga cprng_fast(&mv_s->session_iv, sizeof(mv_s->session_iv));
19571a8031e1Shsuenaga mv_s->sflags |= RDY_CRP_IV;
19581a8031e1Shsuenaga break;
19591a8031e1Shsuenaga case CRYPTO_SHA1_HMAC_96:
19601a8031e1Shsuenaga case CRYPTO_MD5_HMAC_96:
19611a8031e1Shsuenaga /* key */
19621a8031e1Shsuenaga if (mvxpsec_hmac_precomp(c->cri_alg,
19631a8031e1Shsuenaga c->cri_key, c->cri_klen,
19641a8031e1Shsuenaga (uint32_t *)&mv_s->session_header.miv_in,
19651a8031e1Shsuenaga (uint32_t *)&mv_s->session_header.miv_out)) {
19661a8031e1Shsuenaga log(LOG_ERR,
19671a8031e1Shsuenaga "%s: Invalid MAC key\n", __func__);
19681a8031e1Shsuenaga err = EINVAL;
19691a8031e1Shsuenaga goto fail;
19701a8031e1Shsuenaga }
19711a8031e1Shsuenaga if (mv_s->sflags & RDY_MAC_KEY ||
19721a8031e1Shsuenaga mv_s->sflags & RDY_MAC_IV) {
19731a8031e1Shsuenaga log(LOG_ERR,
19741a8031e1Shsuenaga "%s: overwrite HMAC: %s->%s.\n",
19751a8031e1Shsuenaga __func__, s_ctlalg(mv_s->hmac_alg),
19761a8031e1Shsuenaga s_ctlalg(c->cri_alg));
19771a8031e1Shsuenaga }
19781a8031e1Shsuenaga mv_s->sflags |= RDY_MAC_KEY;
19791a8031e1Shsuenaga mv_s->sflags |= RDY_MAC_IV;
19801a8031e1Shsuenaga
19811a8031e1Shsuenaga mv_s->mac_klen = c->cri_klen;
19821a8031e1Shsuenaga mv_s->hmac_alg = c->cri_alg;
19831a8031e1Shsuenaga break;
19841a8031e1Shsuenaga default:
19851a8031e1Shsuenaga log(LOG_ERR, "%s: Unknown algorithm %d\n",
19861a8031e1Shsuenaga __func__, c->cri_alg);
19871a8031e1Shsuenaga err = EINVAL;
19881a8031e1Shsuenaga goto fail;
19891a8031e1Shsuenaga }
19901a8031e1Shsuenaga }
19911a8031e1Shsuenaga MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO,
19921a8031e1Shsuenaga "H/W Crypto session (id:%u) added.\n", session);
19931a8031e1Shsuenaga
19941a8031e1Shsuenaga *sidp = sid;
19951a8031e1Shsuenaga MVXPSEC_EVCNT_INCR(sc, session_new);
19961a8031e1Shsuenaga mutex_exit(&sc->sc_session_mtx);
19971a8031e1Shsuenaga
19981a8031e1Shsuenaga /* sync session header(it's never touched after here) */
19991a8031e1Shsuenaga bus_dmamap_sync(sc->sc_dmat,
20001a8031e1Shsuenaga mv_s->session_header_map,
20011a8031e1Shsuenaga 0, sizeof(mv_s->session_header),
20021a8031e1Shsuenaga BUS_DMASYNC_PREWRITE);
20031a8031e1Shsuenaga
20041a8031e1Shsuenaga return 0;
20051a8031e1Shsuenaga
20061a8031e1Shsuenaga fail:
20071a8031e1Shsuenaga sc->sc_nsessions--;
20081a8031e1Shsuenaga sc->sc_sessions[session] = NULL;
20091a8031e1Shsuenaga hint = session;
20101a8031e1Shsuenaga if (mv_s)
20111a8031e1Shsuenaga mvxpsec_session_dealloc(mv_s);
20121a8031e1Shsuenaga log(LOG_WARNING,
2013176f1e87Sandvar "%s: Failed to add H/W crypto session (id:%u): err=%d\n",
20141a8031e1Shsuenaga __func__, session, err);
20151a8031e1Shsuenaga
20161a8031e1Shsuenaga mutex_exit(&sc->sc_session_mtx);
20171a8031e1Shsuenaga return err;
20181a8031e1Shsuenaga }
20191a8031e1Shsuenaga
20201a8031e1Shsuenaga /*
20211a8031e1Shsuenaga * remove opencrypto session
20221a8031e1Shsuenaga */
20238449a122Sriastradh void
mvxpsec_freesession(void * arg,uint64_t tid)20241a8031e1Shsuenaga mvxpsec_freesession(void *arg, uint64_t tid)
20251a8031e1Shsuenaga {
20261a8031e1Shsuenaga struct mvxpsec_softc *sc = arg;
20271a8031e1Shsuenaga struct mvxpsec_session *mv_s;
20281a8031e1Shsuenaga int session;
20291a8031e1Shsuenaga uint32_t sid = ((uint32_t)tid) & 0xffffffff;
20301a8031e1Shsuenaga
20311a8031e1Shsuenaga session = MVXPSEC_SESSION(sid);
2032f32a8269Sriastradh KASSERTMSG(session >= 0, "session=%d", session);
2033f32a8269Sriastradh KASSERTMSG(session < MVXPSEC_MAX_SESSIONS, "session=%d max=%d",
2034f32a8269Sriastradh session, MVXPSEC_MAX_SESSIONS);
20351a8031e1Shsuenaga
20361a8031e1Shsuenaga mutex_enter(&sc->sc_session_mtx);
2037f32a8269Sriastradh mv_s = sc->sc_sessions[session];
2038f32a8269Sriastradh KASSERT(mv_s != NULL);
20391a8031e1Shsuenaga MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO,
20401a8031e1Shsuenaga "%s: inactivate session %d\n", __func__, session);
20411a8031e1Shsuenaga
20421a8031e1Shsuenaga /* inactivate mvxpsec session */
20431a8031e1Shsuenaga sc->sc_sessions[session] = NULL;
20441a8031e1Shsuenaga sc->sc_nsessions--;
20451a8031e1Shsuenaga sc->sc_last_session = NULL;
20461a8031e1Shsuenaga mutex_exit(&sc->sc_session_mtx);
20471a8031e1Shsuenaga
20481a8031e1Shsuenaga KASSERT(sc->sc_nsessions >= 0);
20491a8031e1Shsuenaga KASSERT(mv_s->sid == sid);
20501a8031e1Shsuenaga
20511a8031e1Shsuenaga mvxpsec_session_dealloc(mv_s);
20521a8031e1Shsuenaga MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO,
20531a8031e1Shsuenaga "H/W Crypto session (id: %d) deleted.\n", session);
20541a8031e1Shsuenaga
20551a8031e1Shsuenaga /* force unblock opencrypto */
20561a8031e1Shsuenaga crypto_unblock(sc->sc_cid, CRYPTO_SYMQ|CRYPTO_ASYMQ);
20571a8031e1Shsuenaga
20581a8031e1Shsuenaga MVXPSEC_EVCNT_INCR(sc, session_free);
20591a8031e1Shsuenaga }
20601a8031e1Shsuenaga
20611a8031e1Shsuenaga /*
20621a8031e1Shsuenaga * process data with existing session
20631a8031e1Shsuenaga */
20641a8031e1Shsuenaga int
mvxpsec_dispatch(void * arg,struct cryptop * crp,int hint)20651a8031e1Shsuenaga mvxpsec_dispatch(void *arg, struct cryptop *crp, int hint)
20661a8031e1Shsuenaga {
20671a8031e1Shsuenaga struct mvxpsec_softc *sc = arg;
20681a8031e1Shsuenaga struct mvxpsec_session *mv_s;
20691a8031e1Shsuenaga struct mvxpsec_packet *mv_p;
20701a8031e1Shsuenaga int q_full;
20711a8031e1Shsuenaga int running;
20721a8031e1Shsuenaga int err;
20731a8031e1Shsuenaga
20741a8031e1Shsuenaga mutex_enter(&sc->sc_queue_mtx);
20751a8031e1Shsuenaga
20761a8031e1Shsuenaga /*
20771a8031e1Shsuenaga * lookup session
20781a8031e1Shsuenaga */
20791a8031e1Shsuenaga mutex_enter(&sc->sc_session_mtx);
20801a8031e1Shsuenaga mv_s = mvxpsec_session_lookup(sc, crp->crp_sid);
20811a8031e1Shsuenaga if (__predict_false(mv_s == NULL)) {
20821a8031e1Shsuenaga err = EINVAL;
20831a8031e1Shsuenaga mv_p = NULL;
20841a8031e1Shsuenaga mutex_exit(&sc->sc_session_mtx);
20851a8031e1Shsuenaga goto fail;
20861a8031e1Shsuenaga }
20871a8031e1Shsuenaga mv_p = mvxpsec_packet_alloc(mv_s);
20881a8031e1Shsuenaga if (__predict_false(mv_p == NULL)) {
20891a8031e1Shsuenaga mutex_exit(&sc->sc_session_mtx);
20901a8031e1Shsuenaga mutex_exit(&sc->sc_queue_mtx);
20911a8031e1Shsuenaga return ERESTART; /* => queued in opencrypto layer */
20921a8031e1Shsuenaga }
20931a8031e1Shsuenaga mutex_exit(&sc->sc_session_mtx);
20941a8031e1Shsuenaga
20951a8031e1Shsuenaga /*
20961a8031e1Shsuenaga * check queue status
20971a8031e1Shsuenaga */
20981a8031e1Shsuenaga #ifdef MVXPSEC_MULTI_PACKET
20991a8031e1Shsuenaga q_full = (sc->sc_wait_qlen >= sc->sc_wait_qlimit) ? 1 : 0;
21001a8031e1Shsuenaga #else
21011a8031e1Shsuenaga q_full = (sc->sc_wait_qlen != 0) ? 1 : 0;
21021a8031e1Shsuenaga #endif
21031a8031e1Shsuenaga running = (sc->sc_flags & HW_RUNNING) ? 1: 0;
21041a8031e1Shsuenaga if (q_full) {
21051a8031e1Shsuenaga /* input queue is full. */
21061a8031e1Shsuenaga if (!running && sc->sc_wait_qlen > 0)
21071a8031e1Shsuenaga mvxpsec_dispatch_queue(sc);
21081a8031e1Shsuenaga MVXPSEC_EVCNT_INCR(sc, queue_full);
21091a8031e1Shsuenaga mvxpsec_packet_dealloc(mv_p);
21101a8031e1Shsuenaga mutex_exit(&sc->sc_queue_mtx);
21111a8031e1Shsuenaga return ERESTART; /* => queued in opencrypto layer */
21121a8031e1Shsuenaga }
21131a8031e1Shsuenaga
21141a8031e1Shsuenaga /*
21151a8031e1Shsuenaga * Load and setup packet data
21161a8031e1Shsuenaga */
21171a8031e1Shsuenaga err = mvxpsec_packet_setcrp(mv_p, crp);
21181a8031e1Shsuenaga if (__predict_false(err))
21191a8031e1Shsuenaga goto fail;
21201a8031e1Shsuenaga
21211a8031e1Shsuenaga /*
21221a8031e1Shsuenaga * Setup DMA descriptor chains
21231a8031e1Shsuenaga */
21241a8031e1Shsuenaga mutex_enter(&sc->sc_dma_mtx);
21251a8031e1Shsuenaga err = mvxpsec_dma_copy_packet(sc, mv_p);
21261a8031e1Shsuenaga mutex_exit(&sc->sc_dma_mtx);
21271a8031e1Shsuenaga if (__predict_false(err))
21281a8031e1Shsuenaga goto fail;
21291a8031e1Shsuenaga
21301a8031e1Shsuenaga #ifdef MVXPSEC_DEBUG
21311a8031e1Shsuenaga mvxpsec_dump_packet(__func__, mv_p);
21321a8031e1Shsuenaga #endif
21331a8031e1Shsuenaga
21341a8031e1Shsuenaga /*
21351a8031e1Shsuenaga * Sync/inval the data cache
21361a8031e1Shsuenaga */
21371a8031e1Shsuenaga err = mvxpsec_dma_sync_packet(sc, mv_p);
21381a8031e1Shsuenaga if (__predict_false(err))
21391a8031e1Shsuenaga goto fail;
21401a8031e1Shsuenaga
21411a8031e1Shsuenaga /*
21421a8031e1Shsuenaga * Enqueue the packet
21431a8031e1Shsuenaga */
21441a8031e1Shsuenaga MVXPSEC_EVCNT_INCR(sc, dispatch_packets);
21451a8031e1Shsuenaga #ifdef MVXPSEC_MULTI_PACKET
21461a8031e1Shsuenaga mvxpsec_packet_enqueue(mv_p);
21471a8031e1Shsuenaga if (!running)
21481a8031e1Shsuenaga mvxpsec_dispatch_queue(sc);
21491a8031e1Shsuenaga #else
21501a8031e1Shsuenaga SIMPLEQ_INSERT_TAIL(&sc->sc_wait_queue, mv_p, queue);
21511a8031e1Shsuenaga sc->sc_wait_qlen++;
21521a8031e1Shsuenaga mv_p->flags |= SETUP_DONE;
21531a8031e1Shsuenaga if (!running)
21541a8031e1Shsuenaga mvxpsec_dispatch_queue(sc);
21551a8031e1Shsuenaga #endif
21561a8031e1Shsuenaga mutex_exit(&sc->sc_queue_mtx);
21571a8031e1Shsuenaga return 0;
21581a8031e1Shsuenaga
21591a8031e1Shsuenaga fail:
21601a8031e1Shsuenaga /* Drop the incoming packet */
21611a8031e1Shsuenaga mvxpsec_drop(sc, crp, mv_p, err);
21621a8031e1Shsuenaga mutex_exit(&sc->sc_queue_mtx);
21631a8031e1Shsuenaga return 0;
21641a8031e1Shsuenaga }
21651a8031e1Shsuenaga
21661a8031e1Shsuenaga /*
21671a8031e1Shsuenaga * back the packet to the IP stack
21681a8031e1Shsuenaga */
21691a8031e1Shsuenaga void
mvxpsec_done(void * arg)21701a8031e1Shsuenaga mvxpsec_done(void *arg)
21711a8031e1Shsuenaga {
21721a8031e1Shsuenaga struct mvxpsec_softc *sc = arg;
21731a8031e1Shsuenaga struct mvxpsec_packet *mv_p;
21741a8031e1Shsuenaga mvxpsec_queue_t ret_queue;
21751a8031e1Shsuenaga int ndone;
21761a8031e1Shsuenaga
21771a8031e1Shsuenaga mutex_enter(&sc->sc_queue_mtx);
21781a8031e1Shsuenaga
21791a8031e1Shsuenaga /* stop wdog timer */
21801a8031e1Shsuenaga callout_stop(&sc->sc_timeout);
21811a8031e1Shsuenaga
21821a8031e1Shsuenaga /* refill MVXPSEC */
21831a8031e1Shsuenaga ret_queue = sc->sc_run_queue;
21841a8031e1Shsuenaga SIMPLEQ_INIT(&sc->sc_run_queue);
21851a8031e1Shsuenaga sc->sc_flags &= ~HW_RUNNING;
21861a8031e1Shsuenaga if (sc->sc_wait_qlen > 0)
21871a8031e1Shsuenaga mvxpsec_dispatch_queue(sc);
21881a8031e1Shsuenaga
21891a8031e1Shsuenaga ndone = 0;
21901a8031e1Shsuenaga while ( (mv_p = SIMPLEQ_FIRST(&ret_queue)) != NULL) {
21911a8031e1Shsuenaga SIMPLEQ_REMOVE_HEAD(&ret_queue, queue);
21921a8031e1Shsuenaga mvxpsec_dma_free(sc, &mv_p->dma_ring);
21931a8031e1Shsuenaga mvxpsec_done_packet(mv_p);
21941a8031e1Shsuenaga ndone++;
21951a8031e1Shsuenaga }
21961a8031e1Shsuenaga MVXPSEC_EVCNT_MAX(sc, max_done, ndone);
21971a8031e1Shsuenaga
21981a8031e1Shsuenaga mutex_exit(&sc->sc_queue_mtx);
21991a8031e1Shsuenaga }
22001a8031e1Shsuenaga
22011a8031e1Shsuenaga /*
22021a8031e1Shsuenaga * drop the packet
22031a8031e1Shsuenaga */
22041a8031e1Shsuenaga INLINE void
mvxpsec_drop(struct mvxpsec_softc * sc,struct cryptop * crp,struct mvxpsec_packet * mv_p,int err)22051a8031e1Shsuenaga mvxpsec_drop(struct mvxpsec_softc *sc, struct cryptop *crp,
22061a8031e1Shsuenaga struct mvxpsec_packet *mv_p, int err)
22071a8031e1Shsuenaga {
22081a8031e1Shsuenaga /* must called with sc->sc_queue_mtx held */
22091a8031e1Shsuenaga KASSERT(mutex_owned(&sc->sc_queue_mtx));
22101a8031e1Shsuenaga
22111a8031e1Shsuenaga if (mv_p)
22121a8031e1Shsuenaga mvxpsec_packet_dealloc(mv_p);
22131a8031e1Shsuenaga if (err < 0)
22141a8031e1Shsuenaga err = EINVAL;
22151a8031e1Shsuenaga crp->crp_etype = err;
22161a8031e1Shsuenaga crypto_done(crp);
22171a8031e1Shsuenaga MVXPSEC_EVCNT_INCR(sc, packet_err);
22181a8031e1Shsuenaga
22191a8031e1Shsuenaga /* dispatch other packets in queue */
22201a8031e1Shsuenaga if (sc->sc_wait_qlen > 0 &&
22211a8031e1Shsuenaga !(sc->sc_flags & HW_RUNNING))
22221a8031e1Shsuenaga mvxpsec_dispatch_queue(sc);
22231a8031e1Shsuenaga
22241a8031e1Shsuenaga /* unblock driver for dropped packet */
22251a8031e1Shsuenaga crypto_unblock(sc->sc_cid, CRYPTO_SYMQ|CRYPTO_ASYMQ);
22261a8031e1Shsuenaga }
22271a8031e1Shsuenaga
22281a8031e1Shsuenaga /* move wait queue entry to run queue */
22291a8031e1Shsuenaga STATIC int
mvxpsec_dispatch_queue(struct mvxpsec_softc * sc)22301a8031e1Shsuenaga mvxpsec_dispatch_queue(struct mvxpsec_softc *sc)
22311a8031e1Shsuenaga {
22321a8031e1Shsuenaga struct mvxpsec_packet *mv_p;
22331a8031e1Shsuenaga paddr_t head;
22341a8031e1Shsuenaga int ndispatch = 0;
22351a8031e1Shsuenaga
22361a8031e1Shsuenaga /* must called with sc->sc_queue_mtx held */
22371a8031e1Shsuenaga KASSERT(mutex_owned(&sc->sc_queue_mtx));
22381a8031e1Shsuenaga
22391a8031e1Shsuenaga /* check there is any task */
22401a8031e1Shsuenaga if (__predict_false(sc->sc_flags & HW_RUNNING)) {
22411a8031e1Shsuenaga log(LOG_WARNING,
22421a8031e1Shsuenaga "%s: another packet already exist.\n", __func__);
22431a8031e1Shsuenaga return 0;
22441a8031e1Shsuenaga }
22451a8031e1Shsuenaga if (__predict_false(SIMPLEQ_EMPTY(&sc->sc_wait_queue))) {
22461a8031e1Shsuenaga log(LOG_WARNING,
22471a8031e1Shsuenaga "%s: no waiting packet yet(qlen=%d).\n",
22481a8031e1Shsuenaga __func__, sc->sc_wait_qlen);
22491a8031e1Shsuenaga return 0;
22501a8031e1Shsuenaga }
22511a8031e1Shsuenaga
22521a8031e1Shsuenaga /* move queue */
22531a8031e1Shsuenaga sc->sc_run_queue = sc->sc_wait_queue;
22541a8031e1Shsuenaga sc->sc_flags |= HW_RUNNING; /* dropped by intr or timeout */
22551a8031e1Shsuenaga SIMPLEQ_INIT(&sc->sc_wait_queue);
22561a8031e1Shsuenaga ndispatch = sc->sc_wait_qlen;
22571a8031e1Shsuenaga sc->sc_wait_qlen = 0;
22581a8031e1Shsuenaga
22591a8031e1Shsuenaga /* get 1st DMA descriptor */
22601a8031e1Shsuenaga mv_p = SIMPLEQ_FIRST(&sc->sc_run_queue);
22611a8031e1Shsuenaga head = mv_p->dma_ring.dma_head->phys_addr;
22621a8031e1Shsuenaga
22631a8031e1Shsuenaga /* terminate last DMA descriptor */
22641a8031e1Shsuenaga mv_p = SIMPLEQ_LAST(&sc->sc_run_queue, mvxpsec_packet, queue);
22651a8031e1Shsuenaga mvxpsec_dma_finalize(sc, &mv_p->dma_ring);
22661a8031e1Shsuenaga
22671a8031e1Shsuenaga /* configure TDMA */
22681a8031e1Shsuenaga if (mvxpsec_dma_wait(sc) < 0) {
22691a8031e1Shsuenaga log(LOG_ERR, "%s: DMA DEVICE not responding", __func__);
22701a8031e1Shsuenaga callout_schedule(&sc->sc_timeout, hz);
22711a8031e1Shsuenaga return 0;
22721a8031e1Shsuenaga }
22731a8031e1Shsuenaga MVXPSEC_WRITE(sc, MV_TDMA_NXT, head);
22741a8031e1Shsuenaga
22751a8031e1Shsuenaga /* trigger ACC */
22761a8031e1Shsuenaga if (mvxpsec_acc_wait(sc) < 0) {
22771a8031e1Shsuenaga log(LOG_ERR, "%s: MVXPSEC not responding", __func__);
22781a8031e1Shsuenaga callout_schedule(&sc->sc_timeout, hz);
22791a8031e1Shsuenaga return 0;
22801a8031e1Shsuenaga }
22811a8031e1Shsuenaga MVXPSEC_WRITE(sc, MV_ACC_COMMAND, MV_ACC_COMMAND_ACT);
22821a8031e1Shsuenaga
22831a8031e1Shsuenaga MVXPSEC_EVCNT_MAX(sc, max_dispatch, ndispatch);
22841a8031e1Shsuenaga MVXPSEC_EVCNT_INCR(sc, dispatch_queue);
22851a8031e1Shsuenaga callout_schedule(&sc->sc_timeout, hz);
22861a8031e1Shsuenaga return 0;
22871a8031e1Shsuenaga }
22881a8031e1Shsuenaga
22891a8031e1Shsuenaga /*
22901a8031e1Shsuenaga * process opencrypto operations(cryptop) for packets.
22911a8031e1Shsuenaga */
22921a8031e1Shsuenaga INLINE int
mvxpsec_parse_crd(struct mvxpsec_packet * mv_p,struct cryptodesc * crd)22931a8031e1Shsuenaga mvxpsec_parse_crd(struct mvxpsec_packet *mv_p, struct cryptodesc *crd)
22941a8031e1Shsuenaga {
22951a8031e1Shsuenaga int ivlen;
22961a8031e1Shsuenaga
22971a8031e1Shsuenaga KASSERT(mv_p->flags & RDY_DATA);
22981a8031e1Shsuenaga
22991a8031e1Shsuenaga /* MAC & Ciphers: set data location and operation */
23001a8031e1Shsuenaga switch (crd->crd_alg) {
23011a8031e1Shsuenaga case CRYPTO_SHA1_HMAC_96:
23021a8031e1Shsuenaga mv_p->pkt_header.desc.acc_config |= MV_ACC_CRYPTO_MAC_96;
23031a8031e1Shsuenaga /* fall through */
23041a8031e1Shsuenaga case CRYPTO_SHA1_HMAC:
23051a8031e1Shsuenaga mv_p->mac_dst = crd->crd_inject;
23061a8031e1Shsuenaga mv_p->mac_off = crd->crd_skip;
23071a8031e1Shsuenaga mv_p->mac_len = crd->crd_len;
23081a8031e1Shsuenaga MV_ACC_CRYPTO_MAC_SET(mv_p->pkt_header.desc.acc_config,
23091a8031e1Shsuenaga MV_ACC_CRYPTO_MAC_HMAC_SHA1);
23101a8031e1Shsuenaga mvxpsec_packet_update_op_order(mv_p, MV_ACC_CRYPTO_OP_MAC);
23111a8031e1Shsuenaga /* No more setup for MAC */
23121a8031e1Shsuenaga return 0;
23131a8031e1Shsuenaga case CRYPTO_MD5_HMAC_96:
23141a8031e1Shsuenaga mv_p->pkt_header.desc.acc_config |= MV_ACC_CRYPTO_MAC_96;
23151a8031e1Shsuenaga /* fall through */
23161a8031e1Shsuenaga case CRYPTO_MD5_HMAC:
23171a8031e1Shsuenaga mv_p->mac_dst = crd->crd_inject;
23181a8031e1Shsuenaga mv_p->mac_off = crd->crd_skip;
23191a8031e1Shsuenaga mv_p->mac_len = crd->crd_len;
23201a8031e1Shsuenaga MV_ACC_CRYPTO_MAC_SET(mv_p->pkt_header.desc.acc_config,
23211a8031e1Shsuenaga MV_ACC_CRYPTO_MAC_HMAC_MD5);
23221a8031e1Shsuenaga mvxpsec_packet_update_op_order(mv_p, MV_ACC_CRYPTO_OP_MAC);
23231a8031e1Shsuenaga /* No more setup for MAC */
23241a8031e1Shsuenaga return 0;
23251a8031e1Shsuenaga case CRYPTO_DES_CBC:
23261a8031e1Shsuenaga mv_p->enc_ivoff = crd->crd_inject;
23271a8031e1Shsuenaga mv_p->enc_off = crd->crd_skip;
23281a8031e1Shsuenaga mv_p->enc_len = crd->crd_len;
23291a8031e1Shsuenaga ivlen = 8;
23301a8031e1Shsuenaga MV_ACC_CRYPTO_ENC_SET(mv_p->pkt_header.desc.acc_config,
23311a8031e1Shsuenaga MV_ACC_CRYPTO_ENC_DES);
23321a8031e1Shsuenaga mv_p->pkt_header.desc.acc_config |= MV_ACC_CRYPTO_CBC;
23331a8031e1Shsuenaga mvxpsec_packet_update_op_order(mv_p, MV_ACC_CRYPTO_OP_ENC);
23341a8031e1Shsuenaga break;
23351a8031e1Shsuenaga case CRYPTO_3DES_CBC:
23361a8031e1Shsuenaga mv_p->enc_ivoff = crd->crd_inject;
23371a8031e1Shsuenaga mv_p->enc_off = crd->crd_skip;
23381a8031e1Shsuenaga mv_p->enc_len = crd->crd_len;
23391a8031e1Shsuenaga ivlen = 8;
23401a8031e1Shsuenaga MV_ACC_CRYPTO_ENC_SET(mv_p->pkt_header.desc.acc_config,
23411a8031e1Shsuenaga MV_ACC_CRYPTO_ENC_3DES);
23421a8031e1Shsuenaga mv_p->pkt_header.desc.acc_config |= MV_ACC_CRYPTO_CBC;
23431a8031e1Shsuenaga mv_p->pkt_header.desc.acc_config |= MV_ACC_CRYPTO_3DES_EDE;
23441a8031e1Shsuenaga mvxpsec_packet_update_op_order(mv_p, MV_ACC_CRYPTO_OP_ENC);
23451a8031e1Shsuenaga break;
23461a8031e1Shsuenaga case CRYPTO_AES_CBC:
23471a8031e1Shsuenaga mv_p->enc_ivoff = crd->crd_inject;
23481a8031e1Shsuenaga mv_p->enc_off = crd->crd_skip;
23491a8031e1Shsuenaga mv_p->enc_len = crd->crd_len;
23501a8031e1Shsuenaga ivlen = 16;
23511a8031e1Shsuenaga MV_ACC_CRYPTO_ENC_SET(mv_p->pkt_header.desc.acc_config,
23521a8031e1Shsuenaga MV_ACC_CRYPTO_ENC_AES);
23531a8031e1Shsuenaga MV_ACC_CRYPTO_AES_KLEN_SET(
23541a8031e1Shsuenaga mv_p->pkt_header.desc.acc_config,
23551a8031e1Shsuenaga mvxpsec_aesklen(mv_p->mv_s->enc_klen));
23561a8031e1Shsuenaga mv_p->pkt_header.desc.acc_config |= MV_ACC_CRYPTO_CBC;
23571a8031e1Shsuenaga mvxpsec_packet_update_op_order(mv_p, MV_ACC_CRYPTO_OP_ENC);
23581a8031e1Shsuenaga break;
23591a8031e1Shsuenaga default:
23601a8031e1Shsuenaga log(LOG_ERR, "%s: Unknown algorithm %d\n",
23611a8031e1Shsuenaga __func__, crd->crd_alg);
23621a8031e1Shsuenaga return EINVAL;
23631a8031e1Shsuenaga }
23641a8031e1Shsuenaga
23651a8031e1Shsuenaga /* Operations only for Cipher, not MAC */
23661a8031e1Shsuenaga if (crd->crd_flags & CRD_F_ENCRYPT) {
23671a8031e1Shsuenaga /* Ciphers: Originate IV for Encryption.*/
23681a8031e1Shsuenaga mv_p->pkt_header.desc.acc_config &= ~MV_ACC_CRYPTO_DECRYPT;
23691a8031e1Shsuenaga mv_p->flags |= DIR_ENCRYPT;
23701a8031e1Shsuenaga
23711a8031e1Shsuenaga if (crd->crd_flags & CRD_F_IV_EXPLICIT) {
23721a8031e1Shsuenaga MVXPSEC_PRINTF(MVXPSEC_DEBUG_ENC_IV, "EXPLICIT IV\n");
23731a8031e1Shsuenaga mv_p->flags |= CRP_EXT_IV;
23741a8031e1Shsuenaga mvxpsec_packet_write_iv(mv_p, crd->crd_iv, ivlen);
23751a8031e1Shsuenaga mv_p->enc_ivoff = MVXPSEC_SRAM_IV_EXT_OFF;
23761a8031e1Shsuenaga }
23771a8031e1Shsuenaga else if (crd->crd_flags & CRD_F_IV_PRESENT) {
23781a8031e1Shsuenaga MVXPSEC_PRINTF(MVXPSEC_DEBUG_ENC_IV, "IV is present\n");
23791a8031e1Shsuenaga mvxpsec_packet_copy_iv(mv_p, crd->crd_inject, ivlen);
23801a8031e1Shsuenaga }
23811a8031e1Shsuenaga else {
23821a8031e1Shsuenaga MVXPSEC_PRINTF(MVXPSEC_DEBUG_ENC_IV, "Create New IV\n");
23831a8031e1Shsuenaga mvxpsec_packet_write_iv(mv_p, NULL, ivlen);
23841a8031e1Shsuenaga }
23851a8031e1Shsuenaga }
23861a8031e1Shsuenaga else {
23871a8031e1Shsuenaga /* Ciphers: IV is loadded from crd_inject when it's present */
23881a8031e1Shsuenaga mv_p->pkt_header.desc.acc_config |= MV_ACC_CRYPTO_DECRYPT;
23891a8031e1Shsuenaga mv_p->flags |= DIR_DECRYPT;
23901a8031e1Shsuenaga
23911a8031e1Shsuenaga if (crd->crd_flags & CRD_F_IV_EXPLICIT) {
23921a8031e1Shsuenaga #ifdef MVXPSEC_DEBUG
23931a8031e1Shsuenaga if (mvxpsec_debug & MVXPSEC_DEBUG_ENC_IV) {
23941a8031e1Shsuenaga MVXPSEC_PRINTF(MVXPSEC_DEBUG_ENC_IV,
23951a8031e1Shsuenaga "EXPLICIT IV(Decrypt)\n");
23961a8031e1Shsuenaga mvxpsec_dump_data(__func__, crd->crd_iv, ivlen);
23971a8031e1Shsuenaga }
23981a8031e1Shsuenaga #endif
23991a8031e1Shsuenaga mv_p->flags |= CRP_EXT_IV;
24001a8031e1Shsuenaga mvxpsec_packet_write_iv(mv_p, crd->crd_iv, ivlen);
24011a8031e1Shsuenaga mv_p->enc_ivoff = MVXPSEC_SRAM_IV_EXT_OFF;
24021a8031e1Shsuenaga }
24031a8031e1Shsuenaga }
24041a8031e1Shsuenaga
24051a8031e1Shsuenaga KASSERT(!((mv_p->flags & DIR_ENCRYPT) && (mv_p->flags & DIR_DECRYPT)));
24061a8031e1Shsuenaga
24071a8031e1Shsuenaga return 0;
24081a8031e1Shsuenaga }
24091a8031e1Shsuenaga
24101a8031e1Shsuenaga INLINE int
mvxpsec_parse_crp(struct mvxpsec_packet * mv_p)24111a8031e1Shsuenaga mvxpsec_parse_crp(struct mvxpsec_packet *mv_p)
24121a8031e1Shsuenaga {
24131a8031e1Shsuenaga struct cryptop *crp = mv_p->crp;
24141a8031e1Shsuenaga struct cryptodesc *crd;
24151a8031e1Shsuenaga int err;
24161a8031e1Shsuenaga
24171a8031e1Shsuenaga KASSERT(crp);
24181a8031e1Shsuenaga
24191a8031e1Shsuenaga mvxpsec_packet_reset_op(mv_p);
24201a8031e1Shsuenaga
24211a8031e1Shsuenaga for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
24221a8031e1Shsuenaga err = mvxpsec_parse_crd(mv_p, crd);
24231a8031e1Shsuenaga if (err)
24241a8031e1Shsuenaga return err;
24251a8031e1Shsuenaga }
24261a8031e1Shsuenaga
24271a8031e1Shsuenaga return 0;
24281a8031e1Shsuenaga }
24291a8031e1Shsuenaga
24301a8031e1Shsuenaga INLINE int
mvxpsec_packet_setcrp(struct mvxpsec_packet * mv_p,struct cryptop * crp)24311a8031e1Shsuenaga mvxpsec_packet_setcrp(struct mvxpsec_packet *mv_p, struct cryptop *crp)
24321a8031e1Shsuenaga {
24331a8031e1Shsuenaga int err = EINVAL;
24341a8031e1Shsuenaga
2435*100a3398Sandvar /* register crp to the MVXPSEC packet */
24361a8031e1Shsuenaga if (crp->crp_flags & CRYPTO_F_IMBUF) {
24371a8031e1Shsuenaga err = mvxpsec_packet_setmbuf(mv_p,
24381a8031e1Shsuenaga (struct mbuf *)crp->crp_buf);
24391a8031e1Shsuenaga mv_p->crp = crp;
24401a8031e1Shsuenaga }
24411a8031e1Shsuenaga else if (crp->crp_flags & CRYPTO_F_IOV) {
24421a8031e1Shsuenaga err = mvxpsec_packet_setuio(mv_p,
24431a8031e1Shsuenaga (struct uio *)crp->crp_buf);
24441a8031e1Shsuenaga mv_p->crp = crp;
24451a8031e1Shsuenaga }
24461a8031e1Shsuenaga else {
24471a8031e1Shsuenaga err = mvxpsec_packet_setdata(mv_p,
24481a8031e1Shsuenaga (struct mbuf *)crp->crp_buf, crp->crp_ilen);
24491a8031e1Shsuenaga mv_p->crp = crp;
24501a8031e1Shsuenaga }
24511a8031e1Shsuenaga if (__predict_false(err))
24521a8031e1Shsuenaga return err;
24531a8031e1Shsuenaga
24541a8031e1Shsuenaga /* parse crp and setup MVXPSEC registers/descriptors */
24551a8031e1Shsuenaga err = mvxpsec_parse_crp(mv_p);
24561a8031e1Shsuenaga if (__predict_false(err))
24571a8031e1Shsuenaga return err;
24581a8031e1Shsuenaga
24591a8031e1Shsuenaga /* fixup data offset to fit MVXPSEC internal SRAM */
24601a8031e1Shsuenaga err = mvxpsec_header_finalize(mv_p);
24611a8031e1Shsuenaga if (__predict_false(err))
24621a8031e1Shsuenaga return err;
24631a8031e1Shsuenaga
24641a8031e1Shsuenaga return 0;
24651a8031e1Shsuenaga }
24661a8031e1Shsuenaga
24671a8031e1Shsuenaga /*
24681a8031e1Shsuenaga * load data for encrypt/decrypt/authentication
24691a8031e1Shsuenaga *
24701a8031e1Shsuenaga * data is raw kernel memory area.
24711a8031e1Shsuenaga */
24721a8031e1Shsuenaga STATIC int
mvxpsec_packet_setdata(struct mvxpsec_packet * mv_p,void * data,uint32_t data_len)24731a8031e1Shsuenaga mvxpsec_packet_setdata(struct mvxpsec_packet *mv_p,
24741a8031e1Shsuenaga void *data, uint32_t data_len)
24751a8031e1Shsuenaga {
24761a8031e1Shsuenaga struct mvxpsec_session *mv_s = mv_p->mv_s;
24771a8031e1Shsuenaga struct mvxpsec_softc *sc = mv_s->sc;
24781a8031e1Shsuenaga
24791a8031e1Shsuenaga if (bus_dmamap_load(sc->sc_dmat, mv_p->data_map, data, data_len,
24801a8031e1Shsuenaga NULL, BUS_DMA_NOWAIT)) {
24811a8031e1Shsuenaga log(LOG_ERR, "%s: cannot load data\n", __func__);
24821a8031e1Shsuenaga return -1;
24831a8031e1Shsuenaga }
24841a8031e1Shsuenaga mv_p->data_type = MVXPSEC_DATA_RAW;
24851a8031e1Shsuenaga mv_p->data_raw = data;
24861a8031e1Shsuenaga mv_p->data_len = data_len;
24871a8031e1Shsuenaga mv_p->flags |= RDY_DATA;
24881a8031e1Shsuenaga
24891a8031e1Shsuenaga return 0;
24901a8031e1Shsuenaga }
24911a8031e1Shsuenaga
24921a8031e1Shsuenaga /*
24931a8031e1Shsuenaga * load data for encrypt/decrypt/authentication
24941a8031e1Shsuenaga *
24951a8031e1Shsuenaga * data is mbuf based network data.
24961a8031e1Shsuenaga */
24971a8031e1Shsuenaga STATIC int
mvxpsec_packet_setmbuf(struct mvxpsec_packet * mv_p,struct mbuf * m)24981a8031e1Shsuenaga mvxpsec_packet_setmbuf(struct mvxpsec_packet *mv_p, struct mbuf *m)
24991a8031e1Shsuenaga {
25001a8031e1Shsuenaga struct mvxpsec_session *mv_s = mv_p->mv_s;
25011a8031e1Shsuenaga struct mvxpsec_softc *sc = mv_s->sc;
25021a8031e1Shsuenaga size_t pktlen = 0;
25031a8031e1Shsuenaga
25041a8031e1Shsuenaga if (__predict_true(m->m_flags & M_PKTHDR))
25051a8031e1Shsuenaga pktlen = m->m_pkthdr.len;
25061a8031e1Shsuenaga else {
25071a8031e1Shsuenaga struct mbuf *mp = m;
25081a8031e1Shsuenaga
25091a8031e1Shsuenaga while (mp != NULL) {
25101a8031e1Shsuenaga pktlen += m->m_len;
25111a8031e1Shsuenaga mp = mp->m_next;
25121a8031e1Shsuenaga }
25131a8031e1Shsuenaga }
25141a8031e1Shsuenaga if (pktlen > SRAM_PAYLOAD_SIZE) {
25152499f515Sriastradh #if NIPSEC > 0
25161a8031e1Shsuenaga extern percpu_t *espstat_percpu;
25171a8031e1Shsuenaga /* XXX:
25181a8031e1Shsuenaga * layer violation. opencrypto knows our max packet size
25191a8031e1Shsuenaga * from crypto_register(9) API.
25201a8031e1Shsuenaga */
25211a8031e1Shsuenaga
25221a8031e1Shsuenaga _NET_STATINC(espstat_percpu, ESP_STAT_TOOBIG);
25232499f515Sriastradh #endif
25241a8031e1Shsuenaga log(LOG_ERR,
25251a8031e1Shsuenaga "%s: ESP Packet too large: %zu [oct.] > %zu [oct.]\n",
25261a8031e1Shsuenaga device_xname(sc->sc_dev),
25271a8031e1Shsuenaga (size_t)pktlen, SRAM_PAYLOAD_SIZE);
25281a8031e1Shsuenaga mv_p->data_type = MVXPSEC_DATA_NONE;
25291a8031e1Shsuenaga mv_p->data_mbuf = NULL;
25301a8031e1Shsuenaga return -1;
25311a8031e1Shsuenaga }
25321a8031e1Shsuenaga
25331a8031e1Shsuenaga if (bus_dmamap_load_mbuf(sc->sc_dmat, mv_p->data_map, m,
25341a8031e1Shsuenaga BUS_DMA_NOWAIT)) {
25351a8031e1Shsuenaga mv_p->data_type = MVXPSEC_DATA_NONE;
25361a8031e1Shsuenaga mv_p->data_mbuf = NULL;
25371a8031e1Shsuenaga log(LOG_ERR, "%s: cannot load mbuf\n", __func__);
25381a8031e1Shsuenaga return -1;
25391a8031e1Shsuenaga }
25401a8031e1Shsuenaga
25411a8031e1Shsuenaga /* set payload buffer */
25421a8031e1Shsuenaga mv_p->data_type = MVXPSEC_DATA_MBUF;
25431a8031e1Shsuenaga mv_p->data_mbuf = m;
25441a8031e1Shsuenaga if (m->m_flags & M_PKTHDR) {
25451a8031e1Shsuenaga mv_p->data_len = m->m_pkthdr.len;
25461a8031e1Shsuenaga }
25471a8031e1Shsuenaga else {
25481a8031e1Shsuenaga mv_p->data_len = 0;
25491a8031e1Shsuenaga while (m) {
25501a8031e1Shsuenaga mv_p->data_len += m->m_len;
25511a8031e1Shsuenaga m = m->m_next;
25521a8031e1Shsuenaga }
25531a8031e1Shsuenaga }
25541a8031e1Shsuenaga mv_p->flags |= RDY_DATA;
25551a8031e1Shsuenaga
25561a8031e1Shsuenaga return 0;
25571a8031e1Shsuenaga }
25581a8031e1Shsuenaga
25591a8031e1Shsuenaga STATIC int
mvxpsec_packet_setuio(struct mvxpsec_packet * mv_p,struct uio * uio)25601a8031e1Shsuenaga mvxpsec_packet_setuio(struct mvxpsec_packet *mv_p, struct uio *uio)
25611a8031e1Shsuenaga {
25621a8031e1Shsuenaga struct mvxpsec_session *mv_s = mv_p->mv_s;
25631a8031e1Shsuenaga struct mvxpsec_softc *sc = mv_s->sc;
25641a8031e1Shsuenaga
25651a8031e1Shsuenaga if (uio->uio_resid > SRAM_PAYLOAD_SIZE) {
25662499f515Sriastradh #if NIPSEC > 0
25671a8031e1Shsuenaga extern percpu_t *espstat_percpu;
25681a8031e1Shsuenaga /* XXX:
25691a8031e1Shsuenaga * layer violation. opencrypto knows our max packet size
25701a8031e1Shsuenaga * from crypto_register(9) API.
25711a8031e1Shsuenaga */
25721a8031e1Shsuenaga
25731a8031e1Shsuenaga _NET_STATINC(espstat_percpu, ESP_STAT_TOOBIG);
25742499f515Sriastradh #endif
25751a8031e1Shsuenaga log(LOG_ERR,
25761a8031e1Shsuenaga "%s: uio request too large: %zu [oct.] > %zu [oct.]\n",
25771a8031e1Shsuenaga device_xname(sc->sc_dev),
25781a8031e1Shsuenaga uio->uio_resid, SRAM_PAYLOAD_SIZE);
25791a8031e1Shsuenaga mv_p->data_type = MVXPSEC_DATA_NONE;
25801a8031e1Shsuenaga mv_p->data_mbuf = NULL;
25811a8031e1Shsuenaga return -1;
25821a8031e1Shsuenaga }
25831a8031e1Shsuenaga
25841a8031e1Shsuenaga if (bus_dmamap_load_uio(sc->sc_dmat, mv_p->data_map, uio,
25851a8031e1Shsuenaga BUS_DMA_NOWAIT)) {
25861a8031e1Shsuenaga mv_p->data_type = MVXPSEC_DATA_NONE;
25871a8031e1Shsuenaga mv_p->data_mbuf = NULL;
25881a8031e1Shsuenaga log(LOG_ERR, "%s: cannot load uio buf\n", __func__);
25891a8031e1Shsuenaga return -1;
25901a8031e1Shsuenaga }
25911a8031e1Shsuenaga
25921a8031e1Shsuenaga /* set payload buffer */
25931a8031e1Shsuenaga mv_p->data_type = MVXPSEC_DATA_UIO;
25941a8031e1Shsuenaga mv_p->data_uio = uio;
25951a8031e1Shsuenaga mv_p->data_len = uio->uio_resid;
25961a8031e1Shsuenaga mv_p->flags |= RDY_DATA;
25971a8031e1Shsuenaga
25981a8031e1Shsuenaga return 0;
25991a8031e1Shsuenaga }
26001a8031e1Shsuenaga
26011a8031e1Shsuenaga STATIC int
mvxpsec_packet_rdata(struct mvxpsec_packet * mv_p,int off,int len,void * cp)26021a8031e1Shsuenaga mvxpsec_packet_rdata(struct mvxpsec_packet *mv_p,
26031a8031e1Shsuenaga int off, int len, void *cp)
26041a8031e1Shsuenaga {
26051a8031e1Shsuenaga uint8_t *p;
26061a8031e1Shsuenaga
26071a8031e1Shsuenaga if (mv_p->data_type == MVXPSEC_DATA_RAW) {
26081a8031e1Shsuenaga p = (uint8_t *)mv_p->data_raw + off;
26091a8031e1Shsuenaga memcpy(cp, p, len);
26101a8031e1Shsuenaga }
26111a8031e1Shsuenaga else if (mv_p->data_type == MVXPSEC_DATA_MBUF) {
26121a8031e1Shsuenaga m_copydata(mv_p->data_mbuf, off, len, cp);
26131a8031e1Shsuenaga }
26141a8031e1Shsuenaga else if (mv_p->data_type == MVXPSEC_DATA_UIO) {
26151a8031e1Shsuenaga cuio_copydata(mv_p->data_uio, off, len, cp);
26161a8031e1Shsuenaga }
26171a8031e1Shsuenaga else
26181a8031e1Shsuenaga return -1;
26191a8031e1Shsuenaga
26201a8031e1Shsuenaga return 0;
26211a8031e1Shsuenaga }
26221a8031e1Shsuenaga
26231a8031e1Shsuenaga STATIC int
mvxpsec_packet_wdata(struct mvxpsec_packet * mv_p,int off,int len,void * cp)26241a8031e1Shsuenaga mvxpsec_packet_wdata(struct mvxpsec_packet *mv_p,
26251a8031e1Shsuenaga int off, int len, void *cp)
26261a8031e1Shsuenaga {
26271a8031e1Shsuenaga uint8_t *p;
26281a8031e1Shsuenaga
26291a8031e1Shsuenaga if (mv_p->data_type == MVXPSEC_DATA_RAW) {
26301a8031e1Shsuenaga p = (uint8_t *)mv_p->data_raw + off;
26311a8031e1Shsuenaga memcpy(p, cp, len);
26321a8031e1Shsuenaga }
26331a8031e1Shsuenaga else if (mv_p->data_type == MVXPSEC_DATA_MBUF) {
26341a8031e1Shsuenaga m_copyback(mv_p->data_mbuf, off, len, cp);
26351a8031e1Shsuenaga }
26361a8031e1Shsuenaga else if (mv_p->data_type == MVXPSEC_DATA_UIO) {
26371a8031e1Shsuenaga cuio_copyback(mv_p->data_uio, off, len, cp);
26381a8031e1Shsuenaga }
26391a8031e1Shsuenaga else
26401a8031e1Shsuenaga return -1;
26411a8031e1Shsuenaga
26421a8031e1Shsuenaga return 0;
26431a8031e1Shsuenaga }
26441a8031e1Shsuenaga
26451a8031e1Shsuenaga /*
26461a8031e1Shsuenaga * Set initial vector of cipher to the session.
26471a8031e1Shsuenaga */
26481a8031e1Shsuenaga STATIC int
mvxpsec_packet_write_iv(struct mvxpsec_packet * mv_p,void * iv,int ivlen)26491a8031e1Shsuenaga mvxpsec_packet_write_iv(struct mvxpsec_packet *mv_p, void *iv, int ivlen)
26501a8031e1Shsuenaga {
26511a8031e1Shsuenaga uint8_t ivbuf[16];
26521a8031e1Shsuenaga
26531a8031e1Shsuenaga KASSERT(ivlen == 8 || ivlen == 16);
26541a8031e1Shsuenaga
26551a8031e1Shsuenaga if (iv == NULL) {
26561a8031e1Shsuenaga if (mv_p->mv_s->sflags & RDY_CRP_IV) {
26571a8031e1Shsuenaga /* use per session IV (compatible with KAME IPsec) */
26581a8031e1Shsuenaga mv_p->pkt_header.crp_iv_work = mv_p->mv_s->session_iv;
26591a8031e1Shsuenaga mv_p->flags |= RDY_CRP_IV;
26601a8031e1Shsuenaga return 0;
26611a8031e1Shsuenaga }
26621a8031e1Shsuenaga cprng_fast(ivbuf, ivlen);
26631a8031e1Shsuenaga iv = ivbuf;
26641a8031e1Shsuenaga }
26651a8031e1Shsuenaga memcpy(&mv_p->pkt_header.crp_iv_work, iv, ivlen);
26661a8031e1Shsuenaga if (mv_p->flags & CRP_EXT_IV) {
26671a8031e1Shsuenaga memcpy(&mv_p->pkt_header.crp_iv_ext, iv, ivlen);
26681a8031e1Shsuenaga mv_p->ext_iv = iv;
26691a8031e1Shsuenaga mv_p->ext_ivlen = ivlen;
26701a8031e1Shsuenaga }
26711a8031e1Shsuenaga mv_p->flags |= RDY_CRP_IV;
26721a8031e1Shsuenaga
26731a8031e1Shsuenaga return 0;
26741a8031e1Shsuenaga }
26751a8031e1Shsuenaga
26761a8031e1Shsuenaga STATIC int
mvxpsec_packet_copy_iv(struct mvxpsec_packet * mv_p,int off,int ivlen)26771a8031e1Shsuenaga mvxpsec_packet_copy_iv(struct mvxpsec_packet *mv_p, int off, int ivlen)
26781a8031e1Shsuenaga {
26791a8031e1Shsuenaga mvxpsec_packet_rdata(mv_p, off, ivlen,
26801a8031e1Shsuenaga &mv_p->pkt_header.crp_iv_work);
26811a8031e1Shsuenaga mv_p->flags |= RDY_CRP_IV;
26821a8031e1Shsuenaga
26831a8031e1Shsuenaga return 0;
26841a8031e1Shsuenaga }
26851a8031e1Shsuenaga
26861a8031e1Shsuenaga /*
26871a8031e1Shsuenaga * set a encryption or decryption key to the session
26881a8031e1Shsuenaga *
26891a8031e1Shsuenaga * Input key material is big endian.
26901a8031e1Shsuenaga */
26911a8031e1Shsuenaga STATIC int
mvxpsec_key_precomp(int alg,void * keymat,int kbitlen,void * key_encrypt,void * key_decrypt)26921a8031e1Shsuenaga mvxpsec_key_precomp(int alg, void *keymat, int kbitlen,
26931a8031e1Shsuenaga void *key_encrypt, void *key_decrypt)
26941a8031e1Shsuenaga {
26951a8031e1Shsuenaga uint32_t *kp = keymat;
26961a8031e1Shsuenaga uint32_t *ekp = key_encrypt;
26971a8031e1Shsuenaga uint32_t *dkp = key_decrypt;
26981a8031e1Shsuenaga int i;
26991a8031e1Shsuenaga
27001a8031e1Shsuenaga switch (alg) {
27011a8031e1Shsuenaga case CRYPTO_DES_CBC:
27021a8031e1Shsuenaga if (kbitlen < 64 || (kbitlen % 8) != 0) {
27031a8031e1Shsuenaga log(LOG_WARNING,
27041a8031e1Shsuenaga "mvxpsec: invalid DES keylen %d\n", kbitlen);
27051a8031e1Shsuenaga return EINVAL;
27061a8031e1Shsuenaga }
27071a8031e1Shsuenaga for (i = 0; i < 2; i++)
27081a8031e1Shsuenaga dkp[i] = ekp[i] = kp[i];
27091a8031e1Shsuenaga for (; i < 8; i++)
27101a8031e1Shsuenaga dkp[i] = ekp[i] = 0;
27111a8031e1Shsuenaga break;
27121a8031e1Shsuenaga case CRYPTO_3DES_CBC:
27131a8031e1Shsuenaga if (kbitlen < 192 || (kbitlen % 8) != 0) {
27141a8031e1Shsuenaga log(LOG_WARNING,
27151a8031e1Shsuenaga "mvxpsec: invalid 3DES keylen %d\n", kbitlen);
27161a8031e1Shsuenaga return EINVAL;
27171a8031e1Shsuenaga }
27181a8031e1Shsuenaga for (i = 0; i < 8; i++)
27191a8031e1Shsuenaga dkp[i] = ekp[i] = kp[i];
27201a8031e1Shsuenaga break;
27211a8031e1Shsuenaga case CRYPTO_AES_CBC:
27221a8031e1Shsuenaga if (kbitlen < 128) {
27231a8031e1Shsuenaga log(LOG_WARNING,
27241a8031e1Shsuenaga "mvxpsec: invalid AES keylen %d\n", kbitlen);
27251a8031e1Shsuenaga return EINVAL;
27261a8031e1Shsuenaga }
27271a8031e1Shsuenaga else if (kbitlen < 192) {
27281a8031e1Shsuenaga /* AES-128 */
27291a8031e1Shsuenaga for (i = 0; i < 4; i++)
27301a8031e1Shsuenaga ekp[i] = kp[i];
27311a8031e1Shsuenaga for (; i < 8; i++)
27321a8031e1Shsuenaga ekp[i] = 0;
27331a8031e1Shsuenaga }
27341a8031e1Shsuenaga else if (kbitlen < 256) {
27351a8031e1Shsuenaga /* AES-192 */
27361a8031e1Shsuenaga for (i = 0; i < 6; i++)
27371a8031e1Shsuenaga ekp[i] = kp[i];
27381a8031e1Shsuenaga for (; i < 8; i++)
27391a8031e1Shsuenaga ekp[i] = 0;
27401a8031e1Shsuenaga }
27411a8031e1Shsuenaga else {
27421a8031e1Shsuenaga /* AES-256 */
27431a8031e1Shsuenaga for (i = 0; i < 8; i++)
27441a8031e1Shsuenaga ekp[i] = kp[i];
27451a8031e1Shsuenaga }
27461a8031e1Shsuenaga /* make decryption key */
27471a8031e1Shsuenaga mv_aes_deckey((uint8_t *)dkp, (uint8_t *)ekp, kbitlen);
27481a8031e1Shsuenaga break;
27491a8031e1Shsuenaga default:
27501a8031e1Shsuenaga for (i = 0; i < 8; i++)
27511a8031e1Shsuenaga ekp[0] = dkp[0] = 0;
27521a8031e1Shsuenaga break;
27531a8031e1Shsuenaga }
27541a8031e1Shsuenaga
27551a8031e1Shsuenaga #ifdef MVXPSEC_DEBUG
27561a8031e1Shsuenaga if (mvxpsec_debug & MVXPSEC_DEBUG_OPENCRYPTO) {
27571a8031e1Shsuenaga MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO,
27581a8031e1Shsuenaga "%s: keyregistered\n", __func__);
27591a8031e1Shsuenaga mvxpsec_dump_data(__func__, ekp, 32);
27601a8031e1Shsuenaga }
27611a8031e1Shsuenaga #endif
27621a8031e1Shsuenaga
27631a8031e1Shsuenaga return 0;
27641a8031e1Shsuenaga }
27651a8031e1Shsuenaga
27661a8031e1Shsuenaga /*
27671a8031e1Shsuenaga * set MAC key to the session
27681a8031e1Shsuenaga *
27691a8031e1Shsuenaga * MAC engine has no register for key itself, but the engine has
27701a8031e1Shsuenaga * inner and outer IV register. software must compute IV before
27711a8031e1Shsuenaga * enable the engine.
27721a8031e1Shsuenaga *
27731a8031e1Shsuenaga * IV is a hash of ipad/opad. these are defined by FIPS-198a
27741a8031e1Shsuenaga * standard.
27751a8031e1Shsuenaga */
27761a8031e1Shsuenaga STATIC int
mvxpsec_hmac_precomp(int alg,void * key,int kbitlen,void * iv_inner,void * iv_outer)27771a8031e1Shsuenaga mvxpsec_hmac_precomp(int alg, void *key, int kbitlen,
27781a8031e1Shsuenaga void *iv_inner, void *iv_outer)
27791a8031e1Shsuenaga {
27801a8031e1Shsuenaga SHA1_CTX sha1;
27811a8031e1Shsuenaga MD5_CTX md5;
27821a8031e1Shsuenaga uint8_t *key8 = key;
27831a8031e1Shsuenaga uint8_t kbuf[64];
27841a8031e1Shsuenaga uint8_t ipad[64];
27851a8031e1Shsuenaga uint8_t opad[64];
27861a8031e1Shsuenaga uint32_t *iv_in = iv_inner;
27871a8031e1Shsuenaga uint32_t *iv_out = iv_outer;
27881a8031e1Shsuenaga int kbytelen;
27891a8031e1Shsuenaga int i;
27901a8031e1Shsuenaga #define HMAC_IPAD 0x36
27911a8031e1Shsuenaga #define HMAC_OPAD 0x5c
27921a8031e1Shsuenaga
27931a8031e1Shsuenaga kbytelen = kbitlen / 8;
27941a8031e1Shsuenaga KASSERT(kbitlen == kbytelen * 8);
27951a8031e1Shsuenaga if (kbytelen > 64) {
27961a8031e1Shsuenaga SHA1Init(&sha1);
27971a8031e1Shsuenaga SHA1Update(&sha1, key, kbytelen);
27981a8031e1Shsuenaga SHA1Final(kbuf, &sha1);
27991a8031e1Shsuenaga key8 = kbuf;
28001a8031e1Shsuenaga kbytelen = 64;
28011a8031e1Shsuenaga }
28021a8031e1Shsuenaga
28031a8031e1Shsuenaga /* make initial 64 oct. string */
28041a8031e1Shsuenaga switch (alg) {
28051a8031e1Shsuenaga case CRYPTO_SHA1_HMAC_96:
28061a8031e1Shsuenaga case CRYPTO_SHA1_HMAC:
28071a8031e1Shsuenaga case CRYPTO_MD5_HMAC_96:
28081a8031e1Shsuenaga case CRYPTO_MD5_HMAC:
28091a8031e1Shsuenaga for (i = 0; i < kbytelen; i++) {
28101a8031e1Shsuenaga ipad[i] = (key8[i] ^ HMAC_IPAD);
28111a8031e1Shsuenaga opad[i] = (key8[i] ^ HMAC_OPAD);
28121a8031e1Shsuenaga }
28131a8031e1Shsuenaga for (; i < 64; i++) {
28141a8031e1Shsuenaga ipad[i] = HMAC_IPAD;
28151a8031e1Shsuenaga opad[i] = HMAC_OPAD;
28161a8031e1Shsuenaga }
28171a8031e1Shsuenaga break;
28181a8031e1Shsuenaga default:
28191a8031e1Shsuenaga break;
28201a8031e1Shsuenaga }
28211a8031e1Shsuenaga #ifdef MVXPSEC_DEBUG
28221a8031e1Shsuenaga if (mvxpsec_debug & MVXPSEC_DEBUG_OPENCRYPTO) {
28231a8031e1Shsuenaga MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO,
28241a8031e1Shsuenaga "%s: HMAC-KEY Pre-comp:\n", __func__);
28251a8031e1Shsuenaga mvxpsec_dump_data(__func__, key, 64);
28261a8031e1Shsuenaga MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO,
28271a8031e1Shsuenaga "%s: ipad:\n", __func__);
28281a8031e1Shsuenaga mvxpsec_dump_data(__func__, ipad, sizeof(ipad));
28291a8031e1Shsuenaga MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO,
28301a8031e1Shsuenaga "%s: opad:\n", __func__);
28311a8031e1Shsuenaga mvxpsec_dump_data(__func__, opad, sizeof(opad));
28321a8031e1Shsuenaga }
28331a8031e1Shsuenaga #endif
28341a8031e1Shsuenaga
28351a8031e1Shsuenaga /* make iv from string */
28361a8031e1Shsuenaga switch (alg) {
28371a8031e1Shsuenaga case CRYPTO_SHA1_HMAC_96:
28381a8031e1Shsuenaga case CRYPTO_SHA1_HMAC:
28391a8031e1Shsuenaga MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO,
28401a8031e1Shsuenaga "%s: Generate iv_in(SHA1)\n", __func__);
28411a8031e1Shsuenaga SHA1Init(&sha1);
28421a8031e1Shsuenaga SHA1Update(&sha1, ipad, 64);
28431a8031e1Shsuenaga /* XXX: private state... (LE) */
28441a8031e1Shsuenaga iv_in[0] = htobe32(sha1.state[0]);
28451a8031e1Shsuenaga iv_in[1] = htobe32(sha1.state[1]);
28461a8031e1Shsuenaga iv_in[2] = htobe32(sha1.state[2]);
28471a8031e1Shsuenaga iv_in[3] = htobe32(sha1.state[3]);
28481a8031e1Shsuenaga iv_in[4] = htobe32(sha1.state[4]);
28491a8031e1Shsuenaga
28501a8031e1Shsuenaga MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO,
28511a8031e1Shsuenaga "%s: Generate iv_out(SHA1)\n", __func__);
28521a8031e1Shsuenaga SHA1Init(&sha1);
28531a8031e1Shsuenaga SHA1Update(&sha1, opad, 64);
28541a8031e1Shsuenaga /* XXX: private state... (LE) */
28551a8031e1Shsuenaga iv_out[0] = htobe32(sha1.state[0]);
28561a8031e1Shsuenaga iv_out[1] = htobe32(sha1.state[1]);
28571a8031e1Shsuenaga iv_out[2] = htobe32(sha1.state[2]);
28581a8031e1Shsuenaga iv_out[3] = htobe32(sha1.state[3]);
28591a8031e1Shsuenaga iv_out[4] = htobe32(sha1.state[4]);
28601a8031e1Shsuenaga break;
28611a8031e1Shsuenaga case CRYPTO_MD5_HMAC_96:
28621a8031e1Shsuenaga case CRYPTO_MD5_HMAC:
28631a8031e1Shsuenaga MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO,
28641a8031e1Shsuenaga "%s: Generate iv_in(MD5)\n", __func__);
28651a8031e1Shsuenaga MD5Init(&md5);
28661a8031e1Shsuenaga MD5Update(&md5, ipad, sizeof(ipad));
28671a8031e1Shsuenaga /* XXX: private state... (LE) */
28681a8031e1Shsuenaga iv_in[0] = htobe32(md5.state[0]);
28691a8031e1Shsuenaga iv_in[1] = htobe32(md5.state[1]);
28701a8031e1Shsuenaga iv_in[2] = htobe32(md5.state[2]);
28711a8031e1Shsuenaga iv_in[3] = htobe32(md5.state[3]);
28721a8031e1Shsuenaga iv_in[4] = 0;
28731a8031e1Shsuenaga
28741a8031e1Shsuenaga MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO,
28751a8031e1Shsuenaga "%s: Generate iv_out(MD5)\n", __func__);
28761a8031e1Shsuenaga MD5Init(&md5);
28771a8031e1Shsuenaga MD5Update(&md5, opad, sizeof(opad));
28781a8031e1Shsuenaga /* XXX: private state... (LE) */
28791a8031e1Shsuenaga iv_out[0] = htobe32(md5.state[0]);
28801a8031e1Shsuenaga iv_out[1] = htobe32(md5.state[1]);
28811a8031e1Shsuenaga iv_out[2] = htobe32(md5.state[2]);
28821a8031e1Shsuenaga iv_out[3] = htobe32(md5.state[3]);
28831a8031e1Shsuenaga iv_out[4] = 0;
28841a8031e1Shsuenaga break;
28851a8031e1Shsuenaga default:
28861a8031e1Shsuenaga break;
28871a8031e1Shsuenaga }
28881a8031e1Shsuenaga
28891a8031e1Shsuenaga #ifdef MVXPSEC_DEBUG
28901a8031e1Shsuenaga if (mvxpsec_debug & MVXPSEC_DEBUG_HASH_IV) {
28911a8031e1Shsuenaga MVXPSEC_PRINTF(MVXPSEC_DEBUG_HASH_IV,
28921a8031e1Shsuenaga "%s: HMAC IV-IN\n", __func__);
28931a8031e1Shsuenaga mvxpsec_dump_data(__func__, (uint8_t *)iv_in, 20);
28941a8031e1Shsuenaga MVXPSEC_PRINTF(MVXPSEC_DEBUG_HASH_IV,
28951a8031e1Shsuenaga "%s: HMAC IV-OUT\n", __func__);
28961a8031e1Shsuenaga mvxpsec_dump_data(__func__, (uint8_t *)iv_out, 20);
28971a8031e1Shsuenaga }
28981a8031e1Shsuenaga #endif
28991a8031e1Shsuenaga
29001a8031e1Shsuenaga return 0;
29011a8031e1Shsuenaga #undef HMAC_IPAD
29021a8031e1Shsuenaga #undef HMAC_OPAD
29031a8031e1Shsuenaga }
29041a8031e1Shsuenaga
29051a8031e1Shsuenaga /*
29061a8031e1Shsuenaga * AES Support routine
29071a8031e1Shsuenaga */
29081a8031e1Shsuenaga static uint8_t AES_SBOX[256] = {
29091a8031e1Shsuenaga 99, 124, 119, 123, 242, 107, 111, 197, 48, 1, 103, 43, 254, 215,
29101a8031e1Shsuenaga 171, 118, 202, 130, 201, 125, 250, 89, 71, 240, 173, 212, 162, 175,
29111a8031e1Shsuenaga 156, 164, 114, 192, 183, 253, 147, 38, 54, 63, 247, 204, 52, 165,
29121a8031e1Shsuenaga 229, 241, 113, 216, 49, 21, 4, 199, 35, 195, 24, 150, 5, 154,
29131a8031e1Shsuenaga 7, 18, 128, 226, 235, 39, 178, 117, 9, 131, 44, 26, 27, 110,
29141a8031e1Shsuenaga 90, 160, 82, 59, 214, 179, 41, 227, 47, 132, 83, 209, 0, 237,
29151a8031e1Shsuenaga 32, 252, 177, 91, 106, 203, 190, 57, 74, 76, 88, 207, 208, 239,
29161a8031e1Shsuenaga 170, 251, 67, 77, 51, 133, 69, 249, 2, 127, 80, 60, 159, 168,
29171a8031e1Shsuenaga 81, 163, 64, 143, 146, 157, 56, 245, 188, 182, 218, 33, 16, 255,
29181a8031e1Shsuenaga 243, 210, 205, 12, 19, 236, 95, 151, 68, 23, 196, 167, 126, 61,
29191a8031e1Shsuenaga 100, 93, 25, 115, 96, 129, 79, 220, 34, 42, 144, 136, 70, 238,
29201a8031e1Shsuenaga 184, 20, 222, 94, 11, 219, 224, 50, 58, 10, 73, 6, 36, 92,
29211a8031e1Shsuenaga 194, 211, 172, 98, 145, 149, 228, 121, 231, 200, 55, 109, 141, 213,
29221a8031e1Shsuenaga 78, 169, 108, 86, 244, 234, 101, 122, 174, 8, 186, 120, 37, 46,
29231a8031e1Shsuenaga 28, 166, 180, 198, 232, 221, 116, 31, 75, 189, 139, 138, 112, 62,
29241a8031e1Shsuenaga 181, 102, 72, 3, 246, 14, 97, 53, 87, 185, 134, 193, 29, 158,
29251a8031e1Shsuenaga 225, 248, 152, 17, 105, 217, 142, 148, 155, 30, 135, 233, 206, 85,
29261a8031e1Shsuenaga 40, 223, 140, 161, 137, 13, 191, 230, 66, 104, 65, 153, 45, 15,
29271a8031e1Shsuenaga 176, 84, 187, 22
29281a8031e1Shsuenaga };
29291a8031e1Shsuenaga
29301a8031e1Shsuenaga static uint32_t AES_RCON[30] = {
29311a8031e1Shsuenaga 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36, 0x6c, 0xd8,
29321a8031e1Shsuenaga 0xab, 0x4d, 0x9a, 0x2f, 0x5e, 0xbc, 0x63, 0xc6, 0x97, 0x35, 0x6a, 0xd4,
29331a8031e1Shsuenaga 0xb3, 0x7d, 0xfa, 0xef, 0xc5, 0x91
29341a8031e1Shsuenaga };
29351a8031e1Shsuenaga
29361a8031e1Shsuenaga STATIC int
mv_aes_ksched(uint8_t k[4][MAXKC],int keyBits,uint8_t W[MAXROUNDS+1][4][MAXBC])29371a8031e1Shsuenaga mv_aes_ksched(uint8_t k[4][MAXKC], int keyBits,
29381a8031e1Shsuenaga uint8_t W[MAXROUNDS+1][4][MAXBC])
29391a8031e1Shsuenaga {
29401a8031e1Shsuenaga int KC, BC, ROUNDS;
29411a8031e1Shsuenaga int i, j, t, rconpointer = 0;
29421a8031e1Shsuenaga uint8_t tk[4][MAXKC];
29431a8031e1Shsuenaga
29441a8031e1Shsuenaga switch (keyBits) {
29451a8031e1Shsuenaga case 128:
29461a8031e1Shsuenaga ROUNDS = 10;
29471a8031e1Shsuenaga KC = 4;
29481a8031e1Shsuenaga break;
29491a8031e1Shsuenaga case 192:
29501a8031e1Shsuenaga ROUNDS = 12;
29511a8031e1Shsuenaga KC = 6;
29521a8031e1Shsuenaga break;
29531a8031e1Shsuenaga case 256:
29541a8031e1Shsuenaga ROUNDS = 14;
29551a8031e1Shsuenaga KC = 8;
29561a8031e1Shsuenaga break;
29571a8031e1Shsuenaga default:
29581a8031e1Shsuenaga return (-1);
29591a8031e1Shsuenaga }
29601a8031e1Shsuenaga BC = 4; /* 128 bits */
29611a8031e1Shsuenaga
29621a8031e1Shsuenaga for(j = 0; j < KC; j++)
29631a8031e1Shsuenaga for(i = 0; i < 4; i++)
29641a8031e1Shsuenaga tk[i][j] = k[i][j];
29651a8031e1Shsuenaga t = 0;
29661a8031e1Shsuenaga
29671a8031e1Shsuenaga /* copy values into round key array */
29681a8031e1Shsuenaga for(j = 0; (j < KC) && (t < (ROUNDS+1)*BC); j++, t++)
29691a8031e1Shsuenaga for(i = 0; i < 4; i++) W[t / BC][i][t % BC] = tk[i][j];
29701a8031e1Shsuenaga
29711a8031e1Shsuenaga while (t < (ROUNDS+1)*BC) { /* while not enough round key material calculated */
29721a8031e1Shsuenaga /* calculate new values */
29731a8031e1Shsuenaga for(i = 0; i < 4; i++)
29741a8031e1Shsuenaga tk[i][0] ^= AES_SBOX[tk[(i+1)%4][KC-1]];
29751a8031e1Shsuenaga tk[0][0] ^= AES_RCON[rconpointer++];
29761a8031e1Shsuenaga
29771a8031e1Shsuenaga if (KC != 8)
29781a8031e1Shsuenaga for(j = 1; j < KC; j++)
29791a8031e1Shsuenaga for(i = 0; i < 4; i++)
29801a8031e1Shsuenaga tk[i][j] ^= tk[i][j-1];
29811a8031e1Shsuenaga else {
29821a8031e1Shsuenaga for(j = 1; j < KC/2; j++)
29831a8031e1Shsuenaga for(i = 0; i < 4; i++)
29841a8031e1Shsuenaga tk[i][j] ^= tk[i][j-1];
29851a8031e1Shsuenaga for(i = 0; i < 4; i++)
29861a8031e1Shsuenaga tk[i][KC/2] ^= AES_SBOX[tk[i][KC/2 - 1]];
29871a8031e1Shsuenaga for(j = KC/2 + 1; j < KC; j++)
29881a8031e1Shsuenaga for(i = 0; i < 4; i++)
29891a8031e1Shsuenaga tk[i][j] ^= tk[i][j-1];
29901a8031e1Shsuenaga }
29911a8031e1Shsuenaga /* copy values into round key array */
29921a8031e1Shsuenaga for(j = 0; (j < KC) && (t < (ROUNDS+1)*BC); j++, t++)
29931a8031e1Shsuenaga for(i = 0; i < 4; i++) W[t / BC][i][t % BC] = tk[i][j];
29941a8031e1Shsuenaga }
29951a8031e1Shsuenaga
29961a8031e1Shsuenaga return 0;
29971a8031e1Shsuenaga }
29981a8031e1Shsuenaga
29991a8031e1Shsuenaga STATIC int
mv_aes_deckey(uint8_t * expandedKey,uint8_t * keyMaterial,int keyLen)30001a8031e1Shsuenaga mv_aes_deckey(uint8_t *expandedKey, uint8_t *keyMaterial, int keyLen)
30011a8031e1Shsuenaga {
30021a8031e1Shsuenaga uint8_t W[MAXROUNDS+1][4][MAXBC];
30031a8031e1Shsuenaga uint8_t k[4][MAXKC];
30041a8031e1Shsuenaga uint8_t j;
30051a8031e1Shsuenaga int i, rounds, KC;
30061a8031e1Shsuenaga
30071a8031e1Shsuenaga if (expandedKey == NULL)
30081a8031e1Shsuenaga return -1;
30091a8031e1Shsuenaga
30101a8031e1Shsuenaga if (!((keyLen == 128) || (keyLen == 192) || (keyLen == 256)))
30111a8031e1Shsuenaga return -1;
30121a8031e1Shsuenaga
30131a8031e1Shsuenaga if (keyMaterial == NULL)
30141a8031e1Shsuenaga return -1;
30151a8031e1Shsuenaga
30161a8031e1Shsuenaga /* initialize key schedule: */
30171a8031e1Shsuenaga for (i=0; i<keyLen/8; i++) {
30181a8031e1Shsuenaga j = keyMaterial[i];
30191a8031e1Shsuenaga k[i % 4][i / 4] = j;
30201a8031e1Shsuenaga }
30211a8031e1Shsuenaga
30221a8031e1Shsuenaga mv_aes_ksched(k, keyLen, W);
30231a8031e1Shsuenaga switch (keyLen) {
30241a8031e1Shsuenaga case 128:
30251a8031e1Shsuenaga rounds = 10;
30261a8031e1Shsuenaga KC = 4;
30271a8031e1Shsuenaga break;
30281a8031e1Shsuenaga case 192:
30291a8031e1Shsuenaga rounds = 12;
30301a8031e1Shsuenaga KC = 6;
30311a8031e1Shsuenaga break;
30321a8031e1Shsuenaga case 256:
30331a8031e1Shsuenaga rounds = 14;
30341a8031e1Shsuenaga KC = 8;
30351a8031e1Shsuenaga break;
30361a8031e1Shsuenaga default:
30371a8031e1Shsuenaga return -1;
30381a8031e1Shsuenaga }
30391a8031e1Shsuenaga
30401a8031e1Shsuenaga for(i=0; i<MAXBC; i++)
30411a8031e1Shsuenaga for(j=0; j<4; j++)
30421a8031e1Shsuenaga expandedKey[i*4+j] = W[rounds][j][i];
30431a8031e1Shsuenaga for(; i<KC; i++)
30441a8031e1Shsuenaga for(j=0; j<4; j++)
30451a8031e1Shsuenaga expandedKey[i*4+j] = W[rounds-1][j][i+MAXBC-KC];
30461a8031e1Shsuenaga
30471a8031e1Shsuenaga return 0;
30481a8031e1Shsuenaga }
30491a8031e1Shsuenaga
30501a8031e1Shsuenaga /*
30511a8031e1Shsuenaga * Clear cipher/mac operation state
30521a8031e1Shsuenaga */
30531a8031e1Shsuenaga INLINE void
mvxpsec_packet_reset_op(struct mvxpsec_packet * mv_p)30541a8031e1Shsuenaga mvxpsec_packet_reset_op(struct mvxpsec_packet *mv_p)
30551a8031e1Shsuenaga {
30561a8031e1Shsuenaga mv_p->pkt_header.desc.acc_config = 0;
30571a8031e1Shsuenaga mv_p->enc_off = mv_p->enc_ivoff = mv_p->enc_len = 0;
30581a8031e1Shsuenaga mv_p->mac_off = mv_p->mac_dst = mv_p->mac_len = 0;
30591a8031e1Shsuenaga }
30601a8031e1Shsuenaga
30611a8031e1Shsuenaga /*
30621a8031e1Shsuenaga * update MVXPSEC operation order
30631a8031e1Shsuenaga */
30641a8031e1Shsuenaga INLINE void
mvxpsec_packet_update_op_order(struct mvxpsec_packet * mv_p,int op)30651a8031e1Shsuenaga mvxpsec_packet_update_op_order(struct mvxpsec_packet *mv_p, int op)
30661a8031e1Shsuenaga {
30671a8031e1Shsuenaga struct mvxpsec_acc_descriptor *acc_desc = &mv_p->pkt_header.desc;
30681a8031e1Shsuenaga uint32_t cur_op = acc_desc->acc_config & MV_ACC_CRYPTO_OP_MASK;
30691a8031e1Shsuenaga
30701a8031e1Shsuenaga KASSERT(op == MV_ACC_CRYPTO_OP_MAC || op == MV_ACC_CRYPTO_OP_ENC);
30711a8031e1Shsuenaga KASSERT((op & MV_ACC_CRYPTO_OP_MASK) == op);
30721a8031e1Shsuenaga
30731a8031e1Shsuenaga if (cur_op == 0)
30741a8031e1Shsuenaga acc_desc->acc_config |= op;
30751a8031e1Shsuenaga else if (cur_op == MV_ACC_CRYPTO_OP_MAC && op == MV_ACC_CRYPTO_OP_ENC) {
30761a8031e1Shsuenaga acc_desc->acc_config &= ~MV_ACC_CRYPTO_OP_MASK;
30771a8031e1Shsuenaga acc_desc->acc_config |= MV_ACC_CRYPTO_OP_MACENC;
30781a8031e1Shsuenaga /* MAC then ENC (= decryption) */
30791a8031e1Shsuenaga }
30801a8031e1Shsuenaga else if (cur_op == MV_ACC_CRYPTO_OP_ENC && op == MV_ACC_CRYPTO_OP_MAC) {
30811a8031e1Shsuenaga acc_desc->acc_config &= ~MV_ACC_CRYPTO_OP_MASK;
30821a8031e1Shsuenaga acc_desc->acc_config |= MV_ACC_CRYPTO_OP_ENCMAC;
30831a8031e1Shsuenaga /* ENC then MAC (= encryption) */
30841a8031e1Shsuenaga }
30851a8031e1Shsuenaga else {
30861a8031e1Shsuenaga log(LOG_ERR, "%s: multiple %s algorithm is not supported.\n",
30871a8031e1Shsuenaga __func__,
30881a8031e1Shsuenaga (op == MV_ACC_CRYPTO_OP_ENC) ? "encryption" : "authentication");
30891a8031e1Shsuenaga }
30901a8031e1Shsuenaga }
30911a8031e1Shsuenaga
30921a8031e1Shsuenaga /*
30931a8031e1Shsuenaga * Parameter Conversions
30941a8031e1Shsuenaga */
30951a8031e1Shsuenaga INLINE uint32_t
mvxpsec_alg2acc(uint32_t alg)30961a8031e1Shsuenaga mvxpsec_alg2acc(uint32_t alg)
30971a8031e1Shsuenaga {
30981a8031e1Shsuenaga uint32_t reg;
30991a8031e1Shsuenaga
31001a8031e1Shsuenaga switch (alg) {
31011a8031e1Shsuenaga case CRYPTO_DES_CBC:
31021a8031e1Shsuenaga reg = MV_ACC_CRYPTO_ENC_DES;
31031a8031e1Shsuenaga reg |= MV_ACC_CRYPTO_CBC;
31041a8031e1Shsuenaga break;
31051a8031e1Shsuenaga case CRYPTO_3DES_CBC:
31061a8031e1Shsuenaga reg = MV_ACC_CRYPTO_ENC_3DES;
31071a8031e1Shsuenaga reg |= MV_ACC_CRYPTO_3DES_EDE;
31081a8031e1Shsuenaga reg |= MV_ACC_CRYPTO_CBC;
31091a8031e1Shsuenaga break;
31101a8031e1Shsuenaga case CRYPTO_AES_CBC:
31111a8031e1Shsuenaga reg = MV_ACC_CRYPTO_ENC_AES;
31121a8031e1Shsuenaga reg |= MV_ACC_CRYPTO_CBC;
31131a8031e1Shsuenaga break;
31141a8031e1Shsuenaga case CRYPTO_SHA1_HMAC_96:
31151a8031e1Shsuenaga reg = MV_ACC_CRYPTO_MAC_HMAC_SHA1;
31161a8031e1Shsuenaga reg |= MV_ACC_CRYPTO_MAC_96;
31171a8031e1Shsuenaga break;
31181a8031e1Shsuenaga case CRYPTO_MD5_HMAC_96:
31191a8031e1Shsuenaga reg = MV_ACC_CRYPTO_MAC_HMAC_MD5;
31201a8031e1Shsuenaga reg |= MV_ACC_CRYPTO_MAC_96;
31211a8031e1Shsuenaga break;
31221a8031e1Shsuenaga default:
31231a8031e1Shsuenaga reg = 0;
31241a8031e1Shsuenaga break;
31251a8031e1Shsuenaga }
31261a8031e1Shsuenaga
31271a8031e1Shsuenaga return reg;
31281a8031e1Shsuenaga }
31291a8031e1Shsuenaga
31301a8031e1Shsuenaga INLINE uint32_t
mvxpsec_aesklen(int klen)31311a8031e1Shsuenaga mvxpsec_aesklen(int klen)
31321a8031e1Shsuenaga {
31331a8031e1Shsuenaga if (klen < 128)
31341a8031e1Shsuenaga return 0;
31351a8031e1Shsuenaga else if (klen < 192)
31361a8031e1Shsuenaga return MV_ACC_CRYPTO_AES_KLEN_128;
31371a8031e1Shsuenaga else if (klen < 256)
31381a8031e1Shsuenaga return MV_ACC_CRYPTO_AES_KLEN_192;
31391a8031e1Shsuenaga else
31401a8031e1Shsuenaga return MV_ACC_CRYPTO_AES_KLEN_256;
31411a8031e1Shsuenaga
31421a8031e1Shsuenaga return 0;
31431a8031e1Shsuenaga }
31441a8031e1Shsuenaga
31451a8031e1Shsuenaga /*
31461a8031e1Shsuenaga * String Conversions
31471a8031e1Shsuenaga */
31481a8031e1Shsuenaga STATIC const char *
s_errreg(uint32_t v)31491a8031e1Shsuenaga s_errreg(uint32_t v)
31501a8031e1Shsuenaga {
31511a8031e1Shsuenaga static char buf[80];
31521a8031e1Shsuenaga
31531a8031e1Shsuenaga snprintf(buf, sizeof(buf),
31541a8031e1Shsuenaga "%sMiss %sDoubleHit %sBothHit %sDataError",
31551a8031e1Shsuenaga (v & MV_TDMA_ERRC_MISS) ? "+" : "-",
31561a8031e1Shsuenaga (v & MV_TDMA_ERRC_DHIT) ? "+" : "-",
31571a8031e1Shsuenaga (v & MV_TDMA_ERRC_BHIT) ? "+" : "-",
31581a8031e1Shsuenaga (v & MV_TDMA_ERRC_DERR) ? "+" : "-");
31591a8031e1Shsuenaga
31601a8031e1Shsuenaga return (const char *)buf;
31611a8031e1Shsuenaga }
31621a8031e1Shsuenaga
31631a8031e1Shsuenaga STATIC const char *
s_winreg(uint32_t v)31641a8031e1Shsuenaga s_winreg(uint32_t v)
31651a8031e1Shsuenaga {
31661a8031e1Shsuenaga static char buf[80];
31671a8031e1Shsuenaga
31681a8031e1Shsuenaga snprintf(buf, sizeof(buf),
31691a8031e1Shsuenaga "%s TGT 0x%x ATTR 0x%02x size %u(0x%04x)[64KB]",
31701a8031e1Shsuenaga (v & MV_TDMA_ATTR_ENABLE) ? "EN" : "DIS",
31711a8031e1Shsuenaga MV_TDMA_ATTR_GET_TARGET(v), MV_TDMA_ATTR_GET_ATTR(v),
31721a8031e1Shsuenaga MV_TDMA_ATTR_GET_SIZE(v), MV_TDMA_ATTR_GET_SIZE(v));
31731a8031e1Shsuenaga
31741a8031e1Shsuenaga return (const char *)buf;
31751a8031e1Shsuenaga }
31761a8031e1Shsuenaga
31771a8031e1Shsuenaga STATIC const char *
s_ctrlreg(uint32_t reg)31781a8031e1Shsuenaga s_ctrlreg(uint32_t reg)
31791a8031e1Shsuenaga {
31801a8031e1Shsuenaga static char buf[80];
31811a8031e1Shsuenaga
31821a8031e1Shsuenaga snprintf(buf, sizeof(buf),
31831a8031e1Shsuenaga "%s: %sFETCH DBURST-%u SBURST-%u %sOUTS %sCHAIN %sBSWAP %sACT",
31841a8031e1Shsuenaga (reg & MV_TDMA_CONTROL_ENABLE) ? "ENABLE" : "DISABLE",
31851a8031e1Shsuenaga (reg & MV_TDMA_CONTROL_FETCH) ? "+" : "-",
31861a8031e1Shsuenaga MV_TDMA_CONTROL_GET_DST_BURST(reg),
31871a8031e1Shsuenaga MV_TDMA_CONTROL_GET_SRC_BURST(reg),
31881a8031e1Shsuenaga (reg & MV_TDMA_CONTROL_OUTS_EN) ? "+" : "-",
31891a8031e1Shsuenaga (reg & MV_TDMA_CONTROL_CHAIN_DIS) ? "-" : "+",
31901a8031e1Shsuenaga (reg & MV_TDMA_CONTROL_BSWAP_DIS) ? "-" : "+",
31911a8031e1Shsuenaga (reg & MV_TDMA_CONTROL_ACT) ? "+" : "-");
31921a8031e1Shsuenaga
31931a8031e1Shsuenaga return (const char *)buf;
31941a8031e1Shsuenaga }
31951a8031e1Shsuenaga
31961a8031e1Shsuenaga _STATIC const char *
s_xpsecintr(uint32_t v)31971a8031e1Shsuenaga s_xpsecintr(uint32_t v)
31981a8031e1Shsuenaga {
31991a8031e1Shsuenaga static char buf[160];
32001a8031e1Shsuenaga
32011a8031e1Shsuenaga snprintf(buf, sizeof(buf),
32021a8031e1Shsuenaga "%sAuth %sDES %sAES-ENC %sAES-DEC %sENC %sSA %sAccAndTDMA "
32031a8031e1Shsuenaga "%sTDMAComp %sTDMAOwn %sAccAndTDMA_Cont",
32041a8031e1Shsuenaga (v & MVXPSEC_INT_AUTH) ? "+" : "-",
32051a8031e1Shsuenaga (v & MVXPSEC_INT_DES) ? "+" : "-",
32061a8031e1Shsuenaga (v & MVXPSEC_INT_AES_ENC) ? "+" : "-",
32071a8031e1Shsuenaga (v & MVXPSEC_INT_AES_DEC) ? "+" : "-",
32081a8031e1Shsuenaga (v & MVXPSEC_INT_ENC) ? "+" : "-",
32091a8031e1Shsuenaga (v & MVXPSEC_INT_SA) ? "+" : "-",
32101a8031e1Shsuenaga (v & MVXPSEC_INT_ACCTDMA) ? "+" : "-",
32111a8031e1Shsuenaga (v & MVXPSEC_INT_TDMA_COMP) ? "+" : "-",
32121a8031e1Shsuenaga (v & MVXPSEC_INT_TDMA_OWN) ? "+" : "-",
32131a8031e1Shsuenaga (v & MVXPSEC_INT_ACCTDMA_CONT) ? "+" : "-");
32141a8031e1Shsuenaga
32151a8031e1Shsuenaga return (const char *)buf;
32161a8031e1Shsuenaga }
32171a8031e1Shsuenaga
32181a8031e1Shsuenaga STATIC const char *
s_ctlalg(uint32_t alg)32191a8031e1Shsuenaga s_ctlalg(uint32_t alg)
32201a8031e1Shsuenaga {
32211a8031e1Shsuenaga switch (alg) {
32221a8031e1Shsuenaga case CRYPTO_SHA1_HMAC_96:
32231a8031e1Shsuenaga return "HMAC-SHA1-96";
32241a8031e1Shsuenaga case CRYPTO_SHA1_HMAC:
32251a8031e1Shsuenaga return "HMAC-SHA1";
32261a8031e1Shsuenaga case CRYPTO_SHA1:
32271a8031e1Shsuenaga return "SHA1";
32281a8031e1Shsuenaga case CRYPTO_MD5_HMAC_96:
32291a8031e1Shsuenaga return "HMAC-MD5-96";
32301a8031e1Shsuenaga case CRYPTO_MD5_HMAC:
32311a8031e1Shsuenaga return "HMAC-MD5";
32321a8031e1Shsuenaga case CRYPTO_MD5:
32331a8031e1Shsuenaga return "MD5";
32341a8031e1Shsuenaga case CRYPTO_DES_CBC:
32351a8031e1Shsuenaga return "DES-CBC";
32361a8031e1Shsuenaga case CRYPTO_3DES_CBC:
32371a8031e1Shsuenaga return "3DES-CBC";
32381a8031e1Shsuenaga case CRYPTO_AES_CBC:
32391a8031e1Shsuenaga return "AES-CBC";
32401a8031e1Shsuenaga default:
32411a8031e1Shsuenaga break;
32421a8031e1Shsuenaga }
32431a8031e1Shsuenaga
32441a8031e1Shsuenaga return "Unknown";
32451a8031e1Shsuenaga }
32461a8031e1Shsuenaga
32471a8031e1Shsuenaga STATIC const char *
s_xpsec_op(uint32_t reg)32481a8031e1Shsuenaga s_xpsec_op(uint32_t reg)
32491a8031e1Shsuenaga {
32501a8031e1Shsuenaga reg &= MV_ACC_CRYPTO_OP_MASK;
32511a8031e1Shsuenaga switch (reg) {
32521a8031e1Shsuenaga case MV_ACC_CRYPTO_OP_ENC:
32531a8031e1Shsuenaga return "ENC";
32541a8031e1Shsuenaga case MV_ACC_CRYPTO_OP_MAC:
32551a8031e1Shsuenaga return "MAC";
32561a8031e1Shsuenaga case MV_ACC_CRYPTO_OP_ENCMAC:
32571a8031e1Shsuenaga return "ENC-MAC";
32581a8031e1Shsuenaga case MV_ACC_CRYPTO_OP_MACENC:
32591a8031e1Shsuenaga return "MAC-ENC";
32601a8031e1Shsuenaga default:
32611a8031e1Shsuenaga break;
32621a8031e1Shsuenaga }
32631a8031e1Shsuenaga
32641a8031e1Shsuenaga return "Unknown";
32651a8031e1Shsuenaga }
32661a8031e1Shsuenaga
32671a8031e1Shsuenaga STATIC const char *
s_xpsec_enc(uint32_t alg)32681a8031e1Shsuenaga s_xpsec_enc(uint32_t alg)
32691a8031e1Shsuenaga {
32701a8031e1Shsuenaga alg <<= MV_ACC_CRYPTO_ENC_SHIFT;
32711a8031e1Shsuenaga switch (alg) {
32721a8031e1Shsuenaga case MV_ACC_CRYPTO_ENC_DES:
32731a8031e1Shsuenaga return "DES";
32741a8031e1Shsuenaga case MV_ACC_CRYPTO_ENC_3DES:
32751a8031e1Shsuenaga return "3DES";
32761a8031e1Shsuenaga case MV_ACC_CRYPTO_ENC_AES:
32771a8031e1Shsuenaga return "AES";
32781a8031e1Shsuenaga default:
32791a8031e1Shsuenaga break;
32801a8031e1Shsuenaga }
32811a8031e1Shsuenaga
32821a8031e1Shsuenaga return "Unknown";
32831a8031e1Shsuenaga }
32841a8031e1Shsuenaga
32851a8031e1Shsuenaga STATIC const char *
s_xpsec_mac(uint32_t alg)32861a8031e1Shsuenaga s_xpsec_mac(uint32_t alg)
32871a8031e1Shsuenaga {
32881a8031e1Shsuenaga alg <<= MV_ACC_CRYPTO_MAC_SHIFT;
32891a8031e1Shsuenaga switch (alg) {
32901a8031e1Shsuenaga case MV_ACC_CRYPTO_MAC_NONE:
32911a8031e1Shsuenaga return "Disabled";
32921a8031e1Shsuenaga case MV_ACC_CRYPTO_MAC_MD5:
32931a8031e1Shsuenaga return "MD5";
32941a8031e1Shsuenaga case MV_ACC_CRYPTO_MAC_SHA1:
32951a8031e1Shsuenaga return "SHA1";
32961a8031e1Shsuenaga case MV_ACC_CRYPTO_MAC_HMAC_MD5:
32971a8031e1Shsuenaga return "HMAC-MD5";
32981a8031e1Shsuenaga case MV_ACC_CRYPTO_MAC_HMAC_SHA1:
32991a8031e1Shsuenaga return "HMAC-SHA1";
33001a8031e1Shsuenaga default:
33011a8031e1Shsuenaga break;
33021a8031e1Shsuenaga }
33031a8031e1Shsuenaga
33041a8031e1Shsuenaga return "Unknown";
33051a8031e1Shsuenaga }
33061a8031e1Shsuenaga
33071a8031e1Shsuenaga STATIC const char *
s_xpsec_frag(uint32_t frag)33081a8031e1Shsuenaga s_xpsec_frag(uint32_t frag)
33091a8031e1Shsuenaga {
33101a8031e1Shsuenaga frag <<= MV_ACC_CRYPTO_FRAG_SHIFT;
33111a8031e1Shsuenaga switch (frag) {
33121a8031e1Shsuenaga case MV_ACC_CRYPTO_NOFRAG:
33131a8031e1Shsuenaga return "NoFragment";
33141a8031e1Shsuenaga case MV_ACC_CRYPTO_FRAG_FIRST:
33151a8031e1Shsuenaga return "FirstFragment";
33161a8031e1Shsuenaga case MV_ACC_CRYPTO_FRAG_MID:
33171a8031e1Shsuenaga return "MiddleFragment";
33181a8031e1Shsuenaga case MV_ACC_CRYPTO_FRAG_LAST:
33191a8031e1Shsuenaga return "LastFragment";
33201a8031e1Shsuenaga default:
33211a8031e1Shsuenaga break;
33221a8031e1Shsuenaga }
33231a8031e1Shsuenaga
33241a8031e1Shsuenaga return "Unknown";
33251a8031e1Shsuenaga }
33261a8031e1Shsuenaga
33271a8031e1Shsuenaga #ifdef MVXPSEC_DEBUG
33281a8031e1Shsuenaga void
mvxpsec_dump_reg(struct mvxpsec_softc * sc)33291a8031e1Shsuenaga mvxpsec_dump_reg(struct mvxpsec_softc *sc)
33301a8031e1Shsuenaga {
33311a8031e1Shsuenaga uint32_t reg;
33321a8031e1Shsuenaga int i;
33331a8031e1Shsuenaga
33341a8031e1Shsuenaga if ((mvxpsec_debug & MVXPSEC_DEBUG_DESC) == 0)
33351a8031e1Shsuenaga return;
33361a8031e1Shsuenaga
33371a8031e1Shsuenaga printf("--- Interrupt Registers ---\n");
33381a8031e1Shsuenaga reg = MVXPSEC_READ(sc, MVXPSEC_INT_CAUSE);
33391a8031e1Shsuenaga printf("MVXPSEC INT CAUSE: 0x%08x\n", reg);
33401a8031e1Shsuenaga printf("MVXPSEC INT CAUSE: %s\n", s_xpsecintr(reg));
33411a8031e1Shsuenaga reg = MVXPSEC_READ(sc, MVXPSEC_INT_MASK);
33421a8031e1Shsuenaga printf("MVXPSEC INT MASK: 0x%08x\n", reg);
33431a8031e1Shsuenaga printf("MVXPSEC INT MASKE: %s\n", s_xpsecintr(reg));
33441a8031e1Shsuenaga
33451a8031e1Shsuenaga printf("--- DMA Configuration Registers ---\n");
33461a8031e1Shsuenaga for (i = 0; i < MV_TDMA_NWINDOW; i++) {
33471a8031e1Shsuenaga reg = MVXPSEC_READ(sc, MV_TDMA_BAR(i));
33481a8031e1Shsuenaga printf("TDMA BAR%d: 0x%08x\n", i, reg);
33491a8031e1Shsuenaga reg = MVXPSEC_READ(sc, MV_TDMA_ATTR(i));
33501a8031e1Shsuenaga printf("TDMA ATTR%d: 0x%08x\n", i, reg);
33511a8031e1Shsuenaga printf(" -> %s\n", s_winreg(reg));
33521a8031e1Shsuenaga }
33531a8031e1Shsuenaga
33541a8031e1Shsuenaga printf("--- DMA Control Registers ---\n");
33551a8031e1Shsuenaga
33561a8031e1Shsuenaga reg = MVXPSEC_READ(sc, MV_TDMA_CONTROL);
33571a8031e1Shsuenaga printf("TDMA CONTROL: 0x%08x\n", reg);
33581a8031e1Shsuenaga printf(" -> %s\n", s_ctrlreg(reg));
33591a8031e1Shsuenaga
33601a8031e1Shsuenaga printf("--- DMA Current Command Descriptors ---\n");
33611a8031e1Shsuenaga
33621a8031e1Shsuenaga reg = MVXPSEC_READ(sc, MV_TDMA_ERR_CAUSE);
33631a8031e1Shsuenaga printf("TDMA ERR CAUSE: 0x%08x\n", reg);
33641a8031e1Shsuenaga
33651a8031e1Shsuenaga reg = MVXPSEC_READ(sc, MV_TDMA_ERR_MASK);
33661a8031e1Shsuenaga printf("TDMA ERR MASK: 0x%08x\n", reg);
33671a8031e1Shsuenaga
33681a8031e1Shsuenaga reg = MVXPSEC_READ(sc, MV_TDMA_CNT);
33691a8031e1Shsuenaga printf("TDMA DATA OWNER: %s\n",
33701a8031e1Shsuenaga (reg & MV_TDMA_CNT_OWN) ? "DMAC" : "CPU");
33711a8031e1Shsuenaga printf("TDMA DATA COUNT: %d(0x%x)\n",
33721a8031e1Shsuenaga (reg & ~MV_TDMA_CNT_OWN), (reg & ~MV_TDMA_CNT_OWN));
33731a8031e1Shsuenaga
33741a8031e1Shsuenaga reg = MVXPSEC_READ(sc, MV_TDMA_SRC);
33751a8031e1Shsuenaga printf("TDMA DATA SRC: 0x%08x\n", reg);
33761a8031e1Shsuenaga
33771a8031e1Shsuenaga reg = MVXPSEC_READ(sc, MV_TDMA_DST);
33781a8031e1Shsuenaga printf("TDMA DATA DST: 0x%08x\n", reg);
33791a8031e1Shsuenaga
33801a8031e1Shsuenaga reg = MVXPSEC_READ(sc, MV_TDMA_NXT);
33811a8031e1Shsuenaga printf("TDMA DATA NXT: 0x%08x\n", reg);
33821a8031e1Shsuenaga
33831a8031e1Shsuenaga reg = MVXPSEC_READ(sc, MV_TDMA_CUR);
33841a8031e1Shsuenaga printf("TDMA DATA CUR: 0x%08x\n", reg);
33851a8031e1Shsuenaga
33861a8031e1Shsuenaga printf("--- ACC Command Register ---\n");
33871a8031e1Shsuenaga reg = MVXPSEC_READ(sc, MV_ACC_COMMAND);
33881a8031e1Shsuenaga printf("ACC COMMAND: 0x%08x\n", reg);
33891a8031e1Shsuenaga printf("ACC: %sACT %sSTOP\n",
33901a8031e1Shsuenaga (reg & MV_ACC_COMMAND_ACT) ? "+" : "-",
33911a8031e1Shsuenaga (reg & MV_ACC_COMMAND_STOP) ? "+" : "-");
33921a8031e1Shsuenaga
33931a8031e1Shsuenaga reg = MVXPSEC_READ(sc, MV_ACC_CONFIG);
33941a8031e1Shsuenaga printf("ACC CONFIG: 0x%08x\n", reg);
33951a8031e1Shsuenaga reg = MVXPSEC_READ(sc, MV_ACC_DESC);
33961a8031e1Shsuenaga printf("ACC DESC: 0x%08x\n", reg);
33971a8031e1Shsuenaga
33981a8031e1Shsuenaga printf("--- DES Key Register ---\n");
33991a8031e1Shsuenaga reg = MVXPSEC_READ(sc, MV_CE_DES_KEY0L);
34001a8031e1Shsuenaga printf("DES KEY0 Low: 0x%08x\n", reg);
34011a8031e1Shsuenaga reg = MVXPSEC_READ(sc, MV_CE_DES_KEY0H);
34021a8031e1Shsuenaga printf("DES KEY0 High: 0x%08x\n", reg);
34031a8031e1Shsuenaga reg = MVXPSEC_READ(sc, MV_CE_DES_KEY1L);
34041a8031e1Shsuenaga printf("DES KEY1 Low: 0x%08x\n", reg);
34051a8031e1Shsuenaga reg = MVXPSEC_READ(sc, MV_CE_DES_KEY1H);
34061a8031e1Shsuenaga printf("DES KEY1 High: 0x%08x\n", reg);
34071a8031e1Shsuenaga reg = MVXPSEC_READ(sc, MV_CE_DES_KEY2L);
34081a8031e1Shsuenaga printf("DES KEY2 Low: 0x%08x\n", reg);
34091a8031e1Shsuenaga reg = MVXPSEC_READ(sc, MV_CE_DES_KEY2H);
34101a8031e1Shsuenaga printf("DES KEY2 High: 0x%08x\n", reg);
34111a8031e1Shsuenaga
34121a8031e1Shsuenaga printf("--- AES Key Register ---\n");
34131a8031e1Shsuenaga for (i = 0; i < 8; i++) {
34141a8031e1Shsuenaga reg = MVXPSEC_READ(sc, MV_CE_AES_EKEY(i));
34151a8031e1Shsuenaga printf("AES ENC KEY COL%d: %08x\n", i, reg);
34161a8031e1Shsuenaga }
34171a8031e1Shsuenaga for (i = 0; i < 8; i++) {
34181a8031e1Shsuenaga reg = MVXPSEC_READ(sc, MV_CE_AES_DKEY(i));
34191a8031e1Shsuenaga printf("AES DEC KEY COL%d: %08x\n", i, reg);
34201a8031e1Shsuenaga }
34211a8031e1Shsuenaga
34221a8031e1Shsuenaga return;
34231a8031e1Shsuenaga }
34241a8031e1Shsuenaga
34251a8031e1Shsuenaga STATIC void
mvxpsec_dump_sram(const char * name,struct mvxpsec_softc * sc,size_t len)34261a8031e1Shsuenaga mvxpsec_dump_sram(const char *name, struct mvxpsec_softc *sc, size_t len)
34271a8031e1Shsuenaga {
34281a8031e1Shsuenaga uint32_t reg;
34291a8031e1Shsuenaga
34301a8031e1Shsuenaga if (sc->sc_sram_va == NULL)
34311a8031e1Shsuenaga return;
34321a8031e1Shsuenaga
34331a8031e1Shsuenaga if (len == 0) {
34341a8031e1Shsuenaga printf("\n%s NO DATA(len=0)\n", name);
34351a8031e1Shsuenaga return;
34361a8031e1Shsuenaga }
34371a8031e1Shsuenaga else if (len > MV_ACC_SRAM_SIZE)
34381a8031e1Shsuenaga len = MV_ACC_SRAM_SIZE;
34391a8031e1Shsuenaga
34401a8031e1Shsuenaga mutex_enter(&sc->sc_dma_mtx);
34411a8031e1Shsuenaga reg = MVXPSEC_READ(sc, MV_TDMA_CONTROL);
34421a8031e1Shsuenaga if (reg & MV_TDMA_CONTROL_ACT) {
34431a8031e1Shsuenaga printf("TDMA is active, cannot access SRAM\n");
34441a8031e1Shsuenaga mutex_exit(&sc->sc_dma_mtx);
34451a8031e1Shsuenaga return;
34461a8031e1Shsuenaga }
34471a8031e1Shsuenaga reg = MVXPSEC_READ(sc, MV_ACC_COMMAND);
34481a8031e1Shsuenaga if (reg & MV_ACC_COMMAND_ACT) {
34491a8031e1Shsuenaga printf("SA is active, cannot access SRAM\n");
34501a8031e1Shsuenaga mutex_exit(&sc->sc_dma_mtx);
34511a8031e1Shsuenaga return;
34521a8031e1Shsuenaga }
34531a8031e1Shsuenaga
34541a8031e1Shsuenaga printf("%s: dump SRAM, %zu bytes\n", name, len);
34551a8031e1Shsuenaga mvxpsec_dump_data(name, sc->sc_sram_va, len);
34561a8031e1Shsuenaga mutex_exit(&sc->sc_dma_mtx);
34571a8031e1Shsuenaga return;
34581a8031e1Shsuenaga }
34591a8031e1Shsuenaga
34601a8031e1Shsuenaga
34611a8031e1Shsuenaga _STATIC void
mvxpsec_dump_dmaq(struct mvxpsec_descriptor_handle * dh)34621a8031e1Shsuenaga mvxpsec_dump_dmaq(struct mvxpsec_descriptor_handle *dh)
34631a8031e1Shsuenaga {
34641a8031e1Shsuenaga struct mvxpsec_descriptor *d =
34651a8031e1Shsuenaga (struct mvxpsec_descriptor *)dh->_desc;
34661a8031e1Shsuenaga
34671a8031e1Shsuenaga printf("--- DMA Command Descriptor ---\n");
34681a8031e1Shsuenaga printf("DESC: VA=%p PA=0x%08x\n",
34691a8031e1Shsuenaga d, (uint32_t)dh->phys_addr);
34701a8031e1Shsuenaga printf("DESC: WORD0 = 0x%08x\n", d->tdma_word0);
34711a8031e1Shsuenaga printf("DESC: SRC = 0x%08x\n", d->tdma_src);
34721a8031e1Shsuenaga printf("DESC: DST = 0x%08x\n", d->tdma_dst);
34731a8031e1Shsuenaga printf("DESC: NXT = 0x%08x\n", d->tdma_nxt);
34741a8031e1Shsuenaga
34751a8031e1Shsuenaga return;
34761a8031e1Shsuenaga }
34771a8031e1Shsuenaga
34781a8031e1Shsuenaga STATIC void
mvxpsec_dump_data(const char * name,void * p,size_t len)34791a8031e1Shsuenaga mvxpsec_dump_data(const char *name, void *p, size_t len)
34801a8031e1Shsuenaga {
34811a8031e1Shsuenaga uint8_t *data = p;
34821a8031e1Shsuenaga off_t off;
34831a8031e1Shsuenaga
34841a8031e1Shsuenaga printf("%s: dump %p, %zu bytes", name, p, len);
34851a8031e1Shsuenaga if (p == NULL || len == 0) {
34861a8031e1Shsuenaga printf("\n%s: NO DATA\n", name);
34871a8031e1Shsuenaga return;
34881a8031e1Shsuenaga }
34891a8031e1Shsuenaga for (off = 0; off < len; off++) {
34901a8031e1Shsuenaga if ((off % 16) == 0) {
34911a8031e1Shsuenaga printf("\n%s: 0x%08x:", name, (uint32_t)off);
34921a8031e1Shsuenaga }
34931a8031e1Shsuenaga if ((off % 4) == 0) {
34941a8031e1Shsuenaga printf(" ");
34951a8031e1Shsuenaga }
34961a8031e1Shsuenaga printf("%02x", data[off]);
34971a8031e1Shsuenaga }
34981a8031e1Shsuenaga printf("\n");
34991a8031e1Shsuenaga
35001a8031e1Shsuenaga return;
35011a8031e1Shsuenaga }
35021a8031e1Shsuenaga
35031a8031e1Shsuenaga _STATIC void
mvxpsec_dump_packet(const char * name,struct mvxpsec_packet * mv_p)35041a8031e1Shsuenaga mvxpsec_dump_packet(const char *name, struct mvxpsec_packet *mv_p)
35051a8031e1Shsuenaga {
35061a8031e1Shsuenaga struct mvxpsec_softc *sc = mv_p->mv_s->sc;
35071a8031e1Shsuenaga
35081a8031e1Shsuenaga printf("%s: packet_data:\n", name);
35091a8031e1Shsuenaga mvxpsec_dump_packet_data(name, mv_p);
35101a8031e1Shsuenaga
35111a8031e1Shsuenaga printf("%s: SRAM:\n", name);
35121a8031e1Shsuenaga mvxpsec_dump_sram(name, sc, 2000);
35131a8031e1Shsuenaga
35141a8031e1Shsuenaga printf("%s: packet_descriptor:\n", name);
35151a8031e1Shsuenaga mvxpsec_dump_packet_desc(name, mv_p);
35161a8031e1Shsuenaga }
35171a8031e1Shsuenaga
35181a8031e1Shsuenaga _STATIC void
mvxpsec_dump_packet_data(const char * name,struct mvxpsec_packet * mv_p)35191a8031e1Shsuenaga mvxpsec_dump_packet_data(const char *name, struct mvxpsec_packet *mv_p)
35201a8031e1Shsuenaga {
35211a8031e1Shsuenaga static char buf[1500];
35221a8031e1Shsuenaga int len;
35231a8031e1Shsuenaga
35241a8031e1Shsuenaga if (mv_p->data_type == MVXPSEC_DATA_MBUF) {
35251a8031e1Shsuenaga struct mbuf *m;
35261a8031e1Shsuenaga
35271a8031e1Shsuenaga m = mv_p->data.mbuf;
35281a8031e1Shsuenaga len = m->m_pkthdr.len;
35291a8031e1Shsuenaga if (len > sizeof(buf))
35301a8031e1Shsuenaga len = sizeof(buf);
35311a8031e1Shsuenaga m_copydata(m, 0, len, buf);
35321a8031e1Shsuenaga }
35331a8031e1Shsuenaga else if (mv_p->data_type == MVXPSEC_DATA_UIO) {
35341a8031e1Shsuenaga struct uio *uio;
35351a8031e1Shsuenaga
35361a8031e1Shsuenaga uio = mv_p->data.uio;
35371a8031e1Shsuenaga len = uio->uio_resid;
35381a8031e1Shsuenaga if (len > sizeof(buf))
35391a8031e1Shsuenaga len = sizeof(buf);
35401a8031e1Shsuenaga cuio_copydata(uio, 0, len, buf);
35411a8031e1Shsuenaga }
35421a8031e1Shsuenaga else if (mv_p->data_type == MVXPSEC_DATA_RAW) {
35431a8031e1Shsuenaga len = mv_p->data_len;
35441a8031e1Shsuenaga if (len > sizeof(buf))
35451a8031e1Shsuenaga len = sizeof(buf);
35461a8031e1Shsuenaga memcpy(buf, mv_p->data.raw, len);
35471a8031e1Shsuenaga }
35481a8031e1Shsuenaga else
35491a8031e1Shsuenaga return;
35501a8031e1Shsuenaga mvxpsec_dump_data(name, buf, len);
35511a8031e1Shsuenaga
35521a8031e1Shsuenaga return;
35531a8031e1Shsuenaga }
35541a8031e1Shsuenaga
35551a8031e1Shsuenaga _STATIC void
mvxpsec_dump_packet_desc(const char * name,struct mvxpsec_packet * mv_p)35561a8031e1Shsuenaga mvxpsec_dump_packet_desc(const char *name, struct mvxpsec_packet *mv_p)
35571a8031e1Shsuenaga {
35581a8031e1Shsuenaga uint32_t *words;
35591a8031e1Shsuenaga
35601a8031e1Shsuenaga if (mv_p == NULL)
35611a8031e1Shsuenaga return;
35621a8031e1Shsuenaga
35631a8031e1Shsuenaga words = &mv_p->pkt_header.desc.acc_desc_dword0;
35641a8031e1Shsuenaga mvxpsec_dump_acc_config(name, words[0]);
35651a8031e1Shsuenaga mvxpsec_dump_acc_encdata(name, words[1], words[2]);
35661a8031e1Shsuenaga mvxpsec_dump_acc_enclen(name, words[2]);
35671a8031e1Shsuenaga mvxpsec_dump_acc_enckey(name, words[3]);
35681a8031e1Shsuenaga mvxpsec_dump_acc_enciv(name, words[4]);
35691a8031e1Shsuenaga mvxpsec_dump_acc_macsrc(name, words[5]);
35701a8031e1Shsuenaga mvxpsec_dump_acc_macdst(name, words[6]);
35711a8031e1Shsuenaga mvxpsec_dump_acc_maciv(name, words[7]);
35721a8031e1Shsuenaga
35731a8031e1Shsuenaga return;
35741a8031e1Shsuenaga }
35751a8031e1Shsuenaga
35761a8031e1Shsuenaga _STATIC void
mvxpsec_dump_acc_config(const char * name,uint32_t w)35771a8031e1Shsuenaga mvxpsec_dump_acc_config(const char *name, uint32_t w)
35781a8031e1Shsuenaga {
35791a8031e1Shsuenaga /* SA: Dword 0 */
35801a8031e1Shsuenaga printf("%s: Dword0=0x%08x\n", name, w);
35811a8031e1Shsuenaga printf("%s: OP = %s\n", name,
35821a8031e1Shsuenaga s_xpsec_op(MV_ACC_CRYPTO_OP(w)));
35831a8031e1Shsuenaga printf("%s: MAC = %s\n", name,
35841a8031e1Shsuenaga s_xpsec_mac(MV_ACC_CRYPTO_MAC(w)));
35851a8031e1Shsuenaga printf("%s: MAC_LEN = %s\n", name,
35861a8031e1Shsuenaga w & MV_ACC_CRYPTO_MAC_96 ? "96-bit" : "full-bit");
35871a8031e1Shsuenaga printf("%s: ENC = %s\n", name,
35881a8031e1Shsuenaga s_xpsec_enc(MV_ACC_CRYPTO_ENC(w)));
35891a8031e1Shsuenaga printf("%s: DIR = %s\n", name,
35901a8031e1Shsuenaga w & MV_ACC_CRYPTO_DECRYPT ? "decryption" : "encryption");
35911a8031e1Shsuenaga printf("%s: CHAIN = %s\n", name,
35921a8031e1Shsuenaga w & MV_ACC_CRYPTO_CBC ? "CBC" : "ECB");
35931a8031e1Shsuenaga printf("%s: 3DES = %s\n", name,
35941a8031e1Shsuenaga w & MV_ACC_CRYPTO_3DES_EDE ? "EDE" : "EEE");
35951a8031e1Shsuenaga printf("%s: FRAGMENT = %s\n", name,
35961a8031e1Shsuenaga s_xpsec_frag(MV_ACC_CRYPTO_FRAG(w)));
35971a8031e1Shsuenaga return;
35981a8031e1Shsuenaga }
35991a8031e1Shsuenaga
36001a8031e1Shsuenaga STATIC void
mvxpsec_dump_acc_encdata(const char * name,uint32_t w,uint32_t w2)36011a8031e1Shsuenaga mvxpsec_dump_acc_encdata(const char *name, uint32_t w, uint32_t w2)
36021a8031e1Shsuenaga {
36031a8031e1Shsuenaga /* SA: Dword 1 */
36041a8031e1Shsuenaga printf("%s: Dword1=0x%08x\n", name, w);
36051a8031e1Shsuenaga printf("%s: ENC SRC = 0x%x\n", name, MV_ACC_DESC_GET_VAL_1(w));
36061a8031e1Shsuenaga printf("%s: ENC DST = 0x%x\n", name, MV_ACC_DESC_GET_VAL_2(w));
36071a8031e1Shsuenaga printf("%s: ENC RANGE = 0x%x - 0x%x\n", name,
36081a8031e1Shsuenaga MV_ACC_DESC_GET_VAL_1(w),
36091a8031e1Shsuenaga MV_ACC_DESC_GET_VAL_1(w) + MV_ACC_DESC_GET_VAL_1(w2) - 1);
36101a8031e1Shsuenaga return;
36111a8031e1Shsuenaga }
36121a8031e1Shsuenaga
36131a8031e1Shsuenaga STATIC void
mvxpsec_dump_acc_enclen(const char * name,uint32_t w)36141a8031e1Shsuenaga mvxpsec_dump_acc_enclen(const char *name, uint32_t w)
36151a8031e1Shsuenaga {
36161a8031e1Shsuenaga /* SA: Dword 2 */
36171a8031e1Shsuenaga printf("%s: Dword2=0x%08x\n", name, w);
36181a8031e1Shsuenaga printf("%s: ENC LEN = %d\n", name,
36191a8031e1Shsuenaga MV_ACC_DESC_GET_VAL_1(w));
36201a8031e1Shsuenaga return;
36211a8031e1Shsuenaga }
36221a8031e1Shsuenaga
36231a8031e1Shsuenaga STATIC void
mvxpsec_dump_acc_enckey(const char * name,uint32_t w)36241a8031e1Shsuenaga mvxpsec_dump_acc_enckey(const char *name, uint32_t w)
36251a8031e1Shsuenaga {
36261a8031e1Shsuenaga /* SA: Dword 3 */
36271a8031e1Shsuenaga printf("%s: Dword3=0x%08x\n", name, w);
36281a8031e1Shsuenaga printf("%s: EKEY = 0x%x\n", name,
36291a8031e1Shsuenaga MV_ACC_DESC_GET_VAL_1(w));
36301a8031e1Shsuenaga return;
36311a8031e1Shsuenaga }
36321a8031e1Shsuenaga
36331a8031e1Shsuenaga STATIC void
mvxpsec_dump_acc_enciv(const char * name,uint32_t w)36341a8031e1Shsuenaga mvxpsec_dump_acc_enciv(const char *name, uint32_t w)
36351a8031e1Shsuenaga {
36361a8031e1Shsuenaga /* SA: Dword 4 */
36371a8031e1Shsuenaga printf("%s: Dword4=0x%08x\n", name, w);
36381a8031e1Shsuenaga printf("%s: EIV = 0x%x\n", name, MV_ACC_DESC_GET_VAL_1(w));
36391a8031e1Shsuenaga printf("%s: EIV_BUF = 0x%x\n", name, MV_ACC_DESC_GET_VAL_2(w));
36401a8031e1Shsuenaga return;
36411a8031e1Shsuenaga }
36421a8031e1Shsuenaga
36431a8031e1Shsuenaga STATIC void
mvxpsec_dump_acc_macsrc(const char * name,uint32_t w)36441a8031e1Shsuenaga mvxpsec_dump_acc_macsrc(const char *name, uint32_t w)
36451a8031e1Shsuenaga {
36461a8031e1Shsuenaga /* SA: Dword 5 */
36471a8031e1Shsuenaga printf("%s: Dword5=0x%08x\n", name, w);
36481a8031e1Shsuenaga printf("%s: MAC_SRC = 0x%x\n", name,
36491a8031e1Shsuenaga MV_ACC_DESC_GET_VAL_1(w));
36501a8031e1Shsuenaga printf("%s: MAC_TOTAL_LEN = %d\n", name,
36511a8031e1Shsuenaga MV_ACC_DESC_GET_VAL_3(w));
36521a8031e1Shsuenaga printf("%s: MAC_RANGE = 0x%0x - 0x%0x\n", name,
36531a8031e1Shsuenaga MV_ACC_DESC_GET_VAL_1(w),
36541a8031e1Shsuenaga MV_ACC_DESC_GET_VAL_1(w) + MV_ACC_DESC_GET_VAL_3(w) - 1);
36551a8031e1Shsuenaga return;
36561a8031e1Shsuenaga }
36571a8031e1Shsuenaga
36581a8031e1Shsuenaga STATIC void
mvxpsec_dump_acc_macdst(const char * name,uint32_t w)36591a8031e1Shsuenaga mvxpsec_dump_acc_macdst(const char *name, uint32_t w)
36601a8031e1Shsuenaga {
36611a8031e1Shsuenaga /* SA: Dword 6 */
36621a8031e1Shsuenaga printf("%s: Dword6=0x%08x\n", name, w);
36631a8031e1Shsuenaga printf("%s: MAC_DST = 0x%x\n", name, MV_ACC_DESC_GET_VAL_1(w));
36641a8031e1Shsuenaga printf("%s: MAC_BLOCK_LEN = %d\n", name,
36651a8031e1Shsuenaga MV_ACC_DESC_GET_VAL_2(w));
36661a8031e1Shsuenaga return;
36671a8031e1Shsuenaga }
36681a8031e1Shsuenaga
36691a8031e1Shsuenaga STATIC void
mvxpsec_dump_acc_maciv(const char * name,uint32_t w)36701a8031e1Shsuenaga mvxpsec_dump_acc_maciv(const char *name, uint32_t w)
36711a8031e1Shsuenaga {
36721a8031e1Shsuenaga /* SA: Dword 7 */
36731a8031e1Shsuenaga printf("%s: Dword7=0x%08x\n", name, w);
36741a8031e1Shsuenaga printf("%s: MAC_INNER_IV = 0x%x\n", name,
36751a8031e1Shsuenaga MV_ACC_DESC_GET_VAL_1(w));
36761a8031e1Shsuenaga printf("%s: MAC_OUTER_IV = 0x%x\n", name,
36771a8031e1Shsuenaga MV_ACC_DESC_GET_VAL_2(w));
36781a8031e1Shsuenaga return;
36791a8031e1Shsuenaga }
36801a8031e1Shsuenaga #endif
3681