Lines Matching +full:inline +full:- +full:crypto +full:- +full:engine
1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
32 * SafeNet SafeXcel-1141 hardware crypto accelerator
102 /* crypto device methods */
117 MODULE_DEPEND(safe, crypto, 1, 1, 1);
133 bus_dmamap_sync((_dma)->dma_tag, (_dma)->dma_map, (_flags))
167 bus_space_read_4((sc)->sc_st, (sc)->sc_sh, (r))
170 bus_space_write_4((sc)->sc_st, (sc)->sc_sh, reg, val)
181 0, "RNG polling buffer size (32-bit words)");
200 switch (pci_get_vendor(sc->sc_dev)) {
202 switch (pci_get_device(sc->sc_dev)) {
203 case PCI_PRODUCT_SAFEXCEL: return "SafeNet SafeXcel-1141";
205 return "SafeNet unknown-part";
207 return "Unknown-vendor unknown-part";
228 sc->sc_dev = dev;
235 * Setup memory-mapping of PCI registers.
238 sc->sc_sr = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
240 if (sc->sc_sr == NULL) {
244 sc->sc_st = rman_get_bustag(sc->sc_sr);
245 sc->sc_sh = rman_get_bushandle(sc->sc_sr);
251 sc->sc_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
253 if (sc->sc_irq == NULL) {
261 if (bus_setup_intr(dev, sc->sc_irq, INTR_TYPE_NET | INTR_MPSAFE,
262 NULL, safe_intr, sc, &sc->sc_ih)) {
267 sc->sc_cid = crypto_get_driverid(dev, sizeof(struct safe_session),
269 if (sc->sc_cid < 0) {
270 device_printf(dev, "could not get crypto driver id\n");
274 sc->sc_chiprev = READ_REG(sc, SAFE_DEVINFO) &
291 &sc->sc_srcdmat)) {
306 &sc->sc_dstdmat)) {
312 * Allocate packet engine descriptors.
316 &sc->sc_ringalloc, 0)) {
318 bus_dma_tag_destroy(sc->sc_srcdmat);
324 sc->sc_ring = (struct safe_ringentry *) sc->sc_ringalloc.dma_vaddr;
325 sc->sc_ringtop = sc->sc_ring + SAFE_MAX_NQUEUE;
326 sc->sc_front = sc->sc_ring;
327 sc->sc_back = sc->sc_ring;
328 raddr = sc->sc_ringalloc.dma_paddr;
329 bzero(sc->sc_ring, SAFE_MAX_NQUEUE * sizeof(struct safe_ringentry));
331 struct safe_ringentry *re = &sc->sc_ring[i];
333 re->re_desc.d_sa = raddr +
335 re->re_sa.sa_staterec = raddr +
340 mtx_init(&sc->sc_ringmtx, device_get_nameunit(dev),
341 "packet engine ring", MTX_DEF);
347 &sc->sc_spalloc, 0)) {
350 mtx_destroy(&sc->sc_ringmtx);
351 safe_dma_free(sc, &sc->sc_ringalloc);
352 bus_dma_tag_destroy(sc->sc_srcdmat);
355 sc->sc_spring = (struct safe_pdesc *) sc->sc_spalloc.dma_vaddr;
356 sc->sc_springtop = sc->sc_spring + SAFE_TOTAL_SPART;
357 sc->sc_spfree = sc->sc_spring;
358 bzero(sc->sc_spring, SAFE_TOTAL_SPART * sizeof(struct safe_pdesc));
361 &sc->sc_dpalloc, 0)) {
364 mtx_destroy(&sc->sc_ringmtx);
365 safe_dma_free(sc, &sc->sc_spalloc);
366 safe_dma_free(sc, &sc->sc_ringalloc);
367 bus_dma_tag_destroy(sc->sc_dstdmat);
370 sc->sc_dpring = (struct safe_pdesc *) sc->sc_dpalloc.dma_vaddr;
371 sc->sc_dpringtop = sc->sc_dpring + SAFE_TOTAL_DPART;
372 sc->sc_dpfree = sc->sc_dpring;
373 bzero(sc->sc_dpring, SAFE_TOTAL_DPART * sizeof(struct safe_pdesc));
375 device_printf(sc->sc_dev, "%s", safe_partname(sc));
377 sc->sc_devinfo = READ_REG(sc, SAFE_DEVINFO);
378 if (sc->sc_devinfo & SAFE_DEVINFO_RNG) {
379 sc->sc_flags |= SAFE_FLAGS_RNG;
382 if (sc->sc_devinfo & SAFE_DEVINFO_PKEY) {
385 sc->sc_flags |= SAFE_FLAGS_KEY;
388 if (sc->sc_devinfo & SAFE_DEVINFO_DES) {
391 if (sc->sc_devinfo & SAFE_DEVINFO_AES) {
394 if (sc->sc_devinfo & SAFE_DEVINFO_MD5) {
397 if (sc->sc_devinfo & SAFE_DEVINFO_SHA1) {
408 if (sc->sc_flags & SAFE_FLAGS_RNG) {
410 sc->sc_rndtest = rndtest_attach(dev);
411 if (sc->sc_rndtest)
412 sc->sc_harvest = rndtest_harvest;
414 sc->sc_harvest = default_harvest;
416 sc->sc_harvest = default_harvest;
420 callout_init(&sc->sc_rngto, 1);
421 callout_reset(&sc->sc_rngto, hz*safe_rnginterval, safe_rng, sc);
429 crypto_unregister_all(sc->sc_cid);
431 bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih);
433 bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq);
435 bus_release_resource(dev, SYS_RES_MEMORY, BS_BAR, sc->sc_sr);
452 callout_stop(&sc->sc_rngto);
454 crypto_unregister_all(sc->sc_cid);
457 if (sc->sc_rndtest)
458 rndtest_detach(sc->sc_rndtest);
462 safe_dma_free(sc, &sc->sc_dpalloc);
463 safe_dma_free(sc, &sc->sc_spalloc);
464 mtx_destroy(&sc->sc_ringmtx);
465 safe_dma_free(sc, &sc->sc_ringalloc);
467 bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih);
468 bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq);
470 bus_dma_tag_destroy(sc->sc_srcdmat);
471 bus_dma_tag_destroy(sc->sc_dstdmat);
472 bus_release_resource(dev, SYS_RES_MEMORY, BS_BAR, sc->sc_sr);
501 sc->sc_suspended = 1;
514 sc->sc_suspended = 0;
538 mtx_lock(&sc->sc_ringmtx);
539 while (sc->sc_back != sc->sc_front) {
540 struct safe_ringentry *re = sc->sc_back;
554 if (re->re_desc.d_csr != 0) {
555 if (!SAFE_PE_CSR_IS_DONE(re->re_desc.d_csr))
557 if (!SAFE_PE_LEN_IS_DONE(re->re_desc.d_len))
559 sc->sc_nqchip--;
562 if (++(sc->sc_back) == sc->sc_ringtop)
563 sc->sc_back = sc->sc_ring;
565 mtx_unlock(&sc->sc_ringmtx);
581 if (sc->sc_needwakeup) { /* XXX check high watermark */
582 int wakeup = sc->sc_needwakeup & CRYPTO_SYMQ;
583 DPRINTF(("%s: wakeup crypto %x\n", __func__,
584 sc->sc_needwakeup));
585 sc->sc_needwakeup &= ~wakeup;
586 crypto_unblock(sc->sc_cid, wakeup);
591 * safe_feed() - post a request to chip
596 bus_dmamap_sync(sc->sc_srcdmat, re->re_src_map, BUS_DMASYNC_PREWRITE);
597 if (re->re_dst_map != NULL)
598 bus_dmamap_sync(sc->sc_dstdmat, re->re_dst_map,
601 safe_dma_sync(&sc->sc_ringalloc,
603 safe_dma_sync(&sc->sc_spalloc, BUS_DMASYNC_PREWRITE);
604 safe_dma_sync(&sc->sc_dpalloc, BUS_DMASYNC_PREWRITE);
612 sc->sc_nqchip++;
613 if (sc->sc_nqchip > safestats.st_maxqchip)
614 safestats.st_maxqchip = sc->sc_nqchip;
625 bcopy(key, ses->ses_key, ses->ses_klen);
627 /* PE is little-endian, insure proper byte order */
628 for (i = 0; i < N(ses->ses_key); i++)
629 ses->ses_key[i] = htole32(ses->ses_key[i]);
640 bcopy(sha1ctx.h.b32, ses->ses_hminner, sizeof(sha1ctx.h.b32));
643 bcopy(sha1ctx.h.b32, ses->ses_hmouter, sizeof(sha1ctx.h.b32));
647 /* PE is little-endian, insure proper byte order */
648 for (i = 0; i < N(ses->ses_hminner); i++) {
649 ses->ses_hminner[i] = htole32(ses->ses_hminner[i]);
650 ses->ses_hmouter[i] = htole32(ses->ses_hmouter[i]);
660 switch (csp->csp_auth_alg) {
662 if ((sc->sc_devinfo & SAFE_DEVINFO_SHA1) == 0)
676 switch (csp->csp_cipher_alg) {
678 if ((sc->sc_devinfo & SAFE_DEVINFO_AES) == 0)
680 if (csp->csp_ivlen != 16)
682 if (csp->csp_cipher_klen != 16 &&
683 csp->csp_cipher_klen != 24 &&
684 csp->csp_cipher_klen != 32)
696 if (csp->csp_flags != 0)
698 switch (csp->csp_mode) {
729 if (csp->csp_cipher_alg != 0) {
730 ses->ses_klen = csp->csp_cipher_klen;
731 if (csp->csp_cipher_key != NULL)
732 safe_setup_enckey(ses, csp->csp_cipher_key);
735 if (csp->csp_auth_alg != 0) {
736 ses->ses_mlen = csp->csp_auth_mlen;
737 if (ses->ses_mlen == 0) {
738 ses->ses_mlen = SHA1_HASH_LEN;
741 if (csp->csp_auth_key != NULL) {
742 safe_setup_mackey(ses, csp->csp_auth_alg,
743 csp->csp_auth_key, csp->csp_auth_klen);
759 op->nsegs = nsegs;
760 bcopy(seg, op->segs, nsegs * sizeof (seg[0]));
777 mtx_lock(&sc->sc_ringmtx);
778 if (sc->sc_front == sc->sc_back && sc->sc_nqchip != 0) {
780 sc->sc_needwakeup |= CRYPTO_SYMQ;
781 mtx_unlock(&sc->sc_ringmtx);
784 re = sc->sc_front;
786 staterec = re->re_sa.sa_staterec; /* save */
788 bzero(&re->re_sa, sizeof(struct safe_ringentry) - sizeof(re->re_desc));
789 re->re_sa.sa_staterec = staterec; /* restore */
791 re->re_crp = crp;
793 sa = &re->re_sa;
794 ses = crypto_get_driver_session(crp->crp_session);
795 csp = crypto_get_params(crp->crp_session);
799 switch (csp->csp_mode) {
811 if (csp->csp_cipher_alg != 0) {
812 if (crp->crp_cipher_key != NULL)
813 safe_setup_enckey(ses, crp->crp_cipher_key);
815 switch (csp->csp_cipher_alg) {
819 if (ses->ses_klen * 8 == 128)
821 else if (ses->ses_klen * 8 == 192)
829 * we can't use an inline IV because hash/crypt offset
836 crypto_read_iv(crp, re->re_sastate.sa_saved_iv);
839 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
852 * This pads results to an 8-byte boundary and
856 * NB: Not sure if the 8-byte pad boundary is a problem.
861 bcopy(ses->ses_key, sa->sa_key, sizeof(sa->sa_key));
864 if (csp->csp_auth_alg != 0) {
865 if (crp->crp_auth_key != NULL) {
866 safe_setup_mackey(ses, csp->csp_auth_alg,
867 crp->crp_auth_key, csp->csp_auth_klen);
870 switch (csp->csp_auth_alg) {
883 bcopy(ses->ses_hminner, sa->sa_indigest,
884 sizeof(sa->sa_indigest));
885 bcopy(ses->ses_hmouter, sa->sa_outdigest,
886 sizeof(sa->sa_outdigest));
889 re->re_flags |= SAFE_QFLAGS_COPYOUTICV;
892 if (csp->csp_mode == CSP_MODE_ETA) {
897 if (crp->crp_aad_length != 0 &&
898 crp->crp_aad_start + crp->crp_aad_length !=
899 crp->crp_payload_start) {
904 if (crp->crp_aad_length != 0)
905 bypass = crp->crp_aad_start;
907 bypass = crp->crp_payload_start;
908 coffset = crp->crp_aad_length;
909 oplen = crp->crp_payload_start + crp->crp_payload_length;
913 crp->crp_aad_start, crp->crp_aad_length,
914 crp->crp_digest_start);
916 crp->crp_payload_start, crp->crp_payload_length,
917 crp->crp_iv_start);
922 if (coffset & 3) { /* offset must be 32-bit aligned */
941 * Typically this is the inline IV. Note that you need
953 bypass = crp->crp_payload_start;
954 oplen = bypass + crp->crp_payload_length;
965 if (bus_dmamap_create(sc->sc_srcdmat, BUS_DMA_NOWAIT, &re->re_src_map)) {
970 if (bus_dmamap_load_crp(sc->sc_srcdmat, re->re_src_map, crp, safe_op_cb,
971 &re->re_src, BUS_DMA_NOWAIT) != 0) {
972 bus_dmamap_destroy(sc->sc_srcdmat, re->re_src_map);
973 re->re_src_map = NULL;
978 re->re_src_mapsize = crypto_buffer_len(&crp->crp_buf);
979 nicealign = safe_dmamap_aligned(&re->re_src);
980 uniform = safe_dmamap_uniform(&re->re_src);
983 nicealign, uniform, re->re_src.nsegs));
984 if (re->re_src.nsegs > 1) {
985 re->re_desc.d_src = sc->sc_spalloc.dma_paddr +
986 ((caddr_t) sc->sc_spfree - (caddr_t) sc->sc_spring);
987 for (i = 0; i < re->re_src_nsegs; i++) {
989 pd = sc->sc_spfree;
990 if (++(sc->sc_spfree) == sc->sc_springtop)
991 sc->sc_spfree = sc->sc_spring;
993 KASSERT((pd->pd_flags&3) == 0 ||
994 (pd->pd_flags&3) == SAFE_PD_DONE,
996 pd->pd_flags));
997 pd->pd_addr = re->re_src_segs[i].ds_addr;
998 pd->pd_size = re->re_src_segs[i].ds_len;
999 pd->pd_flags = SAFE_PD_READY;
1006 re->re_desc.d_src = re->re_src_segs[0].ds_addr;
1009 if (csp->csp_mode == CSP_MODE_DIGEST) {
1019 re->re_dst = re->re_src;
1029 if (bus_dmamap_create(sc->sc_dstdmat, BUS_DMA_NOWAIT,
1030 &re->re_dst_map)) {
1035 if (bus_dmamap_load_crp(sc->sc_dstdmat, re->re_dst_map,
1036 crp, safe_op_cb, &re->re_dst, BUS_DMA_NOWAIT) !=
1038 bus_dmamap_destroy(sc->sc_dstdmat,
1039 re->re_dst_map);
1040 re->re_dst_map = NULL;
1045 } else if (crp->crp_buf.cb_type == CRYPTO_BUF_MBUF) {
1061 totlen = re->re_src_mapsize;
1062 if (crp->crp_buf.cb_mbuf->m_flags & M_PKTHDR) {
1065 if (m && !m_dup_pkthdr(m, crp->crp_buf.cb_mbuf,
1076 err = sc->sc_nqchip ? ERESTART : ENOMEM;
1083 err = sc->sc_nqchip ?
1089 m->m_len = len;
1099 err = sc->sc_nqchip ?
1110 err = sc->sc_nqchip ?
1116 m->m_len = len = min(totlen, len);
1117 totlen -= len;
1119 mp = &m->m_next;
1121 re->re_dst_m = top;
1122 if (bus_dmamap_create(sc->sc_dstdmat,
1123 BUS_DMA_NOWAIT, &re->re_dst_map) != 0) {
1128 if (bus_dmamap_load_mbuf_sg(sc->sc_dstdmat,
1129 re->re_dst_map, top, re->re_dst_segs,
1130 &re->re_dst_nsegs, 0) != 0) {
1131 bus_dmamap_destroy(sc->sc_dstdmat,
1132 re->re_dst_map);
1133 re->re_dst_map = NULL;
1138 re->re_dst_mapsize = re->re_src_mapsize;
1139 if (re->re_src.mapsize > oplen) {
1147 if (!(csp->csp_mode == CSP_MODE_ETA &&
1148 (re->re_src.mapsize-oplen) == ses->ses_mlen &&
1149 crp->crp_digest_start == oplen))
1150 safe_mcopy(crp->crp_buf.cb_mbuf,
1151 re->re_dst_m, oplen);
1180 if (re->re_dst.nsegs > 1) {
1181 re->re_desc.d_dst = sc->sc_dpalloc.dma_paddr +
1182 ((caddr_t) sc->sc_dpfree - (caddr_t) sc->sc_dpring);
1183 for (i = 0; i < re->re_dst_nsegs; i++) {
1184 pd = sc->sc_dpfree;
1185 KASSERT((pd->pd_flags&3) == 0 ||
1186 (pd->pd_flags&3) == SAFE_PD_DONE,
1188 pd->pd_flags));
1189 if (++(sc->sc_dpfree) == sc->sc_dpringtop)
1190 sc->sc_dpfree = sc->sc_dpring;
1191 pd->pd_addr = re->re_dst_segs[i].ds_addr;
1192 pd->pd_flags = SAFE_PD_READY;
1199 re->re_desc.d_dst = re->re_dst_segs[0].ds_addr;
1205 * and the packet engine descriptor. The operation
1208 sa->sa_cmd0 = cmd0 | SAFE_SA_CMD0_IPCI | SAFE_SA_CMD0_OPCI;
1209 sa->sa_cmd1 = cmd1
1222 re->re_desc.d_csr = SAFE_PE_CSR_READY | SAFE_PE_CSR_SAPCI;
1223 if (csp->csp_auth_alg != 0)
1224 re->re_desc.d_csr |= SAFE_PE_CSR_LOADSA | SAFE_PE_CSR_HASHFINAL;
1225 re->re_desc.d_len = oplen
1233 if (++(sc->sc_front) == sc->sc_ringtop)
1234 sc->sc_front = sc->sc_ring;
1238 mtx_unlock(&sc->sc_ringmtx);
1242 if (re->re_dst_m != NULL)
1243 m_freem(re->re_dst_m);
1245 if (re->re_dst_map != NULL && re->re_dst_map != re->re_src_map) {
1246 bus_dmamap_unload(sc->sc_dstdmat, re->re_dst_map);
1247 bus_dmamap_destroy(sc->sc_dstdmat, re->re_dst_map);
1249 if (re->re_src_map != NULL) {
1250 bus_dmamap_unload(sc->sc_srcdmat, re->re_src_map);
1251 bus_dmamap_destroy(sc->sc_srcdmat, re->re_src_map);
1253 mtx_unlock(&sc->sc_ringmtx);
1255 crp->crp_etype = err;
1259 sc->sc_needwakeup |= CRYPTO_SYMQ;
1268 struct cryptop *crp = (struct cryptop *)re->re_crp;
1272 ses = crypto_get_driver_session(crp->crp_session);
1273 csp = crypto_get_params(crp->crp_session);
1276 safestats.st_obytes += re->re_dst.mapsize;
1278 safe_dma_sync(&sc->sc_ringalloc,
1280 if (re->re_desc.d_csr & SAFE_PE_CSR_STATUS) {
1281 device_printf(sc->sc_dev, "csr 0x%x cmd0 0x%x cmd1 0x%x\n",
1282 re->re_desc.d_csr,
1283 re->re_sa.sa_cmd0, re->re_sa.sa_cmd1);
1285 crp->crp_etype = EIO; /* something more meaningful? */
1289 * XXX: Should crp_buf.cb_mbuf be updated to re->re_dst_m if
1290 * it is non-NULL?
1293 if (re->re_dst_map != NULL && re->re_dst_map != re->re_src_map) {
1294 bus_dmamap_sync(sc->sc_dstdmat, re->re_dst_map,
1296 bus_dmamap_unload(sc->sc_dstdmat, re->re_dst_map);
1297 bus_dmamap_destroy(sc->sc_dstdmat, re->re_dst_map);
1299 bus_dmamap_sync(sc->sc_srcdmat, re->re_src_map, BUS_DMASYNC_POSTWRITE);
1300 bus_dmamap_unload(sc->sc_srcdmat, re->re_src_map);
1301 bus_dmamap_destroy(sc->sc_srcdmat, re->re_src_map);
1303 if (re->re_flags & SAFE_QFLAGS_COPYOUTICV) {
1304 if (csp->csp_auth_alg == CRYPTO_SHA1_HMAC) {
1306 * SHA-1 ICV's are byte-swapped; fix 'em up
1309 re->re_sastate.sa_saved_indigest[0] =
1310 bswap32(re->re_sastate.sa_saved_indigest[0]);
1311 re->re_sastate.sa_saved_indigest[1] =
1312 bswap32(re->re_sastate.sa_saved_indigest[1]);
1313 re->re_sastate.sa_saved_indigest[2] =
1314 bswap32(re->re_sastate.sa_saved_indigest[2]);
1317 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
1318 crypto_copydata(crp, crp->crp_digest_start,
1319 ses->ses_mlen, hash);
1320 if (timingsafe_bcmp(re->re_sastate.sa_saved_indigest,
1321 hash, ses->ses_mlen) != 0)
1322 crp->crp_etype = EBADMSG;
1324 crypto_copyback(crp, crp->crp_digest_start,
1325 ses->ses_mlen, re->re_sastate.sa_saved_indigest);
1343 while (j >= srcm->m_len) {
1344 j -= srcm->m_len;
1345 srcm = srcm->m_next;
1350 slen = srcm->m_len - j;
1353 while (j >= dstm->m_len) {
1354 j -= dstm->m_len;
1355 dstm = dstm->m_next;
1360 dlen = dstm->m_len - j;
1369 srcm = srcm->m_next;
1372 sptr = srcm->m_data;
1373 slen = srcm->m_len;
1375 sptr += j, slen -= j;
1377 dstm = dstm->m_next;
1380 dptr = dstm->m_data;
1381 dlen = dstm->m_len;
1383 dptr += j, dlen -= j;
1412 * This could take between 750ms - 1000ms in time.
1512 (*sc->sc_harvest)(sc->sc_rndtest, buf, maxwords*sizeof (u_int32_t));
1513 callout_reset(&sc->sc_rngto,
1522 *paddr = segs->ds_addr;
1535 r = bus_dma_tag_create(bus_get_dma_tag(sc->sc_dev), /* parent */
1545 &dma->dma_tag);
1547 device_printf(sc->sc_dev, "safe_dma_malloc: "
1552 r = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr,
1553 BUS_DMA_NOWAIT, &dma->dma_map);
1555 device_printf(sc->sc_dev, "safe_dma_malloc: "
1561 r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
1564 &dma->dma_paddr,
1567 device_printf(sc->sc_dev, "safe_dma_malloc: "
1572 dma->dma_size = size;
1575 bus_dmamap_unload(dma->dma_tag, dma->dma_map);
1577 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
1579 bus_dma_tag_destroy(dma->dma_tag);
1581 dma->dma_tag = NULL;
1588 bus_dmamap_unload(dma->dma_tag, dma->dma_map);
1589 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
1590 bus_dma_tag_destroy(dma->dma_tag);
1628 | SAFE_PE_DMACFG_ESDESC /* endian-swap descriptors */
1629 | SAFE_PE_DMACFG_ESSA /* endian-swap SA's */
1630 | SAFE_PE_DMACFG_ESPDESC /* endian-swap part. desc's */
1637 if (sc->sc_chiprev == SAFE_REV(1,0)) {
1646 device_printf(sc->sc_dev,
1649 SAFE_REV_MAJ(sc->sc_chiprev),
1650 SAFE_REV_MIN(sc->sc_chiprev));
1654 WRITE_REG(sc, SAFE_PE_PDRBASE, sc->sc_ringalloc.dma_paddr);
1655 WRITE_REG(sc, SAFE_PE_RDRBASE, sc->sc_ringalloc.dma_paddr);
1660 ("PE ring entry not 32-bit aligned!"));
1666 WRITE_REG(sc, SAFE_PE_GRNGBASE, sc->sc_spalloc.dma_paddr);
1667 WRITE_REG(sc, SAFE_PE_SRNGBASE, sc->sc_dpalloc.dma_paddr);
1681 * Configure hardware to use level-triggered interrupts and
1705 if (sc->sc_nqchip != 0) {
1706 struct safe_ringentry *re = sc->sc_back;
1708 while (re != sc->sc_front) {
1709 if (re->re_desc.d_csr != 0)
1711 if (++re == sc->sc_ringtop)
1712 re = sc->sc_ring;
1714 sc->sc_back = re;
1715 sc->sc_nqchip = 0;
1731 if (re->re_dst_m != NULL)
1732 m_freem(re->re_dst_m);
1734 crp = (struct cryptop *)re->re_crp;
1736 re->re_desc.d_csr = 0;
1738 crp->crp_etype = EFAULT;
1757 * segment must be aligned on a 32-bit boundary and all
1765 for (i = 0; i < op->nsegs; i++) {
1766 if (op->segs[i].ds_addr & 3)
1768 if (i != (op->nsegs - 1) && (op->segs[i].ds_len & 3))
1788 if (op->nsegs > 0) {
1791 for (i = 0; i < op->nsegs-1; i++) {
1792 if (op->segs[i].ds_len % SAFE_MAX_DSIZE)
1794 if (op->segs[i].ds_len != SAFE_MAX_DSIZE)
1836 (unsigned long)(sc->sc_back - sc->sc_ring),
1837 (unsigned long)(sc->sc_front - sc->sc_ring));
1845 ix = re - sc->sc_ring;
1849 , re->re_desc.d_csr
1850 , re->re_desc.d_src
1851 , re->re_desc.d_dst
1852 , re->re_desc.d_sa
1853 , re->re_desc.d_len
1855 if (re->re_src.nsegs > 1) {
1856 ix = (re->re_desc.d_src - sc->sc_spalloc.dma_paddr) /
1858 for (nsegs = re->re_src.nsegs; nsegs; nsegs--) {
1860 , ix, &sc->sc_spring[ix]
1861 , (caddr_t)(uintptr_t) sc->sc_spring[ix].pd_addr
1862 , sc->sc_spring[ix].pd_size
1863 , sc->sc_spring[ix].pd_flags
1865 if (sc->sc_spring[ix].pd_size == 0)
1872 if (re->re_dst.nsegs > 1) {
1873 ix = (re->re_desc.d_dst - sc->sc_dpalloc.dma_paddr) /
1875 for (nsegs = re->re_dst.nsegs; nsegs; nsegs--) {
1877 , ix, &sc->sc_dpring[ix]
1878 , (caddr_t)(uintptr_t) sc->sc_dpring[ix].pd_addr
1879 , sc->sc_dpring[ix].pd_flags
1886 re->re_sa.sa_cmd0, re->re_sa.sa_cmd1, re->re_sa.sa_staterec);
1888 , re->re_sa.sa_key[0]
1889 , re->re_sa.sa_key[1]
1890 , re->re_sa.sa_key[2]
1891 , re->re_sa.sa_key[3]
1892 , re->re_sa.sa_key[4]
1893 , re->re_sa.sa_key[5]
1894 , re->re_sa.sa_key[6]
1895 , re->re_sa.sa_key[7]
1898 , re->re_sa.sa_indigest[0]
1899 , re->re_sa.sa_indigest[1]
1900 , re->re_sa.sa_indigest[2]
1901 , re->re_sa.sa_indigest[3]
1902 , re->re_sa.sa_indigest[4]
1905 , re->re_sa.sa_outdigest[0]
1906 , re->re_sa.sa_outdigest[1]
1907 , re->re_sa.sa_outdigest[2]
1908 , re->re_sa.sa_outdigest[3]
1909 , re->re_sa.sa_outdigest[4]
1912 , re->re_sastate.sa_saved_iv[0]
1913 , re->re_sastate.sa_saved_iv[1]
1914 , re->re_sastate.sa_saved_iv[2]
1915 , re->re_sastate.sa_saved_iv[3]
1918 , re->re_sastate.sa_saved_hashbc
1919 , re->re_sastate.sa_saved_indigest[0]
1920 , re->re_sastate.sa_saved_indigest[1]
1921 , re->re_sastate.sa_saved_indigest[2]
1922 , re->re_sastate.sa_saved_indigest[3]
1923 , re->re_sastate.sa_saved_indigest[4]
1930 mtx_lock(&sc->sc_ringmtx);
1935 if (sc->sc_nqchip) {
1936 struct safe_ringentry *re = sc->sc_back;
1939 if (++re == sc->sc_ringtop)
1940 re = sc->sc_ring;
1941 } while (re != sc->sc_front);
1943 mtx_unlock(&sc->sc_ringmtx);
1952 strncpy(dmode, "", sizeof(dmode) - 1);
1953 dmode[sizeof(dmode) - 1] = '\0';
1956 if (error == 0 && req->newptr != NULL) {