1 /* $FreeBSD: src/sys/dev/ubsec/ubsec.c,v 1.6.2.12 2003/06/04 17:56:59 sam Exp $ */ 2 /* $OpenBSD: ubsec.c,v 1.115 2002/09/24 18:33:26 jason Exp $ */ 3 4 /* 5 * Copyright (c) 2000 Jason L. Wright (jason@thought.net) 6 * Copyright (c) 2000 Theo de Raadt (deraadt@openbsd.org) 7 * Copyright (c) 2001 Patrik Lindergren (patrik@ipunplugged.com) 8 * 9 * All rights reserved. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 3. All advertising materials mentioning features or use of this software 20 * must display the following acknowledgement: 21 * This product includes software developed by Jason L. Wright 22 * 4. The name of the author may not be used to endorse or promote products 23 * derived from this software without specific prior written permission. 24 * 25 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 26 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 27 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 28 * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, 29 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 30 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 31 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 32 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 33 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 34 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 35 * POSSIBILITY OF SUCH DAMAGE. 36 * 37 * Effort sponsored in part by the Defense Advanced Research Projects 38 * Agency (DARPA) and Air Force Research Laboratory, Air Force 39 * Materiel Command, USAF, under agreement number F30602-01-2-0537. 40 * 41 */ 42 43 /* 44 * uBsec 5[56]01, 58xx hardware crypto accelerator 45 */ 46 47 #include "opt_ubsec.h" 48 49 #include <sys/param.h> 50 #include <sys/systm.h> 51 #include <sys/proc.h> 52 #include <sys/errno.h> 53 #include <sys/malloc.h> 54 #include <sys/kernel.h> 55 #include <sys/mbuf.h> 56 #include <sys/sysctl.h> 57 #include <sys/endian.h> 58 #include <sys/bus.h> 59 #include <sys/rman.h> 60 #include <sys/md5.h> 61 #include <sys/random.h> 62 #include <sys/thread2.h> 63 64 #include <vm/vm.h> 65 #include <vm/pmap.h> 66 67 #include <machine/clock.h> 68 69 #include <crypto/sha1.h> 70 #include <opencrypto/cryptodev.h> 71 #include <opencrypto/cryptosoft.h> 72 73 #include "cryptodev_if.h" 74 75 #include <bus/pci/pcivar.h> 76 #include <bus/pci/pcireg.h> 77 78 /* grr, #defines for gratuitous incompatibility in queue.h */ 79 #define SIMPLEQ_HEAD STAILQ_HEAD 80 #define SIMPLEQ_ENTRY STAILQ_ENTRY 81 #define SIMPLEQ_INIT STAILQ_INIT 82 #define SIMPLEQ_INSERT_TAIL STAILQ_INSERT_TAIL 83 #define SIMPLEQ_EMPTY STAILQ_EMPTY 84 #define SIMPLEQ_FIRST STAILQ_FIRST 85 #define SIMPLEQ_REMOVE_HEAD STAILQ_REMOVE_HEAD 86 #define SIMPLEQ_FOREACH STAILQ_FOREACH 87 /* ditto for endian.h */ 88 #define letoh16(x) le16toh(x) 89 #define letoh32(x) le32toh(x) 90 91 #ifdef UBSEC_RNDTEST 92 #include "../rndtest/rndtest.h" 93 #endif 94 #include "ubsecreg.h" 95 #include "ubsecvar.h" 96 97 /* 98 * Prototypes and count for the pci_device structure 99 */ 100 static int ubsec_probe(device_t); 101 static int ubsec_attach(device_t); 102 static int ubsec_detach(device_t); 103 static int ubsec_suspend(device_t); 104 static int ubsec_resume(device_t); 105 static void ubsec_shutdown(device_t); 106 static void ubsec_intr(void *); 107 static int ubsec_newsession(void *, u_int32_t *, struct cryptoini *); 108 static int ubsec_freesession(void *, u_int64_t); 109 static int ubsec_process(void *, struct cryptop *, int); 110 static void ubsec_callback(struct ubsec_softc *, struct ubsec_q *); 111 static void ubsec_feed(struct ubsec_softc *); 112 static void ubsec_mcopy(struct mbuf *, struct mbuf *, int, int); 113 static void ubsec_callback2(struct ubsec_softc *, struct ubsec_q2 *); 114 static int ubsec_feed2(struct ubsec_softc *); 115 static void ubsec_rng(void *); 116 static int ubsec_dma_malloc(struct ubsec_softc *, bus_size_t, 117 struct ubsec_dma_alloc *, int); 118 #define ubsec_dma_sync(_dma, _flags) \ 119 bus_dmamap_sync((_dma)->dma_tag, (_dma)->dma_map, (_flags)) 120 static void ubsec_dma_free(struct ubsec_softc *, struct ubsec_dma_alloc *); 121 static int ubsec_dmamap_aligned(struct ubsec_operand *op); 122 123 static void ubsec_reset_board(struct ubsec_softc *sc); 124 static void ubsec_init_board(struct ubsec_softc *sc); 125 static void ubsec_init_pciregs(device_t dev); 126 static void ubsec_totalreset(struct ubsec_softc *sc); 127 128 static int ubsec_free_q(struct ubsec_softc *sc, struct ubsec_q *q); 129 130 static int ubsec_kprocess(void*, struct cryptkop *, int); 131 static int ubsec_kprocess_modexp_hw(struct ubsec_softc *, struct cryptkop *, int); 132 static int ubsec_kprocess_modexp_sw(struct ubsec_softc *, struct cryptkop *, int); 133 static int ubsec_kprocess_rsapriv(struct ubsec_softc *, struct cryptkop *, int); 134 static void ubsec_kfree(struct ubsec_softc *, struct ubsec_q2 *); 135 static int ubsec_ksigbits(struct crparam *); 136 static void ubsec_kshift_r(u_int, u_int8_t *, u_int, u_int8_t *, u_int); 137 static void ubsec_kshift_l(u_int, u_int8_t *, u_int, u_int8_t *, u_int); 138 139 140 static device_method_t ubsec_methods[] = { 141 /* Device interface */ 142 DEVMETHOD(device_probe, ubsec_probe), 143 DEVMETHOD(device_attach, ubsec_attach), 144 DEVMETHOD(device_detach, ubsec_detach), 145 DEVMETHOD(device_suspend, ubsec_suspend), 146 DEVMETHOD(device_resume, ubsec_resume), 147 DEVMETHOD(device_shutdown, ubsec_shutdown), 148 149 /* bus interface */ 150 DEVMETHOD(bus_print_child, bus_generic_print_child), 151 DEVMETHOD(bus_driver_added, bus_generic_driver_added), 152 153 /* crypto device methods */ 154 DEVMETHOD(cryptodev_newsession, ubsec_newsession), 155 DEVMETHOD(cryptodev_freesession,ubsec_freesession), 156 DEVMETHOD(cryptodev_process, ubsec_process), 157 DEVMETHOD(cryptodev_kprocess, ubsec_kprocess), 158 159 { 0, 0 } 160 }; 161 static driver_t ubsec_driver = { 162 "ubsec", 163 ubsec_methods, 164 sizeof (struct ubsec_softc) 165 }; 166 static devclass_t ubsec_devclass; 167 168 DECLARE_DUMMY_MODULE(ubsec); 169 DRIVER_MODULE(ubsec, pci, ubsec_driver, ubsec_devclass, 0, 0); 170 MODULE_DEPEND(ubsec, crypto, 1, 1, 1); 171 #ifdef UBSEC_RNDTEST 172 MODULE_DEPEND(ubsec, rndtest, 1, 1, 1); 173 #endif 174 175 SYSCTL_NODE(_hw, OID_AUTO, ubsec, CTLFLAG_RD, 0, "Broadcom driver parameters"); 176 177 #ifdef UBSEC_DEBUG 178 static void ubsec_dump_pb(volatile struct ubsec_pktbuf *); 179 static void ubsec_dump_mcr(struct ubsec_mcr *); 180 static void ubsec_dump_ctx2(struct ubsec_ctx_keyop *); 181 182 static int ubsec_debug = 0; 183 SYSCTL_INT(_hw_ubsec, OID_AUTO, debug, CTLFLAG_RW, &ubsec_debug, 184 0, "control debugging msgs"); 185 #endif 186 187 #define READ_REG(sc,r) \ 188 bus_space_read_4((sc)->sc_st, (sc)->sc_sh, (r)) 189 190 #define WRITE_REG(sc,reg,val) \ 191 bus_space_write_4((sc)->sc_st, (sc)->sc_sh, reg, val) 192 193 #define SWAP32(x) (x) = htole32(ntohl((x))) 194 #define HTOLE32(x) (x) = htole32(x) 195 196 197 struct ubsec_stats ubsecstats; 198 SYSCTL_STRUCT(_hw_ubsec, OID_AUTO, stats, CTLFLAG_RD, &ubsecstats, 199 ubsec_stats, "driver statistics"); 200 201 static int 202 ubsec_probe(device_t dev) 203 { 204 if (pci_get_vendor(dev) == PCI_VENDOR_SUN && 205 (pci_get_device(dev) == PCI_PRODUCT_SUN_5821 || 206 pci_get_device(dev) == PCI_PRODUCT_SUN_SCA1K)) 207 return (0); 208 if (pci_get_vendor(dev) == PCI_VENDOR_BLUESTEEL && 209 (pci_get_device(dev) == PCI_PRODUCT_BLUESTEEL_5501 || 210 pci_get_device(dev) == PCI_PRODUCT_BLUESTEEL_5601)) 211 return (0); 212 if (pci_get_vendor(dev) == PCI_VENDOR_BROADCOM && 213 (pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5801 || 214 pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5802 || 215 pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5805 || 216 pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5820 || 217 pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5821 || 218 pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5822 || 219 pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5823 220 )) 221 return (0); 222 return (ENXIO); 223 } 224 225 static const char* 226 ubsec_partname(struct ubsec_softc *sc) 227 { 228 /* XXX sprintf numbers when not decoded */ 229 switch (pci_get_vendor(sc->sc_dev)) { 230 case PCI_VENDOR_BROADCOM: 231 switch (pci_get_device(sc->sc_dev)) { 232 case PCI_PRODUCT_BROADCOM_5801: return "Broadcom 5801"; 233 case PCI_PRODUCT_BROADCOM_5802: return "Broadcom 5802"; 234 case PCI_PRODUCT_BROADCOM_5805: return "Broadcom 5805"; 235 case PCI_PRODUCT_BROADCOM_5820: return "Broadcom 5820"; 236 case PCI_PRODUCT_BROADCOM_5821: return "Broadcom 5821"; 237 case PCI_PRODUCT_BROADCOM_5822: return "Broadcom 5822"; 238 case PCI_PRODUCT_BROADCOM_5823: return "Broadcom 5823"; 239 } 240 return "Broadcom unknown-part"; 241 case PCI_VENDOR_BLUESTEEL: 242 switch (pci_get_device(sc->sc_dev)) { 243 case PCI_PRODUCT_BLUESTEEL_5601: return "Bluesteel 5601"; 244 } 245 return "Bluesteel unknown-part"; 246 case PCI_VENDOR_SUN: 247 switch (pci_get_device(sc->sc_dev)) { 248 case PCI_PRODUCT_SUN_5821: return "Sun Crypto 5821"; 249 case PCI_PRODUCT_SUN_SCA1K: return "Sun Crypto 1K"; 250 } 251 return "Sun unknown-part"; 252 } 253 return "Unknown-vendor unknown-part"; 254 } 255 256 static void 257 default_harvest(struct rndtest_state *rsp, void *buf, u_int count) 258 { 259 u_int32_t *p = (u_int32_t *)buf; 260 for (count /= sizeof (u_int32_t); count; count--) 261 add_true_randomness(*p++); 262 } 263 264 static int 265 ubsec_attach(device_t dev) 266 { 267 struct ubsec_softc *sc = device_get_softc(dev); 268 struct ubsec_dma *dmap; 269 u_int32_t cmd, i; 270 int rid; 271 272 KASSERT(sc != NULL, ("ubsec_attach: null software carrier!")); 273 bzero(sc, sizeof (*sc)); 274 sc->sc_dev = dev; 275 276 SIMPLEQ_INIT(&sc->sc_queue); 277 SIMPLEQ_INIT(&sc->sc_qchip); 278 SIMPLEQ_INIT(&sc->sc_queue2); 279 SIMPLEQ_INIT(&sc->sc_qchip2); 280 SIMPLEQ_INIT(&sc->sc_q2free); 281 282 /* XXX handle power management */ 283 284 sc->sc_statmask = BS_STAT_MCR1_DONE | BS_STAT_DMAERR; 285 286 if (pci_get_vendor(dev) == PCI_VENDOR_BLUESTEEL && 287 pci_get_device(dev) == PCI_PRODUCT_BLUESTEEL_5601) 288 sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG; 289 290 if (pci_get_vendor(dev) == PCI_VENDOR_BROADCOM && 291 (pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5802 || 292 pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5805)) 293 sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG; 294 295 if (pci_get_vendor(dev) == PCI_VENDOR_BROADCOM && 296 pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5820) 297 sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG | 298 UBS_FLAGS_LONGCTX | UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY; 299 300 if ((pci_get_vendor(dev) == PCI_VENDOR_BROADCOM && 301 (pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5821 || 302 pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5822 || 303 pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5823)) || 304 (pci_get_vendor(dev) == PCI_VENDOR_SUN && 305 (pci_get_device(dev) == PCI_PRODUCT_SUN_SCA1K || 306 pci_get_device(dev) == PCI_PRODUCT_SUN_5821))) { 307 /* NB: the 5821/5822 defines some additional status bits */ 308 sc->sc_statmask |= BS_STAT_MCR1_ALLEMPTY | 309 BS_STAT_MCR2_ALLEMPTY; 310 sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG | 311 UBS_FLAGS_LONGCTX | UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY; 312 } 313 314 cmd = pci_read_config(dev, PCIR_COMMAND, 4); 315 cmd |= PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN; 316 pci_write_config(dev, PCIR_COMMAND, cmd, 4); 317 cmd = pci_read_config(dev, PCIR_COMMAND, 4); 318 319 if (!(cmd & PCIM_CMD_MEMEN)) { 320 device_printf(dev, "failed to enable memory mapping\n"); 321 goto bad; 322 } 323 324 if (!(cmd & PCIM_CMD_BUSMASTEREN)) { 325 device_printf(dev, "failed to enable bus mastering\n"); 326 goto bad; 327 } 328 329 /* 330 * Setup memory-mapping of PCI registers. 331 */ 332 rid = BS_BAR; 333 sc->sc_sr = bus_alloc_resource(dev, SYS_RES_MEMORY, &rid, 334 0, ~0, 1, RF_ACTIVE); 335 if (sc->sc_sr == NULL) { 336 device_printf(dev, "cannot map register space\n"); 337 goto bad; 338 } 339 sc->sc_st = rman_get_bustag(sc->sc_sr); 340 sc->sc_sh = rman_get_bushandle(sc->sc_sr); 341 342 /* 343 * Arrange interrupt line. 344 */ 345 rid = 0; 346 sc->sc_irq = bus_alloc_resource(dev, SYS_RES_IRQ, &rid, 347 0, ~0, 1, RF_SHAREABLE|RF_ACTIVE); 348 if (sc->sc_irq == NULL) { 349 device_printf(dev, "could not map interrupt\n"); 350 goto bad1; 351 } 352 /* 353 * NB: Network code assumes we are blocked with splimp() 354 * so make sure the IRQ is mapped appropriately. 355 */ 356 if (bus_setup_intr(dev, sc->sc_irq, 0, 357 ubsec_intr, sc, 358 &sc->sc_ih, NULL)) { 359 device_printf(dev, "could not establish interrupt\n"); 360 goto bad2; 361 } 362 363 sc->sc_cid = crypto_get_driverid(dev, CRYPTOCAP_F_HARDWARE); 364 if (sc->sc_cid < 0) { 365 device_printf(dev, "could not get crypto driver id\n"); 366 goto bad3; 367 } 368 369 /* 370 * Setup DMA descriptor area. 371 */ 372 if (bus_dma_tag_create(NULL, /* parent */ 373 1, 0, /* alignment, bounds */ 374 BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ 375 BUS_SPACE_MAXADDR, /* highaddr */ 376 NULL, NULL, /* filter, filterarg */ 377 0x3ffff, /* maxsize */ 378 UBS_MAX_SCATTER, /* nsegments */ 379 0xffff, /* maxsegsize */ 380 BUS_DMA_ALLOCNOW, /* flags */ 381 &sc->sc_dmat)) { 382 device_printf(dev, "cannot allocate DMA tag\n"); 383 goto bad4; 384 } 385 SIMPLEQ_INIT(&sc->sc_freequeue); 386 dmap = sc->sc_dmaa; 387 for (i = 0; i < UBS_MAX_NQUEUE; i++, dmap++) { 388 struct ubsec_q *q; 389 390 q = kmalloc(sizeof(struct ubsec_q), M_DEVBUF, M_WAITOK); 391 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_dmachunk), 392 &dmap->d_alloc, 0)) { 393 device_printf(dev, "cannot allocate dma buffers\n"); 394 kfree(q, M_DEVBUF); 395 break; 396 } 397 dmap->d_dma = (struct ubsec_dmachunk *)dmap->d_alloc.dma_vaddr; 398 399 q->q_dma = dmap; 400 sc->sc_queuea[i] = q; 401 402 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next); 403 } 404 405 device_printf(sc->sc_dev, "%s\n", ubsec_partname(sc)); 406 407 crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0); 408 crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0); 409 crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0); 410 crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0); 411 412 /* 413 * Reset Broadcom chip 414 */ 415 ubsec_reset_board(sc); 416 417 /* 418 * Init Broadcom specific PCI settings 419 */ 420 ubsec_init_pciregs(dev); 421 422 /* 423 * Init Broadcom chip 424 */ 425 ubsec_init_board(sc); 426 427 #ifndef UBSEC_NO_RNG 428 if (sc->sc_flags & UBS_FLAGS_RNG) { 429 sc->sc_statmask |= BS_STAT_MCR2_DONE; 430 #ifdef UBSEC_RNDTEST 431 sc->sc_rndtest = rndtest_attach(dev); 432 if (sc->sc_rndtest) 433 sc->sc_harvest = rndtest_harvest; 434 else 435 sc->sc_harvest = default_harvest; 436 #else 437 sc->sc_harvest = default_harvest; 438 #endif 439 440 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr), 441 &sc->sc_rng.rng_q.q_mcr, 0)) 442 goto skip_rng; 443 444 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_rngbypass), 445 &sc->sc_rng.rng_q.q_ctx, 0)) { 446 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr); 447 goto skip_rng; 448 } 449 450 if (ubsec_dma_malloc(sc, sizeof(u_int32_t) * 451 UBSEC_RNG_BUFSIZ, &sc->sc_rng.rng_buf, 0)) { 452 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_ctx); 453 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr); 454 goto skip_rng; 455 } 456 457 if (hz >= 100) 458 sc->sc_rnghz = hz / 100; 459 else 460 sc->sc_rnghz = 1; 461 callout_init(&sc->sc_rngto); 462 callout_reset(&sc->sc_rngto, sc->sc_rnghz, ubsec_rng, sc); 463 skip_rng: 464 ; 465 } 466 #endif /* UBSEC_NO_RNG */ 467 468 if (sc->sc_flags & UBS_FLAGS_KEY) { 469 sc->sc_statmask |= BS_STAT_MCR2_DONE; 470 471 crypto_kregister(sc->sc_cid, CRK_MOD_EXP, 0); 472 #if 0 473 crypto_kregister(sc->sc_cid, CRK_MOD_EXP_CRT, 0); 474 #endif 475 } 476 return (0); 477 bad4: 478 crypto_unregister_all(sc->sc_cid); 479 bad3: 480 bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih); 481 bad2: 482 bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq); 483 bad1: 484 bus_release_resource(dev, SYS_RES_MEMORY, BS_BAR, sc->sc_sr); 485 bad: 486 return (ENXIO); 487 } 488 489 /* 490 * Detach a device that successfully probed. 491 */ 492 static int 493 ubsec_detach(device_t dev) 494 { 495 struct ubsec_softc *sc = device_get_softc(dev); 496 497 KASSERT(sc != NULL, ("ubsec_detach: null software carrier")); 498 499 /* XXX wait/abort active ops */ 500 501 crit_enter(); 502 503 callout_stop(&sc->sc_rngto); 504 505 crypto_unregister_all(sc->sc_cid); 506 507 #ifdef UBSEC_RNDTEST 508 if (sc->sc_rndtest) 509 rndtest_detach(sc->sc_rndtest); 510 #endif 511 512 while (!SIMPLEQ_EMPTY(&sc->sc_freequeue)) { 513 struct ubsec_q *q; 514 515 q = SIMPLEQ_FIRST(&sc->sc_freequeue); 516 SIMPLEQ_REMOVE_HEAD(&sc->sc_freequeue, q_next); 517 ubsec_dma_free(sc, &q->q_dma->d_alloc); 518 kfree(q, M_DEVBUF); 519 } 520 #ifndef UBSEC_NO_RNG 521 if (sc->sc_flags & UBS_FLAGS_RNG) { 522 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr); 523 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_ctx); 524 ubsec_dma_free(sc, &sc->sc_rng.rng_buf); 525 } 526 #endif /* UBSEC_NO_RNG */ 527 528 bus_generic_detach(dev); 529 bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih); 530 bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq); 531 532 bus_dma_tag_destroy(sc->sc_dmat); 533 bus_release_resource(dev, SYS_RES_MEMORY, BS_BAR, sc->sc_sr); 534 535 crit_exit(); 536 537 return (0); 538 } 539 540 /* 541 * Stop all chip i/o so that the kernel's probe routines don't 542 * get confused by errant DMAs when rebooting. 543 */ 544 static void 545 ubsec_shutdown(device_t dev) 546 { 547 #ifdef notyet 548 ubsec_stop(device_get_softc(dev)); 549 #endif 550 } 551 552 /* 553 * Device suspend routine. 554 */ 555 static int 556 ubsec_suspend(device_t dev) 557 { 558 struct ubsec_softc *sc = device_get_softc(dev); 559 560 KASSERT(sc != NULL, ("ubsec_suspend: null software carrier")); 561 #ifdef notyet 562 /* XXX stop the device and save PCI settings */ 563 #endif 564 sc->sc_suspended = 1; 565 566 return (0); 567 } 568 569 static int 570 ubsec_resume(device_t dev) 571 { 572 struct ubsec_softc *sc = device_get_softc(dev); 573 574 KASSERT(sc != NULL, ("ubsec_resume: null software carrier")); 575 #ifdef notyet 576 /* XXX retore PCI settings and start the device */ 577 #endif 578 sc->sc_suspended = 0; 579 return (0); 580 } 581 582 /* 583 * UBSEC Interrupt routine 584 */ 585 static void 586 ubsec_intr(void *arg) 587 { 588 struct ubsec_softc *sc = arg; 589 volatile u_int32_t stat; 590 struct ubsec_q *q; 591 struct ubsec_dma *dmap; 592 int npkts = 0, i; 593 594 stat = READ_REG(sc, BS_STAT); 595 stat &= sc->sc_statmask; 596 if (stat == 0) { 597 return; 598 } 599 600 WRITE_REG(sc, BS_STAT, stat); /* IACK */ 601 602 /* 603 * Check to see if we have any packets waiting for us 604 */ 605 if ((stat & BS_STAT_MCR1_DONE)) { 606 while (!SIMPLEQ_EMPTY(&sc->sc_qchip)) { 607 q = SIMPLEQ_FIRST(&sc->sc_qchip); 608 dmap = q->q_dma; 609 610 if ((dmap->d_dma->d_mcr.mcr_flags & htole16(UBS_MCR_DONE)) == 0) 611 break; 612 613 SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, q_next); 614 615 npkts = q->q_nstacked_mcrs; 616 sc->sc_nqchip -= 1+npkts; 617 /* 618 * search for further sc_qchip ubsec_q's that share 619 * the same MCR, and complete them too, they must be 620 * at the top. 621 */ 622 for (i = 0; i < npkts; i++) { 623 if(q->q_stacked_mcr[i]) { 624 ubsec_callback(sc, q->q_stacked_mcr[i]); 625 } else { 626 break; 627 } 628 } 629 ubsec_callback(sc, q); 630 } 631 632 /* 633 * Don't send any more packet to chip if there has been 634 * a DMAERR. 635 */ 636 if (!(stat & BS_STAT_DMAERR)) 637 ubsec_feed(sc); 638 } 639 640 /* 641 * Check to see if we have any key setups/rng's waiting for us 642 */ 643 if ((sc->sc_flags & (UBS_FLAGS_KEY|UBS_FLAGS_RNG)) && 644 (stat & BS_STAT_MCR2_DONE)) { 645 struct ubsec_q2 *q2; 646 struct ubsec_mcr *mcr; 647 648 while (!SIMPLEQ_EMPTY(&sc->sc_qchip2)) { 649 q2 = SIMPLEQ_FIRST(&sc->sc_qchip2); 650 651 ubsec_dma_sync(&q2->q_mcr, 652 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 653 654 mcr = (struct ubsec_mcr *)q2->q_mcr.dma_vaddr; 655 if ((mcr->mcr_flags & htole16(UBS_MCR_DONE)) == 0) { 656 ubsec_dma_sync(&q2->q_mcr, 657 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 658 break; 659 } 660 SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip2, q_next); 661 ubsec_callback2(sc, q2); 662 /* 663 * Don't send any more packet to chip if there has been 664 * a DMAERR. 665 */ 666 if (!(stat & BS_STAT_DMAERR)) 667 ubsec_feed2(sc); 668 } 669 } 670 671 /* 672 * Check to see if we got any DMA Error 673 */ 674 if (stat & BS_STAT_DMAERR) { 675 #ifdef UBSEC_DEBUG 676 if (ubsec_debug) { 677 volatile u_int32_t a = READ_REG(sc, BS_ERR); 678 679 kprintf("dmaerr %s@%08x\n", 680 (a & BS_ERR_READ) ? "read" : "write", 681 a & BS_ERR_ADDR); 682 } 683 #endif /* UBSEC_DEBUG */ 684 ubsecstats.hst_dmaerr++; 685 ubsec_totalreset(sc); 686 ubsec_feed(sc); 687 } 688 689 if (sc->sc_needwakeup) { /* XXX check high watermark */ 690 int wakeup = sc->sc_needwakeup & (CRYPTO_SYMQ|CRYPTO_ASYMQ); 691 #ifdef UBSEC_DEBUG 692 if (ubsec_debug) 693 device_printf(sc->sc_dev, "wakeup crypto (%x)\n", 694 sc->sc_needwakeup); 695 #endif /* UBSEC_DEBUG */ 696 sc->sc_needwakeup &= ~wakeup; 697 crypto_unblock(sc->sc_cid, wakeup); 698 } 699 } 700 701 /* 702 * ubsec_feed() - aggregate and post requests to chip 703 */ 704 static void 705 ubsec_feed(struct ubsec_softc *sc) 706 { 707 struct ubsec_q *q, *q2; 708 int npkts, i; 709 void *v; 710 u_int32_t stat; 711 712 /* 713 * Decide how many ops to combine in a single MCR. We cannot 714 * aggregate more than UBS_MAX_AGGR because this is the number 715 * of slots defined in the data structure. Note that 716 * aggregation only happens if ops are marked batch'able. 717 * Aggregating ops reduces the number of interrupts to the host 718 * but also (potentially) increases the latency for processing 719 * completed ops as we only get an interrupt when all aggregated 720 * ops have completed. 721 */ 722 if (sc->sc_nqueue == 0) 723 return; 724 if (sc->sc_nqueue > 1) { 725 npkts = 0; 726 SIMPLEQ_FOREACH(q, &sc->sc_queue, q_next) { 727 npkts++; 728 if ((q->q_crp->crp_flags & CRYPTO_F_BATCH) == 0) 729 break; 730 } 731 } else 732 npkts = 1; 733 /* 734 * Check device status before going any further. 735 */ 736 if ((stat = READ_REG(sc, BS_STAT)) & (BS_STAT_MCR1_FULL | BS_STAT_DMAERR)) { 737 if (stat & BS_STAT_DMAERR) { 738 ubsec_totalreset(sc); 739 ubsecstats.hst_dmaerr++; 740 } else 741 ubsecstats.hst_mcr1full++; 742 return; 743 } 744 if (sc->sc_nqueue > ubsecstats.hst_maxqueue) 745 ubsecstats.hst_maxqueue = sc->sc_nqueue; 746 if (npkts > UBS_MAX_AGGR) 747 npkts = UBS_MAX_AGGR; 748 if (npkts < 2) /* special case 1 op */ 749 goto feed1; 750 751 ubsecstats.hst_totbatch += npkts-1; 752 #ifdef UBSEC_DEBUG 753 if (ubsec_debug) 754 kprintf("merging %d records\n", npkts); 755 #endif /* UBSEC_DEBUG */ 756 757 q = SIMPLEQ_FIRST(&sc->sc_queue); 758 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q_next); 759 --sc->sc_nqueue; 760 761 bus_dmamap_sync(sc->sc_dmat, q->q_src_map, BUS_DMASYNC_PREWRITE); 762 if (q->q_dst_map != NULL) 763 bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, BUS_DMASYNC_PREREAD); 764 765 q->q_nstacked_mcrs = npkts - 1; /* Number of packets stacked */ 766 767 for (i = 0; i < q->q_nstacked_mcrs; i++) { 768 q2 = SIMPLEQ_FIRST(&sc->sc_queue); 769 bus_dmamap_sync(sc->sc_dmat, q2->q_src_map, 770 BUS_DMASYNC_PREWRITE); 771 if (q2->q_dst_map != NULL) 772 bus_dmamap_sync(sc->sc_dmat, q2->q_dst_map, 773 BUS_DMASYNC_PREREAD); 774 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q_next); 775 --sc->sc_nqueue; 776 777 v = (void*)(((char *)&q2->q_dma->d_dma->d_mcr) + sizeof(struct ubsec_mcr) - 778 sizeof(struct ubsec_mcr_add)); 779 bcopy(v, &q->q_dma->d_dma->d_mcradd[i], sizeof(struct ubsec_mcr_add)); 780 q->q_stacked_mcr[i] = q2; 781 } 782 q->q_dma->d_dma->d_mcr.mcr_pkts = htole16(npkts); 783 SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next); 784 sc->sc_nqchip += npkts; 785 if (sc->sc_nqchip > ubsecstats.hst_maxqchip) 786 ubsecstats.hst_maxqchip = sc->sc_nqchip; 787 ubsec_dma_sync(&q->q_dma->d_alloc, 788 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 789 WRITE_REG(sc, BS_MCR1, q->q_dma->d_alloc.dma_paddr + 790 offsetof(struct ubsec_dmachunk, d_mcr)); 791 return; 792 793 feed1: 794 q = SIMPLEQ_FIRST(&sc->sc_queue); 795 796 bus_dmamap_sync(sc->sc_dmat, q->q_src_map, BUS_DMASYNC_PREWRITE); 797 if (q->q_dst_map != NULL) 798 bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, BUS_DMASYNC_PREREAD); 799 ubsec_dma_sync(&q->q_dma->d_alloc, 800 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 801 802 WRITE_REG(sc, BS_MCR1, q->q_dma->d_alloc.dma_paddr + 803 offsetof(struct ubsec_dmachunk, d_mcr)); 804 #ifdef UBSEC_DEBUG 805 if (ubsec_debug) 806 kprintf("feed1: q->chip %p %08x stat %08x\n", 807 q, (u_int32_t)vtophys(&q->q_dma->d_dma->d_mcr), 808 stat); 809 #endif /* UBSEC_DEBUG */ 810 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q_next); 811 --sc->sc_nqueue; 812 SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next); 813 sc->sc_nqchip++; 814 if (sc->sc_nqchip > ubsecstats.hst_maxqchip) 815 ubsecstats.hst_maxqchip = sc->sc_nqchip; 816 return; 817 } 818 819 static void 820 ubsec_setup_enckey(struct ubsec_session *ses, int algo, caddr_t key) 821 { 822 823 /* Go ahead and compute key in ubsec's byte order */ 824 if (algo == CRYPTO_DES_CBC) { 825 bcopy(key, &ses->ses_deskey[0], 8); 826 bcopy(key, &ses->ses_deskey[2], 8); 827 bcopy(key, &ses->ses_deskey[4], 8); 828 } else 829 bcopy(key, ses->ses_deskey, 24); 830 831 SWAP32(ses->ses_deskey[0]); 832 SWAP32(ses->ses_deskey[1]); 833 SWAP32(ses->ses_deskey[2]); 834 SWAP32(ses->ses_deskey[3]); 835 SWAP32(ses->ses_deskey[4]); 836 SWAP32(ses->ses_deskey[5]); 837 } 838 839 static void 840 ubsec_setup_mackey(struct ubsec_session *ses, int algo, caddr_t key, int klen) 841 { 842 MD5_CTX md5ctx; 843 SHA1_CTX sha1ctx; 844 int i; 845 846 for (i = 0; i < klen; i++) 847 key[i] ^= HMAC_IPAD_VAL; 848 849 if (algo == CRYPTO_MD5_HMAC) { 850 MD5Init(&md5ctx); 851 MD5Update(&md5ctx, key, klen); 852 MD5Update(&md5ctx, hmac_ipad_buffer, MD5_HMAC_BLOCK_LEN - klen); 853 bcopy(md5ctx.state, ses->ses_hminner, sizeof(md5ctx.state)); 854 } else { 855 SHA1Init(&sha1ctx); 856 SHA1Update(&sha1ctx, key, klen); 857 SHA1Update(&sha1ctx, hmac_ipad_buffer, 858 SHA1_HMAC_BLOCK_LEN - klen); 859 bcopy(sha1ctx.h.b32, ses->ses_hminner, sizeof(sha1ctx.h.b32)); 860 } 861 862 for (i = 0; i < klen; i++) 863 key[i] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL); 864 865 if (algo == CRYPTO_MD5_HMAC) { 866 MD5Init(&md5ctx); 867 MD5Update(&md5ctx, key, klen); 868 MD5Update(&md5ctx, hmac_opad_buffer, MD5_HMAC_BLOCK_LEN - klen); 869 bcopy(md5ctx.state, ses->ses_hmouter, sizeof(md5ctx.state)); 870 } else { 871 SHA1Init(&sha1ctx); 872 SHA1Update(&sha1ctx, key, klen); 873 SHA1Update(&sha1ctx, hmac_opad_buffer, 874 SHA1_HMAC_BLOCK_LEN - klen); 875 bcopy(sha1ctx.h.b32, ses->ses_hmouter, sizeof(sha1ctx.h.b32)); 876 } 877 878 for (i = 0; i < klen; i++) 879 key[i] ^= HMAC_OPAD_VAL; 880 } 881 882 /* 883 * Allocate a new 'session' and return an encoded session id. 'sidp' 884 * contains our registration id, and should contain an encoded session 885 * id on successful allocation. 886 */ 887 static int 888 ubsec_newsession(void *arg, u_int32_t *sidp, struct cryptoini *cri) 889 { 890 struct cryptoini *c, *encini = NULL, *macini = NULL; 891 struct ubsec_softc *sc = arg; 892 struct ubsec_session *ses = NULL; 893 int sesn; 894 #if 0 895 MD5_CTX md5ctx; 896 SHA1_CTX sha1ctx; 897 int i; 898 #endif 899 900 KASSERT(sc != NULL, ("ubsec_newsession: null softc")); 901 if (sidp == NULL || cri == NULL || sc == NULL) 902 return (EINVAL); 903 904 for (c = cri; c != NULL; c = c->cri_next) { 905 if (c->cri_alg == CRYPTO_MD5_HMAC || 906 c->cri_alg == CRYPTO_SHA1_HMAC) { 907 if (macini) 908 return (EINVAL); 909 macini = c; 910 } else if (c->cri_alg == CRYPTO_DES_CBC || 911 c->cri_alg == CRYPTO_3DES_CBC) { 912 if (encini) 913 return (EINVAL); 914 encini = c; 915 } else 916 return (EINVAL); 917 } 918 if (encini == NULL && macini == NULL) 919 return (EINVAL); 920 921 if (sc->sc_sessions == NULL) { 922 ses = sc->sc_sessions = kmalloc(sizeof(struct ubsec_session), 923 M_DEVBUF, M_INTWAIT); 924 sesn = 0; 925 sc->sc_nsessions = 1; 926 } else { 927 for (sesn = 0; sesn < sc->sc_nsessions; sesn++) { 928 if (sc->sc_sessions[sesn].ses_used == 0) { 929 ses = &sc->sc_sessions[sesn]; 930 break; 931 } 932 } 933 934 if (ses == NULL) { 935 sesn = sc->sc_nsessions; 936 ses = kmalloc((sesn + 1) * sizeof(struct ubsec_session), 937 M_DEVBUF, M_INTWAIT); 938 bcopy(sc->sc_sessions, ses, sesn * 939 sizeof(struct ubsec_session)); 940 bzero(sc->sc_sessions, sesn * 941 sizeof(struct ubsec_session)); 942 kfree(sc->sc_sessions, M_DEVBUF); 943 sc->sc_sessions = ses; 944 ses = &sc->sc_sessions[sesn]; 945 sc->sc_nsessions++; 946 } 947 } 948 949 bzero(ses, sizeof(struct ubsec_session)); 950 ses->ses_used = 1; 951 if (encini) { 952 read_random(ses->ses_iv, sizeof(ses->ses_iv)); 953 if (encini->cri_key != NULL) { 954 ubsec_setup_enckey(ses, encini->cri_alg, 955 encini->cri_key); 956 } 957 #if 0 958 /* get an IV, network byte order */ 959 /* XXX may read fewer than requested */ 960 read_random(ses->ses_iv, sizeof(ses->ses_iv)); 961 962 /* Go ahead and compute key in ubsec's byte order */ 963 if (encini->cri_alg == CRYPTO_DES_CBC) { 964 bcopy(encini->cri_key, &ses->ses_deskey[0], 8); 965 bcopy(encini->cri_key, &ses->ses_deskey[2], 8); 966 bcopy(encini->cri_key, &ses->ses_deskey[4], 8); 967 } else 968 bcopy(encini->cri_key, ses->ses_deskey, 24); 969 970 SWAP32(ses->ses_deskey[0]); 971 SWAP32(ses->ses_deskey[1]); 972 SWAP32(ses->ses_deskey[2]); 973 SWAP32(ses->ses_deskey[3]); 974 SWAP32(ses->ses_deskey[4]); 975 SWAP32(ses->ses_deskey[5]); 976 #endif 977 } 978 979 if (macini) { 980 ses->ses_mlen = macini->cri_mlen; 981 if (ses->ses_mlen == 0) { 982 if (macini->cri_alg == CRYPTO_MD5_HMAC) 983 ses->ses_mlen = MD5_HASH_LEN; 984 else 985 ses->ses_mlen = SHA1_HASH_LEN; 986 } 987 988 if (macini->cri_key != NULL) { 989 ubsec_setup_mackey(ses, macini->cri_alg, 990 macini->cri_key, macini->cri_klen/8); 991 } 992 #if 0 993 for (i = 0; i < macini->cri_klen / 8; i++) 994 macini->cri_key[i] ^= HMAC_IPAD_VAL; 995 996 if (macini->cri_alg == CRYPTO_MD5_HMAC) { 997 MD5Init(&md5ctx); 998 MD5Update(&md5ctx, macini->cri_key, 999 macini->cri_klen / 8); 1000 MD5Update(&md5ctx, hmac_ipad_buffer, 1001 MD5_HMAC_BLOCK_LEN - (macini->cri_klen / 8)); 1002 bcopy(md5ctx.state, ses->ses_hminner, 1003 sizeof(md5ctx.state)); 1004 } else { 1005 SHA1Init(&sha1ctx); 1006 SHA1Update(&sha1ctx, macini->cri_key, 1007 macini->cri_klen / 8); 1008 SHA1Update(&sha1ctx, hmac_ipad_buffer, 1009 SHA1_HMAC_BLOCK_LEN - (macini->cri_klen / 8)); 1010 bcopy(sha1ctx.h.b32, ses->ses_hminner, 1011 sizeof(sha1ctx.h.b32)); 1012 } 1013 1014 for (i = 0; i < macini->cri_klen / 8; i++) 1015 macini->cri_key[i] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL); 1016 1017 if (macini->cri_alg == CRYPTO_MD5_HMAC) { 1018 MD5Init(&md5ctx); 1019 MD5Update(&md5ctx, macini->cri_key, 1020 macini->cri_klen / 8); 1021 MD5Update(&md5ctx, hmac_opad_buffer, 1022 MD5_HMAC_BLOCK_LEN - (macini->cri_klen / 8)); 1023 bcopy(md5ctx.state, ses->ses_hmouter, 1024 sizeof(md5ctx.state)); 1025 } else { 1026 SHA1Init(&sha1ctx); 1027 SHA1Update(&sha1ctx, macini->cri_key, 1028 macini->cri_klen / 8); 1029 SHA1Update(&sha1ctx, hmac_opad_buffer, 1030 SHA1_HMAC_BLOCK_LEN - (macini->cri_klen / 8)); 1031 bcopy(sha1ctx.h.b32, ses->ses_hmouter, 1032 sizeof(sha1ctx.h.b32)); 1033 } 1034 1035 for (i = 0; i < macini->cri_klen / 8; i++) 1036 macini->cri_key[i] ^= HMAC_OPAD_VAL; 1037 #endif 1038 } 1039 1040 *sidp = UBSEC_SID(device_get_unit(sc->sc_dev), sesn); 1041 return (0); 1042 } 1043 1044 /* 1045 * Deallocate a session. 1046 */ 1047 static int 1048 ubsec_freesession(void *arg, u_int64_t tid) 1049 { 1050 struct ubsec_softc *sc = arg; 1051 int session; 1052 u_int32_t sid = ((u_int32_t) tid) & 0xffffffff; 1053 1054 KASSERT(sc != NULL, ("ubsec_freesession: null softc")); 1055 if (sc == NULL) 1056 return (EINVAL); 1057 1058 session = UBSEC_SESSION(sid); 1059 if (session >= sc->sc_nsessions) 1060 return (EINVAL); 1061 1062 bzero(&sc->sc_sessions[session], sizeof(sc->sc_sessions[session])); 1063 return (0); 1064 } 1065 1066 static void 1067 ubsec_op_cb(void *arg, bus_dma_segment_t *seg, int nsegs, bus_size_t mapsize, int error) 1068 { 1069 struct ubsec_operand *op = arg; 1070 1071 KASSERT(nsegs <= UBS_MAX_SCATTER, 1072 ("Too many DMA segments returned when mapping operand")); 1073 #ifdef UBSEC_DEBUG 1074 if (ubsec_debug) 1075 kprintf("ubsec_op_cb: mapsize %u nsegs %d\n", 1076 (u_int) mapsize, nsegs); 1077 #endif 1078 op->mapsize = mapsize; 1079 op->nsegs = nsegs; 1080 bcopy(seg, op->segs, nsegs * sizeof (seg[0])); 1081 } 1082 1083 static int 1084 ubsec_process(void *arg, struct cryptop *crp, int hint) 1085 { 1086 struct ubsec_q *q = NULL; 1087 int err = 0, i, j, nicealign; 1088 struct ubsec_softc *sc = arg; 1089 struct cryptodesc *crd1, *crd2, *maccrd, *enccrd; 1090 int encoffset = 0, macoffset = 0, cpskip, cpoffset; 1091 int sskip, dskip, stheend, dtheend; 1092 int16_t coffset; 1093 struct ubsec_session *ses; 1094 struct ubsec_pktctx ctx; 1095 struct ubsec_dma *dmap = NULL; 1096 1097 if (crp == NULL || crp->crp_callback == NULL || sc == NULL) { 1098 ubsecstats.hst_invalid++; 1099 return (EINVAL); 1100 } 1101 if (UBSEC_SESSION(crp->crp_sid) >= sc->sc_nsessions) { 1102 ubsecstats.hst_badsession++; 1103 return (EINVAL); 1104 } 1105 1106 crit_enter(); 1107 1108 if (SIMPLEQ_EMPTY(&sc->sc_freequeue)) { 1109 ubsecstats.hst_queuefull++; 1110 sc->sc_needwakeup |= CRYPTO_SYMQ; 1111 crit_exit(); 1112 return (ERESTART); 1113 } 1114 q = SIMPLEQ_FIRST(&sc->sc_freequeue); 1115 SIMPLEQ_REMOVE_HEAD(&sc->sc_freequeue, q_next); 1116 crit_exit(); 1117 1118 dmap = q->q_dma; /* Save dma pointer */ 1119 bzero(q, sizeof(struct ubsec_q)); 1120 bzero(&ctx, sizeof(ctx)); 1121 1122 q->q_sesn = UBSEC_SESSION(crp->crp_sid); 1123 q->q_dma = dmap; 1124 ses = &sc->sc_sessions[q->q_sesn]; 1125 1126 if (crp->crp_flags & CRYPTO_F_IMBUF) { 1127 q->q_src_m = (struct mbuf *)crp->crp_buf; 1128 q->q_dst_m = (struct mbuf *)crp->crp_buf; 1129 } else if (crp->crp_flags & CRYPTO_F_IOV) { 1130 q->q_src_io = (struct uio *)crp->crp_buf; 1131 q->q_dst_io = (struct uio *)crp->crp_buf; 1132 } else { 1133 ubsecstats.hst_badflags++; 1134 err = EINVAL; 1135 goto errout; /* XXX we don't handle contiguous blocks! */ 1136 } 1137 1138 bzero(&dmap->d_dma->d_mcr, sizeof(struct ubsec_mcr)); 1139 1140 dmap->d_dma->d_mcr.mcr_pkts = htole16(1); 1141 dmap->d_dma->d_mcr.mcr_flags = 0; 1142 q->q_crp = crp; 1143 1144 crd1 = crp->crp_desc; 1145 if (crd1 == NULL) { 1146 ubsecstats.hst_nodesc++; 1147 err = EINVAL; 1148 goto errout; 1149 } 1150 crd2 = crd1->crd_next; 1151 1152 if (crd2 == NULL) { 1153 if (crd1->crd_alg == CRYPTO_MD5_HMAC || 1154 crd1->crd_alg == CRYPTO_SHA1_HMAC) { 1155 maccrd = crd1; 1156 enccrd = NULL; 1157 } else if (crd1->crd_alg == CRYPTO_DES_CBC || 1158 crd1->crd_alg == CRYPTO_3DES_CBC) { 1159 maccrd = NULL; 1160 enccrd = crd1; 1161 } else { 1162 ubsecstats.hst_badalg++; 1163 err = EINVAL; 1164 goto errout; 1165 } 1166 } else { 1167 if ((crd1->crd_alg == CRYPTO_MD5_HMAC || 1168 crd1->crd_alg == CRYPTO_SHA1_HMAC) && 1169 (crd2->crd_alg == CRYPTO_DES_CBC || 1170 crd2->crd_alg == CRYPTO_3DES_CBC) && 1171 ((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) { 1172 maccrd = crd1; 1173 enccrd = crd2; 1174 } else if ((crd1->crd_alg == CRYPTO_DES_CBC || 1175 crd1->crd_alg == CRYPTO_3DES_CBC) && 1176 (crd2->crd_alg == CRYPTO_MD5_HMAC || 1177 crd2->crd_alg == CRYPTO_SHA1_HMAC) && 1178 (crd1->crd_flags & CRD_F_ENCRYPT)) { 1179 enccrd = crd1; 1180 maccrd = crd2; 1181 } else { 1182 /* 1183 * We cannot order the ubsec as requested 1184 */ 1185 ubsecstats.hst_badalg++; 1186 err = EINVAL; 1187 goto errout; 1188 } 1189 } 1190 1191 if (enccrd) { 1192 encoffset = enccrd->crd_skip; 1193 ctx.pc_flags |= htole16(UBS_PKTCTX_ENC_3DES); 1194 1195 if (enccrd->crd_flags & CRD_F_ENCRYPT) { 1196 q->q_flags |= UBSEC_QFLAGS_COPYOUTIV; 1197 1198 if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) 1199 bcopy(enccrd->crd_iv, ctx.pc_iv, 8); 1200 else { 1201 ctx.pc_iv[0] = ses->ses_iv[0]; 1202 ctx.pc_iv[1] = ses->ses_iv[1]; 1203 } 1204 1205 if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) { 1206 if (crp->crp_flags & CRYPTO_F_IMBUF) 1207 m_copyback(q->q_src_m, 1208 enccrd->crd_inject, 1209 8, (caddr_t)ctx.pc_iv); 1210 else if (crp->crp_flags & CRYPTO_F_IOV) 1211 cuio_copyback(q->q_src_io, 1212 enccrd->crd_inject, 1213 8, (caddr_t)ctx.pc_iv); 1214 } 1215 } else { 1216 ctx.pc_flags |= htole16(UBS_PKTCTX_INBOUND); 1217 1218 if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) 1219 bcopy(enccrd->crd_iv, ctx.pc_iv, 8); 1220 else if (crp->crp_flags & CRYPTO_F_IMBUF) 1221 m_copydata(q->q_src_m, enccrd->crd_inject, 1222 8, (caddr_t)ctx.pc_iv); 1223 else if (crp->crp_flags & CRYPTO_F_IOV) 1224 cuio_copydata(q->q_src_io, 1225 enccrd->crd_inject, 8, 1226 (caddr_t)ctx.pc_iv); 1227 } 1228 1229 ctx.pc_deskey[0] = ses->ses_deskey[0]; 1230 ctx.pc_deskey[1] = ses->ses_deskey[1]; 1231 ctx.pc_deskey[2] = ses->ses_deskey[2]; 1232 ctx.pc_deskey[3] = ses->ses_deskey[3]; 1233 ctx.pc_deskey[4] = ses->ses_deskey[4]; 1234 ctx.pc_deskey[5] = ses->ses_deskey[5]; 1235 SWAP32(ctx.pc_iv[0]); 1236 SWAP32(ctx.pc_iv[1]); 1237 } 1238 1239 if (maccrd) { 1240 macoffset = maccrd->crd_skip; 1241 1242 if (maccrd->crd_alg == CRYPTO_MD5_HMAC) 1243 ctx.pc_flags |= htole16(UBS_PKTCTX_AUTH_MD5); 1244 else 1245 ctx.pc_flags |= htole16(UBS_PKTCTX_AUTH_SHA1); 1246 1247 for (i = 0; i < 5; i++) { 1248 ctx.pc_hminner[i] = ses->ses_hminner[i]; 1249 ctx.pc_hmouter[i] = ses->ses_hmouter[i]; 1250 1251 HTOLE32(ctx.pc_hminner[i]); 1252 HTOLE32(ctx.pc_hmouter[i]); 1253 } 1254 } 1255 1256 if (enccrd && maccrd) { 1257 /* 1258 * ubsec cannot handle packets where the end of encryption 1259 * and authentication are not the same, or where the 1260 * encrypted part begins before the authenticated part. 1261 */ 1262 if ((encoffset + enccrd->crd_len) != 1263 (macoffset + maccrd->crd_len)) { 1264 ubsecstats.hst_lenmismatch++; 1265 err = EINVAL; 1266 goto errout; 1267 } 1268 if (enccrd->crd_skip < maccrd->crd_skip) { 1269 ubsecstats.hst_skipmismatch++; 1270 err = EINVAL; 1271 goto errout; 1272 } 1273 sskip = maccrd->crd_skip; 1274 cpskip = dskip = enccrd->crd_skip; 1275 stheend = maccrd->crd_len; 1276 dtheend = enccrd->crd_len; 1277 coffset = enccrd->crd_skip - maccrd->crd_skip; 1278 cpoffset = cpskip + dtheend; 1279 #ifdef UBSEC_DEBUG 1280 if (ubsec_debug) { 1281 kprintf("mac: skip %d, len %d, inject %d\n", 1282 maccrd->crd_skip, maccrd->crd_len, maccrd->crd_inject); 1283 kprintf("enc: skip %d, len %d, inject %d\n", 1284 enccrd->crd_skip, enccrd->crd_len, enccrd->crd_inject); 1285 kprintf("src: skip %d, len %d\n", sskip, stheend); 1286 kprintf("dst: skip %d, len %d\n", dskip, dtheend); 1287 kprintf("ubs: coffset %d, pktlen %d, cpskip %d, cpoffset %d\n", 1288 coffset, stheend, cpskip, cpoffset); 1289 } 1290 #endif 1291 } else { 1292 cpskip = dskip = sskip = macoffset + encoffset; 1293 dtheend = stheend = (enccrd)?enccrd->crd_len:maccrd->crd_len; 1294 cpoffset = cpskip + dtheend; 1295 coffset = 0; 1296 } 1297 ctx.pc_offset = htole16(coffset >> 2); 1298 1299 if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT, &q->q_src_map)) { 1300 ubsecstats.hst_nomap++; 1301 err = ENOMEM; 1302 goto errout; 1303 } 1304 if (crp->crp_flags & CRYPTO_F_IMBUF) { 1305 if (bus_dmamap_load_mbuf(sc->sc_dmat, q->q_src_map, 1306 q->q_src_m, ubsec_op_cb, &q->q_src, BUS_DMA_NOWAIT) != 0) { 1307 bus_dmamap_destroy(sc->sc_dmat, q->q_src_map); 1308 q->q_src_map = NULL; 1309 ubsecstats.hst_noload++; 1310 err = ENOMEM; 1311 goto errout; 1312 } 1313 } else if (crp->crp_flags & CRYPTO_F_IOV) { 1314 if (bus_dmamap_load_uio(sc->sc_dmat, q->q_src_map, 1315 q->q_src_io, ubsec_op_cb, &q->q_src, BUS_DMA_NOWAIT) != 0) { 1316 bus_dmamap_destroy(sc->sc_dmat, q->q_src_map); 1317 q->q_src_map = NULL; 1318 ubsecstats.hst_noload++; 1319 err = ENOMEM; 1320 goto errout; 1321 } 1322 } 1323 nicealign = ubsec_dmamap_aligned(&q->q_src); 1324 1325 dmap->d_dma->d_mcr.mcr_pktlen = htole16(stheend); 1326 1327 #ifdef UBSEC_DEBUG 1328 if (ubsec_debug) 1329 kprintf("src skip: %d nicealign: %u\n", sskip, nicealign); 1330 #endif 1331 for (i = j = 0; i < q->q_src_nsegs; i++) { 1332 struct ubsec_pktbuf *pb; 1333 bus_size_t packl = q->q_src_segs[i].ds_len; 1334 bus_addr_t packp = q->q_src_segs[i].ds_addr; 1335 1336 if (sskip >= packl) { 1337 sskip -= packl; 1338 continue; 1339 } 1340 1341 packl -= sskip; 1342 packp += sskip; 1343 sskip = 0; 1344 1345 if (packl > 0xfffc) { 1346 err = EIO; 1347 goto errout; 1348 } 1349 1350 if (j == 0) 1351 pb = &dmap->d_dma->d_mcr.mcr_ipktbuf; 1352 else 1353 pb = &dmap->d_dma->d_sbuf[j - 1]; 1354 1355 pb->pb_addr = htole32(packp); 1356 1357 if (stheend) { 1358 if (packl > stheend) { 1359 pb->pb_len = htole32(stheend); 1360 stheend = 0; 1361 } else { 1362 pb->pb_len = htole32(packl); 1363 stheend -= packl; 1364 } 1365 } else 1366 pb->pb_len = htole32(packl); 1367 1368 if ((i + 1) == q->q_src_nsegs) 1369 pb->pb_next = 0; 1370 else 1371 pb->pb_next = htole32(dmap->d_alloc.dma_paddr + 1372 offsetof(struct ubsec_dmachunk, d_sbuf[j])); 1373 j++; 1374 } 1375 1376 if (enccrd == NULL && maccrd != NULL) { 1377 dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr = 0; 1378 dmap->d_dma->d_mcr.mcr_opktbuf.pb_len = 0; 1379 dmap->d_dma->d_mcr.mcr_opktbuf.pb_next = htole32(dmap->d_alloc.dma_paddr + 1380 offsetof(struct ubsec_dmachunk, d_macbuf[0])); 1381 #ifdef UBSEC_DEBUG 1382 if (ubsec_debug) 1383 kprintf("opkt: %x %x %x\n", 1384 dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr, 1385 dmap->d_dma->d_mcr.mcr_opktbuf.pb_len, 1386 dmap->d_dma->d_mcr.mcr_opktbuf.pb_next); 1387 #endif 1388 } else { 1389 if (crp->crp_flags & CRYPTO_F_IOV) { 1390 if (!nicealign) { 1391 ubsecstats.hst_iovmisaligned++; 1392 err = EINVAL; 1393 goto errout; 1394 } 1395 if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT, 1396 &q->q_dst_map)) { 1397 ubsecstats.hst_nomap++; 1398 err = ENOMEM; 1399 goto errout; 1400 } 1401 if (bus_dmamap_load_uio(sc->sc_dmat, q->q_dst_map, 1402 q->q_dst_io, ubsec_op_cb, &q->q_dst, BUS_DMA_NOWAIT) != 0) { 1403 bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map); 1404 q->q_dst_map = NULL; 1405 ubsecstats.hst_noload++; 1406 err = ENOMEM; 1407 goto errout; 1408 } 1409 } else if (crp->crp_flags & CRYPTO_F_IMBUF) { 1410 if (nicealign) { 1411 q->q_dst = q->q_src; 1412 } else { 1413 int totlen, len; 1414 struct mbuf *m, *top, **mp; 1415 1416 ubsecstats.hst_unaligned++; 1417 totlen = q->q_src_mapsize; 1418 if (q->q_src_m->m_flags & M_PKTHDR) { 1419 len = MHLEN; 1420 MGETHDR(m, MB_DONTWAIT, MT_DATA); 1421 if (m && !m_dup_pkthdr(m, q->q_src_m, MB_DONTWAIT)) { 1422 m_free(m); 1423 m = NULL; 1424 } 1425 } else { 1426 len = MLEN; 1427 MGET(m, MB_DONTWAIT, MT_DATA); 1428 } 1429 if (m == NULL) { 1430 ubsecstats.hst_nombuf++; 1431 err = sc->sc_nqueue ? ERESTART : ENOMEM; 1432 goto errout; 1433 } 1434 if (totlen >= MINCLSIZE) { 1435 MCLGET(m, MB_DONTWAIT); 1436 if ((m->m_flags & M_EXT) == 0) { 1437 m_free(m); 1438 ubsecstats.hst_nomcl++; 1439 err = sc->sc_nqueue ? ERESTART : ENOMEM; 1440 goto errout; 1441 } 1442 len = MCLBYTES; 1443 } 1444 m->m_len = len; 1445 top = NULL; 1446 mp = ⊤ 1447 1448 while (totlen > 0) { 1449 if (top) { 1450 MGET(m, MB_DONTWAIT, MT_DATA); 1451 if (m == NULL) { 1452 m_freem(top); 1453 ubsecstats.hst_nombuf++; 1454 err = sc->sc_nqueue ? ERESTART : ENOMEM; 1455 goto errout; 1456 } 1457 len = MLEN; 1458 } 1459 if (top && totlen >= MINCLSIZE) { 1460 MCLGET(m, MB_DONTWAIT); 1461 if ((m->m_flags & M_EXT) == 0) { 1462 *mp = m; 1463 m_freem(top); 1464 ubsecstats.hst_nomcl++; 1465 err = sc->sc_nqueue ? ERESTART : ENOMEM; 1466 goto errout; 1467 } 1468 len = MCLBYTES; 1469 } 1470 m->m_len = len = min(totlen, len); 1471 totlen -= len; 1472 *mp = m; 1473 mp = &m->m_next; 1474 } 1475 q->q_dst_m = top; 1476 ubsec_mcopy(q->q_src_m, q->q_dst_m, 1477 cpskip, cpoffset); 1478 if (bus_dmamap_create(sc->sc_dmat, 1479 BUS_DMA_NOWAIT, &q->q_dst_map) != 0) { 1480 ubsecstats.hst_nomap++; 1481 err = ENOMEM; 1482 goto errout; 1483 } 1484 if (bus_dmamap_load_mbuf(sc->sc_dmat, 1485 q->q_dst_map, q->q_dst_m, 1486 ubsec_op_cb, &q->q_dst, 1487 BUS_DMA_NOWAIT) != 0) { 1488 bus_dmamap_destroy(sc->sc_dmat, 1489 q->q_dst_map); 1490 q->q_dst_map = NULL; 1491 ubsecstats.hst_noload++; 1492 err = ENOMEM; 1493 goto errout; 1494 } 1495 } 1496 } else { 1497 ubsecstats.hst_badflags++; 1498 err = EINVAL; 1499 goto errout; 1500 } 1501 1502 #ifdef UBSEC_DEBUG 1503 if (ubsec_debug) 1504 kprintf("dst skip: %d\n", dskip); 1505 #endif 1506 for (i = j = 0; i < q->q_dst_nsegs; i++) { 1507 struct ubsec_pktbuf *pb; 1508 bus_size_t packl = q->q_dst_segs[i].ds_len; 1509 bus_addr_t packp = q->q_dst_segs[i].ds_addr; 1510 1511 if (dskip >= packl) { 1512 dskip -= packl; 1513 continue; 1514 } 1515 1516 packl -= dskip; 1517 packp += dskip; 1518 dskip = 0; 1519 1520 if (packl > 0xfffc) { 1521 err = EIO; 1522 goto errout; 1523 } 1524 1525 if (j == 0) 1526 pb = &dmap->d_dma->d_mcr.mcr_opktbuf; 1527 else 1528 pb = &dmap->d_dma->d_dbuf[j - 1]; 1529 1530 pb->pb_addr = htole32(packp); 1531 1532 if (dtheend) { 1533 if (packl > dtheend) { 1534 pb->pb_len = htole32(dtheend); 1535 dtheend = 0; 1536 } else { 1537 pb->pb_len = htole32(packl); 1538 dtheend -= packl; 1539 } 1540 } else 1541 pb->pb_len = htole32(packl); 1542 1543 if ((i + 1) == q->q_dst_nsegs) { 1544 if (maccrd) 1545 pb->pb_next = htole32(dmap->d_alloc.dma_paddr + 1546 offsetof(struct ubsec_dmachunk, d_macbuf[0])); 1547 else 1548 pb->pb_next = 0; 1549 } else 1550 pb->pb_next = htole32(dmap->d_alloc.dma_paddr + 1551 offsetof(struct ubsec_dmachunk, d_dbuf[j])); 1552 j++; 1553 } 1554 } 1555 1556 dmap->d_dma->d_mcr.mcr_cmdctxp = htole32(dmap->d_alloc.dma_paddr + 1557 offsetof(struct ubsec_dmachunk, d_ctx)); 1558 1559 if (sc->sc_flags & UBS_FLAGS_LONGCTX) { 1560 struct ubsec_pktctx_long *ctxl; 1561 1562 ctxl = (struct ubsec_pktctx_long *)(dmap->d_alloc.dma_vaddr + 1563 offsetof(struct ubsec_dmachunk, d_ctx)); 1564 1565 /* transform small context into long context */ 1566 ctxl->pc_len = htole16(sizeof(struct ubsec_pktctx_long)); 1567 ctxl->pc_type = htole16(UBS_PKTCTX_TYPE_IPSEC); 1568 ctxl->pc_flags = ctx.pc_flags; 1569 ctxl->pc_offset = ctx.pc_offset; 1570 for (i = 0; i < 6; i++) 1571 ctxl->pc_deskey[i] = ctx.pc_deskey[i]; 1572 for (i = 0; i < 5; i++) 1573 ctxl->pc_hminner[i] = ctx.pc_hminner[i]; 1574 for (i = 0; i < 5; i++) 1575 ctxl->pc_hmouter[i] = ctx.pc_hmouter[i]; 1576 ctxl->pc_iv[0] = ctx.pc_iv[0]; 1577 ctxl->pc_iv[1] = ctx.pc_iv[1]; 1578 } else 1579 bcopy(&ctx, dmap->d_alloc.dma_vaddr + 1580 offsetof(struct ubsec_dmachunk, d_ctx), 1581 sizeof(struct ubsec_pktctx)); 1582 1583 crit_enter(); 1584 SIMPLEQ_INSERT_TAIL(&sc->sc_queue, q, q_next); 1585 sc->sc_nqueue++; 1586 ubsecstats.hst_ipackets++; 1587 ubsecstats.hst_ibytes += dmap->d_alloc.dma_size; 1588 if ((hint & CRYPTO_HINT_MORE) == 0 || sc->sc_nqueue >= UBS_MAX_AGGR) 1589 ubsec_feed(sc); 1590 crit_exit(); 1591 return (0); 1592 1593 errout: 1594 if (q != NULL) { 1595 if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m)) 1596 m_freem(q->q_dst_m); 1597 1598 if (q->q_dst_map != NULL && q->q_dst_map != q->q_src_map) { 1599 bus_dmamap_unload(sc->sc_dmat, q->q_dst_map); 1600 bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map); 1601 } 1602 if (q->q_src_map != NULL) { 1603 bus_dmamap_unload(sc->sc_dmat, q->q_src_map); 1604 bus_dmamap_destroy(sc->sc_dmat, q->q_src_map); 1605 } 1606 1607 crit_enter(); 1608 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next); 1609 crit_exit(); 1610 } 1611 if (err != ERESTART) { 1612 crp->crp_etype = err; 1613 crypto_done(crp); 1614 } else { 1615 sc->sc_needwakeup |= CRYPTO_SYMQ; 1616 } 1617 return (err); 1618 } 1619 1620 static void 1621 ubsec_callback(struct ubsec_softc *sc, struct ubsec_q *q) 1622 { 1623 struct cryptop *crp = (struct cryptop *)q->q_crp; 1624 struct cryptodesc *crd; 1625 struct ubsec_dma *dmap = q->q_dma; 1626 1627 ubsecstats.hst_opackets++; 1628 ubsecstats.hst_obytes += dmap->d_alloc.dma_size; 1629 1630 ubsec_dma_sync(&dmap->d_alloc, 1631 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 1632 if (q->q_dst_map != NULL && q->q_dst_map != q->q_src_map) { 1633 bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, 1634 BUS_DMASYNC_POSTREAD); 1635 bus_dmamap_unload(sc->sc_dmat, q->q_dst_map); 1636 bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map); 1637 } 1638 bus_dmamap_sync(sc->sc_dmat, q->q_src_map, BUS_DMASYNC_POSTWRITE); 1639 bus_dmamap_unload(sc->sc_dmat, q->q_src_map); 1640 bus_dmamap_destroy(sc->sc_dmat, q->q_src_map); 1641 1642 if ((crp->crp_flags & CRYPTO_F_IMBUF) && (q->q_src_m != q->q_dst_m)) { 1643 m_freem(q->q_src_m); 1644 crp->crp_buf = (caddr_t)q->q_dst_m; 1645 } 1646 ubsecstats.hst_obytes += ((struct mbuf *)crp->crp_buf)->m_len; 1647 1648 /* copy out IV for future use */ 1649 if (q->q_flags & UBSEC_QFLAGS_COPYOUTIV) { 1650 for (crd = crp->crp_desc; crd; crd = crd->crd_next) { 1651 if (crd->crd_alg != CRYPTO_DES_CBC && 1652 crd->crd_alg != CRYPTO_3DES_CBC) 1653 continue; 1654 if (crp->crp_flags & CRYPTO_F_IMBUF) 1655 m_copydata((struct mbuf *)crp->crp_buf, 1656 crd->crd_skip + crd->crd_len - 8, 8, 1657 (caddr_t)sc->sc_sessions[q->q_sesn].ses_iv); 1658 else if (crp->crp_flags & CRYPTO_F_IOV) { 1659 cuio_copydata((struct uio *)crp->crp_buf, 1660 crd->crd_skip + crd->crd_len - 8, 8, 1661 (caddr_t)sc->sc_sessions[q->q_sesn].ses_iv); 1662 } 1663 break; 1664 } 1665 } 1666 1667 for (crd = crp->crp_desc; crd; crd = crd->crd_next) { 1668 if (crd->crd_alg != CRYPTO_MD5_HMAC && 1669 crd->crd_alg != CRYPTO_SHA1_HMAC) 1670 continue; 1671 crypto_copyback(crp->crp_flags, crp->crp_buf, crd->crd_inject, 1672 sc->sc_sessions[q->q_sesn].ses_mlen, 1673 (caddr_t)dmap->d_dma->d_macbuf); 1674 #if 0 1675 if (crp->crp_flags & CRYPTO_F_IMBUF) 1676 m_copyback((struct mbuf *)crp->crp_buf, 1677 crd->crd_inject, 12, 1678 (caddr_t)dmap->d_dma->d_macbuf); 1679 else if (crp->crp_flags & CRYPTO_F_IOV && crp->crp_mac) 1680 bcopy((caddr_t)dmap->d_dma->d_macbuf, 1681 crp->crp_mac, 12); 1682 break; 1683 #endif 1684 } 1685 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next); 1686 crypto_done(crp); 1687 } 1688 1689 static void 1690 ubsec_mcopy(struct mbuf *srcm, struct mbuf *dstm, int hoffset, int toffset) 1691 { 1692 int i, j, dlen, slen; 1693 caddr_t dptr, sptr; 1694 1695 j = 0; 1696 sptr = srcm->m_data; 1697 slen = srcm->m_len; 1698 dptr = dstm->m_data; 1699 dlen = dstm->m_len; 1700 1701 while (1) { 1702 for (i = 0; i < min(slen, dlen); i++) { 1703 if (j < hoffset || j >= toffset) 1704 *dptr++ = *sptr++; 1705 slen--; 1706 dlen--; 1707 j++; 1708 } 1709 if (slen == 0) { 1710 srcm = srcm->m_next; 1711 if (srcm == NULL) 1712 return; 1713 sptr = srcm->m_data; 1714 slen = srcm->m_len; 1715 } 1716 if (dlen == 0) { 1717 dstm = dstm->m_next; 1718 if (dstm == NULL) 1719 return; 1720 dptr = dstm->m_data; 1721 dlen = dstm->m_len; 1722 } 1723 } 1724 } 1725 1726 /* 1727 * feed the key generator, must be called at splimp() or higher. 1728 */ 1729 static int 1730 ubsec_feed2(struct ubsec_softc *sc) 1731 { 1732 struct ubsec_q2 *q; 1733 1734 while (!SIMPLEQ_EMPTY(&sc->sc_queue2)) { 1735 if (READ_REG(sc, BS_STAT) & BS_STAT_MCR2_FULL) 1736 break; 1737 q = SIMPLEQ_FIRST(&sc->sc_queue2); 1738 1739 ubsec_dma_sync(&q->q_mcr, 1740 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1741 ubsec_dma_sync(&q->q_ctx, BUS_DMASYNC_PREWRITE); 1742 1743 WRITE_REG(sc, BS_MCR2, q->q_mcr.dma_paddr); 1744 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue2, q_next); 1745 --sc->sc_nqueue2; 1746 SIMPLEQ_INSERT_TAIL(&sc->sc_qchip2, q, q_next); 1747 } 1748 return (0); 1749 } 1750 1751 /* 1752 * Callback for handling random numbers 1753 */ 1754 static void 1755 ubsec_callback2(struct ubsec_softc *sc, struct ubsec_q2 *q) 1756 { 1757 struct cryptkop *krp; 1758 struct ubsec_ctx_keyop *ctx; 1759 1760 ctx = (struct ubsec_ctx_keyop *)q->q_ctx.dma_vaddr; 1761 ubsec_dma_sync(&q->q_ctx, BUS_DMASYNC_POSTWRITE); 1762 1763 switch (q->q_type) { 1764 #ifndef UBSEC_NO_RNG 1765 case UBS_CTXOP_RNGBYPASS: { 1766 struct ubsec_q2_rng *rng = (struct ubsec_q2_rng *)q; 1767 1768 ubsec_dma_sync(&rng->rng_buf, BUS_DMASYNC_POSTREAD); 1769 (*sc->sc_harvest)(sc->sc_rndtest, 1770 rng->rng_buf.dma_vaddr, 1771 UBSEC_RNG_BUFSIZ*sizeof (u_int32_t)); 1772 rng->rng_used = 0; 1773 callout_reset(&sc->sc_rngto, sc->sc_rnghz, ubsec_rng, sc); 1774 break; 1775 } 1776 #endif 1777 case UBS_CTXOP_MODEXP: { 1778 struct ubsec_q2_modexp *me = (struct ubsec_q2_modexp *)q; 1779 u_int rlen, clen; 1780 1781 krp = me->me_krp; 1782 rlen = (me->me_modbits + 7) / 8; 1783 clen = (krp->krp_param[krp->krp_iparams].crp_nbits + 7) / 8; 1784 1785 ubsec_dma_sync(&me->me_M, BUS_DMASYNC_POSTWRITE); 1786 ubsec_dma_sync(&me->me_E, BUS_DMASYNC_POSTWRITE); 1787 ubsec_dma_sync(&me->me_C, BUS_DMASYNC_POSTREAD); 1788 ubsec_dma_sync(&me->me_epb, BUS_DMASYNC_POSTWRITE); 1789 1790 if (clen < rlen) 1791 krp->krp_status = E2BIG; 1792 else { 1793 if (sc->sc_flags & UBS_FLAGS_HWNORM) { 1794 bzero(krp->krp_param[krp->krp_iparams].crp_p, 1795 (krp->krp_param[krp->krp_iparams].crp_nbits 1796 + 7) / 8); 1797 bcopy(me->me_C.dma_vaddr, 1798 krp->krp_param[krp->krp_iparams].crp_p, 1799 (me->me_modbits + 7) / 8); 1800 } else 1801 ubsec_kshift_l(me->me_shiftbits, 1802 me->me_C.dma_vaddr, me->me_normbits, 1803 krp->krp_param[krp->krp_iparams].crp_p, 1804 krp->krp_param[krp->krp_iparams].crp_nbits); 1805 } 1806 1807 crypto_kdone(krp); 1808 1809 /* bzero all potentially sensitive data */ 1810 bzero(me->me_E.dma_vaddr, me->me_E.dma_size); 1811 bzero(me->me_M.dma_vaddr, me->me_M.dma_size); 1812 bzero(me->me_C.dma_vaddr, me->me_C.dma_size); 1813 bzero(me->me_q.q_ctx.dma_vaddr, me->me_q.q_ctx.dma_size); 1814 1815 /* Can't free here, so put us on the free list. */ 1816 SIMPLEQ_INSERT_TAIL(&sc->sc_q2free, &me->me_q, q_next); 1817 break; 1818 } 1819 case UBS_CTXOP_RSAPRIV: { 1820 struct ubsec_q2_rsapriv *rp = (struct ubsec_q2_rsapriv *)q; 1821 u_int len; 1822 1823 krp = rp->rpr_krp; 1824 ubsec_dma_sync(&rp->rpr_msgin, BUS_DMASYNC_POSTWRITE); 1825 ubsec_dma_sync(&rp->rpr_msgout, BUS_DMASYNC_POSTREAD); 1826 1827 len = (krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT].crp_nbits + 7) / 8; 1828 bcopy(rp->rpr_msgout.dma_vaddr, 1829 krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT].crp_p, len); 1830 1831 crypto_kdone(krp); 1832 1833 bzero(rp->rpr_msgin.dma_vaddr, rp->rpr_msgin.dma_size); 1834 bzero(rp->rpr_msgout.dma_vaddr, rp->rpr_msgout.dma_size); 1835 bzero(rp->rpr_q.q_ctx.dma_vaddr, rp->rpr_q.q_ctx.dma_size); 1836 1837 /* Can't free here, so put us on the free list. */ 1838 SIMPLEQ_INSERT_TAIL(&sc->sc_q2free, &rp->rpr_q, q_next); 1839 break; 1840 } 1841 default: 1842 device_printf(sc->sc_dev, "unknown ctx op: %x\n", 1843 letoh16(ctx->ctx_op)); 1844 break; 1845 } 1846 } 1847 1848 #ifndef UBSEC_NO_RNG 1849 static void 1850 ubsec_rng(void *vsc) 1851 { 1852 struct ubsec_softc *sc = vsc; 1853 struct ubsec_q2_rng *rng = &sc->sc_rng; 1854 struct ubsec_mcr *mcr; 1855 struct ubsec_ctx_rngbypass *ctx; 1856 1857 crit_enter(); 1858 if (rng->rng_used) { 1859 crit_exit(); 1860 return; 1861 } 1862 sc->sc_nqueue2++; 1863 if (sc->sc_nqueue2 >= UBS_MAX_NQUEUE) 1864 goto out; 1865 1866 mcr = (struct ubsec_mcr *)rng->rng_q.q_mcr.dma_vaddr; 1867 ctx = (struct ubsec_ctx_rngbypass *)rng->rng_q.q_ctx.dma_vaddr; 1868 1869 mcr->mcr_pkts = htole16(1); 1870 mcr->mcr_flags = 0; 1871 mcr->mcr_cmdctxp = htole32(rng->rng_q.q_ctx.dma_paddr); 1872 mcr->mcr_ipktbuf.pb_addr = mcr->mcr_ipktbuf.pb_next = 0; 1873 mcr->mcr_ipktbuf.pb_len = 0; 1874 mcr->mcr_reserved = mcr->mcr_pktlen = 0; 1875 mcr->mcr_opktbuf.pb_addr = htole32(rng->rng_buf.dma_paddr); 1876 mcr->mcr_opktbuf.pb_len = htole32(((sizeof(u_int32_t) * UBSEC_RNG_BUFSIZ)) & 1877 UBS_PKTBUF_LEN); 1878 mcr->mcr_opktbuf.pb_next = 0; 1879 1880 ctx->rbp_len = htole16(sizeof(struct ubsec_ctx_rngbypass)); 1881 ctx->rbp_op = htole16(UBS_CTXOP_RNGBYPASS); 1882 rng->rng_q.q_type = UBS_CTXOP_RNGBYPASS; 1883 1884 ubsec_dma_sync(&rng->rng_buf, BUS_DMASYNC_PREREAD); 1885 1886 SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &rng->rng_q, q_next); 1887 rng->rng_used = 1; 1888 ubsec_feed2(sc); 1889 ubsecstats.hst_rng++; 1890 crit_exit(); 1891 1892 return; 1893 1894 out: 1895 /* 1896 * Something weird happened, generate our own call back. 1897 */ 1898 sc->sc_nqueue2--; 1899 crit_exit(); 1900 callout_reset(&sc->sc_rngto, sc->sc_rnghz, ubsec_rng, sc); 1901 } 1902 #endif /* UBSEC_NO_RNG */ 1903 1904 static void 1905 ubsec_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 1906 { 1907 bus_addr_t *paddr = (bus_addr_t*) arg; 1908 *paddr = segs->ds_addr; 1909 } 1910 1911 static int 1912 ubsec_dma_malloc( 1913 struct ubsec_softc *sc, 1914 bus_size_t size, 1915 struct ubsec_dma_alloc *dma, 1916 int mapflags 1917 ) 1918 { 1919 int r; 1920 1921 /* XXX could specify sc_dmat as parent but that just adds overhead */ 1922 r = bus_dma_tag_create(NULL, /* parent */ 1923 1, 0, /* alignment, bounds */ 1924 BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ 1925 BUS_SPACE_MAXADDR, /* highaddr */ 1926 NULL, NULL, /* filter, filterarg */ 1927 size, /* maxsize */ 1928 1, /* nsegments */ 1929 size, /* maxsegsize */ 1930 BUS_DMA_ALLOCNOW, /* flags */ 1931 &dma->dma_tag); 1932 if (r != 0) { 1933 device_printf(sc->sc_dev, "ubsec_dma_malloc: " 1934 "bus_dma_tag_create failed; error %u\n", r); 1935 goto fail_0; 1936 } 1937 1938 r = bus_dmamap_create(dma->dma_tag, BUS_DMA_NOWAIT, &dma->dma_map); 1939 if (r != 0) { 1940 device_printf(sc->sc_dev, "ubsec_dma_malloc: " 1941 "bus_dmamap_create failed; error %u\n", r); 1942 goto fail_1; 1943 } 1944 1945 r = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr, 1946 BUS_DMA_NOWAIT, &dma->dma_map); 1947 if (r != 0) { 1948 device_printf(sc->sc_dev, "ubsec_dma_malloc: " 1949 "bus_dmammem_alloc failed; size %ju, error %u\n", 1950 (intmax_t)size, r); 1951 goto fail_2; 1952 } 1953 1954 r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, 1955 size, 1956 ubsec_dmamap_cb, 1957 &dma->dma_paddr, 1958 mapflags | BUS_DMA_NOWAIT); 1959 if (r != 0) { 1960 device_printf(sc->sc_dev, "ubsec_dma_malloc: " 1961 "bus_dmamap_load failed; error %u\n", r); 1962 goto fail_3; 1963 } 1964 1965 dma->dma_size = size; 1966 return (0); 1967 1968 fail_3: 1969 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 1970 fail_2: 1971 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); 1972 fail_1: 1973 bus_dmamap_destroy(dma->dma_tag, dma->dma_map); 1974 bus_dma_tag_destroy(dma->dma_tag); 1975 fail_0: 1976 dma->dma_map = NULL; 1977 dma->dma_tag = NULL; 1978 return (r); 1979 } 1980 1981 static void 1982 ubsec_dma_free(struct ubsec_softc *sc, struct ubsec_dma_alloc *dma) 1983 { 1984 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 1985 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); 1986 bus_dmamap_destroy(dma->dma_tag, dma->dma_map); 1987 bus_dma_tag_destroy(dma->dma_tag); 1988 } 1989 1990 /* 1991 * Resets the board. Values in the regesters are left as is 1992 * from the reset (i.e. initial values are assigned elsewhere). 1993 */ 1994 static void 1995 ubsec_reset_board(struct ubsec_softc *sc) 1996 { 1997 volatile u_int32_t ctrl; 1998 1999 ctrl = READ_REG(sc, BS_CTRL); 2000 ctrl |= BS_CTRL_RESET; 2001 WRITE_REG(sc, BS_CTRL, ctrl); 2002 2003 /* 2004 * Wait aprox. 30 PCI clocks = 900 ns = 0.9 us 2005 */ 2006 DELAY(10); 2007 } 2008 2009 /* 2010 * Init Broadcom registers 2011 */ 2012 static void 2013 ubsec_init_board(struct ubsec_softc *sc) 2014 { 2015 u_int32_t ctrl; 2016 2017 ctrl = READ_REG(sc, BS_CTRL); 2018 ctrl &= ~(BS_CTRL_BE32 | BS_CTRL_BE64); 2019 ctrl |= BS_CTRL_LITTLE_ENDIAN | BS_CTRL_MCR1INT; 2020 2021 if (sc->sc_flags & (UBS_FLAGS_KEY|UBS_FLAGS_RNG)) 2022 ctrl |= BS_CTRL_MCR2INT; 2023 else 2024 ctrl &= ~BS_CTRL_MCR2INT; 2025 2026 if (sc->sc_flags & UBS_FLAGS_HWNORM) 2027 ctrl &= ~BS_CTRL_SWNORM; 2028 2029 WRITE_REG(sc, BS_CTRL, ctrl); 2030 } 2031 2032 /* 2033 * Init Broadcom PCI registers 2034 */ 2035 static void 2036 ubsec_init_pciregs(device_t dev) 2037 { 2038 #if 0 2039 u_int32_t misc; 2040 2041 misc = pci_conf_read(pc, pa->pa_tag, BS_RTY_TOUT); 2042 misc = (misc & ~(UBS_PCI_RTY_MASK << UBS_PCI_RTY_SHIFT)) 2043 | ((UBS_DEF_RTY & 0xff) << UBS_PCI_RTY_SHIFT); 2044 misc = (misc & ~(UBS_PCI_TOUT_MASK << UBS_PCI_TOUT_SHIFT)) 2045 | ((UBS_DEF_TOUT & 0xff) << UBS_PCI_TOUT_SHIFT); 2046 pci_conf_write(pc, pa->pa_tag, BS_RTY_TOUT, misc); 2047 #endif 2048 2049 /* 2050 * This will set the cache line size to 1, this will 2051 * force the BCM58xx chip just to do burst read/writes. 2052 * Cache line read/writes are to slow 2053 */ 2054 pci_write_config(dev, PCIR_CACHELNSZ, UBS_DEF_CACHELINE, 1); 2055 } 2056 2057 /* 2058 * Clean up after a chip crash. 2059 * It is assumed that the caller in splimp() 2060 */ 2061 static void 2062 ubsec_cleanchip(struct ubsec_softc *sc) 2063 { 2064 struct ubsec_q *q; 2065 2066 while (!SIMPLEQ_EMPTY(&sc->sc_qchip)) { 2067 q = SIMPLEQ_FIRST(&sc->sc_qchip); 2068 SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, q_next); 2069 ubsec_free_q(sc, q); 2070 } 2071 sc->sc_nqchip = 0; 2072 } 2073 2074 /* 2075 * free a ubsec_q 2076 * It is assumed that the caller is within spimp() 2077 */ 2078 static int 2079 ubsec_free_q(struct ubsec_softc *sc, struct ubsec_q *q) 2080 { 2081 struct ubsec_q *q2; 2082 struct cryptop *crp; 2083 int npkts; 2084 int i; 2085 2086 npkts = q->q_nstacked_mcrs; 2087 2088 for (i = 0; i < npkts; i++) { 2089 if(q->q_stacked_mcr[i]) { 2090 q2 = q->q_stacked_mcr[i]; 2091 2092 if ((q2->q_dst_m != NULL) && (q2->q_src_m != q2->q_dst_m)) 2093 m_freem(q2->q_dst_m); 2094 2095 crp = (struct cryptop *)q2->q_crp; 2096 2097 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q2, q_next); 2098 2099 crp->crp_etype = EFAULT; 2100 crypto_done(crp); 2101 } else { 2102 break; 2103 } 2104 } 2105 2106 /* 2107 * Free header MCR 2108 */ 2109 if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m)) 2110 m_freem(q->q_dst_m); 2111 2112 crp = (struct cryptop *)q->q_crp; 2113 2114 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next); 2115 2116 crp->crp_etype = EFAULT; 2117 crypto_done(crp); 2118 return(0); 2119 } 2120 2121 /* 2122 * Routine to reset the chip and clean up. 2123 * It is assumed that the caller is in splimp() 2124 */ 2125 static void 2126 ubsec_totalreset(struct ubsec_softc *sc) 2127 { 2128 ubsec_reset_board(sc); 2129 ubsec_init_board(sc); 2130 ubsec_cleanchip(sc); 2131 } 2132 2133 static int 2134 ubsec_dmamap_aligned(struct ubsec_operand *op) 2135 { 2136 int i; 2137 2138 for (i = 0; i < op->nsegs; i++) { 2139 if (op->segs[i].ds_addr & 3) 2140 return (0); 2141 if ((i != (op->nsegs - 1)) && 2142 (op->segs[i].ds_len & 3)) 2143 return (0); 2144 } 2145 return (1); 2146 } 2147 2148 static void 2149 ubsec_kfree(struct ubsec_softc *sc, struct ubsec_q2 *q) 2150 { 2151 switch (q->q_type) { 2152 case UBS_CTXOP_MODEXP: { 2153 struct ubsec_q2_modexp *me = (struct ubsec_q2_modexp *)q; 2154 2155 ubsec_dma_free(sc, &me->me_q.q_mcr); 2156 ubsec_dma_free(sc, &me->me_q.q_ctx); 2157 ubsec_dma_free(sc, &me->me_M); 2158 ubsec_dma_free(sc, &me->me_E); 2159 ubsec_dma_free(sc, &me->me_C); 2160 ubsec_dma_free(sc, &me->me_epb); 2161 kfree(me, M_DEVBUF); 2162 break; 2163 } 2164 case UBS_CTXOP_RSAPRIV: { 2165 struct ubsec_q2_rsapriv *rp = (struct ubsec_q2_rsapriv *)q; 2166 2167 ubsec_dma_free(sc, &rp->rpr_q.q_mcr); 2168 ubsec_dma_free(sc, &rp->rpr_q.q_ctx); 2169 ubsec_dma_free(sc, &rp->rpr_msgin); 2170 ubsec_dma_free(sc, &rp->rpr_msgout); 2171 kfree(rp, M_DEVBUF); 2172 break; 2173 } 2174 default: 2175 device_printf(sc->sc_dev, "invalid kfree 0x%x\n", q->q_type); 2176 break; 2177 } 2178 } 2179 2180 static int 2181 ubsec_kprocess(void *arg, struct cryptkop *krp, int hint) 2182 { 2183 struct ubsec_softc *sc = arg; 2184 int r; 2185 2186 if (krp == NULL || krp->krp_callback == NULL) 2187 return (EINVAL); 2188 2189 while (!SIMPLEQ_EMPTY(&sc->sc_q2free)) { 2190 struct ubsec_q2 *q; 2191 2192 q = SIMPLEQ_FIRST(&sc->sc_q2free); 2193 SIMPLEQ_REMOVE_HEAD(&sc->sc_q2free, q_next); 2194 ubsec_kfree(sc, q); 2195 } 2196 2197 switch (krp->krp_op) { 2198 case CRK_MOD_EXP: 2199 if (sc->sc_flags & UBS_FLAGS_HWNORM) 2200 r = ubsec_kprocess_modexp_hw(sc, krp, hint); 2201 else 2202 r = ubsec_kprocess_modexp_sw(sc, krp, hint); 2203 break; 2204 case CRK_MOD_EXP_CRT: 2205 return (ubsec_kprocess_rsapriv(sc, krp, hint)); 2206 default: 2207 device_printf(sc->sc_dev, "kprocess: invalid op 0x%x\n", 2208 krp->krp_op); 2209 krp->krp_status = EOPNOTSUPP; 2210 crypto_kdone(krp); 2211 return (0); 2212 } 2213 return (0); /* silence compiler */ 2214 } 2215 2216 /* 2217 * Start computation of cr[C] = (cr[M] ^ cr[E]) mod cr[N] (sw normalization) 2218 */ 2219 static int 2220 ubsec_kprocess_modexp_sw(struct ubsec_softc *sc, struct cryptkop *krp, int hint) 2221 { 2222 struct ubsec_q2_modexp *me; 2223 struct ubsec_mcr *mcr; 2224 struct ubsec_ctx_modexp *ctx; 2225 struct ubsec_pktbuf *epb; 2226 int err = 0; 2227 u_int nbits, normbits, mbits, shiftbits, ebits; 2228 2229 me = kmalloc(sizeof *me, M_DEVBUF, M_INTWAIT | M_ZERO); 2230 me->me_krp = krp; 2231 me->me_q.q_type = UBS_CTXOP_MODEXP; 2232 2233 nbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_N]); 2234 if (nbits <= 512) 2235 normbits = 512; 2236 else if (nbits <= 768) 2237 normbits = 768; 2238 else if (nbits <= 1024) 2239 normbits = 1024; 2240 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 1536) 2241 normbits = 1536; 2242 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 2048) 2243 normbits = 2048; 2244 else { 2245 err = E2BIG; 2246 goto errout; 2247 } 2248 2249 shiftbits = normbits - nbits; 2250 2251 me->me_modbits = nbits; 2252 me->me_shiftbits = shiftbits; 2253 me->me_normbits = normbits; 2254 2255 /* Sanity check: result bits must be >= true modulus bits. */ 2256 if (krp->krp_param[krp->krp_iparams].crp_nbits < nbits) { 2257 err = ERANGE; 2258 goto errout; 2259 } 2260 2261 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr), 2262 &me->me_q.q_mcr, 0)) { 2263 err = ENOMEM; 2264 goto errout; 2265 } 2266 mcr = (struct ubsec_mcr *)me->me_q.q_mcr.dma_vaddr; 2267 2268 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_modexp), 2269 &me->me_q.q_ctx, 0)) { 2270 err = ENOMEM; 2271 goto errout; 2272 } 2273 2274 mbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_M]); 2275 if (mbits > nbits) { 2276 err = E2BIG; 2277 goto errout; 2278 } 2279 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_M, 0)) { 2280 err = ENOMEM; 2281 goto errout; 2282 } 2283 ubsec_kshift_r(shiftbits, 2284 krp->krp_param[UBS_MODEXP_PAR_M].crp_p, mbits, 2285 me->me_M.dma_vaddr, normbits); 2286 2287 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_C, 0)) { 2288 err = ENOMEM; 2289 goto errout; 2290 } 2291 bzero(me->me_C.dma_vaddr, me->me_C.dma_size); 2292 2293 ebits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_E]); 2294 if (ebits > nbits) { 2295 err = E2BIG; 2296 goto errout; 2297 } 2298 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_E, 0)) { 2299 err = ENOMEM; 2300 goto errout; 2301 } 2302 ubsec_kshift_r(shiftbits, 2303 krp->krp_param[UBS_MODEXP_PAR_E].crp_p, ebits, 2304 me->me_E.dma_vaddr, normbits); 2305 2306 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_pktbuf), 2307 &me->me_epb, 0)) { 2308 err = ENOMEM; 2309 goto errout; 2310 } 2311 epb = (struct ubsec_pktbuf *)me->me_epb.dma_vaddr; 2312 epb->pb_addr = htole32(me->me_E.dma_paddr); 2313 epb->pb_next = 0; 2314 epb->pb_len = htole32(normbits / 8); 2315 2316 #ifdef UBSEC_DEBUG 2317 if (ubsec_debug) { 2318 kprintf("Epb "); 2319 ubsec_dump_pb(epb); 2320 } 2321 #endif 2322 2323 mcr->mcr_pkts = htole16(1); 2324 mcr->mcr_flags = 0; 2325 mcr->mcr_cmdctxp = htole32(me->me_q.q_ctx.dma_paddr); 2326 mcr->mcr_reserved = 0; 2327 mcr->mcr_pktlen = 0; 2328 2329 mcr->mcr_ipktbuf.pb_addr = htole32(me->me_M.dma_paddr); 2330 mcr->mcr_ipktbuf.pb_len = htole32(normbits / 8); 2331 mcr->mcr_ipktbuf.pb_next = htole32(me->me_epb.dma_paddr); 2332 2333 mcr->mcr_opktbuf.pb_addr = htole32(me->me_C.dma_paddr); 2334 mcr->mcr_opktbuf.pb_next = 0; 2335 mcr->mcr_opktbuf.pb_len = htole32(normbits / 8); 2336 2337 #ifdef DIAGNOSTIC 2338 /* Misaligned output buffer will hang the chip. */ 2339 if ((letoh32(mcr->mcr_opktbuf.pb_addr) & 3) != 0) 2340 panic("%s: modexp invalid addr 0x%x\n", 2341 device_get_nameunit(sc->sc_dev), 2342 letoh32(mcr->mcr_opktbuf.pb_addr)); 2343 if ((letoh32(mcr->mcr_opktbuf.pb_len) & 3) != 0) 2344 panic("%s: modexp invalid len 0x%x\n", 2345 device_get_nameunit(sc->sc_dev), 2346 letoh32(mcr->mcr_opktbuf.pb_len)); 2347 #endif 2348 2349 ctx = (struct ubsec_ctx_modexp *)me->me_q.q_ctx.dma_vaddr; 2350 bzero(ctx, sizeof(*ctx)); 2351 ubsec_kshift_r(shiftbits, 2352 krp->krp_param[UBS_MODEXP_PAR_N].crp_p, nbits, 2353 ctx->me_N, normbits); 2354 ctx->me_len = htole16((normbits / 8) + (4 * sizeof(u_int16_t))); 2355 ctx->me_op = htole16(UBS_CTXOP_MODEXP); 2356 ctx->me_E_len = htole16(nbits); 2357 ctx->me_N_len = htole16(nbits); 2358 2359 #ifdef UBSEC_DEBUG 2360 if (ubsec_debug) { 2361 ubsec_dump_mcr(mcr); 2362 ubsec_dump_ctx2((struct ubsec_ctx_keyop *)ctx); 2363 } 2364 #endif 2365 2366 /* 2367 * ubsec_feed2 will sync mcr and ctx, we just need to sync 2368 * everything else. 2369 */ 2370 ubsec_dma_sync(&me->me_M, BUS_DMASYNC_PREWRITE); 2371 ubsec_dma_sync(&me->me_E, BUS_DMASYNC_PREWRITE); 2372 ubsec_dma_sync(&me->me_C, BUS_DMASYNC_PREREAD); 2373 ubsec_dma_sync(&me->me_epb, BUS_DMASYNC_PREWRITE); 2374 2375 /* Enqueue and we're done... */ 2376 crit_enter(); 2377 SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &me->me_q, q_next); 2378 ubsec_feed2(sc); 2379 ubsecstats.hst_modexp++; 2380 crit_exit(); 2381 2382 return (0); 2383 2384 errout: 2385 if (me != NULL) { 2386 if (me->me_q.q_mcr.dma_map != NULL) 2387 ubsec_dma_free(sc, &me->me_q.q_mcr); 2388 if (me->me_q.q_ctx.dma_map != NULL) { 2389 bzero(me->me_q.q_ctx.dma_vaddr, me->me_q.q_ctx.dma_size); 2390 ubsec_dma_free(sc, &me->me_q.q_ctx); 2391 } 2392 if (me->me_M.dma_map != NULL) { 2393 bzero(me->me_M.dma_vaddr, me->me_M.dma_size); 2394 ubsec_dma_free(sc, &me->me_M); 2395 } 2396 if (me->me_E.dma_map != NULL) { 2397 bzero(me->me_E.dma_vaddr, me->me_E.dma_size); 2398 ubsec_dma_free(sc, &me->me_E); 2399 } 2400 if (me->me_C.dma_map != NULL) { 2401 bzero(me->me_C.dma_vaddr, me->me_C.dma_size); 2402 ubsec_dma_free(sc, &me->me_C); 2403 } 2404 if (me->me_epb.dma_map != NULL) 2405 ubsec_dma_free(sc, &me->me_epb); 2406 kfree(me, M_DEVBUF); 2407 } 2408 krp->krp_status = err; 2409 crypto_kdone(krp); 2410 return (0); 2411 } 2412 2413 /* 2414 * Start computation of cr[C] = (cr[M] ^ cr[E]) mod cr[N] (hw normalization) 2415 */ 2416 static int 2417 ubsec_kprocess_modexp_hw(struct ubsec_softc *sc, struct cryptkop *krp, int hint) 2418 { 2419 struct ubsec_q2_modexp *me; 2420 struct ubsec_mcr *mcr; 2421 struct ubsec_ctx_modexp *ctx; 2422 struct ubsec_pktbuf *epb; 2423 int err = 0; 2424 u_int nbits, normbits, mbits, shiftbits, ebits; 2425 2426 me = kmalloc(sizeof *me, M_DEVBUF, M_INTWAIT | M_ZERO); 2427 me->me_krp = krp; 2428 me->me_q.q_type = UBS_CTXOP_MODEXP; 2429 2430 nbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_N]); 2431 if (nbits <= 512) 2432 normbits = 512; 2433 else if (nbits <= 768) 2434 normbits = 768; 2435 else if (nbits <= 1024) 2436 normbits = 1024; 2437 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 1536) 2438 normbits = 1536; 2439 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 2048) 2440 normbits = 2048; 2441 else { 2442 err = E2BIG; 2443 goto errout; 2444 } 2445 2446 shiftbits = normbits - nbits; 2447 2448 /* XXX ??? */ 2449 me->me_modbits = nbits; 2450 me->me_shiftbits = shiftbits; 2451 me->me_normbits = normbits; 2452 2453 /* Sanity check: result bits must be >= true modulus bits. */ 2454 if (krp->krp_param[krp->krp_iparams].crp_nbits < nbits) { 2455 err = ERANGE; 2456 goto errout; 2457 } 2458 2459 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr), 2460 &me->me_q.q_mcr, 0)) { 2461 err = ENOMEM; 2462 goto errout; 2463 } 2464 mcr = (struct ubsec_mcr *)me->me_q.q_mcr.dma_vaddr; 2465 2466 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_modexp), 2467 &me->me_q.q_ctx, 0)) { 2468 err = ENOMEM; 2469 goto errout; 2470 } 2471 2472 mbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_M]); 2473 if (mbits > nbits) { 2474 err = E2BIG; 2475 goto errout; 2476 } 2477 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_M, 0)) { 2478 err = ENOMEM; 2479 goto errout; 2480 } 2481 bzero(me->me_M.dma_vaddr, normbits / 8); 2482 bcopy(krp->krp_param[UBS_MODEXP_PAR_M].crp_p, 2483 me->me_M.dma_vaddr, (mbits + 7) / 8); 2484 2485 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_C, 0)) { 2486 err = ENOMEM; 2487 goto errout; 2488 } 2489 bzero(me->me_C.dma_vaddr, me->me_C.dma_size); 2490 2491 ebits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_E]); 2492 if (ebits > nbits) { 2493 err = E2BIG; 2494 goto errout; 2495 } 2496 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_E, 0)) { 2497 err = ENOMEM; 2498 goto errout; 2499 } 2500 bzero(me->me_E.dma_vaddr, normbits / 8); 2501 bcopy(krp->krp_param[UBS_MODEXP_PAR_E].crp_p, 2502 me->me_E.dma_vaddr, (ebits + 7) / 8); 2503 2504 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_pktbuf), 2505 &me->me_epb, 0)) { 2506 err = ENOMEM; 2507 goto errout; 2508 } 2509 epb = (struct ubsec_pktbuf *)me->me_epb.dma_vaddr; 2510 epb->pb_addr = htole32(me->me_E.dma_paddr); 2511 epb->pb_next = 0; 2512 epb->pb_len = htole32((ebits + 7) / 8); 2513 2514 #ifdef UBSEC_DEBUG 2515 if (ubsec_debug) { 2516 kprintf("Epb "); 2517 ubsec_dump_pb(epb); 2518 } 2519 #endif 2520 2521 mcr->mcr_pkts = htole16(1); 2522 mcr->mcr_flags = 0; 2523 mcr->mcr_cmdctxp = htole32(me->me_q.q_ctx.dma_paddr); 2524 mcr->mcr_reserved = 0; 2525 mcr->mcr_pktlen = 0; 2526 2527 mcr->mcr_ipktbuf.pb_addr = htole32(me->me_M.dma_paddr); 2528 mcr->mcr_ipktbuf.pb_len = htole32(normbits / 8); 2529 mcr->mcr_ipktbuf.pb_next = htole32(me->me_epb.dma_paddr); 2530 2531 mcr->mcr_opktbuf.pb_addr = htole32(me->me_C.dma_paddr); 2532 mcr->mcr_opktbuf.pb_next = 0; 2533 mcr->mcr_opktbuf.pb_len = htole32(normbits / 8); 2534 2535 #ifdef DIAGNOSTIC 2536 /* Misaligned output buffer will hang the chip. */ 2537 if ((letoh32(mcr->mcr_opktbuf.pb_addr) & 3) != 0) 2538 panic("%s: modexp invalid addr 0x%x\n", 2539 device_get_nameunit(sc->sc_dev), 2540 letoh32(mcr->mcr_opktbuf.pb_addr)); 2541 if ((letoh32(mcr->mcr_opktbuf.pb_len) & 3) != 0) 2542 panic("%s: modexp invalid len 0x%x\n", 2543 device_get_nameunit(sc->sc_dev), 2544 letoh32(mcr->mcr_opktbuf.pb_len)); 2545 #endif 2546 2547 ctx = (struct ubsec_ctx_modexp *)me->me_q.q_ctx.dma_vaddr; 2548 bzero(ctx, sizeof(*ctx)); 2549 bcopy(krp->krp_param[UBS_MODEXP_PAR_N].crp_p, ctx->me_N, 2550 (nbits + 7) / 8); 2551 ctx->me_len = htole16((normbits / 8) + (4 * sizeof(u_int16_t))); 2552 ctx->me_op = htole16(UBS_CTXOP_MODEXP); 2553 ctx->me_E_len = htole16(ebits); 2554 ctx->me_N_len = htole16(nbits); 2555 2556 #ifdef UBSEC_DEBUG 2557 if (ubsec_debug) { 2558 ubsec_dump_mcr(mcr); 2559 ubsec_dump_ctx2((struct ubsec_ctx_keyop *)ctx); 2560 } 2561 #endif 2562 2563 /* 2564 * ubsec_feed2 will sync mcr and ctx, we just need to sync 2565 * everything else. 2566 */ 2567 ubsec_dma_sync(&me->me_M, BUS_DMASYNC_PREWRITE); 2568 ubsec_dma_sync(&me->me_E, BUS_DMASYNC_PREWRITE); 2569 ubsec_dma_sync(&me->me_C, BUS_DMASYNC_PREREAD); 2570 ubsec_dma_sync(&me->me_epb, BUS_DMASYNC_PREWRITE); 2571 2572 /* Enqueue and we're done... */ 2573 crit_enter(); 2574 SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &me->me_q, q_next); 2575 ubsec_feed2(sc); 2576 crit_exit(); 2577 2578 return (0); 2579 2580 errout: 2581 if (me != NULL) { 2582 if (me->me_q.q_mcr.dma_map != NULL) 2583 ubsec_dma_free(sc, &me->me_q.q_mcr); 2584 if (me->me_q.q_ctx.dma_map != NULL) { 2585 bzero(me->me_q.q_ctx.dma_vaddr, me->me_q.q_ctx.dma_size); 2586 ubsec_dma_free(sc, &me->me_q.q_ctx); 2587 } 2588 if (me->me_M.dma_map != NULL) { 2589 bzero(me->me_M.dma_vaddr, me->me_M.dma_size); 2590 ubsec_dma_free(sc, &me->me_M); 2591 } 2592 if (me->me_E.dma_map != NULL) { 2593 bzero(me->me_E.dma_vaddr, me->me_E.dma_size); 2594 ubsec_dma_free(sc, &me->me_E); 2595 } 2596 if (me->me_C.dma_map != NULL) { 2597 bzero(me->me_C.dma_vaddr, me->me_C.dma_size); 2598 ubsec_dma_free(sc, &me->me_C); 2599 } 2600 if (me->me_epb.dma_map != NULL) 2601 ubsec_dma_free(sc, &me->me_epb); 2602 kfree(me, M_DEVBUF); 2603 } 2604 krp->krp_status = err; 2605 crypto_kdone(krp); 2606 return (0); 2607 } 2608 2609 static int 2610 ubsec_kprocess_rsapriv(struct ubsec_softc *sc, struct cryptkop *krp, int hint) 2611 { 2612 struct ubsec_q2_rsapriv *rp = NULL; 2613 struct ubsec_mcr *mcr; 2614 struct ubsec_ctx_rsapriv *ctx; 2615 int err = 0; 2616 u_int padlen, msglen; 2617 2618 msglen = ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_P]); 2619 padlen = ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_Q]); 2620 if (msglen > padlen) 2621 padlen = msglen; 2622 2623 if (padlen <= 256) 2624 padlen = 256; 2625 else if (padlen <= 384) 2626 padlen = 384; 2627 else if (padlen <= 512) 2628 padlen = 512; 2629 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && padlen <= 768) 2630 padlen = 768; 2631 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && padlen <= 1024) 2632 padlen = 1024; 2633 else { 2634 err = E2BIG; 2635 goto errout; 2636 } 2637 2638 if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_DP]) > padlen) { 2639 err = E2BIG; 2640 goto errout; 2641 } 2642 2643 if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_DQ]) > padlen) { 2644 err = E2BIG; 2645 goto errout; 2646 } 2647 2648 if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_PINV]) > padlen) { 2649 err = E2BIG; 2650 goto errout; 2651 } 2652 2653 rp = kmalloc(sizeof *rp, M_DEVBUF, M_INTWAIT | M_ZERO); 2654 rp->rpr_krp = krp; 2655 rp->rpr_q.q_type = UBS_CTXOP_RSAPRIV; 2656 2657 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr), 2658 &rp->rpr_q.q_mcr, 0)) { 2659 err = ENOMEM; 2660 goto errout; 2661 } 2662 mcr = (struct ubsec_mcr *)rp->rpr_q.q_mcr.dma_vaddr; 2663 2664 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_rsapriv), 2665 &rp->rpr_q.q_ctx, 0)) { 2666 err = ENOMEM; 2667 goto errout; 2668 } 2669 ctx = (struct ubsec_ctx_rsapriv *)rp->rpr_q.q_ctx.dma_vaddr; 2670 bzero(ctx, sizeof *ctx); 2671 2672 /* Copy in p */ 2673 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_P].crp_p, 2674 &ctx->rpr_buf[0 * (padlen / 8)], 2675 (krp->krp_param[UBS_RSAPRIV_PAR_P].crp_nbits + 7) / 8); 2676 2677 /* Copy in q */ 2678 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_Q].crp_p, 2679 &ctx->rpr_buf[1 * (padlen / 8)], 2680 (krp->krp_param[UBS_RSAPRIV_PAR_Q].crp_nbits + 7) / 8); 2681 2682 /* Copy in dp */ 2683 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_DP].crp_p, 2684 &ctx->rpr_buf[2 * (padlen / 8)], 2685 (krp->krp_param[UBS_RSAPRIV_PAR_DP].crp_nbits + 7) / 8); 2686 2687 /* Copy in dq */ 2688 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_DQ].crp_p, 2689 &ctx->rpr_buf[3 * (padlen / 8)], 2690 (krp->krp_param[UBS_RSAPRIV_PAR_DQ].crp_nbits + 7) / 8); 2691 2692 /* Copy in pinv */ 2693 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_PINV].crp_p, 2694 &ctx->rpr_buf[4 * (padlen / 8)], 2695 (krp->krp_param[UBS_RSAPRIV_PAR_PINV].crp_nbits + 7) / 8); 2696 2697 msglen = padlen * 2; 2698 2699 /* Copy in input message (aligned buffer/length). */ 2700 if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_MSGIN]) > msglen) { 2701 /* Is this likely? */ 2702 err = E2BIG; 2703 goto errout; 2704 } 2705 if (ubsec_dma_malloc(sc, (msglen + 7) / 8, &rp->rpr_msgin, 0)) { 2706 err = ENOMEM; 2707 goto errout; 2708 } 2709 bzero(rp->rpr_msgin.dma_vaddr, (msglen + 7) / 8); 2710 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_MSGIN].crp_p, 2711 rp->rpr_msgin.dma_vaddr, 2712 (krp->krp_param[UBS_RSAPRIV_PAR_MSGIN].crp_nbits + 7) / 8); 2713 2714 /* Prepare space for output message (aligned buffer/length). */ 2715 if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT]) < msglen) { 2716 /* Is this likely? */ 2717 err = E2BIG; 2718 goto errout; 2719 } 2720 if (ubsec_dma_malloc(sc, (msglen + 7) / 8, &rp->rpr_msgout, 0)) { 2721 err = ENOMEM; 2722 goto errout; 2723 } 2724 bzero(rp->rpr_msgout.dma_vaddr, (msglen + 7) / 8); 2725 2726 mcr->mcr_pkts = htole16(1); 2727 mcr->mcr_flags = 0; 2728 mcr->mcr_cmdctxp = htole32(rp->rpr_q.q_ctx.dma_paddr); 2729 mcr->mcr_ipktbuf.pb_addr = htole32(rp->rpr_msgin.dma_paddr); 2730 mcr->mcr_ipktbuf.pb_next = 0; 2731 mcr->mcr_ipktbuf.pb_len = htole32(rp->rpr_msgin.dma_size); 2732 mcr->mcr_reserved = 0; 2733 mcr->mcr_pktlen = htole16(msglen); 2734 mcr->mcr_opktbuf.pb_addr = htole32(rp->rpr_msgout.dma_paddr); 2735 mcr->mcr_opktbuf.pb_next = 0; 2736 mcr->mcr_opktbuf.pb_len = htole32(rp->rpr_msgout.dma_size); 2737 2738 #ifdef DIAGNOSTIC 2739 if (rp->rpr_msgin.dma_paddr & 3 || rp->rpr_msgin.dma_size & 3) { 2740 panic("%s: rsapriv: invalid msgin %x(0x%jx)", 2741 device_get_nameunit(sc->sc_dev), 2742 rp->rpr_msgin.dma_paddr, 2743 (uintmax_t)rp->rpr_msgin.dma_size); 2744 } 2745 if (rp->rpr_msgout.dma_paddr & 3 || rp->rpr_msgout.dma_size & 3) { 2746 panic("%s: rsapriv: invalid msgout %x(0x%jx)", 2747 device_get_nameunit(sc->sc_dev), 2748 rp->rpr_msgout.dma_paddr, 2749 (uintmax_t)rp->rpr_msgout.dma_size); 2750 } 2751 #endif 2752 2753 ctx->rpr_len = (sizeof(u_int16_t) * 4) + (5 * (padlen / 8)); 2754 ctx->rpr_op = htole16(UBS_CTXOP_RSAPRIV); 2755 ctx->rpr_q_len = htole16(padlen); 2756 ctx->rpr_p_len = htole16(padlen); 2757 2758 /* 2759 * ubsec_feed2 will sync mcr and ctx, we just need to sync 2760 * everything else. 2761 */ 2762 ubsec_dma_sync(&rp->rpr_msgin, BUS_DMASYNC_PREWRITE); 2763 ubsec_dma_sync(&rp->rpr_msgout, BUS_DMASYNC_PREREAD); 2764 2765 /* Enqueue and we're done... */ 2766 crit_enter(); 2767 SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &rp->rpr_q, q_next); 2768 ubsec_feed2(sc); 2769 ubsecstats.hst_modexpcrt++; 2770 crit_exit(); 2771 return (0); 2772 2773 errout: 2774 if (rp != NULL) { 2775 if (rp->rpr_q.q_mcr.dma_map != NULL) 2776 ubsec_dma_free(sc, &rp->rpr_q.q_mcr); 2777 if (rp->rpr_msgin.dma_map != NULL) { 2778 bzero(rp->rpr_msgin.dma_vaddr, rp->rpr_msgin.dma_size); 2779 ubsec_dma_free(sc, &rp->rpr_msgin); 2780 } 2781 if (rp->rpr_msgout.dma_map != NULL) { 2782 bzero(rp->rpr_msgout.dma_vaddr, rp->rpr_msgout.dma_size); 2783 ubsec_dma_free(sc, &rp->rpr_msgout); 2784 } 2785 kfree(rp, M_DEVBUF); 2786 } 2787 krp->krp_status = err; 2788 crypto_kdone(krp); 2789 return (0); 2790 } 2791 2792 #ifdef UBSEC_DEBUG 2793 static void 2794 ubsec_dump_pb(volatile struct ubsec_pktbuf *pb) 2795 { 2796 kprintf("addr 0x%x (0x%x) next 0x%x\n", 2797 pb->pb_addr, pb->pb_len, pb->pb_next); 2798 } 2799 2800 static void 2801 ubsec_dump_ctx2(struct ubsec_ctx_keyop *c) 2802 { 2803 kprintf("CTX (0x%x):\n", c->ctx_len); 2804 switch (letoh16(c->ctx_op)) { 2805 case UBS_CTXOP_RNGBYPASS: 2806 case UBS_CTXOP_RNGSHA1: 2807 break; 2808 case UBS_CTXOP_MODEXP: 2809 { 2810 struct ubsec_ctx_modexp *cx = (void *)c; 2811 int i, len; 2812 2813 kprintf(" Elen %u, Nlen %u\n", 2814 letoh16(cx->me_E_len), letoh16(cx->me_N_len)); 2815 len = (cx->me_N_len + 7)/8; 2816 for (i = 0; i < len; i++) 2817 kprintf("%s%02x", (i == 0) ? " N: " : ":", cx->me_N[i]); 2818 kprintf("\n"); 2819 break; 2820 } 2821 default: 2822 kprintf("unknown context: %x\n", c->ctx_op); 2823 } 2824 kprintf("END CTX\n"); 2825 } 2826 2827 static void 2828 ubsec_dump_mcr(struct ubsec_mcr *mcr) 2829 { 2830 volatile struct ubsec_mcr_add *ma; 2831 int i; 2832 2833 kprintf("MCR:\n"); 2834 kprintf(" pkts: %u, flags 0x%x\n", 2835 letoh16(mcr->mcr_pkts), letoh16(mcr->mcr_flags)); 2836 ma = (volatile struct ubsec_mcr_add *)&mcr->mcr_cmdctxp; 2837 for (i = 0; i < letoh16(mcr->mcr_pkts); i++) { 2838 kprintf(" %d: ctx 0x%x len 0x%x rsvd 0x%x\n", i, 2839 letoh32(ma->mcr_cmdctxp), letoh16(ma->mcr_pktlen), 2840 letoh16(ma->mcr_reserved)); 2841 kprintf(" %d: ipkt ", i); 2842 ubsec_dump_pb(&ma->mcr_ipktbuf); 2843 kprintf(" %d: opkt ", i); 2844 ubsec_dump_pb(&ma->mcr_opktbuf); 2845 ma++; 2846 } 2847 kprintf("END MCR\n"); 2848 } 2849 #endif /* UBSEC_DEBUG */ 2850 2851 /* 2852 * Return the number of significant bits of a big number. 2853 */ 2854 static int 2855 ubsec_ksigbits(struct crparam *cr) 2856 { 2857 u_int plen = (cr->crp_nbits + 7) / 8; 2858 int i, sig = plen * 8; 2859 u_int8_t c, *p = cr->crp_p; 2860 2861 for (i = plen - 1; i >= 0; i--) { 2862 c = p[i]; 2863 if (c != 0) { 2864 while ((c & 0x80) == 0) { 2865 sig--; 2866 c <<= 1; 2867 } 2868 break; 2869 } 2870 sig -= 8; 2871 } 2872 return (sig); 2873 } 2874 2875 static void 2876 ubsec_kshift_r( 2877 u_int shiftbits, 2878 u_int8_t *src, u_int srcbits, 2879 u_int8_t *dst, u_int dstbits) 2880 { 2881 u_int slen, dlen; 2882 int i, si, di, n; 2883 2884 slen = (srcbits + 7) / 8; 2885 dlen = (dstbits + 7) / 8; 2886 2887 for (i = 0; i < slen; i++) 2888 dst[i] = src[i]; 2889 for (i = 0; i < dlen - slen; i++) 2890 dst[slen + i] = 0; 2891 2892 n = shiftbits / 8; 2893 if (n != 0) { 2894 si = dlen - n - 1; 2895 di = dlen - 1; 2896 while (si >= 0) 2897 dst[di--] = dst[si--]; 2898 while (di >= 0) 2899 dst[di--] = 0; 2900 } 2901 2902 n = shiftbits % 8; 2903 if (n != 0) { 2904 for (i = dlen - 1; i > 0; i--) 2905 dst[i] = (dst[i] << n) | 2906 (dst[i - 1] >> (8 - n)); 2907 dst[0] = dst[0] << n; 2908 } 2909 } 2910 2911 static void 2912 ubsec_kshift_l( 2913 u_int shiftbits, 2914 u_int8_t *src, u_int srcbits, 2915 u_int8_t *dst, u_int dstbits) 2916 { 2917 int slen, dlen, i, n; 2918 2919 slen = (srcbits + 7) / 8; 2920 dlen = (dstbits + 7) / 8; 2921 2922 n = shiftbits / 8; 2923 for (i = 0; i < slen; i++) 2924 dst[i] = src[i + n]; 2925 for (i = 0; i < dlen - slen; i++) 2926 dst[slen + i] = 0; 2927 2928 n = shiftbits % 8; 2929 if (n != 0) { 2930 for (i = 0; i < (dlen - 1); i++) 2931 dst[i] = (dst[i] >> n) | (dst[i + 1] << (8 - n)); 2932 dst[dlen - 1] = dst[dlen - 1] >> n; 2933 } 2934 } 2935