1 /* $OpenBSD: crypto.c,v 1.89 2021/10/21 23:03:48 tobhe Exp $ */ 2 /* 3 * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu) 4 * 5 * This code was written by Angelos D. Keromytis in Athens, Greece, in 6 * February 2000. Network Security Technologies Inc. (NSTI) kindly 7 * supported the development of this code. 8 * 9 * Copyright (c) 2000, 2001 Angelos D. Keromytis 10 * 11 * Permission to use, copy, and modify this software with or without fee 12 * is hereby granted, provided that this entire notice is included in 13 * all source code copies of any software which is or includes a copy or 14 * modification of this software. 15 * 16 * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR 17 * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY 18 * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE 19 * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR 20 * PURPOSE. 21 */ 22 23 #include <sys/param.h> 24 #include <sys/systm.h> 25 #include <sys/malloc.h> 26 #include <sys/pool.h> 27 28 #include <crypto/cryptodev.h> 29 30 /* 31 * Locks used to protect struct members in this file: 32 * A allocated during driver attach, no hotplug, no detach 33 * I immutable after creation 34 * K kernel lock 35 */ 36 37 struct cryptocap *crypto_drivers; /* [A] array allocated by driver 38 [K] driver data and session count */ 39 int crypto_drivers_num = 0; /* [A] attached drivers array size */ 40 41 struct pool cryptop_pool; /* [I] set of crypto descriptors */ 42 43 /* 44 * Create a new session. 45 */ 46 int 47 crypto_newsession(u_int64_t *sid, struct cryptoini *cri, int hard) 48 { 49 u_int32_t hid, lid, hid2 = -1; 50 struct cryptocap *cpc; 51 struct cryptoini *cr; 52 int err, s, turn = 0; 53 54 if (crypto_drivers == NULL) 55 return EINVAL; 56 57 KERNEL_ASSERT_LOCKED(); 58 59 s = splvm(); 60 61 /* 62 * The algorithm we use here is pretty stupid; just use the 63 * first driver that supports all the algorithms we need. Do 64 * a double-pass over all the drivers, ignoring software ones 65 * at first, to deal with cases of drivers that register after 66 * the software one(s) --- e.g., PCMCIA crypto cards. 67 * 68 * XXX We need more smarts here (in real life too, but that's 69 * XXX another story altogether). 70 */ 71 do { 72 for (hid = 0; hid < crypto_drivers_num; hid++) { 73 cpc = &crypto_drivers[hid]; 74 75 /* 76 * If it's not initialized or has remaining sessions 77 * referencing it, skip. 78 */ 79 if (cpc->cc_newsession == NULL || 80 (cpc->cc_flags & CRYPTOCAP_F_CLEANUP)) 81 continue; 82 83 if (cpc->cc_flags & CRYPTOCAP_F_SOFTWARE) { 84 /* 85 * First round of search, ignore 86 * software drivers. 87 */ 88 if (turn == 0) 89 continue; 90 } else { /* !CRYPTOCAP_F_SOFTWARE */ 91 /* Second round of search, only software. */ 92 if (turn == 1) 93 continue; 94 } 95 96 /* See if all the algorithms are supported. */ 97 for (cr = cri; cr; cr = cr->cri_next) { 98 if (cpc->cc_alg[cr->cri_alg] == 0) 99 break; 100 } 101 102 /* 103 * If even one algorithm is not supported, 104 * keep searching. 105 */ 106 if (cr != NULL) 107 continue; 108 109 /* 110 * If we had a previous match, see how it compares 111 * to this one. Keep "remembering" whichever is 112 * the best of the two. 113 */ 114 if (hid2 != -1) { 115 /* 116 * Compare session numbers, pick the one 117 * with the lowest. 118 * XXX Need better metrics, this will 119 * XXX just do un-weighted round-robin. 120 */ 121 if (crypto_drivers[hid].cc_sessions <= 122 crypto_drivers[hid2].cc_sessions) 123 hid2 = hid; 124 } else { 125 /* 126 * Remember this one, for future 127 * comparisons. 128 */ 129 hid2 = hid; 130 } 131 } 132 133 /* 134 * If we found something worth remembering, leave. The 135 * side-effect is that we will always prefer a hardware 136 * driver over the software one. 137 */ 138 if (hid2 != -1) 139 break; 140 141 turn++; 142 143 /* If we only want hardware drivers, don't do second pass. */ 144 } while (turn <= 2 && hard == 0); 145 146 hid = hid2; 147 148 /* 149 * Can't do everything in one session. 150 * 151 * XXX Fix this. We need to inject a "virtual" session 152 * XXX layer right about here. 153 */ 154 155 if (hid == -1) { 156 splx(s); 157 return EINVAL; 158 } 159 160 /* Call the driver initialization routine. */ 161 lid = hid; /* Pass the driver ID. */ 162 err = crypto_drivers[hid].cc_newsession(&lid, cri); 163 if (err == 0) { 164 (*sid) = hid; 165 (*sid) <<= 32; 166 (*sid) |= (lid & 0xffffffff); 167 crypto_drivers[hid].cc_sessions++; 168 } 169 170 splx(s); 171 return err; 172 } 173 174 /* 175 * Delete an existing session (or a reserved session on an unregistered 176 * driver). 177 */ 178 int 179 crypto_freesession(u_int64_t sid) 180 { 181 int err = 0, s; 182 u_int32_t hid; 183 184 if (crypto_drivers == NULL) 185 return EINVAL; 186 187 /* Determine two IDs. */ 188 hid = (sid >> 32) & 0xffffffff; 189 190 if (hid >= crypto_drivers_num) 191 return ENOENT; 192 193 KERNEL_ASSERT_LOCKED(); 194 195 s = splvm(); 196 197 if (crypto_drivers[hid].cc_sessions) 198 crypto_drivers[hid].cc_sessions--; 199 200 /* Call the driver cleanup routine, if available. */ 201 if (crypto_drivers[hid].cc_freesession) 202 err = crypto_drivers[hid].cc_freesession(sid); 203 204 /* 205 * If this was the last session of a driver marked as invalid, 206 * make the entry available for reuse. 207 */ 208 if ((crypto_drivers[hid].cc_flags & CRYPTOCAP_F_CLEANUP) && 209 crypto_drivers[hid].cc_sessions == 0) 210 explicit_bzero(&crypto_drivers[hid], sizeof(struct cryptocap)); 211 212 splx(s); 213 return err; 214 } 215 216 /* 217 * Find an empty slot. 218 */ 219 int32_t 220 crypto_get_driverid(u_int8_t flags) 221 { 222 struct cryptocap *newdrv; 223 int i, s; 224 225 /* called from attach routines */ 226 KERNEL_ASSERT_LOCKED(); 227 228 s = splvm(); 229 230 if (crypto_drivers_num == 0) { 231 crypto_drivers_num = CRYPTO_DRIVERS_INITIAL; 232 crypto_drivers = mallocarray(crypto_drivers_num, 233 sizeof(struct cryptocap), M_CRYPTO_DATA, M_NOWAIT | M_ZERO); 234 if (crypto_drivers == NULL) { 235 crypto_drivers_num = 0; 236 splx(s); 237 return -1; 238 } 239 } 240 241 for (i = 0; i < crypto_drivers_num; i++) { 242 if (crypto_drivers[i].cc_process == NULL && 243 !(crypto_drivers[i].cc_flags & CRYPTOCAP_F_CLEANUP) && 244 crypto_drivers[i].cc_sessions == 0) { 245 crypto_drivers[i].cc_sessions = 1; /* Mark */ 246 crypto_drivers[i].cc_flags = flags; 247 splx(s); 248 return i; 249 } 250 } 251 252 /* Out of entries, allocate some more. */ 253 if (crypto_drivers_num >= CRYPTO_DRIVERS_MAX) { 254 splx(s); 255 return -1; 256 } 257 258 newdrv = mallocarray(crypto_drivers_num, 259 2 * sizeof(struct cryptocap), M_CRYPTO_DATA, M_NOWAIT); 260 if (newdrv == NULL) { 261 splx(s); 262 return -1; 263 } 264 265 memcpy(newdrv, crypto_drivers, 266 crypto_drivers_num * sizeof(struct cryptocap)); 267 bzero(&newdrv[crypto_drivers_num], 268 crypto_drivers_num * sizeof(struct cryptocap)); 269 270 newdrv[i].cc_sessions = 1; /* Mark */ 271 newdrv[i].cc_flags = flags; 272 273 free(crypto_drivers, M_CRYPTO_DATA, 274 crypto_drivers_num * sizeof(struct cryptocap)); 275 276 crypto_drivers_num *= 2; 277 crypto_drivers = newdrv; 278 splx(s); 279 return i; 280 } 281 282 /* 283 * Register a crypto driver. It should be called once for each algorithm 284 * supported by the driver. 285 */ 286 int 287 crypto_register(u_int32_t driverid, int *alg, 288 int (*newses)(u_int32_t *, struct cryptoini *), 289 int (*freeses)(u_int64_t), int (*process)(struct cryptop *)) 290 { 291 int s, i; 292 293 if (driverid >= crypto_drivers_num || alg == NULL || 294 crypto_drivers == NULL) 295 return EINVAL; 296 297 /* called from attach routines */ 298 KERNEL_ASSERT_LOCKED(); 299 300 s = splvm(); 301 302 for (i = 0; i <= CRYPTO_ALGORITHM_MAX; i++) { 303 /* 304 * XXX Do some performance testing to determine 305 * placing. We probably need an auxiliary data 306 * structure that describes relative performances. 307 */ 308 309 crypto_drivers[driverid].cc_alg[i] = alg[i]; 310 } 311 312 313 crypto_drivers[driverid].cc_newsession = newses; 314 crypto_drivers[driverid].cc_process = process; 315 crypto_drivers[driverid].cc_freesession = freeses; 316 crypto_drivers[driverid].cc_sessions = 0; /* Unmark */ 317 318 splx(s); 319 320 return 0; 321 } 322 323 /* 324 * Unregister a crypto driver. If there are pending sessions using it, 325 * leave enough information around so that subsequent calls using those 326 * sessions will correctly detect the driver being unregistered and reroute 327 * the request. 328 */ 329 int 330 crypto_unregister(u_int32_t driverid, int alg) 331 { 332 int i = CRYPTO_ALGORITHM_MAX + 1, s; 333 u_int32_t ses; 334 335 /* may be called from detach routines, but not used */ 336 KERNEL_ASSERT_LOCKED(); 337 338 s = splvm(); 339 340 /* Sanity checks. */ 341 if (driverid >= crypto_drivers_num || crypto_drivers == NULL || 342 alg <= 0 || alg > (CRYPTO_ALGORITHM_MAX + 1)) { 343 splx(s); 344 return EINVAL; 345 } 346 347 if (alg != CRYPTO_ALGORITHM_MAX + 1) { 348 if (crypto_drivers[driverid].cc_alg[alg] == 0) { 349 splx(s); 350 return EINVAL; 351 } 352 crypto_drivers[driverid].cc_alg[alg] = 0; 353 354 /* Was this the last algorithm ? */ 355 for (i = 1; i <= CRYPTO_ALGORITHM_MAX; i++) 356 if (crypto_drivers[driverid].cc_alg[i] != 0) 357 break; 358 } 359 360 /* 361 * If a driver unregistered its last algorithm or all of them 362 * (alg == CRYPTO_ALGORITHM_MAX + 1), cleanup its entry. 363 */ 364 if (i == CRYPTO_ALGORITHM_MAX + 1 || alg == CRYPTO_ALGORITHM_MAX + 1) { 365 ses = crypto_drivers[driverid].cc_sessions; 366 bzero(&crypto_drivers[driverid], sizeof(struct cryptocap)); 367 if (ses != 0) { 368 /* 369 * If there are pending sessions, just mark as invalid. 370 */ 371 crypto_drivers[driverid].cc_flags |= CRYPTOCAP_F_CLEANUP; 372 crypto_drivers[driverid].cc_sessions = ses; 373 } 374 } 375 splx(s); 376 return 0; 377 } 378 379 /* 380 * Add crypto request to a queue, to be processed by a kernel thread. 381 */ 382 void 383 crypto_dispatch(struct cryptop *crp) 384 { 385 int lock = 1, s; 386 u_int32_t hid; 387 388 s = splvm(); 389 hid = (crp->crp_sid >> 32) & 0xffffffff; 390 if (hid < crypto_drivers_num) { 391 if (crypto_drivers[hid].cc_flags & CRYPTOCAP_F_MPSAFE) 392 lock = 0; 393 } 394 splx(s); 395 396 /* XXXSMP crypto_invoke() is not MP safe */ 397 lock = 1; 398 399 if (lock) 400 KERNEL_LOCK(); 401 crypto_invoke(crp); 402 if (lock) 403 KERNEL_UNLOCK(); 404 } 405 406 /* 407 * Dispatch a crypto request to the appropriate crypto devices. 408 */ 409 void 410 crypto_invoke(struct cryptop *crp) 411 { 412 u_int64_t nid; 413 u_int32_t hid; 414 int error; 415 int s, i; 416 417 /* Sanity checks. */ 418 KASSERT(crp != NULL); 419 KASSERT(crp->crp_callback != NULL); 420 421 KERNEL_ASSERT_LOCKED(); 422 423 s = splvm(); 424 if (crp->crp_ndesc < 1 || crypto_drivers == NULL) { 425 crp->crp_etype = EINVAL; 426 goto done; 427 } 428 429 hid = (crp->crp_sid >> 32) & 0xffffffff; 430 if (hid >= crypto_drivers_num) 431 goto migrate; 432 433 if (crypto_drivers[hid].cc_flags & CRYPTOCAP_F_CLEANUP) { 434 crypto_freesession(crp->crp_sid); 435 goto migrate; 436 } 437 438 if (crypto_drivers[hid].cc_process == NULL) 439 goto migrate; 440 441 crypto_drivers[hid].cc_operations++; 442 crypto_drivers[hid].cc_bytes += crp->crp_ilen; 443 444 error = crypto_drivers[hid].cc_process(crp); 445 if (error) { 446 if (error == ERESTART) { 447 /* Unregister driver and migrate session. */ 448 crypto_unregister(hid, CRYPTO_ALGORITHM_MAX + 1); 449 goto migrate; 450 } else { 451 crp->crp_etype = error; 452 } 453 } 454 455 splx(s); 456 return; 457 458 migrate: 459 /* Migrate session. */ 460 for (i = 0; i < crp->crp_ndesc - 1; i++) 461 crp->crp_desc[i].CRD_INI.cri_next = &crp->crp_desc[i+1].CRD_INI; 462 crp->crp_desc[crp->crp_ndesc].CRD_INI.cri_next = NULL; 463 464 if (crypto_newsession(&nid, &(crp->crp_desc->CRD_INI), 0) == 0) 465 crp->crp_sid = nid; 466 467 crp->crp_etype = EAGAIN; 468 done: 469 crypto_done(crp); 470 splx(s); 471 } 472 473 /* 474 * Release a set of crypto descriptors. 475 */ 476 void 477 crypto_freereq(struct cryptop *crp) 478 { 479 if (crp == NULL) 480 return; 481 482 if (crp->crp_ndescalloc > 2) 483 free(crp->crp_desc, M_CRYPTO_DATA, 484 crp->crp_ndescalloc * sizeof(struct cryptodesc)); 485 pool_put(&cryptop_pool, crp); 486 } 487 488 /* 489 * Acquire a set of crypto descriptors. 490 */ 491 struct cryptop * 492 crypto_getreq(int num) 493 { 494 struct cryptop *crp; 495 496 crp = pool_get(&cryptop_pool, PR_NOWAIT | PR_ZERO); 497 if (crp == NULL) 498 return NULL; 499 500 crp->crp_desc = crp->crp_sdesc; 501 crp->crp_ndescalloc = crp->crp_ndesc = num; 502 503 if (num > 2) { 504 crp->crp_desc = mallocarray(num, sizeof(struct cryptodesc), 505 M_CRYPTO_DATA, M_NOWAIT | M_ZERO); 506 if (crp->crp_desc == NULL) { 507 pool_put(&cryptop_pool, crp); 508 return NULL; 509 } 510 } 511 512 return crp; 513 } 514 515 void 516 crypto_init(void) 517 { 518 pool_init(&cryptop_pool, sizeof(struct cryptop), 0, IPL_VM, 0, 519 "cryptop", NULL); 520 } 521 522 /* 523 * Invoke the callback on behalf of the driver. 524 */ 525 void 526 crypto_done(struct cryptop *crp) 527 { 528 crp->crp_flags |= CRYPTO_F_DONE; 529 530 crp->crp_callback(crp); 531 } 532