1 /* $NetBSD: cprng_fast.c,v 1.19 2023/08/05 11:39:18 riastradh Exp $ */ 2 3 /*- 4 * Copyright (c) 2014 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Taylor R. Campbell. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 __KERNEL_RCSID(0, "$NetBSD: cprng_fast.c,v 1.19 2023/08/05 11:39:18 riastradh Exp $"); 34 35 #include <sys/types.h> 36 #include <sys/param.h> 37 #include <sys/bitops.h> 38 #include <sys/cprng.h> 39 #include <sys/cpu.h> 40 #include <sys/entropy.h> 41 #include <sys/evcnt.h> 42 #include <sys/kmem.h> 43 #include <sys/percpu.h> 44 #include <sys/pserialize.h> 45 46 #include <crypto/chacha/chacha.h> 47 48 #define CPRNG_FAST_SEED_BYTES CHACHA_STREAM_KEYBYTES 49 50 struct cprng_fast { 51 /* 128-bit vector unit generates 256 bytes at once */ 52 uint8_t buf[256]; 53 uint8_t key[CPRNG_FAST_SEED_BYTES]; 54 uint8_t nonce[CHACHA_STREAM_NONCEBYTES]; 55 unsigned i; 56 struct evcnt *reseed_evcnt; 57 unsigned epoch; 58 }; 59 60 static void cprng_fast_init_cpu(void *, void *, struct cpu_info *); 61 static void cprng_fast_reseed(struct cprng_fast **, unsigned); 62 63 static void cprng_fast_seed(struct cprng_fast *, const void *); 64 static void cprng_fast_buf(struct cprng_fast *, void *, unsigned); 65 66 static void cprng_fast_buf_short(void *, size_t); 67 static void cprng_fast_buf_long(void *, size_t); 68 69 static percpu_t *cprng_fast_percpu __read_mostly; 70 71 void 72 cprng_fast_init(void) 73 { 74 75 cprng_fast_percpu = percpu_create(sizeof(struct cprng_fast), 76 cprng_fast_init_cpu, NULL, NULL); 77 } 78 79 static void 80 cprng_fast_init_cpu(void *p, void *arg __unused, struct cpu_info *ci) 81 { 82 struct cprng_fast *const cprng = p; 83 84 cprng->epoch = 0; 85 86 cprng->reseed_evcnt = kmem_alloc(sizeof(*cprng->reseed_evcnt), 87 KM_SLEEP); 88 evcnt_attach_dynamic(cprng->reseed_evcnt, EVCNT_TYPE_MISC, NULL, 89 ci->ci_cpuname, "cprng_fast reseed"); 90 } 91 92 static int 93 cprng_fast_get(struct cprng_fast **cprngp) 94 { 95 struct cprng_fast *cprng; 96 unsigned epoch; 97 int s; 98 99 KASSERT(!cpu_intr_p()); 100 KASSERT(pserialize_not_in_read_section()); 101 102 *cprngp = cprng = percpu_getref(cprng_fast_percpu); 103 s = splsoftserial(); 104 105 epoch = entropy_epoch(); 106 if (__predict_false(cprng->epoch != epoch)) { 107 splx(s); 108 cprng_fast_reseed(cprngp, epoch); 109 s = splsoftserial(); 110 } 111 112 return s; 113 } 114 115 static void 116 cprng_fast_put(struct cprng_fast *cprng, int s) 117 { 118 119 KASSERT((cprng == percpu_getref(cprng_fast_percpu)) && 120 (percpu_putref(cprng_fast_percpu), true)); 121 splx(s); 122 percpu_putref(cprng_fast_percpu); 123 } 124 125 static void 126 cprng_fast_reseed(struct cprng_fast **cprngp, unsigned epoch) 127 { 128 struct cprng_fast *cprng; 129 uint8_t seed[CPRNG_FAST_SEED_BYTES]; 130 int s; 131 132 /* 133 * Drop the percpu(9) reference to extract a fresh seed from 134 * the entropy pool. cprng_strong may sleep on an adaptive 135 * lock, which invalidates our percpu(9) reference. 136 * 137 * This may race with reseeding in another thread, which is no 138 * big deal -- worst case, we rewind the entropy epoch here and 139 * cause the next caller to reseed again, and in the end we 140 * just reseed a couple more times than necessary. 141 */ 142 percpu_putref(cprng_fast_percpu); 143 cprng_strong(kern_cprng, seed, sizeof(seed), 0); 144 *cprngp = cprng = percpu_getref(cprng_fast_percpu); 145 146 s = splsoftserial(); 147 cprng_fast_seed(cprng, seed); 148 cprng->epoch = epoch; 149 cprng->reseed_evcnt->ev_count++; 150 splx(s); 151 152 explicit_memset(seed, 0, sizeof(seed)); 153 } 154 155 /* CPRNG algorithm */ 156 157 static void 158 cprng_fast_seed(struct cprng_fast *cprng, const void *seed) 159 { 160 161 (void)memset(cprng->buf, 0, sizeof cprng->buf); 162 (void)memcpy(cprng->key, seed, sizeof cprng->key); 163 (void)memset(cprng->nonce, 0, sizeof cprng->nonce); 164 cprng->i = sizeof cprng->buf; 165 } 166 167 static void 168 cprng_fast_buf(struct cprng_fast *cprng, void *buf, unsigned len) 169 { 170 uint8_t *p = buf; 171 unsigned n = len, n0; 172 173 KASSERT(cprng->i <= sizeof(cprng->buf)); 174 KASSERT(len <= sizeof(cprng->buf)); 175 176 n0 = MIN(n, sizeof(cprng->buf) - cprng->i); 177 memcpy(p, &cprng->buf[cprng->i], n0); 178 if ((n -= n0) == 0) { 179 cprng->i += n0; 180 KASSERT(cprng->i <= sizeof(cprng->buf)); 181 return; 182 } 183 p += n0; 184 le64enc(cprng->nonce, 1 + le64dec(cprng->nonce)); 185 chacha_stream(cprng->buf, sizeof(cprng->buf), 0, cprng->nonce, 186 cprng->key, 8); 187 memcpy(p, cprng->buf, n); 188 cprng->i = n; 189 } 190 191 /* Public API */ 192 193 static void 194 cprng_fast_buf_short(void *buf, size_t len) 195 { 196 struct cprng_fast *cprng; 197 int s; 198 199 KASSERT(len <= sizeof(cprng->buf)); 200 201 s = cprng_fast_get(&cprng); 202 cprng_fast_buf(cprng, buf, len); 203 cprng_fast_put(cprng, s); 204 } 205 206 static void 207 cprng_fast_buf_long(void *buf, size_t len) 208 { 209 uint8_t seed[CHACHA_STREAM_KEYBYTES]; 210 uint8_t nonce[CHACHA_STREAM_NONCEBYTES] = {0}; 211 212 CTASSERT(sizeof(seed) <= sizeof(((struct cprng_fast *)0)->buf)); 213 214 #if SIZE_MAX >= 0x3fffffffff 215 /* >=256 GB is not reasonable */ 216 KASSERT(len <= 0x3fffffffff); 217 #endif 218 219 cprng_fast_buf_short(seed, sizeof seed); 220 chacha_stream(buf, len, 0, nonce, seed, 8); 221 222 (void)explicit_memset(seed, 0, sizeof seed); 223 } 224 225 uint32_t 226 cprng_fast32(void) 227 { 228 uint32_t v; 229 230 cprng_fast_buf_short(&v, sizeof v); 231 232 return v; 233 } 234 235 uint64_t 236 cprng_fast64(void) 237 { 238 uint64_t v; 239 240 cprng_fast_buf_short(&v, sizeof v); 241 242 return v; 243 } 244 245 size_t 246 cprng_fast(void *buf, size_t len) 247 { 248 249 /* 250 * We don't want to hog the CPU, so we use the short version, 251 * to generate output without preemption, only if we can do it 252 * with at most one ChaCha call. 253 */ 254 if (len <= sizeof(((struct cprng_fast *)0)->buf)) 255 cprng_fast_buf_short(buf, len); 256 else 257 cprng_fast_buf_long(buf, len); 258 259 return len; /* hysterical raisins */ 260 } 261