1 /* $OpenBSD: subr_percpu.c,v 1.8 2017/09/08 05:36:53 deraadt Exp $ */ 2 3 /* 4 * Copyright (c) 2016 David Gwynne <dlg@openbsd.org> 5 * 6 * Permission to use, copy, modify, and distribute this software for any 7 * purpose with or without fee is hereby granted, provided that the above 8 * copyright notice and this permission notice appear in all copies. 9 * 10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 17 */ 18 19 #include <sys/param.h> 20 #include <sys/systm.h> 21 #include <sys/pool.h> 22 #include <sys/malloc.h> 23 24 #include <sys/percpu.h> 25 26 #ifdef MULTIPROCESSOR 27 struct pool cpumem_pl; 28 29 void 30 percpu_init(void) 31 { 32 pool_init(&cpumem_pl, sizeof(struct cpumem) * ncpusfound, 0, 33 IPL_NONE, PR_WAITOK, "percpumem", &pool_allocator_single); 34 } 35 36 struct cpumem * 37 cpumem_get(struct pool *pp) 38 { 39 struct cpumem *cm; 40 unsigned int cpu; 41 42 cm = pool_get(&cpumem_pl, PR_WAITOK); 43 44 for (cpu = 0; cpu < ncpusfound; cpu++) 45 cm[cpu].mem = pool_get(pp, PR_WAITOK | PR_ZERO); 46 47 return (cm); 48 } 49 50 void 51 cpumem_put(struct pool *pp, struct cpumem *cm) 52 { 53 unsigned int cpu; 54 55 for (cpu = 0; cpu < ncpusfound; cpu++) 56 pool_put(pp, cm[cpu].mem); 57 58 pool_put(&cpumem_pl, cm); 59 } 60 61 struct cpumem * 62 cpumem_malloc(size_t sz, int type) 63 { 64 struct cpumem *cm; 65 unsigned int cpu; 66 67 sz = roundup(sz, CACHELINESIZE); 68 69 cm = pool_get(&cpumem_pl, PR_WAITOK); 70 71 for (cpu = 0; cpu < ncpusfound; cpu++) 72 cm[cpu].mem = malloc(sz, type, M_WAITOK | M_ZERO); 73 74 return (cm); 75 } 76 77 struct cpumem * 78 cpumem_malloc_ncpus(struct cpumem *bootcm, size_t sz, int type) 79 { 80 struct cpumem *cm; 81 unsigned int cpu; 82 83 sz = roundup(sz, CACHELINESIZE); 84 85 cm = pool_get(&cpumem_pl, PR_WAITOK); 86 87 cm[0].mem = bootcm[0].mem; 88 for (cpu = 1; cpu < ncpusfound; cpu++) 89 cm[cpu].mem = malloc(sz, type, M_WAITOK | M_ZERO); 90 91 return (cm); 92 } 93 94 void 95 cpumem_free(struct cpumem *cm, int type, size_t sz) 96 { 97 unsigned int cpu; 98 99 sz = roundup(sz, CACHELINESIZE); 100 101 for (cpu = 0; cpu < ncpusfound; cpu++) 102 free(cm[cpu].mem, type, sz); 103 104 pool_put(&cpumem_pl, cm); 105 } 106 107 void * 108 cpumem_first(struct cpumem_iter *i, struct cpumem *cm) 109 { 110 i->cpu = 0; 111 112 return (cm[0].mem); 113 } 114 115 void * 116 cpumem_next(struct cpumem_iter *i, struct cpumem *cm) 117 { 118 unsigned int cpu = ++i->cpu; 119 120 if (cpu >= ncpusfound) 121 return (NULL); 122 123 return (cm[cpu].mem); 124 } 125 126 struct cpumem * 127 counters_alloc(unsigned int n) 128 { 129 struct cpumem *cm; 130 struct cpumem_iter cmi; 131 uint64_t *counters; 132 unsigned int i; 133 134 KASSERT(n > 0); 135 136 n++; /* add space for a generation number */ 137 cm = cpumem_malloc(n * sizeof(uint64_t), M_COUNTERS); 138 139 CPUMEM_FOREACH(counters, &cmi, cm) { 140 for (i = 0; i < n; i++) 141 counters[i] = 0; 142 } 143 144 return (cm); 145 } 146 147 struct cpumem * 148 counters_alloc_ncpus(struct cpumem *cm, unsigned int n) 149 { 150 n++; /* the generation number */ 151 return (cpumem_malloc_ncpus(cm, n * sizeof(uint64_t), M_COUNTERS)); 152 } 153 154 void 155 counters_free(struct cpumem *cm, unsigned int n) 156 { 157 n++; /* generation number */ 158 cpumem_free(cm, M_COUNTERS, n * sizeof(uint64_t)); 159 } 160 161 void 162 counters_read(struct cpumem *cm, uint64_t *output, unsigned int n) 163 { 164 struct cpumem_iter cmi; 165 uint64_t *gen, *counters, *temp; 166 uint64_t enter, leave; 167 unsigned int i; 168 169 for (i = 0; i < n; i++) 170 output[i] = 0; 171 172 temp = mallocarray(n, sizeof(uint64_t), M_TEMP, M_WAITOK); 173 174 gen = cpumem_first(&cmi, cm); 175 do { 176 counters = gen + 1; 177 178 enter = *gen; 179 for (;;) { 180 /* the generation number is odd during an update */ 181 while (enter & 1) { 182 yield(); 183 enter = *gen; 184 } 185 186 membar_consumer(); 187 for (i = 0; i < n; i++) 188 temp[i] = counters[i]; 189 190 membar_consumer(); 191 leave = *gen; 192 193 if (enter == leave) 194 break; 195 196 enter = leave; 197 } 198 199 for (i = 0; i < n; i++) 200 output[i] += temp[i]; 201 202 gen = cpumem_next(&cmi, cm); 203 } while (gen != NULL); 204 205 free(temp, M_TEMP, n * sizeof(uint64_t)); 206 } 207 208 void 209 counters_zero(struct cpumem *cm, unsigned int n) 210 { 211 struct cpumem_iter cmi; 212 uint64_t *counters; 213 unsigned int i; 214 215 counters = cpumem_first(&cmi, cm); 216 do { 217 for (i = 0; i < n; i++) 218 counters[i] = 0; 219 /* zero the generation numbers too */ 220 membar_producer(); 221 counters[i] = 0; 222 223 counters = cpumem_next(&cmi, cm); 224 } while (counters != NULL); 225 } 226 227 #else /* MULTIPROCESSOR */ 228 229 /* 230 * Uniprocessor implementation of per-CPU data structures. 231 * 232 * UP percpu memory is a single memory allocation cast to/from the 233 * cpumem struct. It is not scaled up to the size of cacheline because 234 * there's no other cache to contend with. 235 */ 236 237 void 238 percpu_init(void) 239 { 240 /* nop */ 241 } 242 243 struct cpumem * 244 cpumem_get(struct pool *pp) 245 { 246 return (pool_get(pp, PR_WAITOK | PR_ZERO)); 247 } 248 249 void 250 cpumem_put(struct pool *pp, struct cpumem *cm) 251 { 252 pool_put(pp, cm); 253 } 254 255 struct cpumem * 256 cpumem_malloc(size_t sz, int type) 257 { 258 return (malloc(sz, type, M_WAITOK | M_ZERO)); 259 } 260 261 struct cpumem * 262 cpumem_malloc_ncpus(struct cpumem *cm, size_t sz, int type) 263 { 264 return (cm); 265 } 266 267 void 268 cpumem_free(struct cpumem *cm, int type, size_t sz) 269 { 270 free(cm, type, sz); 271 } 272 273 void * 274 cpumem_first(struct cpumem_iter *i, struct cpumem *cm) 275 { 276 return (cm); 277 } 278 279 void * 280 cpumem_next(struct cpumem_iter *i, struct cpumem *cm) 281 { 282 return (NULL); 283 } 284 285 struct cpumem * 286 counters_alloc(unsigned int n) 287 { 288 KASSERT(n > 0); 289 290 return (cpumem_malloc(n * sizeof(uint64_t), M_COUNTERS)); 291 } 292 293 struct cpumem * 294 counters_alloc_ncpus(struct cpumem *cm, unsigned int n) 295 { 296 /* this is unecessary, but symmetrical */ 297 return (cpumem_malloc_ncpus(cm, n * sizeof(uint64_t), M_COUNTERS)); 298 } 299 300 void 301 counters_free(struct cpumem *cm, unsigned int n) 302 { 303 cpumem_free(cm, M_COUNTERS, n * sizeof(uint64_t)); 304 } 305 306 void 307 counters_read(struct cpumem *cm, uint64_t *output, unsigned int n) 308 { 309 uint64_t *counters; 310 unsigned int i; 311 int s; 312 313 counters = (uint64_t *)cm; 314 315 s = splhigh(); 316 for (i = 0; i < n; i++) 317 output[i] = counters[i]; 318 splx(s); 319 } 320 321 void 322 counters_zero(struct cpumem *cm, unsigned int n) 323 { 324 uint64_t *counters; 325 unsigned int i; 326 int s; 327 328 counters = (uint64_t *)cm; 329 330 s = splhigh(); 331 for (i = 0; i < n; i++) 332 counters[i] = 0; 333 splx(s); 334 } 335 336 #endif /* MULTIPROCESSOR */ 337