1 /* $OpenBSD: subr_percpu.c,v 1.1 2016/10/21 06:27:50 dlg Exp $ */ 2 3 /* 4 * Copyright (c) 2016 David Gwynne <dlg@openbsd.org> 5 * 6 * Permission to use, copy, modify, and distribute this software for any 7 * purpose with or without fee is hereby granted, provided that the above 8 * copyright notice and this permission notice appear in all copies. 9 * 10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 17 */ 18 19 #include <sys/param.h> 20 #include <sys/systm.h> 21 #include <sys/pool.h> 22 #include <sys/malloc.h> 23 #include <sys/types.h> 24 25 #include <sys/percpu.h> 26 27 #ifdef MULTIPROCESSOR 28 struct pool cpumem_pl; 29 30 void 31 percpu_init(void) 32 { 33 pool_init(&cpumem_pl, sizeof(struct cpumem) * ncpus, 0, IPL_NONE, 34 PR_WAITOK, "percpumem", &pool_allocator_single); 35 } 36 37 struct cpumem * 38 cpumem_get(struct pool *pp) 39 { 40 struct cpumem *cm; 41 unsigned int cpu; 42 43 cm = pool_get(&cpumem_pl, PR_WAITOK); 44 45 for (cpu = 0; cpu < ncpus; cpu++) 46 cm[cpu].mem = pool_get(pp, PR_WAITOK | PR_ZERO); 47 48 return (cm); 49 } 50 51 void 52 cpumem_put(struct pool *pp, struct cpumem *cm) 53 { 54 unsigned int cpu; 55 56 for (cpu = 0; cpu < ncpus; cpu++) 57 pool_put(pp, cm[cpu].mem); 58 59 pool_put(&cpumem_pl, cm); 60 } 61 62 struct cpumem * 63 cpumem_malloc(size_t sz, int type) 64 { 65 struct cpumem *cm; 66 unsigned int cpu; 67 68 sz = roundup(sz, CACHELINESIZE); 69 70 cm = pool_get(&cpumem_pl, PR_WAITOK); 71 72 for (cpu = 0; cpu < ncpus; cpu++) 73 cm[cpu].mem = malloc(sz, type, M_WAITOK | M_ZERO); 74 75 return (cm); 76 } 77 78 struct cpumem * 79 cpumem_realloc(struct cpumem *bootcm, size_t sz, int type) 80 { 81 struct cpumem *cm; 82 unsigned int cpu; 83 84 sz = roundup(sz, CACHELINESIZE); 85 86 cm = pool_get(&cpumem_pl, PR_WAITOK); 87 88 cm[0].mem = bootcm[0].mem; 89 for (cpu = 1; cpu < ncpus; cpu++) 90 cm[cpu].mem = malloc(sz, type, M_WAITOK | M_ZERO); 91 92 return (cm); 93 } 94 95 void 96 cpumem_free(struct cpumem *cm, int type, size_t sz) 97 { 98 unsigned int cpu; 99 100 sz = roundup(sz, CACHELINESIZE); 101 102 for (cpu = 0; cpu < ncpus; cpu++) 103 free(cm[cpu].mem, type, sz); 104 105 pool_put(&cpumem_pl, cm); 106 } 107 108 void * 109 cpumem_first(struct cpumem_iter *i, struct cpumem *cm) 110 { 111 i->cpu = 0; 112 113 return (cm[0].mem); 114 } 115 116 void * 117 cpumem_next(struct cpumem_iter *i, struct cpumem *cm) 118 { 119 unsigned int cpu = ++i->cpu; 120 121 if (cpu >= ncpus) 122 return (NULL); 123 124 return (cm[cpu].mem); 125 } 126 127 struct cpumem * 128 counters_alloc(unsigned int n, int type) 129 { 130 struct cpumem *cm; 131 struct cpumem_iter cmi; 132 uint64_t *counters; 133 unsigned int i; 134 135 KASSERT(n > 0); 136 137 n++; /* add space for a generation number */ 138 cm = cpumem_malloc(n * sizeof(uint64_t), type); 139 140 CPUMEM_FOREACH(counters, &cmi, cm) { 141 for (i = 0; i < n; i++) 142 counters[i] = 0; 143 } 144 145 return (cm); 146 } 147 148 struct cpumem * 149 counters_realloc(struct cpumem *cm, unsigned int n, int type) 150 { 151 n++; /* the generation number */ 152 return (cpumem_realloc(cm, n * sizeof(uint64_t), type)); 153 } 154 155 void 156 counters_free(struct cpumem *cm, int type, unsigned int n) 157 { 158 n++; /* generation number */ 159 cpumem_free(cm, type, n * sizeof(uint64_t)); 160 } 161 162 void 163 counters_read(struct cpumem *cm, uint64_t *output, unsigned int n) 164 { 165 struct cpumem_iter cmi; 166 uint64_t *gen, *counters, *temp; 167 uint64_t enter, leave; 168 unsigned int i; 169 170 for (i = 0; i < n; i++) 171 output[i] = 0; 172 173 temp = mallocarray(n, sizeof(uint64_t), M_TEMP, M_WAITOK); 174 175 gen = cpumem_first(&cmi, cm); 176 do { 177 counters = gen + 1; 178 179 enter = *gen; 180 for (;;) { 181 /* the generation number is odd during an update */ 182 while (enter & 1) { 183 yield(); 184 membar_consumer(); 185 enter = *gen; 186 } 187 188 for (i = 0; i < n; i++) 189 temp[i] = counters[i]; 190 191 membar_consumer(); 192 leave = *gen; 193 194 if (enter == leave) 195 break; 196 197 enter = leave; 198 } 199 200 for (i = 0; i < n; i++) 201 output[i] += temp[i]; 202 203 gen = cpumem_next(&cmi, cm); 204 } while (gen != NULL); 205 206 free(temp, M_TEMP, n * sizeof(uint64_t)); 207 } 208 209 void 210 counters_zero(struct cpumem *cm, unsigned int n) 211 { 212 struct cpumem_iter cmi; 213 uint64_t *counters; 214 unsigned int i; 215 216 n++; /* zero the generation numbers too */ 217 218 counters = cpumem_first(&cmi, cm); 219 do { 220 for (i = 0; i < n; i++) 221 counters[i] = 0; 222 223 counters = cpumem_next(&cmi, cm); 224 } while (counters != NULL); 225 } 226 227 #else /* MULTIPROCESSOR */ 228 229 /* 230 * Uniprocessor implementation of per-CPU data structures. 231 * 232 * UP percpu memory is a single memory allocation cast to/from the 233 * cpumem struct. It is not scaled up to the size of cacheline because 234 * there's no other cache to contend with. 235 */ 236 237 void 238 percpu_init(void) 239 { 240 /* nop */ 241 } 242 243 struct cpumem * 244 cpumem_get(struct pool *pp) 245 { 246 return (pool_get(pp, PR_WAITOK)); 247 } 248 249 void 250 cpumem_put(struct pool *pp, struct cpumem *cm) 251 { 252 pool_put(pp, cm); 253 } 254 255 struct cpumem * 256 cpumem_malloc(size_t sz, int type) 257 { 258 return (malloc(sz, type, M_WAITOK)); 259 } 260 261 struct cpumem * 262 cpumem_realloc(struct cpumem *cm, size_t sz, int type) 263 { 264 return (cm); 265 } 266 267 void 268 cpumem_free(struct cpumem *cm, int type, size_t sz) 269 { 270 free(cm, type, sz); 271 } 272 273 struct cpumem * 274 counters_alloc(unsigned int n, int type) 275 { 276 KASSERT(n > 0); 277 278 return (cpumem_malloc(n * sizeof(uint64_t), type)); 279 } 280 281 struct cpumem * 282 counters_realloc(struct cpumem *cm, unsigned int n, int type) 283 { 284 /* this is unecessary, but symmetrical */ 285 return (cpumem_realloc(cm, n * sizeof(uint64_t), type)); 286 } 287 288 void 289 counters_free(struct cpumem *cm, int type, unsigned int n) 290 { 291 cpumem_free(cm, type, n * sizeof(uint64_t)); 292 } 293 294 void 295 counters_read(struct cpumem *cm, uint64_t *output, unsigned int n) 296 { 297 uint64_t *counters; 298 unsigned int i; 299 int s; 300 301 counters = (uint64_t *)cm; 302 303 s = splhigh(); 304 for (i = 0; i < n; i++) 305 output[i] = counters[i]; 306 splx(s); 307 } 308 309 void 310 counters_zero(struct cpumem *cm, unsigned int n) 311 { 312 uint64_t *counters; 313 unsigned int i; 314 int s; 315 316 counters = (uint64_t *)cm; 317 318 s = splhigh(); 319 for (i = 0; i < n; i++) 320 counters[i] = 0; 321 splx(s); 322 } 323 324 #endif /* MULTIPROCESSOR */ 325 326