1 /* $NetBSD: subr_percpu.c,v 1.11 2011/04/14 05:53:53 matt Exp $ */ 2 3 /*- 4 * Copyright (c)2007,2008 YAMAMOTO Takashi, 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 /* 30 * per-cpu storage. 31 */ 32 33 #include <sys/cdefs.h> 34 __KERNEL_RCSID(0, "$NetBSD: subr_percpu.c,v 1.11 2011/04/14 05:53:53 matt Exp $"); 35 36 #include <sys/param.h> 37 #include <sys/cpu.h> 38 #include <sys/kmem.h> 39 #include <sys/kernel.h> 40 #include <sys/mutex.h> 41 #include <sys/percpu.h> 42 #include <sys/rwlock.h> 43 #include <sys/vmem.h> 44 #include <sys/xcall.h> 45 46 #include <uvm/uvm_extern.h> 47 48 #define PERCPU_QUANTUM_SIZE (ALIGNBYTES + 1) 49 #define PERCPU_QCACHE_MAX 0 50 #define PERCPU_IMPORT_SIZE 2048 51 52 #if defined(DIAGNOSTIC) 53 #define MAGIC 0x50435055 /* "PCPU" */ 54 #define percpu_encrypt(pc) ((pc) ^ MAGIC) 55 #define percpu_decrypt(pc) ((pc) ^ MAGIC) 56 #else /* defined(DIAGNOSTIC) */ 57 #define percpu_encrypt(pc) (pc) 58 #define percpu_decrypt(pc) (pc) 59 #endif /* defined(DIAGNOSTIC) */ 60 61 static krwlock_t percpu_swap_lock; 62 static kmutex_t percpu_allocation_lock; 63 static vmem_t *percpu_offset_arena; 64 static unsigned int percpu_nextoff = PERCPU_QUANTUM_SIZE; 65 66 static percpu_cpu_t * 67 cpu_percpu(struct cpu_info *ci) 68 { 69 70 return &ci->ci_data.cpu_percpu; 71 } 72 73 static unsigned int 74 percpu_offset(percpu_t *pc) 75 { 76 const unsigned int off = percpu_decrypt((uintptr_t)pc); 77 78 KASSERT(off < percpu_nextoff); 79 return off; 80 } 81 82 /* 83 * percpu_cpu_swap: crosscall handler for percpu_cpu_enlarge 84 */ 85 86 static void 87 percpu_cpu_swap(void *p1, void *p2) 88 { 89 struct cpu_info * const ci = p1; 90 percpu_cpu_t * const newpcc = p2; 91 percpu_cpu_t * const pcc = cpu_percpu(ci); 92 93 KASSERT(ci == curcpu()); 94 95 /* 96 * swap *pcc and *newpcc unless anyone has beaten us. 97 */ 98 rw_enter(&percpu_swap_lock, RW_WRITER); 99 if (newpcc->pcc_size > pcc->pcc_size) { 100 percpu_cpu_t tmp; 101 int s; 102 103 tmp = *pcc; 104 105 /* 106 * block interrupts so that we don't lose their modifications. 107 */ 108 109 s = splhigh(); 110 111 /* 112 * copy data to new storage. 113 */ 114 115 memcpy(newpcc->pcc_data, pcc->pcc_data, pcc->pcc_size); 116 117 /* 118 * this assignment needs to be atomic for percpu_getptr_remote. 119 */ 120 121 pcc->pcc_data = newpcc->pcc_data; 122 123 splx(s); 124 125 pcc->pcc_size = newpcc->pcc_size; 126 *newpcc = tmp; 127 } 128 rw_exit(&percpu_swap_lock); 129 } 130 131 /* 132 * percpu_cpu_enlarge: ensure that percpu_cpu_t of each cpus have enough space 133 */ 134 135 static void 136 percpu_cpu_enlarge(size_t size) 137 { 138 CPU_INFO_ITERATOR cii; 139 struct cpu_info *ci; 140 141 for (CPU_INFO_FOREACH(cii, ci)) { 142 percpu_cpu_t pcc; 143 144 pcc.pcc_data = kmem_alloc(size, KM_SLEEP); /* XXX cacheline */ 145 pcc.pcc_size = size; 146 if (!mp_online) { 147 percpu_cpu_swap(ci, &pcc); 148 } else { 149 uint64_t where; 150 151 where = xc_unicast(0, percpu_cpu_swap, ci, &pcc, ci); 152 xc_wait(where); 153 } 154 KASSERT(pcc.pcc_size < size); 155 if (pcc.pcc_data != NULL) { 156 kmem_free(pcc.pcc_data, pcc.pcc_size); 157 } 158 } 159 } 160 161 /* 162 * percpu_backend_alloc: vmem import callback for percpu_offset_arena 163 */ 164 165 static vmem_addr_t 166 percpu_backend_alloc(vmem_t *dummy, vmem_size_t size, vmem_size_t *resultsize, 167 vm_flag_t vmflags) 168 { 169 unsigned int offset; 170 unsigned int nextoff; 171 172 ASSERT_SLEEPABLE(); 173 KASSERT(dummy == NULL); 174 175 if ((vmflags & VM_NOSLEEP) != 0) 176 return VMEM_ADDR_NULL; 177 178 size = roundup(size, PERCPU_IMPORT_SIZE); 179 mutex_enter(&percpu_allocation_lock); 180 offset = percpu_nextoff; 181 percpu_nextoff = nextoff = percpu_nextoff + size; 182 mutex_exit(&percpu_allocation_lock); 183 184 percpu_cpu_enlarge(nextoff); 185 186 *resultsize = size; 187 return (vmem_addr_t)offset; 188 } 189 190 static void 191 percpu_zero_cb(void *vp, void *vp2, struct cpu_info *ci) 192 { 193 size_t sz = (uintptr_t)vp2; 194 195 memset(vp, 0, sz); 196 } 197 198 /* 199 * percpu_zero: initialize percpu storage with zero. 200 */ 201 202 static void 203 percpu_zero(percpu_t *pc, size_t sz) 204 { 205 206 percpu_foreach(pc, percpu_zero_cb, (void *)(uintptr_t)sz); 207 } 208 209 /* 210 * percpu_init: subsystem initialization 211 */ 212 213 void 214 percpu_init(void) 215 { 216 217 ASSERT_SLEEPABLE(); 218 rw_init(&percpu_swap_lock); 219 mutex_init(&percpu_allocation_lock, MUTEX_DEFAULT, IPL_NONE); 220 221 percpu_offset_arena = vmem_create("percpu", 0, 0, PERCPU_QUANTUM_SIZE, 222 percpu_backend_alloc, NULL, NULL, PERCPU_QCACHE_MAX, VM_SLEEP, 223 IPL_NONE); 224 } 225 226 /* 227 * percpu_init_cpu: cpu initialization 228 * 229 * => should be called before the cpu appears on the list for CPU_INFO_FOREACH. 230 */ 231 232 void 233 percpu_init_cpu(struct cpu_info *ci) 234 { 235 percpu_cpu_t * const pcc = cpu_percpu(ci); 236 size_t size = percpu_nextoff; /* XXX racy */ 237 238 ASSERT_SLEEPABLE(); 239 pcc->pcc_size = size; 240 if (size) { 241 pcc->pcc_data = kmem_zalloc(pcc->pcc_size, KM_SLEEP); 242 } 243 } 244 245 /* 246 * percpu_alloc: allocate percpu storage 247 * 248 * => called in thread context. 249 * => considered as an expensive and rare operation. 250 * => allocated storage is initialized with zeros. 251 */ 252 253 percpu_t * 254 percpu_alloc(size_t size) 255 { 256 unsigned int offset; 257 percpu_t *pc; 258 259 ASSERT_SLEEPABLE(); 260 offset = vmem_alloc(percpu_offset_arena, size, VM_SLEEP | VM_BESTFIT); 261 pc = (percpu_t *)percpu_encrypt((uintptr_t)offset); 262 percpu_zero(pc, size); 263 return pc; 264 } 265 266 /* 267 * percpu_free: free percpu storage 268 * 269 * => called in thread context. 270 * => considered as an expensive and rare operation. 271 */ 272 273 void 274 percpu_free(percpu_t *pc, size_t size) 275 { 276 277 ASSERT_SLEEPABLE(); 278 vmem_free(percpu_offset_arena, (vmem_addr_t)percpu_offset(pc), size); 279 } 280 281 /* 282 * percpu_getref: 283 * 284 * => safe to be used in either thread or interrupt context 285 * => disables preemption; must be bracketed with a percpu_putref() 286 */ 287 288 void * 289 percpu_getref(percpu_t *pc) 290 { 291 292 KPREEMPT_DISABLE(curlwp); 293 return percpu_getptr_remote(pc, curcpu()); 294 } 295 296 /* 297 * percpu_putref: 298 * 299 * => drops the preemption-disabled count after caller is done with per-cpu 300 * data 301 */ 302 303 void 304 percpu_putref(percpu_t *pc) 305 { 306 307 KPREEMPT_ENABLE(curlwp); 308 } 309 310 /* 311 * percpu_traverse_enter, percpu_traverse_exit, percpu_getptr_remote: 312 * helpers to access remote cpu's percpu data. 313 * 314 * => called in thread context. 315 * => percpu_traverse_enter can block low-priority xcalls. 316 * => typical usage would be: 317 * 318 * sum = 0; 319 * percpu_traverse_enter(); 320 * for (CPU_INFO_FOREACH(cii, ci)) { 321 * unsigned int *p = percpu_getptr_remote(pc, ci); 322 * sum += *p; 323 * } 324 * percpu_traverse_exit(); 325 */ 326 327 void 328 percpu_traverse_enter(void) 329 { 330 331 ASSERT_SLEEPABLE(); 332 rw_enter(&percpu_swap_lock, RW_READER); 333 } 334 335 void 336 percpu_traverse_exit(void) 337 { 338 339 rw_exit(&percpu_swap_lock); 340 } 341 342 void * 343 percpu_getptr_remote(percpu_t *pc, struct cpu_info *ci) 344 { 345 346 return &((char *)cpu_percpu(ci)->pcc_data)[percpu_offset(pc)]; 347 } 348 349 /* 350 * percpu_foreach: call the specified callback function for each cpus. 351 * 352 * => called in thread context. 353 * => caller should not rely on the cpu iteration order. 354 * => the callback function should be minimum because it is executed with 355 * holding a global lock, which can block low-priority xcalls. 356 * eg. it's illegal for a callback function to sleep for memory allocation. 357 */ 358 void 359 percpu_foreach(percpu_t *pc, percpu_callback_t cb, void *arg) 360 { 361 CPU_INFO_ITERATOR cii; 362 struct cpu_info *ci; 363 364 percpu_traverse_enter(); 365 for (CPU_INFO_FOREACH(cii, ci)) { 366 (*cb)(percpu_getptr_remote(pc, ci), arg, ci); 367 } 368 percpu_traverse_exit(); 369 } 370