xref: /netbsd-src/sys/kern/subr_percpu.c (revision b1c86f5f087524e68db12794ee9c3e3da1ab17a0)
1 /*	$NetBSD: subr_percpu.c,v 1.10 2009/10/21 21:12:06 rmind Exp $	*/
2 
3 /*-
4  * Copyright (c)2007,2008 YAMAMOTO Takashi,
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice, this list of conditions and the following disclaimer.
12  * 2. Redistributions in binary form must reproduce the above copyright
13  *    notice, this list of conditions and the following disclaimer in the
14  *    documentation and/or other materials provided with the distribution.
15  *
16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26  * SUCH DAMAGE.
27  */
28 
29 /*
30  * per-cpu storage.
31  */
32 
33 #include <sys/cdefs.h>
34 __KERNEL_RCSID(0, "$NetBSD: subr_percpu.c,v 1.10 2009/10/21 21:12:06 rmind Exp $");
35 
36 #include <sys/param.h>
37 #include <sys/cpu.h>
38 #include <sys/kmem.h>
39 #include <sys/kernel.h>
40 #include <sys/mutex.h>
41 #include <sys/percpu.h>
42 #include <sys/rwlock.h>
43 #include <sys/vmem.h>
44 #include <sys/xcall.h>
45 
46 #include <uvm/uvm_extern.h>
47 
48 #define	PERCPU_QUANTUM_SIZE	(ALIGNBYTES + 1)
49 #define	PERCPU_QCACHE_MAX	0
50 #define	PERCPU_IMPORT_SIZE	2048
51 
52 #if defined(DIAGNOSTIC)
53 #define	MAGIC	0x50435055	/* "PCPU" */
54 #define	percpu_encrypt(pc)	((pc) ^ MAGIC)
55 #define	percpu_decrypt(pc)	((pc) ^ MAGIC)
56 #else /* defined(DIAGNOSTIC) */
57 #define	percpu_encrypt(pc)	(pc)
58 #define	percpu_decrypt(pc)	(pc)
59 #endif /* defined(DIAGNOSTIC) */
60 
61 static krwlock_t percpu_swap_lock;
62 static kmutex_t percpu_allocation_lock;
63 static vmem_t *percpu_offset_arena;
64 static unsigned int percpu_nextoff = PERCPU_QUANTUM_SIZE;
65 
66 static percpu_cpu_t *
67 cpu_percpu(struct cpu_info *ci)
68 {
69 
70 	return &ci->ci_data.cpu_percpu;
71 }
72 
73 static unsigned int
74 percpu_offset(percpu_t *pc)
75 {
76 	const unsigned int off = percpu_decrypt((uintptr_t)pc);
77 
78 	KASSERT(off < percpu_nextoff);
79 	return off;
80 }
81 
82 /*
83  * percpu_cpu_swap: crosscall handler for percpu_cpu_enlarge
84  */
85 
86 static void
87 percpu_cpu_swap(void *p1, void *p2)
88 {
89 	struct cpu_info * const ci = p1;
90 	percpu_cpu_t * const newpcc = p2;
91 	percpu_cpu_t * const pcc = cpu_percpu(ci);
92 
93 	/*
94 	 * swap *pcc and *newpcc unless anyone has beaten us.
95 	 */
96 
97 	rw_enter(&percpu_swap_lock, RW_WRITER);
98 	if (newpcc->pcc_size > pcc->pcc_size) {
99 		percpu_cpu_t tmp;
100 		int s;
101 
102 		tmp = *pcc;
103 
104 		/*
105 		 * block interrupts so that we don't lose their modifications.
106 		 */
107 
108 		s = splhigh();
109 
110 		/*
111 		 * copy data to new storage.
112 		 */
113 
114 		memcpy(newpcc->pcc_data, pcc->pcc_data, pcc->pcc_size);
115 
116 		/*
117 		 * this assignment needs to be atomic for percpu_getptr_remote.
118 		 */
119 
120 		pcc->pcc_data = newpcc->pcc_data;
121 
122 		splx(s);
123 
124 		pcc->pcc_size = newpcc->pcc_size;
125 		*newpcc = tmp;
126 	}
127 	rw_exit(&percpu_swap_lock);
128 }
129 
130 /*
131  * percpu_cpu_enlarge: ensure that percpu_cpu_t of each cpus have enough space
132  */
133 
134 static void
135 percpu_cpu_enlarge(size_t size)
136 {
137 	CPU_INFO_ITERATOR cii;
138 	struct cpu_info *ci;
139 
140 	for (CPU_INFO_FOREACH(cii, ci)) {
141 		percpu_cpu_t pcc;
142 
143 		pcc.pcc_data = kmem_alloc(size, KM_SLEEP); /* XXX cacheline */
144 		pcc.pcc_size = size;
145 		if (!mp_online) {
146 			percpu_cpu_swap(ci, &pcc);
147 		} else {
148 			uint64_t where;
149 
150 			where = xc_unicast(0, percpu_cpu_swap, ci, &pcc, ci);
151 			xc_wait(where);
152 		}
153 		KASSERT(pcc.pcc_size < size);
154 		if (pcc.pcc_data != NULL) {
155 			kmem_free(pcc.pcc_data, pcc.pcc_size);
156 		}
157 	}
158 }
159 
160 /*
161  * percpu_backend_alloc: vmem import callback for percpu_offset_arena
162  */
163 
164 static vmem_addr_t
165 percpu_backend_alloc(vmem_t *dummy, vmem_size_t size, vmem_size_t *resultsize,
166     vm_flag_t vmflags)
167 {
168 	unsigned int offset;
169 	unsigned int nextoff;
170 
171 	ASSERT_SLEEPABLE();
172 	KASSERT(dummy == NULL);
173 
174 	if ((vmflags & VM_NOSLEEP) != 0)
175 		return VMEM_ADDR_NULL;
176 
177 	size = roundup(size, PERCPU_IMPORT_SIZE);
178 	mutex_enter(&percpu_allocation_lock);
179 	offset = percpu_nextoff;
180 	percpu_nextoff = nextoff = percpu_nextoff + size;
181 	mutex_exit(&percpu_allocation_lock);
182 
183 	percpu_cpu_enlarge(nextoff);
184 
185 	*resultsize = size;
186 	return (vmem_addr_t)offset;
187 }
188 
189 static void
190 percpu_zero_cb(void *vp, void *vp2, struct cpu_info *ci)
191 {
192 	size_t sz = (uintptr_t)vp2;
193 
194 	memset(vp, 0, sz);
195 }
196 
197 /*
198  * percpu_zero: initialize percpu storage with zero.
199  */
200 
201 static void
202 percpu_zero(percpu_t *pc, size_t sz)
203 {
204 
205 	percpu_foreach(pc, percpu_zero_cb, (void *)(uintptr_t)sz);
206 }
207 
208 /*
209  * percpu_init: subsystem initialization
210  */
211 
212 void
213 percpu_init(void)
214 {
215 
216 	ASSERT_SLEEPABLE();
217 	rw_init(&percpu_swap_lock);
218 	mutex_init(&percpu_allocation_lock, MUTEX_DEFAULT, IPL_NONE);
219 
220 	percpu_offset_arena = vmem_create("percpu", 0, 0, PERCPU_QUANTUM_SIZE,
221 	    percpu_backend_alloc, NULL, NULL, PERCPU_QCACHE_MAX, VM_SLEEP,
222 	    IPL_NONE);
223 }
224 
225 /*
226  * percpu_init_cpu: cpu initialization
227  *
228  * => should be called before the cpu appears on the list for CPU_INFO_FOREACH.
229  */
230 
231 void
232 percpu_init_cpu(struct cpu_info *ci)
233 {
234 	percpu_cpu_t * const pcc = cpu_percpu(ci);
235 	size_t size = percpu_nextoff; /* XXX racy */
236 
237 	ASSERT_SLEEPABLE();
238 	pcc->pcc_size = size;
239 	if (size) {
240 		pcc->pcc_data = kmem_zalloc(pcc->pcc_size, KM_SLEEP);
241 	}
242 }
243 
244 /*
245  * percpu_alloc: allocate percpu storage
246  *
247  * => called in thread context.
248  * => considered as an expensive and rare operation.
249  * => allocated storage is initialized with zeros.
250  */
251 
252 percpu_t *
253 percpu_alloc(size_t size)
254 {
255 	unsigned int offset;
256 	percpu_t *pc;
257 
258 	ASSERT_SLEEPABLE();
259 	offset = vmem_alloc(percpu_offset_arena, size, VM_SLEEP | VM_BESTFIT);
260 	pc = (percpu_t *)percpu_encrypt((uintptr_t)offset);
261 	percpu_zero(pc, size);
262 	return pc;
263 }
264 
265 /*
266  * percpu_free: free percpu storage
267  *
268  * => called in thread context.
269  * => considered as an expensive and rare operation.
270  */
271 
272 void
273 percpu_free(percpu_t *pc, size_t size)
274 {
275 
276 	ASSERT_SLEEPABLE();
277 	vmem_free(percpu_offset_arena, (vmem_addr_t)percpu_offset(pc), size);
278 }
279 
280 /*
281  * percpu_getref:
282  *
283  * => safe to be used in either thread or interrupt context
284  * => disables preemption; must be bracketed with a percpu_putref()
285  */
286 
287 void *
288 percpu_getref(percpu_t *pc)
289 {
290 
291 	KPREEMPT_DISABLE(curlwp);
292 	return percpu_getptr_remote(pc, curcpu());
293 }
294 
295 /*
296  * percpu_putref:
297  *
298  * => drops the preemption-disabled count after caller is done with per-cpu
299  *    data
300  */
301 
302 void
303 percpu_putref(percpu_t *pc)
304 {
305 
306 	KPREEMPT_ENABLE(curlwp);
307 }
308 
309 /*
310  * percpu_traverse_enter, percpu_traverse_exit, percpu_getptr_remote:
311  * helpers to access remote cpu's percpu data.
312  *
313  * => called in thread context.
314  * => percpu_traverse_enter can block low-priority xcalls.
315  * => typical usage would be:
316  *
317  *	sum = 0;
318  *	percpu_traverse_enter();
319  *	for (CPU_INFO_FOREACH(cii, ci)) {
320  *		unsigned int *p = percpu_getptr_remote(pc, ci);
321  *		sum += *p;
322  *	}
323  *	percpu_traverse_exit();
324  */
325 
326 void
327 percpu_traverse_enter(void)
328 {
329 
330 	ASSERT_SLEEPABLE();
331 	rw_enter(&percpu_swap_lock, RW_READER);
332 }
333 
334 void
335 percpu_traverse_exit(void)
336 {
337 
338 	rw_exit(&percpu_swap_lock);
339 }
340 
341 void *
342 percpu_getptr_remote(percpu_t *pc, struct cpu_info *ci)
343 {
344 
345 	return &((char *)cpu_percpu(ci)->pcc_data)[percpu_offset(pc)];
346 }
347 
348 /*
349  * percpu_foreach: call the specified callback function for each cpus.
350  *
351  * => called in thread context.
352  * => caller should not rely on the cpu iteration order.
353  * => the callback function should be minimum because it is executed with
354  *    holding a global lock, which can block low-priority xcalls.
355  *    eg. it's illegal for a callback function to sleep for memory allocation.
356  */
357 void
358 percpu_foreach(percpu_t *pc, percpu_callback_t cb, void *arg)
359 {
360 	CPU_INFO_ITERATOR cii;
361 	struct cpu_info *ci;
362 
363 	percpu_traverse_enter();
364 	for (CPU_INFO_FOREACH(cii, ci)) {
365 		(*cb)(percpu_getptr_remote(pc, ci), arg, ci);
366 	}
367 	percpu_traverse_exit();
368 }
369