xref: /netbsd-src/sys/kern/kern_cpu.c (revision 46f5119e40af2e51998f686b2fdcc76b5488f7f3) 
1 /*	$NetBSD: kern_cpu.c,v 1.45 2010/12/22 02:43:23 matt Exp $	*/
2 
3 /*-
4  * Copyright (c) 2007, 2008, 2009, 2010 The NetBSD Foundation, Inc.
5  * All rights reserved.
6  *
7  * This code is derived from software contributed to The NetBSD Foundation
8  * by Andrew Doran.
9  *
10  * Redistribution and use in source and binary forms, with or without
11  * modification, are permitted provided that the following conditions
12  * are met:
13  * 1. Redistributions of source code must retain the above copyright
14  *    notice, this list of conditions and the following disclaimer.
15  * 2. Redistributions in binary form must reproduce the above copyright
16  *    notice, this list of conditions and the following disclaimer in the
17  *    documentation and/or other materials provided with the distribution.
18  *
19  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29  * POSSIBILITY OF SUCH DAMAGE.
30  */
31 
32 /*-
33  * Copyright (c)2007 YAMAMOTO Takashi,
34  * All rights reserved.
35  *
36  * Redistribution and use in source and binary forms, with or without
37  * modification, are permitted provided that the following conditions
38  * are met:
39  * 1. Redistributions of source code must retain the above copyright
40  *    notice, this list of conditions and the following disclaimer.
41  * 2. Redistributions in binary form must reproduce the above copyright
42  *    notice, this list of conditions and the following disclaimer in the
43  *    documentation and/or other materials provided with the distribution.
44  *
45  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
46  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
47  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
48  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
49  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
50  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
51  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
52  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
53  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
54  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
55  * SUCH DAMAGE.
56  */
57 
58 #include <sys/cdefs.h>
59 __KERNEL_RCSID(0, "$NetBSD: kern_cpu.c,v 1.45 2010/12/22 02:43:23 matt Exp $");
60 
61 #include <sys/param.h>
62 #include <sys/systm.h>
63 #include <sys/idle.h>
64 #include <sys/sched.h>
65 #include <sys/intr.h>
66 #include <sys/conf.h>
67 #include <sys/cpu.h>
68 #include <sys/cpuio.h>
69 #include <sys/proc.h>
70 #include <sys/percpu.h>
71 #include <sys/kernel.h>
72 #include <sys/kauth.h>
73 #include <sys/xcall.h>
74 #include <sys/pool.h>
75 #include <sys/kmem.h>
76 #include <sys/select.h>
77 #include <sys/namei.h>
78 #include <sys/callout.h>
79 
80 #include <uvm/uvm_extern.h>
81 
82 /*
83  * If the port has state that cpu_data is the first thing in cpu_info,
84  * verify the claim is true.  This will prevent the from getting out
85  * of sync.
86  */
87 #ifdef __HAVE_CPU_DATA_FIRST
88 CTASSERT(offsetof(struct cpu_info, ci_data) == 0);
89 #else
90 CTASSERT(offsetof(struct cpu_info, ci_data) != 0);
91 #endif
92 
93 void	cpuctlattach(int);
94 
95 static void	cpu_xc_online(struct cpu_info *);
96 static void	cpu_xc_offline(struct cpu_info *);
97 
98 dev_type_ioctl(cpuctl_ioctl);
99 
100 const struct cdevsw cpuctl_cdevsw = {
101 	nullopen, nullclose, nullread, nullwrite, cpuctl_ioctl,
102 	nullstop, notty, nopoll, nommap, nokqfilter,
103 	D_OTHER | D_MPSAFE
104 };
105 
106 kmutex_t cpu_lock;
107 int	ncpu;
108 int	ncpuonline;
109 bool	mp_online;
110 struct	cpuqueue cpu_queue = CIRCLEQ_HEAD_INITIALIZER(cpu_queue);
111 
112 static struct cpu_info **cpu_infos;
113 
114 int
115 mi_cpu_attach(struct cpu_info *ci)
116 {
117 	int error;
118 
119 	KASSERT(maxcpus > 0);
120 
121 	ci->ci_index = ncpu;
122 	CIRCLEQ_INSERT_TAIL(&cpu_queue, ci, ci_data.cpu_qchain);
123 	TAILQ_INIT(&ci->ci_data.cpu_ld_locks);
124 	__cpu_simple_lock_init(&ci->ci_data.cpu_ld_lock);
125 
126 	/* This is useful for eg, per-cpu evcnt */
127 	snprintf(ci->ci_data.cpu_name, sizeof(ci->ci_data.cpu_name), "cpu%d",
128 	    cpu_index(ci));
129 
130 	sched_cpuattach(ci);
131 
132 	error = create_idle_lwp(ci);
133 	if (error != 0) {
134 		/* XXX revert sched_cpuattach */
135 		return error;
136 	}
137 
138 	if (ci == curcpu())
139 		ci->ci_data.cpu_onproc = curlwp;
140 	else
141 		ci->ci_data.cpu_onproc = ci->ci_data.cpu_idlelwp;
142 
143 	percpu_init_cpu(ci);
144 	softint_init(ci);
145 	callout_init_cpu(ci);
146 	xc_init_cpu(ci);
147 	pool_cache_cpu_init(ci);
148 	selsysinit(ci);
149 	cache_cpu_init(ci);
150 	TAILQ_INIT(&ci->ci_data.cpu_biodone);
151 	ncpu++;
152 	ncpuonline++;
153 
154 	if (cpu_infos == NULL) {
155 		cpu_infos =
156 		    kmem_zalloc(sizeof(cpu_infos[0]) * maxcpus, KM_SLEEP);
157 	}
158 	cpu_infos[cpu_index(ci)] = ci;
159 
160 	return 0;
161 }
162 
163 void
164 cpuctlattach(int dummy)
165 {
166 
167 	KASSERT(cpu_infos != NULL);
168 }
169 
170 int
171 cpuctl_ioctl(dev_t dev, u_long cmd, void *data, int flag, lwp_t *l)
172 {
173 	CPU_INFO_ITERATOR cii;
174 	cpustate_t *cs;
175 	struct cpu_info *ci;
176 	int error, i;
177 	u_int id;
178 
179 	error = 0;
180 
181 	mutex_enter(&cpu_lock);
182 	switch (cmd) {
183 	case IOC_CPU_SETSTATE:
184 		if (error == 0)
185 			cs = data;
186 		error = kauth_authorize_system(l->l_cred,
187 		    KAUTH_SYSTEM_CPU, KAUTH_REQ_SYSTEM_CPU_SETSTATE, cs, NULL,
188 		    NULL);
189 		if (error != 0)
190 			break;
191 		if (cs->cs_id >= maxcpus ||
192 		    (ci = cpu_lookup(cs->cs_id)) == NULL) {
193 			error = ESRCH;
194 			break;
195 		}
196 		error = cpu_setintr(ci, cs->cs_intr);
197 		error = cpu_setstate(ci, cs->cs_online);
198 		break;
199 
200 	case IOC_CPU_GETSTATE:
201 		if (error == 0)
202 			cs = data;
203 		id = cs->cs_id;
204 		memset(cs, 0, sizeof(*cs));
205 		cs->cs_id = id;
206 		if (cs->cs_id >= maxcpus ||
207 		    (ci = cpu_lookup(id)) == NULL) {
208 			error = ESRCH;
209 			break;
210 		}
211 		if ((ci->ci_schedstate.spc_flags & SPCF_OFFLINE) != 0)
212 			cs->cs_online = false;
213 		else
214 			cs->cs_online = true;
215 		if ((ci->ci_schedstate.spc_flags & SPCF_NOINTR) != 0)
216 			cs->cs_intr = false;
217 		else
218 			cs->cs_intr = true;
219 		cs->cs_lastmod = (int32_t)ci->ci_schedstate.spc_lastmod;
220 		cs->cs_lastmodhi = (int32_t)
221 		    (ci->ci_schedstate.spc_lastmod >> 32);
222 		cs->cs_intrcnt = cpu_intr_count(ci) + 1;
223 		break;
224 
225 	case IOC_CPU_MAPID:
226 		i = 0;
227 		for (CPU_INFO_FOREACH(cii, ci)) {
228 			if (i++ == *(int *)data)
229 				break;
230 		}
231 		if (ci == NULL)
232 			error = ESRCH;
233 		else
234 			*(int *)data = cpu_index(ci);
235 		break;
236 
237 	case IOC_CPU_GETCOUNT:
238 		*(int *)data = ncpu;
239 		break;
240 
241 	default:
242 		error = ENOTTY;
243 		break;
244 	}
245 	mutex_exit(&cpu_lock);
246 
247 	return error;
248 }
249 
250 struct cpu_info *
251 cpu_lookup(u_int idx)
252 {
253 	struct cpu_info *ci;
254 
255 	KASSERT(idx < maxcpus);
256 
257 	if (__predict_false(cpu_infos == NULL)) {
258 		KASSERT(idx == 0);
259 		return curcpu();
260 	}
261 
262 	ci = cpu_infos[idx];
263 	KASSERT(ci == NULL || cpu_index(ci) == idx);
264 
265 	return ci;
266 }
267 
268 static void
269 cpu_xc_offline(struct cpu_info *ci)
270 {
271 	struct schedstate_percpu *spc, *mspc = NULL;
272 	struct cpu_info *target_ci;
273 	struct lwp *l;
274 	CPU_INFO_ITERATOR cii;
275 	int s;
276 
277 	/*
278 	 * Thread that made the cross call (separate context) holds
279 	 * cpu_lock on our behalf.
280 	 */
281 	spc = &ci->ci_schedstate;
282 	s = splsched();
283 	spc->spc_flags |= SPCF_OFFLINE;
284 	splx(s);
285 
286 	/* Take the first available CPU for the migration. */
287 	for (CPU_INFO_FOREACH(cii, target_ci)) {
288 		mspc = &target_ci->ci_schedstate;
289 		if ((mspc->spc_flags & SPCF_OFFLINE) == 0)
290 			break;
291 	}
292 	KASSERT(target_ci != NULL);
293 
294 	/*
295 	 * Migrate all non-bound threads to the other CPU.  Note that this
296 	 * runs from the xcall thread, thus handling of LSONPROC is not needed.
297 	 */
298 	mutex_enter(proc_lock);
299 	LIST_FOREACH(l, &alllwp, l_list) {
300 		struct cpu_info *mci;
301 
302 		lwp_lock(l);
303 		if (l->l_cpu != ci || (l->l_pflag & (LP_BOUND | LP_INTR))) {
304 			lwp_unlock(l);
305 			continue;
306 		}
307 		/* Normal case - no affinity */
308 		if ((l->l_flag & LW_AFFINITY) == 0) {
309 			lwp_migrate(l, target_ci);
310 			continue;
311 		}
312 		/* Affinity is set, find an online CPU in the set */
313 		KASSERT(l->l_affinity != NULL);
314 		for (CPU_INFO_FOREACH(cii, mci)) {
315 			mspc = &mci->ci_schedstate;
316 			if ((mspc->spc_flags & SPCF_OFFLINE) == 0 &&
317 			    kcpuset_isset(cpu_index(mci), l->l_affinity))
318 				break;
319 		}
320 		if (mci == NULL) {
321 			lwp_unlock(l);
322 			mutex_exit(proc_lock);
323 			goto fail;
324 		}
325 		lwp_migrate(l, mci);
326 	}
327 	mutex_exit(proc_lock);
328 
329 #ifdef __HAVE_MD_CPU_OFFLINE
330 	cpu_offline_md();
331 #endif
332 	return;
333 fail:
334 	/* Just unset the SPCF_OFFLINE flag, caller will check */
335 	s = splsched();
336 	spc->spc_flags &= ~SPCF_OFFLINE;
337 	splx(s);
338 }
339 
340 static void
341 cpu_xc_online(struct cpu_info *ci)
342 {
343 	struct schedstate_percpu *spc;
344 	int s;
345 
346 	spc = &ci->ci_schedstate;
347 	s = splsched();
348 	spc->spc_flags &= ~SPCF_OFFLINE;
349 	splx(s);
350 }
351 
352 int
353 cpu_setstate(struct cpu_info *ci, bool online)
354 {
355 	struct schedstate_percpu *spc;
356 	CPU_INFO_ITERATOR cii;
357 	struct cpu_info *ci2;
358 	uint64_t where;
359 	xcfunc_t func;
360 	int nonline;
361 
362 	spc = &ci->ci_schedstate;
363 
364 	KASSERT(mutex_owned(&cpu_lock));
365 
366 	if (online) {
367 		if ((spc->spc_flags & SPCF_OFFLINE) == 0)
368 			return 0;
369 		func = (xcfunc_t)cpu_xc_online;
370 		ncpuonline++;
371 	} else {
372 		if ((spc->spc_flags & SPCF_OFFLINE) != 0)
373 			return 0;
374 		nonline = 0;
375 		/*
376 		 * Ensure that at least one CPU within the processor set
377 		 * stays online.  Revisit this later.
378 		 */
379 		for (CPU_INFO_FOREACH(cii, ci2)) {
380 			if ((ci2->ci_schedstate.spc_flags & SPCF_OFFLINE) != 0)
381 				continue;
382 			if (ci2->ci_schedstate.spc_psid != spc->spc_psid)
383 				continue;
384 			nonline++;
385 		}
386 		if (nonline == 1)
387 			return EBUSY;
388 		func = (xcfunc_t)cpu_xc_offline;
389 		ncpuonline--;
390 	}
391 
392 	where = xc_unicast(0, func, ci, NULL, ci);
393 	xc_wait(where);
394 	if (online) {
395 		KASSERT((spc->spc_flags & SPCF_OFFLINE) == 0);
396 	} else if ((spc->spc_flags & SPCF_OFFLINE) == 0) {
397 		/* If was not set offline, then it is busy */
398 		return EBUSY;
399 	}
400 
401 	spc->spc_lastmod = time_second;
402 	return 0;
403 }
404 
405 #ifdef __HAVE_INTR_CONTROL
406 static void
407 cpu_xc_intr(struct cpu_info *ci)
408 {
409 	struct schedstate_percpu *spc;
410 	int s;
411 
412 	spc = &ci->ci_schedstate;
413 	s = splsched();
414 	spc->spc_flags &= ~SPCF_NOINTR;
415 	splx(s);
416 }
417 
418 static void
419 cpu_xc_nointr(struct cpu_info *ci)
420 {
421 	struct schedstate_percpu *spc;
422 	int s;
423 
424 	spc = &ci->ci_schedstate;
425 	s = splsched();
426 	spc->spc_flags |= SPCF_NOINTR;
427 	splx(s);
428 }
429 
430 int
431 cpu_setintr(struct cpu_info *ci, bool intr)
432 {
433 	struct schedstate_percpu *spc;
434 	CPU_INFO_ITERATOR cii;
435 	struct cpu_info *ci2;
436 	uint64_t where;
437 	xcfunc_t func;
438 	int nintr;
439 
440 	spc = &ci->ci_schedstate;
441 
442 	KASSERT(mutex_owned(&cpu_lock));
443 
444 	if (intr) {
445 		if ((spc->spc_flags & SPCF_NOINTR) == 0)
446 			return 0;
447 		func = (xcfunc_t)cpu_xc_intr;
448 	} else {
449 		if ((spc->spc_flags & SPCF_NOINTR) != 0)
450 			return 0;
451 		/*
452 		 * Ensure that at least one CPU within the system
453 		 * is handing device interrupts.
454 		 */
455 		nintr = 0;
456 		for (CPU_INFO_FOREACH(cii, ci2)) {
457 			if ((ci2->ci_schedstate.spc_flags & SPCF_NOINTR) != 0)
458 				continue;
459 			if (ci2 == ci)
460 				continue;
461 			nintr++;
462 		}
463 		if (nintr == 0)
464 			return EBUSY;
465 		func = (xcfunc_t)cpu_xc_nointr;
466 	}
467 
468 	where = xc_unicast(0, func, ci, NULL, ci);
469 	xc_wait(where);
470 	if (intr) {
471 		KASSERT((spc->spc_flags & SPCF_NOINTR) == 0);
472 	} else if ((spc->spc_flags & SPCF_NOINTR) == 0) {
473 		/* If was not set offline, then it is busy */
474 		return EBUSY;
475 	}
476 
477 	/* Direct interrupts away from the CPU and record the change. */
478 	cpu_intr_redistribute();
479 	spc->spc_lastmod = time_second;
480 	return 0;
481 }
482 #else	/* __HAVE_INTR_CONTROL */
483 int
484 cpu_setintr(struct cpu_info *ci, bool intr)
485 {
486 
487 	return EOPNOTSUPP;
488 }
489 
490 u_int
491 cpu_intr_count(struct cpu_info *ci)
492 {
493 
494 	return 0;	/* 0 == "don't know" */
495 }
496 #endif	/* __HAVE_INTR_CONTROL */
497 
498 bool
499 cpu_softintr_p(void)
500 {
501 
502 	return (curlwp->l_pflag & LP_INTR) != 0;
503 }
504