xref: /netbsd-src/sys/kern/kern_cpu.c (revision 62a8debe1dc62962e18a1c918def78666141273b) 
1 /*	$NetBSD: kern_cpu.c,v 1.43 2010/01/13 01:57:17 mrg Exp $	*/
2 
3 /*-
4  * Copyright (c) 2007, 2008, 2009 The NetBSD Foundation, Inc.
5  * All rights reserved.
6  *
7  * This code is derived from software contributed to The NetBSD Foundation
8  * by Andrew Doran.
9  *
10  * Redistribution and use in source and binary forms, with or without
11  * modification, are permitted provided that the following conditions
12  * are met:
13  * 1. Redistributions of source code must retain the above copyright
14  *    notice, this list of conditions and the following disclaimer.
15  * 2. Redistributions in binary form must reproduce the above copyright
16  *    notice, this list of conditions and the following disclaimer in the
17  *    documentation and/or other materials provided with the distribution.
18  *
19  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29  * POSSIBILITY OF SUCH DAMAGE.
30  */
31 
32 /*-
33  * Copyright (c)2007 YAMAMOTO Takashi,
34  * All rights reserved.
35  *
36  * Redistribution and use in source and binary forms, with or without
37  * modification, are permitted provided that the following conditions
38  * are met:
39  * 1. Redistributions of source code must retain the above copyright
40  *    notice, this list of conditions and the following disclaimer.
41  * 2. Redistributions in binary form must reproduce the above copyright
42  *    notice, this list of conditions and the following disclaimer in the
43  *    documentation and/or other materials provided with the distribution.
44  *
45  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
46  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
47  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
48  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
49  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
50  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
51  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
52  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
53  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
54  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
55  * SUCH DAMAGE.
56  */
57 
58 #include <sys/cdefs.h>
59 __KERNEL_RCSID(0, "$NetBSD: kern_cpu.c,v 1.43 2010/01/13 01:57:17 mrg Exp $");
60 
61 #include <sys/param.h>
62 #include <sys/systm.h>
63 #include <sys/idle.h>
64 #include <sys/sched.h>
65 #include <sys/intr.h>
66 #include <sys/conf.h>
67 #include <sys/cpu.h>
68 #include <sys/cpuio.h>
69 #include <sys/proc.h>
70 #include <sys/percpu.h>
71 #include <sys/kernel.h>
72 #include <sys/kauth.h>
73 #include <sys/xcall.h>
74 #include <sys/pool.h>
75 #include <sys/kmem.h>
76 #include <sys/select.h>
77 #include <sys/namei.h>
78 #include <sys/callout.h>
79 
80 #include <uvm/uvm_extern.h>
81 
82 void	cpuctlattach(int);
83 
84 static void	cpu_xc_online(struct cpu_info *);
85 static void	cpu_xc_offline(struct cpu_info *);
86 
87 dev_type_ioctl(cpuctl_ioctl);
88 
89 const struct cdevsw cpuctl_cdevsw = {
90 	nullopen, nullclose, nullread, nullwrite, cpuctl_ioctl,
91 	nullstop, notty, nopoll, nommap, nokqfilter,
92 	D_OTHER | D_MPSAFE
93 };
94 
95 kmutex_t cpu_lock;
96 int	ncpu;
97 int	ncpuonline;
98 bool	mp_online;
99 struct	cpuqueue cpu_queue = CIRCLEQ_HEAD_INITIALIZER(cpu_queue);
100 
101 static struct cpu_info *cpu_infos[MAXCPUS];
102 
103 int
104 mi_cpu_attach(struct cpu_info *ci)
105 {
106 	int error;
107 
108 	ci->ci_index = ncpu;
109 	cpu_infos[cpu_index(ci)] = ci;
110 	CIRCLEQ_INSERT_TAIL(&cpu_queue, ci, ci_data.cpu_qchain);
111 	TAILQ_INIT(&ci->ci_data.cpu_ld_locks);
112 	__cpu_simple_lock_init(&ci->ci_data.cpu_ld_lock);
113 
114 	/* This is useful for eg, per-cpu evcnt */
115 	snprintf(ci->ci_data.cpu_name, sizeof(ci->ci_data.cpu_name), "cpu%d",
116 		 cpu_index(ci));
117 
118 	sched_cpuattach(ci);
119 
120 	error = create_idle_lwp(ci);
121 	if (error != 0) {
122 		/* XXX revert sched_cpuattach */
123 		return error;
124 	}
125 
126 	if (ci == curcpu())
127 		ci->ci_data.cpu_onproc = curlwp;
128 	else
129 		ci->ci_data.cpu_onproc = ci->ci_data.cpu_idlelwp;
130 
131 	percpu_init_cpu(ci);
132 	softint_init(ci);
133 	callout_init_cpu(ci);
134 	xc_init_cpu(ci);
135 	pool_cache_cpu_init(ci);
136 	selsysinit(ci);
137 	cache_cpu_init(ci);
138 	TAILQ_INIT(&ci->ci_data.cpu_biodone);
139 	ncpu++;
140 	ncpuonline++;
141 
142 	return 0;
143 }
144 
145 void
146 cpuctlattach(int dummy)
147 {
148 
149 }
150 
151 int
152 cpuctl_ioctl(dev_t dev, u_long cmd, void *data, int flag, lwp_t *l)
153 {
154 	CPU_INFO_ITERATOR cii;
155 	cpustate_t *cs;
156 	struct cpu_info *ci;
157 	int error, i;
158 	u_int id;
159 
160 	error = 0;
161 
162 	mutex_enter(&cpu_lock);
163 	switch (cmd) {
164 	case IOC_CPU_SETSTATE:
165 		if (error == 0)
166 			cs = data;
167 		error = kauth_authorize_system(l->l_cred,
168 		    KAUTH_SYSTEM_CPU, KAUTH_REQ_SYSTEM_CPU_SETSTATE, cs, NULL,
169 		    NULL);
170 		if (error != 0)
171 			break;
172 		if (cs->cs_id >= __arraycount(cpu_infos) ||
173 		    (ci = cpu_lookup(cs->cs_id)) == NULL) {
174 			error = ESRCH;
175 			break;
176 		}
177 		error = cpu_setintr(ci, cs->cs_intr);
178 		error = cpu_setstate(ci, cs->cs_online);
179 		break;
180 
181 	case IOC_CPU_GETSTATE:
182 		if (error == 0)
183 			cs = data;
184 		id = cs->cs_id;
185 		memset(cs, 0, sizeof(*cs));
186 		cs->cs_id = id;
187 		if (cs->cs_id >= __arraycount(cpu_infos) ||
188 		    (ci = cpu_lookup(id)) == NULL) {
189 			error = ESRCH;
190 			break;
191 		}
192 		if ((ci->ci_schedstate.spc_flags & SPCF_OFFLINE) != 0)
193 			cs->cs_online = false;
194 		else
195 			cs->cs_online = true;
196 		if ((ci->ci_schedstate.spc_flags & SPCF_NOINTR) != 0)
197 			cs->cs_intr = false;
198 		else
199 			cs->cs_intr = true;
200 		cs->cs_lastmod = (int32_t)ci->ci_schedstate.spc_lastmod;
201 		cs->cs_lastmodhi = (int32_t)
202 		    (ci->ci_schedstate.spc_lastmod >> 32);
203 		cs->cs_intrcnt = cpu_intr_count(ci) + 1;
204 		break;
205 
206 	case IOC_CPU_MAPID:
207 		i = 0;
208 		for (CPU_INFO_FOREACH(cii, ci)) {
209 			if (i++ == *(int *)data)
210 				break;
211 		}
212 		if (ci == NULL)
213 			error = ESRCH;
214 		else
215 			*(int *)data = cpu_index(ci);
216 		break;
217 
218 	case IOC_CPU_GETCOUNT:
219 		*(int *)data = ncpu;
220 		break;
221 
222 	default:
223 		error = ENOTTY;
224 		break;
225 	}
226 	mutex_exit(&cpu_lock);
227 
228 	return error;
229 }
230 
231 struct cpu_info *
232 cpu_lookup(u_int idx)
233 {
234 	struct cpu_info *ci = cpu_infos[idx];
235 
236 	KASSERT(idx < __arraycount(cpu_infos));
237 	KASSERT(ci == NULL || cpu_index(ci) == idx);
238 
239 	return ci;
240 }
241 
242 static void
243 cpu_xc_offline(struct cpu_info *ci)
244 {
245 	struct schedstate_percpu *spc, *mspc = NULL;
246 	struct cpu_info *target_ci;
247 	struct lwp *l;
248 	CPU_INFO_ITERATOR cii;
249 	int s;
250 
251 	/*
252 	 * Thread that made the cross call (separate context) holds
253 	 * cpu_lock on our behalf.
254 	 */
255 	spc = &ci->ci_schedstate;
256 	s = splsched();
257 	spc->spc_flags |= SPCF_OFFLINE;
258 	splx(s);
259 
260 	/* Take the first available CPU for the migration. */
261 	for (CPU_INFO_FOREACH(cii, target_ci)) {
262 		mspc = &target_ci->ci_schedstate;
263 		if ((mspc->spc_flags & SPCF_OFFLINE) == 0)
264 			break;
265 	}
266 	KASSERT(target_ci != NULL);
267 
268 	/*
269 	 * Migrate all non-bound threads to the other CPU.  Note that this
270 	 * runs from the xcall thread, thus handling of LSONPROC is not needed.
271 	 */
272 	mutex_enter(proc_lock);
273 	LIST_FOREACH(l, &alllwp, l_list) {
274 		struct cpu_info *mci;
275 
276 		lwp_lock(l);
277 		if (l->l_cpu != ci || (l->l_pflag & (LP_BOUND | LP_INTR))) {
278 			lwp_unlock(l);
279 			continue;
280 		}
281 		/* Normal case - no affinity */
282 		if ((l->l_flag & LW_AFFINITY) == 0) {
283 			lwp_migrate(l, target_ci);
284 			continue;
285 		}
286 		/* Affinity is set, find an online CPU in the set */
287 		KASSERT(l->l_affinity != NULL);
288 		for (CPU_INFO_FOREACH(cii, mci)) {
289 			mspc = &mci->ci_schedstate;
290 			if ((mspc->spc_flags & SPCF_OFFLINE) == 0 &&
291 			    kcpuset_isset(cpu_index(mci), l->l_affinity))
292 				break;
293 		}
294 		if (mci == NULL) {
295 			lwp_unlock(l);
296 			mutex_exit(proc_lock);
297 			goto fail;
298 		}
299 		lwp_migrate(l, mci);
300 	}
301 	mutex_exit(proc_lock);
302 
303 #ifdef __HAVE_MD_CPU_OFFLINE
304 	cpu_offline_md();
305 #endif
306 	return;
307 fail:
308 	/* Just unset the SPCF_OFFLINE flag, caller will check */
309 	s = splsched();
310 	spc->spc_flags &= ~SPCF_OFFLINE;
311 	splx(s);
312 }
313 
314 static void
315 cpu_xc_online(struct cpu_info *ci)
316 {
317 	struct schedstate_percpu *spc;
318 	int s;
319 
320 	spc = &ci->ci_schedstate;
321 	s = splsched();
322 	spc->spc_flags &= ~SPCF_OFFLINE;
323 	splx(s);
324 }
325 
326 int
327 cpu_setstate(struct cpu_info *ci, bool online)
328 {
329 	struct schedstate_percpu *spc;
330 	CPU_INFO_ITERATOR cii;
331 	struct cpu_info *ci2;
332 	uint64_t where;
333 	xcfunc_t func;
334 	int nonline;
335 
336 	spc = &ci->ci_schedstate;
337 
338 	KASSERT(mutex_owned(&cpu_lock));
339 
340 	if (online) {
341 		if ((spc->spc_flags & SPCF_OFFLINE) == 0)
342 			return 0;
343 		func = (xcfunc_t)cpu_xc_online;
344 		ncpuonline++;
345 	} else {
346 		if ((spc->spc_flags & SPCF_OFFLINE) != 0)
347 			return 0;
348 		nonline = 0;
349 		/*
350 		 * Ensure that at least one CPU within the processor set
351 		 * stays online.  Revisit this later.
352 		 */
353 		for (CPU_INFO_FOREACH(cii, ci2)) {
354 			if ((ci2->ci_schedstate.spc_flags & SPCF_OFFLINE) != 0)
355 				continue;
356 			if (ci2->ci_schedstate.spc_psid != spc->spc_psid)
357 				continue;
358 			nonline++;
359 		}
360 		if (nonline == 1)
361 			return EBUSY;
362 		func = (xcfunc_t)cpu_xc_offline;
363 		ncpuonline--;
364 	}
365 
366 	where = xc_unicast(0, func, ci, NULL, ci);
367 	xc_wait(where);
368 	if (online) {
369 		KASSERT((spc->spc_flags & SPCF_OFFLINE) == 0);
370 	} else if ((spc->spc_flags & SPCF_OFFLINE) == 0) {
371 		/* If was not set offline, then it is busy */
372 		return EBUSY;
373 	}
374 
375 	spc->spc_lastmod = time_second;
376 	return 0;
377 }
378 
379 #ifdef __HAVE_INTR_CONTROL
380 static void
381 cpu_xc_intr(struct cpu_info *ci)
382 {
383 	struct schedstate_percpu *spc;
384 	int s;
385 
386 	spc = &ci->ci_schedstate;
387 	s = splsched();
388 	spc->spc_flags &= ~SPCF_NOINTR;
389 	splx(s);
390 }
391 
392 static void
393 cpu_xc_nointr(struct cpu_info *ci)
394 {
395 	struct schedstate_percpu *spc;
396 	int s;
397 
398 	spc = &ci->ci_schedstate;
399 	s = splsched();
400 	spc->spc_flags |= SPCF_NOINTR;
401 	splx(s);
402 }
403 
404 int
405 cpu_setintr(struct cpu_info *ci, bool intr)
406 {
407 	struct schedstate_percpu *spc;
408 	CPU_INFO_ITERATOR cii;
409 	struct cpu_info *ci2;
410 	uint64_t where;
411 	xcfunc_t func;
412 	int nintr;
413 
414 	spc = &ci->ci_schedstate;
415 
416 	KASSERT(mutex_owned(&cpu_lock));
417 
418 	if (intr) {
419 		if ((spc->spc_flags & SPCF_NOINTR) == 0)
420 			return 0;
421 		func = (xcfunc_t)cpu_xc_intr;
422 	} else {
423 		if ((spc->spc_flags & SPCF_NOINTR) != 0)
424 			return 0;
425 		/*
426 		 * Ensure that at least one CPU within the system
427 		 * is handing device interrupts.
428 		 */
429 		nintr = 0;
430 		for (CPU_INFO_FOREACH(cii, ci2)) {
431 			if ((ci2->ci_schedstate.spc_flags & SPCF_NOINTR) != 0)
432 				continue;
433 			if (ci2 == ci)
434 				continue;
435 			nintr++;
436 		}
437 		if (nintr == 0)
438 			return EBUSY;
439 		func = (xcfunc_t)cpu_xc_nointr;
440 	}
441 
442 	where = xc_unicast(0, func, ci, NULL, ci);
443 	xc_wait(where);
444 	if (intr) {
445 		KASSERT((spc->spc_flags & SPCF_NOINTR) == 0);
446 	} else if ((spc->spc_flags & SPCF_NOINTR) == 0) {
447 		/* If was not set offline, then it is busy */
448 		return EBUSY;
449 	}
450 
451 	/* Direct interrupts away from the CPU and record the change. */
452 	cpu_intr_redistribute();
453 	spc->spc_lastmod = time_second;
454 	return 0;
455 }
456 #else	/* __HAVE_INTR_CONTROL */
457 int
458 cpu_setintr(struct cpu_info *ci, bool intr)
459 {
460 
461 	return EOPNOTSUPP;
462 }
463 
464 u_int
465 cpu_intr_count(struct cpu_info *ci)
466 {
467 
468 	return 0;	/* 0 == "don't know" */
469 }
470 #endif	/* __HAVE_INTR_CONTROL */
471 
472 bool
473 cpu_softintr_p(void)
474 {
475 
476 	return (curlwp->l_pflag & LP_INTR) != 0;
477 }
478