xref: /netbsd-src/sys/kern/kern_cpu.c (revision 6a493d6bc668897c91594964a732d38505b70cbb) 
1 /*	$NetBSD: kern_cpu.c,v 1.62 2013/12/19 23:36:07 mlelstv Exp $	*/
2 
3 /*-
4  * Copyright (c) 2007, 2008, 2009, 2010, 2012 The NetBSD Foundation, Inc.
5  * All rights reserved.
6  *
7  * This code is derived from software contributed to The NetBSD Foundation
8  * by Andrew Doran.
9  *
10  * Redistribution and use in source and binary forms, with or without
11  * modification, are permitted provided that the following conditions
12  * are met:
13  * 1. Redistributions of source code must retain the above copyright
14  *    notice, this list of conditions and the following disclaimer.
15  * 2. Redistributions in binary form must reproduce the above copyright
16  *    notice, this list of conditions and the following disclaimer in the
17  *    documentation and/or other materials provided with the distribution.
18  *
19  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29  * POSSIBILITY OF SUCH DAMAGE.
30  */
31 
32 /*-
33  * Copyright (c)2007 YAMAMOTO Takashi,
34  * All rights reserved.
35  *
36  * Redistribution and use in source and binary forms, with or without
37  * modification, are permitted provided that the following conditions
38  * are met:
39  * 1. Redistributions of source code must retain the above copyright
40  *    notice, this list of conditions and the following disclaimer.
41  * 2. Redistributions in binary form must reproduce the above copyright
42  *    notice, this list of conditions and the following disclaimer in the
43  *    documentation and/or other materials provided with the distribution.
44  *
45  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
46  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
47  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
48  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
49  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
50  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
51  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
52  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
53  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
54  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
55  * SUCH DAMAGE.
56  */
57 
58 #include <sys/cdefs.h>
59 __KERNEL_RCSID(0, "$NetBSD: kern_cpu.c,v 1.62 2013/12/19 23:36:07 mlelstv Exp $");
60 
61 #include "opt_cpu_ucode.h"
62 #include "opt_compat_netbsd.h"
63 
64 #include <sys/param.h>
65 #include <sys/systm.h>
66 #include <sys/idle.h>
67 #include <sys/sched.h>
68 #include <sys/intr.h>
69 #include <sys/conf.h>
70 #include <sys/cpu.h>
71 #include <sys/cpuio.h>
72 #include <sys/proc.h>
73 #include <sys/percpu.h>
74 #include <sys/kernel.h>
75 #include <sys/kauth.h>
76 #include <sys/xcall.h>
77 #include <sys/pool.h>
78 #include <sys/kmem.h>
79 #include <sys/select.h>
80 #include <sys/namei.h>
81 #include <sys/callout.h>
82 #include <sys/pcu.h>
83 
84 #include <uvm/uvm_extern.h>
85 
86 /*
87  * If the port has stated that cpu_data is the first thing in cpu_info,
88  * verify that the claim is true. This will prevent them from getting out
89  * of sync.
90  */
91 #ifdef __HAVE_CPU_DATA_FIRST
92 CTASSERT(offsetof(struct cpu_info, ci_data) == 0);
93 #else
94 CTASSERT(offsetof(struct cpu_info, ci_data) != 0);
95 #endif
96 
97 void	cpuctlattach(int);
98 
99 static void	cpu_xc_online(struct cpu_info *);
100 static void	cpu_xc_offline(struct cpu_info *);
101 
102 dev_type_ioctl(cpuctl_ioctl);
103 
104 const struct cdevsw cpuctl_cdevsw = {
105 	nullopen, nullclose, nullread, nullwrite, cpuctl_ioctl,
106 	nullstop, notty, nopoll, nommap, nokqfilter,
107 	D_OTHER | D_MPSAFE
108 };
109 
110 kmutex_t	cpu_lock		__cacheline_aligned;
111 int		ncpu			__read_mostly;
112 int		ncpuonline		__read_mostly;
113 bool		mp_online		__read_mostly;
114 
115 /* An array of CPUs.  There are ncpu entries. */
116 struct cpu_info **cpu_infos		__read_mostly;
117 
118 /* Note: set on mi_cpu_attach() and idle_loop(). */
119 kcpuset_t *	kcpuset_attached	__read_mostly	= NULL;
120 kcpuset_t *	kcpuset_running		__read_mostly	= NULL;
121 
122 /*
123  * mi_cpu_init: early initialisation of MI CPU related structures.
124  *
125  * Note: may not block and memory allocator is not yet available.
126  */
127 void
128 mi_cpu_init(void)
129 {
130 
131 	mutex_init(&cpu_lock, MUTEX_DEFAULT, IPL_NONE);
132 
133 	kcpuset_create(&kcpuset_attached, true);
134 	kcpuset_create(&kcpuset_running, true);
135 	kcpuset_set(kcpuset_running, 0);
136 }
137 
138 int
139 mi_cpu_attach(struct cpu_info *ci)
140 {
141 	int error;
142 
143 	KASSERT(maxcpus > 0);
144 
145 	ci->ci_index = ncpu;
146 	kcpuset_set(kcpuset_attached, cpu_index(ci));
147 
148 	/*
149 	 * Create a convenience cpuset of just ourselves.
150 	 */
151 	kcpuset_create(&ci->ci_data.cpu_kcpuset, true);
152 	kcpuset_set(ci->ci_data.cpu_kcpuset, cpu_index(ci));
153 
154 	TAILQ_INIT(&ci->ci_data.cpu_ld_locks);
155 	__cpu_simple_lock_init(&ci->ci_data.cpu_ld_lock);
156 
157 	/* This is useful for eg, per-cpu evcnt */
158 	snprintf(ci->ci_data.cpu_name, sizeof(ci->ci_data.cpu_name), "cpu%d",
159 	    cpu_index(ci));
160 
161 	if (__predict_false(cpu_infos == NULL)) {
162 		size_t ci_bufsize = (maxcpus + 1) * sizeof(struct cpu_info *);
163 		cpu_infos = kmem_zalloc(ci_bufsize, KM_SLEEP);
164 	}
165 	cpu_infos[cpu_index(ci)] = ci;
166 
167 	sched_cpuattach(ci);
168 
169 	error = create_idle_lwp(ci);
170 	if (error != 0) {
171 		/* XXX revert sched_cpuattach */
172 		return error;
173 	}
174 
175 	if (ci == curcpu())
176 		ci->ci_data.cpu_onproc = curlwp;
177 	else
178 		ci->ci_data.cpu_onproc = ci->ci_data.cpu_idlelwp;
179 
180 	percpu_init_cpu(ci);
181 	softint_init(ci);
182 	callout_init_cpu(ci);
183 	xc_init_cpu(ci);
184 	pool_cache_cpu_init(ci);
185 	selsysinit(ci);
186 	cache_cpu_init(ci);
187 	TAILQ_INIT(&ci->ci_data.cpu_biodone);
188 	ncpu++;
189 	ncpuonline++;
190 
191 	return 0;
192 }
193 
194 void
195 cpuctlattach(int dummy)
196 {
197 
198 	KASSERT(cpu_infos != NULL);
199 }
200 
201 int
202 cpuctl_ioctl(dev_t dev, u_long cmd, void *data, int flag, lwp_t *l)
203 {
204 	CPU_INFO_ITERATOR cii;
205 	cpustate_t *cs;
206 	struct cpu_info *ci;
207 	int error, i;
208 	u_int id;
209 
210 	error = 0;
211 
212 	mutex_enter(&cpu_lock);
213 	switch (cmd) {
214 	case IOC_CPU_SETSTATE:
215 		cs = data;
216 		error = kauth_authorize_system(l->l_cred,
217 		    KAUTH_SYSTEM_CPU, KAUTH_REQ_SYSTEM_CPU_SETSTATE, cs, NULL,
218 		    NULL);
219 		if (error != 0)
220 			break;
221 		if (cs->cs_id >= maxcpus ||
222 		    (ci = cpu_lookup(cs->cs_id)) == NULL) {
223 			error = ESRCH;
224 			break;
225 		}
226 		cpu_setintr(ci, cs->cs_intr);
227 		error = cpu_setstate(ci, cs->cs_online);
228 		break;
229 
230 	case IOC_CPU_GETSTATE:
231 		cs = data;
232 		id = cs->cs_id;
233 		memset(cs, 0, sizeof(*cs));
234 		cs->cs_id = id;
235 		if (cs->cs_id >= maxcpus ||
236 		    (ci = cpu_lookup(id)) == NULL) {
237 			error = ESRCH;
238 			break;
239 		}
240 		if ((ci->ci_schedstate.spc_flags & SPCF_OFFLINE) != 0)
241 			cs->cs_online = false;
242 		else
243 			cs->cs_online = true;
244 		if ((ci->ci_schedstate.spc_flags & SPCF_NOINTR) != 0)
245 			cs->cs_intr = false;
246 		else
247 			cs->cs_intr = true;
248 		cs->cs_lastmod = (int32_t)ci->ci_schedstate.spc_lastmod;
249 		cs->cs_lastmodhi = (int32_t)
250 		    (ci->ci_schedstate.spc_lastmod >> 32);
251 		cs->cs_intrcnt = cpu_intr_count(ci) + 1;
252 		cs->cs_hwid = ci->ci_cpuid;
253 		break;
254 
255 	case IOC_CPU_MAPID:
256 		i = 0;
257 		for (CPU_INFO_FOREACH(cii, ci)) {
258 			if (i++ == *(int *)data)
259 				break;
260 		}
261 		if (ci == NULL)
262 			error = ESRCH;
263 		else
264 			*(int *)data = cpu_index(ci);
265 		break;
266 
267 	case IOC_CPU_GETCOUNT:
268 		*(int *)data = ncpu;
269 		break;
270 
271 #ifdef CPU_UCODE
272 	case IOC_CPU_UCODE_GET_VERSION:
273 		error = cpu_ucode_get_version((struct cpu_ucode_version *)data);
274 		break;
275 
276 #ifdef COMPAT_60
277 	case OIOC_CPU_UCODE_GET_VERSION:
278 		error = compat6_cpu_ucode_get_version((struct compat6_cpu_ucode *)data);
279 		break;
280 #endif
281 
282 	case IOC_CPU_UCODE_APPLY:
283 		error = kauth_authorize_machdep(l->l_cred,
284 		    KAUTH_MACHDEP_CPU_UCODE_APPLY,
285 		    NULL, NULL, NULL, NULL);
286 		if (error != 0)
287 			break;
288 		error = cpu_ucode_apply((const struct cpu_ucode *)data);
289 		break;
290 
291 #ifdef COMPAT_60
292 	case OIOC_CPU_UCODE_APPLY:
293 		error = kauth_authorize_machdep(l->l_cred,
294 		    KAUTH_MACHDEP_CPU_UCODE_APPLY,
295 		    NULL, NULL, NULL, NULL);
296 		if (error != 0)
297 			break;
298 		error = compat6_cpu_ucode_apply((const struct compat6_cpu_ucode *)data);
299 		break;
300 #endif
301 #endif
302 
303 	default:
304 		error = ENOTTY;
305 		break;
306 	}
307 	mutex_exit(&cpu_lock);
308 
309 	return error;
310 }
311 
312 struct cpu_info *
313 cpu_lookup(u_int idx)
314 {
315 	struct cpu_info *ci;
316 
317 	KASSERT(idx < maxcpus);
318 
319 	if (__predict_false(cpu_infos == NULL)) {
320 		KASSERT(idx == 0);
321 		return curcpu();
322 	}
323 
324 	ci = cpu_infos[idx];
325 	KASSERT(ci == NULL || cpu_index(ci) == idx);
326 
327 	return ci;
328 }
329 
330 static void
331 cpu_xc_offline(struct cpu_info *ci)
332 {
333 	struct schedstate_percpu *spc, *mspc = NULL;
334 	struct cpu_info *target_ci;
335 	struct lwp *l;
336 	CPU_INFO_ITERATOR cii;
337 	int s;
338 
339 	/*
340 	 * Thread that made the cross call (separate context) holds
341 	 * cpu_lock on our behalf.
342 	 */
343 	spc = &ci->ci_schedstate;
344 	s = splsched();
345 	spc->spc_flags |= SPCF_OFFLINE;
346 	splx(s);
347 
348 	/* Take the first available CPU for the migration. */
349 	for (CPU_INFO_FOREACH(cii, target_ci)) {
350 		mspc = &target_ci->ci_schedstate;
351 		if ((mspc->spc_flags & SPCF_OFFLINE) == 0)
352 			break;
353 	}
354 	KASSERT(target_ci != NULL);
355 
356 	/*
357 	 * Migrate all non-bound threads to the other CPU.  Note that this
358 	 * runs from the xcall thread, thus handling of LSONPROC is not needed.
359 	 */
360 	mutex_enter(proc_lock);
361 	LIST_FOREACH(l, &alllwp, l_list) {
362 		struct cpu_info *mci;
363 
364 		lwp_lock(l);
365 		if (l->l_cpu != ci || (l->l_pflag & (LP_BOUND | LP_INTR))) {
366 			lwp_unlock(l);
367 			continue;
368 		}
369 		/* Regular case - no affinity. */
370 		if (l->l_affinity == NULL) {
371 			lwp_migrate(l, target_ci);
372 			continue;
373 		}
374 		/* Affinity is set, find an online CPU in the set. */
375 		for (CPU_INFO_FOREACH(cii, mci)) {
376 			mspc = &mci->ci_schedstate;
377 			if ((mspc->spc_flags & SPCF_OFFLINE) == 0 &&
378 			    kcpuset_isset(l->l_affinity, cpu_index(mci)))
379 				break;
380 		}
381 		if (mci == NULL) {
382 			lwp_unlock(l);
383 			mutex_exit(proc_lock);
384 			goto fail;
385 		}
386 		lwp_migrate(l, mci);
387 	}
388 	mutex_exit(proc_lock);
389 
390 #if PCU_UNIT_COUNT > 0
391 	pcu_save_all_on_cpu();
392 #endif
393 
394 #ifdef __HAVE_MD_CPU_OFFLINE
395 	cpu_offline_md();
396 #endif
397 	return;
398 fail:
399 	/* Just unset the SPCF_OFFLINE flag, caller will check */
400 	s = splsched();
401 	spc->spc_flags &= ~SPCF_OFFLINE;
402 	splx(s);
403 }
404 
405 static void
406 cpu_xc_online(struct cpu_info *ci)
407 {
408 	struct schedstate_percpu *spc;
409 	int s;
410 
411 	spc = &ci->ci_schedstate;
412 	s = splsched();
413 	spc->spc_flags &= ~SPCF_OFFLINE;
414 	splx(s);
415 }
416 
417 int
418 cpu_setstate(struct cpu_info *ci, bool online)
419 {
420 	struct schedstate_percpu *spc;
421 	CPU_INFO_ITERATOR cii;
422 	struct cpu_info *ci2;
423 	uint64_t where;
424 	xcfunc_t func;
425 	int nonline;
426 
427 	spc = &ci->ci_schedstate;
428 
429 	KASSERT(mutex_owned(&cpu_lock));
430 
431 	if (online) {
432 		if ((spc->spc_flags & SPCF_OFFLINE) == 0)
433 			return 0;
434 		func = (xcfunc_t)cpu_xc_online;
435 		ncpuonline++;
436 	} else {
437 		if ((spc->spc_flags & SPCF_OFFLINE) != 0)
438 			return 0;
439 		nonline = 0;
440 		/*
441 		 * Ensure that at least one CPU within the processor set
442 		 * stays online.  Revisit this later.
443 		 */
444 		for (CPU_INFO_FOREACH(cii, ci2)) {
445 			if ((ci2->ci_schedstate.spc_flags & SPCF_OFFLINE) != 0)
446 				continue;
447 			if (ci2->ci_schedstate.spc_psid != spc->spc_psid)
448 				continue;
449 			nonline++;
450 		}
451 		if (nonline == 1)
452 			return EBUSY;
453 		func = (xcfunc_t)cpu_xc_offline;
454 		ncpuonline--;
455 	}
456 
457 	where = xc_unicast(0, func, ci, NULL, ci);
458 	xc_wait(where);
459 	if (online) {
460 		KASSERT((spc->spc_flags & SPCF_OFFLINE) == 0);
461 	} else if ((spc->spc_flags & SPCF_OFFLINE) == 0) {
462 		/* If was not set offline, then it is busy */
463 		return EBUSY;
464 	}
465 
466 	spc->spc_lastmod = time_second;
467 	return 0;
468 }
469 
470 #ifdef __HAVE_INTR_CONTROL
471 static void
472 cpu_xc_intr(struct cpu_info *ci)
473 {
474 	struct schedstate_percpu *spc;
475 	int s;
476 
477 	spc = &ci->ci_schedstate;
478 	s = splsched();
479 	spc->spc_flags &= ~SPCF_NOINTR;
480 	splx(s);
481 }
482 
483 static void
484 cpu_xc_nointr(struct cpu_info *ci)
485 {
486 	struct schedstate_percpu *spc;
487 	int s;
488 
489 	spc = &ci->ci_schedstate;
490 	s = splsched();
491 	spc->spc_flags |= SPCF_NOINTR;
492 	splx(s);
493 }
494 
495 int
496 cpu_setintr(struct cpu_info *ci, bool intr)
497 {
498 	struct schedstate_percpu *spc;
499 	CPU_INFO_ITERATOR cii;
500 	struct cpu_info *ci2;
501 	uint64_t where;
502 	xcfunc_t func;
503 	int nintr;
504 
505 	spc = &ci->ci_schedstate;
506 
507 	KASSERT(mutex_owned(&cpu_lock));
508 
509 	if (intr) {
510 		if ((spc->spc_flags & SPCF_NOINTR) == 0)
511 			return 0;
512 		func = (xcfunc_t)cpu_xc_intr;
513 	} else {
514 		if ((spc->spc_flags & SPCF_NOINTR) != 0)
515 			return 0;
516 		/*
517 		 * Ensure that at least one CPU within the system
518 		 * is handing device interrupts.
519 		 */
520 		nintr = 0;
521 		for (CPU_INFO_FOREACH(cii, ci2)) {
522 			if ((ci2->ci_schedstate.spc_flags & SPCF_NOINTR) != 0)
523 				continue;
524 			if (ci2 == ci)
525 				continue;
526 			nintr++;
527 		}
528 		if (nintr == 0)
529 			return EBUSY;
530 		func = (xcfunc_t)cpu_xc_nointr;
531 	}
532 
533 	where = xc_unicast(0, func, ci, NULL, ci);
534 	xc_wait(where);
535 	if (intr) {
536 		KASSERT((spc->spc_flags & SPCF_NOINTR) == 0);
537 	} else if ((spc->spc_flags & SPCF_NOINTR) == 0) {
538 		/* If was not set offline, then it is busy */
539 		return EBUSY;
540 	}
541 
542 	/* Direct interrupts away from the CPU and record the change. */
543 	cpu_intr_redistribute();
544 	spc->spc_lastmod = time_second;
545 	return 0;
546 }
547 #else	/* __HAVE_INTR_CONTROL */
548 int
549 cpu_setintr(struct cpu_info *ci, bool intr)
550 {
551 
552 	return EOPNOTSUPP;
553 }
554 
555 u_int
556 cpu_intr_count(struct cpu_info *ci)
557 {
558 
559 	return 0;	/* 0 == "don't know" */
560 }
561 #endif	/* __HAVE_INTR_CONTROL */
562 
563 bool
564 cpu_softintr_p(void)
565 {
566 
567 	return (curlwp->l_pflag & LP_INTR) != 0;
568 }
569 
570 #ifdef CPU_UCODE
571 int
572 cpu_ucode_load(struct cpu_ucode_softc *sc, const char *fwname)
573 {
574 	firmware_handle_t fwh;
575 	int error;
576 
577 	if (sc->sc_blob != NULL) {
578 		firmware_free(sc->sc_blob, 0);
579 		sc->sc_blob = NULL;
580 		sc->sc_blobsize = 0;
581 	}
582 
583 	error = cpu_ucode_md_open(&fwh, sc->loader_version, fwname);
584 	if (error != 0) {
585 		aprint_error("ucode: firmware_open failed: %i\n", error);
586 		goto err0;
587 	}
588 
589 	sc->sc_blobsize = firmware_get_size(fwh);
590 	sc->sc_blob = firmware_malloc(sc->sc_blobsize);
591 	if (sc->sc_blob == NULL) {
592 		error = ENOMEM;
593 		firmware_close(fwh);
594 		goto err0;
595 	}
596 
597 	error = firmware_read(fwh, 0, sc->sc_blob, sc->sc_blobsize);
598 	firmware_close(fwh);
599 	if (error != 0)
600 		goto err1;
601 
602 	return 0;
603 
604 err1:
605 	firmware_free(sc->sc_blob, 0);
606 	sc->sc_blob = NULL;
607 	sc->sc_blobsize = 0;
608 err0:
609 	return error;
610 }
611 #endif
612