xref: /openbsd-src/sys/kern/kern_sched.c (revision 7b3f8d1d68d965ba90edeb8b95711a130dd8d612)
1 /*	$OpenBSD: kern_sched.c,v 1.102 2024/10/08 11:57:59 claudio Exp $	*/
2 /*
3  * Copyright (c) 2007, 2008 Artur Grabowski <art@openbsd.org>
4  *
5  * Permission to use, copy, modify, and distribute this software for any
6  * purpose with or without fee is hereby granted, provided that the above
7  * copyright notice and this permission notice appear in all copies.
8  *
9  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
10  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
11  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
12  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
13  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
14  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
15  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
16  */
17 
18 #include <sys/param.h>
19 
20 #include <sys/sched.h>
21 #include <sys/proc.h>
22 #include <sys/kthread.h>
23 #include <sys/systm.h>
24 #include <sys/clockintr.h>
25 #include <sys/resourcevar.h>
26 #include <sys/task.h>
27 #include <sys/time.h>
28 #include <sys/smr.h>
29 #include <sys/tracepoint.h>
30 
31 #include <uvm/uvm_extern.h>
32 
33 void sched_kthreads_create(void *);
34 
35 int sched_proc_to_cpu_cost(struct cpu_info *ci, struct proc *p);
36 struct proc *sched_steal_proc(struct cpu_info *);
37 
38 /*
39  * To help choosing which cpu should run which process we keep track
40  * of cpus which are currently idle and which cpus have processes
41  * queued.
42  */
43 struct cpuset sched_idle_cpus;
44 struct cpuset sched_queued_cpus;
45 struct cpuset sched_all_cpus;
46 
47 /*
48  * Some general scheduler counters.
49  */
50 uint64_t sched_nmigrations;	/* Cpu migration counter */
51 uint64_t sched_nomigrations;	/* Cpu no migration counter */
52 uint64_t sched_noidle;		/* Times we didn't pick the idle task */
53 uint64_t sched_stolen;		/* Times we stole proc from other cpus */
54 uint64_t sched_choose;		/* Times we chose a cpu */
55 uint64_t sched_wasidle;		/* Times we came out of idle */
56 
57 int sched_smt;
58 
59 /*
60  * A few notes about cpu_switchto that is implemented in MD code.
61  *
62  * cpu_switchto takes two arguments, the old proc and the proc
63  * it should switch to. The new proc will never be NULL, so we always have
64  * a saved state that we need to switch to. The old proc however can
65  * be NULL if the process is exiting. NULL for the old proc simply
66  * means "don't bother saving old state".
67  *
68  * cpu_switchto is supposed to atomically load the new state of the process
69  * including the pcb, pmap and setting curproc, the p_cpu pointer in the
70  * proc and p_stat to SONPROC. Atomically with respect to interrupts, other
71  * cpus in the system must not depend on this state being consistent.
72  * Therefore no locking is necessary in cpu_switchto other than blocking
73  * interrupts during the context switch.
74  */
75 
76 /*
77  * sched_init_cpu is called from main() for the boot cpu, then it's the
78  * responsibility of the MD code to call it for all other cpus.
79  */
80 void
81 sched_init_cpu(struct cpu_info *ci)
82 {
83 	struct schedstate_percpu *spc = &ci->ci_schedstate;
84 	int i;
85 
86 	for (i = 0; i < SCHED_NQS; i++)
87 		TAILQ_INIT(&spc->spc_qs[i]);
88 
89 	spc->spc_idleproc = NULL;
90 
91 	clockintr_bind(&spc->spc_itimer, ci, itimer_update, NULL);
92 	clockintr_bind(&spc->spc_profclock, ci, profclock, NULL);
93 	clockintr_bind(&spc->spc_roundrobin, ci, roundrobin, NULL);
94 	clockintr_bind(&spc->spc_statclock, ci, statclock, NULL);
95 
96 	kthread_create_deferred(sched_kthreads_create, ci);
97 
98 	LIST_INIT(&spc->spc_deadproc);
99 	SIMPLEQ_INIT(&spc->spc_deferred);
100 
101 	/*
102 	 * Slight hack here until the cpuset code handles cpu_info
103 	 * structures.
104 	 */
105 	cpuset_init_cpu(ci);
106 
107 #ifdef __HAVE_CPU_TOPOLOGY
108 	if (!sched_smt && ci->ci_smt_id > 0)
109 		return;
110 #endif
111 	cpuset_add(&sched_all_cpus, ci);
112 }
113 
114 void
115 sched_kthreads_create(void *v)
116 {
117 	struct cpu_info *ci = v;
118 	struct schedstate_percpu *spc = &ci->ci_schedstate;
119 	static int num;
120 
121 	if (fork1(&proc0, FORK_SHAREVM|FORK_SHAREFILES|FORK_NOZOMBIE|
122 	    FORK_SYSTEM|FORK_IDLE, sched_idle, ci, NULL,
123 	    &spc->spc_idleproc))
124 		panic("fork idle");
125 
126 	/* Name it as specified. */
127 	snprintf(spc->spc_idleproc->p_p->ps_comm,
128 	    sizeof(spc->spc_idleproc->p_p->ps_comm),
129 	    "idle%d", num);
130 
131 	num++;
132 }
133 
134 void
135 sched_idle(void *v)
136 {
137 	struct schedstate_percpu *spc;
138 	struct proc *p = curproc;
139 	struct cpu_info *ci = v;
140 
141 	KERNEL_UNLOCK();
142 
143 	spc = &ci->ci_schedstate;
144 
145 	/*
146 	 * First time we enter here, we're not supposed to idle,
147 	 * just go away for a while.
148 	 */
149 	SCHED_LOCK();
150 	cpuset_add(&sched_idle_cpus, ci);
151 	p->p_stat = SSLEEP;
152 	p->p_cpu = ci;
153 	atomic_setbits_int(&p->p_flag, P_CPUPEG);
154 	mi_switch();
155 	cpuset_del(&sched_idle_cpus, ci);
156 	SCHED_UNLOCK();
157 
158 	KASSERT(ci == curcpu());
159 	KASSERT(curproc == spc->spc_idleproc);
160 
161 	while (1) {
162 		while (!cpu_is_idle(curcpu())) {
163 			struct proc *dead;
164 
165 			SCHED_LOCK();
166 			p->p_stat = SSLEEP;
167 			mi_switch();
168 			SCHED_UNLOCK();
169 
170 			while ((dead = LIST_FIRST(&spc->spc_deadproc))) {
171 				LIST_REMOVE(dead, p_hash);
172 				exit2(dead);
173 			}
174 		}
175 
176 		splassert(IPL_NONE);
177 
178 		smr_idle();
179 
180 		cpuset_add(&sched_idle_cpus, ci);
181 		cpu_idle_enter();
182 		while (spc->spc_whichqs == 0) {
183 #ifdef MULTIPROCESSOR
184 			if (spc->spc_schedflags & SPCF_SHOULDHALT &&
185 			    (spc->spc_schedflags & SPCF_HALTED) == 0) {
186 				cpuset_del(&sched_idle_cpus, ci);
187 				SCHED_LOCK();
188 				atomic_setbits_int(&spc->spc_schedflags,
189 				    spc->spc_whichqs ? 0 : SPCF_HALTED);
190 				SCHED_UNLOCK();
191 				wakeup(spc);
192 			}
193 #endif
194 			cpu_idle_cycle();
195 		}
196 		cpu_idle_leave();
197 		cpuset_del(&sched_idle_cpus, ci);
198 	}
199 }
200 
201 /*
202  * To free our address space we have to jump through a few hoops.
203  * The freeing is done by the reaper, but until we have one reaper
204  * per cpu, we have no way of putting this proc on the deadproc list
205  * and waking up the reaper without risking having our address space and
206  * stack torn from under us before we manage to switch to another proc.
207  * Therefore we have a per-cpu list of dead processes where we put this
208  * proc and have idle clean up that list and move it to the reaper list.
209  * All this will be unnecessary once we can bind the reaper this cpu
210  * and not risk having it switch to another in case it sleeps.
211  */
212 void
213 sched_exit(struct proc *p)
214 {
215 	struct schedstate_percpu *spc = &curcpu()->ci_schedstate;
216 
217 	LIST_INSERT_HEAD(&spc->spc_deadproc, p, p_hash);
218 
219 	tuagg_add_runtime();
220 
221 	KERNEL_ASSERT_LOCKED();
222 	sched_toidle();
223 }
224 
225 void
226 sched_toidle(void)
227 {
228 	struct schedstate_percpu *spc = &curcpu()->ci_schedstate;
229 	struct proc *idle;
230 
231 #ifdef MULTIPROCESSOR
232 	/* This process no longer needs to hold the kernel lock. */
233 	if (_kernel_lock_held())
234 		__mp_release_all(&kernel_lock);
235 #endif
236 
237 	if (ISSET(spc->spc_schedflags, SPCF_ITIMER)) {
238 		atomic_clearbits_int(&spc->spc_schedflags, SPCF_ITIMER);
239 		clockintr_cancel(&spc->spc_itimer);
240 	}
241 	if (ISSET(spc->spc_schedflags, SPCF_PROFCLOCK)) {
242 		atomic_clearbits_int(&spc->spc_schedflags, SPCF_PROFCLOCK);
243 		clockintr_cancel(&spc->spc_profclock);
244 	}
245 
246 	atomic_clearbits_int(&spc->spc_schedflags, SPCF_SWITCHCLEAR);
247 
248 	SCHED_LOCK();
249 	idle = spc->spc_idleproc;
250 	idle->p_stat = SRUN;
251 
252 	uvmexp.swtch++;
253 	if (curproc != NULL)
254 		TRACEPOINT(sched, off__cpu, idle->p_tid + THREAD_PID_OFFSET,
255 		    idle->p_p->ps_pid);
256 	cpu_switchto(NULL, idle);
257 	panic("cpu_switchto returned");
258 }
259 
260 /*
261  * Run queue management.
262  */
263 void
264 sched_init_runqueues(void)
265 {
266 }
267 
268 void
269 setrunqueue(struct cpu_info *ci, struct proc *p, uint8_t prio)
270 {
271 	struct schedstate_percpu *spc;
272 	int queue = prio >> 2;
273 
274 	if (ci == NULL)
275 		ci = sched_choosecpu(p);
276 
277 	KASSERT(ci != NULL);
278 	SCHED_ASSERT_LOCKED();
279 	KASSERT(p->p_wchan == NULL);
280 
281 	p->p_cpu = ci;
282 	p->p_stat = SRUN;
283 	p->p_runpri = prio;
284 
285 	spc = &p->p_cpu->ci_schedstate;
286 	spc->spc_nrun++;
287 	TRACEPOINT(sched, enqueue, p->p_tid + THREAD_PID_OFFSET,
288 	    p->p_p->ps_pid);
289 
290 	TAILQ_INSERT_TAIL(&spc->spc_qs[queue], p, p_runq);
291 	spc->spc_whichqs |= (1U << queue);
292 	cpuset_add(&sched_queued_cpus, p->p_cpu);
293 
294 	if (cpuset_isset(&sched_idle_cpus, p->p_cpu))
295 		cpu_unidle(p->p_cpu);
296 	else if (prio < spc->spc_curpriority)
297 		need_resched(ci);
298 }
299 
300 void
301 remrunqueue(struct proc *p)
302 {
303 	struct schedstate_percpu *spc;
304 	int queue = p->p_runpri >> 2;
305 
306 	SCHED_ASSERT_LOCKED();
307 	spc = &p->p_cpu->ci_schedstate;
308 	spc->spc_nrun--;
309 	TRACEPOINT(sched, dequeue, p->p_tid + THREAD_PID_OFFSET,
310 	    p->p_p->ps_pid);
311 
312 	TAILQ_REMOVE(&spc->spc_qs[queue], p, p_runq);
313 	if (TAILQ_EMPTY(&spc->spc_qs[queue])) {
314 		spc->spc_whichqs &= ~(1U << queue);
315 		if (spc->spc_whichqs == 0)
316 			cpuset_del(&sched_queued_cpus, p->p_cpu);
317 	}
318 }
319 
320 struct proc *
321 sched_chooseproc(void)
322 {
323 	struct schedstate_percpu *spc = &curcpu()->ci_schedstate;
324 	struct proc *p;
325 	int queue;
326 
327 	SCHED_ASSERT_LOCKED();
328 
329 #ifdef MULTIPROCESSOR
330 	if (spc->spc_schedflags & SPCF_SHOULDHALT) {
331 		if (spc->spc_whichqs) {
332 			for (queue = 0; queue < SCHED_NQS; queue++) {
333 				while ((p = TAILQ_FIRST(&spc->spc_qs[queue]))) {
334 					remrunqueue(p);
335 					setrunqueue(NULL, p, p->p_runpri);
336 					if (p->p_cpu == curcpu()) {
337 						KASSERT(p->p_flag & P_CPUPEG);
338 						goto again;
339 					}
340 				}
341 			}
342 		}
343 		p = spc->spc_idleproc;
344 		if (p == NULL)
345 			panic("no idleproc set on CPU%d",
346 			    CPU_INFO_UNIT(curcpu()));
347 		p->p_stat = SRUN;
348 		KASSERT(p->p_wchan == NULL);
349 		return (p);
350 	}
351 again:
352 #endif
353 
354 	if (spc->spc_whichqs) {
355 		queue = ffs(spc->spc_whichqs) - 1;
356 		p = TAILQ_FIRST(&spc->spc_qs[queue]);
357 		remrunqueue(p);
358 		sched_noidle++;
359 		if (p->p_stat != SRUN)
360 			panic("thread %d not in SRUN: %d", p->p_tid, p->p_stat);
361 	} else if ((p = sched_steal_proc(curcpu())) == NULL) {
362 		p = spc->spc_idleproc;
363 		if (p == NULL)
364 			panic("no idleproc set on CPU%d",
365 			    CPU_INFO_UNIT(curcpu()));
366 		p->p_stat = SRUN;
367 	}
368 
369 	KASSERT(p->p_wchan == NULL);
370 	return (p);
371 }
372 
373 struct cpu_info *
374 sched_choosecpu_fork(struct proc *parent, int flags)
375 {
376 #ifdef MULTIPROCESSOR
377 	struct cpu_info *choice = NULL;
378 	int run, best_run = INT_MAX;
379 	struct cpu_info *ci;
380 	struct cpuset set;
381 
382 #if 0
383 	/*
384 	 * XXX
385 	 * Don't do this until we have a painless way to move the cpu in exec.
386 	 * Preferably when nuking the old pmap and getting a new one on a
387 	 * new cpu.
388 	 */
389 	/*
390 	 * PPWAIT forks are simple. We know that the parent will not
391 	 * run until we exec and choose another cpu, so we just steal its
392 	 * cpu.
393 	 */
394 	if (flags & FORK_PPWAIT)
395 		return (parent->p_cpu);
396 #endif
397 
398 	/*
399 	 * Look at all cpus that are currently idle and have nothing queued.
400 	 * If there are none, pick the one with least queued procs first,
401 	 * then the one with lowest load average.
402 	 */
403 	cpuset_complement(&set, &sched_queued_cpus, &sched_idle_cpus);
404 	cpuset_intersection(&set, &set, &sched_all_cpus);
405 	if (cpuset_first(&set) == NULL)
406 		cpuset_copy(&set, &sched_all_cpus);
407 
408 	while ((ci = cpuset_first(&set)) != NULL) {
409 		cpuset_del(&set, ci);
410 
411 		run = ci->ci_schedstate.spc_nrun;
412 
413 		if (choice == NULL || run < best_run) {
414 			choice = ci;
415 			best_run = run;
416 		}
417 	}
418 
419 	return (choice);
420 #else
421 	return (curcpu());
422 #endif
423 }
424 
425 struct cpu_info *
426 sched_choosecpu(struct proc *p)
427 {
428 #ifdef MULTIPROCESSOR
429 	struct cpu_info *choice = NULL;
430 	int last_cost = INT_MAX;
431 	struct cpu_info *ci;
432 	struct cpuset set;
433 
434 	/*
435 	 * If pegged to a cpu, don't allow it to move.
436 	 */
437 	if (p->p_flag & P_CPUPEG)
438 		return (p->p_cpu);
439 
440 	sched_choose++;
441 
442 	/*
443 	 * Look at all cpus that are currently idle and have nothing queued.
444 	 * If there are none, pick the cheapest of those.
445 	 * (idle + queued could mean that the cpu is handling an interrupt
446 	 * at this moment and haven't had time to leave idle yet).
447 	 */
448 	cpuset_complement(&set, &sched_queued_cpus, &sched_idle_cpus);
449 	cpuset_intersection(&set, &set, &sched_all_cpus);
450 
451 	/*
452 	 * First, just check if our current cpu is in that set, if it is,
453 	 * this is simple.
454 	 * Also, our cpu might not be idle, but if it's the current cpu
455 	 * and it has nothing else queued and we're curproc, take it.
456 	 */
457 	if (cpuset_isset(&set, p->p_cpu) ||
458 	    (p->p_cpu == curcpu() && p->p_cpu->ci_schedstate.spc_nrun == 0 &&
459 	    (p->p_cpu->ci_schedstate.spc_schedflags & SPCF_SHOULDHALT) == 0 &&
460 	    curproc == p)) {
461 		sched_wasidle++;
462 		return (p->p_cpu);
463 	}
464 
465 	if (cpuset_first(&set) == NULL)
466 		cpuset_copy(&set, &sched_all_cpus);
467 
468 	while ((ci = cpuset_first(&set)) != NULL) {
469 		int cost = sched_proc_to_cpu_cost(ci, p);
470 
471 		if (choice == NULL || cost < last_cost) {
472 			choice = ci;
473 			last_cost = cost;
474 		}
475 		cpuset_del(&set, ci);
476 	}
477 
478 	if (p->p_cpu != choice)
479 		sched_nmigrations++;
480 	else
481 		sched_nomigrations++;
482 
483 	return (choice);
484 #else
485 	return (curcpu());
486 #endif
487 }
488 
489 /*
490  * Attempt to steal a proc from some cpu.
491  */
492 struct proc *
493 sched_steal_proc(struct cpu_info *self)
494 {
495 	struct proc *best = NULL;
496 #ifdef MULTIPROCESSOR
497 	struct schedstate_percpu *spc;
498 	int bestcost = INT_MAX;
499 	struct cpu_info *ci;
500 	struct cpuset set;
501 
502 	KASSERT((self->ci_schedstate.spc_schedflags & SPCF_SHOULDHALT) == 0);
503 
504 	/* Don't steal if we don't want to schedule processes in this CPU. */
505 	if (!cpuset_isset(&sched_all_cpus, self))
506 		return (NULL);
507 
508 	cpuset_copy(&set, &sched_queued_cpus);
509 
510 	while ((ci = cpuset_first(&set)) != NULL) {
511 		struct proc *p;
512 		int queue;
513 		int cost;
514 
515 		cpuset_del(&set, ci);
516 
517 		spc = &ci->ci_schedstate;
518 
519 		queue = ffs(spc->spc_whichqs) - 1;
520 		TAILQ_FOREACH(p, &spc->spc_qs[queue], p_runq) {
521 			if (p->p_flag & P_CPUPEG)
522 				continue;
523 
524 			cost = sched_proc_to_cpu_cost(self, p);
525 
526 			if (best == NULL || cost < bestcost) {
527 				best = p;
528 				bestcost = cost;
529 			}
530 		}
531 	}
532 	if (best == NULL)
533 		return (NULL);
534 
535 	TRACEPOINT(sched, steal, best->p_tid + THREAD_PID_OFFSET,
536 	    best->p_p->ps_pid, CPU_INFO_UNIT(self));
537 
538 	remrunqueue(best);
539 	best->p_cpu = self;
540 
541 	sched_stolen++;
542 #endif
543 	return (best);
544 }
545 
546 #ifdef MULTIPROCESSOR
547 /*
548  * Base 2 logarithm of an int. returns 0 for 0 (yeye, I know).
549  */
550 static int
551 log2(unsigned int i)
552 {
553 	int ret = 0;
554 
555 	while (i >>= 1)
556 		ret++;
557 
558 	return (ret);
559 }
560 
561 /*
562  * Calculate the cost of moving the proc to this cpu.
563  *
564  * What we want is some guesstimate of how much "performance" it will
565  * cost us to move the proc here. Not just for caches and TLBs and NUMA
566  * memory, but also for the proc itself. A highly loaded cpu might not
567  * be the best candidate for this proc since it won't get run.
568  *
569  * Just total guesstimates for now.
570  */
571 
572 int sched_cost_priority = 1;
573 int sched_cost_runnable = 3;
574 int sched_cost_resident = 1;
575 #endif
576 
577 int
578 sched_proc_to_cpu_cost(struct cpu_info *ci, struct proc *p)
579 {
580 	int cost = 0;
581 #ifdef MULTIPROCESSOR
582 	struct schedstate_percpu *spc;
583 	int l2resident = 0;
584 
585 	spc = &ci->ci_schedstate;
586 
587 	/*
588 	 * First, account for the priority of the proc we want to move.
589 	 * More willing to move, the lower the priority of the destination
590 	 * and the higher the priority of the proc.
591 	 */
592 	if (!cpuset_isset(&sched_idle_cpus, ci)) {
593 		cost += (p->p_usrpri - spc->spc_curpriority) *
594 		    sched_cost_priority;
595 		cost += sched_cost_runnable;
596 	}
597 	if (cpuset_isset(&sched_queued_cpus, ci))
598 		cost += spc->spc_nrun * sched_cost_runnable;
599 
600 	/*
601 	 * Try to avoid the primary cpu as it handles hardware interrupts.
602 	 *
603 	 * XXX Needs to be revisited when we distribute interrupts
604 	 * over cpus.
605 	 */
606 	if (CPU_IS_PRIMARY(ci))
607 		cost += sched_cost_runnable;
608 
609 	/*
610 	 * If the proc is on this cpu already, lower the cost by how much
611 	 * it has been running and an estimate of its footprint.
612 	 */
613 	if (p->p_cpu == ci && p->p_slptime == 0) {
614 		l2resident =
615 		    log2(pmap_resident_count(p->p_vmspace->vm_map.pmap));
616 		cost -= l2resident * sched_cost_resident;
617 	}
618 #endif
619 	return (cost);
620 }
621 
622 /*
623  * Peg a proc to a cpu.
624  */
625 void
626 sched_peg_curproc(struct cpu_info *ci)
627 {
628 	struct proc *p = curproc;
629 
630 	SCHED_LOCK();
631 	atomic_setbits_int(&p->p_flag, P_CPUPEG);
632 	setrunqueue(ci, p, p->p_usrpri);
633 	p->p_ru.ru_nvcsw++;
634 	mi_switch();
635 	SCHED_UNLOCK();
636 }
637 
638 void
639 sched_unpeg_curproc(void)
640 {
641 	struct proc *p = curproc;
642 
643 	atomic_clearbits_int(&p->p_flag, P_CPUPEG);
644 }
645 
646 #ifdef MULTIPROCESSOR
647 
648 void
649 sched_start_secondary_cpus(void)
650 {
651 	CPU_INFO_ITERATOR cii;
652 	struct cpu_info *ci;
653 
654 	CPU_INFO_FOREACH(cii, ci) {
655 		struct schedstate_percpu *spc = &ci->ci_schedstate;
656 
657 		if (CPU_IS_PRIMARY(ci) || !CPU_IS_RUNNING(ci))
658 			continue;
659 		atomic_clearbits_int(&spc->spc_schedflags,
660 		    SPCF_SHOULDHALT | SPCF_HALTED);
661 #ifdef __HAVE_CPU_TOPOLOGY
662 		if (!sched_smt && ci->ci_smt_id > 0)
663 			continue;
664 #endif
665 		cpuset_add(&sched_all_cpus, ci);
666 	}
667 }
668 
669 void
670 sched_stop_secondary_cpus(void)
671 {
672 	CPU_INFO_ITERATOR cii;
673 	struct cpu_info *ci;
674 
675 	/*
676 	 * Make sure we stop the secondary CPUs.
677 	 */
678 	CPU_INFO_FOREACH(cii, ci) {
679 		struct schedstate_percpu *spc = &ci->ci_schedstate;
680 
681 		if (CPU_IS_PRIMARY(ci) || !CPU_IS_RUNNING(ci))
682 			continue;
683 		cpuset_del(&sched_all_cpus, ci);
684 		atomic_setbits_int(&spc->spc_schedflags, SPCF_SHOULDHALT);
685 	}
686 	CPU_INFO_FOREACH(cii, ci) {
687 		struct schedstate_percpu *spc = &ci->ci_schedstate;
688 
689 		if (CPU_IS_PRIMARY(ci) || !CPU_IS_RUNNING(ci))
690 			continue;
691 		while ((spc->spc_schedflags & SPCF_HALTED) == 0) {
692 			sleep_setup(spc, PZERO, "schedstate");
693 			sleep_finish(0,
694 			    (spc->spc_schedflags & SPCF_HALTED) == 0);
695 		}
696 	}
697 }
698 
699 struct sched_barrier_state {
700 	struct cpu_info *ci;
701 	struct cond cond;
702 };
703 
704 void
705 sched_barrier_task(void *arg)
706 {
707 	struct sched_barrier_state *sb = arg;
708 	struct cpu_info *ci = sb->ci;
709 
710 	sched_peg_curproc(ci);
711 	cond_signal(&sb->cond);
712 	sched_unpeg_curproc();
713 }
714 
715 void
716 sched_barrier(struct cpu_info *ci)
717 {
718 	struct sched_barrier_state sb;
719 	struct task task;
720 	CPU_INFO_ITERATOR cii;
721 
722 	if (ci == NULL) {
723 		CPU_INFO_FOREACH(cii, ci) {
724 			if (CPU_IS_PRIMARY(ci))
725 				break;
726 		}
727 	}
728 	KASSERT(ci != NULL);
729 
730 	if (ci == curcpu())
731 		return;
732 
733 	sb.ci = ci;
734 	cond_init(&sb.cond);
735 	task_set(&task, sched_barrier_task, &sb);
736 
737 	task_add(systqmp, &task);
738 	cond_wait(&sb.cond, "sbar");
739 }
740 
741 #else
742 
743 void
744 sched_barrier(struct cpu_info *ci)
745 {
746 }
747 
748 #endif
749 
750 /*
751  * Functions to manipulate cpu sets.
752  */
753 struct cpu_info *cpuset_infos[MAXCPUS];
754 static struct cpuset cpuset_all;
755 
756 void
757 cpuset_init_cpu(struct cpu_info *ci)
758 {
759 	cpuset_add(&cpuset_all, ci);
760 	cpuset_infos[CPU_INFO_UNIT(ci)] = ci;
761 }
762 
763 void
764 cpuset_clear(struct cpuset *cs)
765 {
766 	memset(cs, 0, sizeof(*cs));
767 }
768 
769 void
770 cpuset_add(struct cpuset *cs, struct cpu_info *ci)
771 {
772 	unsigned int num = CPU_INFO_UNIT(ci);
773 	atomic_setbits_int(&cs->cs_set[num/32], (1U << (num % 32)));
774 }
775 
776 void
777 cpuset_del(struct cpuset *cs, struct cpu_info *ci)
778 {
779 	unsigned int num = CPU_INFO_UNIT(ci);
780 	atomic_clearbits_int(&cs->cs_set[num/32], (1U << (num % 32)));
781 }
782 
783 int
784 cpuset_isset(struct cpuset *cs, struct cpu_info *ci)
785 {
786 	unsigned int num = CPU_INFO_UNIT(ci);
787 	return (cs->cs_set[num/32] & (1U << (num % 32)));
788 }
789 
790 void
791 cpuset_add_all(struct cpuset *cs)
792 {
793 	cpuset_copy(cs, &cpuset_all);
794 }
795 
796 void
797 cpuset_copy(struct cpuset *to, struct cpuset *from)
798 {
799 	memcpy(to, from, sizeof(*to));
800 }
801 
802 struct cpu_info *
803 cpuset_first(struct cpuset *cs)
804 {
805 	int i;
806 
807 	for (i = 0; i < CPUSET_ASIZE(ncpus); i++)
808 		if (cs->cs_set[i])
809 			return (cpuset_infos[i * 32 + ffs(cs->cs_set[i]) - 1]);
810 
811 	return (NULL);
812 }
813 
814 void
815 cpuset_union(struct cpuset *to, struct cpuset *a, struct cpuset *b)
816 {
817 	int i;
818 
819 	for (i = 0; i < CPUSET_ASIZE(ncpus); i++)
820 		to->cs_set[i] = a->cs_set[i] | b->cs_set[i];
821 }
822 
823 void
824 cpuset_intersection(struct cpuset *to, struct cpuset *a, struct cpuset *b)
825 {
826 	int i;
827 
828 	for (i = 0; i < CPUSET_ASIZE(ncpus); i++)
829 		to->cs_set[i] = a->cs_set[i] & b->cs_set[i];
830 }
831 
832 void
833 cpuset_complement(struct cpuset *to, struct cpuset *a, struct cpuset *b)
834 {
835 	int i;
836 
837 	for (i = 0; i < CPUSET_ASIZE(ncpus); i++)
838 		to->cs_set[i] = b->cs_set[i] & ~a->cs_set[i];
839 }
840 
841 int
842 cpuset_cardinality(struct cpuset *cs)
843 {
844 	int cardinality, i, n;
845 
846 	cardinality = 0;
847 
848 	for (i = 0; i < CPUSET_ASIZE(ncpus); i++)
849 		for (n = cs->cs_set[i]; n != 0; n &= n - 1)
850 			cardinality++;
851 
852 	return (cardinality);
853 }
854 
855 int
856 sysctl_hwncpuonline(void)
857 {
858 	return cpuset_cardinality(&sched_all_cpus);
859 }
860 
861 int
862 cpu_is_online(struct cpu_info *ci)
863 {
864 	return cpuset_isset(&sched_all_cpus, ci);
865 }
866 
867 #ifdef __HAVE_CPU_TOPOLOGY
868 
869 #include <sys/sysctl.h>
870 
871 int
872 sysctl_hwsmt(void *oldp, size_t *oldlenp, void *newp, size_t newlen)
873 {
874 	CPU_INFO_ITERATOR cii;
875 	struct cpu_info *ci;
876 	int err, newsmt;
877 
878 	newsmt = sched_smt;
879 	err = sysctl_int_bounded(oldp, oldlenp, newp, newlen, &newsmt, 0, 1);
880 	if (err)
881 		return err;
882 	if (newsmt == sched_smt)
883 		return 0;
884 
885 	sched_smt = newsmt;
886 	CPU_INFO_FOREACH(cii, ci) {
887 		if (CPU_IS_PRIMARY(ci) || !CPU_IS_RUNNING(ci))
888 			continue;
889 		if (ci->ci_smt_id == 0)
890 			continue;
891 		if (sched_smt)
892 			cpuset_add(&sched_all_cpus, ci);
893 		else
894 			cpuset_del(&sched_all_cpus, ci);
895 	}
896 
897 	return 0;
898 }
899 
900 #endif
901