xref: /netbsd-src/sys/rump/librump/rumpkern/rump.c (revision 413d532bcc3f62d122e56d92e13ac64825a40baf)
1 /*	$NetBSD: rump.c,v 1.304 2014/04/27 15:08:52 pooka Exp $	*/
2 
3 /*
4  * Copyright (c) 2007-2011 Antti Kantee.  All Rights Reserved.
5  *
6  * Redistribution and use in source and binary forms, with or without
7  * modification, are permitted provided that the following conditions
8  * are met:
9  * 1. Redistributions of source code must retain the above copyright
10  *    notice, this list of conditions and the following disclaimer.
11  * 2. Redistributions in binary form must reproduce the above copyright
12  *    notice, this list of conditions and the following disclaimer in the
13  *    documentation and/or other materials provided with the distribution.
14  *
15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
16  * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
17  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
18  * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
21  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25  * SUCH DAMAGE.
26  */
27 
28 #include <sys/cdefs.h>
29 __KERNEL_RCSID(0, "$NetBSD: rump.c,v 1.304 2014/04/27 15:08:52 pooka Exp $");
30 
31 #include <sys/systm.h>
32 #define ELFSIZE ARCH_ELFSIZE
33 
34 #include <sys/param.h>
35 #include <sys/atomic.h>
36 #include <sys/buf.h>
37 #include <sys/callout.h>
38 #include <sys/conf.h>
39 #include <sys/cpu.h>
40 #include <sys/device.h>
41 #include <sys/evcnt.h>
42 #include <sys/event.h>
43 #include <sys/exec_elf.h>
44 #include <sys/filedesc.h>
45 #include <sys/iostat.h>
46 #include <sys/kauth.h>
47 #include <sys/kcpuset.h>
48 #include <sys/kernel.h>
49 #include <sys/kmem.h>
50 #include <sys/kprintf.h>
51 #include <sys/kthread.h>
52 #include <sys/ksyms.h>
53 #include <sys/msgbuf.h>
54 #include <sys/module.h>
55 #include <sys/namei.h>
56 #include <sys/once.h>
57 #include <sys/percpu.h>
58 #include <sys/pipe.h>
59 #include <sys/pool.h>
60 #include <sys/pserialize.h>
61 #include <sys/queue.h>
62 #include <sys/reboot.h>
63 #include <sys/resourcevar.h>
64 #include <sys/select.h>
65 #include <sys/sysctl.h>
66 #include <sys/syscall.h>
67 #include <sys/syscallvar.h>
68 #include <sys/timetc.h>
69 #include <sys/tty.h>
70 #include <sys/uidinfo.h>
71 #include <sys/vmem.h>
72 #include <sys/xcall.h>
73 #include <sys/cprng.h>
74 #include <sys/ktrace.h>
75 
76 #include <rump/rumpuser.h>
77 
78 #include <secmodel/suser/suser.h>
79 
80 #include <prop/proplib.h>
81 
82 #include <uvm/uvm_extern.h>
83 #include <uvm/uvm_readahead.h>
84 
85 #include "rump_private.h"
86 #include "rump_net_private.h"
87 #include "rump_vfs_private.h"
88 #include "rump_dev_private.h"
89 
90 char machine[] = MACHINE;
91 
92 struct proc *initproc;
93 
94 struct device rump_rootdev = {
95 	.dv_class = DV_VIRTUAL
96 };
97 
98 #ifdef RUMP_WITHOUT_THREADS
99 int rump_threads = 0;
100 #else
101 int rump_threads = 1;
102 #endif
103 
104 static int rump_hyp_syscall(int, void *, long *);
105 static int rump_hyp_rfork(void *, int, const char *);
106 static void rump_hyp_lwpexit(void);
107 static void rump_hyp_execnotify(const char *);
108 
109 static void rump_component_addlocal(void);
110 static struct lwp *bootlwp;
111 
112 static char rump_msgbuf[16*1024]; /* 16k should be enough for std rump needs */
113 
114 bool rump_ttycomponent = false;
115 
116 static void
117 rump_aiodone_worker(struct work *wk, void *dummy)
118 {
119 	struct buf *bp = (struct buf *)wk;
120 
121 	KASSERT(&bp->b_work == wk);
122 	bp->b_iodone(bp);
123 }
124 
125 static int rump_inited;
126 
127 void (*rump_vfs_drainbufs)(int) = (void *)nullop;
128 int  (*rump_vfs_makeonedevnode)(dev_t, const char *,
129 				devmajor_t, devminor_t) = (void *)nullop;
130 int  (*rump_vfs_makedevnodes)(dev_t, const char *, char,
131 			      devmajor_t, devminor_t, int) = (void *)nullop;
132 
133 int rump__unavailable(void);
134 int rump__unavailable() {return EOPNOTSUPP;}
135 
136 __weak_alias(biodone,rump__unavailable);
137 
138 rump_proc_vfs_init_fn rump_proc_vfs_init = (void *)nullop;
139 rump_proc_vfs_release_fn rump_proc_vfs_release = (void *)nullop;
140 
141 static void add_linkedin_modules(const struct modinfo *const *, size_t);
142 
143 /*
144  * Create some sysctl nodes.  why only this you ask.  well, init_sysctl
145  * is a kitchen sink in need of some gardening.  but i want to use
146  * others today.  Furthermore, creating a whole kitchen sink full of
147  * sysctl nodes is a waste of cycles for rump kernel bootstrap.
148  */
149 static void
150 mksysctls(void)
151 {
152 
153 	/* hw.pagesize */
154 	sysctl_createv(NULL, 0, NULL, NULL,
155 	    CTLFLAG_PERMANENT|CTLFLAG_IMMEDIATE,
156 	    CTLTYPE_INT, "pagesize",
157 	    SYSCTL_DESCR("Software page size"),
158 	    NULL, PAGE_SIZE, NULL, 0,
159 	    CTL_HW, HW_PAGESIZE, CTL_EOL);
160 }
161 
162 /* there's no convenient kernel entry point for this, so just craft out own */
163 static pid_t
164 spgetpid(void)
165 {
166 
167 	return curproc->p_pid;
168 }
169 
170 static const struct rumpuser_hyperup hyp = {
171 	.hyp_schedule		= rump_schedule,
172 	.hyp_unschedule		= rump_unschedule,
173 	.hyp_backend_unschedule	= rump_user_unschedule,
174 	.hyp_backend_schedule	= rump_user_schedule,
175 	.hyp_lwproc_switch	= rump_lwproc_switch,
176 	.hyp_lwproc_release	= rump_lwproc_releaselwp,
177 	.hyp_lwproc_rfork	= rump_hyp_rfork,
178 	.hyp_lwproc_newlwp	= rump_lwproc_newlwp,
179 	.hyp_lwproc_curlwp	= rump_lwproc_curlwp,
180 	.hyp_lwpexit		= rump_hyp_lwpexit,
181 	.hyp_syscall		= rump_hyp_syscall,
182 	.hyp_execnotify		= rump_hyp_execnotify,
183 	.hyp_getpid		= spgetpid,
184 };
185 
186 int
187 rump_daemonize_begin(void)
188 {
189 
190 	if (rump_inited)
191 		return EALREADY;
192 
193 	return rumpuser_daemonize_begin();
194 }
195 
196 int
197 rump_daemonize_done(int error)
198 {
199 
200 	return rumpuser_daemonize_done(error);
201 }
202 
203 #ifndef RUMP_USE_CTOR
204 RUMP_COMPONENT(RUMP_COMPONENT_POSTINIT)
205 {
206 	__link_set_decl(rump_components, struct rump_component);
207 
208 	/*
209 	 * Trick compiler into generating references so that statically
210 	 * linked rump kernels are generated with the link set symbols.
211 	 */
212 	asm("" :: "r"(__start_link_set_rump_components));
213 	asm("" :: "r"(__stop_link_set_rump_components));
214 }
215 #endif
216 
217 int
218 rump_init(void)
219 {
220 	char buf[256];
221 	struct timespec ts;
222 	int64_t sec;
223 	long nsec;
224 	struct lwp *l, *initlwp;
225 	int i, numcpu;
226 
227 	/* not reentrant */
228 	if (rump_inited)
229 		return 0;
230 	else if (rump_inited == -1)
231 		panic("rump_init: host process restart required");
232 	else
233 		rump_inited = 1;
234 
235 	/* initialize hypervisor */
236 	if (rumpuser_init(RUMPUSER_VERSION, &hyp) != 0) {
237 		rumpuser_dprintf("rumpuser init failed\n");
238 		return EINVAL;
239 	}
240 
241 	/* init minimal lwp/cpu context */
242 	rump_lwproc_init();
243 	l = &lwp0;
244 	l->l_cpu = l->l_target_cpu = rump_cpu;
245 	rump_lwproc_curlwp_set(l);
246 
247 	/* retrieve env vars which affect the early stage of bootstrap */
248 	if (rumpuser_getparam("RUMP_THREADS", buf, sizeof(buf)) == 0) {
249 		rump_threads = *buf != '0';
250 	}
251 	if (rumpuser_getparam("RUMP_VERBOSE", buf, sizeof(buf)) == 0) {
252 		if (*buf != '0')
253 			boothowto = AB_VERBOSE;
254 	}
255 
256 	if (rumpuser_getparam(RUMPUSER_PARAM_NCPU, buf, sizeof(buf)) != 0)
257 		panic("mandatory hypervisor configuration (NCPU) missing");
258 	numcpu = strtoll(buf, NULL, 10);
259 	if (numcpu < 1) {
260 		panic("rump kernels are not lightweight enough for \"%d\" CPUs",
261 		    numcpu);
262 	}
263 
264 	rump_thread_init();
265 	rump_cpus_bootstrap(&numcpu);
266 
267 	rumpuser_clock_gettime(RUMPUSER_CLOCK_RELWALL, &sec, &nsec);
268 	boottime.tv_sec = sec;
269 	boottime.tv_nsec = nsec;
270 
271 	initmsgbuf(rump_msgbuf, sizeof(rump_msgbuf));
272 	aprint_verbose("%s%s", copyright, version);
273 
274 	rump_intr_init(numcpu);
275 
276 	rump_tsleep_init();
277 
278 	rumpuser_mutex_init(&rump_giantlock, RUMPUSER_MTX_SPIN);
279 	ksyms_init();
280 	uvm_init();
281 	evcnt_init();
282 
283 	kcpuset_sysinit();
284 	once_init();
285 	kernconfig_lock_init();
286 	prop_kern_init();
287 
288 	kmem_init();
289 	kmeminit();
290 
291 	uvm_ra_init();
292 	uao_init();
293 
294 	mutex_obj_init();
295 	callout_startup();
296 
297 	kprintf_init();
298 	pserialize_init();
299 	loginit();
300 
301 	kauth_init();
302 
303 	secmodel_init();
304 	sysctl_init();
305 
306 	rnd_init();
307 	cprng_init();
308 	kern_cprng = cprng_strong_create("kernel", IPL_VM,
309 	    CPRNG_INIT_ANY|CPRNG_REKEY_ANY);
310 	rump_hyperentropy_init();
311 
312 	procinit();
313 	proc0_init();
314 	uid_init();
315 	chgproccnt(0, 1);
316 
317 	l->l_proc = &proc0;
318 	lwp_update_creds(l);
319 
320 	lwpinit_specificdata();
321 	lwp_initspecific(&lwp0);
322 
323 	rump_biglock_init();
324 
325 	rump_scheduler_init(numcpu);
326 	/* revert temporary context and schedule a semireal context */
327 	rump_lwproc_curlwp_clear(l);
328 	initproc = &proc0; /* borrow proc0 before we get initproc started */
329 	rump_schedule();
330 	bootlwp = curlwp;
331 
332 	percpu_init();
333 	inittimecounter();
334 	ntp_init();
335 
336 #ifdef KTRACE
337 	ktrinit();
338 #endif
339 
340 	ts = boottime;
341 	tc_setclock(&ts);
342 
343 	extern krwlock_t exec_lock;
344 	rw_init(&exec_lock);
345 
346 	/* we are mostly go.  do per-cpu subsystem init */
347 	for (i = 0; i < numcpu; i++) {
348 		struct cpu_info *ci = cpu_lookup(i);
349 
350 		/* attach non-bootstrap CPUs */
351 		if (i > 0) {
352 			rump_cpu_attach(ci);
353 			ncpu++;
354 		}
355 
356 		callout_init_cpu(ci);
357 		softint_init(ci);
358 		xc_init_cpu(ci);
359 		pool_cache_cpu_init(ci);
360 		selsysinit(ci);
361 		percpu_init_cpu(ci);
362 
363 		TAILQ_INIT(&ci->ci_data.cpu_ld_locks);
364 		__cpu_simple_lock_init(&ci->ci_data.cpu_ld_lock);
365 
366 		aprint_verbose("cpu%d at thinair0: rump virtual cpu\n", i);
367 	}
368 
369 	/* CPUs are up.  allow kernel threads to run */
370 	rump_thread_allow(NULL);
371 
372 	rnd_init_softint();
373 
374 	mksysctls();
375 	kqueue_init();
376 	iostat_init();
377 	fd_sys_init();
378 	module_init();
379 	devsw_init();
380 	pipe_init();
381 	resource_init();
382 	procinit_sysctl();
383 	time_init();
384 	time_init2();
385 
386 	/* start page baroness */
387 	if (rump_threads) {
388 		if (kthread_create(PRI_PGDAEMON, KTHREAD_MPSAFE, NULL,
389 		    uvm_pageout, NULL, &uvm.pagedaemon_lwp, "pdaemon") != 0)
390 			panic("pagedaemon create failed");
391 	} else
392 		uvm.pagedaemon_lwp = NULL; /* doesn't match curlwp */
393 
394 	/* process dso's */
395 	rumpuser_dl_bootstrap(add_linkedin_modules,
396 	    rump_kernelfsym_load, rump_component_load);
397 
398 	rump_component_addlocal();
399 	rump_component_init(RUMP_COMPONENT_KERN);
400 
401 	/* initialize factions, if present */
402 	rump_component_init(RUMP__FACTION_VFS);
403 	/* pnbuf_cache is used even without vfs */
404 	if (rump_component_count(RUMP__FACTION_VFS) == 0) {
405 		pnbuf_cache = pool_cache_init(MAXPATHLEN, 0, 0, 0, "pnbufpl",
406 		    NULL, IPL_NONE, NULL, NULL, NULL);
407 	}
408 	rump_component_init(RUMP__FACTION_NET);
409 	rump_component_init(RUMP__FACTION_DEV);
410 	KASSERT(rump_component_count(RUMP__FACTION_VFS) <= 1
411 	    && rump_component_count(RUMP__FACTION_NET) <= 1
412 	    && rump_component_count(RUMP__FACTION_DEV) <= 1);
413 
414 	rump_component_init(RUMP_COMPONENT_KERN_VFS);
415 
416 	/*
417 	 * if we initialized the tty component above, the tyttymtx is
418 	 * now initialized.  otherwise, we need to initialize it.
419 	 */
420 	if (!rump_ttycomponent)
421 		mutex_init(&tty_lock, MUTEX_DEFAULT, IPL_VM);
422 
423 	cold = 0;
424 
425 	/* aieeeedondest */
426 	if (rump_threads) {
427 		if (workqueue_create(&uvm.aiodone_queue, "aiodoned",
428 		    rump_aiodone_worker, NULL, 0, 0, WQ_MPSAFE))
429 			panic("aiodoned");
430 	}
431 
432 	sysctl_finalize();
433 
434 	module_init_class(MODULE_CLASS_ANY);
435 
436 	if (rumpuser_getparam(RUMPUSER_PARAM_HOSTNAME,
437 	    hostname, MAXHOSTNAMELEN) != 0) {
438 		panic("mandatory hypervisor configuration (HOSTNAME) missing");
439 	}
440 	hostnamelen = strlen(hostname);
441 
442 	sigemptyset(&sigcantmask);
443 
444 	if (rump_threads)
445 		vmem_rehash_start();
446 
447 	/*
448 	 * Create init (proc 1), used to attach implicit threads in rump.
449 	 * (note: must be done after vfsinit to get cwdi)
450 	 */
451 	initlwp = rump__lwproc_alloclwp(NULL);
452 	mutex_enter(proc_lock);
453 	initproc = proc_find_raw(1);
454 	mutex_exit(proc_lock);
455 	if (initproc == NULL)
456 		panic("where in the world is initproc?");
457 
458 	rump_component_init(RUMP_COMPONENT_POSTINIT);
459 
460 	/* load syscalls */
461 	rump_component_init(RUMP_COMPONENT_SYSCALL);
462 
463 	/* component inits done */
464 	bootlwp = NULL;
465 
466 	/* open 0/1/2 for init */
467 	KASSERT(rump_lwproc_curlwp() == NULL);
468 	rump_lwproc_switch(initlwp);
469 	rump_consdev_init();
470 	rump_lwproc_switch(NULL);
471 
472 	/* release cpu */
473 	rump_unschedule();
474 
475 	return 0;
476 }
477 /* historic compat */
478 __strong_alias(rump__init,rump_init);
479 
480 int
481 rump_init_server(const char *url)
482 {
483 
484 	return rumpuser_sp_init(url, ostype, osrelease, MACHINE);
485 }
486 
487 static int compcounter[RUMP_COMPONENT_MAX];
488 static int compinited[RUMP_COMPONENT_MAX];
489 
490 /*
491  * Yea, this is O(n^2), but we're only looking at a handful of components.
492  * Components are always initialized from the thread that called rump_init().
493  */
494 static LIST_HEAD(, rump_component) rchead = LIST_HEAD_INITIALIZER(rchead);
495 
496 #ifdef RUMP_USE_CTOR
497 struct modinfo_boot_chain modinfo_boot_chain \
498     = LIST_HEAD_INITIALIZER(modinfo_boot_chain);
499 
500 static void
501 rump_component_addlocal(void)
502 {
503 	struct modinfo_chain *mc;
504 
505 	while ((mc = LIST_FIRST(&modinfo_boot_chain)) != NULL) {
506 		LIST_REMOVE(mc, mc_entries);
507 		module_builtin_add(&mc->mc_info, 1, false);
508 	}
509 }
510 
511 #else /* RUMP_USE_CTOR */
512 
513 static void
514 rump_component_addlocal(void)
515 {
516 	__link_set_decl(rump_components, struct rump_component);
517 	struct rump_component *const *rc;
518 
519 	__link_set_foreach(rc, rump_components) {
520 		rump_component_load(*rc);
521 	}
522 }
523 #endif /* RUMP_USE_CTOR */
524 
525 void
526 rump_component_load(const struct rump_component *rc_const)
527 {
528 	struct rump_component *rc, *rc_iter;
529 
530 	/*
531 	 * XXX: this is ok since the "const" was removed from the
532 	 * definition of RUMP_COMPONENT().
533 	 *
534 	 * However, to preserve the hypercall interface, the const
535 	 * remains here.  This can be fixed in the next hypercall revision.
536 	 */
537 	rc = __UNCONST(rc_const);
538 
539 	KASSERT(curlwp == bootlwp);
540 
541 	LIST_FOREACH(rc_iter, &rchead, rc_entries) {
542 		if (rc_iter == rc)
543 			return;
544 	}
545 
546 	LIST_INSERT_HEAD(&rchead, rc, rc_entries);
547 	KASSERT(rc->rc_type < RUMP_COMPONENT_MAX);
548 	compcounter[rc->rc_type]++;
549 }
550 
551 int
552 rump_component_count(enum rump_component_type type)
553 {
554 
555 	KASSERT(curlwp == bootlwp);
556 	KASSERT(type < RUMP_COMPONENT_MAX);
557 	return compcounter[type];
558 }
559 
560 void
561 rump_component_init(enum rump_component_type type)
562 {
563 	const struct rump_component *rc, *rc_safe;
564 
565 	KASSERT(curlwp == bootlwp);
566 	KASSERT(!compinited[type]);
567 	LIST_FOREACH_SAFE(rc, &rchead, rc_entries, rc_safe) {
568 		if (rc->rc_type == type) {
569 			rc->rc_init();
570 			LIST_REMOVE(rc, rc_entries);
571 		}
572 	}
573 	compinited[type] = 1;
574 }
575 
576 /*
577  * Initialize a module which has already been loaded and linked
578  * with dlopen(). This is fundamentally the same as a builtin module.
579  *
580  * XXX: this interface does not really work in the RUMP_USE_CTOR case,
581  * but I'm not sure it's anything to cry about.  In feeling blue,
582  * things could somehow be handled via modinfo_boot_chain.
583  */
584 int
585 rump_module_init(const struct modinfo * const *mip, size_t nmodinfo)
586 {
587 
588 	return module_builtin_add(mip, nmodinfo, true);
589 }
590 
591 /*
592  * Finish module (flawless victory, fatality!).
593  */
594 int
595 rump_module_fini(const struct modinfo *mi)
596 {
597 
598 	return module_builtin_remove(mi, true);
599 }
600 
601 /*
602  * Add loaded and linked module to the builtin list.  It will
603  * later be initialized with module_init_class().
604  */
605 
606 static void
607 add_linkedin_modules(const struct modinfo * const *mip, size_t nmodinfo)
608 {
609 
610 	module_builtin_add(mip, nmodinfo, false);
611 }
612 
613 int
614 rump_kernelfsym_load(void *symtab, uint64_t symsize,
615 	char *strtab, uint64_t strsize)
616 {
617 	static int inited = 0;
618 	Elf64_Ehdr ehdr;
619 
620 	if (inited)
621 		return EBUSY;
622 	inited = 1;
623 
624 	/*
625 	 * Use 64bit header since it's bigger.  Shouldn't make a
626 	 * difference, since we're passing in all zeroes anyway.
627 	 */
628 	memset(&ehdr, 0, sizeof(ehdr));
629 	ksyms_addsyms_explicit(&ehdr, symtab, symsize, strtab, strsize);
630 
631 	return 0;
632 }
633 
634 static int
635 rump_hyp_syscall(int num, void *arg, long *retval)
636 {
637 	register_t regrv[2] = {0, 0};
638 	struct lwp *l;
639 	struct sysent *callp;
640 	int rv;
641 
642 	if (__predict_false(num >= SYS_NSYSENT))
643 		return ENOSYS;
644 
645 	/* XXX: always uses native syscall vector */
646 	callp = rump_sysent + num;
647 	l = curlwp;
648 	rv = sy_invoke(callp, l, (void *)arg, regrv, num);
649 	retval[0] = regrv[0];
650 	retval[1] = regrv[1];
651 
652 	return rv;
653 }
654 
655 static int
656 rump_hyp_rfork(void *priv, int flags, const char *comm)
657 {
658 	struct vmspace *newspace;
659 	struct proc *p;
660 	int error;
661 
662 	if ((error = rump_lwproc_rfork(flags)) != 0)
663 		return error;
664 
665 	/*
666 	 * Since it's a proxy proc, adjust the vmspace.
667 	 * Refcount will eternally be 1.
668 	 */
669 	p = curproc;
670 	newspace = kmem_zalloc(sizeof(*newspace), KM_SLEEP);
671 	newspace->vm_refcnt = 1;
672 	newspace->vm_map.pmap = priv;
673 	KASSERT(p->p_vmspace == vmspace_kernel());
674 	p->p_vmspace = newspace;
675 	if (comm)
676 		strlcpy(p->p_comm, comm, sizeof(p->p_comm));
677 
678 	return 0;
679 }
680 
681 /*
682  * Order all lwps in a process to exit.  does *not* wait for them to drain.
683  */
684 static void
685 rump_hyp_lwpexit(void)
686 {
687 	struct proc *p = curproc;
688 	uint64_t where;
689 	struct lwp *l;
690 
691 	mutex_enter(p->p_lock);
692 	/*
693 	 * First pass: mark all lwps in the process with LW_RUMP_QEXIT
694 	 * so that they know they should exit.
695 	 */
696 	LIST_FOREACH(l, &p->p_lwps, l_sibling) {
697 		if (l == curlwp)
698 			continue;
699 		l->l_flag |= LW_RUMP_QEXIT;
700 	}
701 	mutex_exit(p->p_lock);
702 
703 	/*
704 	 * Next, make sure everyone on all CPUs sees our status
705 	 * update.  This keeps threads inside cv_wait() and makes
706 	 * sure we don't access a stale cv pointer later when
707 	 * we wake up the threads.
708 	 */
709 
710 	where = xc_broadcast(0, (xcfunc_t)nullop, NULL, NULL);
711 	xc_wait(where);
712 
713 	/*
714 	 * Ok, all lwps are either:
715 	 *  1) not in the cv code
716 	 *  2) sleeping on l->l_private
717 	 *  3) sleeping on p->p_waitcv
718 	 *
719 	 * Either way, l_private is stable until we set PS_RUMP_LWPEXIT
720 	 * in p->p_sflag.
721 	 */
722 
723 	mutex_enter(p->p_lock);
724 	LIST_FOREACH(l, &p->p_lwps, l_sibling) {
725 		if (l->l_private)
726 			cv_broadcast(l->l_private);
727 	}
728 	p->p_sflag |= PS_RUMP_LWPEXIT;
729 	cv_broadcast(&p->p_waitcv);
730 	mutex_exit(p->p_lock);
731 }
732 
733 /*
734  * Notify process that all threads have been drained and exec is complete.
735  */
736 static void
737 rump_hyp_execnotify(const char *comm)
738 {
739 	struct proc *p = curproc;
740 
741 	fd_closeexec();
742 	mutex_enter(p->p_lock);
743 	KASSERT(p->p_nlwps == 1 && p->p_sflag & PS_RUMP_LWPEXIT);
744 	p->p_sflag &= ~PS_RUMP_LWPEXIT;
745 	mutex_exit(p->p_lock);
746 	strlcpy(p->p_comm, comm, sizeof(p->p_comm));
747 }
748 
749 int
750 rump_boot_gethowto()
751 {
752 
753 	return boothowto;
754 }
755 
756 void
757 rump_boot_sethowto(int howto)
758 {
759 
760 	boothowto = howto;
761 }
762 
763 int
764 rump_getversion(void)
765 {
766 
767 	return __NetBSD_Version__;
768 }
769 /* compat */
770 __strong_alias(rump_pub_getversion,rump_getversion);
771 
772 int
773 rump_nativeabi_p(void)
774 {
775 
776 #ifdef _RUMP_NATIVE_ABI
777 	return 1;
778 #else
779 	return 0;
780 #endif
781 }
782 
783 /*
784  * Note: may be called unscheduled.  Not fully safe since no locking
785  * of allevents (currently that's not even available).
786  */
787 void
788 rump_printevcnts()
789 {
790 	struct evcnt *ev;
791 
792 	TAILQ_FOREACH(ev, &allevents, ev_list)
793 		rumpuser_dprintf("%s / %s: %" PRIu64 "\n",
794 		    ev->ev_group, ev->ev_name, ev->ev_count);
795 }
796 
797 /*
798  * If you use this interface ... well ... all bets are off.
799  * The original purpose is for the p2k fs server library to be
800  * able to use the same pid/lid for VOPs as the host kernel.
801  */
802 void
803 rump_allbetsareoff_setid(pid_t pid, int lid)
804 {
805 	struct lwp *l = curlwp;
806 	struct proc *p = l->l_proc;
807 
808 	l->l_lid = lid;
809 	p->p_pid = pid;
810 }
811 
812 #include <sys/pserialize.h>
813 
814 static void
815 ipiemu(void *a1, void *a2)
816 {
817 
818 	xc__highpri_intr(NULL);
819 	pserialize_switchpoint();
820 }
821 
822 void
823 rump_xc_highpri(struct cpu_info *ci)
824 {
825 
826 	if (ci)
827 		xc_unicast(0, ipiemu, NULL, NULL, ci);
828 	else
829 		xc_broadcast(0, ipiemu, NULL, NULL);
830 }
831 
832 int
833 rump_syscall(int num, void *data, size_t dlen, register_t *retval)
834 {
835 	struct proc *p;
836 	struct emul *e;
837 	struct sysent *callp;
838 	const int *etrans = NULL;
839 	int rv;
840 
841 	rump_schedule();
842 	p = curproc;
843 	e = p->p_emul;
844 #ifndef __HAVE_MINIMAL_EMUL
845 	KASSERT(num > 0 && num < e->e_nsysent);
846 #endif
847 	callp = e->e_sysent + num;
848 
849 	rv = sy_invoke(callp, curlwp, data, retval, num);
850 
851 	/*
852 	 * I hope that (!__HAVE_MINIMAL_EMUL || __HAVE_SYSCALL_INTERN) is
853 	 * an invariant ...
854 	 */
855 #if !defined(__HAVE_MINIMAL_EMUL)
856 	etrans = e->e_errno;
857 #elif defined(__HAVE_SYSCALL_INTERN)
858 	etrans = p->p_emuldata;
859 #endif
860 
861 	if (etrans) {
862 		rv = etrans[rv];
863 		/*
864 		 * XXX: small hack since Linux etrans vectors on some
865 		 * archs contain negative errnos, but rump_syscalls
866 		 * uses the -1 + errno ABI.  Note that these
867 		 * negative values are always the result of translation,
868 		 * otherwise the above translation method would not
869 		 * work very well.
870 		 */
871 		if (rv < 0)
872 			rv = -rv;
873 	}
874 	rump_unschedule();
875 
876 	return rv;
877 }
878 
879 void
880 rump_syscall_boot_establish(const struct rump_onesyscall *calls, size_t ncall)
881 {
882 	struct sysent *callp;
883 	size_t i;
884 
885 	for (i = 0; i < ncall; i++) {
886 		callp = rump_sysent + calls[i].ros_num;
887 		KASSERT(bootlwp != NULL
888 		    && callp->sy_call == (sy_call_t *)enosys);
889 		callp->sy_call = calls[i].ros_handler;
890 	}
891 }
892 
893 /*
894  * Temporary notification that rumpkern_time is obsolete.  This is to
895  * be removed along with obsoleting rumpkern_time in a few months.
896  */
897 #define RUMPKERN_TIME_WARN "rumpkern_time is obsolete, functionality in librump"
898 __warn_references(rumpkern_time_is_obsolete,RUMPKERN_TIME_WARN)
899 void rumpkern_time_is_obsolete(void);
900 void
901 rumpkern_time_is_obsolete(void)
902 {
903 	printf("WARNING: %s\n", RUMPKERN_TIME_WARN);
904 }
905