xref: /netbsd-src/sys/rump/net/lib/libshmif/if_shmem.c (revision deb6f0161a9109e7de9b519dc8dfb9478668dcdd)
1 /*	$NetBSD: if_shmem.c,v 1.75 2018/06/26 06:48:03 msaitoh Exp $	*/
2 
3 /*
4  * Copyright (c) 2009, 2010 Antti Kantee.  All Rights Reserved.
5  *
6  * Development of this software was supported by The Nokia Foundation.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  *
17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
18  * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
19  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
20  * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
23  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27  * SUCH DAMAGE.
28  */
29 
30 #include <sys/cdefs.h>
31 __KERNEL_RCSID(0, "$NetBSD: if_shmem.c,v 1.75 2018/06/26 06:48:03 msaitoh Exp $");
32 
33 #include <sys/param.h>
34 #include <sys/atomic.h>
35 #include <sys/fcntl.h>
36 #include <sys/kmem.h>
37 #include <sys/kthread.h>
38 #include <sys/lock.h>
39 #include <sys/vmem.h>
40 #include <sys/cprng.h>
41 
42 #include <net/bpf.h>
43 #include <net/if.h>
44 #include <net/if_dl.h>
45 #include <net/if_ether.h>
46 
47 #include <netinet/in.h>
48 #include <netinet/in_var.h>
49 
50 #include <rump-sys/kern.h>
51 #include <rump-sys/net.h>
52 
53 #include <rump/rump.h>
54 #include <rump/rumpuser.h>
55 
56 #include "shmif_user.h"
57 
58 static int shmif_clone(struct if_clone *, int);
59 static int shmif_unclone(struct ifnet *);
60 
61 struct if_clone shmif_cloner =
62     IF_CLONE_INITIALIZER("shmif", shmif_clone, shmif_unclone);
63 
64 /*
65  * Do r/w prefault for backend pages when attaching the interface.
66  * At least logically thinking improves performance (although no
67  * mlocking is done, so they might go away).
68  */
69 #define PREFAULT_RW
70 
71 /*
72  * A virtual ethernet interface which uses shared memory from a
73  * memory mapped file as the bus.
74  */
75 
76 static int	shmif_init(struct ifnet *);
77 static int	shmif_ioctl(struct ifnet *, u_long, void *);
78 static void	shmif_start(struct ifnet *);
79 static void	shmif_stop(struct ifnet *, int);
80 
81 #include "shmifvar.h"
82 
83 struct shmif_sc {
84 	struct ethercom sc_ec;
85 	struct shmif_mem *sc_busmem;
86 	int sc_memfd;
87 	int sc_kq;
88 	int sc_unit;
89 
90 	char *sc_backfile;
91 	size_t sc_backfilelen;
92 
93 	uint64_t sc_devgen;
94 	uint32_t sc_nextpacket;
95 
96 	kmutex_t sc_mtx;
97 	kcondvar_t sc_cv;
98 
99 	struct lwp *sc_rcvl;
100 	bool sc_dying;
101 
102 	uint64_t sc_uuid;
103 };
104 
105 static void shmif_rcv(void *);
106 
107 #define LOCK_UNLOCKED	0
108 #define LOCK_LOCKED	1
109 #define LOCK_COOLDOWN	1001
110 
111 vmem_t *shmif_units;
112 
113 static void
114 dowakeup(struct shmif_sc *sc)
115 {
116 	struct rumpuser_iovec iov;
117 	uint32_t ver = SHMIF_VERSION;
118 	size_t n;
119 
120 	iov.iov_base = &ver;
121 	iov.iov_len = sizeof(ver);
122 	rumpuser_iovwrite(sc->sc_memfd, &iov, 1, IFMEM_WAKEUP, &n);
123 }
124 
125 /*
126  * This locking needs work and will misbehave severely if:
127  * 1) the backing memory has to be paged in
128  * 2) some lockholder exits while holding the lock
129  */
130 static void
131 shmif_lockbus(struct shmif_mem *busmem)
132 {
133 	int i = 0;
134 
135 	while (__predict_false(atomic_cas_32(&busmem->shm_lock,
136 	    LOCK_UNLOCKED, LOCK_LOCKED) == LOCK_LOCKED)) {
137 		if (__predict_false(++i > LOCK_COOLDOWN)) {
138 			/* wait 1ms */
139 			rumpuser_clock_sleep(RUMPUSER_CLOCK_RELWALL,
140 			    0, 1000*1000);
141 			i = 0;
142 		}
143 		continue;
144 	}
145 	membar_enter();
146 }
147 
148 static void
149 shmif_unlockbus(struct shmif_mem *busmem)
150 {
151 	unsigned int old __diagused;
152 
153 	membar_exit();
154 	old = atomic_swap_32(&busmem->shm_lock, LOCK_UNLOCKED);
155 	KASSERT(old == LOCK_LOCKED);
156 }
157 
158 static int
159 allocif(int unit, struct shmif_sc **scp)
160 {
161 	uint8_t enaddr[ETHER_ADDR_LEN] = { 0xb2, 0xa0, 0x00, 0x00, 0x00, 0x00 };
162 	struct shmif_sc *sc;
163 	struct ifnet *ifp;
164 	uint32_t randnum;
165 	int error;
166 
167 	randnum = cprng_fast32();
168 	memcpy(&enaddr[2], &randnum, sizeof(randnum));
169 
170 	sc = kmem_zalloc(sizeof(*sc), KM_SLEEP);
171 	sc->sc_memfd = -1;
172 	sc->sc_unit = unit;
173 	sc->sc_uuid = cprng_fast64();
174 
175 	ifp = &sc->sc_ec.ec_if;
176 
177 	snprintf(ifp->if_xname, sizeof(ifp->if_xname), "shmif%d", unit);
178 	ifp->if_softc = sc;
179 	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
180 	ifp->if_init = shmif_init;
181 	ifp->if_ioctl = shmif_ioctl;
182 	ifp->if_start = shmif_start;
183 	ifp->if_stop = shmif_stop;
184 	ifp->if_mtu = ETHERMTU;
185 	ifp->if_dlt = DLT_EN10MB;
186 
187 	mutex_init(&sc->sc_mtx, MUTEX_DEFAULT, IPL_NONE);
188 	cv_init(&sc->sc_cv, "shmifcv");
189 
190 	error = if_initialize(ifp);
191 	if (error != 0) {
192 		aprint_error("shmif%d: if_initialize failed(%d)\n", unit,
193 		    error);
194 		cv_destroy(&sc->sc_cv);
195 		mutex_destroy(&sc->sc_mtx);
196 		kmem_free(sc, sizeof(*sc));
197 
198 		return error;
199 	}
200 	ether_ifattach(ifp, enaddr);
201 	if_register(ifp);
202 
203 	aprint_verbose("shmif%d: Ethernet address %s\n",
204 	    unit, ether_sprintf(enaddr));
205 
206 	if (scp)
207 		*scp = sc;
208 
209 	error = 0;
210 	if (rump_threads) {
211 		error = kthread_create(PRI_NONE,
212 		    KTHREAD_MPSAFE | KTHREAD_MUSTJOIN, NULL,
213 		    shmif_rcv, ifp, &sc->sc_rcvl, "shmif");
214 	} else {
215 		printf("WARNING: threads not enabled, shmif NOT working\n");
216 	}
217 
218 	if (error) {
219 		shmif_unclone(ifp);
220 	}
221 
222 	return error;
223 }
224 
225 static int
226 initbackend(struct shmif_sc *sc, int memfd)
227 {
228 	volatile uint8_t v;
229 	volatile uint8_t *p;
230 	void *mem;
231 	int error;
232 
233 	error = rumpcomp_shmif_mmap(memfd, BUSMEM_SIZE, &mem);
234 	if (error)
235 		return error;
236 	sc->sc_busmem = mem;
237 
238 	if (sc->sc_busmem->shm_magic
239 	    && sc->sc_busmem->shm_magic != SHMIF_MAGIC) {
240 		printf("bus is not magical");
241 		rumpuser_unmap(sc->sc_busmem, BUSMEM_SIZE);
242 		return ENOEXEC;
243 	}
244 
245 	/*
246 	 * Prefault in pages to minimize runtime penalty with buslock.
247 	 * Use 512 instead of PAGE_SIZE to make sure we catch cases where
248 	 * rump kernel PAGE_SIZE > host page size.
249 	 */
250 	for (p = (uint8_t *)sc->sc_busmem;
251 	    p < (uint8_t *)sc->sc_busmem + BUSMEM_SIZE;
252 	    p += 512)
253 		v = *p;
254 
255 	shmif_lockbus(sc->sc_busmem);
256 	/* we're first?  initialize bus */
257 	if (sc->sc_busmem->shm_magic == 0) {
258 		sc->sc_busmem->shm_magic = SHMIF_MAGIC;
259 		sc->sc_busmem->shm_first = BUSMEM_DATASIZE;
260 	}
261 
262 	sc->sc_nextpacket = sc->sc_busmem->shm_last;
263 	sc->sc_devgen = sc->sc_busmem->shm_gen;
264 
265 #ifdef PREFAULT_RW
266 	for (p = (uint8_t *)sc->sc_busmem;
267 	    p < (uint8_t *)sc->sc_busmem + BUSMEM_SIZE;
268 	    p += PAGE_SIZE) {
269 		v = *p;
270 		*p = v;
271 	}
272 #endif
273 	shmif_unlockbus(sc->sc_busmem);
274 
275 	sc->sc_kq = -1;
276 	error = rumpcomp_shmif_watchsetup(&sc->sc_kq, memfd);
277 	if (error) {
278 		rumpuser_unmap(sc->sc_busmem, BUSMEM_SIZE);
279 		return error;
280 	}
281 
282 	sc->sc_memfd = memfd;
283 
284 	return error;
285 }
286 
287 static void
288 finibackend(struct shmif_sc *sc)
289 {
290 
291 	if (sc->sc_backfile == NULL)
292 		return;
293 
294 	if (sc->sc_backfile) {
295 		kmem_free(sc->sc_backfile, sc->sc_backfilelen);
296 		sc->sc_backfile = NULL;
297 		sc->sc_backfilelen = 0;
298 	}
299 
300 	rumpuser_unmap(sc->sc_busmem, BUSMEM_SIZE);
301 	rumpuser_close(sc->sc_memfd);
302 	rumpuser_close(sc->sc_kq);
303 
304 	sc->sc_memfd = -1;
305 }
306 
307 int
308 rump_shmif_create(const char *path, int *ifnum)
309 {
310 	struct shmif_sc *sc;
311 	vmem_addr_t t;
312 	int unit, error;
313 	int memfd = -1; /* XXXgcc */
314 
315 	if (path) {
316 		error = rumpuser_open(path,
317 		    RUMPUSER_OPEN_RDWR | RUMPUSER_OPEN_CREATE, &memfd);
318 		if (error)
319 			return error;
320 	}
321 
322 	error = vmem_xalloc(shmif_units, 1, 0, 0, 0,
323 	    VMEM_ADDR_MIN, VMEM_ADDR_MAX, VM_INSTANTFIT | VM_SLEEP, &t);
324 
325 	if (error != 0) {
326 		if (path)
327 			rumpuser_close(memfd);
328 		return error;
329 	}
330 
331 	unit = t - 1;
332 
333 	if ((error = allocif(unit, &sc)) != 0) {
334 		if (path)
335 			rumpuser_close(memfd);
336 		return error;
337 	}
338 
339 	if (!path)
340 		goto out;
341 
342 	error = initbackend(sc, memfd);
343 	if (error) {
344 		shmif_unclone(&sc->sc_ec.ec_if);
345 		return error;
346 	}
347 
348 	sc->sc_backfilelen = strlen(path)+1;
349 	sc->sc_backfile = kmem_alloc(sc->sc_backfilelen, KM_SLEEP);
350 	strcpy(sc->sc_backfile, path);
351 
352  out:
353 	if (ifnum)
354 		*ifnum = unit;
355 
356 	return 0;
357 }
358 
359 static int
360 shmif_clone(struct if_clone *ifc, int unit)
361 {
362 	int rc __diagused;
363 	vmem_addr_t unit2;
364 
365 	/*
366 	 * Ok, we know the unit number, but we must still reserve it.
367 	 * Otherwise the wildcard-side of things might get the same one.
368 	 * This is slightly offset-happy due to vmem.  First, we offset
369 	 * the range of unit numbers by +1 since vmem cannot deal with
370 	 * ranges starting from 0.  Talk about uuuh.
371 	 */
372 	rc = vmem_xalloc(shmif_units, 1, 0, 0, 0, unit+1, unit+1,
373 	    VM_SLEEP | VM_INSTANTFIT, &unit2);
374 	KASSERT(rc == 0 && unit2-1 == unit);
375 
376 	return allocif(unit, NULL);
377 }
378 
379 static int
380 shmif_unclone(struct ifnet *ifp)
381 {
382 	struct shmif_sc *sc = ifp->if_softc;
383 
384 	shmif_stop(ifp, 1);
385 	if_down(ifp);
386 
387 	mutex_enter(&sc->sc_mtx);
388 	sc->sc_dying = true;
389 	cv_broadcast(&sc->sc_cv);
390 	mutex_exit(&sc->sc_mtx);
391 
392 	if (sc->sc_rcvl)
393 		kthread_join(sc->sc_rcvl);
394 	sc->sc_rcvl = NULL;
395 
396 	/*
397 	 * Need to be called after the kthread left, otherwise closing kqueue
398 	 * (sc_kq) hangs sometimes perhaps because of a race condition between
399 	 * close and kevent in the kthread on the kqueue.
400 	 */
401 	finibackend(sc);
402 
403 	vmem_xfree(shmif_units, sc->sc_unit+1, 1);
404 
405 	ether_ifdetach(ifp);
406 	if_detach(ifp);
407 
408 	cv_destroy(&sc->sc_cv);
409 	mutex_destroy(&sc->sc_mtx);
410 
411 	kmem_free(sc, sizeof(*sc));
412 
413 	return 0;
414 }
415 
416 static int
417 shmif_init(struct ifnet *ifp)
418 {
419 	struct shmif_sc *sc = ifp->if_softc;
420 	int error = 0;
421 
422 	if (sc->sc_memfd == -1)
423 		return ENXIO;
424 	KASSERT(sc->sc_busmem);
425 
426 	ifp->if_flags |= IFF_RUNNING;
427 
428 	mutex_enter(&sc->sc_mtx);
429 	sc->sc_nextpacket = sc->sc_busmem->shm_last;
430 	sc->sc_devgen = sc->sc_busmem->shm_gen;
431 
432 	cv_broadcast(&sc->sc_cv);
433 	mutex_exit(&sc->sc_mtx);
434 
435 	return error;
436 }
437 
438 static int
439 shmif_ioctl(struct ifnet *ifp, u_long cmd, void *data)
440 {
441 	struct shmif_sc *sc = ifp->if_softc;
442 	struct ifdrv *ifd;
443 	char *path;
444 	int s, rv, memfd;
445 
446 	s = splnet();
447 	switch (cmd) {
448 	case SIOCGLINKSTR:
449 		ifd = data;
450 
451 		if (sc->sc_backfilelen == 0) {
452 			rv = ENOENT;
453 			break;
454 		}
455 
456 		ifd->ifd_len = sc->sc_backfilelen;
457 		if (ifd->ifd_cmd == IFLINKSTR_QUERYLEN) {
458 			rv = 0;
459 			break;
460 		}
461 
462 		if (ifd->ifd_cmd != 0) {
463 			rv = EINVAL;
464 			break;
465 		}
466 
467 		rv = copyoutstr(sc->sc_backfile, ifd->ifd_data,
468 		    MIN(sc->sc_backfilelen, ifd->ifd_len), NULL);
469 		break;
470 	case SIOCSLINKSTR:
471 		if (ifp->if_flags & IFF_UP) {
472 			rv = EBUSY;
473 			break;
474 		}
475 
476 		ifd = data;
477 		if (ifd->ifd_cmd == IFLINKSTR_UNSET) {
478 			finibackend(sc);
479 			rv = 0;
480 			break;
481 		} else if (ifd->ifd_cmd != 0) {
482 			rv = EINVAL;
483 			break;
484 		} else if (sc->sc_backfile) {
485 			rv = EBUSY;
486 			break;
487 		}
488 
489 		if (ifd->ifd_len > MAXPATHLEN) {
490 			rv = E2BIG;
491 			break;
492 		} else if (ifd->ifd_len < 1) {
493 			rv = EINVAL;
494 			break;
495 		}
496 
497 		path = kmem_alloc(ifd->ifd_len, KM_SLEEP);
498 		rv = copyinstr(ifd->ifd_data, path, ifd->ifd_len, NULL);
499 		if (rv) {
500 			kmem_free(path, ifd->ifd_len);
501 			break;
502 		}
503 		rv = rumpuser_open(path,
504 		    RUMPUSER_OPEN_RDWR | RUMPUSER_OPEN_CREATE, &memfd);
505 		if (rv) {
506 			kmem_free(path, ifd->ifd_len);
507 			break;
508 		}
509 		rv = initbackend(sc, memfd);
510 		if (rv) {
511 			kmem_free(path, ifd->ifd_len);
512 			rumpuser_close(memfd);
513 			break;
514 		}
515 		sc->sc_backfile = path;
516 		sc->sc_backfilelen = ifd->ifd_len;
517 
518 		break;
519 	default:
520 		rv = ether_ioctl(ifp, cmd, data);
521 		if (rv == ENETRESET)
522 			rv = 0;
523 		break;
524 	}
525 	splx(s);
526 
527 	return rv;
528 }
529 
530 /* send everything in-context since it's just a matter of mem-to-mem copy */
531 static void
532 shmif_start(struct ifnet *ifp)
533 {
534 	struct shmif_sc *sc = ifp->if_softc;
535 	struct shmif_mem *busmem = sc->sc_busmem;
536 	struct mbuf *m, *m0;
537 	uint32_t dataoff;
538 	uint32_t pktsize, pktwrote;
539 	bool wrote = false;
540 	bool wrap;
541 
542 	ifp->if_flags |= IFF_OACTIVE;
543 
544 	for (;;) {
545 		struct shmif_pkthdr sp;
546 		struct timeval tv;
547 
548 		IF_DEQUEUE(&ifp->if_snd, m0);
549 		if (m0 == NULL) {
550 			break;
551 		}
552 
553 		pktsize = 0;
554 		for (m = m0; m != NULL; m = m->m_next) {
555 			pktsize += m->m_len;
556 		}
557 		KASSERT(pktsize <= ETHERMTU + ETHER_HDR_LEN);
558 
559 		getmicrouptime(&tv);
560 		sp.sp_len = pktsize;
561 		sp.sp_sec = tv.tv_sec;
562 		sp.sp_usec = tv.tv_usec;
563 		sp.sp_sender = sc->sc_uuid;
564 
565 		bpf_mtap(ifp, m0, BPF_D_OUT);
566 
567 		shmif_lockbus(busmem);
568 		KASSERT(busmem->shm_magic == SHMIF_MAGIC);
569 		busmem->shm_last = shmif_nextpktoff(busmem, busmem->shm_last);
570 
571 		wrap = false;
572 		dataoff = shmif_buswrite(busmem,
573 		    busmem->shm_last, &sp, sizeof(sp), &wrap);
574 		pktwrote = 0;
575 		for (m = m0; m != NULL; m = m->m_next) {
576 			pktwrote += m->m_len;
577 			dataoff = shmif_buswrite(busmem, dataoff,
578 			    mtod(m, void *), m->m_len, &wrap);
579 		}
580 		KASSERT(pktwrote == pktsize);
581 		if (wrap) {
582 			busmem->shm_gen++;
583 			DPRINTF(("bus generation now %" PRIu64 "\n",
584 			    busmem->shm_gen));
585 		}
586 		shmif_unlockbus(busmem);
587 
588 		m_freem(m0);
589 		wrote = true;
590 		ifp->if_opackets++;
591 
592 		DPRINTF(("shmif_start: send %d bytes at off %d\n",
593 		    pktsize, busmem->shm_last));
594 	}
595 
596 	ifp->if_flags &= ~IFF_OACTIVE;
597 
598 	/* wakeup? */
599 	if (wrote) {
600 		dowakeup(sc);
601 	}
602 }
603 
604 static void
605 shmif_stop(struct ifnet *ifp, int disable)
606 {
607 	struct shmif_sc *sc = ifp->if_softc;
608 
609 	ifp->if_flags &= ~IFF_RUNNING;
610 	membar_producer();
611 
612 	/*
613 	 * wakeup thread.  this will of course wake up all bus
614 	 * listeners, but that's life.
615 	 */
616 	if (sc->sc_memfd != -1) {
617 		dowakeup(sc);
618 	}
619 }
620 
621 
622 /*
623  * Check if we have been sleeping too long.  Basically,
624  * our in-sc nextpkt must by first <= nextpkt <= last"+1".
625  * We use the fact that first is guaranteed to never overlap
626  * with the last frame in the ring.
627  */
628 static __inline bool
629 stillvalid_p(struct shmif_sc *sc)
630 {
631 	struct shmif_mem *busmem = sc->sc_busmem;
632 	unsigned gendiff = busmem->shm_gen - sc->sc_devgen;
633 	uint32_t lastoff, devoff;
634 
635 	KASSERT(busmem->shm_first != busmem->shm_last);
636 
637 	/* normalize onto a 2x busmem chunk */
638 	devoff = sc->sc_nextpacket;
639 	lastoff = shmif_nextpktoff(busmem, busmem->shm_last);
640 
641 	/* trivial case */
642 	if (gendiff > 1)
643 		return false;
644 	KASSERT(gendiff <= 1);
645 
646 	/* Normalize onto 2x busmem chunk */
647 	if (busmem->shm_first >= lastoff) {
648 		lastoff += BUSMEM_DATASIZE;
649 		if (gendiff == 0)
650 			devoff += BUSMEM_DATASIZE;
651 	} else {
652 		if (gendiff)
653 			return false;
654 	}
655 
656 	return devoff >= busmem->shm_first && devoff <= lastoff;
657 }
658 
659 static void
660 shmif_rcv(void *arg)
661 {
662 	struct ifnet *ifp = arg;
663 	struct shmif_sc *sc = ifp->if_softc;
664 	struct shmif_mem *busmem;
665 	struct mbuf *m = NULL;
666 	struct ether_header *eth;
667 	uint32_t nextpkt;
668 	bool wrap, passup;
669 	int error;
670 	const int align
671 	    = ALIGN(sizeof(struct ether_header)) - sizeof(struct ether_header);
672 
673  reup:
674 	mutex_enter(&sc->sc_mtx);
675 	while ((ifp->if_flags & IFF_RUNNING) == 0 && !sc->sc_dying)
676 		cv_wait(&sc->sc_cv, &sc->sc_mtx);
677 	mutex_exit(&sc->sc_mtx);
678 
679 	busmem = sc->sc_busmem;
680 
681 	while (ifp->if_flags & IFF_RUNNING) {
682 		struct shmif_pkthdr sp;
683 
684 		if (m == NULL) {
685 			m = m_gethdr(M_WAIT, MT_DATA);
686 			MCLGET(m, M_WAIT);
687 			m->m_data += align;
688 		}
689 
690 		DPRINTF(("waiting %d/%" PRIu64 "\n",
691 		    sc->sc_nextpacket, sc->sc_devgen));
692 		KASSERT(m->m_flags & M_EXT);
693 
694 		shmif_lockbus(busmem);
695 		KASSERT(busmem->shm_magic == SHMIF_MAGIC);
696 		KASSERT(busmem->shm_gen >= sc->sc_devgen);
697 
698 		/* need more data? */
699 		if (sc->sc_devgen == busmem->shm_gen &&
700 		    shmif_nextpktoff(busmem, busmem->shm_last)
701 		     == sc->sc_nextpacket) {
702 			shmif_unlockbus(busmem);
703 			error = rumpcomp_shmif_watchwait(sc->sc_kq);
704 			if (__predict_false(error))
705 				printf("shmif_rcv: wait failed %d\n", error);
706 			membar_consumer();
707 			continue;
708 		}
709 
710 		if (stillvalid_p(sc)) {
711 			nextpkt = sc->sc_nextpacket;
712 		} else {
713 			KASSERT(busmem->shm_gen > 0);
714 			nextpkt = busmem->shm_first;
715 			if (busmem->shm_first > busmem->shm_last)
716 				sc->sc_devgen = busmem->shm_gen - 1;
717 			else
718 				sc->sc_devgen = busmem->shm_gen;
719 			DPRINTF(("dev %p overrun, new data: %d/%" PRIu64 "\n",
720 			    sc, nextpkt, sc->sc_devgen));
721 		}
722 
723 		/*
724 		 * If our read pointer is ahead the bus last write, our
725 		 * generation must be one behind.
726 		 */
727 		KASSERT(!(nextpkt > busmem->shm_last
728 		    && sc->sc_devgen == busmem->shm_gen));
729 
730 		wrap = false;
731 		nextpkt = shmif_busread(busmem, &sp,
732 		    nextpkt, sizeof(sp), &wrap);
733 		KASSERT(sp.sp_len <= ETHERMTU + ETHER_HDR_LEN);
734 		nextpkt = shmif_busread(busmem, mtod(m, void *),
735 		    nextpkt, sp.sp_len, &wrap);
736 
737 		DPRINTF(("shmif_rcv: read packet of length %d at %d\n",
738 		    sp.sp_len, nextpkt));
739 
740 		sc->sc_nextpacket = nextpkt;
741 		shmif_unlockbus(sc->sc_busmem);
742 
743 		if (wrap) {
744 			sc->sc_devgen++;
745 			DPRINTF(("dev %p generation now %" PRIu64 "\n",
746 			    sc, sc->sc_devgen));
747 		}
748 
749 		/*
750 		 * Ignore packets too short to possibly be valid.
751 		 * This is hit at least for the first frame on a new bus.
752 		 */
753 		if (__predict_false(sp.sp_len < ETHER_HDR_LEN)) {
754 			DPRINTF(("shmif read packet len %d < ETHER_HDR_LEN\n",
755 			    sp.sp_len));
756 			continue;
757 		}
758 
759 		m->m_len = m->m_pkthdr.len = sp.sp_len;
760 		m_set_rcvif(m, ifp);
761 
762 		/*
763 		 * Test if we want to pass the packet upwards
764 		 */
765 		eth = mtod(m, struct ether_header *);
766 		if (sp.sp_sender == sc->sc_uuid) {
767 			passup = false;
768 		} else if (memcmp(eth->ether_dhost, CLLADDR(ifp->if_sadl),
769 		    ETHER_ADDR_LEN) == 0) {
770 			passup = true;
771 		} else if (ETHER_IS_MULTICAST(eth->ether_dhost)) {
772 			passup = true;
773 		} else if (ifp->if_flags & IFF_PROMISC) {
774 			m->m_flags |= M_PROMISC;
775 			passup = true;
776 		} else {
777 			passup = false;
778 		}
779 
780 		if (passup) {
781 			int bound;
782 			KERNEL_LOCK(1, NULL);
783 			/* Prevent LWP migrations between CPUs for psref(9) */
784 			bound = curlwp_bind();
785 			if_input(ifp, m);
786 			curlwp_bindx(bound);
787 			KERNEL_UNLOCK_ONE(NULL);
788 			m = NULL;
789 		}
790 		/* else: reuse mbuf for a future packet */
791 	}
792 	m_freem(m);
793 	m = NULL;
794 
795 	if (!sc->sc_dying)
796 		goto reup;
797 
798 	kthread_exit(0);
799 }
800