xref: /netbsd-src/sys/net/agr/if_agr.c (revision 7f21db1c0118155e0dd40b75182e30c589d9f63e)
1 /*	$NetBSD: if_agr.c,v 1.26 2010/02/08 17:59:06 dyoung Exp $	*/
2 
3 /*-
4  * Copyright (c)2005 YAMAMOTO Takashi,
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice, this list of conditions and the following disclaimer.
12  * 2. Redistributions in binary form must reproduce the above copyright
13  *    notice, this list of conditions and the following disclaimer in the
14  *    documentation and/or other materials provided with the distribution.
15  *
16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26  * SUCH DAMAGE.
27  */
28 
29 #include <sys/cdefs.h>
30 __KERNEL_RCSID(0, "$NetBSD: if_agr.c,v 1.26 2010/02/08 17:59:06 dyoung Exp $");
31 
32 #include "opt_inet.h"
33 
34 #include <sys/param.h>
35 #include <sys/callout.h>
36 #include <sys/malloc.h>
37 #include <sys/mbuf.h>
38 #include <sys/systm.h>
39 #include <sys/types.h>
40 #include <sys/queue.h>
41 #include <sys/sockio.h>
42 #include <sys/proc.h>	/* XXX for curproc */
43 #include <sys/kauth.h>
44 #include <sys/xcall.h>
45 
46 #include <net/bpf.h>
47 #include <net/if.h>
48 #include <net/if_dl.h>
49 #include <net/if_types.h>
50 #include <net/if_ether.h>
51 
52 #if defined(INET)
53 #include <netinet/in.h>
54 #include <netinet/if_inarp.h>
55 #endif
56 
57 #include <net/agr/if_agrvar.h>
58 #include <net/agr/if_agrvar_impl.h>
59 #include <net/agr/if_agrioctl.h>
60 #include <net/agr/if_agrsubr.h>
61 #include <net/agr/if_agrethervar.h>
62 
63 void agrattach(int);
64 
65 static int agr_clone_create(struct if_clone *, int);
66 static int agr_clone_destroy(struct ifnet *);
67 static void agr_start(struct ifnet *);
68 static int agr_setconfig(struct agr_softc *, const struct agrreq *);
69 static int agr_getconfig(struct agr_softc *, struct agrreq *);
70 static int agr_getportlist(struct agr_softc *, struct agrreq *);
71 static int agr_addport(struct ifnet *, struct ifnet *);
72 static int agr_remport(struct ifnet *, struct ifnet *);
73 static int agrreq_copyin(const void *, struct agrreq *);
74 static int agrreq_copyout(void *, struct agrreq *);
75 static int agr_ioctl(struct ifnet *, u_long, void *);
76 static struct agr_port *agr_select_tx_port(struct agr_softc *, struct mbuf *);
77 static int agr_ioctl_filter(struct ifnet *, u_long, void *);
78 static void agr_reset_iftype(struct ifnet *);
79 static int agr_config_promisc(struct agr_softc *);
80 static int agrport_config_promisc_callback(struct agr_port *, void *);
81 static int agrport_config_promisc(struct agr_port *, bool);
82 static int agrport_cleanup(struct agr_softc *, struct agr_port *);
83 
84 static int agr_enter(struct agr_softc *);
85 static void agr_exit(struct agr_softc *);
86 static int agr_pause(struct agr_softc *);
87 static void agr_evacuate(struct agr_softc *);
88 static void agr_sync(void);
89 static void agr_ports_lock(struct agr_softc *);
90 static void agr_ports_unlock(struct agr_softc *);
91 static bool agr_ports_enter(struct agr_softc *);
92 static void agr_ports_exit(struct agr_softc *);
93 
94 static struct if_clone agr_cloner =
95     IF_CLONE_INITIALIZER("agr", agr_clone_create, agr_clone_destroy);
96 
97 /*
98  * EXPORTED FUNCTIONS
99  */
100 
101 /*
102  * agrattch: device attach routine.
103  */
104 
105 void
106 agrattach(int count)
107 {
108 
109 	if_clone_attach(&agr_cloner);
110 }
111 
112 /*
113  * agr_input: frame collector.
114  */
115 
116 void
117 agr_input(struct ifnet *ifp_port, struct mbuf *m)
118 {
119 	struct agr_port *port;
120 	struct ifnet *ifp;
121 #if NVLAN > 0
122 	struct m_tag *mtag;
123 #endif
124 
125 	port = ifp_port->if_agrprivate;
126 	KASSERT(port);
127 	ifp = port->port_agrifp;
128 	if ((port->port_flags & AGRPORT_COLLECTING) == 0) {
129 		m_freem(m);
130 		ifp->if_ierrors++;
131 		return;
132 	}
133 
134 	ifp->if_ipackets++;
135 	m->m_pkthdr.rcvif = ifp;
136 
137 #define DNH_DEBUG
138 #if NVLAN > 0
139 	/* got a vlan packet? */
140 	if ((mtag = m_tag_find(m, PACKET_TAG_VLAN, NULL)) != NULL) {
141 #ifdef DNH_DEBUG
142 		printf("%s: vlan tag %d attached\n",
143 			ifp->if_xname,
144 			htole16((*(u_int *)(mtag + 1)) & 0xffff));
145 		printf("%s: vlan input\n", ifp->if_xname);
146 #endif
147 		vlan_input(ifp, m);
148 		return;
149 #ifdef DNH_DEBUG
150 	} else {
151 		struct ethercom *ec = (void *)ifp;
152 		printf("%s: no vlan tag attached, ec_nvlans=%d\n",
153 			ifp->if_xname, ec->ec_nvlans);
154 #endif
155 	}
156 #endif
157 
158 	if (ifp->if_bpf) {
159 		bpf_ops->bpf_mtap(ifp->if_bpf, m);
160 	}
161 
162 	(*ifp->if_input)(ifp, m);
163 }
164 
165 /*
166  * EXPORTED AGR-INTERNAL FUNCTIONS
167  */
168 
169 void
170 agr_lock(struct agr_softc *sc)
171 {
172 
173 	mutex_enter(&sc->sc_lock);
174 }
175 
176 void
177 agr_unlock(struct agr_softc *sc)
178 {
179 
180 	mutex_exit(&sc->sc_lock);
181 }
182 
183 /*
184  * agr_xmit_frame: transmit a pre-built frame.
185  */
186 
187 int
188 agr_xmit_frame(struct ifnet *ifp_port, struct mbuf *m)
189 {
190 	int error;
191 
192 	struct sockaddr_storage dst0;
193 	struct sockaddr *dst;
194 	int hdrlen;
195 
196 	/*
197 	 * trim off link level header and let if_output re-add it.
198 	 * XXX better to introduce an API to transmit pre-built frames.
199 	 */
200 
201 	hdrlen = ifp_port->if_hdrlen;
202 	if (m->m_pkthdr.len < hdrlen) {
203 		m_freem(m);
204 		return EINVAL;
205 	}
206 	memset(&dst0, 0, sizeof(dst0));
207 	dst = (struct sockaddr *)&dst0;
208 	dst->sa_family = pseudo_AF_HDRCMPLT;
209 	dst->sa_len = hdrlen;
210 	m_copydata(m, 0, hdrlen, &dst->sa_data);
211 	m_adj(m, hdrlen);
212 
213 	error = (*ifp_port->if_output)(ifp_port, m, dst, NULL);
214 
215 	return error;
216 }
217 
218 int
219 agrport_ioctl(struct agr_port *port, u_long cmd, void *arg)
220 {
221 	struct ifnet *ifp = port->port_ifp;
222 
223 	KASSERT(ifp->if_agrprivate == (void *)port);
224 	KASSERT(ifp->if_ioctl == agr_ioctl_filter);
225 
226 	return (*port->port_ioctl)(ifp, cmd, arg);
227 }
228 
229 /*
230  * INTERNAL FUNCTIONS
231  */
232 
233 /*
234  * Enable vlan hardware assist for the specified port.
235  */
236 static int
237 agr_vlan_add(struct agr_port *port, void *arg)
238 {
239 	struct ifnet *ifp = port->port_ifp;
240 	struct ethercom *ec_port = (void *)ifp;
241 	struct ifreq ifr;
242 	int error=0;
243 
244 	if (ec_port->ec_nvlans++ == 0 &&
245 	    (ec_port->ec_capabilities & ETHERCAP_VLAN_MTU) != 0) {
246 		struct ifnet *p = port->port_ifp;
247 		/*
248 		 * Enable Tx/Rx of VLAN-sized frames.
249 		 */
250 		ec_port->ec_capenable |= ETHERCAP_VLAN_MTU;
251 		if (p->if_flags & IFF_UP) {
252 			ifr.ifr_flags = p->if_flags;
253 			error = (*p->if_ioctl)(p, SIOCSIFFLAGS,
254 			    (void *) &ifr);
255 			if (error) {
256 				if (ec_port->ec_nvlans-- == 1)
257 					ec_port->ec_capenable &=
258 					    ~ETHERCAP_VLAN_MTU;
259 				return (error);
260 			}
261 		}
262 	}
263 
264 	return error;
265 }
266 
267 /*
268  * Disable vlan hardware assist for the specified port.
269  */
270 static int
271 agr_vlan_del(struct agr_port *port, void *arg)
272 {
273 	struct ethercom *ec_port = (void *)port->port_ifp;
274 	struct ifreq ifr;
275 
276 	/* Disable vlan support */
277 	if (ec_port->ec_nvlans-- == 1) {
278 		/*
279 		 * Disable Tx/Rx of VLAN-sized frames.
280 		 */
281 		ec_port->ec_capenable &= ~ETHERCAP_VLAN_MTU;
282 		if (port->port_ifp->if_flags & IFF_UP) {
283 			ifr.ifr_flags = port->port_ifp->if_flags;
284 			(void) (*port->port_ifp->if_ioctl)(port->port_ifp,
285 			    SIOCSIFFLAGS, (void *) &ifr);
286 		}
287 	}
288 
289 	return 0;
290 }
291 
292 
293 /*
294  * Check for vlan attach/detach.
295  * ec->ec_nvlans is directly modified by the vlan driver.
296  * We keep a local count in sc (sc->sc_nvlans) to detect
297  * when the vlan driver attaches or detaches.
298  * Note the agr interface must be up for this to work.
299  */
300 static void
301 agr_vlan_check(struct ifnet *ifp, struct agr_softc *sc)
302 {
303 	struct ethercom *ec = (void *)ifp;
304 	int error;
305 
306 	/* vlans in sync? */
307 	if (sc->sc_nvlans == ec->ec_nvlans) {
308 		return;
309 	}
310 
311 	if (sc->sc_nvlans == 0) {
312 		/* vlan added */
313 		error = agr_port_foreach(sc, agr_vlan_add, NULL);
314 		sc->sc_nvlans = ec->ec_nvlans;
315 	} else if (ec->ec_nvlans == 0) {
316 		/* vlan removed */
317 		error = agr_port_foreach(sc, agr_vlan_del, NULL);
318 		sc->sc_nvlans = 0;
319 	}
320 }
321 
322 static int
323 agr_clone_create(struct if_clone *ifc, int unit)
324 {
325 	struct agr_softc *sc;
326 	struct ifnet *ifp;
327 
328 	sc = agr_alloc_softc();
329 	TAILQ_INIT(&sc->sc_ports);
330 	mutex_init(&sc->sc_lock, MUTEX_DEFAULT, IPL_NET);
331 	mutex_init(&sc->sc_entry_mtx, MUTEX_DEFAULT, IPL_NONE);
332 	cv_init(&sc->sc_insc_cv, "agr_softc");
333 	cv_init(&sc->sc_ports_cv, "agrports");
334 	agrtimer_init(sc);
335 	ifp = &sc->sc_if;
336 	snprintf(ifp->if_xname, sizeof(ifp->if_xname), "%s%d",
337 	    ifc->ifc_name, unit);
338 
339 	ifp->if_softc = sc;
340 	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
341 	ifp->if_start = agr_start;
342 	ifp->if_ioctl = agr_ioctl;
343 	IFQ_SET_READY(&ifp->if_snd);
344 
345 	if_attach(ifp);
346 
347 	agr_reset_iftype(ifp);
348 
349 	return 0;
350 }
351 
352 static void
353 agr_reset_iftype(struct ifnet *ifp)
354 {
355 
356 	ifp->if_type = IFT_OTHER;
357 	ifp->if_dlt = DLT_NULL;
358 	ifp->if_addrlen = 0;
359 	if_alloc_sadl(ifp);
360 }
361 
362 static int
363 agr_clone_destroy(struct ifnet *ifp)
364 {
365 	struct agr_softc *sc = ifp->if_softc;
366 	int error;
367 
368 	if ((error = agr_pause(sc)) != 0)
369 		return error;
370 
371 	if_detach(ifp);
372 	agrtimer_destroy(sc);
373 	/* Now that the ifnet has been detached, and our
374 	 * component ifnets are disconnected, there can be
375 	 * no new threads in the softc.  Wait for every
376 	 * thread to get out of the softc.
377 	 */
378 	agr_evacuate(sc);
379 	mutex_destroy(&sc->sc_lock);
380 	mutex_destroy(&sc->sc_entry_mtx);
381 	cv_destroy(&sc->sc_insc_cv);
382 	cv_destroy(&sc->sc_ports_cv);
383 	agr_free_softc(sc);
384 
385 	return 0;
386 }
387 
388 static struct agr_port *
389 agr_select_tx_port(struct agr_softc *sc, struct mbuf *m)
390 {
391 
392 	return (*sc->sc_iftop->iftop_select_tx_port)(sc, m);
393 }
394 
395 #if 0 /* "generic" version */
396 static struct agr_port *
397 agr_select_tx_port(struct agr_softc *sc, struct mbuf *m)
398 {
399 	struct agr_port *port;
400 	uint32_t hash;
401 
402 	hash = (*sc->sc_iftop->iftop_hashmbuf)(sc, m);
403 	if (sc->sc_nports == 0)
404 		return NULL;
405 	hash %= sc->sc_nports;
406 	port = TAILQ_FIRST(&sc->sc_ports);
407 	KASSERT(port != NULL);
408 	while (hash--) {
409 		port = TAILQ_NEXT(port, port_q);
410 		KASSERT(port != NULL);
411 	}
412 
413 	return port;
414 }
415 #endif /* 0 */
416 
417 static void
418 agr_start(struct ifnet *ifp)
419 {
420 	struct agr_softc *sc = ifp->if_softc;
421 	struct mbuf *m;
422 
423 	AGR_LOCK(sc);
424 
425 	while (/* CONSTCOND */ 1) {
426 		struct agr_port *port;
427 
428 		IFQ_DEQUEUE(&ifp->if_snd, m);
429 		if (m == NULL) {
430 			break;
431 		}
432 		if (ifp->if_bpf) {
433 			bpf_ops->bpf_mtap(ifp->if_bpf, m);
434 		}
435 		port = agr_select_tx_port(sc, m);
436 		if (port) {
437 			int error;
438 
439 			error = agr_xmit_frame(port->port_ifp, m);
440 			if (error) {
441 				ifp->if_oerrors++;
442 			} else {
443 				ifp->if_opackets++;
444 			}
445 		} else {
446 			m_freem(m);
447 			ifp->if_oerrors++;
448 		}
449 	}
450 
451 	AGR_UNLOCK(sc);
452 
453 	ifp->if_flags &= ~IFF_OACTIVE;
454 }
455 
456 static int
457 agr_setconfig(struct agr_softc *sc, const struct agrreq *ar)
458 {
459 	struct ifnet *ifp = &sc->sc_if;
460 	int cmd = ar->ar_cmd;
461 	struct ifnet *ifp_port;
462 	int error = 0;
463 	char ifname[IFNAMSIZ];
464 
465 	memset(ifname, 0, sizeof(ifname));
466 	error = copyin(ar->ar_buf, ifname,
467 	    MIN(ar->ar_buflen, sizeof(ifname) - 1));
468 	if (error) {
469 		return error;
470 	}
471 	ifp_port = ifunit(ifname);
472 	if (ifp_port == NULL) {
473 		return ENOENT;
474 	}
475 
476 	agr_ports_lock(sc);
477 	switch (cmd) {
478 	case AGRCMD_ADDPORT:
479 		error = agr_addport(ifp, ifp_port);
480 		break;
481 
482 	case AGRCMD_REMPORT:
483 		error = agr_remport(ifp, ifp_port);
484 		break;
485 
486 	default:
487 		error = EINVAL;
488 		break;
489 	}
490 	agr_ports_unlock(sc);
491 
492 	return error;
493 }
494 
495 static int
496 agr_getportlist(struct agr_softc *sc, struct agrreq *ar)
497 {
498 	struct agr_port *port;
499 	struct agrportlist apl;
500 	struct agrportinfo api;
501 	char *cp = ar->ar_buf;
502 	size_t bufleft = (cp == NULL) ? 0 : ar->ar_buflen;
503 	int error;
504 
505 	if (cp != NULL) {
506 		memset(&apl, 0, sizeof(apl));
507 		memset(&api, 0, sizeof(api));
508 
509 		if (bufleft < sizeof(apl)) {
510 			return E2BIG;
511 		}
512 		apl.apl_nports = sc->sc_nports;
513 		error = copyout(&apl, cp, sizeof(apl));
514 		if (error) {
515 			return error;
516 		}
517 		cp += sizeof(apl);
518 	}
519 	bufleft -= sizeof(apl);
520 
521 	TAILQ_FOREACH(port, &sc->sc_ports, port_q) {
522 		if (cp != NULL) {
523 			if (bufleft < sizeof(api)) {
524 				return E2BIG;
525 			}
526 			memcpy(api.api_ifname, port->port_ifp->if_xname,
527 			    sizeof(api.api_ifname));
528 			api.api_flags = 0;
529 			if (port->port_flags & AGRPORT_COLLECTING) {
530 				api.api_flags |= AGRPORTINFO_COLLECTING;
531 			}
532 			if (port->port_flags & AGRPORT_DISTRIBUTING) {
533 				api.api_flags |= AGRPORTINFO_DISTRIBUTING;
534 			}
535 			error = copyout(&api, cp, sizeof(api));
536 			if (error) {
537 				return error;
538 			}
539 			cp += sizeof(api);
540 		}
541 		bufleft -= sizeof(api);
542 	}
543 
544 	if (cp == NULL) {
545 		ar->ar_buflen = -bufleft; /* necessary buffer size */
546 	}
547 
548 	return 0;
549 }
550 
551 static int
552 agr_getconfig(struct agr_softc *sc, struct agrreq *ar)
553 {
554 	int cmd = ar->ar_cmd;
555 	int error;
556 
557 	(void)agr_ports_enter(sc);
558 	switch (cmd) {
559 	case AGRCMD_PORTLIST:
560 		error = agr_getportlist(sc, ar);
561 		break;
562 
563 	default:
564 		error = EINVAL;
565 		break;
566 	}
567 	agr_ports_exit(sc);
568 
569 	return error;
570 }
571 
572 static int
573 agr_addport(struct ifnet *ifp, struct ifnet *ifp_port)
574 {
575 	const struct ifaddr *ifa;
576 	struct agr_softc *sc = ifp->if_softc;
577 	struct agr_port *port = NULL;
578 	int error = 0;
579 
580 	if (ifp_port->if_ioctl == NULL) {
581 		error = EOPNOTSUPP;
582 		goto out;
583 	}
584 
585 	if (ifp_port->if_agrprivate) {
586 		error = EBUSY;
587 		goto out;
588 	}
589 
590 	if (ifp_port->if_start == agr_start) {
591 		error = EINVAL;
592 		goto out;
593 	}
594 
595 	port = malloc(sizeof(*port) + ifp_port->if_addrlen, M_DEVBUF,
596 	    M_WAITOK | M_ZERO);
597 	if (port == NULL) {
598 		error = ENOMEM;
599 		goto out;
600 	}
601 	port->port_flags = AGRPORT_LARVAL;
602 
603 	IFADDR_FOREACH(ifa, ifp_port) {
604 		if (ifa->ifa_addr->sa_family != AF_LINK) {
605 			error = EBUSY;
606 			goto out;
607 		}
608 	}
609 
610 	if (sc->sc_nports == 0) {
611 		switch (ifp_port->if_type) {
612 		case IFT_ETHER:
613 			sc->sc_iftop = &agrether_ops;
614 			break;
615 
616 		default:
617 			error = EPROTONOSUPPORT; /* XXX */
618 			goto out;
619 		}
620 
621 		error = (*sc->sc_iftop->iftop_ctor)(sc, ifp_port);
622 		if (error)
623 			goto out;
624 		agrtimer_start(sc);
625 	} else {
626 		if (ifp->if_type != ifp_port->if_type) {
627 			error = EINVAL;
628 			goto out;
629 		}
630 		if (ifp->if_addrlen != ifp_port->if_addrlen) {
631 			error = EINVAL;
632 			goto out;
633 		}
634 	}
635 
636 	memcpy(port->port_origlladdr, CLLADDR(ifp_port->if_sadl),
637 	    ifp_port->if_addrlen);
638 
639 	/*
640 	 * start to modify ifp_port.
641 	 */
642 
643 	/*
644 	 * XXX this should probably be SIOCALIFADDR but that doesn't
645 	 * appear to work (ENOTTY). We want to change the mac address
646 	 * of each port to that of the first port. No need for arps
647 	 * since there are no inet addresses assigned to the ports.
648 	 */
649 	error = (*ifp_port->if_ioctl)(ifp_port, SIOCINITIFADDR, ifp->if_dl);
650 
651 	if (error) {
652 		printf("%s: SIOCINITIFADDR error %d\n", __func__, error);
653 		goto cleanup;
654 	}
655 	port->port_flags |= AGRPORT_LADDRCHANGED;
656 
657 	ifp->if_type = ifp_port->if_type;
658 	AGR_LOCK(sc);
659 
660 	port->port_ifp = ifp_port;
661 	ifp_port->if_agrprivate = port;
662 	port->port_agrifp = ifp;
663 	TAILQ_INSERT_TAIL(&sc->sc_ports, port, port_q);
664 	sc->sc_nports++;
665 
666 	port->port_ioctl = ifp_port->if_ioctl;
667 	ifp_port->if_ioctl = agr_ioctl_filter;
668 
669 	port->port_flags |= AGRPORT_ATTACHED;
670 
671 	AGR_UNLOCK(sc);
672 
673 	error = (*sc->sc_iftop->iftop_portinit)(sc, port);
674 	if (error) {
675 		printf("%s: portinit error %d\n", __func__, error);
676 		goto cleanup;
677 	}
678 
679 	ifp->if_flags |= IFF_RUNNING;
680 
681 	agrport_config_promisc(port, (ifp->if_flags & IFF_PROMISC) != 0);
682 	error = (*sc->sc_iftop->iftop_configmulti_port)(sc, port, true);
683 	if (error) {
684 		printf("%s: configmulti error %d\n", __func__, error);
685 		goto cleanup;
686 	}
687 
688 	AGR_LOCK(sc);
689 	port->port_flags &= ~AGRPORT_LARVAL;
690 	AGR_UNLOCK(sc);
691 out:
692 	if (error && port) {
693 		free(port, M_DEVBUF);
694 	}
695 	return error;
696 
697 cleanup:
698 	if (agrport_cleanup(sc, port)) {
699 		printf("%s: error on cleanup\n", __func__);
700 
701 		port = NULL; /* XXX */
702 	}
703 
704 	if (sc->sc_nports == 0) {
705 		KASSERT(TAILQ_EMPTY(&sc->sc_ports));
706 		agrtimer_stop(sc);
707 		(*sc->sc_iftop->iftop_dtor)(sc);
708 		sc->sc_iftop = NULL;
709 		agr_reset_iftype(ifp);
710 	} else {
711 		KASSERT(!TAILQ_EMPTY(&sc->sc_ports));
712 	}
713 
714 	goto out;
715 }
716 
717 static int
718 agr_remport(struct ifnet *ifp, struct ifnet *ifp_port)
719 {
720 	struct agr_softc *sc = ifp->if_softc;
721 	struct agr_port *port;
722 	int error = 0;
723 
724 	if (ifp_port->if_agrprivate == NULL) {
725 		error = ENOENT;
726 		return error;
727 	}
728 
729 	port = ifp_port->if_agrprivate;
730 	if (port->port_agrifp != ifp) {
731 		error = EINVAL;
732 		return error;
733 	}
734 
735 	KASSERT(sc->sc_nports > 0);
736 
737 	AGR_LOCK(sc);
738 	port->port_flags |= AGRPORT_DETACHING;
739 	AGR_UNLOCK(sc);
740 
741 	error = (*sc->sc_iftop->iftop_portfini)(sc, port);
742 	if (error) {
743 		/* XXX XXX */
744 		printf("%s: portfini error %d\n", __func__, error);
745 		goto out;
746 	}
747 
748 	error = (*sc->sc_iftop->iftop_configmulti_port)(sc, port, false);
749 	if (error) {
750 		/* XXX XXX */
751 		printf("%s: configmulti_port error %d\n", __func__, error);
752 		goto out;
753 	}
754 
755 	error = agrport_cleanup(sc, port);
756 	if (error) {
757 		/* XXX XXX */
758 		printf("%s: agrport_cleanup error %d\n", __func__, error);
759 		goto out;
760 	}
761 
762 	free(port, M_DEVBUF);
763 
764 out:
765 	if (sc->sc_nports == 0) {
766 		KASSERT(TAILQ_EMPTY(&sc->sc_ports));
767 		agrtimer_stop(sc);
768 		(*sc->sc_iftop->iftop_dtor)(sc);
769 		sc->sc_iftop = NULL;
770 		/* XXX should purge all addresses? */
771 		agr_reset_iftype(ifp);
772 	} else {
773 		KASSERT(!TAILQ_EMPTY(&sc->sc_ports));
774 	}
775 
776 	return error;
777 }
778 
779 static int
780 agrport_cleanup(struct agr_softc *sc, struct agr_port *port)
781 {
782 	struct ifnet *ifp_port = port->port_ifp;
783 	int error;
784 	int result = 0;
785 
786 	error = agrport_config_promisc(port, false);
787 	if (error) {
788 		printf("%s: config_promisc error %d\n", __func__, error);
789 		result = error;
790 	}
791 
792 	if ((port->port_flags & AGRPORT_LADDRCHANGED)) {
793 #if 0
794 		memcpy(LLADDR(ifp_port->if_sadl), port->port_origlladdr,
795 		    ifp_port->if_addrlen);
796 		if (ifp_port->if_init != NULL) {
797 			error = (*ifp_port->if_init)(ifp_port);
798 		}
799 #else
800 		union {
801 			struct sockaddr sa;
802 			struct sockaddr_dl sdl;
803 			struct sockaddr_storage ss;
804 		} u;
805 		struct ifaddr ifa;
806 
807 		sockaddr_dl_init(&u.sdl, sizeof(u.ss),
808 		    0, ifp_port->if_type, NULL, 0,
809 		    port->port_origlladdr, ifp_port->if_addrlen);
810 		memset(&ifa, 0, sizeof(ifa));
811 		ifa.ifa_addr = &u.sa;
812 		error = agrport_ioctl(port, SIOCINITIFADDR, &ifa);
813 #endif
814 		if (error) {
815 			printf("%s: if_init error %d\n", __func__, error);
816 			result = error;
817 		} else {
818 			port->port_flags &= ~AGRPORT_LADDRCHANGED;
819 		}
820 	}
821 
822 	AGR_LOCK(sc);
823 	if ((port->port_flags & AGRPORT_ATTACHED)) {
824 		ifp_port->if_agrprivate = NULL;
825 
826 		TAILQ_REMOVE(&sc->sc_ports, port, port_q);
827 		sc->sc_nports--;
828 
829 		KASSERT(ifp_port->if_ioctl == agr_ioctl_filter);
830 		ifp_port->if_ioctl = port->port_ioctl;
831 
832 		port->port_flags &= ~AGRPORT_ATTACHED;
833 	}
834 	AGR_UNLOCK(sc);
835 
836 	return result;
837 }
838 
839 static int
840 agr_ioctl_multi(struct ifnet *ifp, u_long cmd, struct ifreq *ifr)
841 {
842 	struct agr_softc *sc = ifp->if_softc;
843 	int error;
844 
845 	error = (*sc->sc_iftop->iftop_configmulti_ifreq)(sc, ifr,
846 	    (cmd == SIOCADDMULTI));
847 
848 	return error;
849 }
850 
851 /*
852  * XXX an incomplete hack; can't filter ioctls handled ifioctl().
853  *
854  * the intention here is to prevent operations on underlying interfaces
855  * so that their states are not changed in the way that agr(4) doesn't
856  * expect.  cf. the BUGS section in the agr(4) manual page.
857  */
858 static int
859 agr_ioctl_filter(struct ifnet *ifp, u_long cmd, void *arg)
860 {
861 	struct agr_port *port = ifp->if_agrprivate;
862 	int error;
863 
864 	KASSERT(port);
865 
866 	switch (cmd) {
867 	case SIOCADDMULTI: /* add m'cast addr */
868 	case SIOCAIFADDR: /* add/chg IF alias */
869 	case SIOCALIFADDR: /* add IF addr */
870 	case SIOCDELMULTI: /* del m'cast addr */
871 	case SIOCDIFADDR: /* delete IF addr */
872 	case SIOCDIFPHYADDR: /* delete gif addrs */
873 	case SIOCDLIFADDR: /* delete IF addr */
874 	case SIOCINITIFADDR:
875 	case SIOCSDRVSPEC: /* set driver-specific parameters */
876 	case SIOCSIFADDR: /* set ifnet address */
877 	case SIOCSIFBRDADDR: /* set broadcast addr */
878 	case SIOCSIFDSTADDR: /* set p-p address */
879 	case SIOCSIFGENERIC: /* generic IF set op */
880 	case SIOCSIFMEDIA: /* set net media */
881 	case SIOCSIFMETRIC: /* set IF metric */
882 	case SIOCSIFMTU: /* set ifnet mtu */
883 	case SIOCSIFNETMASK: /* set net addr mask */
884 	case SIOCSIFPHYADDR: /* set gif addres */
885 	case SIOCSLIFPHYADDR: /* set gif addrs */
886 	case SIOCSVH: /* set carp param */
887 		error = EBUSY;
888 		break;
889 	case SIOCSIFCAP: /* XXX */
890 	case SIOCSIFFLAGS: /* XXX */
891 	default:
892 		error = agrport_ioctl(port, cmd, arg);
893 		break;
894 	}
895 	return error;
896 }
897 
898 static int
899 agrreq_copyin(const void *ubuf, struct agrreq *ar)
900 {
901 	int error;
902 
903 	error = copyin(ubuf, ar, sizeof(*ar));
904 	if (error) {
905 		return error;
906 	}
907 
908 	if (ar->ar_version != AGRREQ_VERSION) {
909 		return EINVAL;
910 	}
911 
912 	return 0;
913 }
914 
915 static int
916 agrreq_copyout(void *ubuf, struct agrreq *ar)
917 {
918 	int error;
919 
920 	KASSERT(ar->ar_version == AGRREQ_VERSION);
921 
922 	error = copyout(ar, ubuf, sizeof(*ar));
923 	if (error) {
924 		return error;
925 	}
926 
927 	return 0;
928 }
929 
930 /* Make sure that if any interrupt handlers are out of the softc. */
931 static void
932 agr_sync(void)
933 {
934 	uint64_t h;
935 
936 	if (!mp_online)
937 		return;
938 
939 	h = xc_broadcast(0, (xcfunc_t)nullop, NULL, NULL);
940 	xc_wait(h);
941 }
942 
943 static int
944 agr_pause(struct agr_softc *sc)
945 {
946 	int error;
947 
948 	mutex_enter(&sc->sc_entry_mtx);
949 	if ((error = sc->sc_noentry) != 0)
950 		goto out;
951 
952 	sc->sc_noentry = EBUSY;
953 
954 	while (sc->sc_insc != 0)
955 		cv_wait(&sc->sc_insc_cv, &sc->sc_entry_mtx);
956 
957 	if (sc->sc_nports == 0) {
958 		sc->sc_noentry = ENXIO;
959 	} else {
960 		sc->sc_noentry = 0;
961 		error = EBUSY;
962 	}
963 	cv_broadcast(&sc->sc_insc_cv);
964 out:
965 	mutex_exit(&sc->sc_entry_mtx);
966 	return error;
967 }
968 
969 static void
970 agr_evacuate(struct agr_softc *sc)
971 {
972 	mutex_enter(&sc->sc_entry_mtx);
973 	cv_broadcast(&sc->sc_insc_cv);
974 	while (sc->sc_insc != 0 || sc->sc_paused != 0)
975 		cv_wait(&sc->sc_insc_cv, &sc->sc_entry_mtx);
976 	mutex_exit(&sc->sc_entry_mtx);
977 
978 	agr_sync();
979 }
980 
981 static int
982 agr_enter(struct agr_softc *sc)
983 {
984 	int error;
985 
986 	mutex_enter(&sc->sc_entry_mtx);
987 	sc->sc_paused++;
988 	while ((error = sc->sc_noentry) == EBUSY)
989 		cv_wait(&sc->sc_insc_cv, &sc->sc_entry_mtx);
990 	sc->sc_paused--;
991 	if (error == 0)
992 		sc->sc_insc++;
993 	mutex_exit(&sc->sc_entry_mtx);
994 
995 	return error;
996 }
997 
998 static void
999 agr_exit(struct agr_softc *sc)
1000 {
1001 	mutex_enter(&sc->sc_entry_mtx);
1002 	if (--sc->sc_insc == 0)
1003 		cv_signal(&sc->sc_insc_cv);
1004 	mutex_exit(&sc->sc_entry_mtx);
1005 }
1006 
1007 static bool
1008 agr_ports_enter(struct agr_softc *sc)
1009 {
1010 	mutex_enter(&sc->sc_entry_mtx);
1011 	while (sc->sc_wrports != 0)
1012 		cv_wait(&sc->sc_ports_cv, &sc->sc_entry_mtx);
1013 	sc->sc_rdports++;
1014 	mutex_exit(&sc->sc_entry_mtx);
1015 
1016 	return true;
1017 }
1018 
1019 static void
1020 agr_ports_exit(struct agr_softc *sc)
1021 {
1022 	mutex_enter(&sc->sc_entry_mtx);
1023 	if (--sc->sc_rdports == 0)
1024 		cv_signal(&sc->sc_ports_cv);
1025 	mutex_exit(&sc->sc_entry_mtx);
1026 }
1027 
1028 static void
1029 agr_ports_lock(struct agr_softc *sc)
1030 {
1031 	mutex_enter(&sc->sc_entry_mtx);
1032 	while (sc->sc_rdports != 0)
1033 		cv_wait(&sc->sc_ports_cv, &sc->sc_entry_mtx);
1034 	sc->sc_wrports = true;
1035 	mutex_exit(&sc->sc_entry_mtx);
1036 }
1037 
1038 static void
1039 agr_ports_unlock(struct agr_softc *sc)
1040 {
1041 	mutex_enter(&sc->sc_entry_mtx);
1042 	sc->sc_wrports = false;
1043 	cv_signal(&sc->sc_ports_cv);
1044 	mutex_exit(&sc->sc_entry_mtx);
1045 }
1046 
1047 static int
1048 agr_ioctl(struct ifnet *ifp, const u_long cmd, void *data)
1049 {
1050 	struct agr_softc *sc = ifp->if_softc;
1051 	struct ifreq *ifr = (struct ifreq *)data;
1052 	struct ifaddr *ifa = (struct ifaddr *)data;
1053 	struct agrreq ar;
1054 	int error;
1055 	bool in_ports = false;
1056 	int s;
1057 
1058 	if ((error = agr_enter(sc)) != 0)
1059 		return error;
1060 
1061 	s = splnet();
1062 
1063 	switch (cmd) {
1064 	case SIOCINITIFADDR:
1065 		in_ports = agr_ports_enter(sc);
1066 		if (sc->sc_nports == 0) {
1067 			error = EINVAL;
1068 			break;
1069 		}
1070 		ifp->if_flags |= IFF_UP;
1071 		switch (ifa->ifa_addr->sa_family) {
1072 #if defined(INET)
1073 		case AF_INET:
1074 			arp_ifinit(ifp, ifa);
1075 			break;
1076 #endif
1077 		default:
1078 			break;
1079 		}
1080 		break;
1081 
1082 #if 0 /* notyet */
1083 	case SIOCSIFMTU:
1084 #endif
1085 
1086 	case SIOCSIFFLAGS:
1087 		/*
1088 		 * Check for a change in vlan status.  This ioctl is the
1089 		 * only way we can tell that a vlan has attached or detached.
1090 		 * Note the agr interface must be up.
1091 		 */
1092 		agr_vlan_check(ifp, sc);
1093 
1094 		if ((error = ifioctl_common(ifp, cmd, data)) != 0)
1095 			break;
1096 		agr_config_promisc(sc);
1097 		break;
1098 
1099 	case SIOCSETAGR:
1100 		splx(s);
1101 		error = kauth_authorize_network(kauth_cred_get(),
1102 		    KAUTH_NETWORK_INTERFACE,
1103 		    KAUTH_REQ_NETWORK_INTERFACE_SETPRIV, ifp, (void *)cmd,
1104 		    NULL);
1105 		if (!error) {
1106 			error = agrreq_copyin(ifr->ifr_data, &ar);
1107 		}
1108 		if (!error) {
1109 			error = agr_setconfig(sc, &ar);
1110 		}
1111 		s = splnet();
1112 		break;
1113 
1114 	case SIOCGETAGR:
1115 		splx(s);
1116 		error = agrreq_copyin(ifr->ifr_data, &ar);
1117 		if (!error) {
1118 			error = agr_getconfig(sc, &ar);
1119 		}
1120 		if (!error) {
1121 			error = agrreq_copyout(ifr->ifr_data, &ar);
1122 		}
1123 		s = splnet();
1124 		break;
1125 
1126 	case SIOCADDMULTI:
1127 	case SIOCDELMULTI:
1128 		in_ports = agr_ports_enter(sc);
1129 		if (sc->sc_nports == 0)
1130 			error = EINVAL;
1131 		else
1132 			error = agr_ioctl_multi(ifp, cmd, ifr);
1133 		break;
1134 
1135 	default:
1136 		error = ifioctl_common(ifp, cmd, data);
1137 		break;
1138 	}
1139 
1140 	if (in_ports)
1141 		agr_ports_exit(sc);
1142 
1143 	splx(s);
1144 
1145 	agr_exit(sc);
1146 
1147 	return error;
1148 }
1149 
1150 static int
1151 agr_config_promisc(struct agr_softc *sc)
1152 {
1153 	int error;
1154 
1155 	agr_port_foreach(sc, agrport_config_promisc_callback, &error);
1156 
1157 	return error;
1158 }
1159 
1160 static int
1161 agrport_config_promisc_callback(struct agr_port *port, void *arg)
1162 {
1163 	struct agr_softc *sc = AGR_SC_FROM_PORT(port);
1164 	int *errorp = arg;
1165 	int error;
1166 	bool promisc;
1167 
1168 	promisc = (sc->sc_if.if_flags & IFF_PROMISC) != 0;
1169 
1170 	error = agrport_config_promisc(port, promisc);
1171 	if (error) {
1172 		*errorp = error;
1173 	}
1174 
1175 	return 0;
1176 }
1177 
1178 static int
1179 agrport_config_promisc(struct agr_port *port, bool promisc)
1180 {
1181 	int error;
1182 
1183 	if (( promisc && (port->port_flags & AGRPORT_PROMISC) != 0) ||
1184 	    (!promisc && (port->port_flags & AGRPORT_PROMISC) == 0)) {
1185 		return 0;
1186 	}
1187 
1188 	error = ifpromisc(port->port_ifp, promisc);
1189 	if (error == 0) {
1190 		if (promisc) {
1191 			port->port_flags |= AGRPORT_PROMISC;
1192 		} else {
1193 			port->port_flags &= ~AGRPORT_PROMISC;
1194 		}
1195 	}
1196 
1197 	return error;
1198 }
1199