xref: /openbsd-src/sys/net/if_mpw.c (revision 99fd087599a8791921855f21bd7e36130f39aadc)
1 /*	$OpenBSD: if_mpw.c,v 1.55 2019/06/26 08:13:13 claudio Exp $ */
2 
3 /*
4  * Copyright (c) 2015 Rafael Zalamena <rzalamena@openbsd.org>
5  *
6  * Permission to use, copy, modify, and distribute this software for any
7  * purpose with or without fee is hereby granted, provided that the above
8  * copyright notice and this permission notice appear in all copies.
9  *
10  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
11  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
13  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
15  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
16  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17  */
18 
19 #include "bpfilter.h"
20 #include "vlan.h"
21 
22 #include <sys/param.h>
23 #include <sys/systm.h>
24 #include <sys/mbuf.h>
25 #include <sys/socket.h>
26 #include <sys/ioctl.h>
27 #include <sys/errno.h>
28 
29 #include <net/if.h>
30 #include <net/if_dl.h>
31 #include <net/if_types.h>
32 #include <net/route.h>
33 
34 #include <netinet/in.h>
35 
36 #include <netinet/if_ether.h>
37 #include <netmpls/mpls.h>
38 
39 #if NBPFILTER > 0
40 #include <net/bpf.h>
41 #endif /* NBPFILTER */
42 
43 #if NVLAN > 0
44 #include <net/if_vlan_var.h>
45 #endif
46 
47 struct mpw_neighbor {
48 	struct shim_hdr		n_rshim;
49 	struct sockaddr_storage	n_nexthop;
50 };
51 
52 struct mpw_softc {
53 	struct arpcom		sc_ac;
54 #define sc_if			sc_ac.ac_if
55 
56 	int			sc_txhprio;
57 	int			sc_rxhprio;
58 	unsigned int		sc_rdomain;
59 	struct ifaddr		sc_ifa;
60 	struct sockaddr_mpls	sc_smpls; /* Local label */
61 
62 	unsigned int		sc_cword;
63 	unsigned int		sc_fword;
64 	uint32_t		sc_flow;
65 	uint32_t		sc_type;
66 	struct mpw_neighbor	*sc_neighbor;
67 
68 	unsigned int		sc_dead;
69 };
70 
71 void	mpwattach(int);
72 int	mpw_clone_create(struct if_clone *, int);
73 int	mpw_clone_destroy(struct ifnet *);
74 int	mpw_ioctl(struct ifnet *, u_long, caddr_t);
75 int	mpw_output(struct ifnet *, struct mbuf *, struct sockaddr *,
76     struct rtentry *);
77 void	mpw_start(struct ifnet *);
78 #if NVLAN > 0
79 struct	mbuf *mpw_vlan_handle(struct mbuf *, struct mpw_softc *);
80 #endif /* NVLAN */
81 
82 struct if_clone mpw_cloner =
83     IF_CLONE_INITIALIZER("mpw", mpw_clone_create, mpw_clone_destroy);
84 
85 void
86 mpwattach(int n)
87 {
88 	if_clone_attach(&mpw_cloner);
89 }
90 
91 int
92 mpw_clone_create(struct if_clone *ifc, int unit)
93 {
94 	struct mpw_softc *sc;
95 	struct ifnet *ifp;
96 
97 	sc = malloc(sizeof(*sc), M_DEVBUF, M_WAITOK|M_CANFAIL|M_ZERO);
98 	if (sc == NULL)
99 		return (ENOMEM);
100 
101 	sc->sc_flow = arc4random();
102 	sc->sc_neighbor = NULL;
103 
104 	ifp = &sc->sc_if;
105 	snprintf(ifp->if_xname, sizeof(ifp->if_xname), "%s%d",
106 	    ifc->ifc_name, unit);
107 	ifp->if_softc = sc;
108 	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
109 	ifp->if_xflags = IFXF_CLONED;
110 	ifp->if_ioctl = mpw_ioctl;
111 	ifp->if_output = mpw_output;
112 	ifp->if_start = mpw_start;
113 	ifp->if_hardmtu = ETHER_MAX_HARDMTU_LEN;
114 	IFQ_SET_MAXLEN(&ifp->if_snd, IFQ_MAXLEN);
115 	ether_fakeaddr(ifp);
116 
117 	sc->sc_dead = 0;
118 
119 	if_counters_alloc(ifp);
120 	if_attach(ifp);
121 	ether_ifattach(ifp);
122 
123 	sc->sc_txhprio = 0;
124 	sc->sc_rxhprio = IF_HDRPRIO_PACKET;
125 	sc->sc_rdomain = 0;
126 	sc->sc_ifa.ifa_ifp = ifp;
127 	sc->sc_ifa.ifa_addr = sdltosa(ifp->if_sadl);
128 	sc->sc_smpls.smpls_len = sizeof(sc->sc_smpls);
129 	sc->sc_smpls.smpls_family = AF_MPLS;
130 
131 	return (0);
132 }
133 
134 int
135 mpw_clone_destroy(struct ifnet *ifp)
136 {
137 	struct mpw_softc *sc = ifp->if_softc;
138 
139 	NET_LOCK();
140 	ifp->if_flags &= ~IFF_RUNNING;
141 	sc->sc_dead = 1;
142 
143 	if (sc->sc_smpls.smpls_label) {
144 		rt_ifa_del(&sc->sc_ifa, RTF_MPLS|RTF_LOCAL,
145 		    smplstosa(&sc->sc_smpls), sc->sc_rdomain);
146 	}
147 	NET_UNLOCK();
148 
149 	ifq_barrier(&ifp->if_snd);
150 
151 	ether_ifdetach(ifp);
152 	if_detach(ifp);
153 
154 	free(sc->sc_neighbor, M_DEVBUF, sizeof(*sc->sc_neighbor));
155 	free(sc, M_DEVBUF, sizeof(*sc));
156 
157 	return (0);
158 }
159 
160 int
161 mpw_set_route(struct mpw_softc *sc, uint32_t label, unsigned int rdomain)
162 {
163 	int error;
164 
165 	if (sc->sc_dead)
166 		return (ENXIO);
167 
168 	if (sc->sc_smpls.smpls_label) {
169 		rt_ifa_del(&sc->sc_ifa, RTF_MPLS|RTF_LOCAL,
170 		    smplstosa(&sc->sc_smpls), sc->sc_rdomain);
171 	}
172 
173 	sc->sc_smpls.smpls_label = label;
174 	sc->sc_rdomain = rdomain;
175 
176 	error = rt_ifa_add(&sc->sc_ifa, RTF_MPLS|RTF_LOCAL,
177 	    smplstosa(&sc->sc_smpls), sc->sc_rdomain);
178 	if (error != 0)
179 		sc->sc_smpls.smpls_label = 0;
180 
181 	return (error);
182 }
183 
184 static int
185 mpw_set_neighbor(struct mpw_softc *sc, const struct if_laddrreq *req)
186 {
187 	struct mpw_neighbor *n, *o;
188 	const struct sockaddr_storage *ss;
189 	const struct sockaddr_mpls *smpls;
190 	uint32_t label;
191 
192 	smpls = (const struct sockaddr_mpls *)&req->dstaddr;
193 
194 	if (smpls->smpls_family != AF_MPLS)
195 		return (EINVAL);
196 	label = smpls->smpls_label;
197 	if (label > MPLS_LABEL_MAX || label <= MPLS_LABEL_RESERVED_MAX)
198 		return (EINVAL);
199 
200 	ss = &req->addr;
201 	switch (ss->ss_family) {
202 	case AF_INET: {
203 		const struct sockaddr_in *sin =
204 		    (const struct sockaddr_in *)ss;
205 
206 		if (in_nullhost(sin->sin_addr) ||
207 		    IN_MULTICAST(sin->sin_addr.s_addr))
208 			return (EINVAL);
209 
210 		break;
211 	}
212 #ifdef INET6
213 	case AF_INET6: {
214 		const struct sockaddr_in6 *sin6 =
215 		    (const struct sockaddr_in6 *)ss;
216 
217 		if (IN6_IS_ADDR_UNSPECIFIED(&sin6->sin6_addr) ||
218 		    IN6_IS_ADDR_MULTICAST(&sin6->sin6_addr))
219 			return (EINVAL);
220 
221 		/* check scope */
222 
223 		break;
224 	}
225 #endif
226 	default:
227 		return (EAFNOSUPPORT);
228 	}
229 
230 	if (sc->sc_dead)
231 		return (ENXIO);
232 
233 	n = malloc(sizeof(*n), M_DEVBUF, M_WAITOK|M_CANFAIL|M_ZERO);
234 	if (n == NULL)
235 		return (ENOMEM);
236 
237 	n->n_rshim.shim_label = MPLS_LABEL2SHIM(label);
238 	n->n_nexthop = *ss;
239 
240 	o = sc->sc_neighbor;
241 	sc->sc_neighbor = n;
242 
243 	NET_UNLOCK();
244 	ifq_barrier(&sc->sc_if.if_snd);
245 	NET_LOCK();
246 
247 	free(o, M_DEVBUF, sizeof(*o));
248 
249 	return (0);
250 }
251 
252 static int
253 mpw_get_neighbor(struct mpw_softc *sc, struct if_laddrreq *req)
254 {
255 	struct sockaddr_mpls *smpls = (struct sockaddr_mpls *)&req->dstaddr;
256 	struct mpw_neighbor *n = sc->sc_neighbor;
257 
258 	if (n == NULL)
259 		return (EADDRNOTAVAIL);
260 
261 	smpls->smpls_len = sizeof(*smpls);
262 	smpls->smpls_family = AF_MPLS;
263 	smpls->smpls_label = MPLS_SHIM2LABEL(n->n_rshim.shim_label);
264 
265 	req->addr = n->n_nexthop;
266 
267 	return (0);
268 }
269 
270 static int
271 mpw_del_neighbor(struct mpw_softc *sc)
272 {
273 	struct mpw_neighbor *o;
274 
275 	if (sc->sc_dead)
276 		return (ENXIO);
277 
278 	o = sc->sc_neighbor;
279 	sc->sc_neighbor = NULL;
280 
281 	NET_UNLOCK();
282 	ifq_barrier(&sc->sc_if.if_snd);
283 	NET_LOCK();
284 
285 	free(o, M_DEVBUF, sizeof(*o));
286 
287 	return (0);
288 }
289 
290 static int
291 mpw_set_label(struct mpw_softc *sc, const struct shim_hdr *label)
292 {
293 	uint32_t shim;
294 
295 	if (label->shim_label > MPLS_LABEL_MAX ||
296 	    label->shim_label <= MPLS_LABEL_RESERVED_MAX)
297 		return (EINVAL);
298 
299 	shim = MPLS_LABEL2SHIM(label->shim_label);
300 	if (sc->sc_smpls.smpls_label == shim)
301 		return (0);
302 
303 	return (mpw_set_route(sc, shim, sc->sc_rdomain));
304 }
305 
306 static int
307 mpw_get_label(struct mpw_softc *sc, struct ifreq *ifr)
308 {
309 	struct shim_hdr label;
310 
311 	label.shim_label = MPLS_SHIM2LABEL(sc->sc_smpls.smpls_label);
312 
313 	if (label.shim_label == 0)
314 		return (EADDRNOTAVAIL);
315 
316 	return (copyout(&label, ifr->ifr_data, sizeof(label)));
317 }
318 
319 static int
320 mpw_del_label(struct mpw_softc *sc)
321 {
322 	if (sc->sc_dead)
323 		return (ENXIO);
324 
325 	if (sc->sc_smpls.smpls_label != MPLS_LABEL2SHIM(0)) {
326 		rt_ifa_del(&sc->sc_ifa, RTF_MPLS | RTF_LOCAL,
327 		    smplstosa(&sc->sc_smpls), 0);
328 	}
329 
330 	sc->sc_smpls.smpls_label = MPLS_LABEL2SHIM(0);
331 
332 	return (0);
333 }
334 
335 static int
336 mpw_set_config(struct mpw_softc *sc, const struct ifreq *ifr)
337 {
338 	struct ifmpwreq imr;
339 	struct if_laddrreq req;
340 	struct sockaddr_mpls *smpls;
341 	struct sockaddr_in *sin;
342 	int error;
343 
344 	error = copyin(ifr->ifr_data, &imr, sizeof(imr));
345 	if (error != 0)
346 		return (error);
347 
348 	/* Teardown all configuration if got no nexthop */
349 	sin = (struct sockaddr_in *)&imr.imr_nexthop;
350 	if (sin->sin_addr.s_addr == 0) {
351 		mpw_del_label(sc);
352 		mpw_del_neighbor(sc);
353 		sc->sc_cword = 0;
354 		sc->sc_type = 0;
355 		return (0);
356 	}
357 
358 	error = mpw_set_label(sc, &imr.imr_lshim);
359 	if (error != 0)
360 		return (error);
361 
362 	smpls = (struct sockaddr_mpls *)&req.dstaddr;
363 	smpls->smpls_family = AF_MPLS;
364 	smpls->smpls_label = imr.imr_rshim.shim_label;
365 	req.addr = imr.imr_nexthop;
366 
367 	error = mpw_set_neighbor(sc, &req);
368 	if (error != 0)
369 		return (error);
370 
371 	sc->sc_cword = ISSET(imr.imr_flags, IMR_FLAG_CONTROLWORD);
372 	sc->sc_type = imr.imr_type;
373 
374 	return (0);
375 }
376 
377 static int
378 mpw_get_config(struct mpw_softc *sc, const struct ifreq *ifr)
379 {
380 	struct ifmpwreq imr;
381 
382 	memset(&imr, 0, sizeof(imr));
383 	imr.imr_flags = sc->sc_cword ? IMR_FLAG_CONTROLWORD : 0;
384 	imr.imr_type = sc->sc_type;
385 
386 	imr.imr_lshim.shim_label = MPLS_SHIM2LABEL(sc->sc_smpls.smpls_label);
387 	if (sc->sc_neighbor) {
388 		imr.imr_rshim.shim_label =
389 		    MPLS_SHIM2LABEL(sc->sc_neighbor->n_rshim.shim_label);
390 		imr.imr_nexthop = sc->sc_neighbor->n_nexthop;
391 	}
392 
393 	return (copyout(&imr, ifr->ifr_data, sizeof(imr)));
394 }
395 
396 int
397 mpw_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
398 {
399 	struct ifreq *ifr = (struct ifreq *) data;
400 	struct mpw_softc *sc = ifp->if_softc;
401 	struct shim_hdr shim;
402 	int error = 0;
403 
404 	switch (cmd) {
405 	case SIOCSIFFLAGS:
406 		if ((ifp->if_flags & IFF_UP))
407 			ifp->if_flags |= IFF_RUNNING;
408 		else
409 			ifp->if_flags &= ~IFF_RUNNING;
410 		break;
411 
412 	case SIOCGPWE3:
413 		ifr->ifr_pwe3 = IF_PWE3_ETHERNET;
414 		break;
415 	case SIOCSPWE3CTRLWORD:
416 		sc->sc_cword = ifr->ifr_pwe3 ? 1 : 0;
417 		break;
418 	case SIOCGPWE3CTRLWORD:
419 		ifr->ifr_pwe3 = sc->sc_cword;
420 		break;
421 	case SIOCSPWE3FAT:
422 		sc->sc_fword = ifr->ifr_pwe3 ? 1 : 0;
423 		break;
424 	case SIOCGPWE3FAT:
425 		ifr->ifr_pwe3 = sc->sc_fword;
426 		break;
427 
428 	case SIOCSPWE3NEIGHBOR:
429 		error = mpw_set_neighbor(sc, (struct if_laddrreq *)data);
430 		break;
431 	case SIOCGPWE3NEIGHBOR:
432 		error = mpw_get_neighbor(sc, (struct if_laddrreq *)data);
433 		break;
434 	case SIOCDPWE3NEIGHBOR:
435 		error = mpw_del_neighbor(sc);
436 		break;
437 
438 	case SIOCGETLABEL:
439 		error = mpw_get_label(sc, ifr);
440 		break;
441 	case SIOCSETLABEL:
442 		error = copyin(ifr->ifr_data, &shim, sizeof(shim));
443 		if (error != 0)
444 			break;
445 		error = mpw_set_label(sc, &shim);
446 		break;
447 
448 	case SIOCSETMPWCFG:
449 		error = mpw_set_config(sc, ifr);
450 		break;
451 
452 	case SIOCGETMPWCFG:
453 		error = mpw_get_config(sc, ifr);
454 		break;
455 
456 	case SIOCSLIFPHYRTABLE:
457 		if (ifr->ifr_rdomainid < 0 ||
458 		    ifr->ifr_rdomainid > RT_TABLEID_MAX ||
459 		    !rtable_exists(ifr->ifr_rdomainid) ||
460 		    ifr->ifr_rdomainid != rtable_l2(ifr->ifr_rdomainid)) {
461 			error = EINVAL;
462 			break;
463 		}
464 		if (sc->sc_rdomain != ifr->ifr_rdomainid) {
465 			error = mpw_set_route(sc, sc->sc_smpls.smpls_label,
466 			    ifr->ifr_rdomainid);
467 		}
468 		break;
469 	case SIOCGLIFPHYRTABLE:
470 		ifr->ifr_rdomainid = sc->sc_rdomain;
471 		break;
472 
473 	case SIOCSTXHPRIO:
474 		error = if_txhprio_l2_check(ifr->ifr_hdrprio);
475 		if (error != 0)
476 			break;
477 
478 		sc->sc_txhprio = ifr->ifr_hdrprio;
479 		break;
480 	case SIOCGTXHPRIO:
481 		ifr->ifr_hdrprio = sc->sc_txhprio;
482 		break;
483 
484 	case SIOCSRXHPRIO:
485 		error = if_rxhprio_l2_check(ifr->ifr_hdrprio);
486 		if (error != 0)
487 			break;
488 
489 		sc->sc_rxhprio = ifr->ifr_hdrprio;
490 		break;
491 	case SIOCGRXHPRIO:
492 		ifr->ifr_hdrprio = sc->sc_rxhprio;
493 		break;
494 
495 	case SIOCADDMULTI:
496 	case SIOCDELMULTI:
497 		break;
498 
499 	default:
500 		error = ether_ioctl(ifp, &sc->sc_ac, cmd, data);
501 		break;
502 	}
503 
504 	return (error);
505 }
506 
507 static void
508 mpw_input(struct mpw_softc *sc, struct mbuf *m)
509 {
510 	struct ifnet *ifp = &sc->sc_if;
511 	struct shim_hdr *shim;
512 	struct mbuf *n;
513 	uint32_t exp;
514 	int rxprio;
515 	int off;
516 
517 	if (!ISSET(ifp->if_flags, IFF_RUNNING))
518 		goto drop;
519 
520 	shim = mtod(m, struct shim_hdr *);
521 	exp = ntohl(shim->shim_label & MPLS_EXP_MASK) >> MPLS_EXP_OFFSET;
522 	if (sc->sc_fword) {
523 		uint32_t flow;
524 
525 		if (MPLS_BOS_ISSET(shim->shim_label))
526 			goto drop;
527 		m_adj(m, sizeof(*shim));
528 
529 		if (m->m_len < sizeof(*shim)) {
530 			m = m_pullup(m, sizeof(*shim));
531 			if (m == NULL)
532 				return;
533 		}
534 		shim = mtod(m, struct shim_hdr *);
535 
536 		if (!MPLS_BOS_ISSET(shim->shim_label))
537 			goto drop;
538 
539 		flow = MPLS_SHIM2LABEL(shim->shim_label);
540 		flow ^= sc->sc_flow;
541 		m->m_pkthdr.ph_flowid = M_FLOWID_VALID | flow;
542 	} else {
543 		if (!MPLS_BOS_ISSET(shim->shim_label))
544 			goto drop;
545 	}
546 	m_adj(m, sizeof(*shim));
547 
548 	if (sc->sc_cword) {
549 		if (m->m_len < sizeof(*shim)) {
550 			m = m_pullup(m, sizeof(*shim));
551 			if (m == NULL)
552 				return;
553 		}
554 		shim = mtod(m, struct shim_hdr *);
555 
556 		/*
557 		 * The first 4 bits identifies that this packet is a
558 		 * control word. If the control word is configured and
559 		 * we received an IP datagram we shall drop it.
560 		 */
561 		if (shim->shim_label & CW_ZERO_MASK) {
562 			ifp->if_ierrors++;
563 			goto drop;
564 		}
565 
566 		/* We don't support fragmentation just yet. */
567 		if (shim->shim_label & CW_FRAG_MASK) {
568 			ifp->if_ierrors++;
569 			goto drop;
570 		}
571 
572 		m_adj(m, MPLS_HDRLEN);
573 	}
574 
575 	if (m->m_len < sizeof(struct ether_header)) {
576 		m = m_pullup(m, sizeof(struct ether_header));
577 		if (m == NULL)
578 			return;
579 	}
580 
581 	n = m_getptr(m, sizeof(struct ether_header), &off);
582 	if (n == NULL) {
583 		ifp->if_ierrors++;
584 		goto drop;
585 	}
586 	if (!ALIGNED_POINTER(mtod(n, caddr_t) + off, uint32_t)) {
587 		n = m_dup_pkt(m, ETHER_ALIGN, M_NOWAIT);
588 		/* Dispose of the original mbuf chain */
589 		m_freem(m);
590 		if (n == NULL)
591 			return;
592 		m = n;
593 	}
594 
595 	rxprio = sc->sc_rxhprio;
596 	switch (rxprio) {
597 	case IF_HDRPRIO_PACKET:
598 		/* nop */
599 		break;
600 	case IF_HDRPRIO_OUTER:
601 		m->m_pkthdr.pf.prio = exp;
602 		break;
603 	default:
604 		m->m_pkthdr.pf.prio = rxprio;
605 		break;
606 	}
607 
608 	m->m_pkthdr.ph_ifidx = ifp->if_index;
609 	m->m_pkthdr.ph_rtableid = ifp->if_rdomain;
610 
611 	/* packet has not been processed by PF yet. */
612 	KASSERT(m->m_pkthdr.pf.statekey == NULL);
613 
614 	if_vinput(ifp, m);
615 	return;
616 drop:
617 	m_freem(m);
618 }
619 
620 int
621 mpw_output(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst,
622     struct rtentry *rt)
623 {
624 	struct mpw_softc *sc = ifp->if_softc;
625 
626 	if (dst->sa_family == AF_LINK &&
627 	    rt != NULL && ISSET(rt->rt_flags, RTF_LOCAL)) {
628 		mpw_input(sc, m);
629 		return (0);
630 	}
631 
632 	return (ether_output(ifp, m, dst, rt));
633 }
634 
635 void
636 mpw_start(struct ifnet *ifp)
637 {
638 	struct mpw_softc *sc = ifp->if_softc;
639 	struct rtentry *rt;
640 	struct ifnet *ifp0;
641 	struct mbuf *m, *m0;
642 	struct shim_hdr *shim;
643 	struct mpw_neighbor *n;
644 	struct sockaddr_mpls smpls = {
645 		.smpls_len = sizeof(smpls),
646 		.smpls_family = AF_MPLS,
647 	};
648 	int txprio = sc->sc_txhprio;
649 	uint8_t prio;
650 	uint32_t exp, bos;
651 
652 	n = sc->sc_neighbor;
653 	if (!ISSET(ifp->if_flags, IFF_RUNNING) ||
654 	    n == NULL) {
655 		IFQ_PURGE(&ifp->if_snd);
656 		return;
657 	}
658 
659 	rt = rtalloc(sstosa(&n->n_nexthop), RT_RESOLVE, sc->sc_rdomain);
660 	if (!rtisvalid(rt)) {
661 		IFQ_PURGE(&ifp->if_snd);
662 		goto rtfree;
663 	}
664 
665 	ifp0 = if_get(rt->rt_ifidx);
666 	if (ifp0 == NULL) {
667 		IFQ_PURGE(&ifp->if_snd);
668 		goto rtfree;
669 	}
670 
671 	while ((m = ifq_dequeue(&ifp->if_snd)) != NULL) {
672 #if NBPFILTER > 0
673 		if (sc->sc_if.if_bpf)
674 			bpf_mtap(sc->sc_if.if_bpf, m, BPF_DIRECTION_OUT);
675 #endif /* NBPFILTER */
676 
677 		m0 = m_get(M_DONTWAIT, m->m_type);
678 		if (m0 == NULL) {
679 			m_freem(m);
680 			continue;
681 		}
682 
683 		M_MOVE_PKTHDR(m0, m);
684 		m0->m_next = m;
685 		m_align(m0, 0);
686 		m0->m_len = 0;
687 
688 		if (sc->sc_cword) {
689 			m0 = m_prepend(m0, sizeof(*shim), M_NOWAIT);
690 			if (m0 == NULL)
691 				continue;
692 
693 			shim = mtod(m0, struct shim_hdr *);
694 			memset(shim, 0, sizeof(*shim));
695 		}
696 
697 		switch (txprio) {
698 		case IF_HDRPRIO_PACKET:
699 			prio = m->m_pkthdr.pf.prio;
700 			break;
701 		default:
702 			prio = txprio;
703 			break;
704 		}
705 		exp = htonl(prio << MPLS_EXP_OFFSET);
706 
707 		bos = MPLS_BOS_MASK;
708 		if (sc->sc_fword) {
709 			uint32_t flow = sc->sc_flow;
710 
711 			m0 = m_prepend(m0, sizeof(*shim), M_NOWAIT);
712 			if (m0 == NULL)
713 				continue;
714 
715 			if (ISSET(m->m_pkthdr.ph_flowid, M_FLOWID_VALID))
716 				flow ^= m->m_pkthdr.ph_flowid & M_FLOWID_MASK;
717 
718 			shim = mtod(m0, struct shim_hdr *);
719 			shim->shim_label = htonl(1) & MPLS_TTL_MASK;
720 			shim->shim_label |= MPLS_LABEL2SHIM(flow) | exp | bos;
721 
722 			bos = 0;
723 		}
724 
725 		m0 = m_prepend(m0, sizeof(*shim), M_NOWAIT);
726 		if (m0 == NULL)
727 			continue;
728 
729 		shim = mtod(m0, struct shim_hdr *);
730 		shim->shim_label = htonl(mpls_defttl) & MPLS_TTL_MASK;
731 		shim->shim_label |= n->n_rshim.shim_label | exp | bos;
732 
733 		m0->m_pkthdr.ph_rtableid = sc->sc_rdomain;
734 		CLR(m0->m_flags, M_BCAST|M_MCAST);
735 
736 		mpls_output(ifp0, m0, (struct sockaddr *)&smpls, rt);
737 	}
738 
739 	if_put(ifp0);
740 rtfree:
741 	rtfree(rt);
742 }
743