xref: /openbsd-src/sys/net/if_mpw.c (revision 1a8dbaac879b9f3335ad7fb25429ce63ac1d6bac)
1 /*	$OpenBSD: if_mpw.c,v 1.59 2020/08/21 22:59:27 kn Exp $ */
2 
3 /*
4  * Copyright (c) 2015 Rafael Zalamena <rzalamena@openbsd.org>
5  *
6  * Permission to use, copy, modify, and distribute this software for any
7  * purpose with or without fee is hereby granted, provided that the above
8  * copyright notice and this permission notice appear in all copies.
9  *
10  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
11  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
13  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
15  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
16  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17  */
18 
19 #include "bpfilter.h"
20 #include "vlan.h"
21 
22 #include <sys/param.h>
23 #include <sys/systm.h>
24 #include <sys/mbuf.h>
25 #include <sys/socket.h>
26 #include <sys/ioctl.h>
27 #include <sys/errno.h>
28 
29 #include <net/if.h>
30 #include <net/if_dl.h>
31 #include <net/if_types.h>
32 #include <net/route.h>
33 
34 #include <netinet/in.h>
35 
36 #include <netinet/if_ether.h>
37 #include <netmpls/mpls.h>
38 
39 #if NBPFILTER > 0
40 #include <net/bpf.h>
41 #endif /* NBPFILTER */
42 
43 #if NVLAN > 0
44 #include <net/if_vlan_var.h>
45 #endif
46 
47 struct mpw_neighbor {
48 	struct shim_hdr		n_rshim;
49 	struct sockaddr_storage	n_nexthop;
50 };
51 
52 struct mpw_softc {
53 	struct arpcom		sc_ac;
54 #define sc_if			sc_ac.ac_if
55 
56 	int			sc_txhprio;
57 	int			sc_rxhprio;
58 	unsigned int		sc_rdomain;
59 	struct ifaddr		sc_ifa;
60 	struct sockaddr_mpls	sc_smpls; /* Local label */
61 
62 	unsigned int		sc_cword;
63 	unsigned int		sc_fword;
64 	uint32_t		sc_flow;
65 	uint32_t		sc_type;
66 	struct mpw_neighbor	*sc_neighbor;
67 
68 	unsigned int		sc_dead;
69 };
70 
71 void	mpwattach(int);
72 int	mpw_clone_create(struct if_clone *, int);
73 int	mpw_clone_destroy(struct ifnet *);
74 int	mpw_ioctl(struct ifnet *, u_long, caddr_t);
75 int	mpw_output(struct ifnet *, struct mbuf *, struct sockaddr *,
76     struct rtentry *);
77 void	mpw_start(struct ifnet *);
78 #if NVLAN > 0
79 struct	mbuf *mpw_vlan_handle(struct mbuf *, struct mpw_softc *);
80 #endif /* NVLAN */
81 
82 struct if_clone mpw_cloner =
83     IF_CLONE_INITIALIZER("mpw", mpw_clone_create, mpw_clone_destroy);
84 
85 void
86 mpwattach(int n)
87 {
88 	if_clone_attach(&mpw_cloner);
89 }
90 
91 int
92 mpw_clone_create(struct if_clone *ifc, int unit)
93 {
94 	struct mpw_softc *sc;
95 	struct ifnet *ifp;
96 
97 	sc = malloc(sizeof(*sc), M_DEVBUF, M_WAITOK|M_CANFAIL|M_ZERO);
98 	if (sc == NULL)
99 		return (ENOMEM);
100 
101 	sc->sc_flow = arc4random();
102 	sc->sc_neighbor = NULL;
103 
104 	ifp = &sc->sc_if;
105 	snprintf(ifp->if_xname, sizeof(ifp->if_xname), "%s%d",
106 	    ifc->ifc_name, unit);
107 	ifp->if_softc = sc;
108 	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
109 	ifp->if_xflags = IFXF_CLONED;
110 	ifp->if_ioctl = mpw_ioctl;
111 	ifp->if_output = mpw_output;
112 	ifp->if_start = mpw_start;
113 	ifp->if_hardmtu = ETHER_MAX_HARDMTU_LEN;
114 	ether_fakeaddr(ifp);
115 
116 	sc->sc_dead = 0;
117 
118 	if_counters_alloc(ifp);
119 	if_attach(ifp);
120 	ether_ifattach(ifp);
121 
122 	sc->sc_txhprio = 0;
123 	sc->sc_rxhprio = IF_HDRPRIO_PACKET;
124 	sc->sc_rdomain = 0;
125 	sc->sc_ifa.ifa_ifp = ifp;
126 	sc->sc_ifa.ifa_addr = sdltosa(ifp->if_sadl);
127 	sc->sc_smpls.smpls_len = sizeof(sc->sc_smpls);
128 	sc->sc_smpls.smpls_family = AF_MPLS;
129 
130 	return (0);
131 }
132 
133 int
134 mpw_clone_destroy(struct ifnet *ifp)
135 {
136 	struct mpw_softc *sc = ifp->if_softc;
137 
138 	NET_LOCK();
139 	ifp->if_flags &= ~IFF_RUNNING;
140 	sc->sc_dead = 1;
141 
142 	if (sc->sc_smpls.smpls_label) {
143 		rt_ifa_del(&sc->sc_ifa, RTF_MPLS|RTF_LOCAL,
144 		    smplstosa(&sc->sc_smpls), sc->sc_rdomain);
145 	}
146 	NET_UNLOCK();
147 
148 	ifq_barrier(&ifp->if_snd);
149 
150 	ether_ifdetach(ifp);
151 	if_detach(ifp);
152 
153 	free(sc->sc_neighbor, M_DEVBUF, sizeof(*sc->sc_neighbor));
154 	free(sc, M_DEVBUF, sizeof(*sc));
155 
156 	return (0);
157 }
158 
159 int
160 mpw_set_route(struct mpw_softc *sc, uint32_t label, unsigned int rdomain)
161 {
162 	int error;
163 
164 	if (sc->sc_dead)
165 		return (ENXIO);
166 
167 	if (sc->sc_smpls.smpls_label) {
168 		rt_ifa_del(&sc->sc_ifa, RTF_MPLS|RTF_LOCAL,
169 		    smplstosa(&sc->sc_smpls), sc->sc_rdomain);
170 	}
171 
172 	sc->sc_smpls.smpls_label = label;
173 	sc->sc_rdomain = rdomain;
174 
175 	error = rt_ifa_add(&sc->sc_ifa, RTF_MPLS|RTF_LOCAL,
176 	    smplstosa(&sc->sc_smpls), sc->sc_rdomain);
177 	if (error != 0)
178 		sc->sc_smpls.smpls_label = 0;
179 
180 	return (error);
181 }
182 
183 static int
184 mpw_set_neighbor(struct mpw_softc *sc, const struct if_laddrreq *req)
185 {
186 	struct mpw_neighbor *n, *o;
187 	const struct sockaddr_storage *ss;
188 	const struct sockaddr_mpls *smpls;
189 	uint32_t label;
190 
191 	smpls = (const struct sockaddr_mpls *)&req->dstaddr;
192 
193 	if (smpls->smpls_family != AF_MPLS)
194 		return (EINVAL);
195 	label = smpls->smpls_label;
196 	if (label > MPLS_LABEL_MAX || label <= MPLS_LABEL_RESERVED_MAX)
197 		return (EINVAL);
198 
199 	ss = &req->addr;
200 	switch (ss->ss_family) {
201 	case AF_INET: {
202 		const struct sockaddr_in *sin =
203 		    (const struct sockaddr_in *)ss;
204 
205 		if (in_nullhost(sin->sin_addr) ||
206 		    IN_MULTICAST(sin->sin_addr.s_addr))
207 			return (EINVAL);
208 
209 		break;
210 	}
211 #ifdef INET6
212 	case AF_INET6: {
213 		const struct sockaddr_in6 *sin6 =
214 		    (const struct sockaddr_in6 *)ss;
215 
216 		if (IN6_IS_ADDR_UNSPECIFIED(&sin6->sin6_addr) ||
217 		    IN6_IS_ADDR_MULTICAST(&sin6->sin6_addr))
218 			return (EINVAL);
219 
220 		/* check scope */
221 
222 		break;
223 	}
224 #endif
225 	default:
226 		return (EAFNOSUPPORT);
227 	}
228 
229 	if (sc->sc_dead)
230 		return (ENXIO);
231 
232 	n = malloc(sizeof(*n), M_DEVBUF, M_WAITOK|M_CANFAIL|M_ZERO);
233 	if (n == NULL)
234 		return (ENOMEM);
235 
236 	n->n_rshim.shim_label = MPLS_LABEL2SHIM(label);
237 	n->n_nexthop = *ss;
238 
239 	o = sc->sc_neighbor;
240 	sc->sc_neighbor = n;
241 
242 	NET_UNLOCK();
243 	ifq_barrier(&sc->sc_if.if_snd);
244 	NET_LOCK();
245 
246 	free(o, M_DEVBUF, sizeof(*o));
247 
248 	return (0);
249 }
250 
251 static int
252 mpw_get_neighbor(struct mpw_softc *sc, struct if_laddrreq *req)
253 {
254 	struct sockaddr_mpls *smpls = (struct sockaddr_mpls *)&req->dstaddr;
255 	struct mpw_neighbor *n = sc->sc_neighbor;
256 
257 	if (n == NULL)
258 		return (EADDRNOTAVAIL);
259 
260 	smpls->smpls_len = sizeof(*smpls);
261 	smpls->smpls_family = AF_MPLS;
262 	smpls->smpls_label = MPLS_SHIM2LABEL(n->n_rshim.shim_label);
263 
264 	req->addr = n->n_nexthop;
265 
266 	return (0);
267 }
268 
269 static int
270 mpw_del_neighbor(struct mpw_softc *sc)
271 {
272 	struct mpw_neighbor *o;
273 
274 	if (sc->sc_dead)
275 		return (ENXIO);
276 
277 	o = sc->sc_neighbor;
278 	sc->sc_neighbor = NULL;
279 
280 	NET_UNLOCK();
281 	ifq_barrier(&sc->sc_if.if_snd);
282 	NET_LOCK();
283 
284 	free(o, M_DEVBUF, sizeof(*o));
285 
286 	return (0);
287 }
288 
289 static int
290 mpw_set_label(struct mpw_softc *sc, const struct shim_hdr *label)
291 {
292 	uint32_t shim;
293 
294 	if (label->shim_label > MPLS_LABEL_MAX ||
295 	    label->shim_label <= MPLS_LABEL_RESERVED_MAX)
296 		return (EINVAL);
297 
298 	shim = MPLS_LABEL2SHIM(label->shim_label);
299 	if (sc->sc_smpls.smpls_label == shim)
300 		return (0);
301 
302 	return (mpw_set_route(sc, shim, sc->sc_rdomain));
303 }
304 
305 static int
306 mpw_get_label(struct mpw_softc *sc, struct ifreq *ifr)
307 {
308 	struct shim_hdr label;
309 
310 	label.shim_label = MPLS_SHIM2LABEL(sc->sc_smpls.smpls_label);
311 
312 	if (label.shim_label == 0)
313 		return (EADDRNOTAVAIL);
314 
315 	return (copyout(&label, ifr->ifr_data, sizeof(label)));
316 }
317 
318 static int
319 mpw_del_label(struct mpw_softc *sc)
320 {
321 	if (sc->sc_dead)
322 		return (ENXIO);
323 
324 	if (sc->sc_smpls.smpls_label != MPLS_LABEL2SHIM(0)) {
325 		rt_ifa_del(&sc->sc_ifa, RTF_MPLS | RTF_LOCAL,
326 		    smplstosa(&sc->sc_smpls), 0);
327 	}
328 
329 	sc->sc_smpls.smpls_label = MPLS_LABEL2SHIM(0);
330 
331 	return (0);
332 }
333 
334 static int
335 mpw_set_config(struct mpw_softc *sc, const struct ifreq *ifr)
336 {
337 	struct ifmpwreq imr;
338 	struct if_laddrreq req;
339 	struct sockaddr_mpls *smpls;
340 	struct sockaddr_in *sin;
341 	int error;
342 
343 	error = copyin(ifr->ifr_data, &imr, sizeof(imr));
344 	if (error != 0)
345 		return (error);
346 
347 	/* Teardown all configuration if got no nexthop */
348 	sin = (struct sockaddr_in *)&imr.imr_nexthop;
349 	if (sin->sin_addr.s_addr == 0) {
350 		mpw_del_label(sc);
351 		mpw_del_neighbor(sc);
352 		sc->sc_cword = 0;
353 		sc->sc_type = 0;
354 		return (0);
355 	}
356 
357 	error = mpw_set_label(sc, &imr.imr_lshim);
358 	if (error != 0)
359 		return (error);
360 
361 	smpls = (struct sockaddr_mpls *)&req.dstaddr;
362 	smpls->smpls_family = AF_MPLS;
363 	smpls->smpls_label = imr.imr_rshim.shim_label;
364 	req.addr = imr.imr_nexthop;
365 
366 	error = mpw_set_neighbor(sc, &req);
367 	if (error != 0)
368 		return (error);
369 
370 	sc->sc_cword = ISSET(imr.imr_flags, IMR_FLAG_CONTROLWORD);
371 	sc->sc_type = imr.imr_type;
372 
373 	return (0);
374 }
375 
376 static int
377 mpw_get_config(struct mpw_softc *sc, const struct ifreq *ifr)
378 {
379 	struct ifmpwreq imr;
380 
381 	memset(&imr, 0, sizeof(imr));
382 	imr.imr_flags = sc->sc_cword ? IMR_FLAG_CONTROLWORD : 0;
383 	imr.imr_type = sc->sc_type;
384 
385 	imr.imr_lshim.shim_label = MPLS_SHIM2LABEL(sc->sc_smpls.smpls_label);
386 	if (sc->sc_neighbor) {
387 		imr.imr_rshim.shim_label =
388 		    MPLS_SHIM2LABEL(sc->sc_neighbor->n_rshim.shim_label);
389 		imr.imr_nexthop = sc->sc_neighbor->n_nexthop;
390 	}
391 
392 	return (copyout(&imr, ifr->ifr_data, sizeof(imr)));
393 }
394 
395 int
396 mpw_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
397 {
398 	struct ifreq *ifr = (struct ifreq *) data;
399 	struct mpw_softc *sc = ifp->if_softc;
400 	struct shim_hdr shim;
401 	int error = 0;
402 
403 	switch (cmd) {
404 	case SIOCSIFFLAGS:
405 		if ((ifp->if_flags & IFF_UP))
406 			ifp->if_flags |= IFF_RUNNING;
407 		else
408 			ifp->if_flags &= ~IFF_RUNNING;
409 		break;
410 
411 	case SIOCGPWE3:
412 		ifr->ifr_pwe3 = IF_PWE3_ETHERNET;
413 		break;
414 	case SIOCSPWE3CTRLWORD:
415 		sc->sc_cword = ifr->ifr_pwe3 ? 1 : 0;
416 		break;
417 	case SIOCGPWE3CTRLWORD:
418 		ifr->ifr_pwe3 = sc->sc_cword;
419 		break;
420 	case SIOCSPWE3FAT:
421 		sc->sc_fword = ifr->ifr_pwe3 ? 1 : 0;
422 		break;
423 	case SIOCGPWE3FAT:
424 		ifr->ifr_pwe3 = sc->sc_fword;
425 		break;
426 
427 	case SIOCSPWE3NEIGHBOR:
428 		error = mpw_set_neighbor(sc, (struct if_laddrreq *)data);
429 		break;
430 	case SIOCGPWE3NEIGHBOR:
431 		error = mpw_get_neighbor(sc, (struct if_laddrreq *)data);
432 		break;
433 	case SIOCDPWE3NEIGHBOR:
434 		error = mpw_del_neighbor(sc);
435 		break;
436 
437 	case SIOCGETLABEL:
438 		error = mpw_get_label(sc, ifr);
439 		break;
440 	case SIOCSETLABEL:
441 		error = copyin(ifr->ifr_data, &shim, sizeof(shim));
442 		if (error != 0)
443 			break;
444 		error = mpw_set_label(sc, &shim);
445 		break;
446 
447 	case SIOCSETMPWCFG:
448 		error = mpw_set_config(sc, ifr);
449 		break;
450 
451 	case SIOCGETMPWCFG:
452 		error = mpw_get_config(sc, ifr);
453 		break;
454 
455 	case SIOCSLIFPHYRTABLE:
456 		if (ifr->ifr_rdomainid < 0 ||
457 		    ifr->ifr_rdomainid > RT_TABLEID_MAX ||
458 		    !rtable_exists(ifr->ifr_rdomainid) ||
459 		    ifr->ifr_rdomainid != rtable_l2(ifr->ifr_rdomainid)) {
460 			error = EINVAL;
461 			break;
462 		}
463 		if (sc->sc_rdomain != ifr->ifr_rdomainid) {
464 			error = mpw_set_route(sc, sc->sc_smpls.smpls_label,
465 			    ifr->ifr_rdomainid);
466 		}
467 		break;
468 	case SIOCGLIFPHYRTABLE:
469 		ifr->ifr_rdomainid = sc->sc_rdomain;
470 		break;
471 
472 	case SIOCSTXHPRIO:
473 		error = if_txhprio_l2_check(ifr->ifr_hdrprio);
474 		if (error != 0)
475 			break;
476 
477 		sc->sc_txhprio = ifr->ifr_hdrprio;
478 		break;
479 	case SIOCGTXHPRIO:
480 		ifr->ifr_hdrprio = sc->sc_txhprio;
481 		break;
482 
483 	case SIOCSRXHPRIO:
484 		error = if_rxhprio_l2_check(ifr->ifr_hdrprio);
485 		if (error != 0)
486 			break;
487 
488 		sc->sc_rxhprio = ifr->ifr_hdrprio;
489 		break;
490 	case SIOCGRXHPRIO:
491 		ifr->ifr_hdrprio = sc->sc_rxhprio;
492 		break;
493 
494 	case SIOCADDMULTI:
495 	case SIOCDELMULTI:
496 		break;
497 
498 	default:
499 		error = ether_ioctl(ifp, &sc->sc_ac, cmd, data);
500 		break;
501 	}
502 
503 	return (error);
504 }
505 
506 static void
507 mpw_input(struct mpw_softc *sc, struct mbuf *m)
508 {
509 	struct ifnet *ifp = &sc->sc_if;
510 	struct shim_hdr *shim;
511 	struct mbuf *n;
512 	uint32_t exp;
513 	int rxprio;
514 	int off;
515 
516 	if (!ISSET(ifp->if_flags, IFF_RUNNING))
517 		goto drop;
518 
519 	shim = mtod(m, struct shim_hdr *);
520 	exp = ntohl(shim->shim_label & MPLS_EXP_MASK) >> MPLS_EXP_OFFSET;
521 	if (sc->sc_fword) {
522 		uint32_t flow;
523 
524 		if (MPLS_BOS_ISSET(shim->shim_label))
525 			goto drop;
526 		m_adj(m, sizeof(*shim));
527 
528 		if (m->m_len < sizeof(*shim)) {
529 			m = m_pullup(m, sizeof(*shim));
530 			if (m == NULL)
531 				return;
532 		}
533 		shim = mtod(m, struct shim_hdr *);
534 
535 		if (!MPLS_BOS_ISSET(shim->shim_label))
536 			goto drop;
537 
538 		flow = MPLS_SHIM2LABEL(shim->shim_label);
539 		flow ^= sc->sc_flow;
540 		SET(m->m_pkthdr.csum_flags, M_FLOWID);
541 		m->m_pkthdr.ph_flowid = flow;
542 	} else {
543 		if (!MPLS_BOS_ISSET(shim->shim_label))
544 			goto drop;
545 	}
546 	m_adj(m, sizeof(*shim));
547 
548 	if (sc->sc_cword) {
549 		if (m->m_len < sizeof(*shim)) {
550 			m = m_pullup(m, sizeof(*shim));
551 			if (m == NULL)
552 				return;
553 		}
554 		shim = mtod(m, struct shim_hdr *);
555 
556 		/*
557 		 * The first 4 bits identifies that this packet is a
558 		 * control word. If the control word is configured and
559 		 * we received an IP datagram we shall drop it.
560 		 */
561 		if (shim->shim_label & CW_ZERO_MASK) {
562 			ifp->if_ierrors++;
563 			goto drop;
564 		}
565 
566 		/* We don't support fragmentation just yet. */
567 		if (shim->shim_label & CW_FRAG_MASK) {
568 			ifp->if_ierrors++;
569 			goto drop;
570 		}
571 
572 		m_adj(m, MPLS_HDRLEN);
573 	}
574 
575 	if (m->m_len < sizeof(struct ether_header)) {
576 		m = m_pullup(m, sizeof(struct ether_header));
577 		if (m == NULL)
578 			return;
579 	}
580 
581 	n = m_getptr(m, sizeof(struct ether_header), &off);
582 	if (n == NULL) {
583 		ifp->if_ierrors++;
584 		goto drop;
585 	}
586 	if (!ALIGNED_POINTER(mtod(n, caddr_t) + off, uint32_t)) {
587 		n = m_dup_pkt(m, ETHER_ALIGN, M_NOWAIT);
588 		/* Dispose of the original mbuf chain */
589 		m_freem(m);
590 		if (n == NULL)
591 			return;
592 		m = n;
593 	}
594 
595 	rxprio = sc->sc_rxhprio;
596 	switch (rxprio) {
597 	case IF_HDRPRIO_PACKET:
598 		/* nop */
599 		break;
600 	case IF_HDRPRIO_OUTER:
601 		m->m_pkthdr.pf.prio = exp;
602 		break;
603 	default:
604 		m->m_pkthdr.pf.prio = rxprio;
605 		break;
606 	}
607 
608 	m->m_pkthdr.ph_ifidx = ifp->if_index;
609 	m->m_pkthdr.ph_rtableid = ifp->if_rdomain;
610 
611 	/* packet has not been processed by PF yet. */
612 	KASSERT(m->m_pkthdr.pf.statekey == NULL);
613 
614 	if_vinput(ifp, m);
615 	return;
616 drop:
617 	m_freem(m);
618 }
619 
620 int
621 mpw_output(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst,
622     struct rtentry *rt)
623 {
624 	struct mpw_softc *sc = ifp->if_softc;
625 
626 	if (dst->sa_family == AF_LINK &&
627 	    rt != NULL && ISSET(rt->rt_flags, RTF_LOCAL)) {
628 		mpw_input(sc, m);
629 		return (0);
630 	}
631 
632 	return (ether_output(ifp, m, dst, rt));
633 }
634 
635 void
636 mpw_start(struct ifnet *ifp)
637 {
638 	struct mpw_softc *sc = ifp->if_softc;
639 	struct rtentry *rt;
640 	struct ifnet *ifp0;
641 	struct mbuf *m, *m0;
642 	struct shim_hdr *shim;
643 	struct mpw_neighbor *n;
644 	struct sockaddr_mpls smpls = {
645 		.smpls_len = sizeof(smpls),
646 		.smpls_family = AF_MPLS,
647 	};
648 	int txprio = sc->sc_txhprio;
649 	uint8_t prio;
650 	uint32_t exp, bos;
651 
652 	n = sc->sc_neighbor;
653 	if (!ISSET(ifp->if_flags, IFF_RUNNING) ||
654 	    n == NULL) {
655 		ifq_purge(&ifp->if_snd);
656 		return;
657 	}
658 
659 	rt = rtalloc(sstosa(&n->n_nexthop), RT_RESOLVE, sc->sc_rdomain);
660 	if (!rtisvalid(rt)) {
661 		ifq_purge(&ifp->if_snd);
662 		goto rtfree;
663 	}
664 
665 	ifp0 = if_get(rt->rt_ifidx);
666 	if (ifp0 == NULL) {
667 		ifq_purge(&ifp->if_snd);
668 		goto rtfree;
669 	}
670 
671 	while ((m = ifq_dequeue(&ifp->if_snd)) != NULL) {
672 #if NBPFILTER > 0
673 		if (sc->sc_if.if_bpf)
674 			bpf_mtap(sc->sc_if.if_bpf, m, BPF_DIRECTION_OUT);
675 #endif /* NBPFILTER */
676 
677 		m0 = m_get(M_DONTWAIT, m->m_type);
678 		if (m0 == NULL) {
679 			m_freem(m);
680 			continue;
681 		}
682 
683 		M_MOVE_PKTHDR(m0, m);
684 		m0->m_next = m;
685 		m_align(m0, 0);
686 		m0->m_len = 0;
687 
688 		if (sc->sc_cword) {
689 			m0 = m_prepend(m0, sizeof(*shim), M_NOWAIT);
690 			if (m0 == NULL)
691 				continue;
692 
693 			shim = mtod(m0, struct shim_hdr *);
694 			memset(shim, 0, sizeof(*shim));
695 		}
696 
697 		switch (txprio) {
698 		case IF_HDRPRIO_PACKET:
699 			prio = m->m_pkthdr.pf.prio;
700 			break;
701 		default:
702 			prio = txprio;
703 			break;
704 		}
705 		exp = htonl(prio << MPLS_EXP_OFFSET);
706 
707 		bos = MPLS_BOS_MASK;
708 		if (sc->sc_fword) {
709 			uint32_t flow = sc->sc_flow;
710 
711 			m0 = m_prepend(m0, sizeof(*shim), M_NOWAIT);
712 			if (m0 == NULL)
713 				continue;
714 
715 			if (ISSET(m->m_pkthdr.csum_flags, M_FLOWID))
716 				flow ^= m->m_pkthdr.ph_flowid;
717 
718 			shim = mtod(m0, struct shim_hdr *);
719 			shim->shim_label = htonl(1) & MPLS_TTL_MASK;
720 			shim->shim_label |= MPLS_LABEL2SHIM(flow) | exp | bos;
721 
722 			bos = 0;
723 		}
724 
725 		m0 = m_prepend(m0, sizeof(*shim), M_NOWAIT);
726 		if (m0 == NULL)
727 			continue;
728 
729 		shim = mtod(m0, struct shim_hdr *);
730 		shim->shim_label = htonl(mpls_defttl) & MPLS_TTL_MASK;
731 		shim->shim_label |= n->n_rshim.shim_label | exp | bos;
732 
733 		m0->m_pkthdr.ph_rtableid = sc->sc_rdomain;
734 		CLR(m0->m_flags, M_BCAST|M_MCAST);
735 
736 		mpls_output(ifp0, m0, (struct sockaddr *)&smpls, rt);
737 	}
738 
739 	if_put(ifp0);
740 rtfree:
741 	rtfree(rt);
742 }
743