Lines Matching defs:m
143 #define MEXT_ISEMBEDDED(m) ((m)->m_ext_ref == (m))
507 struct mbuf *m = object;
510 m->m_paddr = POOL_VTOPHYS(m);
512 m->m_paddr = M_PADDR_INVALID;
521 m_add(struct mbuf *c, struct mbuf *m)
526 return m;
530 n->m_next = m;
537 struct mbuf *m;
541 m = pool_cache_get(mb_cache,
543 if (m == NULL)
545 KASSERTMSG(((vaddr_t)m->m_dat & PAGE_MASK) + MLEN <= PAGE_SIZE,
546 "m=%p m->m_dat=%p"
548 m, m->m_dat,
553 mowner_init(m, type);
554 m->m_ext_ref = m; /* default */
555 m->m_type = type;
556 m->m_len = 0;
557 m->m_next = NULL;
558 m->m_nextpkt = NULL; /* default */
559 m->m_data = m->m_dat;
560 m->m_flags = 0; /* default */
562 return m;
568 struct mbuf *m;
570 m = m_get(how, type);
571 if (m == NULL)
574 m->m_data = m->m_pktdat;
575 m->m_flags = M_PKTHDR;
577 m_reset_rcvif(m);
578 m->m_pkthdr.len = 0;
579 m->m_pkthdr.csum_flags = 0;
580 m->m_pkthdr.csum_data = 0;
581 m->m_pkthdr.segsz = 0;
582 m->m_pkthdr.ether_vtag = 0;
583 m->m_pkthdr.pkthdr_flags = 0;
584 SLIST_INIT(&m->m_pkthdr.tags);
586 m->m_pkthdr.pattr_class = NULL;
587 m->m_pkthdr.pattr_af = AF_UNSPEC;
588 m->m_pkthdr.pattr_hdr = NULL;
590 return m;
596 struct mbuf *m;
600 if ((m = m_get(how, type)) == NULL)
603 m_clget(m, how);
604 if ((m->m_flags & M_EXT) == 0) {
605 m_free(m);
609 m->m_len = alignbytes + nbytes;
610 m_adj(m, alignbytes);
612 return m;
618 struct mbuf *m;
622 if ((m = m_gethdr(how, type)) == NULL)
625 m_clget(m, how);
626 if ((m->m_flags & M_EXT) == 0) {
627 m_free(m);
631 m->m_len = m->m_pkthdr.len = alignbytes + nbytes;
632 m_adj(m, alignbytes);
634 return m;
638 m_clget(struct mbuf *m, int how)
640 m->m_ext_storage.ext_buf = (char *)pool_cache_get_paddr(mcl_cache,
642 &m->m_ext_storage.ext_paddr);
644 if (m->m_ext_storage.ext_buf == NULL)
647 KASSERTMSG((((vaddr_t)m->m_ext_storage.ext_buf & PAGE_MASK) + mclbytes
649 "m=%p m->m_ext_storage.ext_buf=%p"
651 m, m->m_dat,
654 MCLINITREFERENCE(m);
655 m->m_data = m->m_ext.ext_buf;
656 m->m_flags = (m->m_flags & ~M_EXTCOPYFLAGS) |
658 m->m_ext.ext_size = MCLBYTES;
659 m->m_ext.ext_free = NULL;
660 m->m_ext.ext_arg = NULL;
663 mowner_ref(m, M_EXT|M_EXT_CLUSTER);
691 m_prepend(struct mbuf *m, int len, int how)
700 mn = m_get(how, m->m_type);
702 m_freem(m);
706 if (m->m_flags & M_PKTHDR) {
707 m_move_pkthdr(mn, m);
709 MCLAIM(mn, m->m_owner);
711 mn->m_next = m;
712 m = mn;
714 if (m->m_flags & M_PKTHDR) {
716 m_align(m, len);
719 m_align(m, len);
722 m->m_len = len;
723 return m;
727 m_copym(struct mbuf *m, int off, int len, int wait)
730 return m_copy_internal(m, off, len, wait, false);
734 m_dup(struct mbuf *m, int off, int len, int wait)
737 return m_copy_internal(m, off, len, wait, true);
747 m_copy_internal(struct mbuf *m, int off0, int len, int wait, bool deep)
749 struct mbuf *m0 __diagused = m;
758 if (off == 0 && m->m_flags & M_PKTHDR)
761 if (m == NULL)
762 panic("%s: m == NULL, off %d", __func__, off);
763 if (off < m->m_len)
765 off -= m->m_len;
766 m = m->m_next;
772 if (m == NULL) {
774 panic("%s: m == NULL, len %d [!COPYALL]",
779 n = m_get(wait, m->m_type);
783 MCLAIM(n, m->m_owner);
786 m_copy_pkthdr(n, m);
793 n->m_len = m_copylen(len, m->m_len - off);
795 if (m->m_flags & M_EXT) {
797 n->m_data = m->m_data + off;
798 MCLADDREFERENCE(m, n);
808 n->m_len = uimin(n->m_len, m->m_len - off);
809 memcpy(mtod(n, void *), mtod(m, char *) + off,
813 memcpy(mtod(n, void *), mtod(m, char *) + off,
821 KASSERTMSG(off <= m->m_len,
822 "m=%p m->m_len=%d off=%d len=%d m0=%p off0=%d len0=%d",
823 m, m->m_len, off, len, m0, off0, len0);
825 if (off == m->m_len) {
826 m = m->m_next;
841 * An optimization of the common case 'm_copym(m, 0, M_COPYALL, how)'.
844 m_copypacket(struct mbuf *m, int how)
848 if (__predict_false((m->m_flags & M_PKTHDR) == 0)) {
849 panic("%s: no header (m = %p)", __func__, m);
852 n = m_get(how, m->m_type);
857 MCLAIM(n, m->m_owner);
858 m_copy_pkthdr(n, m);
859 n->m_len = m->m_len;
860 if (m->m_flags & M_EXT) {
861 n->m_data = m->m_data;
862 MCLADDREFERENCE(m, n);
864 memcpy(mtod(n, char *), mtod(m, char *), n->m_len);
867 m = m->m_next;
868 while (m) {
869 o = m_get(how, m->m_type);
873 MCLAIM(o, m->m_owner);
877 n->m_len = m->m_len;
878 if (m->m_flags & M_EXT) {
879 n->m_data = m->m_data;
880 MCLADDREFERENCE(m, n);
882 memcpy(mtod(n, char *), mtod(m, char *), n->m_len);
885 m = m->m_next;
895 m_copydata(struct mbuf *m, int off, int len, void *cp)
898 struct mbuf *m0 = m;
907 if (m == NULL)
908 panic("m_copydata(%p,%d,%d,%p): m=NULL, off=%d (%d)",
910 if (off < m->m_len)
912 off -= m->m_len;
913 m = m->m_next;
916 if (m == NULL)
918 "m=NULL, off=%d (%d), len=%d (%d)",
921 count = uimin(m->m_len - off, len);
922 memcpy(cp, mtod(m, char *) + off, count);
926 m = m->m_next;
931 * Concatenate mbuf chain n to m.
932 * n might be copied into m (when n->m_len is small), therefore data portion of
937 m_cat(struct mbuf *m, struct mbuf *n)
940 while (m->m_next)
941 m = m->m_next;
943 if (M_READONLY(m) || n->m_len > M_TRAILINGSPACE(m)) {
945 m->m_next = n;
949 memcpy(mtod(m, char *) + m->m_len, mtod(n, void *),
951 m->m_len += n->m_len;
960 struct mbuf *m;
963 if ((m = mp) == NULL)
969 while (m != NULL && len > 0) {
970 if (m->m_len <= len) {
971 len -= m->m_len;
972 m->m_len = 0;
973 m = m->m_next;
975 m->m_len -= len;
976 m->m_data += len;
993 count += m->m_len;
994 if (m->m_next == NULL)
996 m = m->m_next;
998 if (m->m_len >= len) {
999 m->m_len -= len;
1014 m = mp;
1015 if (m->m_flags & M_PKTHDR)
1016 m->m_pkthdr.len = count;
1017 for (; m; m = m->m_next) {
1018 if (m->m_len >= count) {
1019 m->m_len = count;
1022 count -= m->m_len;
1024 if (m) {
1025 while (m->m_next)
1026 (m = m->m_next)->m_len = 0;
1042 struct mbuf *n = *m0, *m;
1056 m = n;
1058 len -= m->m_len;
1063 m = m_get(M_DONTWAIT, n->m_type);
1064 if (m == NULL) {
1067 MCLAIM(m, n->m_owner);
1069 m_move_pkthdr(m, n);
1072 space = &m->m_dat[MLEN] - (m->m_data + m->m_len);
1075 memcpy(mtod(m, char *) + m->m_len, mtod(n, void *),
1078 m->m_len += count;
1087 m->m_next = n;
1088 *m0 = m;
1099 struct mbuf *m = n;
1102 if (!m_ensure_contig(&m, len)) {
1103 KASSERT(m != NULL);
1104 m_freem(m);
1105 m = NULL;
1107 return m;
1111 * ensure that [off, off + len) is contiguous on the mbuf chain "m".
1116 * on error return (NULL return value), original "m" will be freed.
1121 m_pulldown(struct mbuf *m, int off, int len, int *offp)
1128 if (m == NULL)
1129 panic("%s: m == NULL", __func__);
1131 m_freem(m);
1135 n = m;
1146 m_freem(m);
1176 m_freem(m);
1208 m_freem(m);
1241 o = m_get(M_DONTWAIT, m->m_type);
1250 m_freem(m);
1280 struct mbuf *m;
1286 m = m_get(M_DONTWAIT, n->m_type);
1287 if (m == NULL)
1289 MCLAIM(m, n->m_owner);
1291 m_move_pkthdr(m, n);
1293 m->m_data += dstoff;
1294 space = &m->m_dat[MLEN] - (m->m_data + m->m_len);
1297 memcpy(mtod(m, char *) + m->m_len, mtod(n, void *),
1300 m->m_len += count;
1309 (void) m_free(m);
1312 m->m_next = n;
1313 return m;
1328 struct mbuf *m, *n;
1332 for (m = m0; m && len > m->m_len; m = m->m_next)
1333 len -= m->m_len;
1334 if (m == NULL)
1337 remain = m->m_len - len;
1349 if ((m->m_flags & M_EXT) == 0 && remain > MHLEN) {
1350 /* m can't be the lead packet */
1353 n->m_next = m_split(m, len, wait);
1362 n = m->m_next;
1363 m->m_next = NULL;
1366 n = m_get(wait, m->m_type);
1369 MCLAIM(n, m->m_owner);
1372 if (m->m_flags & M_EXT) {
1373 n->m_data = m->m_data + len;
1374 MCLADDREFERENCE(m, n);
1377 memcpy(mtod(n, void *), mtod(m, char *) + len, remain);
1381 m->m_len = len;
1382 n->m_next = m->m_next;
1383 m->m_next = NULL;
1393 struct mbuf *m;
1409 m = m_gethdr(M_DONTWAIT, MT_DATA);
1410 if (m == NULL)
1412 m_set_rcvif(m, ifp);
1413 m->m_pkthdr.len = totlen;
1414 m->m_len = MHLEN;
1418 m = m_get(M_DONTWAIT, MT_DATA);
1419 if (m == NULL) {
1423 m->m_len = MLEN;
1429 MCLGET(m, M_DONTWAIT);
1430 if ((m->m_flags & M_EXT) == 0) {
1431 m_free(m);
1435 m->m_len = len = uimin(len, MCLBYTES);
1440 if (len < m->m_len) {
1441 if (top == 0 && len + max_linkhdr <= m->m_len)
1442 m->m_data += max_linkhdr;
1443 m->m_len = len;
1445 len = m->m_len;
1448 memcpy(mtod(m, void *), cp, (size_t)len);
1451 *mp = m;
1452 mp = &m->m_next;
1542 struct mbuf *m, *n;
1563 m = *mp;
1564 while (off > (mlen = m->m_len)) {
1567 if (m->m_next == NULL) {
1574 * try to make some space at the end of "m".
1577 mlen = m->m_len;
1579 (m->m_flags & M_EXT) == 0 && m->m_len == 0) {
1580 MCLGET(m, how);
1582 tspace = M_TRAILINGSPACE(m);
1586 memset(mtod(m, char *) + m->m_len, 0,
1588 m->m_len += tspace;
1599 n = m_getcl(how, m->m_type, 0);
1601 n = m_get(how, m->m_type);
1608 m->m_next = n;
1610 mp = &m->m_next;
1611 m = m->m_next;
1614 mlen = m->m_len - off;
1615 if (mlen != 0 && M_READONLY(m)) {
1630 n = m_split_internal(m, off, how, false);
1633 m->m_next = n;
1634 mp = &m->m_next;
1635 m = n;
1648 n = m_get(how, m->m_type);
1651 MCLAIM(n, m->m_owner);
1652 if (off == 0 && (m->m_flags & M_PKTHDR) != 0) {
1653 m_move_pkthdr(n, m);
1673 while (m != NULL && M_READONLY(m) &&
1674 n->m_type == m->m_type && eatlen > 0) {
1675 mlen = uimin(eatlen, m->m_len);
1677 m_copydata(m, 0, mlen, datap);
1680 m->m_data += mlen;
1681 m->m_len -= mlen;
1683 if (m->m_len == 0)
1684 *mp = m = m_free(m);
1688 n->m_next = m;
1689 *mp = m = n;
1694 memcpy(mtod(m, char *) + off, cp, (unsigned)mlen);
1703 if (m->m_next == NULL) {
1706 mp = &m->m_next;
1707 m = m->m_next;
1711 if (((m = *mp0)->m_flags & M_PKTHDR) && (m->m_pkthdr.len < totlen)) {
1713 m->m_pkthdr.len = totlen;
1728 m_defrag(struct mbuf *m, int how)
1733 KASSERT((m->m_flags & M_PKTHDR) != 0);
1735 if (m->m_next == NULL)
1736 return m;
1739 if ((m->m_flags & M_EXT) == 0 && m->m_pkthdr.len <= MCLBYTES) {
1740 if (m->m_pkthdr.len <= MHLEN) {
1741 if (M_TRAILINGSPACE(m) < (m->m_pkthdr.len - m->m_len)) {
1742 KASSERTMSG(M_LEADINGSPACE(m) +
1743 M_TRAILINGSPACE(m) >=
1744 (m->m_pkthdr.len - m->m_len),
1747 (int)M_LEADINGSPACE(m),
1748 (int)M_TRAILINGSPACE(m),
1749 M_READONLY(m),
1750 m->m_pkthdr.len, m->m_len);
1752 memmove(m->m_pktdat, m->m_data, m->m_len);
1753 m->m_data = m->m_pktdat;
1755 KASSERT(M_TRAILINGSPACE(m) >=
1756 (m->m_pkthdr.len - m->m_len));
1763 KASSERTMSG(m->m_len <= MHLEN,
1764 "m=%p m->m_len=%d MHLEN=%u",
1765 m, m->m_len, (unsigned)MHLEN);
1766 m_copydata(m, 0, m->m_len, mtod(m0, void *));
1768 MCLGET(m, how);
1769 if ((m->m_flags & M_EXT) == 0) {
1773 memcpy(m->m_data, mtod(m0, void *), m->m_len);
1776 KASSERTMSG(M_TRAILINGSPACE(m) >= (m->m_pkthdr.len - m->m_len),
1777 "m=%p M_TRAILINGSPACE(m)=%zd m->m_pkthdr.len=%d"
1778 " m->m_len=%d",
1779 m, M_TRAILINGSPACE(m), m->m_pkthdr.len, m->m_len);
1780 m_copydata(m->m_next, 0, m->m_pkthdr.len - m->m_len,
1781 mtod(m, char *) + m->m_len);
1782 m->m_len = m->m_pkthdr.len;
1783 m_freem(m->m_next);
1784 m->m_next = NULL;
1785 return m;
1793 sz = m->m_pkthdr.len - m->m_len;
1807 m_copydata(m, m->m_pkthdr.len - sz, mn->m_len,
1825 m_freem(m->m_next);
1826 m->m_next = m0;
1828 return m;
1832 m_remove_pkthdr(struct mbuf *m)
1834 KASSERT(m->m_flags & M_PKTHDR);
1836 m_tag_delete_chain(m);
1837 m->m_flags &= ~M_PKTHDR;
1838 memset(&m->m_pkthdr, 0, sizeof(m->m_pkthdr));
1877 m_align(struct mbuf *m, int len)
1882 KASSERTMSG(M_LEADINGSPACE(m) == 0, "m=%p M_LEADINGSPACE(m)=%zd",
1883 m, M_LEADINGSPACE(m));
1885 buflen = M_BUFSIZE(m);
1887 KASSERTMSG(len <= buflen, "m=%p len=%d buflen=%d", m, len, buflen);
1889 m->m_data += adjust &~ (sizeof(long)-1);
1897 m_apply(struct mbuf *m, int off, int len,
1908 KASSERT(m != NULL);
1909 if (off < m->m_len)
1911 off -= m->m_len;
1912 m = m->m_next;
1915 KASSERT(m != NULL);
1916 count = uimin(m->m_len - off, len);
1918 rval = (*f)(arg, mtod(m, char *) + off, count);
1924 m = m->m_next;
1934 m_getptr(struct mbuf *m, int loc, int *off)
1939 if (m->m_len > loc) {
1941 return m;
1944 loc -= m->m_len;
1946 if (m->m_next == NULL) {
1949 *off = m->m_len;
1950 return m;
1954 m = m->m_next;
1964 * => free the mbuf m itself as well.
1967 m_ext_free(struct mbuf *m)
1969 const bool embedded = MEXT_ISEMBEDDED(m);
1973 KASSERT((m->m_flags & M_EXT) != 0);
1974 KASSERT(MEXT_ISEMBEDDED(m->m_ext_ref));
1975 KASSERT((m->m_ext_ref->m_flags & M_EXT) != 0);
1976 KASSERT((m->m_flags & M_EXT_CLUSTER) ==
1977 (m->m_ext_ref->m_flags & M_EXT_CLUSTER));
1979 if (__predict_false(m->m_type == MT_FREE)) {
1980 panic("mbuf %p already freed", m);
1983 if (__predict_true(m->m_ext.ext_refcnt == 1)) {
1984 refcnt = m->m_ext.ext_refcnt = 0;
1987 refcnt = atomic_dec_uint_nv(&m->m_ext.ext_refcnt);
1997 m->m_ext_ref = m;
2005 m->m_ext.ext_refcnt++; /* XXX */
2006 m_ext_free(m->m_ext_ref);
2007 m->m_ext_ref = m;
2008 } else if ((m->m_flags & M_EXT_CLUSTER) != 0) {
2010 m->m_ext.ext_buf, m->m_ext.ext_paddr);
2011 } else if (m->m_ext.ext_free) {
2012 (*m->m_ext.ext_free)(m,
2013 m->m_ext.ext_buf, m->m_ext.ext_size,
2014 m->m_ext.ext_arg);
2016 * 'm' is already freed by the ext_free callback.
2020 free(m->m_ext.ext_buf, 0);
2025 m->m_type = MT_FREE;
2026 m->m_data = NULL;
2027 pool_cache_put(mb_cache, m);
2036 m_free(struct mbuf *m)
2040 mowner_revoke(m, 1, m->m_flags);
2041 mbstat_type_add(m->m_type, -1);
2043 if (m->m_flags & M_PKTHDR)
2044 m_tag_delete_chain(m);
2046 n = m->m_next;
2048 if (m->m_flags & M_EXT) {
2049 m_ext_free(m);
2051 if (__predict_false(m->m_type == MT_FREE)) {
2052 panic("mbuf %p already freed", m);
2054 m->m_type = MT_FREE;
2055 m->m_data = NULL;
2056 pool_cache_put(mb_cache, m);
2063 m_freem(struct mbuf *m)
2065 if (m == NULL)
2068 m = m_free(m);
2069 } while (m);
2074 m_print(const struct mbuf *m, const char *modif, void (*pr)(const char *, ...))
2097 m0 = m;
2106 (*pr)("MBUF(%d) %p\n", no, m);
2107 snprintb(buf, sizeof(buf), M_FLAGS_BITS, (u_int)m->m_flags);
2109 m->m_data, m->m_len, m->m_type, buf);
2112 unsigned char *p = m->m_data;
2116 for (i = 0; i < m->m_len; i++) {
2124 (*pr)(" owner=%p, next=%p, nextpkt=%p\n", m->m_owner, m->m_next,
2125 m->m_nextpkt);
2127 (int)M_LEADINGSPACE(m), (int)M_TRAILINGSPACE(m),
2128 (int)M_READONLY(m));
2129 if ((m->m_flags & M_PKTHDR) != 0) {
2130 snprintb(buf, sizeof(buf), M_CSUM_BITS, m->m_pkthdr.csum_flags);
2133 m->m_pkthdr.len, m_get_rcvif_NOMPSAFE(m),
2134 buf, m->m_pkthdr.csum_data, m->m_pkthdr.segsz);
2136 if ((m->m_flags & M_EXT)) {
2139 m->m_ext.ext_refcnt,
2140 m->m_ext.ext_buf, m->m_ext.ext_size,
2141 m->m_ext.ext_free, m->m_ext.ext_arg);
2143 if ((~m->m_flags & (M_EXT|M_EXT_PAGES)) == 0) {
2144 vaddr_t sva = (vaddr_t)m->m_ext.ext_buf;
2145 vaddr_t eva = sva + m->m_ext.ext_size;
2151 (*pr)(" %p", m->m_ext.ext_pgs[i]);
2157 m = m->m_next;
2158 if (m != NULL) {
2205 mowner_init(struct mbuf *m, int type)
2211 m->m_owner = mo = &unknown_mowners[type];
2220 mowner_ref(struct mbuf *m, int flags)
2222 struct mowner *mo = m->m_owner;
2237 mowner_revoke(struct mbuf *m, bool all, int flags)
2239 struct mowner *mo = m->m_owner;
2254 m->m_owner = &revoked_mowner;
2258 mowner_claim(struct mbuf *m, struct mowner *mo)
2261 int flags = m->m_flags;
2273 m->m_owner = mo;
2277 m_claim(struct mbuf *m, struct mowner *mo)
2280 if (m->m_owner == mo || mo == NULL)
2283 mowner_revoke(m, true, m->m_flags);
2284 mowner_claim(m, mo);
2288 m_claimm(struct mbuf *m, struct mowner *mo)
2291 for (; m != NULL; m = m->m_next)
2292 m_claim(m, mo);
2302 m_verify_packet(struct mbuf *m)
2304 struct mbuf *n = m;
2308 if (__predict_false((m->m_flags & M_PKTHDR) == 0)) {
2321 if (__predict_false((n != m) && (n->m_flags & M_PKTHDR) != 0)) {
2348 if (__predict_false(totlen != m->m_pkthdr.len)) {
2350 totlen, m->m_pkthdr.len);
2377 m_tag_prepend(struct mbuf *m, struct m_tag *t)
2379 KASSERT((m->m_flags & M_PKTHDR) != 0);
2380 SLIST_INSERT_HEAD(&m->m_pkthdr.tags, t, m_tag_link);
2384 m_tag_unlink(struct mbuf *m, struct m_tag *t)
2386 KASSERT((m->m_flags & M_PKTHDR) != 0);
2387 SLIST_REMOVE(&m->m_pkthdr.tags, t, m_tag, m_tag_link);
2391 m_tag_delete(struct mbuf *m, struct m_tag *t)
2393 m_tag_unlink(m, t);
2398 m_tag_delete_chain(struct mbuf *m)
2402 KASSERT((m->m_flags & M_PKTHDR) != 0);
2404 p = SLIST_FIRST(&m->m_pkthdr.tags);
2408 m_tag_delete(m, q);
2409 m_tag_delete(m, p);
2413 m_tag_find(const struct mbuf *m, int type)
2417 KASSERT((m->m_flags & M_PKTHDR) != 0);
2419 p = SLIST_FIRST(&m->m_pkthdr.tags);