1 /* $NetBSD: uipc_mbuf.c,v 1.246 2022/04/09 23:38:33 riastradh Exp $ */ 2 3 /* 4 * Copyright (c) 1999, 2001, 2018 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 9 * NASA Ames Research Center, and Maxime Villard. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 * POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 /* 34 * Copyright (c) 1982, 1986, 1988, 1991, 1993 35 * The Regents of the University of California. All rights reserved. 36 * 37 * Redistribution and use in source and binary forms, with or without 38 * modification, are permitted provided that the following conditions 39 * are met: 40 * 1. Redistributions of source code must retain the above copyright 41 * notice, this list of conditions and the following disclaimer. 42 * 2. Redistributions in binary form must reproduce the above copyright 43 * notice, this list of conditions and the following disclaimer in the 44 * documentation and/or other materials provided with the distribution. 45 * 3. Neither the name of the University nor the names of its contributors 46 * may be used to endorse or promote products derived from this software 47 * without specific prior written permission. 48 * 49 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 50 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 51 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 52 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 53 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 54 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 55 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 56 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 57 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 58 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 59 * SUCH DAMAGE. 60 * 61 * @(#)uipc_mbuf.c 8.4 (Berkeley) 2/14/95 62 */ 63 64 #include <sys/cdefs.h> 65 __KERNEL_RCSID(0, "$NetBSD: uipc_mbuf.c,v 1.246 2022/04/09 23:38:33 riastradh Exp $"); 66 67 #ifdef _KERNEL_OPT 68 #include "opt_mbuftrace.h" 69 #include "opt_nmbclusters.h" 70 #include "opt_ddb.h" 71 #include "ether.h" 72 #endif 73 74 #include <sys/param.h> 75 #include <sys/systm.h> 76 #include <sys/atomic.h> 77 #include <sys/cpu.h> 78 #include <sys/proc.h> 79 #include <sys/mbuf.h> 80 #include <sys/kernel.h> 81 #include <sys/syslog.h> 82 #include <sys/domain.h> 83 #include <sys/protosw.h> 84 #include <sys/percpu.h> 85 #include <sys/pool.h> 86 #include <sys/socket.h> 87 #include <sys/sysctl.h> 88 89 #include <net/if.h> 90 91 pool_cache_t mb_cache; /* mbuf cache */ 92 static pool_cache_t mcl_cache; /* mbuf cluster cache */ 93 94 struct mbstat mbstat; 95 int max_linkhdr; 96 int max_protohdr; 97 int max_hdr; 98 int max_datalen; 99 100 static void mb_drain(void *, int); 101 static int mb_ctor(void *, void *, int); 102 103 static void sysctl_kern_mbuf_setup(void); 104 105 static struct sysctllog *mbuf_sysctllog; 106 107 static struct mbuf *m_copy_internal(struct mbuf *, int, int, int, bool); 108 static struct mbuf *m_split_internal(struct mbuf *, int, int, bool); 109 static int m_copyback_internal(struct mbuf **, int, int, const void *, 110 int, int); 111 112 /* Flags for m_copyback_internal. */ 113 #define CB_COPYBACK 0x0001 /* copyback from cp */ 114 #define CB_PRESERVE 0x0002 /* preserve original data */ 115 #define CB_COW 0x0004 /* do copy-on-write */ 116 #define CB_EXTEND 0x0008 /* extend chain */ 117 118 static const char mclpool_warnmsg[] = 119 "WARNING: mclpool limit reached; increase kern.mbuf.nmbclusters"; 120 121 MALLOC_DEFINE(M_MBUF, "mbuf", "mbuf"); 122 123 static percpu_t *mbstat_percpu; 124 125 #ifdef MBUFTRACE 126 struct mownerhead mowners = LIST_HEAD_INITIALIZER(mowners); 127 struct mowner unknown_mowners[] = { 128 MOWNER_INIT("unknown", "free"), 129 MOWNER_INIT("unknown", "data"), 130 MOWNER_INIT("unknown", "header"), 131 MOWNER_INIT("unknown", "soname"), 132 MOWNER_INIT("unknown", "soopts"), 133 MOWNER_INIT("unknown", "ftable"), 134 MOWNER_INIT("unknown", "control"), 135 MOWNER_INIT("unknown", "oobdata"), 136 }; 137 struct mowner revoked_mowner = MOWNER_INIT("revoked", ""); 138 #endif 139 140 #define MEXT_ISEMBEDDED(m) ((m)->m_ext_ref == (m)) 141 142 #define MCLADDREFERENCE(o, n) \ 143 do { \ 144 KASSERT(((o)->m_flags & M_EXT) != 0); \ 145 KASSERT(((n)->m_flags & M_EXT) == 0); \ 146 KASSERT((o)->m_ext.ext_refcnt >= 1); \ 147 (n)->m_flags |= ((o)->m_flags & M_EXTCOPYFLAGS); \ 148 atomic_inc_uint(&(o)->m_ext.ext_refcnt); \ 149 (n)->m_ext_ref = (o)->m_ext_ref; \ 150 mowner_ref((n), (n)->m_flags); \ 151 } while (/* CONSTCOND */ 0) 152 153 static int 154 nmbclusters_limit(void) 155 { 156 #if defined(PMAP_MAP_POOLPAGE) 157 /* direct mapping, doesn't use space in kmem_arena */ 158 vsize_t max_size = physmem / 4; 159 #else 160 vsize_t max_size = MIN(physmem / 4, nkmempages / 4); 161 #endif 162 163 max_size = max_size * PAGE_SIZE / MCLBYTES; 164 #ifdef NMBCLUSTERS_MAX 165 max_size = MIN(max_size, NMBCLUSTERS_MAX); 166 #endif 167 168 return max_size; 169 } 170 171 /* 172 * Initialize the mbuf allocator. 173 */ 174 void 175 mbinit(void) 176 { 177 178 CTASSERT(sizeof(struct _m_ext) <= MHLEN); 179 CTASSERT(sizeof(struct mbuf) == MSIZE); 180 181 sysctl_kern_mbuf_setup(); 182 183 mb_cache = pool_cache_init(msize, 0, 0, 0, "mbpl", 184 NULL, IPL_VM, mb_ctor, NULL, NULL); 185 KASSERT(mb_cache != NULL); 186 187 mcl_cache = pool_cache_init(mclbytes, COHERENCY_UNIT, 0, 0, "mclpl", 188 NULL, IPL_VM, NULL, NULL, NULL); 189 KASSERT(mcl_cache != NULL); 190 191 pool_cache_set_drain_hook(mb_cache, mb_drain, NULL); 192 pool_cache_set_drain_hook(mcl_cache, mb_drain, NULL); 193 194 /* 195 * Set an arbitrary default limit on the number of mbuf clusters. 196 */ 197 #ifdef NMBCLUSTERS 198 nmbclusters = MIN(NMBCLUSTERS, nmbclusters_limit()); 199 #else 200 nmbclusters = MAX(1024, 201 (vsize_t)physmem * PAGE_SIZE / MCLBYTES / 16); 202 nmbclusters = MIN(nmbclusters, nmbclusters_limit()); 203 #endif 204 205 /* 206 * Set the hard limit on the mclpool to the number of 207 * mbuf clusters the kernel is to support. Log the limit 208 * reached message max once a minute. 209 */ 210 pool_cache_sethardlimit(mcl_cache, nmbclusters, mclpool_warnmsg, 60); 211 212 mbstat_percpu = percpu_alloc(sizeof(struct mbstat_cpu)); 213 214 /* 215 * Set a low water mark for both mbufs and clusters. This should 216 * help ensure that they can be allocated in a memory starvation 217 * situation. This is important for e.g. diskless systems which 218 * must allocate mbufs in order for the pagedaemon to clean pages. 219 */ 220 pool_cache_setlowat(mb_cache, mblowat); 221 pool_cache_setlowat(mcl_cache, mcllowat); 222 223 #ifdef MBUFTRACE 224 { 225 /* 226 * Attach the unknown mowners. 227 */ 228 int i; 229 MOWNER_ATTACH(&revoked_mowner); 230 for (i = sizeof(unknown_mowners)/sizeof(unknown_mowners[0]); 231 i-- > 0; ) 232 MOWNER_ATTACH(&unknown_mowners[i]); 233 } 234 #endif 235 } 236 237 static void 238 mb_drain(void *arg, int flags) 239 { 240 struct domain *dp; 241 const struct protosw *pr; 242 struct ifnet *ifp; 243 int s; 244 245 KERNEL_LOCK(1, NULL); 246 s = splvm(); 247 DOMAIN_FOREACH(dp) { 248 for (pr = dp->dom_protosw; 249 pr < dp->dom_protoswNPROTOSW; pr++) 250 if (pr->pr_drain) 251 (*pr->pr_drain)(); 252 } 253 /* XXX we cannot use psref in H/W interrupt */ 254 if (!cpu_intr_p()) { 255 int bound = curlwp_bind(); 256 IFNET_READER_FOREACH(ifp) { 257 struct psref psref; 258 259 if_acquire(ifp, &psref); 260 261 if (ifp->if_drain) 262 (*ifp->if_drain)(ifp); 263 264 if_release(ifp, &psref); 265 } 266 curlwp_bindx(bound); 267 } 268 splx(s); 269 mbstat.m_drain++; 270 KERNEL_UNLOCK_ONE(NULL); 271 } 272 273 /* 274 * sysctl helper routine for the kern.mbuf subtree. 275 * nmbclusters, mblowat and mcllowat need range 276 * checking and pool tweaking after being reset. 277 */ 278 static int 279 sysctl_kern_mbuf(SYSCTLFN_ARGS) 280 { 281 int error, newval; 282 struct sysctlnode node; 283 284 node = *rnode; 285 node.sysctl_data = &newval; 286 switch (rnode->sysctl_num) { 287 case MBUF_NMBCLUSTERS: 288 case MBUF_MBLOWAT: 289 case MBUF_MCLLOWAT: 290 newval = *(int*)rnode->sysctl_data; 291 break; 292 default: 293 return EOPNOTSUPP; 294 } 295 296 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 297 if (error || newp == NULL) 298 return error; 299 if (newval < 0) 300 return EINVAL; 301 302 switch (node.sysctl_num) { 303 case MBUF_NMBCLUSTERS: 304 if (newval < nmbclusters) 305 return EINVAL; 306 if (newval > nmbclusters_limit()) 307 return EINVAL; 308 nmbclusters = newval; 309 pool_cache_sethardlimit(mcl_cache, nmbclusters, 310 mclpool_warnmsg, 60); 311 break; 312 case MBUF_MBLOWAT: 313 mblowat = newval; 314 pool_cache_setlowat(mb_cache, mblowat); 315 break; 316 case MBUF_MCLLOWAT: 317 mcllowat = newval; 318 pool_cache_setlowat(mcl_cache, mcllowat); 319 break; 320 } 321 322 return 0; 323 } 324 325 #ifdef MBUFTRACE 326 static void 327 mowner_convert_to_user_cb(void *v1, void *v2, struct cpu_info *ci) 328 { 329 struct mowner_counter *mc = v1; 330 struct mowner_user *mo_user = v2; 331 int i; 332 333 for (i = 0; i < MOWNER_COUNTER_NCOUNTERS; i++) { 334 mo_user->mo_counter[i] += mc->mc_counter[i]; 335 } 336 } 337 338 static void 339 mowner_convert_to_user(struct mowner *mo, struct mowner_user *mo_user) 340 { 341 342 memset(mo_user, 0, sizeof(*mo_user)); 343 CTASSERT(sizeof(mo_user->mo_name) == sizeof(mo->mo_name)); 344 CTASSERT(sizeof(mo_user->mo_descr) == sizeof(mo->mo_descr)); 345 memcpy(mo_user->mo_name, mo->mo_name, sizeof(mo->mo_name)); 346 memcpy(mo_user->mo_descr, mo->mo_descr, sizeof(mo->mo_descr)); 347 percpu_foreach(mo->mo_counters, mowner_convert_to_user_cb, mo_user); 348 } 349 350 static int 351 sysctl_kern_mbuf_mowners(SYSCTLFN_ARGS) 352 { 353 struct mowner *mo; 354 size_t len = 0; 355 int error = 0; 356 357 if (namelen != 0) 358 return EINVAL; 359 if (newp != NULL) 360 return EPERM; 361 362 LIST_FOREACH(mo, &mowners, mo_link) { 363 struct mowner_user mo_user; 364 365 mowner_convert_to_user(mo, &mo_user); 366 367 if (oldp != NULL) { 368 if (*oldlenp - len < sizeof(mo_user)) { 369 error = ENOMEM; 370 break; 371 } 372 error = copyout(&mo_user, (char *)oldp + len, 373 sizeof(mo_user)); 374 if (error) 375 break; 376 } 377 len += sizeof(mo_user); 378 } 379 380 if (error == 0) 381 *oldlenp = len; 382 383 return error; 384 } 385 #endif /* MBUFTRACE */ 386 387 void 388 mbstat_type_add(int type, int diff) 389 { 390 struct mbstat_cpu *mb; 391 int s; 392 393 s = splvm(); 394 mb = percpu_getref(mbstat_percpu); 395 mb->m_mtypes[type] += diff; 396 percpu_putref(mbstat_percpu); 397 splx(s); 398 } 399 400 static void 401 mbstat_convert_to_user_cb(void *v1, void *v2, struct cpu_info *ci) 402 { 403 struct mbstat_cpu *mbsc = v1; 404 struct mbstat *mbs = v2; 405 int i; 406 407 for (i = 0; i < __arraycount(mbs->m_mtypes); i++) { 408 mbs->m_mtypes[i] += mbsc->m_mtypes[i]; 409 } 410 } 411 412 static void 413 mbstat_convert_to_user(struct mbstat *mbs) 414 { 415 416 memset(mbs, 0, sizeof(*mbs)); 417 mbs->m_drain = mbstat.m_drain; 418 percpu_foreach(mbstat_percpu, mbstat_convert_to_user_cb, mbs); 419 } 420 421 static int 422 sysctl_kern_mbuf_stats(SYSCTLFN_ARGS) 423 { 424 struct sysctlnode node; 425 struct mbstat mbs; 426 427 mbstat_convert_to_user(&mbs); 428 node = *rnode; 429 node.sysctl_data = &mbs; 430 node.sysctl_size = sizeof(mbs); 431 return sysctl_lookup(SYSCTLFN_CALL(&node)); 432 } 433 434 static void 435 sysctl_kern_mbuf_setup(void) 436 { 437 438 KASSERT(mbuf_sysctllog == NULL); 439 sysctl_createv(&mbuf_sysctllog, 0, NULL, NULL, 440 CTLFLAG_PERMANENT, 441 CTLTYPE_NODE, "mbuf", 442 SYSCTL_DESCR("mbuf control variables"), 443 NULL, 0, NULL, 0, 444 CTL_KERN, KERN_MBUF, CTL_EOL); 445 446 sysctl_createv(&mbuf_sysctllog, 0, NULL, NULL, 447 CTLFLAG_PERMANENT|CTLFLAG_IMMEDIATE, 448 CTLTYPE_INT, "msize", 449 SYSCTL_DESCR("mbuf base size"), 450 NULL, msize, NULL, 0, 451 CTL_KERN, KERN_MBUF, MBUF_MSIZE, CTL_EOL); 452 sysctl_createv(&mbuf_sysctllog, 0, NULL, NULL, 453 CTLFLAG_PERMANENT|CTLFLAG_IMMEDIATE, 454 CTLTYPE_INT, "mclbytes", 455 SYSCTL_DESCR("mbuf cluster size"), 456 NULL, mclbytes, NULL, 0, 457 CTL_KERN, KERN_MBUF, MBUF_MCLBYTES, CTL_EOL); 458 sysctl_createv(&mbuf_sysctllog, 0, NULL, NULL, 459 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 460 CTLTYPE_INT, "nmbclusters", 461 SYSCTL_DESCR("Limit on the number of mbuf clusters"), 462 sysctl_kern_mbuf, 0, &nmbclusters, 0, 463 CTL_KERN, KERN_MBUF, MBUF_NMBCLUSTERS, CTL_EOL); 464 sysctl_createv(&mbuf_sysctllog, 0, NULL, NULL, 465 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 466 CTLTYPE_INT, "mblowat", 467 SYSCTL_DESCR("mbuf low water mark"), 468 sysctl_kern_mbuf, 0, &mblowat, 0, 469 CTL_KERN, KERN_MBUF, MBUF_MBLOWAT, CTL_EOL); 470 sysctl_createv(&mbuf_sysctllog, 0, NULL, NULL, 471 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 472 CTLTYPE_INT, "mcllowat", 473 SYSCTL_DESCR("mbuf cluster low water mark"), 474 sysctl_kern_mbuf, 0, &mcllowat, 0, 475 CTL_KERN, KERN_MBUF, MBUF_MCLLOWAT, CTL_EOL); 476 sysctl_createv(&mbuf_sysctllog, 0, NULL, NULL, 477 CTLFLAG_PERMANENT, 478 CTLTYPE_STRUCT, "stats", 479 SYSCTL_DESCR("mbuf allocation statistics"), 480 sysctl_kern_mbuf_stats, 0, NULL, 0, 481 CTL_KERN, KERN_MBUF, MBUF_STATS, CTL_EOL); 482 #ifdef MBUFTRACE 483 sysctl_createv(&mbuf_sysctllog, 0, NULL, NULL, 484 CTLFLAG_PERMANENT, 485 CTLTYPE_STRUCT, "mowners", 486 SYSCTL_DESCR("Information about mbuf owners"), 487 sysctl_kern_mbuf_mowners, 0, NULL, 0, 488 CTL_KERN, KERN_MBUF, MBUF_MOWNERS, CTL_EOL); 489 #endif 490 } 491 492 static int 493 mb_ctor(void *arg, void *object, int flags) 494 { 495 struct mbuf *m = object; 496 497 #ifdef POOL_VTOPHYS 498 m->m_paddr = POOL_VTOPHYS(m); 499 #else 500 m->m_paddr = M_PADDR_INVALID; 501 #endif 502 return 0; 503 } 504 505 /* 506 * Add mbuf to the end of a chain 507 */ 508 struct mbuf * 509 m_add(struct mbuf *c, struct mbuf *m) 510 { 511 struct mbuf *n; 512 513 if (c == NULL) 514 return m; 515 516 for (n = c; n->m_next != NULL; n = n->m_next) 517 continue; 518 n->m_next = m; 519 return c; 520 } 521 522 struct mbuf * 523 m_get(int how, int type) 524 { 525 struct mbuf *m; 526 527 KASSERT(type != MT_FREE); 528 529 m = pool_cache_get(mb_cache, 530 how == M_WAIT ? PR_WAITOK|PR_LIMITFAIL : PR_NOWAIT); 531 if (m == NULL) 532 return NULL; 533 KASSERT(((vaddr_t)m->m_dat & PAGE_MASK) + MLEN <= PAGE_SIZE); 534 535 mbstat_type_add(type, 1); 536 537 mowner_init(m, type); 538 m->m_ext_ref = m; /* default */ 539 m->m_type = type; 540 m->m_len = 0; 541 m->m_next = NULL; 542 m->m_nextpkt = NULL; /* default */ 543 m->m_data = m->m_dat; 544 m->m_flags = 0; /* default */ 545 546 return m; 547 } 548 549 struct mbuf * 550 m_gethdr(int how, int type) 551 { 552 struct mbuf *m; 553 554 m = m_get(how, type); 555 if (m == NULL) 556 return NULL; 557 558 m->m_data = m->m_pktdat; 559 m->m_flags = M_PKTHDR; 560 561 m_reset_rcvif(m); 562 m->m_pkthdr.len = 0; 563 m->m_pkthdr.csum_flags = 0; 564 m->m_pkthdr.csum_data = 0; 565 m->m_pkthdr.segsz = 0; 566 m->m_pkthdr.ether_vtag = 0; 567 m->m_pkthdr.pkthdr_flags = 0; 568 SLIST_INIT(&m->m_pkthdr.tags); 569 570 m->m_pkthdr.pattr_class = NULL; 571 m->m_pkthdr.pattr_af = AF_UNSPEC; 572 m->m_pkthdr.pattr_hdr = NULL; 573 574 return m; 575 } 576 577 void 578 m_clget(struct mbuf *m, int how) 579 { 580 m->m_ext_storage.ext_buf = (char *)pool_cache_get_paddr(mcl_cache, 581 how == M_WAIT ? (PR_WAITOK|PR_LIMITFAIL) : PR_NOWAIT, 582 &m->m_ext_storage.ext_paddr); 583 584 if (m->m_ext_storage.ext_buf == NULL) 585 return; 586 587 KASSERT(((vaddr_t)m->m_ext_storage.ext_buf & PAGE_MASK) + mclbytes 588 <= PAGE_SIZE); 589 590 MCLINITREFERENCE(m); 591 m->m_data = m->m_ext.ext_buf; 592 m->m_flags = (m->m_flags & ~M_EXTCOPYFLAGS) | 593 M_EXT|M_EXT_CLUSTER|M_EXT_RW; 594 m->m_ext.ext_size = MCLBYTES; 595 m->m_ext.ext_free = NULL; 596 m->m_ext.ext_arg = NULL; 597 /* ext_paddr initialized above */ 598 599 mowner_ref(m, M_EXT|M_EXT_CLUSTER); 600 } 601 602 struct mbuf * 603 m_getcl(int how, int type, int flags) 604 { 605 struct mbuf *mp; 606 607 if ((flags & M_PKTHDR) != 0) 608 mp = m_gethdr(how, type); 609 else 610 mp = m_get(how, type); 611 612 if (mp == NULL) 613 return NULL; 614 615 MCLGET(mp, how); 616 if ((mp->m_flags & M_EXT) != 0) 617 return mp; 618 619 m_free(mp); 620 return NULL; 621 } 622 623 /* 624 * Utility function for M_PREPEND. Do *NOT* use it directly. 625 */ 626 struct mbuf * 627 m_prepend(struct mbuf *m, int len, int how) 628 { 629 struct mbuf *mn; 630 631 if (__predict_false(len > MHLEN)) { 632 panic("%s: len > MHLEN", __func__); 633 } 634 635 KASSERT(len != M_COPYALL); 636 mn = m_get(how, m->m_type); 637 if (mn == NULL) { 638 m_freem(m); 639 return NULL; 640 } 641 642 if (m->m_flags & M_PKTHDR) { 643 m_move_pkthdr(mn, m); 644 } else { 645 MCLAIM(mn, m->m_owner); 646 } 647 mn->m_next = m; 648 m = mn; 649 650 if (m->m_flags & M_PKTHDR) { 651 if (len < MHLEN) 652 m_align(m, len); 653 } else { 654 if (len < MLEN) 655 m_align(m, len); 656 } 657 658 m->m_len = len; 659 return m; 660 } 661 662 struct mbuf * 663 m_copym(struct mbuf *m, int off, int len, int wait) 664 { 665 /* Shallow copy on M_EXT. */ 666 return m_copy_internal(m, off, len, wait, false); 667 } 668 669 struct mbuf * 670 m_dup(struct mbuf *m, int off, int len, int wait) 671 { 672 /* Deep copy. */ 673 return m_copy_internal(m, off, len, wait, true); 674 } 675 676 static inline int 677 m_copylen(int len, int copylen) 678 { 679 return (len == M_COPYALL) ? copylen : uimin(len, copylen); 680 } 681 682 static struct mbuf * 683 m_copy_internal(struct mbuf *m, int off0, int len, int wait, bool deep) 684 { 685 struct mbuf *n, **np; 686 int off = off0; 687 struct mbuf *top; 688 int copyhdr = 0; 689 690 if (off < 0 || (len != M_COPYALL && len < 0)) 691 panic("%s: off %d, len %d", __func__, off, len); 692 if (off == 0 && m->m_flags & M_PKTHDR) 693 copyhdr = 1; 694 while (off > 0) { 695 if (m == NULL) 696 panic("%s: m == NULL, off %d", __func__, off); 697 if (off < m->m_len) 698 break; 699 off -= m->m_len; 700 m = m->m_next; 701 } 702 703 np = ⊤ 704 top = NULL; 705 while (len == M_COPYALL || len > 0) { 706 if (m == NULL) { 707 if (len != M_COPYALL) 708 panic("%s: m == NULL, len %d [!COPYALL]", 709 __func__, len); 710 break; 711 } 712 713 n = m_get(wait, m->m_type); 714 *np = n; 715 if (n == NULL) 716 goto nospace; 717 MCLAIM(n, m->m_owner); 718 719 if (copyhdr) { 720 m_copy_pkthdr(n, m); 721 if (len == M_COPYALL) 722 n->m_pkthdr.len -= off0; 723 else 724 n->m_pkthdr.len = len; 725 copyhdr = 0; 726 } 727 n->m_len = m_copylen(len, m->m_len - off); 728 729 if (m->m_flags & M_EXT) { 730 if (!deep) { 731 n->m_data = m->m_data + off; 732 MCLADDREFERENCE(m, n); 733 } else { 734 /* 735 * We don't care if MCLGET fails. n->m_len is 736 * recomputed and handles that. 737 */ 738 MCLGET(n, wait); 739 n->m_len = 0; 740 n->m_len = M_TRAILINGSPACE(n); 741 n->m_len = m_copylen(len, n->m_len); 742 n->m_len = uimin(n->m_len, m->m_len - off); 743 memcpy(mtod(n, void *), mtod(m, char *) + off, 744 (unsigned)n->m_len); 745 } 746 } else { 747 memcpy(mtod(n, void *), mtod(m, char *) + off, 748 (unsigned)n->m_len); 749 } 750 751 if (len != M_COPYALL) 752 len -= n->m_len; 753 off += n->m_len; 754 755 KASSERT(off <= m->m_len); 756 757 if (off == m->m_len) { 758 m = m->m_next; 759 off = 0; 760 } 761 np = &n->m_next; 762 } 763 764 return top; 765 766 nospace: 767 m_freem(top); 768 return NULL; 769 } 770 771 /* 772 * Copy an entire packet, including header (which must be present). 773 * An optimization of the common case 'm_copym(m, 0, M_COPYALL, how)'. 774 */ 775 struct mbuf * 776 m_copypacket(struct mbuf *m, int how) 777 { 778 struct mbuf *top, *n, *o; 779 780 if (__predict_false((m->m_flags & M_PKTHDR) == 0)) { 781 panic("%s: no header (m = %p)", __func__, m); 782 } 783 784 n = m_get(how, m->m_type); 785 top = n; 786 if (!n) 787 goto nospace; 788 789 MCLAIM(n, m->m_owner); 790 m_copy_pkthdr(n, m); 791 n->m_len = m->m_len; 792 if (m->m_flags & M_EXT) { 793 n->m_data = m->m_data; 794 MCLADDREFERENCE(m, n); 795 } else { 796 memcpy(mtod(n, char *), mtod(m, char *), n->m_len); 797 } 798 799 m = m->m_next; 800 while (m) { 801 o = m_get(how, m->m_type); 802 if (!o) 803 goto nospace; 804 805 MCLAIM(o, m->m_owner); 806 n->m_next = o; 807 n = n->m_next; 808 809 n->m_len = m->m_len; 810 if (m->m_flags & M_EXT) { 811 n->m_data = m->m_data; 812 MCLADDREFERENCE(m, n); 813 } else { 814 memcpy(mtod(n, char *), mtod(m, char *), n->m_len); 815 } 816 817 m = m->m_next; 818 } 819 return top; 820 821 nospace: 822 m_freem(top); 823 return NULL; 824 } 825 826 void 827 m_copydata(struct mbuf *m, int off, int len, void *cp) 828 { 829 unsigned int count; 830 struct mbuf *m0 = m; 831 int len0 = len; 832 int off0 = off; 833 void *cp0 = cp; 834 835 KASSERT(len != M_COPYALL); 836 if (off < 0 || len < 0) 837 panic("m_copydata: off %d, len %d", off, len); 838 while (off > 0) { 839 if (m == NULL) 840 panic("m_copydata(%p,%d,%d,%p): m=NULL, off=%d (%d)", 841 m0, len0, off0, cp0, off, off0 - off); 842 if (off < m->m_len) 843 break; 844 off -= m->m_len; 845 m = m->m_next; 846 } 847 while (len > 0) { 848 if (m == NULL) 849 panic("m_copydata(%p,%d,%d,%p): " 850 "m=NULL, off=%d (%d), len=%d (%d)", 851 m0, len0, off0, cp0, 852 off, off0 - off, len, len0 - len); 853 count = uimin(m->m_len - off, len); 854 memcpy(cp, mtod(m, char *) + off, count); 855 len -= count; 856 cp = (char *)cp + count; 857 off = 0; 858 m = m->m_next; 859 } 860 } 861 862 /* 863 * Concatenate mbuf chain n to m. 864 * n might be copied into m (when n->m_len is small), therefore data portion of 865 * n could be copied into an mbuf of different mbuf type. 866 * Any m_pkthdr is not updated. 867 */ 868 void 869 m_cat(struct mbuf *m, struct mbuf *n) 870 { 871 872 while (m->m_next) 873 m = m->m_next; 874 while (n) { 875 if (M_READONLY(m) || n->m_len > M_TRAILINGSPACE(m)) { 876 /* just join the two chains */ 877 m->m_next = n; 878 return; 879 } 880 /* splat the data from one into the other */ 881 memcpy(mtod(m, char *) + m->m_len, mtod(n, void *), 882 (u_int)n->m_len); 883 m->m_len += n->m_len; 884 n = m_free(n); 885 } 886 } 887 888 void 889 m_adj(struct mbuf *mp, int req_len) 890 { 891 int len = req_len; 892 struct mbuf *m; 893 int count; 894 895 if ((m = mp) == NULL) 896 return; 897 if (len >= 0) { 898 /* 899 * Trim from head. 900 */ 901 while (m != NULL && len > 0) { 902 if (m->m_len <= len) { 903 len -= m->m_len; 904 m->m_len = 0; 905 m = m->m_next; 906 } else { 907 m->m_len -= len; 908 m->m_data += len; 909 len = 0; 910 } 911 } 912 if (mp->m_flags & M_PKTHDR) 913 mp->m_pkthdr.len -= (req_len - len); 914 } else { 915 /* 916 * Trim from tail. Scan the mbuf chain, 917 * calculating its length and finding the last mbuf. 918 * If the adjustment only affects this mbuf, then just 919 * adjust and return. Otherwise, rescan and truncate 920 * after the remaining size. 921 */ 922 len = -len; 923 count = 0; 924 for (;;) { 925 count += m->m_len; 926 if (m->m_next == NULL) 927 break; 928 m = m->m_next; 929 } 930 if (m->m_len >= len) { 931 m->m_len -= len; 932 if (mp->m_flags & M_PKTHDR) 933 mp->m_pkthdr.len -= len; 934 return; 935 } 936 937 count -= len; 938 if (count < 0) 939 count = 0; 940 941 /* 942 * Correct length for chain is "count". 943 * Find the mbuf with last data, adjust its length, 944 * and toss data from remaining mbufs on chain. 945 */ 946 m = mp; 947 if (m->m_flags & M_PKTHDR) 948 m->m_pkthdr.len = count; 949 for (; m; m = m->m_next) { 950 if (m->m_len >= count) { 951 m->m_len = count; 952 break; 953 } 954 count -= m->m_len; 955 } 956 if (m) { 957 while (m->m_next) 958 (m = m->m_next)->m_len = 0; 959 } 960 } 961 } 962 963 /* 964 * m_ensure_contig: rearrange an mbuf chain that given length of bytes 965 * would be contiguous and in the data area of an mbuf (therefore, mtod() 966 * would work for a structure of given length). 967 * 968 * => On success, returns true and the resulting mbuf chain; false otherwise. 969 * => The mbuf chain may change, but is always preserved valid. 970 */ 971 bool 972 m_ensure_contig(struct mbuf **m0, int len) 973 { 974 struct mbuf *n = *m0, *m; 975 size_t count, space; 976 977 KASSERT(len != M_COPYALL); 978 /* 979 * If first mbuf has no cluster, and has room for len bytes 980 * without shifting current data, pullup into it, 981 * otherwise allocate a new mbuf to prepend to the chain. 982 */ 983 if ((n->m_flags & M_EXT) == 0 && 984 n->m_data + len < &n->m_dat[MLEN] && n->m_next) { 985 if (n->m_len >= len) { 986 return true; 987 } 988 m = n; 989 n = n->m_next; 990 len -= m->m_len; 991 } else { 992 if (len > MHLEN) { 993 return false; 994 } 995 m = m_get(M_DONTWAIT, n->m_type); 996 if (m == NULL) { 997 return false; 998 } 999 MCLAIM(m, n->m_owner); 1000 if (n->m_flags & M_PKTHDR) { 1001 m_move_pkthdr(m, n); 1002 } 1003 } 1004 space = &m->m_dat[MLEN] - (m->m_data + m->m_len); 1005 do { 1006 count = MIN(MIN(MAX(len, max_protohdr), space), n->m_len); 1007 memcpy(mtod(m, char *) + m->m_len, mtod(n, void *), 1008 (unsigned)count); 1009 len -= count; 1010 m->m_len += count; 1011 n->m_len -= count; 1012 space -= count; 1013 if (n->m_len) 1014 n->m_data += count; 1015 else 1016 n = m_free(n); 1017 } while (len > 0 && n); 1018 1019 m->m_next = n; 1020 *m0 = m; 1021 1022 return len <= 0; 1023 } 1024 1025 /* 1026 * m_pullup: same as m_ensure_contig(), but destroys mbuf chain on error. 1027 */ 1028 struct mbuf * 1029 m_pullup(struct mbuf *n, int len) 1030 { 1031 struct mbuf *m = n; 1032 1033 KASSERT(len != M_COPYALL); 1034 if (!m_ensure_contig(&m, len)) { 1035 KASSERT(m != NULL); 1036 m_freem(m); 1037 m = NULL; 1038 } 1039 return m; 1040 } 1041 1042 /* 1043 * ensure that [off, off + len) is contiguous on the mbuf chain "m". 1044 * packet chain before "off" is kept untouched. 1045 * if offp == NULL, the target will start at <retval, 0> on resulting chain. 1046 * if offp != NULL, the target will start at <retval, *offp> on resulting chain. 1047 * 1048 * on error return (NULL return value), original "m" will be freed. 1049 * 1050 * XXX M_TRAILINGSPACE/M_LEADINGSPACE on shared cluster (sharedcluster) 1051 */ 1052 struct mbuf * 1053 m_pulldown(struct mbuf *m, int off, int len, int *offp) 1054 { 1055 struct mbuf *n, *o; 1056 int hlen, tlen, olen; 1057 int sharedcluster; 1058 1059 /* Check invalid arguments. */ 1060 if (m == NULL) 1061 panic("%s: m == NULL", __func__); 1062 if (len > MCLBYTES) { 1063 m_freem(m); 1064 return NULL; 1065 } 1066 1067 n = m; 1068 while (n != NULL && off > 0) { 1069 if (n->m_len > off) 1070 break; 1071 off -= n->m_len; 1072 n = n->m_next; 1073 } 1074 /* Be sure to point non-empty mbuf. */ 1075 while (n != NULL && n->m_len == 0) 1076 n = n->m_next; 1077 if (!n) { 1078 m_freem(m); 1079 return NULL; /* mbuf chain too short */ 1080 } 1081 1082 sharedcluster = M_READONLY(n); 1083 1084 /* 1085 * The target data is on <n, off>. If we got enough data on the mbuf 1086 * "n", we're done. 1087 */ 1088 #ifdef __NO_STRICT_ALIGNMENT 1089 if ((off == 0 || offp) && len <= n->m_len - off && !sharedcluster) 1090 #else 1091 if ((off == 0 || offp) && len <= n->m_len - off && !sharedcluster && 1092 ALIGNED_POINTER((mtod(n, char *) + off), uint32_t)) 1093 #endif 1094 goto ok; 1095 1096 /* 1097 * When (len <= n->m_len - off) and (off != 0), it is a special case. 1098 * Len bytes from <n, off> sit in single mbuf, but the caller does 1099 * not like the starting position (off). 1100 * 1101 * Chop the current mbuf into two pieces, set off to 0. 1102 */ 1103 if (len <= n->m_len - off) { 1104 struct mbuf *mlast; 1105 1106 o = m_dup(n, off, n->m_len - off, M_DONTWAIT); 1107 if (o == NULL) { 1108 m_freem(m); 1109 return NULL; /* ENOBUFS */ 1110 } 1111 KASSERT(o->m_len >= len); 1112 for (mlast = o; mlast->m_next != NULL; mlast = mlast->m_next) 1113 ; 1114 n->m_len = off; 1115 mlast->m_next = n->m_next; 1116 n->m_next = o; 1117 n = o; 1118 off = 0; 1119 goto ok; 1120 } 1121 1122 /* 1123 * We need to take hlen from <n, off> and tlen from <n->m_next, 0>, 1124 * and construct contiguous mbuf with m_len == len. 1125 * 1126 * Note that hlen + tlen == len, and tlen > 0. 1127 */ 1128 hlen = n->m_len - off; 1129 tlen = len - hlen; 1130 1131 /* 1132 * Ensure that we have enough trailing data on mbuf chain. If not, 1133 * we can do nothing about the chain. 1134 */ 1135 olen = 0; 1136 for (o = n->m_next; o != NULL; o = o->m_next) 1137 olen += o->m_len; 1138 if (hlen + olen < len) { 1139 m_freem(m); 1140 return NULL; /* mbuf chain too short */ 1141 } 1142 1143 /* 1144 * Easy cases first. We need to use m_copydata() to get data from 1145 * <n->m_next, 0>. 1146 */ 1147 if ((off == 0 || offp) && M_TRAILINGSPACE(n) >= tlen && 1148 !sharedcluster) { 1149 m_copydata(n->m_next, 0, tlen, mtod(n, char *) + n->m_len); 1150 n->m_len += tlen; 1151 m_adj(n->m_next, tlen); 1152 goto ok; 1153 } 1154 if ((off == 0 || offp) && M_LEADINGSPACE(n->m_next) >= hlen && 1155 #ifndef __NO_STRICT_ALIGNMENT 1156 ALIGNED_POINTER((n->m_next->m_data - hlen), uint32_t) && 1157 #endif 1158 !sharedcluster && n->m_next->m_len >= tlen) { 1159 n->m_next->m_data -= hlen; 1160 n->m_next->m_len += hlen; 1161 memcpy(mtod(n->m_next, void *), mtod(n, char *) + off, hlen); 1162 n->m_len -= hlen; 1163 n = n->m_next; 1164 off = 0; 1165 goto ok; 1166 } 1167 1168 /* 1169 * Now, we need to do the hard way. Don't copy as there's no room 1170 * on both ends. 1171 */ 1172 o = m_get(M_DONTWAIT, m->m_type); 1173 if (o && len > MLEN) { 1174 MCLGET(o, M_DONTWAIT); 1175 if ((o->m_flags & M_EXT) == 0) { 1176 m_free(o); 1177 o = NULL; 1178 } 1179 } 1180 if (!o) { 1181 m_freem(m); 1182 return NULL; /* ENOBUFS */ 1183 } 1184 /* get hlen from <n, off> into <o, 0> */ 1185 o->m_len = hlen; 1186 memcpy(mtod(o, void *), mtod(n, char *) + off, hlen); 1187 n->m_len -= hlen; 1188 /* get tlen from <n->m_next, 0> into <o, hlen> */ 1189 m_copydata(n->m_next, 0, tlen, mtod(o, char *) + o->m_len); 1190 o->m_len += tlen; 1191 m_adj(n->m_next, tlen); 1192 o->m_next = n->m_next; 1193 n->m_next = o; 1194 n = o; 1195 off = 0; 1196 1197 ok: 1198 if (offp) 1199 *offp = off; 1200 return n; 1201 } 1202 1203 /* 1204 * Like m_pullup(), except a new mbuf is always allocated, and we allow 1205 * the amount of empty space before the data in the new mbuf to be specified 1206 * (in the event that the caller expects to prepend later). 1207 */ 1208 struct mbuf * 1209 m_copyup(struct mbuf *n, int len, int dstoff) 1210 { 1211 struct mbuf *m; 1212 int count, space; 1213 1214 KASSERT(len != M_COPYALL); 1215 if (len > ((int)MHLEN - dstoff)) 1216 goto bad; 1217 m = m_get(M_DONTWAIT, n->m_type); 1218 if (m == NULL) 1219 goto bad; 1220 MCLAIM(m, n->m_owner); 1221 if (n->m_flags & M_PKTHDR) { 1222 m_move_pkthdr(m, n); 1223 } 1224 m->m_data += dstoff; 1225 space = &m->m_dat[MLEN] - (m->m_data + m->m_len); 1226 do { 1227 count = uimin(uimin(uimax(len, max_protohdr), space), n->m_len); 1228 memcpy(mtod(m, char *) + m->m_len, mtod(n, void *), 1229 (unsigned)count); 1230 len -= count; 1231 m->m_len += count; 1232 n->m_len -= count; 1233 space -= count; 1234 if (n->m_len) 1235 n->m_data += count; 1236 else 1237 n = m_free(n); 1238 } while (len > 0 && n); 1239 if (len > 0) { 1240 (void) m_free(m); 1241 goto bad; 1242 } 1243 m->m_next = n; 1244 return m; 1245 bad: 1246 m_freem(n); 1247 return NULL; 1248 } 1249 1250 struct mbuf * 1251 m_split(struct mbuf *m0, int len, int wait) 1252 { 1253 return m_split_internal(m0, len, wait, true); 1254 } 1255 1256 static struct mbuf * 1257 m_split_internal(struct mbuf *m0, int len0, int wait, bool copyhdr) 1258 { 1259 struct mbuf *m, *n; 1260 unsigned len = len0, remain, len_save; 1261 1262 KASSERT(len0 != M_COPYALL); 1263 for (m = m0; m && len > m->m_len; m = m->m_next) 1264 len -= m->m_len; 1265 if (m == NULL) 1266 return NULL; 1267 1268 remain = m->m_len - len; 1269 if (copyhdr && (m0->m_flags & M_PKTHDR)) { 1270 n = m_gethdr(wait, m0->m_type); 1271 if (n == NULL) 1272 return NULL; 1273 1274 MCLAIM(n, m0->m_owner); 1275 m_copy_rcvif(n, m0); 1276 n->m_pkthdr.len = m0->m_pkthdr.len - len0; 1277 len_save = m0->m_pkthdr.len; 1278 m0->m_pkthdr.len = len0; 1279 1280 if (m->m_flags & M_EXT) 1281 goto extpacket; 1282 1283 if (remain > MHLEN) { 1284 /* m can't be the lead packet */ 1285 m_align(n, 0); 1286 n->m_len = 0; 1287 n->m_next = m_split(m, len, wait); 1288 if (n->m_next == NULL) { 1289 (void)m_free(n); 1290 m0->m_pkthdr.len = len_save; 1291 return NULL; 1292 } 1293 return n; 1294 } else { 1295 m_align(n, remain); 1296 } 1297 } else if (remain == 0) { 1298 n = m->m_next; 1299 m->m_next = NULL; 1300 return n; 1301 } else { 1302 n = m_get(wait, m->m_type); 1303 if (n == NULL) 1304 return NULL; 1305 MCLAIM(n, m->m_owner); 1306 m_align(n, remain); 1307 } 1308 1309 extpacket: 1310 if (m->m_flags & M_EXT) { 1311 n->m_data = m->m_data + len; 1312 MCLADDREFERENCE(m, n); 1313 } else { 1314 memcpy(mtod(n, void *), mtod(m, char *) + len, remain); 1315 } 1316 1317 n->m_len = remain; 1318 m->m_len = len; 1319 n->m_next = m->m_next; 1320 m->m_next = NULL; 1321 return n; 1322 } 1323 1324 /* 1325 * Routine to copy from device local memory into mbufs. 1326 */ 1327 struct mbuf * 1328 m_devget(char *buf, int totlen, int off, struct ifnet *ifp) 1329 { 1330 struct mbuf *m; 1331 struct mbuf *top = NULL, **mp = ⊤ 1332 char *cp, *epkt; 1333 int len; 1334 1335 cp = buf; 1336 epkt = cp + totlen; 1337 if (off) { 1338 /* 1339 * If 'off' is non-zero, packet is trailer-encapsulated, 1340 * so we have to skip the type and length fields. 1341 */ 1342 cp += off + 2 * sizeof(uint16_t); 1343 totlen -= 2 * sizeof(uint16_t); 1344 } 1345 1346 m = m_gethdr(M_DONTWAIT, MT_DATA); 1347 if (m == NULL) 1348 return NULL; 1349 m_set_rcvif(m, ifp); 1350 m->m_pkthdr.len = totlen; 1351 m->m_len = MHLEN; 1352 1353 while (totlen > 0) { 1354 if (top) { 1355 m = m_get(M_DONTWAIT, MT_DATA); 1356 if (m == NULL) { 1357 m_freem(top); 1358 return NULL; 1359 } 1360 m->m_len = MLEN; 1361 } 1362 1363 len = uimin(totlen, epkt - cp); 1364 1365 if (len >= MINCLSIZE) { 1366 MCLGET(m, M_DONTWAIT); 1367 if ((m->m_flags & M_EXT) == 0) { 1368 m_free(m); 1369 m_freem(top); 1370 return NULL; 1371 } 1372 m->m_len = len = uimin(len, MCLBYTES); 1373 } else { 1374 /* 1375 * Place initial small packet/header at end of mbuf. 1376 */ 1377 if (len < m->m_len) { 1378 if (top == 0 && len + max_linkhdr <= m->m_len) 1379 m->m_data += max_linkhdr; 1380 m->m_len = len; 1381 } else 1382 len = m->m_len; 1383 } 1384 1385 memcpy(mtod(m, void *), cp, (size_t)len); 1386 1387 cp += len; 1388 *mp = m; 1389 mp = &m->m_next; 1390 totlen -= len; 1391 if (cp == epkt) 1392 cp = buf; 1393 } 1394 1395 return top; 1396 } 1397 1398 /* 1399 * Copy data from a buffer back into the indicated mbuf chain, 1400 * starting "off" bytes from the beginning, extending the mbuf 1401 * chain if necessary. 1402 */ 1403 void 1404 m_copyback(struct mbuf *m0, int off, int len, const void *cp) 1405 { 1406 #if defined(DEBUG) 1407 struct mbuf *origm = m0; 1408 int error; 1409 #endif 1410 1411 if (m0 == NULL) 1412 return; 1413 1414 #if defined(DEBUG) 1415 error = 1416 #endif 1417 m_copyback_internal(&m0, off, len, cp, CB_COPYBACK|CB_EXTEND, 1418 M_DONTWAIT); 1419 1420 #if defined(DEBUG) 1421 if (error != 0 || (m0 != NULL && origm != m0)) 1422 panic("m_copyback"); 1423 #endif 1424 } 1425 1426 struct mbuf * 1427 m_copyback_cow(struct mbuf *m0, int off, int len, const void *cp, int how) 1428 { 1429 int error; 1430 1431 /* don't support chain expansion */ 1432 KASSERT(len != M_COPYALL); 1433 KDASSERT(off + len <= m_length(m0)); 1434 1435 error = m_copyback_internal(&m0, off, len, cp, CB_COPYBACK|CB_COW, 1436 how); 1437 if (error) { 1438 /* 1439 * no way to recover from partial success. 1440 * just free the chain. 1441 */ 1442 m_freem(m0); 1443 return NULL; 1444 } 1445 return m0; 1446 } 1447 1448 int 1449 m_makewritable(struct mbuf **mp, int off, int len, int how) 1450 { 1451 int error; 1452 #if defined(DEBUG) 1453 int origlen = m_length(*mp); 1454 #endif 1455 1456 error = m_copyback_internal(mp, off, len, NULL, CB_PRESERVE|CB_COW, 1457 how); 1458 if (error) 1459 return error; 1460 1461 #if defined(DEBUG) 1462 int reslen = 0; 1463 for (struct mbuf *n = *mp; n; n = n->m_next) 1464 reslen += n->m_len; 1465 if (origlen != reslen) 1466 panic("m_makewritable: length changed"); 1467 if (((*mp)->m_flags & M_PKTHDR) != 0 && reslen != (*mp)->m_pkthdr.len) 1468 panic("m_makewritable: inconsist"); 1469 #endif 1470 1471 return 0; 1472 } 1473 1474 static int 1475 m_copyback_internal(struct mbuf **mp0, int off, int len, const void *vp, 1476 int flags, int how) 1477 { 1478 int mlen; 1479 struct mbuf *m, *n; 1480 struct mbuf **mp; 1481 int totlen = 0; 1482 const char *cp = vp; 1483 1484 KASSERT(mp0 != NULL); 1485 KASSERT(*mp0 != NULL); 1486 KASSERT((flags & CB_PRESERVE) == 0 || cp == NULL); 1487 KASSERT((flags & CB_COPYBACK) == 0 || cp != NULL); 1488 1489 if (len == M_COPYALL) 1490 len = m_length(*mp0) - off; 1491 1492 /* 1493 * we don't bother to update "totlen" in the case of CB_COW, 1494 * assuming that CB_EXTEND and CB_COW are exclusive. 1495 */ 1496 1497 KASSERT((~flags & (CB_EXTEND|CB_COW)) != 0); 1498 1499 mp = mp0; 1500 m = *mp; 1501 while (off > (mlen = m->m_len)) { 1502 off -= mlen; 1503 totlen += mlen; 1504 if (m->m_next == NULL) { 1505 int tspace; 1506 extend: 1507 if ((flags & CB_EXTEND) == 0) 1508 goto out; 1509 1510 /* 1511 * try to make some space at the end of "m". 1512 */ 1513 1514 mlen = m->m_len; 1515 if (off + len >= MINCLSIZE && 1516 (m->m_flags & M_EXT) == 0 && m->m_len == 0) { 1517 MCLGET(m, how); 1518 } 1519 tspace = M_TRAILINGSPACE(m); 1520 if (tspace > 0) { 1521 tspace = uimin(tspace, off + len); 1522 KASSERT(tspace > 0); 1523 memset(mtod(m, char *) + m->m_len, 0, 1524 uimin(off, tspace)); 1525 m->m_len += tspace; 1526 off += mlen; 1527 totlen -= mlen; 1528 continue; 1529 } 1530 1531 /* 1532 * need to allocate an mbuf. 1533 */ 1534 1535 if (off + len >= MINCLSIZE) { 1536 n = m_getcl(how, m->m_type, 0); 1537 } else { 1538 n = m_get(how, m->m_type); 1539 } 1540 if (n == NULL) { 1541 goto out; 1542 } 1543 n->m_len = uimin(M_TRAILINGSPACE(n), off + len); 1544 memset(mtod(n, char *), 0, uimin(n->m_len, off)); 1545 m->m_next = n; 1546 } 1547 mp = &m->m_next; 1548 m = m->m_next; 1549 } 1550 while (len > 0) { 1551 mlen = m->m_len - off; 1552 if (mlen != 0 && M_READONLY(m)) { 1553 /* 1554 * This mbuf is read-only. Allocate a new writable 1555 * mbuf and try again. 1556 */ 1557 char *datap; 1558 int eatlen; 1559 1560 KASSERT((flags & CB_COW) != 0); 1561 1562 /* 1563 * if we're going to write into the middle of 1564 * a mbuf, split it first. 1565 */ 1566 if (off > 0) { 1567 n = m_split_internal(m, off, how, false); 1568 if (n == NULL) 1569 goto enobufs; 1570 m->m_next = n; 1571 mp = &m->m_next; 1572 m = n; 1573 off = 0; 1574 continue; 1575 } 1576 1577 /* 1578 * XXX TODO coalesce into the trailingspace of 1579 * the previous mbuf when possible. 1580 */ 1581 1582 /* 1583 * allocate a new mbuf. copy packet header if needed. 1584 */ 1585 n = m_get(how, m->m_type); 1586 if (n == NULL) 1587 goto enobufs; 1588 MCLAIM(n, m->m_owner); 1589 if (off == 0 && (m->m_flags & M_PKTHDR) != 0) { 1590 m_move_pkthdr(n, m); 1591 n->m_len = MHLEN; 1592 } else { 1593 if (len >= MINCLSIZE) 1594 MCLGET(n, M_DONTWAIT); 1595 n->m_len = 1596 (n->m_flags & M_EXT) ? MCLBYTES : MLEN; 1597 } 1598 if (n->m_len > len) 1599 n->m_len = len; 1600 1601 /* 1602 * free the region which has been overwritten. 1603 * copying data from old mbufs if requested. 1604 */ 1605 if (flags & CB_PRESERVE) 1606 datap = mtod(n, char *); 1607 else 1608 datap = NULL; 1609 eatlen = n->m_len; 1610 while (m != NULL && M_READONLY(m) && 1611 n->m_type == m->m_type && eatlen > 0) { 1612 mlen = uimin(eatlen, m->m_len); 1613 if (datap) { 1614 m_copydata(m, 0, mlen, datap); 1615 datap += mlen; 1616 } 1617 m->m_data += mlen; 1618 m->m_len -= mlen; 1619 eatlen -= mlen; 1620 if (m->m_len == 0) 1621 *mp = m = m_free(m); 1622 } 1623 if (eatlen > 0) 1624 n->m_len -= eatlen; 1625 n->m_next = m; 1626 *mp = m = n; 1627 continue; 1628 } 1629 mlen = uimin(mlen, len); 1630 if (flags & CB_COPYBACK) { 1631 memcpy(mtod(m, char *) + off, cp, (unsigned)mlen); 1632 cp += mlen; 1633 } 1634 len -= mlen; 1635 mlen += off; 1636 off = 0; 1637 totlen += mlen; 1638 if (len == 0) 1639 break; 1640 if (m->m_next == NULL) { 1641 goto extend; 1642 } 1643 mp = &m->m_next; 1644 m = m->m_next; 1645 } 1646 1647 out: 1648 if (((m = *mp0)->m_flags & M_PKTHDR) && (m->m_pkthdr.len < totlen)) { 1649 KASSERT((flags & CB_EXTEND) != 0); 1650 m->m_pkthdr.len = totlen; 1651 } 1652 1653 return 0; 1654 1655 enobufs: 1656 return ENOBUFS; 1657 } 1658 1659 /* 1660 * Compress the mbuf chain. Return the new mbuf chain on success, NULL on 1661 * failure. The first mbuf is preserved, and on success the pointer returned 1662 * is the same as the one passed. 1663 */ 1664 struct mbuf * 1665 m_defrag(struct mbuf *m, int how) 1666 { 1667 struct mbuf *m0, *mn, *n; 1668 int sz; 1669 1670 KASSERT((m->m_flags & M_PKTHDR) != 0); 1671 1672 if (m->m_next == NULL) 1673 return m; 1674 1675 /* Defrag to single mbuf if at all possible */ 1676 if ((m->m_flags & M_EXT) == 0 && m->m_pkthdr.len <= MCLBYTES) { 1677 if (m->m_pkthdr.len <= MHLEN) { 1678 if (M_TRAILINGSPACE(m) < (m->m_pkthdr.len - m->m_len)) { 1679 KASSERTMSG(M_LEADINGSPACE(m) + 1680 M_TRAILINGSPACE(m) >= 1681 (m->m_pkthdr.len - m->m_len), 1682 "too small leading %d trailing %d ro? %d" 1683 " pkthdr.len %d mlen %d", 1684 (int)M_LEADINGSPACE(m), 1685 (int)M_TRAILINGSPACE(m), 1686 M_READONLY(m), 1687 m->m_pkthdr.len, m->m_len); 1688 1689 memmove(m->m_pktdat, m->m_data, m->m_len); 1690 m->m_data = m->m_pktdat; 1691 1692 KASSERT(M_TRAILINGSPACE(m) >= 1693 (m->m_pkthdr.len - m->m_len)); 1694 } 1695 } else { 1696 /* Must copy data before adding cluster */ 1697 m0 = m_get(how, MT_DATA); 1698 if (m0 == NULL) 1699 return NULL; 1700 KASSERT(m->m_len <= MHLEN); 1701 m_copydata(m, 0, m->m_len, mtod(m0, void *)); 1702 1703 MCLGET(m, how); 1704 if ((m->m_flags & M_EXT) == 0) { 1705 m_free(m0); 1706 return NULL; 1707 } 1708 memcpy(m->m_data, mtod(m0, void *), m->m_len); 1709 m_free(m0); 1710 } 1711 KASSERT(M_TRAILINGSPACE(m) >= (m->m_pkthdr.len - m->m_len)); 1712 m_copydata(m->m_next, 0, m->m_pkthdr.len - m->m_len, 1713 mtod(m, char *) + m->m_len); 1714 m->m_len = m->m_pkthdr.len; 1715 m_freem(m->m_next); 1716 m->m_next = NULL; 1717 return m; 1718 } 1719 1720 m0 = m_get(how, MT_DATA); 1721 if (m0 == NULL) 1722 return NULL; 1723 mn = m0; 1724 1725 sz = m->m_pkthdr.len - m->m_len; 1726 KASSERT(sz >= 0); 1727 1728 do { 1729 if (sz > MLEN) { 1730 MCLGET(mn, how); 1731 if ((mn->m_flags & M_EXT) == 0) { 1732 m_freem(m0); 1733 return NULL; 1734 } 1735 } 1736 1737 mn->m_len = MIN(sz, MCLBYTES); 1738 1739 m_copydata(m, m->m_pkthdr.len - sz, mn->m_len, 1740 mtod(mn, void *)); 1741 1742 sz -= mn->m_len; 1743 1744 if (sz > 0) { 1745 /* need more mbufs */ 1746 n = m_get(how, MT_DATA); 1747 if (n == NULL) { 1748 m_freem(m0); 1749 return NULL; 1750 } 1751 1752 mn->m_next = n; 1753 mn = n; 1754 } 1755 } while (sz > 0); 1756 1757 m_freem(m->m_next); 1758 m->m_next = m0; 1759 1760 return m; 1761 } 1762 1763 void 1764 m_remove_pkthdr(struct mbuf *m) 1765 { 1766 KASSERT(m->m_flags & M_PKTHDR); 1767 1768 m_tag_delete_chain(m); 1769 m->m_flags &= ~M_PKTHDR; 1770 memset(&m->m_pkthdr, 0, sizeof(m->m_pkthdr)); 1771 } 1772 1773 void 1774 m_copy_pkthdr(struct mbuf *to, struct mbuf *from) 1775 { 1776 KASSERT((to->m_flags & M_EXT) == 0); 1777 KASSERT((to->m_flags & M_PKTHDR) == 0 || 1778 SLIST_FIRST(&to->m_pkthdr.tags) == NULL); 1779 KASSERT((from->m_flags & M_PKTHDR) != 0); 1780 1781 to->m_pkthdr = from->m_pkthdr; 1782 to->m_flags = from->m_flags & M_COPYFLAGS; 1783 to->m_data = to->m_pktdat; 1784 1785 SLIST_INIT(&to->m_pkthdr.tags); 1786 m_tag_copy_chain(to, from); 1787 } 1788 1789 void 1790 m_move_pkthdr(struct mbuf *to, struct mbuf *from) 1791 { 1792 KASSERT((to->m_flags & M_EXT) == 0); 1793 KASSERT((to->m_flags & M_PKTHDR) == 0 || 1794 SLIST_FIRST(&to->m_pkthdr.tags) == NULL); 1795 KASSERT((from->m_flags & M_PKTHDR) != 0); 1796 1797 to->m_pkthdr = from->m_pkthdr; 1798 to->m_flags = from->m_flags & M_COPYFLAGS; 1799 to->m_data = to->m_pktdat; 1800 1801 from->m_flags &= ~M_PKTHDR; 1802 } 1803 1804 /* 1805 * Set the m_data pointer of a newly-allocated mbuf to place an object of the 1806 * specified size at the end of the mbuf, longword aligned. 1807 */ 1808 void 1809 m_align(struct mbuf *m, int len) 1810 { 1811 int buflen, adjust; 1812 1813 KASSERT(len != M_COPYALL); 1814 KASSERT(M_LEADINGSPACE(m) == 0); 1815 1816 buflen = M_BUFSIZE(m); 1817 1818 KASSERT(len <= buflen); 1819 adjust = buflen - len; 1820 m->m_data += adjust &~ (sizeof(long)-1); 1821 } 1822 1823 /* 1824 * Apply function f to the data in an mbuf chain starting "off" bytes from the 1825 * beginning, continuing for "len" bytes. 1826 */ 1827 int 1828 m_apply(struct mbuf *m, int off, int len, 1829 int (*f)(void *, void *, unsigned int), void *arg) 1830 { 1831 unsigned int count; 1832 int rval; 1833 1834 KASSERT(len != M_COPYALL); 1835 KASSERT(len >= 0); 1836 KASSERT(off >= 0); 1837 1838 while (off > 0) { 1839 KASSERT(m != NULL); 1840 if (off < m->m_len) 1841 break; 1842 off -= m->m_len; 1843 m = m->m_next; 1844 } 1845 while (len > 0) { 1846 KASSERT(m != NULL); 1847 count = uimin(m->m_len - off, len); 1848 1849 rval = (*f)(arg, mtod(m, char *) + off, count); 1850 if (rval) 1851 return rval; 1852 1853 len -= count; 1854 off = 0; 1855 m = m->m_next; 1856 } 1857 1858 return 0; 1859 } 1860 1861 /* 1862 * Return a pointer to mbuf/offset of location in mbuf chain. 1863 */ 1864 struct mbuf * 1865 m_getptr(struct mbuf *m, int loc, int *off) 1866 { 1867 1868 while (loc >= 0) { 1869 /* Normal end of search */ 1870 if (m->m_len > loc) { 1871 *off = loc; 1872 return m; 1873 } 1874 1875 loc -= m->m_len; 1876 1877 if (m->m_next == NULL) { 1878 if (loc == 0) { 1879 /* Point at the end of valid data */ 1880 *off = m->m_len; 1881 return m; 1882 } 1883 return NULL; 1884 } else { 1885 m = m->m_next; 1886 } 1887 } 1888 1889 return NULL; 1890 } 1891 1892 /* 1893 * Release a reference to the mbuf external storage. 1894 * 1895 * => free the mbuf m itself as well. 1896 */ 1897 static void 1898 m_ext_free(struct mbuf *m) 1899 { 1900 const bool embedded = MEXT_ISEMBEDDED(m); 1901 bool dofree = true; 1902 u_int refcnt; 1903 1904 KASSERT((m->m_flags & M_EXT) != 0); 1905 KASSERT(MEXT_ISEMBEDDED(m->m_ext_ref)); 1906 KASSERT((m->m_ext_ref->m_flags & M_EXT) != 0); 1907 KASSERT((m->m_flags & M_EXT_CLUSTER) == 1908 (m->m_ext_ref->m_flags & M_EXT_CLUSTER)); 1909 1910 if (__predict_false(m->m_type == MT_FREE)) { 1911 panic("mbuf %p already freed", m); 1912 } 1913 1914 if (__predict_true(m->m_ext.ext_refcnt == 1)) { 1915 refcnt = m->m_ext.ext_refcnt = 0; 1916 } else { 1917 #ifndef __HAVE_ATOMIC_AS_MEMBAR 1918 membar_release(); 1919 #endif 1920 refcnt = atomic_dec_uint_nv(&m->m_ext.ext_refcnt); 1921 } 1922 1923 if (refcnt > 0) { 1924 if (embedded) { 1925 /* 1926 * other mbuf's m_ext_ref still points to us. 1927 */ 1928 dofree = false; 1929 } else { 1930 m->m_ext_ref = m; 1931 } 1932 } else { 1933 /* 1934 * dropping the last reference 1935 */ 1936 #ifndef __HAVE_ATOMIC_AS_MEMBAR 1937 membar_acquire(); 1938 #endif 1939 if (!embedded) { 1940 m->m_ext.ext_refcnt++; /* XXX */ 1941 m_ext_free(m->m_ext_ref); 1942 m->m_ext_ref = m; 1943 } else if ((m->m_flags & M_EXT_CLUSTER) != 0) { 1944 pool_cache_put_paddr(mcl_cache, 1945 m->m_ext.ext_buf, m->m_ext.ext_paddr); 1946 } else if (m->m_ext.ext_free) { 1947 (*m->m_ext.ext_free)(m, 1948 m->m_ext.ext_buf, m->m_ext.ext_size, 1949 m->m_ext.ext_arg); 1950 /* 1951 * 'm' is already freed by the ext_free callback. 1952 */ 1953 dofree = false; 1954 } else { 1955 free(m->m_ext.ext_buf, 0); 1956 } 1957 } 1958 1959 if (dofree) { 1960 m->m_type = MT_FREE; 1961 m->m_data = NULL; 1962 pool_cache_put(mb_cache, m); 1963 } 1964 } 1965 1966 /* 1967 * Free a single mbuf and associated external storage. Return the 1968 * successor, if any. 1969 */ 1970 struct mbuf * 1971 m_free(struct mbuf *m) 1972 { 1973 struct mbuf *n; 1974 1975 mowner_revoke(m, 1, m->m_flags); 1976 mbstat_type_add(m->m_type, -1); 1977 1978 if (m->m_flags & M_PKTHDR) 1979 m_tag_delete_chain(m); 1980 1981 n = m->m_next; 1982 1983 if (m->m_flags & M_EXT) { 1984 m_ext_free(m); 1985 } else { 1986 if (__predict_false(m->m_type == MT_FREE)) { 1987 panic("mbuf %p already freed", m); 1988 } 1989 m->m_type = MT_FREE; 1990 m->m_data = NULL; 1991 pool_cache_put(mb_cache, m); 1992 } 1993 1994 return n; 1995 } 1996 1997 void 1998 m_freem(struct mbuf *m) 1999 { 2000 if (m == NULL) 2001 return; 2002 do { 2003 m = m_free(m); 2004 } while (m); 2005 } 2006 2007 #if defined(DDB) 2008 void 2009 m_print(const struct mbuf *m, const char *modif, void (*pr)(const char *, ...)) 2010 { 2011 char ch; 2012 bool opt_c = false; 2013 bool opt_d = false; 2014 #if NETHER > 0 2015 bool opt_v = false; 2016 const struct mbuf *m0 = NULL; 2017 #endif 2018 int no = 0; 2019 char buf[512]; 2020 2021 while ((ch = *(modif++)) != '\0') { 2022 switch (ch) { 2023 case 'c': 2024 opt_c = true; 2025 break; 2026 case 'd': 2027 opt_d = true; 2028 break; 2029 #if NETHER > 0 2030 case 'v': 2031 opt_v = true; 2032 m0 = m; 2033 break; 2034 #endif 2035 default: 2036 break; 2037 } 2038 } 2039 2040 nextchain: 2041 (*pr)("MBUF(%d) %p\n", no, m); 2042 snprintb(buf, sizeof(buf), M_FLAGS_BITS, (u_int)m->m_flags); 2043 (*pr)(" data=%p, len=%d, type=%d, flags=%s\n", 2044 m->m_data, m->m_len, m->m_type, buf); 2045 if (opt_d) { 2046 int i; 2047 unsigned char *p = m->m_data; 2048 2049 (*pr)(" data:"); 2050 2051 for (i = 0; i < m->m_len; i++) { 2052 if (i % 16 == 0) 2053 (*pr)("\n"); 2054 (*pr)(" %02x", p[i]); 2055 } 2056 2057 (*pr)("\n"); 2058 } 2059 (*pr)(" owner=%p, next=%p, nextpkt=%p\n", m->m_owner, m->m_next, 2060 m->m_nextpkt); 2061 (*pr)(" leadingspace=%u, trailingspace=%u, readonly=%u\n", 2062 (int)M_LEADINGSPACE(m), (int)M_TRAILINGSPACE(m), 2063 (int)M_READONLY(m)); 2064 if ((m->m_flags & M_PKTHDR) != 0) { 2065 snprintb(buf, sizeof(buf), M_CSUM_BITS, m->m_pkthdr.csum_flags); 2066 (*pr)(" pktlen=%d, rcvif=%p, csum_flags=%s, csum_data=0x%" 2067 PRIx32 ", segsz=%u\n", 2068 m->m_pkthdr.len, m_get_rcvif_NOMPSAFE(m), 2069 buf, m->m_pkthdr.csum_data, m->m_pkthdr.segsz); 2070 } 2071 if ((m->m_flags & M_EXT)) { 2072 (*pr)(" ext_refcnt=%u, ext_buf=%p, ext_size=%zd, " 2073 "ext_free=%p, ext_arg=%p\n", 2074 m->m_ext.ext_refcnt, 2075 m->m_ext.ext_buf, m->m_ext.ext_size, 2076 m->m_ext.ext_free, m->m_ext.ext_arg); 2077 } 2078 if ((~m->m_flags & (M_EXT|M_EXT_PAGES)) == 0) { 2079 vaddr_t sva = (vaddr_t)m->m_ext.ext_buf; 2080 vaddr_t eva = sva + m->m_ext.ext_size; 2081 int n = (round_page(eva) - trunc_page(sva)) >> PAGE_SHIFT; 2082 int i; 2083 2084 (*pr)(" pages:"); 2085 for (i = 0; i < n; i ++) { 2086 (*pr)(" %p", m->m_ext.ext_pgs[i]); 2087 } 2088 (*pr)("\n"); 2089 } 2090 2091 if (opt_c) { 2092 m = m->m_next; 2093 if (m != NULL) { 2094 no++; 2095 goto nextchain; 2096 } 2097 } 2098 2099 #if NETHER > 0 2100 if (opt_v && m0) 2101 m_examine(m0, AF_ETHER, modif, pr); 2102 #endif 2103 } 2104 #endif /* defined(DDB) */ 2105 2106 #if defined(MBUFTRACE) 2107 void 2108 mowner_init_owner(struct mowner *mo, const char *name, const char *descr) 2109 { 2110 memset(mo, 0, sizeof(*mo)); 2111 strlcpy(mo->mo_name, name, sizeof(mo->mo_name)); 2112 strlcpy(mo->mo_descr, descr, sizeof(mo->mo_descr)); 2113 } 2114 2115 void 2116 mowner_attach(struct mowner *mo) 2117 { 2118 2119 KASSERT(mo->mo_counters == NULL); 2120 mo->mo_counters = percpu_alloc(sizeof(struct mowner_counter)); 2121 2122 /* XXX lock */ 2123 LIST_INSERT_HEAD(&mowners, mo, mo_link); 2124 } 2125 2126 void 2127 mowner_detach(struct mowner *mo) 2128 { 2129 2130 KASSERT(mo->mo_counters != NULL); 2131 2132 /* XXX lock */ 2133 LIST_REMOVE(mo, mo_link); 2134 2135 percpu_free(mo->mo_counters, sizeof(struct mowner_counter)); 2136 mo->mo_counters = NULL; 2137 } 2138 2139 void 2140 mowner_init(struct mbuf *m, int type) 2141 { 2142 struct mowner_counter *mc; 2143 struct mowner *mo; 2144 int s; 2145 2146 m->m_owner = mo = &unknown_mowners[type]; 2147 s = splvm(); 2148 mc = percpu_getref(mo->mo_counters); 2149 mc->mc_counter[MOWNER_COUNTER_CLAIMS]++; 2150 percpu_putref(mo->mo_counters); 2151 splx(s); 2152 } 2153 2154 void 2155 mowner_ref(struct mbuf *m, int flags) 2156 { 2157 struct mowner *mo = m->m_owner; 2158 struct mowner_counter *mc; 2159 int s; 2160 2161 s = splvm(); 2162 mc = percpu_getref(mo->mo_counters); 2163 if ((flags & M_EXT) != 0) 2164 mc->mc_counter[MOWNER_COUNTER_EXT_CLAIMS]++; 2165 if ((flags & M_EXT_CLUSTER) != 0) 2166 mc->mc_counter[MOWNER_COUNTER_CLUSTER_CLAIMS]++; 2167 percpu_putref(mo->mo_counters); 2168 splx(s); 2169 } 2170 2171 void 2172 mowner_revoke(struct mbuf *m, bool all, int flags) 2173 { 2174 struct mowner *mo = m->m_owner; 2175 struct mowner_counter *mc; 2176 int s; 2177 2178 s = splvm(); 2179 mc = percpu_getref(mo->mo_counters); 2180 if ((flags & M_EXT) != 0) 2181 mc->mc_counter[MOWNER_COUNTER_EXT_RELEASES]++; 2182 if ((flags & M_EXT_CLUSTER) != 0) 2183 mc->mc_counter[MOWNER_COUNTER_CLUSTER_RELEASES]++; 2184 if (all) 2185 mc->mc_counter[MOWNER_COUNTER_RELEASES]++; 2186 percpu_putref(mo->mo_counters); 2187 splx(s); 2188 if (all) 2189 m->m_owner = &revoked_mowner; 2190 } 2191 2192 static void 2193 mowner_claim(struct mbuf *m, struct mowner *mo) 2194 { 2195 struct mowner_counter *mc; 2196 int flags = m->m_flags; 2197 int s; 2198 2199 s = splvm(); 2200 mc = percpu_getref(mo->mo_counters); 2201 mc->mc_counter[MOWNER_COUNTER_CLAIMS]++; 2202 if ((flags & M_EXT) != 0) 2203 mc->mc_counter[MOWNER_COUNTER_EXT_CLAIMS]++; 2204 if ((flags & M_EXT_CLUSTER) != 0) 2205 mc->mc_counter[MOWNER_COUNTER_CLUSTER_CLAIMS]++; 2206 percpu_putref(mo->mo_counters); 2207 splx(s); 2208 m->m_owner = mo; 2209 } 2210 2211 void 2212 m_claim(struct mbuf *m, struct mowner *mo) 2213 { 2214 2215 if (m->m_owner == mo || mo == NULL) 2216 return; 2217 2218 mowner_revoke(m, true, m->m_flags); 2219 mowner_claim(m, mo); 2220 } 2221 2222 void 2223 m_claimm(struct mbuf *m, struct mowner *mo) 2224 { 2225 2226 for (; m != NULL; m = m->m_next) 2227 m_claim(m, mo); 2228 } 2229 #endif /* defined(MBUFTRACE) */ 2230 2231 #ifdef DIAGNOSTIC 2232 /* 2233 * Verify that the mbuf chain is not malformed. Used only for diagnostic. 2234 * Panics on error. 2235 */ 2236 void 2237 m_verify_packet(struct mbuf *m) 2238 { 2239 struct mbuf *n = m; 2240 char *low, *high, *dat; 2241 int totlen = 0, len; 2242 2243 if (__predict_false((m->m_flags & M_PKTHDR) == 0)) { 2244 panic("%s: mbuf doesn't have M_PKTHDR", __func__); 2245 } 2246 2247 while (n != NULL) { 2248 if (__predict_false(n->m_type == MT_FREE)) { 2249 panic("%s: mbuf already freed (n = %p)", __func__, n); 2250 } 2251 #if 0 2252 /* 2253 * This ought to be a rule of the mbuf API. Unfortunately, 2254 * many places don't respect that rule. 2255 */ 2256 if (__predict_false((n != m) && (n->m_flags & M_PKTHDR) != 0)) { 2257 panic("%s: M_PKTHDR set on secondary mbuf", __func__); 2258 } 2259 #endif 2260 if (__predict_false(n->m_nextpkt != NULL)) { 2261 panic("%s: m_nextpkt not null (m_nextpkt = %p)", 2262 __func__, n->m_nextpkt); 2263 } 2264 2265 dat = n->m_data; 2266 len = n->m_len; 2267 if (__predict_false(len < 0)) { 2268 panic("%s: incorrect length (len = %d)", __func__, len); 2269 } 2270 2271 low = M_BUFADDR(n); 2272 high = low + M_BUFSIZE(n); 2273 if (__predict_false((dat < low) || (dat + len > high))) { 2274 panic("%s: m_data not in packet" 2275 "(dat = %p, len = %d, low = %p, high = %p)", 2276 __func__, dat, len, low, high); 2277 } 2278 2279 totlen += len; 2280 n = n->m_next; 2281 } 2282 2283 if (__predict_false(totlen != m->m_pkthdr.len)) { 2284 panic("%s: inconsistent mbuf length (%d != %d)", __func__, 2285 totlen, m->m_pkthdr.len); 2286 } 2287 } 2288 #endif 2289 2290 struct m_tag * 2291 m_tag_get(int type, int len, int wait) 2292 { 2293 struct m_tag *t; 2294 2295 if (len < 0) 2296 return NULL; 2297 t = malloc(len + sizeof(struct m_tag), M_PACKET_TAGS, wait); 2298 if (t == NULL) 2299 return NULL; 2300 t->m_tag_id = type; 2301 t->m_tag_len = len; 2302 return t; 2303 } 2304 2305 void 2306 m_tag_free(struct m_tag *t) 2307 { 2308 free(t, M_PACKET_TAGS); 2309 } 2310 2311 void 2312 m_tag_prepend(struct mbuf *m, struct m_tag *t) 2313 { 2314 KASSERT((m->m_flags & M_PKTHDR) != 0); 2315 SLIST_INSERT_HEAD(&m->m_pkthdr.tags, t, m_tag_link); 2316 } 2317 2318 void 2319 m_tag_unlink(struct mbuf *m, struct m_tag *t) 2320 { 2321 KASSERT((m->m_flags & M_PKTHDR) != 0); 2322 SLIST_REMOVE(&m->m_pkthdr.tags, t, m_tag, m_tag_link); 2323 } 2324 2325 void 2326 m_tag_delete(struct mbuf *m, struct m_tag *t) 2327 { 2328 m_tag_unlink(m, t); 2329 m_tag_free(t); 2330 } 2331 2332 void 2333 m_tag_delete_chain(struct mbuf *m) 2334 { 2335 struct m_tag *p, *q; 2336 2337 KASSERT((m->m_flags & M_PKTHDR) != 0); 2338 2339 p = SLIST_FIRST(&m->m_pkthdr.tags); 2340 if (p == NULL) 2341 return; 2342 while ((q = SLIST_NEXT(p, m_tag_link)) != NULL) 2343 m_tag_delete(m, q); 2344 m_tag_delete(m, p); 2345 } 2346 2347 struct m_tag * 2348 m_tag_find(const struct mbuf *m, int type) 2349 { 2350 struct m_tag *p; 2351 2352 KASSERT((m->m_flags & M_PKTHDR) != 0); 2353 2354 p = SLIST_FIRST(&m->m_pkthdr.tags); 2355 while (p != NULL) { 2356 if (p->m_tag_id == type) 2357 return p; 2358 p = SLIST_NEXT(p, m_tag_link); 2359 } 2360 return NULL; 2361 } 2362 2363 struct m_tag * 2364 m_tag_copy(struct m_tag *t) 2365 { 2366 struct m_tag *p; 2367 2368 p = m_tag_get(t->m_tag_id, t->m_tag_len, M_NOWAIT); 2369 if (p == NULL) 2370 return NULL; 2371 memcpy(p + 1, t + 1, t->m_tag_len); 2372 return p; 2373 } 2374 2375 /* 2376 * Copy two tag chains. The destination mbuf (to) loses any attached 2377 * tags even if the operation fails. This should not be a problem, as 2378 * m_tag_copy_chain() is typically called with a newly-allocated 2379 * destination mbuf. 2380 */ 2381 int 2382 m_tag_copy_chain(struct mbuf *to, struct mbuf *from) 2383 { 2384 struct m_tag *p, *t, *tprev = NULL; 2385 2386 KASSERT((from->m_flags & M_PKTHDR) != 0); 2387 2388 m_tag_delete_chain(to); 2389 SLIST_FOREACH(p, &from->m_pkthdr.tags, m_tag_link) { 2390 t = m_tag_copy(p); 2391 if (t == NULL) { 2392 m_tag_delete_chain(to); 2393 return 0; 2394 } 2395 if (tprev == NULL) 2396 SLIST_INSERT_HEAD(&to->m_pkthdr.tags, t, m_tag_link); 2397 else 2398 SLIST_INSERT_AFTER(tprev, t, m_tag_link); 2399 tprev = t; 2400 } 2401 return 1; 2402 } 2403