1 /* $NetBSD: uipc_mbuf.c,v 1.53 2001/07/26 19:05:04 thorpej Exp $ */ 2 3 /*- 4 * Copyright (c) 1999, 2001 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 9 * NASA Ames Research Center. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 3. All advertising materials mentioning features or use of this software 20 * must display the following acknowledgement: 21 * This product includes software developed by the NetBSD 22 * Foundation, Inc. and its contributors. 23 * 4. Neither the name of The NetBSD Foundation nor the names of its 24 * contributors may be used to endorse or promote products derived 25 * from this software without specific prior written permission. 26 * 27 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 28 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 29 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 30 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 31 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 32 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 33 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 34 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 35 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 36 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 37 * POSSIBILITY OF SUCH DAMAGE. 38 */ 39 40 /* 41 * Copyright (c) 1982, 1986, 1988, 1991, 1993 42 * The Regents of the University of California. All rights reserved. 43 * 44 * Redistribution and use in source and binary forms, with or without 45 * modification, are permitted provided that the following conditions 46 * are met: 47 * 1. Redistributions of source code must retain the above copyright 48 * notice, this list of conditions and the following disclaimer. 49 * 2. Redistributions in binary form must reproduce the above copyright 50 * notice, this list of conditions and the following disclaimer in the 51 * documentation and/or other materials provided with the distribution. 52 * 3. All advertising materials mentioning features or use of this software 53 * must display the following acknowledgement: 54 * This product includes software developed by the University of 55 * California, Berkeley and its contributors. 56 * 4. Neither the name of the University nor the names of its contributors 57 * may be used to endorse or promote products derived from this software 58 * without specific prior written permission. 59 * 60 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 61 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 62 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 63 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 64 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 65 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 66 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 67 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 68 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 69 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 70 * SUCH DAMAGE. 71 * 72 * @(#)uipc_mbuf.c 8.4 (Berkeley) 2/14/95 73 */ 74 75 #include <sys/param.h> 76 #include <sys/systm.h> 77 #include <sys/proc.h> 78 #include <sys/malloc.h> 79 #include <sys/map.h> 80 #define MBTYPES 81 #include <sys/mbuf.h> 82 #include <sys/kernel.h> 83 #include <sys/syslog.h> 84 #include <sys/domain.h> 85 #include <sys/protosw.h> 86 #include <sys/pool.h> 87 #include <sys/socket.h> 88 #include <net/if.h> 89 90 #include <uvm/uvm_extern.h> 91 92 #include <sys/sysctl.h> 93 94 struct pool mbpool; /* mbuf pool */ 95 struct pool mclpool; /* mbuf cluster pool */ 96 97 struct pool_cache mbpool_cache; 98 struct pool_cache mclpool_cache; 99 100 struct mbstat mbstat; 101 int max_linkhdr; 102 int max_protohdr; 103 int max_hdr; 104 int max_datalen; 105 106 void *mclpool_alloc __P((unsigned long, int, int)); 107 void mclpool_release __P((void *, unsigned long, int)); 108 static struct mbuf *m_copym0 __P((struct mbuf *, int, int, int, int)); 109 110 const char *mclpool_warnmsg = 111 "WARNING: mclpool limit reached; increase NMBCLUSTERS"; 112 113 /* 114 * Initialize the mbuf allcator. 115 */ 116 void 117 mbinit() 118 { 119 120 pool_init(&mbpool, msize, 0, 0, 0, "mbpl", 0, NULL, NULL, 0); 121 pool_init(&mclpool, mclbytes, 0, 0, 0, "mclpl", 0, mclpool_alloc, 122 mclpool_release, 0); 123 124 pool_cache_init(&mbpool_cache, &mbpool, NULL, NULL, NULL); 125 pool_cache_init(&mclpool_cache, &mclpool, NULL, NULL, NULL); 126 127 /* 128 * Set the hard limit on the mclpool to the number of 129 * mbuf clusters the kernel is to support. Log the limit 130 * reached message max once a minute. 131 */ 132 pool_sethardlimit(&mclpool, nmbclusters, mclpool_warnmsg, 60); 133 134 /* 135 * Set a low water mark for both mbufs and clusters. This should 136 * help ensure that they can be allocated in a memory starvation 137 * situation. This is important for e.g. diskless systems which 138 * must allocate mbufs in order for the pagedaemon to clean pages. 139 */ 140 pool_setlowat(&mbpool, mblowat); 141 pool_setlowat(&mclpool, mcllowat); 142 } 143 144 int 145 sysctl_dombuf(name, namelen, oldp, oldlenp, newp, newlen) 146 int *name; 147 u_int namelen; 148 void *oldp; 149 size_t *oldlenp; 150 void *newp; 151 size_t newlen; 152 { 153 int error, newval; 154 155 /* All sysctl names at this level are terminal. */ 156 if (namelen != 1) 157 return (ENOTDIR); /* overloaded */ 158 159 switch (name[0]) { 160 case MBUF_MSIZE: 161 return (sysctl_rdint(oldp, oldlenp, newp, msize)); 162 case MBUF_MCLBYTES: 163 return (sysctl_rdint(oldp, oldlenp, newp, mclbytes)); 164 case MBUF_NMBCLUSTERS: 165 /* 166 * If we have direct-mapped pool pages, we can adjust this 167 * number on the fly. If not, we're limited by the size 168 * of mb_map, and cannot change this value. 169 * 170 * Note: we only allow the value to be increased, never 171 * decreased. 172 */ 173 if (mb_map == NULL) { 174 newval = nmbclusters; 175 error = sysctl_int(oldp, oldlenp, newp, newlen, 176 &newval); 177 if (error != 0) 178 return (error); 179 if (newp != NULL) { 180 if (newval >= nmbclusters) { 181 nmbclusters = newval; 182 pool_sethardlimit(&mclpool, 183 nmbclusters, mclpool_warnmsg, 60); 184 } else 185 error = EINVAL; 186 } 187 return (error); 188 } else 189 return (sysctl_rdint(oldp, oldlenp, newp, nmbclusters)); 190 case MBUF_MBLOWAT: 191 case MBUF_MCLLOWAT: 192 /* New value must be >= 0. */ 193 newval = (name[0] == MBUF_MBLOWAT) ? mblowat : mcllowat; 194 error = sysctl_int(oldp, oldlenp, newp, newlen, &newval); 195 if (error != 0) 196 return (error); 197 if (newp != NULL) { 198 if (newval >= 0) { 199 if (name[0] == MBUF_MBLOWAT) { 200 mblowat = newval; 201 pool_setlowat(&mbpool, newval); 202 } else { 203 mcllowat = newval; 204 pool_setlowat(&mclpool, newval); 205 } 206 } else 207 error = EINVAL; 208 } 209 return (error); 210 default: 211 return (EOPNOTSUPP); 212 } 213 /* NOTREACHED */ 214 } 215 216 void * 217 mclpool_alloc(sz, flags, mtype) 218 unsigned long sz; 219 int flags; 220 int mtype; 221 { 222 boolean_t waitok = (flags & PR_WAITOK) ? TRUE : FALSE; 223 224 return ((void *)uvm_km_alloc_poolpage1(mb_map, uvmexp.mb_object, 225 waitok)); 226 } 227 228 void 229 mclpool_release(v, sz, mtype) 230 void *v; 231 unsigned long sz; 232 int mtype; 233 { 234 235 uvm_km_free_poolpage1(mb_map, (vaddr_t)v); 236 } 237 238 /* 239 * When MGET failes, ask protocols to free space when short of memory, 240 * then re-attempt to allocate an mbuf. 241 */ 242 struct mbuf * 243 m_retry(i, t) 244 int i, t; 245 { 246 struct mbuf *m; 247 248 m_reclaim(i); 249 #define m_retry(i, t) (struct mbuf *)0 250 MGET(m, i, t); 251 #undef m_retry 252 if (m != NULL) 253 mbstat.m_wait++; 254 else 255 mbstat.m_drops++; 256 return (m); 257 } 258 259 /* 260 * As above; retry an MGETHDR. 261 */ 262 struct mbuf * 263 m_retryhdr(i, t) 264 int i, t; 265 { 266 struct mbuf *m; 267 268 m_reclaim(i); 269 #define m_retryhdr(i, t) (struct mbuf *)0 270 MGETHDR(m, i, t); 271 #undef m_retryhdr 272 if (m != NULL) 273 mbstat.m_wait++; 274 else 275 mbstat.m_drops++; 276 return (m); 277 } 278 279 void 280 m_reclaim(how) 281 int how; 282 { 283 struct domain *dp; 284 struct protosw *pr; 285 struct ifnet *ifp; 286 int s = splvm(); 287 288 for (dp = domains; dp; dp = dp->dom_next) 289 for (pr = dp->dom_protosw; 290 pr < dp->dom_protoswNPROTOSW; pr++) 291 if (pr->pr_drain) 292 (*pr->pr_drain)(); 293 for (ifp = TAILQ_FIRST(&ifnet); ifp; ifp = TAILQ_NEXT(ifp, if_list)) 294 if (ifp->if_drain) 295 (*ifp->if_drain)(ifp); 296 splx(s); 297 mbstat.m_drain++; 298 } 299 300 /* 301 * Space allocation routines. 302 * These are also available as macros 303 * for critical paths. 304 */ 305 struct mbuf * 306 m_get(nowait, type) 307 int nowait, type; 308 { 309 struct mbuf *m; 310 311 MGET(m, nowait, type); 312 return (m); 313 } 314 315 struct mbuf * 316 m_gethdr(nowait, type) 317 int nowait, type; 318 { 319 struct mbuf *m; 320 321 MGETHDR(m, nowait, type); 322 return (m); 323 } 324 325 struct mbuf * 326 m_getclr(nowait, type) 327 int nowait, type; 328 { 329 struct mbuf *m; 330 331 MGET(m, nowait, type); 332 if (m == 0) 333 return (0); 334 memset(mtod(m, caddr_t), 0, MLEN); 335 return (m); 336 } 337 338 struct mbuf * 339 m_free(m) 340 struct mbuf *m; 341 { 342 struct mbuf *n; 343 344 MFREE(m, n); 345 return (n); 346 } 347 348 void 349 m_freem(m) 350 struct mbuf *m; 351 { 352 struct mbuf *n; 353 354 if (m == NULL) 355 return; 356 do { 357 MFREE(m, n); 358 m = n; 359 } while (m); 360 } 361 362 /* 363 * Mbuffer utility routines. 364 */ 365 366 /* 367 * Lesser-used path for M_PREPEND: 368 * allocate new mbuf to prepend to chain, 369 * copy junk along. 370 */ 371 struct mbuf * 372 m_prepend(m, len, how) 373 struct mbuf *m; 374 int len, how; 375 { 376 struct mbuf *mn; 377 378 MGET(mn, how, m->m_type); 379 if (mn == (struct mbuf *)NULL) { 380 m_freem(m); 381 return ((struct mbuf *)NULL); 382 } 383 if (m->m_flags & M_PKTHDR) { 384 M_COPY_PKTHDR(mn, m); 385 m->m_flags &= ~M_PKTHDR; 386 } 387 mn->m_next = m; 388 m = mn; 389 if (len < MHLEN) 390 MH_ALIGN(m, len); 391 m->m_len = len; 392 return (m); 393 } 394 395 /* 396 * Make a copy of an mbuf chain starting "off0" bytes from the beginning, 397 * continuing for "len" bytes. If len is M_COPYALL, copy to end of mbuf. 398 * The wait parameter is a choice of M_WAIT/M_DONTWAIT from caller. 399 */ 400 int MCFail; 401 402 struct mbuf * 403 m_copym(m, off0, len, wait) 404 struct mbuf *m; 405 int off0, wait; 406 int len; 407 { 408 return m_copym0(m, off0, len, wait, 0); /* shallow copy on M_EXT */ 409 } 410 411 struct mbuf * 412 m_dup(m, off0, len, wait) 413 struct mbuf *m; 414 int off0, wait; 415 int len; 416 { 417 return m_copym0(m, off0, len, wait, 1); /* deep copy */ 418 } 419 420 static struct mbuf * 421 m_copym0(m, off0, len, wait, deep) 422 struct mbuf *m; 423 int off0, wait; 424 int len; 425 int deep; /* deep copy */ 426 { 427 struct mbuf *n, **np; 428 int off = off0; 429 struct mbuf *top; 430 int copyhdr = 0; 431 432 if (off < 0 || len < 0) 433 panic("m_copym: off %d, len %d", off, len); 434 if (off == 0 && m->m_flags & M_PKTHDR) 435 copyhdr = 1; 436 while (off > 0) { 437 if (m == 0) 438 panic("m_copym: m == 0"); 439 if (off < m->m_len) 440 break; 441 off -= m->m_len; 442 m = m->m_next; 443 } 444 np = ⊤ 445 top = 0; 446 while (len > 0) { 447 if (m == 0) { 448 if (len != M_COPYALL) 449 panic("m_copym: m == 0 and not COPYALL"); 450 break; 451 } 452 MGET(n, wait, m->m_type); 453 *np = n; 454 if (n == 0) 455 goto nospace; 456 if (copyhdr) { 457 M_COPY_PKTHDR(n, m); 458 if (len == M_COPYALL) 459 n->m_pkthdr.len -= off0; 460 else 461 n->m_pkthdr.len = len; 462 copyhdr = 0; 463 } 464 n->m_len = min(len, m->m_len - off); 465 if (m->m_flags & M_EXT) { 466 if (!deep) { 467 n->m_data = m->m_data + off; 468 n->m_ext = m->m_ext; 469 MCLADDREFERENCE(m, n); 470 } else { 471 /* 472 * we are unsure about the way m was allocated. 473 * copy into multiple MCLBYTES cluster mbufs. 474 */ 475 MCLGET(n, wait); 476 n->m_len = 0; 477 n->m_len = M_TRAILINGSPACE(n); 478 n->m_len = min(n->m_len, len); 479 n->m_len = min(n->m_len, m->m_len - off); 480 memcpy(mtod(n, caddr_t), mtod(m, caddr_t) + off, 481 (unsigned)n->m_len); 482 } 483 } else 484 memcpy(mtod(n, caddr_t), mtod(m, caddr_t)+off, 485 (unsigned)n->m_len); 486 if (len != M_COPYALL) 487 len -= n->m_len; 488 off += n->m_len; 489 #ifdef DIAGNOSTIC 490 if (off > m->m_len) 491 panic("m_copym0 overrun"); 492 #endif 493 if (off == m->m_len) { 494 m = m->m_next; 495 off = 0; 496 } 497 np = &n->m_next; 498 } 499 if (top == 0) 500 MCFail++; 501 return (top); 502 nospace: 503 m_freem(top); 504 MCFail++; 505 return (0); 506 } 507 508 /* 509 * Copy an entire packet, including header (which must be present). 510 * An optimization of the common case `m_copym(m, 0, M_COPYALL, how)'. 511 */ 512 struct mbuf * 513 m_copypacket(m, how) 514 struct mbuf *m; 515 int how; 516 { 517 struct mbuf *top, *n, *o; 518 519 MGET(n, how, m->m_type); 520 top = n; 521 if (!n) 522 goto nospace; 523 524 M_COPY_PKTHDR(n, m); 525 n->m_len = m->m_len; 526 if (m->m_flags & M_EXT) { 527 n->m_data = m->m_data; 528 n->m_ext = m->m_ext; 529 MCLADDREFERENCE(m, n); 530 } else { 531 memcpy(mtod(n, char *), mtod(m, char *), n->m_len); 532 } 533 534 m = m->m_next; 535 while (m) { 536 MGET(o, how, m->m_type); 537 if (!o) 538 goto nospace; 539 540 n->m_next = o; 541 n = n->m_next; 542 543 n->m_len = m->m_len; 544 if (m->m_flags & M_EXT) { 545 n->m_data = m->m_data; 546 n->m_ext = m->m_ext; 547 MCLADDREFERENCE(m, n); 548 } else { 549 memcpy(mtod(n, char *), mtod(m, char *), n->m_len); 550 } 551 552 m = m->m_next; 553 } 554 return top; 555 nospace: 556 m_freem(top); 557 MCFail++; 558 return 0; 559 } 560 561 /* 562 * Copy data from an mbuf chain starting "off" bytes from the beginning, 563 * continuing for "len" bytes, into the indicated buffer. 564 */ 565 void 566 m_copydata(m, off, len, cp) 567 struct mbuf *m; 568 int off; 569 int len; 570 caddr_t cp; 571 { 572 unsigned count; 573 574 if (off < 0 || len < 0) 575 panic("m_copydata"); 576 while (off > 0) { 577 if (m == 0) 578 panic("m_copydata"); 579 if (off < m->m_len) 580 break; 581 off -= m->m_len; 582 m = m->m_next; 583 } 584 while (len > 0) { 585 if (m == 0) 586 panic("m_copydata"); 587 count = min(m->m_len - off, len); 588 memcpy(cp, mtod(m, caddr_t) + off, count); 589 len -= count; 590 cp += count; 591 off = 0; 592 m = m->m_next; 593 } 594 } 595 596 /* 597 * Concatenate mbuf chain n to m. 598 * Both chains must be of the same type (e.g. MT_DATA). 599 * Any m_pkthdr is not updated. 600 */ 601 void 602 m_cat(m, n) 603 struct mbuf *m, *n; 604 { 605 while (m->m_next) 606 m = m->m_next; 607 while (n) { 608 if (m->m_flags & M_EXT || 609 m->m_data + m->m_len + n->m_len >= &m->m_dat[MLEN]) { 610 /* just join the two chains */ 611 m->m_next = n; 612 return; 613 } 614 /* splat the data from one into the other */ 615 memcpy(mtod(m, caddr_t) + m->m_len, mtod(n, caddr_t), 616 (u_int)n->m_len); 617 m->m_len += n->m_len; 618 n = m_free(n); 619 } 620 } 621 622 void 623 m_adj(mp, req_len) 624 struct mbuf *mp; 625 int req_len; 626 { 627 int len = req_len; 628 struct mbuf *m; 629 int count; 630 631 if ((m = mp) == NULL) 632 return; 633 if (len >= 0) { 634 /* 635 * Trim from head. 636 */ 637 while (m != NULL && len > 0) { 638 if (m->m_len <= len) { 639 len -= m->m_len; 640 m->m_len = 0; 641 m = m->m_next; 642 } else { 643 m->m_len -= len; 644 m->m_data += len; 645 len = 0; 646 } 647 } 648 m = mp; 649 if (mp->m_flags & M_PKTHDR) 650 m->m_pkthdr.len -= (req_len - len); 651 } else { 652 /* 653 * Trim from tail. Scan the mbuf chain, 654 * calculating its length and finding the last mbuf. 655 * If the adjustment only affects this mbuf, then just 656 * adjust and return. Otherwise, rescan and truncate 657 * after the remaining size. 658 */ 659 len = -len; 660 count = 0; 661 for (;;) { 662 count += m->m_len; 663 if (m->m_next == (struct mbuf *)0) 664 break; 665 m = m->m_next; 666 } 667 if (m->m_len >= len) { 668 m->m_len -= len; 669 if (mp->m_flags & M_PKTHDR) 670 mp->m_pkthdr.len -= len; 671 return; 672 } 673 count -= len; 674 if (count < 0) 675 count = 0; 676 /* 677 * Correct length for chain is "count". 678 * Find the mbuf with last data, adjust its length, 679 * and toss data from remaining mbufs on chain. 680 */ 681 m = mp; 682 if (m->m_flags & M_PKTHDR) 683 m->m_pkthdr.len = count; 684 for (; m; m = m->m_next) { 685 if (m->m_len >= count) { 686 m->m_len = count; 687 break; 688 } 689 count -= m->m_len; 690 } 691 while (m->m_next) 692 (m = m->m_next) ->m_len = 0; 693 } 694 } 695 696 /* 697 * Rearange an mbuf chain so that len bytes are contiguous 698 * and in the data area of an mbuf (so that mtod and dtom 699 * will work for a structure of size len). Returns the resulting 700 * mbuf chain on success, frees it and returns null on failure. 701 * If there is room, it will add up to max_protohdr-len extra bytes to the 702 * contiguous region in an attempt to avoid being called next time. 703 */ 704 int MPFail; 705 706 struct mbuf * 707 m_pullup(n, len) 708 struct mbuf *n; 709 int len; 710 { 711 struct mbuf *m; 712 int count; 713 int space; 714 715 /* 716 * If first mbuf has no cluster, and has room for len bytes 717 * without shifting current data, pullup into it, 718 * otherwise allocate a new mbuf to prepend to the chain. 719 */ 720 if ((n->m_flags & M_EXT) == 0 && 721 n->m_data + len < &n->m_dat[MLEN] && n->m_next) { 722 if (n->m_len >= len) 723 return (n); 724 m = n; 725 n = n->m_next; 726 len -= m->m_len; 727 } else { 728 if (len > MHLEN) 729 goto bad; 730 MGET(m, M_DONTWAIT, n->m_type); 731 if (m == 0) 732 goto bad; 733 m->m_len = 0; 734 if (n->m_flags & M_PKTHDR) { 735 M_COPY_PKTHDR(m, n); 736 n->m_flags &= ~M_PKTHDR; 737 } 738 } 739 space = &m->m_dat[MLEN] - (m->m_data + m->m_len); 740 do { 741 count = min(min(max(len, max_protohdr), space), n->m_len); 742 memcpy(mtod(m, caddr_t) + m->m_len, mtod(n, caddr_t), 743 (unsigned)count); 744 len -= count; 745 m->m_len += count; 746 n->m_len -= count; 747 space -= count; 748 if (n->m_len) 749 n->m_data += count; 750 else 751 n = m_free(n); 752 } while (len > 0 && n); 753 if (len > 0) { 754 (void) m_free(m); 755 goto bad; 756 } 757 m->m_next = n; 758 return (m); 759 bad: 760 m_freem(n); 761 MPFail++; 762 return (0); 763 } 764 765 /* 766 * Partition an mbuf chain in two pieces, returning the tail -- 767 * all but the first len0 bytes. In case of failure, it returns NULL and 768 * attempts to restore the chain to its original state. 769 */ 770 struct mbuf * 771 m_split(m0, len0, wait) 772 struct mbuf *m0; 773 int len0, wait; 774 { 775 struct mbuf *m, *n; 776 unsigned len = len0, remain, len_save; 777 778 for (m = m0; m && len > m->m_len; m = m->m_next) 779 len -= m->m_len; 780 if (m == 0) 781 return (0); 782 remain = m->m_len - len; 783 if (m0->m_flags & M_PKTHDR) { 784 MGETHDR(n, wait, m0->m_type); 785 if (n == 0) 786 return (0); 787 n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif; 788 n->m_pkthdr.len = m0->m_pkthdr.len - len0; 789 len_save = m0->m_pkthdr.len; 790 m0->m_pkthdr.len = len0; 791 if (m->m_flags & M_EXT) 792 goto extpacket; 793 if (remain > MHLEN) { 794 /* m can't be the lead packet */ 795 MH_ALIGN(n, 0); 796 n->m_next = m_split(m, len, wait); 797 if (n->m_next == 0) { 798 (void) m_free(n); 799 m0->m_pkthdr.len = len_save; 800 return (0); 801 } else 802 return (n); 803 } else 804 MH_ALIGN(n, remain); 805 } else if (remain == 0) { 806 n = m->m_next; 807 m->m_next = 0; 808 return (n); 809 } else { 810 MGET(n, wait, m->m_type); 811 if (n == 0) 812 return (0); 813 M_ALIGN(n, remain); 814 } 815 extpacket: 816 if (m->m_flags & M_EXT) { 817 n->m_ext = m->m_ext; 818 MCLADDREFERENCE(m, n); 819 n->m_data = m->m_data + len; 820 } else { 821 memcpy(mtod(n, caddr_t), mtod(m, caddr_t) + len, remain); 822 } 823 n->m_len = remain; 824 m->m_len = len; 825 n->m_next = m->m_next; 826 m->m_next = 0; 827 return (n); 828 } 829 /* 830 * Routine to copy from device local memory into mbufs. 831 */ 832 struct mbuf * 833 m_devget(buf, totlen, off0, ifp, copy) 834 char *buf; 835 int totlen, off0; 836 struct ifnet *ifp; 837 void (*copy) __P((const void *from, void *to, size_t len)); 838 { 839 struct mbuf *m; 840 struct mbuf *top = 0, **mp = ⊤ 841 int off = off0, len; 842 char *cp; 843 char *epkt; 844 845 cp = buf; 846 epkt = cp + totlen; 847 if (off) { 848 /* 849 * If 'off' is non-zero, packet is trailer-encapsulated, 850 * so we have to skip the type and length fields. 851 */ 852 cp += off + 2 * sizeof(u_int16_t); 853 totlen -= 2 * sizeof(u_int16_t); 854 } 855 MGETHDR(m, M_DONTWAIT, MT_DATA); 856 if (m == 0) 857 return (0); 858 m->m_pkthdr.rcvif = ifp; 859 m->m_pkthdr.len = totlen; 860 m->m_len = MHLEN; 861 862 while (totlen > 0) { 863 if (top) { 864 MGET(m, M_DONTWAIT, MT_DATA); 865 if (m == 0) { 866 m_freem(top); 867 return (0); 868 } 869 m->m_len = MLEN; 870 } 871 len = min(totlen, epkt - cp); 872 if (len >= MINCLSIZE) { 873 MCLGET(m, M_DONTWAIT); 874 if ((m->m_flags & M_EXT) == 0) { 875 m_free(m); 876 m_freem(top); 877 return (0); 878 } 879 m->m_len = len = min(len, MCLBYTES); 880 } else { 881 /* 882 * Place initial small packet/header at end of mbuf. 883 */ 884 if (len < m->m_len) { 885 if (top == 0 && len + max_linkhdr <= m->m_len) 886 m->m_data += max_linkhdr; 887 m->m_len = len; 888 } else 889 len = m->m_len; 890 } 891 if (copy) 892 copy(cp, mtod(m, caddr_t), (size_t)len); 893 else 894 memcpy(mtod(m, caddr_t), cp, (size_t)len); 895 cp += len; 896 *mp = m; 897 mp = &m->m_next; 898 totlen -= len; 899 if (cp == epkt) 900 cp = buf; 901 } 902 return (top); 903 } 904 905 /* 906 * Copy data from a buffer back into the indicated mbuf chain, 907 * starting "off" bytes from the beginning, extending the mbuf 908 * chain if necessary. 909 */ 910 void 911 m_copyback(m0, off, len, cp) 912 struct mbuf *m0; 913 int off; 914 int len; 915 caddr_t cp; 916 { 917 int mlen; 918 struct mbuf *m = m0, *n; 919 int totlen = 0; 920 921 if (m0 == 0) 922 return; 923 while (off > (mlen = m->m_len)) { 924 off -= mlen; 925 totlen += mlen; 926 if (m->m_next == 0) { 927 n = m_getclr(M_DONTWAIT, m->m_type); 928 if (n == 0) 929 goto out; 930 n->m_len = min(MLEN, len + off); 931 m->m_next = n; 932 } 933 m = m->m_next; 934 } 935 while (len > 0) { 936 mlen = min (m->m_len - off, len); 937 memcpy(mtod(m, caddr_t) + off, cp, (unsigned)mlen); 938 cp += mlen; 939 len -= mlen; 940 mlen += off; 941 off = 0; 942 totlen += mlen; 943 if (len == 0) 944 break; 945 if (m->m_next == 0) { 946 n = m_get(M_DONTWAIT, m->m_type); 947 if (n == 0) 948 break; 949 n->m_len = min(MLEN, len); 950 m->m_next = n; 951 } 952 m = m->m_next; 953 } 954 out: if (((m = m0)->m_flags & M_PKTHDR) && (m->m_pkthdr.len < totlen)) 955 m->m_pkthdr.len = totlen; 956 } 957