1 /* $OpenBSD: uipc_mbuf.c,v 1.40 2001/06/27 04:49:47 art Exp $ */ 2 /* $NetBSD: uipc_mbuf.c,v 1.15.4.1 1996/06/13 17:11:44 cgd Exp $ */ 3 4 /* 5 * Copyright (c) 1982, 1986, 1988, 1991, 1993 6 * The Regents of the University of California. All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 3. All advertising materials mentioning features or use of this software 17 * must display the following acknowledgement: 18 * This product includes software developed by the University of 19 * California, Berkeley and its contributors. 20 * 4. Neither the name of the University nor the names of its contributors 21 * may be used to endorse or promote products derived from this software 22 * without specific prior written permission. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 27 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 34 * SUCH DAMAGE. 35 * 36 * @(#)uipc_mbuf.c 8.2 (Berkeley) 1/4/94 37 */ 38 39 /* 40 %%% portions-copyright-nrl-95 41 Portions of this software are Copyright 1995-1998 by Randall Atkinson, 42 Ronald Lee, Daniel McDonald, Bao Phan, and Chris Winters. All Rights 43 Reserved. All rights under this copyright have been assigned to the US 44 Naval Research Laboratory (NRL). The NRL Copyright Notice and License 45 Agreement Version 1.1 (January 17, 1995) applies to these portions of the 46 software. 47 You should have received a copy of the license with this software. If you 48 didn't get a copy, you may request one from <license@ipv6.nrl.navy.mil>. 49 */ 50 51 #include <sys/param.h> 52 #include <sys/systm.h> 53 #include <sys/proc.h> 54 #include <sys/malloc.h> 55 #include <sys/map.h> 56 #define MBTYPES 57 #include <sys/mbuf.h> 58 #include <sys/kernel.h> 59 #include <sys/syslog.h> 60 #include <sys/domain.h> 61 #include <sys/protosw.h> 62 #include <sys/pool.h> 63 64 #include <machine/cpu.h> 65 66 #include <vm/vm.h> 67 68 #include <uvm/uvm_extern.h> 69 70 struct pool mbpool; /* mbuf pool */ 71 struct pool mclpool; /* mbuf cluster pool */ 72 73 extern vm_map_t mb_map; 74 struct mbuf *mbutl; 75 int needqueuedrain; 76 77 void *mclpool_alloc __P((unsigned long, int, int)); 78 void mclpool_release __P((void *, unsigned long, int)); 79 struct mbuf *m_copym0 __P((struct mbuf *, int, int, int, int)); 80 81 const char *mclpool_warnmsg = 82 "WARNING: mclpool limit reached; increase NMBCLUSTERS"; 83 84 /* 85 * Initialize the mbuf allcator. 86 */ 87 void 88 mbinit() 89 { 90 pool_init(&mbpool, MSIZE, 0, 0, 0, "mbpl", 0, NULL, NULL, 0); 91 pool_init(&mclpool, MCLBYTES, 0, 0, 0, "mclpl", 0, mclpool_alloc, 92 mclpool_release, 0); 93 94 /* 95 * Set the hard limit on the mclpool to the number of 96 * mbuf clusters the kernel is to support. Log the limit 97 * reached message max once a minute. 98 */ 99 pool_sethardlimit(&mclpool, nmbclusters, mclpool_warnmsg, 60); 100 101 /* 102 * Set a low water mark for both mbufs and clusters. This should 103 * help ensure that they can be allocated in a memory starvation 104 * situation. This is important for e.g. diskless systems which 105 * must allocate mbufs in order for the pagedaemon to clean pages. 106 */ 107 pool_setlowat(&mbpool, mblowat); 108 pool_setlowat(&mclpool, mcllowat); 109 } 110 111 112 void * 113 mclpool_alloc(sz, flags, mtype) 114 unsigned long sz; 115 int flags; 116 int mtype; 117 { 118 boolean_t waitok = (flags & PR_WAITOK) ? TRUE : FALSE; 119 120 return ((void *)uvm_km_alloc_poolpage1(mb_map, uvmexp.mb_object, 121 waitok)); 122 } 123 124 void 125 mclpool_release(v, sz, mtype) 126 void *v; 127 unsigned long sz; 128 int mtype; 129 { 130 uvm_km_free_poolpage1(mb_map, (vaddr_t)v); 131 } 132 133 /* 134 * When MGET failes, ask protocols to free space when short of memory, 135 * then re-attempt to allocate an mbuf. 136 */ 137 struct mbuf * 138 m_retry(i, t) 139 int i, t; 140 { 141 register struct mbuf *m; 142 143 if (i & M_DONTWAIT) { 144 needqueuedrain = 1; 145 setsoftnet(); 146 return (NULL); 147 } 148 m_reclaim(); 149 #define m_retry(i, t) NULL 150 MGET(m, i, t); 151 #undef m_retry 152 if (m != NULL) 153 mbstat.m_wait++; 154 else 155 mbstat.m_drops++; 156 return (m); 157 } 158 159 /* 160 * As above; retry an MGETHDR. 161 */ 162 struct mbuf * 163 m_retryhdr(i, t) 164 int i, t; 165 { 166 register struct mbuf *m; 167 168 if (i & M_DONTWAIT) { 169 needqueuedrain = 1; 170 setsoftnet(); 171 return (NULL); 172 } 173 m_reclaim(); 174 #define m_retryhdr(i, t) NULL 175 MGETHDR(m, i, t); 176 #undef m_retryhdr 177 if (m != NULL) 178 mbstat.m_wait++; 179 else 180 mbstat.m_drops++; 181 return (m); 182 } 183 184 void 185 m_reclaim() 186 { 187 register struct domain *dp; 188 register struct protosw *pr; 189 int s = splimp(); 190 191 needqueuedrain = 0; 192 for (dp = domains; dp; dp = dp->dom_next) 193 for (pr = dp->dom_protosw; pr < dp->dom_protoswNPROTOSW; pr++) 194 if (pr->pr_drain) 195 (*pr->pr_drain)(); 196 splx(s); 197 mbstat.m_drain++; 198 } 199 200 /* 201 * Space allocation routines. 202 * These are also available as macros 203 * for critical paths. 204 */ 205 struct mbuf * 206 m_get(nowait, type) 207 int nowait, type; 208 { 209 register struct mbuf *m; 210 211 MGET(m, nowait, type); 212 return (m); 213 } 214 215 struct mbuf * 216 m_gethdr(nowait, type) 217 int nowait, type; 218 { 219 register struct mbuf *m; 220 221 MGETHDR(m, nowait, type); 222 return (m); 223 } 224 225 struct mbuf * 226 m_getclr(nowait, type) 227 int nowait, type; 228 { 229 register struct mbuf *m; 230 231 MGET(m, nowait, type); 232 if (m == NULL) 233 return (NULL); 234 memset(mtod(m, caddr_t), 0, MLEN); 235 return (m); 236 } 237 238 struct mbuf * 239 m_free(m) 240 struct mbuf *m; 241 { 242 register struct mbuf *n; 243 244 MFREE(m, n); 245 return (n); 246 } 247 248 void 249 m_freem(m) 250 register struct mbuf *m; 251 { 252 register struct mbuf *n; 253 254 if (m == NULL) 255 return; 256 do { 257 MFREE(m, n); 258 } while ((m = n) != NULL); 259 } 260 261 /* 262 * Mbuffer utility routines. 263 */ 264 265 /* 266 * Lesser-used path for M_PREPEND: 267 * allocate new mbuf to prepend to chain, 268 * copy junk along. 269 */ 270 struct mbuf * 271 m_prepend(m, len, how) 272 register struct mbuf *m; 273 int len, how; 274 { 275 struct mbuf *mn; 276 277 MGET(mn, how, m->m_type); 278 if (mn == NULL) { 279 m_freem(m); 280 return (NULL); 281 } 282 if (m->m_flags & M_PKTHDR) 283 M_MOVE_PKTHDR(mn, m); 284 mn->m_next = m; 285 m = mn; 286 if (len < MHLEN) 287 MH_ALIGN(m, len); 288 m->m_len = len; 289 return (m); 290 } 291 292 /* 293 * Make a copy of an mbuf chain starting "off0" bytes from the beginning, 294 * continuing for "len" bytes. If len is M_COPYALL, copy to end of mbuf. 295 * The wait parameter is a choice of M_WAIT/M_DONTWAIT from caller. 296 */ 297 int MCFail; 298 299 struct mbuf * 300 m_copym(m, off0, len, wait) 301 struct mbuf *m; 302 int off0, wait; 303 int len; 304 { 305 return m_copym0(m, off0, len, wait, 0); /* shallow copy on M_EXT */ 306 } 307 308 /* 309 * m_copym2() is like m_copym(), except it COPIES cluster mbufs, instead 310 * of merely bumping the reference count. 311 */ 312 struct mbuf * 313 m_copym2(m, off0, len, wait) 314 struct mbuf *m; 315 int off0, wait; 316 int len; 317 { 318 return m_copym0(m, off0, len, wait, 1); /* deep copy */ 319 } 320 321 struct mbuf * 322 m_copym0(m, off0, len, wait, deep) 323 struct mbuf *m; 324 int off0, wait; 325 int len; 326 int deep; /* deep copy */ 327 { 328 struct mbuf *n, **np; 329 int off = off0; 330 struct mbuf *top; 331 int copyhdr = 0; 332 333 if (off < 0 || len < 0) 334 panic("m_copym0: off %d, len %d", off, len); 335 if (off == 0 && m->m_flags & M_PKTHDR) 336 copyhdr = 1; 337 while (off > 0) { 338 if (m == 0) 339 panic("m_copym0: null mbuf"); 340 if (off < m->m_len) 341 break; 342 off -= m->m_len; 343 m = m->m_next; 344 } 345 np = ⊤ 346 top = 0; 347 while (len > 0) { 348 if (m == 0) { 349 if (len != M_COPYALL) 350 panic("m_copym0: m == 0 and not COPYALL"); 351 break; 352 } 353 MGET(n, wait, m->m_type); 354 *np = n; 355 if (n == 0) 356 goto nospace; 357 if (copyhdr) { 358 M_DUP_PKTHDR(n, m); 359 if (len == M_COPYALL) 360 n->m_pkthdr.len -= off0; 361 else 362 n->m_pkthdr.len = len; 363 copyhdr = 0; 364 } 365 n->m_len = min(len, m->m_len - off); 366 if (m->m_flags & M_EXT) { 367 if (!deep) { 368 n->m_data = m->m_data + off; 369 n->m_ext = m->m_ext; 370 MCLADDREFERENCE(m, n); 371 } else { 372 /* 373 * we are unsure about the way m was allocated. 374 * copy into multiple MCLBYTES cluster mbufs. 375 */ 376 MCLGET(n, wait); 377 n->m_len = 0; 378 n->m_len = M_TRAILINGSPACE(n); 379 n->m_len = min(n->m_len, len); 380 n->m_len = min(n->m_len, m->m_len - off); 381 memcpy(mtod(n, caddr_t), mtod(m, caddr_t) + off, 382 (unsigned)n->m_len); 383 } 384 } else 385 memcpy(mtod(n, caddr_t), mtod(m, caddr_t)+off, 386 (unsigned)n->m_len); 387 if (len != M_COPYALL) 388 len -= n->m_len; 389 off += n->m_len; 390 #ifdef DIAGNOSTIC 391 if (off > m->m_len) 392 panic("m_copym0 overrun"); 393 #endif 394 if (off == m->m_len) { 395 m = m->m_next; 396 off = 0; 397 } 398 np = &n->m_next; 399 } 400 if (top == 0) 401 MCFail++; 402 return (top); 403 nospace: 404 m_freem(top); 405 MCFail++; 406 return (0); 407 } 408 409 /* 410 * Copy data from an mbuf chain starting "off" bytes from the beginning, 411 * continuing for "len" bytes, into the indicated buffer. 412 */ 413 void 414 m_copydata(m, off, len, cp) 415 register struct mbuf *m; 416 register int off; 417 register int len; 418 caddr_t cp; 419 { 420 register unsigned count; 421 422 if (off < 0) 423 panic("m_copydata: off %d < 0", off); 424 if (len < 0) 425 panic("m_copydata: len %d < 0", len); 426 while (off > 0) { 427 if (m == NULL) 428 panic("m_copydata: null mbuf in skip"); 429 if (off < m->m_len) 430 break; 431 off -= m->m_len; 432 m = m->m_next; 433 } 434 while (len > 0) { 435 if (m == NULL) 436 panic("m_copydata: null mbuf"); 437 count = min(m->m_len - off, len); 438 bcopy(mtod(m, caddr_t) + off, cp, count); 439 len -= count; 440 cp += count; 441 off = 0; 442 m = m->m_next; 443 } 444 } 445 446 /* 447 * Concatenate mbuf chain n to m. 448 * Both chains must be of the same type (e.g. MT_DATA). 449 * Any m_pkthdr is not updated. 450 */ 451 void 452 m_cat(m, n) 453 register struct mbuf *m, *n; 454 { 455 while (m->m_next) 456 m = m->m_next; 457 while (n) { 458 if (m->m_flags & M_EXT || 459 m->m_data + m->m_len + n->m_len >= &m->m_dat[MLEN]) { 460 /* just join the two chains */ 461 m->m_next = n; 462 return; 463 } 464 /* splat the data from one into the other */ 465 bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len, 466 (u_int)n->m_len); 467 m->m_len += n->m_len; 468 n = m_free(n); 469 } 470 } 471 472 void 473 m_adj(mp, req_len) 474 struct mbuf *mp; 475 int req_len; 476 { 477 register int len = req_len; 478 register struct mbuf *m; 479 register int count; 480 481 if ((m = mp) == NULL) 482 return; 483 if (len >= 0) { 484 /* 485 * Trim from head. 486 */ 487 while (m != NULL && len > 0) { 488 if (m->m_len <= len) { 489 len -= m->m_len; 490 m->m_len = 0; 491 m = m->m_next; 492 } else { 493 m->m_len -= len; 494 m->m_data += len; 495 len = 0; 496 } 497 } 498 m = mp; 499 if (mp->m_flags & M_PKTHDR) 500 m->m_pkthdr.len -= (req_len - len); 501 } else { 502 /* 503 * Trim from tail. Scan the mbuf chain, 504 * calculating its length and finding the last mbuf. 505 * If the adjustment only affects this mbuf, then just 506 * adjust and return. Otherwise, rescan and truncate 507 * after the remaining size. 508 */ 509 len = -len; 510 count = 0; 511 for (;;) { 512 count += m->m_len; 513 if (m->m_next == NULL) 514 break; 515 m = m->m_next; 516 } 517 if (m->m_len >= len) { 518 m->m_len -= len; 519 if (mp->m_flags & M_PKTHDR) 520 mp->m_pkthdr.len -= len; 521 return; 522 } 523 count -= len; 524 if (count < 0) 525 count = 0; 526 /* 527 * Correct length for chain is "count". 528 * Find the mbuf with last data, adjust its length, 529 * and toss data from remaining mbufs on chain. 530 */ 531 m = mp; 532 if (m->m_flags & M_PKTHDR) 533 m->m_pkthdr.len = count; 534 for (; m; m = m->m_next) { 535 if (m->m_len >= count) { 536 m->m_len = count; 537 break; 538 } 539 count -= m->m_len; 540 } 541 while ((m = m->m_next) != NULL) 542 m->m_len = 0; 543 } 544 } 545 546 /* 547 * Rearange an mbuf chain so that len bytes are contiguous 548 * and in the data area of an mbuf (so that mtod and dtom 549 * will work for a structure of size len). Returns the resulting 550 * mbuf chain on success, frees it and returns null on failure. 551 * If there is room, it will add up to max_protohdr-len extra bytes to the 552 * contiguous region in an attempt to avoid being called next time. 553 */ 554 int MPFail; 555 556 struct mbuf * 557 m_pullup(n, len) 558 register struct mbuf *n; 559 int len; 560 { 561 register struct mbuf *m; 562 register int count; 563 int space; 564 565 /* 566 * If first mbuf has no cluster, and has room for len bytes 567 * without shifting current data, pullup into it, 568 * otherwise allocate a new mbuf to prepend to the chain. 569 */ 570 if ((n->m_flags & M_EXT) == 0 && 571 n->m_data + len < &n->m_dat[MLEN] && n->m_next) { 572 if (n->m_len >= len) 573 return (n); 574 m = n; 575 n = n->m_next; 576 len -= m->m_len; 577 } else { 578 if (len > MHLEN) 579 goto bad; 580 MGET(m, M_DONTWAIT, n->m_type); 581 if (m == NULL) 582 goto bad; 583 m->m_len = 0; 584 if (n->m_flags & M_PKTHDR) 585 M_MOVE_PKTHDR(m, n); 586 } 587 space = &m->m_dat[MLEN] - (m->m_data + m->m_len); 588 do { 589 count = min(min(max(len, max_protohdr), space), n->m_len); 590 bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len, 591 (unsigned)count); 592 len -= count; 593 m->m_len += count; 594 n->m_len -= count; 595 space -= count; 596 if (n->m_len) 597 n->m_data += count; 598 else 599 n = m_free(n); 600 } while (len > 0 && n); 601 if (len > 0) { 602 (void)m_free(m); 603 goto bad; 604 } 605 m->m_next = n; 606 return (m); 607 bad: 608 m_freem(n); 609 MPFail++; 610 return (NULL); 611 } 612 613 /* 614 * m_pullup2() works like m_pullup, save that len can be <= MCLBYTES. 615 * m_pullup2() only works on values of len such that MHLEN < len <= MCLBYTES, 616 * it calls m_pullup() for values <= MHLEN. It also only coagulates the 617 * reqested number of bytes. (For those of us who expect unwieldly option 618 * headers. 619 * 620 * KEBE SAYS: Remember that dtom() calls with data in clusters does not work! 621 */ 622 struct mbuf * 623 m_pullup2(n, len) 624 register struct mbuf *n; 625 int len; 626 { 627 register struct mbuf *m; 628 register int count; 629 int space; 630 631 if (len <= MHLEN) 632 return m_pullup(n, len); 633 if ((n->m_flags & M_EXT) != 0 && 634 n->m_data + len < &n->m_data[MCLBYTES] && n->m_next) { 635 if (n->m_len >= len) 636 return (n); 637 m = n; 638 n = n->m_next; 639 len -= m->m_len; 640 } else { 641 if (len > MCLBYTES) 642 goto bad; 643 MGET(m, M_DONTWAIT, n->m_type); 644 if (m == NULL) 645 goto bad; 646 MCLGET(m, M_DONTWAIT); 647 if ((m->m_flags & M_EXT) == 0) 648 goto bad; 649 m->m_len = 0; 650 if (n->m_flags & M_PKTHDR) { 651 /* Too many adverse side effects. */ 652 /* M_MOVE_PKTHDR(m, n); */ 653 m->m_flags = (n->m_flags & M_COPYFLAGS) | M_EXT; 654 M_MOVE_HDR(m, n); 655 /* n->m_data is cool. */ 656 } 657 } 658 659 do { 660 count = min(len, n->m_len); 661 bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len, 662 (unsigned)count); 663 len -= count; 664 m->m_len += count; 665 n->m_len -= count; 666 space -= count; 667 if (n->m_len) 668 n->m_data += count; 669 else 670 n = m_free(n); 671 } while (len > 0 && n); 672 if (len > 0) { 673 (void)m_free(m); 674 goto bad; 675 } 676 m->m_next = n; 677 678 return (m); 679 bad: 680 m_freem(n); 681 MPFail++; 682 return (NULL); 683 } 684 685 /* 686 * Return a pointer to mbuf/offset of location in mbuf chain. 687 */ 688 struct mbuf * 689 m_getptr(m, loc, off) 690 struct mbuf *m; 691 int loc; 692 int *off; 693 { 694 while (loc >= 0) { 695 /* Normal end of search */ 696 if (m->m_len > loc) { 697 *off = loc; 698 return (m); 699 } 700 else { 701 loc -= m->m_len; 702 703 if (m->m_next == NULL) { 704 if (loc == 0) { 705 /* Point at the end of valid data */ 706 *off = m->m_len; 707 return (m); 708 } 709 else 710 return (NULL); 711 } 712 else 713 m = m->m_next; 714 } 715 } 716 717 return (NULL); 718 } 719 720 /* 721 * Inject a new mbuf chain of length siz in mbuf chain m0 at 722 * position len0. Returns a pointer to the first injected mbuf, or 723 * NULL on failure (m0 is left undisturbed). Note that if there is 724 * enough space for an object of size siz in the appropriate position, 725 * no memory will be allocated. Also, there will be no data movement in 726 * the first len0 bytes (pointers to that will remain valid). 727 * 728 * XXX It is assumed that siz is less than the size of an mbuf at the moment. 729 */ 730 struct mbuf * 731 m_inject(m0, len0, siz, wait) 732 register struct mbuf *m0; 733 int len0, siz, wait; 734 { 735 register struct mbuf *m, *n, *n2 = NULL, *n3; 736 unsigned len = len0, remain; 737 738 if ((siz >= MHLEN) || (len0 <= 0)) 739 return (NULL); 740 for (m = m0; m && len > m->m_len; m = m->m_next) 741 len -= m->m_len; 742 if (m == NULL) 743 return (NULL); 744 remain = m->m_len - len; 745 if (remain == 0) { 746 if ((m->m_next) && (M_LEADINGSPACE(m->m_next) >= siz)) { 747 m->m_next->m_len += siz; 748 if (m0->m_flags & M_PKTHDR) 749 m0->m_pkthdr.len += siz; 750 m->m_next->m_data -= siz; 751 return m->m_next; 752 } 753 } else { 754 n2 = m_copym2(m, len, remain, wait); 755 if (n2 == NULL) 756 return (NULL); 757 } 758 759 MGET(n, wait, MT_DATA); 760 if (n == NULL) { 761 if (n2) 762 m_freem(n2); 763 return (NULL); 764 } 765 766 n->m_len = siz; 767 if (m0->m_flags & M_PKTHDR) 768 m0->m_pkthdr.len += siz; 769 m->m_len -= remain; /* Trim */ 770 if (n2) { 771 for (n3 = n; n3->m_next != NULL; n3 = n3->m_next) 772 ; 773 n3->m_next = n2; 774 } else 775 n3 = n; 776 for (; n3->m_next != NULL; n3 = n3->m_next) 777 ; 778 n3->m_next = m->m_next; 779 m->m_next = n; 780 return n; 781 } 782 783 /* 784 * Partition an mbuf chain in two pieces, returning the tail -- 785 * all but the first len0 bytes. In case of failure, it returns NULL and 786 * attempts to restore the chain to its original state. 787 */ 788 struct mbuf * 789 m_split(m0, len0, wait) 790 register struct mbuf *m0; 791 int len0, wait; 792 { 793 register struct mbuf *m, *n; 794 unsigned len = len0, remain, olen; 795 796 for (m = m0; m && len > m->m_len; m = m->m_next) 797 len -= m->m_len; 798 if (m == NULL) 799 return (NULL); 800 remain = m->m_len - len; 801 if (m0->m_flags & M_PKTHDR) { 802 MGETHDR(n, wait, m0->m_type); 803 if (n == NULL) 804 return (NULL); 805 M_DUP_PKTHDR(n, m0); 806 n->m_pkthdr.len -= len0; 807 olen = m0->m_pkthdr.len; 808 m0->m_pkthdr.len = len0; 809 if (m->m_flags & M_EXT) 810 goto extpacket; 811 if (remain > MHLEN) { 812 /* m can't be the lead packet */ 813 MH_ALIGN(n, 0); 814 n->m_next = m_split(m, len, wait); 815 if (n->m_next == NULL) { 816 (void) m_free(n); 817 m0->m_pkthdr.len = olen; 818 return (NULL); 819 } else 820 return (n); 821 } else 822 MH_ALIGN(n, remain); 823 } else if (remain == 0) { 824 n = m->m_next; 825 m->m_next = NULL; 826 return (n); 827 } else { 828 MGET(n, wait, m->m_type); 829 if (n == NULL) 830 return (NULL); 831 M_ALIGN(n, remain); 832 } 833 extpacket: 834 if (m->m_flags & M_EXT) { 835 n->m_flags |= M_EXT; 836 MCLADDREFERENCE(m, n); 837 n->m_data = m->m_data + len; 838 } else { 839 bcopy(mtod(m, caddr_t) + len, mtod(n, caddr_t), remain); 840 } 841 n->m_len = remain; 842 m->m_len = len; 843 n->m_next = m->m_next; 844 m->m_next = NULL; 845 return (n); 846 } 847 848 /* 849 * Routine to copy from device local memory into mbufs. 850 */ 851 struct mbuf * 852 m_devget(buf, totlen, off0, ifp, copy) 853 char *buf; 854 int totlen, off0; 855 struct ifnet *ifp; 856 void (*copy) __P((const void *, void *, size_t)); 857 { 858 register struct mbuf *m; 859 struct mbuf *top = NULL, **mp = ⊤ 860 register int off = off0, len; 861 register char *cp; 862 char *epkt; 863 864 cp = buf; 865 epkt = cp + totlen; 866 if (off) { 867 /* 868 * If 'off' is non-zero, packet is trailer-encapsulated, 869 * so we have to skip the type and length fields. 870 */ 871 cp += off + 2 * sizeof(u_int16_t); 872 totlen -= 2 * sizeof(u_int16_t); 873 } 874 MGETHDR(m, M_DONTWAIT, MT_DATA); 875 if (m == NULL) 876 return (NULL); 877 m->m_pkthdr.rcvif = ifp; 878 m->m_pkthdr.len = totlen; 879 m->m_len = MHLEN; 880 881 while (totlen > 0) { 882 if (top != NULL) { 883 MGET(m, M_DONTWAIT, MT_DATA); 884 if (m == NULL) { 885 m_freem(top); 886 return (NULL); 887 } 888 m->m_len = MLEN; 889 } 890 len = min(totlen, epkt - cp); 891 if (len >= MINCLSIZE) { 892 MCLGET(m, M_DONTWAIT); 893 if (m->m_flags & M_EXT) 894 m->m_len = len = min(len, MCLBYTES); 895 else 896 len = m->m_len; 897 } else { 898 /* 899 * Place initial small packet/header at end of mbuf. 900 */ 901 if (len < m->m_len) { 902 if (top == NULL && 903 len + max_linkhdr <= m->m_len) 904 m->m_data += max_linkhdr; 905 m->m_len = len; 906 } else 907 len = m->m_len; 908 } 909 if (copy) 910 copy(cp, mtod(m, caddr_t), (size_t)len); 911 else 912 bcopy(cp, mtod(m, caddr_t), (size_t)len); 913 cp += len; 914 *mp = m; 915 mp = &m->m_next; 916 totlen -= len; 917 if (cp == epkt) 918 cp = buf; 919 } 920 return (top); 921 } 922 923 void 924 m_zero(m) 925 struct mbuf *m; 926 { 927 while (m) { 928 if (m->m_flags & M_PKTHDR) 929 memset((void *)(m + sizeof(struct m_hdr) + 930 sizeof(struct pkthdr)), 0, MHLEN); 931 else 932 memset((void *)(m + sizeof(struct m_hdr)), 0, MLEN); 933 if ((m->m_flags & M_EXT) && 934 (m->m_ext.ext_free == NULL) && 935 !MCLISREFERENCED(m)) 936 memset(m->m_ext.ext_buf, 0, m->m_ext.ext_size); 937 m = m->m_next; 938 } 939 } 940 941 /* 942 * Apply function f to the data in an mbuf chain starting "off" bytes from the 943 * beginning, continuing for "len" bytes. 944 */ 945 int 946 m_apply(m, off, len, f, fstate) 947 struct mbuf *m; 948 int off; 949 int len; 950 /* fstate, data, len */ 951 int (*f)(caddr_t, caddr_t, unsigned int); 952 caddr_t fstate; 953 { 954 int rval; 955 unsigned int count; 956 957 if (len < 0) 958 panic("m_apply: len %d < 0", len); 959 if (off < 0) 960 panic("m_apply: off %d < 0", off); 961 while (off > 0) { 962 if (m == NULL) 963 panic("m_apply: null mbuf in skip"); 964 if (off < m->m_len) 965 break; 966 off -= m->m_len; 967 m = m->m_next; 968 } 969 while (len > 0) { 970 if (m == NULL) 971 panic("m_apply: null mbuf"); 972 count = min(m->m_len - off, len); 973 974 rval = f(fstate, mtod(m, caddr_t) + off, count); 975 if (rval) 976 return (rval); 977 978 len -= count; 979 off = 0; 980 m = m->m_next; 981 } 982 983 return (0); 984 } 985 986 #ifdef SMALL_KERNEL 987 /* 988 * The idea of adding code in a small kernel might look absurd, but this is 989 * instead of macros. 990 */ 991 struct mbuf * 992 _sk_mget(int how, int type) 993 { 994 struct mbuf *m; 995 _MGET(m, how, type); 996 return m; 997 } 998 999 struct mbuf * 1000 _sk_mgethdr(int how, int type) 1001 { 1002 struct mbuf *m; 1003 _MGETHDR(m, how, type); 1004 return m; 1005 } 1006 1007 void 1008 _sk_mclget(struct mbuf *m, int how) 1009 { 1010 _MCLGET(m, how); 1011 } 1012 #endif /* SMALL_KERNEL */ 1013