1 /* $NetBSD: uipc_mbuf.c,v 1.63 2003/02/01 06:23:44 thorpej Exp $ */ 2 3 /*- 4 * Copyright (c) 1999, 2001 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 9 * NASA Ames Research Center. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 3. All advertising materials mentioning features or use of this software 20 * must display the following acknowledgement: 21 * This product includes software developed by the NetBSD 22 * Foundation, Inc. and its contributors. 23 * 4. Neither the name of The NetBSD Foundation nor the names of its 24 * contributors may be used to endorse or promote products derived 25 * from this software without specific prior written permission. 26 * 27 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 28 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 29 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 30 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 31 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 32 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 33 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 34 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 35 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 36 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 37 * POSSIBILITY OF SUCH DAMAGE. 38 */ 39 40 /* 41 * Copyright (c) 1982, 1986, 1988, 1991, 1993 42 * The Regents of the University of California. All rights reserved. 43 * 44 * Redistribution and use in source and binary forms, with or without 45 * modification, are permitted provided that the following conditions 46 * are met: 47 * 1. Redistributions of source code must retain the above copyright 48 * notice, this list of conditions and the following disclaimer. 49 * 2. Redistributions in binary form must reproduce the above copyright 50 * notice, this list of conditions and the following disclaimer in the 51 * documentation and/or other materials provided with the distribution. 52 * 3. All advertising materials mentioning features or use of this software 53 * must display the following acknowledgement: 54 * This product includes software developed by the University of 55 * California, Berkeley and its contributors. 56 * 4. Neither the name of the University nor the names of its contributors 57 * may be used to endorse or promote products derived from this software 58 * without specific prior written permission. 59 * 60 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 61 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 62 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 63 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 64 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 65 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 66 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 67 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 68 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 69 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 70 * SUCH DAMAGE. 71 * 72 * @(#)uipc_mbuf.c 8.4 (Berkeley) 2/14/95 73 */ 74 75 #include <sys/cdefs.h> 76 __KERNEL_RCSID(0, "$NetBSD: uipc_mbuf.c,v 1.63 2003/02/01 06:23:44 thorpej Exp $"); 77 78 #include <sys/param.h> 79 #include <sys/systm.h> 80 #include <sys/proc.h> 81 #include <sys/malloc.h> 82 #define MBTYPES 83 #include <sys/mbuf.h> 84 #include <sys/kernel.h> 85 #include <sys/syslog.h> 86 #include <sys/domain.h> 87 #include <sys/protosw.h> 88 #include <sys/pool.h> 89 #include <sys/socket.h> 90 #include <sys/sysctl.h> 91 92 #include <net/if.h> 93 94 #include <uvm/uvm_extern.h> 95 96 97 struct pool mbpool; /* mbuf pool */ 98 struct pool mclpool; /* mbuf cluster pool */ 99 100 struct pool_cache mbpool_cache; 101 struct pool_cache mclpool_cache; 102 103 struct mbstat mbstat; 104 int max_linkhdr; 105 int max_protohdr; 106 int max_hdr; 107 int max_datalen; 108 109 void *mclpool_alloc(struct pool *, int); 110 void mclpool_release(struct pool *, void *); 111 112 struct pool_allocator mclpool_allocator = { 113 mclpool_alloc, mclpool_release, 0, 114 }; 115 116 static struct mbuf *m_copym0 __P((struct mbuf *, int, int, int, int)); 117 118 const char mclpool_warnmsg[] = 119 "WARNING: mclpool limit reached; increase NMBCLUSTERS"; 120 121 MALLOC_DEFINE(M_MBUF, "mbuf", "mbuf"); 122 123 /* 124 * Initialize the mbuf allcator. 125 */ 126 void 127 mbinit(void) 128 { 129 130 pool_init(&mbpool, msize, 0, 0, 0, "mbpl", NULL); 131 pool_init(&mclpool, mclbytes, 0, 0, 0, "mclpl", &mclpool_allocator); 132 133 pool_set_drain_hook(&mbpool, m_reclaim, NULL); 134 pool_set_drain_hook(&mclpool, m_reclaim, NULL); 135 136 pool_cache_init(&mbpool_cache, &mbpool, NULL, NULL, NULL); 137 pool_cache_init(&mclpool_cache, &mclpool, NULL, NULL, NULL); 138 139 /* 140 * Set the hard limit on the mclpool to the number of 141 * mbuf clusters the kernel is to support. Log the limit 142 * reached message max once a minute. 143 */ 144 pool_sethardlimit(&mclpool, nmbclusters, mclpool_warnmsg, 60); 145 146 /* 147 * Set a low water mark for both mbufs and clusters. This should 148 * help ensure that they can be allocated in a memory starvation 149 * situation. This is important for e.g. diskless systems which 150 * must allocate mbufs in order for the pagedaemon to clean pages. 151 */ 152 pool_setlowat(&mbpool, mblowat); 153 pool_setlowat(&mclpool, mcllowat); 154 } 155 156 int 157 sysctl_dombuf(int *name, u_int namelen, void *oldp, size_t *oldlenp, 158 void *newp, size_t newlen) 159 { 160 int error, newval; 161 162 /* All sysctl names at this level are terminal. */ 163 if (namelen != 1) 164 return (ENOTDIR); /* overloaded */ 165 166 switch (name[0]) { 167 case MBUF_MSIZE: 168 return (sysctl_rdint(oldp, oldlenp, newp, msize)); 169 case MBUF_MCLBYTES: 170 return (sysctl_rdint(oldp, oldlenp, newp, mclbytes)); 171 case MBUF_NMBCLUSTERS: 172 /* 173 * If we have direct-mapped pool pages, we can adjust this 174 * number on the fly. If not, we're limited by the size 175 * of mb_map, and cannot change this value. 176 * 177 * Note: we only allow the value to be increased, never 178 * decreased. 179 */ 180 if (mb_map == NULL) { 181 newval = nmbclusters; 182 error = sysctl_int(oldp, oldlenp, newp, newlen, 183 &newval); 184 if (error != 0) 185 return (error); 186 if (newp != NULL) { 187 if (newval >= nmbclusters) { 188 nmbclusters = newval; 189 pool_sethardlimit(&mclpool, 190 nmbclusters, mclpool_warnmsg, 60); 191 } else 192 error = EINVAL; 193 } 194 return (error); 195 } else 196 return (sysctl_rdint(oldp, oldlenp, newp, nmbclusters)); 197 case MBUF_MBLOWAT: 198 case MBUF_MCLLOWAT: 199 /* New value must be >= 0. */ 200 newval = (name[0] == MBUF_MBLOWAT) ? mblowat : mcllowat; 201 error = sysctl_int(oldp, oldlenp, newp, newlen, &newval); 202 if (error != 0) 203 return (error); 204 if (newp != NULL) { 205 if (newval >= 0) { 206 if (name[0] == MBUF_MBLOWAT) { 207 mblowat = newval; 208 pool_setlowat(&mbpool, newval); 209 } else { 210 mcllowat = newval; 211 pool_setlowat(&mclpool, newval); 212 } 213 } else 214 error = EINVAL; 215 } 216 return (error); 217 default: 218 return (EOPNOTSUPP); 219 } 220 /* NOTREACHED */ 221 } 222 223 void * 224 mclpool_alloc(struct pool *pp, int flags) 225 { 226 boolean_t waitok = (flags & PR_WAITOK) ? TRUE : FALSE; 227 228 return ((void *)uvm_km_alloc_poolpage1(mb_map, NULL, waitok)); 229 } 230 231 void 232 mclpool_release(struct pool *pp, void *v) 233 { 234 235 uvm_km_free_poolpage1(mb_map, (vaddr_t)v); 236 } 237 238 void 239 m_reclaim(void *arg, int flags) 240 { 241 struct domain *dp; 242 struct protosw *pr; 243 struct ifnet *ifp; 244 int s = splvm(); 245 246 for (dp = domains; dp; dp = dp->dom_next) 247 for (pr = dp->dom_protosw; 248 pr < dp->dom_protoswNPROTOSW; pr++) 249 if (pr->pr_drain) 250 (*pr->pr_drain)(); 251 for (ifp = TAILQ_FIRST(&ifnet); ifp; ifp = TAILQ_NEXT(ifp, if_list)) 252 if (ifp->if_drain) 253 (*ifp->if_drain)(ifp); 254 splx(s); 255 mbstat.m_drain++; 256 } 257 258 /* 259 * Space allocation routines. 260 * These are also available as macros 261 * for critical paths. 262 */ 263 struct mbuf * 264 m_get(int nowait, int type) 265 { 266 struct mbuf *m; 267 268 MGET(m, nowait, type); 269 return (m); 270 } 271 272 struct mbuf * 273 m_gethdr(int nowait, int type) 274 { 275 struct mbuf *m; 276 277 MGETHDR(m, nowait, type); 278 return (m); 279 } 280 281 struct mbuf * 282 m_getclr(int nowait, int type) 283 { 284 struct mbuf *m; 285 286 MGET(m, nowait, type); 287 if (m == 0) 288 return (0); 289 memset(mtod(m, caddr_t), 0, MLEN); 290 return (m); 291 } 292 293 struct mbuf * 294 m_free(struct mbuf *m) 295 { 296 struct mbuf *n; 297 298 MFREE(m, n); 299 return (n); 300 } 301 302 void 303 m_freem(struct mbuf *m) 304 { 305 struct mbuf *n; 306 307 if (m == NULL) 308 return; 309 do { 310 MFREE(m, n); 311 m = n; 312 } while (m); 313 } 314 315 /* 316 * Mbuffer utility routines. 317 */ 318 319 /* 320 * Lesser-used path for M_PREPEND: 321 * allocate new mbuf to prepend to chain, 322 * copy junk along. 323 */ 324 struct mbuf * 325 m_prepend(struct mbuf *m, int len, int how) 326 { 327 struct mbuf *mn; 328 329 MGET(mn, how, m->m_type); 330 if (mn == (struct mbuf *)NULL) { 331 m_freem(m); 332 return ((struct mbuf *)NULL); 333 } 334 if (m->m_flags & M_PKTHDR) { 335 M_COPY_PKTHDR(mn, m); 336 m->m_flags &= ~M_PKTHDR; 337 } 338 mn->m_next = m; 339 m = mn; 340 if (len < MHLEN) 341 MH_ALIGN(m, len); 342 m->m_len = len; 343 return (m); 344 } 345 346 /* 347 * Make a copy of an mbuf chain starting "off0" bytes from the beginning, 348 * continuing for "len" bytes. If len is M_COPYALL, copy to end of mbuf. 349 * The wait parameter is a choice of M_WAIT/M_DONTWAIT from caller. 350 */ 351 int MCFail; 352 353 struct mbuf * 354 m_copym(struct mbuf *m, int off0, int len, int wait) 355 { 356 return m_copym0(m, off0, len, wait, 0); /* shallow copy on M_EXT */ 357 } 358 359 struct mbuf * 360 m_dup(struct mbuf *m, int off0, int len, int wait) 361 { 362 return m_copym0(m, off0, len, wait, 1); /* deep copy */ 363 } 364 365 static struct mbuf * 366 m_copym0(struct mbuf *m, int off0, int len, int wait, int deep) 367 { 368 struct mbuf *n, **np; 369 int off = off0; 370 struct mbuf *top; 371 int copyhdr = 0; 372 373 if (off < 0 || len < 0) 374 panic("m_copym: off %d, len %d", off, len); 375 if (off == 0 && m->m_flags & M_PKTHDR) 376 copyhdr = 1; 377 while (off > 0) { 378 if (m == 0) 379 panic("m_copym: m == 0"); 380 if (off < m->m_len) 381 break; 382 off -= m->m_len; 383 m = m->m_next; 384 } 385 np = ⊤ 386 top = 0; 387 while (len > 0) { 388 if (m == 0) { 389 if (len != M_COPYALL) 390 panic("m_copym: m == 0 and not COPYALL"); 391 break; 392 } 393 MGET(n, wait, m->m_type); 394 *np = n; 395 if (n == 0) 396 goto nospace; 397 if (copyhdr) { 398 M_COPY_PKTHDR(n, m); 399 if (len == M_COPYALL) 400 n->m_pkthdr.len -= off0; 401 else 402 n->m_pkthdr.len = len; 403 copyhdr = 0; 404 } 405 n->m_len = min(len, m->m_len - off); 406 if (m->m_flags & M_EXT) { 407 if (!deep) { 408 n->m_data = m->m_data + off; 409 n->m_ext = m->m_ext; 410 MCLADDREFERENCE(m, n); 411 } else { 412 /* 413 * we are unsure about the way m was allocated. 414 * copy into multiple MCLBYTES cluster mbufs. 415 */ 416 MCLGET(n, wait); 417 n->m_len = 0; 418 n->m_len = M_TRAILINGSPACE(n); 419 n->m_len = min(n->m_len, len); 420 n->m_len = min(n->m_len, m->m_len - off); 421 memcpy(mtod(n, caddr_t), mtod(m, caddr_t) + off, 422 (unsigned)n->m_len); 423 } 424 } else 425 memcpy(mtod(n, caddr_t), mtod(m, caddr_t)+off, 426 (unsigned)n->m_len); 427 if (len != M_COPYALL) 428 len -= n->m_len; 429 off += n->m_len; 430 #ifdef DIAGNOSTIC 431 if (off > m->m_len) 432 panic("m_copym0 overrun"); 433 #endif 434 if (off == m->m_len) { 435 m = m->m_next; 436 off = 0; 437 } 438 np = &n->m_next; 439 } 440 if (top == 0) 441 MCFail++; 442 return (top); 443 nospace: 444 m_freem(top); 445 MCFail++; 446 return (0); 447 } 448 449 /* 450 * Copy an entire packet, including header (which must be present). 451 * An optimization of the common case `m_copym(m, 0, M_COPYALL, how)'. 452 */ 453 struct mbuf * 454 m_copypacket(struct mbuf *m, int how) 455 { 456 struct mbuf *top, *n, *o; 457 458 MGET(n, how, m->m_type); 459 top = n; 460 if (!n) 461 goto nospace; 462 463 M_COPY_PKTHDR(n, m); 464 n->m_len = m->m_len; 465 if (m->m_flags & M_EXT) { 466 n->m_data = m->m_data; 467 n->m_ext = m->m_ext; 468 MCLADDREFERENCE(m, n); 469 } else { 470 memcpy(mtod(n, char *), mtod(m, char *), n->m_len); 471 } 472 473 m = m->m_next; 474 while (m) { 475 MGET(o, how, m->m_type); 476 if (!o) 477 goto nospace; 478 479 n->m_next = o; 480 n = n->m_next; 481 482 n->m_len = m->m_len; 483 if (m->m_flags & M_EXT) { 484 n->m_data = m->m_data; 485 n->m_ext = m->m_ext; 486 MCLADDREFERENCE(m, n); 487 } else { 488 memcpy(mtod(n, char *), mtod(m, char *), n->m_len); 489 } 490 491 m = m->m_next; 492 } 493 return top; 494 nospace: 495 m_freem(top); 496 MCFail++; 497 return 0; 498 } 499 500 /* 501 * Copy data from an mbuf chain starting "off" bytes from the beginning, 502 * continuing for "len" bytes, into the indicated buffer. 503 */ 504 void 505 m_copydata(struct mbuf *m, int off, int len, caddr_t cp) 506 { 507 unsigned count; 508 509 if (off < 0 || len < 0) 510 panic("m_copydata"); 511 while (off > 0) { 512 if (m == 0) 513 panic("m_copydata"); 514 if (off < m->m_len) 515 break; 516 off -= m->m_len; 517 m = m->m_next; 518 } 519 while (len > 0) { 520 if (m == 0) 521 panic("m_copydata"); 522 count = min(m->m_len - off, len); 523 memcpy(cp, mtod(m, caddr_t) + off, count); 524 len -= count; 525 cp += count; 526 off = 0; 527 m = m->m_next; 528 } 529 } 530 531 /* 532 * Concatenate mbuf chain n to m. 533 * Both chains must be of the same type (e.g. MT_DATA). 534 * Any m_pkthdr is not updated. 535 */ 536 void 537 m_cat(struct mbuf *m, struct mbuf *n) 538 { 539 while (m->m_next) 540 m = m->m_next; 541 while (n) { 542 if (m->m_flags & M_EXT || 543 m->m_data + m->m_len + n->m_len >= &m->m_dat[MLEN]) { 544 /* just join the two chains */ 545 m->m_next = n; 546 return; 547 } 548 /* splat the data from one into the other */ 549 memcpy(mtod(m, caddr_t) + m->m_len, mtod(n, caddr_t), 550 (u_int)n->m_len); 551 m->m_len += n->m_len; 552 n = m_free(n); 553 } 554 } 555 556 void 557 m_adj(struct mbuf *mp, int req_len) 558 { 559 int len = req_len; 560 struct mbuf *m; 561 int count; 562 563 if ((m = mp) == NULL) 564 return; 565 if (len >= 0) { 566 /* 567 * Trim from head. 568 */ 569 while (m != NULL && len > 0) { 570 if (m->m_len <= len) { 571 len -= m->m_len; 572 m->m_len = 0; 573 m = m->m_next; 574 } else { 575 m->m_len -= len; 576 m->m_data += len; 577 len = 0; 578 } 579 } 580 m = mp; 581 if (mp->m_flags & M_PKTHDR) 582 m->m_pkthdr.len -= (req_len - len); 583 } else { 584 /* 585 * Trim from tail. Scan the mbuf chain, 586 * calculating its length and finding the last mbuf. 587 * If the adjustment only affects this mbuf, then just 588 * adjust and return. Otherwise, rescan and truncate 589 * after the remaining size. 590 */ 591 len = -len; 592 count = 0; 593 for (;;) { 594 count += m->m_len; 595 if (m->m_next == (struct mbuf *)0) 596 break; 597 m = m->m_next; 598 } 599 if (m->m_len >= len) { 600 m->m_len -= len; 601 if (mp->m_flags & M_PKTHDR) 602 mp->m_pkthdr.len -= len; 603 return; 604 } 605 count -= len; 606 if (count < 0) 607 count = 0; 608 /* 609 * Correct length for chain is "count". 610 * Find the mbuf with last data, adjust its length, 611 * and toss data from remaining mbufs on chain. 612 */ 613 m = mp; 614 if (m->m_flags & M_PKTHDR) 615 m->m_pkthdr.len = count; 616 for (; m; m = m->m_next) { 617 if (m->m_len >= count) { 618 m->m_len = count; 619 break; 620 } 621 count -= m->m_len; 622 } 623 while (m->m_next) 624 (m = m->m_next) ->m_len = 0; 625 } 626 } 627 628 /* 629 * Rearange an mbuf chain so that len bytes are contiguous 630 * and in the data area of an mbuf (so that mtod and dtom 631 * will work for a structure of size len). Returns the resulting 632 * mbuf chain on success, frees it and returns null on failure. 633 * If there is room, it will add up to max_protohdr-len extra bytes to the 634 * contiguous region in an attempt to avoid being called next time. 635 */ 636 int MPFail; 637 638 struct mbuf * 639 m_pullup(struct mbuf *n, int len) 640 { 641 struct mbuf *m; 642 int count; 643 int space; 644 645 /* 646 * If first mbuf has no cluster, and has room for len bytes 647 * without shifting current data, pullup into it, 648 * otherwise allocate a new mbuf to prepend to the chain. 649 */ 650 if ((n->m_flags & M_EXT) == 0 && 651 n->m_data + len < &n->m_dat[MLEN] && n->m_next) { 652 if (n->m_len >= len) 653 return (n); 654 m = n; 655 n = n->m_next; 656 len -= m->m_len; 657 } else { 658 if (len > MHLEN) 659 goto bad; 660 MGET(m, M_DONTWAIT, n->m_type); 661 if (m == 0) 662 goto bad; 663 m->m_len = 0; 664 if (n->m_flags & M_PKTHDR) { 665 M_COPY_PKTHDR(m, n); 666 n->m_flags &= ~M_PKTHDR; 667 } 668 } 669 space = &m->m_dat[MLEN] - (m->m_data + m->m_len); 670 do { 671 count = min(min(max(len, max_protohdr), space), n->m_len); 672 memcpy(mtod(m, caddr_t) + m->m_len, mtod(n, caddr_t), 673 (unsigned)count); 674 len -= count; 675 m->m_len += count; 676 n->m_len -= count; 677 space -= count; 678 if (n->m_len) 679 n->m_data += count; 680 else 681 n = m_free(n); 682 } while (len > 0 && n); 683 if (len > 0) { 684 (void) m_free(m); 685 goto bad; 686 } 687 m->m_next = n; 688 return (m); 689 bad: 690 m_freem(n); 691 MPFail++; 692 return (0); 693 } 694 695 /* 696 * Like m_pullup(), except a new mbuf is always allocated, and we allow 697 * the amount of empty space before the data in the new mbuf to be specified 698 * (in the event that the caller expects to prepend later). 699 */ 700 int MSFail; 701 702 struct mbuf * 703 m_copyup(struct mbuf *n, int len, int dstoff) 704 { 705 struct mbuf *m; 706 int count, space; 707 708 if (len > (MHLEN - dstoff)) 709 goto bad; 710 MGET(m, M_DONTWAIT, n->m_type); 711 if (m == NULL) 712 goto bad; 713 m->m_len = 0; 714 if (n->m_flags & M_PKTHDR) { 715 M_COPY_PKTHDR(m, n); 716 n->m_flags &= ~M_PKTHDR; 717 } 718 m->m_data += dstoff; 719 space = &m->m_dat[MLEN] - (m->m_data + m->m_len); 720 do { 721 count = min(min(max(len, max_protohdr), space), n->m_len); 722 memcpy(mtod(m, caddr_t) + m->m_len, mtod(n, caddr_t), 723 (unsigned)count); 724 len -= count; 725 m->m_len += count; 726 n->m_len -= count; 727 space -= count; 728 if (n->m_len) 729 n->m_data += count; 730 else 731 n = m_free(n); 732 } while (len > 0 && n); 733 if (len > 0) { 734 (void) m_free(m); 735 goto bad; 736 } 737 m->m_next = n; 738 return (m); 739 bad: 740 m_freem(n); 741 MSFail++; 742 return (NULL); 743 } 744 745 /* 746 * Partition an mbuf chain in two pieces, returning the tail -- 747 * all but the first len0 bytes. In case of failure, it returns NULL and 748 * attempts to restore the chain to its original state. 749 */ 750 struct mbuf * 751 m_split(struct mbuf *m0, int len0, int wait) 752 { 753 struct mbuf *m, *n; 754 unsigned len = len0, remain, len_save; 755 756 for (m = m0; m && len > m->m_len; m = m->m_next) 757 len -= m->m_len; 758 if (m == 0) 759 return (0); 760 remain = m->m_len - len; 761 if (m0->m_flags & M_PKTHDR) { 762 MGETHDR(n, wait, m0->m_type); 763 if (n == 0) 764 return (0); 765 n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif; 766 n->m_pkthdr.len = m0->m_pkthdr.len - len0; 767 len_save = m0->m_pkthdr.len; 768 m0->m_pkthdr.len = len0; 769 if (m->m_flags & M_EXT) 770 goto extpacket; 771 if (remain > MHLEN) { 772 /* m can't be the lead packet */ 773 MH_ALIGN(n, 0); 774 n->m_next = m_split(m, len, wait); 775 if (n->m_next == 0) { 776 (void) m_free(n); 777 m0->m_pkthdr.len = len_save; 778 return (0); 779 } else 780 return (n); 781 } else 782 MH_ALIGN(n, remain); 783 } else if (remain == 0) { 784 n = m->m_next; 785 m->m_next = 0; 786 return (n); 787 } else { 788 MGET(n, wait, m->m_type); 789 if (n == 0) 790 return (0); 791 M_ALIGN(n, remain); 792 } 793 extpacket: 794 if (m->m_flags & M_EXT) { 795 n->m_ext = m->m_ext; 796 MCLADDREFERENCE(m, n); 797 n->m_data = m->m_data + len; 798 } else { 799 memcpy(mtod(n, caddr_t), mtod(m, caddr_t) + len, remain); 800 } 801 n->m_len = remain; 802 m->m_len = len; 803 n->m_next = m->m_next; 804 m->m_next = 0; 805 return (n); 806 } 807 /* 808 * Routine to copy from device local memory into mbufs. 809 */ 810 struct mbuf * 811 m_devget(char *buf, int totlen, int off0, struct ifnet *ifp, 812 void (*copy)(const void *from, void *to, size_t len)) 813 { 814 struct mbuf *m; 815 struct mbuf *top = 0, **mp = ⊤ 816 int off = off0, len; 817 char *cp; 818 char *epkt; 819 820 cp = buf; 821 epkt = cp + totlen; 822 if (off) { 823 /* 824 * If 'off' is non-zero, packet is trailer-encapsulated, 825 * so we have to skip the type and length fields. 826 */ 827 cp += off + 2 * sizeof(u_int16_t); 828 totlen -= 2 * sizeof(u_int16_t); 829 } 830 MGETHDR(m, M_DONTWAIT, MT_DATA); 831 if (m == 0) 832 return (0); 833 m->m_pkthdr.rcvif = ifp; 834 m->m_pkthdr.len = totlen; 835 m->m_len = MHLEN; 836 837 while (totlen > 0) { 838 if (top) { 839 MGET(m, M_DONTWAIT, MT_DATA); 840 if (m == 0) { 841 m_freem(top); 842 return (0); 843 } 844 m->m_len = MLEN; 845 } 846 len = min(totlen, epkt - cp); 847 if (len >= MINCLSIZE) { 848 MCLGET(m, M_DONTWAIT); 849 if ((m->m_flags & M_EXT) == 0) { 850 m_free(m); 851 m_freem(top); 852 return (0); 853 } 854 m->m_len = len = min(len, MCLBYTES); 855 } else { 856 /* 857 * Place initial small packet/header at end of mbuf. 858 */ 859 if (len < m->m_len) { 860 if (top == 0 && len + max_linkhdr <= m->m_len) 861 m->m_data += max_linkhdr; 862 m->m_len = len; 863 } else 864 len = m->m_len; 865 } 866 if (copy) 867 copy(cp, mtod(m, caddr_t), (size_t)len); 868 else 869 memcpy(mtod(m, caddr_t), cp, (size_t)len); 870 cp += len; 871 *mp = m; 872 mp = &m->m_next; 873 totlen -= len; 874 if (cp == epkt) 875 cp = buf; 876 } 877 return (top); 878 } 879 880 /* 881 * Copy data from a buffer back into the indicated mbuf chain, 882 * starting "off" bytes from the beginning, extending the mbuf 883 * chain if necessary. 884 */ 885 void 886 m_copyback(struct mbuf *m0, int off, int len, caddr_t cp) 887 { 888 int mlen; 889 struct mbuf *m = m0, *n; 890 int totlen = 0; 891 892 if (m0 == 0) 893 return; 894 while (off > (mlen = m->m_len)) { 895 off -= mlen; 896 totlen += mlen; 897 if (m->m_next == 0) { 898 n = m_getclr(M_DONTWAIT, m->m_type); 899 if (n == 0) 900 goto out; 901 n->m_len = min(MLEN, len + off); 902 m->m_next = n; 903 } 904 m = m->m_next; 905 } 906 while (len > 0) { 907 mlen = min (m->m_len - off, len); 908 memcpy(mtod(m, caddr_t) + off, cp, (unsigned)mlen); 909 cp += mlen; 910 len -= mlen; 911 mlen += off; 912 off = 0; 913 totlen += mlen; 914 if (len == 0) 915 break; 916 if (m->m_next == 0) { 917 n = m_get(M_DONTWAIT, m->m_type); 918 if (n == 0) 919 break; 920 n->m_len = min(MLEN, len); 921 m->m_next = n; 922 } 923 m = m->m_next; 924 } 925 out: if (((m = m0)->m_flags & M_PKTHDR) && (m->m_pkthdr.len < totlen)) 926 m->m_pkthdr.len = totlen; 927 } 928