1 /* $NetBSD: uipc_mbuf.c,v 1.19 1997/04/24 08:14:06 mycroft Exp $ */ 2 3 /* 4 * Copyright (c) 1982, 1986, 1988, 1991, 1993 5 * The Regents of the University of California. All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. All advertising materials mentioning features or use of this software 16 * must display the following acknowledgement: 17 * This product includes software developed by the University of 18 * California, Berkeley and its contributors. 19 * 4. Neither the name of the University nor the names of its contributors 20 * may be used to endorse or promote products derived from this software 21 * without specific prior written permission. 22 * 23 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 24 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 25 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 26 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 27 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 28 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 29 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 30 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 31 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 32 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 33 * SUCH DAMAGE. 34 * 35 * @(#)uipc_mbuf.c 8.2 (Berkeley) 1/4/94 36 */ 37 38 #include <sys/param.h> 39 #include <sys/systm.h> 40 #include <sys/proc.h> 41 #include <sys/malloc.h> 42 #include <sys/map.h> 43 #define MBTYPES 44 #include <sys/mbuf.h> 45 #include <sys/kernel.h> 46 #include <sys/syslog.h> 47 #include <sys/domain.h> 48 #include <sys/protosw.h> 49 50 #include <vm/vm.h> 51 52 struct mbuf *mbutl; 53 struct mbstat mbstat; 54 union mcluster *mclfree; 55 int max_linkhdr; 56 int max_protohdr; 57 int max_hdr; 58 int max_datalen; 59 60 extern vm_map_t mb_map; 61 62 void 63 mbinit() 64 { 65 int s; 66 67 mclfree = NULL; 68 s = splimp(); 69 if (m_clalloc(max(4096/CLBYTES, 1), M_DONTWAIT) == 0) 70 goto bad; 71 splx(s); 72 return; 73 bad: 74 panic("mbinit"); 75 } 76 77 /* 78 * Allocate some number of mbuf clusters 79 * and place on cluster free list. 80 * Must be called at splimp. 81 */ 82 /* ARGSUSED */ 83 int 84 m_clalloc(ncl, nowait) 85 register int ncl; 86 int nowait; 87 { 88 static volatile struct timeval lastlogged; 89 struct timeval curtime, logdiff; 90 register caddr_t p; 91 register int i; 92 int npg, s; 93 94 npg = ncl * CLSIZE; 95 p = (caddr_t)kmem_malloc(mb_map, ctob(npg), 96 nowait ? M_NOWAIT : M_WAITOK); 97 if (p == NULL) { 98 s = splclock(); 99 curtime = time; 100 splx(s); 101 timersub(&curtime, &lastlogged, &logdiff); 102 if (logdiff.tv_sec >= 60) { 103 lastlogged = curtime; 104 log(LOG_ERR, "mb_map full\n"); 105 } 106 m_reclaim(); 107 return (mclfree != NULL); 108 } 109 ncl = ncl * CLBYTES / MCLBYTES; 110 for (i = 0; i < ncl; i++) { 111 ((union mcluster *)p)->mcl_next = mclfree; 112 mclfree = (union mcluster *)p; 113 p += MCLBYTES; 114 mbstat.m_clfree++; 115 } 116 mbstat.m_clusters += ncl; 117 return (1); 118 } 119 120 /* 121 * When MGET failes, ask protocols to free space when short of memory, 122 * then re-attempt to allocate an mbuf. 123 */ 124 struct mbuf * 125 m_retry(i, t) 126 int i, t; 127 { 128 register struct mbuf *m; 129 130 m_reclaim(); 131 #define m_retry(i, t) (struct mbuf *)0 132 MGET(m, i, t); 133 #undef m_retry 134 if (m != NULL) 135 mbstat.m_wait++; 136 else 137 mbstat.m_drops++; 138 return (m); 139 } 140 141 /* 142 * As above; retry an MGETHDR. 143 */ 144 struct mbuf * 145 m_retryhdr(i, t) 146 int i, t; 147 { 148 register struct mbuf *m; 149 150 m_reclaim(); 151 #define m_retryhdr(i, t) (struct mbuf *)0 152 MGETHDR(m, i, t); 153 #undef m_retryhdr 154 if (m != NULL) 155 mbstat.m_wait++; 156 else 157 mbstat.m_drops++; 158 return (m); 159 } 160 161 void 162 m_reclaim() 163 { 164 register struct domain *dp; 165 register struct protosw *pr; 166 int s = splimp(); 167 168 for (dp = domains; dp; dp = dp->dom_next) 169 for (pr = dp->dom_protosw; pr < dp->dom_protoswNPROTOSW; pr++) 170 if (pr->pr_drain) 171 (*pr->pr_drain)(); 172 splx(s); 173 mbstat.m_drain++; 174 } 175 176 /* 177 * Space allocation routines. 178 * These are also available as macros 179 * for critical paths. 180 */ 181 struct mbuf * 182 m_get(nowait, type) 183 int nowait, type; 184 { 185 register struct mbuf *m; 186 187 MGET(m, nowait, type); 188 return (m); 189 } 190 191 struct mbuf * 192 m_gethdr(nowait, type) 193 int nowait, type; 194 { 195 register struct mbuf *m; 196 197 MGETHDR(m, nowait, type); 198 return (m); 199 } 200 201 struct mbuf * 202 m_getclr(nowait, type) 203 int nowait, type; 204 { 205 register struct mbuf *m; 206 207 MGET(m, nowait, type); 208 if (m == 0) 209 return (0); 210 bzero(mtod(m, caddr_t), MLEN); 211 return (m); 212 } 213 214 struct mbuf * 215 m_free(m) 216 struct mbuf *m; 217 { 218 register struct mbuf *n; 219 220 MFREE(m, n); 221 return (n); 222 } 223 224 void 225 m_freem(m) 226 register struct mbuf *m; 227 { 228 register struct mbuf *n; 229 230 if (m == NULL) 231 return; 232 do { 233 MFREE(m, n); 234 m = n; 235 } while (m); 236 } 237 238 /* 239 * Mbuffer utility routines. 240 */ 241 242 /* 243 * Lesser-used path for M_PREPEND: 244 * allocate new mbuf to prepend to chain, 245 * copy junk along. 246 */ 247 struct mbuf * 248 m_prepend(m, len, how) 249 register struct mbuf *m; 250 int len, how; 251 { 252 struct mbuf *mn; 253 254 MGET(mn, how, m->m_type); 255 if (mn == (struct mbuf *)NULL) { 256 m_freem(m); 257 return ((struct mbuf *)NULL); 258 } 259 if (m->m_flags & M_PKTHDR) { 260 M_COPY_PKTHDR(mn, m); 261 m->m_flags &= ~M_PKTHDR; 262 } 263 mn->m_next = m; 264 m = mn; 265 if (len < MHLEN) 266 MH_ALIGN(m, len); 267 m->m_len = len; 268 return (m); 269 } 270 271 /* 272 * Make a copy of an mbuf chain starting "off0" bytes from the beginning, 273 * continuing for "len" bytes. If len is M_COPYALL, copy to end of mbuf. 274 * The wait parameter is a choice of M_WAIT/M_DONTWAIT from caller. 275 */ 276 int MCFail; 277 278 struct mbuf * 279 m_copym(m, off0, len, wait) 280 register struct mbuf *m; 281 int off0, wait; 282 register int len; 283 { 284 register struct mbuf *n, **np; 285 register int off = off0; 286 struct mbuf *top; 287 int copyhdr = 0; 288 289 if (off < 0 || len < 0) 290 panic("m_copym"); 291 if (off == 0 && m->m_flags & M_PKTHDR) 292 copyhdr = 1; 293 while (off > 0) { 294 if (m == 0) 295 panic("m_copym"); 296 if (off < m->m_len) 297 break; 298 off -= m->m_len; 299 m = m->m_next; 300 } 301 np = ⊤ 302 top = 0; 303 while (len > 0) { 304 if (m == 0) { 305 if (len != M_COPYALL) 306 panic("m_copym"); 307 break; 308 } 309 MGET(n, wait, m->m_type); 310 *np = n; 311 if (n == 0) 312 goto nospace; 313 if (copyhdr) { 314 M_COPY_PKTHDR(n, m); 315 if (len == M_COPYALL) 316 n->m_pkthdr.len -= off0; 317 else 318 n->m_pkthdr.len = len; 319 copyhdr = 0; 320 } 321 n->m_len = min(len, m->m_len - off); 322 if (m->m_flags & M_EXT) { 323 n->m_data = m->m_data + off; 324 n->m_ext = m->m_ext; 325 MCLADDREFERENCE(m, n); 326 } else 327 bcopy(mtod(m, caddr_t)+off, mtod(n, caddr_t), 328 (unsigned)n->m_len); 329 if (len != M_COPYALL) 330 len -= n->m_len; 331 off = 0; 332 m = m->m_next; 333 np = &n->m_next; 334 } 335 if (top == 0) 336 MCFail++; 337 return (top); 338 nospace: 339 m_freem(top); 340 MCFail++; 341 return (0); 342 } 343 344 /* 345 * Copy an entire packet, including header (which must be present). 346 * An optimization of the common case `m_copym(m, 0, M_COPYALL, how)'. 347 */ 348 struct mbuf * 349 m_copypacket(m, how) 350 struct mbuf *m; 351 int how; 352 { 353 struct mbuf *top, *n, *o; 354 355 MGET(n, how, m->m_type); 356 top = n; 357 if (!n) 358 goto nospace; 359 360 M_COPY_PKTHDR(n, m); 361 n->m_len = m->m_len; 362 if (m->m_flags & M_EXT) { 363 n->m_data = m->m_data; 364 n->m_ext = m->m_ext; 365 MCLADDREFERENCE(m, n); 366 } else { 367 bcopy(mtod(m, char *), mtod(n, char *), n->m_len); 368 } 369 370 m = m->m_next; 371 while (m) { 372 MGET(o, how, m->m_type); 373 if (!o) 374 goto nospace; 375 376 n->m_next = o; 377 n = n->m_next; 378 379 n->m_len = m->m_len; 380 if (m->m_flags & M_EXT) { 381 n->m_data = m->m_data; 382 n->m_ext = m->m_ext; 383 MCLADDREFERENCE(m, n); 384 } else { 385 bcopy(mtod(m, char *), mtod(n, char *), n->m_len); 386 } 387 388 m = m->m_next; 389 } 390 return top; 391 nospace: 392 m_freem(top); 393 MCFail++; 394 return 0; 395 } 396 397 /* 398 * Copy data from an mbuf chain starting "off" bytes from the beginning, 399 * continuing for "len" bytes, into the indicated buffer. 400 */ 401 void 402 m_copydata(m, off, len, cp) 403 register struct mbuf *m; 404 register int off; 405 register int len; 406 caddr_t cp; 407 { 408 register unsigned count; 409 410 if (off < 0 || len < 0) 411 panic("m_copydata"); 412 while (off > 0) { 413 if (m == 0) 414 panic("m_copydata"); 415 if (off < m->m_len) 416 break; 417 off -= m->m_len; 418 m = m->m_next; 419 } 420 while (len > 0) { 421 if (m == 0) 422 panic("m_copydata"); 423 count = min(m->m_len - off, len); 424 bcopy(mtod(m, caddr_t) + off, cp, count); 425 len -= count; 426 cp += count; 427 off = 0; 428 m = m->m_next; 429 } 430 } 431 432 /* 433 * Concatenate mbuf chain n to m. 434 * Both chains must be of the same type (e.g. MT_DATA). 435 * Any m_pkthdr is not updated. 436 */ 437 void 438 m_cat(m, n) 439 register struct mbuf *m, *n; 440 { 441 while (m->m_next) 442 m = m->m_next; 443 while (n) { 444 if (m->m_flags & M_EXT || 445 m->m_data + m->m_len + n->m_len >= &m->m_dat[MLEN]) { 446 /* just join the two chains */ 447 m->m_next = n; 448 return; 449 } 450 /* splat the data from one into the other */ 451 bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len, 452 (u_int)n->m_len); 453 m->m_len += n->m_len; 454 n = m_free(n); 455 } 456 } 457 458 void 459 m_adj(mp, req_len) 460 struct mbuf *mp; 461 int req_len; 462 { 463 register int len = req_len; 464 register struct mbuf *m; 465 register count; 466 467 if ((m = mp) == NULL) 468 return; 469 if (len >= 0) { 470 /* 471 * Trim from head. 472 */ 473 while (m != NULL && len > 0) { 474 if (m->m_len <= len) { 475 len -= m->m_len; 476 m->m_len = 0; 477 m = m->m_next; 478 } else { 479 m->m_len -= len; 480 m->m_data += len; 481 len = 0; 482 } 483 } 484 m = mp; 485 if (mp->m_flags & M_PKTHDR) 486 m->m_pkthdr.len -= (req_len - len); 487 } else { 488 /* 489 * Trim from tail. Scan the mbuf chain, 490 * calculating its length and finding the last mbuf. 491 * If the adjustment only affects this mbuf, then just 492 * adjust and return. Otherwise, rescan and truncate 493 * after the remaining size. 494 */ 495 len = -len; 496 count = 0; 497 for (;;) { 498 count += m->m_len; 499 if (m->m_next == (struct mbuf *)0) 500 break; 501 m = m->m_next; 502 } 503 if (m->m_len >= len) { 504 m->m_len -= len; 505 if (mp->m_flags & M_PKTHDR) 506 mp->m_pkthdr.len -= len; 507 return; 508 } 509 count -= len; 510 if (count < 0) 511 count = 0; 512 /* 513 * Correct length for chain is "count". 514 * Find the mbuf with last data, adjust its length, 515 * and toss data from remaining mbufs on chain. 516 */ 517 m = mp; 518 if (m->m_flags & M_PKTHDR) 519 m->m_pkthdr.len = count; 520 for (; m; m = m->m_next) { 521 if (m->m_len >= count) { 522 m->m_len = count; 523 break; 524 } 525 count -= m->m_len; 526 } 527 while (m->m_next) 528 (m = m->m_next) ->m_len = 0; 529 } 530 } 531 532 /* 533 * Rearange an mbuf chain so that len bytes are contiguous 534 * and in the data area of an mbuf (so that mtod and dtom 535 * will work for a structure of size len). Returns the resulting 536 * mbuf chain on success, frees it and returns null on failure. 537 * If there is room, it will add up to max_protohdr-len extra bytes to the 538 * contiguous region in an attempt to avoid being called next time. 539 */ 540 int MPFail; 541 542 struct mbuf * 543 m_pullup(n, len) 544 register struct mbuf *n; 545 int len; 546 { 547 register struct mbuf *m; 548 register int count; 549 int space; 550 551 /* 552 * If first mbuf has no cluster, and has room for len bytes 553 * without shifting current data, pullup into it, 554 * otherwise allocate a new mbuf to prepend to the chain. 555 */ 556 if ((n->m_flags & M_EXT) == 0 && 557 n->m_data + len < &n->m_dat[MLEN] && n->m_next) { 558 if (n->m_len >= len) 559 return (n); 560 m = n; 561 n = n->m_next; 562 len -= m->m_len; 563 } else { 564 if (len > MHLEN) 565 goto bad; 566 MGET(m, M_DONTWAIT, n->m_type); 567 if (m == 0) 568 goto bad; 569 m->m_len = 0; 570 if (n->m_flags & M_PKTHDR) { 571 M_COPY_PKTHDR(m, n); 572 n->m_flags &= ~M_PKTHDR; 573 } 574 } 575 space = &m->m_dat[MLEN] - (m->m_data + m->m_len); 576 do { 577 count = min(min(max(len, max_protohdr), space), n->m_len); 578 bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len, 579 (unsigned)count); 580 len -= count; 581 m->m_len += count; 582 n->m_len -= count; 583 space -= count; 584 if (n->m_len) 585 n->m_data += count; 586 else 587 n = m_free(n); 588 } while (len > 0 && n); 589 if (len > 0) { 590 (void) m_free(m); 591 goto bad; 592 } 593 m->m_next = n; 594 return (m); 595 bad: 596 m_freem(n); 597 MPFail++; 598 return (0); 599 } 600 601 /* 602 * Partition an mbuf chain in two pieces, returning the tail -- 603 * all but the first len0 bytes. In case of failure, it returns NULL and 604 * attempts to restore the chain to its original state. 605 */ 606 struct mbuf * 607 m_split(m0, len0, wait) 608 register struct mbuf *m0; 609 int len0, wait; 610 { 611 register struct mbuf *m, *n; 612 unsigned len = len0, remain; 613 614 for (m = m0; m && len > m->m_len; m = m->m_next) 615 len -= m->m_len; 616 if (m == 0) 617 return (0); 618 remain = m->m_len - len; 619 if (m0->m_flags & M_PKTHDR) { 620 MGETHDR(n, wait, m0->m_type); 621 if (n == 0) 622 return (0); 623 n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif; 624 n->m_pkthdr.len = m0->m_pkthdr.len - len0; 625 m0->m_pkthdr.len = len0; 626 if (m->m_flags & M_EXT) 627 goto extpacket; 628 if (remain > MHLEN) { 629 /* m can't be the lead packet */ 630 MH_ALIGN(n, 0); 631 n->m_next = m_split(m, len, wait); 632 if (n->m_next == 0) { 633 (void) m_free(n); 634 return (0); 635 } else 636 return (n); 637 } else 638 MH_ALIGN(n, remain); 639 } else if (remain == 0) { 640 n = m->m_next; 641 m->m_next = 0; 642 return (n); 643 } else { 644 MGET(n, wait, m->m_type); 645 if (n == 0) 646 return (0); 647 M_ALIGN(n, remain); 648 } 649 extpacket: 650 if (m->m_flags & M_EXT) { 651 n->m_ext = m->m_ext; 652 MCLADDREFERENCE(m, n); 653 n->m_data = m->m_data + len; 654 } else { 655 bcopy(mtod(m, caddr_t) + len, mtod(n, caddr_t), remain); 656 } 657 n->m_len = remain; 658 m->m_len = len; 659 n->m_next = m->m_next; 660 m->m_next = 0; 661 return (n); 662 } 663 /* 664 * Routine to copy from device local memory into mbufs. 665 */ 666 struct mbuf * 667 m_devget(buf, totlen, off0, ifp, copy) 668 char *buf; 669 int totlen, off0; 670 struct ifnet *ifp; 671 void (*copy) __P((const void *from, void *to, size_t len)); 672 { 673 register struct mbuf *m; 674 struct mbuf *top = 0, **mp = ⊤ 675 register int off = off0, len; 676 register char *cp; 677 char *epkt; 678 679 cp = buf; 680 epkt = cp + totlen; 681 if (off) { 682 /* 683 * If 'off' is non-zero, packet is trailer-encapsulated, 684 * so we have to skip the type and length fields. 685 */ 686 cp += off + 2 * sizeof(u_int16_t); 687 totlen -= 2 * sizeof(u_int16_t); 688 } 689 MGETHDR(m, M_DONTWAIT, MT_DATA); 690 if (m == 0) 691 return (0); 692 m->m_pkthdr.rcvif = ifp; 693 m->m_pkthdr.len = totlen; 694 m->m_len = MHLEN; 695 696 while (totlen > 0) { 697 if (top) { 698 MGET(m, M_DONTWAIT, MT_DATA); 699 if (m == 0) { 700 m_freem(top); 701 return (0); 702 } 703 m->m_len = MLEN; 704 } 705 len = min(totlen, epkt - cp); 706 if (len >= MINCLSIZE) { 707 MCLGET(m, M_DONTWAIT); 708 if ((m->m_flags & M_EXT) == 0) { 709 m_freem(top); 710 return (0); 711 } 712 m->m_len = len = min(len, MCLBYTES); 713 } else { 714 /* 715 * Place initial small packet/header at end of mbuf. 716 */ 717 if (len < m->m_len) { 718 if (top == 0 && len + max_linkhdr <= m->m_len) 719 m->m_data += max_linkhdr; 720 m->m_len = len; 721 } else 722 len = m->m_len; 723 } 724 if (copy) 725 copy(cp, mtod(m, caddr_t), (size_t)len); 726 else 727 bcopy(cp, mtod(m, caddr_t), (size_t)len); 728 cp += len; 729 *mp = m; 730 mp = &m->m_next; 731 totlen -= len; 732 if (cp == epkt) 733 cp = buf; 734 } 735 return (top); 736 } 737 738 /* 739 * Copy data from a buffer back into the indicated mbuf chain, 740 * starting "off" bytes from the beginning, extending the mbuf 741 * chain if necessary. 742 */ 743 void 744 m_copyback(m0, off, len, cp) 745 struct mbuf *m0; 746 register int off; 747 register int len; 748 caddr_t cp; 749 { 750 register int mlen; 751 register struct mbuf *m = m0, *n; 752 int totlen = 0; 753 754 if (m0 == 0) 755 return; 756 while (off > (mlen = m->m_len)) { 757 off -= mlen; 758 totlen += mlen; 759 if (m->m_next == 0) { 760 n = m_getclr(M_DONTWAIT, m->m_type); 761 if (n == 0) 762 goto out; 763 n->m_len = min(MLEN, len + off); 764 m->m_next = n; 765 } 766 m = m->m_next; 767 } 768 while (len > 0) { 769 mlen = min (m->m_len - off, len); 770 bcopy(cp, mtod(m, caddr_t) + off, (unsigned)mlen); 771 cp += mlen; 772 len -= mlen; 773 mlen += off; 774 off = 0; 775 totlen += mlen; 776 if (len == 0) 777 break; 778 if (m->m_next == 0) { 779 n = m_get(M_DONTWAIT, m->m_type); 780 if (n == 0) 781 break; 782 n->m_len = min(MLEN, len); 783 m->m_next = n; 784 } 785 m = m->m_next; 786 } 787 out: if (((m = m0)->m_flags & M_PKTHDR) && (m->m_pkthdr.len < totlen)) 788 m->m_pkthdr.len = totlen; 789 } 790