1 /* $NetBSD: uipc_mbuf.c,v 1.21 1997/06/06 10:51:49 pk Exp $ */ 2 3 /* 4 * Copyright (c) 1982, 1986, 1988, 1991, 1993 5 * The Regents of the University of California. All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. All advertising materials mentioning features or use of this software 16 * must display the following acknowledgement: 17 * This product includes software developed by the University of 18 * California, Berkeley and its contributors. 19 * 4. Neither the name of the University nor the names of its contributors 20 * may be used to endorse or promote products derived from this software 21 * without specific prior written permission. 22 * 23 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 24 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 25 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 26 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 27 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 28 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 29 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 30 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 31 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 32 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 33 * SUCH DAMAGE. 34 * 35 * @(#)uipc_mbuf.c 8.2 (Berkeley) 1/4/94 36 */ 37 38 #include <sys/param.h> 39 #include <sys/systm.h> 40 #include <sys/proc.h> 41 #include <sys/malloc.h> 42 #include <sys/map.h> 43 #define MBTYPES 44 #include <sys/mbuf.h> 45 #include <sys/kernel.h> 46 #include <sys/syslog.h> 47 #include <sys/domain.h> 48 #include <sys/protosw.h> 49 50 #include <vm/vm.h> 51 52 struct mbuf *mbutl; 53 struct mbstat mbstat; 54 union mcluster *mclfree; 55 int max_linkhdr; 56 int max_protohdr; 57 int max_hdr; 58 int max_datalen; 59 60 extern vm_map_t mb_map; 61 62 void 63 mbinit() 64 { 65 int s; 66 67 mclfree = NULL; 68 s = splimp(); 69 if (m_clalloc(max(4096/CLBYTES, 1), M_DONTWAIT) == 0) 70 goto bad; 71 splx(s); 72 return; 73 bad: 74 panic("mbinit"); 75 } 76 77 /* 78 * Allocate some number of mbuf clusters 79 * and place on cluster free list. 80 * Must be called at splimp. 81 */ 82 /* ARGSUSED */ 83 int 84 m_clalloc(ncl, nowait) 85 register int ncl; 86 int nowait; 87 { 88 static volatile struct timeval lastlogged; 89 struct timeval curtime, logdiff; 90 register caddr_t p; 91 register int i; 92 int npg, s; 93 94 npg = ncl * CLSIZE; 95 p = (caddr_t)kmem_malloc(mb_map, ctob(npg), nowait == 0); 96 if (p == NULL) { 97 s = splclock(); 98 curtime = time; 99 splx(s); 100 timersub(&curtime, &lastlogged, &logdiff); 101 if (logdiff.tv_sec >= 60) { 102 lastlogged = curtime; 103 log(LOG_ERR, "mb_map full\n"); 104 } 105 m_reclaim(); 106 return (mclfree != NULL); 107 } 108 ncl = ncl * CLBYTES / MCLBYTES; 109 for (i = 0; i < ncl; i++) { 110 ((union mcluster *)p)->mcl_next = mclfree; 111 mclfree = (union mcluster *)p; 112 p += MCLBYTES; 113 mbstat.m_clfree++; 114 } 115 mbstat.m_clusters += ncl; 116 return (1); 117 } 118 119 /* 120 * When MGET failes, ask protocols to free space when short of memory, 121 * then re-attempt to allocate an mbuf. 122 */ 123 struct mbuf * 124 m_retry(i, t) 125 int i, t; 126 { 127 register struct mbuf *m; 128 129 m_reclaim(); 130 #define m_retry(i, t) (struct mbuf *)0 131 MGET(m, i, t); 132 #undef m_retry 133 if (m != NULL) 134 mbstat.m_wait++; 135 else 136 mbstat.m_drops++; 137 return (m); 138 } 139 140 /* 141 * As above; retry an MGETHDR. 142 */ 143 struct mbuf * 144 m_retryhdr(i, t) 145 int i, t; 146 { 147 register struct mbuf *m; 148 149 m_reclaim(); 150 #define m_retryhdr(i, t) (struct mbuf *)0 151 MGETHDR(m, i, t); 152 #undef m_retryhdr 153 if (m != NULL) 154 mbstat.m_wait++; 155 else 156 mbstat.m_drops++; 157 return (m); 158 } 159 160 void 161 m_reclaim() 162 { 163 register struct domain *dp; 164 register struct protosw *pr; 165 int s = splimp(); 166 167 for (dp = domains; dp; dp = dp->dom_next) 168 for (pr = dp->dom_protosw; pr < dp->dom_protoswNPROTOSW; pr++) 169 if (pr->pr_drain) 170 (*pr->pr_drain)(); 171 splx(s); 172 mbstat.m_drain++; 173 } 174 175 /* 176 * Space allocation routines. 177 * These are also available as macros 178 * for critical paths. 179 */ 180 struct mbuf * 181 m_get(nowait, type) 182 int nowait, type; 183 { 184 register struct mbuf *m; 185 186 MGET(m, nowait, type); 187 return (m); 188 } 189 190 struct mbuf * 191 m_gethdr(nowait, type) 192 int nowait, type; 193 { 194 register struct mbuf *m; 195 196 MGETHDR(m, nowait, type); 197 return (m); 198 } 199 200 struct mbuf * 201 m_getclr(nowait, type) 202 int nowait, type; 203 { 204 register struct mbuf *m; 205 206 MGET(m, nowait, type); 207 if (m == 0) 208 return (0); 209 bzero(mtod(m, caddr_t), MLEN); 210 return (m); 211 } 212 213 struct mbuf * 214 m_free(m) 215 struct mbuf *m; 216 { 217 register struct mbuf *n; 218 219 MFREE(m, n); 220 return (n); 221 } 222 223 void 224 m_freem(m) 225 register struct mbuf *m; 226 { 227 register struct mbuf *n; 228 229 if (m == NULL) 230 return; 231 do { 232 MFREE(m, n); 233 m = n; 234 } while (m); 235 } 236 237 /* 238 * Mbuffer utility routines. 239 */ 240 241 /* 242 * Lesser-used path for M_PREPEND: 243 * allocate new mbuf to prepend to chain, 244 * copy junk along. 245 */ 246 struct mbuf * 247 m_prepend(m, len, how) 248 register struct mbuf *m; 249 int len, how; 250 { 251 struct mbuf *mn; 252 253 MGET(mn, how, m->m_type); 254 if (mn == (struct mbuf *)NULL) { 255 m_freem(m); 256 return ((struct mbuf *)NULL); 257 } 258 if (m->m_flags & M_PKTHDR) { 259 M_COPY_PKTHDR(mn, m); 260 m->m_flags &= ~M_PKTHDR; 261 } 262 mn->m_next = m; 263 m = mn; 264 if (len < MHLEN) 265 MH_ALIGN(m, len); 266 m->m_len = len; 267 return (m); 268 } 269 270 /* 271 * Make a copy of an mbuf chain starting "off0" bytes from the beginning, 272 * continuing for "len" bytes. If len is M_COPYALL, copy to end of mbuf. 273 * The wait parameter is a choice of M_WAIT/M_DONTWAIT from caller. 274 */ 275 int MCFail; 276 277 struct mbuf * 278 m_copym(m, off0, len, wait) 279 register struct mbuf *m; 280 int off0, wait; 281 register int len; 282 { 283 register struct mbuf *n, **np; 284 register int off = off0; 285 struct mbuf *top; 286 int copyhdr = 0; 287 288 if (off < 0 || len < 0) 289 panic("m_copym"); 290 if (off == 0 && m->m_flags & M_PKTHDR) 291 copyhdr = 1; 292 while (off > 0) { 293 if (m == 0) 294 panic("m_copym"); 295 if (off < m->m_len) 296 break; 297 off -= m->m_len; 298 m = m->m_next; 299 } 300 np = ⊤ 301 top = 0; 302 while (len > 0) { 303 if (m == 0) { 304 if (len != M_COPYALL) 305 panic("m_copym"); 306 break; 307 } 308 MGET(n, wait, m->m_type); 309 *np = n; 310 if (n == 0) 311 goto nospace; 312 if (copyhdr) { 313 M_COPY_PKTHDR(n, m); 314 if (len == M_COPYALL) 315 n->m_pkthdr.len -= off0; 316 else 317 n->m_pkthdr.len = len; 318 copyhdr = 0; 319 } 320 n->m_len = min(len, m->m_len - off); 321 if (m->m_flags & M_EXT) { 322 n->m_data = m->m_data + off; 323 n->m_ext = m->m_ext; 324 MCLADDREFERENCE(m, n); 325 } else 326 bcopy(mtod(m, caddr_t)+off, mtod(n, caddr_t), 327 (unsigned)n->m_len); 328 if (len != M_COPYALL) 329 len -= n->m_len; 330 off = 0; 331 m = m->m_next; 332 np = &n->m_next; 333 } 334 if (top == 0) 335 MCFail++; 336 return (top); 337 nospace: 338 m_freem(top); 339 MCFail++; 340 return (0); 341 } 342 343 /* 344 * Copy an entire packet, including header (which must be present). 345 * An optimization of the common case `m_copym(m, 0, M_COPYALL, how)'. 346 */ 347 struct mbuf * 348 m_copypacket(m, how) 349 struct mbuf *m; 350 int how; 351 { 352 struct mbuf *top, *n, *o; 353 354 MGET(n, how, m->m_type); 355 top = n; 356 if (!n) 357 goto nospace; 358 359 M_COPY_PKTHDR(n, m); 360 n->m_len = m->m_len; 361 if (m->m_flags & M_EXT) { 362 n->m_data = m->m_data; 363 n->m_ext = m->m_ext; 364 MCLADDREFERENCE(m, n); 365 } else { 366 bcopy(mtod(m, char *), mtod(n, char *), n->m_len); 367 } 368 369 m = m->m_next; 370 while (m) { 371 MGET(o, how, m->m_type); 372 if (!o) 373 goto nospace; 374 375 n->m_next = o; 376 n = n->m_next; 377 378 n->m_len = m->m_len; 379 if (m->m_flags & M_EXT) { 380 n->m_data = m->m_data; 381 n->m_ext = m->m_ext; 382 MCLADDREFERENCE(m, n); 383 } else { 384 bcopy(mtod(m, char *), mtod(n, char *), n->m_len); 385 } 386 387 m = m->m_next; 388 } 389 return top; 390 nospace: 391 m_freem(top); 392 MCFail++; 393 return 0; 394 } 395 396 /* 397 * Copy data from an mbuf chain starting "off" bytes from the beginning, 398 * continuing for "len" bytes, into the indicated buffer. 399 */ 400 void 401 m_copydata(m, off, len, cp) 402 register struct mbuf *m; 403 register int off; 404 register int len; 405 caddr_t cp; 406 { 407 register unsigned count; 408 409 if (off < 0 || len < 0) 410 panic("m_copydata"); 411 while (off > 0) { 412 if (m == 0) 413 panic("m_copydata"); 414 if (off < m->m_len) 415 break; 416 off -= m->m_len; 417 m = m->m_next; 418 } 419 while (len > 0) { 420 if (m == 0) 421 panic("m_copydata"); 422 count = min(m->m_len - off, len); 423 bcopy(mtod(m, caddr_t) + off, cp, count); 424 len -= count; 425 cp += count; 426 off = 0; 427 m = m->m_next; 428 } 429 } 430 431 /* 432 * Concatenate mbuf chain n to m. 433 * Both chains must be of the same type (e.g. MT_DATA). 434 * Any m_pkthdr is not updated. 435 */ 436 void 437 m_cat(m, n) 438 register struct mbuf *m, *n; 439 { 440 while (m->m_next) 441 m = m->m_next; 442 while (n) { 443 if (m->m_flags & M_EXT || 444 m->m_data + m->m_len + n->m_len >= &m->m_dat[MLEN]) { 445 /* just join the two chains */ 446 m->m_next = n; 447 return; 448 } 449 /* splat the data from one into the other */ 450 bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len, 451 (u_int)n->m_len); 452 m->m_len += n->m_len; 453 n = m_free(n); 454 } 455 } 456 457 void 458 m_adj(mp, req_len) 459 struct mbuf *mp; 460 int req_len; 461 { 462 register int len = req_len; 463 register struct mbuf *m; 464 register count; 465 466 if ((m = mp) == NULL) 467 return; 468 if (len >= 0) { 469 /* 470 * Trim from head. 471 */ 472 while (m != NULL && len > 0) { 473 if (m->m_len <= len) { 474 len -= m->m_len; 475 m->m_len = 0; 476 m = m->m_next; 477 } else { 478 m->m_len -= len; 479 m->m_data += len; 480 len = 0; 481 } 482 } 483 m = mp; 484 if (mp->m_flags & M_PKTHDR) 485 m->m_pkthdr.len -= (req_len - len); 486 } else { 487 /* 488 * Trim from tail. Scan the mbuf chain, 489 * calculating its length and finding the last mbuf. 490 * If the adjustment only affects this mbuf, then just 491 * adjust and return. Otherwise, rescan and truncate 492 * after the remaining size. 493 */ 494 len = -len; 495 count = 0; 496 for (;;) { 497 count += m->m_len; 498 if (m->m_next == (struct mbuf *)0) 499 break; 500 m = m->m_next; 501 } 502 if (m->m_len >= len) { 503 m->m_len -= len; 504 if (mp->m_flags & M_PKTHDR) 505 mp->m_pkthdr.len -= len; 506 return; 507 } 508 count -= len; 509 if (count < 0) 510 count = 0; 511 /* 512 * Correct length for chain is "count". 513 * Find the mbuf with last data, adjust its length, 514 * and toss data from remaining mbufs on chain. 515 */ 516 m = mp; 517 if (m->m_flags & M_PKTHDR) 518 m->m_pkthdr.len = count; 519 for (; m; m = m->m_next) { 520 if (m->m_len >= count) { 521 m->m_len = count; 522 break; 523 } 524 count -= m->m_len; 525 } 526 while (m->m_next) 527 (m = m->m_next) ->m_len = 0; 528 } 529 } 530 531 /* 532 * Rearange an mbuf chain so that len bytes are contiguous 533 * and in the data area of an mbuf (so that mtod and dtom 534 * will work for a structure of size len). Returns the resulting 535 * mbuf chain on success, frees it and returns null on failure. 536 * If there is room, it will add up to max_protohdr-len extra bytes to the 537 * contiguous region in an attempt to avoid being called next time. 538 */ 539 int MPFail; 540 541 struct mbuf * 542 m_pullup(n, len) 543 register struct mbuf *n; 544 int len; 545 { 546 register struct mbuf *m; 547 register int count; 548 int space; 549 550 /* 551 * If first mbuf has no cluster, and has room for len bytes 552 * without shifting current data, pullup into it, 553 * otherwise allocate a new mbuf to prepend to the chain. 554 */ 555 if ((n->m_flags & M_EXT) == 0 && 556 n->m_data + len < &n->m_dat[MLEN] && n->m_next) { 557 if (n->m_len >= len) 558 return (n); 559 m = n; 560 n = n->m_next; 561 len -= m->m_len; 562 } else { 563 if (len > MHLEN) 564 goto bad; 565 MGET(m, M_DONTWAIT, n->m_type); 566 if (m == 0) 567 goto bad; 568 m->m_len = 0; 569 if (n->m_flags & M_PKTHDR) { 570 M_COPY_PKTHDR(m, n); 571 n->m_flags &= ~M_PKTHDR; 572 } 573 } 574 space = &m->m_dat[MLEN] - (m->m_data + m->m_len); 575 do { 576 count = min(min(max(len, max_protohdr), space), n->m_len); 577 bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len, 578 (unsigned)count); 579 len -= count; 580 m->m_len += count; 581 n->m_len -= count; 582 space -= count; 583 if (n->m_len) 584 n->m_data += count; 585 else 586 n = m_free(n); 587 } while (len > 0 && n); 588 if (len > 0) { 589 (void) m_free(m); 590 goto bad; 591 } 592 m->m_next = n; 593 return (m); 594 bad: 595 m_freem(n); 596 MPFail++; 597 return (0); 598 } 599 600 /* 601 * Partition an mbuf chain in two pieces, returning the tail -- 602 * all but the first len0 bytes. In case of failure, it returns NULL and 603 * attempts to restore the chain to its original state. 604 */ 605 struct mbuf * 606 m_split(m0, len0, wait) 607 register struct mbuf *m0; 608 int len0, wait; 609 { 610 register struct mbuf *m, *n; 611 unsigned len = len0, remain; 612 613 for (m = m0; m && len > m->m_len; m = m->m_next) 614 len -= m->m_len; 615 if (m == 0) 616 return (0); 617 remain = m->m_len - len; 618 if (m0->m_flags & M_PKTHDR) { 619 MGETHDR(n, wait, m0->m_type); 620 if (n == 0) 621 return (0); 622 n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif; 623 n->m_pkthdr.len = m0->m_pkthdr.len - len0; 624 m0->m_pkthdr.len = len0; 625 if (m->m_flags & M_EXT) 626 goto extpacket; 627 if (remain > MHLEN) { 628 /* m can't be the lead packet */ 629 MH_ALIGN(n, 0); 630 n->m_next = m_split(m, len, wait); 631 if (n->m_next == 0) { 632 (void) m_free(n); 633 return (0); 634 } else 635 return (n); 636 } else 637 MH_ALIGN(n, remain); 638 } else if (remain == 0) { 639 n = m->m_next; 640 m->m_next = 0; 641 return (n); 642 } else { 643 MGET(n, wait, m->m_type); 644 if (n == 0) 645 return (0); 646 M_ALIGN(n, remain); 647 } 648 extpacket: 649 if (m->m_flags & M_EXT) { 650 n->m_ext = m->m_ext; 651 MCLADDREFERENCE(m, n); 652 n->m_data = m->m_data + len; 653 } else { 654 bcopy(mtod(m, caddr_t) + len, mtod(n, caddr_t), remain); 655 } 656 n->m_len = remain; 657 m->m_len = len; 658 n->m_next = m->m_next; 659 m->m_next = 0; 660 return (n); 661 } 662 /* 663 * Routine to copy from device local memory into mbufs. 664 */ 665 struct mbuf * 666 m_devget(buf, totlen, off0, ifp, copy) 667 char *buf; 668 int totlen, off0; 669 struct ifnet *ifp; 670 void (*copy) __P((const void *from, void *to, size_t len)); 671 { 672 register struct mbuf *m; 673 struct mbuf *top = 0, **mp = ⊤ 674 register int off = off0, len; 675 register char *cp; 676 char *epkt; 677 678 cp = buf; 679 epkt = cp + totlen; 680 if (off) { 681 /* 682 * If 'off' is non-zero, packet is trailer-encapsulated, 683 * so we have to skip the type and length fields. 684 */ 685 cp += off + 2 * sizeof(u_int16_t); 686 totlen -= 2 * sizeof(u_int16_t); 687 } 688 MGETHDR(m, M_DONTWAIT, MT_DATA); 689 if (m == 0) 690 return (0); 691 m->m_pkthdr.rcvif = ifp; 692 m->m_pkthdr.len = totlen; 693 m->m_len = MHLEN; 694 695 while (totlen > 0) { 696 if (top) { 697 MGET(m, M_DONTWAIT, MT_DATA); 698 if (m == 0) { 699 m_freem(top); 700 return (0); 701 } 702 m->m_len = MLEN; 703 } 704 len = min(totlen, epkt - cp); 705 if (len >= MINCLSIZE) { 706 MCLGET(m, M_DONTWAIT); 707 if ((m->m_flags & M_EXT) == 0) { 708 m_free(m); 709 m_freem(top); 710 return (0); 711 } 712 m->m_len = len = min(len, MCLBYTES); 713 } else { 714 /* 715 * Place initial small packet/header at end of mbuf. 716 */ 717 if (len < m->m_len) { 718 if (top == 0 && len + max_linkhdr <= m->m_len) 719 m->m_data += max_linkhdr; 720 m->m_len = len; 721 } else 722 len = m->m_len; 723 } 724 if (copy) 725 copy(cp, mtod(m, caddr_t), (size_t)len); 726 else 727 bcopy(cp, mtod(m, caddr_t), (size_t)len); 728 cp += len; 729 *mp = m; 730 mp = &m->m_next; 731 totlen -= len; 732 if (cp == epkt) 733 cp = buf; 734 } 735 return (top); 736 } 737 738 /* 739 * Copy data from a buffer back into the indicated mbuf chain, 740 * starting "off" bytes from the beginning, extending the mbuf 741 * chain if necessary. 742 */ 743 void 744 m_copyback(m0, off, len, cp) 745 struct mbuf *m0; 746 register int off; 747 register int len; 748 caddr_t cp; 749 { 750 register int mlen; 751 register struct mbuf *m = m0, *n; 752 int totlen = 0; 753 754 if (m0 == 0) 755 return; 756 while (off > (mlen = m->m_len)) { 757 off -= mlen; 758 totlen += mlen; 759 if (m->m_next == 0) { 760 n = m_getclr(M_DONTWAIT, m->m_type); 761 if (n == 0) 762 goto out; 763 n->m_len = min(MLEN, len + off); 764 m->m_next = n; 765 } 766 m = m->m_next; 767 } 768 while (len > 0) { 769 mlen = min (m->m_len - off, len); 770 bcopy(cp, mtod(m, caddr_t) + off, (unsigned)mlen); 771 cp += mlen; 772 len -= mlen; 773 mlen += off; 774 off = 0; 775 totlen += mlen; 776 if (len == 0) 777 break; 778 if (m->m_next == 0) { 779 n = m_get(M_DONTWAIT, m->m_type); 780 if (n == 0) 781 break; 782 n->m_len = min(MLEN, len); 783 m->m_next = n; 784 } 785 m = m->m_next; 786 } 787 out: if (((m = m0)->m_flags & M_PKTHDR) && (m->m_pkthdr.len < totlen)) 788 m->m_pkthdr.len = totlen; 789 } 790