1 /* $NetBSD: uipc_mbuf.c,v 1.10 1994/06/29 06:33:34 cgd Exp $ */ 2 3 /* 4 * Copyright (c) 1982, 1986, 1988, 1991, 1993 5 * The Regents of the University of California. All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. All advertising materials mentioning features or use of this software 16 * must display the following acknowledgement: 17 * This product includes software developed by the University of 18 * California, Berkeley and its contributors. 19 * 4. Neither the name of the University nor the names of its contributors 20 * may be used to endorse or promote products derived from this software 21 * without specific prior written permission. 22 * 23 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 24 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 25 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 26 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 27 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 28 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 29 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 30 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 31 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 32 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 33 * SUCH DAMAGE. 34 * 35 * @(#)uipc_mbuf.c 8.2 (Berkeley) 1/4/94 36 */ 37 38 #include <sys/param.h> 39 #include <sys/systm.h> 40 #include <sys/proc.h> 41 #include <sys/malloc.h> 42 #include <sys/map.h> 43 #define MBTYPES 44 #include <sys/mbuf.h> 45 #include <sys/kernel.h> 46 #include <sys/syslog.h> 47 #include <sys/domain.h> 48 #include <sys/protosw.h> 49 50 #include <vm/vm.h> 51 52 extern vm_map_t mb_map; 53 struct mbuf *mbutl; 54 char *mclrefcnt; 55 56 void 57 mbinit() 58 { 59 int s; 60 61 #if CLBYTES < 4096 62 #define NCL_INIT (4096/CLBYTES) 63 #else 64 #define NCL_INIT 1 65 #endif 66 s = splimp(); 67 if (m_clalloc(NCL_INIT, M_DONTWAIT) == 0) 68 goto bad; 69 splx(s); 70 return; 71 bad: 72 panic("mbinit"); 73 } 74 75 /* 76 * Allocate some number of mbuf clusters 77 * and place on cluster free list. 78 * Must be called at splimp. 79 */ 80 /* ARGSUSED */ 81 m_clalloc(ncl, nowait) 82 register int ncl; 83 int nowait; 84 { 85 static int logged; 86 register caddr_t p; 87 register int i; 88 int npg; 89 90 npg = ncl * CLSIZE; 91 p = (caddr_t)kmem_malloc(mb_map, ctob(npg), !nowait); 92 if (p == NULL) { 93 if (logged == 0) { 94 logged++; 95 log(LOG_ERR, "mb_map full\n"); 96 } 97 return (0); 98 } 99 ncl = ncl * CLBYTES / MCLBYTES; 100 for (i = 0; i < ncl; i++) { 101 ((union mcluster *)p)->mcl_next = mclfree; 102 mclfree = (union mcluster *)p; 103 p += MCLBYTES; 104 mbstat.m_clfree++; 105 } 106 mbstat.m_clusters += ncl; 107 return (1); 108 } 109 110 /* 111 * When MGET failes, ask protocols to free space when short of memory, 112 * then re-attempt to allocate an mbuf. 113 */ 114 struct mbuf * 115 m_retry(i, t) 116 int i, t; 117 { 118 register struct mbuf *m; 119 120 m_reclaim(); 121 #define m_retry(i, t) (struct mbuf *)0 122 MGET(m, i, t); 123 #undef m_retry 124 return (m); 125 } 126 127 /* 128 * As above; retry an MGETHDR. 129 */ 130 struct mbuf * 131 m_retryhdr(i, t) 132 int i, t; 133 { 134 register struct mbuf *m; 135 136 m_reclaim(); 137 #define m_retryhdr(i, t) (struct mbuf *)0 138 MGETHDR(m, i, t); 139 #undef m_retryhdr 140 return (m); 141 } 142 143 m_reclaim() 144 { 145 register struct domain *dp; 146 register struct protosw *pr; 147 int s = splimp(); 148 149 for (dp = domains; dp; dp = dp->dom_next) 150 for (pr = dp->dom_protosw; pr < dp->dom_protoswNPROTOSW; pr++) 151 if (pr->pr_drain) 152 (*pr->pr_drain)(); 153 splx(s); 154 mbstat.m_drain++; 155 } 156 157 /* 158 * Space allocation routines. 159 * These are also available as macros 160 * for critical paths. 161 */ 162 struct mbuf * 163 m_get(nowait, type) 164 int nowait, type; 165 { 166 register struct mbuf *m; 167 168 MGET(m, nowait, type); 169 return (m); 170 } 171 172 struct mbuf * 173 m_gethdr(nowait, type) 174 int nowait, type; 175 { 176 register struct mbuf *m; 177 178 MGETHDR(m, nowait, type); 179 return (m); 180 } 181 182 struct mbuf * 183 m_getclr(nowait, type) 184 int nowait, type; 185 { 186 register struct mbuf *m; 187 188 MGET(m, nowait, type); 189 if (m == 0) 190 return (0); 191 bzero(mtod(m, caddr_t), MLEN); 192 return (m); 193 } 194 195 struct mbuf * 196 m_free(m) 197 struct mbuf *m; 198 { 199 register struct mbuf *n; 200 201 MFREE(m, n); 202 return (n); 203 } 204 205 void 206 m_freem(m) 207 register struct mbuf *m; 208 { 209 register struct mbuf *n; 210 211 if (m == NULL) 212 return; 213 do { 214 MFREE(m, n); 215 } while (m = n); 216 } 217 218 /* 219 * Mbuffer utility routines. 220 */ 221 222 /* 223 * Lesser-used path for M_PREPEND: 224 * allocate new mbuf to prepend to chain, 225 * copy junk along. 226 */ 227 struct mbuf * 228 m_prepend(m, len, how) 229 register struct mbuf *m; 230 int len, how; 231 { 232 struct mbuf *mn; 233 234 MGET(mn, how, m->m_type); 235 if (mn == (struct mbuf *)NULL) { 236 m_freem(m); 237 return ((struct mbuf *)NULL); 238 } 239 if (m->m_flags & M_PKTHDR) { 240 M_COPY_PKTHDR(mn, m); 241 m->m_flags &= ~M_PKTHDR; 242 } 243 mn->m_next = m; 244 m = mn; 245 if (len < MHLEN) 246 MH_ALIGN(m, len); 247 m->m_len = len; 248 return (m); 249 } 250 251 /* 252 * Make a copy of an mbuf chain starting "off0" bytes from the beginning, 253 * continuing for "len" bytes. If len is M_COPYALL, copy to end of mbuf. 254 * The wait parameter is a choice of M_WAIT/M_DONTWAIT from caller. 255 */ 256 int MCFail; 257 258 struct mbuf * 259 m_copym(m, off0, len, wait) 260 register struct mbuf *m; 261 int off0, wait; 262 register int len; 263 { 264 register struct mbuf *n, **np; 265 register int off = off0; 266 struct mbuf *top; 267 int copyhdr = 0; 268 269 if (off < 0 || len < 0) 270 panic("m_copym"); 271 if (off == 0 && m->m_flags & M_PKTHDR) 272 copyhdr = 1; 273 while (off > 0) { 274 if (m == 0) 275 panic("m_copym"); 276 if (off < m->m_len) 277 break; 278 off -= m->m_len; 279 m = m->m_next; 280 } 281 np = ⊤ 282 top = 0; 283 while (len > 0) { 284 if (m == 0) { 285 if (len != M_COPYALL) 286 panic("m_copym"); 287 break; 288 } 289 MGET(n, wait, m->m_type); 290 *np = n; 291 if (n == 0) 292 goto nospace; 293 if (copyhdr) { 294 M_COPY_PKTHDR(n, m); 295 if (len == M_COPYALL) 296 n->m_pkthdr.len -= off0; 297 else 298 n->m_pkthdr.len = len; 299 copyhdr = 0; 300 } 301 n->m_len = min(len, m->m_len - off); 302 if (m->m_flags & M_EXT) { 303 n->m_data = m->m_data + off; 304 mclrefcnt[mtocl(m->m_ext.ext_buf)]++; 305 n->m_ext = m->m_ext; 306 n->m_flags |= M_EXT; 307 } else 308 bcopy(mtod(m, caddr_t)+off, mtod(n, caddr_t), 309 (unsigned)n->m_len); 310 if (len != M_COPYALL) 311 len -= n->m_len; 312 off = 0; 313 m = m->m_next; 314 np = &n->m_next; 315 } 316 if (top == 0) 317 MCFail++; 318 return (top); 319 nospace: 320 m_freem(top); 321 MCFail++; 322 return (0); 323 } 324 325 /* 326 * Copy data from an mbuf chain starting "off" bytes from the beginning, 327 * continuing for "len" bytes, into the indicated buffer. 328 */ 329 m_copydata(m, off, len, cp) 330 register struct mbuf *m; 331 register int off; 332 register int len; 333 caddr_t cp; 334 { 335 register unsigned count; 336 337 if (off < 0 || len < 0) 338 panic("m_copydata"); 339 while (off > 0) { 340 if (m == 0) 341 panic("m_copydata"); 342 if (off < m->m_len) 343 break; 344 off -= m->m_len; 345 m = m->m_next; 346 } 347 while (len > 0) { 348 if (m == 0) 349 panic("m_copydata"); 350 count = min(m->m_len - off, len); 351 bcopy(mtod(m, caddr_t) + off, cp, count); 352 len -= count; 353 cp += count; 354 off = 0; 355 m = m->m_next; 356 } 357 } 358 359 /* 360 * Concatenate mbuf chain n to m. 361 * Both chains must be of the same type (e.g. MT_DATA). 362 * Any m_pkthdr is not updated. 363 */ 364 m_cat(m, n) 365 register struct mbuf *m, *n; 366 { 367 while (m->m_next) 368 m = m->m_next; 369 while (n) { 370 if (m->m_flags & M_EXT || 371 m->m_data + m->m_len + n->m_len >= &m->m_dat[MLEN]) { 372 /* just join the two chains */ 373 m->m_next = n; 374 return; 375 } 376 /* splat the data from one into the other */ 377 bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len, 378 (u_int)n->m_len); 379 m->m_len += n->m_len; 380 n = m_free(n); 381 } 382 } 383 384 m_adj(mp, req_len) 385 struct mbuf *mp; 386 int req_len; 387 { 388 register int len = req_len; 389 register struct mbuf *m; 390 register count; 391 392 if ((m = mp) == NULL) 393 return; 394 if (len >= 0) { 395 /* 396 * Trim from head. 397 */ 398 while (m != NULL && len > 0) { 399 if (m->m_len <= len) { 400 len -= m->m_len; 401 m->m_len = 0; 402 m = m->m_next; 403 } else { 404 m->m_len -= len; 405 m->m_data += len; 406 len = 0; 407 } 408 } 409 m = mp; 410 if (mp->m_flags & M_PKTHDR) 411 m->m_pkthdr.len -= (req_len - len); 412 } else { 413 /* 414 * Trim from tail. Scan the mbuf chain, 415 * calculating its length and finding the last mbuf. 416 * If the adjustment only affects this mbuf, then just 417 * adjust and return. Otherwise, rescan and truncate 418 * after the remaining size. 419 */ 420 len = -len; 421 count = 0; 422 for (;;) { 423 count += m->m_len; 424 if (m->m_next == (struct mbuf *)0) 425 break; 426 m = m->m_next; 427 } 428 if (m->m_len >= len) { 429 m->m_len -= len; 430 if (mp->m_flags & M_PKTHDR) 431 mp->m_pkthdr.len -= len; 432 return; 433 } 434 count -= len; 435 if (count < 0) 436 count = 0; 437 /* 438 * Correct length for chain is "count". 439 * Find the mbuf with last data, adjust its length, 440 * and toss data from remaining mbufs on chain. 441 */ 442 m = mp; 443 if (m->m_flags & M_PKTHDR) 444 m->m_pkthdr.len = count; 445 for (; m; m = m->m_next) { 446 if (m->m_len >= count) { 447 m->m_len = count; 448 break; 449 } 450 count -= m->m_len; 451 } 452 while (m = m->m_next) 453 m->m_len = 0; 454 } 455 } 456 457 /* 458 * Rearange an mbuf chain so that len bytes are contiguous 459 * and in the data area of an mbuf (so that mtod and dtom 460 * will work for a structure of size len). Returns the resulting 461 * mbuf chain on success, frees it and returns null on failure. 462 * If there is room, it will add up to max_protohdr-len extra bytes to the 463 * contiguous region in an attempt to avoid being called next time. 464 */ 465 int MPFail; 466 467 struct mbuf * 468 m_pullup(n, len) 469 register struct mbuf *n; 470 int len; 471 { 472 register struct mbuf *m; 473 register int count; 474 int space; 475 476 /* 477 * If first mbuf has no cluster, and has room for len bytes 478 * without shifting current data, pullup into it, 479 * otherwise allocate a new mbuf to prepend to the chain. 480 */ 481 if ((n->m_flags & M_EXT) == 0 && 482 n->m_data + len < &n->m_dat[MLEN] && n->m_next) { 483 if (n->m_len >= len) 484 return (n); 485 m = n; 486 n = n->m_next; 487 len -= m->m_len; 488 } else { 489 if (len > MHLEN) 490 goto bad; 491 MGET(m, M_DONTWAIT, n->m_type); 492 if (m == 0) 493 goto bad; 494 m->m_len = 0; 495 if (n->m_flags & M_PKTHDR) { 496 M_COPY_PKTHDR(m, n); 497 n->m_flags &= ~M_PKTHDR; 498 } 499 } 500 space = &m->m_dat[MLEN] - (m->m_data + m->m_len); 501 do { 502 count = min(min(max(len, max_protohdr), space), n->m_len); 503 bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len, 504 (unsigned)count); 505 len -= count; 506 m->m_len += count; 507 n->m_len -= count; 508 space -= count; 509 if (n->m_len) 510 n->m_data += count; 511 else 512 n = m_free(n); 513 } while (len > 0 && n); 514 if (len > 0) { 515 (void) m_free(m); 516 goto bad; 517 } 518 m->m_next = n; 519 return (m); 520 bad: 521 m_freem(n); 522 MPFail++; 523 return (0); 524 } 525 526 /* 527 * Partition an mbuf chain in two pieces, returning the tail -- 528 * all but the first len0 bytes. In case of failure, it returns NULL and 529 * attempts to restore the chain to its original state. 530 */ 531 struct mbuf * 532 m_split(m0, len0, wait) 533 register struct mbuf *m0; 534 int len0, wait; 535 { 536 register struct mbuf *m, *n; 537 unsigned len = len0, remain; 538 539 for (m = m0; m && len > m->m_len; m = m->m_next) 540 len -= m->m_len; 541 if (m == 0) 542 return (0); 543 remain = m->m_len - len; 544 if (m0->m_flags & M_PKTHDR) { 545 MGETHDR(n, wait, m0->m_type); 546 if (n == 0) 547 return (0); 548 n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif; 549 n->m_pkthdr.len = m0->m_pkthdr.len - len0; 550 m0->m_pkthdr.len = len0; 551 if (m->m_flags & M_EXT) 552 goto extpacket; 553 if (remain > MHLEN) { 554 /* m can't be the lead packet */ 555 MH_ALIGN(n, 0); 556 n->m_next = m_split(m, len, wait); 557 if (n->m_next == 0) { 558 (void) m_free(n); 559 return (0); 560 } else 561 return (n); 562 } else 563 MH_ALIGN(n, remain); 564 } else if (remain == 0) { 565 n = m->m_next; 566 m->m_next = 0; 567 return (n); 568 } else { 569 MGET(n, wait, m->m_type); 570 if (n == 0) 571 return (0); 572 M_ALIGN(n, remain); 573 } 574 extpacket: 575 if (m->m_flags & M_EXT) { 576 n->m_flags |= M_EXT; 577 n->m_ext = m->m_ext; 578 mclrefcnt[mtocl(m->m_ext.ext_buf)]++; 579 m->m_ext.ext_size = 0; /* For Accounting XXXXXX danger */ 580 n->m_data = m->m_data + len; 581 } else { 582 bcopy(mtod(m, caddr_t) + len, mtod(n, caddr_t), remain); 583 } 584 n->m_len = remain; 585 m->m_len = len; 586 n->m_next = m->m_next; 587 m->m_next = 0; 588 return (n); 589 } 590 /* 591 * Routine to copy from device local memory into mbufs. 592 */ 593 struct mbuf * 594 m_devget(buf, totlen, off0, ifp, copy) 595 char *buf; 596 int totlen, off0; 597 struct ifnet *ifp; 598 void (*copy)(); 599 { 600 register struct mbuf *m; 601 struct mbuf *top = 0, **mp = ⊤ 602 register int off = off0, len; 603 register char *cp; 604 char *epkt; 605 606 cp = buf; 607 epkt = cp + totlen; 608 if (off) { 609 cp += off + 2 * sizeof(u_short); 610 totlen -= 2 * sizeof(u_short); 611 } 612 MGETHDR(m, M_DONTWAIT, MT_DATA); 613 if (m == 0) 614 return (0); 615 m->m_pkthdr.rcvif = ifp; 616 m->m_pkthdr.len = totlen; 617 m->m_len = MHLEN; 618 619 while (totlen > 0) { 620 if (top) { 621 MGET(m, M_DONTWAIT, MT_DATA); 622 if (m == 0) { 623 m_freem(top); 624 return (0); 625 } 626 m->m_len = MLEN; 627 } 628 len = min(totlen, epkt - cp); 629 if (len >= MINCLSIZE) { 630 MCLGET(m, M_DONTWAIT); 631 if (m->m_flags & M_EXT) 632 m->m_len = len = min(len, MCLBYTES); 633 else 634 len = m->m_len; 635 } else { 636 /* 637 * Place initial small packet/header at end of mbuf. 638 */ 639 if (len < m->m_len) { 640 if (top == 0 && len + max_linkhdr <= m->m_len) 641 m->m_data += max_linkhdr; 642 m->m_len = len; 643 } else 644 len = m->m_len; 645 } 646 if (copy) 647 copy(cp, mtod(m, caddr_t), (unsigned)len); 648 else 649 bcopy(cp, mtod(m, caddr_t), (unsigned)len); 650 cp += len; 651 *mp = m; 652 mp = &m->m_next; 653 totlen -= len; 654 if (cp == epkt) 655 cp = buf; 656 } 657 return (top); 658 } 659