1 /* $OpenBSD: uipc_mbuf.c,v 1.231 2016/09/15 02:00:16 dlg Exp $ */ 2 /* $NetBSD: uipc_mbuf.c,v 1.15.4.1 1996/06/13 17:11:44 cgd Exp $ */ 3 4 /* 5 * Copyright (c) 1982, 1986, 1988, 1991, 1993 6 * The Regents of the University of California. All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 3. Neither the name of the University nor the names of its contributors 17 * may be used to endorse or promote products derived from this software 18 * without specific prior written permission. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 30 * SUCH DAMAGE. 31 * 32 * @(#)uipc_mbuf.c 8.2 (Berkeley) 1/4/94 33 */ 34 35 /* 36 * @(#)COPYRIGHT 1.1 (NRL) 17 January 1995 37 * 38 * NRL grants permission for redistribution and use in source and binary 39 * forms, with or without modification, of the software and documentation 40 * created at NRL provided that the following conditions are met: 41 * 42 * 1. Redistributions of source code must retain the above copyright 43 * notice, this list of conditions and the following disclaimer. 44 * 2. Redistributions in binary form must reproduce the above copyright 45 * notice, this list of conditions and the following disclaimer in the 46 * documentation and/or other materials provided with the distribution. 47 * 3. All advertising materials mentioning features or use of this software 48 * must display the following acknowledgements: 49 * This product includes software developed by the University of 50 * California, Berkeley and its contributors. 51 * This product includes software developed at the Information 52 * Technology Division, US Naval Research Laboratory. 53 * 4. Neither the name of the NRL nor the names of its contributors 54 * may be used to endorse or promote products derived from this software 55 * without specific prior written permission. 56 * 57 * THE SOFTWARE PROVIDED BY NRL IS PROVIDED BY NRL AND CONTRIBUTORS ``AS 58 * IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 59 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A 60 * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NRL OR 61 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 62 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 63 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 64 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 65 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 66 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 67 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 68 * 69 * The views and conclusions contained in the software and documentation 70 * are those of the authors and should not be interpreted as representing 71 * official policies, either expressed or implied, of the US Naval 72 * Research Laboratory (NRL). 73 */ 74 75 #include "pf.h" 76 77 #include <sys/param.h> 78 #include <sys/systm.h> 79 #include <sys/malloc.h> 80 #include <sys/mbuf.h> 81 #include <sys/kernel.h> 82 #include <sys/syslog.h> 83 #include <sys/domain.h> 84 #include <sys/protosw.h> 85 #include <sys/pool.h> 86 87 #include <sys/socket.h> 88 #include <sys/socketvar.h> 89 #include <net/if.h> 90 91 92 #include <uvm/uvm_extern.h> 93 94 #ifdef DDB 95 #include <machine/db_machdep.h> 96 #endif 97 98 #if NPF > 0 99 #include <net/pfvar.h> 100 #endif /* NPF > 0 */ 101 102 struct mbstat mbstat; /* mbuf stats */ 103 struct mutex mbstatmtx = MUTEX_INITIALIZER(IPL_NET); 104 struct pool mbpool; /* mbuf pool */ 105 struct pool mtagpool; 106 107 /* mbuf cluster pools */ 108 u_int mclsizes[MCLPOOLS] = { 109 MCLBYTES, /* must be at slot 0 */ 110 4 * 1024, 111 8 * 1024, 112 9 * 1024, 113 12 * 1024, 114 16 * 1024, 115 64 * 1024 116 }; 117 static char mclnames[MCLPOOLS][8]; 118 struct pool mclpools[MCLPOOLS]; 119 120 struct pool *m_clpool(u_int); 121 122 int max_linkhdr; /* largest link-level header */ 123 int max_protohdr; /* largest protocol header */ 124 int max_hdr; /* largest link+protocol header */ 125 126 struct mutex m_extref_mtx = MUTEX_INITIALIZER(IPL_NET); 127 128 void m_extfree(struct mbuf *); 129 void nmbclust_update(void); 130 void m_zero(struct mbuf *); 131 132 static void (*mextfree_fns[4])(caddr_t, u_int, void *); 133 static u_int num_extfree_fns; 134 135 const char *mclpool_warnmsg = 136 "WARNING: mclpools limit reached; increase kern.maxclusters"; 137 138 /* 139 * Initialize the mbuf allocator. 140 */ 141 void 142 mbinit(void) 143 { 144 int i; 145 146 #if DIAGNOSTIC 147 if (mclsizes[0] != MCLBYTES) 148 panic("mbinit: the smallest cluster size != MCLBYTES"); 149 if (mclsizes[nitems(mclsizes) - 1] != MAXMCLBYTES) 150 panic("mbinit: the largest cluster size != MAXMCLBYTES"); 151 #endif 152 153 pool_init(&mbpool, MSIZE, 0, IPL_NET, 0, "mbufpl", NULL); 154 pool_set_constraints(&mbpool, &kp_dma_contig); 155 pool_setlowat(&mbpool, mblowat); 156 157 pool_init(&mtagpool, PACKET_TAG_MAXSIZE + sizeof(struct m_tag), 0, 158 IPL_NET, 0, "mtagpl", NULL); 159 160 for (i = 0; i < nitems(mclsizes); i++) { 161 snprintf(mclnames[i], sizeof(mclnames[0]), "mcl%dk", 162 mclsizes[i] >> 10); 163 pool_init(&mclpools[i], mclsizes[i], 0, IPL_NET, 0, 164 mclnames[i], NULL); 165 pool_set_constraints(&mclpools[i], &kp_dma_contig); 166 pool_setlowat(&mclpools[i], mcllowat); 167 } 168 169 (void)mextfree_register(m_extfree_pool); 170 KASSERT(num_extfree_fns == 1); 171 172 nmbclust_update(); 173 } 174 175 void 176 nmbclust_update(void) 177 { 178 unsigned int i, n; 179 180 /* 181 * Set the hard limit on the mclpools to the number of 182 * mbuf clusters the kernel is to support. Log the limit 183 * reached message max once a minute. 184 */ 185 for (i = 0; i < nitems(mclsizes); i++) { 186 n = (unsigned long long)nmbclust * MCLBYTES / mclsizes[i]; 187 (void)pool_sethardlimit(&mclpools[i], n, mclpool_warnmsg, 60); 188 /* 189 * XXX this needs to be reconsidered. 190 * Setting the high water mark to nmbclust is too high 191 * but we need to have enough spare buffers around so that 192 * allocations in interrupt context don't fail or mclgeti() 193 * drivers may end up with empty rings. 194 */ 195 pool_sethiwat(&mclpools[i], n); 196 } 197 pool_sethiwat(&mbpool, nmbclust); 198 } 199 200 /* 201 * Space allocation routines. 202 */ 203 struct mbuf * 204 m_get(int nowait, int type) 205 { 206 struct mbuf *m; 207 208 m = pool_get(&mbpool, nowait == M_WAIT ? PR_WAITOK : PR_NOWAIT); 209 if (m == NULL) 210 return (NULL); 211 212 mtx_enter(&mbstatmtx); 213 mbstat.m_mtypes[type]++; 214 mtx_leave(&mbstatmtx); 215 216 m->m_type = type; 217 m->m_next = NULL; 218 m->m_nextpkt = NULL; 219 m->m_data = m->m_dat; 220 m->m_flags = 0; 221 222 return (m); 223 } 224 225 /* 226 * ATTN: When changing anything here check m_inithdr() and m_defrag() those 227 * may need to change as well. 228 */ 229 struct mbuf * 230 m_gethdr(int nowait, int type) 231 { 232 struct mbuf *m; 233 234 m = pool_get(&mbpool, nowait == M_WAIT ? PR_WAITOK : PR_NOWAIT); 235 if (m == NULL) 236 return (NULL); 237 238 mtx_enter(&mbstatmtx); 239 mbstat.m_mtypes[type]++; 240 mtx_leave(&mbstatmtx); 241 242 m->m_type = type; 243 244 return (m_inithdr(m)); 245 } 246 247 struct mbuf * 248 m_inithdr(struct mbuf *m) 249 { 250 /* keep in sync with m_gethdr */ 251 m->m_next = NULL; 252 m->m_nextpkt = NULL; 253 m->m_data = m->m_pktdat; 254 m->m_flags = M_PKTHDR; 255 memset(&m->m_pkthdr, 0, sizeof(m->m_pkthdr)); 256 m->m_pkthdr.pf.prio = IFQ_DEFPRIO; 257 258 return (m); 259 } 260 261 void 262 m_resethdr(struct mbuf *m) 263 { 264 int len = m->m_pkthdr.len; 265 u_int8_t loopcnt = m->m_pkthdr.ph_loopcnt; 266 267 KASSERT(m->m_flags & M_PKTHDR); 268 m->m_flags &= (M_EXT|M_PKTHDR|M_EOR|M_EXTWR|M_ZEROIZE); 269 270 /* delete all mbuf tags to reset the state */ 271 m_tag_delete_chain(m); 272 273 #if NPF > 0 274 pf_pkt_unlink_state_key(m); 275 #endif /* NPF > 0 */ 276 277 /* like m_inithdr(), but keep any associated data and mbufs */ 278 memset(&m->m_pkthdr, 0, sizeof(m->m_pkthdr)); 279 m->m_pkthdr.pf.prio = IFQ_DEFPRIO; 280 m->m_pkthdr.len = len; 281 m->m_pkthdr.ph_loopcnt = loopcnt; 282 } 283 284 struct mbuf * 285 m_getclr(int nowait, int type) 286 { 287 struct mbuf *m; 288 289 MGET(m, nowait, type); 290 if (m == NULL) 291 return (NULL); 292 memset(mtod(m, caddr_t), 0, MLEN); 293 return (m); 294 } 295 296 struct pool * 297 m_clpool(u_int pktlen) 298 { 299 struct pool *pp; 300 int pi; 301 302 for (pi = 0; pi < nitems(mclpools); pi++) { 303 pp = &mclpools[pi]; 304 if (pktlen <= pp->pr_size) 305 return (pp); 306 } 307 308 return (NULL); 309 } 310 311 struct mbuf * 312 m_clget(struct mbuf *m, int how, u_int pktlen) 313 { 314 struct mbuf *m0 = NULL; 315 struct pool *pp; 316 caddr_t buf; 317 318 pp = m_clpool(pktlen); 319 #ifdef DIAGNOSTIC 320 if (pp == NULL) 321 panic("m_clget: request for %u byte cluster", pktlen); 322 #endif 323 324 if (m == NULL) { 325 m0 = m_gethdr(how, MT_DATA); 326 if (m0 == NULL) 327 return (NULL); 328 329 m = m0; 330 } 331 buf = pool_get(pp, how == M_WAIT ? PR_WAITOK : PR_NOWAIT); 332 if (buf == NULL) { 333 if (m0) 334 m_freem(m0); 335 return (NULL); 336 } 337 338 MEXTADD(m, buf, pp->pr_size, M_EXTWR, MEXTFREE_POOL, pp); 339 return (m); 340 } 341 342 void 343 m_extfree_pool(caddr_t buf, u_int size, void *pp) 344 { 345 pool_put(pp, buf); 346 } 347 348 struct mbuf * 349 m_free(struct mbuf *m) 350 { 351 struct mbuf *n; 352 353 if (m == NULL) 354 return (NULL); 355 356 mtx_enter(&mbstatmtx); 357 mbstat.m_mtypes[m->m_type]--; 358 mtx_leave(&mbstatmtx); 359 360 n = m->m_next; 361 if (m->m_flags & M_ZEROIZE) { 362 m_zero(m); 363 /* propagate M_ZEROIZE to the next mbuf in the chain */ 364 if (n) 365 n->m_flags |= M_ZEROIZE; 366 } 367 if (m->m_flags & M_PKTHDR) { 368 m_tag_delete_chain(m); 369 #if NPF > 0 370 pf_pkt_unlink_state_key(m); 371 #endif /* NPF > 0 */ 372 } 373 if (m->m_flags & M_EXT) 374 m_extfree(m); 375 376 pool_put(&mbpool, m); 377 378 return (n); 379 } 380 381 void 382 m_extref(struct mbuf *o, struct mbuf *n) 383 { 384 int refs = MCLISREFERENCED(o); 385 386 n->m_flags |= o->m_flags & (M_EXT|M_EXTWR); 387 388 if (refs) 389 mtx_enter(&m_extref_mtx); 390 n->m_ext.ext_nextref = o->m_ext.ext_nextref; 391 n->m_ext.ext_prevref = o; 392 o->m_ext.ext_nextref = n; 393 n->m_ext.ext_nextref->m_ext.ext_prevref = n; 394 if (refs) 395 mtx_leave(&m_extref_mtx); 396 397 MCLREFDEBUGN((n), __FILE__, __LINE__); 398 } 399 400 static inline u_int 401 m_extunref(struct mbuf *m) 402 { 403 int refs = 1; 404 405 if (!MCLISREFERENCED(m)) 406 return (0); 407 408 mtx_enter(&m_extref_mtx); 409 if (MCLISREFERENCED(m)) { 410 m->m_ext.ext_nextref->m_ext.ext_prevref = 411 m->m_ext.ext_prevref; 412 m->m_ext.ext_prevref->m_ext.ext_nextref = 413 m->m_ext.ext_nextref; 414 } else 415 refs = 0; 416 mtx_leave(&m_extref_mtx); 417 418 return (refs); 419 } 420 421 /* 422 * Returns a number for use with MEXTADD. 423 * Should only be called once per function. 424 * Drivers can be assured that the index will be non zero. 425 */ 426 u_int 427 mextfree_register(void (*fn)(caddr_t, u_int, void *)) 428 { 429 KASSERT(num_extfree_fns < nitems(mextfree_fns)); 430 mextfree_fns[num_extfree_fns] = fn; 431 return num_extfree_fns++; 432 } 433 434 void 435 m_extfree(struct mbuf *m) 436 { 437 if (m_extunref(m) == 0) { 438 KASSERT(m->m_ext.ext_free_fn < num_extfree_fns); 439 mextfree_fns[m->m_ext.ext_free_fn](m->m_ext.ext_buf, 440 m->m_ext.ext_size, m->m_ext.ext_arg); 441 } 442 443 m->m_flags &= ~(M_EXT|M_EXTWR); 444 } 445 446 struct mbuf * 447 m_freem(struct mbuf *m) 448 { 449 struct mbuf *n; 450 451 if (m == NULL) 452 return (NULL); 453 454 n = m->m_nextpkt; 455 456 do 457 m = m_free(m); 458 while (m != NULL); 459 460 return (n); 461 } 462 463 void 464 m_purge(struct mbuf *m) 465 { 466 while (m != NULL) 467 m = m_freem(m); 468 } 469 470 /* 471 * mbuf chain defragmenter. This function uses some evil tricks to defragment 472 * an mbuf chain into a single buffer without changing the mbuf pointer. 473 * This needs to know a lot of the mbuf internals to make this work. 474 */ 475 int 476 m_defrag(struct mbuf *m, int how) 477 { 478 struct mbuf *m0; 479 480 if (m->m_next == NULL) 481 return (0); 482 483 #ifdef DIAGNOSTIC 484 if (!(m->m_flags & M_PKTHDR)) 485 panic("m_defrag: no packet hdr or not a chain"); 486 #endif 487 488 if ((m0 = m_gethdr(how, m->m_type)) == NULL) 489 return (ENOBUFS); 490 if (m->m_pkthdr.len > MHLEN) { 491 MCLGETI(m0, how, NULL, m->m_pkthdr.len); 492 if (!(m0->m_flags & M_EXT)) { 493 m_free(m0); 494 return (ENOBUFS); 495 } 496 } 497 m_copydata(m, 0, m->m_pkthdr.len, mtod(m0, caddr_t)); 498 m0->m_pkthdr.len = m0->m_len = m->m_pkthdr.len; 499 500 /* free chain behind and possible ext buf on the first mbuf */ 501 m_freem(m->m_next); 502 m->m_next = NULL; 503 if (m->m_flags & M_EXT) 504 m_extfree(m); 505 506 /* 507 * Bounce copy mbuf over to the original mbuf and set everything up. 508 * This needs to reset or clear all pointers that may go into the 509 * original mbuf chain. 510 */ 511 if (m0->m_flags & M_EXT) { 512 memcpy(&m->m_ext, &m0->m_ext, sizeof(struct mbuf_ext)); 513 MCLINITREFERENCE(m); 514 m->m_flags |= m0->m_flags & (M_EXT|M_EXTWR); 515 m->m_data = m->m_ext.ext_buf; 516 } else { 517 m->m_data = m->m_pktdat; 518 memcpy(m->m_data, m0->m_data, m0->m_len); 519 } 520 m->m_pkthdr.len = m->m_len = m0->m_len; 521 522 m0->m_flags &= ~(M_EXT|M_EXTWR); /* cluster is gone */ 523 m_free(m0); 524 525 return (0); 526 } 527 528 /* 529 * Mbuffer utility routines. 530 */ 531 532 /* 533 * Ensure len bytes of contiguous space at the beginning of the mbuf chain 534 */ 535 struct mbuf * 536 m_prepend(struct mbuf *m, int len, int how) 537 { 538 struct mbuf *mn; 539 540 if (len > MHLEN) 541 panic("mbuf prepend length too big"); 542 543 if (M_LEADINGSPACE(m) >= len) { 544 m->m_data -= len; 545 m->m_len += len; 546 } else { 547 MGET(mn, how, m->m_type); 548 if (mn == NULL) { 549 m_freem(m); 550 return (NULL); 551 } 552 if (m->m_flags & M_PKTHDR) 553 M_MOVE_PKTHDR(mn, m); 554 mn->m_next = m; 555 m = mn; 556 MH_ALIGN(m, len); 557 m->m_len = len; 558 } 559 if (m->m_flags & M_PKTHDR) 560 m->m_pkthdr.len += len; 561 return (m); 562 } 563 564 /* 565 * Make a copy of an mbuf chain starting "off" bytes from the beginning, 566 * continuing for "len" bytes. If len is M_COPYALL, copy to end of mbuf. 567 * The wait parameter is a choice of M_WAIT/M_DONTWAIT from caller. 568 */ 569 struct mbuf * 570 m_copym(struct mbuf *m0, int off, int len, int wait) 571 { 572 struct mbuf *m, *n, **np; 573 struct mbuf *top; 574 int copyhdr = 0; 575 576 if (off < 0 || len < 0) 577 panic("m_copym0: off %d, len %d", off, len); 578 if (off == 0 && m0->m_flags & M_PKTHDR) 579 copyhdr = 1; 580 if ((m = m_getptr(m0, off, &off)) == NULL) 581 panic("m_copym0: short mbuf chain"); 582 np = ⊤ 583 top = NULL; 584 while (len > 0) { 585 if (m == NULL) { 586 if (len != M_COPYALL) 587 panic("m_copym0: m == NULL and not COPYALL"); 588 break; 589 } 590 MGET(n, wait, m->m_type); 591 *np = n; 592 if (n == NULL) 593 goto nospace; 594 if (copyhdr) { 595 if (m_dup_pkthdr(n, m0, wait)) 596 goto nospace; 597 if (len != M_COPYALL) 598 n->m_pkthdr.len = len; 599 copyhdr = 0; 600 } 601 n->m_len = min(len, m->m_len - off); 602 if (m->m_flags & M_EXT) { 603 n->m_data = m->m_data + off; 604 n->m_ext = m->m_ext; 605 MCLADDREFERENCE(m, n); 606 } else 607 memcpy(mtod(n, caddr_t), mtod(m, caddr_t) + off, 608 n->m_len); 609 if (len != M_COPYALL) 610 len -= n->m_len; 611 off += n->m_len; 612 #ifdef DIAGNOSTIC 613 if (off > m->m_len) 614 panic("m_copym0 overrun"); 615 #endif 616 if (off == m->m_len) { 617 m = m->m_next; 618 off = 0; 619 } 620 np = &n->m_next; 621 } 622 return (top); 623 nospace: 624 m_freem(top); 625 return (NULL); 626 } 627 628 /* 629 * Copy data from an mbuf chain starting "off" bytes from the beginning, 630 * continuing for "len" bytes, into the indicated buffer. 631 */ 632 void 633 m_copydata(struct mbuf *m, int off, int len, caddr_t cp) 634 { 635 unsigned count; 636 637 if (off < 0) 638 panic("m_copydata: off %d < 0", off); 639 if (len < 0) 640 panic("m_copydata: len %d < 0", len); 641 if ((m = m_getptr(m, off, &off)) == NULL) 642 panic("m_copydata: short mbuf chain"); 643 while (len > 0) { 644 if (m == NULL) 645 panic("m_copydata: null mbuf"); 646 count = min(m->m_len - off, len); 647 memmove(cp, mtod(m, caddr_t) + off, count); 648 len -= count; 649 cp += count; 650 off = 0; 651 m = m->m_next; 652 } 653 } 654 655 /* 656 * Copy data from a buffer back into the indicated mbuf chain, 657 * starting "off" bytes from the beginning, extending the mbuf 658 * chain if necessary. The mbuf needs to be properly initialized 659 * including the setting of m_len. 660 */ 661 int 662 m_copyback(struct mbuf *m0, int off, int len, const void *_cp, int wait) 663 { 664 int mlen, totlen = 0; 665 struct mbuf *m = m0, *n; 666 caddr_t cp = (caddr_t)_cp; 667 int error = 0; 668 669 if (m0 == NULL) 670 return (0); 671 while (off > (mlen = m->m_len)) { 672 off -= mlen; 673 totlen += mlen; 674 if (m->m_next == NULL) { 675 if ((n = m_get(wait, m->m_type)) == NULL) { 676 error = ENOBUFS; 677 goto out; 678 } 679 680 if (off + len > MLEN) { 681 MCLGETI(n, wait, NULL, off + len); 682 if (!(n->m_flags & M_EXT)) { 683 m_free(n); 684 error = ENOBUFS; 685 goto out; 686 } 687 } 688 memset(mtod(n, caddr_t), 0, off); 689 n->m_len = len + off; 690 m->m_next = n; 691 } 692 m = m->m_next; 693 } 694 while (len > 0) { 695 /* extend last packet to be filled fully */ 696 if (m->m_next == NULL && (len > m->m_len - off)) 697 m->m_len += min(len - (m->m_len - off), 698 M_TRAILINGSPACE(m)); 699 mlen = min(m->m_len - off, len); 700 memmove(mtod(m, caddr_t) + off, cp, mlen); 701 cp += mlen; 702 len -= mlen; 703 totlen += mlen + off; 704 if (len == 0) 705 break; 706 off = 0; 707 708 if (m->m_next == NULL) { 709 if ((n = m_get(wait, m->m_type)) == NULL) { 710 error = ENOBUFS; 711 goto out; 712 } 713 714 if (len > MLEN) { 715 MCLGETI(n, wait, NULL, len); 716 if (!(n->m_flags & M_EXT)) { 717 m_free(n); 718 error = ENOBUFS; 719 goto out; 720 } 721 } 722 n->m_len = len; 723 m->m_next = n; 724 } 725 m = m->m_next; 726 } 727 out: 728 if (((m = m0)->m_flags & M_PKTHDR) && (m->m_pkthdr.len < totlen)) 729 m->m_pkthdr.len = totlen; 730 731 return (error); 732 } 733 734 /* 735 * Concatenate mbuf chain n to m. 736 * n might be copied into m (when n->m_len is small), therefore data portion of 737 * n could be copied into an mbuf of different mbuf type. 738 * Therefore both chains should be of the same type (e.g. MT_DATA). 739 * Any m_pkthdr is not updated. 740 */ 741 void 742 m_cat(struct mbuf *m, struct mbuf *n) 743 { 744 while (m->m_next) 745 m = m->m_next; 746 while (n) { 747 if (M_READONLY(m) || n->m_len > M_TRAILINGSPACE(m)) { 748 /* just join the two chains */ 749 m->m_next = n; 750 return; 751 } 752 /* splat the data from one into the other */ 753 memcpy(mtod(m, caddr_t) + m->m_len, mtod(n, caddr_t), 754 n->m_len); 755 m->m_len += n->m_len; 756 n = m_free(n); 757 } 758 } 759 760 void 761 m_adj(struct mbuf *mp, int req_len) 762 { 763 int len = req_len; 764 struct mbuf *m; 765 int count; 766 767 if ((m = mp) == NULL) 768 return; 769 if (len >= 0) { 770 /* 771 * Trim from head. 772 */ 773 while (m != NULL && len > 0) { 774 if (m->m_len <= len) { 775 len -= m->m_len; 776 m->m_len = 0; 777 m = m->m_next; 778 } else { 779 m->m_len -= len; 780 m->m_data += len; 781 len = 0; 782 } 783 } 784 if (mp->m_flags & M_PKTHDR) 785 mp->m_pkthdr.len -= (req_len - len); 786 } else { 787 /* 788 * Trim from tail. Scan the mbuf chain, 789 * calculating its length and finding the last mbuf. 790 * If the adjustment only affects this mbuf, then just 791 * adjust and return. Otherwise, rescan and truncate 792 * after the remaining size. 793 */ 794 len = -len; 795 count = 0; 796 for (;;) { 797 count += m->m_len; 798 if (m->m_next == NULL) 799 break; 800 m = m->m_next; 801 } 802 if (m->m_len >= len) { 803 m->m_len -= len; 804 if (mp->m_flags & M_PKTHDR) 805 mp->m_pkthdr.len -= len; 806 return; 807 } 808 count -= len; 809 if (count < 0) 810 count = 0; 811 /* 812 * Correct length for chain is "count". 813 * Find the mbuf with last data, adjust its length, 814 * and toss data from remaining mbufs on chain. 815 */ 816 m = mp; 817 if (m->m_flags & M_PKTHDR) 818 m->m_pkthdr.len = count; 819 for (; m; m = m->m_next) { 820 if (m->m_len >= count) { 821 m->m_len = count; 822 break; 823 } 824 count -= m->m_len; 825 } 826 while ((m = m->m_next) != NULL) 827 m->m_len = 0; 828 } 829 } 830 831 /* 832 * Rearrange an mbuf chain so that len bytes are contiguous 833 * and in the data area of an mbuf (so that mtod will work 834 * for a structure of size len). Returns the resulting 835 * mbuf chain on success, frees it and returns null on failure. 836 */ 837 struct mbuf * 838 m_pullup(struct mbuf *n, int len) 839 { 840 struct mbuf *m; 841 int count; 842 843 /* 844 * If first mbuf has no cluster, and has room for len bytes 845 * without shifting current data, pullup into it, 846 * otherwise allocate a new mbuf to prepend to the chain. 847 */ 848 if ((n->m_flags & M_EXT) == 0 && n->m_next && 849 n->m_data + len < &n->m_dat[MLEN]) { 850 if (n->m_len >= len) 851 return (n); 852 m = n; 853 n = n->m_next; 854 len -= m->m_len; 855 } else if ((n->m_flags & M_EXT) != 0 && len > MHLEN && n->m_next && 856 n->m_data + len < &n->m_ext.ext_buf[n->m_ext.ext_size]) { 857 if (n->m_len >= len) 858 return (n); 859 m = n; 860 n = n->m_next; 861 len -= m->m_len; 862 } else { 863 if (len > MAXMCLBYTES) 864 goto bad; 865 MGET(m, M_DONTWAIT, n->m_type); 866 if (m == NULL) 867 goto bad; 868 if (len > MHLEN) { 869 MCLGETI(m, M_DONTWAIT, NULL, len); 870 if ((m->m_flags & M_EXT) == 0) { 871 m_free(m); 872 goto bad; 873 } 874 } 875 m->m_len = 0; 876 if (n->m_flags & M_PKTHDR) 877 M_MOVE_PKTHDR(m, n); 878 } 879 880 do { 881 count = min(len, n->m_len); 882 memcpy(mtod(m, caddr_t) + m->m_len, mtod(n, caddr_t), 883 count); 884 len -= count; 885 m->m_len += count; 886 n->m_len -= count; 887 if (n->m_len) 888 n->m_data += count; 889 else 890 n = m_free(n); 891 } while (len > 0 && n); 892 if (len > 0) { 893 (void)m_free(m); 894 goto bad; 895 } 896 m->m_next = n; 897 898 return (m); 899 bad: 900 m_freem(n); 901 return (NULL); 902 } 903 904 /* 905 * Return a pointer to mbuf/offset of location in mbuf chain. 906 */ 907 struct mbuf * 908 m_getptr(struct mbuf *m, int loc, int *off) 909 { 910 while (loc >= 0) { 911 /* Normal end of search */ 912 if (m->m_len > loc) { 913 *off = loc; 914 return (m); 915 } else { 916 loc -= m->m_len; 917 918 if (m->m_next == NULL) { 919 if (loc == 0) { 920 /* Point at the end of valid data */ 921 *off = m->m_len; 922 return (m); 923 } else { 924 return (NULL); 925 } 926 } else { 927 m = m->m_next; 928 } 929 } 930 } 931 932 return (NULL); 933 } 934 935 /* 936 * Partition an mbuf chain in two pieces, returning the tail -- 937 * all but the first len0 bytes. In case of failure, it returns NULL and 938 * attempts to restore the chain to its original state. 939 */ 940 struct mbuf * 941 m_split(struct mbuf *m0, int len0, int wait) 942 { 943 struct mbuf *m, *n; 944 unsigned len = len0, remain, olen; 945 946 for (m = m0; m && len > m->m_len; m = m->m_next) 947 len -= m->m_len; 948 if (m == NULL) 949 return (NULL); 950 remain = m->m_len - len; 951 if (m0->m_flags & M_PKTHDR) { 952 MGETHDR(n, wait, m0->m_type); 953 if (n == NULL) 954 return (NULL); 955 if (m_dup_pkthdr(n, m0, wait)) { 956 m_freem(n); 957 return (NULL); 958 } 959 n->m_pkthdr.len -= len0; 960 olen = m0->m_pkthdr.len; 961 m0->m_pkthdr.len = len0; 962 if (m->m_flags & M_EXT) 963 goto extpacket; 964 if (remain > MHLEN) { 965 /* m can't be the lead packet */ 966 MH_ALIGN(n, 0); 967 n->m_next = m_split(m, len, wait); 968 if (n->m_next == NULL) { 969 (void) m_free(n); 970 m0->m_pkthdr.len = olen; 971 return (NULL); 972 } else 973 return (n); 974 } else 975 MH_ALIGN(n, remain); 976 } else if (remain == 0) { 977 n = m->m_next; 978 m->m_next = NULL; 979 return (n); 980 } else { 981 MGET(n, wait, m->m_type); 982 if (n == NULL) 983 return (NULL); 984 M_ALIGN(n, remain); 985 } 986 extpacket: 987 if (m->m_flags & M_EXT) { 988 n->m_ext = m->m_ext; 989 MCLADDREFERENCE(m, n); 990 n->m_data = m->m_data + len; 991 } else { 992 memcpy(mtod(n, caddr_t), mtod(m, caddr_t) + len, remain); 993 } 994 n->m_len = remain; 995 m->m_len = len; 996 n->m_next = m->m_next; 997 m->m_next = NULL; 998 return (n); 999 } 1000 1001 /* 1002 * Make space for a new header of length hlen at skip bytes 1003 * into the packet. When doing this we allocate new mbufs only 1004 * when absolutely necessary. The mbuf where the new header 1005 * is to go is returned together with an offset into the mbuf. 1006 * If NULL is returned then the mbuf chain may have been modified; 1007 * the caller is assumed to always free the chain. 1008 */ 1009 struct mbuf * 1010 m_makespace(struct mbuf *m0, int skip, int hlen, int *off) 1011 { 1012 struct mbuf *m; 1013 unsigned remain; 1014 1015 KASSERT(m0 != NULL); 1016 KASSERT(hlen < MHLEN); 1017 1018 for (m = m0; m && skip > m->m_len; m = m->m_next) 1019 skip -= m->m_len; 1020 if (m == NULL) 1021 return (NULL); 1022 /* 1023 * At this point skip is the offset into the mbuf m 1024 * where the new header should be placed. Figure out 1025 * if there's space to insert the new header. If so, 1026 * and copying the remainder makese sense then do so. 1027 * Otherwise insert a new mbuf in the chain, splitting 1028 * the contents of m as needed. 1029 */ 1030 remain = m->m_len - skip; /* data to move */ 1031 if (skip < remain && hlen <= M_LEADINGSPACE(m)) { 1032 if (skip) 1033 memmove(m->m_data-hlen, m->m_data, skip); 1034 m->m_data -= hlen; 1035 m->m_len += hlen; 1036 (*off) = skip; 1037 } else if (hlen > M_TRAILINGSPACE(m)) { 1038 struct mbuf *n0, *n, **np; 1039 int todo, len, done, alloc; 1040 1041 n0 = NULL; 1042 np = &n0; 1043 alloc = 0; 1044 done = 0; 1045 todo = remain; 1046 while (todo > 0) { 1047 MGET(n, M_DONTWAIT, m->m_type); 1048 len = MHLEN; 1049 if (n && todo > MHLEN) { 1050 MCLGET(n, M_DONTWAIT); 1051 len = MCLBYTES; 1052 if ((n->m_flags & M_EXT) == 0) { 1053 m_free(n); 1054 n = NULL; 1055 } 1056 } 1057 if (n == NULL) { 1058 m_freem(n0); 1059 return NULL; 1060 } 1061 *np = n; 1062 np = &n->m_next; 1063 alloc++; 1064 len = min(todo, len); 1065 memcpy(n->m_data, mtod(m, char *) + skip + done, len); 1066 n->m_len = len; 1067 done += len; 1068 todo -= len; 1069 } 1070 1071 if (hlen <= M_TRAILINGSPACE(m) + remain) { 1072 m->m_len = skip + hlen; 1073 *off = skip; 1074 if (n0 != NULL) { 1075 *np = m->m_next; 1076 m->m_next = n0; 1077 } 1078 } 1079 else { 1080 n = m_get(M_DONTWAIT, m->m_type); 1081 if (n == NULL) { 1082 m_freem(n0); 1083 return NULL; 1084 } 1085 alloc++; 1086 1087 if ((n->m_next = n0) == NULL) 1088 np = &n->m_next; 1089 n0 = n; 1090 1091 *np = m->m_next; 1092 m->m_next = n0; 1093 1094 n->m_len = hlen; 1095 m->m_len = skip; 1096 1097 m = n; /* header is at front ... */ 1098 *off = 0; /* ... of new mbuf */ 1099 } 1100 } else { 1101 /* 1102 * Copy the remainder to the back of the mbuf 1103 * so there's space to write the new header. 1104 */ 1105 if (remain > 0) 1106 memmove(mtod(m, caddr_t) + skip + hlen, 1107 mtod(m, caddr_t) + skip, remain); 1108 m->m_len += hlen; 1109 *off = skip; 1110 } 1111 m0->m_pkthdr.len += hlen; /* adjust packet length */ 1112 return m; 1113 } 1114 1115 1116 /* 1117 * Routine to copy from device local memory into mbufs. 1118 */ 1119 struct mbuf * 1120 m_devget(char *buf, int totlen, int off) 1121 { 1122 struct mbuf *m; 1123 struct mbuf *top, **mp; 1124 int len; 1125 1126 top = NULL; 1127 mp = ⊤ 1128 1129 if (off < 0 || off > MHLEN) 1130 return (NULL); 1131 1132 MGETHDR(m, M_DONTWAIT, MT_DATA); 1133 if (m == NULL) 1134 return (NULL); 1135 1136 m->m_pkthdr.len = totlen; 1137 1138 len = MHLEN; 1139 1140 while (totlen > 0) { 1141 if (top != NULL) { 1142 MGET(m, M_DONTWAIT, MT_DATA); 1143 if (m == NULL) { 1144 /* 1145 * As we might get called by pfkey, make sure 1146 * we do not leak sensitive data. 1147 */ 1148 top->m_flags |= M_ZEROIZE; 1149 m_freem(top); 1150 return (NULL); 1151 } 1152 len = MLEN; 1153 } 1154 1155 if (totlen + off >= MINCLSIZE) { 1156 MCLGET(m, M_DONTWAIT); 1157 if (m->m_flags & M_EXT) 1158 len = MCLBYTES; 1159 } else { 1160 /* Place initial small packet/header at end of mbuf. */ 1161 if (top == NULL && totlen + off + max_linkhdr <= len) { 1162 m->m_data += max_linkhdr; 1163 len -= max_linkhdr; 1164 } 1165 } 1166 1167 if (off) { 1168 m->m_data += off; 1169 len -= off; 1170 off = 0; 1171 } 1172 1173 m->m_len = len = min(totlen, len); 1174 memcpy(mtod(m, void *), buf, (size_t)len); 1175 1176 buf += len; 1177 *mp = m; 1178 mp = &m->m_next; 1179 totlen -= len; 1180 } 1181 return (top); 1182 } 1183 1184 void 1185 m_zero(struct mbuf *m) 1186 { 1187 #ifdef DIAGNOSTIC 1188 if (M_READONLY(m)) 1189 panic("m_zero: M_READONLY"); 1190 #endif /* DIAGNOSTIC */ 1191 1192 if (m->m_flags & M_EXT) 1193 explicit_bzero(m->m_ext.ext_buf, m->m_ext.ext_size); 1194 else { 1195 if (m->m_flags & M_PKTHDR) 1196 explicit_bzero(m->m_pktdat, MHLEN); 1197 else 1198 explicit_bzero(m->m_dat, MLEN); 1199 } 1200 } 1201 1202 /* 1203 * Apply function f to the data in an mbuf chain starting "off" bytes from the 1204 * beginning, continuing for "len" bytes. 1205 */ 1206 int 1207 m_apply(struct mbuf *m, int off, int len, 1208 int (*f)(caddr_t, caddr_t, unsigned int), caddr_t fstate) 1209 { 1210 int rval; 1211 unsigned int count; 1212 1213 if (len < 0) 1214 panic("m_apply: len %d < 0", len); 1215 if (off < 0) 1216 panic("m_apply: off %d < 0", off); 1217 while (off > 0) { 1218 if (m == NULL) 1219 panic("m_apply: null mbuf in skip"); 1220 if (off < m->m_len) 1221 break; 1222 off -= m->m_len; 1223 m = m->m_next; 1224 } 1225 while (len > 0) { 1226 if (m == NULL) 1227 panic("m_apply: null mbuf"); 1228 count = min(m->m_len - off, len); 1229 1230 rval = f(fstate, mtod(m, caddr_t) + off, count); 1231 if (rval) 1232 return (rval); 1233 1234 len -= count; 1235 off = 0; 1236 m = m->m_next; 1237 } 1238 1239 return (0); 1240 } 1241 1242 int 1243 m_leadingspace(struct mbuf *m) 1244 { 1245 if (M_READONLY(m)) 1246 return 0; 1247 return (m->m_flags & M_EXT ? m->m_data - m->m_ext.ext_buf : 1248 m->m_flags & M_PKTHDR ? m->m_data - m->m_pktdat : 1249 m->m_data - m->m_dat); 1250 } 1251 1252 int 1253 m_trailingspace(struct mbuf *m) 1254 { 1255 if (M_READONLY(m)) 1256 return 0; 1257 return (m->m_flags & M_EXT ? m->m_ext.ext_buf + 1258 m->m_ext.ext_size - (m->m_data + m->m_len) : 1259 &m->m_dat[MLEN] - (m->m_data + m->m_len)); 1260 } 1261 1262 1263 /* 1264 * Duplicate mbuf pkthdr from from to to. 1265 * from must have M_PKTHDR set, and to must be empty. 1266 */ 1267 int 1268 m_dup_pkthdr(struct mbuf *to, struct mbuf *from, int wait) 1269 { 1270 int error; 1271 1272 KASSERT(from->m_flags & M_PKTHDR); 1273 1274 to->m_flags = (to->m_flags & (M_EXT | M_EXTWR)); 1275 to->m_flags |= (from->m_flags & M_COPYFLAGS); 1276 to->m_pkthdr = from->m_pkthdr; 1277 1278 #if NPF > 0 1279 pf_pkt_state_key_ref(to); 1280 #endif /* NPF > 0 */ 1281 1282 SLIST_INIT(&to->m_pkthdr.ph_tags); 1283 1284 if ((error = m_tag_copy_chain(to, from, wait)) != 0) 1285 return (error); 1286 1287 if ((to->m_flags & M_EXT) == 0) 1288 to->m_data = to->m_pktdat; 1289 1290 return (0); 1291 } 1292 1293 struct mbuf * 1294 m_dup_pkt(struct mbuf *m0, unsigned int adj, int wait) 1295 { 1296 struct mbuf *m; 1297 int len; 1298 1299 len = m0->m_pkthdr.len + adj; 1300 if (len > MAXMCLBYTES) /* XXX */ 1301 return (NULL); 1302 1303 m = m_get(wait, m0->m_type); 1304 if (m == NULL) 1305 return (NULL); 1306 1307 if (m_dup_pkthdr(m, m0, wait) != 0) 1308 goto fail; 1309 1310 if (len > MHLEN) { 1311 MCLGETI(m, wait, NULL, len); 1312 if (!ISSET(m->m_flags, M_EXT)) 1313 goto fail; 1314 } 1315 1316 m->m_len = m->m_pkthdr.len = len; 1317 m_adj(m, adj); 1318 m_copydata(m0, 0, m0->m_pkthdr.len, mtod(m, caddr_t)); 1319 1320 return (m); 1321 1322 fail: 1323 m_freem(m); 1324 return (NULL); 1325 } 1326 1327 #ifdef DDB 1328 void 1329 m_print(void *v, 1330 int (*pr)(const char *, ...) __attribute__((__format__(__kprintf__,1,2)))) 1331 { 1332 struct mbuf *m = v; 1333 1334 (*pr)("mbuf %p\n", m); 1335 (*pr)("m_type: %i\tm_flags: %b\n", m->m_type, m->m_flags, M_BITS); 1336 (*pr)("m_next: %p\tm_nextpkt: %p\n", m->m_next, m->m_nextpkt); 1337 (*pr)("m_data: %p\tm_len: %u\n", m->m_data, m->m_len); 1338 (*pr)("m_dat: %p\tm_pktdat: %p\n", m->m_dat, m->m_pktdat); 1339 if (m->m_flags & M_PKTHDR) { 1340 (*pr)("m_ptkhdr.ph_ifidx: %u\tm_pkthdr.len: %i\n", 1341 m->m_pkthdr.ph_ifidx, m->m_pkthdr.len); 1342 (*pr)("m_ptkhdr.ph_tags: %p\tm_pkthdr.ph_tagsset: %b\n", 1343 SLIST_FIRST(&m->m_pkthdr.ph_tags), 1344 m->m_pkthdr.ph_tagsset, MTAG_BITS); 1345 (*pr)("m_pkthdr.ph_flowid: %u\tm_pkthdr.ph_loopcnt: %u\n", 1346 m->m_pkthdr.ph_flowid, m->m_pkthdr.ph_loopcnt); 1347 (*pr)("m_pkthdr.csum_flags: %b\n", 1348 m->m_pkthdr.csum_flags, MCS_BITS); 1349 (*pr)("m_pkthdr.ether_vtag: %u\tm_ptkhdr.ph_rtableid: %u\n", 1350 m->m_pkthdr.ether_vtag, m->m_pkthdr.ph_rtableid); 1351 (*pr)("m_pkthdr.pf.statekey: %p\tm_pkthdr.pf.inp %p\n", 1352 m->m_pkthdr.pf.statekey, m->m_pkthdr.pf.inp); 1353 (*pr)("m_pkthdr.pf.qid: %u\tm_pkthdr.pf.tag: %u\n", 1354 m->m_pkthdr.pf.qid, m->m_pkthdr.pf.tag); 1355 (*pr)("m_pkthdr.pf.flags: %b\n", 1356 m->m_pkthdr.pf.flags, MPF_BITS); 1357 (*pr)("m_pkthdr.pf.routed: %u\tm_pkthdr.pf.prio: %u\n", 1358 m->m_pkthdr.pf.routed, m->m_pkthdr.pf.prio); 1359 } 1360 if (m->m_flags & M_EXT) { 1361 (*pr)("m_ext.ext_buf: %p\tm_ext.ext_size: %u\n", 1362 m->m_ext.ext_buf, m->m_ext.ext_size); 1363 (*pr)("m_ext.ext_free_fn: %u\tm_ext.ext_arg: %p\n", 1364 m->m_ext.ext_free_fn, m->m_ext.ext_arg); 1365 (*pr)("m_ext.ext_nextref: %p\tm_ext.ext_prevref: %p\n", 1366 m->m_ext.ext_nextref, m->m_ext.ext_prevref); 1367 1368 } 1369 } 1370 #endif 1371 1372 /* 1373 * mbuf lists 1374 */ 1375 1376 void 1377 ml_init(struct mbuf_list *ml) 1378 { 1379 ml->ml_head = ml->ml_tail = NULL; 1380 ml->ml_len = 0; 1381 } 1382 1383 void 1384 ml_enqueue(struct mbuf_list *ml, struct mbuf *m) 1385 { 1386 if (ml->ml_tail == NULL) 1387 ml->ml_head = ml->ml_tail = m; 1388 else { 1389 ml->ml_tail->m_nextpkt = m; 1390 ml->ml_tail = m; 1391 } 1392 1393 m->m_nextpkt = NULL; 1394 ml->ml_len++; 1395 } 1396 1397 void 1398 ml_enlist(struct mbuf_list *mla, struct mbuf_list *mlb) 1399 { 1400 if (!ml_empty(mlb)) { 1401 if (ml_empty(mla)) 1402 mla->ml_head = mlb->ml_head; 1403 else 1404 mla->ml_tail->m_nextpkt = mlb->ml_head; 1405 mla->ml_tail = mlb->ml_tail; 1406 mla->ml_len += mlb->ml_len; 1407 1408 ml_init(mlb); 1409 } 1410 } 1411 1412 struct mbuf * 1413 ml_dequeue(struct mbuf_list *ml) 1414 { 1415 struct mbuf *m; 1416 1417 m = ml->ml_head; 1418 if (m != NULL) { 1419 ml->ml_head = m->m_nextpkt; 1420 if (ml->ml_head == NULL) 1421 ml->ml_tail = NULL; 1422 1423 m->m_nextpkt = NULL; 1424 ml->ml_len--; 1425 } 1426 1427 return (m); 1428 } 1429 1430 struct mbuf * 1431 ml_dechain(struct mbuf_list *ml) 1432 { 1433 struct mbuf *m0; 1434 1435 m0 = ml->ml_head; 1436 1437 ml_init(ml); 1438 1439 return (m0); 1440 } 1441 1442 unsigned int 1443 ml_purge(struct mbuf_list *ml) 1444 { 1445 struct mbuf *m, *n; 1446 unsigned int len; 1447 1448 for (m = ml->ml_head; m != NULL; m = n) { 1449 n = m->m_nextpkt; 1450 m_freem(m); 1451 } 1452 1453 len = ml->ml_len; 1454 ml_init(ml); 1455 1456 return (len); 1457 } 1458 1459 /* 1460 * mbuf queues 1461 */ 1462 1463 void 1464 mq_init(struct mbuf_queue *mq, u_int maxlen, int ipl) 1465 { 1466 mtx_init(&mq->mq_mtx, ipl); 1467 ml_init(&mq->mq_list); 1468 mq->mq_maxlen = maxlen; 1469 } 1470 1471 int 1472 mq_enqueue(struct mbuf_queue *mq, struct mbuf *m) 1473 { 1474 int dropped = 0; 1475 1476 mtx_enter(&mq->mq_mtx); 1477 if (mq_len(mq) < mq->mq_maxlen) 1478 ml_enqueue(&mq->mq_list, m); 1479 else { 1480 mq->mq_drops++; 1481 dropped = 1; 1482 } 1483 mtx_leave(&mq->mq_mtx); 1484 1485 if (dropped) 1486 m_freem(m); 1487 1488 return (dropped); 1489 } 1490 1491 struct mbuf * 1492 mq_dequeue(struct mbuf_queue *mq) 1493 { 1494 struct mbuf *m; 1495 1496 mtx_enter(&mq->mq_mtx); 1497 m = ml_dequeue(&mq->mq_list); 1498 mtx_leave(&mq->mq_mtx); 1499 1500 return (m); 1501 } 1502 1503 int 1504 mq_enlist(struct mbuf_queue *mq, struct mbuf_list *ml) 1505 { 1506 struct mbuf *m; 1507 int dropped = 0; 1508 1509 mtx_enter(&mq->mq_mtx); 1510 if (mq_len(mq) < mq->mq_maxlen) 1511 ml_enlist(&mq->mq_list, ml); 1512 else { 1513 dropped = ml_len(ml); 1514 mq->mq_drops += dropped; 1515 } 1516 mtx_leave(&mq->mq_mtx); 1517 1518 if (dropped) { 1519 while ((m = ml_dequeue(ml)) != NULL) 1520 m_freem(m); 1521 } 1522 1523 return (dropped); 1524 } 1525 1526 void 1527 mq_delist(struct mbuf_queue *mq, struct mbuf_list *ml) 1528 { 1529 mtx_enter(&mq->mq_mtx); 1530 *ml = mq->mq_list; 1531 ml_init(&mq->mq_list); 1532 mtx_leave(&mq->mq_mtx); 1533 } 1534 1535 struct mbuf * 1536 mq_dechain(struct mbuf_queue *mq) 1537 { 1538 struct mbuf *m0; 1539 1540 mtx_enter(&mq->mq_mtx); 1541 m0 = ml_dechain(&mq->mq_list); 1542 mtx_leave(&mq->mq_mtx); 1543 1544 return (m0); 1545 } 1546 1547 unsigned int 1548 mq_purge(struct mbuf_queue *mq) 1549 { 1550 struct mbuf_list ml; 1551 1552 mq_delist(mq, &ml); 1553 1554 return (ml_purge(&ml)); 1555 } 1556