1 /* $OpenBSD: ifq.c,v 1.36 2020/01/25 06:31:32 dlg Exp $ */ 2 3 /* 4 * Copyright (c) 2015 David Gwynne <dlg@openbsd.org> 5 * 6 * Permission to use, copy, modify, and distribute this software for any 7 * purpose with or without fee is hereby granted, provided that the above 8 * copyright notice and this permission notice appear in all copies. 9 * 10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 17 */ 18 19 #include "bpfilter.h" 20 21 #include <sys/param.h> 22 #include <sys/systm.h> 23 #include <sys/socket.h> 24 #include <sys/mbuf.h> 25 #include <sys/proc.h> 26 #include <sys/sysctl.h> 27 28 #include <net/if.h> 29 #include <net/if_var.h> 30 31 #if NBPFILTER > 0 32 #include <net/bpf.h> 33 #endif 34 35 /* 36 * priq glue 37 */ 38 unsigned int priq_idx(unsigned int, const struct mbuf *); 39 struct mbuf *priq_enq(struct ifqueue *, struct mbuf *); 40 struct mbuf *priq_deq_begin(struct ifqueue *, void **); 41 void priq_deq_commit(struct ifqueue *, struct mbuf *, void *); 42 void priq_purge(struct ifqueue *, struct mbuf_list *); 43 44 void *priq_alloc(unsigned int, void *); 45 void priq_free(unsigned int, void *); 46 47 const struct ifq_ops priq_ops = { 48 priq_idx, 49 priq_enq, 50 priq_deq_begin, 51 priq_deq_commit, 52 priq_purge, 53 priq_alloc, 54 priq_free, 55 }; 56 57 const struct ifq_ops * const ifq_priq_ops = &priq_ops; 58 59 /* 60 * priq internal structures 61 */ 62 63 struct priq { 64 struct mbuf_list pq_lists[IFQ_NQUEUES]; 65 }; 66 67 /* 68 * ifqueue serialiser 69 */ 70 71 void ifq_start_task(void *); 72 void ifq_restart_task(void *); 73 void ifq_barrier_task(void *); 74 void ifq_bundle_task(void *); 75 76 static inline void 77 ifq_run_start(struct ifqueue *ifq) 78 { 79 ifq_serialize(ifq, &ifq->ifq_start); 80 } 81 82 void 83 ifq_serialize(struct ifqueue *ifq, struct task *t) 84 { 85 struct task work; 86 87 if (ISSET(t->t_flags, TASK_ONQUEUE)) 88 return; 89 90 mtx_enter(&ifq->ifq_task_mtx); 91 if (!ISSET(t->t_flags, TASK_ONQUEUE)) { 92 SET(t->t_flags, TASK_ONQUEUE); 93 TAILQ_INSERT_TAIL(&ifq->ifq_task_list, t, t_entry); 94 } 95 96 if (ifq->ifq_serializer == NULL) { 97 ifq->ifq_serializer = curcpu(); 98 99 while ((t = TAILQ_FIRST(&ifq->ifq_task_list)) != NULL) { 100 TAILQ_REMOVE(&ifq->ifq_task_list, t, t_entry); 101 CLR(t->t_flags, TASK_ONQUEUE); 102 work = *t; /* copy to caller to avoid races */ 103 104 mtx_leave(&ifq->ifq_task_mtx); 105 106 (*work.t_func)(work.t_arg); 107 108 mtx_enter(&ifq->ifq_task_mtx); 109 } 110 111 ifq->ifq_serializer = NULL; 112 } 113 mtx_leave(&ifq->ifq_task_mtx); 114 } 115 116 int 117 ifq_is_serialized(struct ifqueue *ifq) 118 { 119 return (ifq->ifq_serializer == curcpu()); 120 } 121 122 void 123 ifq_start(struct ifqueue *ifq) 124 { 125 if (ifq_len(ifq) >= min(ifq->ifq_if->if_txmit, ifq->ifq_maxlen)) { 126 task_del(ifq->ifq_softnet, &ifq->ifq_bundle); 127 ifq_run_start(ifq); 128 } else 129 task_add(ifq->ifq_softnet, &ifq->ifq_bundle); 130 } 131 132 void 133 ifq_start_task(void *p) 134 { 135 struct ifqueue *ifq = p; 136 struct ifnet *ifp = ifq->ifq_if; 137 138 if (!ISSET(ifp->if_flags, IFF_RUNNING) || 139 ifq_empty(ifq) || ifq_is_oactive(ifq)) 140 return; 141 142 ifp->if_qstart(ifq); 143 } 144 145 void 146 ifq_restart_task(void *p) 147 { 148 struct ifqueue *ifq = p; 149 struct ifnet *ifp = ifq->ifq_if; 150 151 ifq_clr_oactive(ifq); 152 ifp->if_qstart(ifq); 153 } 154 155 void 156 ifq_bundle_task(void *p) 157 { 158 struct ifqueue *ifq = p; 159 160 ifq_run_start(ifq); 161 } 162 163 void 164 ifq_barrier(struct ifqueue *ifq) 165 { 166 struct cond c = COND_INITIALIZER(); 167 struct task t = TASK_INITIALIZER(ifq_barrier_task, &c); 168 169 task_del(ifq->ifq_softnet, &ifq->ifq_bundle); 170 171 if (ifq->ifq_serializer == NULL) 172 return; 173 174 ifq_serialize(ifq, &t); 175 176 cond_wait(&c, "ifqbar"); 177 } 178 179 void 180 ifq_barrier_task(void *p) 181 { 182 struct cond *c = p; 183 184 cond_signal(c); 185 } 186 187 /* 188 * ifqueue mbuf queue API 189 */ 190 191 void 192 ifq_init(struct ifqueue *ifq, struct ifnet *ifp, unsigned int idx) 193 { 194 ifq->ifq_if = ifp; 195 ifq->ifq_softnet = net_tq(ifp->if_index); /* + idx */ 196 ifq->ifq_softc = NULL; 197 198 mtx_init(&ifq->ifq_mtx, IPL_NET); 199 200 /* default to priq */ 201 ifq->ifq_ops = &priq_ops; 202 ifq->ifq_q = priq_ops.ifqop_alloc(idx, NULL); 203 204 ml_init(&ifq->ifq_free); 205 ifq->ifq_len = 0; 206 207 ifq->ifq_packets = 0; 208 ifq->ifq_bytes = 0; 209 ifq->ifq_qdrops = 0; 210 ifq->ifq_errors = 0; 211 ifq->ifq_mcasts = 0; 212 213 mtx_init(&ifq->ifq_task_mtx, IPL_NET); 214 TAILQ_INIT(&ifq->ifq_task_list); 215 ifq->ifq_serializer = NULL; 216 task_set(&ifq->ifq_bundle, ifq_bundle_task, ifq); 217 218 task_set(&ifq->ifq_start, ifq_start_task, ifq); 219 task_set(&ifq->ifq_restart, ifq_restart_task, ifq); 220 221 if (ifq->ifq_maxlen == 0) 222 ifq_set_maxlen(ifq, IFQ_MAXLEN); 223 224 ifq->ifq_idx = idx; 225 } 226 227 void 228 ifq_attach(struct ifqueue *ifq, const struct ifq_ops *newops, void *opsarg) 229 { 230 struct mbuf_list ml = MBUF_LIST_INITIALIZER(); 231 struct mbuf_list free_ml = MBUF_LIST_INITIALIZER(); 232 struct mbuf *m; 233 const struct ifq_ops *oldops; 234 void *newq, *oldq; 235 236 newq = newops->ifqop_alloc(ifq->ifq_idx, opsarg); 237 238 mtx_enter(&ifq->ifq_mtx); 239 ifq->ifq_ops->ifqop_purge(ifq, &ml); 240 ifq->ifq_len = 0; 241 242 oldops = ifq->ifq_ops; 243 oldq = ifq->ifq_q; 244 245 ifq->ifq_ops = newops; 246 ifq->ifq_q = newq; 247 248 while ((m = ml_dequeue(&ml)) != NULL) { 249 m = ifq->ifq_ops->ifqop_enq(ifq, m); 250 if (m != NULL) { 251 ifq->ifq_qdrops++; 252 ml_enqueue(&free_ml, m); 253 } else 254 ifq->ifq_len++; 255 } 256 mtx_leave(&ifq->ifq_mtx); 257 258 oldops->ifqop_free(ifq->ifq_idx, oldq); 259 260 ml_purge(&free_ml); 261 } 262 263 void 264 ifq_destroy(struct ifqueue *ifq) 265 { 266 struct mbuf_list ml = MBUF_LIST_INITIALIZER(); 267 268 NET_ASSERT_UNLOCKED(); 269 if (!task_del(ifq->ifq_softnet, &ifq->ifq_bundle)) 270 taskq_barrier(ifq->ifq_softnet); 271 272 /* don't need to lock because this is the last use of the ifq */ 273 274 ifq->ifq_ops->ifqop_purge(ifq, &ml); 275 ifq->ifq_ops->ifqop_free(ifq->ifq_idx, ifq->ifq_q); 276 277 ml_purge(&ml); 278 } 279 280 void 281 ifq_add_data(struct ifqueue *ifq, struct if_data *data) 282 { 283 mtx_enter(&ifq->ifq_mtx); 284 data->ifi_opackets += ifq->ifq_packets; 285 data->ifi_obytes += ifq->ifq_bytes; 286 data->ifi_oqdrops += ifq->ifq_qdrops; 287 data->ifi_omcasts += ifq->ifq_mcasts; 288 /* ifp->if_data.ifi_oerrors */ 289 mtx_leave(&ifq->ifq_mtx); 290 } 291 292 int 293 ifq_enqueue(struct ifqueue *ifq, struct mbuf *m) 294 { 295 struct mbuf *dm; 296 297 mtx_enter(&ifq->ifq_mtx); 298 dm = ifq->ifq_ops->ifqop_enq(ifq, m); 299 if (dm != m) { 300 ifq->ifq_packets++; 301 ifq->ifq_bytes += m->m_pkthdr.len; 302 if (ISSET(m->m_flags, M_MCAST)) 303 ifq->ifq_mcasts++; 304 } 305 306 if (dm == NULL) 307 ifq->ifq_len++; 308 else 309 ifq->ifq_qdrops++; 310 mtx_leave(&ifq->ifq_mtx); 311 312 if (dm != NULL) 313 m_freem(dm); 314 315 return (dm == m ? ENOBUFS : 0); 316 } 317 318 static inline void 319 ifq_deq_enter(struct ifqueue *ifq) 320 { 321 mtx_enter(&ifq->ifq_mtx); 322 } 323 324 static inline void 325 ifq_deq_leave(struct ifqueue *ifq) 326 { 327 struct mbuf_list ml; 328 329 ml = ifq->ifq_free; 330 ml_init(&ifq->ifq_free); 331 332 mtx_leave(&ifq->ifq_mtx); 333 334 if (!ml_empty(&ml)) 335 ml_purge(&ml); 336 } 337 338 struct mbuf * 339 ifq_deq_begin(struct ifqueue *ifq) 340 { 341 struct mbuf *m = NULL; 342 void *cookie; 343 344 ifq_deq_enter(ifq); 345 if (ifq->ifq_len == 0 || 346 (m = ifq->ifq_ops->ifqop_deq_begin(ifq, &cookie)) == NULL) { 347 ifq_deq_leave(ifq); 348 return (NULL); 349 } 350 351 m->m_pkthdr.ph_cookie = cookie; 352 353 return (m); 354 } 355 356 void 357 ifq_deq_commit(struct ifqueue *ifq, struct mbuf *m) 358 { 359 void *cookie; 360 361 KASSERT(m != NULL); 362 cookie = m->m_pkthdr.ph_cookie; 363 364 ifq->ifq_ops->ifqop_deq_commit(ifq, m, cookie); 365 ifq->ifq_len--; 366 ifq_deq_leave(ifq); 367 } 368 369 void 370 ifq_deq_rollback(struct ifqueue *ifq, struct mbuf *m) 371 { 372 KASSERT(m != NULL); 373 374 ifq_deq_leave(ifq); 375 } 376 377 struct mbuf * 378 ifq_dequeue(struct ifqueue *ifq) 379 { 380 struct mbuf *m; 381 382 m = ifq_deq_begin(ifq); 383 if (m == NULL) 384 return (NULL); 385 386 ifq_deq_commit(ifq, m); 387 388 return (m); 389 } 390 391 int 392 ifq_deq_sleep(struct ifqueue *ifq, struct mbuf **mp, int nbio, int priority, 393 const char *wmesg, volatile unsigned int *sleeping, 394 volatile unsigned int *alive) 395 { 396 struct mbuf *m; 397 void *cookie; 398 int error; 399 400 ifq_deq_enter(ifq); 401 if (ifq->ifq_len == 0 && nbio) 402 error = EWOULDBLOCK; 403 else { 404 for (;;) { 405 m = ifq->ifq_ops->ifqop_deq_begin(ifq, &cookie); 406 if (m != NULL) { 407 ifq->ifq_ops->ifqop_deq_commit(ifq, m, cookie); 408 ifq->ifq_len--; 409 *mp = m; 410 break; 411 } 412 413 (*sleeping)++; 414 error = msleep_nsec(ifq, &ifq->ifq_mtx, 415 priority, wmesg, INFSLP); 416 (*sleeping)--; 417 if (error != 0) 418 break; 419 if (!(*alive)) { 420 error = ENXIO; 421 break; 422 } 423 } 424 } 425 ifq_deq_leave(ifq); 426 427 return (error); 428 } 429 430 int 431 ifq_hdatalen(struct ifqueue *ifq) 432 { 433 struct mbuf *m; 434 int len = 0; 435 436 m = ifq_deq_begin(ifq); 437 if (m != NULL) { 438 len = m->m_pkthdr.len; 439 ifq_deq_rollback(ifq, m); 440 } 441 442 return (len); 443 } 444 445 unsigned int 446 ifq_purge(struct ifqueue *ifq) 447 { 448 struct mbuf_list ml = MBUF_LIST_INITIALIZER(); 449 unsigned int rv; 450 451 mtx_enter(&ifq->ifq_mtx); 452 ifq->ifq_ops->ifqop_purge(ifq, &ml); 453 rv = ifq->ifq_len; 454 ifq->ifq_len = 0; 455 ifq->ifq_qdrops += rv; 456 mtx_leave(&ifq->ifq_mtx); 457 458 KASSERT(rv == ml_len(&ml)); 459 460 ml_purge(&ml); 461 462 return (rv); 463 } 464 465 void * 466 ifq_q_enter(struct ifqueue *ifq, const struct ifq_ops *ops) 467 { 468 mtx_enter(&ifq->ifq_mtx); 469 if (ifq->ifq_ops == ops) 470 return (ifq->ifq_q); 471 472 mtx_leave(&ifq->ifq_mtx); 473 474 return (NULL); 475 } 476 477 void 478 ifq_q_leave(struct ifqueue *ifq, void *q) 479 { 480 KASSERT(q == ifq->ifq_q); 481 mtx_leave(&ifq->ifq_mtx); 482 } 483 484 void 485 ifq_mfreem(struct ifqueue *ifq, struct mbuf *m) 486 { 487 MUTEX_ASSERT_LOCKED(&ifq->ifq_mtx); 488 489 ifq->ifq_len--; 490 ifq->ifq_qdrops++; 491 ml_enqueue(&ifq->ifq_free, m); 492 } 493 494 void 495 ifq_mfreeml(struct ifqueue *ifq, struct mbuf_list *ml) 496 { 497 MUTEX_ASSERT_LOCKED(&ifq->ifq_mtx); 498 499 ifq->ifq_len -= ml_len(ml); 500 ifq->ifq_qdrops += ml_len(ml); 501 ml_enlist(&ifq->ifq_free, ml); 502 } 503 504 /* 505 * ifiq 506 */ 507 508 static void ifiq_process(void *); 509 510 void 511 ifiq_init(struct ifiqueue *ifiq, struct ifnet *ifp, unsigned int idx) 512 { 513 ifiq->ifiq_if = ifp; 514 ifiq->ifiq_softnet = net_tq(ifp->if_index); /* + idx */ 515 ifiq->ifiq_softc = NULL; 516 517 mtx_init(&ifiq->ifiq_mtx, IPL_NET); 518 ml_init(&ifiq->ifiq_ml); 519 task_set(&ifiq->ifiq_task, ifiq_process, ifiq); 520 ifiq->ifiq_pressure = 0; 521 522 ifiq->ifiq_packets = 0; 523 ifiq->ifiq_bytes = 0; 524 ifiq->ifiq_qdrops = 0; 525 ifiq->ifiq_errors = 0; 526 527 ifiq->ifiq_idx = idx; 528 } 529 530 void 531 ifiq_destroy(struct ifiqueue *ifiq) 532 { 533 NET_ASSERT_UNLOCKED(); 534 if (!task_del(ifiq->ifiq_softnet, &ifiq->ifiq_task)) 535 taskq_barrier(ifiq->ifiq_softnet); 536 537 /* don't need to lock because this is the last use of the ifiq */ 538 ml_purge(&ifiq->ifiq_ml); 539 } 540 541 unsigned int ifiq_maxlen_drop = 2048 * 5; 542 unsigned int ifiq_maxlen_return = 2048 * 3; 543 544 int 545 ifiq_input(struct ifiqueue *ifiq, struct mbuf_list *ml) 546 { 547 struct ifnet *ifp = ifiq->ifiq_if; 548 struct mbuf *m; 549 uint64_t packets; 550 uint64_t bytes = 0; 551 unsigned int len; 552 #if NBPFILTER > 0 553 caddr_t if_bpf; 554 #endif 555 556 if (ml_empty(ml)) 557 return (0); 558 559 MBUF_LIST_FOREACH(ml, m) { 560 m->m_pkthdr.ph_ifidx = ifp->if_index; 561 m->m_pkthdr.ph_rtableid = ifp->if_rdomain; 562 bytes += m->m_pkthdr.len; 563 } 564 packets = ml_len(ml); 565 566 #if NBPFILTER > 0 567 if_bpf = ifp->if_bpf; 568 if (if_bpf) { 569 struct mbuf_list ml0 = *ml; 570 571 ml_init(ml); 572 573 while ((m = ml_dequeue(&ml0)) != NULL) { 574 if (bpf_mtap_ether(if_bpf, m, BPF_DIRECTION_IN)) 575 m_freem(m); 576 else 577 ml_enqueue(ml, m); 578 } 579 580 if (ml_empty(ml)) { 581 mtx_enter(&ifiq->ifiq_mtx); 582 ifiq->ifiq_packets += packets; 583 ifiq->ifiq_bytes += bytes; 584 mtx_leave(&ifiq->ifiq_mtx); 585 586 return (0); 587 } 588 } 589 #endif 590 591 mtx_enter(&ifiq->ifiq_mtx); 592 ifiq->ifiq_packets += packets; 593 ifiq->ifiq_bytes += bytes; 594 595 len = ml_len(&ifiq->ifiq_ml); 596 if (len > ifiq_maxlen_drop) 597 ifiq->ifiq_qdrops += ml_len(ml); 598 else 599 ml_enlist(&ifiq->ifiq_ml, ml); 600 mtx_leave(&ifiq->ifiq_mtx); 601 602 if (ml_empty(ml)) 603 task_add(ifiq->ifiq_softnet, &ifiq->ifiq_task); 604 else 605 ml_purge(ml); 606 607 return (len > ifiq_maxlen_return); 608 } 609 610 void 611 ifiq_add_data(struct ifiqueue *ifiq, struct if_data *data) 612 { 613 mtx_enter(&ifiq->ifiq_mtx); 614 data->ifi_ipackets += ifiq->ifiq_packets; 615 data->ifi_ibytes += ifiq->ifiq_bytes; 616 data->ifi_iqdrops += ifiq->ifiq_qdrops; 617 mtx_leave(&ifiq->ifiq_mtx); 618 } 619 620 int 621 ifiq_enqueue(struct ifiqueue *ifiq, struct mbuf *m) 622 { 623 mtx_enter(&ifiq->ifiq_mtx); 624 ml_enqueue(&ifiq->ifiq_ml, m); 625 mtx_leave(&ifiq->ifiq_mtx); 626 627 task_add(ifiq->ifiq_softnet, &ifiq->ifiq_task); 628 629 return (0); 630 } 631 632 static void 633 ifiq_process(void *arg) 634 { 635 struct ifiqueue *ifiq = arg; 636 struct mbuf_list ml; 637 638 if (ifiq_empty(ifiq)) 639 return; 640 641 mtx_enter(&ifiq->ifiq_mtx); 642 ml = ifiq->ifiq_ml; 643 ml_init(&ifiq->ifiq_ml); 644 mtx_leave(&ifiq->ifiq_mtx); 645 646 if_input_process(ifiq->ifiq_if, &ml); 647 } 648 649 int 650 net_ifiq_sysctl(int *name, u_int namelen, void *oldp, size_t *oldlenp, 651 void *newp, size_t newlen) 652 { 653 int error = EOPNOTSUPP; 654 /* pressure is disabled for 6.6-release */ 655 #if 0 656 int val; 657 658 if (namelen != 1) 659 return (EISDIR); 660 661 switch (name[0]) { 662 case NET_LINK_IFRXQ_PRESSURE_RETURN: 663 val = ifiq_pressure_return; 664 error = sysctl_int(oldp, oldlenp, newp, newlen, &val); 665 if (error != 0) 666 return (error); 667 if (val < 1 || val > ifiq_pressure_drop) 668 return (EINVAL); 669 ifiq_pressure_return = val; 670 break; 671 case NET_LINK_IFRXQ_PRESSURE_DROP: 672 val = ifiq_pressure_drop; 673 error = sysctl_int(oldp, oldlenp, newp, newlen, &val); 674 if (error != 0) 675 return (error); 676 if (ifiq_pressure_return > val) 677 return (EINVAL); 678 ifiq_pressure_drop = val; 679 break; 680 default: 681 error = EOPNOTSUPP; 682 break; 683 } 684 #endif 685 686 return (error); 687 } 688 689 /* 690 * priq implementation 691 */ 692 693 unsigned int 694 priq_idx(unsigned int nqueues, const struct mbuf *m) 695 { 696 unsigned int flow = 0; 697 698 if (ISSET(m->m_pkthdr.ph_flowid, M_FLOWID_VALID)) 699 flow = m->m_pkthdr.ph_flowid & M_FLOWID_MASK; 700 701 return (flow % nqueues); 702 } 703 704 void * 705 priq_alloc(unsigned int idx, void *null) 706 { 707 struct priq *pq; 708 int i; 709 710 pq = malloc(sizeof(struct priq), M_DEVBUF, M_WAITOK); 711 for (i = 0; i < IFQ_NQUEUES; i++) 712 ml_init(&pq->pq_lists[i]); 713 return (pq); 714 } 715 716 void 717 priq_free(unsigned int idx, void *pq) 718 { 719 free(pq, M_DEVBUF, sizeof(struct priq)); 720 } 721 722 struct mbuf * 723 priq_enq(struct ifqueue *ifq, struct mbuf *m) 724 { 725 struct priq *pq; 726 struct mbuf_list *pl; 727 struct mbuf *n = NULL; 728 unsigned int prio; 729 730 pq = ifq->ifq_q; 731 KASSERT(m->m_pkthdr.pf.prio <= IFQ_MAXPRIO); 732 733 /* Find a lower priority queue to drop from */ 734 if (ifq_len(ifq) >= ifq->ifq_maxlen) { 735 for (prio = 0; prio < m->m_pkthdr.pf.prio; prio++) { 736 pl = &pq->pq_lists[prio]; 737 if (ml_len(pl) > 0) { 738 n = ml_dequeue(pl); 739 goto enqueue; 740 } 741 } 742 /* 743 * There's no lower priority queue that we can 744 * drop from so don't enqueue this one. 745 */ 746 return (m); 747 } 748 749 enqueue: 750 pl = &pq->pq_lists[m->m_pkthdr.pf.prio]; 751 ml_enqueue(pl, m); 752 753 return (n); 754 } 755 756 struct mbuf * 757 priq_deq_begin(struct ifqueue *ifq, void **cookiep) 758 { 759 struct priq *pq = ifq->ifq_q; 760 struct mbuf_list *pl; 761 unsigned int prio = nitems(pq->pq_lists); 762 struct mbuf *m; 763 764 do { 765 pl = &pq->pq_lists[--prio]; 766 m = MBUF_LIST_FIRST(pl); 767 if (m != NULL) { 768 *cookiep = pl; 769 return (m); 770 } 771 } while (prio > 0); 772 773 return (NULL); 774 } 775 776 void 777 priq_deq_commit(struct ifqueue *ifq, struct mbuf *m, void *cookie) 778 { 779 struct mbuf_list *pl = cookie; 780 781 KASSERT(MBUF_LIST_FIRST(pl) == m); 782 783 ml_dequeue(pl); 784 } 785 786 void 787 priq_purge(struct ifqueue *ifq, struct mbuf_list *ml) 788 { 789 struct priq *pq = ifq->ifq_q; 790 struct mbuf_list *pl; 791 unsigned int prio = nitems(pq->pq_lists); 792 793 do { 794 pl = &pq->pq_lists[--prio]; 795 ml_enlist(ml, pl); 796 } while (prio > 0); 797 } 798