1 /* $OpenBSD: ifq.c,v 1.31 2019/04/16 04:04:19 dlg Exp $ */ 2 3 /* 4 * Copyright (c) 2015 David Gwynne <dlg@openbsd.org> 5 * 6 * Permission to use, copy, modify, and distribute this software for any 7 * purpose with or without fee is hereby granted, provided that the above 8 * copyright notice and this permission notice appear in all copies. 9 * 10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 17 */ 18 19 #include "bpfilter.h" 20 21 #include <sys/param.h> 22 #include <sys/systm.h> 23 #include <sys/socket.h> 24 #include <sys/mbuf.h> 25 #include <sys/proc.h> 26 27 #include <net/if.h> 28 #include <net/if_var.h> 29 30 #if NBPFILTER > 0 31 #include <net/bpf.h> 32 #endif 33 34 /* 35 * priq glue 36 */ 37 unsigned int priq_idx(unsigned int, const struct mbuf *); 38 struct mbuf *priq_enq(struct ifqueue *, struct mbuf *); 39 struct mbuf *priq_deq_begin(struct ifqueue *, void **); 40 void priq_deq_commit(struct ifqueue *, struct mbuf *, void *); 41 void priq_purge(struct ifqueue *, struct mbuf_list *); 42 43 void *priq_alloc(unsigned int, void *); 44 void priq_free(unsigned int, void *); 45 46 const struct ifq_ops priq_ops = { 47 priq_idx, 48 priq_enq, 49 priq_deq_begin, 50 priq_deq_commit, 51 priq_purge, 52 priq_alloc, 53 priq_free, 54 }; 55 56 const struct ifq_ops * const ifq_priq_ops = &priq_ops; 57 58 /* 59 * priq internal structures 60 */ 61 62 struct priq { 63 struct mbuf_list pq_lists[IFQ_NQUEUES]; 64 }; 65 66 /* 67 * ifqueue serialiser 68 */ 69 70 void ifq_start_task(void *); 71 void ifq_restart_task(void *); 72 void ifq_barrier_task(void *); 73 void ifq_bundle_task(void *); 74 75 static inline void 76 ifq_run_start(struct ifqueue *ifq) 77 { 78 ifq_serialize(ifq, &ifq->ifq_start); 79 } 80 81 void 82 ifq_serialize(struct ifqueue *ifq, struct task *t) 83 { 84 struct task work; 85 86 if (ISSET(t->t_flags, TASK_ONQUEUE)) 87 return; 88 89 mtx_enter(&ifq->ifq_task_mtx); 90 if (!ISSET(t->t_flags, TASK_ONQUEUE)) { 91 SET(t->t_flags, TASK_ONQUEUE); 92 TAILQ_INSERT_TAIL(&ifq->ifq_task_list, t, t_entry); 93 } 94 95 if (ifq->ifq_serializer == NULL) { 96 ifq->ifq_serializer = curcpu(); 97 98 while ((t = TAILQ_FIRST(&ifq->ifq_task_list)) != NULL) { 99 TAILQ_REMOVE(&ifq->ifq_task_list, t, t_entry); 100 CLR(t->t_flags, TASK_ONQUEUE); 101 work = *t; /* copy to caller to avoid races */ 102 103 mtx_leave(&ifq->ifq_task_mtx); 104 105 (*work.t_func)(work.t_arg); 106 107 mtx_enter(&ifq->ifq_task_mtx); 108 } 109 110 ifq->ifq_serializer = NULL; 111 } 112 mtx_leave(&ifq->ifq_task_mtx); 113 } 114 115 int 116 ifq_is_serialized(struct ifqueue *ifq) 117 { 118 return (ifq->ifq_serializer == curcpu()); 119 } 120 121 void 122 ifq_start(struct ifqueue *ifq) 123 { 124 if (ifq_len(ifq) >= min(ifq->ifq_if->if_txmit, ifq->ifq_maxlen)) { 125 task_del(ifq->ifq_softnet, &ifq->ifq_bundle); 126 ifq_run_start(ifq); 127 } else 128 task_add(ifq->ifq_softnet, &ifq->ifq_bundle); 129 } 130 131 void 132 ifq_start_task(void *p) 133 { 134 struct ifqueue *ifq = p; 135 struct ifnet *ifp = ifq->ifq_if; 136 137 if (!ISSET(ifp->if_flags, IFF_RUNNING) || 138 ifq_empty(ifq) || ifq_is_oactive(ifq)) 139 return; 140 141 ifp->if_qstart(ifq); 142 } 143 144 void 145 ifq_restart_task(void *p) 146 { 147 struct ifqueue *ifq = p; 148 struct ifnet *ifp = ifq->ifq_if; 149 150 ifq_clr_oactive(ifq); 151 ifp->if_qstart(ifq); 152 } 153 154 void 155 ifq_bundle_task(void *p) 156 { 157 struct ifqueue *ifq = p; 158 159 ifq_run_start(ifq); 160 } 161 162 void 163 ifq_barrier(struct ifqueue *ifq) 164 { 165 struct cond c = COND_INITIALIZER(); 166 struct task t = TASK_INITIALIZER(ifq_barrier_task, &c); 167 168 task_del(ifq->ifq_softnet, &ifq->ifq_bundle); 169 170 if (ifq->ifq_serializer == NULL) 171 return; 172 173 ifq_serialize(ifq, &t); 174 175 cond_wait(&c, "ifqbar"); 176 } 177 178 void 179 ifq_barrier_task(void *p) 180 { 181 struct cond *c = p; 182 183 cond_signal(c); 184 } 185 186 /* 187 * ifqueue mbuf queue API 188 */ 189 190 void 191 ifq_init(struct ifqueue *ifq, struct ifnet *ifp, unsigned int idx) 192 { 193 ifq->ifq_if = ifp; 194 ifq->ifq_softnet = net_tq(ifp->if_index); /* + idx */ 195 ifq->ifq_softc = NULL; 196 197 mtx_init(&ifq->ifq_mtx, IPL_NET); 198 199 /* default to priq */ 200 ifq->ifq_ops = &priq_ops; 201 ifq->ifq_q = priq_ops.ifqop_alloc(idx, NULL); 202 203 ml_init(&ifq->ifq_free); 204 ifq->ifq_len = 0; 205 206 ifq->ifq_packets = 0; 207 ifq->ifq_bytes = 0; 208 ifq->ifq_qdrops = 0; 209 ifq->ifq_errors = 0; 210 ifq->ifq_mcasts = 0; 211 212 mtx_init(&ifq->ifq_task_mtx, IPL_NET); 213 TAILQ_INIT(&ifq->ifq_task_list); 214 ifq->ifq_serializer = NULL; 215 task_set(&ifq->ifq_bundle, ifq_bundle_task, ifq); 216 217 task_set(&ifq->ifq_start, ifq_start_task, ifq); 218 task_set(&ifq->ifq_restart, ifq_restart_task, ifq); 219 220 if (ifq->ifq_maxlen == 0) 221 ifq_set_maxlen(ifq, IFQ_MAXLEN); 222 223 ifq->ifq_idx = idx; 224 } 225 226 void 227 ifq_attach(struct ifqueue *ifq, const struct ifq_ops *newops, void *opsarg) 228 { 229 struct mbuf_list ml = MBUF_LIST_INITIALIZER(); 230 struct mbuf_list free_ml = MBUF_LIST_INITIALIZER(); 231 struct mbuf *m; 232 const struct ifq_ops *oldops; 233 void *newq, *oldq; 234 235 newq = newops->ifqop_alloc(ifq->ifq_idx, opsarg); 236 237 mtx_enter(&ifq->ifq_mtx); 238 ifq->ifq_ops->ifqop_purge(ifq, &ml); 239 ifq->ifq_len = 0; 240 241 oldops = ifq->ifq_ops; 242 oldq = ifq->ifq_q; 243 244 ifq->ifq_ops = newops; 245 ifq->ifq_q = newq; 246 247 while ((m = ml_dequeue(&ml)) != NULL) { 248 m = ifq->ifq_ops->ifqop_enq(ifq, m); 249 if (m != NULL) { 250 ifq->ifq_qdrops++; 251 ml_enqueue(&free_ml, m); 252 } else 253 ifq->ifq_len++; 254 } 255 mtx_leave(&ifq->ifq_mtx); 256 257 oldops->ifqop_free(ifq->ifq_idx, oldq); 258 259 ml_purge(&free_ml); 260 } 261 262 void 263 ifq_destroy(struct ifqueue *ifq) 264 { 265 struct mbuf_list ml = MBUF_LIST_INITIALIZER(); 266 267 NET_ASSERT_UNLOCKED(); 268 if (!task_del(ifq->ifq_softnet, &ifq->ifq_bundle)) 269 taskq_barrier(ifq->ifq_softnet); 270 271 /* don't need to lock because this is the last use of the ifq */ 272 273 ifq->ifq_ops->ifqop_purge(ifq, &ml); 274 ifq->ifq_ops->ifqop_free(ifq->ifq_idx, ifq->ifq_q); 275 276 ml_purge(&ml); 277 } 278 279 void 280 ifq_add_data(struct ifqueue *ifq, struct if_data *data) 281 { 282 mtx_enter(&ifq->ifq_mtx); 283 data->ifi_opackets += ifq->ifq_packets; 284 data->ifi_obytes += ifq->ifq_bytes; 285 data->ifi_oqdrops += ifq->ifq_qdrops; 286 data->ifi_omcasts += ifq->ifq_mcasts; 287 /* ifp->if_data.ifi_oerrors */ 288 mtx_leave(&ifq->ifq_mtx); 289 } 290 291 int 292 ifq_enqueue(struct ifqueue *ifq, struct mbuf *m) 293 { 294 struct mbuf *dm; 295 296 mtx_enter(&ifq->ifq_mtx); 297 dm = ifq->ifq_ops->ifqop_enq(ifq, m); 298 if (dm != m) { 299 ifq->ifq_packets++; 300 ifq->ifq_bytes += m->m_pkthdr.len; 301 if (ISSET(m->m_flags, M_MCAST)) 302 ifq->ifq_mcasts++; 303 } 304 305 if (dm == NULL) 306 ifq->ifq_len++; 307 else 308 ifq->ifq_qdrops++; 309 mtx_leave(&ifq->ifq_mtx); 310 311 if (dm != NULL) 312 m_freem(dm); 313 314 return (dm == m ? ENOBUFS : 0); 315 } 316 317 static inline void 318 ifq_deq_enter(struct ifqueue *ifq) 319 { 320 mtx_enter(&ifq->ifq_mtx); 321 } 322 323 static inline void 324 ifq_deq_leave(struct ifqueue *ifq) 325 { 326 struct mbuf_list ml; 327 328 ml = ifq->ifq_free; 329 ml_init(&ifq->ifq_free); 330 331 mtx_leave(&ifq->ifq_mtx); 332 333 if (!ml_empty(&ml)) 334 ml_purge(&ml); 335 } 336 337 struct mbuf * 338 ifq_deq_begin(struct ifqueue *ifq) 339 { 340 struct mbuf *m = NULL; 341 void *cookie; 342 343 ifq_deq_enter(ifq); 344 if (ifq->ifq_len == 0 || 345 (m = ifq->ifq_ops->ifqop_deq_begin(ifq, &cookie)) == NULL) { 346 ifq_deq_leave(ifq); 347 return (NULL); 348 } 349 350 m->m_pkthdr.ph_cookie = cookie; 351 352 return (m); 353 } 354 355 void 356 ifq_deq_commit(struct ifqueue *ifq, struct mbuf *m) 357 { 358 void *cookie; 359 360 KASSERT(m != NULL); 361 cookie = m->m_pkthdr.ph_cookie; 362 363 ifq->ifq_ops->ifqop_deq_commit(ifq, m, cookie); 364 ifq->ifq_len--; 365 ifq_deq_leave(ifq); 366 } 367 368 void 369 ifq_deq_rollback(struct ifqueue *ifq, struct mbuf *m) 370 { 371 KASSERT(m != NULL); 372 373 ifq_deq_leave(ifq); 374 } 375 376 struct mbuf * 377 ifq_dequeue(struct ifqueue *ifq) 378 { 379 struct mbuf *m; 380 381 m = ifq_deq_begin(ifq); 382 if (m == NULL) 383 return (NULL); 384 385 ifq_deq_commit(ifq, m); 386 387 return (m); 388 } 389 390 int 391 ifq_hdatalen(struct ifqueue *ifq) 392 { 393 struct mbuf *m; 394 int len = 0; 395 396 m = ifq_deq_begin(ifq); 397 if (m != NULL) { 398 len = m->m_pkthdr.len; 399 ifq_deq_commit(ifq, m); 400 } 401 402 return (len); 403 } 404 405 unsigned int 406 ifq_purge(struct ifqueue *ifq) 407 { 408 struct mbuf_list ml = MBUF_LIST_INITIALIZER(); 409 unsigned int rv; 410 411 mtx_enter(&ifq->ifq_mtx); 412 ifq->ifq_ops->ifqop_purge(ifq, &ml); 413 rv = ifq->ifq_len; 414 ifq->ifq_len = 0; 415 ifq->ifq_qdrops += rv; 416 mtx_leave(&ifq->ifq_mtx); 417 418 KASSERT(rv == ml_len(&ml)); 419 420 ml_purge(&ml); 421 422 return (rv); 423 } 424 425 void * 426 ifq_q_enter(struct ifqueue *ifq, const struct ifq_ops *ops) 427 { 428 mtx_enter(&ifq->ifq_mtx); 429 if (ifq->ifq_ops == ops) 430 return (ifq->ifq_q); 431 432 mtx_leave(&ifq->ifq_mtx); 433 434 return (NULL); 435 } 436 437 void 438 ifq_q_leave(struct ifqueue *ifq, void *q) 439 { 440 KASSERT(q == ifq->ifq_q); 441 mtx_leave(&ifq->ifq_mtx); 442 } 443 444 void 445 ifq_mfreem(struct ifqueue *ifq, struct mbuf *m) 446 { 447 MUTEX_ASSERT_LOCKED(&ifq->ifq_mtx); 448 449 ifq->ifq_len--; 450 ifq->ifq_qdrops++; 451 ml_enqueue(&ifq->ifq_free, m); 452 } 453 454 void 455 ifq_mfreeml(struct ifqueue *ifq, struct mbuf_list *ml) 456 { 457 MUTEX_ASSERT_LOCKED(&ifq->ifq_mtx); 458 459 ifq->ifq_len -= ml_len(ml); 460 ifq->ifq_qdrops += ml_len(ml); 461 ml_enlist(&ifq->ifq_free, ml); 462 } 463 464 /* 465 * ifiq 466 */ 467 468 static void ifiq_process(void *); 469 470 void 471 ifiq_init(struct ifiqueue *ifiq, struct ifnet *ifp, unsigned int idx) 472 { 473 ifiq->ifiq_if = ifp; 474 ifiq->ifiq_softnet = net_tq(ifp->if_index); /* + idx */ 475 ifiq->ifiq_softc = NULL; 476 477 mtx_init(&ifiq->ifiq_mtx, IPL_NET); 478 ml_init(&ifiq->ifiq_ml); 479 task_set(&ifiq->ifiq_task, ifiq_process, ifiq); 480 ifiq->ifiq_pressure = 0; 481 482 ifiq->ifiq_packets = 0; 483 ifiq->ifiq_bytes = 0; 484 ifiq->ifiq_qdrops = 0; 485 ifiq->ifiq_errors = 0; 486 487 ifiq->ifiq_idx = idx; 488 } 489 490 void 491 ifiq_destroy(struct ifiqueue *ifiq) 492 { 493 NET_ASSERT_UNLOCKED(); 494 if (!task_del(ifiq->ifiq_softnet, &ifiq->ifiq_task)) 495 taskq_barrier(ifiq->ifiq_softnet); 496 497 /* don't need to lock because this is the last use of the ifiq */ 498 ml_purge(&ifiq->ifiq_ml); 499 } 500 501 unsigned int ifiq_maxlen_drop = 2048 * 5; 502 unsigned int ifiq_maxlen_return = 2048 * 3; 503 504 int 505 ifiq_input(struct ifiqueue *ifiq, struct mbuf_list *ml) 506 { 507 struct ifnet *ifp = ifiq->ifiq_if; 508 struct mbuf *m; 509 uint64_t packets; 510 uint64_t bytes = 0; 511 unsigned int len; 512 #if NBPFILTER > 0 513 caddr_t if_bpf; 514 #endif 515 516 if (ml_empty(ml)) 517 return (0); 518 519 MBUF_LIST_FOREACH(ml, m) { 520 m->m_pkthdr.ph_ifidx = ifp->if_index; 521 m->m_pkthdr.ph_rtableid = ifp->if_rdomain; 522 bytes += m->m_pkthdr.len; 523 } 524 packets = ml_len(ml); 525 526 #if NBPFILTER > 0 527 if_bpf = ifp->if_bpf; 528 if (if_bpf) { 529 struct mbuf_list ml0 = *ml; 530 531 ml_init(ml); 532 533 while ((m = ml_dequeue(&ml0)) != NULL) { 534 if (bpf_mtap_ether(if_bpf, m, BPF_DIRECTION_IN)) 535 m_freem(m); 536 else 537 ml_enqueue(ml, m); 538 } 539 540 if (ml_empty(ml)) { 541 mtx_enter(&ifiq->ifiq_mtx); 542 ifiq->ifiq_packets += packets; 543 ifiq->ifiq_bytes += bytes; 544 mtx_leave(&ifiq->ifiq_mtx); 545 546 return (0); 547 } 548 } 549 #endif 550 551 mtx_enter(&ifiq->ifiq_mtx); 552 ifiq->ifiq_packets += packets; 553 ifiq->ifiq_bytes += bytes; 554 555 len = ml_len(&ifiq->ifiq_ml); 556 if (len > ifiq_maxlen_drop) 557 ifiq->ifiq_qdrops += ml_len(ml); 558 else 559 ml_enlist(&ifiq->ifiq_ml, ml); 560 mtx_leave(&ifiq->ifiq_mtx); 561 562 if (ml_empty(ml)) 563 task_add(ifiq->ifiq_softnet, &ifiq->ifiq_task); 564 else 565 ml_purge(ml); 566 567 return (len > ifiq_maxlen_return); 568 } 569 570 void 571 ifiq_add_data(struct ifiqueue *ifiq, struct if_data *data) 572 { 573 mtx_enter(&ifiq->ifiq_mtx); 574 data->ifi_ipackets += ifiq->ifiq_packets; 575 data->ifi_ibytes += ifiq->ifiq_bytes; 576 data->ifi_iqdrops += ifiq->ifiq_qdrops; 577 mtx_leave(&ifiq->ifiq_mtx); 578 } 579 580 int 581 ifiq_enqueue(struct ifiqueue *ifiq, struct mbuf *m) 582 { 583 mtx_enter(&ifiq->ifiq_mtx); 584 ml_enqueue(&ifiq->ifiq_ml, m); 585 mtx_leave(&ifiq->ifiq_mtx); 586 587 task_add(ifiq->ifiq_softnet, &ifiq->ifiq_task); 588 589 return (0); 590 } 591 592 static void 593 ifiq_process(void *arg) 594 { 595 struct ifiqueue *ifiq = arg; 596 struct mbuf_list ml; 597 598 if (ifiq_empty(ifiq)) 599 return; 600 601 mtx_enter(&ifiq->ifiq_mtx); 602 ml = ifiq->ifiq_ml; 603 ml_init(&ifiq->ifiq_ml); 604 mtx_leave(&ifiq->ifiq_mtx); 605 606 if_input_process(ifiq->ifiq_if, &ml); 607 } 608 609 /* 610 * priq implementation 611 */ 612 613 unsigned int 614 priq_idx(unsigned int nqueues, const struct mbuf *m) 615 { 616 unsigned int flow = 0; 617 618 if (ISSET(m->m_pkthdr.ph_flowid, M_FLOWID_VALID)) 619 flow = m->m_pkthdr.ph_flowid & M_FLOWID_MASK; 620 621 return (flow % nqueues); 622 } 623 624 void * 625 priq_alloc(unsigned int idx, void *null) 626 { 627 struct priq *pq; 628 int i; 629 630 pq = malloc(sizeof(struct priq), M_DEVBUF, M_WAITOK); 631 for (i = 0; i < IFQ_NQUEUES; i++) 632 ml_init(&pq->pq_lists[i]); 633 return (pq); 634 } 635 636 void 637 priq_free(unsigned int idx, void *pq) 638 { 639 free(pq, M_DEVBUF, sizeof(struct priq)); 640 } 641 642 struct mbuf * 643 priq_enq(struct ifqueue *ifq, struct mbuf *m) 644 { 645 struct priq *pq; 646 struct mbuf_list *pl; 647 struct mbuf *n = NULL; 648 unsigned int prio; 649 650 pq = ifq->ifq_q; 651 KASSERT(m->m_pkthdr.pf.prio <= IFQ_MAXPRIO); 652 653 /* Find a lower priority queue to drop from */ 654 if (ifq_len(ifq) >= ifq->ifq_maxlen) { 655 for (prio = 0; prio < m->m_pkthdr.pf.prio; prio++) { 656 pl = &pq->pq_lists[prio]; 657 if (ml_len(pl) > 0) { 658 n = ml_dequeue(pl); 659 goto enqueue; 660 } 661 } 662 /* 663 * There's no lower priority queue that we can 664 * drop from so don't enqueue this one. 665 */ 666 return (m); 667 } 668 669 enqueue: 670 pl = &pq->pq_lists[m->m_pkthdr.pf.prio]; 671 ml_enqueue(pl, m); 672 673 return (n); 674 } 675 676 struct mbuf * 677 priq_deq_begin(struct ifqueue *ifq, void **cookiep) 678 { 679 struct priq *pq = ifq->ifq_q; 680 struct mbuf_list *pl; 681 unsigned int prio = nitems(pq->pq_lists); 682 struct mbuf *m; 683 684 do { 685 pl = &pq->pq_lists[--prio]; 686 m = MBUF_LIST_FIRST(pl); 687 if (m != NULL) { 688 *cookiep = pl; 689 return (m); 690 } 691 } while (prio > 0); 692 693 return (NULL); 694 } 695 696 void 697 priq_deq_commit(struct ifqueue *ifq, struct mbuf *m, void *cookie) 698 { 699 struct mbuf_list *pl = cookie; 700 701 KASSERT(MBUF_LIST_FIRST(pl) == m); 702 703 ml_dequeue(pl); 704 } 705 706 void 707 priq_purge(struct ifqueue *ifq, struct mbuf_list *ml) 708 { 709 struct priq *pq = ifq->ifq_q; 710 struct mbuf_list *pl; 711 unsigned int prio = nitems(pq->pq_lists); 712 713 do { 714 pl = &pq->pq_lists[--prio]; 715 ml_enlist(ml, pl); 716 } while (prio > 0); 717 } 718