1 /* 2 * Copyright (C) 2013-2016 Vincenzo Maffione 3 * Copyright (C) 2013-2016 Luigi Rizzo 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 /* 29 * This module implements netmap support on top of standard, 30 * unmodified device drivers. 31 * 32 * A NIOCREGIF request is handled here if the device does not 33 * have native support. TX and RX rings are emulated as follows: 34 * 35 * NIOCREGIF 36 * We preallocate a block of TX mbufs (roughly as many as 37 * tx descriptors; the number is not critical) to speed up 38 * operation during transmissions. The refcount on most of 39 * these buffers is artificially bumped up so we can recycle 40 * them more easily. Also, the destructor is intercepted 41 * so we use it as an interrupt notification to wake up 42 * processes blocked on a poll(). 43 * 44 * For each receive ring we allocate one "struct mbq" 45 * (an mbuf tailq plus a spinlock). We intercept packets 46 * (through if_input) 47 * on the receive path and put them in the mbq from which 48 * netmap receive routines can grab them. 49 * 50 * TX: 51 * in the generic_txsync() routine, netmap buffers are copied 52 * (or linked, in a future) to the preallocated mbufs 53 * and pushed to the transmit queue. Some of these mbufs 54 * (those with NS_REPORT, or otherwise every half ring) 55 * have the refcount=1, others have refcount=2. 56 * When the destructor is invoked, we take that as 57 * a notification that all mbufs up to that one in 58 * the specific ring have been completed, and generate 59 * the equivalent of a transmit interrupt. 60 * 61 * RX: 62 * 63 */ 64 65 #ifdef __FreeBSD__ 66 67 #include <sys/cdefs.h> /* prerequisite */ 68 __FBSDID("$FreeBSD$"); 69 70 #include <sys/types.h> 71 #include <sys/errno.h> 72 #include <sys/malloc.h> 73 #include <sys/lock.h> /* PROT_EXEC */ 74 #include <sys/rwlock.h> 75 #include <sys/socket.h> /* sockaddrs */ 76 #include <sys/selinfo.h> 77 #include <net/if.h> 78 #include <net/if_var.h> 79 #include <machine/bus.h> /* bus_dmamap_* in netmap_kern.h */ 80 81 // XXX temporary - D() defined here 82 #include <net/netmap.h> 83 #include <dev/netmap/netmap_kern.h> 84 #include <dev/netmap/netmap_mem2.h> 85 86 #define rtnl_lock() ND("rtnl_lock called") 87 #define rtnl_unlock() ND("rtnl_unlock called") 88 #define MBUF_RXQ(m) ((m)->m_pkthdr.flowid) 89 #define smp_mb() 90 91 /* 92 * FreeBSD mbuf allocator/deallocator in emulation mode: 93 */ 94 #if __FreeBSD_version < 1100000 95 96 /* 97 * For older versions of FreeBSD: 98 * 99 * We allocate EXT_PACKET mbuf+clusters, but need to set M_NOFREE 100 * so that the destructor, if invoked, will not free the packet. 101 * In principle we should set the destructor only on demand, 102 * but since there might be a race we better do it on allocation. 103 * As a consequence, we also need to set the destructor or we 104 * would leak buffers. 105 */ 106 107 /* mbuf destructor, also need to change the type to EXT_EXTREF, 108 * add an M_NOFREE flag, and then clear the flag and 109 * chain into uma_zfree(zone_pack, mf) 110 * (or reinstall the buffer ?) 111 */ 112 #define SET_MBUF_DESTRUCTOR(m, fn) do { \ 113 (m)->m_ext.ext_free = (void *)fn; \ 114 (m)->m_ext.ext_type = EXT_EXTREF; \ 115 } while (0) 116 117 static int 118 void_mbuf_dtor(struct mbuf *m, void *arg1, void *arg2) 119 { 120 /* restore original mbuf */ 121 m->m_ext.ext_buf = m->m_data = m->m_ext.ext_arg1; 122 m->m_ext.ext_arg1 = NULL; 123 m->m_ext.ext_type = EXT_PACKET; 124 m->m_ext.ext_free = NULL; 125 if (MBUF_REFCNT(m) == 0) 126 SET_MBUF_REFCNT(m, 1); 127 uma_zfree(zone_pack, m); 128 129 return 0; 130 } 131 132 static inline struct mbuf * 133 nm_os_get_mbuf(struct ifnet *ifp, int len) 134 { 135 struct mbuf *m; 136 137 (void)ifp; 138 m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR); 139 if (m) { 140 /* m_getcl() (mb_ctor_mbuf) has an assert that checks that 141 * M_NOFREE flag is not specified as third argument, 142 * so we have to set M_NOFREE after m_getcl(). */ 143 m->m_flags |= M_NOFREE; 144 m->m_ext.ext_arg1 = m->m_ext.ext_buf; // XXX save 145 m->m_ext.ext_free = (void *)void_mbuf_dtor; 146 m->m_ext.ext_type = EXT_EXTREF; 147 ND(5, "create m %p refcnt %d", m, MBUF_REFCNT(m)); 148 } 149 return m; 150 } 151 152 #else /* __FreeBSD_version >= 1100000 */ 153 154 /* 155 * Newer versions of FreeBSD, using a straightforward scheme. 156 * 157 * We allocate mbufs with m_gethdr(), since the mbuf header is needed 158 * by the driver. We also attach a customly-provided external storage, 159 * which in this case is a netmap buffer. When calling m_extadd(), however 160 * we pass a NULL address, since the real address (and length) will be 161 * filled in by nm_os_generic_xmit_frame() right before calling 162 * if_transmit(). 163 * 164 * The dtor function does nothing, however we need it since mb_free_ext() 165 * has a KASSERT(), checking that the mbuf dtor function is not NULL. 166 */ 167 168 #define SET_MBUF_DESTRUCTOR(m, fn) do { \ 169 (m)->m_ext.ext_free = (void *)fn; \ 170 } while (0) 171 172 static void void_mbuf_dtor(struct mbuf *m, void *arg1, void *arg2) { } 173 174 static inline struct mbuf * 175 nm_os_get_mbuf(struct ifnet *ifp, int len) 176 { 177 struct mbuf *m; 178 179 (void)ifp; 180 (void)len; 181 182 m = m_gethdr(M_NOWAIT, MT_DATA); 183 if (m == NULL) { 184 return m; 185 } 186 187 m_extadd(m, NULL /* buf */, 0 /* size */, void_mbuf_dtor, 188 NULL, NULL, 0, EXT_NET_DRV); 189 190 return m; 191 } 192 193 #endif /* __FreeBSD_version >= 1100000 */ 194 195 #elif defined _WIN32 196 197 #include "win_glue.h" 198 199 #define rtnl_lock() ND("rtnl_lock called") 200 #define rtnl_unlock() ND("rtnl_unlock called") 201 #define MBUF_TXQ(m) 0//((m)->m_pkthdr.flowid) 202 #define MBUF_RXQ(m) 0//((m)->m_pkthdr.flowid) 203 #define smp_mb() //XXX: to be correctly defined 204 205 #else /* linux */ 206 207 #include "bsd_glue.h" 208 209 #include <linux/rtnetlink.h> /* rtnl_[un]lock() */ 210 #include <linux/ethtool.h> /* struct ethtool_ops, get_ringparam */ 211 #include <linux/hrtimer.h> 212 213 static inline struct mbuf * 214 nm_os_get_mbuf(struct ifnet *ifp, int len) 215 { 216 return alloc_skb(ifp->needed_headroom + len + 217 ifp->needed_tailroom, GFP_ATOMIC); 218 } 219 220 #endif /* linux */ 221 222 223 /* Common headers. */ 224 #include <net/netmap.h> 225 #include <dev/netmap/netmap_kern.h> 226 #include <dev/netmap/netmap_mem2.h> 227 228 229 #define for_each_kring_n(_i, _k, _karr, _n) \ 230 for (_k=_karr, _i = 0; _i < _n; (_k)++, (_i)++) 231 232 #define for_each_tx_kring(_i, _k, _na) \ 233 for_each_kring_n(_i, _k, (_na)->tx_rings, (_na)->num_tx_rings) 234 #define for_each_tx_kring_h(_i, _k, _na) \ 235 for_each_kring_n(_i, _k, (_na)->tx_rings, (_na)->num_tx_rings + 1) 236 237 #define for_each_rx_kring(_i, _k, _na) \ 238 for_each_kring_n(_i, _k, (_na)->rx_rings, (_na)->num_rx_rings) 239 #define for_each_rx_kring_h(_i, _k, _na) \ 240 for_each_kring_n(_i, _k, (_na)->rx_rings, (_na)->num_rx_rings + 1) 241 242 243 /* ======================== PERFORMANCE STATISTICS =========================== */ 244 245 #ifdef RATE_GENERIC 246 #define IFRATE(x) x 247 struct rate_stats { 248 unsigned long txpkt; 249 unsigned long txsync; 250 unsigned long txirq; 251 unsigned long txrepl; 252 unsigned long txdrop; 253 unsigned long rxpkt; 254 unsigned long rxirq; 255 unsigned long rxsync; 256 }; 257 258 struct rate_context { 259 unsigned refcount; 260 struct timer_list timer; 261 struct rate_stats new; 262 struct rate_stats old; 263 }; 264 265 #define RATE_PRINTK(_NAME_) \ 266 printk( #_NAME_ " = %lu Hz\n", (cur._NAME_ - ctx->old._NAME_)/RATE_PERIOD); 267 #define RATE_PERIOD 2 268 static void rate_callback(unsigned long arg) 269 { 270 struct rate_context * ctx = (struct rate_context *)arg; 271 struct rate_stats cur = ctx->new; 272 int r; 273 274 RATE_PRINTK(txpkt); 275 RATE_PRINTK(txsync); 276 RATE_PRINTK(txirq); 277 RATE_PRINTK(txrepl); 278 RATE_PRINTK(txdrop); 279 RATE_PRINTK(rxpkt); 280 RATE_PRINTK(rxsync); 281 RATE_PRINTK(rxirq); 282 printk("\n"); 283 284 ctx->old = cur; 285 r = mod_timer(&ctx->timer, jiffies + 286 msecs_to_jiffies(RATE_PERIOD * 1000)); 287 if (unlikely(r)) 288 D("[v1000] Error: mod_timer()"); 289 } 290 291 static struct rate_context rate_ctx; 292 293 void generic_rate(int txp, int txs, int txi, int rxp, int rxs, int rxi) 294 { 295 if (txp) rate_ctx.new.txpkt++; 296 if (txs) rate_ctx.new.txsync++; 297 if (txi) rate_ctx.new.txirq++; 298 if (rxp) rate_ctx.new.rxpkt++; 299 if (rxs) rate_ctx.new.rxsync++; 300 if (rxi) rate_ctx.new.rxirq++; 301 } 302 303 #else /* !RATE */ 304 #define IFRATE(x) 305 #endif /* !RATE */ 306 307 308 /* =============== GENERIC NETMAP ADAPTER SUPPORT ================= */ 309 310 /* 311 * Wrapper used by the generic adapter layer to notify 312 * the poller threads. Differently from netmap_rx_irq(), we check 313 * only NAF_NETMAP_ON instead of NAF_NATIVE_ON to enable the irq. 314 */ 315 void 316 netmap_generic_irq(struct netmap_adapter *na, u_int q, u_int *work_done) 317 { 318 if (unlikely(!nm_netmap_on(na))) 319 return; 320 321 netmap_common_irq(na, q, work_done); 322 #ifdef RATE_GENERIC 323 if (work_done) 324 rate_ctx.new.rxirq++; 325 else 326 rate_ctx.new.txirq++; 327 #endif /* RATE_GENERIC */ 328 } 329 330 static int 331 generic_netmap_unregister(struct netmap_adapter *na) 332 { 333 struct netmap_generic_adapter *gna = (struct netmap_generic_adapter *)na; 334 struct netmap_kring *kring = NULL; 335 int i, r; 336 337 if (na->active_fds == 0) { 338 D("Generic adapter %p goes off", na); 339 rtnl_lock(); 340 341 na->na_flags &= ~NAF_NETMAP_ON; 342 343 /* Release packet steering control. */ 344 nm_os_catch_tx(gna, 0); 345 346 /* Stop intercepting packets on the RX path. */ 347 nm_os_catch_rx(gna, 0); 348 349 rtnl_unlock(); 350 } 351 352 for_each_rx_kring_h(r, kring, na) { 353 if (nm_kring_pending_off(kring)) { 354 D("RX ring %d of generic adapter %p goes off", r, na); 355 kring->nr_mode = NKR_NETMAP_OFF; 356 } 357 } 358 for_each_tx_kring_h(r, kring, na) { 359 if (nm_kring_pending_off(kring)) { 360 kring->nr_mode = NKR_NETMAP_OFF; 361 D("TX ring %d of generic adapter %p goes off", r, na); 362 } 363 } 364 365 for_each_rx_kring(r, kring, na) { 366 /* Free the mbufs still pending in the RX queues, 367 * that did not end up into the corresponding netmap 368 * RX rings. */ 369 mbq_safe_purge(&kring->rx_queue); 370 nm_os_mitigation_cleanup(&gna->mit[r]); 371 } 372 373 /* Decrement reference counter for the mbufs in the 374 * TX pools. These mbufs can be still pending in drivers, 375 * (e.g. this happens with virtio-net driver, which 376 * does lazy reclaiming of transmitted mbufs). */ 377 for_each_tx_kring(r, kring, na) { 378 /* We must remove the destructor on the TX event, 379 * because the destructor invokes netmap code, and 380 * the netmap module may disappear before the 381 * TX event is consumed. */ 382 mtx_lock_spin(&kring->tx_event_lock); 383 if (kring->tx_event) { 384 SET_MBUF_DESTRUCTOR(kring->tx_event, NULL); 385 } 386 kring->tx_event = NULL; 387 mtx_unlock_spin(&kring->tx_event_lock); 388 } 389 390 if (na->active_fds == 0) { 391 free(gna->mit, M_DEVBUF); 392 393 for_each_rx_kring(r, kring, na) { 394 mbq_safe_fini(&kring->rx_queue); 395 } 396 397 for_each_tx_kring(r, kring, na) { 398 mtx_destroy(&kring->tx_event_lock); 399 if (kring->tx_pool == NULL) { 400 continue; 401 } 402 403 for (i=0; i<na->num_tx_desc; i++) { 404 if (kring->tx_pool[i]) { 405 m_freem(kring->tx_pool[i]); 406 } 407 } 408 free(kring->tx_pool, M_DEVBUF); 409 kring->tx_pool = NULL; 410 } 411 412 #ifdef RATE_GENERIC 413 if (--rate_ctx.refcount == 0) { 414 D("del_timer()"); 415 del_timer(&rate_ctx.timer); 416 } 417 #endif 418 } 419 420 return 0; 421 } 422 423 /* Enable/disable netmap mode for a generic network interface. */ 424 static int 425 generic_netmap_register(struct netmap_adapter *na, int enable) 426 { 427 struct netmap_generic_adapter *gna = (struct netmap_generic_adapter *)na; 428 struct netmap_kring *kring = NULL; 429 int error; 430 int i, r; 431 432 if (!na) { 433 return EINVAL; 434 } 435 436 if (!enable) { 437 /* This is actually an unregif. */ 438 return generic_netmap_unregister(na); 439 } 440 441 if (na->active_fds == 0) { 442 D("Generic adapter %p goes on", na); 443 /* Do all memory allocations when (na->active_fds == 0), to 444 * simplify error management. */ 445 446 /* Allocate memory for mitigation support on all the rx queues. */ 447 gna->mit = malloc(na->num_rx_rings * sizeof(struct nm_generic_mit), 448 M_DEVBUF, M_NOWAIT | M_ZERO); 449 if (!gna->mit) { 450 D("mitigation allocation failed"); 451 error = ENOMEM; 452 goto out; 453 } 454 455 for_each_rx_kring(r, kring, na) { 456 /* Init mitigation support. */ 457 nm_os_mitigation_init(&gna->mit[r], r, na); 458 459 /* Initialize the rx queue, as generic_rx_handler() can 460 * be called as soon as nm_os_catch_rx() returns. 461 */ 462 mbq_safe_init(&kring->rx_queue); 463 } 464 465 /* 466 * Prepare mbuf pools (parallel to the tx rings), for packet 467 * transmission. Don't preallocate the mbufs here, it's simpler 468 * to leave this task to txsync. 469 */ 470 for_each_tx_kring(r, kring, na) { 471 kring->tx_pool = NULL; 472 } 473 for_each_tx_kring(r, kring, na) { 474 kring->tx_pool = 475 malloc(na->num_tx_desc * sizeof(struct mbuf *), 476 M_DEVBUF, M_NOWAIT | M_ZERO); 477 if (!kring->tx_pool) { 478 D("tx_pool allocation failed"); 479 error = ENOMEM; 480 goto free_tx_pools; 481 } 482 mtx_init(&kring->tx_event_lock, "tx_event_lock", 483 NULL, MTX_SPIN); 484 } 485 } 486 487 for_each_rx_kring_h(r, kring, na) { 488 if (nm_kring_pending_on(kring)) { 489 D("RX ring %d of generic adapter %p goes on", r, na); 490 kring->nr_mode = NKR_NETMAP_ON; 491 } 492 493 } 494 for_each_tx_kring_h(r, kring, na) { 495 if (nm_kring_pending_on(kring)) { 496 D("TX ring %d of generic adapter %p goes on", r, na); 497 kring->nr_mode = NKR_NETMAP_ON; 498 } 499 } 500 501 for_each_tx_kring(r, kring, na) { 502 /* Initialize tx_pool and tx_event. */ 503 for (i=0; i<na->num_tx_desc; i++) { 504 kring->tx_pool[i] = NULL; 505 } 506 507 kring->tx_event = NULL; 508 } 509 510 if (na->active_fds == 0) { 511 rtnl_lock(); 512 513 /* Prepare to intercept incoming traffic. */ 514 error = nm_os_catch_rx(gna, 1); 515 if (error) { 516 D("nm_os_catch_rx(1) failed (%d)", error); 517 goto register_handler; 518 } 519 520 /* Make netmap control the packet steering. */ 521 error = nm_os_catch_tx(gna, 1); 522 if (error) { 523 D("nm_os_catch_tx(1) failed (%d)", error); 524 goto catch_rx; 525 } 526 527 rtnl_unlock(); 528 529 na->na_flags |= NAF_NETMAP_ON; 530 531 #ifdef RATE_GENERIC 532 if (rate_ctx.refcount == 0) { 533 D("setup_timer()"); 534 memset(&rate_ctx, 0, sizeof(rate_ctx)); 535 setup_timer(&rate_ctx.timer, &rate_callback, (unsigned long)&rate_ctx); 536 if (mod_timer(&rate_ctx.timer, jiffies + msecs_to_jiffies(1500))) { 537 D("Error: mod_timer()"); 538 } 539 } 540 rate_ctx.refcount++; 541 #endif /* RATE */ 542 } 543 544 return 0; 545 546 /* Here (na->active_fds == 0) holds. */ 547 catch_rx: 548 nm_os_catch_rx(gna, 0); 549 register_handler: 550 rtnl_unlock(); 551 free_tx_pools: 552 for_each_tx_kring(r, kring, na) { 553 mtx_destroy(&kring->tx_event_lock); 554 if (kring->tx_pool == NULL) { 555 continue; 556 } 557 free(kring->tx_pool, M_DEVBUF); 558 kring->tx_pool = NULL; 559 } 560 for_each_rx_kring(r, kring, na) { 561 mbq_safe_fini(&kring->rx_queue); 562 } 563 free(gna->mit, M_DEVBUF); 564 out: 565 566 return error; 567 } 568 569 /* 570 * Callback invoked when the device driver frees an mbuf used 571 * by netmap to transmit a packet. This usually happens when 572 * the NIC notifies the driver that transmission is completed. 573 */ 574 static void 575 generic_mbuf_destructor(struct mbuf *m) 576 { 577 struct netmap_adapter *na = NA(GEN_TX_MBUF_IFP(m)); 578 struct netmap_kring *kring; 579 unsigned int r = MBUF_TXQ(m); 580 unsigned int r_orig = r; 581 582 if (unlikely(!nm_netmap_on(na) || r >= na->num_tx_rings)) { 583 D("Error: no netmap adapter on device %p", 584 GEN_TX_MBUF_IFP(m)); 585 return; 586 } 587 588 /* 589 * First, clear the event mbuf. 590 * In principle, the event 'm' should match the one stored 591 * on ring 'r'. However we check it explicitely to stay 592 * safe against lower layers (qdisc, driver, etc.) changing 593 * MBUF_TXQ(m) under our feet. If the match is not found 594 * on 'r', we try to see if it belongs to some other ring. 595 */ 596 for (;;) { 597 bool match = false; 598 599 kring = &na->tx_rings[r]; 600 mtx_lock_spin(&kring->tx_event_lock); 601 if (kring->tx_event == m) { 602 kring->tx_event = NULL; 603 match = true; 604 } 605 mtx_unlock_spin(&kring->tx_event_lock); 606 607 if (match) { 608 if (r != r_orig) { 609 RD(1, "event %p migrated: ring %u --> %u", 610 m, r_orig, r); 611 } 612 break; 613 } 614 615 if (++r == na->num_tx_rings) r = 0; 616 617 if (r == r_orig) { 618 RD(1, "Cannot match event %p", m); 619 return; 620 } 621 } 622 623 /* Second, wake up clients. They will reclaim the event through 624 * txsync. */ 625 netmap_generic_irq(na, r, NULL); 626 #ifdef __FreeBSD__ 627 void_mbuf_dtor(m, NULL, NULL); 628 #endif 629 } 630 631 extern int netmap_adaptive_io; 632 633 /* Record completed transmissions and update hwtail. 634 * 635 * The oldest tx buffer not yet completed is at nr_hwtail + 1, 636 * nr_hwcur is the first unsent buffer. 637 */ 638 static u_int 639 generic_netmap_tx_clean(struct netmap_kring *kring, int txqdisc) 640 { 641 u_int const lim = kring->nkr_num_slots - 1; 642 u_int nm_i = nm_next(kring->nr_hwtail, lim); 643 u_int hwcur = kring->nr_hwcur; 644 u_int n = 0; 645 struct mbuf **tx_pool = kring->tx_pool; 646 647 ND("hwcur = %d, hwtail = %d", kring->nr_hwcur, kring->nr_hwtail); 648 649 while (nm_i != hwcur) { /* buffers not completed */ 650 struct mbuf *m = tx_pool[nm_i]; 651 652 if (txqdisc) { 653 if (m == NULL) { 654 /* Nothing to do, this is going 655 * to be replenished. */ 656 RD(3, "Is this happening?"); 657 658 } else if (MBUF_QUEUED(m)) { 659 break; /* Not dequeued yet. */ 660 661 } else if (MBUF_REFCNT(m) != 1) { 662 /* This mbuf has been dequeued but is still busy 663 * (refcount is 2). 664 * Leave it to the driver and replenish. */ 665 m_freem(m); 666 tx_pool[nm_i] = NULL; 667 } 668 669 } else { 670 if (unlikely(m == NULL)) { 671 int event_consumed; 672 673 /* This slot was used to place an event. */ 674 mtx_lock_spin(&kring->tx_event_lock); 675 event_consumed = (kring->tx_event == NULL); 676 mtx_unlock_spin(&kring->tx_event_lock); 677 if (!event_consumed) { 678 /* The event has not been consumed yet, 679 * still busy in the driver. */ 680 break; 681 } 682 /* The event has been consumed, we can go 683 * ahead. */ 684 685 } else if (MBUF_REFCNT(m) != 1) { 686 /* This mbuf is still busy: its refcnt is 2. */ 687 break; 688 } 689 } 690 691 n++; 692 nm_i = nm_next(nm_i, lim); 693 #if 0 /* rate adaptation */ 694 if (netmap_adaptive_io > 1) { 695 if (n >= netmap_adaptive_io) 696 break; 697 } else if (netmap_adaptive_io) { 698 /* if hwcur - nm_i < lim/8 do an early break 699 * so we prevent the sender from stalling. See CVT. 700 */ 701 if (hwcur >= nm_i) { 702 if (hwcur - nm_i < lim/2) 703 break; 704 } else { 705 if (hwcur + lim + 1 - nm_i < lim/2) 706 break; 707 } 708 } 709 #endif 710 } 711 kring->nr_hwtail = nm_prev(nm_i, lim); 712 ND("tx completed [%d] -> hwtail %d", n, kring->nr_hwtail); 713 714 return n; 715 } 716 717 /* Compute a slot index in the middle between inf and sup. */ 718 static inline u_int 719 ring_middle(u_int inf, u_int sup, u_int lim) 720 { 721 u_int n = lim + 1; 722 u_int e; 723 724 if (sup >= inf) { 725 e = (sup + inf) / 2; 726 } else { /* wrap around */ 727 e = (sup + n + inf) / 2; 728 if (e >= n) { 729 e -= n; 730 } 731 } 732 733 if (unlikely(e >= n)) { 734 D("This cannot happen"); 735 e = 0; 736 } 737 738 return e; 739 } 740 741 static void 742 generic_set_tx_event(struct netmap_kring *kring, u_int hwcur) 743 { 744 u_int lim = kring->nkr_num_slots - 1; 745 struct mbuf *m; 746 u_int e; 747 u_int ntc = nm_next(kring->nr_hwtail, lim); /* next to clean */ 748 749 if (ntc == hwcur) { 750 return; /* all buffers are free */ 751 } 752 753 /* 754 * We have pending packets in the driver between hwtail+1 755 * and hwcur, and we have to chose one of these slot to 756 * generate a notification. 757 * There is a race but this is only called within txsync which 758 * does a double check. 759 */ 760 #if 0 761 /* Choose a slot in the middle, so that we don't risk ending 762 * up in a situation where the client continuously wake up, 763 * fills one or a few TX slots and go to sleep again. */ 764 e = ring_middle(ntc, hwcur, lim); 765 #else 766 /* Choose the first pending slot, to be safe against driver 767 * reordering mbuf transmissions. */ 768 e = ntc; 769 #endif 770 771 m = kring->tx_pool[e]; 772 if (m == NULL) { 773 /* An event is already in place. */ 774 return; 775 } 776 777 mtx_lock_spin(&kring->tx_event_lock); 778 if (kring->tx_event) { 779 /* An event is already in place. */ 780 mtx_unlock_spin(&kring->tx_event_lock); 781 return; 782 } 783 784 SET_MBUF_DESTRUCTOR(m, generic_mbuf_destructor); 785 kring->tx_event = m; 786 mtx_unlock_spin(&kring->tx_event_lock); 787 788 kring->tx_pool[e] = NULL; 789 790 ND(5, "Request Event at %d mbuf %p refcnt %d", e, m, m ? MBUF_REFCNT(m) : -2 ); 791 792 /* Decrement the refcount. This will free it if we lose the race 793 * with the driver. */ 794 m_freem(m); 795 smp_mb(); 796 } 797 798 799 /* 800 * generic_netmap_txsync() transforms netmap buffers into mbufs 801 * and passes them to the standard device driver 802 * (ndo_start_xmit() or ifp->if_transmit() ). 803 * On linux this is not done directly, but using dev_queue_xmit(), 804 * since it implements the TX flow control (and takes some locks). 805 */ 806 static int 807 generic_netmap_txsync(struct netmap_kring *kring, int flags) 808 { 809 struct netmap_adapter *na = kring->na; 810 struct netmap_generic_adapter *gna = (struct netmap_generic_adapter *)na; 811 struct ifnet *ifp = na->ifp; 812 struct netmap_ring *ring = kring->ring; 813 u_int nm_i; /* index into the netmap ring */ // j 814 u_int const lim = kring->nkr_num_slots - 1; 815 u_int const head = kring->rhead; 816 u_int ring_nr = kring->ring_id; 817 818 IFRATE(rate_ctx.new.txsync++); 819 820 rmb(); 821 822 /* 823 * First part: process new packets to send. 824 */ 825 nm_i = kring->nr_hwcur; 826 if (nm_i != head) { /* we have new packets to send */ 827 struct nm_os_gen_arg a; 828 u_int event = -1; 829 830 if (gna->txqdisc && nm_kr_txempty(kring)) { 831 /* In txqdisc mode, we ask for a delayed notification, 832 * but only when cur == hwtail, which means that the 833 * client is going to block. */ 834 event = ring_middle(nm_i, head, lim); 835 ND(3, "Place txqdisc event (hwcur=%u,event=%u," 836 "head=%u,hwtail=%u)", nm_i, event, head, 837 kring->nr_hwtail); 838 } 839 840 a.ifp = ifp; 841 a.ring_nr = ring_nr; 842 a.head = a.tail = NULL; 843 844 while (nm_i != head) { 845 struct netmap_slot *slot = &ring->slot[nm_i]; 846 u_int len = slot->len; 847 void *addr = NMB(na, slot); 848 /* device-specific */ 849 struct mbuf *m; 850 int tx_ret; 851 852 NM_CHECK_ADDR_LEN(na, addr, len); 853 854 /* Tale a mbuf from the tx pool (replenishing the pool 855 * entry if necessary) and copy in the user packet. */ 856 m = kring->tx_pool[nm_i]; 857 if (unlikely(m == NULL)) { 858 kring->tx_pool[nm_i] = m = 859 nm_os_get_mbuf(ifp, NETMAP_BUF_SIZE(na)); 860 if (m == NULL) { 861 RD(2, "Failed to replenish mbuf"); 862 /* Here we could schedule a timer which 863 * retries to replenish after a while, 864 * and notifies the client when it 865 * manages to replenish some slots. In 866 * any case we break early to avoid 867 * crashes. */ 868 break; 869 } 870 IFRATE(rate_ctx.new.txrepl++); 871 } 872 873 a.m = m; 874 a.addr = addr; 875 a.len = len; 876 a.qevent = (nm_i == event); 877 /* When not in txqdisc mode, we should ask 878 * notifications when NS_REPORT is set, or roughly 879 * every half ring. To optimize this, we set a 880 * notification event when the client runs out of 881 * TX ring space, or when transmission fails. In 882 * the latter case we also break early. 883 */ 884 tx_ret = nm_os_generic_xmit_frame(&a); 885 if (unlikely(tx_ret)) { 886 if (!gna->txqdisc) { 887 /* 888 * No room for this mbuf in the device driver. 889 * Request a notification FOR A PREVIOUS MBUF, 890 * then call generic_netmap_tx_clean(kring) to do the 891 * double check and see if we can free more buffers. 892 * If there is space continue, else break; 893 * NOTE: the double check is necessary if the problem 894 * occurs in the txsync call after selrecord(). 895 * Also, we need some way to tell the caller that not 896 * all buffers were queued onto the device (this was 897 * not a problem with native netmap driver where space 898 * is preallocated). The bridge has a similar problem 899 * and we solve it there by dropping the excess packets. 900 */ 901 generic_set_tx_event(kring, nm_i); 902 if (generic_netmap_tx_clean(kring, gna->txqdisc)) { 903 /* space now available */ 904 continue; 905 } else { 906 break; 907 } 908 } 909 910 /* In txqdisc mode, the netmap-aware qdisc 911 * queue has the same length as the number of 912 * netmap slots (N). Since tail is advanced 913 * only when packets are dequeued, qdisc 914 * queue overrun cannot happen, so 915 * nm_os_generic_xmit_frame() did not fail 916 * because of that. 917 * However, packets can be dropped because 918 * carrier is off, or because our qdisc is 919 * being deactivated, or possibly for other 920 * reasons. In these cases, we just let the 921 * packet to be dropped. */ 922 IFRATE(rate_ctx.new.txdrop++); 923 } 924 925 slot->flags &= ~(NS_REPORT | NS_BUF_CHANGED); 926 nm_i = nm_next(nm_i, lim); 927 IFRATE(rate_ctx.new.txpkt++); 928 } 929 if (a.head != NULL) { 930 a.addr = NULL; 931 nm_os_generic_xmit_frame(&a); 932 } 933 /* Update hwcur to the next slot to transmit. Here nm_i 934 * is not necessarily head, we could break early. */ 935 kring->nr_hwcur = nm_i; 936 } 937 938 /* 939 * Second, reclaim completed buffers 940 */ 941 if (!gna->txqdisc && (flags & NAF_FORCE_RECLAIM || nm_kr_txempty(kring))) { 942 /* No more available slots? Set a notification event 943 * on a netmap slot that will be cleaned in the future. 944 * No doublecheck is performed, since txsync() will be 945 * called twice by netmap_poll(). 946 */ 947 generic_set_tx_event(kring, nm_i); 948 } 949 950 generic_netmap_tx_clean(kring, gna->txqdisc); 951 952 return 0; 953 } 954 955 956 /* 957 * This handler is registered (through nm_os_catch_rx()) 958 * within the attached network interface 959 * in the RX subsystem, so that every mbuf passed up by 960 * the driver can be stolen to the network stack. 961 * Stolen packets are put in a queue where the 962 * generic_netmap_rxsync() callback can extract them. 963 * Returns 1 if the packet was stolen, 0 otherwise. 964 */ 965 int 966 generic_rx_handler(struct ifnet *ifp, struct mbuf *m) 967 { 968 struct netmap_adapter *na = NA(ifp); 969 struct netmap_generic_adapter *gna = (struct netmap_generic_adapter *)na; 970 struct netmap_kring *kring; 971 u_int work_done; 972 u_int r = MBUF_RXQ(m); /* receive ring number */ 973 974 if (r >= na->num_rx_rings) { 975 r = r % na->num_rx_rings; 976 } 977 978 kring = &na->rx_rings[r]; 979 980 if (kring->nr_mode == NKR_NETMAP_OFF) { 981 /* We must not intercept this mbuf. */ 982 return 0; 983 } 984 985 /* limit the size of the queue */ 986 if (unlikely(!gna->rxsg && MBUF_LEN(m) > NETMAP_BUF_SIZE(na))) { 987 /* This may happen when GRO/LRO features are enabled for 988 * the NIC driver when the generic adapter does not 989 * support RX scatter-gather. */ 990 RD(2, "Warning: driver pushed up big packet " 991 "(size=%d)", (int)MBUF_LEN(m)); 992 m_freem(m); 993 } else if (unlikely(mbq_len(&kring->rx_queue) > 1024)) { 994 m_freem(m); 995 } else { 996 mbq_safe_enqueue(&kring->rx_queue, m); 997 } 998 999 if (netmap_generic_mit < 32768) { 1000 /* no rx mitigation, pass notification up */ 1001 netmap_generic_irq(na, r, &work_done); 1002 } else { 1003 /* same as send combining, filter notification if there is a 1004 * pending timer, otherwise pass it up and start a timer. 1005 */ 1006 if (likely(nm_os_mitigation_active(&gna->mit[r]))) { 1007 /* Record that there is some pending work. */ 1008 gna->mit[r].mit_pending = 1; 1009 } else { 1010 netmap_generic_irq(na, r, &work_done); 1011 nm_os_mitigation_start(&gna->mit[r]); 1012 } 1013 } 1014 1015 /* We have intercepted the mbuf. */ 1016 return 1; 1017 } 1018 1019 /* 1020 * generic_netmap_rxsync() extracts mbufs from the queue filled by 1021 * generic_netmap_rx_handler() and puts their content in the netmap 1022 * receive ring. 1023 * Access must be protected because the rx handler is asynchronous, 1024 */ 1025 static int 1026 generic_netmap_rxsync(struct netmap_kring *kring, int flags) 1027 { 1028 struct netmap_ring *ring = kring->ring; 1029 struct netmap_adapter *na = kring->na; 1030 u_int nm_i; /* index into the netmap ring */ //j, 1031 u_int n; 1032 u_int const lim = kring->nkr_num_slots - 1; 1033 u_int const head = kring->rhead; 1034 int force_update = (flags & NAF_FORCE_READ) || kring->nr_kflags & NKR_PENDINTR; 1035 1036 /* Adapter-specific variables. */ 1037 uint16_t slot_flags = kring->nkr_slot_flags; 1038 u_int nm_buf_len = NETMAP_BUF_SIZE(na); 1039 struct mbq tmpq; 1040 struct mbuf *m; 1041 int avail; /* in bytes */ 1042 int mlen; 1043 int copy; 1044 1045 if (head > lim) 1046 return netmap_ring_reinit(kring); 1047 1048 IFRATE(rate_ctx.new.rxsync++); 1049 1050 /* 1051 * First part: skip past packets that userspace has released. 1052 * This can possibly make room for the second part. 1053 */ 1054 nm_i = kring->nr_hwcur; 1055 if (nm_i != head) { 1056 /* Userspace has released some packets. */ 1057 for (n = 0; nm_i != head; n++) { 1058 struct netmap_slot *slot = &ring->slot[nm_i]; 1059 1060 slot->flags &= ~NS_BUF_CHANGED; 1061 nm_i = nm_next(nm_i, lim); 1062 } 1063 kring->nr_hwcur = head; 1064 } 1065 1066 /* 1067 * Second part: import newly received packets. 1068 */ 1069 if (!netmap_no_pendintr && !force_update) { 1070 return 0; 1071 } 1072 1073 nm_i = kring->nr_hwtail; /* First empty slot in the receive ring. */ 1074 1075 /* Compute the available space (in bytes) in this netmap ring. 1076 * The first slot that is not considered in is the one before 1077 * nr_hwcur. */ 1078 1079 avail = nm_prev(kring->nr_hwcur, lim) - nm_i; 1080 if (avail < 0) 1081 avail += lim + 1; 1082 avail *= nm_buf_len; 1083 1084 /* First pass: While holding the lock on the RX mbuf queue, 1085 * extract as many mbufs as they fit the available space, 1086 * and put them in a temporary queue. 1087 * To avoid performing a per-mbuf division (mlen / nm_buf_len) to 1088 * to update avail, we do the update in a while loop that we 1089 * also use to set the RX slots, but without performing the copy. */ 1090 mbq_init(&tmpq); 1091 mbq_lock(&kring->rx_queue); 1092 for (n = 0;; n++) { 1093 m = mbq_peek(&kring->rx_queue); 1094 if (!m) { 1095 /* No more packets from the driver. */ 1096 break; 1097 } 1098 1099 mlen = MBUF_LEN(m); 1100 if (mlen > avail) { 1101 /* No more space in the ring. */ 1102 break; 1103 } 1104 1105 mbq_dequeue(&kring->rx_queue); 1106 1107 while (mlen) { 1108 copy = nm_buf_len; 1109 if (mlen < copy) { 1110 copy = mlen; 1111 } 1112 mlen -= copy; 1113 avail -= nm_buf_len; 1114 1115 ring->slot[nm_i].len = copy; 1116 ring->slot[nm_i].flags = slot_flags | (mlen ? NS_MOREFRAG : 0); 1117 nm_i = nm_next(nm_i, lim); 1118 } 1119 1120 mbq_enqueue(&tmpq, m); 1121 } 1122 mbq_unlock(&kring->rx_queue); 1123 1124 /* Second pass: Drain the temporary queue, going over the used RX slots, 1125 * and perform the copy out of the RX queue lock. */ 1126 nm_i = kring->nr_hwtail; 1127 1128 for (;;) { 1129 void *nmaddr; 1130 int ofs = 0; 1131 int morefrag; 1132 1133 m = mbq_dequeue(&tmpq); 1134 if (!m) { 1135 break; 1136 } 1137 1138 do { 1139 nmaddr = NMB(na, &ring->slot[nm_i]); 1140 /* We only check the address here on generic rx rings. */ 1141 if (nmaddr == NETMAP_BUF_BASE(na)) { /* Bad buffer */ 1142 m_freem(m); 1143 mbq_purge(&tmpq); 1144 mbq_fini(&tmpq); 1145 return netmap_ring_reinit(kring); 1146 } 1147 1148 copy = ring->slot[nm_i].len; 1149 m_copydata(m, ofs, copy, nmaddr); 1150 ofs += copy; 1151 morefrag = ring->slot[nm_i].flags & NS_MOREFRAG; 1152 nm_i = nm_next(nm_i, lim); 1153 } while (morefrag); 1154 1155 m_freem(m); 1156 } 1157 1158 mbq_fini(&tmpq); 1159 1160 if (n) { 1161 kring->nr_hwtail = nm_i; 1162 IFRATE(rate_ctx.new.rxpkt += n); 1163 } 1164 kring->nr_kflags &= ~NKR_PENDINTR; 1165 1166 return 0; 1167 } 1168 1169 static void 1170 generic_netmap_dtor(struct netmap_adapter *na) 1171 { 1172 struct netmap_generic_adapter *gna = (struct netmap_generic_adapter*)na; 1173 struct ifnet *ifp = netmap_generic_getifp(gna); 1174 struct netmap_adapter *prev_na = gna->prev; 1175 1176 if (prev_na != NULL) { 1177 D("Released generic NA %p", gna); 1178 netmap_adapter_put(prev_na); 1179 if (nm_iszombie(na)) { 1180 /* 1181 * The driver has been removed without releasing 1182 * the reference so we need to do it here. 1183 */ 1184 netmap_adapter_put(prev_na); 1185 } 1186 } 1187 NM_ATTACH_NA(ifp, prev_na); 1188 /* 1189 * netmap_detach_common(), that it's called after this function, 1190 * overrides WNA(ifp) if na->ifp is not NULL. 1191 */ 1192 na->ifp = NULL; 1193 D("Restored native NA %p", prev_na); 1194 } 1195 1196 /* 1197 * generic_netmap_attach() makes it possible to use netmap on 1198 * a device without native netmap support. 1199 * This is less performant than native support but potentially 1200 * faster than raw sockets or similar schemes. 1201 * 1202 * In this "emulated" mode, netmap rings do not necessarily 1203 * have the same size as those in the NIC. We use a default 1204 * value and possibly override it if the OS has ways to fetch the 1205 * actual configuration. 1206 */ 1207 int 1208 generic_netmap_attach(struct ifnet *ifp) 1209 { 1210 struct netmap_adapter *na; 1211 struct netmap_generic_adapter *gna; 1212 int retval; 1213 u_int num_tx_desc, num_rx_desc; 1214 1215 num_tx_desc = num_rx_desc = netmap_generic_ringsize; /* starting point */ 1216 1217 nm_os_generic_find_num_desc(ifp, &num_tx_desc, &num_rx_desc); /* ignore errors */ 1218 ND("Netmap ring size: TX = %d, RX = %d", num_tx_desc, num_rx_desc); 1219 if (num_tx_desc == 0 || num_rx_desc == 0) { 1220 D("Device has no hw slots (tx %u, rx %u)", num_tx_desc, num_rx_desc); 1221 return EINVAL; 1222 } 1223 1224 gna = malloc(sizeof(*gna), M_DEVBUF, M_NOWAIT | M_ZERO); 1225 if (gna == NULL) { 1226 D("no memory on attach, give up"); 1227 return ENOMEM; 1228 } 1229 na = (struct netmap_adapter *)gna; 1230 strncpy(na->name, ifp->if_xname, sizeof(na->name)); 1231 na->ifp = ifp; 1232 na->num_tx_desc = num_tx_desc; 1233 na->num_rx_desc = num_rx_desc; 1234 na->nm_register = &generic_netmap_register; 1235 na->nm_txsync = &generic_netmap_txsync; 1236 na->nm_rxsync = &generic_netmap_rxsync; 1237 na->nm_dtor = &generic_netmap_dtor; 1238 /* when using generic, NAF_NETMAP_ON is set so we force 1239 * NAF_SKIP_INTR to use the regular interrupt handler 1240 */ 1241 na->na_flags = NAF_SKIP_INTR | NAF_HOST_RINGS; 1242 1243 ND("[GNA] num_tx_queues(%d), real_num_tx_queues(%d), len(%lu)", 1244 ifp->num_tx_queues, ifp->real_num_tx_queues, 1245 ifp->tx_queue_len); 1246 ND("[GNA] num_rx_queues(%d), real_num_rx_queues(%d)", 1247 ifp->num_rx_queues, ifp->real_num_rx_queues); 1248 1249 nm_os_generic_find_num_queues(ifp, &na->num_tx_rings, &na->num_rx_rings); 1250 1251 retval = netmap_attach_common(na); 1252 if (retval) { 1253 free(gna, M_DEVBUF); 1254 return retval; 1255 } 1256 1257 gna->prev = NA(ifp); /* save old na */ 1258 if (gna->prev != NULL) { 1259 netmap_adapter_get(gna->prev); 1260 } 1261 NM_ATTACH_NA(ifp, na); 1262 1263 nm_os_generic_set_features(gna); 1264 1265 D("Created generic NA %p (prev %p)", gna, gna->prev); 1266 1267 return retval; 1268 } 1269