Lines Matching defs:txq
439 #define TXQ_AVAIL(txq) (txq->ift_size - get_inuse(txq->ift_size, txq->ift_cidx, txq->ift_pidx, txq->ift_gen))
550 #define CALLOUT_LOCK(txq) mtx_lock(&txq->ift_mtx)
551 #define CALLOUT_UNLOCK(txq) mtx_unlock(&txq->ift_mtx)
644 &iflib_encap_txq_avail_fail, 0, "# txq avail failures");
701 static int iflib_tx_credits_update(if_ctx_t ctx, iflib_txq_t txq);
706 static void iflib_txq_check_drain(iflib_txq_t txq, int budget);
719 static void iflib_ifmp_purge(iflib_txq_t txq);
1008 iflib_txq_t txq = &ctx->ifc_txqs[kring->ring_id];
1010 bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map,
1040 pi.ipi_segs = txq->ift_segs;
1045 __builtin_prefetch(&txq->ift_sds.ifsd_m[nic_i]);
1046 __builtin_prefetch(&txq->ift_sds.ifsd_map[nic_i]);
1095 __builtin_prefetch(&txq->ift_sds.ifsd_m[nic_i + 1]);
1096 __builtin_prefetch(&txq->ift_sds.ifsd_map[nic_i + 1]);
1102 netmap_reload_map(na, txq->ift_buf_tag,
1103 txq->ift_sds.ifsd_map[nic_i], addr);
1106 bus_dmamap_sync(txq->ift_buf_tag,
1107 txq->ift_sds.ifsd_map[nic_i],
1117 bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map,
1121 ctx->isc_txd_flush(ctx->ifc_softc, txq->ift_id, nic_i);
1132 if (iflib_tx_credits_update(ctx, txq)) {
1134 nic_i = txq->ift_cidx_processed;
1141 callout_reset_sbt_on(&txq->ift_netmap_timer,
1143 iflib_netmap_timer, txq,
1144 txq->ift_netmap_timer.c_cpu, 0);
1334 iflib_netmap_txq_init(if_ctx_t ctx, iflib_txq_t txq)
1339 slot = netmap_reset(na, NR_TX, txq->ift_id, 0);
1350 int si = netmap_idx_n2k(na->tx_rings[txq->ift_id], i);
1351 netmap_load_map(na, txq->ift_buf_tag, txq->ift_sds.ifsd_map[i],
1375 iflib_txq_t txq = arg;
1376 if_ctx_t ctx = txq->ift_ctx;
1382 netmap_tx_irq(ctx->ifc_ifp, txq->ift_id);
1388 #define iflib_netmap_txq_init(ctx, txq) (0)
1590 iflib_txq_t txq;
1610 txq = &ctx->ifc_txqs[txqid];
1611 bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map,
1620 GROUPTASK_ENQUEUE(&txq->ift_task);
1705 iflib_txsd_alloc(iflib_txq_t txq)
1707 if_ctx_t ctx = txq->ift_ctx;
1722 MPASS(scctx->isc_ntxd[txq->ift_br_offset] > 0);
1745 &txq->ift_buf_tag))) {
1763 &txq->ift_tso_buf_tag))) {
1770 if (!(txq->ift_sds.ifsd_m =
1772 scctx->isc_ntxd[txq->ift_br_offset], M_IFLIB, M_NOWAIT | M_ZERO))) {
1781 if ((txq->ift_sds.ifsd_map = (bus_dmamap_t *)malloc(
1782 sizeof(bus_dmamap_t) * scctx->isc_ntxd[txq->ift_br_offset],
1789 if (tso && (txq->ift_sds.ifsd_tso_map = (bus_dmamap_t *)malloc(
1790 sizeof(bus_dmamap_t) * scctx->isc_ntxd[txq->ift_br_offset],
1797 for (int i = 0; i < scctx->isc_ntxd[txq->ift_br_offset]; i++) {
1798 err = bus_dmamap_create(txq->ift_buf_tag, 0,
1799 &txq->ift_sds.ifsd_map[i]);
1806 err = bus_dmamap_create(txq->ift_tso_buf_tag, 0,
1807 &txq->ift_sds.ifsd_tso_map[i]);
1821 iflib_txsd_destroy(if_ctx_t ctx, iflib_txq_t txq, int i)
1825 if (txq->ift_sds.ifsd_map != NULL) {
1826 map = txq->ift_sds.ifsd_map[i];
1827 bus_dmamap_sync(txq->ift_buf_tag, map, BUS_DMASYNC_POSTWRITE);
1828 bus_dmamap_unload(txq->ift_buf_tag, map);
1829 bus_dmamap_destroy(txq->ift_buf_tag, map);
1830 txq->ift_sds.ifsd_map[i] = NULL;
1833 if (txq->ift_sds.ifsd_tso_map != NULL) {
1834 map = txq->ift_sds.ifsd_tso_map[i];
1835 bus_dmamap_sync(txq->ift_tso_buf_tag, map,
1837 bus_dmamap_unload(txq->ift_tso_buf_tag, map);
1838 bus_dmamap_destroy(txq->ift_tso_buf_tag, map);
1839 txq->ift_sds.ifsd_tso_map[i] = NULL;
1844 iflib_txq_destroy(iflib_txq_t txq)
1846 if_ctx_t ctx = txq->ift_ctx;
1848 for (int i = 0; i < txq->ift_size; i++)
1849 iflib_txsd_destroy(ctx, txq, i);
1851 if (txq->ift_br != NULL) {
1852 ifmp_ring_free(txq->ift_br);
1853 txq->ift_br = NULL;
1856 mtx_destroy(&txq->ift_mtx);
1858 if (txq->ift_sds.ifsd_map != NULL) {
1859 free(txq->ift_sds.ifsd_map, M_IFLIB);
1860 txq->ift_sds.ifsd_map = NULL;
1862 if (txq->ift_sds.ifsd_tso_map != NULL) {
1863 free(txq->ift_sds.ifsd_tso_map, M_IFLIB);
1864 txq->ift_sds.ifsd_tso_map = NULL;
1866 if (txq->ift_sds.ifsd_m != NULL) {
1867 free(txq->ift_sds.ifsd_m, M_IFLIB);
1868 txq->ift_sds.ifsd_m = NULL;
1870 if (txq->ift_buf_tag != NULL) {
1871 bus_dma_tag_destroy(txq->ift_buf_tag);
1872 txq->ift_buf_tag = NULL;
1874 if (txq->ift_tso_buf_tag != NULL) {
1875 bus_dma_tag_destroy(txq->ift_tso_buf_tag);
1876 txq->ift_tso_buf_tag = NULL;
1878 if (txq->ift_ifdi != NULL) {
1879 free(txq->ift_ifdi, M_IFLIB);
1884 iflib_txsd_free(if_ctx_t ctx, iflib_txq_t txq, int i)
1888 mp = &txq->ift_sds.ifsd_m[i];
1892 if (txq->ift_sds.ifsd_map != NULL) {
1893 bus_dmamap_sync(txq->ift_buf_tag,
1894 txq->ift_sds.ifsd_map[i], BUS_DMASYNC_POSTWRITE);
1895 bus_dmamap_unload(txq->ift_buf_tag, txq->ift_sds.ifsd_map[i]);
1897 if (txq->ift_sds.ifsd_tso_map != NULL) {
1898 bus_dmamap_sync(txq->ift_tso_buf_tag,
1899 txq->ift_sds.ifsd_tso_map[i], BUS_DMASYNC_POSTWRITE);
1900 bus_dmamap_unload(txq->ift_tso_buf_tag,
1901 txq->ift_sds.ifsd_tso_map[i]);
1909 iflib_txq_setup(iflib_txq_t txq)
1911 if_ctx_t ctx = txq->ift_ctx;
1918 txq->ift_qstatus = IFLIB_QUEUE_IDLE;
1920 txq->ift_update_freq = IFLIB_DEFAULT_TX_UPDATE_FREQ;
1923 txq->ift_cidx_processed = 0;
1924 txq->ift_pidx = txq->ift_cidx = txq->ift_npending = 0;
1925 txq->ift_size = scctx->isc_ntxd[txq->ift_br_offset];
1927 for (i = 0, di = txq->ift_ifdi; i < sctx->isc_ntxqs; i++, di++)
1930 IFDI_TXQ_SETUP(ctx, txq->ift_id);
1931 for (i = 0, di = txq->ift_ifdi; i < sctx->isc_ntxqs; i++, di++)
2404 iflib_txq_t txq = arg;
2405 if_ctx_t ctx = txq->ift_ctx;
2417 if (this_tick - txq->ift_last_timer_tick >= iflib_timer_default) {
2418 txq->ift_last_timer_tick = this_tick;
2419 IFDI_TIMER(ctx, txq->ift_id);
2420 if ((txq->ift_qstatus == IFLIB_QUEUE_HUNG) &&
2421 ((txq->ift_cleaned_prev == txq->ift_cleaned) ||
2425 if (txq->ift_qstatus != IFLIB_QUEUE_IDLE &&
2426 ifmp_ring_is_stalled(txq->ift_br)) {
2429 txq->ift_qstatus = IFLIB_QUEUE_HUNG;
2431 txq->ift_cleaned_prev = txq->ift_cleaned;
2434 if (txq->ift_db_pending)
2435 GROUPTASK_ENQUEUE(&txq->ift_task);
2439 callout_reset_on(&txq->ift_timer, iflib_timer_default, iflib_timer,
2440 txq, txq->ift_timer.c_cpu);
2446 txq->ift_id, TXQ_AVAIL(txq), txq->ift_pidx);
2490 iflib_txq_t txq;
2516 for (i = 0, txq = ctx->ifc_txqs; i < scctx->isc_ntxqsets; i++, txq++) {
2517 CALLOUT_LOCK(txq);
2518 callout_stop(&txq->ift_timer);
2520 callout_stop(&txq->ift_netmap_timer);
2522 CALLOUT_UNLOCK(txq);
2523 (void)iflib_netmap_txq_init(ctx, txq);
2555 txq = ctx->ifc_txqs;
2556 for (i = 0; i < scctx->isc_ntxqsets; i++, txq++)
2557 callout_reset_on(&txq->ift_timer, iflib_timer_default, iflib_timer, txq,
2558 txq->ift_timer.c_cpu);
2591 iflib_txq_t txq = ctx->ifc_txqs;
2616 for (i = 0; i < scctx->isc_ntxqsets; i++, txq++) {
2619 CALLOUT_LOCK(txq);
2620 callout_stop(&txq->ift_timer);
2622 callout_stop(&txq->ift_netmap_timer);
2624 CALLOUT_UNLOCK(txq);
2627 iflib_ifmp_purge(txq);
2629 for (j = 0; j < txq->ift_size; j++) {
2630 iflib_txsd_free(ctx, txq, j);
2632 txq->ift_processed = txq->ift_cleaned = txq->ift_cidx_processed = 0;
2633 txq->ift_in_use = txq->ift_gen = txq->ift_no_desc_avail = 0;
2635 txq->ift_cidx = txq->ift_pidx;
2637 txq->ift_cidx = txq->ift_pidx = 0;
2639 txq->ift_closed = txq->ift_mbuf_defrag = txq->ift_mbuf_defrag_failed = 0;
2640 txq->ift_no_tx_dma_setup = txq->ift_txd_encap_efbig = txq->ift_map_failed = 0;
2641 txq->ift_pullups = 0;
2642 ifmp_ring_reset_stats(txq->ift_br);
2643 for (j = 0, di = txq->ift_ifdi; j < sctx->isc_ntxqs; j++, di++)
3108 #define TXD_NOTIFY_COUNT(txq) (((txq)->ift_size / (txq)->ift_update_freq) - 1)
3110 txq_max_db_deferred(iflib_txq_t txq, qidx_t in_use)
3112 qidx_t notify_count = TXD_NOTIFY_COUNT(txq);
3113 qidx_t minthresh = txq->ift_size / 8;
3124 txq_max_rs_deferred(iflib_txq_t txq)
3126 qidx_t notify_count = TXD_NOTIFY_COUNT(txq);
3127 qidx_t minthresh = txq->ift_size / 8;
3128 if (txq->ift_in_use > 4 * minthresh)
3130 if (txq->ift_in_use > 2 * minthresh)
3132 if (txq->ift_in_use > minthresh)
3140 #define TXQ_MAX_DB_DEFERRED(txq, in_use) txq_max_db_deferred((txq), (in_use))
3141 #define TXQ_MAX_RS_DEFERRED(txq) txq_max_rs_deferred(txq)
3157 iflib_txd_db_check(iflib_txq_t txq, int ring)
3159 if_ctx_t ctx = txq->ift_ctx;
3162 max = TXQ_MAX_DB_DEFERRED(txq, txq->ift_in_use);
3165 if (ring || (txq->ift_db_pending >= max) || (TXQ_AVAIL(txq) <= MAX_TX_DESC(ctx) + 2)) {
3172 dbval = txq->ift_npending ? txq->ift_npending : txq->ift_pidx;
3173 bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map,
3175 ctx->isc_txd_flush(ctx->ifc_softc, txq->ift_id, dbval);
3180 txq->ift_db_pending = txq->ift_npending = 0;
3347 iflib_parse_header(iflib_txq_t txq, if_pkt_info_t pi, struct mbuf **mp)
3349 if_shared_ctx_t sctx = txq->ift_ctx->ifc_sctx;
3366 err = iflib_parse_ether_header(pi, mp, &txq->ift_pullups);
3394 txq->ift_pullups++;
3400 txq->ift_pullups++;
3421 txq->ift_pullups++;
3460 txq->ift_pullups++;
3475 txq->ift_pullups++;
3513 iflib_remove_mbuf(iflib_txq_t txq)
3518 ifsd_m = txq->ift_sds.ifsd_m;
3519 ntxd = txq->ift_size;
3520 pidx = txq->ift_pidx & (ntxd - 1);
3521 ifsd_m = txq->ift_sds.ifsd_m;
3524 bus_dmamap_unload(txq->ift_buf_tag, txq->ift_sds.ifsd_map[pidx]);
3525 if (txq->ift_sds.ifsd_tso_map != NULL)
3526 bus_dmamap_unload(txq->ift_tso_buf_tag,
3527 txq->ift_sds.ifsd_tso_map[pidx]);
3529 txq->ift_dequeued++;
3535 calc_next_txd(iflib_txq_t txq, int cidx, uint8_t qid)
3541 ntxd = txq->ift_size;
3542 size = txq->ift_txd_size[qid];
3543 start = txq->ift_ifdi[qid].idi_vaddr;
3598 iflib_encap(iflib_txq_t txq, struct mbuf **m_headp)
3612 ctx = txq->ift_ctx;
3615 segs = txq->ift_segs;
3616 ntxd = txq->ift_size;
3623 cidx = txq->ift_cidx;
3624 pidx = txq->ift_pidx;
3628 next_txd = calc_next_txd(txq, cidx, 0);
3633 prefetch(&txq->ift_sds.ifsd_m[next]);
3634 prefetch(&txq->ift_sds.ifsd_map[next]);
3637 map = txq->ift_sds.ifsd_map[pidx];
3638 ifsd_m = txq->ift_sds.ifsd_m;
3641 buf_tag = txq->ift_tso_buf_tag;
3643 map = txq->ift_sds.ifsd_tso_map[pidx];
3647 buf_tag = txq->ift_buf_tag;
3649 map = txq->ift_sds.ifsd_map[pidx];
3664 pi.ipi_qsidx = txq->ift_id;
3671 if (__predict_false((err = iflib_parse_header(txq, &pi, m_headp)) != 0)) {
3693 txq->ift_mbuf_defrag++;
3708 txq->ift_no_tx_dma_setup++;
3711 txq->ift_no_tx_dma_setup++;
3717 txq->ift_map_failed++;
3728 if (__predict_false(nsegs + 2 > TXQ_AVAIL(txq))) {
3729 txq->ift_no_desc_avail++;
3733 if ((txq->ift_task.gt_task.ta_flags & TASK_ENQUEUED) == 0)
3734 GROUPTASK_ENQUEUE(&txq->ift_task);
3743 txq->ift_rs_pending += nsegs + 1;
3744 if (txq->ift_rs_pending > TXQ_MAX_RS_DEFERRED(txq) ||
3745 iflib_no_tx_batch || (TXQ_AVAIL(txq) - nsegs) <= MAX_TX_DESC(ctx) + 2) {
3747 txq->ift_rs_pending = 0;
3753 MPASS(pidx >= 0 && pidx < txq->ift_size);
3760 MPASS(pi.ipi_new_pidx < txq->ift_size);
3764 ndesc += txq->ift_size;
3765 txq->ift_gen = 1;
3774 txq->ift_in_use += ndesc;
3775 txq->ift_db_pending += ndesc;
3781 txq->ift_pidx = pi.ipi_new_pidx;
3782 txq->ift_npending += pi.ipi_ndescs;
3784 *m_headp = m_head = iflib_remove_mbuf(txq);
3786 txq->ift_txd_encap_efbig++;
3801 txq->ift_mbuf_defrag_failed++;
3802 txq->ift_map_failed++;
3811 iflib_tx_desc_free(iflib_txq_t txq, int n)
3817 cidx = txq->ift_cidx;
3818 gen = txq->ift_gen;
3819 qsize = txq->ift_size;
3821 ifsd_m = txq->ift_sds.ifsd_m;
3822 do_prefetch = (txq->ift_ctx->ifc_flags & IFC_PREFETCH);
3832 bus_dmamap_sync(txq->ift_tso_buf_tag,
3833 txq->ift_sds.ifsd_tso_map[cidx],
3835 bus_dmamap_unload(txq->ift_tso_buf_tag,
3836 txq->ift_sds.ifsd_tso_map[cidx]);
3838 bus_dmamap_sync(txq->ift_buf_tag,
3839 txq->ift_sds.ifsd_map[cidx],
3841 bus_dmamap_unload(txq->ift_buf_tag,
3842 txq->ift_sds.ifsd_map[cidx]);
3849 txq->ift_dequeued++;
3858 txq->ift_cidx = cidx;
3859 txq->ift_gen = gen;
3863 iflib_completed_tx_reclaim(iflib_txq_t txq, int thresh)
3866 if_ctx_t ctx = txq->ift_ctx;
3869 MPASS(thresh /*+ MAX_TX_DESC(txq->ift_ctx) */ < txq->ift_size);
3874 iflib_tx_credits_update(ctx, txq);
3875 reclaim = DESC_RECLAIMABLE(txq);
3877 if (reclaim <= thresh /* + MAX_TX_DESC(txq->ift_ctx) */) {
3881 txq->ift_processed, txq->ift_cleaned, txq->ift_ctx->ifc_softc_ctx.isc_tx_nsegments,
3887 iflib_tx_desc_free(txq, reclaim);
3888 txq->ift_cleaned += reclaim;
3889 txq->ift_in_use -= reclaim;
3915 iflib_txq_check_drain(iflib_txq_t txq, int budget)
3918 ifmp_ring_check_drainage(txq->ift_br, budget);
3924 iflib_txq_t txq = r->cookie;
3925 if_ctx_t ctx = txq->ift_ctx;
3927 if (TXQ_AVAIL(txq) > MAX_TX_DESC(ctx) + 2)
3929 bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map,
3931 return (ctx->isc_txd_credits_update(ctx->ifc_softc, txq->ift_id,
3938 iflib_txq_t txq = r->cookie;
3939 if_ctx_t ctx = txq->ift_ctx;
3951 reclaimed = iflib_completed_tx_reclaim(txq, RECLAIM_THRESH(ctx));
3952 rang = iflib_txd_db_check(txq, reclaimed && txq->ift_db_pending);
3961 if (__predict_true(r->items[(cidx + i) & (r->size - 1)] != (void *)txq))
3969 txq->ift_qstatus = IFLIB_QUEUE_IDLE;
3970 CALLOUT_LOCK(txq);
3971 callout_stop(&txq->ift_timer);
3972 CALLOUT_UNLOCK(txq);
3981 txq->ift_qstatus = IFLIB_QUEUE_IDLE;
3987 avail, ctx->ifc_flags, TXQ_AVAIL(txq));
3991 for (i = 0; i < count && TXQ_AVAIL(txq) >= MAX_TX_DESC(ctx) + 2; i++) {
3998 * Completion interrupts will use the address of the txq
4004 if (__predict_false(*mp == (struct mbuf *)txq)) {
4008 err = iflib_encap(txq, mp);
4026 rang = iflib_txd_db_check(txq, false);
4031 iflib_txd_db_check(txq, ring);
4054 iflib_txq_t txq;
4056 txq = r->cookie;
4058 txq->ift_qstatus = IFLIB_QUEUE_IDLE;
4059 CALLOUT_LOCK(txq);
4060 callout_stop(&txq->ift_timer);
4061 CALLOUT_UNLOCK(txq);
4066 if (__predict_false(*mp == (struct mbuf *)txq))
4076 iflib_ifmp_purge(iflib_txq_t txq)
4080 r = txq->ift_br;
4093 iflib_txq_t txq = context;
4094 if_ctx_t ctx = txq->ift_ctx;
4099 txq->ift_cpu_exec_count[curcpu]++;
4105 netmap_tx_irq(ifp, txq->ift_id))
4112 if (txq->ift_db_pending)
4113 ifmp_ring_enqueue(txq->ift_br, (void **)&txq, 1, TX_BATCH_SIZE, abdicate);
4115 ifmp_ring_check_drainage(txq->ift_br, TX_BATCH_SIZE);
4120 ifmp_ring_check_drainage(txq->ift_br, TX_BATCH_SIZE);
4127 IFDI_TX_QUEUE_INTR_ENABLE(ctx, txq->ift_id);
4183 iflib_txq_t txq;
4202 for (txq = ctx->ifc_txqs, i = 0; i < sctx->isc_ntxqsets; i++, txq++) {
4203 CALLOUT_LOCK(txq);
4204 callout_stop(&txq->ift_timer);
4205 CALLOUT_UNLOCK(txq);
4214 for (txq = ctx->ifc_txqs, i = 0; i < sctx->isc_ntxqsets; i++, txq++) {
4215 callout_reset_on(&txq->ift_timer, iflib_timer_default, iflib_timer, txq,
4216 txq->ift_timer.c_cpu);
4225 for (txq = ctx->ifc_txqs, i = 0; i < sctx->isc_ntxqsets; i++, txq++)
4226 iflib_txq_check_drain(txq, IFLIB_RESTART_BUDGET);
4287 iflib_txq_t txq;
4325 txq = &ctx->ifc_txqs[qidx];
4328 if (txq->ift_closed) {
4365 err = ifmp_ring_enqueue(txq->ift_br, (void **)&m, 1, TX_BATCH_SIZE, abdicate);
4368 GROUPTASK_ENQUEUE(&txq->ift_task);
4371 GROUPTASK_ENQUEUE(&txq->ift_task);
4374 txq->ift_closed = TRUE;
4376 ifmp_ring_check_drainage(txq->ift_br, TX_BATCH_SIZE);
4441 iflib_txq_t txq = ctx->ifc_txqs;
4447 for (i = 0; i < NTXQSETS(ctx); i++, txq++)
4448 while (!(ifmp_ring_is_idle(txq->ift_br) || ifmp_ring_is_stalled(txq->ift_br)))
4449 iflib_txq_check_drain(txq, 0);
5549 iflib_txq_t txq;
5556 for (txq = ctx->ifc_txqs, i = 0; i < NTXQSETS(ctx); i++, txq++) {
5557 callout_drain(&txq->ift_timer);
5559 callout_drain(&txq->ift_netmap_timer);
5561 if (txq->ift_task.gt_uniq != NULL)
5562 taskqgroup_detach(tqg, &txq->ift_task);
5622 iflib_txq_t txq = ctx->ifc_txqs;
5628 for (int i = 0; i < NTXQSETS(ctx); i++, txq++)
5629 iflib_txq_check_drain(txq, IFLIB_RESTART_BUDGET);
5854 iflib_txq_t txq;
5891 txq = ctx->ifc_txqs;
5897 for (txconf = i = 0, cpu = CPU_FIRST(); i < ntxqsets; i++, txconf++, txq++, cpu = CPU_NEXT(cpu)) {
5907 txq->ift_ifdi = ifdip;
5915 txq->ift_txd_size[j] = scctx->isc_txd_size[j];
5918 txq->ift_ctx = ctx;
5919 txq->ift_id = i;
5921 txq->ift_br_offset = 1;
5923 txq->ift_br_offset = 0;
5926 if (iflib_txsd_alloc(txq)) {
5933 snprintf(txq->ift_mtx_name, MTX_NAME_LEN, "%s:TX(%d):callout",
5934 device_get_nameunit(dev), txq->ift_id);
5935 mtx_init(&txq->ift_mtx, txq->ift_mtx_name, NULL, MTX_DEF);
5936 callout_init_mtx(&txq->ift_timer, &txq->ift_mtx, 0);
5937 txq->ift_timer.c_cpu = cpu;
5939 callout_init_mtx(&txq->ift_netmap_timer, &txq->ift_mtx, 0);
5940 txq->ift_netmap_timer.c_cpu = cpu;
5943 err = ifmp_ring_alloc(&txq->ift_br, 2048, txq, iflib_txq_drain,
6070 iflib_txq_t txq = ctx->ifc_txqs;
6073 for (i = 0; i < NTXQSETS(ctx); i++, txq++)
6074 iflib_txq_setup(txq);
6082 iflib_txq_t txq = ctx->ifc_txqs;
6086 for (i = 0; i < NTXQSETS(ctx); i++, txq++) {
6088 iflib_dma_free(&txq->ift_ifdi[j]);
6089 iflib_txq_destroy(txq);
6432 iflib_txq_t txq = ctx->ifc_txqs;
6463 GROUPTASK_INIT(&txq->ift_task, 0, _task_fn_tx, txq);
6464 taskqgroup_attach(qgroup_if_io_tqg, &txq->ift_task, txq, dev, res,
6529 iflib_txq_t txq = ctx->ifc_txqs;
6539 for (int i = 0; i < ctx->ifc_softc_ctx.isc_ntxqsets; i++, txq++)
6540 txq->ift_qstatus = IFLIB_QUEUE_IDLE;
6547 iflib_tx_credits_update(if_ctx_t ctx, iflib_txq_t txq)
6551 int credits_pre = txq->ift_cidx_processed;
6554 bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map,
6556 if ((credits = ctx->isc_txd_credits_update(ctx->ifc_softc, txq->ift_id, true)) == 0)
6559 txq->ift_processed += credits;
6560 txq->ift_cidx_processed += credits;
6562 MPASS(credits_pre + credits == txq->ift_cidx_processed);
6563 if (txq->ift_cidx_processed >= txq->ift_size)
6564 txq->ift_cidx_processed -= txq->ift_size;
6868 "permit #txq != #rxq");
6914 iflib_txq_t txq;
6927 qfmt = "txq%03d";
6929 qfmt = "txq%02d";
6931 qfmt = "txq%d";
6932 for (i = 0, txq = ctx->ifc_txqs; i < scctx->isc_ntxqsets; i++, txq++) {
6938 CTLFLAG_RD, &txq->ift_task.gt_cpu, 0,
6942 CTLFLAG_RD, &txq->ift_dequeued, "total mbufs freed");
6944 CTLFLAG_RD, &txq->ift_enqueued, "total mbufs enqueued");
6947 CTLFLAG_RD, &txq->ift_mbuf_defrag,
6950 CTLFLAG_RD, &txq->ift_pullups,
6954 &txq->ift_mbuf_defrag_failed, "# of times m_defrag failed");
6956 "no_desc_avail", CTLFLAG_RD, &txq->ift_no_desc_avail,
6959 "tx_map_failed", CTLFLAG_RD, &txq->ift_map_failed,
6962 "txd_encap_efbig", CTLFLAG_RD, &txq->ift_txd_encap_efbig,
6965 "no_tx_dma_setup", CTLFLAG_RD, &txq->ift_no_tx_dma_setup,
6968 CTLFLAG_RD, &txq->ift_pidx, 1, "Producer Index");
6970 CTLFLAG_RD, &txq->ift_cidx, 1, "Consumer Index");
6972 "txq_cidx_processed", CTLFLAG_RD, &txq->ift_cidx_processed,
6975 CTLFLAG_RD, &txq->ift_in_use, 1, "descriptors in use");
6977 "txq_processed", CTLFLAG_RD, &txq->ift_processed,
6980 CTLFLAG_RD, &txq->ift_cleaned, "total cleaned");
6983 __DEVOLATILE(uint64_t *, &txq->ift_br->state), 0,
6986 "r_enqueues", CTLFLAG_RD, &txq->ift_br->enqueues,
6989 "r_drops", CTLFLAG_RD, &txq->ift_br->drops,
6992 "r_starts", CTLFLAG_RD, &txq->ift_br->starts,
6995 "r_stalls", CTLFLAG_RD, &txq->ift_br->stalls,
6998 "r_restarts", CTLFLAG_RD, &txq->ift_br->restarts,
7001 "r_abdications", CTLFLAG_RD, &txq->ift_br->abdications,
7141 iflib_txq_t txq;
7149 txq = &ctx->ifc_txqs[0];
7150 error = iflib_encap(txq, &m);
7152 (void)iflib_txd_db_check(txq, true);
7162 iflib_txq_t txq;
7172 txq = &ctx->ifc_txqs[0];
7173 (void)iflib_completed_tx_reclaim(txq, RECLAIM_THRESH(ctx));