Lines Matching +full:max +full:- +full:outbound +full:- +full:regions
1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
272 * Each tunable is set to a default value here if it's known at compile-time.
273 * Otherwise it is set to -n as an indication to tweak_tunables() that it should
290 int t4_ntxq = -NTXQ;
296 int t4_nrxq = -NRXQ;
302 static int t4_ntxq_vi = -NTXQ_VI;
307 static int t4_nrxq_vi = -NRXQ_VI;
313 0, "Reserve TX queue 0 of each VI for non-flowid packets");
317 static int t4_nofldtxq = -NOFLDTXQ;
322 static int t4_nofldtxq_vi = -NOFLDTXQ_VI;
329 static int t4_nofldrxq = -NOFLDRXQ;
334 static int t4_nofldrxq_vi = -NOFLDRXQ_VI;
343 #define PKTC_IDX_OFLD (-1)
348 /* 0 means chip/fw default, non-zero number is value in microseconds */
353 /* 0 means chip/fw default, non-zero number is value in microseconds */
358 /* 0 means chip/fw default, non-zero number is # of keepalives before abort */
363 /* 0 means chip/fw default, non-zero number is value in microseconds */
368 /* 0 means chip/fw default, non-zero number is value in microseconds */
373 /* 0 means chip/fw default, non-zero number is # of rexmt before abort */
378 /* -1 means chip/fw default, other values are raw backoff values to use */
380 -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1
436 static int t4_nnmtxq = -NNMTXQ;
441 static int t4_nnmrxq = -NNMRXQ;
446 static int t4_nnmtxq_vi = -NNMTXQ_VI;
451 static int t4_nnmrxq_vi = -NNMRXQ_VI;
465 #define PKTC_IDX (-1)
483 * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively).
487 0, "Interrupt types allowed (bit 0 = INTx, 1 = MSI, 2 = MSI-X)");
493 #define BUILTIN_CF "built-in"
517 * -1 to run with the firmware default. Same as FEC_AUTO (bit 5)
520 static int t4_fec = -1;
528 * -1 to set FORCE_FEC iff requested_fec != AUTO. Multiple FEC bits are okay.
535 static int t4_force_fec = -1;
541 * -1 to run with the firmware default.
545 static int t4_autoneg = -1;
550 * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed,
551 * encouraged respectively). '-n' is the same as 'n' except the firmware
556 "Firmware auto-install (0 = prohibited, 1 = allowed, 2 = encouraged)");
585 static int t4_toecaps_allowed = -1;
589 static int t4_rdmacaps_allowed = -1;
593 static int t4_cryptocaps_allowed = -1;
597 static int t4_iscsicaps_allowed = -1;
620 * -1: driver should figure out a good value.
625 static int pcie_relaxed_ordering = -1;
647 * Set to non-zero to enable the attack filter. A packet that matches any of
731 uint16_t intr_type; /* INTx, MSI, or MSI-X */
898 {0x4400, "Chelsio T440-dbg"},
899 {0x4401, "Chelsio T420-CR"},
900 {0x4402, "Chelsio T422-CR"},
901 {0x4403, "Chelsio T440-CR"},
902 {0x4404, "Chelsio T420-BCH"},
903 {0x4405, "Chelsio T440-BCH"},
904 {0x4406, "Chelsio T440-CH"},
905 {0x4407, "Chelsio T420-SO"},
906 {0x4408, "Chelsio T420-CX"},
907 {0x4409, "Chelsio T420-BT"},
908 {0x440a, "Chelsio T404-BT"},
909 {0x440e, "Chelsio T440-LP-CR"},
912 {0x5400, "Chelsio T580-dbg"},
913 {0x5401, "Chelsio T520-CR"}, /* 2 x 10G */
914 {0x5402, "Chelsio T522-CR"}, /* 2 x 10G, 2 X 1G */
915 {0x5403, "Chelsio T540-CR"}, /* 4 x 10G */
916 {0x5407, "Chelsio T520-SO"}, /* 2 x 10G, nomem */
917 {0x5409, "Chelsio T520-BT"}, /* 2 x 10GBaseT */
918 {0x540a, "Chelsio T504-BT"}, /* 4 x 1G */
919 {0x540d, "Chelsio T580-CR"}, /* 2 x 40G */
920 {0x540e, "Chelsio T540-LP-CR"}, /* 4 x 10G */
921 {0x5410, "Chelsio T580-LP-CR"}, /* 2 x 40G */
922 {0x5411, "Chelsio T520-LL-CR"}, /* 2 x 10G */
923 {0x5412, "Chelsio T560-CR"}, /* 1 x 40G, 2 x 10G */
924 {0x5414, "Chelsio T580-LP-SO-CR"}, /* 2 x 40G, nomem */
925 {0x5415, "Chelsio T502-BT"}, /* 2 x 1G */
926 {0x5418, "Chelsio T540-BT"}, /* 4 x 10GBaseT */
927 {0x5419, "Chelsio T540-LP-BT"}, /* 4 x 10GBaseT */
928 {0x541a, "Chelsio T540-SO-BT"}, /* 4 x 10GBaseT, nomem */
929 {0x541b, "Chelsio T540-SO-CR"}, /* 4 x 10G, nomem */
932 {0x5483, "Custom T540-CR"},
933 {0x5484, "Custom T540-BT"},
936 {0x6400, "Chelsio T6-DBG-25"}, /* 2 x 10/25G, debug */
937 {0x6401, "Chelsio T6225-CR"}, /* 2 x 10/25G */
938 {0x6402, "Chelsio T6225-SO-CR"}, /* 2 x 10/25G, nomem */
939 {0x6403, "Chelsio T6425-CR"}, /* 4 x 10/25G */
940 {0x6404, "Chelsio T6425-SO-CR"}, /* 4 x 10/25G, nomem */
941 {0x6405, "Chelsio T6225-SO-OCP3"}, /* 2 x 10/25G, nomem */
942 {0x6406, "Chelsio T6225-OCP3"}, /* 2 x 10/25G */
943 {0x6407, "Chelsio T62100-LP-CR"}, /* 2 x 40/50/100G */
944 {0x6408, "Chelsio T62100-SO-CR"}, /* 2 x 40/50/100G, nomem */
945 {0x6409, "Chelsio T6210-BT"}, /* 2 x 10GBASE-T */
946 {0x640d, "Chelsio T62100-CR"}, /* 2 x 40/50/100G */
947 {0x6410, "Chelsio T6-DBG-100"}, /* 2 x 40/50/100G, debug */
948 {0x6411, "Chelsio T6225-LL-CR"}, /* 2 x 10/25G */
949 {0x6414, "Chelsio T62100-SO-OCP3"}, /* 2 x 40/50/100G, nomem */
950 {0x6415, "Chelsio T6201-BT"}, /* 2 x 1000BASE-T */
953 {0x6480, "Custom T6225-CR"},
954 {0x6481, "Custom T62100-CR"},
955 {0x6482, "Custom T6225-CR"},
956 {0x6483, "Custom T62100-CR"},
957 {0x6484, "Custom T64100-CR"},
958 {0x6485, "Custom T6240-SO"},
959 {0x6486, "Custom T6225-SO-CR"},
960 {0x6487, "Custom T6225-CR"},
1101 if (id >= CHELSIO_T4 && id - CHELSIO_T4 < nitems(devnames))
1102 sc->names = &devnames[id - CHELSIO_T4];
1104 device_printf(sc->dev, "chip id %d is not supported.\n", id);
1105 sc->names = NULL;
1117 parent = device_get_nameunit(sc->dev);
1118 name = sc->names->ifnet_name;
1121 value == pi->port_id)
1124 return (-1);
1142 cur = &sc->cal_info[sc->cal_current];
1143 next_up = (sc->cal_current + 1) % CNT_CAL_INFO;
1144 nex = &sc->cal_info[next_up];
1145 if (__predict_false(sc->cal_count == 0)) {
1147 cur->hw_cur = hw;
1148 cur->sbt_cur = sbt;
1149 sc->cal_count++;
1153 if (cur->hw_cur == hw) {
1155 sc->cal_count = 0;
1156 atomic_store_rel_int(&cur->gen, 0);
1160 seqc_write_begin(&nex->gen);
1161 nex->hw_prev = cur->hw_cur;
1162 nex->sbt_prev = cur->sbt_cur;
1163 nex->hw_cur = hw;
1164 nex->sbt_cur = sbt;
1165 seqc_write_end(&nex->gen);
1166 sc->cal_current = next_up;
1168 callout_reset_sbt_curcpu(&sc->cal_callout, SBT_1S, 0, t4_calibration,
1183 sc->cal_info[i].gen = 0;
1185 sc->cal_current = 0;
1186 sc->cal_count = 0;
1187 sc->cal_gen = 0;
1212 sc->dev = dev;
1213 sysctl_ctx_init(&sc->ctx);
1214 TUNABLE_INT_FETCH("hw.cxgbe.dflags", &sc->debug_flags);
1224 sc->params.pci.mps = 128 << ((v & PCIEM_CTL_MAX_PAYLOAD) >> 5);
1236 sc->sge_gts_reg = MYPF_REG(A_SGE_PF_GTS);
1237 sc->sge_kdoorbell_reg = MYPF_REG(A_SGE_PF_KDOORBELL);
1238 sc->traceq = -1;
1239 mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF);
1240 snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer",
1243 snprintf(sc->lockname, sizeof(sc->lockname), "%s",
1245 mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF);
1248 mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF);
1249 TAILQ_INIT(&sc->sfl);
1250 callout_init_mtx(&sc->sfl_callout, &sc->sfl_lock, 0);
1252 mtx_init(&sc->reg_lock, "indirect register access", 0, MTX_DEF);
1254 sc->policy = NULL;
1255 rw_init(&sc->policy_lock, "connection offload policy");
1257 callout_init(&sc->ktls_tick, 1);
1259 callout_init(&sc->cal_callout, 1);
1261 refcount_init(&sc->vxlan_refcount, 0);
1263 TASK_INIT(&sc->reset_task, 0, reset_adapter_task, sc);
1264 TASK_INIT(&sc->fatal_error_task, 0, fatal_error_task, sc);
1266 sc->ctrlq_oid = SYSCTL_ADD_NODE(&sc->ctx,
1267 SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev)), OID_AUTO, "ctrlq",
1269 sc->fwq_oid = SYSCTL_ADD_NODE(&sc->ctx,
1270 SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev)), OID_AUTO, "fwq",
1277 memset(sc->chan_map, 0xff, sizeof(sc->chan_map));
1281 rc = -t4_prep_adapter(sc, buf);
1295 sc->pf = chip_id(sc) <= CHELSIO_T5 ? G_SOURCEPF(j) : G_T6_SOURCEPF(j);
1296 sc->mbox = sc->pf;
1299 if (sc->names == NULL) {
1318 rc = make_dev_s(&mda, &sc->cdev, "%s", device_get_nameunit(dev));
1341 MPASS(sc->flags & FW_OK);
1347 if (sc->flags & MASTER_PF) {
1374 * First pass over all the ports - allocate VIs and initialize some
1381 sc->port[i] = pi;
1384 pi->adapter = sc;
1385 pi->port_id = i;
1388 * pi->nvi's final value is known.
1390 pi->vi = malloc(sizeof(struct vi_info) * t4_num_vis, M_CXGBE,
1397 rc = -t4_port_init(sc, sc->mbox, sc->pf, 0, i);
1401 free(pi->vi, M_CXGBE);
1403 sc->port[i] = NULL;
1407 if (is_bt(pi->port_type))
1408 setbit(&sc->bt_map, pi->tx_chan);
1410 MPASS(!isset(&sc->bt_map, pi->tx_chan));
1412 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d",
1414 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF);
1415 sc->chan_map[pi->tx_chan] = i;
1424 pi->fcs_reg = -1;
1426 pi->fcs_reg = t4_port_reg(sc, pi->tx_chan,
1429 pi->fcs_base = 0;
1432 ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change,
1440 pi->flags |= FIXED_IFMEDIA;
1443 pi->dev = device_add_child(dev, sc->names->ifnet_name,
1445 if (pi->dev == NULL) {
1451 pi->vi[0].dev = pi->dev;
1452 device_set_softc(pi->dev, pi);
1458 nports = sc->params.nports;
1464 sc->intr_type = iaq.intr_type;
1465 sc->intr_count = iaq.nirq;
1467 s = &sc->sge;
1468 s->nrxq = nports * iaq.nrxq;
1469 s->ntxq = nports * iaq.ntxq;
1471 s->nrxq += nports * (num_vis - 1) * iaq.nrxq_vi;
1472 s->ntxq += nports * (num_vis - 1) * iaq.ntxq_vi;
1474 s->neq = s->ntxq + s->nrxq; /* the free list in an rxq is an eq */
1475 s->neq += nports; /* ctrl queues: 1 per port */
1476 s->niq = s->nrxq + 1; /* 1 extra for firmware event queue */
1479 s->nofldtxq = nports * iaq.nofldtxq;
1481 s->nofldtxq += nports * (num_vis - 1) * iaq.nofldtxq_vi;
1482 s->neq += s->nofldtxq;
1484 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_ofld_txq),
1490 s->nofldrxq = nports * iaq.nofldrxq;
1492 s->nofldrxq += nports * (num_vis - 1) * iaq.nofldrxq_vi;
1493 s->neq += s->nofldrxq; /* free list */
1494 s->niq += s->nofldrxq;
1496 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq),
1501 s->nnmrxq = 0;
1502 s->nnmtxq = 0;
1504 s->nnmrxq += nports * iaq.nnmrxq;
1505 s->nnmtxq += nports * iaq.nnmtxq;
1508 s->nnmrxq += nports * (num_vis - 1) * iaq.nnmrxq_vi;
1509 s->nnmtxq += nports * (num_vis - 1) * iaq.nnmtxq_vi;
1511 s->neq += s->nnmtxq + s->nnmrxq;
1512 s->niq += s->nnmrxq;
1514 s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq),
1516 s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq),
1519 MPASS(s->niq <= s->iqmap_sz);
1520 MPASS(s->neq <= s->eqmap_sz);
1522 s->ctrlq = malloc(nports * sizeof(struct sge_wrq), M_CXGBE,
1524 s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE,
1526 s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE,
1528 s->iqmap = malloc(s->iqmap_sz * sizeof(struct sge_iq *), M_CXGBE,
1530 s->eqmap = malloc(s->eqmap_sz * sizeof(struct sge_eq *), M_CXGBE,
1533 sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE,
1546 if (sc->vres.key.size != 0)
1547 sc->key_map = vmem_create("T4TLS key map", sc->vres.key.start,
1548 sc->vres.key.size, 32, 0, M_FIRSTFIT | M_WAITOK);
1565 struct port_info *pi = sc->port[i];
1571 pi->nvi = num_vis;
1573 vi->pi = pi;
1574 vi->adapter = sc;
1575 vi->first_intr = -1;
1576 vi->qsize_rxq = t4_qsize_rxq;
1577 vi->qsize_txq = t4_qsize_txq;
1579 vi->first_rxq = rqidx;
1580 vi->first_txq = tqidx;
1581 vi->tmr_idx = t4_tmr_idx;
1582 vi->pktc_idx = t4_pktc_idx;
1583 vi->nrxq = j == 0 ? iaq.nrxq : iaq.nrxq_vi;
1584 vi->ntxq = j == 0 ? iaq.ntxq : iaq.ntxq_vi;
1586 rqidx += vi->nrxq;
1587 tqidx += vi->ntxq;
1589 if (j == 0 && vi->ntxq > 1)
1590 vi->rsrv_noflowq = t4_rsrv_noflowq ? 1 : 0;
1592 vi->rsrv_noflowq = 0;
1595 vi->first_ofld_txq = ofld_tqidx;
1596 vi->nofldtxq = j == 0 ? iaq.nofldtxq : iaq.nofldtxq_vi;
1597 ofld_tqidx += vi->nofldtxq;
1600 vi->ofld_tmr_idx = t4_tmr_idx_ofld;
1601 vi->ofld_pktc_idx = t4_pktc_idx_ofld;
1602 vi->first_ofld_rxq = ofld_rqidx;
1603 vi->nofldrxq = j == 0 ? iaq.nofldrxq : iaq.nofldrxq_vi;
1605 ofld_rqidx += vi->nofldrxq;
1608 vi->first_nm_rxq = nm_rqidx;
1609 vi->first_nm_txq = nm_tqidx;
1611 vi->nnmrxq = iaq.nnmrxq;
1612 vi->nnmtxq = iaq.nnmtxq;
1614 vi->nnmrxq = iaq.nnmrxq_vi;
1615 vi->nnmtxq = iaq.nnmtxq_vi;
1617 nm_rqidx += vi->nnmrxq;
1618 nm_tqidx += vi->nnmtxq;
1633 * Ensure thread-safe mailbox access (in debug builds).
1639 sc->flags |= CHK_MBOX_ACCESS;
1646 sc->params.pci.speed, sc->params.pci.width, sc->params.nports,
1647 sc->intr_count, sc->intr_type == INTR_MSIX ? "MSI-X" :
1648 (sc->intr_type == INTR_MSI ? "MSI" : "INTx"),
1649 sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq);
1656 if (rc != 0 && sc->cdev) {
1680 pi = sc->port[i];
1681 if (pi != NULL && pi->dev == dev) {
1682 sbuf_printf(sb, "port=%d", pi->port_id);
1695 if (sc->flags & FW_OK)
1709 pi = sc->port[port];
1710 if (pi == NULL || pi->dev == NULL)
1712 *child = pi->dev;
1776 if (sc->cdev) {
1777 destroy_dev(sc->cdev);
1778 sc->cdev = NULL;
1785 sc->flags &= ~CHK_MBOX_ACCESS;
1786 if (sc->flags & FULL_INIT_DONE) {
1787 if (!(sc->flags & IS_VF))
1800 for (i = 0; i < sc->intr_count; i++)
1801 t4_free_irq(sc, &sc->irq[i]);
1803 if ((sc->flags & (IS_VF | FW_OK)) == FW_OK)
1807 pi = sc->port[i];
1809 t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->vi[0].viid);
1811 mtx_destroy(&pi->pi_lock);
1812 free(pi->vi, M_CXGBE);
1816 callout_stop(&sc->cal_callout);
1817 callout_drain(&sc->cal_callout);
1819 sysctl_ctx_free(&sc->ctx);
1822 if ((sc->flags & (IS_VF | FW_OK)) == FW_OK)
1823 t4_fw_bye(sc, sc->mbox);
1825 if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX)
1828 if (sc->regs_res)
1829 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid,
1830 sc->regs_res);
1832 if (sc->udbs_res)
1833 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid,
1834 sc->udbs_res);
1836 if (sc->msix_res)
1837 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid,
1838 sc->msix_res);
1840 if (sc->l2t)
1842 if (sc->smt)
1843 t4_free_smt(sc->smt);
1848 if (sc->key_map)
1849 vmem_destroy(sc->key_map);
1855 free(sc->sge.ofld_txq, M_CXGBE);
1858 free(sc->sge.ofld_rxq, M_CXGBE);
1861 free(sc->sge.nm_rxq, M_CXGBE);
1862 free(sc->sge.nm_txq, M_CXGBE);
1864 free(sc->irq, M_CXGBE);
1865 free(sc->sge.rxq, M_CXGBE);
1866 free(sc->sge.txq, M_CXGBE);
1867 free(sc->sge.ctrlq, M_CXGBE);
1868 free(sc->sge.iqmap, M_CXGBE);
1869 free(sc->sge.eqmap, M_CXGBE);
1870 free(sc->tids.ftid_tab, M_CXGBE);
1871 free(sc->tids.hpftid_tab, M_CXGBE);
1872 free_hftid_hash(&sc->tids);
1873 free(sc->tids.tid_tab, M_CXGBE);
1876 callout_drain(&sc->ktls_tick);
1877 callout_drain(&sc->sfl_callout);
1878 if (mtx_initialized(&sc->tids.ftid_lock)) {
1879 mtx_destroy(&sc->tids.ftid_lock);
1880 cv_destroy(&sc->tids.ftid_cv);
1882 if (mtx_initialized(&sc->tids.atid_lock))
1883 mtx_destroy(&sc->tids.atid_lock);
1884 if (mtx_initialized(&sc->ifp_lock))
1885 mtx_destroy(&sc->ifp_lock);
1887 if (rw_initialized(&sc->policy_lock)) {
1888 rw_destroy(&sc->policy_lock);
1890 if (sc->policy != NULL)
1891 free_offload_policy(sc->policy);
1896 struct memwin *mw = &sc->memwin[i];
1898 if (rw_initialized(&mw->mw_lock))
1899 rw_destroy(&mw->mw_lock);
1902 mtx_destroy(&sc->sfl_lock);
1903 mtx_destroy(&sc->reg_lock);
1904 mtx_destroy(&sc->sc_lock);
1917 if (atomic_testandset_int(&sc->error_flags, ilog2(ADAP_STOPPED))) {
1919 __func__, curthread, sc->flags, sc->error_flags);
1923 sc->flags, sc->error_flags);
1926 pi = sc->port[i];
1928 if (pi->up_vis > 0 && pi->link_cfg.link_ok) {
1935 pi->link_cfg.link_ok = false;
1949 if (!atomic_testandclear_int(&sc->error_flags, ilog2(ADAP_STOPPED))) {
1951 __func__, curthread, sc->flags, sc->error_flags);
1955 sc->flags, sc->error_flags);
1958 MPASS((sc->flags & FW_OK) == 0);
1959 MPASS((sc->flags & MASTER_PF) == 0);
1960 MPASS(sc->reset_thread == NULL);
1967 sc->reset_thread = curthread;
1971 sc->reset_thread = NULL;
1972 atomic_set_int(&sc->error_flags, ADAP_STOPPED);
1975 atomic_clear_int(&sc->error_flags, ADAP_FATAL_ERR);
1976 atomic_add_int(&sc->incarnation, 1);
1977 atomic_add_int(&sc->num_resets, 1);
1988 MPASS(sc->reset_thread == curthread);
1989 mtx_lock(&sc->reg_lock);
1990 atomic_clear_int(&sc->error_flags, HW_OFF_LIMITS);
1991 mtx_unlock(&sc->reg_lock);
1995 mtx_lock(&sc->reg_lock);
1996 atomic_set_int(&sc->error_flags, HW_OFF_LIMITS);
1997 mtx_unlock(&sc->reg_lock);
1998 sc->flags &= ~(FW_OK | MASTER_PF);
1999 sc->reset_thread = NULL;
2037 pi = sc->port[i];
2038 pi->vxlan_tcam_entry = false;
2040 vi->xact_addr_filt = -1;
2041 mtx_lock(&vi->tick_mtx);
2042 vi->flags |= VI_SKIP_STATS;
2043 mtx_unlock(&vi->tick_mtx);
2044 if (!(vi->flags & VI_INIT_DONE))
2047 ifp = vi->ifp;
2049 mtx_lock(&vi->tick_mtx);
2050 callout_stop(&vi->tick);
2051 mtx_unlock(&vi->tick_mtx);
2052 callout_drain(&vi->tick);
2060 txq->eq.flags &= ~(EQ_ENABLED | EQ_HW_ALLOCATED);
2065 TXQ_LOCK(&ofld_txq->wrq);
2066 ofld_txq->wrq.eq.flags &= ~EQ_HW_ALLOCATED;
2067 TXQ_UNLOCK(&ofld_txq->wrq);
2071 rxq->iq.flags &= ~IQ_HW_ALLOCATED;
2075 ofld_rxq->iq.flags &= ~IQ_HW_ALLOCATED;
2082 if (sc->flags & FULL_INIT_DONE) {
2084 wrq = &sc->sge.ctrlq[i];
2086 wrq->eq.flags &= ~EQ_HW_ALLOCATED;
2091 if (pi->flags & HAS_TRACEQ) {
2092 pi->flags &= ~HAS_TRACEQ;
2093 sc->traceq = -1;
2094 sc->tracer_valid = 0;
2095 sc->tracer_enabled = 0;
2098 if (sc->flags & FULL_INIT_DONE) {
2100 sc->sge.fwq.flags &= ~IQ_HW_ALLOCATED;
2101 quiesce_iq_fl(sc, &sc->sge.fwq, NULL);
2105 callout_stop(&sc->cal_callout);
2106 callout_drain(&sc->cal_callout);
2179 o->flags = sc->flags;
2181 o->nbmcaps = sc->nbmcaps;
2182 o->linkcaps = sc->linkcaps;
2183 o->switchcaps = sc->switchcaps;
2184 o->niccaps = sc->niccaps;
2185 o->toecaps = sc->toecaps;
2186 o->rdmacaps = sc->rdmacaps;
2187 o->cryptocaps = sc->cryptocaps;
2188 o->iscsicaps = sc->iscsicaps;
2189 o->fcoecaps = sc->fcoecaps;
2191 o->cfcsum = sc->cfcsum;
2192 MPASS(sizeof(o->cfg_file) == sizeof(sc->cfg_file));
2193 memcpy(o->cfg_file, sc->cfg_file, sizeof(o->cfg_file));
2195 o->params = sc->params;
2196 o->vres = sc->vres;
2197 o->tids = sc->tids;
2198 o->sge = sc->sge;
2200 o->rawf_base = sc->rawf_base;
2201 o->nrawf = sc->nrawf;
2213 if (o->c##caps != sc->c##caps) { \
2214 CH_ERR(sc, "%scaps 0x%04x -> 0x%04x.\n", #c, o->c##caps, \
2215 sc->c##caps); \
2231 if (o->cfcsum != sc->cfcsum) {
2232 CH_ERR(sc, "config file %s (0x%x) -> %s (0x%x)\n", o->cfg_file,
2233 o->cfcsum, sc->cfg_file, sc->cfcsum);
2238 if (o->p != sc->p) { \
2239 CH_ERR(sc, #name " %d -> %d\n", o->p, sc->p); \
2331 MPASS(sc->flags & FW_OK);
2333 if (sc->flags & MASTER_PF) {
2352 pi = sc->port[i];
2354 MPASS(pi->vi != NULL);
2355 MPASS(pi->vi[0].dev == pi->dev);
2357 rc = -t4_port_init(sc, sc->mbox, sc->pf, 0, i);
2360 "failed to re-initialize port %d: %d\n", i, rc);
2363 MPASS(sc->chan_map[pi->tx_chan] == i);
2375 "failed to re-allocate extra VI: %d\n", rc);
2388 if (sc->flags & FULL_INIT_DONE) {
2391 CH_ERR(sc, "failed to re-initialize adapter: %d\n", rc);
2395 if (sc->vxlan_refcount > 0)
2399 pi = sc->port[i];
2401 mtx_lock(&vi->tick_mtx);
2402 vi->flags &= ~VI_SKIP_STATS;
2403 mtx_unlock(&vi->tick_mtx);
2404 if (!(vi->flags & VI_INIT_DONE))
2408 CH_ERR(vi, "failed to re-initialize "
2412 if (sc->traceq < 0 && IS_MAIN_VI(vi)) {
2413 sc->traceq = sc->sge.rxq[vi->first_rxq].iq.abs_id;
2417 V_RSSCONTROL(pi->tx_chan) |
2418 V_QUEUENUMBER(sc->traceq));
2419 pi->flags |= HAS_TRACEQ;
2422 ifp = vi->ifp;
2434 CH_ERR(vi, "failed to re-configure MAC: %d\n", rc);
2437 rc = -t4_enable_vi(sc, sc->mbox, vi->viid, true,
2440 CH_ERR(vi, "failed to re-enable VI: %d\n", rc);
2445 txq->eq.flags |= EQ_ENABLED;
2448 mtx_lock(&vi->tick_mtx);
2449 callout_schedule(&vi->tick, hz);
2450 mtx_unlock(&vi->tick_mtx);
2453 if (pi->up_vis > 0) {
2458 if (pi->link_cfg.link_ok)
2466 pi = sc->port[i];
2468 if (!(vi->flags & VI_INIT_DONE))
2470 ifp = vi->ifp;
2475 CH_ERR(vi, "failed to re-configure MCAST MACs: %d\n", rc);
2542 rc = BUS_RESET_CHILD(device_get_parent(sc->dev), sc->dev, 0);
2553 MPASS(sc->error_flags & HW_OFF_LIMITS);
2554 bus_space_write_4(sc->bt, sc->bh, A_PL_RST,
2576 const int flags = sc->flags;
2577 const int eflags = sc->error_flags;
2585 "flags 0x%08x -> 0x%08x, err_flags 0x%08x -> 0x%08x.\n",
2586 rc, flags, sc->flags, eflags, sc->error_flags);
2595 device_set_descf(dev, "port %d", pi->port_id);
2611 struct sysctl_ctx_list *ctx = &vi->ctx;
2614 struct adapter *sc = vi->adapter;
2617 children = SYSCTL_CHILDREN(device_get_sysctl_tree(vi->dev));
2618 vi->rxq_oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "rxq",
2620 vi->txq_oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "txq",
2623 vi->nm_rxq_oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "nm_rxq",
2625 vi->nm_txq_oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "nm_txq",
2629 vi->ofld_rxq_oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "ofld_rxq",
2633 vi->ofld_txq_oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "ofld_txq",
2637 vi->xact_addr_filt = -1;
2638 mtx_init(&vi->tick_mtx, "vi tick", NULL, MTX_DEF);
2639 callout_init_mtx(&vi->tick, &vi->tick_mtx, 0);
2640 if (sc->flags & IS_VF || t4_tx_vm_wr != 0)
2641 vi->flags |= TX_USES_VM_WR;
2645 vi->ifp = ifp;
2655 if (vi->pi->nvi > 1 || sc->flags & IS_VF)
2679 if (vi->nofldrxq != 0)
2683 if (is_ethoffload(sc) && vi->nofldtxq != 0) {
2690 if (vi->flags & TX_USES_VM_WR)
2695 if (is_ethoffload(sc) && vi->nofldtxq != 0)
2702 if (sc->flags & KERN_TLS_ON || !is_t6(sc))
2707 ether_ifattach(ifp, vi->hw_addr);
2709 if (vi->nnmrxq != 0)
2713 sbuf_printf(sb, "%d txq, %d rxq (NIC)", vi->ntxq, vi->nrxq);
2717 sbuf_printf(sb, "; %d txq (TOE)", vi->nofldtxq);
2720 sbuf_printf(sb, "; %d txq (TOE/ETHOFLD)", vi->nofldtxq);
2723 sbuf_printf(sb, "; %d txq (ETHOFLD)", vi->nofldtxq);
2729 sbuf_printf(sb, ", %d rxq (TOE)", vi->nofldrxq);
2734 vi->nnmtxq, vi->nnmrxq);
2746 vi->pfil = pfil_head_register(&pa);
2753 struct adapter *sc = pi->adapter;
2757 sysctl_ctx_init(&pi->ctx);
2759 cxgbe_vi_attach(dev, &pi->vi[0]);
2764 vi->dev = device_add_child(dev, sc->names->vi_ifnet_name, DEVICE_UNIT_ANY);
2765 if (vi->dev == NULL) {
2769 device_set_softc(vi->dev, vi);
2782 if_t ifp = vi->ifp;
2784 if (vi->pfil != NULL) {
2785 pfil_head_unregister(vi->pfil);
2786 vi->pfil = NULL;
2797 callout_drain(&vi->tick);
2798 mtx_destroy(&vi->tick_mtx);
2799 sysctl_ctx_free(&vi->ctx);
2802 if_free(vi->ifp);
2803 vi->ifp = NULL;
2810 struct adapter *sc = pi->adapter;
2818 sysctl_ctx_free(&pi->ctx);
2819 begin_vi_detach(sc, &pi->vi[0]);
2820 if (pi->flags & HAS_TRACEQ) {
2821 sc->traceq = -1; /* cloner should not create ifnet */
2824 cxgbe_vi_detach(&pi->vi[0]);
2825 ifmedia_removeall(&pi->media);
2826 end_vi_detach(sc, &pi->vi[0]);
2835 struct adapter *sc = vi->adapter;
2848 struct port_info *pi = vi->pi;
2849 struct adapter *sc = pi->adapter;
2855 mtu = ifr->ifr_mtu;
2863 if (vi->flags & VI_INIT_DONE) {
2884 flags = vi->if_flags;
2893 vi->if_flags = if_getflags(ifp);
2915 mask = ifr->ifr_reqcap ^ if_getcapenable(ifp);
2925 "tso4 disabled due to -txcsum.\n");
2937 "tso6 disabled due to -txcsum6.\n");
2977 rxq->iq.flags |= IQ_LRO_ENABLED;
2979 rxq->iq.flags &= ~IQ_LRO_ENABLED;
3002 /* Need to find out how to disable auto-mtu-inflation */
3019 rxq->iq.flags |= IQ_RX_TIMESTAMP;
3021 rxq->iq.flags &= ~IQ_RX_TIMESTAMP;
3060 rc = ifmedia_ioctl(ifp, ifr, &pi->media, cmd);
3083 rc = -t4_i2c_rd(sc, sc->mbox, pi->port_id, i2c.dev_addr,
3102 struct port_info *pi = vi->pi;
3109 MPASS(m->m_nextpkt == NULL); /* not quite ready for this yet */
3111 if (m->m_pkthdr.csum_flags & CSUM_SND_TAG)
3112 MPASS(m->m_pkthdr.snd_tag->ifp == ifp);
3115 if (__predict_false(pi->link_cfg.link_ok == false)) {
3120 rc = parse_pkt(&m, vi->flags & TX_USES_VM_WR);
3129 atomic_add_int(&pi->tx_parse_error, 1); /* rare, atomic is ok */
3134 sc = vi->adapter;
3135 txq = &sc->sge.txq[vi->first_txq];
3137 txq += ((m->m_pkthdr.flowid % (vi->ntxq - vi->rsrv_noflowq)) +
3138 vi->rsrv_noflowq);
3141 rc = mp_ring_enqueue(txq->r, items, 1, 256);
3156 if (vi->flags & VI_INIT_DONE) {
3159 txq->eq.flags |= EQ_QFLUSH;
3161 while (!mp_ring_is_idle(txq->r)) {
3162 mp_ring_check_drainage(txq->r, 4096);
3166 txq->eq.flags &= ~EQ_QFLUSH;
3177 struct fw_vi_stats_vf *s = &vi->stats;
3179 mtx_lock(&vi->tick_mtx);
3181 mtx_unlock(&vi->tick_mtx);
3185 return (s->rx_bcast_frames + s->rx_mcast_frames +
3186 s->rx_ucast_frames);
3188 return (s->rx_err_frames);
3190 return (s->tx_bcast_frames + s->tx_mcast_frames +
3191 s->tx_ucast_frames + s->tx_offload_frames);
3193 return (s->tx_drop_frames);
3195 return (s->rx_bcast_bytes + s->rx_mcast_bytes +
3196 s->rx_ucast_bytes);
3198 return (s->tx_bcast_bytes + s->tx_mcast_bytes +
3199 s->tx_ucast_bytes + s->tx_offload_bytes);
3201 return (s->rx_mcast_frames);
3203 return (s->tx_mcast_frames);
3208 if (vi->flags & VI_INIT_DONE) {
3213 drops += counter_u64_fetch(txq->r->dropped);
3229 struct port_info *pi = vi->pi;
3230 struct port_stats *s = &pi->stats;
3232 mtx_lock(&vi->tick_mtx);
3234 mtx_unlock(&vi->tick_mtx);
3238 return (s->rx_frames);
3241 return (s->rx_jabber + s->rx_runt + s->rx_too_long +
3242 s->rx_fcs_err + s->rx_len_err);
3245 return (s->tx_frames);
3248 return (s->tx_error_frames);
3251 return (s->rx_octets);
3254 return (s->tx_octets);
3257 return (s->rx_mcast_frames);
3260 return (s->tx_mcast_frames);
3263 return (s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 +
3264 s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 +
3265 s->rx_trunc3 + pi->tnl_cong_drops);
3270 drops = s->tx_drop;
3271 if (vi->flags & VI_INIT_DONE) {
3276 drops += counter_u64_fetch(txq->r->dropped);
3295 switch (params->hdr.type) {
3306 if (is_t6(vi->pi->adapter))
3328 struct port_info *pi = vi->pi;
3329 struct ifmedia *ifm = &pi->media;
3330 struct link_config *lc = &pi->link_cfg;
3331 struct adapter *sc = pi->adapter;
3338 if (IFM_SUBTYPE(ifm->ifm_media) == IFM_AUTO) {
3340 if (!(lc->pcaps & FW_PORT_CAP32_ANEG)) {
3344 lc->requested_aneg = AUTONEG_ENABLE;
3345 lc->requested_speed = 0;
3346 lc->requested_fc |= PAUSE_AUTONEG;
3348 lc->requested_aneg = AUTONEG_DISABLE;
3349 lc->requested_speed =
3350 ifmedia_baudrate(ifm->ifm_media) / 1000000;
3351 lc->requested_fc = 0;
3352 if (IFM_OPTIONS(ifm->ifm_media) & IFM_ETH_RXPAUSE)
3353 lc->requested_fc |= PAUSE_RX;
3354 if (IFM_OPTIONS(ifm->ifm_media) & IFM_ETH_TXPAUSE)
3355 lc->requested_fc |= PAUSE_TX;
3357 if (pi->up_vis > 0 && !hw_off_limits(sc)) {
3378 switch(pi->port_type) {
3437 switch (pi->mod_type) {
3517 struct port_info *pi = vi->pi;
3518 struct adapter *sc = pi->adapter;
3519 struct link_config *lc = &pi->link_cfg;
3525 if (pi->up_vis == 0 && !hw_off_limits(sc)) {
3538 ifmr->ifm_status = IFM_AVALID;
3539 if (lc->link_ok == false)
3541 ifmr->ifm_status |= IFM_ACTIVE;
3544 ifmr->ifm_active = IFM_ETHER | IFM_FDX;
3545 ifmr->ifm_active &= ~(IFM_ETH_TXPAUSE | IFM_ETH_RXPAUSE);
3546 if (lc->fc & PAUSE_RX)
3547 ifmr->ifm_active |= IFM_ETH_RXPAUSE;
3548 if (lc->fc & PAUSE_TX)
3549 ifmr->ifm_active |= IFM_ETH_TXPAUSE;
3550 ifmr->ifm_active |= port_mword(pi, speed_to_fwcap(lc->speed));
3561 device_set_descf(dev, "port %d vi %td", vi->pi->port_id,
3562 vi - vi->pi->vi);
3575 index = vi - pi->vi;
3579 device_get_nameunit(vi->dev)));
3581 rc = t4_alloc_vi_func(sc, sc->mbox, pi->tx_chan, sc->pf, 0, 1,
3582 vi->hw_addr, &vi->rss_size, &vi->vfvld, &vi->vin, func, 0);
3585 "for port %d: %d\n", index, pi->port_id, -rc);
3586 return (-rc);
3588 vi->viid = rc;
3590 if (vi->rss_size == 1) {
3597 device_printf(vi->dev, "RSS table not available.\n");
3598 vi->rss_base = 0xffff;
3605 V_FW_PARAMS_PARAM_YZ(vi->viid);
3606 rc = t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val);
3608 vi->rss_base = 0xffff;
3610 MPASS((val >> 16) == vi->rss_size);
3611 vi->rss_base = val & 0xffff;
3626 pi = vi->pi;
3627 sc = pi->adapter;
3649 sc = vi->adapter;
3653 t4_free_vi(sc, sc->mbox, sc->pf, 0, vi->viid);
3667 panic("%s: panic on fatal error", device_get_nameunit(sc->dev));
3676 if (atomic_testandclear_int(&sc->error_flags, ilog2(ADAP_CIM_ERR))) {
3701 const bool verbose = (sc->debug_flags & DF_VERBOSE_SLOWINTR) != 0;
3704 if (atomic_testandset_int(&sc->error_flags, ilog2(ADAP_FATAL_ERR)))
3716 atomic_set_int(&sc->error_flags, ADAP_CIM_ERR);
3720 device_get_nameunit(sc->dev), fw_error);
3721 taskqueue_enqueue(reset_tq, &sc->fatal_error_task);
3735 sc->regs_rid = PCIR_BAR(0);
3736 sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
3737 &sc->regs_rid, RF_ACTIVE);
3738 if (sc->regs_res == NULL) {
3739 device_printf(sc->dev, "cannot map registers.\n");
3742 sc->bt = rman_get_bustag(sc->regs_res);
3743 sc->bh = rman_get_bushandle(sc->regs_res);
3744 sc->mmio_len = rman_get_size(sc->regs_res);
3745 setbit(&sc->doorbells, DOORBELL_KDB);
3747 sc->msix_rid = PCIR_BAR(4);
3748 sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
3749 &sc->msix_rid, RF_ACTIVE);
3750 if (sc->msix_res == NULL) {
3751 device_printf(sc->dev, "cannot map MSI-X BAR.\n");
3766 if (is_t4(sc) && sc->rdmacaps == 0)
3769 sc->udbs_rid = PCIR_BAR(2);
3770 sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
3771 &sc->udbs_rid, RF_ACTIVE);
3772 if (sc->udbs_res == NULL) {
3773 device_printf(sc->dev, "cannot map doorbell BAR.\n");
3776 sc->udbs_base = rman_get_virtual(sc->udbs_res);
3779 setbit(&sc->doorbells, DOORBELL_UDB);
3787 * (UDBS_SEG_SIZE) doorbell regions, each associated
3793 rc = pmap_change_attr((vm_offset_t)sc->udbs_base,
3794 rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING);
3796 clrbit(&sc->doorbells, DOORBELL_UDB);
3797 setbit(&sc->doorbells, DOORBELL_WCWR);
3798 setbit(&sc->doorbells, DOORBELL_UDBWC);
3800 device_printf(sc->dev,
3811 sc->iwt.wc_en = isset(&sc->doorbells, DOORBELL_UDBWC) ? 1 : 0;
3819 if ((sc->doorbells & t4_doorbells_allowed) != 0) {
3820 sc->doorbells &= t4_doorbells_allowed;
3824 sc->doorbells, t4_doorbells_allowed);
3872 for (i = 0, mw = &sc->memwin[0]; i < NUM_MEMWIN; i++, mw_init++, mw++) {
3873 if (!rw_initialized(&mw->mw_lock)) {
3874 rw_init(&mw->mw_lock, "memory window access");
3875 mw->mw_base = mw_init->base;
3876 mw->mw_aperture = mw_init->aperture;
3877 mw->mw_curpos = 0;
3881 (mw->mw_base + bar0) | V_BIR(0) |
3882 V_WINDOW(ilog2(mw->mw_aperture) - 10));
3883 rw_wlock(&mw->mw_lock);
3884 position_memwin(sc, i, mw->mw_curpos);
3885 rw_wunlock(&mw->mw_lock);
3895 * address prior to the requested address. mw->mw_curpos always has the actual
3906 mw = &sc->memwin[idx];
3907 rw_assert(&mw->mw_lock, RA_WLOCKED);
3911 mw->mw_curpos = addr & ~0xf; /* start must be 16B aligned */
3913 pf = V_PFNUM(sc->pf);
3914 mw->mw_curpos = addr & ~0x7f; /* start must be 128B aligned */
3917 t4_write_reg(sc, reg, mw->mw_curpos | pf);
3934 mw = &sc->memwin[idx];
3936 rw_rlock(&mw->mw_lock);
3937 mw_end = mw->mw_curpos + mw->mw_aperture;
3938 if (addr >= mw_end || addr < mw->mw_curpos) {
3940 if (!rw_try_upgrade(&mw->mw_lock)) {
3941 rw_runlock(&mw->mw_lock);
3942 rw_wlock(&mw->mw_lock);
3944 rw_assert(&mw->mw_lock, RA_WLOCKED);
3946 rw_downgrade(&mw->mw_lock);
3947 mw_end = mw->mw_curpos + mw->mw_aperture;
3949 rw_assert(&mw->mw_lock, RA_RLOCKED);
3952 v = t4_read_reg(sc, mw->mw_base + addr -
3953 mw->mw_curpos);
3957 t4_write_reg(sc, mw->mw_base + addr -
3958 mw->mw_curpos, htole32(v));
3961 len -= 4;
3963 rw_runlock(&mw->mw_lock);
3978 t = &sc->tids;
3979 if (t->natids == 0)
3982 MPASS(t->atid_tab == NULL);
3984 t->atid_tab = malloc(t->natids * sizeof(*t->atid_tab), M_CXGBE,
3986 mtx_init(&t->atid_lock, "atid lock", NULL, MTX_DEF);
3987 t->afree = t->atid_tab;
3988 t->atids_in_use = 0;
3989 t->atid_alloc_stopped = false;
3990 for (i = 1; i < t->natids; i++)
3991 t->atid_tab[i - 1].next = &t->atid_tab[i];
3992 t->atid_tab[t->natids - 1].next = NULL;
4000 t = &sc->tids;
4002 KASSERT(t->atids_in_use == 0,
4003 ("%s: %d atids still in use.", __func__, t->atids_in_use));
4005 if (mtx_initialized(&t->atid_lock))
4006 mtx_destroy(&t->atid_lock);
4007 free(t->atid_tab, M_CXGBE);
4008 t->atid_tab = NULL;
4014 struct tid_info *t = &sc->tids;
4016 mtx_lock(&t->atid_lock);
4017 t->atid_alloc_stopped = true;
4018 mtx_unlock(&t->atid_lock);
4024 struct tid_info *t = &sc->tids;
4026 mtx_lock(&t->atid_lock);
4027 KASSERT(t->atids_in_use == 0,
4028 ("%s: %d atids still in use.", __func__, t->atids_in_use));
4029 t->atid_alloc_stopped = false;
4030 mtx_unlock(&t->atid_lock);
4036 struct tid_info *t = &sc->tids;
4037 int atid = -1;
4039 mtx_lock(&t->atid_lock);
4040 if (t->afree && !t->atid_alloc_stopped) {
4041 union aopen_entry *p = t->afree;
4043 atid = p - t->atid_tab;
4045 t->afree = p->next;
4046 p->data = ctx;
4047 t->atids_in_use++;
4049 mtx_unlock(&t->atid_lock);
4056 struct tid_info *t = &sc->tids;
4058 return (t->atid_tab[atid].data);
4064 struct tid_info *t = &sc->tids;
4065 union aopen_entry *p = &t->atid_tab[atid];
4067 mtx_lock(&t->atid_lock);
4068 p->next = t->afree;
4069 t->afree = p;
4070 t->atids_in_use--;
4071 mtx_unlock(&t->atid_lock);
4102 return ((const struct t4_range *)a)->start -
4103 ((const struct t4_range *)b)->start;
4129 r->size = G_EDRAM0_SIZE(addr_len) << 20;
4130 if (r->size > 0) {
4131 r->start = G_EDRAM0_BASE(addr_len) << 20;
4132 if (addr >= r->start &&
4133 addr + len <= r->start + r->size)
4141 r->size = G_EDRAM1_SIZE(addr_len) << 20;
4142 if (r->size > 0) {
4143 r->start = G_EDRAM1_BASE(addr_len) << 20;
4144 if (addr >= r->start &&
4145 addr + len <= r->start + r->size)
4153 r->size = G_EXT_MEM_SIZE(addr_len) << 20;
4154 if (r->size > 0) {
4155 r->start = G_EXT_MEM_BASE(addr_len) << 20;
4156 if (addr >= r->start &&
4157 addr + len <= r->start + r->size)
4165 r->size = G_EXT_MEM1_SIZE(addr_len) << 20;
4166 if (r->size > 0) {
4167 r->start = G_EXT_MEM1_BASE(addr_len) << 20;
4168 if (addr >= r->start &&
4169 addr + len <= r->start + r->size)
4181 /* Start from index 0 and examine the next n - 1 entries. */
4183 for (remaining = n - 1; remaining > 0; remaining--, r++) {
4185 MPASS(r->size > 0); /* r is a valid entry. */
4187 MPASS(next->size > 0); /* and so is the next one. */
4189 while (r->start + r->size >= next->start) {
4191 r->size = max(r->start + r->size,
4192 next->start + next->size) - r->start;
4193 n--; /* One fewer entry in total. */
4194 if (--remaining == 0)
4204 MPASS(next->size > 0); /* must be valid */
4208 * This so that the foo->size assertion in the
4214 bzero(&mem_ranges[n], (nitems(mem_ranges) - n) *
4224 if (addr >= r->start &&
4225 addr + len <= r->start + r->size)
4303 struct devlog_params *dparams = &sc->params.devlog;
4306 rc = validate_mt_off_len(sc, dparams->memtype, dparams->start,
4307 dparams->size, &dparams->addr);
4316 iaq->nirq = T4_EXTRA_INTR;
4317 iaq->nirq += nports * max(iaq->nrxq, iaq->nnmrxq);
4318 iaq->nirq += nports * iaq->nofldrxq;
4319 iaq->nirq += nports * (iaq->num_vis - 1) *
4320 max(iaq->nrxq_vi, iaq->nnmrxq_vi);
4321 iaq->nirq += nports * (iaq->num_vis - 1) * iaq->nofldrxq_vi;
4332 const int nports = sc->params.nports;
4338 iaq->intr_type = itype;
4339 iaq->num_vis = t4_num_vis;
4340 iaq->ntxq = t4_ntxq;
4341 iaq->ntxq_vi = t4_ntxq_vi;
4342 iaq->nrxq = t4_nrxq;
4343 iaq->nrxq_vi = t4_nrxq_vi;
4346 iaq->nofldtxq = t4_nofldtxq;
4347 iaq->nofldtxq_vi = t4_nofldtxq_vi;
4352 iaq->nofldrxq = t4_nofldrxq;
4353 iaq->nofldrxq_vi = t4_nofldrxq_vi;
4358 iaq->nnmtxq = t4_nnmtxq;
4359 iaq->nnmrxq = t4_nnmrxq;
4362 iaq->nnmtxq_vi = t4_nnmtxq_vi;
4363 iaq->nnmrxq_vi = t4_nnmrxq_vi;
4368 if (iaq->nirq <= navail &&
4369 (itype != INTR_MSI || powerof2(iaq->nirq))) {
4371 * This is the normal case -- there are enough interrupts for
4381 while (iaq->num_vis > 1) {
4382 iaq->num_vis--;
4384 if (iaq->nirq <= navail &&
4385 (itype != INTR_MSI || powerof2(iaq->nirq))) {
4386 device_printf(sc->dev, "virtual interfaces per port "
4390 iaq->num_vis, t4_num_vis, iaq->nrxq, iaq->nofldrxq,
4391 iaq->nrxq_vi, iaq->nofldrxq_vi, iaq->nnmrxq_vi,
4392 itype, navail, iaq->nirq);
4400 MPASS(iaq->num_vis == 1);
4401 iaq->ntxq_vi = iaq->nrxq_vi = 0;
4402 iaq->nofldtxq_vi = iaq->nofldrxq_vi = 0;
4403 iaq->nnmtxq_vi = iaq->nnmrxq_vi = 0;
4404 if (iaq->num_vis != t4_num_vis) {
4405 device_printf(sc->dev, "extra virtual interfaces disabled. "
4408 iaq->nrxq, iaq->nofldrxq, iaq->nrxq_vi, iaq->nofldrxq_vi,
4409 iaq->nnmrxq_vi, itype, navail, iaq->nirq);
4418 if (iaq->nrxq > 1) {
4419 iaq->nrxq = rounddown_pow_of_two(iaq->nrxq - 1);
4420 if (iaq->nnmrxq > iaq->nrxq)
4421 iaq->nnmrxq = iaq->nrxq;
4423 if (iaq->nofldrxq > 1)
4424 iaq->nofldrxq >>= 1;
4426 old_nirq = iaq->nirq;
4428 if (iaq->nirq <= navail &&
4429 (itype != INTR_MSI || powerof2(iaq->nirq))) {
4430 device_printf(sc->dev, "running with reduced number of "
4433 "itype %d, navail %u, nirq %d.\n", iaq->nrxq,
4434 iaq->nofldrxq, itype, navail, iaq->nirq);
4437 } while (old_nirq != iaq->nirq);
4440 device_printf(sc->dev, "running with minimal number of queues. "
4442 iaq->nirq = 1;
4443 iaq->nrxq = 1;
4444 iaq->ntxq = 1;
4445 if (iaq->nofldrxq > 0) {
4446 iaq->nofldrxq = 1;
4447 iaq->nofldtxq = 1;
4449 iaq->nnmtxq = 0;
4450 iaq->nnmrxq = 0;
4452 MPASS(iaq->num_vis > 0);
4453 if (iaq->num_vis > 1) {
4454 MPASS(iaq->nrxq_vi > 0);
4455 MPASS(iaq->ntxq_vi > 0);
4457 MPASS(iaq->nirq > 0);
4458 MPASS(iaq->nrxq > 0);
4459 MPASS(iaq->ntxq > 0);
4461 MPASS(powerof2(iaq->nirq));
4476 navail = pci_msix_count(sc->dev);
4478 navail = pci_msi_count(sc->dev);
4486 nalloc = iaq->nirq;
4489 rc = pci_alloc_msix(sc->dev, &nalloc);
4491 rc = pci_alloc_msi(sc->dev, &nalloc);
4494 if (nalloc == iaq->nirq)
4501 device_printf(sc->dev, "fewer vectors than requested, "
4503 itype, iaq->nirq, nalloc);
4504 pci_release_msi(sc->dev);
4509 device_printf(sc->dev,
4511 itype, rc, iaq->nirq, nalloc);
4514 device_printf(sc->dev,
4516 "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types,
4517 pci_msix_count(sc->dev), pci_msi_count(sc->dev));
4628 if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver)
4635 #define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x)
4636 if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) &&
4657 device_printf(sc->dev,
4663 *dcfg = firmware_get(fw_info->kld_name);
4666 *fw = firmware_get(fw_info->fw_mod_name);
4695 const uint32_t c = be32toh(card_fw->fw_ver);
4703 fw_install = t4_fw_install < 0 ? -t4_fw_install : t4_fw_install;
4709 device_printf(sc->dev,
4711 " will use compiled-in firmware version for"
4715 memcpy(&bundled_fw, fw->data, sizeof(bundled_fw));
4724 if ((sc->flags & FW_OK) == 0) {
4760 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
4776 device_printf(sc->dev,
4783 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
4788 rc = sc->flags & FW_OK ? 0 : ENOENT;
4791 k = be32toh(((const struct fw_hdr *)fw->data)->fw_ver);
4794 device_printf(sc->dev,
4801 rc = sc->flags & FW_OK ? 0 : EINVAL;
4805 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
4812 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0);
4814 device_printf(sc->dev, "failed to install firmware: %d\n", rc);
4818 memcpy(card_fw, fw->data, sizeof(*card_fw));
4843 device_printf(sc->dev,
4848 drv_fw = &fw_info->fw_h;
4853 rc = -t4_get_fw_hdr(sc, card_fw);
4855 device_printf(sc->dev,
4868 rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state);
4870 rc = -rc;
4871 device_printf(sc->dev,
4881 MPASS(be32toh(card_fw->flags) & FW_HDR_FLAGS_RESET_HALT);
4882 sc->flags |= FW_OK; /* The firmware responded to the FW_HELLO. */
4884 if (rc == sc->pf) {
4885 sc->flags |= MASTER_PF;
4898 device_printf(sc->dev, "couldn't be master(%d), "
4908 device_printf(sc->dev, "PF%d is master, device state %d. "
4910 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", rc);
4911 sc->cfcsum = 0;
4915 if (rc != 0 && sc->flags & FW_OK) {
4916 t4_fw_bye(sc, sc->mbox);
4917 sc->flags &= ~FW_OK;
4937 if (pci_get_device(sc->dev) == 0x440a)
4945 device_printf(sc->dev,
4950 cfdata = dcfg->data;
4951 cflen = dcfg->datasize & ~3;
4957 device_printf(sc->dev,
4963 snprintf(s, sizeof(s), "%s_%s", fw_info->kld_name, cfg_file);
4967 device_printf(sc->dev,
4973 cfdata = rcfg->data;
4974 cflen = rcfg->datasize & ~3;
4978 device_printf(sc->dev,
4979 "config file too long (%d, max allowed is %d).\n",
4987 device_printf(sc->dev,
5033 rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST);
5035 device_printf(sc->dev, "firmware reset failed: %d.\n", rc);
5058 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val);
5061 device_printf(sc->dev,
5074 device_printf(sc->dev,
5079 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
5081 device_printf(sc->dev, "failed to pre-process config file: %d "
5089 device_printf(sc->dev,
5093 sc->cfcsum = cfcsum;
5094 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", cfg_file);
5101 caps.x##caps &= htobe16(caps_allowed->x##caps); \
5117 * to cope with the situation in non-debug builds by disabling
5130 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL);
5132 device_printf(sc->dev,
5141 rc = -t4_fw_initialize(sc, sc->mbox);
5143 device_printf(sc->dev, "fw_initialize failed: %d.\n", rc);
5162 MPASS(sc->flags & MASTER_PF);
5177 fallback = sc->debug_flags & DF_DISABLE_CFG_RETRY ? false : true;
5183 device_printf(sc->dev,
5209 snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u",
5210 G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers),
5211 G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers),
5212 G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers),
5213 G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers));
5215 snprintf(sc->bs_version, sizeof(sc->bs_version), "%u.%u.%u.%u",
5216 G_FW_HDR_FW_VER_MAJOR(sc->params.bs_vers),
5217 G_FW_HDR_FW_VER_MINOR(sc->params.bs_vers),
5218 G_FW_HDR_FW_VER_MICRO(sc->params.bs_vers),
5219 G_FW_HDR_FW_VER_BUILD(sc->params.bs_vers));
5221 snprintf(sc->tp_version, sizeof(sc->tp_version), "%u.%u.%u.%u",
5222 G_FW_HDR_FW_VER_MAJOR(sc->params.tp_vers),
5223 G_FW_HDR_FW_VER_MINOR(sc->params.tp_vers),
5224 G_FW_HDR_FW_VER_MICRO(sc->params.tp_vers),
5225 G_FW_HDR_FW_VER_BUILD(sc->params.tp_vers));
5227 snprintf(sc->er_version, sizeof(sc->er_version), "%u.%u.%u.%u",
5228 G_FW_HDR_FW_VER_MAJOR(sc->params.er_vers),
5229 G_FW_HDR_FW_VER_MINOR(sc->params.er_vers),
5230 G_FW_HDR_FW_VER_MICRO(sc->params.er_vers),
5231 G_FW_HDR_FW_VER_BUILD(sc->params.er_vers));
5235 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
5237 device_printf(sc->dev,
5242 sc->params.portvec = val[0];
5243 sc->params.nports = bitcount32(val[0]);
5244 sc->params.vpd.cclk = val[1];
5247 rc = -t4_init_devlog_params(sc, 1);
5251 device_printf(sc->dev,
5271 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val);
5274 sc->params.fw_vers < FW_VERSION32(1, 20, 1, 0)) {
5278 device_printf(sc->dev,
5284 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val);
5286 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m,
5289 device_printf(sc->dev,
5298 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val);
5300 sc->params.viid_smt_extn_support = true;
5302 sc->params.viid_smt_extn_support = false;
5327 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 7, param, val);
5329 device_printf(sc->dev,
5334 sc->sge.iq_start = val[0];
5335 sc->sge.eq_start = val[1];
5337 sc->tids.ftid_base = val[2];
5338 sc->tids.ftid_end = val[3];
5339 sc->tids.nftids = val[3] - val[2] + 1;
5341 sc->vres.l2t.start = val[4];
5342 sc->vres.l2t.size = val[5] - val[4] + 1;
5344 if (sc->vres.l2t.size > 0)
5346 sc->params.core_vdd = val[6];
5350 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
5352 device_printf(sc->dev,
5356 MPASS((int)val[0] >= sc->sge.iq_start);
5357 sc->sge.iqmap_sz = val[0] - sc->sge.iq_start + 1;
5358 MPASS((int)val[1] >= sc->sge.eq_start);
5359 sc->sge.eqmap_sz = val[1] - sc->sge.eq_start + 1;
5363 sc->tids.tid_base = t4_read_reg(sc,
5368 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
5370 device_printf(sc->dev,
5375 sc->tids.hpftid_base = val[0];
5376 sc->tids.hpftid_end = val[1];
5377 sc->tids.nhpftids = val[1] - val[0] + 1;
5383 MPASS(sc->tids.hpftid_base == 0);
5384 MPASS(sc->tids.tid_base == sc->tids.nhpftids);
5389 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
5391 device_printf(sc->dev,
5396 sc->rawf_base = val[0];
5397 sc->nrawf = val[1] - val[0] + 1;
5412 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, param, val);
5414 sc->params.mps_bg_map = val[0];
5416 sc->params.mps_bg_map = UINT32_MAX; /* Not a legal value. */
5420 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, param, val);
5422 sc->params.tp_ch_map = val[0];
5424 sc->params.tp_ch_map = UINT32_MAX; /* Not a legal value. */
5431 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, param, val);
5433 sc->params.filter2_wr_support = val[0] != 0;
5435 sc->params.filter2_wr_support = 0;
5442 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, param, val);
5444 sc->params.ulptx_memwrite_dsgl = val[0] != 0;
5446 sc->params.ulptx_memwrite_dsgl = false;
5450 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, param, val);
5452 sc->params.fr_nsmr_tpte_wr_support = val[0] != 0;
5454 sc->params.fr_nsmr_tpte_wr_support = false;
5458 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, param, val);
5460 sc->params.dev_512sgl_mr = val[0] != 0;
5462 sc->params.dev_512sgl_mr = false;
5465 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, param, val);
5467 sc->params.max_pkts_per_eth_tx_pkts_wr = val[0];
5469 sc->params.max_pkts_per_eth_tx_pkts_wr = 15;
5472 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, param, val);
5475 sc->params.nsched_cls = val[0];
5477 sc->params.nsched_cls = sc->chip_params->nsched_cls;
5484 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
5486 device_printf(sc->dev,
5492 sc->x = htobe16(caps.x); \
5504 if (sc->niccaps & FW_CAPS_CONFIG_NIC_HASHFILTER) {
5506 MPASS(sc->toecaps == 0);
5507 sc->toecaps = 0;
5510 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, param, val);
5512 device_printf(sc->dev,
5516 sc->tids.ntids = val[0];
5517 if (sc->params.fw_vers < FW_VERSION32(1, 20, 5, 0)) {
5518 MPASS(sc->tids.ntids >= sc->tids.nhpftids);
5519 sc->tids.ntids -= sc->tids.nhpftids;
5521 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS);
5522 sc->params.hash_filter = 1;
5524 if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) {
5528 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val);
5530 device_printf(sc->dev,
5535 sc->tids.etid_base = val[0];
5536 sc->tids.etid_end = val[1];
5537 sc->tids.netids = val[1] - val[0] + 1;
5538 sc->params.eo_wr_cred = val[2];
5539 sc->params.ethoffload = 1;
5542 if (sc->toecaps) {
5543 /* query offload-related parameters */
5550 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
5552 device_printf(sc->dev,
5556 sc->tids.ntids = val[0];
5557 if (sc->params.fw_vers < FW_VERSION32(1, 20, 5, 0)) {
5558 MPASS(sc->tids.ntids >= sc->tids.nhpftids);
5559 sc->tids.ntids -= sc->tids.nhpftids;
5561 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS);
5563 sc->tids.stid_base = val[1];
5564 sc->tids.nstids = val[2] - val[1] + 1;
5566 sc->vres.ddp.start = val[3];
5567 sc->vres.ddp.size = val[4] - val[3] + 1;
5568 sc->params.ofldq_wr_cred = val[5];
5569 sc->params.offload = 1;
5572 * The firmware attempts memfree TOE configuration for -SO cards
5579 sc->iscsicaps = 0;
5580 sc->rdmacaps = 0;
5582 if (sc->rdmacaps) {
5589 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
5591 device_printf(sc->dev,
5595 sc->vres.stag.start = val[0];
5596 sc->vres.stag.size = val[1] - val[0] + 1;
5597 sc->vres.rq.start = val[2];
5598 sc->vres.rq.size = val[3] - val[2] + 1;
5599 sc->vres.pbl.start = val[4];
5600 sc->vres.pbl.size = val[5] - val[4] + 1;
5608 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
5610 device_printf(sc->dev,
5614 sc->vres.qp.start = val[0];
5615 sc->vres.qp.size = val[1] - val[0] + 1;
5616 sc->vres.cq.start = val[2];
5617 sc->vres.cq.size = val[3] - val[2] + 1;
5618 sc->vres.ocq.start = val[4];
5619 sc->vres.ocq.size = val[5] - val[4] + 1;
5625 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 4, param, val);
5627 device_printf(sc->dev,
5631 sc->vres.srq.start = val[0];
5632 sc->vres.srq.size = val[1] - val[0] + 1;
5633 sc->params.max_ordird_qp = val[2];
5634 sc->params.max_ird_adapter = val[3];
5636 if (sc->iscsicaps) {
5639 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
5641 device_printf(sc->dev,
5645 sc->vres.iscsi.start = val[0];
5646 sc->vres.iscsi.size = val[1] - val[0] + 1;
5648 if (sc->cryptocaps & FW_CAPS_CONFIG_TLSKEYS) {
5651 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
5653 device_printf(sc->dev,
5657 sc->vres.key.start = val[0];
5658 sc->vres.key.size = val[1] - val[0] + 1;
5667 t4_read_mtu_tbl(sc, sc->params.mtus, NULL);
5668 t4_load_mtus(sc, sc->params.mtus, sc->params.a_wnd, sc->params.b_wnd);
5689 callout_schedule_sbt(&sc->ktls_tick, SBT_1MS, 0, C_HARDCLOCK);
5701 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, ¶m);
5709 sc->flags |= KERN_TLS_ON;
5710 callout_reset_sbt(&sc->ktls_tick, SBT_1MS, 0, ktls_tick, sc,
5713 sc->flags &= ~KERN_TLS_ON;
5714 callout_stop(&sc->ktls_tick);
5732 (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val);
5737 if (t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val) == 0)
5738 sc->params.port_caps32 = 1;
5741 val = 1 << (G_MASKSIZE(t4_read_reg(sc, A_TP_RSS_CONFIG_TNL)) - 1);
5743 V_MASKFILTER(val - 1));
5819 if (t4_toe_rexmt_backoff[i] != -1) {
5838 sc->tlst.inline_keys = t4_tls_inline_keys;
5839 sc->tlst.combo_wrs = t4_tls_combo_wrs;
5853 struct adapter_params *p = &sc->params;
5855 device_set_descf(sc->dev, "Chelsio %s", p->vpd.id);
5884 ifm = &pi->media;
5885 if (ifm->ifm_cur != NULL &&
5886 IFM_SUBTYPE(ifm->ifm_cur->ifm_media) == IFM_NONE)
5889 lc = &pi->link_cfg;
5890 if (lc->requested_aneg != AUTONEG_DISABLE &&
5891 lc->pcaps & FW_PORT_CAP32_ANEG) {
5896 if (lc->requested_fc & PAUSE_TX)
5898 if (lc->requested_fc & PAUSE_RX)
5900 if (lc->requested_speed == 0)
5901 speed = port_top_speed(pi) * 1000; /* Gbps -> Mbps */
5903 speed = lc->requested_speed;
5915 return (pi->port_type == FW_PORT_TYPE_BT_SGMII ||
5916 pi->port_type == FW_PORT_TYPE_BT_XFI ||
5917 pi->port_type == FW_PORT_TYPE_BT_XAUI ||
5918 pi->port_type == FW_PORT_TYPE_KX4 ||
5919 pi->port_type == FW_PORT_TYPE_KX ||
5920 pi->port_type == FW_PORT_TYPE_KR ||
5921 pi->port_type == FW_PORT_TYPE_BP_AP ||
5922 pi->port_type == FW_PORT_TYPE_BP4_AP ||
5923 pi->port_type == FW_PORT_TYPE_BP40_BA ||
5924 pi->port_type == FW_PORT_TYPE_KR4_100G ||
5925 pi->port_type == FW_PORT_TYPE_KR_SFP28 ||
5926 pi->port_type == FW_PORT_TYPE_KR_XLAUI);
5939 if (pi->flags & FIXED_IFMEDIA)
5945 ifm = &pi->media;
5947 lc = &pi->link_cfg;
5948 ss = G_FW_PORT_CAP32_SPEED(lc->pcaps); /* Supported Speeds */
5952 MPASS(LIST_EMPTY(&ifm->ifm_list));
5974 if (lc->pcaps & FW_PORT_CAP32_ANEG)
5986 struct link_config *lc = &pi->link_cfg;
5990 lc->requested_caps = 0;
5991 lc->requested_speed = 0;
5994 lc->requested_aneg = AUTONEG_DISABLE;
5996 lc->requested_aneg = AUTONEG_ENABLE;
5998 lc->requested_aneg = AUTONEG_AUTO;
6000 lc->requested_fc = t4_pause_settings & (PAUSE_TX | PAUSE_RX |
6004 lc->requested_fec = FEC_AUTO;
6006 lc->requested_fec = FEC_NONE;
6008 /* -1 is handled by the FEC_AUTO block above and not here. */
6009 lc->requested_fec = t4_fec &
6011 if (lc->requested_fec == 0)
6012 lc->requested_fec = FEC_AUTO;
6015 lc->force_fec = -1;
6017 lc->force_fec = 1;
6019 lc->force_fec = 0;
6030 struct link_config *lc = &pi->link_cfg;
6036 if (lc->requested_speed != 0) {
6037 fwspeed = speed_to_fwcap(lc->requested_speed);
6038 if ((fwspeed & lc->pcaps) == 0) {
6040 lc->requested_speed = 0;
6045 MPASS(lc->requested_aneg == AUTONEG_ENABLE ||
6046 lc->requested_aneg == AUTONEG_DISABLE ||
6047 lc->requested_aneg == AUTONEG_AUTO);
6048 if (lc->requested_aneg == AUTONEG_ENABLE &&
6049 !(lc->pcaps & FW_PORT_CAP32_ANEG)) {
6051 lc->requested_aneg = AUTONEG_AUTO;
6055 MPASS((lc->requested_fc & ~(PAUSE_TX | PAUSE_RX | PAUSE_AUTONEG)) == 0);
6056 if (lc->requested_fc & PAUSE_TX &&
6057 !(lc->pcaps & FW_PORT_CAP32_FC_TX)) {
6059 lc->requested_fc &= ~PAUSE_TX;
6061 if (lc->requested_fc & PAUSE_RX &&
6062 !(lc->pcaps & FW_PORT_CAP32_FC_RX)) {
6064 lc->requested_fc &= ~PAUSE_RX;
6066 if (!(lc->requested_fc & PAUSE_AUTONEG) &&
6067 !(lc->pcaps & FW_PORT_CAP32_FORCE_PAUSE)) {
6069 lc->requested_fc |= PAUSE_AUTONEG;
6073 if ((lc->requested_fec & FEC_RS &&
6074 !(lc->pcaps & FW_PORT_CAP32_FEC_RS)) ||
6075 (lc->requested_fec & FEC_BASER_RS &&
6076 !(lc->pcaps & FW_PORT_CAP32_FEC_BASER_RS))) {
6078 lc->requested_fec = FEC_AUTO;
6091 struct adapter *sc = pi->adapter;
6092 struct link_config *lc = &pi->link_cfg;
6099 if (lc->requested_aneg == AUTONEG_ENABLE)
6100 MPASS(lc->pcaps & FW_PORT_CAP32_ANEG);
6101 if (!(lc->requested_fc & PAUSE_AUTONEG))
6102 MPASS(lc->pcaps & FW_PORT_CAP32_FORCE_PAUSE);
6103 if (lc->requested_fc & PAUSE_TX)
6104 MPASS(lc->pcaps & FW_PORT_CAP32_FC_TX);
6105 if (lc->requested_fc & PAUSE_RX)
6106 MPASS(lc->pcaps & FW_PORT_CAP32_FC_RX);
6107 if (lc->requested_fec & FEC_RS)
6108 MPASS(lc->pcaps & FW_PORT_CAP32_FEC_RS);
6109 if (lc->requested_fec & FEC_BASER_RS)
6110 MPASS(lc->pcaps & FW_PORT_CAP32_FEC_BASER_RS);
6112 if (!(sc->flags & IS_VF)) {
6113 rc = -t4_link_l1cfg(sc, sc->mbox, pi->tx_chan, lc);
6115 device_printf(pi->dev, "l1cfg failed: %d\n", rc);
6121 * An L1_CFG will almost always result in a link-change event if the
6129 if (lc->link_ok && !(lc->requested_fc & PAUSE_AUTONEG))
6130 lc->fc = lc->requested_fc & (PAUSE_TX | PAUSE_RX);
6149 struct vi_info *vi = if_getsoftc(ctx->ifp);
6150 struct port_info *pi = vi->pi;
6151 struct adapter *sc = pi->adapter;
6153 if (ctx->rc < 0)
6156 ctx->mcaddr[ctx->i] = LLADDR(sdl);
6157 MPASS(ETHER_IS_MULTICAST(ctx->mcaddr[ctx->i]));
6158 ctx->i++;
6160 if (ctx->i == FW_MAC_EXACT_CHUNK) {
6161 ctx->rc = t4_alloc_mac_filt(sc, sc->mbox, vi->viid, ctx->del,
6162 ctx->i, ctx->mcaddr, NULL, &ctx->hash, 0);
6163 if (ctx->rc < 0) {
6166 for (j = 0; j < ctx->i; j++) {
6167 if_printf(ctx->ifp,
6171 ctx->mcaddr[j][0], ctx->mcaddr[j][1],
6172 ctx->mcaddr[j][2], ctx->mcaddr[j][3],
6173 ctx->mcaddr[j][4], ctx->mcaddr[j][5],
6174 -ctx->rc);
6178 ctx->del = 0;
6179 ctx->i = 0;
6194 struct port_info *pi = vi->pi;
6195 struct adapter *sc = pi->adapter;
6196 int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1;
6215 rc = -t4_set_rxmode(sc, sc->mbox, vi->viid, mtu, promisc,
6228 rc = t4_change_mac(sc, sc->mbox, vi->viid, vi->xact_addr_filt,
6229 ucaddr, true, &vi->smt_idx);
6231 rc = -rc;
6235 vi->xact_addr_filt = rc;
6260 rc = -ctx.rc;
6264 rc = t4_alloc_mac_filt(sc, sc->mbox, vi->viid,
6268 rc = -rc;
6285 rc = -t4_set_addr_hash(sc, sc->mbox, vi->viid, 0, ctx.hash, 0);
6291 pi->vxlan_tcam_entry = false;
6295 if (IS_MAIN_VI(vi) && sc->vxlan_refcount > 0 &&
6296 pi->vxlan_tcam_entry == false) {
6297 rc = t4_alloc_raw_mac_filt(sc, vi->viid, match_all_mac,
6298 match_all_mac, sc->rawf_base + pi->port_id, 1, pi->port_id,
6301 rc = -rc;
6305 MPASS(rc == sc->rawf_base + pi->port_id);
6307 pi->vxlan_tcam_entry = true;
6353 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) {
6362 sc->last_op = wmesg;
6363 sc->last_op_thr = curthread;
6364 sc->last_op_flags = flags;
6384 wakeup(&sc->flags);
6386 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0);
6389 sc->last_op = "t4detach";
6390 sc->last_op_thr = curthread;
6391 sc->last_op_flags = 0;
6403 wakeup(&sc->flags);
6421 wakeup(&sc->flags);
6428 struct port_info *pi = vi->pi;
6429 struct adapter *sc = pi->adapter;
6430 if_t ifp = vi->ifp;
6439 if (!(sc->flags & FULL_INIT_DONE) && ((rc = adapter_init(sc)) != 0))
6442 if (!(vi->flags & VI_INIT_DONE) && ((rc = vi_init(vi)) != 0))
6450 if (pi->up_vis == 0) {
6457 rc = -t4_enable_vi(sc, sc->mbox, vi->viid, true, true);
6471 txq->eq.flags |= EQ_ENABLED;
6478 if (sc->traceq < 0 && IS_MAIN_VI(vi)) {
6479 sc->traceq = sc->sge.rxq[vi->first_rxq].iq.abs_id;
6481 A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) |
6482 V_QUEUENUMBER(sc->traceq));
6483 pi->flags |= HAS_TRACEQ;
6487 pi->up_vis++;
6489 if (pi->link_cfg.link_ok)
6493 mtx_lock(&vi->tick_mtx);
6494 if (vi->pi->nvi > 1 || sc->flags & IS_VF)
6495 callout_reset(&vi->tick, hz, vi_tick, vi);
6497 callout_reset(&vi->tick, hz, cxgbe_tick, vi);
6498 mtx_unlock(&vi->tick_mtx);
6512 struct port_info *pi = vi->pi;
6513 struct adapter *sc = pi->adapter;
6514 if_t ifp = vi->ifp;
6520 if (!(vi->flags & VI_INIT_DONE)) {
6524 "vi->flags 0x%016lx, if_flags 0x%08x, "
6525 "if_drv_flags 0x%08x\n", vi->flags, if_getflags(ifp),
6538 rc = -t4_enable_vi(sc, sc->mbox, vi->viid, false, false);
6546 txq->eq.flags &= ~EQ_ENABLED;
6550 mtx_lock(&vi->tick_mtx);
6551 callout_stop(&vi->tick);
6552 mtx_unlock(&vi->tick_mtx);
6560 pi->up_vis--;
6561 if (pi->up_vis > 0) {
6566 pi->link_cfg.link_ok = false;
6567 pi->link_cfg.speed = 0;
6568 pi->link_cfg.link_down_rc = 255;
6577 * will walk the entire sc->irq list and clean up whatever is valid.
6587 struct sge *sge = &sc->sge;
6602 irq = &sc->irq[0];
6603 rid = sc->intr_type == INTR_INTX ? 0 : 1;
6608 if (sc->flags & IS_VF)
6609 KASSERT(sc->intr_count >= T4VF_EXTRA_INTR + sc->params.nports,
6612 KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports,
6616 if (!(sc->flags & IS_VF)) {
6625 rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sge->fwq, "evt");
6632 pi = sc->port[p];
6634 vi->first_intr = rid - 1;
6636 if (vi->nnmrxq > 0) {
6637 int n = max(vi->nrxq, vi->nnmrxq);
6639 rxq = &sge->rxq[vi->first_rxq];
6641 nm_rxq = &sge->nm_rxq[vi->first_nm_rxq];
6646 if (q < vi->nrxq)
6647 irq->rxq = rxq++;
6649 if (q < vi->nnmrxq)
6650 irq->nm_rxq = nm_rxq++;
6652 if (irq->nm_rxq != NULL &&
6653 irq->rxq == NULL) {
6656 t4_nm_intr, irq->nm_rxq, s);
6658 if (irq->nm_rxq != NULL &&
6659 irq->rxq != NULL) {
6665 if (irq->rxq != NULL &&
6666 irq->nm_rxq == NULL) {
6669 t4_intr, irq->rxq, s);
6674 if (q < vi->nrxq) {
6675 bus_bind_intr(sc->dev, irq->res,
6681 vi->nintr++;
6692 bus_bind_intr(sc->dev, irq->res,
6697 vi->nintr++;
6709 vi->nintr++;
6714 MPASS(irq == &sc->irq[sc->intr_count]);
6731 rss_key[i] = htobe32(raw_rss_key[nitems(rss_key) - 1 - i]);
6733 t4_write_rss_key(sc, &rss_key[0], -1, 1);
6754 MPASS(sc->params.nports <= nitems(sc->tq));
6755 for (i = 0; i < sc->params.nports; i++) {
6756 if (sc->tq[i] != NULL)
6758 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT,
6759 taskqueue_thread_enqueue, &sc->tq[i]);
6760 if (sc->tq[i] == NULL) {
6764 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d",
6765 device_get_nameunit(sc->dev), i);
6768 if (!(sc->flags & IS_VF)) {
6782 KASSERT((sc->flags & FULL_INIT_DONE) == 0,
6789 sc->flags |= FULL_INIT_DONE;
6804 for (i = 0; i < nitems(sc->tq); i++) {
6805 if (sc->tq[i] == NULL)
6807 taskqueue_free(sc->tq[i]);
6808 sc->tq[i] = NULL;
6811 sc->flags &= ~FULL_INIT_DONE;
6856 * enabling any 4-tuple hash is nonsense configuration.
6885 struct adapter *sc = vi->adapter;
6906 if (vi->nrxq > vi->rss_size) {
6908 "some queues will never receive traffic.\n", vi->nrxq,
6909 vi->rss_size);
6910 } else if (vi->rss_size % vi->nrxq) {
6912 "expect uneven traffic distribution.\n", vi->nrxq,
6913 vi->rss_size);
6916 if (vi->nrxq != nbuckets) {
6918 "performance will be impacted.\n", vi->nrxq, nbuckets);
6921 if (vi->rss == NULL)
6922 vi->rss = malloc(vi->rss_size * sizeof (*vi->rss), M_CXGBE,
6924 for (i = 0; i < vi->rss_size;) {
6927 j %= vi->nrxq;
6928 rxq = &sc->sge.rxq[vi->first_rxq + j];
6929 vi->rss[i++] = rxq->iq.abs_id;
6932 vi->rss[i++] = rxq->iq.abs_id;
6933 if (i == vi->rss_size)
6939 rc = -t4_config_rss_range(sc, sc->mbox, vi->viid, 0, vi->rss_size,
6940 vi->rss, vi->rss_size);
6947 vi->hashen = hashconfig_to_hashen(hashconfig);
6954 extra = hashen_to_hashconfig(vi->hashen) ^ hashconfig;
6971 CH_ALERT(vi, "IPv4 2-tuple hashing forced on.\n");
6973 CH_ALERT(vi, "TCP/IPv4 4-tuple hashing forced on.\n");
6975 CH_ALERT(vi, "IPv6 2-tuple hashing forced on.\n");
6977 CH_ALERT(vi, "TCP/IPv6 4-tuple hashing forced on.\n");
6979 CH_ALERT(vi, "UDP/IPv4 4-tuple hashing forced on.\n");
6981 CH_ALERT(vi, "UDP/IPv6 4-tuple hashing forced on.\n");
6983 vi->hashen = F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN |
6988 rc = -t4_config_vi_rss(sc, sc->mbox, vi->viid, vi->hashen, vi->rss[0],
7003 ASSERT_SYNCHRONIZED_OP(vi->adapter);
7004 KASSERT((vi->flags & VI_INIT_DONE) == 0,
7011 vi->flags |= VI_INIT_DONE;
7023 if (vi->flags & VI_INIT_DONE) {
7025 free(vi->rss, M_CXGBE);
7026 free(vi->nm_rss, M_CXGBE);
7030 vi->flags &= ~VI_INIT_DONE;
7036 struct sge_eq *eq = &txq->eq;
7037 struct sge_qstat *spg = (void *)&eq->desc[eq->sidx];
7039 MPASS(eq->flags & EQ_SW_ALLOCATED);
7040 MPASS(!(eq->flags & EQ_ENABLED));
7043 while (!mp_ring_is_idle(txq->r)) {
7044 mp_ring_check_drainage(txq->r, 4096);
7047 MPASS(txq->txp.npkt == 0);
7049 if (eq->flags & EQ_HW_ALLOCATED) {
7055 while (spg->cidx != htobe16(eq->pidx))
7057 while (eq->cidx != eq->pidx)
7065 while (eq->cidx != eq->pidx) {
7069 txsd = &txq->sdesc[eq->cidx];
7070 for (m = txsd->m; m != NULL; m = nextpkt) {
7071 nextpkt = m->m_nextpkt;
7072 m->m_nextpkt = NULL;
7075 IDXINCR(eq->cidx, txsd->desc_used, eq->sidx);
7077 spg->pidx = spg->cidx = htobe16(eq->cidx);
7088 while ((wr = STAILQ_FIRST(&wrq->wr_list)) != NULL) {
7089 STAILQ_REMOVE_HEAD(&wrq->wr_list, link);
7091 wrq->nwr_pending--;
7092 wrq->ndesc_needed -= howmany(wr->wr_len, EQ_ESIZE);
7096 MPASS(wrq->nwr_pending == 0);
7097 MPASS(wrq->ndesc_needed == 0);
7098 wrq->nwr_pending = 0;
7099 wrq->ndesc_needed = 0;
7107 while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED))
7111 MPASS(iq->flags & IQ_HAS_FL);
7113 mtx_lock(&sc->sfl_lock);
7115 fl->flags |= FL_DOOMED;
7117 callout_stop(&sc->sfl_callout);
7118 mtx_unlock(&sc->sfl_lock);
7120 KASSERT((fl->flags & FL_STARVING) == 0,
7124 if (!(iq->flags & IQ_HW_ALLOCATED))
7138 struct adapter *sc = vi->adapter;
7148 if (!(vi->flags & VI_INIT_DONE))
7157 quiesce_wrq(&ofld_txq->wrq);
7162 quiesce_iq_fl(sc, &rxq->iq, &rxq->fl);
7167 quiesce_iq_fl(sc, &ofld_rxq->iq, &ofld_rxq->fl);
7178 irq->rid = rid;
7179 irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid,
7181 if (irq->res == NULL) {
7182 device_printf(sc->dev,
7187 rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET,
7188 NULL, handler, arg, &irq->tag);
7190 device_printf(sc->dev,
7194 bus_describe_intr(sc->dev, irq->res, irq->tag, "%s", name);
7202 if (irq->tag)
7203 bus_teardown_intr(sc->dev, irq->res, irq->tag);
7204 if (irq->res)
7205 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res);
7216 regs->version = chip_id(sc) | chip_rev(sc) << 10;
7217 t4_get_regs(sc, buf, regs->len);
7244 if (sc->flags & IS_VF) {
7248 mtx_assert(&sc->reg_lock, MA_OWNED);
7264 if (!(sc->flags & IS_VF))
7265 mtx_lock(&sc->reg_lock);
7266 stats->tx_bcast_bytes = GET_STAT(TX_VF_BCAST_BYTES);
7267 stats->tx_bcast_frames = GET_STAT(TX_VF_BCAST_FRAMES);
7268 stats->tx_mcast_bytes = GET_STAT(TX_VF_MCAST_BYTES);
7269 stats->tx_mcast_frames = GET_STAT(TX_VF_MCAST_FRAMES);
7270 stats->tx_ucast_bytes = GET_STAT(TX_VF_UCAST_BYTES);
7271 stats->tx_ucast_frames = GET_STAT(TX_VF_UCAST_FRAMES);
7272 stats->tx_drop_frames = GET_STAT(TX_VF_DROP_FRAMES);
7273 stats->tx_offload_bytes = GET_STAT(TX_VF_OFFLOAD_BYTES);
7274 stats->tx_offload_frames = GET_STAT(TX_VF_OFFLOAD_FRAMES);
7275 stats->rx_bcast_bytes = GET_STAT(RX_VF_BCAST_BYTES);
7276 stats->rx_bcast_frames = GET_STAT(RX_VF_BCAST_FRAMES);
7277 stats->rx_mcast_bytes = GET_STAT(RX_VF_MCAST_BYTES);
7278 stats->rx_mcast_frames = GET_STAT(RX_VF_MCAST_FRAMES);
7279 stats->rx_ucast_bytes = GET_STAT(RX_VF_UCAST_BYTES);
7280 stats->rx_ucast_frames = GET_STAT(RX_VF_UCAST_FRAMES);
7281 stats->rx_err_frames = GET_STAT(RX_VF_ERR_FRAMES);
7282 if (!(sc->flags & IS_VF))
7283 mtx_unlock(&sc->reg_lock);
7306 mtx_assert(&vi->tick_mtx, MA_OWNED);
7308 if (vi->flags & VI_SKIP_STATS)
7313 if (timevalcmp(&tv, &vi->last_refreshed, <))
7316 t4_get_vi_stats(vi->adapter, vi->vin, &vi->stats);
7317 getmicrotime(&vi->last_refreshed);
7329 mtx_assert(&vi->tick_mtx, MA_OWNED);
7331 if (vi->flags & VI_SKIP_STATS)
7336 if (timevalcmp(&tv, &vi->last_refreshed, <))
7339 pi = vi->pi;
7340 sc = vi->adapter;
7342 t4_get_port_stats(sc, pi->port_id, &pi->stats);
7343 chan_map = pi->rx_e_chan_map;
7345 i = ffs(chan_map) - 1;
7346 mtx_lock(&sc->reg_lock);
7349 mtx_unlock(&sc->reg_lock);
7353 pi->tnl_cong_drops = tnl_cong_drops;
7354 getmicrotime(&vi->last_refreshed);
7363 mtx_assert(&vi->tick_mtx, MA_OWNED);
7366 callout_schedule(&vi->tick, hz);
7374 mtx_assert(&vi->tick_mtx, MA_OWNED);
7377 callout_schedule(&vi->tick, hz);
7405 struct sysctl_ctx_list *ctx = &sc->ctx;
7413 oid = device_get_sysctl_tree(sc->dev);
7416 sc->sc_do_rxcopy = 1;
7418 &sc->sc_do_rxcopy, 1, "Do RX copy of small frames");
7421 sc->params.nports, "# of ports");
7425 (uintptr_t)&sc->doorbells, sysctl_bitfield_8b, "A",
7429 sc->params.vpd.cclk, "core clock frequency (in KHz)");
7433 sc->params.sge.timer_val, sizeof(sc->params.sge.timer_val),
7438 sc->params.sge.counter_val, sizeof(sc->params.sge.counter_val),
7443 sc->lro_timeout = 100;
7445 &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)");
7448 &sc->debug_flags, 0, "flags to enable runtime debugging");
7451 CTLFLAG_RD, sc->tp_version, 0, "TP microcode version");
7454 CTLFLAG_RD, sc->fw_version, 0, "firmware version");
7456 if (sc->flags & IS_VF)
7463 CTLFLAG_RD, sc->params.vpd.sn, 0, "serial number");
7466 CTLFLAG_RD, sc->params.vpd.pn, 0, "part number");
7469 CTLFLAG_RD, sc->params.vpd.ec, 0, "engineering change");
7472 CTLFLAG_RD, sc->params.vpd.md, 0, "manufacturing diags version");
7475 CTLFLAG_RD, sc->params.vpd.na, 0, "network address");
7478 sc->er_version, 0, "expansion ROM version");
7481 sc->bs_version, 0, "bootstrap firmware version");
7484 NULL, sc->params.scfg_vers, "serial config version");
7487 NULL, sc->params.vpd_vers, "VPD version");
7490 CTLFLAG_RD, sc->cfg_file, 0, "configuration file");
7493 sc->cfcsum, "config file checksum");
7498 (uintptr_t)&sc->name, sysctl_bitfield_16b, "A", \
7513 NULL, sc->tids.nftids, "number of filters");
7540 &sc->swintr, 0, "software triggered interrupts");
7618 "CIM OBQ 6 (SGE0-RX)");
7623 "CIM OBQ 7 (SGE1-RX)");
7640 sysctl_ddp_stats, "A", "non-TCP DDP statistics");
7678 sysctl_meminfo, "A", "memory regions");
7745 CTLFLAG_RW, &sc->tlst.inline_keys, 0, "Always pass TLS "
7751 CTLFLAG_RW, &sc->tlst.combo_wrs, 0, "Attempt to "
7769 sc->tt.cong_algorithm = -1;
7771 CTLFLAG_RW, &sc->tt.cong_algorithm, 0, "congestion control "
7772 "(-1 = default, 0 = reno, 1 = tahoe, 2 = newreno, "
7775 sc->tt.sndbuf = -1;
7777 &sc->tt.sndbuf, 0, "hardware send buffer");
7779 sc->tt.ddp = 0;
7781 CTLFLAG_RW | CTLFLAG_SKIP, &sc->tt.ddp, 0, "");
7783 &sc->tt.ddp, 0, "Enable zero-copy aio_read(2)");
7785 sc->tt.rx_coalesce = -1;
7787 CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing");
7789 sc->tt.tls = 1;
7794 sc->tt.tx_align = -1;
7796 CTLFLAG_RW, &sc->tt.tx_align, 0, "chop and align payload");
7798 sc->tt.tx_zcopy = 0;
7800 CTLFLAG_RW, &sc->tt.tx_zcopy, 0,
7801 "Enable zero-copy aio_write(2)");
7803 sc->tt.cop_managed_offloading = !!t4_cop_managed_offloading;
7806 &sc->tt.cop_managed_offloading, 0,
7809 sc->tt.autorcvbuf_inc = 16 * 1024;
7811 CTLFLAG_RW, &sc->tt.autorcvbuf_inc, 0,
7814 sc->tt.update_hc_on_pmtu_change = 1;
7817 &sc->tt.update_hc_on_pmtu_change, 0,
7820 sc->tt.iso = 1;
7822 &sc->tt.iso, 0, "Enable iSCSI segmentation offload");
7858 "Persist timer max (us)");
7912 struct sysctl_ctx_list *ctx = &vi->ctx;
7919 oid = device_get_sysctl_tree(vi->dev);
7923 vi->viid, "VI identifer");
7925 &vi->nrxq, 0, "# of rx queues");
7927 &vi->ntxq, 0, "# of tx queues");
7929 &vi->first_rxq, 0, "index of first rx queue");
7931 &vi->first_txq, 0, "index of first tx queue");
7933 vi->rss_base, "start of RSS indirection table");
7935 vi->rss_size, "size of RSS indirection table");
7941 "Reserve queue 0 for non-flowid packets");
7944 if (vi->adapter->flags & IS_VF) {
7945 MPASS(vi->flags & TX_USES_VM_WR);
7955 if (vi->nofldrxq != 0) {
7957 &vi->nofldrxq, 0,
7960 CTLFLAG_RD, &vi->first_ofld_rxq, 0,
7973 if (vi->nofldtxq != 0) {
7975 &vi->nofldtxq, 0,
7978 CTLFLAG_RD, &vi->first_ofld_txq, 0,
7983 if (vi->nnmrxq != 0) {
7985 &vi->nnmrxq, 0, "# of netmap rx queues");
7987 &vi->nnmtxq, 0, "# of netmap tx queues");
7989 CTLFLAG_RD, &vi->first_nm_rxq, 0,
7992 CTLFLAG_RD, &vi->first_nm_txq, 0,
8015 struct sysctl_ctx_list *ctx = &pi->ctx;
8018 struct adapter *sc = pi->adapter;
8026 oid = device_get_sysctl_tree(pi->dev);
8032 if (pi->port_type == FW_PORT_TYPE_BT_XAUI) {
8058 "autonegotiation (-1 = not supported)");
8064 &pi->link_cfg.requested_caps, 0, "L1 config requested by driver");
8066 &pi->link_cfg.pcaps, 0, "port capabilities");
8068 &pi->link_cfg.acaps, 0, "advertised capabilities");
8070 &pi->link_cfg.lpacaps, 0, "link partner advertised capabilities");
8073 port_top_speed(pi), "max speed (in Gbps)");
8075 pi->mps_bg_map, "MPS buffer group map");
8077 NULL, pi->rx_e_chan_map, "TP rx e-channel map");
8079 pi->tx_chan, "TP tx c-channel");
8081 pi->rx_chan, "TP rx c-channel");
8083 if (sc->flags & IS_VF)
8094 CTLFLAG_RW, &pi->sched_params->pktsize, 0,
8095 "pktsize for per-flow cl-rl (0 means up to the driver )");
8097 CTLFLAG_RW, &pi->sched_params->burstsize, 0,
8098 "burstsize for per-flow cl-rl (0 means up to the driver)");
8099 for (i = 0; i < sc->params.nsched_cls; i++) {
8100 struct tx_cl_rl_params *tc = &pi->sched_params->cl_rl[i];
8107 CTLFLAG_RD, &tc->state, 0, "current state");
8110 (uintptr_t)&tc->flags, sysctl_bitfield_8b, "A", "flags");
8112 CTLFLAG_RD, &tc->refcount, 0, "references to this class");
8115 (pi->port_id << 16) | i, sysctl_tc_params, "A",
8126 &pi->tx_parse_error, 0,
8132 t4_port_reg(sc, pi->tx_chan, A_MPS_PORT_STAT_##stat##_L), \
8138 &pi->stats.name, desc)
8198 T4_PORTSTAT(rx_ovflow0, "# drops due to buffer-group 0 overflows");
8199 T4_PORTSTAT(rx_ovflow1, "# drops due to buffer-group 1 overflows");
8200 T4_PORTSTAT(rx_ovflow2, "# drops due to buffer-group 2 overflows");
8201 T4_PORTSTAT(rx_ovflow3, "# drops due to buffer-group 3 overflows");
8202 T4_PORTSTAT(rx_trunc0, "# of buffer-group 0 truncated packets");
8203 T4_PORTSTAT(rx_trunc1, "# of buffer-group 1 truncated packets");
8204 T4_PORTSTAT(rx_trunc2, "# of buffer-group 2 truncated packets");
8205 T4_PORTSTAT(rx_trunc3, "# of buffer-group 3 truncated packets");
8218 for (i = arg1; arg2; arg2 -= sizeof(int), i++) {
8268 struct adapter *sc = pi->adapter;
8272 rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK | INTR_OK, "t4btt");
8279 rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e,
8298 val = vi->rsrv_noflowq;
8300 if (rc != 0 || req->newptr == NULL)
8303 if ((val >= 1) && (vi->ntxq > 1))
8304 vi->rsrv_noflowq = 1;
8306 vi->rsrv_noflowq = 0;
8315 struct adapter *sc = vi->adapter;
8318 MPASS(!(sc->flags & IS_VF));
8320 val = vi->flags & TX_USES_VM_WR ? 1 : 0;
8322 if (rc != 0 || req->newptr == NULL)
8334 else if (if_getdrvflags(vi->ifp) & IFF_DRV_RUNNING) {
8342 struct port_info *pi = vi->pi;
8345 uint8_t npkt = sc->params.max_pkts_per_eth_tx_pkts_wr;
8348 vi->flags |= TX_USES_VM_WR;
8349 if_sethwtsomaxsegcount(vi->ifp, TX_SGL_SEGS_VM_TSO);
8351 V_TXPKT_INTF(pi->tx_chan));
8352 if (!(sc->flags & IS_VF))
8353 npkt--;
8355 vi->flags &= ~TX_USES_VM_WR;
8356 if_sethwtsomaxsegcount(vi->ifp, TX_SGL_SEGS_TSO);
8358 V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_PF(sc->pf) |
8359 V_TXPKT_VF(vi->vin) | V_TXPKT_VF_VLD(vi->vfvld));
8362 txq->cpl_ctrl0 = ctrl0;
8363 txq->txp.max_npkt = npkt;
8374 struct adapter *sc = vi->adapter;
8379 idx = vi->tmr_idx;
8382 if (rc != 0 || req->newptr == NULL)
8393 v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(vi->pktc_idx != -1);
8396 atomic_store_rel_8(&rxq->iq.intr_params, v);
8398 rxq->iq.intr_params = v;
8401 vi->tmr_idx = idx;
8411 struct adapter *sc = vi->adapter;
8414 idx = vi->pktc_idx;
8417 if (rc != 0 || req->newptr == NULL)
8420 if (idx < -1 || idx >= SGE_NCOUNTERS)
8428 if (vi->flags & VI_INIT_DONE)
8431 vi->pktc_idx = idx;
8441 struct adapter *sc = vi->adapter;
8444 qsize = vi->qsize_rxq;
8447 if (rc != 0 || req->newptr == NULL)
8458 if (vi->flags & VI_INIT_DONE)
8461 vi->qsize_rxq = qsize;
8471 struct adapter *sc = vi->adapter;
8474 qsize = vi->qsize_txq;
8477 if (rc != 0 || req->newptr == NULL)
8488 if (vi->flags & VI_INIT_DONE)
8491 vi->qsize_txq = qsize;
8501 struct adapter *sc = pi->adapter;
8502 struct link_config *lc = &pi->link_cfg;
8505 if (req->newptr == NULL) {
8513 if (lc->link_ok) {
8514 sbuf_printf(sb, "%b", (lc->fc & (PAUSE_TX | PAUSE_RX)) |
8515 (lc->requested_fc & PAUSE_AUTONEG), bits);
8517 sbuf_printf(sb, "%b", lc->requested_fc & (PAUSE_TX |
8526 s[0] = '0' + (lc->requested_fc & (PAUSE_TX | PAUSE_RX |
8538 n = s[0] - '0';
8542 rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK | INTR_OK,
8548 lc->requested_fc = n;
8550 if (pi->up_vis > 0)
8565 struct link_config *lc = &pi->link_cfg;
8568 static char *bits = "\20\1RS-FEC\2FC-FEC\3NO-FEC\4RSVD1\5RSVD2";
8573 if (lc->link_ok)
8574 sbuf_printf(sb, "%b", lc->fec, bits);
8587 struct adapter *sc = pi->adapter;
8588 struct link_config *lc = &pi->link_cfg;
8592 if (req->newptr == NULL) {
8594 static char *bits = "\20\1RS-FEC\2FC-FEC\3NO-FEC\4RSVD2"
8601 sbuf_printf(sb, "%b", lc->requested_fec, bits);
8609 lc->requested_fec == FEC_AUTO ? -1 :
8610 lc->requested_fec & (M_FW_PORT_CAP32_FEC | FEC_MODULE));
8622 rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK | INTR_OK,
8627 old = lc->requested_fec;
8629 lc->requested_fec = FEC_AUTO;
8631 lc->requested_fec = FEC_NONE;
8633 if ((lc->pcaps |
8635 lc->pcaps) {
8639 lc->requested_fec = n & (M_FW_PORT_CAP32_FEC |
8644 if (pi->up_vis > 0) {
8647 lc->requested_fec = old;
8665 struct adapter *sc = pi->adapter;
8666 struct link_config *lc = &pi->link_cfg;
8670 static char *bits = "\20\1RS-FEC\2FC-FEC\3NO-FEC\4RSVD2\5RSVD3";
8685 if (pi->up_vis == 0) {
8695 fec = lc->fec_hint;
8696 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE ||
8697 !fec_supported(lc->pcaps)) {
8718 struct adapter *sc = pi->adapter;
8719 struct link_config *lc = &pi->link_cfg;
8722 if (lc->pcaps & FW_PORT_CAP32_ANEG)
8723 val = lc->requested_aneg == AUTONEG_DISABLE ? 0 : 1;
8725 val = -1;
8727 if (rc != 0 || req->newptr == NULL)
8736 rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK | INTR_OK,
8741 if (val == AUTONEG_ENABLE && !(lc->pcaps & FW_PORT_CAP32_ANEG)) {
8745 lc->requested_aneg = val;
8748 if (pi->up_vis > 0)
8762 struct adapter *sc = pi->adapter;
8763 struct link_config *lc = &pi->link_cfg;
8766 val = lc->force_fec;
8767 MPASS(val >= -1 && val <= 1);
8769 if (rc != 0 || req->newptr == NULL)
8771 if (!(lc->pcaps & FW_PORT_CAP32_FORCE_FEC))
8773 if (val < -1 || val > 1)
8776 rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK | INTR_OK, "t4ff");
8780 lc->force_fec = val;
8783 if (pi->up_vis > 0)
8798 mtx_lock(&sc->reg_lock);
8805 mtx_unlock(&sc->reg_lock);
8827 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val);
8833 /* unknown is returned as 0 but we display -1 in that case */
8834 t = val == 0 ? -1 : val;
8847 if (sc->params.core_vdd == 0) {
8858 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1,
8864 sc->params.core_vdd = val;
8867 return (sysctl_handle_int(oidp, &sc->params.core_vdd, 0, req));
8877 v = sc->sensor_resets;
8879 if (rc != 0 || req->newptr == NULL || v <= 0)
8882 if (sc->params.fw_vers < FW_VERSION32(1, 24, 7, 0) ||
8896 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val);
8900 sc->sensor_resets++;
8920 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val);
8960 mtx_lock(&sc->reg_lock);
8965 mtx_unlock(&sc->reg_lock);
8976 sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]);
8986 "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI", /* ibq's */
8987 "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */
8988 "SGE0-RX", "SGE1-RX" /* additional obq's (T5 onwards) */
8999 u_int cim_num_obq = sc->chip_params->cim_num_obq;
9009 mtx_lock(&sc->reg_lock);
9011 rc = -ENXIO;
9014 mtx_unlock(&sc->reg_lock);
9016 /* outbound queue */
9018 qid -= CIM_NUM_IBQ;
9021 mtx_lock(&sc->reg_lock);
9023 rc = -ENXIO;
9026 mtx_unlock(&sc->reg_lock);
9030 rc = -rc;
9062 for (p = buf; p <= &buf[sc->params.cim_la_size - 8]; p += 8) {
9092 for (p = buf; p <= &buf[sc->params.cim_la_size - 10]; p += 10) {
9122 buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE,
9127 mtx_lock(&sc->reg_lock);
9131 rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg);
9133 rc = -t4_cim_read_la(sc, buf, NULL);
9135 mtx_unlock(&sc->reg_lock);
9168 device_get_nameunit(sc->dev),
9175 device_get_nameunit(sc->dev),
9191 device_get_nameunit(sc->dev));
9199 device_get_nameunit(sc->dev), sbuf_data(&sb));
9208 atomic_set_int(&sc->error_flags, ADAP_CIM_ERR);
9228 mtx_lock(&sc->reg_lock);
9233 mtx_unlock(&sc->reg_lock);
9276 mtx_lock(&sc->reg_lock);
9281 mtx_unlock(&sc->reg_lock);
9319 cim_num_obq = sc->chip_params->cim_num_obq;
9329 mtx_lock(&sc->reg_lock);
9333 rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat);
9335 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq,
9341 mtx_unlock(&sc->reg_lock);
9360 wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
9382 mtx_lock(&sc->reg_lock);
9387 mtx_unlock(&sc->reg_lock);
9391 if (sc->chip_params->nchan > 2) {
9425 mtx_lock(&sc->reg_lock);
9430 mtx_unlock(&sc->reg_lock);
9455 mtx_lock(&sc->reg_lock);
9460 mtx_unlock(&sc->reg_lock);
9514 struct devlog_params *dparams = &sc->params.devlog;
9518 if (dparams->addr == 0)
9522 buf = malloc(dparams->size, M_CXGBE, M_ZERO | flags);
9526 mtx_lock(&sc->reg_lock);
9530 rc = read_via_memwin(sc, 1, dparams->addr, (void *)buf,
9531 dparams->size);
9532 mtx_unlock(&sc->reg_lock);
9536 nentries = dparams->size / sizeof(struct fw_devlog_e);
9540 if (e->timestamp == 0)
9543 e->timestamp = be64toh(e->timestamp);
9544 e->seqno = be32toh(e->seqno);
9546 e->params[j] = be32toh(e->params[j]);
9548 if (e->timestamp < ftstamp) {
9549 ftstamp = e->timestamp;
9563 if (e->timestamp == 0)
9567 e->seqno, e->timestamp,
9568 (e->level < nitems(devlog_level_strings) ?
9569 devlog_level_strings[e->level] : "UNKNOWN"),
9570 (e->facility < nitems(devlog_facility_strings) ?
9571 devlog_facility_strings[e->facility] : "UNKNOWN"));
9572 sbuf_printf(sb, e->fmt, e->params[0], e->params[1],
9573 e->params[2], e->params[3], e->params[4],
9574 e->params[5], e->params[6], e->params[7]);
9610 device_get_nameunit(sc->dev));
9618 device_get_nameunit(sc->dev), sbuf_data(&sb));
9631 int i, nchan = sc->chip_params->nchan;
9634 mtx_lock(&sc->reg_lock);
9641 mtx_unlock(&sc->reg_lock);
9690 mtx_lock(&sc->reg_lock);
9692 mtx_unlock(&sc->reg_lock);
9700 mtx_unlock(&sc->reg_lock);
9707 sbuf_printf(sb, "\n %u %-5s %u ", i,
9755 for (i = 0; i < sc->chip_params->nchan; i += 2) {
9756 mtx_lock(&sc->reg_lock);
9763 mtx_unlock(&sc->reg_lock);
9773 sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j],
9789 struct link_config *lc = &pi->link_cfg;
9796 if (lc->link_ok || lc->link_down_rc == 255)
9799 sbuf_printf(sb, "%s", t4_link_down_rc_str(lc->link_down_rc));
9816 const u_int v1 = ((const struct mem_desc *)a)->base;
9817 const u_int v2 = ((const struct mem_desc *)b)->base;
9820 return (-1);
9836 size = to - from + 1;
9841 sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size);
9861 "ULPTX state:", "On-chip queues:",
9880 mtx_lock(&sc->reg_lock);
9929 (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR);
9930 (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR);
9931 (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR);
9932 (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
9933 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE);
9934 (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE);
9935 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE);
9936 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE);
9937 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE);
9940 md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE);
9941 md->limit = md->base - 1 +
9946 md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE);
9947 md->limit = md->base - 1 +
9954 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE);
9956 md->base = t4_read_reg(sc, A_LE_DB_HASH_TBL_BASE_ADDR);
9957 md->limit = 0;
9959 md->base = 0;
9960 md->idx = nitems(region); /* hide it */
9965 md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\
9966 (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT)
9976 if (sc->cryptocaps & FW_CAPS_CONFIG_TLSKEYS) {
9981 md->base = 0;
9983 md->idx = nitems(region);
9996 md->base = t4_read_reg(sc, A_SGE_DBVFIFO_BADDR);
9997 md->limit = md->base + size - 1;
9999 md->idx = nitems(region);
10003 md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE);
10004 md->limit = 0;
10006 md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE);
10007 md->limit = 0;
10010 md->base = sc->vres.ocq.start;
10011 if (sc->vres.ocq.size)
10012 md->limit = md->base + sc->vres.ocq.size - 1;
10014 md->idx = nitems(region); /* hide it */
10017 /* add any address-space holes, there can be up to 3 */
10018 for (n = 0; n < i - 1; n++)
10020 (md++)->base = avail[n].limit;
10022 (md++)->base = avail[n].limit;
10024 n = md - mem;
10030 avail[lo].limit - 1);
10037 mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0;
10044 hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1;
10048 hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1;
10067 sbuf_printf(sb, "%u p-structs (%u free)\n",
10087 for (i = 0; i < sc->chip_params->nchan; i++) {
10105 mtx_unlock(&sc->reg_lock);
10137 for (i = 0; i < sc->chip_params->mps_tcam_size; i++) {
10142 mtx_lock(&sc->reg_lock);
10149 mtx_unlock(&sc->reg_lock);
10155 mtx_lock(&sc->reg_lock);
10162 mtx_unlock(&sc->reg_lock);
10170 (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1);
10192 rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd,
10240 for (i = 0; i < sc->chip_params->mps_tcam_size; i++) {
10251 ctl |= V_CTLTCAMINDEX(i - 256) | V_CTLTCAMSEL(1);
10252 mtx_lock(&sc->reg_lock);
10262 mtx_unlock(&sc->reg_lock);
10282 mtx_lock(&sc->reg_lock);
10292 mtx_unlock(&sc->reg_lock);
10307 mtx_lock(&sc->reg_lock);
10314 mtx_unlock(&sc->reg_lock);
10320 "%012jx %06x %06x - - %3c"
10326 cls_lo & F_T6_VF_VALID ? G_T6_VF(cls_lo) : -1);
10329 "%012jx - - ", i, addr[0], addr[1],
10336 sbuf_printf(sb, " - N ");
10338 sbuf_printf(sb, "- %3c %4x %3c %#x%4u%4d",
10342 cls_lo & F_T6_VF_VALID ? G_T6_VF(cls_lo) : -1);
10366 rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd,
10410 mtx_lock(&sc->reg_lock);
10415 mtx_unlock(&sc->reg_lock);
10452 mtx_lock(&sc->reg_lock);
10459 mtx_unlock(&sc->reg_lock);
10469 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], tx_cnt[i],
10475 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], rx_cnt[i],
10482 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], tx_cnt[i],
10484 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], rx_cnt[i],
10492 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], tx_cnt[i],
10494 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], rx_cnt[i],
10513 mtx_lock(&sc->reg_lock);
10518 mtx_unlock(&sc->reg_lock);
10544 mtx_lock(&sc->reg_lock);
10549 mtx_unlock(&sc->reg_lock);
10581 struct tid_info *t = &sc->tids;
10588 if (t->natids) {
10589 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1,
10590 t->atids_in_use);
10593 if (t->nhpftids) {
10594 sbuf_printf(sb, "HPFTID range: %u-%u, in use: %u\n",
10595 t->hpftid_base, t->hpftid_end, t->hpftids_in_use);
10598 if (t->ntids) {
10601 mtx_lock(&sc->reg_lock);
10614 mtx_unlock(&sc->reg_lock);
10621 sbuf_printf(sb, "%u-%u, ", t->tid_base, x - 1);
10622 sbuf_printf(sb, "%u-%u", y, t->ntids - 1);
10624 sbuf_printf(sb, "%u-%u", t->tid_base, t->tid_base +
10625 t->ntids - 1);
10628 atomic_load_acq_int(&t->tids_in_use));
10631 if (t->nstids) {
10632 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base,
10633 t->stid_base + t->nstids - 1, t->stids_in_use);
10636 if (t->nftids) {
10637 sbuf_printf(sb, "FTID range: %u-%u, in use: %u\n", t->ftid_base,
10638 t->ftid_end, t->ftids_in_use);
10641 if (t->netids) {
10642 sbuf_printf(sb, "ETID range: %u-%u, in use: %u\n", t->etid_base,
10643 t->etid_base + t->netids - 1, t->etids_in_use);
10646 mtx_lock(&sc->reg_lock);
10653 mtx_unlock(&sc->reg_lock);
10676 mtx_lock(&sc->reg_lock);
10681 mtx_unlock(&sc->reg_lock);
10689 if (sc->chip_params->nchan > 2) {
10754 mtx_lock(&sc->reg_lock);
10759 mtx_unlock(&sc->reg_lock);
10767 if (sc->chip_params->nchan > 2) {
10794 struct tp_params *tpp = &sc->params.tp;
10798 mask = tpp->la_mask >> 16;
10800 if (rc != 0 || req->newptr == NULL)
10804 mtx_lock(&sc->reg_lock);
10808 tpp->la_mask = mask << 16;
10810 tpp->la_mask);
10812 mtx_unlock(&sc->reg_lock);
10829 while (f->name) {
10830 uint64_t mask = (1ULL << f->width) - 1;
10831 int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name,
10832 ((uintmax_t)v >> f->start) & mask);
10992 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
11003 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
11024 mtx_lock(&sc->reg_lock);
11043 mtx_unlock(&sc->reg_lock);
11066 mtx_lock(&sc->reg_lock);
11071 mtx_unlock(&sc->reg_lock);
11079 if (sc->chip_params->nchan > 2) {
11116 mtx_lock(&sc->reg_lock);
11121 mtx_unlock(&sc->reg_lock);
11150 mtx_lock(&sc->reg_lock);
11158 mtx_unlock(&sc->reg_lock);
11195 rc = bus_get_cpus(sc->dev, op, sizeof(cpuset), &cpuset);
11218 val = atomic_load_int(&sc->num_resets);
11220 if (rc != 0 || req->newptr == NULL)
11225 atomic_store_int(&sc->num_resets, 0);
11235 taskqueue_enqueue(reset_tq, &sc->reset_task);
11247 v = sc->tt.tls;
11249 if (rc != 0 || req->newptr == NULL)
11252 if (v != 0 && !(sc->cryptocaps & FW_CAPS_CONFIG_TLSKEYS))
11261 sc->tt.tls = !!v;
11263 for_each_vi(sc->port[i], j, vi) {
11264 if (vi->flags & VI_INIT_DONE)
11265 t4_update_fl_bufsize(vi->ifp);
11295 u_int cclk_ps = 1000000000 / sc->params.vpd.cclk;
11297 mtx_lock(&sc->reg_lock);
11299 res = (u_int)-1;
11302 mtx_unlock(&sc->reg_lock);
11303 if (res == (u_int)-1)
11334 u_int cclk_ps = 1000000000 / sc->params.vpd.cclk;
11336 mtx_lock(&sc->reg_lock);
11345 mtx_unlock(&sc->reg_lock);
11361 u_int cclk_ps = 1000000000 / sc->params.vpd.cclk;
11368 mtx_lock(&sc->reg_lock);
11380 mtx_unlock(&sc->reg_lock);
11400 mtx_lock(&sc->reg_lock);
11407 mtx_unlock(&sc->reg_lock);
11425 mtx_lock(&sc->reg_lock);
11432 mtx_unlock(&sc->reg_lock);
11443 struct adapter *sc = vi->adapter;
11448 idx = vi->ofld_tmr_idx;
11451 if (rc != 0 || req->newptr == NULL)
11462 v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(vi->ofld_pktc_idx != -1);
11465 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v);
11467 ofld_rxq->iq.intr_params = v;
11470 vi->ofld_tmr_idx = idx;
11480 struct adapter *sc = vi->adapter;
11483 idx = vi->ofld_pktc_idx;
11486 if (rc != 0 || req->newptr == NULL)
11489 if (idx < -1 || idx >= SGE_NCOUNTERS)
11497 if (vi->flags & VI_INIT_DONE)
11500 vi->ofld_pktc_idx = idx;
11512 if (cntxt->cid > M_CTXTQID)
11515 if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS &&
11516 cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM)
11528 if (sc->flags & FW_OK) {
11529 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id,
11530 &cntxt->data[0]);
11539 rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]);
11566 if (sc->flags & FULL_INIT_DONE &&
11567 (sc->debug_flags & DF_LOAD_FW_ANYTIME) == 0) {
11572 fw_data = malloc(fw->len, M_CXGBE, M_WAITOK);
11574 rc = copyin(fw->data, fw_data, fw->len);
11576 rc = -t4_load_fw(sc, fw_data, fw->len);
11599 if (cfg->len == 0) {
11601 rc = -t4_load_cfg(sc, NULL, 0);
11605 cfg_data = malloc(cfg->len, M_CXGBE, M_WAITOK);
11607 rc = copyin(cfg->data, cfg_data, cfg->len);
11609 rc = -t4_load_cfg(sc, cfg_data, cfg->len);
11624 if (br->len > 1024 * 1024)
11627 if (br->pf_offset == 0) {
11629 if (br->pfidx_addr > 7)
11631 offset = G_OFFSET(t4_read_reg(sc, PF_REG(br->pfidx_addr,
11633 } else if (br->pf_offset == 1) {
11635 offset = G_OFFSET(br->pfidx_addr);
11649 if (br->len == 0) {
11651 rc = -t4_load_boot(sc, NULL, offset, 0);
11655 br_data = malloc(br->len, M_CXGBE, M_WAITOK);
11657 rc = copyin(br->data, br_data, br->len);
11659 rc = -t4_load_boot(sc, br_data, offset, br->len);
11682 if (bc->len == 0) {
11684 rc = -t4_load_bootcfg(sc, NULL, 0);
11688 bc_data = malloc(bc->len, M_CXGBE, M_WAITOK);
11690 rc = copyin(bc->data, bc_data, bc->len);
11692 rc = -t4_load_bootcfg(sc, bc_data, bc->len);
11708 buf = malloc(dump->len, M_CXGBE, M_NOWAIT | M_ZERO);
11719 cudbg->adap = sc;
11720 cudbg->print = (cudbg_print_cb)printf;
11723 device_printf(sc->dev, "%s: wr_flash %u, len %u, data %p.\n",
11724 __func__, dump->wr_flash, dump->len, dump->data);
11727 if (dump->wr_flash)
11728 cudbg->use_flash = 1;
11729 MPASS(sizeof(cudbg->dbg_bitmap) == sizeof(dump->bitmap));
11730 memcpy(cudbg->dbg_bitmap, dump->bitmap, sizeof(cudbg->dbg_bitmap));
11732 rc = cudbg_collect(handle, buf, &dump->len);
11736 rc = copyout(buf, dump->data, dump->len);
11752 r = &op->rule[0];
11753 for (i = 0; i < op->nrules; i++, r++) {
11754 free(r->bpf_prog.bf_insns, M_CXGBE);
11756 free(op->rule, M_CXGBE);
11773 if (uop->nrules == 0) {
11777 } else if (uop->nrules > 256) { /* arbitrary */
11783 op->nrules = uop->nrules;
11784 len = op->nrules * sizeof(struct offload_rule);
11785 op->rule = malloc(len, M_CXGBE, M_ZERO | M_WAITOK);
11786 rc = copyin(uop->rule, op->rule, len);
11788 free(op->rule, M_CXGBE);
11793 r = &op->rule[0];
11794 for (i = 0; i < op->nrules; i++, r++) {
11797 if (r->open_type != OPEN_TYPE_LISTEN &&
11798 r->open_type != OPEN_TYPE_ACTIVE &&
11799 r->open_type != OPEN_TYPE_PASSIVE &&
11800 r->open_type != OPEN_TYPE_DONTCARE) {
11807 op->nrules = i;
11813 s = &r->settings;
11814 if ((s->offload != 0 && s->offload != 1) ||
11815 s->cong_algo < -1 || s->cong_algo > CONG_ALG_HIGHSPEED ||
11816 s->sched_class < -1 ||
11817 s->sched_class >= sc->params.nsched_cls) {
11822 bf = &r->bpf_prog;
11823 u = bf->bf_insns; /* userspace ptr */
11824 bf->bf_insns = NULL;
11825 if (bf->bf_len == 0) {
11829 len = bf->bf_len * sizeof(*bf->bf_insns);
11830 bf->bf_insns = malloc(len, M_CXGBE, M_ZERO | M_WAITOK);
11831 rc = copyin(u, bf->bf_insns, len);
11835 if (!bpf_validate(bf->bf_insns, bf->bf_len)) {
11841 rw_wlock(&sc->policy_lock);
11842 old = sc->policy;
11843 sc->policy = op;
11844 rw_wunlock(&sc->policy_lock);
11859 mtx_lock(&sc->reg_lock);
11863 rc = validate_mem_range(sc, mr->addr, mr->len);
11864 mtx_unlock(&sc->reg_lock);
11868 buf = malloc(min(mr->len, MAX_READ_BUF_SIZE), M_CXGBE, M_WAITOK);
11869 addr = mr->addr;
11870 remaining = mr->len;
11871 dst = (void *)mr->data;
11875 mtx_lock(&sc->reg_lock);
11880 mtx_unlock(&sc->reg_lock);
11889 remaining -= n;
11903 if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports)
11906 if (i2cd->len > sizeof(i2cd->data))
11915 rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr,
11916 i2cd->offset, i2cd->len, &i2cd->data[0]);
11938 if (port_id >= sc->params.nports)
11940 pi = sc->port[port_id];
11944 mtx_lock(&sc->reg_lock);
11947 t4_clr_port_stats(sc, pi->tx_chan);
11949 if (pi->fcs_reg != -1)
11950 pi->fcs_base = t4_read_reg64(sc, pi->fcs_reg);
11952 pi->stats.rx_fcs_err = 0;
11955 if (vi->flags & VI_INIT_DONE)
11956 t4_clr_vi_stats(sc, vi->vin);
11958 chan_map = pi->rx_e_chan_map;
11961 i = ffs(chan_map) - 1;
11967 mtx_unlock(&sc->reg_lock);
11968 pi->tx_parse_error = 0;
11969 pi->tnl_cong_drops = 0;
11976 if (vi->flags & VI_INIT_DONE) {
11980 rxq->lro.lro_queued = 0;
11981 rxq->lro.lro_flushed = 0;
11983 rxq->rxcsum = 0;
11984 rxq->vlan_extraction = 0;
11985 rxq->vxlan_rxcsum = 0;
11987 rxq->fl.cl_allocated = 0;
11988 rxq->fl.cl_recycled = 0;
11989 rxq->fl.cl_fast_recycled = 0;
11993 txq->txcsum = 0;
11994 txq->tso_wrs = 0;
11995 txq->vlan_insertion = 0;
11996 txq->imm_wrs = 0;
11997 txq->sgl_wrs = 0;
11998 txq->txpkt_wrs = 0;
11999 txq->txpkts0_wrs = 0;
12000 txq->txpkts1_wrs = 0;
12001 txq->txpkts0_pkts = 0;
12002 txq->txpkts1_pkts = 0;
12003 txq->txpkts_flush = 0;
12004 txq->raw_wrs = 0;
12005 txq->vxlan_tso_wrs = 0;
12006 txq->vxlan_txcsum = 0;
12007 txq->kern_tls_records = 0;
12008 txq->kern_tls_short = 0;
12009 txq->kern_tls_partial = 0;
12010 txq->kern_tls_full = 0;
12011 txq->kern_tls_octets = 0;
12012 txq->kern_tls_waste = 0;
12013 txq->kern_tls_options = 0;
12014 txq->kern_tls_header = 0;
12015 txq->kern_tls_fin = 0;
12016 txq->kern_tls_fin_short = 0;
12017 txq->kern_tls_cbc = 0;
12018 txq->kern_tls_gcm = 0;
12019 mp_ring_reset_stats(txq->r);
12024 ofld_txq->wrq.tx_wrs_direct = 0;
12025 ofld_txq->wrq.tx_wrs_copied = 0;
12026 counter_u64_zero(ofld_txq->tx_iscsi_pdus);
12027 counter_u64_zero(ofld_txq->tx_iscsi_octets);
12028 counter_u64_zero(ofld_txq->tx_iscsi_iso_wrs);
12029 counter_u64_zero(ofld_txq->tx_aio_jobs);
12030 counter_u64_zero(ofld_txq->tx_aio_octets);
12031 counter_u64_zero(ofld_txq->tx_toe_tls_records);
12032 counter_u64_zero(ofld_txq->tx_toe_tls_octets);
12037 ofld_rxq->fl.cl_allocated = 0;
12038 ofld_rxq->fl.cl_recycled = 0;
12039 ofld_rxq->fl.cl_fast_recycled = 0;
12041 ofld_rxq->rx_iscsi_ddp_setup_ok);
12043 ofld_rxq->rx_iscsi_ddp_setup_error);
12044 ofld_rxq->rx_iscsi_ddp_pdus = 0;
12045 ofld_rxq->rx_iscsi_ddp_octets = 0;
12046 ofld_rxq->rx_iscsi_fl_pdus = 0;
12047 ofld_rxq->rx_iscsi_fl_octets = 0;
12048 ofld_rxq->rx_aio_ddp_jobs = 0;
12049 ofld_rxq->rx_aio_ddp_octets = 0;
12050 ofld_rxq->rx_toe_tls_records = 0;
12051 ofld_rxq->rx_toe_tls_octets = 0;
12052 ofld_rxq->rx_toe_ddp_octets = 0;
12053 counter_u64_zero(ofld_rxq->ddp_buffer_alloc);
12054 counter_u64_zero(ofld_rxq->ddp_buffer_reuse);
12055 counter_u64_zero(ofld_rxq->ddp_buffer_free);
12060 wrq = &sc->sge.ctrlq[pi->port_id];
12061 wrq->tx_wrs_direct = 0;
12062 wrq->tx_wrs_copied = 0;
12076 bcopy(&ca->addr[0], &in6.s6_addr[0], sizeof(in6.s6_addr));
12092 bcopy(&ca->addr[0], &in6.s6_addr[0], sizeof(in6.s6_addr));
12104 return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0);
12113 dev = sc->dev;
12126 dev = sc->dev;
12136 struct adapter *sc = pi->adapter;
12143 KASSERT((pi->flags & FIXED_IFMEDIA) == 0,
12144 ("%s: port_type %u", __func__, pi->port_type));
12146 vi = &pi->vi[0];
12150 if (pi->mod_type != FW_PORT_MOD_TYPE_NONE) {
12158 ifp = vi->ifp;
12159 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
12161 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN)
12163 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED)
12165 else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) {
12167 port_top_speed(pi), mod_str[pi->mod_type]);
12170 pi->mod_type);
12179 struct link_config *lc = &pi->link_cfg;
12180 struct adapter *sc = pi->adapter;
12186 if (lc->link_ok) {
12187 if (lc->speed > 25000 ||
12188 (lc->speed == 25000 && lc->fec == FEC_RS)) {
12189 pi->fcs_reg = T5_PORT_REG(pi->tx_chan,
12192 pi->fcs_reg = T5_PORT_REG(pi->tx_chan,
12195 pi->fcs_base = t4_read_reg64(sc, pi->fcs_reg);
12196 pi->stats.rx_fcs_err = 0;
12198 pi->fcs_reg = -1;
12201 MPASS(pi->fcs_reg != -1);
12202 MPASS(pi->fcs_base == 0);
12206 ifp = vi->ifp;
12210 if (lc->link_ok) {
12211 if_setbaudrate(ifp, IF_Mbps(lc->speed));
12228 * in - the only guarantee is that sc->sc_lock is a valid lock.
12240 struct adapter *sc = dev->si_drv1;
12250 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
12253 mtx_lock(&sc->reg_lock);
12256 else if (edata->size == 4)
12257 edata->val = t4_read_reg(sc, edata->addr);
12258 else if (edata->size == 8)
12259 edata->val = t4_read_reg64(sc, edata->addr);
12262 mtx_unlock(&sc->reg_lock);
12269 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
12272 mtx_lock(&sc->reg_lock);
12275 else if (edata->size == 4) {
12276 if (edata->val & 0xffffffff00000000)
12278 t4_write_reg(sc, edata->addr, (uint32_t) edata->val);
12279 } else if (edata->size == 8)
12280 t4_write_reg64(sc, edata->addr, edata->val);
12283 mtx_unlock(&sc->reg_lock);
12292 if (regs->len < reglen) {
12293 regs->len = reglen; /* hint to the caller */
12297 regs->len = reglen;
12299 mtx_lock(&sc->reg_lock);
12304 mtx_unlock(&sc->reg_lock);
12306 rc = copyout(buf, regs->data, reglen);
12388 struct port_info *pi = vi->pi;
12389 struct adapter *sc = pi->adapter;
12400 if (sc->flags & KERN_TLS_ON && is_t6(sc)) {
12411 p = sc->port[i];
12413 if (if_getcapenable(v->ifp) & IFCAP_TXTLS) {
12416 device_get_nameunit(v->dev));
12432 if ((if_getcapenable(vi->ifp) & IFCAP_TOE) != 0) {
12442 if (!(vi->flags & VI_INIT_DONE) && ((rc = vi_init(vi)) != 0))
12444 if (!(pi->vi[0].flags & VI_INIT_DONE) &&
12445 ((rc = vi_init(&pi->vi[0])) != 0))
12448 if (isset(&sc->offload_map, pi->port_id)) {
12450 MPASS(pi->uld_vis > 0);
12451 pi->uld_vis++;
12464 KASSERT(sc->tom_softc != NULL,
12476 if (pi->uld_vis++ == 0)
12477 setbit(&sc->offload_map, pi->port_id);
12479 if ((if_getcapenable(vi->ifp) & IFCAP_TOE) == 0) {
12483 MPASS(isset(&sc->offload_map, pi->port_id));
12484 MPASS(pi->uld_vis > 0);
12485 if (--pi->uld_vis == 0)
12486 clrbit(&sc->offload_map, pi->port_id);
12537 if (!(sc->flags & FULL_INIT_DONE)) {
12547 rc = t4_uld_list[id]->uld_activate(sc);
12549 setbit(&sc->active_ulds, id);
12570 rc = t4_uld_list[id]->uld_deactivate(sc);
12572 clrbit(&sc->active_ulds, id);
12591 rc = t4_uld_list[i]->uld_deactivate(sc);
12594 clrbit(&sc->active_ulds, i);
12612 t4_uld_list[i]->uld_stop == NULL)
12614 (void) t4_uld_list[i]->uld_stop(sc);
12630 t4_uld_list[i]->uld_restart == NULL)
12632 (void) t4_uld_list[i]->uld_restart(sc);
12644 return (isset(&sc->active_ulds, id));
12662 if (sc->flags & KERN_TLS_ON)
12664 if (sc->offload_map != 0) {
12693 nq = *t < 0 ? -*t : c;
12736 if (t4_toecaps_allowed == -1)
12739 if (t4_toecaps_allowed == -1)
12744 if (t4_rdmacaps_allowed == -1) {
12749 if (t4_iscsicaps_allowed == -1) {
12758 if (t4_pktc_idx_ofld < -1 || t4_pktc_idx_ofld >= SGE_NCOUNTERS)
12761 if (t4_rdmacaps_allowed == -1)
12764 if (t4_iscsicaps_allowed == -1)
12778 if (t4_pktc_idx < -1 || t4_pktc_idx >= SGE_NCOUNTERS)
12792 * Number of VIs to create per-port. The first VI is the "main" regular
12824 base = sc->memwin[2].mw_base;
12830 pf = V_PFNUM(sc->pf);
12833 off = addr - win_pos;
12848 len -= sizeof(buf);
12869 struct devlog_params *dparams = &sc->params.devlog;
12874 if (dparams->start == 0) {
12879 nentries = dparams->size / sizeof(struct fw_devlog_e);
12880 m = fwmtype_to_hwmtype(dparams->memtype);
12883 first = -1;
12885 rc = -t4_mem_read(sc, m, dparams->start + i * sizeof(e),
12900 if (first == -1)
12905 rc = -t4_mem_read(sc, m, dparams->start + i * sizeof(e),
13061 t4_write_reg(sc, A_MPS_RX_VXLAN_TYPE, V_VXLAN(sc->vxlan_port) |
13064 pi = sc->port[i];
13065 if (pi->vxlan_tcam_entry == true)
13067 rc = t4_alloc_raw_mac_filt(sc, pi->vi[0].viid, match_all_mac,
13068 match_all_mac, sc->rawf_base + pi->port_id, 1, pi->port_id,
13071 rc = -rc;
13072 CH_ERR(&pi->vi[0],
13075 MPASS(rc == sc->rawf_base + pi->port_id);
13076 pi->vxlan_tcam_entry = true;
13086 if (sc->nrawf == 0 || chip_id(sc) <= CHELSIO_T5)
13091 if (sc->vxlan_refcount == 0) {
13092 sc->vxlan_port = v->port;
13093 sc->vxlan_refcount = 1;
13096 } else if (sc->vxlan_port == v->port) {
13097 sc->vxlan_refcount++;
13101 sc->vxlan_port, v->port);
13111 if (sc->nrawf == 0 || chip_id(sc) <= CHELSIO_T5)
13121 if (sc->vxlan_port != v->port)
13123 if (sc->vxlan_refcount == 0) {
13125 "ignoring attempt to stop it again.\n", sc->vxlan_port);
13126 } else if (--sc->vxlan_refcount == 0 && !hw_off_limits(sc))
13219 if (--loaded == 0) {