Lines Matching defs:sq
50 struct mlx5e_sq_param sq;
866 sq_stats = &pch->sq[j].stats;
884 struct mlx5e_sq *sq = channel->sq;
886 if (sq == NULL)
889 sq_stats = &sq->stats;
1568 mlx5e_free_sq_db(struct mlx5e_sq *sq)
1570 int wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
1574 if (sq->mbuf[x].mbuf != NULL) {
1575 bus_dmamap_unload(sq->dma_tag, sq->mbuf[x].dma_map);
1576 m_freem(sq->mbuf[x].mbuf);
1578 if (sq->mbuf[x].mst != NULL) {
1579 m_snd_tag_rele(sq->mbuf[x].mst);
1580 sq->mbuf[x].mst = NULL;
1582 bus_dmamap_destroy(sq->dma_tag, sq->mbuf[x].dma_map);
1584 free(sq->mbuf, M_MLX5EN);
1588 mlx5e_alloc_sq_db(struct mlx5e_sq *sq)
1590 int wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
1594 sq->mbuf = malloc_domainset(wq_sz * sizeof(sq->mbuf[0]), M_MLX5EN,
1595 mlx5_dev_domainset(sq->priv->mdev), M_WAITOK | M_ZERO);
1599 err = -bus_dmamap_create(sq->dma_tag, 0, &sq->mbuf[x].dma_map);
1602 bus_dmamap_destroy(sq->dma_tag, sq->mbuf[x].dma_map);
1603 free(sq->mbuf, M_MLX5EN);
1615 mlx5e_update_sq_inline(struct mlx5e_sq *sq)
1617 sq->max_inline = sq->priv->params.tx_max_inline;
1618 sq->min_inline_mode = sq->priv->params.tx_min_inline_mode;
1624 if (sq->priv->params_ethtool.trust_state != MLX5_QPTS_TRUST_PCP ||
1625 sq->min_inline_mode == MLX5_INLINE_MODE_NONE) {
1626 if (MLX5_CAP_ETH(sq->priv->mdev, wqe_vlan_insert))
1627 sq->min_insert_caps = MLX5E_INSERT_VLAN | MLX5E_INSERT_NON_VLAN;
1629 sq->min_insert_caps = MLX5E_INSERT_NON_VLAN;
1631 sq->min_insert_caps = 0;
1641 mtx_lock(&c->sq[i].lock);
1642 mlx5e_update_sq_inline(&c->sq[i]);
1643 mtx_unlock(&c->sq[i].lock);
1664 struct mlx5e_sq *sq)
1686 &sq->dma_tag)))
1689 sq->mkey_be = cpu_to_be32(priv->mr.key);
1690 sq->ifp = priv->ifp;
1691 sq->priv = priv;
1692 sq->tc = tc;
1694 err = mlx5_wq_cyc_create(mdev, ¶m->wq, sqc_wq, &sq->wq,
1695 &sq->wq_ctrl);
1699 sq->wq.db = &sq->wq.db[MLX5_SND_DBR];
1701 err = mlx5e_alloc_sq_db(sq);
1705 mlx5e_update_sq_inline(sq);
1708 mlx5e_create_stats(&sq->stats.ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
1710 sq->stats.arg);
1715 mlx5_wq_destroy(&sq->wq_ctrl);
1718 bus_dma_tag_destroy(sq->dma_tag);
1724 mlx5e_destroy_sq(struct mlx5e_sq *sq)
1727 sysctl_ctx_free(&sq->stats.ctx);
1729 mlx5e_free_sq_db(sq);
1730 mlx5_wq_destroy(&sq->wq_ctrl);
1731 bus_dma_tag_destroy(sq->dma_tag);
1735 mlx5e_enable_sq(struct mlx5e_sq *sq, struct mlx5e_sq_param *param,
1746 sizeof(u64) * sq->wq_ctrl.buf.npages;
1751 sq->uar_map = bfreg->map;
1753 ts_format = mlx5_get_sq_default_ts(sq->priv->mdev);
1760 MLX5_SET(sqc, sqc, cqn, sq->cq.mcq.cqn);
1769 MLX5_SET(wq, wq, log_wq_pg_sz, sq->wq_ctrl.buf.page_shift -
1771 MLX5_SET64(wq, wq, dbr_addr, sq->wq_ctrl.db.dma);
1773 mlx5_fill_page_array(&sq->wq_ctrl.buf,
1776 err = mlx5_core_create_sq(sq->priv->mdev, in, inlen, &sq->sqn);
1784 mlx5e_modify_sq(struct mlx5e_sq *sq, int curr_state, int next_state)
1798 MLX5_SET(modify_sq_in, in, sqn, sq->sqn);
1802 err = mlx5_core_modify_sq(sq->priv->mdev, in, inlen);
1810 mlx5e_disable_sq(struct mlx5e_sq *sq)
1813 mlx5_core_destroy_sq(sq->priv->mdev, sq->sqn);
1820 struct mlx5e_sq *sq)
1824 sq->cev_factor = c->priv->params_ethtool.tx_completion_fact;
1827 if (sq->cev_factor == 0)
1828 sq->cev_factor = 1;
1830 err = mlx5e_create_sq(c, tc, param, sq);
1834 err = mlx5e_enable_sq(sq, param, &c->bfreg, c->priv->tisn[tc]);
1838 err = mlx5e_modify_sq(sq, MLX5_SQC_STATE_RST, MLX5_SQC_STATE_RDY);
1842 WRITE_ONCE(sq->running, 1);
1847 mlx5e_disable_sq(sq);
1849 mlx5e_destroy_sq(sq);
1855 mlx5e_sq_send_nops_locked(struct mlx5e_sq *sq, int can_sleep)
1858 while (sq->cev_counter != 0) {
1859 while (!mlx5e_sq_has_room_for(sq, 1)) {
1861 mtx_unlock(&sq->lock);
1863 mtx_lock(&sq->lock);
1869 mlx5e_send_nop(sq, 1);
1873 mlx5e_tx_notify_hw(sq, false);
1879 struct mlx5e_sq *sq = arg;
1881 mtx_assert(&sq->lock, MA_OWNED);
1884 switch (sq->cev_next_state) {
1887 mlx5e_sq_send_nops_locked(sq, 0);
1890 if (sq->cev_counter == 0) {
1891 sq->cev_next_state = MLX5E_CEV_STATE_INITIAL;
1897 sq->cev_next_state = MLX5E_CEV_STATE_SEND_NOPS;
1902 callout_reset_curcpu(&sq->cev_callout, hz, mlx5e_sq_cev_timeout, sq);
1906 mlx5e_drain_sq(struct mlx5e_sq *sq)
1909 struct mlx5_core_dev *mdev= sq->priv->mdev;
1920 if (READ_ONCE(sq->running) == 0)
1924 WRITE_ONCE(sq->running, 0);
1927 mtx_lock(&sq->lock);
1930 sq->cev_next_state = MLX5E_CEV_STATE_HOLD_NOPS;
1931 callout_stop(&sq->cev_callout);
1934 mlx5e_sq_send_nops_locked(sq, 1);
1935 mtx_unlock(&sq->lock);
1938 mtx_lock(&sq->lock);
1939 while (sq->cc != sq->pc &&
1940 (sq->priv->media_status_last & IFM_ACTIVE) != 0 &&
1943 mtx_unlock(&sq->lock);
1945 sq->cq.mcq.comp(&sq->cq.mcq, NULL);
1946 mtx_lock(&sq->lock);
1948 mtx_unlock(&sq->lock);
1951 error = mlx5e_modify_sq(sq, MLX5_SQC_STATE_RDY, MLX5_SQC_STATE_ERR);
1953 mlx5_en_err(sq->ifp,
1958 mtx_lock(&sq->lock);
1959 while (sq->cc != sq->pc &&
1962 mtx_unlock(&sq->lock);
1964 sq->cq.mcq.comp(&sq->cq.mcq, NULL);
1965 mtx_lock(&sq->lock);
1967 mtx_unlock(&sq->lock);
1971 mlx5e_close_sq_wait(struct mlx5e_sq *sq)
1974 mlx5e_drain_sq(sq);
1975 mlx5e_disable_sq(sq);
1976 mlx5e_destroy_sq(sq);
2121 err = mlx5e_open_cq(c->priv, &cparam->tx_cq, &c->sq[tc].cq,
2130 mlx5e_close_cq(&c->sq[tc].cq);
2141 mlx5e_close_cq(&c->sq[tc].cq);
2152 err = mlx5e_open_sq(c, tc, &cparam->sq, &c->sq[tc]);
2161 mlx5e_close_sq_wait(&c->sq[tc]);
2172 mlx5e_close_sq_wait(&c->sq[tc]);
2194 struct mlx5e_sq *sq = c->sq + tc;
2196 mtx_init(&sq->lock, "mlx5tx",
2198 mtx_init(&sq->comp_lock, "mlx5comp",
2201 callout_init_mtx(&sq->cev_callout, &sq->lock, 0);
2234 callout_drain(&c->sq[tc].cev_callout);
2235 mtx_destroy(&c->sq[tc].lock);
2236 mtx_destroy(&c->sq[tc].comp_lock);
2253 MLX5E_ZERO(&c->sq[i], mlx5e_sq_zero_start);
2271 err = mlx5e_iq_open(c, &cparam->sq, &cparam->tx_cq, &c->iq);
2496 mlx5e_build_sq_param(priv, &cparam->sq);
2564 mlx5e_refresh_sq_params(struct mlx5e_priv *priv, struct mlx5e_sq *sq)
2580 return (mlx5_core_modify_cq_moderation_mode(priv->mdev, &sq->cq.mcq,
2586 return (mlx5_core_modify_cq_moderation(priv->mdev, &sq->cq.mcq,
2659 err = mlx5e_refresh_sq_params(priv, &c->sq[i]);
4086 mlx5e_drain_sq(&ch->sq[i]);
4090 mlx5e_reset_sq_doorbell_record(struct mlx5e_sq *sq)
4093 sq->doorbell.d32[0] = cpu_to_be32(MLX5_OPCODE_NOP);
4094 sq->doorbell.d32[1] = cpu_to_be32(sq->sqn << 8);
4095 mlx5e_tx_notify_hw(sq, true);
4099 mlx5e_resume_sq(struct mlx5e_sq *sq)
4104 if (READ_ONCE(sq->running) != 0)
4107 err = mlx5e_modify_sq(sq, MLX5_SQC_STATE_ERR,
4110 mlx5_en_err(sq->ifp,
4114 sq->cc = 0;
4115 sq->pc = 0;
4118 mlx5e_reset_sq_doorbell_record(sq);
4120 err = mlx5e_modify_sq(sq, MLX5_SQC_STATE_RST,
4123 mlx5_en_err(sq->ifp,
4127 sq->cev_next_state = MLX5E_CEV_STATE_INITIAL;
4128 WRITE_ONCE(sq->running, 1);
4137 mlx5e_resume_sq(&ch->sq[i]);
4426 if (unlikely(pch->sq[0].running == 0))
4441 params->unlimited.queue_level = mlx5e_sq_queue_level(&pch->sq[0]);
5066 struct mlx5e_sq *sq;
5073 sq = &priv->channel[0].sq[0];
5075 if (sq->running == 0) {
5080 if (mlx5e_sq_xmit(sq, &m) != 0) {
5087 mlx5e_tx_notify_hw(sq, true);