xref: /dpdk/drivers/net/gve/gve_ethdev.h (revision eb8ec5c3452a35e02dbb928b19d6c394e5e7e658)

/* SPDX-License-Identifier: BSD-3-Clause
 * Copyright(C) 2022 Intel Corporation
 */

#ifndef _GVE_ETHDEV_H_
#define _GVE_ETHDEV_H_

#include <ethdev_driver.h>
#include <ethdev_pci.h>
#include <rte_ether.h>
#include <rte_pci.h>

#include "base/gve.h"

/* TODO: this is a workaround to ensure that Tx complq is enough */
#define DQO_TX_MULTIPLIER 4

#define GVE_DEFAULT_MAX_RING_SIZE	1024
#define GVE_DEFAULT_MIN_RX_RING_SIZE	512
#define GVE_DEFAULT_MIN_TX_RING_SIZE	256

#define GVE_DEFAULT_RX_FREE_THRESH	64
#define GVE_DEFAULT_TX_FREE_THRESH	32
#define GVE_DEFAULT_TX_RS_THRESH	32
#define GVE_TX_MAX_FREE_SZ		512

#define GVE_RX_BUF_ALIGN_DQO		128
#define GVE_RX_MIN_BUF_SIZE_DQO		1024
#define GVE_RX_MAX_BUF_SIZE_DQO		((16 * 1024) - GVE_RX_BUF_ALIGN_DQO)
#define GVE_MAX_QUEUE_SIZE_DQO		4096

#define GVE_RX_BUF_ALIGN_GQI		2048
#define GVE_RX_MIN_BUF_SIZE_GQI		2048
#define GVE_RX_MAX_BUF_SIZE_GQI		4096

#define GVE_RSS_HASH_KEY_SIZE 40
#define GVE_RSS_INDIR_SIZE 128

#define GVE_TX_CKSUM_OFFLOAD_MASK (		\
		RTE_MBUF_F_TX_L4_MASK  |	\
		RTE_MBUF_F_TX_TCP_SEG)

#define GVE_TX_CKSUM_OFFLOAD_MASK_DQO (		\
		GVE_TX_CKSUM_OFFLOAD_MASK |	\
		RTE_MBUF_F_TX_IP_CKSUM)

#define GVE_RTE_RSS_OFFLOAD_ALL (	\
	RTE_ETH_RSS_IPV4 |		\
	RTE_ETH_RSS_NONFRAG_IPV4_TCP |	\
	RTE_ETH_RSS_IPV6 |		\
	RTE_ETH_RSS_IPV6_EX |		\
	RTE_ETH_RSS_NONFRAG_IPV6_TCP |	\
	RTE_ETH_RSS_IPV6_TCP_EX |	\
	RTE_ETH_RSS_NONFRAG_IPV4_UDP |	\
	RTE_ETH_RSS_NONFRAG_IPV6_UDP |	\
	RTE_ETH_RSS_IPV6_UDP_EX)

/* A list of pages registered with the device during setup and used by a queue
 * as buffers
 */
struct gve_queue_page_list {
	uint32_t id; /* unique id */
	uint32_t num_entries;
	dma_addr_t *page_buses; /* the dma addrs of the pages */
	const struct rte_memzone *mz;
};

/* A TX desc ring entry */
union gve_tx_desc {
	struct gve_tx_pkt_desc pkt; /* first desc for a packet */
	struct gve_tx_seg_desc seg; /* subsequent descs for a packet */
};

/* Tx desc for DQO format */
union gve_tx_desc_dqo {
	struct gve_tx_pkt_desc_dqo pkt;
	struct gve_tx_tso_context_desc_dqo tso_ctx;
	struct gve_tx_general_context_desc_dqo general_ctx;
};

/* Offload features */
union gve_tx_offload {
	uint64_t data;
	struct {
		uint64_t l2_len:7; /* L2 (MAC) Header Length. */
		uint64_t l3_len:9; /* L3 (IP) Header Length. */
		uint64_t l4_len:8; /* L4 Header Length. */
		uint64_t tso_segsz:16; /* TCP TSO segment size */
		/* uint64_t unused : 24; */
	};
};

struct gve_tx_iovec {
	uint32_t iov_base; /* offset in fifo */
	uint32_t iov_len;
};

struct gve_tx_stats {
	uint64_t packets;
	uint64_t bytes;
	uint64_t errors;
};

struct gve_rx_stats {
	uint64_t packets;
	uint64_t bytes;
	uint64_t errors;
	uint64_t no_mbufs;
	uint64_t no_mbufs_bulk;
	uint64_t imissed;
};

struct gve_xstats_name_offset {
	char name[RTE_ETH_XSTATS_NAME_SIZE];
	unsigned int offset;
};

struct gve_tx_queue {
	volatile union gve_tx_desc *tx_desc_ring;
	const struct rte_memzone *mz;
	uint64_t tx_ring_phys_addr;
	struct rte_mbuf **sw_ring;
	volatile rte_be32_t *qtx_tail;
	volatile rte_be32_t *qtx_head;

	uint32_t tx_tail;
	uint16_t nb_tx_desc;
	uint16_t nb_free;
	uint16_t nb_used;
	uint32_t next_to_clean;
	uint16_t free_thresh;
	uint16_t rs_thresh;

	/* Only valid for DQO_QPL queue format */
	uint16_t sw_tail;
	uint16_t sw_ntc;
	uint16_t sw_nb_free;
	uint32_t fifo_size;
	uint32_t fifo_head;
	uint32_t fifo_avail;
	uint64_t fifo_base;
	struct gve_queue_page_list *qpl;
	struct gve_tx_iovec *iov_ring;

	/* stats items */
	struct gve_tx_stats stats;

	uint16_t port_id;
	uint16_t queue_id;

	uint16_t ntfy_id;
	volatile rte_be32_t *ntfy_addr;

	struct gve_priv *hw;
	const struct rte_memzone *qres_mz;
	struct gve_queue_resources *qres;

	/* newly added for DQO */
	volatile union gve_tx_desc_dqo *tx_ring;
	struct gve_tx_compl_desc *compl_ring;
	const struct rte_memzone *compl_ring_mz;
	uint64_t compl_ring_phys_addr;
	uint32_t complq_tail;
	uint16_t sw_size;
	uint8_t cur_gen_bit;
	uint32_t last_desc_cleaned;
	void **txqs;
	uint16_t re_cnt;

	/* Only valid for DQO_RDA queue format */
	struct gve_tx_queue *complq;

	uint8_t is_gqi_qpl;
};

struct gve_rx_ctx {
	struct rte_mbuf *mbuf_head;
	struct rte_mbuf *mbuf_tail;
	uint16_t total_frags;
	bool drop_pkt;
};

struct gve_rx_queue {
	volatile struct gve_rx_desc *rx_desc_ring;
	volatile union gve_rx_data_slot *rx_data_ring;
	const struct rte_memzone *mz;
	const struct rte_memzone *data_mz;
	uint64_t rx_ring_phys_addr;
	struct rte_mbuf **sw_ring;
	struct rte_mempool *mpool;
	struct gve_rx_ctx ctx;

	uint16_t rx_tail;
	uint16_t nb_rx_desc;
	uint16_t expected_seqno; /* the next expected seqno */
	uint16_t free_thresh;
	uint16_t nb_rx_hold;
	uint32_t next_avail;
	uint32_t nb_avail;

	volatile rte_be32_t *qrx_tail;
	volatile rte_be32_t *ntfy_addr;

	/* only valid for GQI_QPL queue format */
	struct gve_queue_page_list *qpl;

	/* stats items */
	struct gve_rx_stats stats;

	struct gve_priv *hw;
	const struct rte_memzone *qres_mz;
	struct gve_queue_resources *qres;

	uint16_t port_id;
	uint16_t queue_id;
	uint16_t ntfy_id;
	uint16_t rx_buf_len;

	/* newly added for DQO */
	volatile struct gve_rx_desc_dqo *rx_ring;
	struct gve_rx_compl_desc_dqo *compl_ring;
	const struct rte_memzone *compl_ring_mz;
	uint64_t compl_ring_phys_addr;
	uint8_t cur_gen_bit;
	uint16_t bufq_tail;

	/* Only valid for DQO_RDA queue format */
	struct gve_rx_queue *bufq;

	uint8_t is_gqi_qpl;
};

struct gve_priv {
	struct gve_irq_db *irq_dbs; /* array of num_ntfy_blks */
	const struct rte_memzone *irq_dbs_mz;
	uint32_t mgmt_msix_idx;
	rte_be32_t *cnt_array; /* array of num_event_counters */
	const struct rte_memzone *cnt_array_mz;

	uint16_t num_event_counters;

	/* TX ring size default and limits. */
	uint16_t default_tx_desc_cnt;
	uint16_t max_tx_desc_cnt;
	uint16_t min_tx_desc_cnt;

	/* RX ring size default and limits. */
	uint16_t default_rx_desc_cnt;
	uint16_t max_rx_desc_cnt;
	uint16_t min_rx_desc_cnt;

	uint16_t tx_pages_per_qpl;

	/* Only valid for DQO_RDA queue format */
	uint16_t tx_compq_size; /* tx completion queue size */
	uint16_t rx_bufq_size; /* rx buff queue size */

	uint64_t max_registered_pages;
	uint64_t num_registered_pages; /* num pages registered with NIC */
	uint16_t default_num_queues; /* default num queues to set up */
	enum gve_queue_format queue_format; /* see enum gve_queue_format */
	uint8_t enable_rsc;

	uint16_t max_nb_txq;
	uint16_t max_nb_rxq;
	uint32_t num_ntfy_blks; /* spilt between TX and RX so must be even */

	struct gve_registers __iomem *reg_bar0; /* see gve_register.h */
	rte_be32_t __iomem *db_bar2; /* "array" of doorbells */
	struct rte_pci_device *pci_dev;

	/* Admin queue - see gve_adminq.h*/
	union gve_adminq_command *adminq;
	struct gve_dma_mem adminq_dma_mem;
	uint32_t adminq_mask; /* masks prod_cnt to adminq size */
	uint32_t adminq_prod_cnt; /* free-running count of AQ cmds executed */
	uint32_t adminq_cmd_fail; /* free-running count of AQ cmds failed */
	uint32_t adminq_timeouts; /* free-running count of AQ cmds timeouts */
	/* free-running count of per AQ cmd executed */
	uint32_t adminq_describe_device_cnt;
	uint32_t adminq_cfg_device_resources_cnt;
	uint32_t adminq_register_page_list_cnt;
	uint32_t adminq_unregister_page_list_cnt;
	uint32_t adminq_create_tx_queue_cnt;
	uint32_t adminq_create_rx_queue_cnt;
	uint32_t adminq_destroy_tx_queue_cnt;
	uint32_t adminq_destroy_rx_queue_cnt;
	uint32_t adminq_dcfg_device_resources_cnt;
	uint32_t adminq_cfg_rss_cnt;
	uint32_t adminq_set_driver_parameter_cnt;
	uint32_t adminq_report_stats_cnt;
	uint32_t adminq_report_link_speed_cnt;
	uint32_t adminq_get_ptype_map_cnt;
	uint32_t adminq_verify_driver_compatibility_cnt;
	volatile uint32_t state_flags;

	/* Gvnic device link speed from hypervisor. */
	uint64_t link_speed;

	uint16_t max_mtu;
	struct rte_ether_addr dev_addr; /* mac address */

	struct gve_tx_queue **txqs;
	struct gve_rx_queue **rxqs;

	uint32_t stats_report_len;
	const struct rte_memzone *stats_report_mem;
	uint16_t stats_start_idx; /* start index of array of stats written by NIC */
	uint16_t stats_end_idx; /* end index of array of stats written by NIC */

	struct gve_rss_config rss_config;
	struct gve_ptype_lut *ptype_lut_dqo;
};

static inline bool
gve_is_gqi(struct gve_priv *priv)
{
	return priv->queue_format == GVE_GQI_RDA_FORMAT ||
		priv->queue_format == GVE_GQI_QPL_FORMAT;
}

static inline bool
gve_get_admin_queue_ok(struct gve_priv *priv)
{
	return !!rte_bit_relaxed_get32(GVE_PRIV_FLAGS_ADMIN_QUEUE_OK,
				       &priv->state_flags);
}

static inline void
gve_set_admin_queue_ok(struct gve_priv *priv)
{
	rte_bit_relaxed_set32(GVE_PRIV_FLAGS_ADMIN_QUEUE_OK,
			      &priv->state_flags);
}

static inline void
gve_clear_admin_queue_ok(struct gve_priv *priv)
{
	rte_bit_relaxed_clear32(GVE_PRIV_FLAGS_ADMIN_QUEUE_OK,
				&priv->state_flags);
}

static inline bool
gve_get_device_resources_ok(struct gve_priv *priv)
{
	return !!rte_bit_relaxed_get32(GVE_PRIV_FLAGS_DEVICE_RESOURCES_OK,
				       &priv->state_flags);
}

static inline void
gve_set_device_resources_ok(struct gve_priv *priv)
{
	rte_bit_relaxed_set32(GVE_PRIV_FLAGS_DEVICE_RESOURCES_OK,
			      &priv->state_flags);
}

static inline void
gve_clear_device_resources_ok(struct gve_priv *priv)
{
	rte_bit_relaxed_clear32(GVE_PRIV_FLAGS_DEVICE_RESOURCES_OK,
				&priv->state_flags);
}

static inline bool
gve_get_device_rings_ok(struct gve_priv *priv)
{
	return !!rte_bit_relaxed_get32(GVE_PRIV_FLAGS_DEVICE_RINGS_OK,
				       &priv->state_flags);
}

static inline void
gve_set_device_rings_ok(struct gve_priv *priv)
{
	rte_bit_relaxed_set32(GVE_PRIV_FLAGS_DEVICE_RINGS_OK,
			      &priv->state_flags);
}

static inline void
gve_clear_device_rings_ok(struct gve_priv *priv)
{
	rte_bit_relaxed_clear32(GVE_PRIV_FLAGS_DEVICE_RINGS_OK,
				&priv->state_flags);
}

int
gve_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id, uint16_t nb_desc,
		   unsigned int socket_id, const struct rte_eth_rxconf *conf,
		   struct rte_mempool *pool);
int
gve_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id, uint16_t nb_desc,
		   unsigned int socket_id, const struct rte_eth_txconf *conf);

void
gve_tx_queue_release(struct rte_eth_dev *dev, uint16_t qid);

void
gve_rx_queue_release(struct rte_eth_dev *dev, uint16_t qid);

int
gve_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id);

int
gve_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id);

int
gve_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id);

int
gve_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id);

void
gve_stop_tx_queues(struct rte_eth_dev *dev);

void
gve_stop_rx_queues(struct rte_eth_dev *dev);

uint16_t
gve_rx_burst(void *rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts);

uint16_t
gve_tx_burst(void *txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts);

void
gve_set_rx_function(struct rte_eth_dev *dev);

void
gve_set_tx_function(struct rte_eth_dev *dev);

struct gve_queue_page_list *
gve_setup_queue_page_list(struct gve_priv *priv, uint16_t queue_id, bool is_rx,
	uint32_t num_pages);
int
gve_teardown_queue_page_list(struct gve_priv *priv,
	struct gve_queue_page_list *qpl);

/* Below functions are used for DQO */

int
gve_rx_queue_setup_dqo(struct rte_eth_dev *dev, uint16_t queue_id,
		       uint16_t nb_desc, unsigned int socket_id,
		       const struct rte_eth_rxconf *conf,
		       struct rte_mempool *pool);
int
gve_tx_queue_setup_dqo(struct rte_eth_dev *dev, uint16_t queue_id,
		       uint16_t nb_desc, unsigned int socket_id,
		       const struct rte_eth_txconf *conf);

void
gve_tx_queue_release_dqo(struct rte_eth_dev *dev, uint16_t qid);

void
gve_rx_queue_release_dqo(struct rte_eth_dev *dev, uint16_t qid);

int
gve_rx_queue_start_dqo(struct rte_eth_dev *dev, uint16_t rx_queue_id);

int
gve_tx_queue_start_dqo(struct rte_eth_dev *dev, uint16_t tx_queue_id);

int
gve_rx_queue_stop_dqo(struct rte_eth_dev *dev, uint16_t rx_queue_id);

int
gve_tx_queue_stop_dqo(struct rte_eth_dev *dev, uint16_t tx_queue_id);

void
gve_stop_tx_queues_dqo(struct rte_eth_dev *dev);

void
gve_stop_rx_queues_dqo(struct rte_eth_dev *dev);

uint16_t
gve_rx_burst_dqo(void *rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts);

uint16_t
gve_tx_burst_dqo(void *txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts);

void
gve_set_rx_function_dqo(struct rte_eth_dev *dev);

void
gve_set_tx_function_dqo(struct rte_eth_dev *dev);

#endif /* _GVE_ETHDEV_H_ */