1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2008-2017 Cisco Systems, Inc. All rights reserved. 3 * Copyright 2007 Nuova Systems, Inc. All rights reserved. 4 */ 5 6 #ifndef _ENIC_H_ 7 #define _ENIC_H_ 8 9 #include "vnic_enet.h" 10 #include "vnic_dev.h" 11 #include "vnic_wq.h" 12 #include "vnic_rq.h" 13 #include "vnic_cq.h" 14 #include "vnic_intr.h" 15 #include "vnic_stats.h" 16 #include "vnic_nic.h" 17 #include "vnic_rss.h" 18 #include "enic_res.h" 19 #include "cq_enet_desc.h" 20 #include <stdbool.h> 21 #include <sys/queue.h> 22 #include <rte_spinlock.h> 23 24 #define DRV_NAME "enic_pmd" 25 #define DRV_DESCRIPTION "Cisco VIC Ethernet NIC Poll-mode Driver" 26 #define DRV_COPYRIGHT "Copyright 2008-2015 Cisco Systems, Inc" 27 28 #define VLAN_ETH_HLEN 18 29 30 #define ENICPMD_SETTING(enic, f) ((enic->config.flags & VENETF_##f) ? 1 : 0) 31 32 #define ENICPMD_BDF_LENGTH 13 /* 0000:00:00.0'\0' */ 33 #define ENIC_CALC_IP_CKSUM 1 34 #define ENIC_CALC_TCP_UDP_CKSUM 2 35 #define ENIC_MAX_MTU 9000 36 #define ENIC_PAGE_SIZE 4096 37 #define PAGE_ROUND_UP(x) \ 38 ((((unsigned long)(x)) + ENIC_PAGE_SIZE-1) & (~(ENIC_PAGE_SIZE-1))) 39 40 #define ENICPMD_VFIO_PATH "/dev/vfio/vfio" 41 /*#define ENIC_DESC_COUNT_MAKE_ODD (x) do{if ((~(x)) & 1) { (x)--; } }while(0)*/ 42 43 #define PCI_DEVICE_ID_CISCO_VIC_ENET 0x0043 /* ethernet vnic */ 44 #define PCI_DEVICE_ID_CISCO_VIC_ENET_VF 0x0071 /* enet SRIOV VF */ 45 46 /* Special Filter id for non-specific packet flagging. Don't change value */ 47 #define ENIC_MAGIC_FILTER_ID 0xffff 48 49 #define ENICPMD_FDIR_MAX 64 50 51 /* HW default VXLAN port */ 52 #define ENIC_DEFAULT_VXLAN_PORT 4789 53 54 /* 55 * Interrupt 0: LSC and errors 56 * Interrupt 1: rx queue 0 57 * Interrupt 2: rx queue 1 58 * ... 59 */ 60 #define ENICPMD_LSC_INTR_OFFSET 0 61 #define ENICPMD_RXQ_INTR_OFFSET 1 62 63 struct enic_fdir_node { 64 struct rte_eth_fdir_filter filter; 65 u16 fltr_id; 66 u16 rq_index; 67 }; 68 69 struct enic_fdir { 70 struct rte_eth_fdir_stats stats; 71 struct rte_hash *hash; 72 struct enic_fdir_node *nodes[ENICPMD_FDIR_MAX]; 73 u32 modes; 74 u32 types_mask; 75 void (*copy_fltr_fn)(struct filter_v2 *filt, 76 const struct rte_eth_fdir_input *input, 77 const struct rte_eth_fdir_masks *masks); 78 }; 79 80 struct enic_soft_stats { 81 rte_atomic64_t rx_nombuf; 82 rte_atomic64_t rx_packet_errors; 83 rte_atomic64_t tx_oversized; 84 }; 85 86 struct enic_memzone_entry { 87 const struct rte_memzone *rz; 88 LIST_ENTRY(enic_memzone_entry) entries; 89 }; 90 91 struct rte_flow { 92 LIST_ENTRY(rte_flow) next; 93 u16 enic_filter_id; 94 struct filter_v2 enic_filter; 95 }; 96 97 /* Per-instance private data structure */ 98 struct enic { 99 struct enic *next; 100 struct rte_pci_device *pdev; 101 struct vnic_enet_config config; 102 struct vnic_dev_bar bar0; 103 struct vnic_dev *vdev; 104 105 /* 106 * mbuf_initializer contains 64 bits of mbuf rearm_data, used by 107 * the avx2 handler at this time. 108 */ 109 uint64_t mbuf_initializer; 110 unsigned int port_id; 111 bool overlay_offload; 112 struct rte_eth_dev *rte_dev; 113 struct enic_fdir fdir; 114 char bdf_name[ENICPMD_BDF_LENGTH]; 115 int dev_fd; 116 int iommu_group_fd; 117 int iommu_groupid; 118 int eventfd; 119 uint8_t mac_addr[ETH_ALEN]; 120 pthread_t err_intr_thread; 121 int promisc; 122 int allmulti; 123 u8 ig_vlan_strip_en; 124 int link_status; 125 u8 hw_ip_checksum; 126 u16 max_mtu; 127 u8 adv_filters; 128 u32 flow_filter_mode; 129 u8 filter_actions; /* HW supported actions */ 130 bool vxlan; 131 bool disable_overlay; /* devargs disable_overlay=1 */ 132 uint8_t enable_avx2_rx; /* devargs enable-avx2-rx=1 */ 133 bool nic_cfg_chk; /* NIC_CFG_CHK available */ 134 bool udp_rss_weak; /* Bodega style UDP RSS */ 135 uint8_t ig_vlan_rewrite_mode; /* devargs ig-vlan-rewrite */ 136 uint16_t vxlan_port; /* current vxlan port pushed to NIC */ 137 138 unsigned int flags; 139 unsigned int priv_flags; 140 141 /* work queue (len = conf_wq_count) */ 142 struct vnic_wq *wq; 143 unsigned int wq_count; /* equals eth_dev nb_tx_queues */ 144 145 /* receive queue (len = conf_rq_count) */ 146 struct vnic_rq *rq; 147 unsigned int rq_count; /* equals eth_dev nb_rx_queues */ 148 149 /* completion queue (len = conf_cq_count) */ 150 struct vnic_cq *cq; 151 unsigned int cq_count; /* equals rq_count + wq_count */ 152 153 /* interrupt vectors (len = conf_intr_count) */ 154 struct vnic_intr *intr; 155 unsigned int intr_count; /* equals enabled interrupts (lsc + rxqs) */ 156 157 /* software counters */ 158 struct enic_soft_stats soft_stats; 159 160 /* configured resources on vic */ 161 unsigned int conf_rq_count; 162 unsigned int conf_wq_count; 163 unsigned int conf_cq_count; 164 unsigned int conf_intr_count; 165 166 /* linked list storing memory allocations */ 167 LIST_HEAD(enic_memzone_list, enic_memzone_entry) memzone_list; 168 rte_spinlock_t memzone_list_lock; 169 rte_spinlock_t mtu_lock; 170 171 LIST_HEAD(enic_flows, rte_flow) flows; 172 173 /* RSS */ 174 uint16_t reta_size; 175 uint8_t hash_key_size; 176 uint64_t flow_type_rss_offloads; /* 0 indicates RSS not supported */ 177 /* 178 * Keep a copy of current RSS config for queries, as we cannot retrieve 179 * it from the NIC. 180 */ 181 uint8_t rss_hash_type; /* NIC_CFG_RSS_HASH_TYPE flags */ 182 uint8_t rss_enable; 183 uint64_t rss_hf; /* ETH_RSS flags */ 184 union vnic_rss_key rss_key; 185 union vnic_rss_cpu rss_cpu; 186 187 uint64_t rx_offload_capa; /* DEV_RX_OFFLOAD flags */ 188 uint64_t tx_offload_capa; /* DEV_TX_OFFLOAD flags */ 189 uint64_t tx_queue_offload_capa; /* DEV_TX_OFFLOAD flags */ 190 uint64_t tx_offload_mask; /* PKT_TX flags accepted */ 191 192 /* Multicast MAC addresses added to the NIC */ 193 uint32_t mc_count; 194 struct rte_ether_addr mc_addrs[ENIC_MULTICAST_PERFECT_FILTERS]; 195 }; 196 197 /* Compute ethdev's max packet size from MTU */ 198 static inline uint32_t enic_mtu_to_max_rx_pktlen(uint32_t mtu) 199 { 200 /* ethdev max size includes eth whereas NIC MTU does not */ 201 return mtu + RTE_ETHER_HDR_LEN; 202 } 203 204 /* Get the CQ index from a Start of Packet(SOP) RQ index */ 205 static inline unsigned int enic_sop_rq_idx_to_cq_idx(unsigned int sop_idx) 206 { 207 return sop_idx / 2; 208 } 209 210 /* Get the RTE RQ index from a Start of Packet(SOP) RQ index */ 211 static inline unsigned int enic_sop_rq_idx_to_rte_idx(unsigned int sop_idx) 212 { 213 return sop_idx / 2; 214 } 215 216 /* Get the Start of Packet(SOP) RQ index from a RTE RQ index */ 217 static inline unsigned int enic_rte_rq_idx_to_sop_idx(unsigned int rte_idx) 218 { 219 return rte_idx * 2; 220 } 221 222 /* Get the Data RQ index from a RTE RQ index */ 223 static inline unsigned int enic_rte_rq_idx_to_data_idx(unsigned int rte_idx) 224 { 225 return rte_idx * 2 + 1; 226 } 227 228 static inline unsigned int enic_vnic_rq_count(struct enic *enic) 229 { 230 return enic->rq_count * 2; 231 } 232 233 static inline unsigned int enic_cq_rq(__rte_unused struct enic *enic, unsigned int rq) 234 { 235 /* Scatter rx uses two receive queues together with one 236 * completion queue, so the completion queue number is no 237 * longer the same as the rq number. 238 */ 239 return rq / 2; 240 } 241 242 static inline unsigned int enic_cq_wq(struct enic *enic, unsigned int wq) 243 { 244 return enic->rq_count + wq; 245 } 246 247 static inline struct enic *pmd_priv(struct rte_eth_dev *eth_dev) 248 { 249 return eth_dev->data->dev_private; 250 } 251 252 static inline uint32_t 253 enic_ring_add(uint32_t n_descriptors, uint32_t i0, uint32_t i1) 254 { 255 uint32_t d = i0 + i1; 256 d -= (d >= n_descriptors) ? n_descriptors : 0; 257 return d; 258 } 259 260 static inline uint32_t 261 enic_ring_sub(uint32_t n_descriptors, uint32_t i0, uint32_t i1) 262 { 263 int32_t d = i1 - i0; 264 return (uint32_t)((d < 0) ? ((int32_t)n_descriptors + d) : d); 265 } 266 267 static inline uint32_t 268 enic_ring_incr(uint32_t n_descriptors, uint32_t idx) 269 { 270 idx++; 271 if (unlikely(idx == n_descriptors)) 272 idx = 0; 273 return idx; 274 } 275 276 void enic_fdir_stats_get(struct enic *enic, 277 struct rte_eth_fdir_stats *stats); 278 int enic_fdir_add_fltr(struct enic *enic, 279 struct rte_eth_fdir_filter *params); 280 int enic_fdir_del_fltr(struct enic *enic, 281 struct rte_eth_fdir_filter *params); 282 void enic_free_wq(void *txq); 283 int enic_alloc_intr_resources(struct enic *enic); 284 int enic_setup_finish(struct enic *enic); 285 int enic_alloc_wq(struct enic *enic, uint16_t queue_idx, 286 unsigned int socket_id, uint16_t nb_desc); 287 void enic_start_wq(struct enic *enic, uint16_t queue_idx); 288 int enic_stop_wq(struct enic *enic, uint16_t queue_idx); 289 void enic_start_rq(struct enic *enic, uint16_t queue_idx); 290 int enic_stop_rq(struct enic *enic, uint16_t queue_idx); 291 void enic_free_rq(void *rxq); 292 int enic_alloc_rq(struct enic *enic, uint16_t queue_idx, 293 unsigned int socket_id, struct rte_mempool *mp, 294 uint16_t nb_desc, uint16_t free_thresh); 295 int enic_set_vnic_res(struct enic *enic); 296 int enic_init_rss_nic_cfg(struct enic *enic); 297 int enic_set_rss_conf(struct enic *enic, 298 struct rte_eth_rss_conf *rss_conf); 299 int enic_set_rss_reta(struct enic *enic, union vnic_rss_cpu *rss_cpu); 300 int enic_set_vlan_strip(struct enic *enic); 301 int enic_enable(struct enic *enic); 302 int enic_disable(struct enic *enic); 303 void enic_remove(struct enic *enic); 304 int enic_get_link_status(struct enic *enic); 305 int enic_dev_stats_get(struct enic *enic, 306 struct rte_eth_stats *r_stats); 307 void enic_dev_stats_clear(struct enic *enic); 308 void enic_add_packet_filter(struct enic *enic); 309 int enic_set_mac_address(struct enic *enic, uint8_t *mac_addr); 310 int enic_del_mac_address(struct enic *enic, int mac_index); 311 unsigned int enic_cleanup_wq(struct enic *enic, struct vnic_wq *wq); 312 void enic_send_pkt(struct enic *enic, struct vnic_wq *wq, 313 struct rte_mbuf *tx_pkt, unsigned short len, 314 uint8_t sop, uint8_t eop, uint8_t cq_entry, 315 uint16_t ol_flags, uint16_t vlan_tag); 316 317 void enic_post_wq_index(struct vnic_wq *wq); 318 int enic_probe(struct enic *enic); 319 int enic_clsf_init(struct enic *enic); 320 void enic_clsf_destroy(struct enic *enic); 321 uint16_t enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, 322 uint16_t nb_pkts); 323 uint16_t enic_noscatter_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, 324 uint16_t nb_pkts); 325 uint16_t enic_dummy_recv_pkts(void *rx_queue, 326 struct rte_mbuf **rx_pkts, 327 uint16_t nb_pkts); 328 uint16_t enic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 329 uint16_t nb_pkts); 330 uint16_t enic_simple_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 331 uint16_t nb_pkts); 332 uint16_t enic_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 333 uint16_t nb_pkts); 334 int enic_set_mtu(struct enic *enic, uint16_t new_mtu); 335 int enic_link_update(struct enic *enic); 336 bool enic_use_vector_rx_handler(struct enic *enic); 337 void enic_fdir_info(struct enic *enic); 338 void enic_fdir_info_get(struct enic *enic, struct rte_eth_fdir_info *stats); 339 extern const struct rte_flow_ops enic_flow_ops; 340 #endif /* _ENIC_H_ */ 341