1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) 2015-2020 Amazon.com, Inc. or its affiliates. 3 * All rights reserved. 4 */ 5 6 #ifndef _ENA_ETHDEV_H_ 7 #define _ENA_ETHDEV_H_ 8 9 #include <rte_atomic.h> 10 #include <rte_ether.h> 11 #include <ethdev_driver.h> 12 #include <ethdev_pci.h> 13 #include <rte_cycles.h> 14 #include <rte_pci.h> 15 #include <rte_bus_pci.h> 16 #include <rte_timer.h> 17 #include <rte_dev.h> 18 #include <rte_net.h> 19 20 #include "ena_com.h" 21 22 #define ENA_REGS_BAR 0 23 #define ENA_MEM_BAR 2 24 25 #define ENA_MAX_NUM_QUEUES 128 26 #define ENA_MIN_FRAME_LEN 64 27 #define ENA_NAME_MAX_LEN 20 28 #define ENA_PKT_MAX_BUFS 17 29 #define ENA_RX_BUF_MIN_SIZE 1400 30 #define ENA_DEFAULT_RING_SIZE 1024 31 32 #define ENA_RX_RSS_TABLE_LOG_SIZE 7 33 #define ENA_RX_RSS_TABLE_SIZE (1 << ENA_RX_RSS_TABLE_LOG_SIZE) 34 35 #define ENA_MIN_MTU 128 36 37 #define ENA_MMIO_DISABLE_REG_READ BIT(0) 38 39 #define ENA_WD_TIMEOUT_SEC 3 40 #define ENA_DEVICE_KALIVE_TIMEOUT (ENA_WD_TIMEOUT_SEC * rte_get_timer_hz()) 41 42 #define ENA_TX_TIMEOUT (5 * rte_get_timer_hz()) 43 #define ENA_MAX_TX_TIMEOUT_SECONDS 60 44 #define ENA_MONITORED_TX_QUEUES 3 45 #define ENA_DEFAULT_MISSING_COMP 256U 46 47 /* While processing submitted and completed descriptors (rx and tx path 48 * respectively) in a loop it is desired to: 49 * - perform batch submissions while populating submission queue 50 * - avoid blocking transmission of other packets during cleanup phase 51 * Hence the utilization ratio of 1/8 of a queue size or max value if the size 52 * of the ring is very big - like 8k Rx rings. 53 */ 54 #define ENA_REFILL_THRESH_DIVIDER 8 55 #define ENA_REFILL_THRESH_PACKET 256 56 57 #define ENA_IDX_NEXT_MASKED(idx, mask) (((idx) + 1) & (mask)) 58 #define ENA_IDX_ADD_MASKED(idx, n, mask) (((idx) + (n)) & (mask)) 59 60 #define ENA_RX_RSS_TABLE_LOG_SIZE 7 61 #define ENA_RX_RSS_TABLE_SIZE (1 << ENA_RX_RSS_TABLE_LOG_SIZE) 62 63 #define ENA_HASH_KEY_SIZE 40 64 65 #define ENA_ALL_RSS_HF (RTE_ETH_RSS_NONFRAG_IPV4_TCP | RTE_ETH_RSS_NONFRAG_IPV4_UDP | \ 66 RTE_ETH_RSS_NONFRAG_IPV6_TCP | RTE_ETH_RSS_NONFRAG_IPV6_UDP) 67 68 #define ENA_IO_TXQ_IDX(q) (2 * (q)) 69 #define ENA_IO_RXQ_IDX(q) (2 * (q) + 1) 70 /* Reversed version of ENA_IO_RXQ_IDX */ 71 #define ENA_IO_RXQ_IDX_REV(q) (((q) - 1) / 2) 72 73 extern struct ena_shared_data *ena_shared_data; 74 75 struct ena_adapter; 76 77 enum ena_ring_type { 78 ENA_RING_TYPE_RX = 1, 79 ENA_RING_TYPE_TX = 2, 80 }; 81 82 struct ena_tx_buffer { 83 struct rte_mbuf *mbuf; 84 unsigned int tx_descs; 85 unsigned int num_of_bufs; 86 uint64_t timestamp; 87 bool print_once; 88 struct ena_com_buf bufs[ENA_PKT_MAX_BUFS]; 89 }; 90 91 /* Rx buffer holds only pointer to the mbuf - may be expanded in the future */ 92 struct ena_rx_buffer { 93 struct rte_mbuf *mbuf; 94 struct ena_com_buf ena_buf; 95 }; 96 97 struct ena_calc_queue_size_ctx { 98 struct ena_com_dev_get_features_ctx *get_feat_ctx; 99 struct ena_com_dev *ena_dev; 100 u32 max_rx_queue_size; 101 u32 max_tx_queue_size; 102 u16 max_tx_sgl_size; 103 u16 max_rx_sgl_size; 104 }; 105 106 struct ena_stats_tx { 107 u64 cnt; 108 u64 bytes; 109 u64 prepare_ctx_err; 110 u64 tx_poll; 111 u64 doorbells; 112 u64 bad_req_id; 113 u64 available_desc; 114 u64 missed_tx; 115 }; 116 117 struct ena_stats_rx { 118 u64 cnt; 119 u64 bytes; 120 u64 refill_partial; 121 u64 l3_csum_bad; 122 u64 l4_csum_bad; 123 u64 l4_csum_good; 124 u64 mbuf_alloc_fail; 125 u64 bad_desc_num; 126 u64 bad_req_id; 127 }; 128 129 struct ena_ring { 130 u16 next_to_use; 131 u16 next_to_clean; 132 uint64_t last_cleanup_ticks; 133 134 enum ena_ring_type type; 135 enum ena_admin_placement_policy_type tx_mem_queue_type; 136 137 /* Indicate there are Tx packets pushed to the device and wait for db */ 138 bool pkts_without_db; 139 140 /* Holds the empty requests for TX/RX OOO completions */ 141 union { 142 uint16_t *empty_tx_reqs; 143 uint16_t *empty_rx_reqs; 144 }; 145 146 union { 147 struct ena_tx_buffer *tx_buffer_info; /* contex of tx packet */ 148 struct ena_rx_buffer *rx_buffer_info; /* contex of rx packet */ 149 }; 150 struct rte_mbuf **rx_refill_buffer; 151 unsigned int ring_size; /* number of tx/rx_buffer_info's entries */ 152 unsigned int size_mask; 153 154 struct ena_com_io_cq *ena_com_io_cq; 155 struct ena_com_io_sq *ena_com_io_sq; 156 157 union { 158 uint16_t tx_free_thresh; 159 uint16_t rx_free_thresh; 160 }; 161 162 struct ena_com_rx_buf_info ena_bufs[ENA_PKT_MAX_BUFS] 163 __rte_cache_aligned; 164 165 struct rte_mempool *mb_pool; 166 unsigned int port_id; 167 unsigned int id; 168 /* Max length PMD can push to device for LLQ */ 169 uint8_t tx_max_header_size; 170 int configured; 171 172 uint8_t *push_buf_intermediate_buf; 173 174 struct ena_adapter *adapter; 175 uint64_t offloads; 176 u16 sgl_size; 177 178 bool disable_meta_caching; 179 180 union { 181 struct ena_stats_rx rx_stats; 182 struct ena_stats_tx tx_stats; 183 }; 184 185 unsigned int numa_socket_id; 186 187 uint32_t missing_tx_completion_threshold; 188 } __rte_cache_aligned; 189 190 enum ena_adapter_state { 191 ENA_ADAPTER_STATE_FREE = 0, 192 ENA_ADAPTER_STATE_INIT = 1, 193 ENA_ADAPTER_STATE_RUNNING = 2, 194 ENA_ADAPTER_STATE_STOPPED = 3, 195 ENA_ADAPTER_STATE_CONFIG = 4, 196 ENA_ADAPTER_STATE_CLOSED = 5, 197 }; 198 199 struct ena_driver_stats { 200 rte_atomic64_t ierrors; 201 rte_atomic64_t oerrors; 202 rte_atomic64_t rx_nombuf; 203 u64 rx_drops; 204 }; 205 206 struct ena_stats_dev { 207 u64 wd_expired; 208 u64 dev_start; 209 u64 dev_stop; 210 /* 211 * Tx drops cannot be reported as the driver statistic, because DPDK 212 * rte_eth_stats structure isn't providing appropriate field for that. 213 * As a workaround it is being published as an extended statistic. 214 */ 215 u64 tx_drops; 216 }; 217 218 struct ena_stats_eni { 219 /* 220 * The number of packets shaped due to inbound aggregate BW 221 * allowance being exceeded 222 */ 223 uint64_t bw_in_allowance_exceeded; 224 /* 225 * The number of packets shaped due to outbound aggregate BW 226 * allowance being exceeded 227 */ 228 uint64_t bw_out_allowance_exceeded; 229 /* The number of packets shaped due to PPS allowance being exceeded */ 230 uint64_t pps_allowance_exceeded; 231 /* 232 * The number of packets shaped due to connection tracking 233 * allowance being exceeded and leading to failure in establishment 234 * of new connections 235 */ 236 uint64_t conntrack_allowance_exceeded; 237 /* 238 * The number of packets shaped due to linklocal packet rate 239 * allowance being exceeded 240 */ 241 uint64_t linklocal_allowance_exceeded; 242 }; 243 244 struct ena_offloads { 245 uint32_t tx_offloads; 246 uint32_t rx_offloads; 247 }; 248 249 /* board specific private data structure */ 250 struct ena_adapter { 251 /* OS defined structs */ 252 struct rte_eth_dev_data *edev_data; 253 254 struct ena_com_dev ena_dev __rte_cache_aligned; 255 256 /* TX */ 257 struct ena_ring tx_ring[ENA_MAX_NUM_QUEUES] __rte_cache_aligned; 258 u32 max_tx_ring_size; 259 u16 max_tx_sgl_size; 260 261 /* RX */ 262 struct ena_ring rx_ring[ENA_MAX_NUM_QUEUES] __rte_cache_aligned; 263 u32 max_rx_ring_size; 264 u16 max_rx_sgl_size; 265 266 u32 max_num_io_queues; 267 u16 max_mtu; 268 struct ena_offloads offloads; 269 270 /* The admin queue isn't protected by the lock and is used to 271 * retrieve statistics from the device. As there is no guarantee that 272 * application won't try to get statistics from multiple threads, it is 273 * safer to lock the queue to avoid admin queue failure. 274 */ 275 rte_spinlock_t admin_lock; 276 277 int id_number; 278 char name[ENA_NAME_MAX_LEN]; 279 u8 mac_addr[RTE_ETHER_ADDR_LEN]; 280 281 void *regs; 282 void *dev_mem_base; 283 284 struct ena_driver_stats *drv_stats; 285 enum ena_adapter_state state; 286 287 bool link_status; 288 289 enum ena_regs_reset_reason_types reset_reason; 290 291 struct rte_timer timer_wd; 292 uint64_t timestamp_wd; 293 uint64_t keep_alive_timeout; 294 295 struct ena_stats_dev dev_stats; 296 struct ena_stats_eni eni_stats; 297 struct ena_admin_basic_stats basic_stats; 298 299 u32 indirect_table[ENA_RX_RSS_TABLE_SIZE]; 300 301 uint32_t all_aenq_groups; 302 uint32_t active_aenq_groups; 303 304 bool trigger_reset; 305 306 bool use_large_llq_hdr; 307 308 uint32_t last_tx_comp_qid; 309 uint64_t missing_tx_completion_to; 310 uint64_t missing_tx_completion_budget; 311 uint64_t tx_cleanup_stall_delay; 312 313 uint64_t memzone_cnt; 314 }; 315 316 int ena_mp_indirect_table_set(struct ena_adapter *adapter); 317 int ena_mp_indirect_table_get(struct ena_adapter *adapter, 318 uint32_t *indirect_table); 319 int ena_rss_reta_update(struct rte_eth_dev *dev, 320 struct rte_eth_rss_reta_entry64 *reta_conf, 321 uint16_t reta_size); 322 int ena_rss_reta_query(struct rte_eth_dev *dev, 323 struct rte_eth_rss_reta_entry64 *reta_conf, 324 uint16_t reta_size); 325 int ena_rss_hash_update(struct rte_eth_dev *dev, 326 struct rte_eth_rss_conf *rss_conf); 327 int ena_rss_hash_conf_get(struct rte_eth_dev *dev, 328 struct rte_eth_rss_conf *rss_conf); 329 int ena_rss_configure(struct ena_adapter *adapter); 330 331 #endif /* _ENA_ETHDEV_H_ */ 332