1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) 2015-2020 Amazon.com, Inc. or its affiliates. 3 * All rights reserved. 4 */ 5 6 #ifndef _ENA_ETHDEV_H_ 7 #define _ENA_ETHDEV_H_ 8 9 #include <rte_atomic.h> 10 #include <rte_ether.h> 11 #include <ethdev_driver.h> 12 #include <ethdev_pci.h> 13 #include <rte_cycles.h> 14 #include <rte_pci.h> 15 #include <rte_bus_pci.h> 16 #include <rte_timer.h> 17 #include <rte_dev.h> 18 #include <rte_net.h> 19 20 #include "ena_com.h" 21 22 #define ENA_REGS_BAR 0 23 #define ENA_MEM_BAR 2 24 25 #define ENA_MAX_NUM_QUEUES 128 26 #define ENA_MIN_FRAME_LEN 64 27 #define ENA_NAME_MAX_LEN 20 28 #define ENA_PKT_MAX_BUFS 17 29 #define ENA_RX_BUF_MIN_SIZE 1400 30 #define ENA_DEFAULT_RING_SIZE 1024 31 32 #define ENA_RX_RSS_TABLE_LOG_SIZE 7 33 #define ENA_RX_RSS_TABLE_SIZE (1 << ENA_RX_RSS_TABLE_LOG_SIZE) 34 35 #define ENA_MIN_MTU 128 36 37 #define ENA_MMIO_DISABLE_REG_READ BIT(0) 38 39 #define ENA_WD_TIMEOUT_SEC 3 40 #define ENA_DEVICE_KALIVE_TIMEOUT (ENA_WD_TIMEOUT_SEC * rte_get_timer_hz()) 41 42 #define ENA_TX_TIMEOUT (5 * rte_get_timer_hz()) 43 #define ENA_MAX_TX_TIMEOUT_SECONDS 60 44 #define ENA_MONITORED_TX_QUEUES 3 45 #define ENA_DEFAULT_MISSING_COMP 256U 46 47 #define ENA_MAX_CONTROL_PATH_POLL_INTERVAL_MSEC 1000 48 49 /* While processing submitted and completed descriptors (rx and tx path 50 * respectively) in a loop it is desired to: 51 * - perform batch submissions while populating submission queue 52 * - avoid blocking transmission of other packets during cleanup phase 53 * Hence the utilization ratio of 1/8 of a queue size or max value if the size 54 * of the ring is very big - like 8k Rx rings. 55 */ 56 #define ENA_REFILL_THRESH_DIVIDER 8 57 #define ENA_REFILL_THRESH_PACKET 256 58 59 /* 60 * The max customer metrics is equal or bigger than the ENI metrics. That 61 * assumption simplifies the fallback to the legacy metrics mechanism. 62 */ 63 #define ENA_MAX_CUSTOMER_METRICS 6 64 65 #define ENA_IDX_NEXT_MASKED(idx, mask) (((idx) + 1) & (mask)) 66 #define ENA_IDX_ADD_MASKED(idx, n, mask) (((idx) + (n)) & (mask)) 67 68 #define ENA_RX_RSS_TABLE_LOG_SIZE 7 69 #define ENA_RX_RSS_TABLE_SIZE (1 << ENA_RX_RSS_TABLE_LOG_SIZE) 70 71 #define ENA_HASH_KEY_SIZE 40 72 73 #define ENA_ALL_RSS_HF (RTE_ETH_RSS_NONFRAG_IPV4_TCP | RTE_ETH_RSS_NONFRAG_IPV4_UDP | \ 74 RTE_ETH_RSS_NONFRAG_IPV6_TCP | RTE_ETH_RSS_NONFRAG_IPV6_UDP) 75 76 #define ENA_IO_TXQ_IDX(q) (2 * (q)) 77 #define ENA_IO_RXQ_IDX(q) (2 * (q) + 1) 78 /* Reversed version of ENA_IO_RXQ_IDX */ 79 #define ENA_IO_RXQ_IDX_REV(q) (((q) - 1) / 2) 80 81 extern struct ena_shared_data *ena_shared_data; 82 83 struct ena_adapter; 84 85 enum ena_ring_type { 86 ENA_RING_TYPE_RX = 1, 87 ENA_RING_TYPE_TX = 2, 88 }; 89 90 typedef enum ena_llq_policy_t { 91 ENA_LLQ_POLICY_DISABLED = 0, /* Host queues */ 92 ENA_LLQ_POLICY_RECOMMENDED = 1, /* Device recommendation */ 93 ENA_LLQ_POLICY_NORMAL = 2, /* 128B long LLQ entry */ 94 ENA_LLQ_POLICY_LARGE = 3, /* 256B long LLQ entry */ 95 ENA_LLQ_POLICY_LAST, 96 } ena_llq_policy; 97 98 struct ena_tx_buffer { 99 struct rte_mbuf *mbuf; 100 unsigned int tx_descs; 101 unsigned int num_of_bufs; 102 uint64_t timestamp; 103 bool print_once; 104 struct ena_com_buf bufs[ENA_PKT_MAX_BUFS]; 105 }; 106 107 /* Rx buffer holds only pointer to the mbuf - may be expanded in the future */ 108 struct ena_rx_buffer { 109 struct rte_mbuf *mbuf; 110 struct ena_com_buf ena_buf; 111 }; 112 113 struct ena_calc_queue_size_ctx { 114 struct ena_com_dev_get_features_ctx *get_feat_ctx; 115 struct ena_com_dev *ena_dev; 116 u32 max_rx_queue_size; 117 u32 max_tx_queue_size; 118 u16 max_tx_sgl_size; 119 u16 max_rx_sgl_size; 120 }; 121 122 struct ena_stats_tx { 123 u64 cnt; 124 u64 bytes; 125 u64 prepare_ctx_err; 126 u64 tx_poll; 127 u64 doorbells; 128 u64 bad_req_id; 129 u64 available_desc; 130 u64 missed_tx; 131 }; 132 133 struct ena_stats_rx { 134 u64 cnt; 135 u64 bytes; 136 u64 refill_partial; 137 u64 l3_csum_bad; 138 u64 l4_csum_bad; 139 u64 l4_csum_good; 140 u64 mbuf_alloc_fail; 141 u64 bad_desc_num; 142 u64 bad_req_id; 143 }; 144 145 struct __rte_cache_aligned ena_ring { 146 u16 next_to_use; 147 u16 next_to_clean; 148 uint64_t last_cleanup_ticks; 149 150 enum ena_ring_type type; 151 enum ena_admin_placement_policy_type tx_mem_queue_type; 152 153 /* Indicate there are Tx packets pushed to the device and wait for db */ 154 bool pkts_without_db; 155 156 /* Holds the empty requests for TX/RX OOO completions */ 157 union { 158 uint16_t *empty_tx_reqs; 159 uint16_t *empty_rx_reqs; 160 }; 161 162 union { 163 struct ena_tx_buffer *tx_buffer_info; /* contex of tx packet */ 164 struct ena_rx_buffer *rx_buffer_info; /* contex of rx packet */ 165 }; 166 struct rte_mbuf **rx_refill_buffer; 167 unsigned int ring_size; /* number of tx/rx_buffer_info's entries */ 168 unsigned int size_mask; 169 170 struct ena_com_io_cq *ena_com_io_cq; 171 struct ena_com_io_sq *ena_com_io_sq; 172 173 union { 174 uint16_t tx_free_thresh; 175 uint16_t rx_free_thresh; 176 }; 177 178 alignas(RTE_CACHE_LINE_SIZE) struct ena_com_rx_buf_info ena_bufs[ENA_PKT_MAX_BUFS]; 179 180 struct rte_mempool *mb_pool; 181 unsigned int port_id; 182 unsigned int id; 183 /* Max length PMD can push to device for LLQ */ 184 uint8_t tx_max_header_size; 185 int configured; 186 187 uint8_t *push_buf_intermediate_buf; 188 189 struct ena_adapter *adapter; 190 uint64_t offloads; 191 u16 sgl_size; 192 193 bool disable_meta_caching; 194 195 union { 196 struct ena_stats_rx rx_stats; 197 struct ena_stats_tx tx_stats; 198 }; 199 200 unsigned int numa_socket_id; 201 202 uint32_t missing_tx_completion_threshold; 203 }; 204 205 enum ena_adapter_state { 206 ENA_ADAPTER_STATE_FREE = 0, 207 ENA_ADAPTER_STATE_INIT = 1, 208 ENA_ADAPTER_STATE_RUNNING = 2, 209 ENA_ADAPTER_STATE_STOPPED = 3, 210 ENA_ADAPTER_STATE_CONFIG = 4, 211 ENA_ADAPTER_STATE_CLOSED = 5, 212 }; 213 214 struct ena_driver_stats { 215 rte_atomic64_t ierrors; 216 rte_atomic64_t oerrors; 217 rte_atomic64_t rx_nombuf; 218 u64 rx_drops; 219 }; 220 221 struct ena_stats_dev { 222 u64 wd_expired; 223 u64 dev_start; 224 u64 dev_stop; 225 /* 226 * Tx drops cannot be reported as the driver statistic, because DPDK 227 * rte_eth_stats structure isn't providing appropriate field for that. 228 * As a workaround it is being published as an extended statistic. 229 */ 230 u64 tx_drops; 231 }; 232 233 struct ena_stats_metrics { 234 /* 235 * The number of packets shaped due to inbound aggregate BW 236 * allowance being exceeded 237 */ 238 uint64_t bw_in_allowance_exceeded; 239 /* 240 * The number of packets shaped due to outbound aggregate BW 241 * allowance being exceeded 242 */ 243 uint64_t bw_out_allowance_exceeded; 244 /* The number of packets shaped due to PPS allowance being exceeded */ 245 uint64_t pps_allowance_exceeded; 246 /* 247 * The number of packets shaped due to connection tracking 248 * allowance being exceeded and leading to failure in establishment 249 * of new connections 250 */ 251 uint64_t conntrack_allowance_exceeded; 252 /* 253 * The number of packets shaped due to linklocal packet rate 254 * allowance being exceeded 255 */ 256 uint64_t linklocal_allowance_exceeded; 257 /* 258 * The number of available connections 259 */ 260 uint64_t conntrack_allowance_available; 261 }; 262 263 struct ena_stats_srd { 264 /* Describes which ENA Express features are enabled */ 265 uint64_t ena_srd_mode; 266 267 /* Number of packets transmitted over ENA SRD */ 268 uint64_t ena_srd_tx_pkts; 269 270 /* Number of packets transmitted or could have been transmitted over ENA SRD */ 271 uint64_t ena_srd_eligible_tx_pkts; 272 273 /* Number of packets received over ENA SRD */ 274 uint64_t ena_srd_rx_pkts; 275 276 /* Percentage of the ENA SRD resources that is in use */ 277 uint64_t ena_srd_resource_utilization; 278 }; 279 280 struct ena_offloads { 281 uint32_t tx_offloads; 282 uint32_t rx_offloads; 283 }; 284 285 /* board specific private data structure */ 286 struct ena_adapter { 287 /* OS defined structs */ 288 struct rte_eth_dev_data *edev_data; 289 290 alignas(RTE_CACHE_LINE_SIZE) struct ena_com_dev ena_dev; 291 292 /* TX */ 293 alignas(RTE_CACHE_LINE_SIZE) struct ena_ring tx_ring[ENA_MAX_NUM_QUEUES]; 294 u32 max_tx_ring_size; 295 u16 max_tx_sgl_size; 296 297 /* RX */ 298 alignas(RTE_CACHE_LINE_SIZE) struct ena_ring rx_ring[ENA_MAX_NUM_QUEUES]; 299 u32 max_rx_ring_size; 300 u16 max_rx_sgl_size; 301 302 u32 max_num_io_queues; 303 u16 max_mtu; 304 struct ena_offloads offloads; 305 306 /* The admin queue isn't protected by the lock and is used to 307 * retrieve statistics from the device. As there is no guarantee that 308 * application won't try to get statistics from multiple threads, it is 309 * safer to lock the queue to avoid admin queue failure. 310 */ 311 rte_spinlock_t admin_lock; 312 313 int id_number; 314 char name[ENA_NAME_MAX_LEN]; 315 u8 mac_addr[RTE_ETHER_ADDR_LEN]; 316 317 void *regs; 318 void *dev_mem_base; 319 320 struct ena_driver_stats *drv_stats; 321 enum ena_adapter_state state; 322 323 bool link_status; 324 325 enum ena_regs_reset_reason_types reset_reason; 326 327 struct rte_timer timer_wd; 328 uint64_t timestamp_wd; 329 uint64_t keep_alive_timeout; 330 331 struct ena_stats_dev dev_stats; 332 struct ena_admin_basic_stats basic_stats; 333 334 u32 indirect_table[ENA_RX_RSS_TABLE_SIZE]; 335 336 uint32_t all_aenq_groups; 337 uint32_t active_aenq_groups; 338 339 bool trigger_reset; 340 ena_llq_policy llq_header_policy; 341 342 uint32_t last_tx_comp_qid; 343 uint64_t missing_tx_completion_to; 344 uint64_t missing_tx_completion_budget; 345 uint64_t tx_cleanup_stall_delay; 346 347 uint64_t memzone_cnt; 348 349 /* Time (in microseconds) of the control path queues monitoring interval */ 350 uint64_t control_path_poll_interval; 351 352 /* 353 * Helper variables for holding the information about the supported 354 * metrics. 355 */ 356 alignas(RTE_CACHE_LINE_SIZE) uint64_t metrics_stats[ENA_MAX_CUSTOMER_METRICS]; 357 uint16_t metrics_num; 358 alignas(RTE_CACHE_LINE_SIZE) struct ena_stats_srd srd_stats; 359 }; 360 361 int ena_mp_indirect_table_set(struct ena_adapter *adapter); 362 int ena_mp_indirect_table_get(struct ena_adapter *adapter, 363 uint32_t *indirect_table); 364 int ena_rss_reta_update(struct rte_eth_dev *dev, 365 struct rte_eth_rss_reta_entry64 *reta_conf, 366 uint16_t reta_size); 367 int ena_rss_reta_query(struct rte_eth_dev *dev, 368 struct rte_eth_rss_reta_entry64 *reta_conf, 369 uint16_t reta_size); 370 int ena_rss_hash_update(struct rte_eth_dev *dev, 371 struct rte_eth_rss_conf *rss_conf); 372 int ena_rss_hash_conf_get(struct rte_eth_dev *dev, 373 struct rte_eth_rss_conf *rss_conf); 374 int ena_rss_configure(struct ena_adapter *adapter); 375 376 #endif /* _ENA_ETHDEV_H_ */ 377