1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) 2015-2020 Amazon.com, Inc. or its affiliates. 3 * All rights reserved. 4 */ 5 6 #ifndef _ENA_ETHDEV_H_ 7 #define _ENA_ETHDEV_H_ 8 9 #include <rte_atomic.h> 10 #include <rte_ether.h> 11 #include <ethdev_driver.h> 12 #include <ethdev_pci.h> 13 #include <rte_cycles.h> 14 #include <rte_pci.h> 15 #include <rte_bus_pci.h> 16 #include <rte_timer.h> 17 #include <rte_dev.h> 18 #include <rte_net.h> 19 20 #include "ena_com.h" 21 22 #define ENA_REGS_BAR 0 23 #define ENA_MEM_BAR 2 24 25 #define ENA_MAX_NUM_QUEUES 128 26 #define ENA_MIN_FRAME_LEN 64 27 #define ENA_NAME_MAX_LEN 20 28 #define ENA_PKT_MAX_BUFS 17 29 #define ENA_RX_BUF_MIN_SIZE 1400 30 #define ENA_DEFAULT_RING_SIZE 1024 31 32 #define ENA_RX_RSS_TABLE_LOG_SIZE 7 33 #define ENA_RX_RSS_TABLE_SIZE (1 << ENA_RX_RSS_TABLE_LOG_SIZE) 34 35 #define ENA_MIN_MTU 128 36 37 #define ENA_MMIO_DISABLE_REG_READ BIT(0) 38 39 #define ENA_WD_TIMEOUT_SEC 3 40 #define ENA_DEVICE_KALIVE_TIMEOUT (ENA_WD_TIMEOUT_SEC * rte_get_timer_hz()) 41 42 #define ENA_TX_TIMEOUT (5 * rte_get_timer_hz()) 43 #define ENA_MAX_TX_TIMEOUT_SECONDS 60 44 #define ENA_MONITORED_TX_QUEUES 3 45 #define ENA_DEFAULT_MISSING_COMP 256U 46 47 #define ENA_MAX_CONTROL_PATH_POLL_INTERVAL_MSEC 1000 48 49 /* While processing submitted and completed descriptors (rx and tx path 50 * respectively) in a loop it is desired to: 51 * - perform batch submissions while populating submission queue 52 * - avoid blocking transmission of other packets during cleanup phase 53 * Hence the utilization ratio of 1/8 of a queue size or max value if the size 54 * of the ring is very big - like 8k Rx rings. 55 */ 56 #define ENA_REFILL_THRESH_DIVIDER 8 57 #define ENA_REFILL_THRESH_PACKET 256 58 59 /* 60 * The max customer metrics is equal or bigger than the ENI metrics. That 61 * assumption simplifies the fallback to the legacy metrics mechanism. 62 */ 63 #define ENA_MAX_CUSTOMER_METRICS 6 64 65 #define ENA_IDX_NEXT_MASKED(idx, mask) (((idx) + 1) & (mask)) 66 #define ENA_IDX_ADD_MASKED(idx, n, mask) (((idx) + (n)) & (mask)) 67 68 #define ENA_RX_RSS_TABLE_LOG_SIZE 7 69 #define ENA_RX_RSS_TABLE_SIZE (1 << ENA_RX_RSS_TABLE_LOG_SIZE) 70 71 #define ENA_HASH_KEY_SIZE 40 72 73 #define ENA_ALL_RSS_HF (RTE_ETH_RSS_NONFRAG_IPV4_TCP | RTE_ETH_RSS_NONFRAG_IPV4_UDP | \ 74 RTE_ETH_RSS_NONFRAG_IPV6_TCP | RTE_ETH_RSS_NONFRAG_IPV6_UDP) 75 76 #define ENA_IO_TXQ_IDX(q) (2 * (q)) 77 #define ENA_IO_RXQ_IDX(q) (2 * (q) + 1) 78 /* Reversed version of ENA_IO_RXQ_IDX */ 79 #define ENA_IO_RXQ_IDX_REV(q) (((q) - 1) / 2) 80 81 extern struct ena_shared_data *ena_shared_data; 82 83 struct ena_adapter; 84 85 enum ena_ring_type { 86 ENA_RING_TYPE_RX = 1, 87 ENA_RING_TYPE_TX = 2, 88 }; 89 90 typedef enum ena_llq_policy_t { 91 ENA_LLQ_POLICY_DISABLED = 0, /* Host queues */ 92 ENA_LLQ_POLICY_RECOMMENDED = 1, /* Device recommendation */ 93 ENA_LLQ_POLICY_NORMAL = 2, /* 128B long LLQ entry */ 94 ENA_LLQ_POLICY_LARGE = 3, /* 256B long LLQ entry */ 95 ENA_LLQ_POLICY_LAST, 96 } ena_llq_policy; 97 98 struct ena_tx_buffer { 99 struct rte_mbuf *mbuf; 100 unsigned int tx_descs; 101 unsigned int num_of_bufs; 102 uint64_t timestamp; 103 bool print_once; 104 struct ena_com_buf bufs[ENA_PKT_MAX_BUFS]; 105 }; 106 107 /* Rx buffer holds only pointer to the mbuf - may be expanded in the future */ 108 struct ena_rx_buffer { 109 struct rte_mbuf *mbuf; 110 struct ena_com_buf ena_buf; 111 }; 112 113 struct ena_calc_queue_size_ctx { 114 struct ena_com_dev_get_features_ctx *get_feat_ctx; 115 struct ena_com_dev *ena_dev; 116 u32 max_rx_queue_size; 117 u32 max_tx_queue_size; 118 u16 max_tx_sgl_size; 119 u16 max_rx_sgl_size; 120 }; 121 122 struct ena_stats_tx { 123 u64 cnt; 124 u64 bytes; 125 u64 prepare_ctx_err; 126 u64 tx_poll; 127 u64 doorbells; 128 u64 bad_req_id; 129 u64 available_desc; 130 u64 missed_tx; 131 }; 132 133 struct ena_stats_rx { 134 u64 cnt; 135 u64 bytes; 136 u64 refill_partial; 137 u64 l3_csum_bad; 138 u64 l4_csum_bad; 139 u64 l4_csum_good; 140 u64 mbuf_alloc_fail; 141 u64 bad_desc_num; 142 u64 bad_req_id; 143 u64 bad_desc; 144 u64 unknown_error; 145 }; 146 147 struct __rte_cache_aligned ena_ring { 148 u16 next_to_use; 149 u16 next_to_clean; 150 uint64_t last_cleanup_ticks; 151 152 enum ena_ring_type type; 153 enum ena_admin_placement_policy_type tx_mem_queue_type; 154 155 /* Indicate there are Tx packets pushed to the device and wait for db */ 156 bool pkts_without_db; 157 158 /* Holds the empty requests for TX/RX OOO completions */ 159 union { 160 uint16_t *empty_tx_reqs; 161 uint16_t *empty_rx_reqs; 162 }; 163 164 union { 165 struct ena_tx_buffer *tx_buffer_info; /* contex of tx packet */ 166 struct ena_rx_buffer *rx_buffer_info; /* contex of rx packet */ 167 }; 168 struct rte_mbuf **rx_refill_buffer; 169 unsigned int ring_size; /* number of tx/rx_buffer_info's entries */ 170 unsigned int size_mask; 171 172 struct ena_com_io_cq *ena_com_io_cq; 173 struct ena_com_io_sq *ena_com_io_sq; 174 175 union { 176 uint16_t tx_free_thresh; 177 uint16_t rx_free_thresh; 178 }; 179 180 alignas(RTE_CACHE_LINE_SIZE) struct ena_com_rx_buf_info ena_bufs[ENA_PKT_MAX_BUFS]; 181 182 struct rte_mempool *mb_pool; 183 unsigned int port_id; 184 unsigned int id; 185 /* Max length PMD can push to device for LLQ */ 186 uint8_t tx_max_header_size; 187 int configured; 188 189 uint8_t *push_buf_intermediate_buf; 190 191 struct ena_adapter *adapter; 192 uint64_t offloads; 193 u16 sgl_size; 194 195 bool disable_meta_caching; 196 197 union { 198 struct ena_stats_rx rx_stats; 199 struct ena_stats_tx tx_stats; 200 }; 201 202 unsigned int numa_socket_id; 203 204 uint32_t missing_tx_completion_threshold; 205 }; 206 207 enum ena_adapter_state { 208 ENA_ADAPTER_STATE_FREE = 0, 209 ENA_ADAPTER_STATE_INIT = 1, 210 ENA_ADAPTER_STATE_RUNNING = 2, 211 ENA_ADAPTER_STATE_STOPPED = 3, 212 ENA_ADAPTER_STATE_CONFIG = 4, 213 ENA_ADAPTER_STATE_CLOSED = 5, 214 }; 215 216 struct ena_driver_stats { 217 rte_atomic64_t ierrors; 218 rte_atomic64_t oerrors; 219 rte_atomic64_t rx_nombuf; 220 u64 rx_drops; 221 }; 222 223 struct ena_stats_dev { 224 u64 wd_expired; 225 u64 dev_start; 226 u64 dev_stop; 227 /* 228 * Tx drops cannot be reported as the driver statistic, because DPDK 229 * rte_eth_stats structure isn't providing appropriate field for that. 230 * As a workaround it is being published as an extended statistic. 231 */ 232 u64 tx_drops; 233 }; 234 235 struct ena_stats_metrics { 236 /* 237 * The number of packets shaped due to inbound aggregate BW 238 * allowance being exceeded 239 */ 240 uint64_t bw_in_allowance_exceeded; 241 /* 242 * The number of packets shaped due to outbound aggregate BW 243 * allowance being exceeded 244 */ 245 uint64_t bw_out_allowance_exceeded; 246 /* The number of packets shaped due to PPS allowance being exceeded */ 247 uint64_t pps_allowance_exceeded; 248 /* 249 * The number of packets shaped due to connection tracking 250 * allowance being exceeded and leading to failure in establishment 251 * of new connections 252 */ 253 uint64_t conntrack_allowance_exceeded; 254 /* 255 * The number of packets shaped due to linklocal packet rate 256 * allowance being exceeded 257 */ 258 uint64_t linklocal_allowance_exceeded; 259 /* 260 * The number of available connections 261 */ 262 uint64_t conntrack_allowance_available; 263 }; 264 265 struct ena_stats_srd { 266 /* Describes which ENA Express features are enabled */ 267 uint64_t ena_srd_mode; 268 269 /* Number of packets transmitted over ENA SRD */ 270 uint64_t ena_srd_tx_pkts; 271 272 /* Number of packets transmitted or could have been transmitted over ENA SRD */ 273 uint64_t ena_srd_eligible_tx_pkts; 274 275 /* Number of packets received over ENA SRD */ 276 uint64_t ena_srd_rx_pkts; 277 278 /* Percentage of the ENA SRD resources that is in use */ 279 uint64_t ena_srd_resource_utilization; 280 }; 281 282 struct ena_offloads { 283 uint32_t tx_offloads; 284 uint32_t rx_offloads; 285 }; 286 287 /* board specific private data structure */ 288 struct ena_adapter { 289 /* OS defined structs */ 290 struct rte_eth_dev_data *edev_data; 291 292 alignas(RTE_CACHE_LINE_SIZE) struct ena_com_dev ena_dev; 293 294 /* TX */ 295 alignas(RTE_CACHE_LINE_SIZE) struct ena_ring tx_ring[ENA_MAX_NUM_QUEUES]; 296 u32 max_tx_ring_size; 297 u16 max_tx_sgl_size; 298 299 /* RX */ 300 alignas(RTE_CACHE_LINE_SIZE) struct ena_ring rx_ring[ENA_MAX_NUM_QUEUES]; 301 u32 max_rx_ring_size; 302 u16 max_rx_sgl_size; 303 304 u32 max_num_io_queues; 305 u16 max_mtu; 306 struct ena_offloads offloads; 307 308 /* The admin queue isn't protected by the lock and is used to 309 * retrieve statistics from the device. As there is no guarantee that 310 * application won't try to get statistics from multiple threads, it is 311 * safer to lock the queue to avoid admin queue failure. 312 */ 313 rte_spinlock_t admin_lock; 314 315 int id_number; 316 char name[ENA_NAME_MAX_LEN]; 317 u8 mac_addr[RTE_ETHER_ADDR_LEN]; 318 319 void *regs; 320 void *dev_mem_base; 321 322 struct ena_driver_stats *drv_stats; 323 enum ena_adapter_state state; 324 325 bool link_status; 326 327 enum ena_regs_reset_reason_types reset_reason; 328 329 struct rte_timer timer_wd; 330 uint64_t timestamp_wd; 331 uint64_t keep_alive_timeout; 332 333 struct ena_stats_dev dev_stats; 334 struct ena_admin_basic_stats basic_stats; 335 336 u32 indirect_table[ENA_RX_RSS_TABLE_SIZE]; 337 338 uint32_t all_aenq_groups; 339 uint32_t active_aenq_groups; 340 341 bool trigger_reset; 342 ena_llq_policy llq_header_policy; 343 344 uint32_t last_tx_comp_qid; 345 uint64_t missing_tx_completion_to; 346 uint64_t missing_tx_completion_budget; 347 uint64_t tx_cleanup_stall_delay; 348 349 uint64_t memzone_cnt; 350 351 /* Time (in microseconds) of the control path queues monitoring interval */ 352 uint64_t control_path_poll_interval; 353 354 /* 355 * Helper variables for holding the information about the supported 356 * metrics. 357 */ 358 alignas(RTE_CACHE_LINE_SIZE) uint64_t metrics_stats[ENA_MAX_CUSTOMER_METRICS]; 359 uint16_t metrics_num; 360 alignas(RTE_CACHE_LINE_SIZE) struct ena_stats_srd srd_stats; 361 }; 362 363 int ena_mp_indirect_table_set(struct ena_adapter *adapter); 364 int ena_mp_indirect_table_get(struct ena_adapter *adapter, 365 uint32_t *indirect_table); 366 int ena_rss_reta_update(struct rte_eth_dev *dev, 367 struct rte_eth_rss_reta_entry64 *reta_conf, 368 uint16_t reta_size); 369 int ena_rss_reta_query(struct rte_eth_dev *dev, 370 struct rte_eth_rss_reta_entry64 *reta_conf, 371 uint16_t reta_size); 372 int ena_rss_hash_update(struct rte_eth_dev *dev, 373 struct rte_eth_rss_conf *rss_conf); 374 int ena_rss_hash_conf_get(struct rte_eth_dev *dev, 375 struct rte_eth_rss_conf *rss_conf); 376 int ena_rss_configure(struct ena_adapter *adapter); 377 378 #endif /* _ENA_ETHDEV_H_ */ 379