1 /* SPDX-License-Identifier: BSD-3-Clause
2 *
3 * Copyright(c) 2019-2021 Xilinx, Inc.
4 * Copyright(c) 2016-2019 Solarflare Communications Inc.
5 *
6 * This software was jointly developed between OKTET Labs (under contract
7 * for Solarflare) and Solarflare Communications, Inc.
8 */
9
10 #ifndef _SFC_H
11 #define _SFC_H
12
13 #include <stdbool.h>
14
15 #include <rte_pci.h>
16 #include <bus_pci_driver.h>
17 #include <ethdev_driver.h>
18 #include <rte_kvargs.h>
19 #include <rte_spinlock.h>
20 #include <rte_atomic.h>
21
22 #include "efx.h"
23
24 #include "sfc_efx_mcdi.h"
25 #include "sfc_efx.h"
26
27 #include "sfc_debug.h"
28 #include "sfc_log.h"
29 #include "sfc_filter.h"
30 #include "sfc_flow_rss.h"
31 #include "sfc_flow_tunnel.h"
32 #include "sfc_sriov.h"
33 #include "sfc_mae.h"
34 #include "sfc_tbls.h"
35 #include "sfc_dp.h"
36 #include "sfc_sw_stats.h"
37 #include "sfc_repr_proxy.h"
38 #include "sfc_service.h"
39 #include "sfc_ethdev_state.h"
40 #include "sfc_nic_dma_dp.h"
41
42 #ifdef __cplusplus
43 extern "C" {
44 #endif
45
46 enum sfc_dev_filter_mode {
47 SFC_DEV_FILTER_MODE_PROMISC = 0,
48 SFC_DEV_FILTER_MODE_ALLMULTI,
49
50 SFC_DEV_FILTER_NMODES
51 };
52
53 struct sfc_intr {
54 efx_intr_type_t type;
55 rte_intr_callback_fn handler;
56 boolean_t lsc_intr;
57 boolean_t rxq_intr;
58 };
59
60 struct sfc_rxq;
61 struct sfc_txq;
62
63 struct sfc_rxq_info;
64 struct sfc_txq_info;
65 struct sfc_dp_rx;
66
67 struct sfc_port {
68 unsigned int lsc_seq;
69
70 uint32_t phy_adv_cap_mask;
71 uint32_t phy_adv_cap;
72 uint32_t fec_cfg;
73 bool fec_auto;
74
75 unsigned int flow_ctrl;
76 boolean_t flow_ctrl_autoneg;
77 boolean_t include_fcs;
78 boolean_t vlan_strip;
79 size_t pdu;
80
81 /*
82 * Flow API isolated mode overrides promisc and allmulti settings;
83 * they won't be applied if isolated mode is active
84 */
85 boolean_t promisc;
86 boolean_t allmulti;
87
88 struct rte_ether_addr default_mac_addr;
89
90 unsigned int max_mcast_addrs;
91 unsigned int nb_mcast_addrs;
92 uint8_t *mcast_addrs;
93
94 uint64_t *mac_stats_buf;
95 unsigned int mac_stats_nb_supported;
96 efsys_mem_t mac_stats_dma_mem;
97 boolean_t mac_stats_reset_pending;
98 uint16_t mac_stats_update_period_ms;
99 uint32_t mac_stats_update_generation;
100 boolean_t mac_stats_periodic_dma_supported;
101 uint64_t mac_stats_last_request_timestamp;
102
103 uint32_t mac_stats_mask[EFX_MAC_STATS_MASK_NPAGES];
104
105 unsigned int mac_stats_by_id[EFX_MAC_NSTATS];
106
107 uint64_t ipackets;
108 };
109
110 struct sfc_rss_hf_rte_to_efx {
111 uint64_t rte;
112 efx_rx_hash_type_t efx;
113 };
114
115 struct sfc_rss {
116 unsigned int channels;
117 efx_rx_scale_context_type_t context_type;
118 efx_rx_hash_support_t hash_support;
119 efx_rx_hash_alg_t hash_alg;
120 unsigned int hf_map_nb_entries;
121 struct sfc_rss_hf_rte_to_efx *hf_map;
122
123 efx_rx_hash_type_t hash_types;
124 unsigned int tbl[EFX_RSS_TBL_SIZE];
125 uint8_t key[EFX_RSS_KEY_SIZE];
126
127 struct sfc_flow_rss_ctx dummy_ctx;
128 };
129
130 /* Adapter private data shared by primary and secondary processes */
131 struct sfc_adapter_shared {
132 unsigned int rxq_count;
133 struct sfc_rxq_info *rxq_info;
134 unsigned int ethdev_rxq_count;
135
136 unsigned int txq_count;
137 struct sfc_txq_info *txq_info;
138 unsigned int ethdev_txq_count;
139
140 struct sfc_rss rss;
141
142 boolean_t isolated;
143 uint32_t tunnel_encaps;
144
145 char log_prefix[SFC_LOG_PREFIX_MAX];
146 struct rte_pci_addr pci_addr;
147 uint16_t port_id;
148
149 char *dp_rx_name;
150 char *dp_tx_name;
151
152 bool counters_rxq_allocated;
153 unsigned int nb_repr_rxq;
154 unsigned int nb_repr_txq;
155
156 struct sfc_nic_dma_info nic_dma_info;
157 };
158
159 /* Adapter process private data */
160 struct sfc_adapter_priv {
161 struct sfc_adapter_shared *shared;
162 const struct sfc_dp_rx *dp_rx;
163 const struct sfc_dp_tx *dp_tx;
164 uint32_t logtype_main;
165 };
166
167 static inline struct sfc_adapter_priv *
sfc_adapter_priv_by_eth_dev(struct rte_eth_dev * eth_dev)168 sfc_adapter_priv_by_eth_dev(struct rte_eth_dev *eth_dev)
169 {
170 struct sfc_adapter_priv *sap = eth_dev->process_private;
171
172 SFC_ASSERT(sap != NULL);
173 return sap;
174 }
175
176 /* RxQ dedicated for counters (counter only RxQ) data */
177 struct sfc_counter_rxq {
178 unsigned int state;
179 #define SFC_COUNTER_RXQ_ATTACHED 0x1
180 #define SFC_COUNTER_RXQ_INITIALIZED 0x2
181 sfc_sw_index_t sw_index;
182 struct rte_mempool *mp;
183 };
184
185 struct sfc_sw_stat_data {
186 const struct sfc_sw_stat_descr *descr;
187 /* Cache fragment */
188 uint64_t *cache;
189 };
190
191 struct sfc_sw_stats {
192 /* Number extended statistics provided by SW stats */
193 unsigned int xstats_count;
194 /* Supported SW statistics */
195 struct sfc_sw_stat_data *supp;
196 unsigned int supp_count;
197
198 /* Cache for all supported SW statistics */
199 uint64_t *cache;
200 unsigned int cache_count;
201
202 uint64_t *reset_vals;
203 /* Location of per-queue reset values for packets/bytes in reset_vals */
204 uint64_t *reset_rx_pkts;
205 uint64_t *reset_rx_bytes;
206 uint64_t *reset_tx_pkts;
207 uint64_t *reset_tx_bytes;
208
209 rte_spinlock_t queues_bitmap_lock;
210 void *queues_bitmap_mem;
211 struct rte_bitmap *queues_bitmap;
212 };
213
214 /* Adapter private data */
215 struct sfc_adapter {
216 /*
217 * It must be the first field of the sfc_adapter structure since
218 * sfc_adapter is the primary process private data (i.e. process
219 * private data plus additional primary process specific data).
220 */
221 struct sfc_adapter_priv priv;
222
223 /*
224 * PMD setup and configuration is not thread safe. Since it is not
225 * performance sensitive, it is better to guarantee thread-safety
226 * and add device level lock. Adapter control operations which
227 * change its state should acquire the lock.
228 */
229 rte_spinlock_t lock;
230 enum sfc_ethdev_state state;
231 struct rte_eth_dev *eth_dev;
232 struct rte_kvargs *kvargs;
233 int socket_id;
234 efsys_bar_t mem_bar;
235 /* Function control window offset */
236 efsys_dma_addr_t fcw_offset;
237 efx_family_t family;
238 efx_nic_t *nic;
239 rte_spinlock_t nic_lock;
240 rte_atomic32_t restart_required;
241
242 struct sfc_efx_mcdi mcdi;
243 struct sfc_sriov sriov;
244 struct sfc_intr intr;
245 struct sfc_port port;
246 struct sfc_sw_stats sw_stats;
247 struct sfc_flow_rss flow_rss;
248 /* Registry of contexts used in Flow Tunnel (FT) offload */
249 struct sfc_ft_ctx ft_ctx_pool[SFC_FT_MAX_NTUNNELS];
250 struct sfc_filter filter;
251 struct sfc_mae mae;
252 struct sfc_tbls hw_tables;
253 struct sfc_repr_proxy repr_proxy;
254
255 struct sfc_flow_indir_actions flow_indir_actions;
256 struct sfc_flow_list flow_list;
257
258 unsigned int rxq_max;
259 unsigned int txq_max;
260
261 unsigned int rxq_max_entries;
262 unsigned int rxq_min_entries;
263
264 unsigned int txq_max_entries;
265 unsigned int txq_min_entries;
266
267 unsigned int evq_max_entries;
268 unsigned int evq_min_entries;
269
270 uint32_t evq_flags;
271 unsigned int evq_count;
272
273 unsigned int mgmt_evq_index;
274 /*
275 * The lock is used to serialise management event queue polling
276 * which can be done from different context. Also the lock
277 * guarantees that mgmt_evq_running is preserved while the lock
278 * is held. It is used to serialise polling and start/stop
279 * operations.
280 *
281 * Locks which may be held when the lock is acquired:
282 * - adapter lock, when:
283 * - device start/stop to change mgmt_evq_running
284 * - any control operations in client side MCDI proxy handling to
285 * poll management event queue waiting for proxy response
286 * - MCDI lock, when:
287 * - any control operations in client side MCDI proxy handling to
288 * poll management event queue waiting for proxy response
289 *
290 * Locks which are acquired with the lock held:
291 * - nic_lock, when:
292 * - MC event processing on management event queue polling
293 * (e.g. MC REBOOT or BADASSERT events)
294 */
295 rte_spinlock_t mgmt_evq_lock;
296 bool mgmt_evq_running;
297 struct sfc_evq *mgmt_evq;
298
299 struct sfc_counter_rxq counter_rxq;
300
301 struct sfc_rxq *rxq_ctrl;
302 struct sfc_txq *txq_ctrl;
303
304 boolean_t tso;
305 boolean_t tso_encap;
306
307 uint64_t negotiated_rx_metadata;
308
309 uint32_t rxd_wait_timeout_ns;
310
311 bool switchdev;
312 };
313
314 static inline struct sfc_adapter_shared *
sfc_adapter_shared_by_eth_dev(struct rte_eth_dev * eth_dev)315 sfc_adapter_shared_by_eth_dev(struct rte_eth_dev *eth_dev)
316 {
317 struct sfc_adapter_shared *sas = eth_dev->data->dev_private;
318
319 return sas;
320 }
321
322 static inline struct sfc_adapter *
sfc_adapter_by_eth_dev(struct rte_eth_dev * eth_dev)323 sfc_adapter_by_eth_dev(struct rte_eth_dev *eth_dev)
324 {
325 struct sfc_adapter_priv *sap = sfc_adapter_priv_by_eth_dev(eth_dev);
326
327 SFC_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY);
328
329 return container_of(sap, struct sfc_adapter, priv);
330 }
331
332 static inline struct sfc_adapter_shared *
sfc_sa2shared(struct sfc_adapter * sa)333 sfc_sa2shared(struct sfc_adapter *sa)
334 {
335 return sa->priv.shared;
336 }
337
338 /*
339 * Add wrapper functions to acquire/release lock to be able to remove or
340 * change the lock in one place.
341 */
342
343 #define sfc_adapter_lock_init(sa) rte_spinlock_init(&(sa)->lock)
344 #define sfc_adapter_is_locked(sa) rte_spinlock_is_locked(&(sa)->lock)
345 #define sfc_adapter_lock(sa) rte_spinlock_lock(&(sa)->lock)
346 #define sfc_adapter_trylock(sa) rte_spinlock_trylock(&(sa)->lock)
347 #define sfc_adapter_unlock(sa) rte_spinlock_unlock(&(sa)->lock)
348 #define sfc_adapter_lock_fini(sa) RTE_SET_USED(sa)
349
350 static inline unsigned int
sfc_nb_counter_rxq(const struct sfc_adapter_shared * sas)351 sfc_nb_counter_rxq(const struct sfc_adapter_shared *sas)
352 {
353 return sas->counters_rxq_allocated ? 1 : 0;
354 }
355
356 bool sfc_repr_supported(const struct sfc_adapter *sa);
357 bool sfc_repr_available(const struct sfc_adapter_shared *sas);
358
359 static inline unsigned int
sfc_repr_nb_rxq(const struct sfc_adapter_shared * sas)360 sfc_repr_nb_rxq(const struct sfc_adapter_shared *sas)
361 {
362 return sas->nb_repr_rxq;
363 }
364
365 static inline unsigned int
sfc_repr_nb_txq(const struct sfc_adapter_shared * sas)366 sfc_repr_nb_txq(const struct sfc_adapter_shared *sas)
367 {
368 return sas->nb_repr_txq;
369 }
370
371 /** Get the number of milliseconds since boot from the default timer */
372 static inline uint64_t
sfc_get_system_msecs(void)373 sfc_get_system_msecs(void)
374 {
375 return rte_get_timer_cycles() * MS_PER_S / rte_get_timer_hz();
376 }
377
378 int sfc_dma_alloc(struct sfc_adapter *sa, const char *name, uint16_t id,
379 efx_nic_dma_addr_type_t addr_type, size_t len, int socket_id,
380 efsys_mem_t *esmp);
381 void sfc_dma_free(const struct sfc_adapter *sa, efsys_mem_t *esmp);
382
383 uint32_t sfc_register_logtype(const struct rte_pci_addr *pci_addr,
384 const char *lt_prefix_str,
385 uint32_t ll_default);
386
387 int sfc_probe(struct sfc_adapter *sa);
388 void sfc_unprobe(struct sfc_adapter *sa);
389 int sfc_attach(struct sfc_adapter *sa);
390 void sfc_pre_detach(struct sfc_adapter *sa);
391 void sfc_detach(struct sfc_adapter *sa);
392 int sfc_start(struct sfc_adapter *sa);
393 void sfc_stop(struct sfc_adapter *sa);
394
395 void sfc_schedule_restart(struct sfc_adapter *sa);
396
397 int sfc_mcdi_init(struct sfc_adapter *sa);
398 void sfc_mcdi_fini(struct sfc_adapter *sa);
399
400 int sfc_configure(struct sfc_adapter *sa);
401 void sfc_close(struct sfc_adapter *sa);
402
403 int sfc_intr_attach(struct sfc_adapter *sa);
404 void sfc_intr_detach(struct sfc_adapter *sa);
405 int sfc_intr_configure(struct sfc_adapter *sa);
406 void sfc_intr_close(struct sfc_adapter *sa);
407 int sfc_intr_start(struct sfc_adapter *sa);
408 void sfc_intr_stop(struct sfc_adapter *sa);
409
410 int sfc_port_attach(struct sfc_adapter *sa);
411 void sfc_port_detach(struct sfc_adapter *sa);
412 int sfc_port_configure(struct sfc_adapter *sa);
413 void sfc_port_close(struct sfc_adapter *sa);
414 int sfc_port_start(struct sfc_adapter *sa);
415 void sfc_port_stop(struct sfc_adapter *sa);
416 void sfc_port_link_mode_to_info(efx_link_mode_t link_mode,
417 struct rte_eth_link *link_info);
418 int sfc_port_update_mac_stats(struct sfc_adapter *sa, boolean_t manual_update);
419 int sfc_port_get_mac_stats(struct sfc_adapter *sa, struct rte_eth_xstat *xstats,
420 unsigned int xstats_count, unsigned int *nb_written);
421 int sfc_port_get_mac_stats_by_id(struct sfc_adapter *sa, const uint64_t *ids,
422 uint64_t *values, unsigned int n);
423 int sfc_port_reset_mac_stats(struct sfc_adapter *sa);
424 int sfc_set_rx_mode(struct sfc_adapter *sa);
425 int sfc_set_rx_mode_unchecked(struct sfc_adapter *sa);
426
427 struct sfc_hw_switch_id;
428
429 int sfc_hw_switch_id_init(struct sfc_adapter *sa,
430 struct sfc_hw_switch_id **idp);
431 void sfc_hw_switch_id_fini(struct sfc_adapter *sa,
432 struct sfc_hw_switch_id *idp);
433 bool sfc_hw_switch_ids_equal(const struct sfc_hw_switch_id *left,
434 const struct sfc_hw_switch_id *right);
435
436 #ifdef __cplusplus
437 }
438 #endif
439
440 #endif /* _SFC_H */
441