1 /* SPDX-License-Identifier: BSD-3-Clause
2 *
3 * Copyright(c) 2019-2021 Xilinx, Inc.
4 * Copyright(c) 2017-2019 Solarflare Communications Inc.
5 *
6 * This software was jointly developed between OKTET Labs (under contract
7 * for Solarflare) and Solarflare Communications, Inc.
8 */
9
10 #ifndef _SFC_EF10_H
11 #define _SFC_EF10_H
12
13 #include "sfc_debug.h"
14
15 #ifdef __cplusplus
16 extern "C" {
17 #endif
18
19 /* Number of events in one cache line */
20 #define SFC_EF10_EV_PER_CACHE_LINE \
21 (RTE_CACHE_LINE_SIZE / sizeof(efx_qword_t))
22
23 #define SFC_EF10_EV_QCLEAR_MASK (~(SFC_EF10_EV_PER_CACHE_LINE - 1))
24
25 /*
26 * Use simple libefx-based implementation of the
27 * sfc_ef10_ev_qclear_cache_line() if SSE2 is not available
28 * since optimized implementation uses __m128i intrinsics.
29 */
30 #ifndef __SSE2__
31 #define SFC_EF10_EV_QCLEAR_USE_EFX
32 #endif
33
34 #if defined(SFC_EF10_EV_QCLEAR_USE_EFX)
35 static inline void
sfc_ef10_ev_qclear_cache_line(void * ptr)36 sfc_ef10_ev_qclear_cache_line(void *ptr)
37 {
38 efx_qword_t *entry = ptr;
39 unsigned int i;
40
41 for (i = 0; i < SFC_EF10_EV_PER_CACHE_LINE; ++i)
42 EFX_SET_QWORD(entry[i]);
43 }
44 #else
45 /*
46 * It is possible to do it using AVX2 and AVX512F, but it shows less
47 * performance.
48 */
49 static inline void
50 sfc_ef10_ev_qclear_cache_line(void *ptr)
51 {
52 const efsys_uint128_t val = _mm_set1_epi64x(UINT64_MAX);
53 efsys_uint128_t *addr = ptr;
54 unsigned int i;
55
56 RTE_BUILD_BUG_ON(sizeof(val) > RTE_CACHE_LINE_SIZE);
57 RTE_BUILD_BUG_ON(RTE_CACHE_LINE_SIZE % sizeof(val) != 0);
58
59 for (i = 0; i < RTE_CACHE_LINE_SIZE / sizeof(val); ++i)
60 _mm_store_si128(&addr[i], val);
61 }
62 #endif
63
64 static inline void
sfc_ef10_ev_qclear(efx_qword_t * hw_ring,unsigned int ptr_mask,unsigned int old_read_ptr,unsigned int read_ptr)65 sfc_ef10_ev_qclear(efx_qword_t *hw_ring, unsigned int ptr_mask,
66 unsigned int old_read_ptr, unsigned int read_ptr)
67 {
68 const unsigned int clear_ptr = read_ptr & SFC_EF10_EV_QCLEAR_MASK;
69 unsigned int old_clear_ptr = old_read_ptr & SFC_EF10_EV_QCLEAR_MASK;
70
71 while (old_clear_ptr != clear_ptr) {
72 sfc_ef10_ev_qclear_cache_line(
73 &hw_ring[old_clear_ptr & ptr_mask]);
74 old_clear_ptr += SFC_EF10_EV_PER_CACHE_LINE;
75 }
76
77 /*
78 * No barriers here.
79 * Functions which push doorbell should care about correct
80 * ordering: store instructions which fill in EvQ ring should be
81 * retired from CPU and DMA sync before doorbell which will allow
82 * to use these event entries.
83 */
84 }
85
86 static inline bool
sfc_ef10_ev_present(const efx_qword_t ev)87 sfc_ef10_ev_present(const efx_qword_t ev)
88 {
89 return ~EFX_QWORD_FIELD(ev, EFX_DWORD_0) |
90 ~EFX_QWORD_FIELD(ev, EFX_DWORD_1);
91 }
92
93
94 /**
95 * Alignment requirement for value written to RX WPTR:
96 * the WPTR must be aligned to an 8 descriptor boundary.
97 */
98 #define SFC_EF10_RX_WPTR_ALIGN 8u
99
100 static inline void
sfc_ef10_rx_qpush(volatile void * doorbell,unsigned int added,unsigned int ptr_mask,uint32_t * dbell_counter)101 sfc_ef10_rx_qpush(volatile void *doorbell, unsigned int added,
102 unsigned int ptr_mask, uint32_t *dbell_counter)
103 {
104 efx_dword_t dword;
105
106 /* Hardware has alignment restriction for WPTR */
107 RTE_BUILD_BUG_ON(SFC_RX_REFILL_BULK % SFC_EF10_RX_WPTR_ALIGN != 0);
108 SFC_ASSERT(RTE_ALIGN(added, SFC_EF10_RX_WPTR_ALIGN) == added);
109
110 EFX_POPULATE_DWORD_1(dword, ERF_DZ_RX_DESC_WPTR, added & ptr_mask);
111
112 /* DMA sync to device is not required */
113
114 /*
115 * rte_write32() has rte_io_wmb() which guarantees that the STORE
116 * operations (i.e. Rx and event descriptor updates) that precede
117 * the rte_io_wmb() call are visible to NIC before the STORE
118 * operations that follow it (i.e. doorbell write).
119 */
120 rte_write32(dword.ed_u32[0], doorbell);
121 (*dbell_counter)++;
122 }
123
124 static inline void
sfc_ef10_ev_qprime(volatile void * qprime,unsigned int read_ptr,unsigned int ptr_mask)125 sfc_ef10_ev_qprime(volatile void *qprime, unsigned int read_ptr,
126 unsigned int ptr_mask)
127 {
128 efx_dword_t dword;
129
130 EFX_POPULATE_DWORD_1(dword, ERF_DZ_EVQ_RPTR, read_ptr & ptr_mask);
131
132 rte_write32_relaxed(dword.ed_u32[0], qprime);
133 rte_wmb();
134 }
135
136
137 const uint32_t *sfc_ef10_supported_ptypes_get(uint32_t tunnel_encaps,
138 size_t *no_of_elements);
139
140
141 #ifdef __cplusplus
142 }
143 #endif
144 #endif /* _SFC_EF10_H */
145