1 /* SPDX-License-Identifier: BSD-3-Clause 2 * 3 * Copyright(c) 2019-2021 Xilinx, Inc. 4 * Copyright(c) 2017-2019 Solarflare Communications Inc. 5 * 6 * This software was jointly developed between OKTET Labs (under contract 7 * for Solarflare) and Solarflare Communications, Inc. 8 */ 9 10 #ifndef _SFC_EF10_H 11 #define _SFC_EF10_H 12 13 #include "sfc_debug.h" 14 15 #ifdef __cplusplus 16 extern "C" { 17 #endif 18 19 /* Number of events in one cache line */ 20 #define SFC_EF10_EV_PER_CACHE_LINE \ 21 (RTE_CACHE_LINE_SIZE / sizeof(efx_qword_t)) 22 23 #define SFC_EF10_EV_QCLEAR_MASK (~(SFC_EF10_EV_PER_CACHE_LINE - 1)) 24 25 /* 26 * Use simple libefx-based implementation of the 27 * sfc_ef10_ev_qclear_cache_line() if SSE2 is not available 28 * since optimized implementation uses __m128i intrinsics. 29 */ 30 #ifndef __SSE2__ 31 #define SFC_EF10_EV_QCLEAR_USE_EFX 32 #endif 33 34 #if defined(SFC_EF10_EV_QCLEAR_USE_EFX) 35 static inline void 36 sfc_ef10_ev_qclear_cache_line(void *ptr) 37 { 38 efx_qword_t *entry = ptr; 39 unsigned int i; 40 41 for (i = 0; i < SFC_EF10_EV_PER_CACHE_LINE; ++i) 42 EFX_SET_QWORD(entry[i]); 43 } 44 #else 45 /* 46 * It is possible to do it using AVX2 and AVX512F, but it shows less 47 * performance. 48 */ 49 static inline void 50 sfc_ef10_ev_qclear_cache_line(void *ptr) 51 { 52 const efsys_uint128_t val = _mm_set1_epi64x(UINT64_MAX); 53 efsys_uint128_t *addr = ptr; 54 unsigned int i; 55 56 RTE_BUILD_BUG_ON(sizeof(val) > RTE_CACHE_LINE_SIZE); 57 RTE_BUILD_BUG_ON(RTE_CACHE_LINE_SIZE % sizeof(val) != 0); 58 59 for (i = 0; i < RTE_CACHE_LINE_SIZE / sizeof(val); ++i) 60 _mm_store_si128(&addr[i], val); 61 } 62 #endif 63 64 static inline void 65 sfc_ef10_ev_qclear(efx_qword_t *hw_ring, unsigned int ptr_mask, 66 unsigned int old_read_ptr, unsigned int read_ptr) 67 { 68 const unsigned int clear_ptr = read_ptr & SFC_EF10_EV_QCLEAR_MASK; 69 unsigned int old_clear_ptr = old_read_ptr & SFC_EF10_EV_QCLEAR_MASK; 70 71 while (old_clear_ptr != clear_ptr) { 72 sfc_ef10_ev_qclear_cache_line( 73 &hw_ring[old_clear_ptr & ptr_mask]); 74 old_clear_ptr += SFC_EF10_EV_PER_CACHE_LINE; 75 } 76 77 /* 78 * No barriers here. 79 * Functions which push doorbell should care about correct 80 * ordering: store instructions which fill in EvQ ring should be 81 * retired from CPU and DMA sync before doorbell which will allow 82 * to use these event entries. 83 */ 84 } 85 86 static inline bool 87 sfc_ef10_ev_present(const efx_qword_t ev) 88 { 89 return ~EFX_QWORD_FIELD(ev, EFX_DWORD_0) | 90 ~EFX_QWORD_FIELD(ev, EFX_DWORD_1); 91 } 92 93 94 /** 95 * Alignment requirement for value written to RX WPTR: 96 * the WPTR must be aligned to an 8 descriptor boundary. 97 */ 98 #define SFC_EF10_RX_WPTR_ALIGN 8u 99 100 static inline void 101 sfc_ef10_rx_qpush(volatile void *doorbell, unsigned int added, 102 unsigned int ptr_mask) 103 { 104 efx_dword_t dword; 105 106 /* Hardware has alignment restriction for WPTR */ 107 RTE_BUILD_BUG_ON(SFC_RX_REFILL_BULK % SFC_EF10_RX_WPTR_ALIGN != 0); 108 SFC_ASSERT(RTE_ALIGN(added, SFC_EF10_RX_WPTR_ALIGN) == added); 109 110 EFX_POPULATE_DWORD_1(dword, ERF_DZ_RX_DESC_WPTR, added & ptr_mask); 111 112 /* DMA sync to device is not required */ 113 114 /* 115 * rte_write32() has rte_io_wmb() which guarantees that the STORE 116 * operations (i.e. Rx and event descriptor updates) that precede 117 * the rte_io_wmb() call are visible to NIC before the STORE 118 * operations that follow it (i.e. doorbell write). 119 */ 120 rte_write32(dword.ed_u32[0], doorbell); 121 } 122 123 static inline void 124 sfc_ef10_ev_qprime(volatile void *qprime, unsigned int read_ptr, 125 unsigned int ptr_mask) 126 { 127 efx_dword_t dword; 128 129 EFX_POPULATE_DWORD_1(dword, ERF_DZ_EVQ_RPTR, read_ptr & ptr_mask); 130 131 rte_write32_relaxed(dword.ed_u32[0], qprime); 132 rte_wmb(); 133 } 134 135 136 const uint32_t * sfc_ef10_supported_ptypes_get(uint32_t tunnel_encaps); 137 138 139 #ifdef __cplusplus 140 } 141 #endif 142 #endif /* _SFC_EF10_H */ 143