1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(C) 2022 Marvell. 3 */ 4 5 #ifndef __CN10K_RXTX_H__ 6 #define __CN10K_RXTX_H__ 7 8 #include <rte_security.h> 9 10 /* ROC Constants */ 11 #include "roc_constants.h" 12 13 /* Platform definition */ 14 #include "roc_platform.h" 15 16 /* IO */ 17 #if defined(__aarch64__) 18 #include "roc_io.h" 19 #else 20 #include "roc_io_generic.h" 21 #endif 22 23 /* HW structure definition */ 24 #include "hw/cpt.h" 25 #include "hw/nix.h" 26 #include "hw/npa.h" 27 #include "hw/npc.h" 28 #include "hw/ssow.h" 29 30 #include "roc_ie_ot.h" 31 32 /* NPA */ 33 #include "roc_npa_dp.h" 34 35 /* SSO */ 36 #include "roc_sso_dp.h" 37 38 /* CPT */ 39 #include "roc_cpt.h" 40 41 /* NIX Inline dev */ 42 #include "roc_nix_inl_dp.h" 43 44 #include "cnxk_ethdev_dp.h" 45 46 struct cn10k_eth_txq { 47 uint64_t send_hdr_w0; 48 int64_t fc_cache_pkts; 49 uint64_t *fc_mem; 50 uintptr_t lmt_base; 51 rte_iova_t io_addr; 52 uint16_t sqes_per_sqb_log2; 53 int16_t nb_sqb_bufs_adj; 54 uint8_t flag; 55 rte_iova_t cpt_io_addr; 56 uint64_t sa_base; 57 uint64_t *cpt_fc; 58 uint16_t cpt_desc; 59 int32_t *cpt_fc_sw; 60 uint64_t lso_tun_fmt; 61 uint64_t ts_mem; 62 uint64_t mark_flag : 8; 63 uint64_t mark_fmt : 48; 64 struct cnxk_eth_txq_comp tx_compl; 65 uint16_t tx_offload_flags; 66 } __plt_cache_aligned; 67 68 struct cn10k_eth_rxq { 69 uint64_t mbuf_initializer; 70 uintptr_t desc; 71 void *lookup_mem; 72 uintptr_t cq_door; 73 uint64_t wdata; 74 int64_t *cq_status; 75 uint32_t head; 76 uint32_t qmask; 77 uint32_t available; 78 uint16_t data_off; 79 uint64_t sa_base; 80 uint64_t lmt_base; 81 uint64_t meta_aura; 82 uintptr_t meta_pool; 83 uint16_t rq; 84 struct cnxk_timesync_info *tstamp; 85 } __plt_cache_aligned; 86 87 /* Private data in sw rsvd area of struct roc_ot_ipsec_inb_sa */ 88 struct cn10k_inb_priv_data { 89 void *userdata; 90 int reass_dynfield_off; 91 int reass_dynflag_bit; 92 struct cnxk_eth_sec_sess *eth_sec; 93 }; 94 95 struct __rte_packed_begin cn10k_sec_sess_priv { 96 union { 97 struct { 98 uint32_t sa_idx; 99 uint8_t inb_sa : 1; 100 uint8_t outer_ip_ver : 1; 101 uint8_t mode : 1; 102 uint8_t roundup_byte : 5; 103 uint8_t roundup_len; 104 uint16_t partial_len : 10; 105 uint16_t chksum : 2; 106 uint16_t dec_ttl : 1; 107 uint16_t nixtx_off : 1; 108 uint16_t rsvd : 2; 109 }; 110 111 uint64_t u64; 112 }; 113 } __rte_packed_end; 114 115 #define LMT_OFF(lmt_addr, lmt_num, offset) \ 116 (void *)((uintptr_t)(lmt_addr) + \ 117 ((uint64_t)(lmt_num) << ROC_LMT_LINE_SIZE_LOG2) + (offset)) 118 119 static inline uint16_t 120 nix_tx_compl_nb_pkts(struct cn10k_eth_txq *txq, const uint64_t wdata, 121 const uint32_t qmask) 122 { 123 uint16_t available = txq->tx_compl.available; 124 125 /* Update the available count if cached value is not enough */ 126 if (!unlikely(available)) { 127 uint64_t reg, head, tail; 128 129 /* Use LDADDA version to avoid reorder */ 130 reg = roc_atomic64_add_sync(wdata, txq->tx_compl.cq_status); 131 /* CQ_OP_STATUS operation error */ 132 if (reg & BIT_ULL(NIX_CQ_OP_STAT_OP_ERR) || 133 reg & BIT_ULL(NIX_CQ_OP_STAT_CQ_ERR)) 134 return 0; 135 136 tail = reg & 0xFFFFF; 137 head = (reg >> 20) & 0xFFFFF; 138 if (tail < head) 139 available = tail - head + qmask + 1; 140 else 141 available = tail - head; 142 143 txq->tx_compl.available = available; 144 } 145 return available; 146 } 147 148 static inline void 149 handle_tx_completion_pkts(struct cn10k_eth_txq *txq, uint8_t mt_safe) 150 { 151 #define CNXK_NIX_CQ_ENTRY_SZ 128 152 #define CQE_SZ(x) ((x) * CNXK_NIX_CQ_ENTRY_SZ) 153 154 uint16_t tx_pkts = 0, nb_pkts; 155 const uintptr_t desc = txq->tx_compl.desc_base; 156 const uint64_t wdata = txq->tx_compl.wdata; 157 const uint32_t qmask = txq->tx_compl.qmask; 158 uint32_t head = txq->tx_compl.head; 159 struct nix_cqe_hdr_s *tx_compl_cq; 160 struct nix_send_comp_s *tx_compl_s0; 161 struct rte_mbuf *m_next, *m; 162 163 if (mt_safe) 164 rte_spinlock_lock(&txq->tx_compl.ext_buf_lock); 165 166 nb_pkts = nix_tx_compl_nb_pkts(txq, wdata, qmask); 167 while (tx_pkts < nb_pkts) { 168 rte_prefetch_non_temporal((void *)(desc + 169 (CQE_SZ((head + 2) & qmask)))); 170 tx_compl_cq = (struct nix_cqe_hdr_s *) 171 (desc + CQE_SZ(head)); 172 tx_compl_s0 = (struct nix_send_comp_s *) 173 ((uint64_t *)tx_compl_cq + 1); 174 m = txq->tx_compl.ptr[tx_compl_s0->sqe_id]; 175 while (m->next != NULL) { 176 m_next = m->next; 177 rte_pktmbuf_free_seg(m); 178 m = m_next; 179 } 180 rte_pktmbuf_free_seg(m); 181 txq->tx_compl.ptr[tx_compl_s0->sqe_id] = NULL; 182 183 head++; 184 head &= qmask; 185 tx_pkts++; 186 } 187 txq->tx_compl.head = head; 188 txq->tx_compl.available -= nb_pkts; 189 190 plt_write64((wdata | nb_pkts), txq->tx_compl.cq_door); 191 192 if (mt_safe) 193 rte_spinlock_unlock(&txq->tx_compl.ext_buf_lock); 194 } 195 196 static __rte_always_inline uint64_t 197 cn10k_cpt_tx_steor_data(void) 198 { 199 /* We have two CPT instructions per LMTLine */ 200 const uint64_t dw_m1 = ROC_CN10K_TWO_CPT_INST_DW_M1; 201 uint64_t data; 202 203 /* This will be moved to addr area */ 204 data = dw_m1 << 16; 205 data |= dw_m1 << 19; 206 data |= dw_m1 << 22; 207 data |= dw_m1 << 25; 208 data |= dw_m1 << 28; 209 data |= dw_m1 << 31; 210 data |= dw_m1 << 34; 211 data |= dw_m1 << 37; 212 data |= dw_m1 << 40; 213 data |= dw_m1 << 43; 214 data |= dw_m1 << 46; 215 data |= dw_m1 << 49; 216 data |= dw_m1 << 52; 217 data |= dw_m1 << 55; 218 data |= dw_m1 << 58; 219 data |= dw_m1 << 61; 220 221 return data; 222 } 223 224 static __rte_always_inline void 225 cn10k_nix_sec_steorl(uintptr_t io_addr, uint32_t lmt_id, uint8_t lnum, 226 uint8_t loff, uint8_t shft) 227 { 228 uint64_t data; 229 uintptr_t pa; 230 231 /* Check if there is any CPT instruction to submit */ 232 if (!lnum && !loff) 233 return; 234 235 data = cn10k_cpt_tx_steor_data(); 236 /* Update lmtline use for partial end line */ 237 if (loff) { 238 data &= ~(0x7ULL << shft); 239 /* Update it to half full i.e 64B */ 240 data |= (0x3UL << shft); 241 } 242 243 pa = io_addr | ((data >> 16) & 0x7) << 4; 244 data &= ~(0x7ULL << 16); 245 /* Update lines - 1 that contain valid data */ 246 data |= ((uint64_t)(lnum + loff - 1)) << 12; 247 data |= (uint64_t)lmt_id; 248 249 /* STEOR */ 250 roc_lmt_submit_steorl(data, pa); 251 } 252 253 #endif /* __CN10K_RXTX_H__ */ 254