1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(C) 2024 Marvell. 3 */ 4 5 #ifndef __CN20K_RXTX_H__ 6 #define __CN20K_RXTX_H__ 7 8 #include <rte_security.h> 9 10 /* ROC Constants */ 11 #include "roc_constants.h" 12 13 /* Platform definition */ 14 #include "roc_platform.h" 15 16 /* IO */ 17 #if defined(__aarch64__) 18 #include "roc_io.h" 19 #else 20 #include "roc_io_generic.h" 21 #endif 22 23 /* HW structure definition */ 24 #include "hw/cpt.h" 25 #include "hw/nix.h" 26 #include "hw/npa.h" 27 #include "hw/npc.h" 28 #include "hw/ssow.h" 29 30 #include "roc_ie_ot.h" 31 32 /* NPA */ 33 #include "roc_npa_dp.h" 34 35 /* SSO */ 36 #include "roc_sso_dp.h" 37 38 /* CPT */ 39 #include "roc_cpt.h" 40 41 /* NIX Inline dev */ 42 #include "roc_nix_inl_dp.h" 43 44 #include "cnxk_ethdev_dp.h" 45 46 struct cn20k_eth_txq { 47 uint64_t send_hdr_w0; 48 int64_t fc_cache_pkts; 49 uint64_t *fc_mem; 50 uintptr_t lmt_base; 51 rte_iova_t io_addr; 52 uint16_t sqes_per_sqb_log2; 53 int16_t nb_sqb_bufs_adj; 54 uint8_t flag; 55 rte_iova_t cpt_io_addr; 56 uint64_t sa_base; 57 uint64_t *cpt_fc; 58 uint16_t cpt_desc; 59 int32_t *cpt_fc_sw; 60 uint64_t lso_tun_fmt; 61 uint64_t ts_mem; 62 uint64_t mark_flag : 8; 63 uint64_t mark_fmt : 48; 64 struct cnxk_eth_txq_comp tx_compl; 65 } __plt_cache_aligned; 66 67 struct cn20k_eth_rxq { 68 uint64_t mbuf_initializer; 69 uintptr_t desc; 70 void *lookup_mem; 71 uintptr_t cq_door; 72 uint64_t wdata; 73 int64_t *cq_status; 74 uint32_t head; 75 uint32_t qmask; 76 uint32_t available; 77 uint16_t data_off; 78 uint64_t sa_base; 79 uint64_t lmt_base; 80 uint64_t meta_aura; 81 uintptr_t meta_pool; 82 uint16_t rq; 83 struct cnxk_timesync_info *tstamp; 84 } __plt_cache_aligned; 85 86 /* Private data in sw rsvd area of struct roc_ot_ipsec_inb_sa */ 87 struct cn20k_inb_priv_data { 88 void *userdata; 89 int reass_dynfield_off; 90 int reass_dynflag_bit; 91 struct cnxk_eth_sec_sess *eth_sec; 92 }; 93 94 struct __rte_packed_begin cn20k_sec_sess_priv { 95 union { 96 struct { 97 uint32_t sa_idx; 98 uint8_t inb_sa : 1; 99 uint8_t outer_ip_ver : 1; 100 uint8_t mode : 1; 101 uint8_t roundup_byte : 5; 102 uint8_t roundup_len; 103 uint16_t partial_len : 10; 104 uint16_t chksum : 2; 105 uint16_t dec_ttl : 1; 106 uint16_t nixtx_off : 1; 107 uint16_t rsvd : 2; 108 }; 109 110 uint64_t u64; 111 }; 112 } __rte_packed_end; 113 114 #define LMT_OFF(lmt_addr, lmt_num, offset) \ 115 (void *)((uintptr_t)(lmt_addr) + ((uint64_t)(lmt_num) << ROC_LMT_LINE_SIZE_LOG2) + (offset)) 116 117 static inline uint16_t 118 nix_tx_compl_nb_pkts(struct cn20k_eth_txq *txq, const uint64_t wdata, const uint32_t qmask) 119 { 120 uint16_t available = txq->tx_compl.available; 121 122 /* Update the available count if cached value is not enough */ 123 if (!unlikely(available)) { 124 uint64_t reg, head, tail; 125 126 /* Use LDADDA version to avoid reorder */ 127 reg = roc_atomic64_add_sync(wdata, txq->tx_compl.cq_status); 128 /* CQ_OP_STATUS operation error */ 129 if (reg & BIT_ULL(NIX_CQ_OP_STAT_OP_ERR) || reg & BIT_ULL(NIX_CQ_OP_STAT_CQ_ERR)) 130 return 0; 131 132 tail = reg & 0xFFFFF; 133 head = (reg >> 20) & 0xFFFFF; 134 if (tail < head) 135 available = tail - head + qmask + 1; 136 else 137 available = tail - head; 138 139 txq->tx_compl.available = available; 140 } 141 return available; 142 } 143 144 static inline void 145 handle_tx_completion_pkts(struct cn20k_eth_txq *txq, uint8_t mt_safe) 146 { 147 #define CNXK_NIX_CQ_ENTRY_SZ 128 148 #define CQE_SZ(x) ((x) * CNXK_NIX_CQ_ENTRY_SZ) 149 150 uint16_t tx_pkts = 0, nb_pkts; 151 const uintptr_t desc = txq->tx_compl.desc_base; 152 const uint64_t wdata = txq->tx_compl.wdata; 153 const uint32_t qmask = txq->tx_compl.qmask; 154 uint32_t head = txq->tx_compl.head; 155 struct nix_cqe_hdr_s *tx_compl_cq; 156 struct nix_send_comp_s *tx_compl_s0; 157 struct rte_mbuf *m_next, *m; 158 159 if (mt_safe) 160 rte_spinlock_lock(&txq->tx_compl.ext_buf_lock); 161 162 nb_pkts = nix_tx_compl_nb_pkts(txq, wdata, qmask); 163 while (tx_pkts < nb_pkts) { 164 rte_prefetch_non_temporal((void *)(desc + (CQE_SZ((head + 2) & qmask)))); 165 tx_compl_cq = (struct nix_cqe_hdr_s *)(desc + CQE_SZ(head)); 166 tx_compl_s0 = (struct nix_send_comp_s *)((uint64_t *)tx_compl_cq + 1); 167 m = txq->tx_compl.ptr[tx_compl_s0->sqe_id]; 168 while (m->next != NULL) { 169 m_next = m->next; 170 rte_pktmbuf_free_seg(m); 171 m = m_next; 172 } 173 rte_pktmbuf_free_seg(m); 174 txq->tx_compl.ptr[tx_compl_s0->sqe_id] = NULL; 175 176 head++; 177 head &= qmask; 178 tx_pkts++; 179 } 180 txq->tx_compl.head = head; 181 txq->tx_compl.available -= nb_pkts; 182 183 plt_write64((wdata | nb_pkts), txq->tx_compl.cq_door); 184 185 if (mt_safe) 186 rte_spinlock_unlock(&txq->tx_compl.ext_buf_lock); 187 } 188 189 static __rte_always_inline uint64_t 190 cn20k_cpt_tx_steor_data(void) 191 { 192 /* We have two CPT instructions per LMTLine TODO */ 193 const uint64_t dw_m1 = ROC_CN10K_TWO_CPT_INST_DW_M1; 194 uint64_t data; 195 196 /* This will be moved to addr area */ 197 data = dw_m1 << 16; 198 data |= dw_m1 << 19; 199 data |= dw_m1 << 22; 200 data |= dw_m1 << 25; 201 data |= dw_m1 << 28; 202 data |= dw_m1 << 31; 203 data |= dw_m1 << 34; 204 data |= dw_m1 << 37; 205 data |= dw_m1 << 40; 206 data |= dw_m1 << 43; 207 data |= dw_m1 << 46; 208 data |= dw_m1 << 49; 209 data |= dw_m1 << 52; 210 data |= dw_m1 << 55; 211 data |= dw_m1 << 58; 212 data |= dw_m1 << 61; 213 214 return data; 215 } 216 217 static __rte_always_inline void 218 cn20k_nix_sec_steorl(uintptr_t io_addr, uint32_t lmt_id, uint8_t lnum, uint8_t loff, uint8_t shft) 219 { 220 uint64_t data; 221 uintptr_t pa; 222 223 /* Check if there is any CPT instruction to submit */ 224 if (!lnum && !loff) 225 return; 226 227 data = cn20k_cpt_tx_steor_data(); 228 /* Update lmtline use for partial end line */ 229 if (loff) { 230 data &= ~(0x7ULL << shft); 231 /* Update it to half full i.e 64B */ 232 data |= (0x3UL << shft); 233 } 234 235 pa = io_addr | ((data >> 16) & 0x7) << 4; 236 data &= ~(0x7ULL << 16); 237 /* Update lines - 1 that contain valid data */ 238 data |= ((uint64_t)(lnum + loff - 1)) << 12; 239 data |= (uint64_t)lmt_id; 240 241 /* STEOR */ 242 roc_lmt_submit_steorl(data, pa); 243 } 244 245 #endif /* __CN20K_RXTX_H__ */ 246