1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(C) 2021 Marvell. 3 */ 4 #ifndef __CN10K_RX_H__ 5 #define __CN10K_RX_H__ 6 7 #include <rte_ethdev.h> 8 #include <rte_security_driver.h> 9 #include <rte_vect.h> 10 #include "cn10k_rxtx.h" 11 12 #define NSEC_PER_SEC 1000000000L 13 14 #define NIX_RX_OFFLOAD_NONE (0) 15 #define NIX_RX_OFFLOAD_RSS_F BIT(0) 16 #define NIX_RX_OFFLOAD_PTYPE_F BIT(1) 17 #define NIX_RX_OFFLOAD_CHECKSUM_F BIT(2) 18 #define NIX_RX_OFFLOAD_MARK_UPDATE_F BIT(3) 19 #define NIX_RX_OFFLOAD_TSTAMP_F BIT(4) 20 #define NIX_RX_OFFLOAD_VLAN_STRIP_F BIT(5) 21 #define NIX_RX_OFFLOAD_SECURITY_F BIT(6) 22 #define NIX_RX_OFFLOAD_MAX (NIX_RX_OFFLOAD_SECURITY_F << 1) 23 24 /* Flags to control cqe_to_mbuf conversion function. 25 * Defining it from backwards to denote its been 26 * not used as offload flags to pick function 27 */ 28 #define NIX_RX_REAS_F BIT(12) 29 #define NIX_RX_VWQE_F BIT(13) 30 #define NIX_RX_MULTI_SEG_F BIT(14) 31 32 #define CNXK_NIX_CQ_ENTRY_SZ 128 33 #define NIX_DESCS_PER_LOOP 4 34 #define CQE_CAST(x) ((struct nix_cqe_hdr_s *)(x)) 35 #define CQE_SZ(x) ((x) * CNXK_NIX_CQ_ENTRY_SZ) 36 37 #define CQE_PTR_OFF(b, i, o, f) \ 38 (((f) & NIX_RX_VWQE_F) ? \ 39 (uint64_t *)(((uintptr_t)((uint64_t *)(b))[i]) + (o)) : \ 40 (uint64_t *)(((uintptr_t)(b)) + CQE_SZ(i) + (o))) 41 #define CQE_PTR_DIFF(b, i, o, f) \ 42 (((f) & NIX_RX_VWQE_F) ? \ 43 (uint64_t *)(((uintptr_t)((uint64_t *)(b))[i]) - (o)) : \ 44 (uint64_t *)(((uintptr_t)(b)) + CQE_SZ(i) - (o))) 45 46 #define NIX_RX_SEC_UCC_CONST \ 47 ((RTE_MBUF_F_RX_IP_CKSUM_BAD >> 1) | \ 48 ((RTE_MBUF_F_RX_IP_CKSUM_GOOD | RTE_MBUF_F_RX_L4_CKSUM_GOOD) >> 1) << 8 | \ 49 ((RTE_MBUF_F_RX_IP_CKSUM_GOOD | RTE_MBUF_F_RX_L4_CKSUM_BAD) >> 1) << 16 | \ 50 ((RTE_MBUF_F_RX_IP_CKSUM_GOOD | RTE_MBUF_F_RX_L4_CKSUM_GOOD) >> 1) << 32 | \ 51 ((RTE_MBUF_F_RX_IP_CKSUM_GOOD | RTE_MBUF_F_RX_L4_CKSUM_GOOD) >> 1) << 48) 52 53 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG 54 static inline void 55 nix_mbuf_validate_next(struct rte_mbuf *m) 56 { 57 if (m->nb_segs == 1 && m->next) { 58 rte_panic("mbuf->next[%p] valid when mbuf->nb_segs is %d", 59 m->next, m->nb_segs); 60 } 61 } 62 #else 63 static inline void 64 nix_mbuf_validate_next(struct rte_mbuf *m) 65 { 66 RTE_SET_USED(m); 67 } 68 #endif 69 70 #define NIX_RX_SEC_REASSEMBLY_F \ 71 (NIX_RX_REAS_F | NIX_RX_OFFLOAD_SECURITY_F) 72 73 static inline rte_eth_ip_reassembly_dynfield_t * 74 cnxk_ip_reassembly_dynfield(struct rte_mbuf *mbuf, 75 int ip_reassembly_dynfield_offset) 76 { 77 return RTE_MBUF_DYNFIELD(mbuf, ip_reassembly_dynfield_offset, 78 rte_eth_ip_reassembly_dynfield_t *); 79 } 80 81 union mbuf_initializer { 82 struct { 83 uint16_t data_off; 84 uint16_t refcnt; 85 uint16_t nb_segs; 86 uint16_t port; 87 } fields; 88 uint64_t value; 89 }; 90 91 static __rte_always_inline uint64_t 92 nix_clear_data_off(uint64_t oldval) 93 { 94 union mbuf_initializer mbuf_init = {.value = oldval}; 95 96 mbuf_init.fields.data_off = 0; 97 return mbuf_init.value; 98 } 99 100 static __rte_always_inline struct rte_mbuf * 101 nix_get_mbuf_from_cqe(void *cq, const uint64_t data_off) 102 { 103 rte_iova_t buff; 104 105 /* Skip CQE, NIX_RX_PARSE_S and SG HDR(9 DWORDs) and peek buff addr */ 106 buff = *((rte_iova_t *)((uint64_t *)cq + 9)); 107 return (struct rte_mbuf *)(buff - data_off); 108 } 109 110 static __rte_always_inline void 111 nix_sec_flush_meta_burst(uint16_t lmt_id, uint64_t data, uint16_t lnum, 112 uintptr_t aura_handle) 113 { 114 uint64_t pa; 115 116 /* Prepare PA and Data */ 117 pa = roc_npa_aura_handle_to_base(aura_handle) + NPA_LF_AURA_BATCH_FREE0; 118 pa |= ((data & 0x7) << 4); 119 120 data >>= 3; 121 data <<= 19; 122 data |= (uint64_t)lmt_id; 123 data |= (uint64_t)(lnum - 1) << 12; 124 125 roc_lmt_submit_steorl(data, pa); 126 } 127 128 static __rte_always_inline void 129 nix_sec_flush_meta(uintptr_t laddr, uint16_t lmt_id, uint8_t loff, 130 uintptr_t aura_handle) 131 { 132 uint64_t pa; 133 134 /* laddr is pointing to first pointer */ 135 laddr -= 8; 136 137 /* Trigger free either on lmtline full or different aura handle */ 138 pa = roc_npa_aura_handle_to_base(aura_handle) + NPA_LF_AURA_BATCH_FREE0; 139 140 /* Update aura handle */ 141 *(uint64_t *)laddr = (((uint64_t)(loff & 0x1) << 32) | 142 roc_npa_aura_handle_to_aura(aura_handle)); 143 144 pa |= ((uint64_t)(loff >> 1) << 4); 145 roc_lmt_submit_steorl(lmt_id, pa); 146 } 147 148 #if defined(RTE_ARCH_ARM64) 149 static __rte_always_inline uint64_t 150 nix_sec_reass_frags_get(const struct cpt_parse_hdr_s *hdr, struct rte_mbuf **next_mbufs) 151 { 152 const struct cpt_frag_info_s *finfo; 153 uint32_t offset = hdr->w2.fi_offset; 154 const uint64_t *frag_ptr; 155 uint64x2_t frags23; 156 uint16x4_t fsz_w1; 157 158 /* offset of 0 implies 256B, otherwise it implies offset*8B */ 159 offset = (((offset - 1) & 0x1f) + 1) * 8; 160 finfo = RTE_PTR_ADD(hdr, offset); 161 frag_ptr = (const uint64_t *)(finfo + 1); 162 frags23 = vrev64q_u8(vld1q_u64(frag_ptr)); 163 164 next_mbufs[0] = ((struct rte_mbuf *)rte_be_to_cpu_64(hdr->frag1_wqe_ptr) - 1); 165 next_mbufs[1] = ((struct rte_mbuf *)vgetq_lane_u64(frags23, 0) - 1); 166 next_mbufs[2] = ((struct rte_mbuf *)vgetq_lane_u64(frags23, 1) - 1); 167 168 fsz_w1 = vreinterpret_u16_u64(vdup_n_u64(finfo->w1.u64)); 169 fsz_w1 = vrev16_u8(fsz_w1); 170 return vget_lane_u64(vreinterpret_u64_u16(fsz_w1), 0); 171 } 172 173 static __rte_always_inline void 174 nix_sec_reass_first_frag_update(struct rte_mbuf *head, const uint8_t *m_ipptr, 175 uint64_t fsz, uint64_t cq_w1, uint16_t *ihl) 176 { 177 union nix_rx_parse_u *rx = (union nix_rx_parse_u *)((uintptr_t)(head + 1) + 8); 178 uint16_t fragx_sum = vaddv_u16(vreinterpret_u16_u64(vdup_n_u64(fsz))); 179 uint8_t lcptr = rx->lcptr; 180 uint16_t tot_len; 181 uint32_t cksum; 182 uint8_t *ipptr; 183 184 ipptr = (uint8_t *)head->buf_addr + head->data_off + lcptr; 185 /* Find the L3 header length and update inner pkt based on meta lc type */ 186 if (((cq_w1 >> 40) & 0xF) == NPC_LT_LC_IP) { 187 const struct rte_ipv4_hdr *m_hdr = (const struct rte_ipv4_hdr *)m_ipptr; 188 struct rte_ipv4_hdr *hdr = (struct rte_ipv4_hdr *)ipptr; 189 190 *ihl = (m_hdr->version_ihl & 0xf) << 2; 191 192 hdr->fragment_offset = 0; 193 tot_len = rte_cpu_to_be_16(fragx_sum + *ihl); 194 hdr->total_length = tot_len; 195 /* Perform incremental checksum based on meta pkt ip hdr */ 196 cksum = m_hdr->hdr_checksum; 197 cksum += m_hdr->fragment_offset; 198 cksum += 0xFFFF; 199 cksum += m_hdr->total_length; 200 cksum += (uint16_t)(~tot_len); 201 cksum = (cksum & 0xFFFF) + ((cksum & 0xFFFF0000) >> 16); 202 hdr->hdr_checksum = cksum; 203 204 head->pkt_len = lcptr + *ihl + fragx_sum; 205 } else { 206 struct rte_ipv6_hdr *hdr = (struct rte_ipv6_hdr *)ipptr; 207 size_t ext_len = sizeof(struct rte_ipv6_hdr); 208 uint8_t *nxt_hdr = (uint8_t *)hdr; 209 uint8_t *nxt_proto = &hdr->proto; 210 int nh = hdr->proto; 211 212 *ihl = 0; 213 tot_len = 0; 214 while (nh != -EINVAL) { 215 nxt_hdr += ext_len; 216 *ihl += ext_len; 217 if (nh == IPPROTO_FRAGMENT) { 218 *nxt_proto = *nxt_hdr; 219 tot_len = *ihl; 220 } 221 nh = rte_ipv6_get_next_ext(nxt_hdr, nh, &ext_len); 222 nxt_proto = nxt_hdr; 223 } 224 225 /* Remove the frag header by moving header 8 bytes forward */ 226 hdr->payload_len = rte_cpu_to_be_16(fragx_sum + *ihl - 227 8 - sizeof(struct rte_ipv6_hdr)); 228 229 /* tot_len is sum of all IP header's length before fragment header */ 230 rte_memcpy(rte_pktmbuf_mtod_offset(head, void *, 8), 231 rte_pktmbuf_mtod(head, void *), 232 lcptr + tot_len); 233 234 head->data_len -= 8; 235 head->data_off += 8; 236 head->pkt_len = lcptr + *ihl - 8 + fragx_sum; 237 /* ihl l3hdr size value should be up to fragment header for next frags */ 238 *ihl = tot_len + 8; 239 } 240 } 241 242 #else 243 static __rte_always_inline uint64_t 244 nix_sec_reass_frags_get(const struct cpt_parse_hdr_s *hdr, struct rte_mbuf **next_mbufs) 245 { 246 RTE_SET_USED(hdr); 247 next_mbufs[0] = NULL; 248 next_mbufs[1] = NULL; 249 next_mbufs[2] = NULL; 250 return 0; 251 } 252 253 static __rte_always_inline void 254 nix_sec_reass_first_frag_update(struct rte_mbuf *head, const uint8_t *m_ipptr, 255 uint64_t fsz, uint64_t cq_w1, uint16_t *ihl) 256 { 257 RTE_SET_USED(head); 258 RTE_SET_USED(m_ipptr); 259 RTE_SET_USED(fsz); 260 RTE_SET_USED(cq_w1); 261 *ihl = 0; 262 } 263 #endif 264 265 static struct rte_mbuf * 266 nix_sec_attach_frags(const struct cpt_parse_hdr_s *hdr, 267 struct rte_mbuf *head, 268 struct cn10k_inb_priv_data *inb_priv, 269 const uint64_t mbuf_init) 270 { 271 uint8_t num_frags = hdr->w0.num_frags; 272 struct rte_mbuf *next_mbufs[3]; 273 union nix_rx_parse_u *frag_rx; 274 struct rte_mbuf *mbuf; 275 uint64_t ol_flags; 276 uint16_t frag_size; 277 uint8_t frag_i = 0; 278 uint16_t rlen; 279 uint64_t *wqe; 280 int off; 281 282 off = inb_priv->reass_dynfield_off; 283 ol_flags = BIT_ULL(inb_priv->reass_dynflag_bit); 284 ol_flags |= RTE_MBUF_F_RX_SEC_OFFLOAD; 285 286 /* Get frags list */ 287 nix_sec_reass_frags_get(hdr, next_mbufs); 288 289 /* Frag-0: */ 290 wqe = (uint64_t *)(head + 1); 291 rlen = ((*(wqe + 10)) >> 16) & 0xFFFF; 292 293 frag_rx = (union nix_rx_parse_u *)(wqe + 1); 294 295 head->ol_flags = ol_flags; 296 /* Update dynamic field with userdata */ 297 *rte_security_dynfield(head) = (uint64_t)inb_priv->userdata; 298 299 num_frags--; 300 mbuf = head; 301 302 /* Frag-1+: */ 303 while (num_frags) { 304 cnxk_ip_reassembly_dynfield(mbuf, off)->next_frag = next_mbufs[frag_i]; 305 cnxk_ip_reassembly_dynfield(mbuf, off)->nb_frags = num_frags; 306 mbuf = next_mbufs[frag_i]; 307 wqe = (uint64_t *)(mbuf + 1); 308 rlen = ((*(wqe + 10)) >> 16) & 0xFFFF; 309 310 frag_rx = (union nix_rx_parse_u *)(wqe + 1); 311 frag_size = rlen + frag_rx->lcptr - frag_rx->laptr; 312 313 *(uint64_t *)(&mbuf->rearm_data) = mbuf_init; 314 mbuf->data_len = frag_size; 315 mbuf->pkt_len = frag_size; 316 mbuf->ol_flags = ol_flags; 317 318 /* Update dynamic field with userdata */ 319 *rte_security_dynfield(mbuf) = (uint64_t)inb_priv->userdata; 320 321 /* Mark frag as get */ 322 RTE_MEMPOOL_CHECK_COOKIES(mbuf->pool, (void **)&mbuf, 1, 1); 323 324 num_frags--; 325 frag_i++; 326 } 327 cnxk_ip_reassembly_dynfield(mbuf, off)->nb_frags = 0; 328 cnxk_ip_reassembly_dynfield(mbuf, off)->next_frag = NULL; 329 330 return head; 331 } 332 333 static __rte_always_inline struct rte_mbuf * 334 nix_sec_reassemble_frags(const struct cpt_parse_hdr_s *hdr, struct rte_mbuf *head, 335 uint64_t cq_w1, uint64_t cq_w5, uint64_t mbuf_init) 336 { 337 uint8_t num_frags = hdr->w0.num_frags; 338 union nix_rx_parse_u *frag_rx; 339 struct rte_mbuf *next_mbufs[3]; 340 uint16_t data_off, b_off; 341 const uint8_t *m_ipptr; 342 uint16_t l3_hdr_size; 343 struct rte_mbuf *mbuf; 344 uint16_t frag_size; 345 uint64_t fsz_w1; 346 uint64_t *wqe; 347 348 /* Base data offset */ 349 b_off = mbuf_init & 0xFFFFUL; 350 mbuf_init &= ~0xFFFFUL; 351 352 /* Get list of all fragments and frag sizes */ 353 fsz_w1 = nix_sec_reass_frags_get(hdr, next_mbufs); 354 355 /* Frag-0: */ 356 wqe = (uint64_t *)(head + 1); 357 358 /* First fragment data len is already update by caller */ 359 m_ipptr = ((const uint8_t *)hdr + ((cq_w5 >> 16) & 0xFF)); 360 nix_sec_reass_first_frag_update(head, m_ipptr, fsz_w1, cq_w1, &l3_hdr_size); 361 fsz_w1 >>= 16; 362 363 /* Frag-1: */ 364 head->next = next_mbufs[0]; 365 mbuf = next_mbufs[0]; 366 wqe = (uint64_t *)(mbuf + 1); 367 frag_rx = (union nix_rx_parse_u *)(wqe + 1); 368 frag_size = fsz_w1 & 0xFFFF; 369 fsz_w1 >>= 16; 370 371 data_off = b_off + frag_rx->lcptr + l3_hdr_size; 372 *(uint64_t *)(&mbuf->rearm_data) = mbuf_init | data_off; 373 mbuf->data_len = frag_size; 374 375 /* Mark frag as get */ 376 RTE_MEMPOOL_CHECK_COOKIES(mbuf->pool, (void **)&mbuf, 1, 1); 377 378 /* Frag-2: */ 379 if (num_frags > 2) { 380 mbuf->next = next_mbufs[1]; 381 mbuf = next_mbufs[1]; 382 wqe = (uint64_t *)(mbuf + 1); 383 frag_rx = (union nix_rx_parse_u *)(wqe + 1); 384 frag_size = fsz_w1 & 0xFFFF; 385 fsz_w1 >>= 16; 386 387 data_off = b_off + frag_rx->lcptr + l3_hdr_size; 388 *(uint64_t *)(&mbuf->rearm_data) = mbuf_init | data_off; 389 mbuf->data_len = frag_size; 390 391 /* Mark frag as get */ 392 RTE_MEMPOOL_CHECK_COOKIES(mbuf->pool, (void **)&mbuf, 1, 1); 393 } 394 395 /* Frag-3: */ 396 if (num_frags > 3) { 397 mbuf->next = next_mbufs[2]; 398 mbuf = next_mbufs[2]; 399 wqe = (uint64_t *)(mbuf + 1); 400 frag_rx = (union nix_rx_parse_u *)(wqe + 1); 401 frag_size = fsz_w1 & 0xFFFF; 402 fsz_w1 >>= 16; 403 404 data_off = b_off + frag_rx->lcptr + l3_hdr_size; 405 *(uint64_t *)(&mbuf->rearm_data) = mbuf_init | data_off; 406 mbuf->data_len = frag_size; 407 408 /* Mark frag as get */ 409 RTE_MEMPOOL_CHECK_COOKIES(mbuf->pool, (void **)&mbuf, 1, 1); 410 } 411 412 head->nb_segs = num_frags; 413 return head; 414 } 415 416 static inline struct rte_mbuf * 417 nix_sec_oop_process(const struct cpt_parse_hdr_s *hdr, struct rte_mbuf *mbuf, uint64_t *mbuf_init) 418 { 419 uintptr_t wqe = rte_be_to_cpu_64(hdr->wqe_ptr); 420 union nix_rx_parse_u *inner_rx; 421 struct rte_mbuf *inner; 422 uint16_t data_off; 423 424 inner = ((struct rte_mbuf *)wqe) - 1; 425 426 inner_rx = (union nix_rx_parse_u *)(wqe + 8); 427 inner->pkt_len = inner_rx->pkt_lenm1 + 1; 428 inner->data_len = inner_rx->pkt_lenm1 + 1; 429 430 /* Mark inner mbuf as get */ 431 RTE_MEMPOOL_CHECK_COOKIES(inner->pool, 432 (void **)&inner, 1, 1); 433 /* Update rearm data for full mbuf as it has 434 * cpt parse header that needs to be skipped. 435 * 436 * Since meta pool will not have private area while 437 * ethdev RQ's first skip would be considering private area 438 * calculate actual data off and update in meta mbuf. 439 */ 440 data_off = (uintptr_t)hdr - (uintptr_t)mbuf->buf_addr; 441 data_off += sizeof(struct cpt_parse_hdr_s); 442 data_off += hdr->w0.pad_len; 443 *mbuf_init &= ~0xFFFFUL; 444 *mbuf_init |= (uint64_t)data_off; 445 446 *rte_security_oop_dynfield(mbuf) = inner; 447 /* Return outer instead of inner mbuf as inner mbuf would have original encrypted packet */ 448 return mbuf; 449 } 450 451 static __rte_always_inline struct rte_mbuf * 452 nix_sec_meta_to_mbuf_sc(uint64_t cq_w1, uint64_t cq_w5, const uint64_t sa_base, 453 uintptr_t laddr, uint8_t *loff, struct rte_mbuf *mbuf, 454 uint16_t data_off, const uint16_t flags, 455 uint64_t mbuf_init) 456 { 457 const void *__p = (void *)((uintptr_t)mbuf + (uint16_t)data_off); 458 const struct cpt_parse_hdr_s *hdr = (const struct cpt_parse_hdr_s *)__p; 459 struct cn10k_inb_priv_data *inb_priv; 460 struct rte_mbuf *inner = NULL; 461 uint32_t sa_idx; 462 uint16_t ucc; 463 uint32_t len; 464 uintptr_t ip; 465 void *inb_sa; 466 uint64_t w0; 467 468 if (!(cq_w1 & BIT(11))) 469 return mbuf; 470 471 if (flags & NIX_RX_REAS_F && hdr->w0.pkt_fmt == ROC_IE_OT_SA_PKT_FMT_FULL) { 472 inner = nix_sec_oop_process(hdr, mbuf, &mbuf_init); 473 } else { 474 inner = (struct rte_mbuf *)(rte_be_to_cpu_64(hdr->wqe_ptr) - 475 sizeof(struct rte_mbuf)); 476 477 /* Store meta in lmtline to free 478 * Assume all meta's from same aura. 479 */ 480 *(uint64_t *)(laddr + (*loff << 3)) = (uint64_t)mbuf; 481 *loff = *loff + 1; 482 } 483 484 /* Get SPI from CPT_PARSE_S's cookie(already swapped) */ 485 w0 = hdr->w0.u64; 486 sa_idx = w0 >> 32; 487 488 inb_sa = roc_nix_inl_ot_ipsec_inb_sa(sa_base, sa_idx); 489 inb_priv = roc_nix_inl_ot_ipsec_inb_sa_sw_rsvd(inb_sa); 490 491 /* Cryptodev injected packet can be identified from SA IDX 0xFFFFFFFF, and 492 * Ethdev injected packet can be identified with match ID 0xFFFF. 493 */ 494 if (flags & NIX_RX_REAS_F && (sa_idx == 0xFFFFFFFF || hdr->w0.match_id == 0xFFFFU)) { 495 *(uint64_t *)(&inner->rearm_data) = (mbuf_init & ~(BIT_ULL(16) - 1)) | 496 inner->data_off; 497 if (hdr->w0.match_id == 0xFFFFU) 498 *rte_security_dynfield(inner) = (uint64_t)inb_priv->userdata; 499 } else { 500 /* Update dynamic field with userdata */ 501 *rte_security_dynfield(inner) = (uint64_t)inb_priv->userdata; 502 *(uint64_t *)(&inner->rearm_data) = mbuf_init; 503 } 504 505 /* Get ucc from cpt parse header */ 506 ucc = hdr->w3.hw_ccode; 507 508 /* Calculate inner packet length as IP total len + l2 len */ 509 ip = (uintptr_t)hdr + ((cq_w5 >> 16) & 0xFF); 510 ip += ((cq_w1 >> 40) & 0x6); 511 len = rte_be_to_cpu_16(*(uint16_t *)ip); 512 len += ((cq_w5 >> 16) & 0xFF) - (cq_w5 & 0xFF); 513 len += (cq_w1 & BIT(42)) ? 40 : 0; 514 515 inner->pkt_len = len; 516 inner->data_len = len; 517 518 inner->ol_flags = ((CPT_COMP_HWGOOD_MASK & (1U << ucc)) ? 519 RTE_MBUF_F_RX_SEC_OFFLOAD : 520 (RTE_MBUF_F_RX_SEC_OFFLOAD | 521 RTE_MBUF_F_RX_SEC_OFFLOAD_FAILED)); 522 523 ucc = hdr->w3.uc_ccode; 524 525 if (ucc && ucc < 0xED) { 526 inner->ol_flags |= RTE_MBUF_F_RX_SEC_OFFLOAD_FAILED; 527 } else { 528 ucc += 3; /* To make codes in 0xFx series except 0 */ 529 inner->ol_flags |= ((ucc & 0xF0) == 0xF0) ? 530 ((NIX_RX_SEC_UCC_CONST >> ((ucc & 0xF) << 3)) 531 & 0xFF) << 1 : RTE_MBUF_F_RX_IP_CKSUM_GOOD; 532 } 533 534 if (!(flags & NIX_RX_REAS_F) || hdr->w0.pkt_fmt != ROC_IE_OT_SA_PKT_FMT_FULL) { 535 /* Mark meta mbuf as put */ 536 RTE_MEMPOOL_CHECK_COOKIES(mbuf->pool, (void **)&mbuf, 1, 0); 537 538 /* Mark inner mbuf as get */ 539 RTE_MEMPOOL_CHECK_COOKIES(inner->pool, (void **)&inner, 1, 1); 540 } 541 542 /* Skip reassembly processing when multi-seg is enabled */ 543 if (!(flags & NIX_RX_MULTI_SEG_F) && (flags & NIX_RX_REAS_F) && hdr->w0.num_frags) { 544 if ((!(hdr->w0.err_sum) || roc_ie_ot_ucc_is_success(hdr->w3.uc_ccode)) && 545 !(hdr->w0.reas_sts)) { 546 /* Reassembly success */ 547 nix_sec_reassemble_frags(hdr, inner, cq_w1, cq_w5, mbuf_init); 548 549 /* Update dynamic field with userdata */ 550 *rte_security_dynfield(inner) = 551 (uint64_t)inb_priv->userdata; 552 553 /* Assume success */ 554 inner->ol_flags = RTE_MBUF_F_RX_SEC_OFFLOAD; 555 } else { 556 /* Reassembly failure */ 557 nix_sec_attach_frags(hdr, inner, inb_priv, mbuf_init); 558 } 559 } 560 return inner; 561 } 562 563 #if defined(RTE_ARCH_ARM64) 564 565 static __rte_always_inline void 566 nix_sec_meta_to_mbuf(uint64_t cq_w1, uint64_t cq_w5, uintptr_t inb_sa, 567 uintptr_t cpth, struct rte_mbuf *inner, 568 uint8x16_t *rx_desc_field1, uint64_t *ol_flags, 569 const uint16_t flags, uint64x2_t *rearm) 570 { 571 const struct cpt_parse_hdr_s *hdr = 572 (const struct cpt_parse_hdr_s *)cpth; 573 uint64_t mbuf_init = vgetq_lane_u64(*rearm, 0); 574 struct cn10k_inb_priv_data *inb_priv; 575 uintptr_t p; 576 577 /* Clear checksum flags */ 578 *ol_flags &= ~(RTE_MBUF_F_RX_L4_CKSUM_MASK | 579 RTE_MBUF_F_RX_IP_CKSUM_MASK); 580 581 582 if (flags & NIX_RX_REAS_F && !inb_sa) { 583 /* Clear and update original lower 16 bit of data offset */ 584 *rearm = (*rearm & ~(BIT_ULL(16) - 1)) | inner->data_off; 585 } else { 586 /* Get SPI from CPT_PARSE_S's cookie(already swapped) */ 587 inb_priv = roc_nix_inl_ot_ipsec_inb_sa_sw_rsvd((void *)inb_sa); 588 /* Update dynamic field with userdata */ 589 *rte_security_dynfield(inner) = (uint64_t)inb_priv->userdata; 590 } 591 592 /* Clear and update original lower 16 bit of data offset */ 593 if (flags & NIX_RX_REAS_F && hdr->w0.match_id == 0xFFFFU) 594 *rearm = (*rearm & ~(BIT_ULL(16) - 1)) | inner->data_off; 595 596 /* Mark inner mbuf as get */ 597 if (!(flags & NIX_RX_REAS_F) || 598 hdr->w0.pkt_fmt != ROC_IE_OT_SA_PKT_FMT_FULL) 599 RTE_MEMPOOL_CHECK_COOKIES(inner->pool, (void **)&inner, 1, 1); 600 601 if (!(flags & NIX_RX_MULTI_SEG_F) && flags & NIX_RX_REAS_F && hdr->w0.num_frags) { 602 if ((!(hdr->w0.err_sum) || roc_ie_ot_ucc_is_success(hdr->w3.uc_ccode)) && 603 !(hdr->w0.reas_sts)) { 604 /* First frag len */ 605 inner->pkt_len = vgetq_lane_u16(*rx_desc_field1, 2); 606 inner->data_len = vgetq_lane_u16(*rx_desc_field1, 4); 607 p = (uintptr_t)&inner->rearm_data; 608 *(uint64_t *)p = mbuf_init; 609 610 /* Reassembly success */ 611 nix_sec_reassemble_frags(hdr, inner, cq_w1, cq_w5, mbuf_init); 612 613 /* Assume success */ 614 *ol_flags |= RTE_MBUF_F_RX_SEC_OFFLOAD; 615 616 /* Update pkt_len and data_len */ 617 *rx_desc_field1 = vsetq_lane_u16(inner->pkt_len, 618 *rx_desc_field1, 2); 619 *rx_desc_field1 = vsetq_lane_u16(inner->data_len, 620 *rx_desc_field1, 4); 621 622 /* Data offset might be updated */ 623 mbuf_init = *(uint64_t *)p; 624 *rearm = vsetq_lane_u64(mbuf_init, *rearm, 0); 625 } else { 626 /* Reassembly failure */ 627 if (inb_sa) { 628 nix_sec_attach_frags(hdr, inner, inb_priv, mbuf_init); 629 *ol_flags |= inner->ol_flags; 630 } 631 } 632 } else if (flags & NIX_RX_REAS_F) { 633 /* Without fragmentation but may have to handle OOP session */ 634 if (hdr->w0.pkt_fmt == ROC_IE_OT_SA_PKT_FMT_FULL) { 635 uint64_t mbuf_init = 0; 636 637 /* Caller has already prepared to return second pass 638 * mbuf and inner mbuf is actually outer. 639 * Store original buffer pointer in dynfield. 640 */ 641 nix_sec_oop_process(hdr, inner, &mbuf_init); 642 /* Clear and update lower 16 bit of data offset */ 643 *rearm = (*rearm & ~(BIT_ULL(16) - 1)) | mbuf_init; 644 } 645 } 646 } 647 #endif 648 649 static __rte_always_inline uint32_t 650 nix_ptype_get(const void *const lookup_mem, const uint64_t in) 651 { 652 const uint16_t *const ptype = lookup_mem; 653 const uint16_t lh_lg_lf = (in & 0xFFF0000000000000) >> 52; 654 const uint16_t tu_l2 = ptype[(in & 0x000FFFF000000000) >> 36]; 655 const uint16_t il4_tu = ptype[PTYPE_NON_TUNNEL_ARRAY_SZ + lh_lg_lf]; 656 657 return (il4_tu << PTYPE_NON_TUNNEL_WIDTH) | tu_l2; 658 } 659 660 static __rte_always_inline uint32_t 661 nix_rx_olflags_get(const void *const lookup_mem, const uint64_t in) 662 { 663 const uint32_t *const ol_flags = 664 (const uint32_t *)((const uint8_t *)lookup_mem + 665 PTYPE_ARRAY_SZ); 666 667 return ol_flags[(in & 0xfff00000) >> 20]; 668 } 669 670 static inline uint64_t 671 nix_update_match_id(const uint16_t match_id, uint64_t ol_flags, 672 struct rte_mbuf *mbuf) 673 { 674 /* There is no separate bit to check match_id 675 * is valid or not? and no flag to identify it is an 676 * RTE_FLOW_ACTION_TYPE_FLAG vs RTE_FLOW_ACTION_TYPE_MARK 677 * action. The former case addressed through 0 being invalid 678 * value and inc/dec match_id pair when MARK is activated. 679 * The later case addressed through defining 680 * CNXK_FLOW_MARK_DEFAULT as value for 681 * RTE_FLOW_ACTION_TYPE_MARK. 682 * This would translate to not use 683 * CNXK_FLOW_ACTION_FLAG_DEFAULT - 1 and 684 * CNXK_FLOW_ACTION_FLAG_DEFAULT for match_id. 685 * i.e valid mark_id's are from 686 * 0 to CNXK_FLOW_ACTION_FLAG_DEFAULT - 2 687 */ 688 if (likely(match_id)) { 689 ol_flags |= RTE_MBUF_F_RX_FDIR; 690 if (match_id != CNXK_FLOW_ACTION_FLAG_DEFAULT) { 691 ol_flags |= RTE_MBUF_F_RX_FDIR_ID; 692 mbuf->hash.fdir.hi = match_id - 1; 693 } 694 } 695 696 return ol_flags; 697 } 698 699 static __rte_always_inline void 700 nix_cqe_xtract_mseg(const union nix_rx_parse_u *rx, struct rte_mbuf *mbuf, 701 uint64_t rearm, uintptr_t cpth, uintptr_t sa_base, const uint16_t flags) 702 { 703 const struct cpt_parse_hdr_s *hdr = (const struct cpt_parse_hdr_s *)cpth; 704 struct cn10k_inb_priv_data *inb_priv = NULL; 705 uint8_t num_frags = 0, frag_i = 0; 706 struct rte_mbuf *next_mbufs[3]; 707 const rte_iova_t *iova_list; 708 bool reas_success = false; 709 uint16_t later_skip = 0; 710 struct rte_mbuf *head; 711 const rte_iova_t *eol; 712 bool rx_inj = false; 713 uint64_t cq_w5 = 0; 714 uint16_t ihl = 0; 715 uint64_t fsz = 0; 716 int dyn_off = 0; 717 uint8_t nb_segs; 718 uint16_t sg_len; 719 uint64_t cq_w1; 720 int64_t len; 721 uint64_t sg; 722 uintptr_t p; 723 724 cq_w1 = *(const uint64_t *)rx; 725 if (flags & NIX_RX_REAS_F) 726 cq_w5 = *((const uint64_t *)rx + 4); 727 /* Use inner rx parse for meta pkts sg list */ 728 if (cq_w1 & BIT(11) && flags & NIX_RX_OFFLOAD_SECURITY_F) { 729 const uint64_t *wqe; 730 /* Rx Inject packet must have Match ID 0xFFFF and for this 731 * wqe will get from address stored at mbuf+1 location 732 */ 733 rx_inj = ((flags & NIX_RX_REAS_F) && ((hdr->w0.match_id == 0xFFFFU) || 734 (hdr->w0.cookie == 0xFFFFFFFFU))); 735 if (rx_inj) 736 wqe = (const uint64_t *)*((uint64_t *)(mbuf + 1)); 737 else 738 wqe = (const uint64_t *)(mbuf + 1); 739 740 if (!(flags & NIX_RX_REAS_F) || hdr->w0.pkt_fmt != ROC_IE_OT_SA_PKT_FMT_FULL) 741 rx = (const union nix_rx_parse_u *)(wqe + 1); 742 } 743 744 sg = *(const uint64_t *)(rx + 1); 745 nb_segs = (sg >> 48) & 0x3; 746 747 if (nb_segs == 1 && !(flags & NIX_RX_REAS_F)) 748 return; 749 750 /* For security we have already updated right pkt_len */ 751 if (cq_w1 & BIT(11) && flags & NIX_RX_OFFLOAD_SECURITY_F) { 752 len = mbuf->pkt_len; 753 754 /* Handle reassembly with multi segs */ 755 if (flags & NIX_RX_REAS_F && hdr->w0.num_frags) { 756 void *inb_sa; 757 758 num_frags = hdr->w0.num_frags; 759 inb_sa = roc_nix_inl_ot_ipsec_inb_sa(sa_base, hdr->w0.u64 >> 32); 760 inb_priv = roc_nix_inl_ot_ipsec_inb_sa_sw_rsvd(inb_sa); 761 ihl = 0; 762 763 dyn_off = inb_priv->reass_dynfield_off; 764 fsz = nix_sec_reass_frags_get(hdr, next_mbufs); 765 num_frags -= 1; 766 767 if (!(hdr->w0.reas_sts) && 768 (!(hdr->w0.err_sum) || 769 roc_ie_ot_ucc_is_success(hdr->w3.uc_ccode))) 770 reas_success = true; 771 } 772 } else { 773 len = rx->pkt_lenm1 + 1; 774 } 775 776 mbuf->pkt_len = len - (flags & NIX_RX_OFFLOAD_TSTAMP_F ? CNXK_NIX_TIMESYNC_RX_OFFSET : 0); 777 mbuf->nb_segs = nb_segs; 778 head = mbuf; 779 mbuf->data_len = 780 (sg & 0xFFFF) - (flags & NIX_RX_OFFLOAD_TSTAMP_F ? CNXK_NIX_TIMESYNC_RX_OFFSET : 0); 781 eol = ((const rte_iova_t *)(rx + 1) + ((rx->desc_sizem1 + 1) << 1)); 782 again: 783 len -= mbuf->data_len; 784 sg = sg >> 16; 785 /* Skip SG_S and first IOVA*/ 786 iova_list = ((const rte_iova_t *)(rx + 1)) + 2; 787 nb_segs--; 788 789 later_skip = (uintptr_t)mbuf->buf_addr - (uintptr_t)mbuf; 790 791 while (nb_segs) { 792 if (!(flags & NIX_RX_REAS_F) || !rx_inj) 793 mbuf->next = (struct rte_mbuf *)(*iova_list - later_skip); 794 mbuf = mbuf->next; 795 796 RTE_MEMPOOL_CHECK_COOKIES(mbuf->pool, (void **)&mbuf, 1, 1); 797 798 sg_len = sg & 0XFFFF; 799 if (flags & NIX_RX_OFFLOAD_SECURITY_F) { 800 /* Adjust last mbuf data length with negative offset for 801 * security pkts if needed. 802 */ 803 len -= sg_len; 804 sg_len = (len > 0) ? sg_len : (sg_len + len); 805 len = (len > 0) ? len : 0; 806 } 807 808 mbuf->data_len = sg_len; 809 sg = sg >> 16; 810 p = (uintptr_t)&mbuf->rearm_data; 811 if (!(flags & NIX_RX_REAS_F) || !rx_inj) 812 *(uint64_t *)p = rearm & ~0xFFFF; 813 nb_segs--; 814 iova_list++; 815 816 if (!nb_segs && (iova_list + 1 < eol)) { 817 sg = *(const uint64_t *)(iova_list); 818 nb_segs = (sg >> 48) & 0x3; 819 head->nb_segs += nb_segs; 820 iova_list = (const rte_iova_t *)(iova_list + 1); 821 } 822 } 823 824 if ((flags & NIX_RX_REAS_F) && (cq_w1 & BIT(11)) && num_frags) { 825 struct rte_mbuf *next_frag = next_mbufs[frag_i]; 826 uint16_t lcptr, ldptr = 0; 827 828 rx = (const union nix_rx_parse_u *)((uintptr_t)(next_frag + 1) + 8); 829 lcptr = (*((const uint64_t *)rx + 4) >> 16) & 0xFF; 830 eol = ((const rte_iova_t *)(rx + 1) + ((rx->desc_sizem1 + 1) << 1)); 831 sg = *(const uint64_t *)(rx + 1); 832 nb_segs = (sg >> 48) & 0x3; 833 834 if (reas_success) { 835 /* Update first fragment info */ 836 if (!frag_i) { 837 const uint8_t *ipptr; 838 839 ipptr = ((const uint8_t *)hdr + ((cq_w5 >> 16) & 0xFF)); 840 nix_sec_reass_first_frag_update(head, ipptr, fsz, cq_w1, &ihl); 841 fsz >>= 16; 842 } 843 mbuf->next = next_frag; 844 head->nb_segs += nb_segs; 845 len = fsz & 0xFFFF; 846 fsz >>= 16; 847 ldptr = ihl + lcptr; 848 } else { 849 len = ((eol[0] >> 16) & 0xFFFF) + lcptr; 850 head->ol_flags |= BIT_ULL(inb_priv->reass_dynflag_bit) | 851 RTE_MBUF_F_RX_SEC_OFFLOAD; 852 cnxk_ip_reassembly_dynfield(head, dyn_off)->next_frag = next_frag; 853 cnxk_ip_reassembly_dynfield(head, dyn_off)->nb_frags = num_frags; 854 /* Update dynamic field with userdata from prev head */ 855 *rte_security_dynfield(next_frag) = *rte_security_dynfield(head); 856 head = next_frag; 857 head->pkt_len = len - (flags & NIX_RX_OFFLOAD_TSTAMP_F ? 858 CNXK_NIX_TIMESYNC_RX_OFFSET : 0); 859 head->nb_segs = nb_segs; 860 } 861 mbuf = next_frag; 862 p = (uintptr_t)&mbuf->rearm_data; 863 *(uint64_t *)p = rearm + ldptr; 864 mbuf->data_len = (sg & 0xFFFF) - ldptr - 865 (flags & NIX_RX_OFFLOAD_TSTAMP_F ? 866 CNXK_NIX_TIMESYNC_RX_OFFSET : 0); 867 RTE_MEMPOOL_CHECK_COOKIES(mbuf->pool, (void **)&mbuf, 1, 1); 868 num_frags--; 869 frag_i++; 870 goto again; 871 } else if ((flags & NIX_RX_REAS_F) && (cq_w1 & BIT(11)) && !reas_success && 872 hdr->w0.pkt_fmt == ROC_IE_OT_SA_PKT_FMT_FULL) { 873 uintptr_t wqe = rte_be_to_cpu_64(hdr->wqe_ptr); 874 875 /* Process OOP packet inner buffer mseg. reas_success flag is used here only 876 * to avoid looping. 877 */ 878 mbuf = ((struct rte_mbuf *)wqe) - 1; 879 rx = (const union nix_rx_parse_u *)(wqe + 8); 880 eol = ((const rte_iova_t *)(rx + 1) + ((rx->desc_sizem1 + 1) << 1)); 881 sg = *(const uint64_t *)(rx + 1); 882 nb_segs = (sg >> 48) & 0x3; 883 884 885 len = mbuf->pkt_len; 886 p = (uintptr_t)&mbuf->rearm_data; 887 *(uint64_t *)p = rearm; 888 mbuf->data_len = (sg & 0xFFFF) - 889 (flags & NIX_RX_OFFLOAD_TSTAMP_F ? 890 CNXK_NIX_TIMESYNC_RX_OFFSET : 0); 891 head = mbuf; 892 head->nb_segs = nb_segs; 893 /* Using this flag to avoid looping in case of OOP */ 894 reas_success = true; 895 goto again; 896 } 897 898 /* Update for last failure fragment */ 899 if ((flags & NIX_RX_REAS_F) && frag_i && !reas_success) { 900 cnxk_ip_reassembly_dynfield(head, dyn_off)->next_frag = NULL; 901 cnxk_ip_reassembly_dynfield(head, dyn_off)->nb_frags = 0; 902 } 903 } 904 905 static __rte_always_inline void 906 cn10k_nix_cqe_to_mbuf(const struct nix_cqe_hdr_s *cq, const uint32_t tag, 907 struct rte_mbuf *mbuf, const void *lookup_mem, 908 const uint64_t val, const uintptr_t cpth, const uintptr_t sa_base, 909 const uint16_t flag) 910 { 911 const union nix_rx_parse_u *rx = 912 (const union nix_rx_parse_u *)((const uint64_t *)cq + 1); 913 const uint64_t w1 = *(const uint64_t *)rx; 914 uint16_t len = rx->pkt_lenm1 + 1; 915 uint64_t ol_flags = 0; 916 uintptr_t p; 917 918 if (flag & NIX_RX_OFFLOAD_PTYPE_F) 919 mbuf->packet_type = nix_ptype_get(lookup_mem, w1); 920 else 921 mbuf->packet_type = 0; 922 923 if (flag & NIX_RX_OFFLOAD_RSS_F) { 924 mbuf->hash.rss = tag; 925 ol_flags |= RTE_MBUF_F_RX_RSS_HASH; 926 } 927 928 /* Skip rx ol flags extraction for Security packets */ 929 if ((!(flag & NIX_RX_SEC_REASSEMBLY_F) || !(w1 & BIT(11))) && 930 flag & NIX_RX_OFFLOAD_CHECKSUM_F) 931 ol_flags |= (uint64_t)nix_rx_olflags_get(lookup_mem, w1); 932 933 if (flag & NIX_RX_OFFLOAD_VLAN_STRIP_F) { 934 if (rx->vtag0_gone) { 935 ol_flags |= RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED; 936 mbuf->vlan_tci = rx->vtag0_tci; 937 } 938 if (rx->vtag1_gone) { 939 ol_flags |= RTE_MBUF_F_RX_QINQ | RTE_MBUF_F_RX_QINQ_STRIPPED; 940 mbuf->vlan_tci_outer = rx->vtag1_tci; 941 } 942 } 943 944 if (flag & NIX_RX_OFFLOAD_MARK_UPDATE_F) 945 ol_flags = nix_update_match_id(rx->match_id, ol_flags, mbuf); 946 947 /* Packet data length and ol flags is already updated for sec */ 948 if (flag & NIX_RX_SEC_REASSEMBLY_F && w1 & BIT_ULL(11)) { 949 mbuf->ol_flags |= ol_flags; 950 } else { 951 mbuf->ol_flags = ol_flags; 952 mbuf->pkt_len = len; 953 mbuf->data_len = len; 954 p = (uintptr_t)&mbuf->rearm_data; 955 *(uint64_t *)p = val; 956 } 957 958 if (flag & NIX_RX_MULTI_SEG_F) 959 /* 960 * For multi segment packets, mbuf length correction according 961 * to Rx timestamp length will be handled later during 962 * timestamp data process. 963 * Hence, timestamp flag argument is not required. 964 */ 965 nix_cqe_xtract_mseg(rx, mbuf, val, cpth, sa_base, flag & ~NIX_RX_OFFLOAD_TSTAMP_F); 966 } 967 968 static inline uint16_t 969 nix_rx_nb_pkts(struct cn10k_eth_rxq *rxq, const uint64_t wdata, 970 const uint16_t pkts, const uint32_t qmask) 971 { 972 uint32_t available = rxq->available; 973 974 /* Update the available count if cached value is not enough */ 975 if (unlikely(available < pkts)) { 976 uint64_t reg, head, tail; 977 978 /* Use LDADDA version to avoid reorder */ 979 reg = roc_atomic64_add_sync(wdata, rxq->cq_status); 980 /* CQ_OP_STATUS operation error */ 981 if (reg & BIT_ULL(NIX_CQ_OP_STAT_OP_ERR) || 982 reg & BIT_ULL(NIX_CQ_OP_STAT_CQ_ERR)) 983 return 0; 984 985 tail = reg & 0xFFFFF; 986 head = (reg >> 20) & 0xFFFFF; 987 if (tail < head) 988 available = tail - head + qmask + 1; 989 else 990 available = tail - head; 991 992 rxq->available = available; 993 } 994 995 return RTE_MIN(pkts, available); 996 } 997 998 static __rte_always_inline void 999 cn10k_nix_mbuf_to_tstamp(struct rte_mbuf *mbuf, 1000 struct cnxk_timesync_info *tstamp, 1001 const uint8_t ts_enable, uint64_t *tstamp_ptr) 1002 { 1003 if (ts_enable) { 1004 mbuf->pkt_len -= CNXK_NIX_TIMESYNC_RX_OFFSET; 1005 mbuf->data_len -= CNXK_NIX_TIMESYNC_RX_OFFSET; 1006 1007 /* Reading the rx timestamp inserted by CGX, viz at 1008 * starting of the packet data. 1009 */ 1010 *tstamp_ptr = ((*tstamp_ptr >> 32) * NSEC_PER_SEC) + 1011 (*tstamp_ptr & 0xFFFFFFFFUL); 1012 *cnxk_nix_timestamp_dynfield(mbuf, tstamp) = 1013 rte_be_to_cpu_64(*tstamp_ptr); 1014 /* RTE_MBUF_F_RX_IEEE1588_TMST flag needs to be set only in case 1015 * PTP packets are received. 1016 */ 1017 if (mbuf->packet_type == RTE_PTYPE_L2_ETHER_TIMESYNC) { 1018 tstamp->rx_tstamp = 1019 *cnxk_nix_timestamp_dynfield(mbuf, tstamp); 1020 tstamp->rx_ready = 1; 1021 mbuf->ol_flags |= RTE_MBUF_F_RX_IEEE1588_PTP | 1022 RTE_MBUF_F_RX_IEEE1588_TMST | 1023 tstamp->rx_tstamp_dynflag; 1024 } 1025 } 1026 } 1027 1028 static __rte_always_inline uint16_t 1029 cn10k_nix_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts, 1030 const uint16_t flags) 1031 { 1032 struct cn10k_eth_rxq *rxq = rx_queue; 1033 const uint64_t mbuf_init = rxq->mbuf_initializer; 1034 const void *lookup_mem = rxq->lookup_mem; 1035 const uint64_t data_off = rxq->data_off; 1036 struct rte_mempool *meta_pool = NULL; 1037 const uintptr_t desc = rxq->desc; 1038 const uint64_t wdata = rxq->wdata; 1039 const uint32_t qmask = rxq->qmask; 1040 uint64_t lbase = rxq->lmt_base; 1041 uint16_t packets = 0, nb_pkts; 1042 uint8_t loff = 0, lnum = 0; 1043 uint32_t head = rxq->head; 1044 struct nix_cqe_hdr_s *cq; 1045 struct rte_mbuf *mbuf; 1046 uint64_t aura_handle; 1047 uint64_t sa_base = 0; 1048 uintptr_t cpth = 0; 1049 uint16_t lmt_id; 1050 uint64_t laddr; 1051 1052 nb_pkts = nix_rx_nb_pkts(rxq, wdata, pkts, qmask); 1053 1054 if (flags & NIX_RX_OFFLOAD_SECURITY_F) { 1055 aura_handle = rxq->meta_aura; 1056 sa_base = rxq->sa_base; 1057 sa_base &= ~(ROC_NIX_INL_SA_BASE_ALIGN - 1); 1058 ROC_LMT_BASE_ID_GET(lbase, lmt_id); 1059 laddr = lbase; 1060 laddr += 8; 1061 if (flags & NIX_RX_REAS_F) 1062 meta_pool = (struct rte_mempool *)rxq->meta_pool; 1063 } 1064 1065 while (packets < nb_pkts) { 1066 /* Prefetch N desc ahead */ 1067 rte_prefetch_non_temporal( 1068 (void *)(desc + (CQE_SZ((head + 2) & qmask)))); 1069 cq = (struct nix_cqe_hdr_s *)(desc + CQE_SZ(head)); 1070 1071 mbuf = nix_get_mbuf_from_cqe(cq, data_off); 1072 1073 /* Mark mempool obj as "get" as it is alloc'ed by NIX */ 1074 RTE_MEMPOOL_CHECK_COOKIES(mbuf->pool, (void **)&mbuf, 1, 1); 1075 1076 /* Translate meta to mbuf */ 1077 if (flags & NIX_RX_OFFLOAD_SECURITY_F) { 1078 const uint64_t cq_w1 = *((const uint64_t *)cq + 1); 1079 const uint64_t cq_w5 = *((const uint64_t *)cq + 5); 1080 1081 cpth = ((uintptr_t)mbuf + (uint16_t)data_off); 1082 1083 /* Update mempool pointer for full mode pkt */ 1084 if ((flags & NIX_RX_REAS_F) && (cq_w1 & BIT(11)) && 1085 !((*(uint64_t *)cpth) & BIT(15))) 1086 mbuf->pool = meta_pool; 1087 1088 mbuf = nix_sec_meta_to_mbuf_sc(cq_w1, cq_w5, sa_base, laddr, 1089 &loff, mbuf, data_off, 1090 flags, mbuf_init); 1091 } 1092 1093 cn10k_nix_cqe_to_mbuf(cq, cq->tag, mbuf, lookup_mem, mbuf_init, 1094 cpth, sa_base, flags); 1095 cn10k_nix_mbuf_to_tstamp(mbuf, rxq->tstamp, 1096 (flags & NIX_RX_OFFLOAD_TSTAMP_F), 1097 (uint64_t *)((uint8_t *)mbuf 1098 + data_off)); 1099 rx_pkts[packets++] = mbuf; 1100 roc_prefetch_store_keep(mbuf); 1101 head++; 1102 head &= qmask; 1103 1104 if (flags & NIX_RX_OFFLOAD_SECURITY_F) { 1105 /* Flush when we don't have space for 4 meta */ 1106 if ((15 - loff) < 1) { 1107 nix_sec_flush_meta(laddr, lmt_id + lnum, loff, 1108 aura_handle); 1109 lnum++; 1110 lnum &= BIT_ULL(ROC_LMT_LINES_PER_CORE_LOG2) - 1111 1; 1112 /* First pointer starts at 8B offset */ 1113 laddr = (uintptr_t)LMT_OFF(lbase, lnum, 8); 1114 loff = 0; 1115 } 1116 } 1117 } 1118 1119 rxq->head = head; 1120 rxq->available -= nb_pkts; 1121 1122 /* Free all the CQs that we've processed */ 1123 plt_write64((wdata | nb_pkts), rxq->cq_door); 1124 1125 /* Free remaining meta buffers if any */ 1126 if (flags & NIX_RX_OFFLOAD_SECURITY_F && loff) 1127 nix_sec_flush_meta(laddr, lmt_id + lnum, loff, aura_handle); 1128 1129 if (flags & NIX_RX_OFFLOAD_SECURITY_F) 1130 rte_io_wmb(); 1131 1132 return nb_pkts; 1133 } 1134 1135 static __rte_always_inline uint16_t 1136 cn10k_nix_flush_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts, 1137 const uint16_t flags) 1138 { 1139 struct cn10k_eth_rxq *rxq = rx_queue; 1140 const uint64_t mbuf_init = rxq->mbuf_initializer; 1141 const void *lookup_mem = rxq->lookup_mem; 1142 const uint64_t data_off = rxq->data_off; 1143 struct rte_mempool *meta_pool = NULL; 1144 const uint64_t wdata = rxq->wdata; 1145 const uint32_t qmask = rxq->qmask; 1146 const uintptr_t desc = rxq->desc; 1147 uint64_t lbase = rxq->lmt_base; 1148 uint16_t packets = 0, nb_pkts; 1149 uint16_t lmt_id __rte_unused; 1150 uint32_t head = rxq->head; 1151 struct nix_cqe_hdr_s *cq; 1152 struct rte_mbuf *mbuf; 1153 uint64_t sa_base = 0; 1154 uintptr_t cpth = 0; 1155 uint8_t loff = 0; 1156 uint64_t laddr; 1157 1158 nb_pkts = nix_rx_nb_pkts(rxq, wdata, pkts, qmask); 1159 1160 if (flags & NIX_RX_OFFLOAD_SECURITY_F) { 1161 sa_base = rxq->sa_base; 1162 sa_base &= ~(ROC_NIX_INL_SA_BASE_ALIGN - 1); 1163 ROC_LMT_BASE_ID_GET(lbase, lmt_id); 1164 laddr = lbase; 1165 laddr += 8; 1166 if (flags & NIX_RX_REAS_F) 1167 meta_pool = (struct rte_mempool *)rxq->meta_pool; 1168 } 1169 1170 while (packets < nb_pkts) { 1171 /* Prefetch N desc ahead */ 1172 rte_prefetch_non_temporal((void *)(desc + (CQE_SZ((head + 2) & qmask)))); 1173 cq = (struct nix_cqe_hdr_s *)(desc + CQE_SZ(head)); 1174 1175 mbuf = nix_get_mbuf_from_cqe(cq, data_off); 1176 1177 /* Mark mempool obj as "get" as it is alloc'ed by NIX */ 1178 RTE_MEMPOOL_CHECK_COOKIES(mbuf->pool, (void **)&mbuf, 1, 1); 1179 1180 /* Translate meta to mbuf */ 1181 if (flags & NIX_RX_OFFLOAD_SECURITY_F) { 1182 const uint64_t cq_w1 = *((const uint64_t *)cq + 1); 1183 const uint64_t cq_w5 = *((const uint64_t *)cq + 5); 1184 struct rte_mbuf *meta_buf = mbuf; 1185 1186 cpth = ((uintptr_t)meta_buf + (uint16_t)data_off); 1187 1188 /* Update mempool pointer for full mode pkt */ 1189 if ((flags & NIX_RX_REAS_F) && (cq_w1 & BIT(11)) && 1190 !((*(uint64_t *)cpth) & BIT(15))) 1191 meta_buf->pool = meta_pool; 1192 1193 mbuf = nix_sec_meta_to_mbuf_sc(cq_w1, cq_w5, sa_base, laddr, &loff, 1194 meta_buf, data_off, flags, mbuf_init); 1195 /* Free Meta mbuf, not use LMT line for flush as this will be called 1196 * from non-datapath i.e. dev_stop case. 1197 */ 1198 if (loff) { 1199 roc_npa_aura_op_free(meta_buf->pool->pool_id, 0, 1200 (uint64_t)meta_buf); 1201 loff = 0; 1202 } 1203 } 1204 1205 cn10k_nix_cqe_to_mbuf(cq, cq->tag, mbuf, lookup_mem, mbuf_init, 1206 cpth, sa_base, flags); 1207 cn10k_nix_mbuf_to_tstamp(mbuf, rxq->tstamp, 1208 (flags & NIX_RX_OFFLOAD_TSTAMP_F), 1209 (uint64_t *)((uint8_t *)mbuf + data_off)); 1210 rx_pkts[packets++] = mbuf; 1211 roc_prefetch_store_keep(mbuf); 1212 head++; 1213 head &= qmask; 1214 } 1215 1216 rxq->head = head; 1217 rxq->available -= nb_pkts; 1218 1219 /* Free all the CQs that we've processed */ 1220 plt_write64((wdata | nb_pkts), rxq->cq_door); 1221 1222 if (flags & NIX_RX_OFFLOAD_SECURITY_F) 1223 rte_io_wmb(); 1224 1225 return nb_pkts; 1226 } 1227 1228 #if defined(RTE_ARCH_ARM64) 1229 static __rte_always_inline uint16_t 1230 cn10k_nix_rx_inj_prepare_mseg(struct rte_mbuf *m, uint64_t *cmd) 1231 { 1232 union nix_send_sg_s *sg, l_sg; 1233 struct rte_mbuf *m_next; 1234 uint16_t segdw, nb_segs; 1235 uint64_t len, dlen; 1236 uint64_t *slist; 1237 1238 sg = (union nix_send_sg_s *)cmd; 1239 l_sg.u = sg->u; 1240 l_sg.u &= 0xC00000000000000; /* LD_TYPE */ 1241 l_sg.subdc = NIX_SUBDC_SG; 1242 nb_segs = m->nb_segs; 1243 len = m->pkt_len; 1244 slist = &cmd[1]; 1245 1246 /* Fill mbuf segments */ 1247 do { 1248 *slist = rte_pktmbuf_iova(m); 1249 dlen = m->data_len; 1250 len -= dlen; 1251 1252 /* Set the segment length */ 1253 l_sg.u |= ((uint64_t)dlen << (l_sg.segs << 4)); 1254 l_sg.segs += 1; 1255 slist++; 1256 nb_segs--; 1257 if (l_sg.segs > 2 && nb_segs) { 1258 sg->u = l_sg.u; 1259 /* Next SG subdesc */ 1260 sg = (union nix_send_sg_s *)slist; 1261 l_sg.u = sg->u; 1262 l_sg.u &= 0xC00000000000000; /* LD_TYPE */ 1263 l_sg.subdc = NIX_SUBDC_SG; 1264 slist++; 1265 } 1266 m_next = m->next; 1267 m = m_next; 1268 } while (nb_segs); 1269 1270 /* Add remaining bytes of data to last seg */ 1271 if (len) { 1272 uint8_t shft = (l_sg.subdc == NIX_SUBDC_SG) ? ((l_sg.segs - 1) << 4) : 0; 1273 dlen = ((l_sg.u >> shft) & 0xFFFFULL) + len; 1274 l_sg.u = l_sg.u & ~(0xFFFFULL << shft); 1275 l_sg.u |= dlen << shft; 1276 } 1277 /* Write the last subdc out */ 1278 sg->u = l_sg.u; 1279 1280 segdw = (uint64_t *)slist - cmd; 1281 /* Roundup extra dwords to multiple of 2 */ 1282 segdw = (segdw >> 1) + (segdw & 0x1); 1283 return segdw; 1284 } 1285 1286 static __rte_always_inline uint16_t 1287 cn10k_nix_inj_pkts(struct rte_security_session **sess, struct cnxk_ethdev_inj_cfg *inj_cfg, 1288 struct rte_mbuf **tx_pkts, uint16_t nb_pkts) 1289 { 1290 uintptr_t c_lbase = inj_cfg->lmt_base; 1291 struct cn10k_sec_sess_priv sess_priv; 1292 uint64_t sa_base = inj_cfg->sa_base; 1293 uint16_t c_lmt_id, burst, left, i; 1294 uintptr_t cptres, rxphdr, dptr; 1295 struct rte_mbuf *m, *last; 1296 uint8_t lnum, shft, loff; 1297 uint64x2_t cmd01, cmd23; 1298 uint64_t ucode_cmd[4]; 1299 rte_iova_t c_io_addr; 1300 uint64_t *laddr; 1301 uint64_t sa, w0; 1302 uint16_t segdw; 1303 1304 /* Get LMT base address and LMT ID as lcore id */ 1305 ROC_LMT_CPT_BASE_ID_GET(c_lbase, c_lmt_id); 1306 c_io_addr = inj_cfg->io_addr; 1307 1308 left = nb_pkts; 1309 again: 1310 burst = left > 32 ? 32 : left; 1311 1312 lnum = 0; 1313 loff = 0; 1314 shft = 16; 1315 1316 for (i = 0; i < burst; i++) { 1317 m = tx_pkts[i]; 1318 sess_priv.u64 = sess[i]->fast_mdata; 1319 last = rte_pktmbuf_lastseg(m); 1320 1321 cptres = rte_pktmbuf_mtod_offset(last, uintptr_t, last->data_len); 1322 cptres += BIT_ULL(7); 1323 cptres = (cptres - 1) & ~(BIT_ULL(7) - 1); 1324 1325 if (m->nb_segs > 1) { 1326 /* Will reserve NIX Rx descriptor with SG list after end of 1327 * last mbuf data location. and pointer to this will be 1328 * stored at 1st mbuf space for Rx path multi-seg processing. 1329 */ 1330 /* Pointer to WQE header */ 1331 *(uint64_t *)(m + 1) = cptres; 1332 /* Reserve 8 Dwords of WQE Hdr + Rx Parse Hdr */ 1333 rxphdr = cptres + 8; 1334 dptr = rxphdr + 7 * 8; 1335 /* Prepare Multiseg SG list */ 1336 segdw = cn10k_nix_rx_inj_prepare_mseg(m, (uint64_t *)(dptr)); 1337 *(uint64_t *)rxphdr = (uint64_t)(segdw - 1) << 12; 1338 cptres += 64 + segdw * 16; 1339 ucode_cmd[1] = dptr | ((uint64_t)m->nb_segs << 60); 1340 } else { 1341 dptr = (uint64_t)rte_pktmbuf_iova(m); 1342 ucode_cmd[1] = dptr; 1343 } 1344 1345 /* Prepare CPT instruction */ 1346 /* CPT word 0 and 1 */ 1347 cmd01 = vdupq_n_u64(0); 1348 w0 = inj_cfg->cmd_w0 | ((uint64_t)m->l2_len - 2) << 24 | (uint64_t)m->l2_len << 16; 1349 cmd01 = vsetq_lane_u64(w0, cmd01, 0); 1350 cmd01 = vsetq_lane_u64(cptres, cmd01, 1); 1351 1352 /* CPT word 2 and 3 */ 1353 cmd23 = vdupq_n_u64(0); 1354 /* Set PF func */ 1355 w0 &= 0xFFFF000000000000UL; 1356 cmd23 = vsetq_lane_u64(w0, cmd23, 0); 1357 cmd23 = vsetq_lane_u64(((uint64_t)m + sizeof(struct rte_mbuf)) | 1, cmd23, 1); 1358 1359 sa_base &= ~0xFFFFUL; 1360 sa = (uintptr_t)roc_nix_inl_ot_ipsec_inb_sa(sa_base, sess_priv.sa_idx); 1361 ucode_cmd[0] = (ROC_IE_OT_MAJOR_OP_PROCESS_INBOUND_IPSEC << 48 | 1UL << 54 | 1362 ((uint64_t)sess_priv.chksum) << 32 | 1363 ((uint64_t)sess_priv.dec_ttl) << 34 | m->pkt_len); 1364 1365 ucode_cmd[2] = 0; 1366 ucode_cmd[3] = (ROC_CPT_DFLT_ENG_GRP_SE_IE << 61 | 1UL << 60 | sa); 1367 1368 /* Move to our line */ 1369 laddr = LMT_OFF(c_lbase, lnum, loff ? 64 : 0); 1370 1371 /* Write CPT instruction to lmt line */ 1372 vst1q_u64(laddr, cmd01); 1373 vst1q_u64((laddr + 2), cmd23); 1374 1375 *(__uint128_t *)(laddr + 4) = *(__uint128_t *)ucode_cmd; 1376 *(__uint128_t *)(laddr + 6) = *(__uint128_t *)(ucode_cmd + 2); 1377 1378 loff = !loff; 1379 lnum = lnum + (loff ? 0 : 1); 1380 shft = shft + (loff ? 0 : 3); 1381 } 1382 1383 left -= burst; 1384 tx_pkts += burst; 1385 sess += burst; 1386 1387 cn10k_nix_sec_steorl(c_io_addr, c_lmt_id, lnum, loff, shft); 1388 1389 rte_io_wmb(); 1390 if (left) 1391 goto again; 1392 1393 return nb_pkts; 1394 } 1395 #else 1396 static __rte_always_inline uint16_t 1397 cn10k_nix_inj_pkts(struct rte_security_session **sess, struct cnxk_ethdev_inj_cfg *inj_cfg, 1398 struct rte_mbuf **tx_pkts, uint16_t nb_pkts) 1399 { 1400 RTE_SET_USED(sess); 1401 RTE_SET_USED(inj_cfg); 1402 RTE_SET_USED(tx_pkts); 1403 RTE_SET_USED(nb_pkts); 1404 return 0; 1405 } 1406 #endif 1407 1408 #if defined(RTE_ARCH_ARM64) 1409 1410 static __rte_always_inline uint64_t 1411 nix_vlan_update(const uint64_t w2, uint64_t ol_flags, uint8x16_t *f) 1412 { 1413 if (w2 & BIT_ULL(21) /* vtag0_gone */) { 1414 ol_flags |= RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED; 1415 *f = vsetq_lane_u16((uint16_t)(w2 >> 32), *f, 5); 1416 } 1417 1418 return ol_flags; 1419 } 1420 1421 static __rte_always_inline uint64_t 1422 nix_qinq_update(const uint64_t w2, uint64_t ol_flags, struct rte_mbuf *mbuf) 1423 { 1424 if (w2 & BIT_ULL(23) /* vtag1_gone */) { 1425 ol_flags |= RTE_MBUF_F_RX_QINQ | RTE_MBUF_F_RX_QINQ_STRIPPED; 1426 mbuf->vlan_tci_outer = (uint16_t)(w2 >> 48); 1427 } 1428 1429 return ol_flags; 1430 } 1431 1432 #define NIX_PUSH_META_TO_FREE(_mbuf, _laddr, _loff_p) \ 1433 do { \ 1434 *(uint64_t *)((_laddr) + (*(_loff_p) << 3)) = (uint64_t)_mbuf; \ 1435 *(_loff_p) = *(_loff_p) + 1; \ 1436 /* Mark meta mbuf as put */ \ 1437 RTE_MEMPOOL_CHECK_COOKIES(_mbuf->pool, (void **)&_mbuf, 1, 0); \ 1438 } while (0) 1439 1440 static __rte_always_inline uint16_t 1441 cn10k_nix_recv_pkts_vector(void *args, struct rte_mbuf **mbufs, uint16_t pkts, 1442 const uint16_t flags, void *lookup_mem, 1443 struct cnxk_timesync_info *tstamp, 1444 uintptr_t lmt_base, uint64_t meta_aura) 1445 { 1446 struct cn10k_eth_rxq *rxq = args; 1447 const uint64_t mbuf_initializer = (flags & NIX_RX_VWQE_F) ? 1448 *(uint64_t *)args : 1449 rxq->mbuf_initializer; 1450 const uint64x2_t data_off = flags & NIX_RX_VWQE_F ? 1451 vdupq_n_u64(RTE_PKTMBUF_HEADROOM) : 1452 vdupq_n_u64(rxq->data_off); 1453 const uint32_t qmask = flags & NIX_RX_VWQE_F ? 0 : rxq->qmask; 1454 const uint64_t wdata = flags & NIX_RX_VWQE_F ? 0 : rxq->wdata; 1455 const uintptr_t desc = flags & NIX_RX_VWQE_F ? 0 : rxq->desc; 1456 uint64x2_t cq0_w8, cq1_w8, cq2_w8, cq3_w8, mbuf01, mbuf23; 1457 uintptr_t cpth0 = 0, cpth1 = 0, cpth2 = 0, cpth3 = 0; 1458 uint64_t ol_flags0, ol_flags1, ol_flags2, ol_flags3; 1459 uint64x2_t rearm0 = vdupq_n_u64(mbuf_initializer); 1460 uint64x2_t rearm1 = vdupq_n_u64(mbuf_initializer); 1461 uint64x2_t rearm2 = vdupq_n_u64(mbuf_initializer); 1462 uint64x2_t rearm3 = vdupq_n_u64(mbuf_initializer); 1463 struct rte_mbuf *mbuf0, *mbuf1, *mbuf2, *mbuf3; 1464 uint8_t loff = 0, lnum = 0, shft = 0; 1465 struct rte_mempool *meta_pool = NULL; 1466 uint8x16_t f0, f1, f2, f3; 1467 uint16_t lmt_id, d_off; 1468 uint64_t lbase, laddr; 1469 uintptr_t sa_base = 0; 1470 uint16_t packets = 0; 1471 uint16_t pkts_left; 1472 uint32_t head; 1473 uintptr_t cq0; 1474 1475 if (!(flags & NIX_RX_VWQE_F)) { 1476 lookup_mem = rxq->lookup_mem; 1477 head = rxq->head; 1478 1479 pkts = nix_rx_nb_pkts(rxq, wdata, pkts, qmask); 1480 pkts_left = pkts & (NIX_DESCS_PER_LOOP - 1); 1481 /* Packets has to be floor-aligned to NIX_DESCS_PER_LOOP */ 1482 pkts = RTE_ALIGN_FLOOR(pkts, NIX_DESCS_PER_LOOP); 1483 if (flags & NIX_RX_OFFLOAD_TSTAMP_F) 1484 tstamp = rxq->tstamp; 1485 1486 cq0 = desc + CQE_SZ(head); 1487 rte_prefetch0(CQE_PTR_OFF(cq0, 0, 64, flags)); 1488 rte_prefetch0(CQE_PTR_OFF(cq0, 1, 64, flags)); 1489 rte_prefetch0(CQE_PTR_OFF(cq0, 2, 64, flags)); 1490 rte_prefetch0(CQE_PTR_OFF(cq0, 3, 64, flags)); 1491 } else { 1492 RTE_SET_USED(head); 1493 } 1494 1495 if (flags & NIX_RX_OFFLOAD_SECURITY_F) { 1496 if (flags & NIX_RX_VWQE_F) { 1497 uint64_t sg_w1; 1498 uint16_t port; 1499 1500 mbuf0 = (struct rte_mbuf *)((uintptr_t)mbufs[0] - 1501 sizeof(struct rte_mbuf)); 1502 /* Pick first mbuf's aura handle assuming all 1503 * mbufs are from a vec and are from same RQ. 1504 */ 1505 if (!meta_aura) 1506 meta_aura = mbuf0->pool->pool_id; 1507 /* Calculate offset from mbuf to actual data area */ 1508 /* Zero aura's first skip i.e mbuf setup might not match the actual 1509 * offset as first skip is taken from second pass RQ. So compute 1510 * using diff b/w first SG pointer and mbuf addr. 1511 */ 1512 sg_w1 = *(uint64_t *)((uintptr_t)mbufs[0] + 72); 1513 d_off = (sg_w1 - (uint64_t)mbuf0); 1514 1515 /* Get SA Base from lookup tbl using port_id */ 1516 port = mbuf_initializer >> 48; 1517 sa_base = cnxk_nix_sa_base_get(port, lookup_mem); 1518 if (flags & NIX_RX_REAS_F) 1519 meta_pool = (struct rte_mempool *)cnxk_nix_inl_metapool_get(port, 1520 lookup_mem); 1521 1522 lbase = lmt_base; 1523 } else { 1524 meta_aura = rxq->meta_aura; 1525 d_off = rxq->data_off; 1526 sa_base = rxq->sa_base; 1527 lbase = rxq->lmt_base; 1528 if (flags & NIX_RX_REAS_F) 1529 meta_pool = (struct rte_mempool *)rxq->meta_pool; 1530 } 1531 sa_base &= ~(ROC_NIX_INL_SA_BASE_ALIGN - 1); 1532 ROC_LMT_BASE_ID_GET(lbase, lmt_id); 1533 lnum = 0; 1534 laddr = lbase; 1535 laddr += 8; 1536 } 1537 1538 while (packets < pkts) { 1539 if (!(flags & NIX_RX_VWQE_F)) { 1540 /* Exit loop if head is about to wrap and become 1541 * unaligned. 1542 */ 1543 if (((head + NIX_DESCS_PER_LOOP - 1) & qmask) < 1544 NIX_DESCS_PER_LOOP) { 1545 pkts_left += (pkts - packets); 1546 break; 1547 } 1548 1549 cq0 = desc + CQE_SZ(head); 1550 } else { 1551 cq0 = (uintptr_t)&mbufs[packets]; 1552 } 1553 1554 if (flags & NIX_RX_VWQE_F) { 1555 if (pkts - packets > 4) { 1556 rte_prefetch_non_temporal(CQE_PTR_OFF(cq0, 1557 4, 0, flags)); 1558 rte_prefetch_non_temporal(CQE_PTR_OFF(cq0, 1559 5, 0, flags)); 1560 rte_prefetch_non_temporal(CQE_PTR_OFF(cq0, 1561 6, 0, flags)); 1562 rte_prefetch_non_temporal(CQE_PTR_OFF(cq0, 1563 7, 0, flags)); 1564 1565 if (likely(pkts - packets > 8)) { 1566 rte_prefetch1(CQE_PTR_OFF(cq0, 1567 8, 0, flags)); 1568 rte_prefetch1(CQE_PTR_OFF(cq0, 1569 9, 0, flags)); 1570 rte_prefetch1(CQE_PTR_OFF(cq0, 1571 10, 0, flags)); 1572 rte_prefetch1(CQE_PTR_OFF(cq0, 1573 11, 0, flags)); 1574 if (pkts - packets > 12) { 1575 rte_prefetch1(CQE_PTR_OFF(cq0, 1576 12, 0, flags)); 1577 rte_prefetch1(CQE_PTR_OFF(cq0, 1578 13, 0, flags)); 1579 rte_prefetch1(CQE_PTR_OFF(cq0, 1580 14, 0, flags)); 1581 rte_prefetch1(CQE_PTR_OFF(cq0, 1582 15, 0, flags)); 1583 } 1584 } 1585 1586 rte_prefetch0(CQE_PTR_DIFF(cq0, 1587 4, RTE_PKTMBUF_HEADROOM, flags)); 1588 rte_prefetch0(CQE_PTR_DIFF(cq0, 1589 5, RTE_PKTMBUF_HEADROOM, flags)); 1590 rte_prefetch0(CQE_PTR_DIFF(cq0, 1591 6, RTE_PKTMBUF_HEADROOM, flags)); 1592 rte_prefetch0(CQE_PTR_DIFF(cq0, 1593 7, RTE_PKTMBUF_HEADROOM, flags)); 1594 1595 if (likely(pkts - packets > 8)) { 1596 rte_prefetch0(CQE_PTR_DIFF(cq0, 1597 8, RTE_PKTMBUF_HEADROOM, flags)); 1598 rte_prefetch0(CQE_PTR_DIFF(cq0, 1599 9, RTE_PKTMBUF_HEADROOM, flags)); 1600 rte_prefetch0(CQE_PTR_DIFF(cq0, 1601 10, RTE_PKTMBUF_HEADROOM, flags)); 1602 rte_prefetch0(CQE_PTR_DIFF(cq0, 1603 11, RTE_PKTMBUF_HEADROOM, flags)); 1604 } 1605 } 1606 } else { 1607 if (flags & NIX_RX_OFFLOAD_SECURITY_F && 1608 pkts - packets > 4) { 1609 /* Fetch cpt parse header */ 1610 void *p0 = 1611 (void *)*CQE_PTR_OFF(cq0, 4, 72, flags); 1612 void *p1 = 1613 (void *)*CQE_PTR_OFF(cq0, 5, 72, flags); 1614 void *p2 = 1615 (void *)*CQE_PTR_OFF(cq0, 6, 72, flags); 1616 void *p3 = 1617 (void *)*CQE_PTR_OFF(cq0, 7, 72, flags); 1618 rte_prefetch0(p0); 1619 rte_prefetch0(p1); 1620 rte_prefetch0(p2); 1621 rte_prefetch0(p3); 1622 } 1623 1624 if (pkts - packets > 8) { 1625 if (flags) { 1626 rte_prefetch0(CQE_PTR_OFF(cq0, 8, 0, flags)); 1627 rte_prefetch0(CQE_PTR_OFF(cq0, 9, 0, flags)); 1628 rte_prefetch0(CQE_PTR_OFF(cq0, 10, 0, flags)); 1629 rte_prefetch0(CQE_PTR_OFF(cq0, 11, 0, flags)); 1630 } 1631 rte_prefetch0(CQE_PTR_OFF(cq0, 8, 64, flags)); 1632 rte_prefetch0(CQE_PTR_OFF(cq0, 9, 64, flags)); 1633 rte_prefetch0(CQE_PTR_OFF(cq0, 10, 64, flags)); 1634 rte_prefetch0(CQE_PTR_OFF(cq0, 11, 64, flags)); 1635 } 1636 } 1637 1638 if (!(flags & NIX_RX_VWQE_F)) { 1639 /* Get NIX_RX_SG_S for size and buffer pointer */ 1640 cq0_w8 = vld1q_u64(CQE_PTR_OFF(cq0, 0, 64, flags)); 1641 cq1_w8 = vld1q_u64(CQE_PTR_OFF(cq0, 1, 64, flags)); 1642 cq2_w8 = vld1q_u64(CQE_PTR_OFF(cq0, 2, 64, flags)); 1643 cq3_w8 = vld1q_u64(CQE_PTR_OFF(cq0, 3, 64, flags)); 1644 1645 /* Extract mbuf from NIX_RX_SG_S */ 1646 mbuf01 = vzip2q_u64(cq0_w8, cq1_w8); 1647 mbuf23 = vzip2q_u64(cq2_w8, cq3_w8); 1648 mbuf01 = vqsubq_u64(mbuf01, data_off); 1649 mbuf23 = vqsubq_u64(mbuf23, data_off); 1650 } else { 1651 mbuf01 = 1652 vsubq_u64(vld1q_u64((uint64_t *)cq0), 1653 vdupq_n_u64(sizeof(struct rte_mbuf))); 1654 mbuf23 = 1655 vsubq_u64(vld1q_u64((uint64_t *)(cq0 + 16)), 1656 vdupq_n_u64(sizeof(struct rte_mbuf))); 1657 } 1658 1659 /* Move mbufs to scalar registers for future use */ 1660 mbuf0 = (struct rte_mbuf *)vgetq_lane_u64(mbuf01, 0); 1661 mbuf1 = (struct rte_mbuf *)vgetq_lane_u64(mbuf01, 1); 1662 mbuf2 = (struct rte_mbuf *)vgetq_lane_u64(mbuf23, 0); 1663 mbuf3 = (struct rte_mbuf *)vgetq_lane_u64(mbuf23, 1); 1664 1665 /* Mark mempool obj as "get" as it is alloc'ed by NIX */ 1666 RTE_MEMPOOL_CHECK_COOKIES(mbuf0->pool, (void **)&mbuf0, 1, 1); 1667 RTE_MEMPOOL_CHECK_COOKIES(mbuf1->pool, (void **)&mbuf1, 1, 1); 1668 RTE_MEMPOOL_CHECK_COOKIES(mbuf2->pool, (void **)&mbuf2, 1, 1); 1669 RTE_MEMPOOL_CHECK_COOKIES(mbuf3->pool, (void **)&mbuf3, 1, 1); 1670 1671 if (!(flags & NIX_RX_VWQE_F)) { 1672 /* Mask to get packet len from NIX_RX_SG_S */ 1673 const uint8x16_t shuf_msk = { 1674 0xFF, 0xFF, /* pkt_type set as unknown */ 1675 0xFF, 0xFF, /* pkt_type set as unknown */ 1676 0, 1, /* octet 1~0, low 16 bits pkt_len */ 1677 0xFF, 0xFF, /* skip high 16it pkt_len, zero out */ 1678 0, 1, /* octet 1~0, 16 bits data_len */ 1679 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}; 1680 1681 /* Form the rx_descriptor_fields1 with pkt_len and data_len */ 1682 f0 = vqtbl1q_u8(cq0_w8, shuf_msk); 1683 f1 = vqtbl1q_u8(cq1_w8, shuf_msk); 1684 f2 = vqtbl1q_u8(cq2_w8, shuf_msk); 1685 f3 = vqtbl1q_u8(cq3_w8, shuf_msk); 1686 } 1687 1688 /* Load CQE word0 and word 1 */ 1689 const uint64_t cq0_w0 = *CQE_PTR_OFF(cq0, 0, 0, flags); 1690 const uint64_t cq0_w1 = *CQE_PTR_OFF(cq0, 0, 8, flags); 1691 const uint64_t cq0_w2 = *CQE_PTR_OFF(cq0, 0, 16, flags); 1692 const uint64_t cq1_w0 = *CQE_PTR_OFF(cq0, 1, 0, flags); 1693 const uint64_t cq1_w1 = *CQE_PTR_OFF(cq0, 1, 8, flags); 1694 const uint64_t cq1_w2 = *CQE_PTR_OFF(cq0, 1, 16, flags); 1695 const uint64_t cq2_w0 = *CQE_PTR_OFF(cq0, 2, 0, flags); 1696 const uint64_t cq2_w1 = *CQE_PTR_OFF(cq0, 2, 8, flags); 1697 const uint64_t cq2_w2 = *CQE_PTR_OFF(cq0, 2, 16, flags); 1698 const uint64_t cq3_w0 = *CQE_PTR_OFF(cq0, 3, 0, flags); 1699 const uint64_t cq3_w1 = *CQE_PTR_OFF(cq0, 3, 8, flags); 1700 const uint64_t cq3_w2 = *CQE_PTR_OFF(cq0, 3, 16, flags); 1701 1702 if (flags & NIX_RX_VWQE_F) { 1703 uint16_t psize0, psize1, psize2, psize3; 1704 1705 psize0 = (cq0_w2 & 0xFFFF) + 1; 1706 psize1 = (cq1_w2 & 0xFFFF) + 1; 1707 psize2 = (cq2_w2 & 0xFFFF) + 1; 1708 psize3 = (cq3_w2 & 0xFFFF) + 1; 1709 1710 f0 = vdupq_n_u64(0); 1711 f1 = vdupq_n_u64(0); 1712 f2 = vdupq_n_u64(0); 1713 f3 = vdupq_n_u64(0); 1714 1715 f0 = vsetq_lane_u16(psize0, f0, 2); 1716 f0 = vsetq_lane_u16(psize0, f0, 4); 1717 1718 f1 = vsetq_lane_u16(psize1, f1, 2); 1719 f1 = vsetq_lane_u16(psize1, f1, 4); 1720 1721 f2 = vsetq_lane_u16(psize2, f2, 2); 1722 f2 = vsetq_lane_u16(psize2, f2, 4); 1723 1724 f3 = vsetq_lane_u16(psize3, f3, 2); 1725 f3 = vsetq_lane_u16(psize3, f3, 4); 1726 } 1727 1728 if (flags & NIX_RX_OFFLOAD_RSS_F) { 1729 /* Fill rss in the rx_descriptor_fields1 */ 1730 f0 = vsetq_lane_u32(cq0_w0, f0, 3); 1731 f1 = vsetq_lane_u32(cq1_w0, f1, 3); 1732 f2 = vsetq_lane_u32(cq2_w0, f2, 3); 1733 f3 = vsetq_lane_u32(cq3_w0, f3, 3); 1734 ol_flags0 = RTE_MBUF_F_RX_RSS_HASH; 1735 ol_flags1 = RTE_MBUF_F_RX_RSS_HASH; 1736 ol_flags2 = RTE_MBUF_F_RX_RSS_HASH; 1737 ol_flags3 = RTE_MBUF_F_RX_RSS_HASH; 1738 } else { 1739 ol_flags0 = 0; 1740 ol_flags1 = 0; 1741 ol_flags2 = 0; 1742 ol_flags3 = 0; 1743 } 1744 1745 if (flags & NIX_RX_OFFLOAD_PTYPE_F) { 1746 /* Fill packet_type in the rx_descriptor_fields1 */ 1747 f0 = vsetq_lane_u32(nix_ptype_get(lookup_mem, cq0_w1), 1748 f0, 0); 1749 f1 = vsetq_lane_u32(nix_ptype_get(lookup_mem, cq1_w1), 1750 f1, 0); 1751 f2 = vsetq_lane_u32(nix_ptype_get(lookup_mem, cq2_w1), 1752 f2, 0); 1753 f3 = vsetq_lane_u32(nix_ptype_get(lookup_mem, cq3_w1), 1754 f3, 0); 1755 } 1756 1757 if (flags & NIX_RX_OFFLOAD_CHECKSUM_F) { 1758 ol_flags0 |= (uint64_t)nix_rx_olflags_get(lookup_mem, cq0_w1); 1759 ol_flags1 |= (uint64_t)nix_rx_olflags_get(lookup_mem, cq1_w1); 1760 ol_flags2 |= (uint64_t)nix_rx_olflags_get(lookup_mem, cq2_w1); 1761 ol_flags3 |= (uint64_t)nix_rx_olflags_get(lookup_mem, cq3_w1); 1762 } 1763 1764 /* Translate meta to mbuf */ 1765 if (flags & NIX_RX_OFFLOAD_SECURITY_F) { 1766 uint64_t cq0_w5 = *CQE_PTR_OFF(cq0, 0, 40, flags); 1767 uint64_t cq1_w5 = *CQE_PTR_OFF(cq0, 1, 40, flags); 1768 uint64_t cq2_w5 = *CQE_PTR_OFF(cq0, 2, 40, flags); 1769 uint64_t cq3_w5 = *CQE_PTR_OFF(cq0, 3, 40, flags); 1770 uint8_t code; 1771 1772 uint64x2_t inner0, inner1, inner2, inner3; 1773 uint64x2_t wqe01, wqe23, sa01, sa23; 1774 uint16x4_t lens, l2lens, ltypes; 1775 uint64x2_t mask01, mask23; 1776 uint8x8_t ucc; 1777 1778 cpth0 = (uintptr_t)mbuf0 + d_off; 1779 cpth1 = (uintptr_t)mbuf1 + d_off; 1780 cpth2 = (uintptr_t)mbuf2 + d_off; 1781 cpth3 = (uintptr_t)mbuf3 + d_off; 1782 1783 inner0 = vld1q_u64((const uint64_t *)cpth0); 1784 inner1 = vld1q_u64((const uint64_t *)cpth1); 1785 inner2 = vld1q_u64((const uint64_t *)cpth2); 1786 inner3 = vld1q_u64((const uint64_t *)cpth3); 1787 1788 /* Extract and reverse wqe pointers */ 1789 wqe01 = vzip2q_u64(inner0, inner1); 1790 wqe23 = vzip2q_u64(inner2, inner3); 1791 wqe01 = vrev64q_u8(wqe01); 1792 wqe23 = vrev64q_u8(wqe23); 1793 /* Adjust wqe pointers to point to mbuf */ 1794 wqe01 = vsubq_u64(wqe01, 1795 vdupq_n_u64(sizeof(struct rte_mbuf))); 1796 wqe23 = vsubq_u64(wqe23, 1797 vdupq_n_u64(sizeof(struct rte_mbuf))); 1798 1799 /* Extract sa idx from cookie area and add to sa_base */ 1800 sa01 = vzip1q_u64(inner0, inner1); 1801 sa23 = vzip1q_u64(inner2, inner3); 1802 1803 sa01 = vshrq_n_u64(sa01, 32); 1804 sa23 = vshrq_n_u64(sa23, 32); 1805 1806 /* Crypto Look-aside Rx Inject case */ 1807 mask01 = vceqq_u64(sa01, vdupq_n_u64(0xFFFFFFFF)); 1808 mask23 = vceqq_u64(sa23, vdupq_n_u64(0xFFFFFFFF)); 1809 1810 sa01 = vshlq_n_u64(sa01, 1811 ROC_NIX_INL_OT_IPSEC_INB_SA_SZ_LOG2); 1812 sa23 = vshlq_n_u64(sa23, 1813 ROC_NIX_INL_OT_IPSEC_INB_SA_SZ_LOG2); 1814 sa01 = vaddq_u64(sa01, vdupq_n_u64(sa_base)); 1815 sa23 = vaddq_u64(sa23, vdupq_n_u64(sa_base)); 1816 1817 if (flags & NIX_RX_REAS_F) { 1818 sa01 = vbicq_u64(sa01, mask01); 1819 sa23 = vbicq_u64(sa23, mask23); 1820 } 1821 1822 const uint8x16x2_t tbl = {{ 1823 { 1824 /* ROC_IE_OT_UCC_SUCCESS_PKT_IP_BADCSUM */ 1825 RTE_MBUF_F_RX_IP_CKSUM_BAD >> 1, 1826 /* ROC_IE_OT_UCC_SUCCESS_PKT_L4_GOODCSUM */ 1827 (RTE_MBUF_F_RX_IP_CKSUM_GOOD | 1828 RTE_MBUF_F_RX_L4_CKSUM_GOOD) >> 1, 1829 /* ROC_IE_OT_UCC_SUCCESS_PKT_L4_BADCSUM */ 1830 (RTE_MBUF_F_RX_IP_CKSUM_GOOD | 1831 RTE_MBUF_F_RX_L4_CKSUM_BAD) >> 1, 1832 1, 1833 /* ROC_IE_OT_UCC_SUCCESS_PKT_UDPESP_NZCSUM */ 1834 (RTE_MBUF_F_RX_IP_CKSUM_GOOD | 1835 RTE_MBUF_F_RX_L4_CKSUM_GOOD) >> 1, 1836 1, 1837 /* ROC_IE_OT_UCC_SUCCESS_PKT_UDP_ZEROCSUM */ 1838 (RTE_MBUF_F_RX_IP_CKSUM_GOOD | 1839 RTE_MBUF_F_RX_L4_CKSUM_GOOD) >> 1, 1840 3, 1, 3, 3, 3, 3, 1, 3, 1, 1841 }, 1842 { 1843 1, 1, 1, 1844 /* ROC_IE_OT_UCC_SUCCESS_PKT_IP_GOODCSUM */ 1845 RTE_MBUF_F_RX_IP_CKSUM_GOOD >> 1, 1846 /* Rest 0 to indicate RTE_MBUF_F_RX_SEC_OFFLOAD_FAILED */ 1847 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1848 }, 1849 }}; 1850 1851 const uint8x8_t err_off = { 1852 /* UCC */ 1853 0xED, 1854 /* HW_CCODE 0:6 -> 7:D */ 1855 -7, 1856 0xED, 1857 -7, 1858 0xED, 1859 -7, 1860 0xED, 1861 -7, 1862 }; 1863 1864 ucc = vdup_n_u8(0); 1865 ucc = vset_lane_u16(*(uint16_t *)(cpth0 + 30), ucc, 0); 1866 ucc = vset_lane_u16(*(uint16_t *)(cpth1 + 30), ucc, 1); 1867 ucc = vset_lane_u16(*(uint16_t *)(cpth2 + 30), ucc, 2); 1868 ucc = vset_lane_u16(*(uint16_t *)(cpth3 + 30), ucc, 3); 1869 ucc = vsub_u8(ucc, err_off); 1870 1871 /* Table lookup to get the corresponding flags, Out of the range 1872 * from this lookup will have value 0 and consider as 1873 * RTE_MBUF_F_RX_SEC_OFFLOAD_FAILED. 1874 */ 1875 ucc = vqtbl2_u8(tbl, ucc); 1876 1877 RTE_BUILD_BUG_ON(NPC_LT_LC_IP != 2); 1878 RTE_BUILD_BUG_ON(NPC_LT_LC_IP_OPT != 3); 1879 RTE_BUILD_BUG_ON(NPC_LT_LC_IP6 != 4); 1880 RTE_BUILD_BUG_ON(NPC_LT_LC_IP6_EXT != 5); 1881 1882 ltypes = vdup_n_u16(0); 1883 ltypes = vset_lane_u16((cq0_w1 >> 40) & 0x6, ltypes, 0); 1884 ltypes = vset_lane_u16((cq1_w1 >> 40) & 0x6, ltypes, 1); 1885 ltypes = vset_lane_u16((cq2_w1 >> 40) & 0x6, ltypes, 2); 1886 ltypes = vset_lane_u16((cq3_w1 >> 40) & 0x6, ltypes, 3); 1887 1888 /* Extract and reverse l3 length from IPv4/IPv6 hdr 1889 * that is in same cacheline most probably as cpth. 1890 */ 1891 cpth0 += ((cq0_w5 >> 16) & 0xFF) + 1892 vget_lane_u16(ltypes, 0); 1893 cpth1 += ((cq1_w5 >> 16) & 0xFF) + 1894 vget_lane_u16(ltypes, 1); 1895 cpth2 += ((cq2_w5 >> 16) & 0xFF) + 1896 vget_lane_u16(ltypes, 2); 1897 cpth3 += ((cq3_w5 >> 16) & 0xFF) + 1898 vget_lane_u16(ltypes, 3); 1899 lens = vdup_n_u16(0); 1900 lens = vset_lane_u16(*(uint16_t *)cpth0, lens, 0); 1901 lens = vset_lane_u16(*(uint16_t *)cpth1, lens, 1); 1902 lens = vset_lane_u16(*(uint16_t *)cpth2, lens, 2); 1903 lens = vset_lane_u16(*(uint16_t *)cpth3, lens, 3); 1904 lens = vrev16_u8(lens); 1905 1906 /* Add l2 length to l3 lengths */ 1907 l2lens = vdup_n_u16(0); 1908 l2lens = vset_lane_u16(((cq0_w5 >> 16) & 0xFF) - 1909 (cq0_w5 & 0xFF), 1910 l2lens, 0); 1911 l2lens = vset_lane_u16(((cq1_w5 >> 16) & 0xFF) - 1912 (cq1_w5 & 0xFF), 1913 l2lens, 1); 1914 l2lens = vset_lane_u16(((cq2_w5 >> 16) & 0xFF) - 1915 (cq2_w5 & 0xFF), 1916 l2lens, 2); 1917 l2lens = vset_lane_u16(((cq3_w5 >> 16) & 0xFF) - 1918 (cq3_w5 & 0xFF), 1919 l2lens, 3); 1920 lens = vadd_u16(lens, l2lens); 1921 1922 /* L3 header adjust */ 1923 const int8x8_t l3adj = { 1924 0, 0, 0, 0, 40, 0, 0, 0, 1925 }; 1926 lens = vadd_u16(lens, vtbl1_u8(l3adj, ltypes)); 1927 1928 /* Initialize rearm data when reassembly is enabled as 1929 * data offset might change. 1930 */ 1931 if (flags & NIX_RX_REAS_F) { 1932 rearm0 = vdupq_n_u64(mbuf_initializer); 1933 rearm1 = vdupq_n_u64(mbuf_initializer); 1934 rearm2 = vdupq_n_u64(mbuf_initializer); 1935 rearm3 = vdupq_n_u64(mbuf_initializer); 1936 } 1937 1938 /* Checksum ol_flags will be cleared if mbuf is meta */ 1939 if (cq0_w1 & BIT(11)) { 1940 uintptr_t wqe = vgetq_lane_u64(wqe01, 0); 1941 uintptr_t sa = vgetq_lane_u64(sa01, 0); 1942 uint16_t len = vget_lane_u16(lens, 0); 1943 1944 cpth0 = (uintptr_t)mbuf0 + d_off; 1945 1946 /* Free meta to aura */ 1947 if (!(flags & NIX_RX_REAS_F) || 1948 *(uint64_t *)cpth0 & BIT_ULL(15)) { 1949 /* Free meta to aura */ 1950 NIX_PUSH_META_TO_FREE(mbuf0, laddr, 1951 &loff); 1952 mbuf01 = vsetq_lane_u64(wqe, mbuf01, 0); 1953 mbuf0 = (struct rte_mbuf *)wqe; 1954 } else if (flags & NIX_RX_REAS_F) { 1955 /* Update meta pool for full mode pkts */ 1956 mbuf0->pool = meta_pool; 1957 } 1958 1959 /* Update pkt_len and data_len */ 1960 f0 = vsetq_lane_u16(len, f0, 2); 1961 f0 = vsetq_lane_u16(len, f0, 4); 1962 1963 nix_sec_meta_to_mbuf(cq0_w1, cq0_w5, sa, cpth0, 1964 mbuf0, &f0, &ol_flags0, 1965 flags, &rearm0); 1966 code = vget_lane_u8(ucc, 0); 1967 ol_flags0 |= code ? (code > 1 ? ((uint64_t)code) << 1 : 0) : 1968 RTE_MBUF_F_RX_SEC_OFFLOAD_FAILED; 1969 1970 ol_flags0 |= ((uint64_t)(vget_lane_u8(ucc, 1)) << 18); 1971 } 1972 1973 if (cq1_w1 & BIT(11)) { 1974 uintptr_t wqe = vgetq_lane_u64(wqe01, 1); 1975 uintptr_t sa = vgetq_lane_u64(sa01, 1); 1976 uint16_t len = vget_lane_u16(lens, 1); 1977 1978 cpth1 = (uintptr_t)mbuf1 + d_off; 1979 1980 /* Free meta to aura */ 1981 if (!(flags & NIX_RX_REAS_F) || 1982 *(uint64_t *)cpth1 & BIT_ULL(15)) { 1983 NIX_PUSH_META_TO_FREE(mbuf1, laddr, 1984 &loff); 1985 mbuf01 = vsetq_lane_u64(wqe, mbuf01, 1); 1986 mbuf1 = (struct rte_mbuf *)wqe; 1987 } else if (flags & NIX_RX_REAS_F) { 1988 /* Update meta pool for full mode pkts */ 1989 mbuf1->pool = meta_pool; 1990 } 1991 1992 /* Update pkt_len and data_len */ 1993 f1 = vsetq_lane_u16(len, f1, 2); 1994 f1 = vsetq_lane_u16(len, f1, 4); 1995 1996 nix_sec_meta_to_mbuf(cq1_w1, cq1_w5, sa, cpth1, 1997 mbuf1, &f1, &ol_flags1, 1998 flags, &rearm1); 1999 code = vget_lane_u8(ucc, 2); 2000 ol_flags1 |= code ? (code > 1 ? ((uint64_t)code) << 1 : 0) : 2001 RTE_MBUF_F_RX_SEC_OFFLOAD_FAILED; 2002 ol_flags1 |= ((uint64_t)(vget_lane_u8(ucc, 3)) << 18); 2003 } 2004 2005 if (cq2_w1 & BIT(11)) { 2006 uintptr_t wqe = vgetq_lane_u64(wqe23, 0); 2007 uintptr_t sa = vgetq_lane_u64(sa23, 0); 2008 uint16_t len = vget_lane_u16(lens, 2); 2009 2010 cpth2 = (uintptr_t)mbuf2 + d_off; 2011 2012 /* Free meta to aura */ 2013 if (!(flags & NIX_RX_REAS_F) || 2014 *(uint64_t *)cpth2 & BIT_ULL(15)) { 2015 NIX_PUSH_META_TO_FREE(mbuf2, laddr, 2016 &loff); 2017 mbuf23 = vsetq_lane_u64(wqe, mbuf23, 0); 2018 mbuf2 = (struct rte_mbuf *)wqe; 2019 } else if (flags & NIX_RX_REAS_F) { 2020 /* Update meta pool for full mode pkts */ 2021 mbuf2->pool = meta_pool; 2022 } 2023 2024 /* Update pkt_len and data_len */ 2025 f2 = vsetq_lane_u16(len, f2, 2); 2026 f2 = vsetq_lane_u16(len, f2, 4); 2027 2028 nix_sec_meta_to_mbuf(cq2_w1, cq2_w5, sa, cpth2, 2029 mbuf2, &f2, &ol_flags2, 2030 flags, &rearm2); 2031 code = vget_lane_u8(ucc, 4); 2032 ol_flags2 |= code ? (code > 1 ? ((uint64_t)code) << 1 : 0) : 2033 RTE_MBUF_F_RX_SEC_OFFLOAD_FAILED; 2034 ol_flags2 |= ((uint64_t)(vget_lane_u8(ucc, 5)) << 18); 2035 } 2036 2037 if (cq3_w1 & BIT(11)) { 2038 uintptr_t wqe = vgetq_lane_u64(wqe23, 1); 2039 uintptr_t sa = vgetq_lane_u64(sa23, 1); 2040 uint16_t len = vget_lane_u16(lens, 3); 2041 2042 cpth3 = (uintptr_t)mbuf3 + d_off; 2043 2044 /* Free meta to aura */ 2045 if (!(flags & NIX_RX_REAS_F) || 2046 *(uint64_t *)cpth3 & BIT_ULL(15)) { 2047 NIX_PUSH_META_TO_FREE(mbuf3, laddr, 2048 &loff); 2049 mbuf23 = vsetq_lane_u64(wqe, mbuf23, 1); 2050 mbuf3 = (struct rte_mbuf *)wqe; 2051 } else if (flags & NIX_RX_REAS_F) { 2052 /* Update meta pool for full mode pkts */ 2053 mbuf3->pool = meta_pool; 2054 } 2055 2056 /* Update pkt_len and data_len */ 2057 f3 = vsetq_lane_u16(len, f3, 2); 2058 f3 = vsetq_lane_u16(len, f3, 4); 2059 2060 nix_sec_meta_to_mbuf(cq3_w1, cq3_w5, sa, cpth3, 2061 mbuf3, &f3, &ol_flags3, 2062 flags, &rearm3); 2063 code = vget_lane_u8(ucc, 6); 2064 ol_flags3 |= code ? (code > 1 ? ((uint64_t)code) << 1 : 0) : 2065 RTE_MBUF_F_RX_SEC_OFFLOAD_FAILED; 2066 ol_flags3 |= ((uint64_t)(vget_lane_u8(ucc, 7)) << 18); 2067 } 2068 } 2069 2070 if (flags & NIX_RX_OFFLOAD_VLAN_STRIP_F) { 2071 2072 ol_flags0 = nix_vlan_update(cq0_w2, ol_flags0, &f0); 2073 ol_flags1 = nix_vlan_update(cq1_w2, ol_flags1, &f1); 2074 ol_flags2 = nix_vlan_update(cq2_w2, ol_flags2, &f2); 2075 ol_flags3 = nix_vlan_update(cq3_w2, ol_flags3, &f3); 2076 2077 ol_flags0 = nix_qinq_update(cq0_w2, ol_flags0, mbuf0); 2078 ol_flags1 = nix_qinq_update(cq1_w2, ol_flags1, mbuf1); 2079 ol_flags2 = nix_qinq_update(cq2_w2, ol_flags2, mbuf2); 2080 ol_flags3 = nix_qinq_update(cq3_w2, ol_flags3, mbuf3); 2081 } 2082 2083 if (flags & NIX_RX_OFFLOAD_MARK_UPDATE_F) { 2084 ol_flags0 = nix_update_match_id( 2085 *(uint16_t *)CQE_PTR_OFF(cq0, 0, 38, flags), 2086 ol_flags0, mbuf0); 2087 ol_flags1 = nix_update_match_id( 2088 *(uint16_t *)CQE_PTR_OFF(cq0, 1, 38, flags), 2089 ol_flags1, mbuf1); 2090 ol_flags2 = nix_update_match_id( 2091 *(uint16_t *)CQE_PTR_OFF(cq0, 2, 38, flags), 2092 ol_flags2, mbuf2); 2093 ol_flags3 = nix_update_match_id( 2094 *(uint16_t *)CQE_PTR_OFF(cq0, 3, 38, flags), 2095 ol_flags3, mbuf3); 2096 } 2097 2098 if ((flags & NIX_RX_OFFLOAD_TSTAMP_F) && 2099 ((flags & NIX_RX_VWQE_F) && tstamp)) { 2100 const uint16x8_t len_off = { 2101 0, /* ptype 0:15 */ 2102 0, /* ptype 16:32 */ 2103 CNXK_NIX_TIMESYNC_RX_OFFSET, /* pktlen 0:15*/ 2104 0, /* pktlen 16:32 */ 2105 CNXK_NIX_TIMESYNC_RX_OFFSET, /* datalen 0:15 */ 2106 0, 2107 0, 2108 0}; 2109 const uint32x4_t ptype = {RTE_PTYPE_L2_ETHER_TIMESYNC, 2110 RTE_PTYPE_L2_ETHER_TIMESYNC, 2111 RTE_PTYPE_L2_ETHER_TIMESYNC, 2112 RTE_PTYPE_L2_ETHER_TIMESYNC}; 2113 const uint64_t ts_olf = RTE_MBUF_F_RX_IEEE1588_PTP | 2114 RTE_MBUF_F_RX_IEEE1588_TMST | 2115 tstamp->rx_tstamp_dynflag; 2116 const uint32x4_t and_mask = {0x1, 0x2, 0x4, 0x8}; 2117 uint64x2_t ts01, ts23, mask; 2118 uint64_t ts[4]; 2119 uint8_t res; 2120 2121 /* Subtract timesync length from total pkt length. */ 2122 f0 = vsubq_u16(f0, len_off); 2123 f1 = vsubq_u16(f1, len_off); 2124 f2 = vsubq_u16(f2, len_off); 2125 f3 = vsubq_u16(f3, len_off); 2126 2127 /* Get the address of actual timestamp. */ 2128 ts01 = vaddq_u64(mbuf01, data_off); 2129 ts23 = vaddq_u64(mbuf23, data_off); 2130 /* Load timestamp from address. */ 2131 ts01 = vsetq_lane_u64(*(uint64_t *)vgetq_lane_u64(ts01, 2132 0), 2133 ts01, 0); 2134 ts01 = vsetq_lane_u64(*(uint64_t *)vgetq_lane_u64(ts01, 2135 1), 2136 ts01, 1); 2137 ts23 = vsetq_lane_u64(*(uint64_t *)vgetq_lane_u64(ts23, 2138 0), 2139 ts23, 0); 2140 ts23 = vsetq_lane_u64(*(uint64_t *)vgetq_lane_u64(ts23, 2141 1), 2142 ts23, 1); 2143 /* Convert from be to cpu byteorder. */ 2144 ts01 = vrev64q_u8(ts01); 2145 ts23 = vrev64q_u8(ts23); 2146 /* Store timestamp into scalar for later use. */ 2147 ts[0] = vgetq_lane_u64(ts01, 0); 2148 ts[1] = vgetq_lane_u64(ts01, 1); 2149 ts[2] = vgetq_lane_u64(ts23, 0); 2150 ts[3] = vgetq_lane_u64(ts23, 1); 2151 2152 /* Store timestamp into dynfield. */ 2153 *cnxk_nix_timestamp_dynfield(mbuf0, tstamp) = ts[0]; 2154 *cnxk_nix_timestamp_dynfield(mbuf1, tstamp) = ts[1]; 2155 *cnxk_nix_timestamp_dynfield(mbuf2, tstamp) = ts[2]; 2156 *cnxk_nix_timestamp_dynfield(mbuf3, tstamp) = ts[3]; 2157 2158 /* Generate ptype mask to filter L2 ether timesync */ 2159 mask = vdupq_n_u32(vgetq_lane_u32(f0, 0)); 2160 mask = vsetq_lane_u32(vgetq_lane_u32(f1, 0), mask, 1); 2161 mask = vsetq_lane_u32(vgetq_lane_u32(f2, 0), mask, 2); 2162 mask = vsetq_lane_u32(vgetq_lane_u32(f3, 0), mask, 3); 2163 2164 /* Match against L2 ether timesync. */ 2165 mask = vceqq_u32(mask, ptype); 2166 /* Convert from vector from scalar mask */ 2167 res = vaddvq_u32(vandq_u32(mask, and_mask)); 2168 res &= 0xF; 2169 2170 if (res) { 2171 /* Fill in the ol_flags for any packets that 2172 * matched. 2173 */ 2174 ol_flags0 |= ((res & 0x1) ? ts_olf : 0); 2175 ol_flags1 |= ((res & 0x2) ? ts_olf : 0); 2176 ol_flags2 |= ((res & 0x4) ? ts_olf : 0); 2177 ol_flags3 |= ((res & 0x8) ? ts_olf : 0); 2178 2179 /* Update Rxq timestamp with the latest 2180 * timestamp. 2181 */ 2182 tstamp->rx_ready = 1; 2183 tstamp->rx_tstamp = ts[31 - rte_clz32(res)]; 2184 } 2185 } 2186 2187 /* Form rearm_data with ol_flags */ 2188 rearm0 = vsetq_lane_u64(ol_flags0, rearm0, 1); 2189 rearm1 = vsetq_lane_u64(ol_flags1, rearm1, 1); 2190 rearm2 = vsetq_lane_u64(ol_flags2, rearm2, 1); 2191 rearm3 = vsetq_lane_u64(ol_flags3, rearm3, 1); 2192 2193 /* Update rx_descriptor_fields1 */ 2194 vst1q_u64((uint64_t *)mbuf0->rx_descriptor_fields1, f0); 2195 vst1q_u64((uint64_t *)mbuf1->rx_descriptor_fields1, f1); 2196 vst1q_u64((uint64_t *)mbuf2->rx_descriptor_fields1, f2); 2197 vst1q_u64((uint64_t *)mbuf3->rx_descriptor_fields1, f3); 2198 2199 /* Update rearm_data */ 2200 vst1q_u64((uint64_t *)mbuf0->rearm_data, rearm0); 2201 vst1q_u64((uint64_t *)mbuf1->rearm_data, rearm1); 2202 vst1q_u64((uint64_t *)mbuf2->rearm_data, rearm2); 2203 vst1q_u64((uint64_t *)mbuf3->rearm_data, rearm3); 2204 2205 if (flags & NIX_RX_MULTI_SEG_F) { 2206 /* Multi segment is enable build mseg list for 2207 * individual mbufs in scalar mode. 2208 */ 2209 nix_cqe_xtract_mseg((union nix_rx_parse_u *) 2210 (CQE_PTR_OFF(cq0, 0, 8, flags)), 2211 mbuf0, mbuf_initializer, cpth0, sa_base, flags); 2212 nix_cqe_xtract_mseg((union nix_rx_parse_u *) 2213 (CQE_PTR_OFF(cq0, 1, 8, flags)), 2214 mbuf1, mbuf_initializer, cpth1, sa_base, flags); 2215 nix_cqe_xtract_mseg((union nix_rx_parse_u *) 2216 (CQE_PTR_OFF(cq0, 2, 8, flags)), 2217 mbuf2, mbuf_initializer, cpth2, sa_base, flags); 2218 nix_cqe_xtract_mseg((union nix_rx_parse_u *) 2219 (CQE_PTR_OFF(cq0, 3, 8, flags)), 2220 mbuf3, mbuf_initializer, cpth3, sa_base, flags); 2221 } 2222 2223 /* Store the mbufs to rx_pkts */ 2224 vst1q_u64((uint64_t *)&mbufs[packets], mbuf01); 2225 vst1q_u64((uint64_t *)&mbufs[packets + 2], mbuf23); 2226 2227 nix_mbuf_validate_next(mbuf0); 2228 nix_mbuf_validate_next(mbuf1); 2229 nix_mbuf_validate_next(mbuf2); 2230 nix_mbuf_validate_next(mbuf3); 2231 2232 packets += NIX_DESCS_PER_LOOP; 2233 2234 if (!(flags & NIX_RX_VWQE_F)) { 2235 /* Advance head pointer and packets */ 2236 head += NIX_DESCS_PER_LOOP; 2237 head &= qmask; 2238 } 2239 2240 if (flags & NIX_RX_OFFLOAD_SECURITY_F) { 2241 /* Check if lmtline border is crossed and adjust lnum */ 2242 if (loff > 15) { 2243 /* Update aura handle */ 2244 *(uint64_t *)(laddr - 8) = 2245 (((uint64_t)(15 & 0x1) << 32) | 2246 roc_npa_aura_handle_to_aura(meta_aura)); 2247 loff = loff - 15; 2248 shft += 3; 2249 2250 lnum++; 2251 laddr = (uintptr_t)LMT_OFF(lbase, lnum, 8); 2252 /* Pick the pointer from 16th index and put it 2253 * at end of this new line. 2254 */ 2255 *(uint64_t *)(laddr + (loff << 3) - 8) = 2256 *(uint64_t *)(laddr - 8); 2257 } 2258 2259 /* Flush it when we are in 16th line and might 2260 * overflow it 2261 */ 2262 if (lnum >= 15 && loff >= 12) { 2263 /* 16 LMT Line size m1 */ 2264 uint64_t data = BIT_ULL(48) - 1; 2265 2266 /* Update aura handle */ 2267 *(uint64_t *)(laddr - 8) = 2268 (((uint64_t)(loff & 0x1) << 32) | 2269 roc_npa_aura_handle_to_aura(meta_aura)); 2270 2271 data = (data & ~(0x7UL << shft)) | 2272 (((uint64_t)loff >> 1) << shft); 2273 2274 /* Send up to 16 lmt lines of pointers */ 2275 nix_sec_flush_meta_burst(lmt_id, data, lnum + 1, 2276 meta_aura); 2277 rte_io_wmb(); 2278 lnum = 0; 2279 loff = 0; 2280 shft = 0; 2281 /* First pointer starts at 8B offset */ 2282 laddr = (uintptr_t)LMT_OFF(lbase, lnum, 8); 2283 } 2284 } 2285 } 2286 2287 if (flags & NIX_RX_OFFLOAD_SECURITY_F && loff) { 2288 /* 16 LMT Line size m1 */ 2289 uint64_t data = BIT_ULL(48) - 1; 2290 2291 /* Update aura handle */ 2292 *(uint64_t *)(laddr - 8) = 2293 (((uint64_t)(loff & 0x1) << 32) | 2294 roc_npa_aura_handle_to_aura(meta_aura)); 2295 2296 data = (data & ~(0x7UL << shft)) | 2297 (((uint64_t)loff >> 1) << shft); 2298 2299 /* Send up to 16 lmt lines of pointers */ 2300 nix_sec_flush_meta_burst(lmt_id, data, lnum + 1, meta_aura); 2301 if (flags & NIX_RX_VWQE_F) 2302 plt_io_wmb(); 2303 } 2304 2305 if (flags & NIX_RX_VWQE_F) 2306 return packets; 2307 2308 rxq->head = head; 2309 rxq->available -= packets; 2310 2311 rte_io_wmb(); 2312 /* Free all the CQs that we've processed */ 2313 plt_write64((rxq->wdata | packets), rxq->cq_door); 2314 2315 if (unlikely(pkts_left)) 2316 packets += cn10k_nix_recv_pkts(args, &mbufs[packets], pkts_left, 2317 flags); 2318 2319 return packets; 2320 } 2321 2322 #else 2323 2324 static inline uint16_t 2325 cn10k_nix_recv_pkts_vector(void *args, struct rte_mbuf **mbufs, uint16_t pkts, 2326 const uint16_t flags, void *lookup_mem, 2327 struct cnxk_timesync_info *tstamp, 2328 uintptr_t lmt_base, uint64_t meta_aura) 2329 { 2330 RTE_SET_USED(args); 2331 RTE_SET_USED(mbufs); 2332 RTE_SET_USED(pkts); 2333 RTE_SET_USED(flags); 2334 RTE_SET_USED(lookup_mem); 2335 RTE_SET_USED(tstamp); 2336 RTE_SET_USED(lmt_base); 2337 RTE_SET_USED(meta_aura); 2338 2339 return 0; 2340 } 2341 2342 #endif 2343 2344 2345 #define RSS_F NIX_RX_OFFLOAD_RSS_F 2346 #define PTYPE_F NIX_RX_OFFLOAD_PTYPE_F 2347 #define CKSUM_F NIX_RX_OFFLOAD_CHECKSUM_F 2348 #define MARK_F NIX_RX_OFFLOAD_MARK_UPDATE_F 2349 #define TS_F NIX_RX_OFFLOAD_TSTAMP_F 2350 #define RX_VLAN_F NIX_RX_OFFLOAD_VLAN_STRIP_F 2351 #define R_SEC_F NIX_RX_OFFLOAD_SECURITY_F 2352 2353 /* [R_SEC_F] [RX_VLAN_F] [TS] [MARK] [CKSUM] [PTYPE] [RSS] */ 2354 #define NIX_RX_FASTPATH_MODES_0_15 \ 2355 R(no_offload, NIX_RX_OFFLOAD_NONE) \ 2356 R(rss, RSS_F) \ 2357 R(ptype, PTYPE_F) \ 2358 R(ptype_rss, PTYPE_F | RSS_F) \ 2359 R(cksum, CKSUM_F) \ 2360 R(cksum_rss, CKSUM_F | RSS_F) \ 2361 R(cksum_ptype, CKSUM_F | PTYPE_F) \ 2362 R(cksum_ptype_rss, CKSUM_F | PTYPE_F | RSS_F) \ 2363 R(mark, MARK_F) \ 2364 R(mark_rss, MARK_F | RSS_F) \ 2365 R(mark_ptype, MARK_F | PTYPE_F) \ 2366 R(mark_ptype_rss, MARK_F | PTYPE_F | RSS_F) \ 2367 R(mark_cksum, MARK_F | CKSUM_F) \ 2368 R(mark_cksum_rss, MARK_F | CKSUM_F | RSS_F) \ 2369 R(mark_cksum_ptype, MARK_F | CKSUM_F | PTYPE_F) \ 2370 R(mark_cksum_ptype_rss, MARK_F | CKSUM_F | PTYPE_F | RSS_F) 2371 2372 #define NIX_RX_FASTPATH_MODES_16_31 \ 2373 R(ts, TS_F) \ 2374 R(ts_rss, TS_F | RSS_F) \ 2375 R(ts_ptype, TS_F | PTYPE_F) \ 2376 R(ts_ptype_rss, TS_F | PTYPE_F | RSS_F) \ 2377 R(ts_cksum, TS_F | CKSUM_F) \ 2378 R(ts_cksum_rss, TS_F | CKSUM_F | RSS_F) \ 2379 R(ts_cksum_ptype, TS_F | CKSUM_F | PTYPE_F) \ 2380 R(ts_cksum_ptype_rss, TS_F | CKSUM_F | PTYPE_F | RSS_F) \ 2381 R(ts_mark, TS_F | MARK_F) \ 2382 R(ts_mark_rss, TS_F | MARK_F | RSS_F) \ 2383 R(ts_mark_ptype, TS_F | MARK_F | PTYPE_F) \ 2384 R(ts_mark_ptype_rss, TS_F | MARK_F | PTYPE_F | RSS_F) \ 2385 R(ts_mark_cksum, TS_F | MARK_F | CKSUM_F) \ 2386 R(ts_mark_cksum_rss, TS_F | MARK_F | CKSUM_F | RSS_F) \ 2387 R(ts_mark_cksum_ptype, TS_F | MARK_F | CKSUM_F | PTYPE_F) \ 2388 R(ts_mark_cksum_ptype_rss, TS_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F) 2389 2390 #define NIX_RX_FASTPATH_MODES_32_47 \ 2391 R(vlan, RX_VLAN_F) \ 2392 R(vlan_rss, RX_VLAN_F | RSS_F) \ 2393 R(vlan_ptype, RX_VLAN_F | PTYPE_F) \ 2394 R(vlan_ptype_rss, RX_VLAN_F | PTYPE_F | RSS_F) \ 2395 R(vlan_cksum, RX_VLAN_F | CKSUM_F) \ 2396 R(vlan_cksum_rss, RX_VLAN_F | CKSUM_F | RSS_F) \ 2397 R(vlan_cksum_ptype, RX_VLAN_F | CKSUM_F | PTYPE_F) \ 2398 R(vlan_cksum_ptype_rss, RX_VLAN_F | CKSUM_F | PTYPE_F | RSS_F) \ 2399 R(vlan_mark, RX_VLAN_F | MARK_F) \ 2400 R(vlan_mark_rss, RX_VLAN_F | MARK_F | RSS_F) \ 2401 R(vlan_mark_ptype, RX_VLAN_F | MARK_F | PTYPE_F) \ 2402 R(vlan_mark_ptype_rss, RX_VLAN_F | MARK_F | PTYPE_F | RSS_F) \ 2403 R(vlan_mark_cksum, RX_VLAN_F | MARK_F | CKSUM_F) \ 2404 R(vlan_mark_cksum_rss, RX_VLAN_F | MARK_F | CKSUM_F | RSS_F) \ 2405 R(vlan_mark_cksum_ptype, RX_VLAN_F | MARK_F | CKSUM_F | PTYPE_F) \ 2406 R(vlan_mark_cksum_ptype_rss, \ 2407 RX_VLAN_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F) 2408 2409 #define NIX_RX_FASTPATH_MODES_48_63 \ 2410 R(vlan_ts, RX_VLAN_F | TS_F) \ 2411 R(vlan_ts_rss, RX_VLAN_F | TS_F | RSS_F) \ 2412 R(vlan_ts_ptype, RX_VLAN_F | TS_F | PTYPE_F) \ 2413 R(vlan_ts_ptype_rss, RX_VLAN_F | TS_F | PTYPE_F | RSS_F) \ 2414 R(vlan_ts_cksum, RX_VLAN_F | TS_F | CKSUM_F) \ 2415 R(vlan_ts_cksum_rss, RX_VLAN_F | TS_F | CKSUM_F | RSS_F) \ 2416 R(vlan_ts_cksum_ptype, RX_VLAN_F | TS_F | CKSUM_F | PTYPE_F) \ 2417 R(vlan_ts_cksum_ptype_rss, \ 2418 RX_VLAN_F | TS_F | CKSUM_F | PTYPE_F | RSS_F) \ 2419 R(vlan_ts_mark, RX_VLAN_F | TS_F | MARK_F) \ 2420 R(vlan_ts_mark_rss, RX_VLAN_F | TS_F | MARK_F | RSS_F) \ 2421 R(vlan_ts_mark_ptype, RX_VLAN_F | TS_F | MARK_F | PTYPE_F) \ 2422 R(vlan_ts_mark_ptype_rss, RX_VLAN_F | TS_F | MARK_F | PTYPE_F | RSS_F) \ 2423 R(vlan_ts_mark_cksum, RX_VLAN_F | TS_F | MARK_F | CKSUM_F) \ 2424 R(vlan_ts_mark_cksum_rss, RX_VLAN_F | TS_F | MARK_F | CKSUM_F | RSS_F) \ 2425 R(vlan_ts_mark_cksum_ptype, \ 2426 RX_VLAN_F | TS_F | MARK_F | CKSUM_F | PTYPE_F) \ 2427 R(vlan_ts_mark_cksum_ptype_rss, \ 2428 RX_VLAN_F | TS_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F) 2429 2430 #define NIX_RX_FASTPATH_MODES_64_79 \ 2431 R(sec, R_SEC_F) \ 2432 R(sec_rss, R_SEC_F | RSS_F) \ 2433 R(sec_ptype, R_SEC_F | PTYPE_F) \ 2434 R(sec_ptype_rss, R_SEC_F | PTYPE_F | RSS_F) \ 2435 R(sec_cksum, R_SEC_F | CKSUM_F) \ 2436 R(sec_cksum_rss, R_SEC_F | CKSUM_F | RSS_F) \ 2437 R(sec_cksum_ptype, R_SEC_F | CKSUM_F | PTYPE_F) \ 2438 R(sec_cksum_ptype_rss, R_SEC_F | CKSUM_F | PTYPE_F | RSS_F) \ 2439 R(sec_mark, R_SEC_F | MARK_F) \ 2440 R(sec_mark_rss, R_SEC_F | MARK_F | RSS_F) \ 2441 R(sec_mark_ptype, R_SEC_F | MARK_F | PTYPE_F) \ 2442 R(sec_mark_ptype_rss, R_SEC_F | MARK_F | PTYPE_F | RSS_F) \ 2443 R(sec_mark_cksum, R_SEC_F | MARK_F | CKSUM_F) \ 2444 R(sec_mark_cksum_rss, R_SEC_F | MARK_F | CKSUM_F | RSS_F) \ 2445 R(sec_mark_cksum_ptype, R_SEC_F | MARK_F | CKSUM_F | PTYPE_F) \ 2446 R(sec_mark_cksum_ptype_rss, \ 2447 R_SEC_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F) 2448 2449 #define NIX_RX_FASTPATH_MODES_80_95 \ 2450 R(sec_ts, R_SEC_F | TS_F) \ 2451 R(sec_ts_rss, R_SEC_F | TS_F | RSS_F) \ 2452 R(sec_ts_ptype, R_SEC_F | TS_F | PTYPE_F) \ 2453 R(sec_ts_ptype_rss, R_SEC_F | TS_F | PTYPE_F | RSS_F) \ 2454 R(sec_ts_cksum, R_SEC_F | TS_F | CKSUM_F) \ 2455 R(sec_ts_cksum_rss, R_SEC_F | TS_F | CKSUM_F | RSS_F) \ 2456 R(sec_ts_cksum_ptype, R_SEC_F | TS_F | CKSUM_F | PTYPE_F) \ 2457 R(sec_ts_cksum_ptype_rss, R_SEC_F | TS_F | CKSUM_F | PTYPE_F | RSS_F) \ 2458 R(sec_ts_mark, R_SEC_F | TS_F | MARK_F) \ 2459 R(sec_ts_mark_rss, R_SEC_F | TS_F | MARK_F | RSS_F) \ 2460 R(sec_ts_mark_ptype, R_SEC_F | TS_F | MARK_F | PTYPE_F) \ 2461 R(sec_ts_mark_ptype_rss, R_SEC_F | TS_F | MARK_F | PTYPE_F | RSS_F) \ 2462 R(sec_ts_mark_cksum, R_SEC_F | TS_F | MARK_F | CKSUM_F) \ 2463 R(sec_ts_mark_cksum_rss, R_SEC_F | TS_F | MARK_F | CKSUM_F | RSS_F) \ 2464 R(sec_ts_mark_cksum_ptype, \ 2465 R_SEC_F | TS_F | MARK_F | CKSUM_F | PTYPE_F) \ 2466 R(sec_ts_mark_cksum_ptype_rss, \ 2467 R_SEC_F | TS_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F) 2468 2469 #define NIX_RX_FASTPATH_MODES_96_111 \ 2470 R(sec_vlan, R_SEC_F | RX_VLAN_F) \ 2471 R(sec_vlan_rss, R_SEC_F | RX_VLAN_F | RSS_F) \ 2472 R(sec_vlan_ptype, R_SEC_F | RX_VLAN_F | PTYPE_F) \ 2473 R(sec_vlan_ptype_rss, R_SEC_F | RX_VLAN_F | PTYPE_F | RSS_F) \ 2474 R(sec_vlan_cksum, R_SEC_F | RX_VLAN_F | CKSUM_F) \ 2475 R(sec_vlan_cksum_rss, R_SEC_F | RX_VLAN_F | CKSUM_F | RSS_F) \ 2476 R(sec_vlan_cksum_ptype, R_SEC_F | RX_VLAN_F | CKSUM_F | PTYPE_F) \ 2477 R(sec_vlan_cksum_ptype_rss, \ 2478 R_SEC_F | RX_VLAN_F | CKSUM_F | PTYPE_F | RSS_F) \ 2479 R(sec_vlan_mark, R_SEC_F | RX_VLAN_F | MARK_F) \ 2480 R(sec_vlan_mark_rss, R_SEC_F | RX_VLAN_F | MARK_F | RSS_F) \ 2481 R(sec_vlan_mark_ptype, R_SEC_F | RX_VLAN_F | MARK_F | PTYPE_F) \ 2482 R(sec_vlan_mark_ptype_rss, \ 2483 R_SEC_F | RX_VLAN_F | MARK_F | PTYPE_F | RSS_F) \ 2484 R(sec_vlan_mark_cksum, R_SEC_F | RX_VLAN_F | MARK_F | CKSUM_F) \ 2485 R(sec_vlan_mark_cksum_rss, \ 2486 R_SEC_F | RX_VLAN_F | MARK_F | CKSUM_F | RSS_F) \ 2487 R(sec_vlan_mark_cksum_ptype, \ 2488 R_SEC_F | RX_VLAN_F | MARK_F | CKSUM_F | PTYPE_F) \ 2489 R(sec_vlan_mark_cksum_ptype_rss, \ 2490 R_SEC_F | RX_VLAN_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F) 2491 2492 #define NIX_RX_FASTPATH_MODES_112_127 \ 2493 R(sec_vlan_ts, R_SEC_F | RX_VLAN_F | TS_F) \ 2494 R(sec_vlan_ts_rss, R_SEC_F | RX_VLAN_F | TS_F | RSS_F) \ 2495 R(sec_vlan_ts_ptype, R_SEC_F | RX_VLAN_F | TS_F | PTYPE_F) \ 2496 R(sec_vlan_ts_ptype_rss, R_SEC_F | RX_VLAN_F | TS_F | PTYPE_F | RSS_F) \ 2497 R(sec_vlan_ts_cksum, R_SEC_F | RX_VLAN_F | TS_F | CKSUM_F) \ 2498 R(sec_vlan_ts_cksum_rss, R_SEC_F | RX_VLAN_F | TS_F | CKSUM_F | RSS_F) \ 2499 R(sec_vlan_ts_cksum_ptype, \ 2500 R_SEC_F | RX_VLAN_F | TS_F | CKSUM_F | PTYPE_F) \ 2501 R(sec_vlan_ts_cksum_ptype_rss, \ 2502 R_SEC_F | RX_VLAN_F | TS_F | CKSUM_F | PTYPE_F | RSS_F) \ 2503 R(sec_vlan_ts_mark, R_SEC_F | RX_VLAN_F | TS_F | MARK_F) \ 2504 R(sec_vlan_ts_mark_rss, R_SEC_F | RX_VLAN_F | TS_F | MARK_F | RSS_F) \ 2505 R(sec_vlan_ts_mark_ptype, \ 2506 R_SEC_F | RX_VLAN_F | TS_F | MARK_F | PTYPE_F) \ 2507 R(sec_vlan_ts_mark_ptype_rss, \ 2508 R_SEC_F | RX_VLAN_F | TS_F | MARK_F | PTYPE_F | RSS_F) \ 2509 R(sec_vlan_ts_mark_cksum, \ 2510 R_SEC_F | RX_VLAN_F | TS_F | MARK_F | CKSUM_F) \ 2511 R(sec_vlan_ts_mark_cksum_rss, \ 2512 R_SEC_F | RX_VLAN_F | TS_F | MARK_F | CKSUM_F | RSS_F) \ 2513 R(sec_vlan_ts_mark_cksum_ptype, \ 2514 R_SEC_F | RX_VLAN_F | TS_F | MARK_F | CKSUM_F | PTYPE_F) \ 2515 R(sec_vlan_ts_mark_cksum_ptype_rss, \ 2516 R_SEC_F | RX_VLAN_F | TS_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F) 2517 2518 2519 #define NIX_RX_FASTPATH_MODES \ 2520 NIX_RX_FASTPATH_MODES_0_15 \ 2521 NIX_RX_FASTPATH_MODES_16_31 \ 2522 NIX_RX_FASTPATH_MODES_32_47 \ 2523 NIX_RX_FASTPATH_MODES_48_63 \ 2524 NIX_RX_FASTPATH_MODES_64_79 \ 2525 NIX_RX_FASTPATH_MODES_80_95 \ 2526 NIX_RX_FASTPATH_MODES_96_111 \ 2527 NIX_RX_FASTPATH_MODES_112_127 \ 2528 2529 #define R(name, flags) \ 2530 uint16_t __rte_noinline __rte_hot cn10k_nix_recv_pkts_##name( \ 2531 void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts); \ 2532 uint16_t __rte_noinline __rte_hot cn10k_nix_recv_pkts_mseg_##name( \ 2533 void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts); \ 2534 uint16_t __rte_noinline __rte_hot cn10k_nix_recv_pkts_vec_##name( \ 2535 void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts); \ 2536 uint16_t __rte_noinline __rte_hot cn10k_nix_recv_pkts_vec_mseg_##name( \ 2537 void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts); \ 2538 uint16_t __rte_noinline __rte_hot cn10k_nix_recv_pkts_reas_##name( \ 2539 void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts); \ 2540 uint16_t __rte_noinline __rte_hot cn10k_nix_recv_pkts_reas_mseg_##name(\ 2541 void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts); \ 2542 uint16_t __rte_noinline __rte_hot cn10k_nix_recv_pkts_reas_vec_##name( \ 2543 void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts); \ 2544 uint16_t __rte_noinline __rte_hot cn10k_nix_recv_pkts_reas_vec_mseg_##name( \ 2545 void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts); 2546 2547 NIX_RX_FASTPATH_MODES 2548 #undef R 2549 2550 #define NIX_RX_RECV(fn, flags) \ 2551 uint16_t __rte_noinline __rte_hot fn( \ 2552 void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts) \ 2553 { \ 2554 return cn10k_nix_recv_pkts(rx_queue, rx_pkts, pkts, (flags)); \ 2555 } 2556 2557 #define NIX_RX_RECV_MSEG(fn, flags) NIX_RX_RECV(fn, flags | NIX_RX_MULTI_SEG_F) 2558 2559 #define NIX_RX_RECV_VEC(fn, flags) \ 2560 uint16_t __rte_noinline __rte_hot fn( \ 2561 void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts) \ 2562 { \ 2563 return cn10k_nix_recv_pkts_vector(rx_queue, rx_pkts, pkts, \ 2564 (flags), NULL, NULL, 0, 0); \ 2565 } 2566 2567 #define NIX_RX_RECV_VEC_MSEG(fn, flags) \ 2568 NIX_RX_RECV_VEC(fn, flags | NIX_RX_MULTI_SEG_F) 2569 2570 uint16_t __rte_noinline __rte_hot cn10k_nix_recv_pkts_all_offload(void *rx_queue, 2571 struct rte_mbuf **rx_pkts, 2572 uint16_t pkts); 2573 2574 uint16_t __rte_noinline __rte_hot cn10k_nix_recv_pkts_vec_all_offload(void *rx_queue, 2575 struct rte_mbuf **rx_pkts, 2576 uint16_t pkts); 2577 2578 uint16_t __rte_noinline __rte_hot cn10k_nix_recv_pkts_all_offload_tst(void *rx_queue, 2579 struct rte_mbuf **rx_pkts, 2580 uint16_t pkts); 2581 2582 uint16_t __rte_noinline __rte_hot cn10k_nix_recv_pkts_vec_all_offload_tst(void *rx_queue, 2583 struct rte_mbuf **rx_pkts, 2584 uint16_t pkts); 2585 2586 #endif /* __CN10K_RX_H__ */ 2587