1 /* SPDX-License-Identifier: BSD-3-Clause 2 * 3 * Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved. 4 * Copyright 2016-2020 NXP 5 * 6 */ 7 8 #include <time.h> 9 #include <net/if.h> 10 11 #include <rte_mbuf.h> 12 #include <rte_ethdev_driver.h> 13 #include <rte_malloc.h> 14 #include <rte_memcpy.h> 15 #include <rte_string_fns.h> 16 #include <rte_dev.h> 17 18 #include <rte_fslmc.h> 19 #include <fslmc_vfio.h> 20 #include <dpaa2_hw_pvt.h> 21 #include <dpaa2_hw_dpio.h> 22 #include <dpaa2_hw_mempool.h> 23 24 #include "dpaa2_pmd_logs.h" 25 #include "dpaa2_ethdev.h" 26 #include "base/dpaa2_hw_dpni_annot.h" 27 28 static inline uint32_t __rte_hot 29 dpaa2_dev_rx_parse_slow(struct rte_mbuf *mbuf, 30 struct dpaa2_annot_hdr *annotation); 31 32 static void enable_tx_tstamp(struct qbman_fd *fd) __rte_unused; 33 34 static inline rte_mbuf_timestamp_t * 35 dpaa2_timestamp_dynfield(struct rte_mbuf *mbuf) 36 { 37 return RTE_MBUF_DYNFIELD(mbuf, 38 dpaa2_timestamp_dynfield_offset, rte_mbuf_timestamp_t *); 39 } 40 41 #define DPAA2_MBUF_TO_CONTIG_FD(_mbuf, _fd, _bpid) do { \ 42 DPAA2_SET_FD_ADDR(_fd, DPAA2_MBUF_VADDR_TO_IOVA(_mbuf)); \ 43 DPAA2_SET_FD_LEN(_fd, _mbuf->data_len); \ 44 DPAA2_SET_ONLY_FD_BPID(_fd, _bpid); \ 45 DPAA2_SET_FD_OFFSET(_fd, _mbuf->data_off); \ 46 DPAA2_SET_FD_FRC(_fd, 0); \ 47 DPAA2_RESET_FD_CTRL(_fd); \ 48 DPAA2_RESET_FD_FLC(_fd); \ 49 } while (0) 50 51 static inline void __rte_hot 52 dpaa2_dev_rx_parse_new(struct rte_mbuf *m, const struct qbman_fd *fd, 53 void *hw_annot_addr) 54 { 55 uint16_t frc = DPAA2_GET_FD_FRC_PARSE_SUM(fd); 56 struct dpaa2_annot_hdr *annotation = 57 (struct dpaa2_annot_hdr *)hw_annot_addr; 58 59 m->packet_type = RTE_PTYPE_UNKNOWN; 60 switch (frc) { 61 case DPAA2_PKT_TYPE_ETHER: 62 m->packet_type = RTE_PTYPE_L2_ETHER; 63 break; 64 case DPAA2_PKT_TYPE_IPV4: 65 m->packet_type = RTE_PTYPE_L2_ETHER | 66 RTE_PTYPE_L3_IPV4; 67 break; 68 case DPAA2_PKT_TYPE_IPV6: 69 m->packet_type = RTE_PTYPE_L2_ETHER | 70 RTE_PTYPE_L3_IPV6; 71 break; 72 case DPAA2_PKT_TYPE_IPV4_EXT: 73 m->packet_type = RTE_PTYPE_L2_ETHER | 74 RTE_PTYPE_L3_IPV4_EXT; 75 break; 76 case DPAA2_PKT_TYPE_IPV6_EXT: 77 m->packet_type = RTE_PTYPE_L2_ETHER | 78 RTE_PTYPE_L3_IPV6_EXT; 79 break; 80 case DPAA2_PKT_TYPE_IPV4_TCP: 81 m->packet_type = RTE_PTYPE_L2_ETHER | 82 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_TCP; 83 break; 84 case DPAA2_PKT_TYPE_IPV6_TCP: 85 m->packet_type = RTE_PTYPE_L2_ETHER | 86 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_TCP; 87 break; 88 case DPAA2_PKT_TYPE_IPV4_UDP: 89 m->packet_type = RTE_PTYPE_L2_ETHER | 90 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_UDP; 91 break; 92 case DPAA2_PKT_TYPE_IPV6_UDP: 93 m->packet_type = RTE_PTYPE_L2_ETHER | 94 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_UDP; 95 break; 96 case DPAA2_PKT_TYPE_IPV4_SCTP: 97 m->packet_type = RTE_PTYPE_L2_ETHER | 98 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_SCTP; 99 break; 100 case DPAA2_PKT_TYPE_IPV6_SCTP: 101 m->packet_type = RTE_PTYPE_L2_ETHER | 102 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_SCTP; 103 break; 104 case DPAA2_PKT_TYPE_IPV4_ICMP: 105 m->packet_type = RTE_PTYPE_L2_ETHER | 106 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_ICMP; 107 break; 108 case DPAA2_PKT_TYPE_IPV6_ICMP: 109 m->packet_type = RTE_PTYPE_L2_ETHER | 110 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_ICMP; 111 break; 112 default: 113 m->packet_type = dpaa2_dev_rx_parse_slow(m, annotation); 114 } 115 m->hash.rss = fd->simple.flc_hi; 116 m->ol_flags |= PKT_RX_RSS_HASH; 117 118 if (dpaa2_enable_ts[m->port]) { 119 *dpaa2_timestamp_dynfield(m) = annotation->word2; 120 m->ol_flags |= dpaa2_timestamp_rx_dynflag; 121 DPAA2_PMD_DP_DEBUG("pkt timestamp:0x%" PRIx64 "", 122 *dpaa2_timestamp_dynfield(m)); 123 } 124 125 DPAA2_PMD_DP_DEBUG("HW frc = 0x%x\t packet type =0x%x " 126 "ol_flags =0x%" PRIx64 "", 127 frc, m->packet_type, m->ol_flags); 128 } 129 130 static inline uint32_t __rte_hot 131 dpaa2_dev_rx_parse_slow(struct rte_mbuf *mbuf, 132 struct dpaa2_annot_hdr *annotation) 133 { 134 uint32_t pkt_type = RTE_PTYPE_UNKNOWN; 135 uint16_t *vlan_tci; 136 137 DPAA2_PMD_DP_DEBUG("(slow parse)annotation(3)=0x%" PRIx64 "\t" 138 "(4)=0x%" PRIx64 "\t", 139 annotation->word3, annotation->word4); 140 141 #if defined(RTE_LIBRTE_IEEE1588) 142 if (BIT_ISSET_AT_POS(annotation->word1, DPAA2_ETH_FAS_PTP)) 143 mbuf->ol_flags |= PKT_RX_IEEE1588_PTP; 144 #endif 145 146 if (BIT_ISSET_AT_POS(annotation->word3, L2_VLAN_1_PRESENT)) { 147 vlan_tci = rte_pktmbuf_mtod_offset(mbuf, uint16_t *, 148 (VLAN_TCI_OFFSET_1(annotation->word5) >> 16)); 149 mbuf->vlan_tci = rte_be_to_cpu_16(*vlan_tci); 150 mbuf->ol_flags |= PKT_RX_VLAN; 151 pkt_type |= RTE_PTYPE_L2_ETHER_VLAN; 152 } else if (BIT_ISSET_AT_POS(annotation->word3, L2_VLAN_N_PRESENT)) { 153 vlan_tci = rte_pktmbuf_mtod_offset(mbuf, uint16_t *, 154 (VLAN_TCI_OFFSET_1(annotation->word5) >> 16)); 155 mbuf->vlan_tci = rte_be_to_cpu_16(*vlan_tci); 156 mbuf->ol_flags |= PKT_RX_VLAN | PKT_RX_QINQ; 157 pkt_type |= RTE_PTYPE_L2_ETHER_QINQ; 158 } 159 160 if (BIT_ISSET_AT_POS(annotation->word3, L2_ARP_PRESENT)) { 161 pkt_type |= RTE_PTYPE_L2_ETHER_ARP; 162 goto parse_done; 163 } else if (BIT_ISSET_AT_POS(annotation->word3, L2_ETH_MAC_PRESENT)) { 164 pkt_type |= RTE_PTYPE_L2_ETHER; 165 } else { 166 goto parse_done; 167 } 168 169 if (BIT_ISSET_AT_POS(annotation->word4, L3_IPV4_1_PRESENT | 170 L3_IPV4_N_PRESENT)) { 171 pkt_type |= RTE_PTYPE_L3_IPV4; 172 if (BIT_ISSET_AT_POS(annotation->word4, L3_IP_1_OPT_PRESENT | 173 L3_IP_N_OPT_PRESENT)) 174 pkt_type |= RTE_PTYPE_L3_IPV4_EXT; 175 176 } else if (BIT_ISSET_AT_POS(annotation->word4, L3_IPV6_1_PRESENT | 177 L3_IPV6_N_PRESENT)) { 178 pkt_type |= RTE_PTYPE_L3_IPV6; 179 if (BIT_ISSET_AT_POS(annotation->word4, L3_IP_1_OPT_PRESENT | 180 L3_IP_N_OPT_PRESENT)) 181 pkt_type |= RTE_PTYPE_L3_IPV6_EXT; 182 } else { 183 goto parse_done; 184 } 185 186 if (BIT_ISSET_AT_POS(annotation->word8, DPAA2_ETH_FAS_L3CE)) 187 mbuf->ol_flags |= PKT_RX_IP_CKSUM_BAD; 188 else if (BIT_ISSET_AT_POS(annotation->word8, DPAA2_ETH_FAS_L4CE)) 189 mbuf->ol_flags |= PKT_RX_L4_CKSUM_BAD; 190 191 if (BIT_ISSET_AT_POS(annotation->word4, L3_IP_1_FIRST_FRAGMENT | 192 L3_IP_1_MORE_FRAGMENT | 193 L3_IP_N_FIRST_FRAGMENT | 194 L3_IP_N_MORE_FRAGMENT)) { 195 pkt_type |= RTE_PTYPE_L4_FRAG; 196 goto parse_done; 197 } else { 198 pkt_type |= RTE_PTYPE_L4_NONFRAG; 199 } 200 201 if (BIT_ISSET_AT_POS(annotation->word4, L3_PROTO_UDP_PRESENT)) 202 pkt_type |= RTE_PTYPE_L4_UDP; 203 204 else if (BIT_ISSET_AT_POS(annotation->word4, L3_PROTO_TCP_PRESENT)) 205 pkt_type |= RTE_PTYPE_L4_TCP; 206 207 else if (BIT_ISSET_AT_POS(annotation->word4, L3_PROTO_SCTP_PRESENT)) 208 pkt_type |= RTE_PTYPE_L4_SCTP; 209 210 else if (BIT_ISSET_AT_POS(annotation->word4, L3_PROTO_ICMP_PRESENT)) 211 pkt_type |= RTE_PTYPE_L4_ICMP; 212 213 else if (BIT_ISSET_AT_POS(annotation->word4, L3_IP_UNKNOWN_PROTOCOL)) 214 pkt_type |= RTE_PTYPE_UNKNOWN; 215 216 parse_done: 217 return pkt_type; 218 } 219 220 static inline uint32_t __rte_hot 221 dpaa2_dev_rx_parse(struct rte_mbuf *mbuf, void *hw_annot_addr) 222 { 223 struct dpaa2_annot_hdr *annotation = 224 (struct dpaa2_annot_hdr *)hw_annot_addr; 225 226 DPAA2_PMD_DP_DEBUG("(fast parse) Annotation = 0x%" PRIx64 "\t", 227 annotation->word4); 228 229 if (BIT_ISSET_AT_POS(annotation->word8, DPAA2_ETH_FAS_L3CE)) 230 mbuf->ol_flags |= PKT_RX_IP_CKSUM_BAD; 231 else if (BIT_ISSET_AT_POS(annotation->word8, DPAA2_ETH_FAS_L4CE)) 232 mbuf->ol_flags |= PKT_RX_L4_CKSUM_BAD; 233 234 if (dpaa2_enable_ts[mbuf->port]) { 235 *dpaa2_timestamp_dynfield(mbuf) = annotation->word2; 236 mbuf->ol_flags |= dpaa2_timestamp_rx_dynflag; 237 DPAA2_PMD_DP_DEBUG("pkt timestamp: 0x%" PRIx64 "", 238 *dpaa2_timestamp_dynfield(mbuf)); 239 } 240 241 /* Check detailed parsing requirement */ 242 if (annotation->word3 & 0x7FFFFC3FFFF) 243 return dpaa2_dev_rx_parse_slow(mbuf, annotation); 244 245 /* Return some common types from parse processing */ 246 switch (annotation->word4) { 247 case DPAA2_L3_IPv4: 248 return RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4; 249 case DPAA2_L3_IPv6: 250 return RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6; 251 case DPAA2_L3_IPv4_TCP: 252 return RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 | 253 RTE_PTYPE_L4_TCP; 254 case DPAA2_L3_IPv4_UDP: 255 return RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 | 256 RTE_PTYPE_L4_UDP; 257 case DPAA2_L3_IPv6_TCP: 258 return RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 | 259 RTE_PTYPE_L4_TCP; 260 case DPAA2_L3_IPv6_UDP: 261 return RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 | 262 RTE_PTYPE_L4_UDP; 263 default: 264 break; 265 } 266 267 return dpaa2_dev_rx_parse_slow(mbuf, annotation); 268 } 269 270 static inline struct rte_mbuf *__rte_hot 271 eth_sg_fd_to_mbuf(const struct qbman_fd *fd, 272 int port_id) 273 { 274 struct qbman_sge *sgt, *sge; 275 size_t sg_addr, fd_addr; 276 int i = 0; 277 void *hw_annot_addr; 278 struct rte_mbuf *first_seg, *next_seg, *cur_seg, *temp; 279 280 fd_addr = (size_t)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd)); 281 hw_annot_addr = (void *)(fd_addr + DPAA2_FD_PTA_SIZE); 282 283 /* Get Scatter gather table address */ 284 sgt = (struct qbman_sge *)(fd_addr + DPAA2_GET_FD_OFFSET(fd)); 285 286 sge = &sgt[i++]; 287 sg_addr = (size_t)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FLE_ADDR(sge)); 288 289 /* First Scatter gather entry */ 290 first_seg = DPAA2_INLINE_MBUF_FROM_BUF(sg_addr, 291 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size); 292 /* Prepare all the metadata for first segment */ 293 first_seg->buf_addr = (uint8_t *)sg_addr; 294 first_seg->ol_flags = 0; 295 first_seg->data_off = DPAA2_GET_FLE_OFFSET(sge); 296 first_seg->data_len = sge->length & 0x1FFFF; 297 first_seg->pkt_len = DPAA2_GET_FD_LEN(fd); 298 first_seg->nb_segs = 1; 299 first_seg->next = NULL; 300 first_seg->port = port_id; 301 if (dpaa2_svr_family == SVR_LX2160A) 302 dpaa2_dev_rx_parse_new(first_seg, fd, hw_annot_addr); 303 else 304 first_seg->packet_type = 305 dpaa2_dev_rx_parse(first_seg, hw_annot_addr); 306 307 rte_mbuf_refcnt_set(first_seg, 1); 308 cur_seg = first_seg; 309 while (!DPAA2_SG_IS_FINAL(sge)) { 310 sge = &sgt[i++]; 311 sg_addr = (size_t)DPAA2_IOVA_TO_VADDR( 312 DPAA2_GET_FLE_ADDR(sge)); 313 next_seg = DPAA2_INLINE_MBUF_FROM_BUF(sg_addr, 314 rte_dpaa2_bpid_info[DPAA2_GET_FLE_BPID(sge)].meta_data_size); 315 next_seg->buf_addr = (uint8_t *)sg_addr; 316 next_seg->data_off = DPAA2_GET_FLE_OFFSET(sge); 317 next_seg->data_len = sge->length & 0x1FFFF; 318 first_seg->nb_segs += 1; 319 rte_mbuf_refcnt_set(next_seg, 1); 320 cur_seg->next = next_seg; 321 next_seg->next = NULL; 322 cur_seg = next_seg; 323 } 324 temp = DPAA2_INLINE_MBUF_FROM_BUF(fd_addr, 325 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size); 326 rte_mbuf_refcnt_set(temp, 1); 327 rte_pktmbuf_free_seg(temp); 328 329 return (void *)first_seg; 330 } 331 332 static inline struct rte_mbuf *__rte_hot 333 eth_fd_to_mbuf(const struct qbman_fd *fd, 334 int port_id) 335 { 336 void *v_addr = DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd)); 337 void *hw_annot_addr = (void *)((size_t)v_addr + DPAA2_FD_PTA_SIZE); 338 struct rte_mbuf *mbuf = DPAA2_INLINE_MBUF_FROM_BUF(v_addr, 339 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size); 340 341 /* need to repopulated some of the fields, 342 * as they may have changed in last transmission 343 */ 344 mbuf->nb_segs = 1; 345 mbuf->ol_flags = 0; 346 mbuf->data_off = DPAA2_GET_FD_OFFSET(fd); 347 mbuf->data_len = DPAA2_GET_FD_LEN(fd); 348 mbuf->pkt_len = mbuf->data_len; 349 mbuf->port = port_id; 350 mbuf->next = NULL; 351 rte_mbuf_refcnt_set(mbuf, 1); 352 353 /* Parse the packet */ 354 /* parse results for LX2 are there in FRC field of FD. 355 * For other DPAA2 platforms , parse results are after 356 * the private - sw annotation area 357 */ 358 359 if (dpaa2_svr_family == SVR_LX2160A) 360 dpaa2_dev_rx_parse_new(mbuf, fd, hw_annot_addr); 361 else 362 mbuf->packet_type = dpaa2_dev_rx_parse(mbuf, hw_annot_addr); 363 364 DPAA2_PMD_DP_DEBUG("to mbuf - mbuf =%p, mbuf->buf_addr =%p, off = %d," 365 "fd_off=%d fd =%" PRIx64 ", meta = %d bpid =%d, len=%d\n", 366 mbuf, mbuf->buf_addr, mbuf->data_off, 367 DPAA2_GET_FD_OFFSET(fd), DPAA2_GET_FD_ADDR(fd), 368 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size, 369 DPAA2_GET_FD_BPID(fd), DPAA2_GET_FD_LEN(fd)); 370 371 return mbuf; 372 } 373 374 static int __rte_noinline __rte_hot 375 eth_mbuf_to_sg_fd(struct rte_mbuf *mbuf, 376 struct qbman_fd *fd, uint16_t bpid) 377 { 378 struct rte_mbuf *cur_seg = mbuf, *prev_seg, *mi, *temp; 379 struct qbman_sge *sgt, *sge = NULL; 380 int i; 381 382 temp = rte_pktmbuf_alloc(mbuf->pool); 383 if (temp == NULL) { 384 DPAA2_PMD_DP_DEBUG("No memory to allocate S/G table\n"); 385 return -ENOMEM; 386 } 387 388 DPAA2_SET_FD_ADDR(fd, DPAA2_MBUF_VADDR_TO_IOVA(temp)); 389 DPAA2_SET_FD_LEN(fd, mbuf->pkt_len); 390 DPAA2_SET_ONLY_FD_BPID(fd, bpid); 391 DPAA2_SET_FD_OFFSET(fd, temp->data_off); 392 DPAA2_FD_SET_FORMAT(fd, qbman_fd_sg); 393 DPAA2_RESET_FD_FRC(fd); 394 DPAA2_RESET_FD_CTRL(fd); 395 /*Set Scatter gather table and Scatter gather entries*/ 396 sgt = (struct qbman_sge *)( 397 (size_t)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd)) 398 + DPAA2_GET_FD_OFFSET(fd)); 399 400 for (i = 0; i < mbuf->nb_segs; i++) { 401 sge = &sgt[i]; 402 /*Resetting the buffer pool id and offset field*/ 403 sge->fin_bpid_offset = 0; 404 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(cur_seg)); 405 DPAA2_SET_FLE_OFFSET(sge, cur_seg->data_off); 406 sge->length = cur_seg->data_len; 407 if (RTE_MBUF_DIRECT(cur_seg)) { 408 if (rte_mbuf_refcnt_read(cur_seg) > 1) { 409 /* If refcnt > 1, invalid bpid is set to ensure 410 * buffer is not freed by HW 411 */ 412 DPAA2_SET_FLE_IVP(sge); 413 rte_mbuf_refcnt_update(cur_seg, -1); 414 } else 415 DPAA2_SET_FLE_BPID(sge, 416 mempool_to_bpid(cur_seg->pool)); 417 cur_seg = cur_seg->next; 418 } else { 419 /* Get owner MBUF from indirect buffer */ 420 mi = rte_mbuf_from_indirect(cur_seg); 421 if (rte_mbuf_refcnt_read(mi) > 1) { 422 /* If refcnt > 1, invalid bpid is set to ensure 423 * owner buffer is not freed by HW 424 */ 425 DPAA2_SET_FLE_IVP(sge); 426 } else { 427 DPAA2_SET_FLE_BPID(sge, 428 mempool_to_bpid(mi->pool)); 429 rte_mbuf_refcnt_update(mi, 1); 430 } 431 prev_seg = cur_seg; 432 cur_seg = cur_seg->next; 433 prev_seg->next = NULL; 434 rte_pktmbuf_free(prev_seg); 435 } 436 } 437 DPAA2_SG_SET_FINAL(sge, true); 438 return 0; 439 } 440 441 static void 442 eth_mbuf_to_fd(struct rte_mbuf *mbuf, 443 struct qbman_fd *fd, uint16_t bpid) __rte_unused; 444 445 static void __rte_noinline __rte_hot 446 eth_mbuf_to_fd(struct rte_mbuf *mbuf, 447 struct qbman_fd *fd, uint16_t bpid) 448 { 449 DPAA2_MBUF_TO_CONTIG_FD(mbuf, fd, bpid); 450 451 DPAA2_PMD_DP_DEBUG("mbuf =%p, mbuf->buf_addr =%p, off = %d," 452 "fd_off=%d fd =%" PRIx64 ", meta = %d bpid =%d, len=%d\n", 453 mbuf, mbuf->buf_addr, mbuf->data_off, 454 DPAA2_GET_FD_OFFSET(fd), DPAA2_GET_FD_ADDR(fd), 455 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size, 456 DPAA2_GET_FD_BPID(fd), DPAA2_GET_FD_LEN(fd)); 457 if (RTE_MBUF_DIRECT(mbuf)) { 458 if (rte_mbuf_refcnt_read(mbuf) > 1) { 459 DPAA2_SET_FD_IVP(fd); 460 rte_mbuf_refcnt_update(mbuf, -1); 461 } 462 } else { 463 struct rte_mbuf *mi; 464 465 mi = rte_mbuf_from_indirect(mbuf); 466 if (rte_mbuf_refcnt_read(mi) > 1) 467 DPAA2_SET_FD_IVP(fd); 468 else 469 rte_mbuf_refcnt_update(mi, 1); 470 rte_pktmbuf_free(mbuf); 471 } 472 } 473 474 static inline int __rte_hot 475 eth_copy_mbuf_to_fd(struct rte_mbuf *mbuf, 476 struct qbman_fd *fd, uint16_t bpid) 477 { 478 struct rte_mbuf *m; 479 void *mb = NULL; 480 481 if (rte_dpaa2_mbuf_alloc_bulk( 482 rte_dpaa2_bpid_info[bpid].bp_list->mp, &mb, 1)) { 483 DPAA2_PMD_DP_DEBUG("Unable to allocated DPAA2 buffer\n"); 484 return -1; 485 } 486 m = (struct rte_mbuf *)mb; 487 memcpy((char *)m->buf_addr + mbuf->data_off, 488 (void *)((char *)mbuf->buf_addr + mbuf->data_off), 489 mbuf->pkt_len); 490 491 /* Copy required fields */ 492 m->data_off = mbuf->data_off; 493 m->ol_flags = mbuf->ol_flags; 494 m->packet_type = mbuf->packet_type; 495 m->tx_offload = mbuf->tx_offload; 496 497 DPAA2_MBUF_TO_CONTIG_FD(m, fd, bpid); 498 499 DPAA2_PMD_DP_DEBUG( 500 "mbuf: %p, BMAN buf addr: %p, fdaddr: %" PRIx64 ", bpid: %d," 501 " meta: %d, off: %d, len: %d\n", 502 (void *)mbuf, 503 mbuf->buf_addr, 504 DPAA2_GET_FD_ADDR(fd), 505 DPAA2_GET_FD_BPID(fd), 506 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size, 507 DPAA2_GET_FD_OFFSET(fd), 508 DPAA2_GET_FD_LEN(fd)); 509 510 return 0; 511 } 512 513 /* This function assumes that caller will be keep the same value for nb_pkts 514 * across calls per queue, if that is not the case, better use non-prefetch 515 * version of rx call. 516 * It will return the packets as requested in previous call without honoring 517 * the current nb_pkts or bufs space. 518 */ 519 uint16_t 520 dpaa2_dev_prefetch_rx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) 521 { 522 /* Function receive frames for a given device and VQ*/ 523 struct dpaa2_queue *dpaa2_q = (struct dpaa2_queue *)queue; 524 struct qbman_result *dq_storage, *dq_storage1 = NULL; 525 uint32_t fqid = dpaa2_q->fqid; 526 int ret, num_rx = 0, pull_size; 527 uint8_t pending, status; 528 struct qbman_swp *swp; 529 const struct qbman_fd *fd; 530 struct qbman_pull_desc pulldesc; 531 struct queue_storage_info_t *q_storage = dpaa2_q->q_storage; 532 struct rte_eth_dev_data *eth_data = dpaa2_q->eth_data; 533 #if defined(RTE_LIBRTE_IEEE1588) 534 struct dpaa2_dev_priv *priv = eth_data->dev_private; 535 #endif 536 537 if (unlikely(!DPAA2_PER_LCORE_ETHRX_DPIO)) { 538 ret = dpaa2_affine_qbman_ethrx_swp(); 539 if (ret) { 540 DPAA2_PMD_ERR("Failure in affining portal"); 541 return 0; 542 } 543 } 544 545 if (unlikely(!rte_dpaa2_bpid_info && 546 rte_eal_process_type() == RTE_PROC_SECONDARY)) 547 rte_dpaa2_bpid_info = dpaa2_q->bp_array; 548 549 swp = DPAA2_PER_LCORE_ETHRX_PORTAL; 550 pull_size = (nb_pkts > dpaa2_dqrr_size) ? dpaa2_dqrr_size : nb_pkts; 551 if (unlikely(!q_storage->active_dqs)) { 552 q_storage->toggle = 0; 553 dq_storage = q_storage->dq_storage[q_storage->toggle]; 554 q_storage->last_num_pkts = pull_size; 555 qbman_pull_desc_clear(&pulldesc); 556 qbman_pull_desc_set_numframes(&pulldesc, 557 q_storage->last_num_pkts); 558 qbman_pull_desc_set_fq(&pulldesc, fqid); 559 qbman_pull_desc_set_storage(&pulldesc, dq_storage, 560 (uint64_t)(DPAA2_VADDR_TO_IOVA(dq_storage)), 1); 561 if (check_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index)) { 562 while (!qbman_check_command_complete( 563 get_swp_active_dqs( 564 DPAA2_PER_LCORE_ETHRX_DPIO->index))) 565 ; 566 clear_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index); 567 } 568 while (1) { 569 if (qbman_swp_pull(swp, &pulldesc)) { 570 DPAA2_PMD_DP_DEBUG("VDQ command is not issued." 571 " QBMAN is busy (1)\n"); 572 /* Portal was busy, try again */ 573 continue; 574 } 575 break; 576 } 577 q_storage->active_dqs = dq_storage; 578 q_storage->active_dpio_id = DPAA2_PER_LCORE_ETHRX_DPIO->index; 579 set_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index, 580 dq_storage); 581 } 582 583 dq_storage = q_storage->active_dqs; 584 rte_prefetch0((void *)(size_t)(dq_storage)); 585 rte_prefetch0((void *)(size_t)(dq_storage + 1)); 586 587 /* Prepare next pull descriptor. This will give space for the 588 * prefething done on DQRR entries 589 */ 590 q_storage->toggle ^= 1; 591 dq_storage1 = q_storage->dq_storage[q_storage->toggle]; 592 qbman_pull_desc_clear(&pulldesc); 593 qbman_pull_desc_set_numframes(&pulldesc, pull_size); 594 qbman_pull_desc_set_fq(&pulldesc, fqid); 595 qbman_pull_desc_set_storage(&pulldesc, dq_storage1, 596 (uint64_t)(DPAA2_VADDR_TO_IOVA(dq_storage1)), 1); 597 598 /* Check if the previous issued command is completed. 599 * Also seems like the SWP is shared between the Ethernet Driver 600 * and the SEC driver. 601 */ 602 while (!qbman_check_command_complete(dq_storage)) 603 ; 604 if (dq_storage == get_swp_active_dqs(q_storage->active_dpio_id)) 605 clear_swp_active_dqs(q_storage->active_dpio_id); 606 607 pending = 1; 608 609 do { 610 /* Loop until the dq_storage is updated with 611 * new token by QBMAN 612 */ 613 while (!qbman_check_new_result(dq_storage)) 614 ; 615 rte_prefetch0((void *)((size_t)(dq_storage + 2))); 616 /* Check whether Last Pull command is Expired and 617 * setting Condition for Loop termination 618 */ 619 if (qbman_result_DQ_is_pull_complete(dq_storage)) { 620 pending = 0; 621 /* Check for valid frame. */ 622 status = qbman_result_DQ_flags(dq_storage); 623 if (unlikely((status & QBMAN_DQ_STAT_VALIDFRAME) == 0)) 624 continue; 625 } 626 fd = qbman_result_DQ_fd(dq_storage); 627 628 #ifndef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA 629 if (dpaa2_svr_family != SVR_LX2160A) { 630 const struct qbman_fd *next_fd = 631 qbman_result_DQ_fd(dq_storage + 1); 632 /* Prefetch Annotation address for the parse results */ 633 rte_prefetch0(DPAA2_IOVA_TO_VADDR((DPAA2_GET_FD_ADDR( 634 next_fd) + DPAA2_FD_PTA_SIZE + 16))); 635 } 636 #endif 637 638 if (unlikely(DPAA2_FD_GET_FORMAT(fd) == qbman_fd_sg)) 639 bufs[num_rx] = eth_sg_fd_to_mbuf(fd, eth_data->port_id); 640 else 641 bufs[num_rx] = eth_fd_to_mbuf(fd, eth_data->port_id); 642 #if defined(RTE_LIBRTE_IEEE1588) 643 priv->rx_timestamp = *dpaa2_timestamp_dynfield(bufs[num_rx]); 644 #endif 645 646 if (eth_data->dev_conf.rxmode.offloads & 647 DEV_RX_OFFLOAD_VLAN_STRIP) 648 rte_vlan_strip(bufs[num_rx]); 649 650 dq_storage++; 651 num_rx++; 652 } while (pending); 653 654 if (check_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index)) { 655 while (!qbman_check_command_complete( 656 get_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index))) 657 ; 658 clear_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index); 659 } 660 /* issue a volatile dequeue command for next pull */ 661 while (1) { 662 if (qbman_swp_pull(swp, &pulldesc)) { 663 DPAA2_PMD_DP_DEBUG("VDQ command is not issued." 664 "QBMAN is busy (2)\n"); 665 continue; 666 } 667 break; 668 } 669 q_storage->active_dqs = dq_storage1; 670 q_storage->active_dpio_id = DPAA2_PER_LCORE_ETHRX_DPIO->index; 671 set_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index, dq_storage1); 672 673 dpaa2_q->rx_pkts += num_rx; 674 675 return num_rx; 676 } 677 678 void __rte_hot 679 dpaa2_dev_process_parallel_event(struct qbman_swp *swp, 680 const struct qbman_fd *fd, 681 const struct qbman_result *dq, 682 struct dpaa2_queue *rxq, 683 struct rte_event *ev) 684 { 685 rte_prefetch0((void *)(size_t)(DPAA2_GET_FD_ADDR(fd) + 686 DPAA2_FD_PTA_SIZE + 16)); 687 688 ev->flow_id = rxq->ev.flow_id; 689 ev->sub_event_type = rxq->ev.sub_event_type; 690 ev->event_type = RTE_EVENT_TYPE_ETHDEV; 691 ev->op = RTE_EVENT_OP_NEW; 692 ev->sched_type = rxq->ev.sched_type; 693 ev->queue_id = rxq->ev.queue_id; 694 ev->priority = rxq->ev.priority; 695 696 ev->mbuf = eth_fd_to_mbuf(fd, rxq->eth_data->port_id); 697 698 qbman_swp_dqrr_consume(swp, dq); 699 } 700 701 void __rte_hot 702 dpaa2_dev_process_atomic_event(struct qbman_swp *swp __rte_unused, 703 const struct qbman_fd *fd, 704 const struct qbman_result *dq, 705 struct dpaa2_queue *rxq, 706 struct rte_event *ev) 707 { 708 uint8_t dqrr_index; 709 710 rte_prefetch0((void *)(size_t)(DPAA2_GET_FD_ADDR(fd) + 711 DPAA2_FD_PTA_SIZE + 16)); 712 713 ev->flow_id = rxq->ev.flow_id; 714 ev->sub_event_type = rxq->ev.sub_event_type; 715 ev->event_type = RTE_EVENT_TYPE_ETHDEV; 716 ev->op = RTE_EVENT_OP_NEW; 717 ev->sched_type = rxq->ev.sched_type; 718 ev->queue_id = rxq->ev.queue_id; 719 ev->priority = rxq->ev.priority; 720 721 ev->mbuf = eth_fd_to_mbuf(fd, rxq->eth_data->port_id); 722 723 dqrr_index = qbman_get_dqrr_idx(dq); 724 *dpaa2_seqn(ev->mbuf) = dqrr_index + 1; 725 DPAA2_PER_LCORE_DQRR_SIZE++; 726 DPAA2_PER_LCORE_DQRR_HELD |= 1 << dqrr_index; 727 DPAA2_PER_LCORE_DQRR_MBUF(dqrr_index) = ev->mbuf; 728 } 729 730 void __rte_hot 731 dpaa2_dev_process_ordered_event(struct qbman_swp *swp, 732 const struct qbman_fd *fd, 733 const struct qbman_result *dq, 734 struct dpaa2_queue *rxq, 735 struct rte_event *ev) 736 { 737 rte_prefetch0((void *)(size_t)(DPAA2_GET_FD_ADDR(fd) + 738 DPAA2_FD_PTA_SIZE + 16)); 739 740 ev->flow_id = rxq->ev.flow_id; 741 ev->sub_event_type = rxq->ev.sub_event_type; 742 ev->event_type = RTE_EVENT_TYPE_ETHDEV; 743 ev->op = RTE_EVENT_OP_NEW; 744 ev->sched_type = rxq->ev.sched_type; 745 ev->queue_id = rxq->ev.queue_id; 746 ev->priority = rxq->ev.priority; 747 748 ev->mbuf = eth_fd_to_mbuf(fd, rxq->eth_data->port_id); 749 750 *dpaa2_seqn(ev->mbuf) = DPAA2_ENQUEUE_FLAG_ORP; 751 *dpaa2_seqn(ev->mbuf) |= qbman_result_DQ_odpid(dq) << DPAA2_EQCR_OPRID_SHIFT; 752 *dpaa2_seqn(ev->mbuf) |= qbman_result_DQ_seqnum(dq) << DPAA2_EQCR_SEQNUM_SHIFT; 753 754 qbman_swp_dqrr_consume(swp, dq); 755 } 756 757 uint16_t 758 dpaa2_dev_rx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) 759 { 760 /* Function receive frames for a given device and VQ */ 761 struct dpaa2_queue *dpaa2_q = (struct dpaa2_queue *)queue; 762 struct qbman_result *dq_storage; 763 uint32_t fqid = dpaa2_q->fqid; 764 int ret, num_rx = 0, next_pull = nb_pkts, num_pulled; 765 uint8_t pending, status; 766 struct qbman_swp *swp; 767 const struct qbman_fd *fd; 768 struct qbman_pull_desc pulldesc; 769 struct rte_eth_dev_data *eth_data = dpaa2_q->eth_data; 770 771 if (unlikely(!DPAA2_PER_LCORE_DPIO)) { 772 ret = dpaa2_affine_qbman_swp(); 773 if (ret) { 774 DPAA2_PMD_ERR( 775 "Failed to allocate IO portal, tid: %d\n", 776 rte_gettid()); 777 return 0; 778 } 779 } 780 swp = DPAA2_PER_LCORE_PORTAL; 781 782 do { 783 dq_storage = dpaa2_q->q_storage->dq_storage[0]; 784 qbman_pull_desc_clear(&pulldesc); 785 qbman_pull_desc_set_fq(&pulldesc, fqid); 786 qbman_pull_desc_set_storage(&pulldesc, dq_storage, 787 (size_t)(DPAA2_VADDR_TO_IOVA(dq_storage)), 1); 788 789 if (next_pull > dpaa2_dqrr_size) { 790 qbman_pull_desc_set_numframes(&pulldesc, 791 dpaa2_dqrr_size); 792 next_pull -= dpaa2_dqrr_size; 793 } else { 794 qbman_pull_desc_set_numframes(&pulldesc, next_pull); 795 next_pull = 0; 796 } 797 798 while (1) { 799 if (qbman_swp_pull(swp, &pulldesc)) { 800 DPAA2_PMD_DP_DEBUG( 801 "VDQ command is not issued.QBMAN is busy\n"); 802 /* Portal was busy, try again */ 803 continue; 804 } 805 break; 806 } 807 808 rte_prefetch0((void *)((size_t)(dq_storage + 1))); 809 /* Check if the previous issued command is completed. */ 810 while (!qbman_check_command_complete(dq_storage)) 811 ; 812 813 num_pulled = 0; 814 pending = 1; 815 do { 816 /* Loop until the dq_storage is updated with 817 * new token by QBMAN 818 */ 819 while (!qbman_check_new_result(dq_storage)) 820 ; 821 rte_prefetch0((void *)((size_t)(dq_storage + 2))); 822 /* Check whether Last Pull command is Expired and 823 * setting Condition for Loop termination 824 */ 825 if (qbman_result_DQ_is_pull_complete(dq_storage)) { 826 pending = 0; 827 /* Check for valid frame. */ 828 status = qbman_result_DQ_flags(dq_storage); 829 if (unlikely((status & 830 QBMAN_DQ_STAT_VALIDFRAME) == 0)) 831 continue; 832 } 833 fd = qbman_result_DQ_fd(dq_storage); 834 835 #ifndef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA 836 if (dpaa2_svr_family != SVR_LX2160A) { 837 const struct qbman_fd *next_fd = 838 qbman_result_DQ_fd(dq_storage + 1); 839 840 /* Prefetch Annotation address for the parse 841 * results. 842 */ 843 rte_prefetch0((DPAA2_IOVA_TO_VADDR( 844 DPAA2_GET_FD_ADDR(next_fd) + 845 DPAA2_FD_PTA_SIZE + 16))); 846 } 847 #endif 848 849 if (unlikely(DPAA2_FD_GET_FORMAT(fd) == qbman_fd_sg)) 850 bufs[num_rx] = eth_sg_fd_to_mbuf(fd, 851 eth_data->port_id); 852 else 853 bufs[num_rx] = eth_fd_to_mbuf(fd, 854 eth_data->port_id); 855 856 if (eth_data->dev_conf.rxmode.offloads & 857 DEV_RX_OFFLOAD_VLAN_STRIP) { 858 rte_vlan_strip(bufs[num_rx]); 859 } 860 861 dq_storage++; 862 num_rx++; 863 num_pulled++; 864 } while (pending); 865 /* Last VDQ provided all packets and more packets are requested */ 866 } while (next_pull && num_pulled == dpaa2_dqrr_size); 867 868 dpaa2_q->rx_pkts += num_rx; 869 870 return num_rx; 871 } 872 873 uint16_t dpaa2_dev_tx_conf(void *queue) 874 { 875 /* Function receive frames for a given device and VQ */ 876 struct dpaa2_queue *dpaa2_q = (struct dpaa2_queue *)queue; 877 struct qbman_result *dq_storage; 878 uint32_t fqid = dpaa2_q->fqid; 879 int ret, num_tx_conf = 0, num_pulled; 880 uint8_t pending, status; 881 struct qbman_swp *swp; 882 const struct qbman_fd *fd, *next_fd; 883 struct qbman_pull_desc pulldesc; 884 struct qbman_release_desc releasedesc; 885 uint32_t bpid; 886 uint64_t buf; 887 #if defined(RTE_LIBRTE_IEEE1588) 888 struct rte_eth_dev_data *eth_data = dpaa2_q->eth_data; 889 struct dpaa2_dev_priv *priv = eth_data->dev_private; 890 struct dpaa2_annot_hdr *annotation; 891 #endif 892 893 if (unlikely(!DPAA2_PER_LCORE_DPIO)) { 894 ret = dpaa2_affine_qbman_swp(); 895 if (ret) { 896 DPAA2_PMD_ERR( 897 "Failed to allocate IO portal, tid: %d\n", 898 rte_gettid()); 899 return 0; 900 } 901 } 902 swp = DPAA2_PER_LCORE_PORTAL; 903 904 do { 905 dq_storage = dpaa2_q->q_storage->dq_storage[0]; 906 qbman_pull_desc_clear(&pulldesc); 907 qbman_pull_desc_set_fq(&pulldesc, fqid); 908 qbman_pull_desc_set_storage(&pulldesc, dq_storage, 909 (size_t)(DPAA2_VADDR_TO_IOVA(dq_storage)), 1); 910 911 qbman_pull_desc_set_numframes(&pulldesc, dpaa2_dqrr_size); 912 913 while (1) { 914 if (qbman_swp_pull(swp, &pulldesc)) { 915 DPAA2_PMD_DP_DEBUG("VDQ command is not issued." 916 "QBMAN is busy\n"); 917 /* Portal was busy, try again */ 918 continue; 919 } 920 break; 921 } 922 923 rte_prefetch0((void *)((size_t)(dq_storage + 1))); 924 /* Check if the previous issued command is completed. */ 925 while (!qbman_check_command_complete(dq_storage)) 926 ; 927 928 num_pulled = 0; 929 pending = 1; 930 do { 931 /* Loop until the dq_storage is updated with 932 * new token by QBMAN 933 */ 934 while (!qbman_check_new_result(dq_storage)) 935 ; 936 rte_prefetch0((void *)((size_t)(dq_storage + 2))); 937 /* Check whether Last Pull command is Expired and 938 * setting Condition for Loop termination 939 */ 940 if (qbman_result_DQ_is_pull_complete(dq_storage)) { 941 pending = 0; 942 /* Check for valid frame. */ 943 status = qbman_result_DQ_flags(dq_storage); 944 if (unlikely((status & 945 QBMAN_DQ_STAT_VALIDFRAME) == 0)) 946 continue; 947 } 948 fd = qbman_result_DQ_fd(dq_storage); 949 950 next_fd = qbman_result_DQ_fd(dq_storage + 1); 951 /* Prefetch Annotation address for the parse results */ 952 rte_prefetch0((void *)(size_t) 953 (DPAA2_GET_FD_ADDR(next_fd) + 954 DPAA2_FD_PTA_SIZE + 16)); 955 956 bpid = DPAA2_GET_FD_BPID(fd); 957 958 /* Create a release descriptor required for releasing 959 * buffers into QBMAN 960 */ 961 qbman_release_desc_clear(&releasedesc); 962 qbman_release_desc_set_bpid(&releasedesc, bpid); 963 964 buf = DPAA2_GET_FD_ADDR(fd); 965 /* feed them to bman */ 966 do { 967 ret = qbman_swp_release(swp, &releasedesc, 968 &buf, 1); 969 } while (ret == -EBUSY); 970 971 dq_storage++; 972 num_tx_conf++; 973 num_pulled++; 974 #if defined(RTE_LIBRTE_IEEE1588) 975 annotation = (struct dpaa2_annot_hdr *)((size_t) 976 DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd)) + 977 DPAA2_FD_PTA_SIZE); 978 priv->tx_timestamp = annotation->word2; 979 #endif 980 } while (pending); 981 982 /* Last VDQ provided all packets and more packets are requested */ 983 } while (num_pulled == dpaa2_dqrr_size); 984 985 dpaa2_q->rx_pkts += num_tx_conf; 986 987 return num_tx_conf; 988 } 989 990 /* Configure the egress frame annotation for timestamp update */ 991 static void enable_tx_tstamp(struct qbman_fd *fd) 992 { 993 struct dpaa2_faead *fd_faead; 994 995 /* Set frame annotation status field as valid */ 996 (fd)->simple.frc |= DPAA2_FD_FRC_FASV; 997 998 /* Set frame annotation egress action descriptor as valid */ 999 (fd)->simple.frc |= DPAA2_FD_FRC_FAEADV; 1000 1001 /* Set Annotation Length as 128B */ 1002 (fd)->simple.ctrl |= DPAA2_FD_CTRL_ASAL; 1003 1004 /* enable update of confirmation frame annotation */ 1005 fd_faead = (struct dpaa2_faead *)((size_t) 1006 DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd)) + 1007 DPAA2_FD_PTA_SIZE + DPAA2_FD_HW_ANNOT_FAEAD_OFFSET); 1008 fd_faead->ctrl = DPAA2_ANNOT_FAEAD_A2V | DPAA2_ANNOT_FAEAD_UPDV | 1009 DPAA2_ANNOT_FAEAD_UPD; 1010 } 1011 1012 /* 1013 * Callback to handle sending packets through WRIOP based interface 1014 */ 1015 uint16_t 1016 dpaa2_dev_tx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) 1017 { 1018 /* Function to transmit the frames to given device and VQ*/ 1019 uint32_t loop, retry_count; 1020 int32_t ret; 1021 struct qbman_fd fd_arr[MAX_TX_RING_SLOTS]; 1022 struct rte_mbuf *mi; 1023 uint32_t frames_to_send; 1024 struct rte_mempool *mp; 1025 struct qbman_eq_desc eqdesc; 1026 struct dpaa2_queue *dpaa2_q = (struct dpaa2_queue *)queue; 1027 struct qbman_swp *swp; 1028 uint16_t num_tx = 0; 1029 uint16_t bpid; 1030 struct rte_eth_dev_data *eth_data = dpaa2_q->eth_data; 1031 struct dpaa2_dev_priv *priv = eth_data->dev_private; 1032 uint32_t flags[MAX_TX_RING_SLOTS] = {0}; 1033 1034 if (unlikely(!DPAA2_PER_LCORE_DPIO)) { 1035 ret = dpaa2_affine_qbman_swp(); 1036 if (ret) { 1037 DPAA2_PMD_ERR( 1038 "Failed to allocate IO portal, tid: %d\n", 1039 rte_gettid()); 1040 return 0; 1041 } 1042 } 1043 swp = DPAA2_PER_LCORE_PORTAL; 1044 1045 DPAA2_PMD_DP_DEBUG("===> eth_data =%p, fqid =%d\n", 1046 eth_data, dpaa2_q->fqid); 1047 1048 #ifdef RTE_LIBRTE_IEEE1588 1049 /* IEEE1588 driver need pointer to tx confirmation queue 1050 * corresponding to last packet transmitted for reading 1051 * the timestamp 1052 */ 1053 priv->next_tx_conf_queue = dpaa2_q->tx_conf_queue; 1054 dpaa2_dev_tx_conf(dpaa2_q->tx_conf_queue); 1055 #endif 1056 1057 /*Prepare enqueue descriptor*/ 1058 qbman_eq_desc_clear(&eqdesc); 1059 qbman_eq_desc_set_no_orp(&eqdesc, DPAA2_EQ_RESP_ERR_FQ); 1060 qbman_eq_desc_set_fq(&eqdesc, dpaa2_q->fqid); 1061 1062 /*Clear the unused FD fields before sending*/ 1063 while (nb_pkts) { 1064 /*Check if the queue is congested*/ 1065 retry_count = 0; 1066 while (qbman_result_SCN_state(dpaa2_q->cscn)) { 1067 retry_count++; 1068 /* Retry for some time before giving up */ 1069 if (retry_count > CONG_RETRY_COUNT) 1070 goto skip_tx; 1071 } 1072 1073 frames_to_send = (nb_pkts > dpaa2_eqcr_size) ? 1074 dpaa2_eqcr_size : nb_pkts; 1075 1076 for (loop = 0; loop < frames_to_send; loop++) { 1077 if (*dpaa2_seqn(*bufs)) { 1078 uint8_t dqrr_index = *dpaa2_seqn(*bufs) - 1; 1079 1080 flags[loop] = QBMAN_ENQUEUE_FLAG_DCA | 1081 dqrr_index; 1082 DPAA2_PER_LCORE_DQRR_SIZE--; 1083 DPAA2_PER_LCORE_DQRR_HELD &= ~(1 << dqrr_index); 1084 *dpaa2_seqn(*bufs) = DPAA2_INVALID_MBUF_SEQN; 1085 } 1086 1087 if (likely(RTE_MBUF_DIRECT(*bufs))) { 1088 mp = (*bufs)->pool; 1089 /* Check the basic scenario and set 1090 * the FD appropriately here itself. 1091 */ 1092 if (likely(mp && mp->ops_index == 1093 priv->bp_list->dpaa2_ops_index && 1094 (*bufs)->nb_segs == 1 && 1095 rte_mbuf_refcnt_read((*bufs)) == 1)) { 1096 if (unlikely(((*bufs)->ol_flags 1097 & PKT_TX_VLAN_PKT) || 1098 (eth_data->dev_conf.txmode.offloads 1099 & DEV_TX_OFFLOAD_VLAN_INSERT))) { 1100 ret = rte_vlan_insert(bufs); 1101 if (ret) 1102 goto send_n_return; 1103 } 1104 DPAA2_MBUF_TO_CONTIG_FD((*bufs), 1105 &fd_arr[loop], mempool_to_bpid(mp)); 1106 bufs++; 1107 #ifdef RTE_LIBRTE_IEEE1588 1108 enable_tx_tstamp(&fd_arr[loop]); 1109 #endif 1110 continue; 1111 } 1112 } else { 1113 mi = rte_mbuf_from_indirect(*bufs); 1114 mp = mi->pool; 1115 } 1116 /* Not a hw_pkt pool allocated frame */ 1117 if (unlikely(!mp || !priv->bp_list)) { 1118 DPAA2_PMD_ERR("Err: No buffer pool attached"); 1119 goto send_n_return; 1120 } 1121 1122 if (unlikely(((*bufs)->ol_flags & PKT_TX_VLAN_PKT) || 1123 (eth_data->dev_conf.txmode.offloads 1124 & DEV_TX_OFFLOAD_VLAN_INSERT))) { 1125 int ret = rte_vlan_insert(bufs); 1126 if (ret) 1127 goto send_n_return; 1128 } 1129 if (mp->ops_index != priv->bp_list->dpaa2_ops_index) { 1130 DPAA2_PMD_WARN("Non DPAA2 buffer pool"); 1131 /* alloc should be from the default buffer pool 1132 * attached to this interface 1133 */ 1134 bpid = priv->bp_list->buf_pool.bpid; 1135 1136 if (unlikely((*bufs)->nb_segs > 1)) { 1137 DPAA2_PMD_ERR("S/G support not added" 1138 " for non hw offload buffer"); 1139 goto send_n_return; 1140 } 1141 if (eth_copy_mbuf_to_fd(*bufs, 1142 &fd_arr[loop], bpid)) { 1143 goto send_n_return; 1144 } 1145 /* free the original packet */ 1146 rte_pktmbuf_free(*bufs); 1147 } else { 1148 bpid = mempool_to_bpid(mp); 1149 if (unlikely((*bufs)->nb_segs > 1)) { 1150 if (eth_mbuf_to_sg_fd(*bufs, 1151 &fd_arr[loop], bpid)) 1152 goto send_n_return; 1153 } else { 1154 eth_mbuf_to_fd(*bufs, 1155 &fd_arr[loop], bpid); 1156 } 1157 } 1158 #ifdef RTE_LIBRTE_IEEE1588 1159 enable_tx_tstamp(&fd_arr[loop]); 1160 #endif 1161 bufs++; 1162 } 1163 1164 loop = 0; 1165 retry_count = 0; 1166 while (loop < frames_to_send) { 1167 ret = qbman_swp_enqueue_multiple(swp, &eqdesc, 1168 &fd_arr[loop], &flags[loop], 1169 frames_to_send - loop); 1170 if (unlikely(ret < 0)) { 1171 retry_count++; 1172 if (retry_count > DPAA2_MAX_TX_RETRY_COUNT) { 1173 num_tx += loop; 1174 nb_pkts -= loop; 1175 goto send_n_return; 1176 } 1177 } else { 1178 loop += ret; 1179 retry_count = 0; 1180 } 1181 } 1182 1183 num_tx += loop; 1184 nb_pkts -= loop; 1185 } 1186 dpaa2_q->tx_pkts += num_tx; 1187 return num_tx; 1188 1189 send_n_return: 1190 /* send any already prepared fd */ 1191 if (loop) { 1192 unsigned int i = 0; 1193 1194 retry_count = 0; 1195 while (i < loop) { 1196 ret = qbman_swp_enqueue_multiple(swp, &eqdesc, 1197 &fd_arr[i], 1198 &flags[i], 1199 loop - i); 1200 if (unlikely(ret < 0)) { 1201 retry_count++; 1202 if (retry_count > DPAA2_MAX_TX_RETRY_COUNT) 1203 break; 1204 } else { 1205 i += ret; 1206 retry_count = 0; 1207 } 1208 } 1209 num_tx += i; 1210 } 1211 skip_tx: 1212 dpaa2_q->tx_pkts += num_tx; 1213 return num_tx; 1214 } 1215 1216 void 1217 dpaa2_dev_free_eqresp_buf(uint16_t eqresp_ci) 1218 { 1219 struct dpaa2_dpio_dev *dpio_dev = DPAA2_PER_LCORE_DPIO; 1220 struct qbman_fd *fd; 1221 struct rte_mbuf *m; 1222 1223 fd = qbman_result_eqresp_fd(&dpio_dev->eqresp[eqresp_ci]); 1224 1225 /* Setting port id does not matter as we are to free the mbuf */ 1226 m = eth_fd_to_mbuf(fd, 0); 1227 rte_pktmbuf_free(m); 1228 } 1229 1230 static void 1231 dpaa2_set_enqueue_descriptor(struct dpaa2_queue *dpaa2_q, 1232 struct rte_mbuf *m, 1233 struct qbman_eq_desc *eqdesc) 1234 { 1235 struct rte_eth_dev_data *eth_data = dpaa2_q->eth_data; 1236 struct dpaa2_dev_priv *priv = eth_data->dev_private; 1237 struct dpaa2_dpio_dev *dpio_dev = DPAA2_PER_LCORE_DPIO; 1238 struct eqresp_metadata *eqresp_meta; 1239 uint16_t orpid, seqnum; 1240 uint8_t dq_idx; 1241 1242 qbman_eq_desc_set_fq(eqdesc, dpaa2_q->fqid); 1243 1244 if (*dpaa2_seqn(m) & DPAA2_ENQUEUE_FLAG_ORP) { 1245 orpid = (*dpaa2_seqn(m) & DPAA2_EQCR_OPRID_MASK) >> 1246 DPAA2_EQCR_OPRID_SHIFT; 1247 seqnum = (*dpaa2_seqn(m) & DPAA2_EQCR_SEQNUM_MASK) >> 1248 DPAA2_EQCR_SEQNUM_SHIFT; 1249 1250 if (!priv->en_loose_ordered) { 1251 qbman_eq_desc_set_orp(eqdesc, 1, orpid, seqnum, 0); 1252 qbman_eq_desc_set_response(eqdesc, (uint64_t) 1253 DPAA2_VADDR_TO_IOVA(&dpio_dev->eqresp[ 1254 dpio_dev->eqresp_pi]), 1); 1255 qbman_eq_desc_set_token(eqdesc, 1); 1256 1257 eqresp_meta = &dpio_dev->eqresp_meta[ 1258 dpio_dev->eqresp_pi]; 1259 eqresp_meta->dpaa2_q = dpaa2_q; 1260 eqresp_meta->mp = m->pool; 1261 1262 dpio_dev->eqresp_pi + 1 < MAX_EQ_RESP_ENTRIES ? 1263 dpio_dev->eqresp_pi++ : 1264 (dpio_dev->eqresp_pi = 0); 1265 } else { 1266 qbman_eq_desc_set_orp(eqdesc, 0, orpid, seqnum, 0); 1267 } 1268 } else { 1269 dq_idx = *dpaa2_seqn(m) - 1; 1270 qbman_eq_desc_set_dca(eqdesc, 1, dq_idx, 0); 1271 DPAA2_PER_LCORE_DQRR_SIZE--; 1272 DPAA2_PER_LCORE_DQRR_HELD &= ~(1 << dq_idx); 1273 } 1274 *dpaa2_seqn(m) = DPAA2_INVALID_MBUF_SEQN; 1275 } 1276 1277 /* Callback to handle sending ordered packets through WRIOP based interface */ 1278 uint16_t 1279 dpaa2_dev_tx_ordered(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) 1280 { 1281 /* Function to transmit the frames to given device and VQ*/ 1282 struct dpaa2_queue *dpaa2_q = (struct dpaa2_queue *)queue; 1283 struct rte_eth_dev_data *eth_data = dpaa2_q->eth_data; 1284 struct dpaa2_dev_priv *priv = eth_data->dev_private; 1285 struct dpaa2_queue *order_sendq = (struct dpaa2_queue *)priv->tx_vq[0]; 1286 struct qbman_fd fd_arr[MAX_TX_RING_SLOTS]; 1287 struct rte_mbuf *mi; 1288 struct rte_mempool *mp; 1289 struct qbman_eq_desc eqdesc[MAX_TX_RING_SLOTS]; 1290 struct qbman_swp *swp; 1291 uint32_t frames_to_send, num_free_eq_desc; 1292 uint32_t loop, retry_count; 1293 int32_t ret; 1294 uint16_t num_tx = 0; 1295 uint16_t bpid; 1296 1297 if (unlikely(!DPAA2_PER_LCORE_DPIO)) { 1298 ret = dpaa2_affine_qbman_swp(); 1299 if (ret) { 1300 DPAA2_PMD_ERR( 1301 "Failed to allocate IO portal, tid: %d\n", 1302 rte_gettid()); 1303 return 0; 1304 } 1305 } 1306 swp = DPAA2_PER_LCORE_PORTAL; 1307 1308 DPAA2_PMD_DP_DEBUG("===> eth_data =%p, fqid =%d\n", 1309 eth_data, dpaa2_q->fqid); 1310 1311 /* This would also handle normal and atomic queues as any type 1312 * of packet can be enqueued when ordered queues are being used. 1313 */ 1314 while (nb_pkts) { 1315 /*Check if the queue is congested*/ 1316 retry_count = 0; 1317 while (qbman_result_SCN_state(dpaa2_q->cscn)) { 1318 retry_count++; 1319 /* Retry for some time before giving up */ 1320 if (retry_count > CONG_RETRY_COUNT) 1321 goto skip_tx; 1322 } 1323 1324 frames_to_send = (nb_pkts > dpaa2_eqcr_size) ? 1325 dpaa2_eqcr_size : nb_pkts; 1326 1327 if (!priv->en_loose_ordered) { 1328 if (*dpaa2_seqn(*bufs) & DPAA2_ENQUEUE_FLAG_ORP) { 1329 num_free_eq_desc = dpaa2_free_eq_descriptors(); 1330 if (num_free_eq_desc < frames_to_send) 1331 frames_to_send = num_free_eq_desc; 1332 } 1333 } 1334 1335 for (loop = 0; loop < frames_to_send; loop++) { 1336 /*Prepare enqueue descriptor*/ 1337 qbman_eq_desc_clear(&eqdesc[loop]); 1338 1339 if (*dpaa2_seqn(*bufs)) { 1340 /* Use only queue 0 for Tx in case of atomic/ 1341 * ordered packets as packets can get unordered 1342 * when being tranmitted out from the interface 1343 */ 1344 dpaa2_set_enqueue_descriptor(order_sendq, 1345 (*bufs), 1346 &eqdesc[loop]); 1347 } else { 1348 qbman_eq_desc_set_no_orp(&eqdesc[loop], 1349 DPAA2_EQ_RESP_ERR_FQ); 1350 qbman_eq_desc_set_fq(&eqdesc[loop], 1351 dpaa2_q->fqid); 1352 } 1353 1354 if (likely(RTE_MBUF_DIRECT(*bufs))) { 1355 mp = (*bufs)->pool; 1356 /* Check the basic scenario and set 1357 * the FD appropriately here itself. 1358 */ 1359 if (likely(mp && mp->ops_index == 1360 priv->bp_list->dpaa2_ops_index && 1361 (*bufs)->nb_segs == 1 && 1362 rte_mbuf_refcnt_read((*bufs)) == 1)) { 1363 if (unlikely((*bufs)->ol_flags 1364 & PKT_TX_VLAN_PKT)) { 1365 ret = rte_vlan_insert(bufs); 1366 if (ret) 1367 goto send_n_return; 1368 } 1369 DPAA2_MBUF_TO_CONTIG_FD((*bufs), 1370 &fd_arr[loop], 1371 mempool_to_bpid(mp)); 1372 bufs++; 1373 continue; 1374 } 1375 } else { 1376 mi = rte_mbuf_from_indirect(*bufs); 1377 mp = mi->pool; 1378 } 1379 /* Not a hw_pkt pool allocated frame */ 1380 if (unlikely(!mp || !priv->bp_list)) { 1381 DPAA2_PMD_ERR("Err: No buffer pool attached"); 1382 goto send_n_return; 1383 } 1384 1385 if (mp->ops_index != priv->bp_list->dpaa2_ops_index) { 1386 DPAA2_PMD_WARN("Non DPAA2 buffer pool"); 1387 /* alloc should be from the default buffer pool 1388 * attached to this interface 1389 */ 1390 bpid = priv->bp_list->buf_pool.bpid; 1391 1392 if (unlikely((*bufs)->nb_segs > 1)) { 1393 DPAA2_PMD_ERR( 1394 "S/G not supp for non hw offload buffer"); 1395 goto send_n_return; 1396 } 1397 if (eth_copy_mbuf_to_fd(*bufs, 1398 &fd_arr[loop], bpid)) { 1399 goto send_n_return; 1400 } 1401 /* free the original packet */ 1402 rte_pktmbuf_free(*bufs); 1403 } else { 1404 bpid = mempool_to_bpid(mp); 1405 if (unlikely((*bufs)->nb_segs > 1)) { 1406 if (eth_mbuf_to_sg_fd(*bufs, 1407 &fd_arr[loop], 1408 bpid)) 1409 goto send_n_return; 1410 } else { 1411 eth_mbuf_to_fd(*bufs, 1412 &fd_arr[loop], bpid); 1413 } 1414 } 1415 bufs++; 1416 } 1417 1418 loop = 0; 1419 retry_count = 0; 1420 while (loop < frames_to_send) { 1421 ret = qbman_swp_enqueue_multiple_desc(swp, 1422 &eqdesc[loop], &fd_arr[loop], 1423 frames_to_send - loop); 1424 if (unlikely(ret < 0)) { 1425 retry_count++; 1426 if (retry_count > DPAA2_MAX_TX_RETRY_COUNT) { 1427 num_tx += loop; 1428 nb_pkts -= loop; 1429 goto send_n_return; 1430 } 1431 } else { 1432 loop += ret; 1433 retry_count = 0; 1434 } 1435 } 1436 1437 num_tx += loop; 1438 nb_pkts -= loop; 1439 } 1440 dpaa2_q->tx_pkts += num_tx; 1441 return num_tx; 1442 1443 send_n_return: 1444 /* send any already prepared fd */ 1445 if (loop) { 1446 unsigned int i = 0; 1447 1448 retry_count = 0; 1449 while (i < loop) { 1450 ret = qbman_swp_enqueue_multiple_desc(swp, 1451 &eqdesc[loop], &fd_arr[i], loop - i); 1452 if (unlikely(ret < 0)) { 1453 retry_count++; 1454 if (retry_count > DPAA2_MAX_TX_RETRY_COUNT) 1455 break; 1456 } else { 1457 i += ret; 1458 retry_count = 0; 1459 } 1460 } 1461 num_tx += i; 1462 } 1463 skip_tx: 1464 dpaa2_q->tx_pkts += num_tx; 1465 return num_tx; 1466 } 1467 1468 /** 1469 * Dummy DPDK callback for TX. 1470 * 1471 * This function is used to temporarily replace the real callback during 1472 * unsafe control operations on the queue, or in case of error. 1473 * 1474 * @param dpdk_txq 1475 * Generic pointer to TX queue structure. 1476 * @param[in] pkts 1477 * Packets to transmit. 1478 * @param pkts_n 1479 * Number of packets in array. 1480 * 1481 * @return 1482 * Number of packets successfully transmitted (<= pkts_n). 1483 */ 1484 uint16_t 1485 dummy_dev_tx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) 1486 { 1487 (void)queue; 1488 (void)bufs; 1489 (void)nb_pkts; 1490 return 0; 1491 } 1492 1493 #if defined(RTE_TOOLCHAIN_GCC) 1494 #pragma GCC diagnostic push 1495 #pragma GCC diagnostic ignored "-Wcast-qual" 1496 #elif defined(RTE_TOOLCHAIN_CLANG) 1497 #pragma clang diagnostic push 1498 #pragma clang diagnostic ignored "-Wcast-qual" 1499 #endif 1500 1501 /* This function loopbacks all the received packets.*/ 1502 uint16_t 1503 dpaa2_dev_loopback_rx(void *queue, 1504 struct rte_mbuf **bufs __rte_unused, 1505 uint16_t nb_pkts) 1506 { 1507 /* Function receive frames for a given device and VQ*/ 1508 struct dpaa2_queue *dpaa2_q = (struct dpaa2_queue *)queue; 1509 struct qbman_result *dq_storage, *dq_storage1 = NULL; 1510 uint32_t fqid = dpaa2_q->fqid; 1511 int ret, num_rx = 0, num_tx = 0, pull_size; 1512 uint8_t pending, status; 1513 struct qbman_swp *swp; 1514 struct qbman_fd *fd[DPAA2_LX2_DQRR_RING_SIZE]; 1515 struct qbman_pull_desc pulldesc; 1516 struct qbman_eq_desc eqdesc; 1517 struct queue_storage_info_t *q_storage = dpaa2_q->q_storage; 1518 struct rte_eth_dev_data *eth_data = dpaa2_q->eth_data; 1519 struct dpaa2_dev_priv *priv = eth_data->dev_private; 1520 struct dpaa2_queue *tx_q = priv->tx_vq[0]; 1521 /* todo - currently we are using 1st TX queue only for loopback*/ 1522 1523 if (unlikely(!DPAA2_PER_LCORE_ETHRX_DPIO)) { 1524 ret = dpaa2_affine_qbman_ethrx_swp(); 1525 if (ret) { 1526 DPAA2_PMD_ERR("Failure in affining portal"); 1527 return 0; 1528 } 1529 } 1530 swp = DPAA2_PER_LCORE_ETHRX_PORTAL; 1531 pull_size = (nb_pkts > dpaa2_dqrr_size) ? dpaa2_dqrr_size : nb_pkts; 1532 if (unlikely(!q_storage->active_dqs)) { 1533 q_storage->toggle = 0; 1534 dq_storage = q_storage->dq_storage[q_storage->toggle]; 1535 q_storage->last_num_pkts = pull_size; 1536 qbman_pull_desc_clear(&pulldesc); 1537 qbman_pull_desc_set_numframes(&pulldesc, 1538 q_storage->last_num_pkts); 1539 qbman_pull_desc_set_fq(&pulldesc, fqid); 1540 qbman_pull_desc_set_storage(&pulldesc, dq_storage, 1541 (size_t)(DPAA2_VADDR_TO_IOVA(dq_storage)), 1); 1542 if (check_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index)) { 1543 while (!qbman_check_command_complete( 1544 get_swp_active_dqs( 1545 DPAA2_PER_LCORE_ETHRX_DPIO->index))) 1546 ; 1547 clear_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index); 1548 } 1549 while (1) { 1550 if (qbman_swp_pull(swp, &pulldesc)) { 1551 DPAA2_PMD_DP_DEBUG( 1552 "VDQ command not issued.QBMAN busy\n"); 1553 /* Portal was busy, try again */ 1554 continue; 1555 } 1556 break; 1557 } 1558 q_storage->active_dqs = dq_storage; 1559 q_storage->active_dpio_id = DPAA2_PER_LCORE_ETHRX_DPIO->index; 1560 set_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index, 1561 dq_storage); 1562 } 1563 1564 dq_storage = q_storage->active_dqs; 1565 rte_prefetch0((void *)(size_t)(dq_storage)); 1566 rte_prefetch0((void *)(size_t)(dq_storage + 1)); 1567 1568 /* Prepare next pull descriptor. This will give space for the 1569 * prefething done on DQRR entries 1570 */ 1571 q_storage->toggle ^= 1; 1572 dq_storage1 = q_storage->dq_storage[q_storage->toggle]; 1573 qbman_pull_desc_clear(&pulldesc); 1574 qbman_pull_desc_set_numframes(&pulldesc, pull_size); 1575 qbman_pull_desc_set_fq(&pulldesc, fqid); 1576 qbman_pull_desc_set_storage(&pulldesc, dq_storage1, 1577 (size_t)(DPAA2_VADDR_TO_IOVA(dq_storage1)), 1); 1578 1579 /*Prepare enqueue descriptor*/ 1580 qbman_eq_desc_clear(&eqdesc); 1581 qbman_eq_desc_set_no_orp(&eqdesc, DPAA2_EQ_RESP_ERR_FQ); 1582 qbman_eq_desc_set_response(&eqdesc, 0, 0); 1583 qbman_eq_desc_set_fq(&eqdesc, tx_q->fqid); 1584 1585 /* Check if the previous issued command is completed. 1586 * Also seems like the SWP is shared between the Ethernet Driver 1587 * and the SEC driver. 1588 */ 1589 while (!qbman_check_command_complete(dq_storage)) 1590 ; 1591 if (dq_storage == get_swp_active_dqs(q_storage->active_dpio_id)) 1592 clear_swp_active_dqs(q_storage->active_dpio_id); 1593 1594 pending = 1; 1595 1596 do { 1597 /* Loop until the dq_storage is updated with 1598 * new token by QBMAN 1599 */ 1600 while (!qbman_check_new_result(dq_storage)) 1601 ; 1602 rte_prefetch0((void *)((size_t)(dq_storage + 2))); 1603 /* Check whether Last Pull command is Expired and 1604 * setting Condition for Loop termination 1605 */ 1606 if (qbman_result_DQ_is_pull_complete(dq_storage)) { 1607 pending = 0; 1608 /* Check for valid frame. */ 1609 status = qbman_result_DQ_flags(dq_storage); 1610 if (unlikely((status & QBMAN_DQ_STAT_VALIDFRAME) == 0)) 1611 continue; 1612 } 1613 fd[num_rx] = (struct qbman_fd *)qbman_result_DQ_fd(dq_storage); 1614 1615 dq_storage++; 1616 num_rx++; 1617 } while (pending); 1618 1619 while (num_tx < num_rx) { 1620 num_tx += qbman_swp_enqueue_multiple_fd(swp, &eqdesc, 1621 &fd[num_tx], 0, num_rx - num_tx); 1622 } 1623 1624 if (check_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index)) { 1625 while (!qbman_check_command_complete( 1626 get_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index))) 1627 ; 1628 clear_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index); 1629 } 1630 /* issue a volatile dequeue command for next pull */ 1631 while (1) { 1632 if (qbman_swp_pull(swp, &pulldesc)) { 1633 DPAA2_PMD_DP_DEBUG("VDQ command is not issued." 1634 "QBMAN is busy (2)\n"); 1635 continue; 1636 } 1637 break; 1638 } 1639 q_storage->active_dqs = dq_storage1; 1640 q_storage->active_dpio_id = DPAA2_PER_LCORE_ETHRX_DPIO->index; 1641 set_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index, dq_storage1); 1642 1643 dpaa2_q->rx_pkts += num_rx; 1644 dpaa2_q->tx_pkts += num_tx; 1645 1646 return 0; 1647 } 1648 #if defined(RTE_TOOLCHAIN_GCC) 1649 #pragma GCC diagnostic pop 1650 #elif defined(RTE_TOOLCHAIN_CLANG) 1651 #pragma clang diagnostic pop 1652 #endif 1653