1 /* SPDX-License-Identifier: BSD-3-Clause 2 * 3 * Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved. 4 * Copyright 2016-2021 NXP 5 * 6 */ 7 8 #include <time.h> 9 #include <net/if.h> 10 11 #include <rte_mbuf.h> 12 #include <ethdev_driver.h> 13 #include <rte_malloc.h> 14 #include <rte_memcpy.h> 15 #include <rte_string_fns.h> 16 #include <rte_dev.h> 17 #include <rte_hexdump.h> 18 19 #include <rte_fslmc.h> 20 #include <fslmc_vfio.h> 21 #include <dpaa2_hw_pvt.h> 22 #include <dpaa2_hw_dpio.h> 23 #include <dpaa2_hw_mempool.h> 24 25 #include "dpaa2_pmd_logs.h" 26 #include "dpaa2_ethdev.h" 27 #include "base/dpaa2_hw_dpni_annot.h" 28 29 static inline uint32_t __rte_hot 30 dpaa2_dev_rx_parse_slow(struct rte_mbuf *mbuf, 31 struct dpaa2_annot_hdr *annotation); 32 33 static void enable_tx_tstamp(struct qbman_fd *fd) __rte_unused; 34 35 static inline rte_mbuf_timestamp_t * 36 dpaa2_timestamp_dynfield(struct rte_mbuf *mbuf) 37 { 38 return RTE_MBUF_DYNFIELD(mbuf, 39 dpaa2_timestamp_dynfield_offset, rte_mbuf_timestamp_t *); 40 } 41 42 #define DPAA2_MBUF_TO_CONTIG_FD(_mbuf, _fd, _bpid) do { \ 43 DPAA2_SET_FD_ADDR(_fd, DPAA2_MBUF_VADDR_TO_IOVA(_mbuf)); \ 44 DPAA2_SET_FD_LEN(_fd, _mbuf->data_len); \ 45 DPAA2_SET_ONLY_FD_BPID(_fd, _bpid); \ 46 DPAA2_SET_FD_OFFSET(_fd, _mbuf->data_off); \ 47 DPAA2_SET_FD_FRC(_fd, 0); \ 48 DPAA2_RESET_FD_CTRL(_fd); \ 49 DPAA2_RESET_FD_FLC(_fd); \ 50 } while (0) 51 52 static inline void __rte_hot 53 dpaa2_dev_rx_parse_new(struct rte_mbuf *m, const struct qbman_fd *fd, 54 void *hw_annot_addr) 55 { 56 uint16_t frc = DPAA2_GET_FD_FRC_PARSE_SUM(fd); 57 struct dpaa2_annot_hdr *annotation = 58 (struct dpaa2_annot_hdr *)hw_annot_addr; 59 60 m->packet_type = RTE_PTYPE_UNKNOWN; 61 switch (frc) { 62 case DPAA2_PKT_TYPE_ETHER: 63 m->packet_type = RTE_PTYPE_L2_ETHER; 64 break; 65 case DPAA2_PKT_TYPE_IPV4: 66 m->packet_type = RTE_PTYPE_L2_ETHER | 67 RTE_PTYPE_L3_IPV4; 68 break; 69 case DPAA2_PKT_TYPE_IPV6: 70 m->packet_type = RTE_PTYPE_L2_ETHER | 71 RTE_PTYPE_L3_IPV6; 72 break; 73 case DPAA2_PKT_TYPE_IPV4_EXT: 74 m->packet_type = RTE_PTYPE_L2_ETHER | 75 RTE_PTYPE_L3_IPV4_EXT; 76 break; 77 case DPAA2_PKT_TYPE_IPV6_EXT: 78 m->packet_type = RTE_PTYPE_L2_ETHER | 79 RTE_PTYPE_L3_IPV6_EXT; 80 break; 81 case DPAA2_PKT_TYPE_IPV4_TCP: 82 m->packet_type = RTE_PTYPE_L2_ETHER | 83 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_TCP; 84 break; 85 case DPAA2_PKT_TYPE_IPV6_TCP: 86 m->packet_type = RTE_PTYPE_L2_ETHER | 87 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_TCP; 88 break; 89 case DPAA2_PKT_TYPE_IPV4_UDP: 90 m->packet_type = RTE_PTYPE_L2_ETHER | 91 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_UDP; 92 break; 93 case DPAA2_PKT_TYPE_IPV6_UDP: 94 m->packet_type = RTE_PTYPE_L2_ETHER | 95 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_UDP; 96 break; 97 case DPAA2_PKT_TYPE_IPV4_SCTP: 98 m->packet_type = RTE_PTYPE_L2_ETHER | 99 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_SCTP; 100 break; 101 case DPAA2_PKT_TYPE_IPV6_SCTP: 102 m->packet_type = RTE_PTYPE_L2_ETHER | 103 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_SCTP; 104 break; 105 case DPAA2_PKT_TYPE_IPV4_ICMP: 106 m->packet_type = RTE_PTYPE_L2_ETHER | 107 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_ICMP; 108 break; 109 case DPAA2_PKT_TYPE_IPV6_ICMP: 110 m->packet_type = RTE_PTYPE_L2_ETHER | 111 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_ICMP; 112 break; 113 default: 114 m->packet_type = dpaa2_dev_rx_parse_slow(m, annotation); 115 } 116 m->hash.rss = fd->simple.flc_hi; 117 m->ol_flags |= RTE_MBUF_F_RX_RSS_HASH; 118 119 if (dpaa2_enable_ts[m->port]) { 120 *dpaa2_timestamp_dynfield(m) = annotation->word2; 121 m->ol_flags |= dpaa2_timestamp_rx_dynflag; 122 DPAA2_PMD_DP_DEBUG("pkt timestamp:0x%" PRIx64 "", 123 *dpaa2_timestamp_dynfield(m)); 124 } 125 126 DPAA2_PMD_DP_DEBUG("HW frc = 0x%x\t packet type =0x%x " 127 "ol_flags =0x%" PRIx64 "", 128 frc, m->packet_type, m->ol_flags); 129 } 130 131 static inline uint32_t __rte_hot 132 dpaa2_dev_rx_parse_slow(struct rte_mbuf *mbuf, 133 struct dpaa2_annot_hdr *annotation) 134 { 135 uint32_t pkt_type = RTE_PTYPE_UNKNOWN; 136 uint16_t *vlan_tci; 137 138 DPAA2_PMD_DP_DEBUG("(slow parse)annotation(3)=0x%" PRIx64 "\t" 139 "(4)=0x%" PRIx64 "\t", 140 annotation->word3, annotation->word4); 141 142 #if defined(RTE_LIBRTE_IEEE1588) 143 if (BIT_ISSET_AT_POS(annotation->word1, DPAA2_ETH_FAS_PTP)) { 144 mbuf->ol_flags |= RTE_MBUF_F_RX_IEEE1588_PTP; 145 mbuf->ol_flags |= RTE_MBUF_F_RX_IEEE1588_TMST; 146 } 147 #endif 148 149 if (BIT_ISSET_AT_POS(annotation->word3, L2_VLAN_1_PRESENT)) { 150 vlan_tci = rte_pktmbuf_mtod_offset(mbuf, uint16_t *, 151 (VLAN_TCI_OFFSET_1(annotation->word5) >> 16)); 152 mbuf->vlan_tci = rte_be_to_cpu_16(*vlan_tci); 153 mbuf->ol_flags |= RTE_MBUF_F_RX_VLAN; 154 pkt_type |= RTE_PTYPE_L2_ETHER_VLAN; 155 } else if (BIT_ISSET_AT_POS(annotation->word3, L2_VLAN_N_PRESENT)) { 156 vlan_tci = rte_pktmbuf_mtod_offset(mbuf, uint16_t *, 157 (VLAN_TCI_OFFSET_1(annotation->word5) >> 16)); 158 mbuf->vlan_tci = rte_be_to_cpu_16(*vlan_tci); 159 mbuf->ol_flags |= RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_QINQ; 160 pkt_type |= RTE_PTYPE_L2_ETHER_QINQ; 161 } 162 163 if (BIT_ISSET_AT_POS(annotation->word3, L2_ARP_PRESENT)) { 164 pkt_type |= RTE_PTYPE_L2_ETHER_ARP; 165 goto parse_done; 166 } else if (BIT_ISSET_AT_POS(annotation->word3, L2_ETH_MAC_PRESENT)) { 167 pkt_type |= RTE_PTYPE_L2_ETHER; 168 } else { 169 goto parse_done; 170 } 171 172 if (BIT_ISSET_AT_POS(annotation->word3, L2_MPLS_1_PRESENT | 173 L2_MPLS_N_PRESENT)) 174 pkt_type |= RTE_PTYPE_L2_ETHER_MPLS; 175 176 if (BIT_ISSET_AT_POS(annotation->word4, L3_IPV4_1_PRESENT | 177 L3_IPV4_N_PRESENT)) { 178 pkt_type |= RTE_PTYPE_L3_IPV4; 179 if (BIT_ISSET_AT_POS(annotation->word4, L3_IP_1_OPT_PRESENT | 180 L3_IP_N_OPT_PRESENT)) 181 pkt_type |= RTE_PTYPE_L3_IPV4_EXT; 182 if (BIT_ISSET_AT_POS(annotation->word4, L3_PROTO_IPSEC_ESP_PRESENT | 183 L3_PROTO_ESP_PRESENT)) 184 pkt_type |= RTE_PTYPE_TUNNEL_ESP; 185 186 } else if (BIT_ISSET_AT_POS(annotation->word4, L3_IPV6_1_PRESENT | 187 L3_IPV6_N_PRESENT)) { 188 pkt_type |= RTE_PTYPE_L3_IPV6; 189 if (BIT_ISSET_AT_POS(annotation->word4, L3_IP_1_OPT_PRESENT | 190 L3_IP_N_OPT_PRESENT)) 191 pkt_type |= RTE_PTYPE_L3_IPV6_EXT; 192 if (BIT_ISSET_AT_POS(annotation->word4, L3_PROTO_IPSEC_ESP_PRESENT | 193 L3_PROTO_ESP_PRESENT)) 194 pkt_type |= RTE_PTYPE_TUNNEL_ESP; 195 } else { 196 goto parse_done; 197 } 198 199 if (BIT_ISSET_AT_POS(annotation->word8, DPAA2_ETH_FAS_L3CE)) 200 mbuf->ol_flags |= RTE_MBUF_F_RX_IP_CKSUM_BAD; 201 else if (BIT_ISSET_AT_POS(annotation->word8, DPAA2_ETH_FAS_L4CE)) 202 mbuf->ol_flags |= RTE_MBUF_F_RX_L4_CKSUM_BAD; 203 204 if (BIT_ISSET_AT_POS(annotation->word4, L3_IP_1_FIRST_FRAGMENT | 205 L3_IP_1_MORE_FRAGMENT | 206 L3_IP_N_FIRST_FRAGMENT | 207 L3_IP_N_MORE_FRAGMENT)) { 208 pkt_type |= RTE_PTYPE_L4_FRAG; 209 goto parse_done; 210 } else { 211 pkt_type |= RTE_PTYPE_L4_NONFRAG; 212 } 213 214 if (BIT_ISSET_AT_POS(annotation->word4, L3_PROTO_UDP_PRESENT)) 215 pkt_type |= RTE_PTYPE_L4_UDP; 216 217 else if (BIT_ISSET_AT_POS(annotation->word4, L3_PROTO_TCP_PRESENT)) 218 pkt_type |= RTE_PTYPE_L4_TCP; 219 220 else if (BIT_ISSET_AT_POS(annotation->word4, L3_PROTO_SCTP_PRESENT)) 221 pkt_type |= RTE_PTYPE_L4_SCTP; 222 223 else if (BIT_ISSET_AT_POS(annotation->word4, L3_PROTO_ICMP_PRESENT)) 224 pkt_type |= RTE_PTYPE_L4_ICMP; 225 226 else if (BIT_ISSET_AT_POS(annotation->word4, L3_IP_UNKNOWN_PROTOCOL)) 227 pkt_type |= RTE_PTYPE_UNKNOWN; 228 229 parse_done: 230 return pkt_type; 231 } 232 233 static inline uint32_t __rte_hot 234 dpaa2_dev_rx_parse(struct rte_mbuf *mbuf, void *hw_annot_addr) 235 { 236 struct dpaa2_annot_hdr *annotation = 237 (struct dpaa2_annot_hdr *)hw_annot_addr; 238 239 DPAA2_PMD_DP_DEBUG("(fast parse) Annotation = 0x%" PRIx64 "\t", 240 annotation->word4); 241 242 if (BIT_ISSET_AT_POS(annotation->word8, DPAA2_ETH_FAS_L3CE)) 243 mbuf->ol_flags |= RTE_MBUF_F_RX_IP_CKSUM_BAD; 244 else if (BIT_ISSET_AT_POS(annotation->word8, DPAA2_ETH_FAS_L4CE)) 245 mbuf->ol_flags |= RTE_MBUF_F_RX_L4_CKSUM_BAD; 246 247 if (dpaa2_enable_ts[mbuf->port]) { 248 *dpaa2_timestamp_dynfield(mbuf) = annotation->word2; 249 mbuf->ol_flags |= dpaa2_timestamp_rx_dynflag; 250 DPAA2_PMD_DP_DEBUG("pkt timestamp: 0x%" PRIx64 "", 251 *dpaa2_timestamp_dynfield(mbuf)); 252 } 253 254 /* Check detailed parsing requirement */ 255 if (annotation->word3 & 0x7FFFFC3FFFF) 256 return dpaa2_dev_rx_parse_slow(mbuf, annotation); 257 258 /* Return some common types from parse processing */ 259 switch (annotation->word4) { 260 case DPAA2_L3_IPv4: 261 return RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4; 262 case DPAA2_L3_IPv6: 263 return RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6; 264 case DPAA2_L3_IPv4_TCP: 265 return RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 | 266 RTE_PTYPE_L4_TCP; 267 case DPAA2_L3_IPv4_UDP: 268 return RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 | 269 RTE_PTYPE_L4_UDP; 270 case DPAA2_L3_IPv6_TCP: 271 return RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 | 272 RTE_PTYPE_L4_TCP; 273 case DPAA2_L3_IPv6_UDP: 274 return RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 | 275 RTE_PTYPE_L4_UDP; 276 default: 277 break; 278 } 279 280 return dpaa2_dev_rx_parse_slow(mbuf, annotation); 281 } 282 283 static inline struct rte_mbuf *__rte_hot 284 eth_sg_fd_to_mbuf(const struct qbman_fd *fd, 285 int port_id) 286 { 287 struct qbman_sge *sgt, *sge; 288 size_t sg_addr, fd_addr; 289 int i = 0; 290 void *hw_annot_addr; 291 struct rte_mbuf *first_seg, *next_seg, *cur_seg, *temp; 292 293 fd_addr = (size_t)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd)); 294 hw_annot_addr = (void *)(fd_addr + DPAA2_FD_PTA_SIZE); 295 296 /* Get Scatter gather table address */ 297 sgt = (struct qbman_sge *)(fd_addr + DPAA2_GET_FD_OFFSET(fd)); 298 299 sge = &sgt[i++]; 300 sg_addr = (size_t)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FLE_ADDR(sge)); 301 302 /* First Scatter gather entry */ 303 first_seg = DPAA2_INLINE_MBUF_FROM_BUF(sg_addr, 304 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size); 305 /* Prepare all the metadata for first segment */ 306 first_seg->buf_addr = (uint8_t *)sg_addr; 307 first_seg->ol_flags = 0; 308 first_seg->data_off = DPAA2_GET_FLE_OFFSET(sge); 309 first_seg->data_len = sge->length & 0x1FFFF; 310 first_seg->pkt_len = DPAA2_GET_FD_LEN(fd); 311 first_seg->nb_segs = 1; 312 first_seg->next = NULL; 313 first_seg->port = port_id; 314 if (dpaa2_svr_family == SVR_LX2160A) 315 dpaa2_dev_rx_parse_new(first_seg, fd, hw_annot_addr); 316 else 317 first_seg->packet_type = 318 dpaa2_dev_rx_parse(first_seg, hw_annot_addr); 319 320 rte_mbuf_refcnt_set(first_seg, 1); 321 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG 322 rte_mempool_check_cookies(rte_mempool_from_obj((void *)first_seg), 323 (void **)&first_seg, 1, 1); 324 #endif 325 cur_seg = first_seg; 326 while (!DPAA2_SG_IS_FINAL(sge)) { 327 sge = &sgt[i++]; 328 sg_addr = (size_t)DPAA2_IOVA_TO_VADDR( 329 DPAA2_GET_FLE_ADDR(sge)); 330 next_seg = DPAA2_INLINE_MBUF_FROM_BUF(sg_addr, 331 rte_dpaa2_bpid_info[DPAA2_GET_FLE_BPID(sge)].meta_data_size); 332 next_seg->buf_addr = (uint8_t *)sg_addr; 333 next_seg->data_off = DPAA2_GET_FLE_OFFSET(sge); 334 next_seg->data_len = sge->length & 0x1FFFF; 335 first_seg->nb_segs += 1; 336 rte_mbuf_refcnt_set(next_seg, 1); 337 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG 338 rte_mempool_check_cookies(rte_mempool_from_obj((void *)next_seg), 339 (void **)&next_seg, 1, 1); 340 #endif 341 cur_seg->next = next_seg; 342 next_seg->next = NULL; 343 cur_seg = next_seg; 344 } 345 temp = DPAA2_INLINE_MBUF_FROM_BUF(fd_addr, 346 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size); 347 rte_mbuf_refcnt_set(temp, 1); 348 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG 349 rte_mempool_check_cookies(rte_mempool_from_obj((void *)temp), 350 (void **)&temp, 1, 1); 351 #endif 352 rte_pktmbuf_free_seg(temp); 353 354 return (void *)first_seg; 355 } 356 357 static inline struct rte_mbuf *__rte_hot 358 eth_fd_to_mbuf(const struct qbman_fd *fd, 359 int port_id) 360 { 361 void *v_addr = DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd)); 362 void *hw_annot_addr = (void *)((size_t)v_addr + DPAA2_FD_PTA_SIZE); 363 struct rte_mbuf *mbuf = DPAA2_INLINE_MBUF_FROM_BUF(v_addr, 364 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size); 365 366 /* need to repopulated some of the fields, 367 * as they may have changed in last transmission 368 */ 369 mbuf->nb_segs = 1; 370 mbuf->ol_flags = 0; 371 mbuf->data_off = DPAA2_GET_FD_OFFSET(fd); 372 mbuf->data_len = DPAA2_GET_FD_LEN(fd); 373 mbuf->pkt_len = mbuf->data_len; 374 mbuf->port = port_id; 375 mbuf->next = NULL; 376 rte_mbuf_refcnt_set(mbuf, 1); 377 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG 378 rte_mempool_check_cookies(rte_mempool_from_obj((void *)mbuf), 379 (void **)&mbuf, 1, 1); 380 #endif 381 382 /* Parse the packet */ 383 /* parse results for LX2 are there in FRC field of FD. 384 * For other DPAA2 platforms , parse results are after 385 * the private - sw annotation area 386 */ 387 388 if (dpaa2_svr_family == SVR_LX2160A) 389 dpaa2_dev_rx_parse_new(mbuf, fd, hw_annot_addr); 390 else 391 mbuf->packet_type = dpaa2_dev_rx_parse(mbuf, hw_annot_addr); 392 393 DPAA2_PMD_DP_DEBUG("to mbuf - mbuf =%p, mbuf->buf_addr =%p, off = %d," 394 "fd_off=%d fd =%" PRIx64 ", meta = %d bpid =%d, len=%d\n", 395 mbuf, mbuf->buf_addr, mbuf->data_off, 396 DPAA2_GET_FD_OFFSET(fd), DPAA2_GET_FD_ADDR(fd), 397 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size, 398 DPAA2_GET_FD_BPID(fd), DPAA2_GET_FD_LEN(fd)); 399 400 return mbuf; 401 } 402 403 static int __rte_noinline __rte_hot 404 eth_mbuf_to_sg_fd(struct rte_mbuf *mbuf, 405 struct qbman_fd *fd, 406 struct rte_mempool *mp, uint16_t bpid) 407 { 408 struct rte_mbuf *cur_seg = mbuf, *prev_seg, *mi, *temp; 409 struct qbman_sge *sgt, *sge = NULL; 410 int i, offset = 0; 411 412 #ifdef RTE_LIBRTE_IEEE1588 413 /* annotation area for timestamp in first buffer */ 414 offset = 0x64; 415 #endif 416 if (RTE_MBUF_DIRECT(mbuf) && 417 (mbuf->data_off > (mbuf->nb_segs * sizeof(struct qbman_sge) 418 + offset))) { 419 temp = mbuf; 420 if (rte_mbuf_refcnt_read(temp) > 1) { 421 /* If refcnt > 1, invalid bpid is set to ensure 422 * buffer is not freed by HW 423 */ 424 fd->simple.bpid_offset = 0; 425 DPAA2_SET_FD_IVP(fd); 426 rte_mbuf_refcnt_update(temp, -1); 427 } else { 428 DPAA2_SET_ONLY_FD_BPID(fd, bpid); 429 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG 430 rte_mempool_check_cookies(rte_mempool_from_obj((void *)temp), 431 (void **)&temp, 1, 0); 432 #endif 433 } 434 DPAA2_SET_FD_OFFSET(fd, offset); 435 } else { 436 temp = rte_pktmbuf_alloc(mp); 437 if (temp == NULL) { 438 DPAA2_PMD_DP_DEBUG("No memory to allocate S/G table\n"); 439 return -ENOMEM; 440 } 441 DPAA2_SET_ONLY_FD_BPID(fd, bpid); 442 DPAA2_SET_FD_OFFSET(fd, temp->data_off); 443 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG 444 rte_mempool_check_cookies(rte_mempool_from_obj((void *)temp), 445 (void **)&temp, 1, 0); 446 #endif 447 } 448 DPAA2_SET_FD_ADDR(fd, DPAA2_MBUF_VADDR_TO_IOVA(temp)); 449 DPAA2_SET_FD_LEN(fd, mbuf->pkt_len); 450 DPAA2_FD_SET_FORMAT(fd, qbman_fd_sg); 451 DPAA2_RESET_FD_FRC(fd); 452 DPAA2_RESET_FD_CTRL(fd); 453 DPAA2_RESET_FD_FLC(fd); 454 /*Set Scatter gather table and Scatter gather entries*/ 455 sgt = (struct qbman_sge *)( 456 (size_t)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd)) 457 + DPAA2_GET_FD_OFFSET(fd)); 458 459 for (i = 0; i < mbuf->nb_segs; i++) { 460 sge = &sgt[i]; 461 /*Resetting the buffer pool id and offset field*/ 462 sge->fin_bpid_offset = 0; 463 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(cur_seg)); 464 DPAA2_SET_FLE_OFFSET(sge, cur_seg->data_off); 465 sge->length = cur_seg->data_len; 466 if (RTE_MBUF_DIRECT(cur_seg)) { 467 /* if we are using inline SGT in same buffers 468 * set the FLE FMT as Frame Data Section 469 */ 470 if (temp == cur_seg) { 471 DPAA2_SG_SET_FORMAT(sge, qbman_fd_list); 472 DPAA2_SET_FLE_IVP(sge); 473 } else { 474 if (rte_mbuf_refcnt_read(cur_seg) > 1) { 475 /* If refcnt > 1, invalid bpid is set to ensure 476 * buffer is not freed by HW 477 */ 478 DPAA2_SET_FLE_IVP(sge); 479 rte_mbuf_refcnt_update(cur_seg, -1); 480 } else { 481 DPAA2_SET_FLE_BPID(sge, 482 mempool_to_bpid(cur_seg->pool)); 483 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG 484 rte_mempool_check_cookies(rte_mempool_from_obj((void *)cur_seg), 485 (void **)&cur_seg, 1, 0); 486 #endif 487 } 488 } 489 cur_seg = cur_seg->next; 490 } else if (RTE_MBUF_HAS_EXTBUF(cur_seg)) { 491 DPAA2_SET_FLE_IVP(sge); 492 cur_seg = cur_seg->next; 493 } else { 494 /* Get owner MBUF from indirect buffer */ 495 mi = rte_mbuf_from_indirect(cur_seg); 496 if (rte_mbuf_refcnt_read(mi) > 1) { 497 /* If refcnt > 1, invalid bpid is set to ensure 498 * owner buffer is not freed by HW 499 */ 500 DPAA2_SET_FLE_IVP(sge); 501 } else { 502 DPAA2_SET_FLE_BPID(sge, 503 mempool_to_bpid(mi->pool)); 504 rte_mbuf_refcnt_update(mi, 1); 505 } 506 prev_seg = cur_seg; 507 cur_seg = cur_seg->next; 508 prev_seg->next = NULL; 509 rte_pktmbuf_free(prev_seg); 510 } 511 } 512 DPAA2_SG_SET_FINAL(sge, true); 513 return 0; 514 } 515 516 static void 517 eth_mbuf_to_fd(struct rte_mbuf *mbuf, 518 struct qbman_fd *fd, uint16_t bpid) __rte_unused; 519 520 static void __rte_noinline __rte_hot 521 eth_mbuf_to_fd(struct rte_mbuf *mbuf, 522 struct qbman_fd *fd, uint16_t bpid) 523 { 524 DPAA2_MBUF_TO_CONTIG_FD(mbuf, fd, bpid); 525 526 DPAA2_PMD_DP_DEBUG("mbuf =%p, mbuf->buf_addr =%p, off = %d," 527 "fd_off=%d fd =%" PRIx64 ", meta = %d bpid =%d, len=%d\n", 528 mbuf, mbuf->buf_addr, mbuf->data_off, 529 DPAA2_GET_FD_OFFSET(fd), DPAA2_GET_FD_ADDR(fd), 530 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size, 531 DPAA2_GET_FD_BPID(fd), DPAA2_GET_FD_LEN(fd)); 532 if (RTE_MBUF_DIRECT(mbuf)) { 533 if (rte_mbuf_refcnt_read(mbuf) > 1) { 534 DPAA2_SET_FD_IVP(fd); 535 rte_mbuf_refcnt_update(mbuf, -1); 536 } 537 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG 538 else 539 rte_mempool_check_cookies(rte_mempool_from_obj((void *)mbuf), 540 (void **)&mbuf, 1, 0); 541 #endif 542 } else if (RTE_MBUF_HAS_EXTBUF(mbuf)) { 543 DPAA2_SET_FD_IVP(fd); 544 } else { 545 struct rte_mbuf *mi; 546 547 mi = rte_mbuf_from_indirect(mbuf); 548 if (rte_mbuf_refcnt_read(mi) > 1) 549 DPAA2_SET_FD_IVP(fd); 550 else 551 rte_mbuf_refcnt_update(mi, 1); 552 rte_pktmbuf_free(mbuf); 553 } 554 } 555 556 static inline int __rte_hot 557 eth_copy_mbuf_to_fd(struct rte_mbuf *mbuf, 558 struct qbman_fd *fd, uint16_t bpid) 559 { 560 struct rte_mbuf *m; 561 void *mb = NULL; 562 563 if (rte_dpaa2_mbuf_alloc_bulk( 564 rte_dpaa2_bpid_info[bpid].bp_list->mp, &mb, 1)) { 565 DPAA2_PMD_DP_DEBUG("Unable to allocated DPAA2 buffer\n"); 566 return -1; 567 } 568 m = (struct rte_mbuf *)mb; 569 memcpy((char *)m->buf_addr + mbuf->data_off, 570 (void *)((char *)mbuf->buf_addr + mbuf->data_off), 571 mbuf->pkt_len); 572 573 /* Copy required fields */ 574 m->data_off = mbuf->data_off; 575 m->ol_flags = mbuf->ol_flags; 576 m->packet_type = mbuf->packet_type; 577 m->tx_offload = mbuf->tx_offload; 578 579 DPAA2_MBUF_TO_CONTIG_FD(m, fd, bpid); 580 581 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG 582 rte_mempool_check_cookies(rte_mempool_from_obj((void *)m), 583 (void **)&m, 1, 0); 584 #endif 585 DPAA2_PMD_DP_DEBUG( 586 "mbuf: %p, BMAN buf addr: %p, fdaddr: %" PRIx64 ", bpid: %d," 587 " meta: %d, off: %d, len: %d\n", 588 (void *)mbuf, 589 mbuf->buf_addr, 590 DPAA2_GET_FD_ADDR(fd), 591 DPAA2_GET_FD_BPID(fd), 592 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size, 593 DPAA2_GET_FD_OFFSET(fd), 594 DPAA2_GET_FD_LEN(fd)); 595 596 return 0; 597 } 598 599 static void 600 dump_err_pkts(struct dpaa2_queue *dpaa2_q) 601 { 602 /* Function receive frames for a given device and VQ */ 603 struct qbman_result *dq_storage; 604 uint32_t fqid = dpaa2_q->fqid; 605 int ret, num_rx = 0, num_pulled; 606 uint8_t pending, status; 607 struct qbman_swp *swp; 608 const struct qbman_fd *fd; 609 struct qbman_pull_desc pulldesc; 610 struct rte_eth_dev_data *eth_data = dpaa2_q->eth_data; 611 uint32_t lcore_id = rte_lcore_id(); 612 void *v_addr, *hw_annot_addr; 613 struct dpaa2_fas *fas; 614 615 if (unlikely(!DPAA2_PER_LCORE_DPIO)) { 616 ret = dpaa2_affine_qbman_swp(); 617 if (ret) { 618 DPAA2_PMD_ERR("Failed to allocate IO portal, tid: %d\n", 619 rte_gettid()); 620 return; 621 } 622 } 623 swp = DPAA2_PER_LCORE_PORTAL; 624 625 dq_storage = dpaa2_q->q_storage[lcore_id].dq_storage[0]; 626 qbman_pull_desc_clear(&pulldesc); 627 qbman_pull_desc_set_fq(&pulldesc, fqid); 628 qbman_pull_desc_set_storage(&pulldesc, dq_storage, 629 (size_t)(DPAA2_VADDR_TO_IOVA(dq_storage)), 1); 630 qbman_pull_desc_set_numframes(&pulldesc, dpaa2_dqrr_size); 631 632 while (1) { 633 if (qbman_swp_pull(swp, &pulldesc)) { 634 DPAA2_PMD_DP_DEBUG("VDQ command is not issued.QBMAN is busy\n"); 635 /* Portal was busy, try again */ 636 continue; 637 } 638 break; 639 } 640 641 /* Check if the previous issued command is completed. */ 642 while (!qbman_check_command_complete(dq_storage)) 643 ; 644 645 num_pulled = 0; 646 pending = 1; 647 do { 648 /* Loop until the dq_storage is updated with 649 * new token by QBMAN 650 */ 651 while (!qbman_check_new_result(dq_storage)) 652 ; 653 654 /* Check whether Last Pull command is Expired and 655 * setting Condition for Loop termination 656 */ 657 if (qbman_result_DQ_is_pull_complete(dq_storage)) { 658 pending = 0; 659 /* Check for valid frame. */ 660 status = qbman_result_DQ_flags(dq_storage); 661 if (unlikely((status & 662 QBMAN_DQ_STAT_VALIDFRAME) == 0)) 663 continue; 664 } 665 fd = qbman_result_DQ_fd(dq_storage); 666 v_addr = DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd)); 667 hw_annot_addr = (void *)((size_t)v_addr + DPAA2_FD_PTA_SIZE); 668 fas = hw_annot_addr; 669 670 DPAA2_PMD_ERR("\n\n[%d] error packet on port[%d]:" 671 " fd_off: %d, fd_err: %x, fas_status: %x", 672 rte_lcore_id(), eth_data->port_id, 673 DPAA2_GET_FD_OFFSET(fd), DPAA2_GET_FD_ERR(fd), 674 fas->status); 675 rte_hexdump(stderr, "Error packet", v_addr, 676 DPAA2_GET_FD_OFFSET(fd) + DPAA2_GET_FD_LEN(fd)); 677 678 dq_storage++; 679 num_rx++; 680 num_pulled++; 681 } while (pending); 682 683 dpaa2_q->err_pkts += num_rx; 684 } 685 686 /* This function assumes that caller will be keep the same value for nb_pkts 687 * across calls per queue, if that is not the case, better use non-prefetch 688 * version of rx call. 689 * It will return the packets as requested in previous call without honoring 690 * the current nb_pkts or bufs space. 691 */ 692 uint16_t 693 dpaa2_dev_prefetch_rx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) 694 { 695 /* Function receive frames for a given device and VQ*/ 696 struct dpaa2_queue *dpaa2_q = (struct dpaa2_queue *)queue; 697 struct qbman_result *dq_storage, *dq_storage1 = NULL; 698 uint32_t fqid = dpaa2_q->fqid; 699 int ret, num_rx = 0, pull_size; 700 uint8_t pending, status; 701 struct qbman_swp *swp; 702 const struct qbman_fd *fd; 703 struct qbman_pull_desc pulldesc; 704 struct queue_storage_info_t *q_storage = dpaa2_q->q_storage; 705 struct rte_eth_dev_data *eth_data = dpaa2_q->eth_data; 706 struct dpaa2_dev_priv *priv = eth_data->dev_private; 707 708 if (unlikely(dpaa2_enable_err_queue)) 709 dump_err_pkts(priv->rx_err_vq); 710 711 if (unlikely(!DPAA2_PER_LCORE_ETHRX_DPIO)) { 712 ret = dpaa2_affine_qbman_ethrx_swp(); 713 if (ret) { 714 DPAA2_PMD_ERR("Failure in affining portal"); 715 return 0; 716 } 717 } 718 719 if (unlikely(!rte_dpaa2_bpid_info && 720 rte_eal_process_type() == RTE_PROC_SECONDARY)) 721 rte_dpaa2_bpid_info = dpaa2_q->bp_array; 722 723 swp = DPAA2_PER_LCORE_ETHRX_PORTAL; 724 pull_size = (nb_pkts > dpaa2_dqrr_size) ? dpaa2_dqrr_size : nb_pkts; 725 if (unlikely(!q_storage->active_dqs)) { 726 q_storage->toggle = 0; 727 dq_storage = q_storage->dq_storage[q_storage->toggle]; 728 q_storage->last_num_pkts = pull_size; 729 qbman_pull_desc_clear(&pulldesc); 730 qbman_pull_desc_set_numframes(&pulldesc, 731 q_storage->last_num_pkts); 732 qbman_pull_desc_set_fq(&pulldesc, fqid); 733 qbman_pull_desc_set_storage(&pulldesc, dq_storage, 734 (uint64_t)(DPAA2_VADDR_TO_IOVA(dq_storage)), 1); 735 if (check_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index)) { 736 while (!qbman_check_command_complete( 737 get_swp_active_dqs( 738 DPAA2_PER_LCORE_ETHRX_DPIO->index))) 739 ; 740 clear_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index); 741 } 742 while (1) { 743 if (qbman_swp_pull(swp, &pulldesc)) { 744 DPAA2_PMD_DP_DEBUG("VDQ command is not issued." 745 " QBMAN is busy (1)\n"); 746 /* Portal was busy, try again */ 747 continue; 748 } 749 break; 750 } 751 q_storage->active_dqs = dq_storage; 752 q_storage->active_dpio_id = DPAA2_PER_LCORE_ETHRX_DPIO->index; 753 set_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index, 754 dq_storage); 755 } 756 757 dq_storage = q_storage->active_dqs; 758 rte_prefetch0((void *)(size_t)(dq_storage)); 759 rte_prefetch0((void *)(size_t)(dq_storage + 1)); 760 761 /* Prepare next pull descriptor. This will give space for the 762 * prefetching done on DQRR entries 763 */ 764 q_storage->toggle ^= 1; 765 dq_storage1 = q_storage->dq_storage[q_storage->toggle]; 766 qbman_pull_desc_clear(&pulldesc); 767 qbman_pull_desc_set_numframes(&pulldesc, pull_size); 768 qbman_pull_desc_set_fq(&pulldesc, fqid); 769 qbman_pull_desc_set_storage(&pulldesc, dq_storage1, 770 (uint64_t)(DPAA2_VADDR_TO_IOVA(dq_storage1)), 1); 771 772 /* Check if the previous issued command is completed. 773 * Also seems like the SWP is shared between the Ethernet Driver 774 * and the SEC driver. 775 */ 776 while (!qbman_check_command_complete(dq_storage)) 777 ; 778 if (dq_storage == get_swp_active_dqs(q_storage->active_dpio_id)) 779 clear_swp_active_dqs(q_storage->active_dpio_id); 780 781 pending = 1; 782 783 do { 784 /* Loop until the dq_storage is updated with 785 * new token by QBMAN 786 */ 787 while (!qbman_check_new_result(dq_storage)) 788 ; 789 rte_prefetch0((void *)((size_t)(dq_storage + 2))); 790 /* Check whether Last Pull command is Expired and 791 * setting Condition for Loop termination 792 */ 793 if (qbman_result_DQ_is_pull_complete(dq_storage)) { 794 pending = 0; 795 /* Check for valid frame. */ 796 status = qbman_result_DQ_flags(dq_storage); 797 if (unlikely((status & QBMAN_DQ_STAT_VALIDFRAME) == 0)) 798 continue; 799 } 800 fd = qbman_result_DQ_fd(dq_storage); 801 802 #ifndef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA 803 if (dpaa2_svr_family != SVR_LX2160A) { 804 const struct qbman_fd *next_fd = 805 qbman_result_DQ_fd(dq_storage + 1); 806 /* Prefetch Annotation address for the parse results */ 807 rte_prefetch0(DPAA2_IOVA_TO_VADDR((DPAA2_GET_FD_ADDR( 808 next_fd) + DPAA2_FD_PTA_SIZE + 16))); 809 } 810 #endif 811 812 if (unlikely(DPAA2_FD_GET_FORMAT(fd) == qbman_fd_sg)) 813 bufs[num_rx] = eth_sg_fd_to_mbuf(fd, eth_data->port_id); 814 else 815 bufs[num_rx] = eth_fd_to_mbuf(fd, eth_data->port_id); 816 #if defined(RTE_LIBRTE_IEEE1588) 817 if (bufs[num_rx]->ol_flags & PKT_RX_IEEE1588_TMST) { 818 priv->rx_timestamp = 819 *dpaa2_timestamp_dynfield(bufs[num_rx]); 820 } 821 #endif 822 823 if (eth_data->dev_conf.rxmode.offloads & 824 RTE_ETH_RX_OFFLOAD_VLAN_STRIP) 825 rte_vlan_strip(bufs[num_rx]); 826 827 dq_storage++; 828 num_rx++; 829 } while (pending); 830 831 if (check_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index)) { 832 while (!qbman_check_command_complete( 833 get_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index))) 834 ; 835 clear_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index); 836 } 837 /* issue a volatile dequeue command for next pull */ 838 while (1) { 839 if (qbman_swp_pull(swp, &pulldesc)) { 840 DPAA2_PMD_DP_DEBUG("VDQ command is not issued." 841 "QBMAN is busy (2)\n"); 842 continue; 843 } 844 break; 845 } 846 q_storage->active_dqs = dq_storage1; 847 q_storage->active_dpio_id = DPAA2_PER_LCORE_ETHRX_DPIO->index; 848 set_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index, dq_storage1); 849 850 dpaa2_q->rx_pkts += num_rx; 851 852 return num_rx; 853 } 854 855 void __rte_hot 856 dpaa2_dev_process_parallel_event(struct qbman_swp *swp, 857 const struct qbman_fd *fd, 858 const struct qbman_result *dq, 859 struct dpaa2_queue *rxq, 860 struct rte_event *ev) 861 { 862 rte_prefetch0((void *)(size_t)(DPAA2_GET_FD_ADDR(fd) + 863 DPAA2_FD_PTA_SIZE + 16)); 864 865 ev->flow_id = rxq->ev.flow_id; 866 ev->sub_event_type = rxq->ev.sub_event_type; 867 ev->event_type = RTE_EVENT_TYPE_ETHDEV; 868 ev->op = RTE_EVENT_OP_NEW; 869 ev->sched_type = rxq->ev.sched_type; 870 ev->queue_id = rxq->ev.queue_id; 871 ev->priority = rxq->ev.priority; 872 873 ev->mbuf = eth_fd_to_mbuf(fd, rxq->eth_data->port_id); 874 875 qbman_swp_dqrr_consume(swp, dq); 876 } 877 878 void __rte_hot 879 dpaa2_dev_process_atomic_event(struct qbman_swp *swp __rte_unused, 880 const struct qbman_fd *fd, 881 const struct qbman_result *dq, 882 struct dpaa2_queue *rxq, 883 struct rte_event *ev) 884 { 885 uint8_t dqrr_index; 886 887 rte_prefetch0((void *)(size_t)(DPAA2_GET_FD_ADDR(fd) + 888 DPAA2_FD_PTA_SIZE + 16)); 889 890 ev->flow_id = rxq->ev.flow_id; 891 ev->sub_event_type = rxq->ev.sub_event_type; 892 ev->event_type = RTE_EVENT_TYPE_ETHDEV; 893 ev->op = RTE_EVENT_OP_NEW; 894 ev->sched_type = rxq->ev.sched_type; 895 ev->queue_id = rxq->ev.queue_id; 896 ev->priority = rxq->ev.priority; 897 898 ev->mbuf = eth_fd_to_mbuf(fd, rxq->eth_data->port_id); 899 900 dqrr_index = qbman_get_dqrr_idx(dq); 901 *dpaa2_seqn(ev->mbuf) = dqrr_index + 1; 902 DPAA2_PER_LCORE_DQRR_SIZE++; 903 DPAA2_PER_LCORE_DQRR_HELD |= 1 << dqrr_index; 904 DPAA2_PER_LCORE_DQRR_MBUF(dqrr_index) = ev->mbuf; 905 } 906 907 void __rte_hot 908 dpaa2_dev_process_ordered_event(struct qbman_swp *swp, 909 const struct qbman_fd *fd, 910 const struct qbman_result *dq, 911 struct dpaa2_queue *rxq, 912 struct rte_event *ev) 913 { 914 rte_prefetch0((void *)(size_t)(DPAA2_GET_FD_ADDR(fd) + 915 DPAA2_FD_PTA_SIZE + 16)); 916 917 ev->flow_id = rxq->ev.flow_id; 918 ev->sub_event_type = rxq->ev.sub_event_type; 919 ev->event_type = RTE_EVENT_TYPE_ETHDEV; 920 ev->op = RTE_EVENT_OP_NEW; 921 ev->sched_type = rxq->ev.sched_type; 922 ev->queue_id = rxq->ev.queue_id; 923 ev->priority = rxq->ev.priority; 924 925 ev->mbuf = eth_fd_to_mbuf(fd, rxq->eth_data->port_id); 926 927 *dpaa2_seqn(ev->mbuf) = DPAA2_ENQUEUE_FLAG_ORP; 928 *dpaa2_seqn(ev->mbuf) |= qbman_result_DQ_odpid(dq) << DPAA2_EQCR_OPRID_SHIFT; 929 *dpaa2_seqn(ev->mbuf) |= qbman_result_DQ_seqnum(dq) << DPAA2_EQCR_SEQNUM_SHIFT; 930 931 qbman_swp_dqrr_consume(swp, dq); 932 } 933 934 uint16_t 935 dpaa2_dev_rx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) 936 { 937 /* Function receive frames for a given device and VQ */ 938 struct dpaa2_queue *dpaa2_q = (struct dpaa2_queue *)queue; 939 struct qbman_result *dq_storage; 940 uint32_t fqid = dpaa2_q->fqid; 941 int ret, num_rx = 0, next_pull = nb_pkts, num_pulled; 942 uint8_t pending, status; 943 struct qbman_swp *swp; 944 const struct qbman_fd *fd; 945 struct qbman_pull_desc pulldesc; 946 struct rte_eth_dev_data *eth_data = dpaa2_q->eth_data; 947 struct dpaa2_dev_priv *priv = eth_data->dev_private; 948 949 if (unlikely(dpaa2_enable_err_queue)) 950 dump_err_pkts(priv->rx_err_vq); 951 952 if (unlikely(!DPAA2_PER_LCORE_DPIO)) { 953 ret = dpaa2_affine_qbman_swp(); 954 if (ret) { 955 DPAA2_PMD_ERR( 956 "Failed to allocate IO portal, tid: %d\n", 957 rte_gettid()); 958 return 0; 959 } 960 } 961 swp = DPAA2_PER_LCORE_PORTAL; 962 963 do { 964 dq_storage = dpaa2_q->q_storage->dq_storage[0]; 965 qbman_pull_desc_clear(&pulldesc); 966 qbman_pull_desc_set_fq(&pulldesc, fqid); 967 qbman_pull_desc_set_storage(&pulldesc, dq_storage, 968 (size_t)(DPAA2_VADDR_TO_IOVA(dq_storage)), 1); 969 970 if (next_pull > dpaa2_dqrr_size) { 971 qbman_pull_desc_set_numframes(&pulldesc, 972 dpaa2_dqrr_size); 973 next_pull -= dpaa2_dqrr_size; 974 } else { 975 qbman_pull_desc_set_numframes(&pulldesc, next_pull); 976 next_pull = 0; 977 } 978 979 while (1) { 980 if (qbman_swp_pull(swp, &pulldesc)) { 981 DPAA2_PMD_DP_DEBUG( 982 "VDQ command is not issued.QBMAN is busy\n"); 983 /* Portal was busy, try again */ 984 continue; 985 } 986 break; 987 } 988 989 rte_prefetch0((void *)((size_t)(dq_storage + 1))); 990 /* Check if the previous issued command is completed. */ 991 while (!qbman_check_command_complete(dq_storage)) 992 ; 993 994 num_pulled = 0; 995 pending = 1; 996 do { 997 /* Loop until the dq_storage is updated with 998 * new token by QBMAN 999 */ 1000 while (!qbman_check_new_result(dq_storage)) 1001 ; 1002 rte_prefetch0((void *)((size_t)(dq_storage + 2))); 1003 /* Check whether Last Pull command is Expired and 1004 * setting Condition for Loop termination 1005 */ 1006 if (qbman_result_DQ_is_pull_complete(dq_storage)) { 1007 pending = 0; 1008 /* Check for valid frame. */ 1009 status = qbman_result_DQ_flags(dq_storage); 1010 if (unlikely((status & 1011 QBMAN_DQ_STAT_VALIDFRAME) == 0)) 1012 continue; 1013 } 1014 fd = qbman_result_DQ_fd(dq_storage); 1015 1016 #ifndef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA 1017 if (dpaa2_svr_family != SVR_LX2160A) { 1018 const struct qbman_fd *next_fd = 1019 qbman_result_DQ_fd(dq_storage + 1); 1020 1021 /* Prefetch Annotation address for the parse 1022 * results. 1023 */ 1024 rte_prefetch0((DPAA2_IOVA_TO_VADDR( 1025 DPAA2_GET_FD_ADDR(next_fd) + 1026 DPAA2_FD_PTA_SIZE + 16))); 1027 } 1028 #endif 1029 1030 if (unlikely(DPAA2_FD_GET_FORMAT(fd) == qbman_fd_sg)) 1031 bufs[num_rx] = eth_sg_fd_to_mbuf(fd, 1032 eth_data->port_id); 1033 else 1034 bufs[num_rx] = eth_fd_to_mbuf(fd, 1035 eth_data->port_id); 1036 1037 #if defined(RTE_LIBRTE_IEEE1588) 1038 if (bufs[num_rx]->ol_flags & PKT_RX_IEEE1588_TMST) { 1039 priv->rx_timestamp = 1040 *dpaa2_timestamp_dynfield(bufs[num_rx]); 1041 } 1042 #endif 1043 1044 if (eth_data->dev_conf.rxmode.offloads & 1045 RTE_ETH_RX_OFFLOAD_VLAN_STRIP) { 1046 rte_vlan_strip(bufs[num_rx]); 1047 } 1048 1049 dq_storage++; 1050 num_rx++; 1051 num_pulled++; 1052 } while (pending); 1053 /* Last VDQ provided all packets and more packets are requested */ 1054 } while (next_pull && num_pulled == dpaa2_dqrr_size); 1055 1056 dpaa2_q->rx_pkts += num_rx; 1057 1058 return num_rx; 1059 } 1060 1061 uint16_t dpaa2_dev_tx_conf(void *queue) 1062 { 1063 /* Function receive frames for a given device and VQ */ 1064 struct dpaa2_queue *dpaa2_q = (struct dpaa2_queue *)queue; 1065 struct qbman_result *dq_storage; 1066 uint32_t fqid = dpaa2_q->fqid; 1067 int ret, num_tx_conf = 0, num_pulled; 1068 uint8_t pending, status; 1069 struct qbman_swp *swp; 1070 const struct qbman_fd *fd, *next_fd; 1071 struct qbman_pull_desc pulldesc; 1072 struct qbman_release_desc releasedesc; 1073 uint32_t bpid; 1074 uint64_t buf; 1075 #if defined(RTE_LIBRTE_IEEE1588) 1076 struct rte_eth_dev_data *eth_data = dpaa2_q->eth_data; 1077 struct dpaa2_dev_priv *priv = eth_data->dev_private; 1078 struct dpaa2_annot_hdr *annotation; 1079 void *v_addr; 1080 struct rte_mbuf *mbuf; 1081 #endif 1082 1083 if (unlikely(!DPAA2_PER_LCORE_DPIO)) { 1084 ret = dpaa2_affine_qbman_swp(); 1085 if (ret) { 1086 DPAA2_PMD_ERR( 1087 "Failed to allocate IO portal, tid: %d\n", 1088 rte_gettid()); 1089 return 0; 1090 } 1091 } 1092 swp = DPAA2_PER_LCORE_PORTAL; 1093 1094 do { 1095 dq_storage = dpaa2_q->q_storage->dq_storage[0]; 1096 qbman_pull_desc_clear(&pulldesc); 1097 qbman_pull_desc_set_fq(&pulldesc, fqid); 1098 qbman_pull_desc_set_storage(&pulldesc, dq_storage, 1099 (size_t)(DPAA2_VADDR_TO_IOVA(dq_storage)), 1); 1100 1101 qbman_pull_desc_set_numframes(&pulldesc, dpaa2_dqrr_size); 1102 1103 while (1) { 1104 if (qbman_swp_pull(swp, &pulldesc)) { 1105 DPAA2_PMD_DP_DEBUG("VDQ command is not issued." 1106 "QBMAN is busy\n"); 1107 /* Portal was busy, try again */ 1108 continue; 1109 } 1110 break; 1111 } 1112 1113 rte_prefetch0((void *)((size_t)(dq_storage + 1))); 1114 /* Check if the previous issued command is completed. */ 1115 while (!qbman_check_command_complete(dq_storage)) 1116 ; 1117 1118 num_pulled = 0; 1119 pending = 1; 1120 do { 1121 /* Loop until the dq_storage is updated with 1122 * new token by QBMAN 1123 */ 1124 while (!qbman_check_new_result(dq_storage)) 1125 ; 1126 rte_prefetch0((void *)((size_t)(dq_storage + 2))); 1127 /* Check whether Last Pull command is Expired and 1128 * setting Condition for Loop termination 1129 */ 1130 if (qbman_result_DQ_is_pull_complete(dq_storage)) { 1131 pending = 0; 1132 /* Check for valid frame. */ 1133 status = qbman_result_DQ_flags(dq_storage); 1134 if (unlikely((status & 1135 QBMAN_DQ_STAT_VALIDFRAME) == 0)) 1136 continue; 1137 } 1138 fd = qbman_result_DQ_fd(dq_storage); 1139 1140 next_fd = qbman_result_DQ_fd(dq_storage + 1); 1141 /* Prefetch Annotation address for the parse results */ 1142 rte_prefetch0((void *)(size_t) 1143 (DPAA2_GET_FD_ADDR(next_fd) + 1144 DPAA2_FD_PTA_SIZE + 16)); 1145 1146 bpid = DPAA2_GET_FD_BPID(fd); 1147 1148 /* Create a release descriptor required for releasing 1149 * buffers into QBMAN 1150 */ 1151 qbman_release_desc_clear(&releasedesc); 1152 qbman_release_desc_set_bpid(&releasedesc, bpid); 1153 1154 buf = DPAA2_GET_FD_ADDR(fd); 1155 /* feed them to bman */ 1156 do { 1157 ret = qbman_swp_release(swp, &releasedesc, 1158 &buf, 1); 1159 } while (ret == -EBUSY); 1160 1161 dq_storage++; 1162 num_tx_conf++; 1163 num_pulled++; 1164 #if defined(RTE_LIBRTE_IEEE1588) 1165 v_addr = DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd)); 1166 mbuf = DPAA2_INLINE_MBUF_FROM_BUF(v_addr, 1167 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size); 1168 1169 if (mbuf->ol_flags & PKT_TX_IEEE1588_TMST) { 1170 annotation = (struct dpaa2_annot_hdr *)((size_t) 1171 DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd)) + 1172 DPAA2_FD_PTA_SIZE); 1173 priv->tx_timestamp = annotation->word2; 1174 } 1175 #endif 1176 } while (pending); 1177 1178 /* Last VDQ provided all packets and more packets are requested */ 1179 } while (num_pulled == dpaa2_dqrr_size); 1180 1181 dpaa2_q->rx_pkts += num_tx_conf; 1182 1183 return num_tx_conf; 1184 } 1185 1186 /* Configure the egress frame annotation for timestamp update */ 1187 static void enable_tx_tstamp(struct qbman_fd *fd) 1188 { 1189 struct dpaa2_faead *fd_faead; 1190 1191 /* Set frame annotation status field as valid */ 1192 (fd)->simple.frc |= DPAA2_FD_FRC_FASV; 1193 1194 /* Set frame annotation egress action descriptor as valid */ 1195 (fd)->simple.frc |= DPAA2_FD_FRC_FAEADV; 1196 1197 /* Set Annotation Length as 128B */ 1198 (fd)->simple.ctrl |= DPAA2_FD_CTRL_ASAL; 1199 1200 /* enable update of confirmation frame annotation */ 1201 fd_faead = (struct dpaa2_faead *)((size_t) 1202 DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd)) + 1203 DPAA2_FD_PTA_SIZE + DPAA2_FD_HW_ANNOT_FAEAD_OFFSET); 1204 fd_faead->ctrl = DPAA2_ANNOT_FAEAD_A2V | DPAA2_ANNOT_FAEAD_UPDV | 1205 DPAA2_ANNOT_FAEAD_UPD; 1206 } 1207 1208 /* 1209 * Callback to handle sending packets through WRIOP based interface 1210 */ 1211 uint16_t 1212 dpaa2_dev_tx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) 1213 { 1214 /* Function to transmit the frames to given device and VQ*/ 1215 uint32_t loop, retry_count; 1216 int32_t ret; 1217 struct qbman_fd fd_arr[MAX_TX_RING_SLOTS]; 1218 struct rte_mbuf *mi; 1219 uint32_t frames_to_send; 1220 struct rte_mempool *mp; 1221 struct qbman_eq_desc eqdesc; 1222 struct dpaa2_queue *dpaa2_q = (struct dpaa2_queue *)queue; 1223 struct qbman_swp *swp; 1224 uint16_t num_tx = 0; 1225 uint16_t bpid; 1226 struct rte_eth_dev_data *eth_data = dpaa2_q->eth_data; 1227 struct dpaa2_dev_priv *priv = eth_data->dev_private; 1228 uint32_t flags[MAX_TX_RING_SLOTS] = {0}; 1229 struct rte_mbuf **orig_bufs = bufs; 1230 1231 if (unlikely(!DPAA2_PER_LCORE_DPIO)) { 1232 ret = dpaa2_affine_qbman_swp(); 1233 if (ret) { 1234 DPAA2_PMD_ERR( 1235 "Failed to allocate IO portal, tid: %d\n", 1236 rte_gettid()); 1237 return 0; 1238 } 1239 } 1240 swp = DPAA2_PER_LCORE_PORTAL; 1241 1242 DPAA2_PMD_DP_DEBUG("===> eth_data =%p, fqid =%d\n", 1243 eth_data, dpaa2_q->fqid); 1244 1245 #ifdef RTE_LIBRTE_IEEE1588 1246 /* IEEE1588 driver need pointer to tx confirmation queue 1247 * corresponding to last packet transmitted for reading 1248 * the timestamp 1249 */ 1250 if ((*bufs)->ol_flags & PKT_TX_IEEE1588_TMST) { 1251 priv->next_tx_conf_queue = dpaa2_q->tx_conf_queue; 1252 dpaa2_dev_tx_conf(dpaa2_q->tx_conf_queue); 1253 priv->tx_timestamp = 0; 1254 } 1255 #endif 1256 1257 /*Prepare enqueue descriptor*/ 1258 qbman_eq_desc_clear(&eqdesc); 1259 qbman_eq_desc_set_no_orp(&eqdesc, DPAA2_EQ_RESP_ERR_FQ); 1260 qbman_eq_desc_set_fq(&eqdesc, dpaa2_q->fqid); 1261 1262 /*Clear the unused FD fields before sending*/ 1263 while (nb_pkts) { 1264 /*Check if the queue is congested*/ 1265 retry_count = 0; 1266 while (qbman_result_SCN_state(dpaa2_q->cscn)) { 1267 retry_count++; 1268 /* Retry for some time before giving up */ 1269 if (retry_count > CONG_RETRY_COUNT) 1270 goto skip_tx; 1271 } 1272 1273 frames_to_send = (nb_pkts > dpaa2_eqcr_size) ? 1274 dpaa2_eqcr_size : nb_pkts; 1275 1276 for (loop = 0; loop < frames_to_send; loop++) { 1277 if (*dpaa2_seqn(*bufs)) { 1278 uint8_t dqrr_index = *dpaa2_seqn(*bufs) - 1; 1279 1280 flags[loop] = QBMAN_ENQUEUE_FLAG_DCA | 1281 dqrr_index; 1282 DPAA2_PER_LCORE_DQRR_SIZE--; 1283 DPAA2_PER_LCORE_DQRR_HELD &= ~(1 << dqrr_index); 1284 *dpaa2_seqn(*bufs) = DPAA2_INVALID_MBUF_SEQN; 1285 } 1286 1287 if (likely(RTE_MBUF_DIRECT(*bufs))) { 1288 mp = (*bufs)->pool; 1289 /* Check the basic scenario and set 1290 * the FD appropriately here itself. 1291 */ 1292 if (likely(mp && mp->ops_index == 1293 priv->bp_list->dpaa2_ops_index && 1294 (*bufs)->nb_segs == 1 && 1295 rte_mbuf_refcnt_read((*bufs)) == 1)) { 1296 if (unlikely(((*bufs)->ol_flags 1297 & RTE_MBUF_F_TX_VLAN) || 1298 (eth_data->dev_conf.txmode.offloads 1299 & RTE_ETH_TX_OFFLOAD_VLAN_INSERT))) { 1300 ret = rte_vlan_insert(bufs); 1301 if (ret) 1302 goto send_n_return; 1303 } 1304 DPAA2_MBUF_TO_CONTIG_FD((*bufs), 1305 &fd_arr[loop], mempool_to_bpid(mp)); 1306 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG 1307 rte_mempool_check_cookies 1308 (rte_mempool_from_obj((void *)*bufs), 1309 (void **)bufs, 1, 0); 1310 #endif 1311 bufs++; 1312 #ifdef RTE_LIBRTE_IEEE1588 1313 enable_tx_tstamp(&fd_arr[loop]); 1314 #endif 1315 continue; 1316 } 1317 } else { 1318 mi = rte_mbuf_from_indirect(*bufs); 1319 mp = mi->pool; 1320 } 1321 1322 if (unlikely(RTE_MBUF_HAS_EXTBUF(*bufs))) { 1323 if (unlikely((*bufs)->nb_segs > 1)) { 1324 if (eth_mbuf_to_sg_fd(*bufs, 1325 &fd_arr[loop], 1326 mp, 0)) 1327 goto send_n_return; 1328 } else { 1329 eth_mbuf_to_fd(*bufs, 1330 &fd_arr[loop], 0); 1331 } 1332 bufs++; 1333 #ifdef RTE_LIBRTE_IEEE1588 1334 enable_tx_tstamp(&fd_arr[loop]); 1335 #endif 1336 continue; 1337 } 1338 1339 /* Not a hw_pkt pool allocated frame */ 1340 if (unlikely(!mp || !priv->bp_list)) { 1341 DPAA2_PMD_ERR("Err: No buffer pool attached"); 1342 goto send_n_return; 1343 } 1344 1345 if (unlikely(((*bufs)->ol_flags & RTE_MBUF_F_TX_VLAN) || 1346 (eth_data->dev_conf.txmode.offloads 1347 & RTE_ETH_TX_OFFLOAD_VLAN_INSERT))) { 1348 int ret = rte_vlan_insert(bufs); 1349 if (ret) 1350 goto send_n_return; 1351 } 1352 if (mp->ops_index != priv->bp_list->dpaa2_ops_index) { 1353 DPAA2_PMD_WARN("Non DPAA2 buffer pool"); 1354 /* alloc should be from the default buffer pool 1355 * attached to this interface 1356 */ 1357 bpid = priv->bp_list->buf_pool.bpid; 1358 1359 if (unlikely((*bufs)->nb_segs > 1)) { 1360 DPAA2_PMD_ERR("S/G support not added" 1361 " for non hw offload buffer"); 1362 goto send_n_return; 1363 } 1364 if (eth_copy_mbuf_to_fd(*bufs, 1365 &fd_arr[loop], bpid)) { 1366 goto send_n_return; 1367 } 1368 /* free the original packet */ 1369 rte_pktmbuf_free(*bufs); 1370 } else { 1371 bpid = mempool_to_bpid(mp); 1372 if (unlikely((*bufs)->nb_segs > 1)) { 1373 if (eth_mbuf_to_sg_fd(*bufs, 1374 &fd_arr[loop], 1375 mp, bpid)) 1376 goto send_n_return; 1377 } else { 1378 eth_mbuf_to_fd(*bufs, 1379 &fd_arr[loop], bpid); 1380 } 1381 } 1382 #ifdef RTE_LIBRTE_IEEE1588 1383 enable_tx_tstamp(&fd_arr[loop]); 1384 #endif 1385 bufs++; 1386 } 1387 1388 loop = 0; 1389 retry_count = 0; 1390 while (loop < frames_to_send) { 1391 ret = qbman_swp_enqueue_multiple(swp, &eqdesc, 1392 &fd_arr[loop], &flags[loop], 1393 frames_to_send - loop); 1394 if (unlikely(ret < 0)) { 1395 retry_count++; 1396 if (retry_count > DPAA2_MAX_TX_RETRY_COUNT) { 1397 num_tx += loop; 1398 nb_pkts -= loop; 1399 goto send_n_return; 1400 } 1401 } else { 1402 loop += ret; 1403 retry_count = 0; 1404 } 1405 } 1406 1407 num_tx += loop; 1408 nb_pkts -= loop; 1409 } 1410 dpaa2_q->tx_pkts += num_tx; 1411 1412 loop = 0; 1413 while (loop < num_tx) { 1414 if (unlikely(RTE_MBUF_HAS_EXTBUF(*orig_bufs))) 1415 rte_pktmbuf_free(*orig_bufs); 1416 orig_bufs++; 1417 loop++; 1418 } 1419 1420 return num_tx; 1421 1422 send_n_return: 1423 /* send any already prepared fd */ 1424 if (loop) { 1425 unsigned int i = 0; 1426 1427 retry_count = 0; 1428 while (i < loop) { 1429 ret = qbman_swp_enqueue_multiple(swp, &eqdesc, 1430 &fd_arr[i], 1431 &flags[i], 1432 loop - i); 1433 if (unlikely(ret < 0)) { 1434 retry_count++; 1435 if (retry_count > DPAA2_MAX_TX_RETRY_COUNT) 1436 break; 1437 } else { 1438 i += ret; 1439 retry_count = 0; 1440 } 1441 } 1442 num_tx += i; 1443 } 1444 skip_tx: 1445 dpaa2_q->tx_pkts += num_tx; 1446 1447 loop = 0; 1448 while (loop < num_tx) { 1449 if (unlikely(RTE_MBUF_HAS_EXTBUF(*orig_bufs))) 1450 rte_pktmbuf_free(*orig_bufs); 1451 orig_bufs++; 1452 loop++; 1453 } 1454 1455 return num_tx; 1456 } 1457 1458 void 1459 dpaa2_dev_free_eqresp_buf(uint16_t eqresp_ci, 1460 __rte_unused struct dpaa2_queue *dpaa2_q) 1461 { 1462 struct dpaa2_dpio_dev *dpio_dev = DPAA2_PER_LCORE_DPIO; 1463 struct qbman_fd *fd; 1464 struct rte_mbuf *m; 1465 1466 fd = qbman_result_eqresp_fd(&dpio_dev->eqresp[eqresp_ci]); 1467 1468 /* Setting port id does not matter as we are to free the mbuf */ 1469 m = eth_fd_to_mbuf(fd, 0); 1470 rte_pktmbuf_free(m); 1471 } 1472 1473 static void 1474 dpaa2_set_enqueue_descriptor(struct dpaa2_queue *dpaa2_q, 1475 struct rte_mbuf *m, 1476 struct qbman_eq_desc *eqdesc) 1477 { 1478 struct rte_eth_dev_data *eth_data = dpaa2_q->eth_data; 1479 struct dpaa2_dev_priv *priv = eth_data->dev_private; 1480 struct dpaa2_dpio_dev *dpio_dev = DPAA2_PER_LCORE_DPIO; 1481 struct eqresp_metadata *eqresp_meta; 1482 uint16_t orpid, seqnum; 1483 uint8_t dq_idx; 1484 1485 qbman_eq_desc_set_fq(eqdesc, dpaa2_q->fqid); 1486 1487 if (*dpaa2_seqn(m) & DPAA2_ENQUEUE_FLAG_ORP) { 1488 orpid = (*dpaa2_seqn(m) & DPAA2_EQCR_OPRID_MASK) >> 1489 DPAA2_EQCR_OPRID_SHIFT; 1490 seqnum = (*dpaa2_seqn(m) & DPAA2_EQCR_SEQNUM_MASK) >> 1491 DPAA2_EQCR_SEQNUM_SHIFT; 1492 1493 if (!priv->en_loose_ordered) { 1494 qbman_eq_desc_set_orp(eqdesc, 1, orpid, seqnum, 0); 1495 qbman_eq_desc_set_response(eqdesc, (uint64_t) 1496 DPAA2_VADDR_TO_IOVA(&dpio_dev->eqresp[ 1497 dpio_dev->eqresp_pi]), 1); 1498 qbman_eq_desc_set_token(eqdesc, 1); 1499 1500 eqresp_meta = &dpio_dev->eqresp_meta[ 1501 dpio_dev->eqresp_pi]; 1502 eqresp_meta->dpaa2_q = dpaa2_q; 1503 eqresp_meta->mp = m->pool; 1504 1505 dpio_dev->eqresp_pi + 1 < MAX_EQ_RESP_ENTRIES ? 1506 dpio_dev->eqresp_pi++ : 1507 (dpio_dev->eqresp_pi = 0); 1508 } else { 1509 qbman_eq_desc_set_orp(eqdesc, 0, orpid, seqnum, 0); 1510 } 1511 } else { 1512 dq_idx = *dpaa2_seqn(m) - 1; 1513 qbman_eq_desc_set_dca(eqdesc, 1, dq_idx, 0); 1514 DPAA2_PER_LCORE_DQRR_SIZE--; 1515 DPAA2_PER_LCORE_DQRR_HELD &= ~(1 << dq_idx); 1516 } 1517 *dpaa2_seqn(m) = DPAA2_INVALID_MBUF_SEQN; 1518 } 1519 1520 uint16_t 1521 dpaa2_dev_tx_multi_txq_ordered(void **queue, 1522 struct rte_mbuf **bufs, uint16_t nb_pkts) 1523 { 1524 /* Function to transmit the frames to multiple queues respectively.*/ 1525 uint32_t loop, retry_count; 1526 int32_t ret; 1527 struct qbman_fd fd_arr[MAX_TX_RING_SLOTS]; 1528 uint32_t frames_to_send; 1529 struct rte_mempool *mp; 1530 struct qbman_eq_desc eqdesc[MAX_TX_RING_SLOTS]; 1531 struct dpaa2_queue *dpaa2_q[MAX_TX_RING_SLOTS]; 1532 struct qbman_swp *swp; 1533 uint16_t bpid; 1534 struct rte_mbuf *mi; 1535 struct rte_eth_dev_data *eth_data; 1536 struct dpaa2_dev_priv *priv; 1537 struct dpaa2_queue *order_sendq; 1538 1539 if (unlikely(!DPAA2_PER_LCORE_DPIO)) { 1540 ret = dpaa2_affine_qbman_swp(); 1541 if (ret) { 1542 DPAA2_PMD_ERR( 1543 "Failed to allocate IO portal, tid: %d\n", 1544 rte_gettid()); 1545 return 0; 1546 } 1547 } 1548 swp = DPAA2_PER_LCORE_PORTAL; 1549 1550 for (loop = 0; loop < nb_pkts; loop++) { 1551 dpaa2_q[loop] = (struct dpaa2_queue *)queue[loop]; 1552 eth_data = dpaa2_q[loop]->eth_data; 1553 priv = eth_data->dev_private; 1554 qbman_eq_desc_clear(&eqdesc[loop]); 1555 if (*dpaa2_seqn(*bufs) && priv->en_ordered) { 1556 order_sendq = (struct dpaa2_queue *)priv->tx_vq[0]; 1557 dpaa2_set_enqueue_descriptor(order_sendq, 1558 (*bufs), 1559 &eqdesc[loop]); 1560 } else { 1561 qbman_eq_desc_set_no_orp(&eqdesc[loop], 1562 DPAA2_EQ_RESP_ERR_FQ); 1563 qbman_eq_desc_set_fq(&eqdesc[loop], 1564 dpaa2_q[loop]->fqid); 1565 } 1566 1567 retry_count = 0; 1568 while (qbman_result_SCN_state(dpaa2_q[loop]->cscn)) { 1569 retry_count++; 1570 /* Retry for some time before giving up */ 1571 if (retry_count > CONG_RETRY_COUNT) 1572 goto send_frames; 1573 } 1574 1575 if (likely(RTE_MBUF_DIRECT(*bufs))) { 1576 mp = (*bufs)->pool; 1577 /* Check the basic scenario and set 1578 * the FD appropriately here itself. 1579 */ 1580 if (likely(mp && mp->ops_index == 1581 priv->bp_list->dpaa2_ops_index && 1582 (*bufs)->nb_segs == 1 && 1583 rte_mbuf_refcnt_read((*bufs)) == 1)) { 1584 if (unlikely((*bufs)->ol_flags 1585 & RTE_MBUF_F_TX_VLAN)) { 1586 ret = rte_vlan_insert(bufs); 1587 if (ret) 1588 goto send_frames; 1589 } 1590 DPAA2_MBUF_TO_CONTIG_FD((*bufs), 1591 &fd_arr[loop], 1592 mempool_to_bpid(mp)); 1593 bufs++; 1594 dpaa2_q[loop]++; 1595 continue; 1596 } 1597 } else { 1598 mi = rte_mbuf_from_indirect(*bufs); 1599 mp = mi->pool; 1600 } 1601 /* Not a hw_pkt pool allocated frame */ 1602 if (unlikely(!mp || !priv->bp_list)) { 1603 DPAA2_PMD_ERR("Err: No buffer pool attached"); 1604 goto send_frames; 1605 } 1606 1607 if (mp->ops_index != priv->bp_list->dpaa2_ops_index) { 1608 DPAA2_PMD_WARN("Non DPAA2 buffer pool"); 1609 /* alloc should be from the default buffer pool 1610 * attached to this interface 1611 */ 1612 bpid = priv->bp_list->buf_pool.bpid; 1613 1614 if (unlikely((*bufs)->nb_segs > 1)) { 1615 DPAA2_PMD_ERR( 1616 "S/G not supp for non hw offload buffer"); 1617 goto send_frames; 1618 } 1619 if (eth_copy_mbuf_to_fd(*bufs, 1620 &fd_arr[loop], bpid)) { 1621 goto send_frames; 1622 } 1623 /* free the original packet */ 1624 rte_pktmbuf_free(*bufs); 1625 } else { 1626 bpid = mempool_to_bpid(mp); 1627 if (unlikely((*bufs)->nb_segs > 1)) { 1628 if (eth_mbuf_to_sg_fd(*bufs, 1629 &fd_arr[loop], 1630 mp, 1631 bpid)) 1632 goto send_frames; 1633 } else { 1634 eth_mbuf_to_fd(*bufs, 1635 &fd_arr[loop], bpid); 1636 } 1637 } 1638 1639 bufs++; 1640 dpaa2_q[loop]++; 1641 } 1642 1643 send_frames: 1644 frames_to_send = loop; 1645 loop = 0; 1646 while (loop < frames_to_send) { 1647 ret = qbman_swp_enqueue_multiple_desc(swp, &eqdesc[loop], 1648 &fd_arr[loop], 1649 frames_to_send - loop); 1650 if (likely(ret > 0)) { 1651 loop += ret; 1652 } else { 1653 retry_count++; 1654 if (retry_count > DPAA2_MAX_TX_RETRY_COUNT) 1655 break; 1656 } 1657 } 1658 1659 return loop; 1660 } 1661 1662 /* Callback to handle sending ordered packets through WRIOP based interface */ 1663 uint16_t 1664 dpaa2_dev_tx_ordered(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) 1665 { 1666 /* Function to transmit the frames to given device and VQ*/ 1667 struct dpaa2_queue *dpaa2_q = (struct dpaa2_queue *)queue; 1668 struct rte_eth_dev_data *eth_data = dpaa2_q->eth_data; 1669 struct dpaa2_dev_priv *priv = eth_data->dev_private; 1670 struct dpaa2_queue *order_sendq = (struct dpaa2_queue *)priv->tx_vq[0]; 1671 struct qbman_fd fd_arr[MAX_TX_RING_SLOTS]; 1672 struct rte_mbuf *mi; 1673 struct rte_mempool *mp; 1674 struct qbman_eq_desc eqdesc[MAX_TX_RING_SLOTS]; 1675 struct qbman_swp *swp; 1676 uint32_t frames_to_send, num_free_eq_desc; 1677 uint32_t loop, retry_count; 1678 int32_t ret; 1679 uint16_t num_tx = 0; 1680 uint16_t bpid; 1681 1682 if (unlikely(!DPAA2_PER_LCORE_DPIO)) { 1683 ret = dpaa2_affine_qbman_swp(); 1684 if (ret) { 1685 DPAA2_PMD_ERR( 1686 "Failed to allocate IO portal, tid: %d\n", 1687 rte_gettid()); 1688 return 0; 1689 } 1690 } 1691 swp = DPAA2_PER_LCORE_PORTAL; 1692 1693 DPAA2_PMD_DP_DEBUG("===> eth_data =%p, fqid =%d\n", 1694 eth_data, dpaa2_q->fqid); 1695 1696 /* This would also handle normal and atomic queues as any type 1697 * of packet can be enqueued when ordered queues are being used. 1698 */ 1699 while (nb_pkts) { 1700 /*Check if the queue is congested*/ 1701 retry_count = 0; 1702 while (qbman_result_SCN_state(dpaa2_q->cscn)) { 1703 retry_count++; 1704 /* Retry for some time before giving up */ 1705 if (retry_count > CONG_RETRY_COUNT) 1706 goto skip_tx; 1707 } 1708 1709 frames_to_send = (nb_pkts > dpaa2_eqcr_size) ? 1710 dpaa2_eqcr_size : nb_pkts; 1711 1712 if (!priv->en_loose_ordered) { 1713 if (*dpaa2_seqn(*bufs) & DPAA2_ENQUEUE_FLAG_ORP) { 1714 num_free_eq_desc = dpaa2_free_eq_descriptors(); 1715 if (num_free_eq_desc < frames_to_send) 1716 frames_to_send = num_free_eq_desc; 1717 } 1718 } 1719 1720 for (loop = 0; loop < frames_to_send; loop++) { 1721 /*Prepare enqueue descriptor*/ 1722 qbman_eq_desc_clear(&eqdesc[loop]); 1723 1724 if (*dpaa2_seqn(*bufs)) { 1725 /* Use only queue 0 for Tx in case of atomic/ 1726 * ordered packets as packets can get unordered 1727 * when being transmitted out from the interface 1728 */ 1729 dpaa2_set_enqueue_descriptor(order_sendq, 1730 (*bufs), 1731 &eqdesc[loop]); 1732 } else { 1733 qbman_eq_desc_set_no_orp(&eqdesc[loop], 1734 DPAA2_EQ_RESP_ERR_FQ); 1735 qbman_eq_desc_set_fq(&eqdesc[loop], 1736 dpaa2_q->fqid); 1737 } 1738 1739 if (likely(RTE_MBUF_DIRECT(*bufs))) { 1740 mp = (*bufs)->pool; 1741 /* Check the basic scenario and set 1742 * the FD appropriately here itself. 1743 */ 1744 if (likely(mp && mp->ops_index == 1745 priv->bp_list->dpaa2_ops_index && 1746 (*bufs)->nb_segs == 1 && 1747 rte_mbuf_refcnt_read((*bufs)) == 1)) { 1748 if (unlikely((*bufs)->ol_flags 1749 & RTE_MBUF_F_TX_VLAN)) { 1750 ret = rte_vlan_insert(bufs); 1751 if (ret) 1752 goto send_n_return; 1753 } 1754 DPAA2_MBUF_TO_CONTIG_FD((*bufs), 1755 &fd_arr[loop], 1756 mempool_to_bpid(mp)); 1757 bufs++; 1758 continue; 1759 } 1760 } else { 1761 mi = rte_mbuf_from_indirect(*bufs); 1762 mp = mi->pool; 1763 } 1764 /* Not a hw_pkt pool allocated frame */ 1765 if (unlikely(!mp || !priv->bp_list)) { 1766 DPAA2_PMD_ERR("Err: No buffer pool attached"); 1767 goto send_n_return; 1768 } 1769 1770 if (mp->ops_index != priv->bp_list->dpaa2_ops_index) { 1771 DPAA2_PMD_WARN("Non DPAA2 buffer pool"); 1772 /* alloc should be from the default buffer pool 1773 * attached to this interface 1774 */ 1775 bpid = priv->bp_list->buf_pool.bpid; 1776 1777 if (unlikely((*bufs)->nb_segs > 1)) { 1778 DPAA2_PMD_ERR( 1779 "S/G not supp for non hw offload buffer"); 1780 goto send_n_return; 1781 } 1782 if (eth_copy_mbuf_to_fd(*bufs, 1783 &fd_arr[loop], bpid)) { 1784 goto send_n_return; 1785 } 1786 /* free the original packet */ 1787 rte_pktmbuf_free(*bufs); 1788 } else { 1789 bpid = mempool_to_bpid(mp); 1790 if (unlikely((*bufs)->nb_segs > 1)) { 1791 if (eth_mbuf_to_sg_fd(*bufs, 1792 &fd_arr[loop], 1793 mp, 1794 bpid)) 1795 goto send_n_return; 1796 } else { 1797 eth_mbuf_to_fd(*bufs, 1798 &fd_arr[loop], bpid); 1799 } 1800 } 1801 bufs++; 1802 } 1803 1804 loop = 0; 1805 retry_count = 0; 1806 while (loop < frames_to_send) { 1807 ret = qbman_swp_enqueue_multiple_desc(swp, 1808 &eqdesc[loop], &fd_arr[loop], 1809 frames_to_send - loop); 1810 if (unlikely(ret < 0)) { 1811 retry_count++; 1812 if (retry_count > DPAA2_MAX_TX_RETRY_COUNT) { 1813 num_tx += loop; 1814 nb_pkts -= loop; 1815 goto send_n_return; 1816 } 1817 } else { 1818 loop += ret; 1819 retry_count = 0; 1820 } 1821 } 1822 1823 num_tx += loop; 1824 nb_pkts -= loop; 1825 } 1826 dpaa2_q->tx_pkts += num_tx; 1827 return num_tx; 1828 1829 send_n_return: 1830 /* send any already prepared fd */ 1831 if (loop) { 1832 unsigned int i = 0; 1833 1834 retry_count = 0; 1835 while (i < loop) { 1836 ret = qbman_swp_enqueue_multiple_desc(swp, 1837 &eqdesc[loop], &fd_arr[i], loop - i); 1838 if (unlikely(ret < 0)) { 1839 retry_count++; 1840 if (retry_count > DPAA2_MAX_TX_RETRY_COUNT) 1841 break; 1842 } else { 1843 i += ret; 1844 retry_count = 0; 1845 } 1846 } 1847 num_tx += i; 1848 } 1849 skip_tx: 1850 dpaa2_q->tx_pkts += num_tx; 1851 return num_tx; 1852 } 1853 1854 #if defined(RTE_TOOLCHAIN_GCC) 1855 #pragma GCC diagnostic push 1856 #pragma GCC diagnostic ignored "-Wcast-qual" 1857 #elif defined(RTE_TOOLCHAIN_CLANG) 1858 #pragma clang diagnostic push 1859 #pragma clang diagnostic ignored "-Wcast-qual" 1860 #endif 1861 1862 /* This function loopbacks all the received packets.*/ 1863 uint16_t 1864 dpaa2_dev_loopback_rx(void *queue, 1865 struct rte_mbuf **bufs __rte_unused, 1866 uint16_t nb_pkts) 1867 { 1868 /* Function receive frames for a given device and VQ*/ 1869 struct dpaa2_queue *dpaa2_q = (struct dpaa2_queue *)queue; 1870 struct qbman_result *dq_storage, *dq_storage1 = NULL; 1871 uint32_t fqid = dpaa2_q->fqid; 1872 int ret, num_rx = 0, num_tx = 0, pull_size; 1873 uint8_t pending, status; 1874 struct qbman_swp *swp; 1875 struct qbman_fd *fd[DPAA2_LX2_DQRR_RING_SIZE]; 1876 struct qbman_pull_desc pulldesc; 1877 struct qbman_eq_desc eqdesc; 1878 struct queue_storage_info_t *q_storage = dpaa2_q->q_storage; 1879 struct rte_eth_dev_data *eth_data = dpaa2_q->eth_data; 1880 struct dpaa2_dev_priv *priv = eth_data->dev_private; 1881 struct dpaa2_queue *tx_q = priv->tx_vq[0]; 1882 /* todo - currently we are using 1st TX queue only for loopback*/ 1883 1884 if (unlikely(!DPAA2_PER_LCORE_ETHRX_DPIO)) { 1885 ret = dpaa2_affine_qbman_ethrx_swp(); 1886 if (ret) { 1887 DPAA2_PMD_ERR("Failure in affining portal"); 1888 return 0; 1889 } 1890 } 1891 swp = DPAA2_PER_LCORE_ETHRX_PORTAL; 1892 pull_size = (nb_pkts > dpaa2_dqrr_size) ? dpaa2_dqrr_size : nb_pkts; 1893 if (unlikely(!q_storage->active_dqs)) { 1894 q_storage->toggle = 0; 1895 dq_storage = q_storage->dq_storage[q_storage->toggle]; 1896 q_storage->last_num_pkts = pull_size; 1897 qbman_pull_desc_clear(&pulldesc); 1898 qbman_pull_desc_set_numframes(&pulldesc, 1899 q_storage->last_num_pkts); 1900 qbman_pull_desc_set_fq(&pulldesc, fqid); 1901 qbman_pull_desc_set_storage(&pulldesc, dq_storage, 1902 (size_t)(DPAA2_VADDR_TO_IOVA(dq_storage)), 1); 1903 if (check_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index)) { 1904 while (!qbman_check_command_complete( 1905 get_swp_active_dqs( 1906 DPAA2_PER_LCORE_ETHRX_DPIO->index))) 1907 ; 1908 clear_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index); 1909 } 1910 while (1) { 1911 if (qbman_swp_pull(swp, &pulldesc)) { 1912 DPAA2_PMD_DP_DEBUG( 1913 "VDQ command not issued.QBMAN busy\n"); 1914 /* Portal was busy, try again */ 1915 continue; 1916 } 1917 break; 1918 } 1919 q_storage->active_dqs = dq_storage; 1920 q_storage->active_dpio_id = DPAA2_PER_LCORE_ETHRX_DPIO->index; 1921 set_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index, 1922 dq_storage); 1923 } 1924 1925 dq_storage = q_storage->active_dqs; 1926 rte_prefetch0((void *)(size_t)(dq_storage)); 1927 rte_prefetch0((void *)(size_t)(dq_storage + 1)); 1928 1929 /* Prepare next pull descriptor. This will give space for the 1930 * prefetching done on DQRR entries 1931 */ 1932 q_storage->toggle ^= 1; 1933 dq_storage1 = q_storage->dq_storage[q_storage->toggle]; 1934 qbman_pull_desc_clear(&pulldesc); 1935 qbman_pull_desc_set_numframes(&pulldesc, pull_size); 1936 qbman_pull_desc_set_fq(&pulldesc, fqid); 1937 qbman_pull_desc_set_storage(&pulldesc, dq_storage1, 1938 (size_t)(DPAA2_VADDR_TO_IOVA(dq_storage1)), 1); 1939 1940 /*Prepare enqueue descriptor*/ 1941 qbman_eq_desc_clear(&eqdesc); 1942 qbman_eq_desc_set_no_orp(&eqdesc, DPAA2_EQ_RESP_ERR_FQ); 1943 qbman_eq_desc_set_response(&eqdesc, 0, 0); 1944 qbman_eq_desc_set_fq(&eqdesc, tx_q->fqid); 1945 1946 /* Check if the previous issued command is completed. 1947 * Also seems like the SWP is shared between the Ethernet Driver 1948 * and the SEC driver. 1949 */ 1950 while (!qbman_check_command_complete(dq_storage)) 1951 ; 1952 if (dq_storage == get_swp_active_dqs(q_storage->active_dpio_id)) 1953 clear_swp_active_dqs(q_storage->active_dpio_id); 1954 1955 pending = 1; 1956 1957 do { 1958 /* Loop until the dq_storage is updated with 1959 * new token by QBMAN 1960 */ 1961 while (!qbman_check_new_result(dq_storage)) 1962 ; 1963 rte_prefetch0((void *)((size_t)(dq_storage + 2))); 1964 /* Check whether Last Pull command is Expired and 1965 * setting Condition for Loop termination 1966 */ 1967 if (qbman_result_DQ_is_pull_complete(dq_storage)) { 1968 pending = 0; 1969 /* Check for valid frame. */ 1970 status = qbman_result_DQ_flags(dq_storage); 1971 if (unlikely((status & QBMAN_DQ_STAT_VALIDFRAME) == 0)) 1972 continue; 1973 } 1974 fd[num_rx] = (struct qbman_fd *)qbman_result_DQ_fd(dq_storage); 1975 1976 dq_storage++; 1977 num_rx++; 1978 } while (pending); 1979 1980 while (num_tx < num_rx) { 1981 num_tx += qbman_swp_enqueue_multiple_fd(swp, &eqdesc, 1982 &fd[num_tx], 0, num_rx - num_tx); 1983 } 1984 1985 if (check_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index)) { 1986 while (!qbman_check_command_complete( 1987 get_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index))) 1988 ; 1989 clear_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index); 1990 } 1991 /* issue a volatile dequeue command for next pull */ 1992 while (1) { 1993 if (qbman_swp_pull(swp, &pulldesc)) { 1994 DPAA2_PMD_DP_DEBUG("VDQ command is not issued." 1995 "QBMAN is busy (2)\n"); 1996 continue; 1997 } 1998 break; 1999 } 2000 q_storage->active_dqs = dq_storage1; 2001 q_storage->active_dpio_id = DPAA2_PER_LCORE_ETHRX_DPIO->index; 2002 set_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index, dq_storage1); 2003 2004 dpaa2_q->rx_pkts += num_rx; 2005 dpaa2_q->tx_pkts += num_tx; 2006 2007 return 0; 2008 } 2009 #if defined(RTE_TOOLCHAIN_GCC) 2010 #pragma GCC diagnostic pop 2011 #elif defined(RTE_TOOLCHAIN_CLANG) 2012 #pragma clang diagnostic pop 2013 #endif 2014