1 /* SPDX-License-Identifier: BSD-3-Clause 2 * 3 * Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved. 4 * Copyright 2016-2021 NXP 5 * 6 */ 7 8 #include <time.h> 9 #include <net/if.h> 10 11 #include <rte_mbuf.h> 12 #include <ethdev_driver.h> 13 #include <rte_malloc.h> 14 #include <rte_memcpy.h> 15 #include <rte_string_fns.h> 16 #include <dev_driver.h> 17 #include <rte_hexdump.h> 18 19 #include <bus_fslmc_driver.h> 20 #include <fslmc_vfio.h> 21 #include <dpaa2_hw_pvt.h> 22 #include <dpaa2_hw_dpio.h> 23 #include <dpaa2_hw_mempool.h> 24 25 #include "dpaa2_pmd_logs.h" 26 #include "dpaa2_ethdev.h" 27 #include "base/dpaa2_hw_dpni_annot.h" 28 29 static inline uint32_t __rte_hot 30 dpaa2_dev_rx_parse_slow(struct rte_mbuf *mbuf, 31 struct dpaa2_annot_hdr *annotation); 32 33 static void enable_tx_tstamp(struct qbman_fd *fd) __rte_unused; 34 35 static inline rte_mbuf_timestamp_t * 36 dpaa2_timestamp_dynfield(struct rte_mbuf *mbuf) 37 { 38 return RTE_MBUF_DYNFIELD(mbuf, 39 dpaa2_timestamp_dynfield_offset, rte_mbuf_timestamp_t *); 40 } 41 42 #define DPAA2_MBUF_TO_CONTIG_FD(_mbuf, _fd, _bpid) do { \ 43 DPAA2_SET_FD_ADDR(_fd, DPAA2_MBUF_VADDR_TO_IOVA(_mbuf)); \ 44 DPAA2_SET_FD_LEN(_fd, _mbuf->data_len); \ 45 DPAA2_SET_ONLY_FD_BPID(_fd, _bpid); \ 46 DPAA2_SET_FD_OFFSET(_fd, _mbuf->data_off); \ 47 DPAA2_SET_FD_FRC(_fd, 0); \ 48 DPAA2_RESET_FD_CTRL(_fd); \ 49 DPAA2_RESET_FD_FLC(_fd); \ 50 } while (0) 51 52 static inline void __rte_hot 53 dpaa2_dev_rx_parse_new(struct rte_mbuf *m, const struct qbman_fd *fd, 54 void *hw_annot_addr) 55 { 56 uint16_t frc = DPAA2_GET_FD_FRC_PARSE_SUM(fd); 57 struct dpaa2_annot_hdr *annotation = 58 (struct dpaa2_annot_hdr *)hw_annot_addr; 59 60 m->packet_type = RTE_PTYPE_UNKNOWN; 61 switch (frc) { 62 case DPAA2_PKT_TYPE_ETHER: 63 m->packet_type = RTE_PTYPE_L2_ETHER; 64 break; 65 case DPAA2_PKT_TYPE_IPV4: 66 m->packet_type = RTE_PTYPE_L2_ETHER | 67 RTE_PTYPE_L3_IPV4; 68 break; 69 case DPAA2_PKT_TYPE_IPV6: 70 m->packet_type = RTE_PTYPE_L2_ETHER | 71 RTE_PTYPE_L3_IPV6; 72 break; 73 case DPAA2_PKT_TYPE_IPV4_EXT: 74 m->packet_type = RTE_PTYPE_L2_ETHER | 75 RTE_PTYPE_L3_IPV4_EXT; 76 break; 77 case DPAA2_PKT_TYPE_IPV6_EXT: 78 m->packet_type = RTE_PTYPE_L2_ETHER | 79 RTE_PTYPE_L3_IPV6_EXT; 80 break; 81 case DPAA2_PKT_TYPE_IPV4_TCP: 82 m->packet_type = RTE_PTYPE_L2_ETHER | 83 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_TCP; 84 break; 85 case DPAA2_PKT_TYPE_IPV6_TCP: 86 m->packet_type = RTE_PTYPE_L2_ETHER | 87 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_TCP; 88 break; 89 case DPAA2_PKT_TYPE_IPV4_UDP: 90 m->packet_type = RTE_PTYPE_L2_ETHER | 91 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_UDP; 92 break; 93 case DPAA2_PKT_TYPE_IPV6_UDP: 94 m->packet_type = RTE_PTYPE_L2_ETHER | 95 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_UDP; 96 break; 97 case DPAA2_PKT_TYPE_IPV4_SCTP: 98 m->packet_type = RTE_PTYPE_L2_ETHER | 99 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_SCTP; 100 break; 101 case DPAA2_PKT_TYPE_IPV6_SCTP: 102 m->packet_type = RTE_PTYPE_L2_ETHER | 103 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_SCTP; 104 break; 105 case DPAA2_PKT_TYPE_IPV4_ICMP: 106 m->packet_type = RTE_PTYPE_L2_ETHER | 107 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_ICMP; 108 break; 109 case DPAA2_PKT_TYPE_IPV6_ICMP: 110 m->packet_type = RTE_PTYPE_L2_ETHER | 111 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_ICMP; 112 break; 113 default: 114 m->packet_type = dpaa2_dev_rx_parse_slow(m, annotation); 115 } 116 m->hash.rss = fd->simple.flc_hi; 117 m->ol_flags |= RTE_MBUF_F_RX_RSS_HASH; 118 119 if (dpaa2_enable_ts[m->port]) { 120 *dpaa2_timestamp_dynfield(m) = annotation->word2; 121 m->ol_flags |= dpaa2_timestamp_rx_dynflag; 122 DPAA2_PMD_DP_DEBUG("pkt timestamp:0x%" PRIx64 "", 123 *dpaa2_timestamp_dynfield(m)); 124 } 125 126 DPAA2_PMD_DP_DEBUG("HW frc = 0x%x\t packet type =0x%x " 127 "ol_flags =0x%" PRIx64 "", 128 frc, m->packet_type, m->ol_flags); 129 } 130 131 static inline uint32_t __rte_hot 132 dpaa2_dev_rx_parse_slow(struct rte_mbuf *mbuf, 133 struct dpaa2_annot_hdr *annotation) 134 { 135 uint32_t pkt_type = RTE_PTYPE_UNKNOWN; 136 uint16_t *vlan_tci; 137 138 DPAA2_PMD_DP_DEBUG("(slow parse)annotation(3)=0x%" PRIx64 "\t" 139 "(4)=0x%" PRIx64 "\t", 140 annotation->word3, annotation->word4); 141 142 #if defined(RTE_LIBRTE_IEEE1588) 143 if (BIT_ISSET_AT_POS(annotation->word1, DPAA2_ETH_FAS_PTP)) { 144 mbuf->ol_flags |= RTE_MBUF_F_RX_IEEE1588_PTP; 145 mbuf->ol_flags |= RTE_MBUF_F_RX_IEEE1588_TMST; 146 } 147 #endif 148 149 if (BIT_ISSET_AT_POS(annotation->word3, L2_VLAN_1_PRESENT)) { 150 vlan_tci = rte_pktmbuf_mtod_offset(mbuf, uint16_t *, 151 (VLAN_TCI_OFFSET_1(annotation->word5) >> 16)); 152 mbuf->vlan_tci = rte_be_to_cpu_16(*vlan_tci); 153 mbuf->ol_flags |= RTE_MBUF_F_RX_VLAN; 154 pkt_type |= RTE_PTYPE_L2_ETHER_VLAN; 155 } else if (BIT_ISSET_AT_POS(annotation->word3, L2_VLAN_N_PRESENT)) { 156 vlan_tci = rte_pktmbuf_mtod_offset(mbuf, uint16_t *, 157 (VLAN_TCI_OFFSET_1(annotation->word5) >> 16)); 158 mbuf->vlan_tci = rte_be_to_cpu_16(*vlan_tci); 159 mbuf->ol_flags |= RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_QINQ; 160 pkt_type |= RTE_PTYPE_L2_ETHER_QINQ; 161 } 162 163 if (BIT_ISSET_AT_POS(annotation->word3, L2_ARP_PRESENT)) { 164 pkt_type |= RTE_PTYPE_L2_ETHER_ARP; 165 goto parse_done; 166 } else if (BIT_ISSET_AT_POS(annotation->word3, L2_ETH_MAC_PRESENT)) { 167 pkt_type |= RTE_PTYPE_L2_ETHER; 168 } else { 169 goto parse_done; 170 } 171 172 if (BIT_ISSET_AT_POS(annotation->word3, L2_MPLS_1_PRESENT | 173 L2_MPLS_N_PRESENT)) 174 pkt_type |= RTE_PTYPE_L2_ETHER_MPLS; 175 176 if (BIT_ISSET_AT_POS(annotation->word4, L3_IPV4_1_PRESENT | 177 L3_IPV4_N_PRESENT)) { 178 pkt_type |= RTE_PTYPE_L3_IPV4; 179 if (BIT_ISSET_AT_POS(annotation->word4, L3_IP_1_OPT_PRESENT | 180 L3_IP_N_OPT_PRESENT)) 181 pkt_type |= RTE_PTYPE_L3_IPV4_EXT; 182 if (BIT_ISSET_AT_POS(annotation->word4, L3_PROTO_IPSEC_ESP_PRESENT | 183 L3_PROTO_ESP_PRESENT)) 184 pkt_type |= RTE_PTYPE_TUNNEL_ESP; 185 186 } else if (BIT_ISSET_AT_POS(annotation->word4, L3_IPV6_1_PRESENT | 187 L3_IPV6_N_PRESENT)) { 188 pkt_type |= RTE_PTYPE_L3_IPV6; 189 if (BIT_ISSET_AT_POS(annotation->word4, L3_IP_1_OPT_PRESENT | 190 L3_IP_N_OPT_PRESENT)) 191 pkt_type |= RTE_PTYPE_L3_IPV6_EXT; 192 if (BIT_ISSET_AT_POS(annotation->word4, L3_PROTO_IPSEC_ESP_PRESENT | 193 L3_PROTO_ESP_PRESENT)) 194 pkt_type |= RTE_PTYPE_TUNNEL_ESP; 195 } else { 196 goto parse_done; 197 } 198 199 if (BIT_ISSET_AT_POS(annotation->word8, DPAA2_ETH_FAS_L3CE)) 200 mbuf->ol_flags |= RTE_MBUF_F_RX_IP_CKSUM_BAD; 201 else if (BIT_ISSET_AT_POS(annotation->word8, DPAA2_ETH_FAS_L4CE)) 202 mbuf->ol_flags |= RTE_MBUF_F_RX_L4_CKSUM_BAD; 203 204 if (BIT_ISSET_AT_POS(annotation->word4, L3_IP_1_FIRST_FRAGMENT | 205 L3_IP_1_MORE_FRAGMENT | 206 L3_IP_N_FIRST_FRAGMENT | 207 L3_IP_N_MORE_FRAGMENT)) { 208 pkt_type |= RTE_PTYPE_L4_FRAG; 209 goto parse_done; 210 } else { 211 pkt_type |= RTE_PTYPE_L4_NONFRAG; 212 } 213 214 if (BIT_ISSET_AT_POS(annotation->word4, L3_PROTO_UDP_PRESENT)) 215 pkt_type |= RTE_PTYPE_L4_UDP; 216 217 else if (BIT_ISSET_AT_POS(annotation->word4, L3_PROTO_TCP_PRESENT)) 218 pkt_type |= RTE_PTYPE_L4_TCP; 219 220 else if (BIT_ISSET_AT_POS(annotation->word4, L3_PROTO_SCTP_PRESENT)) 221 pkt_type |= RTE_PTYPE_L4_SCTP; 222 223 else if (BIT_ISSET_AT_POS(annotation->word4, L3_PROTO_ICMP_PRESENT)) 224 pkt_type |= RTE_PTYPE_L4_ICMP; 225 226 else if (BIT_ISSET_AT_POS(annotation->word4, L3_IP_UNKNOWN_PROTOCOL)) 227 pkt_type |= RTE_PTYPE_UNKNOWN; 228 229 parse_done: 230 return pkt_type; 231 } 232 233 static inline uint32_t __rte_hot 234 dpaa2_dev_rx_parse(struct rte_mbuf *mbuf, void *hw_annot_addr) 235 { 236 struct dpaa2_annot_hdr *annotation = 237 (struct dpaa2_annot_hdr *)hw_annot_addr; 238 239 DPAA2_PMD_DP_DEBUG("(fast parse) Annotation = 0x%" PRIx64 "\t", 240 annotation->word4); 241 242 if (BIT_ISSET_AT_POS(annotation->word8, DPAA2_ETH_FAS_L3CE)) 243 mbuf->ol_flags |= RTE_MBUF_F_RX_IP_CKSUM_BAD; 244 else if (BIT_ISSET_AT_POS(annotation->word8, DPAA2_ETH_FAS_L4CE)) 245 mbuf->ol_flags |= RTE_MBUF_F_RX_L4_CKSUM_BAD; 246 247 if (dpaa2_enable_ts[mbuf->port]) { 248 *dpaa2_timestamp_dynfield(mbuf) = annotation->word2; 249 mbuf->ol_flags |= dpaa2_timestamp_rx_dynflag; 250 DPAA2_PMD_DP_DEBUG("pkt timestamp: 0x%" PRIx64 "", 251 *dpaa2_timestamp_dynfield(mbuf)); 252 } 253 254 /* Check detailed parsing requirement */ 255 if (annotation->word3 & 0x7FFFFC3FFFF) 256 return dpaa2_dev_rx_parse_slow(mbuf, annotation); 257 258 /* Return some common types from parse processing */ 259 switch (annotation->word4) { 260 case DPAA2_L3_IPv4: 261 return RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4; 262 case DPAA2_L3_IPv6: 263 return RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6; 264 case DPAA2_L3_IPv4_TCP: 265 return RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 | 266 RTE_PTYPE_L4_TCP; 267 case DPAA2_L3_IPv4_UDP: 268 return RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 | 269 RTE_PTYPE_L4_UDP; 270 case DPAA2_L3_IPv6_TCP: 271 return RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 | 272 RTE_PTYPE_L4_TCP; 273 case DPAA2_L3_IPv6_UDP: 274 return RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 | 275 RTE_PTYPE_L4_UDP; 276 default: 277 break; 278 } 279 280 return dpaa2_dev_rx_parse_slow(mbuf, annotation); 281 } 282 283 static inline struct rte_mbuf *__rte_hot 284 eth_sg_fd_to_mbuf(const struct qbman_fd *fd, 285 int port_id) 286 { 287 struct qbman_sge *sgt, *sge; 288 size_t sg_addr, fd_addr; 289 int i = 0; 290 void *hw_annot_addr; 291 struct rte_mbuf *first_seg, *next_seg, *cur_seg, *temp; 292 293 fd_addr = (size_t)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd)); 294 hw_annot_addr = (void *)(fd_addr + DPAA2_FD_PTA_SIZE); 295 296 /* Get Scatter gather table address */ 297 sgt = (struct qbman_sge *)(fd_addr + DPAA2_GET_FD_OFFSET(fd)); 298 299 sge = &sgt[i++]; 300 sg_addr = (size_t)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FLE_ADDR(sge)); 301 302 /* First Scatter gather entry */ 303 first_seg = DPAA2_INLINE_MBUF_FROM_BUF(sg_addr, 304 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size); 305 /* Prepare all the metadata for first segment */ 306 first_seg->buf_addr = (uint8_t *)sg_addr; 307 first_seg->ol_flags = 0; 308 first_seg->data_off = DPAA2_GET_FLE_OFFSET(sge); 309 first_seg->data_len = sge->length & 0x1FFFF; 310 first_seg->pkt_len = DPAA2_GET_FD_LEN(fd); 311 first_seg->nb_segs = 1; 312 first_seg->next = NULL; 313 first_seg->port = port_id; 314 if (dpaa2_svr_family == SVR_LX2160A) 315 dpaa2_dev_rx_parse_new(first_seg, fd, hw_annot_addr); 316 else 317 first_seg->packet_type = 318 dpaa2_dev_rx_parse(first_seg, hw_annot_addr); 319 320 rte_mbuf_refcnt_set(first_seg, 1); 321 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG 322 rte_mempool_check_cookies(rte_mempool_from_obj((void *)first_seg), 323 (void **)&first_seg, 1, 1); 324 #endif 325 cur_seg = first_seg; 326 while (!DPAA2_SG_IS_FINAL(sge)) { 327 sge = &sgt[i++]; 328 sg_addr = (size_t)DPAA2_IOVA_TO_VADDR( 329 DPAA2_GET_FLE_ADDR(sge)); 330 next_seg = DPAA2_INLINE_MBUF_FROM_BUF(sg_addr, 331 rte_dpaa2_bpid_info[DPAA2_GET_FLE_BPID(sge)].meta_data_size); 332 next_seg->buf_addr = (uint8_t *)sg_addr; 333 next_seg->data_off = DPAA2_GET_FLE_OFFSET(sge); 334 next_seg->data_len = sge->length & 0x1FFFF; 335 first_seg->nb_segs += 1; 336 rte_mbuf_refcnt_set(next_seg, 1); 337 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG 338 rte_mempool_check_cookies(rte_mempool_from_obj((void *)next_seg), 339 (void **)&next_seg, 1, 1); 340 #endif 341 cur_seg->next = next_seg; 342 next_seg->next = NULL; 343 cur_seg = next_seg; 344 } 345 temp = DPAA2_INLINE_MBUF_FROM_BUF(fd_addr, 346 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size); 347 rte_mbuf_refcnt_set(temp, 1); 348 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG 349 rte_mempool_check_cookies(rte_mempool_from_obj((void *)temp), 350 (void **)&temp, 1, 1); 351 #endif 352 rte_pktmbuf_free_seg(temp); 353 354 return (void *)first_seg; 355 } 356 357 static inline struct rte_mbuf *__rte_hot 358 eth_fd_to_mbuf(const struct qbman_fd *fd, 359 int port_id) 360 { 361 void *v_addr = DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd)); 362 void *hw_annot_addr = (void *)((size_t)v_addr + DPAA2_FD_PTA_SIZE); 363 struct rte_mbuf *mbuf = DPAA2_INLINE_MBUF_FROM_BUF(v_addr, 364 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size); 365 366 /* need to repopulated some of the fields, 367 * as they may have changed in last transmission 368 */ 369 mbuf->nb_segs = 1; 370 mbuf->ol_flags = 0; 371 mbuf->data_off = DPAA2_GET_FD_OFFSET(fd); 372 mbuf->data_len = DPAA2_GET_FD_LEN(fd); 373 mbuf->pkt_len = mbuf->data_len; 374 mbuf->port = port_id; 375 mbuf->next = NULL; 376 rte_mbuf_refcnt_set(mbuf, 1); 377 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG 378 rte_mempool_check_cookies(rte_mempool_from_obj((void *)mbuf), 379 (void **)&mbuf, 1, 1); 380 #endif 381 382 /* Parse the packet */ 383 /* parse results for LX2 are there in FRC field of FD. 384 * For other DPAA2 platforms , parse results are after 385 * the private - sw annotation area 386 */ 387 388 if (dpaa2_svr_family == SVR_LX2160A) 389 dpaa2_dev_rx_parse_new(mbuf, fd, hw_annot_addr); 390 else 391 mbuf->packet_type = dpaa2_dev_rx_parse(mbuf, hw_annot_addr); 392 393 DPAA2_PMD_DP_DEBUG("to mbuf - mbuf =%p, mbuf->buf_addr =%p, off = %d," 394 "fd_off=%d fd =%" PRIx64 ", meta = %d bpid =%d, len=%d\n", 395 mbuf, mbuf->buf_addr, mbuf->data_off, 396 DPAA2_GET_FD_OFFSET(fd), DPAA2_GET_FD_ADDR(fd), 397 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size, 398 DPAA2_GET_FD_BPID(fd), DPAA2_GET_FD_LEN(fd)); 399 400 return mbuf; 401 } 402 403 static int __rte_noinline __rte_hot 404 eth_mbuf_to_sg_fd(struct rte_mbuf *mbuf, 405 struct qbman_fd *fd, 406 struct sw_buf_free *free_buf, 407 uint32_t *free_count, 408 uint32_t pkt_id, 409 uint16_t bpid) 410 { 411 struct rte_mbuf *cur_seg = mbuf, *mi, *temp; 412 struct qbman_sge *sgt, *sge = NULL; 413 int i, offset = 0; 414 415 #ifdef RTE_LIBRTE_IEEE1588 416 /* annotation area for timestamp in first buffer */ 417 offset = 0x64; 418 #endif 419 if (RTE_MBUF_DIRECT(mbuf) && 420 (mbuf->data_off > (mbuf->nb_segs * sizeof(struct qbman_sge) 421 + offset))) { 422 temp = mbuf; 423 if (rte_mbuf_refcnt_read(temp) > 1) { 424 /* If refcnt > 1, invalid bpid is set to ensure 425 * buffer is not freed by HW 426 */ 427 fd->simple.bpid_offset = 0; 428 DPAA2_SET_FD_IVP(fd); 429 rte_mbuf_refcnt_update(temp, -1); 430 } else { 431 DPAA2_SET_ONLY_FD_BPID(fd, bpid); 432 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG 433 rte_mempool_check_cookies(rte_mempool_from_obj((void *)temp), 434 (void **)&temp, 1, 0); 435 #endif 436 } 437 DPAA2_SET_FD_OFFSET(fd, offset); 438 } else { 439 temp = rte_pktmbuf_alloc(dpaa2_tx_sg_pool); 440 if (temp == NULL) { 441 DPAA2_PMD_DP_DEBUG("No memory to allocate S/G table\n"); 442 return -ENOMEM; 443 } 444 DPAA2_SET_ONLY_FD_BPID(fd, mempool_to_bpid(dpaa2_tx_sg_pool)); 445 DPAA2_SET_FD_OFFSET(fd, temp->data_off); 446 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG 447 rte_mempool_check_cookies(rte_mempool_from_obj((void *)temp), 448 (void **)&temp, 1, 0); 449 #endif 450 } 451 DPAA2_SET_FD_ADDR(fd, DPAA2_MBUF_VADDR_TO_IOVA(temp)); 452 DPAA2_SET_FD_LEN(fd, mbuf->pkt_len); 453 DPAA2_FD_SET_FORMAT(fd, qbman_fd_sg); 454 DPAA2_RESET_FD_FRC(fd); 455 DPAA2_RESET_FD_CTRL(fd); 456 DPAA2_RESET_FD_FLC(fd); 457 /*Set Scatter gather table and Scatter gather entries*/ 458 sgt = (struct qbman_sge *)( 459 (size_t)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd)) 460 + DPAA2_GET_FD_OFFSET(fd)); 461 462 for (i = 0; i < mbuf->nb_segs; i++) { 463 sge = &sgt[i]; 464 /*Resetting the buffer pool id and offset field*/ 465 sge->fin_bpid_offset = 0; 466 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(cur_seg)); 467 DPAA2_SET_FLE_OFFSET(sge, cur_seg->data_off); 468 sge->length = cur_seg->data_len; 469 if (RTE_MBUF_DIRECT(cur_seg)) { 470 /* if we are using inline SGT in same buffers 471 * set the FLE FMT as Frame Data Section 472 */ 473 if (temp == cur_seg) { 474 DPAA2_SG_SET_FORMAT(sge, qbman_fd_list); 475 DPAA2_SET_FLE_IVP(sge); 476 } else { 477 if (rte_mbuf_refcnt_read(cur_seg) > 1) { 478 /* If refcnt > 1, invalid bpid is set to ensure 479 * buffer is not freed by HW 480 */ 481 DPAA2_SET_FLE_IVP(sge); 482 rte_mbuf_refcnt_update(cur_seg, -1); 483 } else { 484 DPAA2_SET_FLE_BPID(sge, 485 mempool_to_bpid(cur_seg->pool)); 486 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG 487 rte_mempool_check_cookies(rte_mempool_from_obj((void *)cur_seg), 488 (void **)&cur_seg, 1, 0); 489 #endif 490 } 491 } 492 } else if (RTE_MBUF_HAS_EXTBUF(cur_seg)) { 493 free_buf[*free_count].seg = cur_seg; 494 free_buf[*free_count].pkt_id = pkt_id; 495 ++*free_count; 496 DPAA2_SET_FLE_IVP(sge); 497 } else { 498 /* Get owner MBUF from indirect buffer */ 499 mi = rte_mbuf_from_indirect(cur_seg); 500 if (rte_mbuf_refcnt_read(mi) > 1) { 501 /* If refcnt > 1, invalid bpid is set to ensure 502 * owner buffer is not freed by HW 503 */ 504 DPAA2_SET_FLE_IVP(sge); 505 } else { 506 DPAA2_SET_FLE_BPID(sge, 507 mempool_to_bpid(mi->pool)); 508 rte_mbuf_refcnt_update(mi, 1); 509 } 510 free_buf[*free_count].seg = cur_seg; 511 free_buf[*free_count].pkt_id = pkt_id; 512 ++*free_count; 513 } 514 cur_seg = cur_seg->next; 515 } 516 DPAA2_SG_SET_FINAL(sge, true); 517 return 0; 518 } 519 520 static void 521 eth_mbuf_to_fd(struct rte_mbuf *mbuf, 522 struct qbman_fd *fd, 523 struct sw_buf_free *buf_to_free, 524 uint32_t *free_count, 525 uint32_t pkt_id, 526 uint16_t bpid) __rte_unused; 527 528 static void __rte_noinline __rte_hot 529 eth_mbuf_to_fd(struct rte_mbuf *mbuf, 530 struct qbman_fd *fd, 531 struct sw_buf_free *buf_to_free, 532 uint32_t *free_count, 533 uint32_t pkt_id, 534 uint16_t bpid) 535 { 536 DPAA2_MBUF_TO_CONTIG_FD(mbuf, fd, bpid); 537 538 DPAA2_PMD_DP_DEBUG("mbuf =%p, mbuf->buf_addr =%p, off = %d," 539 "fd_off=%d fd =%" PRIx64 ", meta = %d bpid =%d, len=%d\n", 540 mbuf, mbuf->buf_addr, mbuf->data_off, 541 DPAA2_GET_FD_OFFSET(fd), DPAA2_GET_FD_ADDR(fd), 542 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size, 543 DPAA2_GET_FD_BPID(fd), DPAA2_GET_FD_LEN(fd)); 544 if (RTE_MBUF_DIRECT(mbuf)) { 545 if (rte_mbuf_refcnt_read(mbuf) > 1) { 546 DPAA2_SET_FD_IVP(fd); 547 rte_mbuf_refcnt_update(mbuf, -1); 548 } 549 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG 550 else 551 rte_mempool_check_cookies(rte_mempool_from_obj((void *)mbuf), 552 (void **)&mbuf, 1, 0); 553 #endif 554 } else if (RTE_MBUF_HAS_EXTBUF(mbuf)) { 555 buf_to_free[*free_count].seg = mbuf; 556 buf_to_free[*free_count].pkt_id = pkt_id; 557 ++*free_count; 558 DPAA2_SET_FD_IVP(fd); 559 } else { 560 struct rte_mbuf *mi; 561 562 mi = rte_mbuf_from_indirect(mbuf); 563 if (rte_mbuf_refcnt_read(mi) > 1) 564 DPAA2_SET_FD_IVP(fd); 565 else 566 rte_mbuf_refcnt_update(mi, 1); 567 568 buf_to_free[*free_count].seg = mbuf; 569 buf_to_free[*free_count].pkt_id = pkt_id; 570 ++*free_count; 571 } 572 } 573 574 static inline int __rte_hot 575 eth_copy_mbuf_to_fd(struct rte_mbuf *mbuf, 576 struct qbman_fd *fd, uint16_t bpid) 577 { 578 struct rte_mbuf *m; 579 void *mb = NULL; 580 581 if (rte_dpaa2_mbuf_alloc_bulk( 582 rte_dpaa2_bpid_info[bpid].bp_list->mp, &mb, 1)) { 583 DPAA2_PMD_DP_DEBUG("Unable to allocated DPAA2 buffer\n"); 584 return -1; 585 } 586 m = (struct rte_mbuf *)mb; 587 memcpy((char *)m->buf_addr + mbuf->data_off, 588 (void *)((char *)mbuf->buf_addr + mbuf->data_off), 589 mbuf->pkt_len); 590 591 /* Copy required fields */ 592 m->data_off = mbuf->data_off; 593 m->ol_flags = mbuf->ol_flags; 594 m->packet_type = mbuf->packet_type; 595 m->tx_offload = mbuf->tx_offload; 596 597 DPAA2_MBUF_TO_CONTIG_FD(m, fd, bpid); 598 599 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG 600 rte_mempool_check_cookies(rte_mempool_from_obj((void *)m), 601 (void **)&m, 1, 0); 602 #endif 603 DPAA2_PMD_DP_DEBUG( 604 "mbuf: %p, BMAN buf addr: %p, fdaddr: %" PRIx64 ", bpid: %d," 605 " meta: %d, off: %d, len: %d\n", 606 (void *)mbuf, 607 mbuf->buf_addr, 608 DPAA2_GET_FD_ADDR(fd), 609 DPAA2_GET_FD_BPID(fd), 610 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size, 611 DPAA2_GET_FD_OFFSET(fd), 612 DPAA2_GET_FD_LEN(fd)); 613 614 return 0; 615 } 616 617 static void 618 dump_err_pkts(struct dpaa2_queue *dpaa2_q) 619 { 620 /* Function receive frames for a given device and VQ */ 621 struct qbman_result *dq_storage; 622 uint32_t fqid = dpaa2_q->fqid; 623 int ret, num_rx = 0, num_pulled; 624 uint8_t pending, status; 625 struct qbman_swp *swp; 626 const struct qbman_fd *fd; 627 struct qbman_pull_desc pulldesc; 628 struct rte_eth_dev_data *eth_data = dpaa2_q->eth_data; 629 uint32_t lcore_id = rte_lcore_id(); 630 void *v_addr, *hw_annot_addr; 631 struct dpaa2_fas *fas; 632 633 if (unlikely(!DPAA2_PER_LCORE_DPIO)) { 634 ret = dpaa2_affine_qbman_swp(); 635 if (ret) { 636 DPAA2_PMD_ERR("Failed to allocate IO portal, tid: %d\n", 637 rte_gettid()); 638 return; 639 } 640 } 641 swp = DPAA2_PER_LCORE_PORTAL; 642 643 dq_storage = dpaa2_q->q_storage[lcore_id].dq_storage[0]; 644 qbman_pull_desc_clear(&pulldesc); 645 qbman_pull_desc_set_fq(&pulldesc, fqid); 646 qbman_pull_desc_set_storage(&pulldesc, dq_storage, 647 (size_t)(DPAA2_VADDR_TO_IOVA(dq_storage)), 1); 648 qbman_pull_desc_set_numframes(&pulldesc, dpaa2_dqrr_size); 649 650 while (1) { 651 if (qbman_swp_pull(swp, &pulldesc)) { 652 DPAA2_PMD_DP_DEBUG("VDQ command is not issued.QBMAN is busy\n"); 653 /* Portal was busy, try again */ 654 continue; 655 } 656 break; 657 } 658 659 /* Check if the previous issued command is completed. */ 660 while (!qbman_check_command_complete(dq_storage)) 661 ; 662 663 num_pulled = 0; 664 pending = 1; 665 do { 666 /* Loop until the dq_storage is updated with 667 * new token by QBMAN 668 */ 669 while (!qbman_check_new_result(dq_storage)) 670 ; 671 672 /* Check whether Last Pull command is Expired and 673 * setting Condition for Loop termination 674 */ 675 if (qbman_result_DQ_is_pull_complete(dq_storage)) { 676 pending = 0; 677 /* Check for valid frame. */ 678 status = qbman_result_DQ_flags(dq_storage); 679 if (unlikely((status & 680 QBMAN_DQ_STAT_VALIDFRAME) == 0)) 681 continue; 682 } 683 fd = qbman_result_DQ_fd(dq_storage); 684 v_addr = DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd)); 685 hw_annot_addr = (void *)((size_t)v_addr + DPAA2_FD_PTA_SIZE); 686 fas = hw_annot_addr; 687 688 DPAA2_PMD_ERR("\n\n[%d] error packet on port[%d]:" 689 " fd_off: %d, fd_err: %x, fas_status: %x", 690 rte_lcore_id(), eth_data->port_id, 691 DPAA2_GET_FD_OFFSET(fd), DPAA2_GET_FD_ERR(fd), 692 fas->status); 693 rte_hexdump(stderr, "Error packet", v_addr, 694 DPAA2_GET_FD_OFFSET(fd) + DPAA2_GET_FD_LEN(fd)); 695 696 dq_storage++; 697 num_rx++; 698 num_pulled++; 699 } while (pending); 700 701 dpaa2_q->err_pkts += num_rx; 702 } 703 704 /* This function assumes that caller will be keep the same value for nb_pkts 705 * across calls per queue, if that is not the case, better use non-prefetch 706 * version of rx call. 707 * It will return the packets as requested in previous call without honoring 708 * the current nb_pkts or bufs space. 709 */ 710 uint16_t 711 dpaa2_dev_prefetch_rx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) 712 { 713 /* Function receive frames for a given device and VQ*/ 714 struct dpaa2_queue *dpaa2_q = (struct dpaa2_queue *)queue; 715 struct qbman_result *dq_storage, *dq_storage1 = NULL; 716 uint32_t fqid = dpaa2_q->fqid; 717 int ret, num_rx = 0, pull_size; 718 uint8_t pending, status; 719 struct qbman_swp *swp; 720 const struct qbman_fd *fd; 721 struct qbman_pull_desc pulldesc; 722 struct queue_storage_info_t *q_storage = dpaa2_q->q_storage; 723 struct rte_eth_dev_data *eth_data = dpaa2_q->eth_data; 724 struct dpaa2_dev_priv *priv = eth_data->dev_private; 725 726 if (unlikely(dpaa2_enable_err_queue)) 727 dump_err_pkts(priv->rx_err_vq); 728 729 if (unlikely(!DPAA2_PER_LCORE_ETHRX_DPIO)) { 730 ret = dpaa2_affine_qbman_ethrx_swp(); 731 if (ret) { 732 DPAA2_PMD_ERR("Failure in affining portal"); 733 return 0; 734 } 735 } 736 737 if (unlikely(!rte_dpaa2_bpid_info && 738 rte_eal_process_type() == RTE_PROC_SECONDARY)) 739 rte_dpaa2_bpid_info = dpaa2_q->bp_array; 740 741 swp = DPAA2_PER_LCORE_ETHRX_PORTAL; 742 pull_size = (nb_pkts > dpaa2_dqrr_size) ? dpaa2_dqrr_size : nb_pkts; 743 if (unlikely(!q_storage->active_dqs)) { 744 q_storage->toggle = 0; 745 dq_storage = q_storage->dq_storage[q_storage->toggle]; 746 q_storage->last_num_pkts = pull_size; 747 qbman_pull_desc_clear(&pulldesc); 748 qbman_pull_desc_set_numframes(&pulldesc, 749 q_storage->last_num_pkts); 750 qbman_pull_desc_set_fq(&pulldesc, fqid); 751 qbman_pull_desc_set_storage(&pulldesc, dq_storage, 752 (uint64_t)(DPAA2_VADDR_TO_IOVA(dq_storage)), 1); 753 if (check_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index)) { 754 while (!qbman_check_command_complete( 755 get_swp_active_dqs( 756 DPAA2_PER_LCORE_ETHRX_DPIO->index))) 757 ; 758 clear_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index); 759 } 760 while (1) { 761 if (qbman_swp_pull(swp, &pulldesc)) { 762 DPAA2_PMD_DP_DEBUG("VDQ command is not issued." 763 " QBMAN is busy (1)\n"); 764 /* Portal was busy, try again */ 765 continue; 766 } 767 break; 768 } 769 q_storage->active_dqs = dq_storage; 770 q_storage->active_dpio_id = DPAA2_PER_LCORE_ETHRX_DPIO->index; 771 set_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index, 772 dq_storage); 773 } 774 775 dq_storage = q_storage->active_dqs; 776 rte_prefetch0((void *)(size_t)(dq_storage)); 777 rte_prefetch0((void *)(size_t)(dq_storage + 1)); 778 779 /* Prepare next pull descriptor. This will give space for the 780 * prefetching done on DQRR entries 781 */ 782 q_storage->toggle ^= 1; 783 dq_storage1 = q_storage->dq_storage[q_storage->toggle]; 784 qbman_pull_desc_clear(&pulldesc); 785 qbman_pull_desc_set_numframes(&pulldesc, pull_size); 786 qbman_pull_desc_set_fq(&pulldesc, fqid); 787 qbman_pull_desc_set_storage(&pulldesc, dq_storage1, 788 (uint64_t)(DPAA2_VADDR_TO_IOVA(dq_storage1)), 1); 789 790 /* Check if the previous issued command is completed. 791 * Also seems like the SWP is shared between the Ethernet Driver 792 * and the SEC driver. 793 */ 794 while (!qbman_check_command_complete(dq_storage)) 795 ; 796 if (dq_storage == get_swp_active_dqs(q_storage->active_dpio_id)) 797 clear_swp_active_dqs(q_storage->active_dpio_id); 798 799 pending = 1; 800 801 do { 802 /* Loop until the dq_storage is updated with 803 * new token by QBMAN 804 */ 805 while (!qbman_check_new_result(dq_storage)) 806 ; 807 rte_prefetch0((void *)((size_t)(dq_storage + 2))); 808 /* Check whether Last Pull command is Expired and 809 * setting Condition for Loop termination 810 */ 811 if (qbman_result_DQ_is_pull_complete(dq_storage)) { 812 pending = 0; 813 /* Check for valid frame. */ 814 status = qbman_result_DQ_flags(dq_storage); 815 if (unlikely((status & QBMAN_DQ_STAT_VALIDFRAME) == 0)) 816 continue; 817 } 818 fd = qbman_result_DQ_fd(dq_storage); 819 820 #ifndef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA 821 if (dpaa2_svr_family != SVR_LX2160A) { 822 const struct qbman_fd *next_fd = 823 qbman_result_DQ_fd(dq_storage + 1); 824 /* Prefetch Annotation address for the parse results */ 825 rte_prefetch0(DPAA2_IOVA_TO_VADDR((DPAA2_GET_FD_ADDR( 826 next_fd) + DPAA2_FD_PTA_SIZE + 16))); 827 } 828 #endif 829 830 if (unlikely(DPAA2_FD_GET_FORMAT(fd) == qbman_fd_sg)) 831 bufs[num_rx] = eth_sg_fd_to_mbuf(fd, eth_data->port_id); 832 else 833 bufs[num_rx] = eth_fd_to_mbuf(fd, eth_data->port_id); 834 #if defined(RTE_LIBRTE_IEEE1588) 835 if (bufs[num_rx]->ol_flags & RTE_MBUF_F_RX_IEEE1588_TMST) { 836 priv->rx_timestamp = 837 *dpaa2_timestamp_dynfield(bufs[num_rx]); 838 } 839 #endif 840 841 if (eth_data->dev_conf.rxmode.offloads & 842 RTE_ETH_RX_OFFLOAD_VLAN_STRIP) 843 rte_vlan_strip(bufs[num_rx]); 844 845 dq_storage++; 846 num_rx++; 847 } while (pending); 848 849 if (check_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index)) { 850 while (!qbman_check_command_complete( 851 get_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index))) 852 ; 853 clear_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index); 854 } 855 /* issue a volatile dequeue command for next pull */ 856 while (1) { 857 if (qbman_swp_pull(swp, &pulldesc)) { 858 DPAA2_PMD_DP_DEBUG("VDQ command is not issued." 859 "QBMAN is busy (2)\n"); 860 continue; 861 } 862 break; 863 } 864 q_storage->active_dqs = dq_storage1; 865 q_storage->active_dpio_id = DPAA2_PER_LCORE_ETHRX_DPIO->index; 866 set_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index, dq_storage1); 867 868 dpaa2_q->rx_pkts += num_rx; 869 870 return num_rx; 871 } 872 873 void __rte_hot 874 dpaa2_dev_process_parallel_event(struct qbman_swp *swp, 875 const struct qbman_fd *fd, 876 const struct qbman_result *dq, 877 struct dpaa2_queue *rxq, 878 struct rte_event *ev) 879 { 880 rte_prefetch0((void *)(size_t)(DPAA2_GET_FD_ADDR(fd) + 881 DPAA2_FD_PTA_SIZE + 16)); 882 883 ev->flow_id = rxq->ev.flow_id; 884 ev->sub_event_type = rxq->ev.sub_event_type; 885 ev->event_type = RTE_EVENT_TYPE_ETHDEV; 886 ev->op = RTE_EVENT_OP_NEW; 887 ev->sched_type = rxq->ev.sched_type; 888 ev->queue_id = rxq->ev.queue_id; 889 ev->priority = rxq->ev.priority; 890 891 ev->mbuf = eth_fd_to_mbuf(fd, rxq->eth_data->port_id); 892 893 qbman_swp_dqrr_consume(swp, dq); 894 } 895 896 void __rte_hot 897 dpaa2_dev_process_atomic_event(struct qbman_swp *swp __rte_unused, 898 const struct qbman_fd *fd, 899 const struct qbman_result *dq, 900 struct dpaa2_queue *rxq, 901 struct rte_event *ev) 902 { 903 uint8_t dqrr_index; 904 905 rte_prefetch0((void *)(size_t)(DPAA2_GET_FD_ADDR(fd) + 906 DPAA2_FD_PTA_SIZE + 16)); 907 908 ev->flow_id = rxq->ev.flow_id; 909 ev->sub_event_type = rxq->ev.sub_event_type; 910 ev->event_type = RTE_EVENT_TYPE_ETHDEV; 911 ev->op = RTE_EVENT_OP_NEW; 912 ev->sched_type = rxq->ev.sched_type; 913 ev->queue_id = rxq->ev.queue_id; 914 ev->priority = rxq->ev.priority; 915 916 ev->mbuf = eth_fd_to_mbuf(fd, rxq->eth_data->port_id); 917 918 dqrr_index = qbman_get_dqrr_idx(dq); 919 *dpaa2_seqn(ev->mbuf) = dqrr_index + 1; 920 DPAA2_PER_LCORE_DQRR_SIZE++; 921 DPAA2_PER_LCORE_DQRR_HELD |= 1 << dqrr_index; 922 DPAA2_PER_LCORE_DQRR_MBUF(dqrr_index) = ev->mbuf; 923 } 924 925 void __rte_hot 926 dpaa2_dev_process_ordered_event(struct qbman_swp *swp, 927 const struct qbman_fd *fd, 928 const struct qbman_result *dq, 929 struct dpaa2_queue *rxq, 930 struct rte_event *ev) 931 { 932 rte_prefetch0((void *)(size_t)(DPAA2_GET_FD_ADDR(fd) + 933 DPAA2_FD_PTA_SIZE + 16)); 934 935 ev->flow_id = rxq->ev.flow_id; 936 ev->sub_event_type = rxq->ev.sub_event_type; 937 ev->event_type = RTE_EVENT_TYPE_ETHDEV; 938 ev->op = RTE_EVENT_OP_NEW; 939 ev->sched_type = rxq->ev.sched_type; 940 ev->queue_id = rxq->ev.queue_id; 941 ev->priority = rxq->ev.priority; 942 943 ev->mbuf = eth_fd_to_mbuf(fd, rxq->eth_data->port_id); 944 945 *dpaa2_seqn(ev->mbuf) = DPAA2_ENQUEUE_FLAG_ORP; 946 *dpaa2_seqn(ev->mbuf) |= qbman_result_DQ_odpid(dq) << DPAA2_EQCR_OPRID_SHIFT; 947 *dpaa2_seqn(ev->mbuf) |= qbman_result_DQ_seqnum(dq) << DPAA2_EQCR_SEQNUM_SHIFT; 948 949 qbman_swp_dqrr_consume(swp, dq); 950 } 951 952 uint16_t 953 dpaa2_dev_rx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) 954 { 955 /* Function receive frames for a given device and VQ */ 956 struct dpaa2_queue *dpaa2_q = (struct dpaa2_queue *)queue; 957 struct qbman_result *dq_storage; 958 uint32_t fqid = dpaa2_q->fqid; 959 int ret, num_rx = 0, next_pull = nb_pkts, num_pulled; 960 uint8_t pending, status; 961 struct qbman_swp *swp; 962 const struct qbman_fd *fd; 963 struct qbman_pull_desc pulldesc; 964 struct rte_eth_dev_data *eth_data = dpaa2_q->eth_data; 965 struct dpaa2_dev_priv *priv = eth_data->dev_private; 966 967 if (unlikely(dpaa2_enable_err_queue)) 968 dump_err_pkts(priv->rx_err_vq); 969 970 if (unlikely(!DPAA2_PER_LCORE_DPIO)) { 971 ret = dpaa2_affine_qbman_swp(); 972 if (ret) { 973 DPAA2_PMD_ERR( 974 "Failed to allocate IO portal, tid: %d\n", 975 rte_gettid()); 976 return 0; 977 } 978 } 979 swp = DPAA2_PER_LCORE_PORTAL; 980 981 do { 982 dq_storage = dpaa2_q->q_storage->dq_storage[0]; 983 qbman_pull_desc_clear(&pulldesc); 984 qbman_pull_desc_set_fq(&pulldesc, fqid); 985 qbman_pull_desc_set_storage(&pulldesc, dq_storage, 986 (size_t)(DPAA2_VADDR_TO_IOVA(dq_storage)), 1); 987 988 if (next_pull > dpaa2_dqrr_size) { 989 qbman_pull_desc_set_numframes(&pulldesc, 990 dpaa2_dqrr_size); 991 next_pull -= dpaa2_dqrr_size; 992 } else { 993 qbman_pull_desc_set_numframes(&pulldesc, next_pull); 994 next_pull = 0; 995 } 996 997 while (1) { 998 if (qbman_swp_pull(swp, &pulldesc)) { 999 DPAA2_PMD_DP_DEBUG( 1000 "VDQ command is not issued.QBMAN is busy\n"); 1001 /* Portal was busy, try again */ 1002 continue; 1003 } 1004 break; 1005 } 1006 1007 rte_prefetch0((void *)((size_t)(dq_storage + 1))); 1008 /* Check if the previous issued command is completed. */ 1009 while (!qbman_check_command_complete(dq_storage)) 1010 ; 1011 1012 num_pulled = 0; 1013 pending = 1; 1014 do { 1015 /* Loop until the dq_storage is updated with 1016 * new token by QBMAN 1017 */ 1018 while (!qbman_check_new_result(dq_storage)) 1019 ; 1020 rte_prefetch0((void *)((size_t)(dq_storage + 2))); 1021 /* Check whether Last Pull command is Expired and 1022 * setting Condition for Loop termination 1023 */ 1024 if (qbman_result_DQ_is_pull_complete(dq_storage)) { 1025 pending = 0; 1026 /* Check for valid frame. */ 1027 status = qbman_result_DQ_flags(dq_storage); 1028 if (unlikely((status & 1029 QBMAN_DQ_STAT_VALIDFRAME) == 0)) 1030 continue; 1031 } 1032 fd = qbman_result_DQ_fd(dq_storage); 1033 1034 #ifndef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA 1035 if (dpaa2_svr_family != SVR_LX2160A) { 1036 const struct qbman_fd *next_fd = 1037 qbman_result_DQ_fd(dq_storage + 1); 1038 1039 /* Prefetch Annotation address for the parse 1040 * results. 1041 */ 1042 rte_prefetch0((DPAA2_IOVA_TO_VADDR( 1043 DPAA2_GET_FD_ADDR(next_fd) + 1044 DPAA2_FD_PTA_SIZE + 16))); 1045 } 1046 #endif 1047 1048 if (unlikely(DPAA2_FD_GET_FORMAT(fd) == qbman_fd_sg)) 1049 bufs[num_rx] = eth_sg_fd_to_mbuf(fd, 1050 eth_data->port_id); 1051 else 1052 bufs[num_rx] = eth_fd_to_mbuf(fd, 1053 eth_data->port_id); 1054 1055 #if defined(RTE_LIBRTE_IEEE1588) 1056 if (bufs[num_rx]->ol_flags & RTE_MBUF_F_RX_IEEE1588_TMST) { 1057 priv->rx_timestamp = 1058 *dpaa2_timestamp_dynfield(bufs[num_rx]); 1059 } 1060 #endif 1061 1062 if (eth_data->dev_conf.rxmode.offloads & 1063 RTE_ETH_RX_OFFLOAD_VLAN_STRIP) { 1064 rte_vlan_strip(bufs[num_rx]); 1065 } 1066 1067 dq_storage++; 1068 num_rx++; 1069 num_pulled++; 1070 } while (pending); 1071 /* Last VDQ provided all packets and more packets are requested */ 1072 } while (next_pull && num_pulled == dpaa2_dqrr_size); 1073 1074 dpaa2_q->rx_pkts += num_rx; 1075 1076 return num_rx; 1077 } 1078 1079 uint16_t dpaa2_dev_tx_conf(void *queue) 1080 { 1081 /* Function receive frames for a given device and VQ */ 1082 struct dpaa2_queue *dpaa2_q = (struct dpaa2_queue *)queue; 1083 struct qbman_result *dq_storage; 1084 uint32_t fqid = dpaa2_q->fqid; 1085 int ret, num_tx_conf = 0, num_pulled; 1086 uint8_t pending, status; 1087 struct qbman_swp *swp; 1088 const struct qbman_fd *fd, *next_fd; 1089 struct qbman_pull_desc pulldesc; 1090 struct qbman_release_desc releasedesc; 1091 uint32_t bpid; 1092 uint64_t buf; 1093 #if defined(RTE_LIBRTE_IEEE1588) 1094 struct rte_eth_dev_data *eth_data = dpaa2_q->eth_data; 1095 struct dpaa2_dev_priv *priv = eth_data->dev_private; 1096 struct dpaa2_annot_hdr *annotation; 1097 void *v_addr; 1098 struct rte_mbuf *mbuf; 1099 #endif 1100 1101 if (unlikely(!DPAA2_PER_LCORE_DPIO)) { 1102 ret = dpaa2_affine_qbman_swp(); 1103 if (ret) { 1104 DPAA2_PMD_ERR( 1105 "Failed to allocate IO portal, tid: %d\n", 1106 rte_gettid()); 1107 return 0; 1108 } 1109 } 1110 swp = DPAA2_PER_LCORE_PORTAL; 1111 1112 do { 1113 dq_storage = dpaa2_q->q_storage->dq_storage[0]; 1114 qbman_pull_desc_clear(&pulldesc); 1115 qbman_pull_desc_set_fq(&pulldesc, fqid); 1116 qbman_pull_desc_set_storage(&pulldesc, dq_storage, 1117 (size_t)(DPAA2_VADDR_TO_IOVA(dq_storage)), 1); 1118 1119 qbman_pull_desc_set_numframes(&pulldesc, dpaa2_dqrr_size); 1120 1121 while (1) { 1122 if (qbman_swp_pull(swp, &pulldesc)) { 1123 DPAA2_PMD_DP_DEBUG("VDQ command is not issued." 1124 "QBMAN is busy\n"); 1125 /* Portal was busy, try again */ 1126 continue; 1127 } 1128 break; 1129 } 1130 1131 rte_prefetch0((void *)((size_t)(dq_storage + 1))); 1132 /* Check if the previous issued command is completed. */ 1133 while (!qbman_check_command_complete(dq_storage)) 1134 ; 1135 1136 num_pulled = 0; 1137 pending = 1; 1138 do { 1139 /* Loop until the dq_storage is updated with 1140 * new token by QBMAN 1141 */ 1142 while (!qbman_check_new_result(dq_storage)) 1143 ; 1144 rte_prefetch0((void *)((size_t)(dq_storage + 2))); 1145 /* Check whether Last Pull command is Expired and 1146 * setting Condition for Loop termination 1147 */ 1148 if (qbman_result_DQ_is_pull_complete(dq_storage)) { 1149 pending = 0; 1150 /* Check for valid frame. */ 1151 status = qbman_result_DQ_flags(dq_storage); 1152 if (unlikely((status & 1153 QBMAN_DQ_STAT_VALIDFRAME) == 0)) 1154 continue; 1155 } 1156 fd = qbman_result_DQ_fd(dq_storage); 1157 1158 next_fd = qbman_result_DQ_fd(dq_storage + 1); 1159 /* Prefetch Annotation address for the parse results */ 1160 rte_prefetch0((void *)(size_t) 1161 (DPAA2_GET_FD_ADDR(next_fd) + 1162 DPAA2_FD_PTA_SIZE + 16)); 1163 1164 bpid = DPAA2_GET_FD_BPID(fd); 1165 1166 /* Create a release descriptor required for releasing 1167 * buffers into QBMAN 1168 */ 1169 qbman_release_desc_clear(&releasedesc); 1170 qbman_release_desc_set_bpid(&releasedesc, bpid); 1171 1172 buf = DPAA2_GET_FD_ADDR(fd); 1173 /* feed them to bman */ 1174 do { 1175 ret = qbman_swp_release(swp, &releasedesc, 1176 &buf, 1); 1177 } while (ret == -EBUSY); 1178 1179 dq_storage++; 1180 num_tx_conf++; 1181 num_pulled++; 1182 #if defined(RTE_LIBRTE_IEEE1588) 1183 v_addr = DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd)); 1184 mbuf = DPAA2_INLINE_MBUF_FROM_BUF(v_addr, 1185 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size); 1186 1187 if (mbuf->ol_flags & RTE_MBUF_F_TX_IEEE1588_TMST) { 1188 annotation = (struct dpaa2_annot_hdr *)((size_t) 1189 DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd)) + 1190 DPAA2_FD_PTA_SIZE); 1191 priv->tx_timestamp = annotation->word2; 1192 } 1193 #endif 1194 } while (pending); 1195 1196 /* Last VDQ provided all packets and more packets are requested */ 1197 } while (num_pulled == dpaa2_dqrr_size); 1198 1199 dpaa2_q->rx_pkts += num_tx_conf; 1200 1201 return num_tx_conf; 1202 } 1203 1204 /* Configure the egress frame annotation for timestamp update */ 1205 static void enable_tx_tstamp(struct qbman_fd *fd) 1206 { 1207 struct dpaa2_faead *fd_faead; 1208 1209 /* Set frame annotation status field as valid */ 1210 (fd)->simple.frc |= DPAA2_FD_FRC_FASV; 1211 1212 /* Set frame annotation egress action descriptor as valid */ 1213 (fd)->simple.frc |= DPAA2_FD_FRC_FAEADV; 1214 1215 /* Set Annotation Length as 128B */ 1216 (fd)->simple.ctrl |= DPAA2_FD_CTRL_ASAL; 1217 1218 /* enable update of confirmation frame annotation */ 1219 fd_faead = (struct dpaa2_faead *)((size_t) 1220 DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd)) + 1221 DPAA2_FD_PTA_SIZE + DPAA2_FD_HW_ANNOT_FAEAD_OFFSET); 1222 fd_faead->ctrl = DPAA2_ANNOT_FAEAD_A2V | DPAA2_ANNOT_FAEAD_UPDV | 1223 DPAA2_ANNOT_FAEAD_UPD; 1224 } 1225 1226 /* 1227 * Callback to handle sending packets through WRIOP based interface 1228 */ 1229 uint16_t 1230 dpaa2_dev_tx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) 1231 { 1232 /* Function to transmit the frames to given device and VQ*/ 1233 uint32_t loop, retry_count; 1234 int32_t ret; 1235 struct qbman_fd fd_arr[MAX_TX_RING_SLOTS]; 1236 struct rte_mbuf *mi; 1237 uint32_t frames_to_send; 1238 struct rte_mempool *mp; 1239 struct qbman_eq_desc eqdesc; 1240 struct dpaa2_queue *dpaa2_q = (struct dpaa2_queue *)queue; 1241 struct qbman_swp *swp; 1242 uint16_t num_tx = 0; 1243 uint16_t bpid; 1244 struct rte_eth_dev_data *eth_data = dpaa2_q->eth_data; 1245 struct dpaa2_dev_priv *priv = eth_data->dev_private; 1246 uint32_t flags[MAX_TX_RING_SLOTS] = {0}; 1247 struct sw_buf_free buf_to_free[DPAA2_MAX_SGS * dpaa2_dqrr_size]; 1248 uint32_t free_count = 0; 1249 1250 if (unlikely(!DPAA2_PER_LCORE_DPIO)) { 1251 ret = dpaa2_affine_qbman_swp(); 1252 if (ret) { 1253 DPAA2_PMD_ERR( 1254 "Failed to allocate IO portal, tid: %d\n", 1255 rte_gettid()); 1256 return 0; 1257 } 1258 } 1259 swp = DPAA2_PER_LCORE_PORTAL; 1260 1261 DPAA2_PMD_DP_DEBUG("===> eth_data =%p, fqid =%d\n", 1262 eth_data, dpaa2_q->fqid); 1263 1264 #ifdef RTE_LIBRTE_IEEE1588 1265 /* IEEE1588 driver need pointer to tx confirmation queue 1266 * corresponding to last packet transmitted for reading 1267 * the timestamp 1268 */ 1269 if ((*bufs)->ol_flags & RTE_MBUF_F_TX_IEEE1588_TMST) { 1270 priv->next_tx_conf_queue = dpaa2_q->tx_conf_queue; 1271 dpaa2_dev_tx_conf(dpaa2_q->tx_conf_queue); 1272 priv->tx_timestamp = 0; 1273 } 1274 #endif 1275 1276 /*Prepare enqueue descriptor*/ 1277 qbman_eq_desc_clear(&eqdesc); 1278 qbman_eq_desc_set_no_orp(&eqdesc, DPAA2_EQ_RESP_ERR_FQ); 1279 qbman_eq_desc_set_fq(&eqdesc, dpaa2_q->fqid); 1280 1281 /*Clear the unused FD fields before sending*/ 1282 while (nb_pkts) { 1283 /*Check if the queue is congested*/ 1284 retry_count = 0; 1285 while (qbman_result_SCN_state(dpaa2_q->cscn)) { 1286 retry_count++; 1287 /* Retry for some time before giving up */ 1288 if (retry_count > CONG_RETRY_COUNT) 1289 goto skip_tx; 1290 } 1291 1292 frames_to_send = (nb_pkts > dpaa2_eqcr_size) ? 1293 dpaa2_eqcr_size : nb_pkts; 1294 1295 for (loop = 0; loop < frames_to_send; loop++) { 1296 if (*dpaa2_seqn(*bufs)) { 1297 uint8_t dqrr_index = *dpaa2_seqn(*bufs) - 1; 1298 1299 flags[loop] = QBMAN_ENQUEUE_FLAG_DCA | 1300 dqrr_index; 1301 DPAA2_PER_LCORE_DQRR_SIZE--; 1302 DPAA2_PER_LCORE_DQRR_HELD &= ~(1 << dqrr_index); 1303 *dpaa2_seqn(*bufs) = DPAA2_INVALID_MBUF_SEQN; 1304 } 1305 1306 if (likely(RTE_MBUF_DIRECT(*bufs))) { 1307 mp = (*bufs)->pool; 1308 /* Check the basic scenario and set 1309 * the FD appropriately here itself. 1310 */ 1311 if (likely(mp && mp->ops_index == 1312 priv->bp_list->dpaa2_ops_index && 1313 (*bufs)->nb_segs == 1 && 1314 rte_mbuf_refcnt_read((*bufs)) == 1)) { 1315 if (unlikely(((*bufs)->ol_flags 1316 & RTE_MBUF_F_TX_VLAN) || 1317 (eth_data->dev_conf.txmode.offloads 1318 & RTE_ETH_TX_OFFLOAD_VLAN_INSERT))) { 1319 ret = rte_vlan_insert(bufs); 1320 if (ret) 1321 goto send_n_return; 1322 } 1323 DPAA2_MBUF_TO_CONTIG_FD((*bufs), 1324 &fd_arr[loop], mempool_to_bpid(mp)); 1325 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG 1326 rte_mempool_check_cookies 1327 (rte_mempool_from_obj((void *)*bufs), 1328 (void **)bufs, 1, 0); 1329 #endif 1330 bufs++; 1331 #ifdef RTE_LIBRTE_IEEE1588 1332 enable_tx_tstamp(&fd_arr[loop]); 1333 #endif 1334 continue; 1335 } 1336 } else { 1337 mi = rte_mbuf_from_indirect(*bufs); 1338 mp = mi->pool; 1339 } 1340 1341 if (unlikely(RTE_MBUF_HAS_EXTBUF(*bufs))) { 1342 if (unlikely((*bufs)->nb_segs > 1)) { 1343 mp = (*bufs)->pool; 1344 if (eth_mbuf_to_sg_fd(*bufs, 1345 &fd_arr[loop], 1346 buf_to_free, 1347 &free_count, 1348 loop, 1349 mempool_to_bpid(mp))) 1350 goto send_n_return; 1351 } else { 1352 eth_mbuf_to_fd(*bufs, 1353 &fd_arr[loop], 1354 buf_to_free, 1355 &free_count, 1356 loop, 0); 1357 } 1358 bufs++; 1359 #ifdef RTE_LIBRTE_IEEE1588 1360 enable_tx_tstamp(&fd_arr[loop]); 1361 #endif 1362 continue; 1363 } 1364 1365 /* Not a hw_pkt pool allocated frame */ 1366 if (unlikely(!mp || !priv->bp_list)) { 1367 DPAA2_PMD_ERR("Err: No buffer pool attached"); 1368 goto send_n_return; 1369 } 1370 1371 if (unlikely(((*bufs)->ol_flags & RTE_MBUF_F_TX_VLAN) || 1372 (eth_data->dev_conf.txmode.offloads 1373 & RTE_ETH_TX_OFFLOAD_VLAN_INSERT))) { 1374 int ret = rte_vlan_insert(bufs); 1375 if (ret) 1376 goto send_n_return; 1377 } 1378 if (mp->ops_index != priv->bp_list->dpaa2_ops_index) { 1379 DPAA2_PMD_WARN("Non DPAA2 buffer pool"); 1380 /* alloc should be from the default buffer pool 1381 * attached to this interface 1382 */ 1383 bpid = priv->bp_list->buf_pool.bpid; 1384 1385 if (unlikely((*bufs)->nb_segs > 1)) { 1386 DPAA2_PMD_ERR("S/G support not added" 1387 " for non hw offload buffer"); 1388 goto send_n_return; 1389 } 1390 if (eth_copy_mbuf_to_fd(*bufs, 1391 &fd_arr[loop], bpid)) { 1392 goto send_n_return; 1393 } 1394 /* free the original packet */ 1395 rte_pktmbuf_free(*bufs); 1396 } else { 1397 bpid = mempool_to_bpid(mp); 1398 if (unlikely((*bufs)->nb_segs > 1)) { 1399 if (eth_mbuf_to_sg_fd(*bufs, 1400 &fd_arr[loop], 1401 buf_to_free, 1402 &free_count, 1403 loop, 1404 bpid)) 1405 goto send_n_return; 1406 } else { 1407 eth_mbuf_to_fd(*bufs, 1408 &fd_arr[loop], 1409 buf_to_free, 1410 &free_count, 1411 loop, bpid); 1412 } 1413 } 1414 #ifdef RTE_LIBRTE_IEEE1588 1415 enable_tx_tstamp(&fd_arr[loop]); 1416 #endif 1417 bufs++; 1418 } 1419 1420 loop = 0; 1421 retry_count = 0; 1422 while (loop < frames_to_send) { 1423 ret = qbman_swp_enqueue_multiple(swp, &eqdesc, 1424 &fd_arr[loop], &flags[loop], 1425 frames_to_send - loop); 1426 if (unlikely(ret < 0)) { 1427 retry_count++; 1428 if (retry_count > DPAA2_MAX_TX_RETRY_COUNT) { 1429 num_tx += loop; 1430 nb_pkts -= loop; 1431 goto send_n_return; 1432 } 1433 } else { 1434 loop += ret; 1435 retry_count = 0; 1436 } 1437 } 1438 1439 num_tx += loop; 1440 nb_pkts -= loop; 1441 } 1442 dpaa2_q->tx_pkts += num_tx; 1443 1444 for (loop = 0; loop < free_count; loop++) { 1445 if (buf_to_free[loop].pkt_id < num_tx) 1446 rte_pktmbuf_free_seg(buf_to_free[loop].seg); 1447 } 1448 1449 return num_tx; 1450 1451 send_n_return: 1452 /* send any already prepared fd */ 1453 if (loop) { 1454 unsigned int i = 0; 1455 1456 retry_count = 0; 1457 while (i < loop) { 1458 ret = qbman_swp_enqueue_multiple(swp, &eqdesc, 1459 &fd_arr[i], 1460 &flags[i], 1461 loop - i); 1462 if (unlikely(ret < 0)) { 1463 retry_count++; 1464 if (retry_count > DPAA2_MAX_TX_RETRY_COUNT) 1465 break; 1466 } else { 1467 i += ret; 1468 retry_count = 0; 1469 } 1470 } 1471 num_tx += i; 1472 } 1473 skip_tx: 1474 dpaa2_q->tx_pkts += num_tx; 1475 1476 for (loop = 0; loop < free_count; loop++) { 1477 if (buf_to_free[loop].pkt_id < num_tx) 1478 rte_pktmbuf_free_seg(buf_to_free[loop].seg); 1479 } 1480 1481 return num_tx; 1482 } 1483 1484 void 1485 dpaa2_dev_free_eqresp_buf(uint16_t eqresp_ci, 1486 __rte_unused struct dpaa2_queue *dpaa2_q) 1487 { 1488 struct dpaa2_dpio_dev *dpio_dev = DPAA2_PER_LCORE_DPIO; 1489 struct qbman_fd *fd; 1490 struct rte_mbuf *m; 1491 1492 fd = qbman_result_eqresp_fd(&dpio_dev->eqresp[eqresp_ci]); 1493 1494 /* Setting port id does not matter as we are to free the mbuf */ 1495 m = eth_fd_to_mbuf(fd, 0); 1496 rte_pktmbuf_free(m); 1497 } 1498 1499 static void 1500 dpaa2_set_enqueue_descriptor(struct dpaa2_queue *dpaa2_q, 1501 struct rte_mbuf *m, 1502 struct qbman_eq_desc *eqdesc) 1503 { 1504 struct rte_eth_dev_data *eth_data = dpaa2_q->eth_data; 1505 struct dpaa2_dev_priv *priv = eth_data->dev_private; 1506 struct dpaa2_dpio_dev *dpio_dev = DPAA2_PER_LCORE_DPIO; 1507 struct eqresp_metadata *eqresp_meta; 1508 uint16_t orpid, seqnum; 1509 uint8_t dq_idx; 1510 1511 qbman_eq_desc_set_fq(eqdesc, dpaa2_q->fqid); 1512 1513 if (*dpaa2_seqn(m) & DPAA2_ENQUEUE_FLAG_ORP) { 1514 orpid = (*dpaa2_seqn(m) & DPAA2_EQCR_OPRID_MASK) >> 1515 DPAA2_EQCR_OPRID_SHIFT; 1516 seqnum = (*dpaa2_seqn(m) & DPAA2_EQCR_SEQNUM_MASK) >> 1517 DPAA2_EQCR_SEQNUM_SHIFT; 1518 1519 if (!priv->en_loose_ordered) { 1520 qbman_eq_desc_set_orp(eqdesc, 1, orpid, seqnum, 0); 1521 qbman_eq_desc_set_response(eqdesc, (uint64_t) 1522 DPAA2_VADDR_TO_IOVA(&dpio_dev->eqresp[ 1523 dpio_dev->eqresp_pi]), 1); 1524 qbman_eq_desc_set_token(eqdesc, 1); 1525 1526 eqresp_meta = &dpio_dev->eqresp_meta[ 1527 dpio_dev->eqresp_pi]; 1528 eqresp_meta->dpaa2_q = dpaa2_q; 1529 eqresp_meta->mp = m->pool; 1530 1531 dpio_dev->eqresp_pi + 1 < MAX_EQ_RESP_ENTRIES ? 1532 dpio_dev->eqresp_pi++ : 1533 (dpio_dev->eqresp_pi = 0); 1534 } else { 1535 qbman_eq_desc_set_orp(eqdesc, 0, orpid, seqnum, 0); 1536 } 1537 } else { 1538 dq_idx = *dpaa2_seqn(m) - 1; 1539 qbman_eq_desc_set_dca(eqdesc, 1, dq_idx, 0); 1540 DPAA2_PER_LCORE_DQRR_SIZE--; 1541 DPAA2_PER_LCORE_DQRR_HELD &= ~(1 << dq_idx); 1542 } 1543 *dpaa2_seqn(m) = DPAA2_INVALID_MBUF_SEQN; 1544 } 1545 1546 uint16_t 1547 dpaa2_dev_tx_multi_txq_ordered(void **queue, 1548 struct rte_mbuf **bufs, uint16_t nb_pkts) 1549 { 1550 /* Function to transmit the frames to multiple queues respectively.*/ 1551 uint32_t loop, i, retry_count; 1552 int32_t ret; 1553 struct qbman_fd fd_arr[MAX_TX_RING_SLOTS]; 1554 uint32_t frames_to_send, num_free_eq_desc = 0; 1555 struct rte_mempool *mp; 1556 struct qbman_eq_desc eqdesc[MAX_TX_RING_SLOTS]; 1557 struct dpaa2_queue *dpaa2_q[MAX_TX_RING_SLOTS]; 1558 struct qbman_swp *swp; 1559 uint16_t bpid; 1560 struct rte_mbuf *mi; 1561 struct rte_eth_dev_data *eth_data; 1562 struct dpaa2_dev_priv *priv; 1563 struct dpaa2_queue *order_sendq; 1564 struct sw_buf_free buf_to_free[DPAA2_MAX_SGS * dpaa2_dqrr_size]; 1565 uint32_t free_count = 0; 1566 1567 if (unlikely(!DPAA2_PER_LCORE_DPIO)) { 1568 ret = dpaa2_affine_qbman_swp(); 1569 if (ret) { 1570 DPAA2_PMD_ERR( 1571 "Failed to allocate IO portal, tid: %d\n", 1572 rte_gettid()); 1573 return 0; 1574 } 1575 } 1576 swp = DPAA2_PER_LCORE_PORTAL; 1577 1578 frames_to_send = (nb_pkts > dpaa2_eqcr_size) ? 1579 dpaa2_eqcr_size : nb_pkts; 1580 1581 for (loop = 0; loop < frames_to_send; loop++) { 1582 dpaa2_q[loop] = (struct dpaa2_queue *)queue[loop]; 1583 eth_data = dpaa2_q[loop]->eth_data; 1584 priv = eth_data->dev_private; 1585 if (!priv->en_loose_ordered) { 1586 if (*dpaa2_seqn(*bufs) & DPAA2_ENQUEUE_FLAG_ORP) { 1587 if (!num_free_eq_desc) { 1588 num_free_eq_desc = dpaa2_free_eq_descriptors(); 1589 if (!num_free_eq_desc) 1590 goto send_frames; 1591 } 1592 num_free_eq_desc--; 1593 } 1594 } 1595 1596 DPAA2_PMD_DP_DEBUG("===> eth_data =%p, fqid =%d\n", 1597 eth_data, dpaa2_q[loop]->fqid); 1598 1599 /* Check if the queue is congested */ 1600 retry_count = 0; 1601 while (qbman_result_SCN_state(dpaa2_q[loop]->cscn)) { 1602 retry_count++; 1603 /* Retry for some time before giving up */ 1604 if (retry_count > CONG_RETRY_COUNT) 1605 goto send_frames; 1606 } 1607 1608 /* Prepare enqueue descriptor */ 1609 qbman_eq_desc_clear(&eqdesc[loop]); 1610 1611 if (*dpaa2_seqn(*bufs) && priv->en_ordered) { 1612 order_sendq = (struct dpaa2_queue *)priv->tx_vq[0]; 1613 dpaa2_set_enqueue_descriptor(order_sendq, 1614 (*bufs), 1615 &eqdesc[loop]); 1616 } else { 1617 qbman_eq_desc_set_no_orp(&eqdesc[loop], 1618 DPAA2_EQ_RESP_ERR_FQ); 1619 qbman_eq_desc_set_fq(&eqdesc[loop], 1620 dpaa2_q[loop]->fqid); 1621 } 1622 1623 if (likely(RTE_MBUF_DIRECT(*bufs))) { 1624 mp = (*bufs)->pool; 1625 /* Check the basic scenario and set 1626 * the FD appropriately here itself. 1627 */ 1628 if (likely(mp && mp->ops_index == 1629 priv->bp_list->dpaa2_ops_index && 1630 (*bufs)->nb_segs == 1 && 1631 rte_mbuf_refcnt_read((*bufs)) == 1)) { 1632 if (unlikely((*bufs)->ol_flags 1633 & RTE_MBUF_F_TX_VLAN)) { 1634 ret = rte_vlan_insert(bufs); 1635 if (ret) 1636 goto send_frames; 1637 } 1638 DPAA2_MBUF_TO_CONTIG_FD((*bufs), 1639 &fd_arr[loop], 1640 mempool_to_bpid(mp)); 1641 bufs++; 1642 continue; 1643 } 1644 } else { 1645 mi = rte_mbuf_from_indirect(*bufs); 1646 mp = mi->pool; 1647 } 1648 /* Not a hw_pkt pool allocated frame */ 1649 if (unlikely(!mp || !priv->bp_list)) { 1650 DPAA2_PMD_ERR("Err: No buffer pool attached"); 1651 goto send_frames; 1652 } 1653 1654 if (mp->ops_index != priv->bp_list->dpaa2_ops_index) { 1655 DPAA2_PMD_WARN("Non DPAA2 buffer pool"); 1656 /* alloc should be from the default buffer pool 1657 * attached to this interface 1658 */ 1659 bpid = priv->bp_list->buf_pool.bpid; 1660 1661 if (unlikely((*bufs)->nb_segs > 1)) { 1662 DPAA2_PMD_ERR( 1663 "S/G not supp for non hw offload buffer"); 1664 goto send_frames; 1665 } 1666 if (eth_copy_mbuf_to_fd(*bufs, 1667 &fd_arr[loop], bpid)) { 1668 goto send_frames; 1669 } 1670 /* free the original packet */ 1671 rte_pktmbuf_free(*bufs); 1672 } else { 1673 bpid = mempool_to_bpid(mp); 1674 if (unlikely((*bufs)->nb_segs > 1)) { 1675 if (eth_mbuf_to_sg_fd(*bufs, 1676 &fd_arr[loop], 1677 buf_to_free, 1678 &free_count, 1679 loop, 1680 bpid)) 1681 goto send_frames; 1682 } else { 1683 eth_mbuf_to_fd(*bufs, 1684 &fd_arr[loop], 1685 buf_to_free, 1686 &free_count, 1687 loop, bpid); 1688 } 1689 } 1690 1691 bufs++; 1692 } 1693 1694 send_frames: 1695 frames_to_send = loop; 1696 loop = 0; 1697 retry_count = 0; 1698 while (loop < frames_to_send) { 1699 ret = qbman_swp_enqueue_multiple_desc(swp, &eqdesc[loop], 1700 &fd_arr[loop], 1701 frames_to_send - loop); 1702 if (likely(ret > 0)) { 1703 loop += ret; 1704 retry_count = 0; 1705 } else { 1706 retry_count++; 1707 if (retry_count > DPAA2_MAX_TX_RETRY_COUNT) 1708 break; 1709 } 1710 } 1711 1712 for (i = 0; i < free_count; i++) { 1713 if (buf_to_free[i].pkt_id < loop) 1714 rte_pktmbuf_free_seg(buf_to_free[i].seg); 1715 } 1716 return loop; 1717 } 1718 1719 /* Callback to handle sending ordered packets through WRIOP based interface */ 1720 uint16_t 1721 dpaa2_dev_tx_ordered(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) 1722 { 1723 /* Function to transmit the frames to given device and VQ*/ 1724 struct dpaa2_queue *dpaa2_q = (struct dpaa2_queue *)queue; 1725 struct rte_eth_dev_data *eth_data = dpaa2_q->eth_data; 1726 struct dpaa2_dev_priv *priv = eth_data->dev_private; 1727 struct dpaa2_queue *order_sendq = (struct dpaa2_queue *)priv->tx_vq[0]; 1728 struct qbman_fd fd_arr[MAX_TX_RING_SLOTS]; 1729 struct rte_mbuf *mi; 1730 struct rte_mempool *mp; 1731 struct qbman_eq_desc eqdesc[MAX_TX_RING_SLOTS]; 1732 struct qbman_swp *swp; 1733 uint32_t frames_to_send, num_free_eq_desc; 1734 uint32_t loop, retry_count; 1735 int32_t ret; 1736 uint16_t num_tx = 0; 1737 uint16_t bpid; 1738 struct sw_buf_free buf_to_free[DPAA2_MAX_SGS * dpaa2_dqrr_size]; 1739 uint32_t free_count = 0; 1740 1741 if (unlikely(!DPAA2_PER_LCORE_DPIO)) { 1742 ret = dpaa2_affine_qbman_swp(); 1743 if (ret) { 1744 DPAA2_PMD_ERR( 1745 "Failed to allocate IO portal, tid: %d\n", 1746 rte_gettid()); 1747 return 0; 1748 } 1749 } 1750 swp = DPAA2_PER_LCORE_PORTAL; 1751 1752 DPAA2_PMD_DP_DEBUG("===> eth_data =%p, fqid =%d\n", 1753 eth_data, dpaa2_q->fqid); 1754 1755 /* This would also handle normal and atomic queues as any type 1756 * of packet can be enqueued when ordered queues are being used. 1757 */ 1758 while (nb_pkts) { 1759 /*Check if the queue is congested*/ 1760 retry_count = 0; 1761 while (qbman_result_SCN_state(dpaa2_q->cscn)) { 1762 retry_count++; 1763 /* Retry for some time before giving up */ 1764 if (retry_count > CONG_RETRY_COUNT) 1765 goto skip_tx; 1766 } 1767 1768 frames_to_send = (nb_pkts > dpaa2_eqcr_size) ? 1769 dpaa2_eqcr_size : nb_pkts; 1770 1771 if (!priv->en_loose_ordered) { 1772 if (*dpaa2_seqn(*bufs) & DPAA2_ENQUEUE_FLAG_ORP) { 1773 num_free_eq_desc = dpaa2_free_eq_descriptors(); 1774 if (num_free_eq_desc < frames_to_send) 1775 frames_to_send = num_free_eq_desc; 1776 } 1777 } 1778 1779 for (loop = 0; loop < frames_to_send; loop++) { 1780 /*Prepare enqueue descriptor*/ 1781 qbman_eq_desc_clear(&eqdesc[loop]); 1782 1783 if (*dpaa2_seqn(*bufs)) { 1784 /* Use only queue 0 for Tx in case of atomic/ 1785 * ordered packets as packets can get unordered 1786 * when being transmitted out from the interface 1787 */ 1788 dpaa2_set_enqueue_descriptor(order_sendq, 1789 (*bufs), 1790 &eqdesc[loop]); 1791 } else { 1792 qbman_eq_desc_set_no_orp(&eqdesc[loop], 1793 DPAA2_EQ_RESP_ERR_FQ); 1794 qbman_eq_desc_set_fq(&eqdesc[loop], 1795 dpaa2_q->fqid); 1796 } 1797 1798 if (likely(RTE_MBUF_DIRECT(*bufs))) { 1799 mp = (*bufs)->pool; 1800 /* Check the basic scenario and set 1801 * the FD appropriately here itself. 1802 */ 1803 if (likely(mp && mp->ops_index == 1804 priv->bp_list->dpaa2_ops_index && 1805 (*bufs)->nb_segs == 1 && 1806 rte_mbuf_refcnt_read((*bufs)) == 1)) { 1807 if (unlikely((*bufs)->ol_flags 1808 & RTE_MBUF_F_TX_VLAN)) { 1809 ret = rte_vlan_insert(bufs); 1810 if (ret) 1811 goto send_n_return; 1812 } 1813 DPAA2_MBUF_TO_CONTIG_FD((*bufs), 1814 &fd_arr[loop], 1815 mempool_to_bpid(mp)); 1816 bufs++; 1817 continue; 1818 } 1819 } else { 1820 mi = rte_mbuf_from_indirect(*bufs); 1821 mp = mi->pool; 1822 } 1823 /* Not a hw_pkt pool allocated frame */ 1824 if (unlikely(!mp || !priv->bp_list)) { 1825 DPAA2_PMD_ERR("Err: No buffer pool attached"); 1826 goto send_n_return; 1827 } 1828 1829 if (mp->ops_index != priv->bp_list->dpaa2_ops_index) { 1830 DPAA2_PMD_WARN("Non DPAA2 buffer pool"); 1831 /* alloc should be from the default buffer pool 1832 * attached to this interface 1833 */ 1834 bpid = priv->bp_list->buf_pool.bpid; 1835 1836 if (unlikely((*bufs)->nb_segs > 1)) { 1837 DPAA2_PMD_ERR( 1838 "S/G not supp for non hw offload buffer"); 1839 goto send_n_return; 1840 } 1841 if (eth_copy_mbuf_to_fd(*bufs, 1842 &fd_arr[loop], bpid)) { 1843 goto send_n_return; 1844 } 1845 /* free the original packet */ 1846 rte_pktmbuf_free(*bufs); 1847 } else { 1848 bpid = mempool_to_bpid(mp); 1849 if (unlikely((*bufs)->nb_segs > 1)) { 1850 if (eth_mbuf_to_sg_fd(*bufs, 1851 &fd_arr[loop], 1852 buf_to_free, 1853 &free_count, 1854 loop, 1855 bpid)) 1856 goto send_n_return; 1857 } else { 1858 eth_mbuf_to_fd(*bufs, 1859 &fd_arr[loop], 1860 buf_to_free, 1861 &free_count, 1862 loop, bpid); 1863 } 1864 } 1865 bufs++; 1866 } 1867 1868 loop = 0; 1869 retry_count = 0; 1870 while (loop < frames_to_send) { 1871 ret = qbman_swp_enqueue_multiple_desc(swp, 1872 &eqdesc[loop], &fd_arr[loop], 1873 frames_to_send - loop); 1874 if (unlikely(ret < 0)) { 1875 retry_count++; 1876 if (retry_count > DPAA2_MAX_TX_RETRY_COUNT) { 1877 num_tx += loop; 1878 nb_pkts -= loop; 1879 goto send_n_return; 1880 } 1881 } else { 1882 loop += ret; 1883 retry_count = 0; 1884 } 1885 } 1886 1887 num_tx += loop; 1888 nb_pkts -= loop; 1889 } 1890 dpaa2_q->tx_pkts += num_tx; 1891 for (loop = 0; loop < free_count; loop++) { 1892 if (buf_to_free[loop].pkt_id < num_tx) 1893 rte_pktmbuf_free_seg(buf_to_free[loop].seg); 1894 } 1895 1896 return num_tx; 1897 1898 send_n_return: 1899 /* send any already prepared fd */ 1900 if (loop) { 1901 unsigned int i = 0; 1902 1903 retry_count = 0; 1904 while (i < loop) { 1905 ret = qbman_swp_enqueue_multiple_desc(swp, 1906 &eqdesc[i], &fd_arr[i], loop - i); 1907 if (unlikely(ret < 0)) { 1908 retry_count++; 1909 if (retry_count > DPAA2_MAX_TX_RETRY_COUNT) 1910 break; 1911 } else { 1912 i += ret; 1913 retry_count = 0; 1914 } 1915 } 1916 num_tx += i; 1917 } 1918 skip_tx: 1919 dpaa2_q->tx_pkts += num_tx; 1920 for (loop = 0; loop < free_count; loop++) { 1921 if (buf_to_free[loop].pkt_id < num_tx) 1922 rte_pktmbuf_free_seg(buf_to_free[loop].seg); 1923 } 1924 1925 return num_tx; 1926 } 1927 1928 #if defined(RTE_TOOLCHAIN_GCC) 1929 #pragma GCC diagnostic push 1930 #pragma GCC diagnostic ignored "-Wcast-qual" 1931 #elif defined(RTE_TOOLCHAIN_CLANG) 1932 #pragma clang diagnostic push 1933 #pragma clang diagnostic ignored "-Wcast-qual" 1934 #endif 1935 1936 /* This function loopbacks all the received packets.*/ 1937 uint16_t 1938 dpaa2_dev_loopback_rx(void *queue, 1939 struct rte_mbuf **bufs __rte_unused, 1940 uint16_t nb_pkts) 1941 { 1942 /* Function receive frames for a given device and VQ*/ 1943 struct dpaa2_queue *dpaa2_q = (struct dpaa2_queue *)queue; 1944 struct qbman_result *dq_storage, *dq_storage1 = NULL; 1945 uint32_t fqid = dpaa2_q->fqid; 1946 int ret, num_rx = 0, num_tx = 0, pull_size; 1947 uint8_t pending, status; 1948 struct qbman_swp *swp; 1949 struct qbman_fd *fd[DPAA2_LX2_DQRR_RING_SIZE]; 1950 struct qbman_pull_desc pulldesc; 1951 struct qbman_eq_desc eqdesc; 1952 struct queue_storage_info_t *q_storage = dpaa2_q->q_storage; 1953 struct rte_eth_dev_data *eth_data = dpaa2_q->eth_data; 1954 struct dpaa2_dev_priv *priv = eth_data->dev_private; 1955 struct dpaa2_queue *tx_q = priv->tx_vq[0]; 1956 /* todo - currently we are using 1st TX queue only for loopback*/ 1957 1958 if (unlikely(!DPAA2_PER_LCORE_ETHRX_DPIO)) { 1959 ret = dpaa2_affine_qbman_ethrx_swp(); 1960 if (ret) { 1961 DPAA2_PMD_ERR("Failure in affining portal"); 1962 return 0; 1963 } 1964 } 1965 swp = DPAA2_PER_LCORE_ETHRX_PORTAL; 1966 pull_size = (nb_pkts > dpaa2_dqrr_size) ? dpaa2_dqrr_size : nb_pkts; 1967 if (unlikely(!q_storage->active_dqs)) { 1968 q_storage->toggle = 0; 1969 dq_storage = q_storage->dq_storage[q_storage->toggle]; 1970 q_storage->last_num_pkts = pull_size; 1971 qbman_pull_desc_clear(&pulldesc); 1972 qbman_pull_desc_set_numframes(&pulldesc, 1973 q_storage->last_num_pkts); 1974 qbman_pull_desc_set_fq(&pulldesc, fqid); 1975 qbman_pull_desc_set_storage(&pulldesc, dq_storage, 1976 (size_t)(DPAA2_VADDR_TO_IOVA(dq_storage)), 1); 1977 if (check_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index)) { 1978 while (!qbman_check_command_complete( 1979 get_swp_active_dqs( 1980 DPAA2_PER_LCORE_ETHRX_DPIO->index))) 1981 ; 1982 clear_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index); 1983 } 1984 while (1) { 1985 if (qbman_swp_pull(swp, &pulldesc)) { 1986 DPAA2_PMD_DP_DEBUG( 1987 "VDQ command not issued.QBMAN busy\n"); 1988 /* Portal was busy, try again */ 1989 continue; 1990 } 1991 break; 1992 } 1993 q_storage->active_dqs = dq_storage; 1994 q_storage->active_dpio_id = DPAA2_PER_LCORE_ETHRX_DPIO->index; 1995 set_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index, 1996 dq_storage); 1997 } 1998 1999 dq_storage = q_storage->active_dqs; 2000 rte_prefetch0((void *)(size_t)(dq_storage)); 2001 rte_prefetch0((void *)(size_t)(dq_storage + 1)); 2002 2003 /* Prepare next pull descriptor. This will give space for the 2004 * prefetching done on DQRR entries 2005 */ 2006 q_storage->toggle ^= 1; 2007 dq_storage1 = q_storage->dq_storage[q_storage->toggle]; 2008 qbman_pull_desc_clear(&pulldesc); 2009 qbman_pull_desc_set_numframes(&pulldesc, pull_size); 2010 qbman_pull_desc_set_fq(&pulldesc, fqid); 2011 qbman_pull_desc_set_storage(&pulldesc, dq_storage1, 2012 (size_t)(DPAA2_VADDR_TO_IOVA(dq_storage1)), 1); 2013 2014 /*Prepare enqueue descriptor*/ 2015 qbman_eq_desc_clear(&eqdesc); 2016 qbman_eq_desc_set_no_orp(&eqdesc, DPAA2_EQ_RESP_ERR_FQ); 2017 qbman_eq_desc_set_response(&eqdesc, 0, 0); 2018 qbman_eq_desc_set_fq(&eqdesc, tx_q->fqid); 2019 2020 /* Check if the previous issued command is completed. 2021 * Also seems like the SWP is shared between the Ethernet Driver 2022 * and the SEC driver. 2023 */ 2024 while (!qbman_check_command_complete(dq_storage)) 2025 ; 2026 if (dq_storage == get_swp_active_dqs(q_storage->active_dpio_id)) 2027 clear_swp_active_dqs(q_storage->active_dpio_id); 2028 2029 pending = 1; 2030 2031 do { 2032 /* Loop until the dq_storage is updated with 2033 * new token by QBMAN 2034 */ 2035 while (!qbman_check_new_result(dq_storage)) 2036 ; 2037 rte_prefetch0((void *)((size_t)(dq_storage + 2))); 2038 /* Check whether Last Pull command is Expired and 2039 * setting Condition for Loop termination 2040 */ 2041 if (qbman_result_DQ_is_pull_complete(dq_storage)) { 2042 pending = 0; 2043 /* Check for valid frame. */ 2044 status = qbman_result_DQ_flags(dq_storage); 2045 if (unlikely((status & QBMAN_DQ_STAT_VALIDFRAME) == 0)) 2046 continue; 2047 } 2048 fd[num_rx] = (struct qbman_fd *)qbman_result_DQ_fd(dq_storage); 2049 2050 dq_storage++; 2051 num_rx++; 2052 } while (pending); 2053 2054 while (num_tx < num_rx) { 2055 num_tx += qbman_swp_enqueue_multiple_fd(swp, &eqdesc, 2056 &fd[num_tx], 0, num_rx - num_tx); 2057 } 2058 2059 if (check_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index)) { 2060 while (!qbman_check_command_complete( 2061 get_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index))) 2062 ; 2063 clear_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index); 2064 } 2065 /* issue a volatile dequeue command for next pull */ 2066 while (1) { 2067 if (qbman_swp_pull(swp, &pulldesc)) { 2068 DPAA2_PMD_DP_DEBUG("VDQ command is not issued." 2069 "QBMAN is busy (2)\n"); 2070 continue; 2071 } 2072 break; 2073 } 2074 q_storage->active_dqs = dq_storage1; 2075 q_storage->active_dpio_id = DPAA2_PER_LCORE_ETHRX_DPIO->index; 2076 set_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index, dq_storage1); 2077 2078 dpaa2_q->rx_pkts += num_rx; 2079 dpaa2_q->tx_pkts += num_tx; 2080 2081 return 0; 2082 } 2083 #if defined(RTE_TOOLCHAIN_GCC) 2084 #pragma GCC diagnostic pop 2085 #elif defined(RTE_TOOLCHAIN_CLANG) 2086 #pragma clang diagnostic pop 2087 #endif 2088