1 /* SPDX-License-Identifier: BSD-3-Clause 2 * 3 * Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved. 4 * Copyright 2016 NXP 5 * 6 */ 7 8 #include <time.h> 9 #include <net/if.h> 10 11 #include <rte_mbuf.h> 12 #include <rte_ethdev.h> 13 #include <rte_malloc.h> 14 #include <rte_memcpy.h> 15 #include <rte_string_fns.h> 16 #include <rte_dev.h> 17 18 #include <fslmc_logs.h> 19 #include <fslmc_vfio.h> 20 #include <dpaa2_hw_pvt.h> 21 #include <dpaa2_hw_dpio.h> 22 #include <dpaa2_hw_mempool.h> 23 24 #include "dpaa2_ethdev.h" 25 #include "base/dpaa2_hw_dpni_annot.h" 26 27 static inline uint32_t __attribute__((hot)) 28 dpaa2_dev_rx_parse(uint64_t hw_annot_addr) 29 { 30 uint32_t pkt_type = RTE_PTYPE_UNKNOWN; 31 struct dpaa2_annot_hdr *annotation = 32 (struct dpaa2_annot_hdr *)hw_annot_addr; 33 34 PMD_RX_LOG(DEBUG, "annotation = 0x%lx ", annotation->word4); 35 36 if (BIT_ISSET_AT_POS(annotation->word3, L2_ARP_PRESENT)) { 37 pkt_type = RTE_PTYPE_L2_ETHER_ARP; 38 goto parse_done; 39 } else if (BIT_ISSET_AT_POS(annotation->word3, L2_ETH_MAC_PRESENT)) { 40 pkt_type = RTE_PTYPE_L2_ETHER; 41 } else { 42 goto parse_done; 43 } 44 45 if (BIT_ISSET_AT_POS(annotation->word4, L3_IPV4_1_PRESENT | 46 L3_IPV4_N_PRESENT)) { 47 pkt_type |= RTE_PTYPE_L3_IPV4; 48 if (BIT_ISSET_AT_POS(annotation->word4, L3_IP_1_OPT_PRESENT | 49 L3_IP_N_OPT_PRESENT)) 50 pkt_type |= RTE_PTYPE_L3_IPV4_EXT; 51 52 } else if (BIT_ISSET_AT_POS(annotation->word4, L3_IPV6_1_PRESENT | 53 L3_IPV6_N_PRESENT)) { 54 pkt_type |= RTE_PTYPE_L3_IPV6; 55 if (BIT_ISSET_AT_POS(annotation->word4, L3_IP_1_OPT_PRESENT | 56 L3_IP_N_OPT_PRESENT)) 57 pkt_type |= RTE_PTYPE_L3_IPV6_EXT; 58 } else { 59 goto parse_done; 60 } 61 62 if (BIT_ISSET_AT_POS(annotation->word4, L3_IP_1_FIRST_FRAGMENT | 63 L3_IP_1_MORE_FRAGMENT | 64 L3_IP_N_FIRST_FRAGMENT | 65 L3_IP_N_MORE_FRAGMENT)) { 66 pkt_type |= RTE_PTYPE_L4_FRAG; 67 goto parse_done; 68 } else { 69 pkt_type |= RTE_PTYPE_L4_NONFRAG; 70 } 71 72 if (BIT_ISSET_AT_POS(annotation->word4, L3_PROTO_UDP_PRESENT)) 73 pkt_type |= RTE_PTYPE_L4_UDP; 74 75 else if (BIT_ISSET_AT_POS(annotation->word4, L3_PROTO_TCP_PRESENT)) 76 pkt_type |= RTE_PTYPE_L4_TCP; 77 78 else if (BIT_ISSET_AT_POS(annotation->word4, L3_PROTO_SCTP_PRESENT)) 79 pkt_type |= RTE_PTYPE_L4_SCTP; 80 81 else if (BIT_ISSET_AT_POS(annotation->word4, L3_PROTO_ICMP_PRESENT)) 82 pkt_type |= RTE_PTYPE_L4_ICMP; 83 84 else if (BIT_ISSET_AT_POS(annotation->word4, L3_IP_UNKNOWN_PROTOCOL)) 85 pkt_type |= RTE_PTYPE_UNKNOWN; 86 87 parse_done: 88 return pkt_type; 89 } 90 91 static inline void __attribute__((hot)) 92 dpaa2_dev_rx_offload(uint64_t hw_annot_addr, struct rte_mbuf *mbuf) 93 { 94 struct dpaa2_annot_hdr *annotation = 95 (struct dpaa2_annot_hdr *)hw_annot_addr; 96 97 if (BIT_ISSET_AT_POS(annotation->word3, 98 L2_VLAN_1_PRESENT | L2_VLAN_N_PRESENT)) 99 mbuf->ol_flags |= PKT_RX_VLAN; 100 101 if (BIT_ISSET_AT_POS(annotation->word8, DPAA2_ETH_FAS_L3CE)) 102 mbuf->ol_flags |= PKT_RX_IP_CKSUM_BAD; 103 104 if (BIT_ISSET_AT_POS(annotation->word8, DPAA2_ETH_FAS_L4CE)) 105 mbuf->ol_flags |= PKT_RX_L4_CKSUM_BAD; 106 } 107 108 static inline struct rte_mbuf *__attribute__((hot)) 109 eth_sg_fd_to_mbuf(const struct qbman_fd *fd) 110 { 111 struct qbman_sge *sgt, *sge; 112 dma_addr_t sg_addr; 113 int i = 0; 114 uint64_t fd_addr; 115 struct rte_mbuf *first_seg, *next_seg, *cur_seg, *temp; 116 117 fd_addr = (uint64_t)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd)); 118 119 /* Get Scatter gather table address */ 120 sgt = (struct qbman_sge *)(fd_addr + DPAA2_GET_FD_OFFSET(fd)); 121 122 sge = &sgt[i++]; 123 sg_addr = (uint64_t)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FLE_ADDR(sge)); 124 125 /* First Scatter gather entry */ 126 first_seg = DPAA2_INLINE_MBUF_FROM_BUF(sg_addr, 127 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size); 128 /* Prepare all the metadata for first segment */ 129 first_seg->buf_addr = (uint8_t *)sg_addr; 130 first_seg->ol_flags = 0; 131 first_seg->data_off = DPAA2_GET_FLE_OFFSET(sge); 132 first_seg->data_len = sge->length & 0x1FFFF; 133 first_seg->pkt_len = DPAA2_GET_FD_LEN(fd); 134 first_seg->nb_segs = 1; 135 first_seg->next = NULL; 136 137 first_seg->packet_type = dpaa2_dev_rx_parse( 138 (uint64_t)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd)) 139 + DPAA2_FD_PTA_SIZE); 140 dpaa2_dev_rx_offload((uint64_t)DPAA2_IOVA_TO_VADDR( 141 DPAA2_GET_FD_ADDR(fd)) + 142 DPAA2_FD_PTA_SIZE, first_seg); 143 rte_mbuf_refcnt_set(first_seg, 1); 144 cur_seg = first_seg; 145 while (!DPAA2_SG_IS_FINAL(sge)) { 146 sge = &sgt[i++]; 147 sg_addr = (uint64_t)DPAA2_IOVA_TO_VADDR( 148 DPAA2_GET_FLE_ADDR(sge)); 149 next_seg = DPAA2_INLINE_MBUF_FROM_BUF(sg_addr, 150 rte_dpaa2_bpid_info[DPAA2_GET_FLE_BPID(sge)].meta_data_size); 151 next_seg->buf_addr = (uint8_t *)sg_addr; 152 next_seg->data_off = DPAA2_GET_FLE_OFFSET(sge); 153 next_seg->data_len = sge->length & 0x1FFFF; 154 first_seg->nb_segs += 1; 155 rte_mbuf_refcnt_set(next_seg, 1); 156 cur_seg->next = next_seg; 157 next_seg->next = NULL; 158 cur_seg = next_seg; 159 } 160 temp = DPAA2_INLINE_MBUF_FROM_BUF(fd_addr, 161 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size); 162 rte_mbuf_refcnt_set(temp, 1); 163 rte_pktmbuf_free_seg(temp); 164 165 return (void *)first_seg; 166 } 167 168 static inline struct rte_mbuf *__attribute__((hot)) 169 eth_fd_to_mbuf(const struct qbman_fd *fd) 170 { 171 struct rte_mbuf *mbuf = DPAA2_INLINE_MBUF_FROM_BUF( 172 DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd)), 173 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size); 174 175 /* need to repopulated some of the fields, 176 * as they may have changed in last transmission 177 */ 178 mbuf->nb_segs = 1; 179 mbuf->ol_flags = 0; 180 mbuf->data_off = DPAA2_GET_FD_OFFSET(fd); 181 mbuf->data_len = DPAA2_GET_FD_LEN(fd); 182 mbuf->pkt_len = mbuf->data_len; 183 184 /* Parse the packet */ 185 /* parse results are after the private - sw annotation area */ 186 mbuf->packet_type = dpaa2_dev_rx_parse( 187 (uint64_t)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd)) 188 + DPAA2_FD_PTA_SIZE); 189 190 dpaa2_dev_rx_offload((uint64_t)DPAA2_IOVA_TO_VADDR( 191 DPAA2_GET_FD_ADDR(fd)) + 192 DPAA2_FD_PTA_SIZE, mbuf); 193 194 mbuf->next = NULL; 195 rte_mbuf_refcnt_set(mbuf, 1); 196 197 PMD_RX_LOG(DEBUG, "to mbuf - mbuf =%p, mbuf->buf_addr =%p, off = %d," 198 "fd_off=%d fd =%lx, meta = %d bpid =%d, len=%d\n", 199 mbuf, mbuf->buf_addr, mbuf->data_off, 200 DPAA2_GET_FD_OFFSET(fd), DPAA2_GET_FD_ADDR(fd), 201 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size, 202 DPAA2_GET_FD_BPID(fd), DPAA2_GET_FD_LEN(fd)); 203 204 return mbuf; 205 } 206 207 static int __attribute__ ((noinline)) __attribute__((hot)) 208 eth_mbuf_to_sg_fd(struct rte_mbuf *mbuf, 209 struct qbman_fd *fd, uint16_t bpid) 210 { 211 struct rte_mbuf *cur_seg = mbuf, *prev_seg, *mi, *temp; 212 struct qbman_sge *sgt, *sge = NULL; 213 int i; 214 215 /* First Prepare FD to be transmited*/ 216 /* Resetting the buffer pool id and offset field*/ 217 fd->simple.bpid_offset = 0; 218 219 temp = rte_pktmbuf_alloc(mbuf->pool); 220 if (temp == NULL) { 221 PMD_TX_LOG(ERR, "No memory to allocate S/G table"); 222 return -ENOMEM; 223 } 224 225 DPAA2_SET_FD_ADDR(fd, DPAA2_MBUF_VADDR_TO_IOVA(temp)); 226 DPAA2_SET_FD_LEN(fd, mbuf->pkt_len); 227 DPAA2_SET_FD_OFFSET(fd, temp->data_off); 228 DPAA2_SET_FD_BPID(fd, bpid); 229 DPAA2_SET_FD_ASAL(fd, DPAA2_ASAL_VAL); 230 DPAA2_FD_SET_FORMAT(fd, qbman_fd_sg); 231 /*Set Scatter gather table and Scatter gather entries*/ 232 sgt = (struct qbman_sge *)( 233 (uint64_t)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd)) 234 + DPAA2_GET_FD_OFFSET(fd)); 235 236 for (i = 0; i < mbuf->nb_segs; i++) { 237 sge = &sgt[i]; 238 /*Resetting the buffer pool id and offset field*/ 239 sge->fin_bpid_offset = 0; 240 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(cur_seg)); 241 DPAA2_SET_FLE_OFFSET(sge, cur_seg->data_off); 242 sge->length = cur_seg->data_len; 243 if (RTE_MBUF_DIRECT(cur_seg)) { 244 if (rte_mbuf_refcnt_read(cur_seg) > 1) { 245 /* If refcnt > 1, invalid bpid is set to ensure 246 * buffer is not freed by HW 247 */ 248 DPAA2_SET_FLE_IVP(sge); 249 rte_mbuf_refcnt_update(cur_seg, -1); 250 } else 251 DPAA2_SET_FLE_BPID(sge, 252 mempool_to_bpid(cur_seg->pool)); 253 cur_seg = cur_seg->next; 254 } else { 255 /* Get owner MBUF from indirect buffer */ 256 mi = rte_mbuf_from_indirect(cur_seg); 257 if (rte_mbuf_refcnt_read(mi) > 1) { 258 /* If refcnt > 1, invalid bpid is set to ensure 259 * owner buffer is not freed by HW 260 */ 261 DPAA2_SET_FLE_IVP(sge); 262 } else { 263 DPAA2_SET_FLE_BPID(sge, 264 mempool_to_bpid(mi->pool)); 265 rte_mbuf_refcnt_update(mi, 1); 266 } 267 prev_seg = cur_seg; 268 cur_seg = cur_seg->next; 269 prev_seg->next = NULL; 270 rte_pktmbuf_free(prev_seg); 271 } 272 } 273 DPAA2_SG_SET_FINAL(sge, true); 274 return 0; 275 } 276 277 static void 278 eth_mbuf_to_fd(struct rte_mbuf *mbuf, 279 struct qbman_fd *fd, uint16_t bpid) __attribute__((unused)); 280 281 static void __attribute__ ((noinline)) __attribute__((hot)) 282 eth_mbuf_to_fd(struct rte_mbuf *mbuf, 283 struct qbman_fd *fd, uint16_t bpid) 284 { 285 /*Resetting the buffer pool id and offset field*/ 286 fd->simple.bpid_offset = 0; 287 288 DPAA2_SET_FD_ADDR(fd, DPAA2_MBUF_VADDR_TO_IOVA(mbuf)); 289 DPAA2_SET_FD_LEN(fd, mbuf->data_len); 290 DPAA2_SET_FD_BPID(fd, bpid); 291 DPAA2_SET_FD_OFFSET(fd, mbuf->data_off); 292 DPAA2_SET_FD_ASAL(fd, DPAA2_ASAL_VAL); 293 294 PMD_TX_LOG(DEBUG, "mbuf =%p, mbuf->buf_addr =%p, off = %d," 295 "fd_off=%d fd =%lx, meta = %d bpid =%d, len=%d\n", 296 mbuf, mbuf->buf_addr, mbuf->data_off, 297 DPAA2_GET_FD_OFFSET(fd), DPAA2_GET_FD_ADDR(fd), 298 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size, 299 DPAA2_GET_FD_BPID(fd), DPAA2_GET_FD_LEN(fd)); 300 if (RTE_MBUF_DIRECT(mbuf)) { 301 if (rte_mbuf_refcnt_read(mbuf) > 1) { 302 DPAA2_SET_FD_IVP(fd); 303 rte_mbuf_refcnt_update(mbuf, -1); 304 } 305 } else { 306 struct rte_mbuf *mi; 307 308 mi = rte_mbuf_from_indirect(mbuf); 309 if (rte_mbuf_refcnt_read(mi) > 1) 310 DPAA2_SET_FD_IVP(fd); 311 else 312 rte_mbuf_refcnt_update(mi, 1); 313 rte_pktmbuf_free(mbuf); 314 } 315 } 316 317 static inline int __attribute__((hot)) 318 eth_copy_mbuf_to_fd(struct rte_mbuf *mbuf, 319 struct qbman_fd *fd, uint16_t bpid) 320 { 321 struct rte_mbuf *m; 322 void *mb = NULL; 323 324 if (rte_dpaa2_mbuf_alloc_bulk( 325 rte_dpaa2_bpid_info[bpid].bp_list->mp, &mb, 1)) { 326 PMD_TX_LOG(WARNING, "Unable to allocated DPAA2 buffer"); 327 return -1; 328 } 329 m = (struct rte_mbuf *)mb; 330 memcpy((char *)m->buf_addr + mbuf->data_off, 331 (void *)((char *)mbuf->buf_addr + mbuf->data_off), 332 mbuf->pkt_len); 333 334 /* Copy required fields */ 335 m->data_off = mbuf->data_off; 336 m->ol_flags = mbuf->ol_flags; 337 m->packet_type = mbuf->packet_type; 338 m->tx_offload = mbuf->tx_offload; 339 340 /*Resetting the buffer pool id and offset field*/ 341 fd->simple.bpid_offset = 0; 342 343 DPAA2_SET_FD_ADDR(fd, DPAA2_MBUF_VADDR_TO_IOVA(m)); 344 DPAA2_SET_FD_LEN(fd, mbuf->data_len); 345 DPAA2_SET_FD_BPID(fd, bpid); 346 DPAA2_SET_FD_OFFSET(fd, mbuf->data_off); 347 DPAA2_SET_FD_ASAL(fd, DPAA2_ASAL_VAL); 348 349 PMD_TX_LOG(DEBUG, " mbuf %p BMAN buf addr %p", 350 (void *)mbuf, mbuf->buf_addr); 351 352 PMD_TX_LOG(DEBUG, " fdaddr =%lx bpid =%d meta =%d off =%d, len =%d", 353 DPAA2_GET_FD_ADDR(fd), 354 DPAA2_GET_FD_BPID(fd), 355 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size, 356 DPAA2_GET_FD_OFFSET(fd), 357 DPAA2_GET_FD_LEN(fd)); 358 359 return 0; 360 } 361 362 uint16_t 363 dpaa2_dev_prefetch_rx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) 364 { 365 /* Function receive frames for a given device and VQ*/ 366 struct dpaa2_queue *dpaa2_q = (struct dpaa2_queue *)queue; 367 struct qbman_result *dq_storage; 368 uint32_t fqid = dpaa2_q->fqid; 369 int ret, num_rx = 0; 370 uint8_t is_last = 0, status; 371 struct qbman_swp *swp; 372 const struct qbman_fd *fd[DPAA2_DQRR_RING_SIZE]; 373 struct qbman_pull_desc pulldesc; 374 struct queue_storage_info_t *q_storage = dpaa2_q->q_storage; 375 struct rte_eth_dev *dev = dpaa2_q->dev; 376 377 if (unlikely(!DPAA2_PER_LCORE_DPIO)) { 378 ret = dpaa2_affine_qbman_swp(); 379 if (ret) { 380 RTE_LOG(ERR, PMD, "Failure in affining portal\n"); 381 return 0; 382 } 383 } 384 swp = DPAA2_PER_LCORE_PORTAL; 385 if (!q_storage->active_dqs) { 386 q_storage->toggle = 0; 387 dq_storage = q_storage->dq_storage[q_storage->toggle]; 388 qbman_pull_desc_clear(&pulldesc); 389 qbman_pull_desc_set_numframes(&pulldesc, 390 (nb_pkts > DPAA2_DQRR_RING_SIZE) ? 391 DPAA2_DQRR_RING_SIZE : nb_pkts); 392 qbman_pull_desc_set_fq(&pulldesc, fqid); 393 qbman_pull_desc_set_storage(&pulldesc, dq_storage, 394 (dma_addr_t)(DPAA2_VADDR_TO_IOVA(dq_storage)), 1); 395 if (check_swp_active_dqs(DPAA2_PER_LCORE_DPIO->index)) { 396 while (!qbman_check_command_complete( 397 get_swp_active_dqs(DPAA2_PER_LCORE_DPIO->index))) 398 ; 399 clear_swp_active_dqs(DPAA2_PER_LCORE_DPIO->index); 400 } 401 while (1) { 402 if (qbman_swp_pull(swp, &pulldesc)) { 403 PMD_RX_LOG(WARNING, "VDQ command is not issued." 404 "QBMAN is busy\n"); 405 /* Portal was busy, try again */ 406 continue; 407 } 408 break; 409 } 410 q_storage->active_dqs = dq_storage; 411 q_storage->active_dpio_id = DPAA2_PER_LCORE_DPIO->index; 412 set_swp_active_dqs(DPAA2_PER_LCORE_DPIO->index, dq_storage); 413 } 414 dq_storage = q_storage->active_dqs; 415 /* Check if the previous issued command is completed. 416 * Also seems like the SWP is shared between the Ethernet Driver 417 * and the SEC driver. 418 */ 419 while (!qbman_check_command_complete(dq_storage)) 420 ; 421 if (dq_storage == get_swp_active_dqs(q_storage->active_dpio_id)) 422 clear_swp_active_dqs(q_storage->active_dpio_id); 423 while (!is_last) { 424 /* Loop until the dq_storage is updated with 425 * new token by QBMAN 426 */ 427 while (!qbman_check_new_result(dq_storage)) 428 ; 429 rte_prefetch0((void *)((uint64_t)(dq_storage + 1))); 430 /* Check whether Last Pull command is Expired and 431 * setting Condition for Loop termination 432 */ 433 if (qbman_result_DQ_is_pull_complete(dq_storage)) { 434 is_last = 1; 435 /* Check for valid frame. */ 436 status = (uint8_t)qbman_result_DQ_flags(dq_storage); 437 if (unlikely((status & QBMAN_DQ_STAT_VALIDFRAME) == 0)) 438 continue; 439 } 440 fd[num_rx] = qbman_result_DQ_fd(dq_storage); 441 442 /* Prefetch Annotation address for the parse results */ 443 rte_prefetch0((void *)((uint64_t)DPAA2_GET_FD_ADDR(fd[num_rx]) 444 + DPAA2_FD_PTA_SIZE + 16)); 445 446 if (unlikely(DPAA2_FD_GET_FORMAT(fd[num_rx]) == qbman_fd_sg)) 447 bufs[num_rx] = eth_sg_fd_to_mbuf(fd[num_rx]); 448 else 449 bufs[num_rx] = eth_fd_to_mbuf(fd[num_rx]); 450 bufs[num_rx]->port = dev->data->port_id; 451 452 if (dev->data->dev_conf.rxmode.hw_vlan_strip) 453 rte_vlan_strip(bufs[num_rx]); 454 455 dq_storage++; 456 num_rx++; 457 } 458 459 if (check_swp_active_dqs(DPAA2_PER_LCORE_DPIO->index)) { 460 while (!qbman_check_command_complete( 461 get_swp_active_dqs(DPAA2_PER_LCORE_DPIO->index))) 462 ; 463 clear_swp_active_dqs(DPAA2_PER_LCORE_DPIO->index); 464 } 465 q_storage->toggle ^= 1; 466 dq_storage = q_storage->dq_storage[q_storage->toggle]; 467 qbman_pull_desc_clear(&pulldesc); 468 qbman_pull_desc_set_numframes(&pulldesc, DPAA2_DQRR_RING_SIZE); 469 qbman_pull_desc_set_fq(&pulldesc, fqid); 470 qbman_pull_desc_set_storage(&pulldesc, dq_storage, 471 (dma_addr_t)(DPAA2_VADDR_TO_IOVA(dq_storage)), 1); 472 /* Issue a volatile dequeue command. */ 473 while (1) { 474 if (qbman_swp_pull(swp, &pulldesc)) { 475 PMD_RX_LOG(WARNING, "VDQ command is not issued." 476 "QBMAN is busy\n"); 477 continue; 478 } 479 break; 480 } 481 q_storage->active_dqs = dq_storage; 482 q_storage->active_dpio_id = DPAA2_PER_LCORE_DPIO->index; 483 set_swp_active_dqs(DPAA2_PER_LCORE_DPIO->index, dq_storage); 484 485 dpaa2_q->rx_pkts += num_rx; 486 487 /* Return the total number of packets received to DPAA2 app */ 488 return num_rx; 489 } 490 491 void __attribute__((hot)) 492 dpaa2_dev_process_parallel_event(struct qbman_swp *swp, 493 const struct qbman_fd *fd, 494 const struct qbman_result *dq, 495 struct dpaa2_queue *rxq, 496 struct rte_event *ev) 497 { 498 ev->mbuf = eth_fd_to_mbuf(fd); 499 500 ev->flow_id = rxq->ev.flow_id; 501 ev->sub_event_type = rxq->ev.sub_event_type; 502 ev->event_type = RTE_EVENT_TYPE_ETHDEV; 503 ev->op = RTE_EVENT_OP_NEW; 504 ev->sched_type = rxq->ev.sched_type; 505 ev->queue_id = rxq->ev.queue_id; 506 ev->priority = rxq->ev.priority; 507 508 qbman_swp_dqrr_consume(swp, dq); 509 } 510 511 /* 512 * Callback to handle sending packets through WRIOP based interface 513 */ 514 uint16_t 515 dpaa2_dev_tx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) 516 { 517 /* Function to transmit the frames to given device and VQ*/ 518 uint32_t loop, retry_count; 519 int32_t ret; 520 struct qbman_fd fd_arr[MAX_TX_RING_SLOTS]; 521 struct rte_mbuf *mi; 522 uint32_t frames_to_send; 523 struct rte_mempool *mp; 524 struct qbman_eq_desc eqdesc; 525 struct dpaa2_queue *dpaa2_q = (struct dpaa2_queue *)queue; 526 struct qbman_swp *swp; 527 uint16_t num_tx = 0; 528 uint16_t bpid; 529 struct rte_eth_dev *dev = dpaa2_q->dev; 530 struct dpaa2_dev_priv *priv = dev->data->dev_private; 531 532 if (unlikely(!DPAA2_PER_LCORE_DPIO)) { 533 ret = dpaa2_affine_qbman_swp(); 534 if (ret) { 535 RTE_LOG(ERR, PMD, "Failure in affining portal\n"); 536 return 0; 537 } 538 } 539 swp = DPAA2_PER_LCORE_PORTAL; 540 541 PMD_TX_LOG(DEBUG, "===> dev =%p, fqid =%d", dev, dpaa2_q->fqid); 542 543 /*Prepare enqueue descriptor*/ 544 qbman_eq_desc_clear(&eqdesc); 545 qbman_eq_desc_set_no_orp(&eqdesc, DPAA2_EQ_RESP_ERR_FQ); 546 qbman_eq_desc_set_response(&eqdesc, 0, 0); 547 qbman_eq_desc_set_qd(&eqdesc, priv->qdid, 548 dpaa2_q->flow_id, dpaa2_q->tc_index); 549 550 /*Clear the unused FD fields before sending*/ 551 while (nb_pkts) { 552 /*Check if the queue is congested*/ 553 retry_count = 0; 554 while (qbman_result_SCN_state(dpaa2_q->cscn)) { 555 retry_count++; 556 /* Retry for some time before giving up */ 557 if (retry_count > CONG_RETRY_COUNT) 558 goto skip_tx; 559 } 560 561 frames_to_send = (nb_pkts >> 3) ? MAX_TX_RING_SLOTS : nb_pkts; 562 563 for (loop = 0; loop < frames_to_send; loop++) { 564 fd_arr[loop].simple.frc = 0; 565 DPAA2_RESET_FD_CTRL((&fd_arr[loop])); 566 DPAA2_SET_FD_FLC((&fd_arr[loop]), NULL); 567 if (RTE_MBUF_DIRECT(*bufs)) { 568 mp = (*bufs)->pool; 569 } else { 570 mi = rte_mbuf_from_indirect(*bufs); 571 mp = mi->pool; 572 } 573 /* Not a hw_pkt pool allocated frame */ 574 if (unlikely(!mp || !priv->bp_list)) { 575 PMD_TX_LOG(ERR, "err: no bpool attached"); 576 goto send_n_return; 577 } 578 579 if (mp->ops_index != priv->bp_list->dpaa2_ops_index) { 580 PMD_TX_LOG(ERR, "non hw offload bufffer "); 581 /* alloc should be from the default buffer pool 582 * attached to this interface 583 */ 584 bpid = priv->bp_list->buf_pool.bpid; 585 586 if (unlikely((*bufs)->nb_segs > 1)) { 587 PMD_TX_LOG(ERR, "S/G support not added" 588 " for non hw offload buffer"); 589 goto send_n_return; 590 } 591 if (eth_copy_mbuf_to_fd(*bufs, 592 &fd_arr[loop], bpid)) { 593 goto send_n_return; 594 } 595 /* free the original packet */ 596 rte_pktmbuf_free(*bufs); 597 } else { 598 bpid = mempool_to_bpid(mp); 599 if (unlikely((*bufs)->nb_segs > 1)) { 600 if (eth_mbuf_to_sg_fd(*bufs, 601 &fd_arr[loop], bpid)) 602 goto send_n_return; 603 } else { 604 eth_mbuf_to_fd(*bufs, 605 &fd_arr[loop], bpid); 606 } 607 } 608 bufs++; 609 } 610 loop = 0; 611 while (loop < frames_to_send) { 612 loop += qbman_swp_enqueue_multiple(swp, &eqdesc, 613 &fd_arr[loop], frames_to_send - loop); 614 } 615 616 num_tx += frames_to_send; 617 dpaa2_q->tx_pkts += frames_to_send; 618 nb_pkts -= frames_to_send; 619 } 620 return num_tx; 621 622 send_n_return: 623 /* send any already prepared fd */ 624 if (loop) { 625 unsigned int i = 0; 626 627 while (i < loop) { 628 i += qbman_swp_enqueue_multiple(swp, &eqdesc, 629 &fd_arr[i], loop - i); 630 } 631 num_tx += loop; 632 dpaa2_q->tx_pkts += loop; 633 } 634 skip_tx: 635 return num_tx; 636 } 637 638 /** 639 * Dummy DPDK callback for TX. 640 * 641 * This function is used to temporarily replace the real callback during 642 * unsafe control operations on the queue, or in case of error. 643 * 644 * @param dpdk_txq 645 * Generic pointer to TX queue structure. 646 * @param[in] pkts 647 * Packets to transmit. 648 * @param pkts_n 649 * Number of packets in array. 650 * 651 * @return 652 * Number of packets successfully transmitted (<= pkts_n). 653 */ 654 uint16_t 655 dummy_dev_tx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) 656 { 657 (void)queue; 658 (void)bufs; 659 (void)nb_pkts; 660 return 0; 661 } 662