1 /* SPDX-License-Identifier: BSD-3-Clause 2 * 3 * Copyright 2016 Freescale Semiconductor, Inc. All rights reserved. 4 * Copyright 2017,2019-2021 NXP 5 * 6 */ 7 8 /* System headers */ 9 #include <inttypes.h> 10 #include <unistd.h> 11 #include <stdio.h> 12 #include <limits.h> 13 #include <sched.h> 14 #include <pthread.h> 15 16 #include <rte_byteorder.h> 17 #include <rte_common.h> 18 #include <rte_interrupts.h> 19 #include <rte_log.h> 20 #include <rte_debug.h> 21 #include <rte_pci.h> 22 #include <rte_atomic.h> 23 #include <rte_branch_prediction.h> 24 #include <rte_memory.h> 25 #include <rte_tailq.h> 26 #include <rte_eal.h> 27 #include <rte_alarm.h> 28 #include <rte_ether.h> 29 #include <ethdev_driver.h> 30 #include <rte_malloc.h> 31 #include <rte_ring.h> 32 #include <rte_ip.h> 33 #include <rte_tcp.h> 34 #include <rte_udp.h> 35 #include <rte_net.h> 36 #include <rte_eventdev.h> 37 38 #include "dpaa_ethdev.h" 39 #include "dpaa_rxtx.h" 40 #include <bus_dpaa_driver.h> 41 #include <dpaa_mempool.h> 42 43 #include <qman.h> 44 #include <fsl_usd.h> 45 #include <fsl_qman.h> 46 #include <fsl_bman.h> 47 #include <dpaa_of.h> 48 #include <netcfg.h> 49 50 #define DPAA_MBUF_TO_CONTIG_FD(_mbuf, _fd, _bpid) \ 51 do { \ 52 (_fd)->cmd = 0; \ 53 (_fd)->opaque_addr = 0; \ 54 (_fd)->opaque = QM_FD_CONTIG << DPAA_FD_FORMAT_SHIFT; \ 55 (_fd)->opaque |= ((_mbuf)->data_off) << DPAA_FD_OFFSET_SHIFT; \ 56 (_fd)->opaque |= (_mbuf)->pkt_len; \ 57 (_fd)->addr = (_mbuf)->buf_iova; \ 58 (_fd)->bpid = _bpid; \ 59 } while (0) 60 61 #ifdef RTE_LIBRTE_DPAA_DEBUG_DRIVER 62 #define DISPLAY_PRINT printf 63 static void dpaa_display_frame_info(const struct qm_fd *fd, 64 uint32_t fqid, bool rx) 65 { 66 int ii; 67 char *ptr; 68 struct annotations_t *annot = rte_dpaa_mem_ptov(fd->addr); 69 uint8_t format; 70 71 if (!fd->status) { 72 /* Do not display correct packets.*/ 73 return; 74 } 75 76 format = (fd->opaque & DPAA_FD_FORMAT_MASK) >> 77 DPAA_FD_FORMAT_SHIFT; 78 79 DISPLAY_PRINT("fqid %d bpid %d addr 0x%lx, format %d\r\n", 80 fqid, fd->bpid, (unsigned long)fd->addr, fd->format); 81 DISPLAY_PRINT("off %d, len %d stat 0x%x\r\n", 82 fd->offset, fd->length20, fd->status); 83 if (rx) { 84 ptr = (char *)&annot->parse; 85 DISPLAY_PRINT("RX parser result:\r\n"); 86 for (ii = 0; ii < (int)sizeof(struct dpaa_eth_parse_results_t); 87 ii++) { 88 DISPLAY_PRINT("%02x ", ptr[ii]); 89 if (((ii + 1) % 16) == 0) 90 DISPLAY_PRINT("\n"); 91 } 92 DISPLAY_PRINT("\n"); 93 } 94 95 if (unlikely(format == qm_fd_sg)) { 96 /*TBD:S/G display: to be implemented*/ 97 return; 98 } 99 100 DISPLAY_PRINT("Frame payload:\r\n"); 101 ptr = (char *)annot; 102 ptr += fd->offset; 103 for (ii = 0; ii < fd->length20; ii++) { 104 DISPLAY_PRINT("%02x ", ptr[ii]); 105 if (((ii + 1) % 16) == 0) 106 DISPLAY_PRINT("\n"); 107 } 108 DISPLAY_PRINT("\n"); 109 } 110 #else 111 #define dpaa_display_frame_info(a, b, c) 112 #endif 113 114 static inline void dpaa_slow_parsing(struct rte_mbuf *m __rte_unused, 115 uint64_t prs __rte_unused) 116 { 117 DPAA_DP_LOG(DEBUG, "Slow parsing"); 118 /*TBD:XXX: to be implemented*/ 119 } 120 121 static inline void dpaa_eth_packet_info(struct rte_mbuf *m, void *fd_virt_addr) 122 { 123 struct annotations_t *annot = GET_ANNOTATIONS(fd_virt_addr); 124 uint64_t prs = *((uintptr_t *)(&annot->parse)) & DPAA_PARSE_MASK; 125 126 DPAA_DP_LOG(DEBUG, " Parsing mbuf: %p with annotations: %p", m, annot); 127 128 m->ol_flags = RTE_MBUF_F_RX_RSS_HASH | RTE_MBUF_F_RX_IP_CKSUM_GOOD | 129 RTE_MBUF_F_RX_L4_CKSUM_GOOD; 130 131 switch (prs) { 132 case DPAA_PKT_TYPE_IPV4: 133 m->packet_type = RTE_PTYPE_L2_ETHER | 134 RTE_PTYPE_L3_IPV4; 135 break; 136 case DPAA_PKT_TYPE_IPV6: 137 m->packet_type = RTE_PTYPE_L2_ETHER | 138 RTE_PTYPE_L3_IPV6; 139 break; 140 case DPAA_PKT_TYPE_ETHER: 141 m->packet_type = RTE_PTYPE_L2_ETHER; 142 break; 143 case DPAA_PKT_TYPE_IPV4_FRAG: 144 case DPAA_PKT_TYPE_IPV4_FRAG_UDP: 145 case DPAA_PKT_TYPE_IPV4_FRAG_TCP: 146 case DPAA_PKT_TYPE_IPV4_FRAG_SCTP: 147 m->packet_type = RTE_PTYPE_L2_ETHER | 148 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_FRAG; 149 break; 150 case DPAA_PKT_TYPE_IPV6_FRAG: 151 case DPAA_PKT_TYPE_IPV6_FRAG_UDP: 152 case DPAA_PKT_TYPE_IPV6_FRAG_TCP: 153 case DPAA_PKT_TYPE_IPV6_FRAG_SCTP: 154 m->packet_type = RTE_PTYPE_L2_ETHER | 155 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_FRAG; 156 break; 157 case DPAA_PKT_TYPE_IPV4_EXT: 158 m->packet_type = RTE_PTYPE_L2_ETHER | 159 RTE_PTYPE_L3_IPV4_EXT; 160 break; 161 case DPAA_PKT_TYPE_IPV6_EXT: 162 m->packet_type = RTE_PTYPE_L2_ETHER | 163 RTE_PTYPE_L3_IPV6_EXT; 164 break; 165 case DPAA_PKT_TYPE_IPV4_TCP: 166 m->packet_type = RTE_PTYPE_L2_ETHER | 167 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_TCP; 168 break; 169 case DPAA_PKT_TYPE_IPV6_TCP: 170 m->packet_type = RTE_PTYPE_L2_ETHER | 171 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_TCP; 172 break; 173 case DPAA_PKT_TYPE_IPV4_UDP: 174 m->packet_type = RTE_PTYPE_L2_ETHER | 175 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_UDP; 176 break; 177 case DPAA_PKT_TYPE_IPV6_UDP: 178 m->packet_type = RTE_PTYPE_L2_ETHER | 179 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_UDP; 180 break; 181 case DPAA_PKT_TYPE_IPSEC_IPV4: 182 if (*((uintptr_t *)&annot->parse) & DPAA_PARSE_ESP_MASK) 183 m->packet_type = RTE_PTYPE_L2_ETHER | 184 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_ESP; 185 break; 186 case DPAA_PKT_TYPE_IPSEC_IPV6: 187 if (*((uintptr_t *)&annot->parse) & DPAA_PARSE_ESP_MASK) 188 m->packet_type = RTE_PTYPE_L2_ETHER | 189 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_TUNNEL_ESP; 190 break; 191 case DPAA_PKT_TYPE_IPV4_EXT_UDP: 192 m->packet_type = RTE_PTYPE_L2_ETHER | 193 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_UDP; 194 break; 195 case DPAA_PKT_TYPE_IPV6_EXT_UDP: 196 m->packet_type = RTE_PTYPE_L2_ETHER | 197 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_UDP; 198 break; 199 case DPAA_PKT_TYPE_IPV4_EXT_TCP: 200 m->packet_type = RTE_PTYPE_L2_ETHER | 201 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_TCP; 202 break; 203 case DPAA_PKT_TYPE_IPV6_EXT_TCP: 204 m->packet_type = RTE_PTYPE_L2_ETHER | 205 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_TCP; 206 break; 207 case DPAA_PKT_TYPE_IPV4_SCTP: 208 m->packet_type = RTE_PTYPE_L2_ETHER | 209 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_SCTP; 210 break; 211 case DPAA_PKT_TYPE_IPV6_SCTP: 212 m->packet_type = RTE_PTYPE_L2_ETHER | 213 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_SCTP; 214 break; 215 case DPAA_PKT_TYPE_IPV4_CSUM_ERR: 216 case DPAA_PKT_TYPE_IPV6_CSUM_ERR: 217 m->ol_flags = RTE_MBUF_F_RX_RSS_HASH | RTE_MBUF_F_RX_IP_CKSUM_BAD; 218 break; 219 case DPAA_PKT_TYPE_IPV4_TCP_CSUM_ERR: 220 case DPAA_PKT_TYPE_IPV6_TCP_CSUM_ERR: 221 case DPAA_PKT_TYPE_IPV4_UDP_CSUM_ERR: 222 case DPAA_PKT_TYPE_IPV6_UDP_CSUM_ERR: 223 m->ol_flags = RTE_MBUF_F_RX_RSS_HASH | RTE_MBUF_F_RX_L4_CKSUM_BAD; 224 break; 225 case DPAA_PKT_TYPE_NONE: 226 m->packet_type = 0; 227 break; 228 /* More switch cases can be added */ 229 default: 230 dpaa_slow_parsing(m, prs); 231 } 232 233 m->tx_offload = annot->parse.ip_off[0]; 234 m->tx_offload |= (annot->parse.l4_off - annot->parse.ip_off[0]) 235 << DPAA_PKT_L3_LEN_SHIFT; 236 237 /* Set the hash values */ 238 m->hash.rss = (uint32_t)(annot->hash); 239 240 /* Check if Vlan is present */ 241 if (prs & DPAA_PARSE_VLAN_MASK) 242 m->ol_flags |= RTE_MBUF_F_RX_VLAN; 243 /* Packet received without stripping the vlan */ 244 } 245 246 static inline void dpaa_checksum(struct rte_mbuf *mbuf) 247 { 248 struct rte_ether_hdr *eth_hdr = 249 rte_pktmbuf_mtod(mbuf, struct rte_ether_hdr *); 250 char *l3_hdr = (char *)eth_hdr + mbuf->l2_len; 251 struct rte_ipv4_hdr *ipv4_hdr = (struct rte_ipv4_hdr *)l3_hdr; 252 struct rte_ipv6_hdr *ipv6_hdr = (struct rte_ipv6_hdr *)l3_hdr; 253 254 DPAA_DP_LOG(DEBUG, "Calculating checksum for mbuf: %p", mbuf); 255 256 if (((mbuf->packet_type & RTE_PTYPE_L3_MASK) == RTE_PTYPE_L3_IPV4) || 257 ((mbuf->packet_type & RTE_PTYPE_L3_MASK) == 258 RTE_PTYPE_L3_IPV4_EXT)) { 259 ipv4_hdr = (struct rte_ipv4_hdr *)l3_hdr; 260 ipv4_hdr->hdr_checksum = 0; 261 ipv4_hdr->hdr_checksum = rte_ipv4_cksum(ipv4_hdr); 262 } else if (((mbuf->packet_type & RTE_PTYPE_L3_MASK) == 263 RTE_PTYPE_L3_IPV6) || 264 ((mbuf->packet_type & RTE_PTYPE_L3_MASK) == 265 RTE_PTYPE_L3_IPV6_EXT)) 266 ipv6_hdr = (struct rte_ipv6_hdr *)l3_hdr; 267 268 if ((mbuf->packet_type & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_TCP) { 269 struct rte_tcp_hdr *tcp_hdr = (struct rte_tcp_hdr *)(l3_hdr + 270 mbuf->l3_len); 271 tcp_hdr->cksum = 0; 272 if (eth_hdr->ether_type == htons(RTE_ETHER_TYPE_IPV4)) 273 tcp_hdr->cksum = rte_ipv4_udptcp_cksum(ipv4_hdr, 274 tcp_hdr); 275 else /* assume ethertype == RTE_ETHER_TYPE_IPV6 */ 276 tcp_hdr->cksum = rte_ipv6_udptcp_cksum(ipv6_hdr, 277 tcp_hdr); 278 } else if ((mbuf->packet_type & RTE_PTYPE_L4_MASK) == 279 RTE_PTYPE_L4_UDP) { 280 struct rte_udp_hdr *udp_hdr = (struct rte_udp_hdr *)(l3_hdr + 281 mbuf->l3_len); 282 udp_hdr->dgram_cksum = 0; 283 if (eth_hdr->ether_type == htons(RTE_ETHER_TYPE_IPV4)) 284 udp_hdr->dgram_cksum = rte_ipv4_udptcp_cksum(ipv4_hdr, 285 udp_hdr); 286 else /* assume ethertype == RTE_ETHER_TYPE_IPV6 */ 287 udp_hdr->dgram_cksum = rte_ipv6_udptcp_cksum(ipv6_hdr, 288 udp_hdr); 289 } 290 } 291 292 static inline void dpaa_checksum_offload(struct rte_mbuf *mbuf, 293 struct qm_fd *fd, char *prs_buf) 294 { 295 struct dpaa_eth_parse_results_t *prs; 296 297 DPAA_DP_LOG(DEBUG, " Offloading checksum for mbuf: %p", mbuf); 298 299 prs = GET_TX_PRS(prs_buf); 300 prs->l3r = 0; 301 prs->l4r = 0; 302 if (((mbuf->packet_type & RTE_PTYPE_L3_MASK) == RTE_PTYPE_L3_IPV4) || 303 ((mbuf->packet_type & RTE_PTYPE_L3_MASK) == 304 RTE_PTYPE_L3_IPV4_EXT)) 305 prs->l3r = DPAA_L3_PARSE_RESULT_IPV4; 306 else if (((mbuf->packet_type & RTE_PTYPE_L3_MASK) == 307 RTE_PTYPE_L3_IPV6) || 308 ((mbuf->packet_type & RTE_PTYPE_L3_MASK) == 309 RTE_PTYPE_L3_IPV6_EXT)) 310 prs->l3r = DPAA_L3_PARSE_RESULT_IPV6; 311 312 if ((mbuf->packet_type & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_TCP) 313 prs->l4r = DPAA_L4_PARSE_RESULT_TCP; 314 else if ((mbuf->packet_type & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_UDP) 315 prs->l4r = DPAA_L4_PARSE_RESULT_UDP; 316 317 prs->ip_off[0] = mbuf->l2_len; 318 prs->l4_off = mbuf->l3_len + mbuf->l2_len; 319 /* Enable L3 (and L4, if TCP or UDP) HW checksum*/ 320 fd->cmd = DPAA_FD_CMD_RPD | DPAA_FD_CMD_DTC; 321 } 322 323 static inline void 324 dpaa_unsegmented_checksum(struct rte_mbuf *mbuf, struct qm_fd *fd_arr) 325 { 326 if (!mbuf->packet_type) { 327 struct rte_net_hdr_lens hdr_lens; 328 329 mbuf->packet_type = rte_net_get_ptype(mbuf, &hdr_lens, 330 RTE_PTYPE_L2_MASK | RTE_PTYPE_L3_MASK 331 | RTE_PTYPE_L4_MASK); 332 mbuf->l2_len = hdr_lens.l2_len; 333 mbuf->l3_len = hdr_lens.l3_len; 334 } 335 if (mbuf->data_off < (DEFAULT_TX_ICEOF + 336 sizeof(struct dpaa_eth_parse_results_t))) { 337 DPAA_DP_LOG(DEBUG, "Checksum offload Err: " 338 "Not enough Headroom " 339 "space for correct Checksum offload." 340 "So Calculating checksum in Software."); 341 dpaa_checksum(mbuf); 342 } else { 343 dpaa_checksum_offload(mbuf, fd_arr, mbuf->buf_addr); 344 } 345 } 346 347 static struct rte_mbuf * 348 dpaa_eth_sg_to_mbuf(const struct qm_fd *fd, uint32_t ifid) 349 { 350 struct dpaa_bp_info *bp_info = DPAA_BPID_TO_POOL_INFO(fd->bpid); 351 struct rte_mbuf *first_seg, *prev_seg, *cur_seg, *temp; 352 struct qm_sg_entry *sgt, *sg_temp; 353 void *vaddr, *sg_vaddr; 354 int i = 0; 355 uint16_t fd_offset = fd->offset; 356 357 vaddr = DPAA_MEMPOOL_PTOV(bp_info, qm_fd_addr(fd)); 358 if (!vaddr) { 359 DPAA_PMD_ERR("unable to convert physical address"); 360 return NULL; 361 } 362 sgt = vaddr + fd_offset; 363 sg_temp = &sgt[i++]; 364 hw_sg_to_cpu(sg_temp); 365 temp = (struct rte_mbuf *)((char *)vaddr - bp_info->meta_data_size); 366 sg_vaddr = DPAA_MEMPOOL_PTOV(bp_info, qm_sg_entry_get64(sg_temp)); 367 368 first_seg = (struct rte_mbuf *)((char *)sg_vaddr - 369 bp_info->meta_data_size); 370 first_seg->data_off = sg_temp->offset; 371 first_seg->data_len = sg_temp->length; 372 first_seg->pkt_len = sg_temp->length; 373 rte_mbuf_refcnt_set(first_seg, 1); 374 375 first_seg->port = ifid; 376 first_seg->nb_segs = 1; 377 first_seg->ol_flags = 0; 378 prev_seg = first_seg; 379 while (i < DPAA_SGT_MAX_ENTRIES) { 380 sg_temp = &sgt[i++]; 381 hw_sg_to_cpu(sg_temp); 382 sg_vaddr = DPAA_MEMPOOL_PTOV(bp_info, 383 qm_sg_entry_get64(sg_temp)); 384 cur_seg = (struct rte_mbuf *)((char *)sg_vaddr - 385 bp_info->meta_data_size); 386 cur_seg->data_off = sg_temp->offset; 387 cur_seg->data_len = sg_temp->length; 388 first_seg->pkt_len += sg_temp->length; 389 first_seg->nb_segs += 1; 390 rte_mbuf_refcnt_set(cur_seg, 1); 391 prev_seg->next = cur_seg; 392 if (sg_temp->final) { 393 cur_seg->next = NULL; 394 break; 395 } 396 prev_seg = cur_seg; 397 } 398 DPAA_DP_LOG(DEBUG, "Received an SG frame len =%d, num_sg =%d", 399 first_seg->pkt_len, first_seg->nb_segs); 400 401 dpaa_eth_packet_info(first_seg, vaddr); 402 rte_pktmbuf_free_seg(temp); 403 404 return first_seg; 405 } 406 407 static inline struct rte_mbuf * 408 dpaa_eth_fd_to_mbuf(const struct qm_fd *fd, uint32_t ifid) 409 { 410 struct rte_mbuf *mbuf; 411 struct dpaa_bp_info *bp_info = DPAA_BPID_TO_POOL_INFO(fd->bpid); 412 void *ptr; 413 uint8_t format = 414 (fd->opaque & DPAA_FD_FORMAT_MASK) >> DPAA_FD_FORMAT_SHIFT; 415 uint16_t offset; 416 uint32_t length; 417 418 if (unlikely(format == qm_fd_sg)) 419 return dpaa_eth_sg_to_mbuf(fd, ifid); 420 421 offset = (fd->opaque & DPAA_FD_OFFSET_MASK) >> DPAA_FD_OFFSET_SHIFT; 422 length = fd->opaque & DPAA_FD_LENGTH_MASK; 423 424 DPAA_DP_LOG(DEBUG, " FD--->MBUF off %d len = %d", offset, length); 425 426 /* Ignoring case when format != qm_fd_contig */ 427 ptr = DPAA_MEMPOOL_PTOV(bp_info, qm_fd_addr(fd)); 428 429 mbuf = (struct rte_mbuf *)((char *)ptr - bp_info->meta_data_size); 430 /* Prefetch the Parse results and packet data to L1 */ 431 rte_prefetch0((void *)((uint8_t *)ptr + DEFAULT_RX_ICEOF)); 432 433 mbuf->data_off = offset; 434 mbuf->data_len = length; 435 mbuf->pkt_len = length; 436 437 mbuf->port = ifid; 438 mbuf->nb_segs = 1; 439 mbuf->ol_flags = 0; 440 mbuf->next = NULL; 441 rte_mbuf_refcnt_set(mbuf, 1); 442 dpaa_eth_packet_info(mbuf, mbuf->buf_addr); 443 444 return mbuf; 445 } 446 447 uint16_t 448 dpaa_free_mbuf(const struct qm_fd *fd) 449 { 450 struct rte_mbuf *mbuf; 451 struct dpaa_bp_info *bp_info; 452 uint8_t format; 453 void *ptr; 454 455 bp_info = DPAA_BPID_TO_POOL_INFO(fd->bpid); 456 format = (fd->opaque & DPAA_FD_FORMAT_MASK) >> DPAA_FD_FORMAT_SHIFT; 457 if (unlikely(format == qm_fd_sg)) { 458 struct rte_mbuf *first_seg, *cur_seg; 459 struct qm_sg_entry *sgt, *sg_temp; 460 void *vaddr, *sg_vaddr; 461 int i = 0; 462 uint16_t fd_offset = fd->offset; 463 464 vaddr = DPAA_MEMPOOL_PTOV(bp_info, qm_fd_addr(fd)); 465 if (!vaddr) { 466 DPAA_PMD_ERR("unable to convert physical address"); 467 return -1; 468 } 469 sgt = vaddr + fd_offset; 470 sg_temp = &sgt[i++]; 471 hw_sg_to_cpu(sg_temp); 472 sg_vaddr = DPAA_MEMPOOL_PTOV(bp_info, 473 qm_sg_entry_get64(sg_temp)); 474 first_seg = (struct rte_mbuf *)((char *)sg_vaddr - 475 bp_info->meta_data_size); 476 first_seg->nb_segs = 1; 477 while (i < DPAA_SGT_MAX_ENTRIES) { 478 sg_temp = &sgt[i++]; 479 hw_sg_to_cpu(sg_temp); 480 if (sg_temp->bpid != 0xFF) { 481 bp_info = DPAA_BPID_TO_POOL_INFO(sg_temp->bpid); 482 sg_vaddr = DPAA_MEMPOOL_PTOV(bp_info, 483 qm_sg_entry_get64(sg_temp)); 484 cur_seg = (struct rte_mbuf *)((char *)sg_vaddr - 485 bp_info->meta_data_size); 486 rte_pktmbuf_free_seg(cur_seg); 487 } 488 if (sg_temp->final) 489 break; 490 } 491 rte_pktmbuf_free_seg(first_seg); 492 return 0; 493 } 494 495 ptr = DPAA_MEMPOOL_PTOV(bp_info, qm_fd_addr(fd)); 496 mbuf = (struct rte_mbuf *)((char *)ptr - bp_info->meta_data_size); 497 498 rte_pktmbuf_free(mbuf); 499 500 return 0; 501 } 502 503 /* Specific for LS1043 */ 504 void 505 dpaa_rx_cb_no_prefetch(struct qman_fq **fq, struct qm_dqrr_entry **dqrr, 506 void **bufs, int num_bufs) 507 { 508 struct rte_mbuf *mbuf; 509 struct dpaa_bp_info *bp_info; 510 const struct qm_fd *fd; 511 void *ptr; 512 struct dpaa_if *dpaa_intf; 513 uint16_t offset, i; 514 uint32_t length; 515 uint8_t format; 516 517 bp_info = DPAA_BPID_TO_POOL_INFO(dqrr[0]->fd.bpid); 518 ptr = rte_dpaa_mem_ptov(qm_fd_addr(&dqrr[0]->fd)); 519 rte_prefetch0((void *)((uint8_t *)ptr + DEFAULT_RX_ICEOF)); 520 bufs[0] = (struct rte_mbuf *)((char *)ptr - bp_info->meta_data_size); 521 522 for (i = 0; i < num_bufs; i++) { 523 if (i < num_bufs - 1) { 524 bp_info = DPAA_BPID_TO_POOL_INFO(dqrr[i + 1]->fd.bpid); 525 ptr = rte_dpaa_mem_ptov(qm_fd_addr(&dqrr[i + 1]->fd)); 526 rte_prefetch0((void *)((uint8_t *)ptr + 527 DEFAULT_RX_ICEOF)); 528 bufs[i + 1] = (struct rte_mbuf *)((char *)ptr - 529 bp_info->meta_data_size); 530 } 531 532 fd = &dqrr[i]->fd; 533 dpaa_intf = fq[0]->dpaa_intf; 534 format = (fd->opaque & DPAA_FD_FORMAT_MASK) >> 535 DPAA_FD_FORMAT_SHIFT; 536 if (unlikely(format == qm_fd_sg)) { 537 bufs[i] = dpaa_eth_sg_to_mbuf(fd, dpaa_intf->ifid); 538 continue; 539 } 540 541 offset = (fd->opaque & DPAA_FD_OFFSET_MASK) >> 542 DPAA_FD_OFFSET_SHIFT; 543 length = fd->opaque & DPAA_FD_LENGTH_MASK; 544 545 mbuf = bufs[i]; 546 mbuf->data_off = offset; 547 mbuf->data_len = length; 548 mbuf->pkt_len = length; 549 mbuf->port = dpaa_intf->ifid; 550 551 mbuf->nb_segs = 1; 552 mbuf->ol_flags = 0; 553 mbuf->next = NULL; 554 rte_mbuf_refcnt_set(mbuf, 1); 555 dpaa_eth_packet_info(mbuf, mbuf->buf_addr); 556 dpaa_display_frame_info(fd, fq[0]->fqid, true); 557 } 558 } 559 560 void 561 dpaa_rx_cb(struct qman_fq **fq, struct qm_dqrr_entry **dqrr, 562 void **bufs, int num_bufs) 563 { 564 struct rte_mbuf *mbuf; 565 const struct qm_fd *fd; 566 struct dpaa_if *dpaa_intf; 567 uint16_t offset, i; 568 uint32_t length; 569 uint8_t format; 570 571 for (i = 0; i < num_bufs; i++) { 572 fd = &dqrr[i]->fd; 573 dpaa_intf = fq[0]->dpaa_intf; 574 format = (fd->opaque & DPAA_FD_FORMAT_MASK) >> 575 DPAA_FD_FORMAT_SHIFT; 576 if (unlikely(format == qm_fd_sg)) { 577 bufs[i] = dpaa_eth_sg_to_mbuf(fd, dpaa_intf->ifid); 578 continue; 579 } 580 581 offset = (fd->opaque & DPAA_FD_OFFSET_MASK) >> 582 DPAA_FD_OFFSET_SHIFT; 583 length = fd->opaque & DPAA_FD_LENGTH_MASK; 584 585 mbuf = bufs[i]; 586 mbuf->data_off = offset; 587 mbuf->data_len = length; 588 mbuf->pkt_len = length; 589 mbuf->port = dpaa_intf->ifid; 590 591 mbuf->nb_segs = 1; 592 mbuf->ol_flags = 0; 593 mbuf->next = NULL; 594 rte_mbuf_refcnt_set(mbuf, 1); 595 dpaa_eth_packet_info(mbuf, mbuf->buf_addr); 596 dpaa_display_frame_info(fd, fq[0]->fqid, true); 597 } 598 } 599 600 void dpaa_rx_cb_prepare(struct qm_dqrr_entry *dq, void **bufs) 601 { 602 struct dpaa_bp_info *bp_info = DPAA_BPID_TO_POOL_INFO(dq->fd.bpid); 603 void *ptr = rte_dpaa_mem_ptov(qm_fd_addr(&dq->fd)); 604 605 /* In case of LS1046, annotation stashing is disabled due to L2 cache 606 * being bottleneck in case of multicore scenario for this platform. 607 * So we prefetch the annotation beforehand, so that it is available 608 * in cache when accessed. 609 */ 610 rte_prefetch0((void *)((uint8_t *)ptr + DEFAULT_RX_ICEOF)); 611 612 *bufs = (struct rte_mbuf *)((char *)ptr - bp_info->meta_data_size); 613 } 614 615 static uint16_t 616 dpaa_eth_queue_portal_rx(struct qman_fq *fq, 617 struct rte_mbuf **bufs, 618 uint16_t nb_bufs) 619 { 620 int ret; 621 622 if (unlikely(!fq->qp_initialized)) { 623 ret = rte_dpaa_portal_fq_init((void *)0, fq); 624 if (ret) { 625 DPAA_PMD_ERR("Failure in affining portal %d", ret); 626 return 0; 627 } 628 fq->qp_initialized = 1; 629 } 630 631 return qman_portal_poll_rx(nb_bufs, (void **)bufs, fq->qp); 632 } 633 634 enum qman_cb_dqrr_result 635 dpaa_rx_cb_parallel(void *event, 636 struct qman_portal *qm __always_unused, 637 struct qman_fq *fq, 638 const struct qm_dqrr_entry *dqrr, 639 void **bufs) 640 { 641 u32 ifid = ((struct dpaa_if *)fq->dpaa_intf)->ifid; 642 struct rte_mbuf *mbuf; 643 struct rte_event *ev = (struct rte_event *)event; 644 645 mbuf = dpaa_eth_fd_to_mbuf(&dqrr->fd, ifid); 646 ev->event_ptr = (void *)mbuf; 647 ev->flow_id = fq->ev.flow_id; 648 ev->sub_event_type = fq->ev.sub_event_type; 649 ev->event_type = RTE_EVENT_TYPE_ETHDEV; 650 ev->op = RTE_EVENT_OP_NEW; 651 ev->sched_type = fq->ev.sched_type; 652 ev->queue_id = fq->ev.queue_id; 653 ev->priority = fq->ev.priority; 654 ev->impl_opaque = (uint8_t)DPAA_INVALID_MBUF_SEQN; 655 *dpaa_seqn(mbuf) = DPAA_INVALID_MBUF_SEQN; 656 *bufs = mbuf; 657 658 return qman_cb_dqrr_consume; 659 } 660 661 enum qman_cb_dqrr_result 662 dpaa_rx_cb_atomic(void *event, 663 struct qman_portal *qm __always_unused, 664 struct qman_fq *fq, 665 const struct qm_dqrr_entry *dqrr, 666 void **bufs) 667 { 668 u8 index; 669 u32 ifid = ((struct dpaa_if *)fq->dpaa_intf)->ifid; 670 struct rte_mbuf *mbuf; 671 struct rte_event *ev = (struct rte_event *)event; 672 673 mbuf = dpaa_eth_fd_to_mbuf(&dqrr->fd, ifid); 674 ev->event_ptr = (void *)mbuf; 675 ev->flow_id = fq->ev.flow_id; 676 ev->sub_event_type = fq->ev.sub_event_type; 677 ev->event_type = RTE_EVENT_TYPE_ETHDEV; 678 ev->op = RTE_EVENT_OP_NEW; 679 ev->sched_type = fq->ev.sched_type; 680 ev->queue_id = fq->ev.queue_id; 681 ev->priority = fq->ev.priority; 682 683 /* Save active dqrr entries */ 684 index = DQRR_PTR2IDX(dqrr); 685 DPAA_PER_LCORE_DQRR_SIZE++; 686 DPAA_PER_LCORE_DQRR_HELD |= 1 << index; 687 DPAA_PER_LCORE_DQRR_MBUF(index) = mbuf; 688 ev->impl_opaque = index + 1; 689 *dpaa_seqn(mbuf) = (uint32_t)index + 1; 690 *bufs = mbuf; 691 692 return qman_cb_dqrr_defer; 693 } 694 695 #ifdef RTE_LIBRTE_DPAA_DEBUG_DRIVER 696 static inline void dpaa_eth_err_queue(struct dpaa_if *dpaa_intf) 697 { 698 struct rte_mbuf *mbuf; 699 struct qman_fq *debug_fq; 700 int ret, i; 701 struct qm_dqrr_entry *dq; 702 struct qm_fd *fd; 703 704 if (unlikely(!RTE_PER_LCORE(dpaa_io))) { 705 ret = rte_dpaa_portal_init((void *)0); 706 if (ret) { 707 DPAA_PMD_ERR("Failure in affining portal"); 708 return; 709 } 710 } 711 for (i = 0; i <= DPAA_DEBUG_FQ_TX_ERROR; i++) { 712 debug_fq = &dpaa_intf->debug_queues[i]; 713 ret = qman_set_vdq(debug_fq, 4, QM_VDQCR_EXACT); 714 if (ret) 715 return; 716 717 do { 718 dq = qman_dequeue(debug_fq); 719 if (!dq) 720 continue; 721 fd = &dq->fd; 722 if (i == DPAA_DEBUG_FQ_RX_ERROR) 723 DPAA_PMD_ERR("RX ERROR status: 0x%08x", 724 fd->status); 725 else 726 DPAA_PMD_ERR("TX ERROR status: 0x%08x", 727 fd->status); 728 dpaa_display_frame_info(fd, debug_fq->fqid, 729 i == DPAA_DEBUG_FQ_RX_ERROR); 730 731 mbuf = dpaa_eth_fd_to_mbuf(fd, dpaa_intf->ifid); 732 rte_pktmbuf_free(mbuf); 733 qman_dqrr_consume(debug_fq, dq); 734 } while (debug_fq->flags & QMAN_FQ_STATE_VDQCR); 735 } 736 } 737 #endif 738 739 uint16_t dpaa_eth_queue_rx(void *q, 740 struct rte_mbuf **bufs, 741 uint16_t nb_bufs) 742 { 743 struct qman_fq *fq = q; 744 struct qm_dqrr_entry *dq; 745 uint32_t num_rx = 0, ifid = ((struct dpaa_if *)fq->dpaa_intf)->ifid; 746 int num_rx_bufs, ret; 747 uint32_t vdqcr_flags = 0; 748 749 if (unlikely(rte_dpaa_bpid_info == NULL && 750 rte_eal_process_type() == RTE_PROC_SECONDARY)) 751 rte_dpaa_bpid_info = fq->bp_array; 752 753 #ifdef RTE_LIBRTE_DPAA_DEBUG_DRIVER 754 if (fq->fqid == ((struct dpaa_if *)fq->dpaa_intf)->rx_queues[0].fqid) 755 dpaa_eth_err_queue((struct dpaa_if *)fq->dpaa_intf); 756 #endif 757 758 if (likely(fq->is_static)) 759 return dpaa_eth_queue_portal_rx(fq, bufs, nb_bufs); 760 761 if (unlikely(!DPAA_PER_LCORE_PORTAL)) { 762 ret = rte_dpaa_portal_init((void *)0); 763 if (ret) { 764 DPAA_PMD_ERR("Failure in affining portal"); 765 return 0; 766 } 767 } 768 769 /* Until request for four buffers, we provide exact number of buffers. 770 * Otherwise we do not set the QM_VDQCR_EXACT flag. 771 * Not setting QM_VDQCR_EXACT flag can provide two more buffers than 772 * requested, so we request two less in this case. 773 */ 774 if (nb_bufs < 4) { 775 vdqcr_flags = QM_VDQCR_EXACT; 776 num_rx_bufs = nb_bufs; 777 } else { 778 num_rx_bufs = nb_bufs > DPAA_MAX_DEQUEUE_NUM_FRAMES ? 779 (DPAA_MAX_DEQUEUE_NUM_FRAMES - 2) : (nb_bufs - 2); 780 } 781 ret = qman_set_vdq(fq, num_rx_bufs, vdqcr_flags); 782 if (ret) 783 return 0; 784 785 do { 786 dq = qman_dequeue(fq); 787 if (!dq) 788 continue; 789 bufs[num_rx++] = dpaa_eth_fd_to_mbuf(&dq->fd, ifid); 790 dpaa_display_frame_info(&dq->fd, fq->fqid, true); 791 qman_dqrr_consume(fq, dq); 792 } while (fq->flags & QMAN_FQ_STATE_VDQCR); 793 794 return num_rx; 795 } 796 797 static int 798 dpaa_eth_mbuf_to_sg_fd(struct rte_mbuf *mbuf, 799 struct qm_fd *fd, 800 struct dpaa_sw_buf_free *free_buf, 801 uint32_t *free_count, 802 uint32_t pkt_id) 803 { 804 struct rte_mbuf *cur_seg = mbuf; 805 struct rte_mbuf *temp, *mi; 806 struct qm_sg_entry *sg_temp, *sgt; 807 int i = 0; 808 809 DPAA_DP_LOG(DEBUG, "Creating SG FD to transmit"); 810 811 temp = rte_pktmbuf_alloc(dpaa_tx_sg_pool); 812 if (!temp) { 813 DPAA_PMD_ERR("Failure in allocation of mbuf"); 814 return -1; 815 } 816 if (temp->buf_len < ((mbuf->nb_segs * sizeof(struct qm_sg_entry)) 817 + temp->data_off)) { 818 DPAA_PMD_ERR("Insufficient space in mbuf for SG entries"); 819 return -1; 820 } 821 822 fd->cmd = 0; 823 fd->opaque_addr = 0; 824 825 if (mbuf->ol_flags & DPAA_TX_CKSUM_OFFLOAD_MASK) { 826 if (!mbuf->packet_type) { 827 struct rte_net_hdr_lens hdr_lens; 828 829 mbuf->packet_type = rte_net_get_ptype(mbuf, &hdr_lens, 830 RTE_PTYPE_L2_MASK | RTE_PTYPE_L3_MASK 831 | RTE_PTYPE_L4_MASK); 832 mbuf->l2_len = hdr_lens.l2_len; 833 mbuf->l3_len = hdr_lens.l3_len; 834 } 835 if (temp->data_off < DEFAULT_TX_ICEOF 836 + sizeof(struct dpaa_eth_parse_results_t)) 837 temp->data_off = DEFAULT_TX_ICEOF 838 + sizeof(struct dpaa_eth_parse_results_t); 839 dcbz_64(temp->buf_addr); 840 dpaa_checksum_offload(mbuf, fd, temp->buf_addr); 841 } 842 843 sgt = temp->buf_addr + temp->data_off; 844 fd->format = QM_FD_SG; 845 fd->addr = temp->buf_iova; 846 fd->offset = temp->data_off; 847 fd->bpid = DPAA_MEMPOOL_TO_BPID(dpaa_tx_sg_pool); 848 fd->length20 = mbuf->pkt_len; 849 850 while (i < DPAA_SGT_MAX_ENTRIES) { 851 sg_temp = &sgt[i++]; 852 sg_temp->opaque = 0; 853 sg_temp->val = 0; 854 sg_temp->addr = cur_seg->buf_iova; 855 sg_temp->offset = cur_seg->data_off; 856 sg_temp->length = cur_seg->data_len; 857 if (RTE_MBUF_DIRECT(cur_seg)) { 858 if (rte_mbuf_refcnt_read(cur_seg) > 1) { 859 /*If refcnt > 1, invalid bpid is set to ensure 860 * buffer is not freed by HW. 861 */ 862 sg_temp->bpid = 0xff; 863 rte_mbuf_refcnt_update(cur_seg, -1); 864 } else { 865 sg_temp->bpid = 866 DPAA_MEMPOOL_TO_BPID(cur_seg->pool); 867 } 868 } else if (RTE_MBUF_HAS_EXTBUF(cur_seg)) { 869 free_buf[*free_count].seg = cur_seg; 870 free_buf[*free_count].pkt_id = pkt_id; 871 ++*free_count; 872 sg_temp->bpid = 0xff; 873 } else { 874 /* Get owner MBUF from indirect buffer */ 875 mi = rte_mbuf_from_indirect(cur_seg); 876 if (rte_mbuf_refcnt_read(mi) > 1) { 877 /*If refcnt > 1, invalid bpid is set to ensure 878 * owner buffer is not freed by HW. 879 */ 880 sg_temp->bpid = 0xff; 881 } else { 882 sg_temp->bpid = DPAA_MEMPOOL_TO_BPID(mi->pool); 883 rte_mbuf_refcnt_update(mi, 1); 884 } 885 free_buf[*free_count].seg = cur_seg; 886 free_buf[*free_count].pkt_id = pkt_id; 887 ++*free_count; 888 } 889 cur_seg = cur_seg->next; 890 if (cur_seg == NULL) { 891 sg_temp->final = 1; 892 cpu_to_hw_sg(sg_temp); 893 break; 894 } 895 cpu_to_hw_sg(sg_temp); 896 } 897 return 0; 898 } 899 900 /* Handle mbufs which are not segmented (non SG) */ 901 static inline void 902 tx_on_dpaa_pool_unsegmented(struct rte_mbuf *mbuf, 903 struct dpaa_bp_info *bp_info, 904 struct qm_fd *fd_arr, 905 struct dpaa_sw_buf_free *buf_to_free, 906 uint32_t *free_count, 907 uint32_t pkt_id) 908 { 909 struct rte_mbuf *mi = NULL; 910 911 if (RTE_MBUF_DIRECT(mbuf)) { 912 if (rte_mbuf_refcnt_read(mbuf) > 1) { 913 /* In case of direct mbuf and mbuf being cloned, 914 * BMAN should _not_ release buffer. 915 */ 916 DPAA_MBUF_TO_CONTIG_FD(mbuf, fd_arr, 0xff); 917 /* Buffer should be releasd by EAL */ 918 rte_mbuf_refcnt_update(mbuf, -1); 919 } else { 920 /* In case of direct mbuf and no cloning, mbuf can be 921 * released by BMAN. 922 */ 923 DPAA_MBUF_TO_CONTIG_FD(mbuf, fd_arr, bp_info->bpid); 924 } 925 } else if (RTE_MBUF_HAS_EXTBUF(mbuf)) { 926 buf_to_free[*free_count].seg = mbuf; 927 buf_to_free[*free_count].pkt_id = pkt_id; 928 ++*free_count; 929 DPAA_MBUF_TO_CONTIG_FD(mbuf, fd_arr, 930 bp_info ? bp_info->bpid : 0xff); 931 } else { 932 /* This is data-containing core mbuf: 'mi' */ 933 mi = rte_mbuf_from_indirect(mbuf); 934 if (rte_mbuf_refcnt_read(mi) > 1) { 935 /* In case of indirect mbuf, and mbuf being cloned, 936 * BMAN should _not_ release it and let EAL release 937 * it through pktmbuf_free below. 938 */ 939 DPAA_MBUF_TO_CONTIG_FD(mbuf, fd_arr, 0xff); 940 } else { 941 /* In case of indirect mbuf, and no cloning, core mbuf 942 * should be released by BMAN. 943 * Increate refcnt of core mbuf so that when 944 * pktmbuf_free is called and mbuf is released, EAL 945 * doesn't try to release core mbuf which would have 946 * been released by BMAN. 947 */ 948 rte_mbuf_refcnt_update(mi, 1); 949 DPAA_MBUF_TO_CONTIG_FD(mbuf, fd_arr, 950 bp_info ? bp_info->bpid : 0xff); 951 } 952 buf_to_free[*free_count].seg = mbuf; 953 buf_to_free[*free_count].pkt_id = pkt_id; 954 ++*free_count; 955 } 956 957 if (mbuf->ol_flags & DPAA_TX_CKSUM_OFFLOAD_MASK) 958 dpaa_unsegmented_checksum(mbuf, fd_arr); 959 } 960 961 /* Handle all mbufs on dpaa BMAN managed pool */ 962 static inline uint16_t 963 tx_on_dpaa_pool(struct rte_mbuf *mbuf, 964 struct dpaa_bp_info *bp_info, 965 struct qm_fd *fd_arr, 966 struct dpaa_sw_buf_free *buf_to_free, 967 uint32_t *free_count, 968 uint32_t pkt_id) 969 { 970 DPAA_DP_LOG(DEBUG, "BMAN offloaded buffer, mbuf: %p", mbuf); 971 972 if (mbuf->nb_segs == 1) { 973 /* Case for non-segmented buffers */ 974 tx_on_dpaa_pool_unsegmented(mbuf, bp_info, fd_arr, 975 buf_to_free, free_count, pkt_id); 976 } else if (mbuf->nb_segs > 1 && 977 mbuf->nb_segs <= DPAA_SGT_MAX_ENTRIES) { 978 if (dpaa_eth_mbuf_to_sg_fd(mbuf, fd_arr, buf_to_free, 979 free_count, pkt_id)) { 980 DPAA_PMD_DEBUG("Unable to create Scatter Gather FD"); 981 return 1; 982 } 983 } else { 984 DPAA_PMD_DEBUG("Number of Segments not supported"); 985 return 1; 986 } 987 988 return 0; 989 } 990 991 /* Handle all mbufs on an external pool (non-dpaa) */ 992 static inline struct rte_mbuf * 993 reallocate_mbuf(struct qman_fq *txq, struct rte_mbuf *mbuf) 994 { 995 struct dpaa_if *dpaa_intf = txq->dpaa_intf; 996 struct dpaa_bp_info *bp_info = dpaa_intf->bp_info; 997 struct rte_mbuf *new_mbufs[DPAA_SGT_MAX_ENTRIES + 1] = {0}; 998 struct rte_mbuf *temp_mbuf; 999 int num_new_segs, mbuf_greater, ret, extra_seg = 0, i = 0; 1000 uint64_t mbufs_size, bytes_to_copy, offset1 = 0, offset2 = 0; 1001 char *data; 1002 1003 DPAA_DP_LOG(DEBUG, "Reallocating transmit buffer"); 1004 1005 mbufs_size = bp_info->size - 1006 bp_info->meta_data_size - RTE_PKTMBUF_HEADROOM; 1007 extra_seg = !!(mbuf->pkt_len % mbufs_size); 1008 num_new_segs = (mbuf->pkt_len / mbufs_size) + extra_seg; 1009 1010 ret = rte_pktmbuf_alloc_bulk(bp_info->mp, new_mbufs, num_new_segs); 1011 if (ret != 0) { 1012 DPAA_DP_LOG(DEBUG, "Allocation for new buffers failed"); 1013 return NULL; 1014 } 1015 1016 temp_mbuf = mbuf; 1017 1018 while (temp_mbuf) { 1019 /* If mbuf data is less than new mbuf remaining memory */ 1020 if ((temp_mbuf->data_len - offset1) < (mbufs_size - offset2)) { 1021 bytes_to_copy = temp_mbuf->data_len - offset1; 1022 mbuf_greater = -1; 1023 /* If mbuf data is greater than new mbuf remaining memory */ 1024 } else if ((temp_mbuf->data_len - offset1) > 1025 (mbufs_size - offset2)) { 1026 bytes_to_copy = mbufs_size - offset2; 1027 mbuf_greater = 1; 1028 /* if mbuf data is equal to new mbuf remaining memory */ 1029 } else { 1030 bytes_to_copy = temp_mbuf->data_len - offset1; 1031 mbuf_greater = 0; 1032 } 1033 1034 /* Copy the data */ 1035 data = rte_pktmbuf_append(new_mbufs[0], bytes_to_copy); 1036 1037 rte_memcpy((uint8_t *)data, rte_pktmbuf_mtod_offset(mbuf, 1038 void *, offset1), bytes_to_copy); 1039 1040 /* Set new offsets and the temp buffers */ 1041 if (mbuf_greater == -1) { 1042 offset1 = 0; 1043 offset2 += bytes_to_copy; 1044 temp_mbuf = temp_mbuf->next; 1045 } else if (mbuf_greater == 1) { 1046 offset2 = 0; 1047 offset1 += bytes_to_copy; 1048 new_mbufs[i]->next = new_mbufs[i + 1]; 1049 new_mbufs[0]->nb_segs++; 1050 i++; 1051 } else { 1052 offset1 = 0; 1053 offset2 = 0; 1054 temp_mbuf = temp_mbuf->next; 1055 new_mbufs[i]->next = new_mbufs[i + 1]; 1056 if (new_mbufs[i + 1]) 1057 new_mbufs[0]->nb_segs++; 1058 i++; 1059 } 1060 } 1061 1062 /* Copy other required fields */ 1063 new_mbufs[0]->ol_flags = mbuf->ol_flags; 1064 new_mbufs[0]->packet_type = mbuf->packet_type; 1065 new_mbufs[0]->tx_offload = mbuf->tx_offload; 1066 1067 rte_pktmbuf_free(mbuf); 1068 1069 return new_mbufs[0]; 1070 } 1071 1072 uint16_t 1073 dpaa_eth_queue_tx(void *q, struct rte_mbuf **bufs, uint16_t nb_bufs) 1074 { 1075 struct rte_mbuf *mbuf, *mi = NULL; 1076 struct rte_mempool *mp; 1077 struct dpaa_bp_info *bp_info; 1078 struct qm_fd fd_arr[DPAA_TX_BURST_SIZE]; 1079 uint32_t frames_to_send, loop, sent = 0; 1080 uint16_t state; 1081 int ret, realloc_mbuf = 0; 1082 uint32_t seqn, index, flags[DPAA_TX_BURST_SIZE] = {0}; 1083 struct dpaa_sw_buf_free buf_to_free[DPAA_MAX_SGS * DPAA_MAX_DEQUEUE_NUM_FRAMES]; 1084 uint32_t free_count = 0; 1085 1086 if (unlikely(!DPAA_PER_LCORE_PORTAL)) { 1087 ret = rte_dpaa_portal_init((void *)0); 1088 if (ret) { 1089 DPAA_PMD_ERR("Failure in affining portal"); 1090 return 0; 1091 } 1092 } 1093 1094 DPAA_DP_LOG(DEBUG, "Transmitting %d buffers on queue: %p", nb_bufs, q); 1095 1096 while (nb_bufs) { 1097 frames_to_send = (nb_bufs > DPAA_TX_BURST_SIZE) ? 1098 DPAA_TX_BURST_SIZE : nb_bufs; 1099 for (loop = 0; loop < frames_to_send; loop++) { 1100 mbuf = *(bufs++); 1101 /* In case the data offset is not multiple of 16, 1102 * FMAN can stall because of an errata. So reallocate 1103 * the buffer in such case. 1104 */ 1105 if (dpaa_svr_family == SVR_LS1043A_FAMILY && 1106 (mbuf->data_off & 0x7F) != 0x0) 1107 realloc_mbuf = 1; 1108 seqn = *dpaa_seqn(mbuf); 1109 if (seqn != DPAA_INVALID_MBUF_SEQN) { 1110 index = seqn - 1; 1111 if (DPAA_PER_LCORE_DQRR_HELD & (1 << index)) { 1112 flags[loop] = 1113 ((index & QM_EQCR_DCA_IDXMASK) << 8); 1114 flags[loop] |= QMAN_ENQUEUE_FLAG_DCA; 1115 DPAA_PER_LCORE_DQRR_SIZE--; 1116 DPAA_PER_LCORE_DQRR_HELD &= 1117 ~(1 << index); 1118 } 1119 } 1120 1121 if (likely(RTE_MBUF_DIRECT(mbuf))) { 1122 mp = mbuf->pool; 1123 bp_info = DPAA_MEMPOOL_TO_POOL_INFO(mp); 1124 if (likely(mp->ops_index == 1125 bp_info->dpaa_ops_index && 1126 mbuf->nb_segs == 1 && 1127 realloc_mbuf == 0 && 1128 rte_mbuf_refcnt_read(mbuf) == 1)) { 1129 DPAA_MBUF_TO_CONTIG_FD(mbuf, 1130 &fd_arr[loop], bp_info->bpid); 1131 if (mbuf->ol_flags & 1132 DPAA_TX_CKSUM_OFFLOAD_MASK) 1133 dpaa_unsegmented_checksum(mbuf, 1134 &fd_arr[loop]); 1135 continue; 1136 } 1137 } else { 1138 mi = rte_mbuf_from_indirect(mbuf); 1139 mp = mi->pool; 1140 } 1141 1142 if (unlikely(RTE_MBUF_HAS_EXTBUF(mbuf))) { 1143 bp_info = NULL; 1144 goto indirect_buf; 1145 } 1146 1147 bp_info = DPAA_MEMPOOL_TO_POOL_INFO(mp); 1148 if (unlikely(mp->ops_index != bp_info->dpaa_ops_index || 1149 realloc_mbuf == 1)) { 1150 struct rte_mbuf *temp_mbuf; 1151 1152 temp_mbuf = reallocate_mbuf(q, mbuf); 1153 if (!temp_mbuf) { 1154 /* Set frames_to_send & nb_bufs so 1155 * that packets are transmitted till 1156 * previous frame. 1157 */ 1158 frames_to_send = loop; 1159 nb_bufs = loop; 1160 goto send_pkts; 1161 } 1162 mbuf = temp_mbuf; 1163 realloc_mbuf = 0; 1164 } 1165 indirect_buf: 1166 state = tx_on_dpaa_pool(mbuf, bp_info, 1167 &fd_arr[loop], 1168 buf_to_free, 1169 &free_count, 1170 loop); 1171 if (unlikely(state)) { 1172 /* Set frames_to_send & nb_bufs so 1173 * that packets are transmitted till 1174 * previous frame. 1175 */ 1176 frames_to_send = loop; 1177 nb_bufs = loop; 1178 goto send_pkts; 1179 } 1180 } 1181 1182 send_pkts: 1183 loop = 0; 1184 while (loop < frames_to_send) { 1185 loop += qman_enqueue_multi(q, &fd_arr[loop], 1186 &flags[loop], 1187 frames_to_send - loop); 1188 } 1189 nb_bufs -= frames_to_send; 1190 sent += frames_to_send; 1191 } 1192 1193 DPAA_DP_LOG(DEBUG, "Transmitted %d buffers on queue: %p", sent, q); 1194 1195 for (loop = 0; loop < free_count; loop++) { 1196 if (buf_to_free[loop].pkt_id < sent) 1197 rte_pktmbuf_free_seg(buf_to_free[loop].seg); 1198 } 1199 1200 return sent; 1201 } 1202 1203 uint16_t 1204 dpaa_eth_queue_tx_slow(void *q, struct rte_mbuf **bufs, uint16_t nb_bufs) 1205 { 1206 qman_ern_poll_free(); 1207 1208 return dpaa_eth_queue_tx(q, bufs, nb_bufs); 1209 } 1210 1211 uint16_t dpaa_eth_tx_drop_all(void *q __rte_unused, 1212 struct rte_mbuf **bufs __rte_unused, 1213 uint16_t nb_bufs __rte_unused) 1214 { 1215 DPAA_DP_LOG(DEBUG, "Drop all packets"); 1216 1217 /* Drop all incoming packets. No need to free packets here 1218 * because the rte_eth f/w frees up the packets through tx_buffer 1219 * callback in case this functions returns count less than nb_bufs 1220 */ 1221 return 0; 1222 } 1223