1 /* SPDX-License-Identifier: BSD-3-Clause 2 * 3 * Copyright 2016 Freescale Semiconductor, Inc. All rights reserved. 4 * Copyright 2017,2019-2024 NXP 5 * 6 */ 7 8 /* System headers */ 9 #include <inttypes.h> 10 #include <unistd.h> 11 #include <stdio.h> 12 #include <limits.h> 13 #include <sched.h> 14 #include <pthread.h> 15 16 #include <rte_byteorder.h> 17 #include <rte_common.h> 18 #include <rte_interrupts.h> 19 #include <rte_log.h> 20 #include <rte_debug.h> 21 #include <rte_pci.h> 22 #include <rte_atomic.h> 23 #include <rte_branch_prediction.h> 24 #include <rte_memory.h> 25 #include <rte_tailq.h> 26 #include <rte_eal.h> 27 #include <rte_alarm.h> 28 #include <rte_ether.h> 29 #include <ethdev_driver.h> 30 #include <rte_malloc.h> 31 #include <rte_ring.h> 32 #include <rte_ip.h> 33 #include <rte_tcp.h> 34 #include <rte_udp.h> 35 #include <rte_net.h> 36 #include <rte_eventdev.h> 37 38 #include "dpaa_ethdev.h" 39 #include "dpaa_rxtx.h" 40 #include <bus_dpaa_driver.h> 41 #include <dpaa_mempool.h> 42 43 #include <qman.h> 44 #include <fsl_usd.h> 45 #include <fsl_qman.h> 46 #include <fsl_bman.h> 47 #include <dpaa_of.h> 48 #include <netcfg.h> 49 50 #ifdef RTE_LIBRTE_DPAA_DEBUG_DRIVER 51 static int s_force_display_frm; 52 #endif 53 54 #define DPAA_MBUF_TO_CONTIG_FD(_mbuf, _fd, _bpid) \ 55 do { \ 56 (_fd)->opaque_addr = 0; \ 57 (_fd)->opaque = QM_FD_CONTIG << DPAA_FD_FORMAT_SHIFT; \ 58 (_fd)->opaque |= ((_mbuf)->data_off) << DPAA_FD_OFFSET_SHIFT; \ 59 (_fd)->opaque |= (_mbuf)->pkt_len; \ 60 (_fd)->addr = (_mbuf)->buf_iova; \ 61 (_fd)->bpid = _bpid; \ 62 } while (0) 63 64 #ifdef RTE_LIBRTE_DPAA_DEBUG_DRIVER 65 void 66 dpaa_force_display_frame_set(int set) 67 { 68 s_force_display_frm = set; 69 } 70 71 #define DISPLAY_PRINT printf 72 static void 73 dpaa_display_frame_info(const struct qm_fd *fd, 74 uint32_t fqid, bool rx) 75 { 76 int pos, offset = 0; 77 char *ptr, info[1024]; 78 struct annotations_t *annot = rte_dpaa_mem_ptov(fd->addr); 79 uint8_t format; 80 const struct dpaa_eth_parse_results_t *psr; 81 82 if (!fd->status && !s_force_display_frm) { 83 /* Do not display correct packets unless force display.*/ 84 return; 85 } 86 psr = &annot->parse; 87 88 format = (fd->opaque & DPAA_FD_FORMAT_MASK) >> DPAA_FD_FORMAT_SHIFT; 89 if (format == qm_fd_contig) 90 sprintf(info, "simple"); 91 else if (format == qm_fd_sg) 92 sprintf(info, "sg"); 93 else 94 sprintf(info, "unknown format(%d)", format); 95 96 DISPLAY_PRINT("%s: fqid=%08x, bpid=%d, phy addr=0x%lx ", 97 rx ? "RX" : "TX", fqid, fd->bpid, (unsigned long)fd->addr); 98 DISPLAY_PRINT("format=%s offset=%d, len=%d, stat=0x%x\r\n", 99 info, fd->offset, fd->length20, fd->status); 100 if (rx) { 101 DISPLAY_PRINT("Display usual RX parser result:\r\n"); 102 if (psr->eth_frame_type == 0) 103 offset += sprintf(&info[offset], "unicast"); 104 else if (psr->eth_frame_type == 1) 105 offset += sprintf(&info[offset], "multicast"); 106 else if (psr->eth_frame_type == 3) 107 offset += sprintf(&info[offset], "broadcast"); 108 else 109 offset += sprintf(&info[offset], "unknown eth type(%d)", 110 psr->eth_frame_type); 111 if (psr->l2r_err) { 112 offset += sprintf(&info[offset], " L2 error(%d)", 113 psr->l2r_err); 114 } else { 115 offset += sprintf(&info[offset], " L2 non error"); 116 } 117 DISPLAY_PRINT("L2: %s, %s, ethernet type:%s\r\n", 118 psr->ethernet ? "is ethernet" : "non ethernet", 119 psr->vlan ? "is vlan" : "non vlan", info); 120 121 offset = 0; 122 DISPLAY_PRINT("L3: %s/%s, %s/%s, %s, %s\r\n", 123 psr->first_ipv4 ? "first IPv4" : "non first IPv4", 124 psr->last_ipv4 ? "last IPv4" : "non last IPv4", 125 psr->first_ipv6 ? "first IPv6" : "non first IPv6", 126 psr->last_ipv6 ? "last IPv6" : "non last IPv6", 127 psr->gre ? "GRE" : "non GRE", 128 psr->l3_err ? "L3 has error" : "L3 non error"); 129 130 if (psr->l4_type == DPAA_PR_L4_TCP_TYPE) { 131 offset += sprintf(&info[offset], "tcp"); 132 } else if (psr->l4_type == DPAA_PR_L4_UDP_TYPE) { 133 offset += sprintf(&info[offset], "udp"); 134 } else if (psr->l4_type == DPAA_PR_L4_IPSEC_TYPE) { 135 offset += sprintf(&info[offset], "IPSec "); 136 if (psr->esp_sum) 137 offset += sprintf(&info[offset], "ESP"); 138 if (psr->ah) 139 offset += sprintf(&info[offset], "AH"); 140 } else if (psr->l4_type == DPAA_PR_L4_SCTP_TYPE) { 141 offset += sprintf(&info[offset], "sctp"); 142 } else if (psr->l4_type == DPAA_PR_L4_DCCP_TYPE) { 143 offset += sprintf(&info[offset], "dccp"); 144 } else { 145 offset += sprintf(&info[offset], "unknown l4 type(%d)", 146 psr->l4_type); 147 } 148 DISPLAY_PRINT("L4: type:%s, L4 validation %s\r\n", 149 info, psr->l4cv ? "Performed" : "NOT performed"); 150 151 offset = 0; 152 if (psr->ethernet) { 153 offset += sprintf(&info[offset], 154 "Eth offset=%d, ethtype offset=%d, ", 155 psr->eth_off, psr->etype_off); 156 } 157 if (psr->vlan) { 158 offset += sprintf(&info[offset], "vLAN offset=%d, ", 159 psr->vlan_off[0]); 160 } 161 if (psr->first_ipv4 || psr->first_ipv6) { 162 offset += sprintf(&info[offset], "first IP offset=%d, ", 163 psr->ip_off[0]); 164 } 165 if (psr->last_ipv4 || psr->last_ipv6) { 166 offset += sprintf(&info[offset], "last IP offset=%d, ", 167 psr->ip_off[1]); 168 } 169 if (psr->gre) { 170 offset += sprintf(&info[offset], "GRE offset=%d, ", 171 psr->gre_off); 172 } 173 if (psr->l4_type >= DPAA_PR_L4_TCP_TYPE) { 174 offset += sprintf(&info[offset], "L4 offset=%d, ", 175 psr->l4_off); 176 } 177 offset += sprintf(&info[offset], "Next HDR(0x%04x) offset=%d.", 178 rte_be_to_cpu_16(psr->nxthdr), psr->nxthdr_off); 179 180 DISPLAY_PRINT("%s\r\n", info); 181 } 182 183 if (unlikely(format == qm_fd_sg)) { 184 /*TBD:S/G display: to be implemented*/ 185 return; 186 } 187 188 DISPLAY_PRINT("Frame payload:\r\n"); 189 ptr = (char *)annot; 190 ptr += fd->offset; 191 for (pos = 0; pos < fd->length20; pos++) { 192 DISPLAY_PRINT("%02x ", ptr[pos]); 193 if (((pos + 1) % 16) == 0) 194 DISPLAY_PRINT("\n"); 195 } 196 DISPLAY_PRINT("\n"); 197 } 198 199 #else 200 #define dpaa_display_frame_info(a, b, c) 201 #endif 202 203 static inline void 204 dpaa_slow_parsing(struct rte_mbuf *m, 205 const struct annotations_t *annot) 206 { 207 const struct dpaa_eth_parse_results_t *parse; 208 209 DPAA_DP_LOG(DEBUG, "Slow parsing"); 210 parse = &annot->parse; 211 212 if (parse->ethernet) 213 m->packet_type |= RTE_PTYPE_L2_ETHER; 214 if (parse->vlan) 215 m->packet_type |= RTE_PTYPE_L2_ETHER_VLAN; 216 if (parse->first_ipv4) 217 m->packet_type |= RTE_PTYPE_L3_IPV4; 218 if (parse->first_ipv6) 219 m->packet_type |= RTE_PTYPE_L3_IPV6; 220 if (parse->gre) 221 m->packet_type |= RTE_PTYPE_TUNNEL_GRE; 222 if (parse->last_ipv4) 223 m->packet_type |= RTE_PTYPE_L3_IPV4_EXT; 224 if (parse->last_ipv6) 225 m->packet_type |= RTE_PTYPE_L3_IPV6_EXT; 226 if (parse->l4_type == DPAA_PR_L4_TCP_TYPE) 227 m->packet_type |= RTE_PTYPE_L4_TCP; 228 else if (parse->l4_type == DPAA_PR_L4_UDP_TYPE) 229 m->packet_type |= RTE_PTYPE_L4_UDP; 230 else if (parse->l4_type == DPAA_PR_L4_IPSEC_TYPE && 231 !parse->l4_info_err && parse->esp_sum) 232 m->packet_type |= RTE_PTYPE_TUNNEL_ESP; 233 else if (parse->l4_type == DPAA_PR_L4_SCTP_TYPE) 234 m->packet_type |= RTE_PTYPE_L4_SCTP; 235 } 236 237 static inline void dpaa_eth_packet_info(struct rte_mbuf *m, void *fd_virt_addr) 238 { 239 struct annotations_t *annot = GET_ANNOTATIONS(fd_virt_addr); 240 uint64_t prs = *((uintptr_t *)(&annot->parse)) & DPAA_PARSE_MASK; 241 struct rte_ether_hdr *eth_hdr = 242 rte_pktmbuf_mtod(m, struct rte_ether_hdr *); 243 244 DPAA_DP_LOG(DEBUG, " Parsing mbuf: %p with annotations: %p", m, annot); 245 246 m->ol_flags = RTE_MBUF_F_RX_RSS_HASH | RTE_MBUF_F_RX_IP_CKSUM_GOOD | 247 RTE_MBUF_F_RX_L4_CKSUM_GOOD; 248 249 switch (prs) { 250 case DPAA_PKT_TYPE_IPV4: 251 m->packet_type = RTE_PTYPE_L2_ETHER | 252 RTE_PTYPE_L3_IPV4; 253 break; 254 case DPAA_PKT_TYPE_IPV6: 255 m->packet_type = RTE_PTYPE_L2_ETHER | 256 RTE_PTYPE_L3_IPV6; 257 break; 258 case DPAA_PKT_TYPE_ETHER: 259 m->packet_type = RTE_PTYPE_L2_ETHER; 260 break; 261 case DPAA_PKT_TYPE_IPV4_FRAG: 262 case DPAA_PKT_TYPE_IPV4_FRAG_UDP: 263 case DPAA_PKT_TYPE_IPV4_FRAG_TCP: 264 case DPAA_PKT_TYPE_IPV4_FRAG_SCTP: 265 m->packet_type = RTE_PTYPE_L2_ETHER | 266 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_FRAG; 267 break; 268 case DPAA_PKT_TYPE_IPV6_FRAG: 269 case DPAA_PKT_TYPE_IPV6_FRAG_UDP: 270 case DPAA_PKT_TYPE_IPV6_FRAG_TCP: 271 case DPAA_PKT_TYPE_IPV6_FRAG_SCTP: 272 m->packet_type = RTE_PTYPE_L2_ETHER | 273 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_FRAG; 274 break; 275 case DPAA_PKT_TYPE_IPV4_EXT: 276 m->packet_type = RTE_PTYPE_L2_ETHER | 277 RTE_PTYPE_L3_IPV4_EXT; 278 break; 279 case DPAA_PKT_TYPE_IPV6_EXT: 280 m->packet_type = RTE_PTYPE_L2_ETHER | 281 RTE_PTYPE_L3_IPV6_EXT; 282 break; 283 case DPAA_PKT_TYPE_IPV4_TCP: 284 m->packet_type = RTE_PTYPE_L2_ETHER | 285 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_TCP; 286 break; 287 case DPAA_PKT_TYPE_IPV6_TCP: 288 m->packet_type = RTE_PTYPE_L2_ETHER | 289 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_TCP; 290 break; 291 case DPAA_PKT_TYPE_IPV4_UDP: 292 m->packet_type = RTE_PTYPE_L2_ETHER | 293 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_UDP; 294 break; 295 case DPAA_PKT_TYPE_IPV6_UDP: 296 m->packet_type = RTE_PTYPE_L2_ETHER | 297 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_UDP; 298 break; 299 case DPAA_PKT_TYPE_IPSEC_IPV4: 300 if (*((uintptr_t *)&annot->parse) & DPAA_PARSE_ESP_MASK) 301 m->packet_type = RTE_PTYPE_L2_ETHER | 302 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_ESP; 303 break; 304 case DPAA_PKT_TYPE_IPSEC_IPV6: 305 if (*((uintptr_t *)&annot->parse) & DPAA_PARSE_ESP_MASK) 306 m->packet_type = RTE_PTYPE_L2_ETHER | 307 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_TUNNEL_ESP; 308 break; 309 case DPAA_PKT_TYPE_IPV4_EXT_UDP: 310 m->packet_type = RTE_PTYPE_L2_ETHER | 311 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_UDP; 312 break; 313 case DPAA_PKT_TYPE_IPV6_EXT_UDP: 314 m->packet_type = RTE_PTYPE_L2_ETHER | 315 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_UDP; 316 break; 317 case DPAA_PKT_TYPE_IPV4_EXT_TCP: 318 m->packet_type = RTE_PTYPE_L2_ETHER | 319 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_TCP; 320 break; 321 case DPAA_PKT_TYPE_IPV6_EXT_TCP: 322 m->packet_type = RTE_PTYPE_L2_ETHER | 323 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_TCP; 324 break; 325 case DPAA_PKT_TYPE_IPV4_SCTP: 326 m->packet_type = RTE_PTYPE_L2_ETHER | 327 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_SCTP; 328 break; 329 case DPAA_PKT_TYPE_IPV6_SCTP: 330 m->packet_type = RTE_PTYPE_L2_ETHER | 331 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_SCTP; 332 break; 333 case DPAA_PKT_TYPE_IPV4_CSUM_ERR: 334 case DPAA_PKT_TYPE_IPV6_CSUM_ERR: 335 m->ol_flags = RTE_MBUF_F_RX_RSS_HASH | RTE_MBUF_F_RX_IP_CKSUM_BAD; 336 break; 337 case DPAA_PKT_TYPE_IPV4_TCP_CSUM_ERR: 338 case DPAA_PKT_TYPE_IPV6_TCP_CSUM_ERR: 339 case DPAA_PKT_TYPE_IPV4_UDP_CSUM_ERR: 340 case DPAA_PKT_TYPE_IPV6_UDP_CSUM_ERR: 341 m->ol_flags = RTE_MBUF_F_RX_RSS_HASH | RTE_MBUF_F_RX_L4_CKSUM_BAD; 342 break; 343 case DPAA_PKT_TYPE_NONE: 344 m->packet_type = 0; 345 break; 346 /* More switch cases can be added */ 347 default: 348 dpaa_slow_parsing(m, annot); 349 } 350 351 m->tx_offload = annot->parse.ip_off[0]; 352 m->tx_offload |= (annot->parse.l4_off - annot->parse.ip_off[0]) 353 << DPAA_PKT_L3_LEN_SHIFT; 354 355 /* Set the hash values */ 356 m->hash.rss = (uint32_t)(annot->hash); 357 358 /* Check if Vlan is present */ 359 if (prs & DPAA_PARSE_VLAN_MASK) 360 m->ol_flags |= RTE_MBUF_F_RX_VLAN; 361 /* Packet received without stripping the vlan */ 362 363 if (eth_hdr->ether_type == htons(RTE_ETHER_TYPE_1588)) { 364 m->ol_flags |= RTE_MBUF_F_RX_IEEE1588_PTP; 365 m->ol_flags |= RTE_MBUF_F_RX_IEEE1588_TMST; 366 } 367 } 368 369 static inline void dpaa_checksum(struct rte_mbuf *mbuf) 370 { 371 struct rte_ether_hdr *eth_hdr = 372 rte_pktmbuf_mtod(mbuf, struct rte_ether_hdr *); 373 char *l3_hdr = (char *)eth_hdr + mbuf->l2_len; 374 struct rte_ipv4_hdr *ipv4_hdr = (struct rte_ipv4_hdr *)l3_hdr; 375 struct rte_ipv6_hdr *ipv6_hdr = (struct rte_ipv6_hdr *)l3_hdr; 376 377 DPAA_DP_LOG(DEBUG, "Calculating checksum for mbuf: %p", mbuf); 378 379 if (((mbuf->packet_type & RTE_PTYPE_L3_MASK) == RTE_PTYPE_L3_IPV4) || 380 ((mbuf->packet_type & RTE_PTYPE_L3_MASK) == 381 RTE_PTYPE_L3_IPV4_EXT)) { 382 ipv4_hdr = (struct rte_ipv4_hdr *)l3_hdr; 383 ipv4_hdr->hdr_checksum = 0; 384 ipv4_hdr->hdr_checksum = rte_ipv4_cksum(ipv4_hdr); 385 } else if (((mbuf->packet_type & RTE_PTYPE_L3_MASK) == 386 RTE_PTYPE_L3_IPV6) || 387 ((mbuf->packet_type & RTE_PTYPE_L3_MASK) == 388 RTE_PTYPE_L3_IPV6_EXT)) 389 ipv6_hdr = (struct rte_ipv6_hdr *)l3_hdr; 390 391 if ((mbuf->packet_type & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_TCP) { 392 struct rte_tcp_hdr *tcp_hdr = (struct rte_tcp_hdr *)(l3_hdr + 393 mbuf->l3_len); 394 tcp_hdr->cksum = 0; 395 if (eth_hdr->ether_type == htons(RTE_ETHER_TYPE_IPV4)) 396 tcp_hdr->cksum = rte_ipv4_udptcp_cksum(ipv4_hdr, 397 tcp_hdr); 398 else /* assume ethertype == RTE_ETHER_TYPE_IPV6 */ 399 tcp_hdr->cksum = rte_ipv6_udptcp_cksum(ipv6_hdr, 400 tcp_hdr); 401 } else if ((mbuf->packet_type & RTE_PTYPE_L4_MASK) == 402 RTE_PTYPE_L4_UDP) { 403 struct rte_udp_hdr *udp_hdr = (struct rte_udp_hdr *)(l3_hdr + 404 mbuf->l3_len); 405 udp_hdr->dgram_cksum = 0; 406 if (eth_hdr->ether_type == htons(RTE_ETHER_TYPE_IPV4)) 407 udp_hdr->dgram_cksum = rte_ipv4_udptcp_cksum(ipv4_hdr, 408 udp_hdr); 409 else /* assume ethertype == RTE_ETHER_TYPE_IPV6 */ 410 udp_hdr->dgram_cksum = rte_ipv6_udptcp_cksum(ipv6_hdr, 411 udp_hdr); 412 } 413 } 414 415 static inline void dpaa_checksum_offload(struct rte_mbuf *mbuf, 416 struct qm_fd *fd, char *prs_buf) 417 { 418 struct dpaa_eth_parse_results_t *prs; 419 420 DPAA_DP_LOG(DEBUG, " Offloading checksum for mbuf: %p", mbuf); 421 422 prs = GET_TX_PRS(prs_buf); 423 prs->l3r = 0; 424 prs->l4r = 0; 425 if (((mbuf->packet_type & RTE_PTYPE_L3_MASK) == RTE_PTYPE_L3_IPV4) || 426 ((mbuf->packet_type & RTE_PTYPE_L3_MASK) == 427 RTE_PTYPE_L3_IPV4_EXT)) 428 prs->l3r = DPAA_L3_PARSE_RESULT_IPV4; 429 else if (((mbuf->packet_type & RTE_PTYPE_L3_MASK) == 430 RTE_PTYPE_L3_IPV6) || 431 ((mbuf->packet_type & RTE_PTYPE_L3_MASK) == 432 RTE_PTYPE_L3_IPV6_EXT)) 433 prs->l3r = DPAA_L3_PARSE_RESULT_IPV6; 434 435 if ((mbuf->packet_type & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_TCP) 436 prs->l4r = DPAA_L4_PARSE_RESULT_TCP; 437 else if ((mbuf->packet_type & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_UDP) 438 prs->l4r = DPAA_L4_PARSE_RESULT_UDP; 439 440 prs->ip_off[0] = mbuf->l2_len; 441 prs->l4_off = mbuf->l3_len + mbuf->l2_len; 442 /* Enable L3 (and L4, if TCP or UDP) HW checksum*/ 443 fd->cmd |= DPAA_FD_CMD_RPD | DPAA_FD_CMD_DTC; 444 } 445 446 static inline void 447 dpaa_unsegmented_checksum(struct rte_mbuf *mbuf, struct qm_fd *fd_arr) 448 { 449 if (!mbuf->packet_type) { 450 struct rte_net_hdr_lens hdr_lens; 451 452 mbuf->packet_type = rte_net_get_ptype(mbuf, &hdr_lens, 453 RTE_PTYPE_L2_MASK | RTE_PTYPE_L3_MASK 454 | RTE_PTYPE_L4_MASK); 455 mbuf->l2_len = hdr_lens.l2_len; 456 mbuf->l3_len = hdr_lens.l3_len; 457 } 458 if (mbuf->data_off < (DEFAULT_TX_ICEOF + 459 sizeof(struct dpaa_eth_parse_results_t))) { 460 DPAA_DP_LOG(DEBUG, "Checksum offload Err: " 461 "Not enough Headroom " 462 "space for correct Checksum offload." 463 "So Calculating checksum in Software."); 464 dpaa_checksum(mbuf); 465 } else { 466 dpaa_checksum_offload(mbuf, fd_arr, mbuf->buf_addr); 467 } 468 } 469 470 static struct rte_mbuf * 471 dpaa_eth_sg_to_mbuf(const struct qm_fd *fd, uint32_t ifid) 472 { 473 struct dpaa_bp_info *bp_info = DPAA_BPID_TO_POOL_INFO(fd->bpid); 474 struct rte_mbuf *first_seg, *prev_seg, *cur_seg, *temp; 475 struct qm_sg_entry *sgt, *sg_temp; 476 void *vaddr, *sg_vaddr; 477 int i = 0; 478 uint16_t fd_offset = fd->offset; 479 480 vaddr = DPAA_MEMPOOL_PTOV(bp_info, qm_fd_addr(fd)); 481 if (!vaddr) { 482 DPAA_PMD_ERR("unable to convert physical address"); 483 return NULL; 484 } 485 sgt = vaddr + fd_offset; 486 sg_temp = &sgt[i++]; 487 hw_sg_to_cpu(sg_temp); 488 temp = (struct rte_mbuf *)((char *)vaddr - bp_info->meta_data_size); 489 sg_vaddr = DPAA_MEMPOOL_PTOV(bp_info, qm_sg_entry_get64(sg_temp)); 490 491 first_seg = (struct rte_mbuf *)((char *)sg_vaddr - 492 bp_info->meta_data_size); 493 first_seg->data_off = sg_temp->offset; 494 first_seg->data_len = sg_temp->length; 495 first_seg->pkt_len = sg_temp->length; 496 rte_mbuf_refcnt_set(first_seg, 1); 497 498 first_seg->port = ifid; 499 first_seg->nb_segs = 1; 500 first_seg->ol_flags = 0; 501 prev_seg = first_seg; 502 while (i < DPAA_SGT_MAX_ENTRIES) { 503 sg_temp = &sgt[i++]; 504 hw_sg_to_cpu(sg_temp); 505 sg_vaddr = DPAA_MEMPOOL_PTOV(bp_info, 506 qm_sg_entry_get64(sg_temp)); 507 cur_seg = (struct rte_mbuf *)((char *)sg_vaddr - 508 bp_info->meta_data_size); 509 cur_seg->data_off = sg_temp->offset; 510 cur_seg->data_len = sg_temp->length; 511 first_seg->pkt_len += sg_temp->length; 512 first_seg->nb_segs += 1; 513 rte_mbuf_refcnt_set(cur_seg, 1); 514 prev_seg->next = cur_seg; 515 if (sg_temp->final) { 516 cur_seg->next = NULL; 517 break; 518 } 519 prev_seg = cur_seg; 520 } 521 DPAA_DP_LOG(DEBUG, "Received an SG frame len =%d, num_sg =%d", 522 first_seg->pkt_len, first_seg->nb_segs); 523 524 dpaa_eth_packet_info(first_seg, vaddr); 525 rte_pktmbuf_free_seg(temp); 526 527 return first_seg; 528 } 529 530 static inline struct rte_mbuf * 531 dpaa_eth_fd_to_mbuf(const struct qm_fd *fd, uint32_t ifid) 532 { 533 struct rte_mbuf *mbuf; 534 struct dpaa_bp_info *bp_info = DPAA_BPID_TO_POOL_INFO(fd->bpid); 535 void *ptr; 536 uint8_t format = 537 (fd->opaque & DPAA_FD_FORMAT_MASK) >> DPAA_FD_FORMAT_SHIFT; 538 uint16_t offset; 539 uint32_t length; 540 541 if (unlikely(format == qm_fd_sg)) 542 return dpaa_eth_sg_to_mbuf(fd, ifid); 543 544 offset = (fd->opaque & DPAA_FD_OFFSET_MASK) >> DPAA_FD_OFFSET_SHIFT; 545 length = fd->opaque & DPAA_FD_LENGTH_MASK; 546 547 DPAA_DP_LOG(DEBUG, " FD--->MBUF off %d len = %d", offset, length); 548 549 /* Ignoring case when format != qm_fd_contig */ 550 ptr = DPAA_MEMPOOL_PTOV(bp_info, qm_fd_addr(fd)); 551 552 mbuf = (struct rte_mbuf *)((char *)ptr - bp_info->meta_data_size); 553 /* Prefetch the Parse results and packet data to L1 */ 554 rte_prefetch0((void *)((uint8_t *)ptr + DEFAULT_RX_ICEOF)); 555 556 mbuf->data_off = offset; 557 mbuf->data_len = length; 558 mbuf->pkt_len = length; 559 560 mbuf->port = ifid; 561 mbuf->nb_segs = 1; 562 mbuf->ol_flags = 0; 563 mbuf->next = NULL; 564 rte_mbuf_refcnt_set(mbuf, 1); 565 dpaa_eth_packet_info(mbuf, mbuf->buf_addr); 566 567 return mbuf; 568 } 569 570 uint16_t 571 dpaa_free_mbuf(const struct qm_fd *fd) 572 { 573 struct rte_mbuf *mbuf; 574 struct dpaa_bp_info *bp_info; 575 uint8_t format; 576 void *ptr; 577 578 bp_info = DPAA_BPID_TO_POOL_INFO(fd->bpid); 579 format = (fd->opaque & DPAA_FD_FORMAT_MASK) >> DPAA_FD_FORMAT_SHIFT; 580 if (unlikely(format == qm_fd_sg)) { 581 struct rte_mbuf *first_seg, *cur_seg; 582 struct qm_sg_entry *sgt, *sg_temp; 583 void *vaddr, *sg_vaddr; 584 int i = 0; 585 uint16_t fd_offset = fd->offset; 586 587 vaddr = DPAA_MEMPOOL_PTOV(bp_info, qm_fd_addr(fd)); 588 if (!vaddr) { 589 DPAA_PMD_ERR("unable to convert physical address"); 590 return -1; 591 } 592 sgt = vaddr + fd_offset; 593 sg_temp = &sgt[i++]; 594 hw_sg_to_cpu(sg_temp); 595 sg_vaddr = DPAA_MEMPOOL_PTOV(bp_info, 596 qm_sg_entry_get64(sg_temp)); 597 first_seg = (struct rte_mbuf *)((char *)sg_vaddr - 598 bp_info->meta_data_size); 599 first_seg->nb_segs = 1; 600 while (i < DPAA_SGT_MAX_ENTRIES) { 601 sg_temp = &sgt[i++]; 602 hw_sg_to_cpu(sg_temp); 603 if (sg_temp->bpid != 0xFF) { 604 bp_info = DPAA_BPID_TO_POOL_INFO(sg_temp->bpid); 605 sg_vaddr = DPAA_MEMPOOL_PTOV(bp_info, 606 qm_sg_entry_get64(sg_temp)); 607 cur_seg = (struct rte_mbuf *)((char *)sg_vaddr - 608 bp_info->meta_data_size); 609 rte_pktmbuf_free_seg(cur_seg); 610 } 611 if (sg_temp->final) 612 break; 613 } 614 rte_pktmbuf_free_seg(first_seg); 615 return 0; 616 } 617 618 ptr = DPAA_MEMPOOL_PTOV(bp_info, qm_fd_addr(fd)); 619 mbuf = (struct rte_mbuf *)((char *)ptr - bp_info->meta_data_size); 620 621 rte_pktmbuf_free(mbuf); 622 623 return 0; 624 } 625 626 /* Specific for LS1043 */ 627 void 628 dpaa_rx_cb_no_prefetch(struct qman_fq **fq, struct qm_dqrr_entry **dqrr, 629 void **bufs, int num_bufs) 630 { 631 struct rte_mbuf *mbuf; 632 struct dpaa_bp_info *bp_info; 633 const struct qm_fd *fd; 634 void *ptr; 635 struct dpaa_if *dpaa_intf; 636 uint16_t offset, i; 637 uint32_t length; 638 uint8_t format; 639 struct annotations_t *annot; 640 641 bp_info = DPAA_BPID_TO_POOL_INFO(dqrr[0]->fd.bpid); 642 ptr = rte_dpaa_mem_ptov(qm_fd_addr(&dqrr[0]->fd)); 643 rte_prefetch0((void *)((uint8_t *)ptr + DEFAULT_RX_ICEOF)); 644 bufs[0] = (struct rte_mbuf *)((char *)ptr - bp_info->meta_data_size); 645 646 for (i = 0; i < num_bufs; i++) { 647 if (i < num_bufs - 1) { 648 bp_info = DPAA_BPID_TO_POOL_INFO(dqrr[i + 1]->fd.bpid); 649 ptr = rte_dpaa_mem_ptov(qm_fd_addr(&dqrr[i + 1]->fd)); 650 rte_prefetch0((void *)((uint8_t *)ptr + 651 DEFAULT_RX_ICEOF)); 652 bufs[i + 1] = (struct rte_mbuf *)((char *)ptr - 653 bp_info->meta_data_size); 654 } 655 656 fd = &dqrr[i]->fd; 657 dpaa_intf = fq[0]->dpaa_intf; 658 format = (fd->opaque & DPAA_FD_FORMAT_MASK) >> 659 DPAA_FD_FORMAT_SHIFT; 660 if (unlikely(format == qm_fd_sg)) { 661 bufs[i] = dpaa_eth_sg_to_mbuf(fd, dpaa_intf->ifid); 662 continue; 663 } 664 665 offset = (fd->opaque & DPAA_FD_OFFSET_MASK) >> 666 DPAA_FD_OFFSET_SHIFT; 667 length = fd->opaque & DPAA_FD_LENGTH_MASK; 668 669 mbuf = bufs[i]; 670 mbuf->data_off = offset; 671 mbuf->data_len = length; 672 mbuf->pkt_len = length; 673 mbuf->port = dpaa_intf->ifid; 674 675 mbuf->nb_segs = 1; 676 mbuf->ol_flags = 0; 677 mbuf->next = NULL; 678 rte_mbuf_refcnt_set(mbuf, 1); 679 dpaa_eth_packet_info(mbuf, mbuf->buf_addr); 680 dpaa_display_frame_info(fd, fq[0]->fqid, true); 681 if (dpaa_ieee_1588) { 682 annot = GET_ANNOTATIONS(mbuf->buf_addr); 683 dpaa_intf->rx_timestamp = 684 rte_cpu_to_be_64(annot->timestamp); 685 } 686 } 687 } 688 689 void 690 dpaa_rx_cb(struct qman_fq **fq, struct qm_dqrr_entry **dqrr, 691 void **bufs, int num_bufs) 692 { 693 struct rte_mbuf *mbuf; 694 const struct qm_fd *fd; 695 struct dpaa_if *dpaa_intf; 696 uint16_t offset, i; 697 uint32_t length; 698 uint8_t format; 699 struct annotations_t *annot; 700 701 for (i = 0; i < num_bufs; i++) { 702 fd = &dqrr[i]->fd; 703 dpaa_intf = fq[0]->dpaa_intf; 704 format = (fd->opaque & DPAA_FD_FORMAT_MASK) >> 705 DPAA_FD_FORMAT_SHIFT; 706 if (unlikely(format == qm_fd_sg)) { 707 bufs[i] = dpaa_eth_sg_to_mbuf(fd, dpaa_intf->ifid); 708 continue; 709 } 710 711 offset = (fd->opaque & DPAA_FD_OFFSET_MASK) >> 712 DPAA_FD_OFFSET_SHIFT; 713 length = fd->opaque & DPAA_FD_LENGTH_MASK; 714 715 mbuf = bufs[i]; 716 mbuf->data_off = offset; 717 mbuf->data_len = length; 718 mbuf->pkt_len = length; 719 mbuf->port = dpaa_intf->ifid; 720 721 mbuf->nb_segs = 1; 722 mbuf->ol_flags = 0; 723 mbuf->next = NULL; 724 rte_mbuf_refcnt_set(mbuf, 1); 725 dpaa_eth_packet_info(mbuf, mbuf->buf_addr); 726 dpaa_display_frame_info(fd, fq[0]->fqid, true); 727 if (dpaa_ieee_1588) { 728 annot = GET_ANNOTATIONS(mbuf->buf_addr); 729 dpaa_intf->rx_timestamp = 730 rte_cpu_to_be_64(annot->timestamp); 731 } 732 } 733 } 734 735 void dpaa_rx_cb_prepare(struct qm_dqrr_entry *dq, void **bufs) 736 { 737 struct dpaa_bp_info *bp_info = DPAA_BPID_TO_POOL_INFO(dq->fd.bpid); 738 void *ptr = rte_dpaa_mem_ptov(qm_fd_addr(&dq->fd)); 739 740 /* In case of LS1046, annotation stashing is disabled due to L2 cache 741 * being bottleneck in case of multicore scenario for this platform. 742 * So we prefetch the annotation beforehand, so that it is available 743 * in cache when accessed. 744 */ 745 rte_prefetch0((void *)((uint8_t *)ptr + DEFAULT_RX_ICEOF)); 746 747 *bufs = (struct rte_mbuf *)((char *)ptr - bp_info->meta_data_size); 748 } 749 750 static uint16_t 751 dpaa_eth_queue_portal_rx(struct qman_fq *fq, 752 struct rte_mbuf **bufs, 753 uint16_t nb_bufs) 754 { 755 int ret; 756 757 if (unlikely(!fq->qp_initialized)) { 758 ret = rte_dpaa_portal_fq_init((void *)0, fq); 759 if (ret) { 760 DPAA_PMD_ERR("Failure in affining portal %d", ret); 761 return 0; 762 } 763 fq->qp_initialized = 1; 764 } 765 766 return qman_portal_poll_rx(nb_bufs, (void **)bufs, fq->qp); 767 } 768 769 enum qman_cb_dqrr_result 770 dpaa_rx_cb_parallel(void *event, 771 struct qman_portal *qm __always_unused, 772 struct qman_fq *fq, 773 const struct qm_dqrr_entry *dqrr, 774 void **bufs) 775 { 776 u32 ifid = ((struct dpaa_if *)fq->dpaa_intf)->ifid; 777 struct rte_mbuf *mbuf; 778 struct rte_event *ev = (struct rte_event *)event; 779 780 mbuf = dpaa_eth_fd_to_mbuf(&dqrr->fd, ifid); 781 ev->event_ptr = (void *)mbuf; 782 ev->flow_id = fq->ev.flow_id; 783 ev->sub_event_type = fq->ev.sub_event_type; 784 ev->event_type = RTE_EVENT_TYPE_ETHDEV; 785 ev->op = RTE_EVENT_OP_NEW; 786 ev->sched_type = fq->ev.sched_type; 787 ev->queue_id = fq->ev.queue_id; 788 ev->priority = fq->ev.priority; 789 ev->impl_opaque = (uint8_t)DPAA_INVALID_MBUF_SEQN; 790 *dpaa_seqn(mbuf) = DPAA_INVALID_MBUF_SEQN; 791 *bufs = mbuf; 792 793 return qman_cb_dqrr_consume; 794 } 795 796 enum qman_cb_dqrr_result 797 dpaa_rx_cb_atomic(void *event, 798 struct qman_portal *qm __always_unused, 799 struct qman_fq *fq, 800 const struct qm_dqrr_entry *dqrr, 801 void **bufs) 802 { 803 u8 index; 804 u32 ifid = ((struct dpaa_if *)fq->dpaa_intf)->ifid; 805 struct rte_mbuf *mbuf; 806 struct rte_event *ev = (struct rte_event *)event; 807 808 mbuf = dpaa_eth_fd_to_mbuf(&dqrr->fd, ifid); 809 ev->event_ptr = (void *)mbuf; 810 ev->flow_id = fq->ev.flow_id; 811 ev->sub_event_type = fq->ev.sub_event_type; 812 ev->event_type = RTE_EVENT_TYPE_ETHDEV; 813 ev->op = RTE_EVENT_OP_NEW; 814 ev->sched_type = fq->ev.sched_type; 815 ev->queue_id = fq->ev.queue_id; 816 ev->priority = fq->ev.priority; 817 818 /* Save active dqrr entries */ 819 index = DQRR_PTR2IDX(dqrr); 820 DPAA_PER_LCORE_DQRR_SIZE++; 821 DPAA_PER_LCORE_DQRR_HELD |= 1 << index; 822 DPAA_PER_LCORE_DQRR_MBUF(index) = mbuf; 823 ev->impl_opaque = index + 1; 824 *dpaa_seqn(mbuf) = (uint32_t)index + 1; 825 *bufs = mbuf; 826 827 return qman_cb_dqrr_defer; 828 } 829 830 #ifdef RTE_LIBRTE_DPAA_DEBUG_DRIVER 831 static inline void 832 dpaa_eth_err_queue(struct qman_fq *fq) 833 { 834 struct rte_mbuf *mbuf; 835 struct qman_fq *debug_fq; 836 int ret, i; 837 struct qm_dqrr_entry *dq; 838 struct qm_fd *fd; 839 struct dpaa_if *dpaa_intf; 840 841 dpaa_intf = fq->dpaa_intf; 842 if (fq != &dpaa_intf->rx_queues[0]) { 843 /* Associate error queues to the first RXQ.*/ 844 return; 845 } 846 847 if (dpaa_intf->cfg->fman_if->is_shared_mac) { 848 /* Error queues of shared MAC are handled in kernel. */ 849 return; 850 } 851 852 if (unlikely(!RTE_PER_LCORE(dpaa_io))) { 853 ret = rte_dpaa_portal_init((void *)0); 854 if (ret) { 855 DPAA_PMD_ERR("Failure in affining portal"); 856 return; 857 } 858 } 859 for (i = 0; i < DPAA_DEBUG_FQ_MAX_NUM; i++) { 860 debug_fq = &dpaa_intf->debug_queues[i]; 861 ret = qman_set_vdq(debug_fq, 4, QM_VDQCR_EXACT); 862 if (ret) 863 return; 864 865 do { 866 dq = qman_dequeue(debug_fq); 867 if (!dq) 868 continue; 869 fd = &dq->fd; 870 if (i == DPAA_DEBUG_FQ_RX_ERROR) 871 DPAA_PMD_ERR("RX ERROR status: 0x%08x", 872 fd->status); 873 else 874 DPAA_PMD_ERR("TX ERROR status: 0x%08x", 875 fd->status); 876 dpaa_display_frame_info(fd, debug_fq->fqid, 877 i == DPAA_DEBUG_FQ_RX_ERROR); 878 879 mbuf = dpaa_eth_fd_to_mbuf(fd, dpaa_intf->ifid); 880 rte_pktmbuf_free(mbuf); 881 qman_dqrr_consume(debug_fq, dq); 882 } while (debug_fq->flags & QMAN_FQ_STATE_VDQCR); 883 } 884 } 885 #endif 886 887 uint16_t dpaa_eth_queue_rx(void *q, 888 struct rte_mbuf **bufs, 889 uint16_t nb_bufs) 890 { 891 struct qman_fq *fq = q; 892 struct qm_dqrr_entry *dq; 893 uint32_t num_rx = 0, ifid = ((struct dpaa_if *)fq->dpaa_intf)->ifid; 894 int num_rx_bufs, ret; 895 uint32_t vdqcr_flags = 0; 896 struct annotations_t *annot; 897 struct dpaa_if *dpaa_intf = fq->dpaa_intf; 898 899 if (unlikely(rte_dpaa_bpid_info == NULL && 900 rte_eal_process_type() == RTE_PROC_SECONDARY)) 901 rte_dpaa_bpid_info = fq->bp_array; 902 903 #ifdef RTE_LIBRTE_DPAA_DEBUG_DRIVER 904 dpaa_eth_err_queue(fq); 905 #endif 906 907 if (likely(fq->is_static)) 908 return dpaa_eth_queue_portal_rx(fq, bufs, nb_bufs); 909 910 if (unlikely(!DPAA_PER_LCORE_PORTAL)) { 911 ret = rte_dpaa_portal_init((void *)0); 912 if (ret) { 913 DPAA_PMD_ERR("Failure in affining portal"); 914 return 0; 915 } 916 } 917 918 /* Until request for four buffers, we provide exact number of buffers. 919 * Otherwise we do not set the QM_VDQCR_EXACT flag. 920 * Not setting QM_VDQCR_EXACT flag can provide two more buffers than 921 * requested, so we request two less in this case. 922 */ 923 if (nb_bufs < 4) { 924 vdqcr_flags = QM_VDQCR_EXACT; 925 num_rx_bufs = nb_bufs; 926 } else { 927 num_rx_bufs = nb_bufs > DPAA_MAX_DEQUEUE_NUM_FRAMES ? 928 (DPAA_MAX_DEQUEUE_NUM_FRAMES - 2) : (nb_bufs - 2); 929 } 930 ret = qman_set_vdq(fq, num_rx_bufs, vdqcr_flags); 931 if (ret) 932 return 0; 933 934 do { 935 dq = qman_dequeue(fq); 936 if (!dq) 937 continue; 938 bufs[num_rx++] = dpaa_eth_fd_to_mbuf(&dq->fd, ifid); 939 dpaa_display_frame_info(&dq->fd, fq->fqid, true); 940 if (dpaa_ieee_1588) { 941 annot = GET_ANNOTATIONS(bufs[num_rx - 1]->buf_addr); 942 dpaa_intf->rx_timestamp = rte_cpu_to_be_64(annot->timestamp); 943 } 944 qman_dqrr_consume(fq, dq); 945 } while (fq->flags & QMAN_FQ_STATE_VDQCR); 946 947 return num_rx; 948 } 949 950 static int 951 dpaa_eth_mbuf_to_sg_fd(struct rte_mbuf *mbuf, 952 struct qm_fd *fd, 953 struct dpaa_sw_buf_free *free_buf, 954 uint32_t *free_count, 955 uint32_t pkt_id) 956 { 957 struct rte_mbuf *cur_seg = mbuf; 958 struct rte_mbuf *temp, *mi; 959 struct qm_sg_entry *sg_temp, *sgt; 960 int i = 0; 961 962 DPAA_DP_LOG(DEBUG, "Creating SG FD to transmit"); 963 964 temp = rte_pktmbuf_alloc(dpaa_tx_sg_pool); 965 if (!temp) { 966 DPAA_PMD_ERR("Failure in allocation of mbuf"); 967 return -1; 968 } 969 if (temp->buf_len < ((mbuf->nb_segs * sizeof(struct qm_sg_entry)) 970 + temp->data_off)) { 971 DPAA_PMD_ERR("Insufficient space in mbuf for SG entries"); 972 return -1; 973 } 974 975 fd->cmd = 0; 976 fd->opaque_addr = 0; 977 978 if (mbuf->ol_flags & DPAA_TX_CKSUM_OFFLOAD_MASK) { 979 if (!mbuf->packet_type) { 980 struct rte_net_hdr_lens hdr_lens; 981 982 mbuf->packet_type = rte_net_get_ptype(mbuf, &hdr_lens, 983 RTE_PTYPE_L2_MASK | RTE_PTYPE_L3_MASK 984 | RTE_PTYPE_L4_MASK); 985 mbuf->l2_len = hdr_lens.l2_len; 986 mbuf->l3_len = hdr_lens.l3_len; 987 } 988 if (temp->data_off < DEFAULT_TX_ICEOF 989 + sizeof(struct dpaa_eth_parse_results_t)) 990 temp->data_off = DEFAULT_TX_ICEOF 991 + sizeof(struct dpaa_eth_parse_results_t); 992 dcbz_64(temp->buf_addr); 993 dpaa_checksum_offload(mbuf, fd, temp->buf_addr); 994 } 995 996 sgt = temp->buf_addr + temp->data_off; 997 fd->format = QM_FD_SG; 998 fd->addr = temp->buf_iova; 999 fd->offset = temp->data_off; 1000 fd->bpid = DPAA_MEMPOOL_TO_BPID(dpaa_tx_sg_pool); 1001 fd->length20 = mbuf->pkt_len; 1002 1003 while (i < DPAA_SGT_MAX_ENTRIES) { 1004 sg_temp = &sgt[i++]; 1005 sg_temp->opaque = 0; 1006 sg_temp->val = 0; 1007 sg_temp->addr = cur_seg->buf_iova; 1008 sg_temp->offset = cur_seg->data_off; 1009 sg_temp->length = cur_seg->data_len; 1010 if (RTE_MBUF_DIRECT(cur_seg)) { 1011 if (rte_mbuf_refcnt_read(cur_seg) > 1) { 1012 /*If refcnt > 1, invalid bpid is set to ensure 1013 * buffer is not freed by HW. 1014 */ 1015 sg_temp->bpid = 0xff; 1016 rte_mbuf_refcnt_update(cur_seg, -1); 1017 } else { 1018 sg_temp->bpid = 1019 DPAA_MEMPOOL_TO_BPID(cur_seg->pool); 1020 } 1021 } else if (RTE_MBUF_HAS_EXTBUF(cur_seg)) { 1022 free_buf[*free_count].seg = cur_seg; 1023 free_buf[*free_count].pkt_id = pkt_id; 1024 ++*free_count; 1025 sg_temp->bpid = 0xff; 1026 } else { 1027 /* Get owner MBUF from indirect buffer */ 1028 mi = rte_mbuf_from_indirect(cur_seg); 1029 if (rte_mbuf_refcnt_read(mi) > 1) { 1030 /*If refcnt > 1, invalid bpid is set to ensure 1031 * owner buffer is not freed by HW. 1032 */ 1033 sg_temp->bpid = 0xff; 1034 } else { 1035 sg_temp->bpid = DPAA_MEMPOOL_TO_BPID(mi->pool); 1036 rte_mbuf_refcnt_update(mi, 1); 1037 } 1038 free_buf[*free_count].seg = cur_seg; 1039 free_buf[*free_count].pkt_id = pkt_id; 1040 ++*free_count; 1041 } 1042 cur_seg = cur_seg->next; 1043 if (cur_seg == NULL) { 1044 sg_temp->final = 1; 1045 cpu_to_hw_sg(sg_temp); 1046 break; 1047 } 1048 cpu_to_hw_sg(sg_temp); 1049 } 1050 return 0; 1051 } 1052 1053 /* Handle mbufs which are not segmented (non SG) */ 1054 static inline void 1055 tx_on_dpaa_pool_unsegmented(struct rte_mbuf *mbuf, 1056 struct dpaa_bp_info *bp_info, 1057 struct qm_fd *fd_arr, 1058 struct dpaa_sw_buf_free *buf_to_free, 1059 uint32_t *free_count, 1060 uint32_t pkt_id) 1061 { 1062 struct rte_mbuf *mi = NULL; 1063 1064 if (RTE_MBUF_DIRECT(mbuf)) { 1065 if (rte_mbuf_refcnt_read(mbuf) > 1) { 1066 /* In case of direct mbuf and mbuf being cloned, 1067 * BMAN should _not_ release buffer. 1068 */ 1069 DPAA_MBUF_TO_CONTIG_FD(mbuf, fd_arr, 0xff); 1070 /* Buffer should be releasd by EAL */ 1071 rte_mbuf_refcnt_update(mbuf, -1); 1072 } else { 1073 /* In case of direct mbuf and no cloning, mbuf can be 1074 * released by BMAN. 1075 */ 1076 DPAA_MBUF_TO_CONTIG_FD(mbuf, fd_arr, bp_info->bpid); 1077 } 1078 } else if (RTE_MBUF_HAS_EXTBUF(mbuf)) { 1079 buf_to_free[*free_count].seg = mbuf; 1080 buf_to_free[*free_count].pkt_id = pkt_id; 1081 ++*free_count; 1082 DPAA_MBUF_TO_CONTIG_FD(mbuf, fd_arr, 1083 bp_info ? bp_info->bpid : 0xff); 1084 } else { 1085 /* This is data-containing core mbuf: 'mi' */ 1086 mi = rte_mbuf_from_indirect(mbuf); 1087 if (rte_mbuf_refcnt_read(mi) > 1) { 1088 /* In case of indirect mbuf, and mbuf being cloned, 1089 * BMAN should _not_ release it and let EAL release 1090 * it through pktmbuf_free below. 1091 */ 1092 DPAA_MBUF_TO_CONTIG_FD(mbuf, fd_arr, 0xff); 1093 } else { 1094 /* In case of indirect mbuf, and no cloning, core mbuf 1095 * should be released by BMAN. 1096 * Increate refcnt of core mbuf so that when 1097 * pktmbuf_free is called and mbuf is released, EAL 1098 * doesn't try to release core mbuf which would have 1099 * been released by BMAN. 1100 */ 1101 rte_mbuf_refcnt_update(mi, 1); 1102 DPAA_MBUF_TO_CONTIG_FD(mbuf, fd_arr, 1103 bp_info ? bp_info->bpid : 0xff); 1104 } 1105 buf_to_free[*free_count].seg = mbuf; 1106 buf_to_free[*free_count].pkt_id = pkt_id; 1107 ++*free_count; 1108 } 1109 1110 if (mbuf->ol_flags & DPAA_TX_CKSUM_OFFLOAD_MASK) 1111 dpaa_unsegmented_checksum(mbuf, fd_arr); 1112 } 1113 1114 /* Handle all mbufs on dpaa BMAN managed pool */ 1115 static inline uint16_t 1116 tx_on_dpaa_pool(struct rte_mbuf *mbuf, 1117 struct dpaa_bp_info *bp_info, 1118 struct qm_fd *fd_arr, 1119 struct dpaa_sw_buf_free *buf_to_free, 1120 uint32_t *free_count, 1121 uint32_t pkt_id) 1122 { 1123 DPAA_DP_LOG(DEBUG, "BMAN offloaded buffer, mbuf: %p", mbuf); 1124 1125 if (mbuf->nb_segs == 1) { 1126 /* Case for non-segmented buffers */ 1127 tx_on_dpaa_pool_unsegmented(mbuf, bp_info, fd_arr, 1128 buf_to_free, free_count, pkt_id); 1129 } else if (mbuf->nb_segs > 1 && 1130 mbuf->nb_segs <= DPAA_SGT_MAX_ENTRIES) { 1131 if (dpaa_eth_mbuf_to_sg_fd(mbuf, fd_arr, buf_to_free, 1132 free_count, pkt_id)) { 1133 DPAA_PMD_DEBUG("Unable to create Scatter Gather FD"); 1134 return 1; 1135 } 1136 } else { 1137 DPAA_PMD_DEBUG("Number of Segments not supported"); 1138 return 1; 1139 } 1140 1141 return 0; 1142 } 1143 1144 /* Handle all mbufs on an external pool (non-dpaa) */ 1145 static inline struct rte_mbuf * 1146 reallocate_mbuf(struct qman_fq *txq, struct rte_mbuf *mbuf) 1147 { 1148 struct dpaa_if *dpaa_intf = txq->dpaa_intf; 1149 struct dpaa_bp_info *bp_info = dpaa_intf->bp_info; 1150 struct rte_mbuf *new_mbufs[DPAA_SGT_MAX_ENTRIES + 1] = {0}; 1151 struct rte_mbuf *temp_mbuf; 1152 int num_new_segs, mbuf_greater, ret, extra_seg = 0, i = 0; 1153 uint64_t mbufs_size, bytes_to_copy, offset1 = 0, offset2 = 0; 1154 char *data; 1155 1156 DPAA_DP_LOG(DEBUG, "Reallocating transmit buffer"); 1157 1158 mbufs_size = bp_info->size - 1159 bp_info->meta_data_size - RTE_PKTMBUF_HEADROOM; 1160 extra_seg = !!(mbuf->pkt_len % mbufs_size); 1161 num_new_segs = (mbuf->pkt_len / mbufs_size) + extra_seg; 1162 1163 ret = rte_pktmbuf_alloc_bulk(bp_info->mp, new_mbufs, num_new_segs); 1164 if (ret != 0) { 1165 DPAA_DP_LOG(DEBUG, "Allocation for new buffers failed"); 1166 return NULL; 1167 } 1168 1169 temp_mbuf = mbuf; 1170 1171 while (temp_mbuf) { 1172 /* If mbuf data is less than new mbuf remaining memory */ 1173 if ((temp_mbuf->data_len - offset1) < (mbufs_size - offset2)) { 1174 bytes_to_copy = temp_mbuf->data_len - offset1; 1175 mbuf_greater = -1; 1176 /* If mbuf data is greater than new mbuf remaining memory */ 1177 } else if ((temp_mbuf->data_len - offset1) > 1178 (mbufs_size - offset2)) { 1179 bytes_to_copy = mbufs_size - offset2; 1180 mbuf_greater = 1; 1181 /* if mbuf data is equal to new mbuf remaining memory */ 1182 } else { 1183 bytes_to_copy = temp_mbuf->data_len - offset1; 1184 mbuf_greater = 0; 1185 } 1186 1187 /* Copy the data */ 1188 data = rte_pktmbuf_append(new_mbufs[0], bytes_to_copy); 1189 1190 rte_memcpy((uint8_t *)data, rte_pktmbuf_mtod_offset(mbuf, 1191 void *, offset1), bytes_to_copy); 1192 1193 /* Set new offsets and the temp buffers */ 1194 if (mbuf_greater == -1) { 1195 offset1 = 0; 1196 offset2 += bytes_to_copy; 1197 temp_mbuf = temp_mbuf->next; 1198 } else if (mbuf_greater == 1) { 1199 offset2 = 0; 1200 offset1 += bytes_to_copy; 1201 new_mbufs[i]->next = new_mbufs[i + 1]; 1202 new_mbufs[0]->nb_segs++; 1203 i++; 1204 } else { 1205 offset1 = 0; 1206 offset2 = 0; 1207 temp_mbuf = temp_mbuf->next; 1208 new_mbufs[i]->next = new_mbufs[i + 1]; 1209 if (new_mbufs[i + 1]) 1210 new_mbufs[0]->nb_segs++; 1211 i++; 1212 } 1213 } 1214 1215 /* Copy other required fields */ 1216 new_mbufs[0]->ol_flags = mbuf->ol_flags; 1217 new_mbufs[0]->packet_type = mbuf->packet_type; 1218 new_mbufs[0]->tx_offload = mbuf->tx_offload; 1219 1220 rte_pktmbuf_free(mbuf); 1221 1222 return new_mbufs[0]; 1223 } 1224 1225 uint16_t 1226 dpaa_eth_queue_tx(void *q, struct rte_mbuf **bufs, uint16_t nb_bufs) 1227 { 1228 struct rte_mbuf *mbuf, *mi = NULL; 1229 struct rte_mempool *mp; 1230 struct dpaa_bp_info *bp_info; 1231 struct qm_fd fd_arr[DPAA_TX_BURST_SIZE]; 1232 uint32_t frames_to_send, loop, sent = 0; 1233 uint16_t state; 1234 int ret, realloc_mbuf = 0; 1235 uint32_t seqn, index, flags[DPAA_TX_BURST_SIZE] = {0}; 1236 struct dpaa_sw_buf_free buf_to_free[DPAA_MAX_SGS * DPAA_MAX_DEQUEUE_NUM_FRAMES]; 1237 uint32_t free_count = 0; 1238 struct qman_fq *fq = q; 1239 struct dpaa_if *dpaa_intf = fq->dpaa_intf; 1240 struct qman_fq *fq_txconf = fq->tx_conf_queue; 1241 1242 if (unlikely(!DPAA_PER_LCORE_PORTAL)) { 1243 ret = rte_dpaa_portal_init((void *)0); 1244 if (ret) { 1245 DPAA_PMD_ERR("Failure in affining portal"); 1246 return 0; 1247 } 1248 } 1249 1250 DPAA_DP_LOG(DEBUG, "Transmitting %d buffers on queue: %p", nb_bufs, q); 1251 1252 if (dpaa_ieee_1588) { 1253 dpaa_intf->next_tx_conf_queue = fq_txconf; 1254 dpaa_eth_tx_conf(fq_txconf); 1255 dpaa_intf->tx_timestamp = 0; 1256 } 1257 1258 while (nb_bufs) { 1259 frames_to_send = (nb_bufs > DPAA_TX_BURST_SIZE) ? 1260 DPAA_TX_BURST_SIZE : nb_bufs; 1261 for (loop = 0; loop < frames_to_send; loop++) { 1262 mbuf = *(bufs++); 1263 /* In case the data offset is not multiple of 16, 1264 * FMAN can stall because of an errata. So reallocate 1265 * the buffer in such case. 1266 */ 1267 if (dpaa_svr_family == SVR_LS1043A_FAMILY && 1268 (mbuf->data_off & 0x7F) != 0x0) 1269 realloc_mbuf = 1; 1270 1271 fd_arr[loop].cmd = 0; 1272 if (dpaa_ieee_1588) { 1273 fd_arr[loop].cmd |= DPAA_FD_CMD_FCO | 1274 qman_fq_fqid(fq_txconf); 1275 fd_arr[loop].cmd |= DPAA_FD_CMD_RPD | 1276 DPAA_FD_CMD_UPD; 1277 } 1278 seqn = *dpaa_seqn(mbuf); 1279 if (seqn != DPAA_INVALID_MBUF_SEQN) { 1280 index = seqn - 1; 1281 if (DPAA_PER_LCORE_DQRR_HELD & (1 << index)) { 1282 flags[loop] = 1283 ((index & QM_EQCR_DCA_IDXMASK) << 8); 1284 flags[loop] |= QMAN_ENQUEUE_FLAG_DCA; 1285 DPAA_PER_LCORE_DQRR_SIZE--; 1286 DPAA_PER_LCORE_DQRR_HELD &= 1287 ~(1 << index); 1288 } 1289 } 1290 1291 if (likely(RTE_MBUF_DIRECT(mbuf))) { 1292 mp = mbuf->pool; 1293 bp_info = DPAA_MEMPOOL_TO_POOL_INFO(mp); 1294 if (likely(mp->ops_index == 1295 bp_info->dpaa_ops_index && 1296 mbuf->nb_segs == 1 && 1297 realloc_mbuf == 0 && 1298 rte_mbuf_refcnt_read(mbuf) == 1)) { 1299 DPAA_MBUF_TO_CONTIG_FD(mbuf, 1300 &fd_arr[loop], bp_info->bpid); 1301 if (mbuf->ol_flags & 1302 DPAA_TX_CKSUM_OFFLOAD_MASK) 1303 dpaa_unsegmented_checksum(mbuf, 1304 &fd_arr[loop]); 1305 continue; 1306 } 1307 } else { 1308 mi = rte_mbuf_from_indirect(mbuf); 1309 mp = mi->pool; 1310 } 1311 1312 if (unlikely(RTE_MBUF_HAS_EXTBUF(mbuf))) { 1313 bp_info = NULL; 1314 goto indirect_buf; 1315 } 1316 1317 bp_info = DPAA_MEMPOOL_TO_POOL_INFO(mp); 1318 if (unlikely(mp->ops_index != bp_info->dpaa_ops_index || 1319 realloc_mbuf == 1)) { 1320 struct rte_mbuf *temp_mbuf; 1321 1322 temp_mbuf = reallocate_mbuf(q, mbuf); 1323 if (!temp_mbuf) { 1324 /* Set frames_to_send & nb_bufs so 1325 * that packets are transmitted till 1326 * previous frame. 1327 */ 1328 frames_to_send = loop; 1329 nb_bufs = loop; 1330 goto send_pkts; 1331 } 1332 mbuf = temp_mbuf; 1333 realloc_mbuf = 0; 1334 } 1335 indirect_buf: 1336 state = tx_on_dpaa_pool(mbuf, bp_info, 1337 &fd_arr[loop], 1338 buf_to_free, 1339 &free_count, 1340 loop); 1341 if (unlikely(state)) { 1342 /* Set frames_to_send & nb_bufs so 1343 * that packets are transmitted till 1344 * previous frame. 1345 */ 1346 frames_to_send = loop; 1347 nb_bufs = loop; 1348 goto send_pkts; 1349 } 1350 } 1351 1352 send_pkts: 1353 loop = 0; 1354 while (loop < frames_to_send) { 1355 loop += qman_enqueue_multi(q, &fd_arr[loop], 1356 &flags[loop], 1357 frames_to_send - loop); 1358 } 1359 nb_bufs -= frames_to_send; 1360 sent += frames_to_send; 1361 } 1362 1363 DPAA_DP_LOG(DEBUG, "Transmitted %d buffers on queue: %p", sent, q); 1364 1365 for (loop = 0; loop < free_count; loop++) { 1366 if (buf_to_free[loop].pkt_id < sent) 1367 rte_pktmbuf_free_seg(buf_to_free[loop].seg); 1368 } 1369 1370 return sent; 1371 } 1372 1373 void 1374 dpaa_eth_tx_conf(void *q) 1375 { 1376 struct qman_fq *fq = q; 1377 struct qm_dqrr_entry *dq; 1378 int num_tx_conf, ret, dq_num; 1379 uint32_t vdqcr_flags = 0; 1380 struct dpaa_if *dpaa_intf = fq->dpaa_intf; 1381 struct qm_dqrr_entry *dqrr; 1382 struct dpaa_bp_info *bp_info; 1383 struct rte_mbuf *mbuf; 1384 void *ptr; 1385 struct annotations_t *annot; 1386 1387 if (unlikely(rte_dpaa_bpid_info == NULL && 1388 rte_eal_process_type() == RTE_PROC_SECONDARY)) 1389 rte_dpaa_bpid_info = fq->bp_array; 1390 1391 if (unlikely(!DPAA_PER_LCORE_PORTAL)) { 1392 ret = rte_dpaa_portal_init((void *)0); 1393 if (ret) { 1394 DPAA_PMD_ERR("Failure in affining portal"); 1395 return; 1396 } 1397 } 1398 1399 num_tx_conf = DPAA_MAX_DEQUEUE_NUM_FRAMES - 2; 1400 1401 do { 1402 dq_num = 0; 1403 ret = qman_set_vdq(fq, num_tx_conf, vdqcr_flags); 1404 if (ret) 1405 return; 1406 do { 1407 dq = qman_dequeue(fq); 1408 if (!dq) 1409 continue; 1410 dqrr = dq; 1411 dq_num++; 1412 bp_info = DPAA_BPID_TO_POOL_INFO(dqrr->fd.bpid); 1413 ptr = rte_dpaa_mem_ptov(qm_fd_addr(&dqrr->fd)); 1414 rte_prefetch0((void *)((uint8_t *)ptr 1415 + DEFAULT_RX_ICEOF)); 1416 mbuf = (struct rte_mbuf *) 1417 ((char *)ptr - bp_info->meta_data_size); 1418 1419 if (mbuf->ol_flags & RTE_MBUF_F_TX_IEEE1588_TMST) { 1420 annot = GET_ANNOTATIONS(mbuf->buf_addr); 1421 dpaa_intf->tx_timestamp = 1422 rte_cpu_to_be_64(annot->timestamp); 1423 } 1424 dpaa_display_frame_info(&dq->fd, fq->fqid, true); 1425 qman_dqrr_consume(fq, dq); 1426 dpaa_free_mbuf(&dq->fd); 1427 } while (fq->flags & QMAN_FQ_STATE_VDQCR); 1428 } while (dq_num == num_tx_conf); 1429 } 1430 1431 uint16_t 1432 dpaa_eth_queue_tx_slow(void *q, struct rte_mbuf **bufs, uint16_t nb_bufs) 1433 { 1434 qman_ern_poll_free(); 1435 1436 return dpaa_eth_queue_tx(q, bufs, nb_bufs); 1437 } 1438 1439 uint16_t dpaa_eth_tx_drop_all(void *q __rte_unused, 1440 struct rte_mbuf **bufs __rte_unused, 1441 uint16_t nb_bufs __rte_unused) 1442 { 1443 DPAA_DP_LOG(DEBUG, "Drop all packets"); 1444 1445 /* Drop all incoming packets. No need to free packets here 1446 * because the rte_eth f/w frees up the packets through tx_buffer 1447 * callback in case this functions returns count less than nb_bufs 1448 */ 1449 return 0; 1450 } 1451