1 /* SPDX-License-Identifier: BSD-3-Clause 2 * 3 * Copyright 2016 Freescale Semiconductor, Inc. All rights reserved. 4 * Copyright 2017,2019 NXP 5 * 6 */ 7 8 /* System headers */ 9 #include <inttypes.h> 10 #include <unistd.h> 11 #include <stdio.h> 12 #include <limits.h> 13 #include <sched.h> 14 #include <pthread.h> 15 16 #include <rte_byteorder.h> 17 #include <rte_common.h> 18 #include <rte_interrupts.h> 19 #include <rte_log.h> 20 #include <rte_debug.h> 21 #include <rte_pci.h> 22 #include <rte_atomic.h> 23 #include <rte_branch_prediction.h> 24 #include <rte_memory.h> 25 #include <rte_tailq.h> 26 #include <rte_eal.h> 27 #include <rte_alarm.h> 28 #include <rte_ether.h> 29 #include <rte_ethdev_driver.h> 30 #include <rte_malloc.h> 31 #include <rte_ring.h> 32 #include <rte_ip.h> 33 #include <rte_tcp.h> 34 #include <rte_udp.h> 35 #include <rte_net.h> 36 #include <rte_eventdev.h> 37 38 #include "dpaa_ethdev.h" 39 #include "dpaa_rxtx.h" 40 #include <rte_dpaa_bus.h> 41 #include <dpaa_mempool.h> 42 43 #include <qman.h> 44 #include <fsl_usd.h> 45 #include <fsl_qman.h> 46 #include <fsl_bman.h> 47 #include <of.h> 48 #include <netcfg.h> 49 50 #define DPAA_MBUF_TO_CONTIG_FD(_mbuf, _fd, _bpid) \ 51 do { \ 52 (_fd)->cmd = 0; \ 53 (_fd)->opaque_addr = 0; \ 54 (_fd)->opaque = QM_FD_CONTIG << DPAA_FD_FORMAT_SHIFT; \ 55 (_fd)->opaque |= ((_mbuf)->data_off) << DPAA_FD_OFFSET_SHIFT; \ 56 (_fd)->opaque |= (_mbuf)->pkt_len; \ 57 (_fd)->addr = (_mbuf)->buf_iova; \ 58 (_fd)->bpid = _bpid; \ 59 } while (0) 60 61 #if (defined RTE_LIBRTE_DPAA_DEBUG_DRIVER) 62 static void dpaa_display_frame(const struct qm_fd *fd) 63 { 64 int ii; 65 char *ptr; 66 67 printf("%s::bpid %x addr %08x%08x, format %d off %d, len %d stat %x\n", 68 __func__, fd->bpid, fd->addr_hi, fd->addr_lo, fd->format, 69 fd->offset, fd->length20, fd->status); 70 71 ptr = (char *)rte_dpaa_mem_ptov(fd->addr); 72 ptr += fd->offset; 73 printf("%02x ", *ptr); 74 for (ii = 1; ii < fd->length20; ii++) { 75 printf("%02x ", *ptr); 76 if ((ii % 16) == 0) 77 printf("\n"); 78 ptr++; 79 } 80 printf("\n"); 81 } 82 #else 83 #define dpaa_display_frame(a) 84 #endif 85 86 static inline void dpaa_slow_parsing(struct rte_mbuf *m __rte_unused, 87 uint64_t prs __rte_unused) 88 { 89 DPAA_DP_LOG(DEBUG, "Slow parsing"); 90 /*TBD:XXX: to be implemented*/ 91 } 92 93 static inline void dpaa_eth_packet_info(struct rte_mbuf *m, void *fd_virt_addr) 94 { 95 struct annotations_t *annot = GET_ANNOTATIONS(fd_virt_addr); 96 uint64_t prs = *((uintptr_t *)(&annot->parse)) & DPAA_PARSE_MASK; 97 98 DPAA_DP_LOG(DEBUG, " Parsing mbuf: %p with annotations: %p", m, annot); 99 100 switch (prs) { 101 case DPAA_PKT_TYPE_IPV4: 102 m->packet_type = RTE_PTYPE_L2_ETHER | 103 RTE_PTYPE_L3_IPV4; 104 break; 105 case DPAA_PKT_TYPE_IPV6: 106 m->packet_type = RTE_PTYPE_L2_ETHER | 107 RTE_PTYPE_L3_IPV6; 108 break; 109 case DPAA_PKT_TYPE_ETHER: 110 m->packet_type = RTE_PTYPE_L2_ETHER; 111 break; 112 case DPAA_PKT_TYPE_IPV4_FRAG: 113 case DPAA_PKT_TYPE_IPV4_FRAG_UDP: 114 case DPAA_PKT_TYPE_IPV4_FRAG_TCP: 115 case DPAA_PKT_TYPE_IPV4_FRAG_SCTP: 116 m->packet_type = RTE_PTYPE_L2_ETHER | 117 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_FRAG; 118 break; 119 case DPAA_PKT_TYPE_IPV6_FRAG: 120 case DPAA_PKT_TYPE_IPV6_FRAG_UDP: 121 case DPAA_PKT_TYPE_IPV6_FRAG_TCP: 122 case DPAA_PKT_TYPE_IPV6_FRAG_SCTP: 123 m->packet_type = RTE_PTYPE_L2_ETHER | 124 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_FRAG; 125 break; 126 case DPAA_PKT_TYPE_IPV4_EXT: 127 m->packet_type = RTE_PTYPE_L2_ETHER | 128 RTE_PTYPE_L3_IPV4_EXT; 129 break; 130 case DPAA_PKT_TYPE_IPV6_EXT: 131 m->packet_type = RTE_PTYPE_L2_ETHER | 132 RTE_PTYPE_L3_IPV6_EXT; 133 break; 134 case DPAA_PKT_TYPE_IPV4_TCP: 135 m->packet_type = RTE_PTYPE_L2_ETHER | 136 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_TCP; 137 break; 138 case DPAA_PKT_TYPE_IPV6_TCP: 139 m->packet_type = RTE_PTYPE_L2_ETHER | 140 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_TCP; 141 break; 142 case DPAA_PKT_TYPE_IPV4_UDP: 143 m->packet_type = RTE_PTYPE_L2_ETHER | 144 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_UDP; 145 break; 146 case DPAA_PKT_TYPE_IPV6_UDP: 147 m->packet_type = RTE_PTYPE_L2_ETHER | 148 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_UDP; 149 break; 150 case DPAA_PKT_TYPE_IPV4_EXT_UDP: 151 m->packet_type = RTE_PTYPE_L2_ETHER | 152 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_UDP; 153 break; 154 case DPAA_PKT_TYPE_IPV6_EXT_UDP: 155 m->packet_type = RTE_PTYPE_L2_ETHER | 156 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_UDP; 157 break; 158 case DPAA_PKT_TYPE_IPV4_EXT_TCP: 159 m->packet_type = RTE_PTYPE_L2_ETHER | 160 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_TCP; 161 break; 162 case DPAA_PKT_TYPE_IPV6_EXT_TCP: 163 m->packet_type = RTE_PTYPE_L2_ETHER | 164 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_TCP; 165 break; 166 case DPAA_PKT_TYPE_IPV4_SCTP: 167 m->packet_type = RTE_PTYPE_L2_ETHER | 168 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_SCTP; 169 break; 170 case DPAA_PKT_TYPE_IPV6_SCTP: 171 m->packet_type = RTE_PTYPE_L2_ETHER | 172 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_SCTP; 173 break; 174 case DPAA_PKT_TYPE_NONE: 175 m->packet_type = 0; 176 break; 177 /* More switch cases can be added */ 178 default: 179 dpaa_slow_parsing(m, prs); 180 } 181 182 m->tx_offload = annot->parse.ip_off[0]; 183 m->tx_offload |= (annot->parse.l4_off - annot->parse.ip_off[0]) 184 << DPAA_PKT_L3_LEN_SHIFT; 185 186 /* Set the hash values */ 187 m->hash.rss = (uint32_t)(annot->hash); 188 /* All packets with Bad checksum are dropped by interface (and 189 * corresponding notification issued to RX error queues). 190 */ 191 m->ol_flags = PKT_RX_RSS_HASH | PKT_RX_IP_CKSUM_GOOD; 192 193 /* Check if Vlan is present */ 194 if (prs & DPAA_PARSE_VLAN_MASK) 195 m->ol_flags |= PKT_RX_VLAN; 196 /* Packet received without stripping the vlan */ 197 } 198 199 static inline void dpaa_checksum(struct rte_mbuf *mbuf) 200 { 201 struct ether_hdr *eth_hdr = rte_pktmbuf_mtod(mbuf, struct ether_hdr *); 202 char *l3_hdr = (char *)eth_hdr + mbuf->l2_len; 203 struct ipv4_hdr *ipv4_hdr = (struct ipv4_hdr *)l3_hdr; 204 struct ipv6_hdr *ipv6_hdr = (struct ipv6_hdr *)l3_hdr; 205 206 DPAA_DP_LOG(DEBUG, "Calculating checksum for mbuf: %p", mbuf); 207 208 if (((mbuf->packet_type & RTE_PTYPE_L3_MASK) == RTE_PTYPE_L3_IPV4) || 209 ((mbuf->packet_type & RTE_PTYPE_L3_MASK) == 210 RTE_PTYPE_L3_IPV4_EXT)) { 211 ipv4_hdr = (struct ipv4_hdr *)l3_hdr; 212 ipv4_hdr->hdr_checksum = 0; 213 ipv4_hdr->hdr_checksum = rte_ipv4_cksum(ipv4_hdr); 214 } else if (((mbuf->packet_type & RTE_PTYPE_L3_MASK) == 215 RTE_PTYPE_L3_IPV6) || 216 ((mbuf->packet_type & RTE_PTYPE_L3_MASK) == 217 RTE_PTYPE_L3_IPV6_EXT)) 218 ipv6_hdr = (struct ipv6_hdr *)l3_hdr; 219 220 if ((mbuf->packet_type & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_TCP) { 221 struct tcp_hdr *tcp_hdr = (struct tcp_hdr *)(l3_hdr + 222 mbuf->l3_len); 223 tcp_hdr->cksum = 0; 224 if (eth_hdr->ether_type == htons(ETHER_TYPE_IPv4)) 225 tcp_hdr->cksum = rte_ipv4_udptcp_cksum(ipv4_hdr, 226 tcp_hdr); 227 else /* assume ethertype == ETHER_TYPE_IPv6 */ 228 tcp_hdr->cksum = rte_ipv6_udptcp_cksum(ipv6_hdr, 229 tcp_hdr); 230 } else if ((mbuf->packet_type & RTE_PTYPE_L4_MASK) == 231 RTE_PTYPE_L4_UDP) { 232 struct udp_hdr *udp_hdr = (struct udp_hdr *)(l3_hdr + 233 mbuf->l3_len); 234 udp_hdr->dgram_cksum = 0; 235 if (eth_hdr->ether_type == htons(ETHER_TYPE_IPv4)) 236 udp_hdr->dgram_cksum = rte_ipv4_udptcp_cksum(ipv4_hdr, 237 udp_hdr); 238 else /* assume ethertype == ETHER_TYPE_IPv6 */ 239 udp_hdr->dgram_cksum = rte_ipv6_udptcp_cksum(ipv6_hdr, 240 udp_hdr); 241 } 242 } 243 244 static inline void dpaa_checksum_offload(struct rte_mbuf *mbuf, 245 struct qm_fd *fd, char *prs_buf) 246 { 247 struct dpaa_eth_parse_results_t *prs; 248 249 DPAA_DP_LOG(DEBUG, " Offloading checksum for mbuf: %p", mbuf); 250 251 prs = GET_TX_PRS(prs_buf); 252 prs->l3r = 0; 253 prs->l4r = 0; 254 if (((mbuf->packet_type & RTE_PTYPE_L3_MASK) == RTE_PTYPE_L3_IPV4) || 255 ((mbuf->packet_type & RTE_PTYPE_L3_MASK) == 256 RTE_PTYPE_L3_IPV4_EXT)) 257 prs->l3r = DPAA_L3_PARSE_RESULT_IPV4; 258 else if (((mbuf->packet_type & RTE_PTYPE_L3_MASK) == 259 RTE_PTYPE_L3_IPV6) || 260 ((mbuf->packet_type & RTE_PTYPE_L3_MASK) == 261 RTE_PTYPE_L3_IPV6_EXT)) 262 prs->l3r = DPAA_L3_PARSE_RESULT_IPV6; 263 264 if ((mbuf->packet_type & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_TCP) 265 prs->l4r = DPAA_L4_PARSE_RESULT_TCP; 266 else if ((mbuf->packet_type & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_UDP) 267 prs->l4r = DPAA_L4_PARSE_RESULT_UDP; 268 269 prs->ip_off[0] = mbuf->l2_len; 270 prs->l4_off = mbuf->l3_len + mbuf->l2_len; 271 /* Enable L3 (and L4, if TCP or UDP) HW checksum*/ 272 fd->cmd = DPAA_FD_CMD_RPD | DPAA_FD_CMD_DTC; 273 } 274 275 static inline void 276 dpaa_unsegmented_checksum(struct rte_mbuf *mbuf, struct qm_fd *fd_arr) 277 { 278 if (!mbuf->packet_type) { 279 struct rte_net_hdr_lens hdr_lens; 280 281 mbuf->packet_type = rte_net_get_ptype(mbuf, &hdr_lens, 282 RTE_PTYPE_L2_MASK | RTE_PTYPE_L3_MASK 283 | RTE_PTYPE_L4_MASK); 284 mbuf->l2_len = hdr_lens.l2_len; 285 mbuf->l3_len = hdr_lens.l3_len; 286 } 287 if (mbuf->data_off < (DEFAULT_TX_ICEOF + 288 sizeof(struct dpaa_eth_parse_results_t))) { 289 DPAA_DP_LOG(DEBUG, "Checksum offload Err: " 290 "Not enough Headroom " 291 "space for correct Checksum offload." 292 "So Calculating checksum in Software."); 293 dpaa_checksum(mbuf); 294 } else { 295 dpaa_checksum_offload(mbuf, fd_arr, mbuf->buf_addr); 296 } 297 } 298 299 struct rte_mbuf * 300 dpaa_eth_sg_to_mbuf(const struct qm_fd *fd, uint32_t ifid) 301 { 302 struct dpaa_bp_info *bp_info = DPAA_BPID_TO_POOL_INFO(fd->bpid); 303 struct rte_mbuf *first_seg, *prev_seg, *cur_seg, *temp; 304 struct qm_sg_entry *sgt, *sg_temp; 305 void *vaddr, *sg_vaddr; 306 int i = 0; 307 uint8_t fd_offset = fd->offset; 308 309 vaddr = DPAA_MEMPOOL_PTOV(bp_info, qm_fd_addr(fd)); 310 if (!vaddr) { 311 DPAA_PMD_ERR("unable to convert physical address"); 312 return NULL; 313 } 314 sgt = vaddr + fd_offset; 315 sg_temp = &sgt[i++]; 316 hw_sg_to_cpu(sg_temp); 317 temp = (struct rte_mbuf *)((char *)vaddr - bp_info->meta_data_size); 318 sg_vaddr = DPAA_MEMPOOL_PTOV(bp_info, qm_sg_entry_get64(sg_temp)); 319 320 first_seg = (struct rte_mbuf *)((char *)sg_vaddr - 321 bp_info->meta_data_size); 322 first_seg->data_off = sg_temp->offset; 323 first_seg->data_len = sg_temp->length; 324 first_seg->pkt_len = sg_temp->length; 325 rte_mbuf_refcnt_set(first_seg, 1); 326 327 first_seg->port = ifid; 328 first_seg->nb_segs = 1; 329 first_seg->ol_flags = 0; 330 prev_seg = first_seg; 331 while (i < DPAA_SGT_MAX_ENTRIES) { 332 sg_temp = &sgt[i++]; 333 hw_sg_to_cpu(sg_temp); 334 sg_vaddr = DPAA_MEMPOOL_PTOV(bp_info, 335 qm_sg_entry_get64(sg_temp)); 336 cur_seg = (struct rte_mbuf *)((char *)sg_vaddr - 337 bp_info->meta_data_size); 338 cur_seg->data_off = sg_temp->offset; 339 cur_seg->data_len = sg_temp->length; 340 first_seg->pkt_len += sg_temp->length; 341 first_seg->nb_segs += 1; 342 rte_mbuf_refcnt_set(cur_seg, 1); 343 prev_seg->next = cur_seg; 344 if (sg_temp->final) { 345 cur_seg->next = NULL; 346 break; 347 } 348 prev_seg = cur_seg; 349 } 350 DPAA_DP_LOG(DEBUG, "Received an SG frame len =%d, num_sg =%d", 351 first_seg->pkt_len, first_seg->nb_segs); 352 353 dpaa_eth_packet_info(first_seg, vaddr); 354 rte_pktmbuf_free_seg(temp); 355 356 return first_seg; 357 } 358 359 static inline struct rte_mbuf * 360 dpaa_eth_fd_to_mbuf(const struct qm_fd *fd, uint32_t ifid) 361 { 362 struct rte_mbuf *mbuf; 363 struct dpaa_bp_info *bp_info = DPAA_BPID_TO_POOL_INFO(fd->bpid); 364 void *ptr; 365 uint8_t format = 366 (fd->opaque & DPAA_FD_FORMAT_MASK) >> DPAA_FD_FORMAT_SHIFT; 367 uint16_t offset; 368 uint32_t length; 369 370 if (unlikely(format == qm_fd_sg)) 371 return dpaa_eth_sg_to_mbuf(fd, ifid); 372 373 offset = (fd->opaque & DPAA_FD_OFFSET_MASK) >> DPAA_FD_OFFSET_SHIFT; 374 length = fd->opaque & DPAA_FD_LENGTH_MASK; 375 376 DPAA_DP_LOG(DEBUG, " FD--->MBUF off %d len = %d", offset, length); 377 378 /* Ignoring case when format != qm_fd_contig */ 379 dpaa_display_frame(fd); 380 ptr = DPAA_MEMPOOL_PTOV(bp_info, qm_fd_addr(fd)); 381 382 mbuf = (struct rte_mbuf *)((char *)ptr - bp_info->meta_data_size); 383 /* Prefetch the Parse results and packet data to L1 */ 384 rte_prefetch0((void *)((uint8_t *)ptr + DEFAULT_RX_ICEOF)); 385 386 mbuf->data_off = offset; 387 mbuf->data_len = length; 388 mbuf->pkt_len = length; 389 390 mbuf->port = ifid; 391 mbuf->nb_segs = 1; 392 mbuf->ol_flags = 0; 393 mbuf->next = NULL; 394 rte_mbuf_refcnt_set(mbuf, 1); 395 dpaa_eth_packet_info(mbuf, mbuf->buf_addr); 396 397 return mbuf; 398 } 399 400 /* Specific for LS1043 */ 401 void 402 dpaa_rx_cb_no_prefetch(struct qman_fq **fq, struct qm_dqrr_entry **dqrr, 403 void **bufs, int num_bufs) 404 { 405 struct rte_mbuf *mbuf; 406 struct dpaa_bp_info *bp_info; 407 const struct qm_fd *fd; 408 void *ptr; 409 struct dpaa_if *dpaa_intf; 410 uint16_t offset, i; 411 uint32_t length; 412 uint8_t format; 413 414 bp_info = DPAA_BPID_TO_POOL_INFO(dqrr[0]->fd.bpid); 415 ptr = rte_dpaa_mem_ptov(qm_fd_addr(&dqrr[0]->fd)); 416 rte_prefetch0((void *)((uint8_t *)ptr + DEFAULT_RX_ICEOF)); 417 bufs[0] = (struct rte_mbuf *)((char *)ptr - bp_info->meta_data_size); 418 419 for (i = 0; i < num_bufs; i++) { 420 if (i < num_bufs - 1) { 421 bp_info = DPAA_BPID_TO_POOL_INFO(dqrr[i + 1]->fd.bpid); 422 ptr = rte_dpaa_mem_ptov(qm_fd_addr(&dqrr[i + 1]->fd)); 423 rte_prefetch0((void *)((uint8_t *)ptr + 424 DEFAULT_RX_ICEOF)); 425 bufs[i + 1] = (struct rte_mbuf *)((char *)ptr - 426 bp_info->meta_data_size); 427 } 428 429 fd = &dqrr[i]->fd; 430 dpaa_intf = fq[0]->dpaa_intf; 431 432 format = (fd->opaque & DPAA_FD_FORMAT_MASK) >> 433 DPAA_FD_FORMAT_SHIFT; 434 if (unlikely(format == qm_fd_sg)) { 435 bufs[i] = dpaa_eth_sg_to_mbuf(fd, dpaa_intf->ifid); 436 continue; 437 } 438 439 offset = (fd->opaque & DPAA_FD_OFFSET_MASK) >> 440 DPAA_FD_OFFSET_SHIFT; 441 length = fd->opaque & DPAA_FD_LENGTH_MASK; 442 443 mbuf = bufs[i]; 444 mbuf->data_off = offset; 445 mbuf->data_len = length; 446 mbuf->pkt_len = length; 447 mbuf->port = dpaa_intf->ifid; 448 449 mbuf->nb_segs = 1; 450 mbuf->ol_flags = 0; 451 mbuf->next = NULL; 452 rte_mbuf_refcnt_set(mbuf, 1); 453 dpaa_eth_packet_info(mbuf, mbuf->buf_addr); 454 } 455 } 456 457 void 458 dpaa_rx_cb(struct qman_fq **fq, struct qm_dqrr_entry **dqrr, 459 void **bufs, int num_bufs) 460 { 461 struct rte_mbuf *mbuf; 462 const struct qm_fd *fd; 463 struct dpaa_if *dpaa_intf; 464 uint16_t offset, i; 465 uint32_t length; 466 uint8_t format; 467 468 for (i = 0; i < num_bufs; i++) { 469 fd = &dqrr[i]->fd; 470 dpaa_intf = fq[0]->dpaa_intf; 471 472 format = (fd->opaque & DPAA_FD_FORMAT_MASK) >> 473 DPAA_FD_FORMAT_SHIFT; 474 if (unlikely(format == qm_fd_sg)) { 475 bufs[i] = dpaa_eth_sg_to_mbuf(fd, dpaa_intf->ifid); 476 continue; 477 } 478 479 offset = (fd->opaque & DPAA_FD_OFFSET_MASK) >> 480 DPAA_FD_OFFSET_SHIFT; 481 length = fd->opaque & DPAA_FD_LENGTH_MASK; 482 483 mbuf = bufs[i]; 484 mbuf->data_off = offset; 485 mbuf->data_len = length; 486 mbuf->pkt_len = length; 487 mbuf->port = dpaa_intf->ifid; 488 489 mbuf->nb_segs = 1; 490 mbuf->ol_flags = 0; 491 mbuf->next = NULL; 492 rte_mbuf_refcnt_set(mbuf, 1); 493 dpaa_eth_packet_info(mbuf, mbuf->buf_addr); 494 } 495 } 496 497 void dpaa_rx_cb_prepare(struct qm_dqrr_entry *dq, void **bufs) 498 { 499 struct dpaa_bp_info *bp_info = DPAA_BPID_TO_POOL_INFO(dq->fd.bpid); 500 void *ptr = rte_dpaa_mem_ptov(qm_fd_addr(&dq->fd)); 501 502 /* In case of LS1046, annotation stashing is disabled due to L2 cache 503 * being bottleneck in case of multicore scanario for this platform. 504 * So we prefetch the annoation beforehand, so that it is available 505 * in cache when accessed. 506 */ 507 rte_prefetch0((void *)((uint8_t *)ptr + DEFAULT_RX_ICEOF)); 508 509 *bufs = (struct rte_mbuf *)((char *)ptr - bp_info->meta_data_size); 510 } 511 512 static uint16_t 513 dpaa_eth_queue_portal_rx(struct qman_fq *fq, 514 struct rte_mbuf **bufs, 515 uint16_t nb_bufs) 516 { 517 int ret; 518 519 if (unlikely(fq->qp == NULL)) { 520 ret = rte_dpaa_portal_fq_init((void *)0, fq); 521 if (ret) { 522 DPAA_PMD_ERR("Failure in affining portal %d", ret); 523 return 0; 524 } 525 } 526 527 return qman_portal_poll_rx(nb_bufs, (void **)bufs, fq->qp); 528 } 529 530 enum qman_cb_dqrr_result 531 dpaa_rx_cb_parallel(void *event, 532 struct qman_portal *qm __always_unused, 533 struct qman_fq *fq, 534 const struct qm_dqrr_entry *dqrr, 535 void **bufs) 536 { 537 u32 ifid = ((struct dpaa_if *)fq->dpaa_intf)->ifid; 538 struct rte_mbuf *mbuf; 539 struct rte_event *ev = (struct rte_event *)event; 540 541 mbuf = dpaa_eth_fd_to_mbuf(&dqrr->fd, ifid); 542 ev->event_ptr = (void *)mbuf; 543 ev->flow_id = fq->ev.flow_id; 544 ev->sub_event_type = fq->ev.sub_event_type; 545 ev->event_type = RTE_EVENT_TYPE_ETHDEV; 546 ev->op = RTE_EVENT_OP_NEW; 547 ev->sched_type = fq->ev.sched_type; 548 ev->queue_id = fq->ev.queue_id; 549 ev->priority = fq->ev.priority; 550 ev->impl_opaque = (uint8_t)DPAA_INVALID_MBUF_SEQN; 551 mbuf->seqn = DPAA_INVALID_MBUF_SEQN; 552 *bufs = mbuf; 553 554 return qman_cb_dqrr_consume; 555 } 556 557 enum qman_cb_dqrr_result 558 dpaa_rx_cb_atomic(void *event, 559 struct qman_portal *qm __always_unused, 560 struct qman_fq *fq, 561 const struct qm_dqrr_entry *dqrr, 562 void **bufs) 563 { 564 u8 index; 565 u32 ifid = ((struct dpaa_if *)fq->dpaa_intf)->ifid; 566 struct rte_mbuf *mbuf; 567 struct rte_event *ev = (struct rte_event *)event; 568 569 mbuf = dpaa_eth_fd_to_mbuf(&dqrr->fd, ifid); 570 ev->event_ptr = (void *)mbuf; 571 ev->flow_id = fq->ev.flow_id; 572 ev->sub_event_type = fq->ev.sub_event_type; 573 ev->event_type = RTE_EVENT_TYPE_ETHDEV; 574 ev->op = RTE_EVENT_OP_NEW; 575 ev->sched_type = fq->ev.sched_type; 576 ev->queue_id = fq->ev.queue_id; 577 ev->priority = fq->ev.priority; 578 579 /* Save active dqrr entries */ 580 index = DQRR_PTR2IDX(dqrr); 581 DPAA_PER_LCORE_DQRR_SIZE++; 582 DPAA_PER_LCORE_DQRR_HELD |= 1 << index; 583 DPAA_PER_LCORE_DQRR_MBUF(index) = mbuf; 584 ev->impl_opaque = index + 1; 585 mbuf->seqn = (uint32_t)index + 1; 586 *bufs = mbuf; 587 588 return qman_cb_dqrr_defer; 589 } 590 591 uint16_t dpaa_eth_queue_rx(void *q, 592 struct rte_mbuf **bufs, 593 uint16_t nb_bufs) 594 { 595 struct qman_fq *fq = q; 596 struct qm_dqrr_entry *dq; 597 uint32_t num_rx = 0, ifid = ((struct dpaa_if *)fq->dpaa_intf)->ifid; 598 int num_rx_bufs, ret; 599 uint32_t vdqcr_flags = 0; 600 601 if (unlikely(rte_dpaa_bpid_info == NULL && 602 rte_eal_process_type() == RTE_PROC_SECONDARY)) 603 rte_dpaa_bpid_info = fq->bp_array; 604 605 if (likely(fq->is_static)) 606 return dpaa_eth_queue_portal_rx(fq, bufs, nb_bufs); 607 608 if (unlikely(!RTE_PER_LCORE(dpaa_io))) { 609 ret = rte_dpaa_portal_init((void *)0); 610 if (ret) { 611 DPAA_PMD_ERR("Failure in affining portal"); 612 return 0; 613 } 614 } 615 616 /* Until request for four buffers, we provide exact number of buffers. 617 * Otherwise we do not set the QM_VDQCR_EXACT flag. 618 * Not setting QM_VDQCR_EXACT flag can provide two more buffers than 619 * requested, so we request two less in this case. 620 */ 621 if (nb_bufs < 4) { 622 vdqcr_flags = QM_VDQCR_EXACT; 623 num_rx_bufs = nb_bufs; 624 } else { 625 num_rx_bufs = nb_bufs > DPAA_MAX_DEQUEUE_NUM_FRAMES ? 626 (DPAA_MAX_DEQUEUE_NUM_FRAMES - 2) : (nb_bufs - 2); 627 } 628 ret = qman_set_vdq(fq, num_rx_bufs, vdqcr_flags); 629 if (ret) 630 return 0; 631 632 do { 633 dq = qman_dequeue(fq); 634 if (!dq) 635 continue; 636 bufs[num_rx++] = dpaa_eth_fd_to_mbuf(&dq->fd, ifid); 637 qman_dqrr_consume(fq, dq); 638 } while (fq->flags & QMAN_FQ_STATE_VDQCR); 639 640 return num_rx; 641 } 642 643 static void *dpaa_get_pktbuf(struct dpaa_bp_info *bp_info) 644 { 645 int ret; 646 size_t buf = 0; 647 struct bm_buffer bufs; 648 649 ret = bman_acquire(bp_info->bp, &bufs, 1, 0); 650 if (ret <= 0) { 651 DPAA_PMD_WARN("Failed to allocate buffers %d", ret); 652 return (void *)buf; 653 } 654 655 DPAA_DP_LOG(DEBUG, "got buffer 0x%" PRIx64 " from pool %d", 656 (uint64_t)bufs.addr, bufs.bpid); 657 658 buf = (size_t)DPAA_MEMPOOL_PTOV(bp_info, bufs.addr) 659 - bp_info->meta_data_size; 660 if (!buf) 661 goto out; 662 663 out: 664 return (void *)buf; 665 } 666 667 static struct rte_mbuf *dpaa_get_dmable_mbuf(struct rte_mbuf *mbuf, 668 struct dpaa_if *dpaa_intf) 669 { 670 struct rte_mbuf *dpaa_mbuf; 671 672 /* allocate pktbuffer on bpid for dpaa port */ 673 dpaa_mbuf = dpaa_get_pktbuf(dpaa_intf->bp_info); 674 if (!dpaa_mbuf) 675 return NULL; 676 677 memcpy((uint8_t *)(dpaa_mbuf->buf_addr) + RTE_PKTMBUF_HEADROOM, (void *) 678 ((uint8_t *)(mbuf->buf_addr) + mbuf->data_off), mbuf->pkt_len); 679 680 /* Copy only the required fields */ 681 dpaa_mbuf->data_off = RTE_PKTMBUF_HEADROOM; 682 dpaa_mbuf->pkt_len = mbuf->pkt_len; 683 dpaa_mbuf->ol_flags = mbuf->ol_flags; 684 dpaa_mbuf->packet_type = mbuf->packet_type; 685 dpaa_mbuf->tx_offload = mbuf->tx_offload; 686 rte_pktmbuf_free(mbuf); 687 return dpaa_mbuf; 688 } 689 690 int 691 dpaa_eth_mbuf_to_sg_fd(struct rte_mbuf *mbuf, 692 struct qm_fd *fd, 693 uint32_t bpid) 694 { 695 struct rte_mbuf *cur_seg = mbuf, *prev_seg = NULL; 696 struct dpaa_bp_info *bp_info = DPAA_BPID_TO_POOL_INFO(bpid); 697 struct rte_mbuf *temp, *mi; 698 struct qm_sg_entry *sg_temp, *sgt; 699 int i = 0; 700 701 DPAA_DP_LOG(DEBUG, "Creating SG FD to transmit"); 702 703 temp = rte_pktmbuf_alloc(bp_info->mp); 704 if (!temp) { 705 DPAA_PMD_ERR("Failure in allocation of mbuf"); 706 return -1; 707 } 708 if (temp->buf_len < ((mbuf->nb_segs * sizeof(struct qm_sg_entry)) 709 + temp->data_off)) { 710 DPAA_PMD_ERR("Insufficient space in mbuf for SG entries"); 711 return -1; 712 } 713 714 fd->cmd = 0; 715 fd->opaque_addr = 0; 716 717 if (mbuf->ol_flags & DPAA_TX_CKSUM_OFFLOAD_MASK) { 718 if (!mbuf->packet_type) { 719 struct rte_net_hdr_lens hdr_lens; 720 721 mbuf->packet_type = rte_net_get_ptype(mbuf, &hdr_lens, 722 RTE_PTYPE_L2_MASK | RTE_PTYPE_L3_MASK 723 | RTE_PTYPE_L4_MASK); 724 mbuf->l2_len = hdr_lens.l2_len; 725 mbuf->l3_len = hdr_lens.l3_len; 726 } 727 if (temp->data_off < DEFAULT_TX_ICEOF 728 + sizeof(struct dpaa_eth_parse_results_t)) 729 temp->data_off = DEFAULT_TX_ICEOF 730 + sizeof(struct dpaa_eth_parse_results_t); 731 dcbz_64(temp->buf_addr); 732 dpaa_checksum_offload(mbuf, fd, temp->buf_addr); 733 } 734 735 sgt = temp->buf_addr + temp->data_off; 736 fd->format = QM_FD_SG; 737 fd->addr = temp->buf_iova; 738 fd->offset = temp->data_off; 739 fd->bpid = bpid; 740 fd->length20 = mbuf->pkt_len; 741 742 while (i < DPAA_SGT_MAX_ENTRIES) { 743 sg_temp = &sgt[i++]; 744 sg_temp->opaque = 0; 745 sg_temp->val = 0; 746 sg_temp->addr = cur_seg->buf_iova; 747 sg_temp->offset = cur_seg->data_off; 748 sg_temp->length = cur_seg->data_len; 749 if (RTE_MBUF_DIRECT(cur_seg)) { 750 if (rte_mbuf_refcnt_read(cur_seg) > 1) { 751 /*If refcnt > 1, invalid bpid is set to ensure 752 * buffer is not freed by HW. 753 */ 754 sg_temp->bpid = 0xff; 755 rte_mbuf_refcnt_update(cur_seg, -1); 756 } else { 757 sg_temp->bpid = 758 DPAA_MEMPOOL_TO_BPID(cur_seg->pool); 759 } 760 cur_seg = cur_seg->next; 761 } else { 762 /* Get owner MBUF from indirect buffer */ 763 mi = rte_mbuf_from_indirect(cur_seg); 764 if (rte_mbuf_refcnt_read(mi) > 1) { 765 /*If refcnt > 1, invalid bpid is set to ensure 766 * owner buffer is not freed by HW. 767 */ 768 sg_temp->bpid = 0xff; 769 } else { 770 sg_temp->bpid = DPAA_MEMPOOL_TO_BPID(mi->pool); 771 rte_mbuf_refcnt_update(mi, 1); 772 } 773 prev_seg = cur_seg; 774 cur_seg = cur_seg->next; 775 prev_seg->next = NULL; 776 rte_pktmbuf_free(prev_seg); 777 } 778 if (cur_seg == NULL) { 779 sg_temp->final = 1; 780 cpu_to_hw_sg(sg_temp); 781 break; 782 } 783 cpu_to_hw_sg(sg_temp); 784 } 785 return 0; 786 } 787 788 /* Handle mbufs which are not segmented (non SG) */ 789 static inline void 790 tx_on_dpaa_pool_unsegmented(struct rte_mbuf *mbuf, 791 struct dpaa_bp_info *bp_info, 792 struct qm_fd *fd_arr) 793 { 794 struct rte_mbuf *mi = NULL; 795 796 if (RTE_MBUF_DIRECT(mbuf)) { 797 if (rte_mbuf_refcnt_read(mbuf) > 1) { 798 /* In case of direct mbuf and mbuf being cloned, 799 * BMAN should _not_ release buffer. 800 */ 801 DPAA_MBUF_TO_CONTIG_FD(mbuf, fd_arr, 0xff); 802 /* Buffer should be releasd by EAL */ 803 rte_mbuf_refcnt_update(mbuf, -1); 804 } else { 805 /* In case of direct mbuf and no cloning, mbuf can be 806 * released by BMAN. 807 */ 808 DPAA_MBUF_TO_CONTIG_FD(mbuf, fd_arr, bp_info->bpid); 809 } 810 } else { 811 /* This is data-containing core mbuf: 'mi' */ 812 mi = rte_mbuf_from_indirect(mbuf); 813 if (rte_mbuf_refcnt_read(mi) > 1) { 814 /* In case of indirect mbuf, and mbuf being cloned, 815 * BMAN should _not_ release it and let EAL release 816 * it through pktmbuf_free below. 817 */ 818 DPAA_MBUF_TO_CONTIG_FD(mbuf, fd_arr, 0xff); 819 } else { 820 /* In case of indirect mbuf, and no cloning, core mbuf 821 * should be released by BMAN. 822 * Increate refcnt of core mbuf so that when 823 * pktmbuf_free is called and mbuf is released, EAL 824 * doesn't try to release core mbuf which would have 825 * been released by BMAN. 826 */ 827 rte_mbuf_refcnt_update(mi, 1); 828 DPAA_MBUF_TO_CONTIG_FD(mbuf, fd_arr, bp_info->bpid); 829 } 830 rte_pktmbuf_free(mbuf); 831 } 832 833 if (mbuf->ol_flags & DPAA_TX_CKSUM_OFFLOAD_MASK) 834 dpaa_unsegmented_checksum(mbuf, fd_arr); 835 } 836 837 /* Handle all mbufs on dpaa BMAN managed pool */ 838 static inline uint16_t 839 tx_on_dpaa_pool(struct rte_mbuf *mbuf, 840 struct dpaa_bp_info *bp_info, 841 struct qm_fd *fd_arr) 842 { 843 DPAA_DP_LOG(DEBUG, "BMAN offloaded buffer, mbuf: %p", mbuf); 844 845 if (mbuf->nb_segs == 1) { 846 /* Case for non-segmented buffers */ 847 tx_on_dpaa_pool_unsegmented(mbuf, bp_info, fd_arr); 848 } else if (mbuf->nb_segs > 1 && 849 mbuf->nb_segs <= DPAA_SGT_MAX_ENTRIES) { 850 if (dpaa_eth_mbuf_to_sg_fd(mbuf, fd_arr, bp_info->bpid)) { 851 DPAA_PMD_DEBUG("Unable to create Scatter Gather FD"); 852 return 1; 853 } 854 } else { 855 DPAA_PMD_DEBUG("Number of Segments not supported"); 856 return 1; 857 } 858 859 return 0; 860 } 861 862 /* Handle all mbufs on an external pool (non-dpaa) */ 863 static inline uint16_t 864 tx_on_external_pool(struct qman_fq *txq, struct rte_mbuf *mbuf, 865 struct qm_fd *fd_arr) 866 { 867 struct dpaa_if *dpaa_intf = txq->dpaa_intf; 868 struct rte_mbuf *dmable_mbuf; 869 870 DPAA_DP_LOG(DEBUG, "Non-BMAN offloaded buffer." 871 "Allocating an offloaded buffer"); 872 dmable_mbuf = dpaa_get_dmable_mbuf(mbuf, dpaa_intf); 873 if (!dmable_mbuf) { 874 DPAA_DP_LOG(DEBUG, "no dpaa buffers."); 875 return 1; 876 } 877 878 DPAA_MBUF_TO_CONTIG_FD(dmable_mbuf, fd_arr, dpaa_intf->bp_info->bpid); 879 if (mbuf->ol_flags & DPAA_TX_CKSUM_OFFLOAD_MASK) 880 dpaa_unsegmented_checksum(mbuf, fd_arr); 881 882 return 0; 883 } 884 885 uint16_t 886 dpaa_eth_queue_tx(void *q, struct rte_mbuf **bufs, uint16_t nb_bufs) 887 { 888 struct rte_mbuf *mbuf, *mi = NULL; 889 struct rte_mempool *mp; 890 struct dpaa_bp_info *bp_info; 891 struct qm_fd fd_arr[DPAA_TX_BURST_SIZE]; 892 uint32_t frames_to_send, loop, sent = 0; 893 uint16_t state; 894 int ret; 895 uint32_t seqn, index, flags[DPAA_TX_BURST_SIZE] = {0}; 896 897 if (unlikely(!RTE_PER_LCORE(dpaa_io))) { 898 ret = rte_dpaa_portal_init((void *)0); 899 if (ret) { 900 DPAA_PMD_ERR("Failure in affining portal"); 901 return 0; 902 } 903 } 904 905 DPAA_DP_LOG(DEBUG, "Transmitting %d buffers on queue: %p", nb_bufs, q); 906 907 while (nb_bufs) { 908 frames_to_send = (nb_bufs > DPAA_TX_BURST_SIZE) ? 909 DPAA_TX_BURST_SIZE : nb_bufs; 910 for (loop = 0; loop < frames_to_send; loop++) { 911 mbuf = *(bufs++); 912 seqn = mbuf->seqn; 913 if (seqn != DPAA_INVALID_MBUF_SEQN) { 914 index = seqn - 1; 915 if (DPAA_PER_LCORE_DQRR_HELD & (1 << index)) { 916 flags[loop] = 917 ((index & QM_EQCR_DCA_IDXMASK) << 8); 918 flags[loop] |= QMAN_ENQUEUE_FLAG_DCA; 919 DPAA_PER_LCORE_DQRR_SIZE--; 920 DPAA_PER_LCORE_DQRR_HELD &= 921 ~(1 << index); 922 } 923 } 924 925 if (likely(RTE_MBUF_DIRECT(mbuf))) { 926 mp = mbuf->pool; 927 bp_info = DPAA_MEMPOOL_TO_POOL_INFO(mp); 928 if (likely(mp->ops_index == 929 bp_info->dpaa_ops_index && 930 mbuf->nb_segs == 1 && 931 rte_mbuf_refcnt_read(mbuf) == 1)) { 932 DPAA_MBUF_TO_CONTIG_FD(mbuf, 933 &fd_arr[loop], bp_info->bpid); 934 if (mbuf->ol_flags & 935 DPAA_TX_CKSUM_OFFLOAD_MASK) 936 dpaa_unsegmented_checksum(mbuf, 937 &fd_arr[loop]); 938 continue; 939 } 940 } else { 941 mi = rte_mbuf_from_indirect(mbuf); 942 mp = mi->pool; 943 } 944 945 bp_info = DPAA_MEMPOOL_TO_POOL_INFO(mp); 946 if (likely(mp->ops_index == bp_info->dpaa_ops_index)) { 947 state = tx_on_dpaa_pool(mbuf, bp_info, 948 &fd_arr[loop]); 949 if (unlikely(state)) { 950 /* Set frames_to_send & nb_bufs so 951 * that packets are transmitted till 952 * previous frame. 953 */ 954 frames_to_send = loop; 955 nb_bufs = loop; 956 goto send_pkts; 957 } 958 } else { 959 state = tx_on_external_pool(q, mbuf, 960 &fd_arr[loop]); 961 if (unlikely(state)) { 962 /* Set frames_to_send & nb_bufs so 963 * that packets are transmitted till 964 * previous frame. 965 */ 966 frames_to_send = loop; 967 nb_bufs = loop; 968 goto send_pkts; 969 } 970 } 971 } 972 973 send_pkts: 974 loop = 0; 975 while (loop < frames_to_send) { 976 loop += qman_enqueue_multi(q, &fd_arr[loop], 977 &flags[loop], 978 frames_to_send - loop); 979 } 980 nb_bufs -= frames_to_send; 981 sent += frames_to_send; 982 } 983 984 DPAA_DP_LOG(DEBUG, "Transmitted %d buffers on queue: %p", sent, q); 985 986 return sent; 987 } 988 989 uint16_t dpaa_eth_tx_drop_all(void *q __rte_unused, 990 struct rte_mbuf **bufs __rte_unused, 991 uint16_t nb_bufs __rte_unused) 992 { 993 DPAA_DP_LOG(DEBUG, "Drop all packets"); 994 995 /* Drop all incoming packets. No need to free packets here 996 * because the rte_eth f/w frees up the packets through tx_buffer 997 * callback in case this functions returns count less than nb_bufs 998 */ 999 return 0; 1000 } 1001