1 /* SPDX-License-Identifier: BSD-3-Clause 2 * 3 * Copyright 2016 Freescale Semiconductor, Inc. All rights reserved. 4 * Copyright 2017 NXP 5 * 6 */ 7 8 /* System headers */ 9 #include <inttypes.h> 10 #include <unistd.h> 11 #include <stdio.h> 12 #include <limits.h> 13 #include <sched.h> 14 #include <pthread.h> 15 16 #include <rte_byteorder.h> 17 #include <rte_common.h> 18 #include <rte_interrupts.h> 19 #include <rte_log.h> 20 #include <rte_debug.h> 21 #include <rte_pci.h> 22 #include <rte_atomic.h> 23 #include <rte_branch_prediction.h> 24 #include <rte_memory.h> 25 #include <rte_tailq.h> 26 #include <rte_eal.h> 27 #include <rte_alarm.h> 28 #include <rte_ether.h> 29 #include <rte_ethdev.h> 30 #include <rte_malloc.h> 31 #include <rte_ring.h> 32 #include <rte_ip.h> 33 #include <rte_tcp.h> 34 #include <rte_udp.h> 35 #include <rte_net.h> 36 37 #include "dpaa_ethdev.h" 38 #include "dpaa_rxtx.h" 39 #include <rte_dpaa_bus.h> 40 #include <dpaa_mempool.h> 41 42 #include <fsl_usd.h> 43 #include <fsl_qman.h> 44 #include <fsl_bman.h> 45 #include <of.h> 46 #include <netcfg.h> 47 48 #define DPAA_MBUF_TO_CONTIG_FD(_mbuf, _fd, _bpid) \ 49 do { \ 50 (_fd)->cmd = 0; \ 51 (_fd)->opaque_addr = 0; \ 52 (_fd)->opaque = QM_FD_CONTIG << DPAA_FD_FORMAT_SHIFT; \ 53 (_fd)->opaque |= ((_mbuf)->data_off) << DPAA_FD_OFFSET_SHIFT; \ 54 (_fd)->opaque |= (_mbuf)->pkt_len; \ 55 (_fd)->addr = (_mbuf)->buf_iova; \ 56 (_fd)->bpid = _bpid; \ 57 } while (0) 58 59 #if (defined RTE_LIBRTE_DPAA_DEBUG_DRIVER) 60 void dpaa_display_frame(const struct qm_fd *fd) 61 { 62 int ii; 63 char *ptr; 64 65 printf("%s::bpid %x addr %08x%08x, format %d off %d, len %d stat %x\n", 66 __func__, fd->bpid, fd->addr_hi, fd->addr_lo, fd->format, 67 fd->offset, fd->length20, fd->status); 68 69 ptr = (char *)rte_dpaa_mem_ptov(fd->addr); 70 ptr += fd->offset; 71 printf("%02x ", *ptr); 72 for (ii = 1; ii < fd->length20; ii++) { 73 printf("%02x ", *ptr); 74 if ((ii % 16) == 0) 75 printf("\n"); 76 ptr++; 77 } 78 printf("\n"); 79 } 80 #else 81 #define dpaa_display_frame(a) 82 #endif 83 84 static inline void dpaa_slow_parsing(struct rte_mbuf *m __rte_unused, 85 uint64_t prs __rte_unused) 86 { 87 DPAA_DP_LOG(DEBUG, "Slow parsing"); 88 /*TBD:XXX: to be implemented*/ 89 } 90 91 static inline void dpaa_eth_packet_info(struct rte_mbuf *m, 92 uint64_t fd_virt_addr) 93 { 94 struct annotations_t *annot = GET_ANNOTATIONS(fd_virt_addr); 95 uint64_t prs = *((uint64_t *)(&annot->parse)) & DPAA_PARSE_MASK; 96 97 DPAA_DP_LOG(DEBUG, " Parsing mbuf: %p with annotations: %p", m, annot); 98 99 switch (prs) { 100 case DPAA_PKT_TYPE_IPV4: 101 m->packet_type = RTE_PTYPE_L2_ETHER | 102 RTE_PTYPE_L3_IPV4; 103 break; 104 case DPAA_PKT_TYPE_IPV6: 105 m->packet_type = RTE_PTYPE_L2_ETHER | 106 RTE_PTYPE_L3_IPV6; 107 break; 108 case DPAA_PKT_TYPE_ETHER: 109 m->packet_type = RTE_PTYPE_L2_ETHER; 110 break; 111 case DPAA_PKT_TYPE_IPV4_FRAG: 112 case DPAA_PKT_TYPE_IPV4_FRAG_UDP: 113 case DPAA_PKT_TYPE_IPV4_FRAG_TCP: 114 case DPAA_PKT_TYPE_IPV4_FRAG_SCTP: 115 m->packet_type = RTE_PTYPE_L2_ETHER | 116 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_FRAG; 117 break; 118 case DPAA_PKT_TYPE_IPV6_FRAG: 119 case DPAA_PKT_TYPE_IPV6_FRAG_UDP: 120 case DPAA_PKT_TYPE_IPV6_FRAG_TCP: 121 case DPAA_PKT_TYPE_IPV6_FRAG_SCTP: 122 m->packet_type = RTE_PTYPE_L2_ETHER | 123 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_FRAG; 124 break; 125 case DPAA_PKT_TYPE_IPV4_EXT: 126 m->packet_type = RTE_PTYPE_L2_ETHER | 127 RTE_PTYPE_L3_IPV4_EXT; 128 break; 129 case DPAA_PKT_TYPE_IPV6_EXT: 130 m->packet_type = RTE_PTYPE_L2_ETHER | 131 RTE_PTYPE_L3_IPV6_EXT; 132 break; 133 case DPAA_PKT_TYPE_IPV4_TCP: 134 m->packet_type = RTE_PTYPE_L2_ETHER | 135 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_TCP; 136 break; 137 case DPAA_PKT_TYPE_IPV6_TCP: 138 m->packet_type = RTE_PTYPE_L2_ETHER | 139 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_TCP; 140 break; 141 case DPAA_PKT_TYPE_IPV4_UDP: 142 m->packet_type = RTE_PTYPE_L2_ETHER | 143 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_UDP; 144 break; 145 case DPAA_PKT_TYPE_IPV6_UDP: 146 m->packet_type = RTE_PTYPE_L2_ETHER | 147 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_UDP; 148 break; 149 case DPAA_PKT_TYPE_IPV4_EXT_UDP: 150 m->packet_type = RTE_PTYPE_L2_ETHER | 151 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_UDP; 152 break; 153 case DPAA_PKT_TYPE_IPV6_EXT_UDP: 154 m->packet_type = RTE_PTYPE_L2_ETHER | 155 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_UDP; 156 break; 157 case DPAA_PKT_TYPE_IPV4_EXT_TCP: 158 m->packet_type = RTE_PTYPE_L2_ETHER | 159 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_TCP; 160 break; 161 case DPAA_PKT_TYPE_IPV6_EXT_TCP: 162 m->packet_type = RTE_PTYPE_L2_ETHER | 163 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_TCP; 164 break; 165 case DPAA_PKT_TYPE_IPV4_SCTP: 166 m->packet_type = RTE_PTYPE_L2_ETHER | 167 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_SCTP; 168 break; 169 case DPAA_PKT_TYPE_IPV6_SCTP: 170 m->packet_type = RTE_PTYPE_L2_ETHER | 171 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_SCTP; 172 break; 173 case DPAA_PKT_TYPE_NONE: 174 m->packet_type = 0; 175 break; 176 /* More switch cases can be added */ 177 default: 178 dpaa_slow_parsing(m, prs); 179 } 180 181 m->tx_offload = annot->parse.ip_off[0]; 182 m->tx_offload |= (annot->parse.l4_off - annot->parse.ip_off[0]) 183 << DPAA_PKT_L3_LEN_SHIFT; 184 185 /* Set the hash values */ 186 m->hash.rss = (uint32_t)(annot->hash); 187 /* All packets with Bad checksum are dropped by interface (and 188 * corresponding notification issued to RX error queues). 189 */ 190 m->ol_flags = PKT_RX_RSS_HASH | PKT_RX_IP_CKSUM_GOOD; 191 192 /* Check if Vlan is present */ 193 if (prs & DPAA_PARSE_VLAN_MASK) 194 m->ol_flags |= PKT_RX_VLAN; 195 /* Packet received without stripping the vlan */ 196 } 197 198 static inline void dpaa_checksum(struct rte_mbuf *mbuf) 199 { 200 struct ether_hdr *eth_hdr = rte_pktmbuf_mtod(mbuf, struct ether_hdr *); 201 char *l3_hdr = (char *)eth_hdr + mbuf->l2_len; 202 struct ipv4_hdr *ipv4_hdr = (struct ipv4_hdr *)l3_hdr; 203 struct ipv6_hdr *ipv6_hdr = (struct ipv6_hdr *)l3_hdr; 204 205 DPAA_DP_LOG(DEBUG, "Calculating checksum for mbuf: %p", mbuf); 206 207 if (((mbuf->packet_type & RTE_PTYPE_L3_MASK) == RTE_PTYPE_L3_IPV4) || 208 ((mbuf->packet_type & RTE_PTYPE_L3_MASK) == 209 RTE_PTYPE_L3_IPV4_EXT)) { 210 ipv4_hdr = (struct ipv4_hdr *)l3_hdr; 211 ipv4_hdr->hdr_checksum = 0; 212 ipv4_hdr->hdr_checksum = rte_ipv4_cksum(ipv4_hdr); 213 } else if (((mbuf->packet_type & RTE_PTYPE_L3_MASK) == 214 RTE_PTYPE_L3_IPV6) || 215 ((mbuf->packet_type & RTE_PTYPE_L3_MASK) == 216 RTE_PTYPE_L3_IPV6_EXT)) 217 ipv6_hdr = (struct ipv6_hdr *)l3_hdr; 218 219 if ((mbuf->packet_type & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_TCP) { 220 struct tcp_hdr *tcp_hdr = (struct tcp_hdr *)(l3_hdr + 221 mbuf->l3_len); 222 tcp_hdr->cksum = 0; 223 if (eth_hdr->ether_type == htons(ETHER_TYPE_IPv4)) 224 tcp_hdr->cksum = rte_ipv4_udptcp_cksum(ipv4_hdr, 225 tcp_hdr); 226 else /* assume ethertype == ETHER_TYPE_IPv6 */ 227 tcp_hdr->cksum = rte_ipv6_udptcp_cksum(ipv6_hdr, 228 tcp_hdr); 229 } else if ((mbuf->packet_type & RTE_PTYPE_L4_MASK) == 230 RTE_PTYPE_L4_UDP) { 231 struct udp_hdr *udp_hdr = (struct udp_hdr *)(l3_hdr + 232 mbuf->l3_len); 233 udp_hdr->dgram_cksum = 0; 234 if (eth_hdr->ether_type == htons(ETHER_TYPE_IPv4)) 235 udp_hdr->dgram_cksum = rte_ipv4_udptcp_cksum(ipv4_hdr, 236 udp_hdr); 237 else /* assume ethertype == ETHER_TYPE_IPv6 */ 238 udp_hdr->dgram_cksum = rte_ipv6_udptcp_cksum(ipv6_hdr, 239 udp_hdr); 240 } 241 } 242 243 static inline void dpaa_checksum_offload(struct rte_mbuf *mbuf, 244 struct qm_fd *fd, char *prs_buf) 245 { 246 struct dpaa_eth_parse_results_t *prs; 247 248 DPAA_DP_LOG(DEBUG, " Offloading checksum for mbuf: %p", mbuf); 249 250 prs = GET_TX_PRS(prs_buf); 251 prs->l3r = 0; 252 prs->l4r = 0; 253 if (((mbuf->packet_type & RTE_PTYPE_L3_MASK) == RTE_PTYPE_L3_IPV4) || 254 ((mbuf->packet_type & RTE_PTYPE_L3_MASK) == 255 RTE_PTYPE_L3_IPV4_EXT)) 256 prs->l3r = DPAA_L3_PARSE_RESULT_IPV4; 257 else if (((mbuf->packet_type & RTE_PTYPE_L3_MASK) == 258 RTE_PTYPE_L3_IPV6) || 259 ((mbuf->packet_type & RTE_PTYPE_L3_MASK) == 260 RTE_PTYPE_L3_IPV6_EXT)) 261 prs->l3r = DPAA_L3_PARSE_RESULT_IPV6; 262 263 if ((mbuf->packet_type & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_TCP) 264 prs->l4r = DPAA_L4_PARSE_RESULT_TCP; 265 else if ((mbuf->packet_type & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_UDP) 266 prs->l4r = DPAA_L4_PARSE_RESULT_UDP; 267 268 prs->ip_off[0] = mbuf->l2_len; 269 prs->l4_off = mbuf->l3_len + mbuf->l2_len; 270 /* Enable L3 (and L4, if TCP or UDP) HW checksum*/ 271 fd->cmd = DPAA_FD_CMD_RPD | DPAA_FD_CMD_DTC; 272 } 273 274 static inline void 275 dpaa_unsegmented_checksum(struct rte_mbuf *mbuf, struct qm_fd *fd_arr) 276 { 277 if (!mbuf->packet_type) { 278 struct rte_net_hdr_lens hdr_lens; 279 280 mbuf->packet_type = rte_net_get_ptype(mbuf, &hdr_lens, 281 RTE_PTYPE_L2_MASK | RTE_PTYPE_L3_MASK 282 | RTE_PTYPE_L4_MASK); 283 mbuf->l2_len = hdr_lens.l2_len; 284 mbuf->l3_len = hdr_lens.l3_len; 285 } 286 if (mbuf->data_off < (DEFAULT_TX_ICEOF + 287 sizeof(struct dpaa_eth_parse_results_t))) { 288 DPAA_DP_LOG(DEBUG, "Checksum offload Err: " 289 "Not enough Headroom " 290 "space for correct Checksum offload." 291 "So Calculating checksum in Software."); 292 dpaa_checksum(mbuf); 293 } else { 294 dpaa_checksum_offload(mbuf, fd_arr, mbuf->buf_addr); 295 } 296 } 297 298 struct rte_mbuf * 299 dpaa_eth_sg_to_mbuf(const struct qm_fd *fd, uint32_t ifid) 300 { 301 struct dpaa_bp_info *bp_info = DPAA_BPID_TO_POOL_INFO(fd->bpid); 302 struct rte_mbuf *first_seg, *prev_seg, *cur_seg, *temp; 303 struct qm_sg_entry *sgt, *sg_temp; 304 void *vaddr, *sg_vaddr; 305 int i = 0; 306 uint8_t fd_offset = fd->offset; 307 308 DPAA_DP_LOG(DEBUG, "Received an SG frame"); 309 310 vaddr = rte_dpaa_mem_ptov(qm_fd_addr(fd)); 311 if (!vaddr) { 312 DPAA_PMD_ERR("unable to convert physical address"); 313 return NULL; 314 } 315 sgt = vaddr + fd_offset; 316 sg_temp = &sgt[i++]; 317 hw_sg_to_cpu(sg_temp); 318 temp = (struct rte_mbuf *)((char *)vaddr - bp_info->meta_data_size); 319 sg_vaddr = rte_dpaa_mem_ptov(qm_sg_entry_get64(sg_temp)); 320 321 first_seg = (struct rte_mbuf *)((char *)sg_vaddr - 322 bp_info->meta_data_size); 323 first_seg->data_off = sg_temp->offset; 324 first_seg->data_len = sg_temp->length; 325 first_seg->pkt_len = sg_temp->length; 326 rte_mbuf_refcnt_set(first_seg, 1); 327 328 first_seg->port = ifid; 329 first_seg->nb_segs = 1; 330 first_seg->ol_flags = 0; 331 prev_seg = first_seg; 332 while (i < DPAA_SGT_MAX_ENTRIES) { 333 sg_temp = &sgt[i++]; 334 hw_sg_to_cpu(sg_temp); 335 sg_vaddr = rte_dpaa_mem_ptov(qm_sg_entry_get64(sg_temp)); 336 cur_seg = (struct rte_mbuf *)((char *)sg_vaddr - 337 bp_info->meta_data_size); 338 cur_seg->data_off = sg_temp->offset; 339 cur_seg->data_len = sg_temp->length; 340 first_seg->pkt_len += sg_temp->length; 341 first_seg->nb_segs += 1; 342 rte_mbuf_refcnt_set(cur_seg, 1); 343 prev_seg->next = cur_seg; 344 if (sg_temp->final) { 345 cur_seg->next = NULL; 346 break; 347 } 348 prev_seg = cur_seg; 349 } 350 351 dpaa_eth_packet_info(first_seg, (uint64_t)vaddr); 352 rte_pktmbuf_free_seg(temp); 353 354 return first_seg; 355 } 356 357 static inline struct rte_mbuf * 358 dpaa_eth_fd_to_mbuf(const struct qm_fd *fd, uint32_t ifid) 359 { 360 struct rte_mbuf *mbuf; 361 struct dpaa_bp_info *bp_info = DPAA_BPID_TO_POOL_INFO(fd->bpid); 362 void *ptr = rte_dpaa_mem_ptov(qm_fd_addr(fd)); 363 uint8_t format = 364 (fd->opaque & DPAA_FD_FORMAT_MASK) >> DPAA_FD_FORMAT_SHIFT; 365 uint16_t offset; 366 uint32_t length; 367 368 DPAA_DP_LOG(DEBUG, " FD--->MBUF"); 369 370 if (unlikely(format == qm_fd_sg)) 371 return dpaa_eth_sg_to_mbuf(fd, ifid); 372 373 rte_prefetch0((void *)((uint8_t *)ptr + DEFAULT_RX_ICEOF)); 374 375 offset = (fd->opaque & DPAA_FD_OFFSET_MASK) >> DPAA_FD_OFFSET_SHIFT; 376 length = fd->opaque & DPAA_FD_LENGTH_MASK; 377 378 /* Ignoring case when format != qm_fd_contig */ 379 dpaa_display_frame(fd); 380 381 mbuf = (struct rte_mbuf *)((char *)ptr - bp_info->meta_data_size); 382 383 mbuf->data_off = offset; 384 mbuf->data_len = length; 385 mbuf->pkt_len = length; 386 387 mbuf->port = ifid; 388 mbuf->nb_segs = 1; 389 mbuf->ol_flags = 0; 390 mbuf->next = NULL; 391 rte_mbuf_refcnt_set(mbuf, 1); 392 dpaa_eth_packet_info(mbuf, (uint64_t)mbuf->buf_addr); 393 394 return mbuf; 395 } 396 397 enum qman_cb_dqrr_result dpaa_rx_cb(void *event __always_unused, 398 struct qman_portal *qm __always_unused, 399 struct qman_fq *fq, 400 const struct qm_dqrr_entry *dqrr, 401 void **bufs) 402 { 403 const struct qm_fd *fd = &dqrr->fd; 404 405 *bufs = dpaa_eth_fd_to_mbuf(fd, 406 ((struct dpaa_if *)fq->dpaa_intf)->ifid); 407 return qman_cb_dqrr_consume; 408 } 409 410 static uint16_t 411 dpaa_eth_queue_portal_rx(struct qman_fq *fq, 412 struct rte_mbuf **bufs, 413 uint16_t nb_bufs) 414 { 415 int ret; 416 417 if (unlikely(fq->qp == NULL)) { 418 ret = rte_dpaa_portal_fq_init((void *)0, fq); 419 if (ret) { 420 DPAA_PMD_ERR("Failure in affining portal %d", ret); 421 return 0; 422 } 423 } 424 425 return qman_portal_poll_rx(nb_bufs, (void **)bufs, fq->qp); 426 } 427 428 uint16_t dpaa_eth_queue_rx(void *q, 429 struct rte_mbuf **bufs, 430 uint16_t nb_bufs) 431 { 432 struct qman_fq *fq = q; 433 struct qm_dqrr_entry *dq; 434 uint32_t num_rx = 0, ifid = ((struct dpaa_if *)fq->dpaa_intf)->ifid; 435 int ret; 436 437 if (likely(fq->is_static)) 438 return dpaa_eth_queue_portal_rx(fq, bufs, nb_bufs); 439 440 ret = rte_dpaa_portal_init((void *)0); 441 if (ret) { 442 DPAA_PMD_ERR("Failure in affining portal"); 443 return 0; 444 } 445 446 ret = qman_set_vdq(fq, (nb_bufs > DPAA_MAX_DEQUEUE_NUM_FRAMES) ? 447 DPAA_MAX_DEQUEUE_NUM_FRAMES : nb_bufs); 448 if (ret) 449 return 0; 450 451 do { 452 dq = qman_dequeue(fq); 453 if (!dq) 454 continue; 455 bufs[num_rx++] = dpaa_eth_fd_to_mbuf(&dq->fd, ifid); 456 qman_dqrr_consume(fq, dq); 457 } while (fq->flags & QMAN_FQ_STATE_VDQCR); 458 459 return num_rx; 460 } 461 462 static void *dpaa_get_pktbuf(struct dpaa_bp_info *bp_info) 463 { 464 int ret; 465 uint64_t buf = 0; 466 struct bm_buffer bufs; 467 468 ret = bman_acquire(bp_info->bp, &bufs, 1, 0); 469 if (ret <= 0) { 470 DPAA_PMD_WARN("Failed to allocate buffers %d", ret); 471 return (void *)buf; 472 } 473 474 DPAA_DP_LOG(DEBUG, "got buffer 0x%lx from pool %d", 475 (uint64_t)bufs.addr, bufs.bpid); 476 477 buf = (uint64_t)rte_dpaa_mem_ptov(bufs.addr) - bp_info->meta_data_size; 478 if (!buf) 479 goto out; 480 481 out: 482 return (void *)buf; 483 } 484 485 static struct rte_mbuf *dpaa_get_dmable_mbuf(struct rte_mbuf *mbuf, 486 struct dpaa_if *dpaa_intf) 487 { 488 struct rte_mbuf *dpaa_mbuf; 489 490 /* allocate pktbuffer on bpid for dpaa port */ 491 dpaa_mbuf = dpaa_get_pktbuf(dpaa_intf->bp_info); 492 if (!dpaa_mbuf) 493 return NULL; 494 495 memcpy((uint8_t *)(dpaa_mbuf->buf_addr) + RTE_PKTMBUF_HEADROOM, (void *) 496 ((uint8_t *)(mbuf->buf_addr) + mbuf->data_off), mbuf->pkt_len); 497 498 /* Copy only the required fields */ 499 dpaa_mbuf->data_off = RTE_PKTMBUF_HEADROOM; 500 dpaa_mbuf->pkt_len = mbuf->pkt_len; 501 dpaa_mbuf->ol_flags = mbuf->ol_flags; 502 dpaa_mbuf->packet_type = mbuf->packet_type; 503 dpaa_mbuf->tx_offload = mbuf->tx_offload; 504 rte_pktmbuf_free(mbuf); 505 return dpaa_mbuf; 506 } 507 508 int 509 dpaa_eth_mbuf_to_sg_fd(struct rte_mbuf *mbuf, 510 struct qm_fd *fd, 511 uint32_t bpid) 512 { 513 struct rte_mbuf *cur_seg = mbuf, *prev_seg = NULL; 514 struct dpaa_bp_info *bp_info = DPAA_BPID_TO_POOL_INFO(bpid); 515 struct rte_mbuf *temp, *mi; 516 struct qm_sg_entry *sg_temp, *sgt; 517 int i = 0; 518 519 DPAA_DP_LOG(DEBUG, "Creating SG FD to transmit"); 520 521 temp = rte_pktmbuf_alloc(bp_info->mp); 522 if (!temp) { 523 DPAA_PMD_ERR("Failure in allocation of mbuf"); 524 return -1; 525 } 526 if (temp->buf_len < ((mbuf->nb_segs * sizeof(struct qm_sg_entry)) 527 + temp->data_off)) { 528 DPAA_PMD_ERR("Insufficient space in mbuf for SG entries"); 529 return -1; 530 } 531 532 fd->cmd = 0; 533 fd->opaque_addr = 0; 534 535 if (mbuf->ol_flags & DPAA_TX_CKSUM_OFFLOAD_MASK) { 536 if (!mbuf->packet_type) { 537 struct rte_net_hdr_lens hdr_lens; 538 539 mbuf->packet_type = rte_net_get_ptype(mbuf, &hdr_lens, 540 RTE_PTYPE_L2_MASK | RTE_PTYPE_L3_MASK 541 | RTE_PTYPE_L4_MASK); 542 mbuf->l2_len = hdr_lens.l2_len; 543 mbuf->l3_len = hdr_lens.l3_len; 544 } 545 if (temp->data_off < DEFAULT_TX_ICEOF 546 + sizeof(struct dpaa_eth_parse_results_t)) 547 temp->data_off = DEFAULT_TX_ICEOF 548 + sizeof(struct dpaa_eth_parse_results_t); 549 dcbz_64(temp->buf_addr); 550 dpaa_checksum_offload(mbuf, fd, temp->buf_addr); 551 } 552 553 sgt = temp->buf_addr + temp->data_off; 554 fd->format = QM_FD_SG; 555 fd->addr = temp->buf_iova; 556 fd->offset = temp->data_off; 557 fd->bpid = bpid; 558 fd->length20 = mbuf->pkt_len; 559 560 while (i < DPAA_SGT_MAX_ENTRIES) { 561 sg_temp = &sgt[i++]; 562 sg_temp->opaque = 0; 563 sg_temp->val = 0; 564 sg_temp->addr = cur_seg->buf_iova; 565 sg_temp->offset = cur_seg->data_off; 566 sg_temp->length = cur_seg->data_len; 567 if (RTE_MBUF_DIRECT(cur_seg)) { 568 if (rte_mbuf_refcnt_read(cur_seg) > 1) { 569 /*If refcnt > 1, invalid bpid is set to ensure 570 * buffer is not freed by HW. 571 */ 572 sg_temp->bpid = 0xff; 573 rte_mbuf_refcnt_update(cur_seg, -1); 574 } else { 575 sg_temp->bpid = 576 DPAA_MEMPOOL_TO_BPID(cur_seg->pool); 577 } 578 cur_seg = cur_seg->next; 579 } else { 580 /* Get owner MBUF from indirect buffer */ 581 mi = rte_mbuf_from_indirect(cur_seg); 582 if (rte_mbuf_refcnt_read(mi) > 1) { 583 /*If refcnt > 1, invalid bpid is set to ensure 584 * owner buffer is not freed by HW. 585 */ 586 sg_temp->bpid = 0xff; 587 } else { 588 sg_temp->bpid = DPAA_MEMPOOL_TO_BPID(mi->pool); 589 rte_mbuf_refcnt_update(mi, 1); 590 } 591 prev_seg = cur_seg; 592 cur_seg = cur_seg->next; 593 prev_seg->next = NULL; 594 rte_pktmbuf_free(prev_seg); 595 } 596 if (cur_seg == NULL) { 597 sg_temp->final = 1; 598 cpu_to_hw_sg(sg_temp); 599 break; 600 } 601 cpu_to_hw_sg(sg_temp); 602 } 603 return 0; 604 } 605 606 /* Handle mbufs which are not segmented (non SG) */ 607 static inline void 608 tx_on_dpaa_pool_unsegmented(struct rte_mbuf *mbuf, 609 struct dpaa_bp_info *bp_info, 610 struct qm_fd *fd_arr) 611 { 612 struct rte_mbuf *mi = NULL; 613 614 if (RTE_MBUF_DIRECT(mbuf)) { 615 if (rte_mbuf_refcnt_read(mbuf) > 1) { 616 /* In case of direct mbuf and mbuf being cloned, 617 * BMAN should _not_ release buffer. 618 */ 619 DPAA_MBUF_TO_CONTIG_FD(mbuf, fd_arr, 0xff); 620 /* Buffer should be releasd by EAL */ 621 rte_mbuf_refcnt_update(mbuf, -1); 622 } else { 623 /* In case of direct mbuf and no cloning, mbuf can be 624 * released by BMAN. 625 */ 626 DPAA_MBUF_TO_CONTIG_FD(mbuf, fd_arr, bp_info->bpid); 627 } 628 } else { 629 /* This is data-containing core mbuf: 'mi' */ 630 mi = rte_mbuf_from_indirect(mbuf); 631 if (rte_mbuf_refcnt_read(mi) > 1) { 632 /* In case of indirect mbuf, and mbuf being cloned, 633 * BMAN should _not_ release it and let EAL release 634 * it through pktmbuf_free below. 635 */ 636 DPAA_MBUF_TO_CONTIG_FD(mbuf, fd_arr, 0xff); 637 } else { 638 /* In case of indirect mbuf, and no cloning, core mbuf 639 * should be released by BMAN. 640 * Increate refcnt of core mbuf so that when 641 * pktmbuf_free is called and mbuf is released, EAL 642 * doesn't try to release core mbuf which would have 643 * been released by BMAN. 644 */ 645 rte_mbuf_refcnt_update(mi, 1); 646 DPAA_MBUF_TO_CONTIG_FD(mbuf, fd_arr, bp_info->bpid); 647 } 648 rte_pktmbuf_free(mbuf); 649 } 650 651 if (mbuf->ol_flags & DPAA_TX_CKSUM_OFFLOAD_MASK) 652 dpaa_unsegmented_checksum(mbuf, fd_arr); 653 } 654 655 /* Handle all mbufs on dpaa BMAN managed pool */ 656 static inline uint16_t 657 tx_on_dpaa_pool(struct rte_mbuf *mbuf, 658 struct dpaa_bp_info *bp_info, 659 struct qm_fd *fd_arr) 660 { 661 DPAA_DP_LOG(DEBUG, "BMAN offloaded buffer, mbuf: %p", mbuf); 662 663 if (mbuf->nb_segs == 1) { 664 /* Case for non-segmented buffers */ 665 tx_on_dpaa_pool_unsegmented(mbuf, bp_info, fd_arr); 666 } else if (mbuf->nb_segs > 1 && 667 mbuf->nb_segs <= DPAA_SGT_MAX_ENTRIES) { 668 if (dpaa_eth_mbuf_to_sg_fd(mbuf, fd_arr, bp_info->bpid)) { 669 DPAA_PMD_DEBUG("Unable to create Scatter Gather FD"); 670 return 1; 671 } 672 } else { 673 DPAA_PMD_DEBUG("Number of Segments not supported"); 674 return 1; 675 } 676 677 return 0; 678 } 679 680 /* Handle all mbufs on an external pool (non-dpaa) */ 681 static inline uint16_t 682 tx_on_external_pool(struct qman_fq *txq, struct rte_mbuf *mbuf, 683 struct qm_fd *fd_arr) 684 { 685 struct dpaa_if *dpaa_intf = txq->dpaa_intf; 686 struct rte_mbuf *dmable_mbuf; 687 688 DPAA_DP_LOG(DEBUG, "Non-BMAN offloaded buffer." 689 "Allocating an offloaded buffer"); 690 dmable_mbuf = dpaa_get_dmable_mbuf(mbuf, dpaa_intf); 691 if (!dmable_mbuf) { 692 DPAA_DP_LOG(DEBUG, "no dpaa buffers."); 693 return 1; 694 } 695 696 DPAA_MBUF_TO_CONTIG_FD(dmable_mbuf, fd_arr, dpaa_intf->bp_info->bpid); 697 698 return 0; 699 } 700 701 uint16_t 702 dpaa_eth_queue_tx(void *q, struct rte_mbuf **bufs, uint16_t nb_bufs) 703 { 704 struct rte_mbuf *mbuf, *mi = NULL; 705 struct rte_mempool *mp; 706 struct dpaa_bp_info *bp_info; 707 struct qm_fd fd_arr[DPAA_TX_BURST_SIZE]; 708 uint32_t frames_to_send, loop, sent = 0; 709 uint16_t state; 710 int ret; 711 712 ret = rte_dpaa_portal_init((void *)0); 713 if (ret) { 714 DPAA_PMD_ERR("Failure in affining portal"); 715 return 0; 716 } 717 718 DPAA_DP_LOG(DEBUG, "Transmitting %d buffers on queue: %p", nb_bufs, q); 719 720 while (nb_bufs) { 721 frames_to_send = (nb_bufs > DPAA_TX_BURST_SIZE) ? 722 DPAA_TX_BURST_SIZE : nb_bufs; 723 for (loop = 0; loop < frames_to_send; loop++) { 724 mbuf = *(bufs++); 725 if (likely(RTE_MBUF_DIRECT(mbuf))) { 726 mp = mbuf->pool; 727 bp_info = DPAA_MEMPOOL_TO_POOL_INFO(mp); 728 if (likely(mp->ops_index == 729 bp_info->dpaa_ops_index && 730 mbuf->nb_segs == 1 && 731 rte_mbuf_refcnt_read(mbuf) == 1)) { 732 DPAA_MBUF_TO_CONTIG_FD(mbuf, 733 &fd_arr[loop], bp_info->bpid); 734 if (mbuf->ol_flags & 735 DPAA_TX_CKSUM_OFFLOAD_MASK) 736 dpaa_unsegmented_checksum(mbuf, 737 &fd_arr[loop]); 738 continue; 739 } 740 } else { 741 mi = rte_mbuf_from_indirect(mbuf); 742 mp = mi->pool; 743 } 744 745 bp_info = DPAA_MEMPOOL_TO_POOL_INFO(mp); 746 if (likely(mp->ops_index == bp_info->dpaa_ops_index)) { 747 state = tx_on_dpaa_pool(mbuf, bp_info, 748 &fd_arr[loop]); 749 if (unlikely(state)) { 750 /* Set frames_to_send & nb_bufs so 751 * that packets are transmitted till 752 * previous frame. 753 */ 754 frames_to_send = loop; 755 nb_bufs = loop; 756 goto send_pkts; 757 } 758 } else { 759 state = tx_on_external_pool(q, mbuf, 760 &fd_arr[loop]); 761 if (unlikely(state)) { 762 /* Set frames_to_send & nb_bufs so 763 * that packets are transmitted till 764 * previous frame. 765 */ 766 frames_to_send = loop; 767 nb_bufs = loop; 768 goto send_pkts; 769 } 770 } 771 } 772 773 send_pkts: 774 loop = 0; 775 while (loop < frames_to_send) { 776 loop += qman_enqueue_multi(q, &fd_arr[loop], 777 frames_to_send - loop); 778 } 779 nb_bufs -= frames_to_send; 780 sent += frames_to_send; 781 } 782 783 DPAA_DP_LOG(DEBUG, "Transmitted %d buffers on queue: %p", sent, q); 784 785 return sent; 786 } 787 788 uint16_t dpaa_eth_tx_drop_all(void *q __rte_unused, 789 struct rte_mbuf **bufs __rte_unused, 790 uint16_t nb_bufs __rte_unused) 791 { 792 DPAA_DP_LOG(DEBUG, "Drop all packets"); 793 794 /* Drop all incoming packets. No need to free packets here 795 * because the rte_eth f/w frees up the packets through tx_buffer 796 * callback in case this functions returns count less than nb_bufs 797 */ 798 return 0; 799 } 800