1 /* SPDX-License-Identifier: BSD-3-Clause 2 * 3 * Copyright 2016 Freescale Semiconductor, Inc. All rights reserved. 4 * Copyright 2017 NXP 5 * 6 */ 7 8 /* System headers */ 9 #include <inttypes.h> 10 #include <unistd.h> 11 #include <stdio.h> 12 #include <limits.h> 13 #include <sched.h> 14 #include <pthread.h> 15 16 #include <rte_byteorder.h> 17 #include <rte_common.h> 18 #include <rte_interrupts.h> 19 #include <rte_log.h> 20 #include <rte_debug.h> 21 #include <rte_pci.h> 22 #include <rte_atomic.h> 23 #include <rte_branch_prediction.h> 24 #include <rte_memory.h> 25 #include <rte_tailq.h> 26 #include <rte_eal.h> 27 #include <rte_alarm.h> 28 #include <rte_ether.h> 29 #include <rte_ethdev_driver.h> 30 #include <rte_malloc.h> 31 #include <rte_ring.h> 32 #include <rte_ip.h> 33 #include <rte_tcp.h> 34 #include <rte_udp.h> 35 #include <rte_net.h> 36 #include <rte_eventdev.h> 37 38 #include "dpaa_ethdev.h" 39 #include "dpaa_rxtx.h" 40 #include <rte_dpaa_bus.h> 41 #include <dpaa_mempool.h> 42 43 #include <qman.h> 44 #include <fsl_usd.h> 45 #include <fsl_qman.h> 46 #include <fsl_bman.h> 47 #include <of.h> 48 #include <netcfg.h> 49 50 #define DPAA_MBUF_TO_CONTIG_FD(_mbuf, _fd, _bpid) \ 51 do { \ 52 (_fd)->cmd = 0; \ 53 (_fd)->opaque_addr = 0; \ 54 (_fd)->opaque = QM_FD_CONTIG << DPAA_FD_FORMAT_SHIFT; \ 55 (_fd)->opaque |= ((_mbuf)->data_off) << DPAA_FD_OFFSET_SHIFT; \ 56 (_fd)->opaque |= (_mbuf)->pkt_len; \ 57 (_fd)->addr = (_mbuf)->buf_iova; \ 58 (_fd)->bpid = _bpid; \ 59 } while (0) 60 61 #if (defined RTE_LIBRTE_DPAA_DEBUG_DRIVER) 62 void dpaa_display_frame(const struct qm_fd *fd) 63 { 64 int ii; 65 char *ptr; 66 67 printf("%s::bpid %x addr %08x%08x, format %d off %d, len %d stat %x\n", 68 __func__, fd->bpid, fd->addr_hi, fd->addr_lo, fd->format, 69 fd->offset, fd->length20, fd->status); 70 71 ptr = (char *)rte_dpaa_mem_ptov(fd->addr); 72 ptr += fd->offset; 73 printf("%02x ", *ptr); 74 for (ii = 1; ii < fd->length20; ii++) { 75 printf("%02x ", *ptr); 76 if ((ii % 16) == 0) 77 printf("\n"); 78 ptr++; 79 } 80 printf("\n"); 81 } 82 #else 83 #define dpaa_display_frame(a) 84 #endif 85 86 static inline void dpaa_slow_parsing(struct rte_mbuf *m __rte_unused, 87 uint64_t prs __rte_unused) 88 { 89 DPAA_DP_LOG(DEBUG, "Slow parsing"); 90 /*TBD:XXX: to be implemented*/ 91 } 92 93 static inline void dpaa_eth_packet_info(struct rte_mbuf *m, 94 uint64_t fd_virt_addr) 95 { 96 struct annotations_t *annot = GET_ANNOTATIONS(fd_virt_addr); 97 uint64_t prs = *((uint64_t *)(&annot->parse)) & DPAA_PARSE_MASK; 98 99 DPAA_DP_LOG(DEBUG, " Parsing mbuf: %p with annotations: %p", m, annot); 100 101 switch (prs) { 102 case DPAA_PKT_TYPE_IPV4: 103 m->packet_type = RTE_PTYPE_L2_ETHER | 104 RTE_PTYPE_L3_IPV4; 105 break; 106 case DPAA_PKT_TYPE_IPV6: 107 m->packet_type = RTE_PTYPE_L2_ETHER | 108 RTE_PTYPE_L3_IPV6; 109 break; 110 case DPAA_PKT_TYPE_ETHER: 111 m->packet_type = RTE_PTYPE_L2_ETHER; 112 break; 113 case DPAA_PKT_TYPE_IPV4_FRAG: 114 case DPAA_PKT_TYPE_IPV4_FRAG_UDP: 115 case DPAA_PKT_TYPE_IPV4_FRAG_TCP: 116 case DPAA_PKT_TYPE_IPV4_FRAG_SCTP: 117 m->packet_type = RTE_PTYPE_L2_ETHER | 118 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_FRAG; 119 break; 120 case DPAA_PKT_TYPE_IPV6_FRAG: 121 case DPAA_PKT_TYPE_IPV6_FRAG_UDP: 122 case DPAA_PKT_TYPE_IPV6_FRAG_TCP: 123 case DPAA_PKT_TYPE_IPV6_FRAG_SCTP: 124 m->packet_type = RTE_PTYPE_L2_ETHER | 125 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_FRAG; 126 break; 127 case DPAA_PKT_TYPE_IPV4_EXT: 128 m->packet_type = RTE_PTYPE_L2_ETHER | 129 RTE_PTYPE_L3_IPV4_EXT; 130 break; 131 case DPAA_PKT_TYPE_IPV6_EXT: 132 m->packet_type = RTE_PTYPE_L2_ETHER | 133 RTE_PTYPE_L3_IPV6_EXT; 134 break; 135 case DPAA_PKT_TYPE_IPV4_TCP: 136 m->packet_type = RTE_PTYPE_L2_ETHER | 137 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_TCP; 138 break; 139 case DPAA_PKT_TYPE_IPV6_TCP: 140 m->packet_type = RTE_PTYPE_L2_ETHER | 141 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_TCP; 142 break; 143 case DPAA_PKT_TYPE_IPV4_UDP: 144 m->packet_type = RTE_PTYPE_L2_ETHER | 145 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_UDP; 146 break; 147 case DPAA_PKT_TYPE_IPV6_UDP: 148 m->packet_type = RTE_PTYPE_L2_ETHER | 149 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_UDP; 150 break; 151 case DPAA_PKT_TYPE_IPV4_EXT_UDP: 152 m->packet_type = RTE_PTYPE_L2_ETHER | 153 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_UDP; 154 break; 155 case DPAA_PKT_TYPE_IPV6_EXT_UDP: 156 m->packet_type = RTE_PTYPE_L2_ETHER | 157 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_UDP; 158 break; 159 case DPAA_PKT_TYPE_IPV4_EXT_TCP: 160 m->packet_type = RTE_PTYPE_L2_ETHER | 161 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_TCP; 162 break; 163 case DPAA_PKT_TYPE_IPV6_EXT_TCP: 164 m->packet_type = RTE_PTYPE_L2_ETHER | 165 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_TCP; 166 break; 167 case DPAA_PKT_TYPE_IPV4_SCTP: 168 m->packet_type = RTE_PTYPE_L2_ETHER | 169 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_SCTP; 170 break; 171 case DPAA_PKT_TYPE_IPV6_SCTP: 172 m->packet_type = RTE_PTYPE_L2_ETHER | 173 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_SCTP; 174 break; 175 case DPAA_PKT_TYPE_NONE: 176 m->packet_type = 0; 177 break; 178 /* More switch cases can be added */ 179 default: 180 dpaa_slow_parsing(m, prs); 181 } 182 183 m->tx_offload = annot->parse.ip_off[0]; 184 m->tx_offload |= (annot->parse.l4_off - annot->parse.ip_off[0]) 185 << DPAA_PKT_L3_LEN_SHIFT; 186 187 /* Set the hash values */ 188 m->hash.rss = (uint32_t)(annot->hash); 189 /* All packets with Bad checksum are dropped by interface (and 190 * corresponding notification issued to RX error queues). 191 */ 192 m->ol_flags = PKT_RX_RSS_HASH | PKT_RX_IP_CKSUM_GOOD; 193 194 /* Check if Vlan is present */ 195 if (prs & DPAA_PARSE_VLAN_MASK) 196 m->ol_flags |= PKT_RX_VLAN; 197 /* Packet received without stripping the vlan */ 198 } 199 200 static inline void dpaa_checksum(struct rte_mbuf *mbuf) 201 { 202 struct ether_hdr *eth_hdr = rte_pktmbuf_mtod(mbuf, struct ether_hdr *); 203 char *l3_hdr = (char *)eth_hdr + mbuf->l2_len; 204 struct ipv4_hdr *ipv4_hdr = (struct ipv4_hdr *)l3_hdr; 205 struct ipv6_hdr *ipv6_hdr = (struct ipv6_hdr *)l3_hdr; 206 207 DPAA_DP_LOG(DEBUG, "Calculating checksum for mbuf: %p", mbuf); 208 209 if (((mbuf->packet_type & RTE_PTYPE_L3_MASK) == RTE_PTYPE_L3_IPV4) || 210 ((mbuf->packet_type & RTE_PTYPE_L3_MASK) == 211 RTE_PTYPE_L3_IPV4_EXT)) { 212 ipv4_hdr = (struct ipv4_hdr *)l3_hdr; 213 ipv4_hdr->hdr_checksum = 0; 214 ipv4_hdr->hdr_checksum = rte_ipv4_cksum(ipv4_hdr); 215 } else if (((mbuf->packet_type & RTE_PTYPE_L3_MASK) == 216 RTE_PTYPE_L3_IPV6) || 217 ((mbuf->packet_type & RTE_PTYPE_L3_MASK) == 218 RTE_PTYPE_L3_IPV6_EXT)) 219 ipv6_hdr = (struct ipv6_hdr *)l3_hdr; 220 221 if ((mbuf->packet_type & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_TCP) { 222 struct tcp_hdr *tcp_hdr = (struct tcp_hdr *)(l3_hdr + 223 mbuf->l3_len); 224 tcp_hdr->cksum = 0; 225 if (eth_hdr->ether_type == htons(ETHER_TYPE_IPv4)) 226 tcp_hdr->cksum = rte_ipv4_udptcp_cksum(ipv4_hdr, 227 tcp_hdr); 228 else /* assume ethertype == ETHER_TYPE_IPv6 */ 229 tcp_hdr->cksum = rte_ipv6_udptcp_cksum(ipv6_hdr, 230 tcp_hdr); 231 } else if ((mbuf->packet_type & RTE_PTYPE_L4_MASK) == 232 RTE_PTYPE_L4_UDP) { 233 struct udp_hdr *udp_hdr = (struct udp_hdr *)(l3_hdr + 234 mbuf->l3_len); 235 udp_hdr->dgram_cksum = 0; 236 if (eth_hdr->ether_type == htons(ETHER_TYPE_IPv4)) 237 udp_hdr->dgram_cksum = rte_ipv4_udptcp_cksum(ipv4_hdr, 238 udp_hdr); 239 else /* assume ethertype == ETHER_TYPE_IPv6 */ 240 udp_hdr->dgram_cksum = rte_ipv6_udptcp_cksum(ipv6_hdr, 241 udp_hdr); 242 } 243 } 244 245 static inline void dpaa_checksum_offload(struct rte_mbuf *mbuf, 246 struct qm_fd *fd, char *prs_buf) 247 { 248 struct dpaa_eth_parse_results_t *prs; 249 250 DPAA_DP_LOG(DEBUG, " Offloading checksum for mbuf: %p", mbuf); 251 252 prs = GET_TX_PRS(prs_buf); 253 prs->l3r = 0; 254 prs->l4r = 0; 255 if (((mbuf->packet_type & RTE_PTYPE_L3_MASK) == RTE_PTYPE_L3_IPV4) || 256 ((mbuf->packet_type & RTE_PTYPE_L3_MASK) == 257 RTE_PTYPE_L3_IPV4_EXT)) 258 prs->l3r = DPAA_L3_PARSE_RESULT_IPV4; 259 else if (((mbuf->packet_type & RTE_PTYPE_L3_MASK) == 260 RTE_PTYPE_L3_IPV6) || 261 ((mbuf->packet_type & RTE_PTYPE_L3_MASK) == 262 RTE_PTYPE_L3_IPV6_EXT)) 263 prs->l3r = DPAA_L3_PARSE_RESULT_IPV6; 264 265 if ((mbuf->packet_type & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_TCP) 266 prs->l4r = DPAA_L4_PARSE_RESULT_TCP; 267 else if ((mbuf->packet_type & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_UDP) 268 prs->l4r = DPAA_L4_PARSE_RESULT_UDP; 269 270 prs->ip_off[0] = mbuf->l2_len; 271 prs->l4_off = mbuf->l3_len + mbuf->l2_len; 272 /* Enable L3 (and L4, if TCP or UDP) HW checksum*/ 273 fd->cmd = DPAA_FD_CMD_RPD | DPAA_FD_CMD_DTC; 274 } 275 276 static inline void 277 dpaa_unsegmented_checksum(struct rte_mbuf *mbuf, struct qm_fd *fd_arr) 278 { 279 if (!mbuf->packet_type) { 280 struct rte_net_hdr_lens hdr_lens; 281 282 mbuf->packet_type = rte_net_get_ptype(mbuf, &hdr_lens, 283 RTE_PTYPE_L2_MASK | RTE_PTYPE_L3_MASK 284 | RTE_PTYPE_L4_MASK); 285 mbuf->l2_len = hdr_lens.l2_len; 286 mbuf->l3_len = hdr_lens.l3_len; 287 } 288 if (mbuf->data_off < (DEFAULT_TX_ICEOF + 289 sizeof(struct dpaa_eth_parse_results_t))) { 290 DPAA_DP_LOG(DEBUG, "Checksum offload Err: " 291 "Not enough Headroom " 292 "space for correct Checksum offload." 293 "So Calculating checksum in Software."); 294 dpaa_checksum(mbuf); 295 } else { 296 dpaa_checksum_offload(mbuf, fd_arr, mbuf->buf_addr); 297 } 298 } 299 300 struct rte_mbuf * 301 dpaa_eth_sg_to_mbuf(const struct qm_fd *fd, uint32_t ifid) 302 { 303 struct dpaa_bp_info *bp_info = DPAA_BPID_TO_POOL_INFO(fd->bpid); 304 struct rte_mbuf *first_seg, *prev_seg, *cur_seg, *temp; 305 struct qm_sg_entry *sgt, *sg_temp; 306 void *vaddr, *sg_vaddr; 307 int i = 0; 308 uint8_t fd_offset = fd->offset; 309 310 DPAA_DP_LOG(DEBUG, "Received an SG frame"); 311 312 vaddr = DPAA_MEMPOOL_PTOV(bp_info, qm_fd_addr(fd)); 313 if (!vaddr) { 314 DPAA_PMD_ERR("unable to convert physical address"); 315 return NULL; 316 } 317 sgt = vaddr + fd_offset; 318 sg_temp = &sgt[i++]; 319 hw_sg_to_cpu(sg_temp); 320 temp = (struct rte_mbuf *)((char *)vaddr - bp_info->meta_data_size); 321 sg_vaddr = DPAA_MEMPOOL_PTOV(bp_info, qm_sg_entry_get64(sg_temp)); 322 323 first_seg = (struct rte_mbuf *)((char *)sg_vaddr - 324 bp_info->meta_data_size); 325 first_seg->data_off = sg_temp->offset; 326 first_seg->data_len = sg_temp->length; 327 first_seg->pkt_len = sg_temp->length; 328 rte_mbuf_refcnt_set(first_seg, 1); 329 330 first_seg->port = ifid; 331 first_seg->nb_segs = 1; 332 first_seg->ol_flags = 0; 333 prev_seg = first_seg; 334 while (i < DPAA_SGT_MAX_ENTRIES) { 335 sg_temp = &sgt[i++]; 336 hw_sg_to_cpu(sg_temp); 337 sg_vaddr = DPAA_MEMPOOL_PTOV(bp_info, 338 qm_sg_entry_get64(sg_temp)); 339 cur_seg = (struct rte_mbuf *)((char *)sg_vaddr - 340 bp_info->meta_data_size); 341 cur_seg->data_off = sg_temp->offset; 342 cur_seg->data_len = sg_temp->length; 343 first_seg->pkt_len += sg_temp->length; 344 first_seg->nb_segs += 1; 345 rte_mbuf_refcnt_set(cur_seg, 1); 346 prev_seg->next = cur_seg; 347 if (sg_temp->final) { 348 cur_seg->next = NULL; 349 break; 350 } 351 prev_seg = cur_seg; 352 } 353 354 dpaa_eth_packet_info(first_seg, (uint64_t)vaddr); 355 rte_pktmbuf_free_seg(temp); 356 357 return first_seg; 358 } 359 360 static inline struct rte_mbuf * 361 dpaa_eth_fd_to_mbuf(const struct qm_fd *fd, uint32_t ifid) 362 { 363 struct rte_mbuf *mbuf; 364 struct dpaa_bp_info *bp_info = DPAA_BPID_TO_POOL_INFO(fd->bpid); 365 void *ptr; 366 uint8_t format = 367 (fd->opaque & DPAA_FD_FORMAT_MASK) >> DPAA_FD_FORMAT_SHIFT; 368 uint16_t offset; 369 uint32_t length; 370 371 DPAA_DP_LOG(DEBUG, " FD--->MBUF"); 372 373 if (unlikely(format == qm_fd_sg)) 374 return dpaa_eth_sg_to_mbuf(fd, ifid); 375 376 ptr = DPAA_MEMPOOL_PTOV(bp_info, qm_fd_addr(fd)); 377 378 rte_prefetch0((void *)((uint8_t *)ptr + DEFAULT_RX_ICEOF)); 379 380 offset = (fd->opaque & DPAA_FD_OFFSET_MASK) >> DPAA_FD_OFFSET_SHIFT; 381 length = fd->opaque & DPAA_FD_LENGTH_MASK; 382 383 /* Ignoring case when format != qm_fd_contig */ 384 dpaa_display_frame(fd); 385 386 mbuf = (struct rte_mbuf *)((char *)ptr - bp_info->meta_data_size); 387 388 mbuf->data_off = offset; 389 mbuf->data_len = length; 390 mbuf->pkt_len = length; 391 392 mbuf->port = ifid; 393 mbuf->nb_segs = 1; 394 mbuf->ol_flags = 0; 395 mbuf->next = NULL; 396 rte_mbuf_refcnt_set(mbuf, 1); 397 dpaa_eth_packet_info(mbuf, (uint64_t)mbuf->buf_addr); 398 399 return mbuf; 400 } 401 402 enum qman_cb_dqrr_result dpaa_rx_cb(void *event __always_unused, 403 struct qman_portal *qm __always_unused, 404 struct qman_fq *fq, 405 const struct qm_dqrr_entry *dqrr, 406 void **bufs) 407 { 408 const struct qm_fd *fd = &dqrr->fd; 409 410 *bufs = dpaa_eth_fd_to_mbuf(fd, 411 ((struct dpaa_if *)fq->dpaa_intf)->ifid); 412 return qman_cb_dqrr_consume; 413 } 414 415 static uint16_t 416 dpaa_eth_queue_portal_rx(struct qman_fq *fq, 417 struct rte_mbuf **bufs, 418 uint16_t nb_bufs) 419 { 420 int ret; 421 422 if (unlikely(fq->qp == NULL)) { 423 ret = rte_dpaa_portal_fq_init((void *)0, fq); 424 if (ret) { 425 DPAA_PMD_ERR("Failure in affining portal %d", ret); 426 return 0; 427 } 428 } 429 430 return qman_portal_poll_rx(nb_bufs, (void **)bufs, fq->qp); 431 } 432 433 enum qman_cb_dqrr_result 434 dpaa_rx_cb_parallel(void *event, 435 struct qman_portal *qm __always_unused, 436 struct qman_fq *fq, 437 const struct qm_dqrr_entry *dqrr, 438 void **bufs) 439 { 440 u32 ifid = ((struct dpaa_if *)fq->dpaa_intf)->ifid; 441 struct rte_mbuf *mbuf; 442 struct rte_event *ev = (struct rte_event *)event; 443 444 mbuf = dpaa_eth_fd_to_mbuf(&dqrr->fd, ifid); 445 ev->event_ptr = (void *)mbuf; 446 ev->flow_id = fq->ev.flow_id; 447 ev->sub_event_type = fq->ev.sub_event_type; 448 ev->event_type = RTE_EVENT_TYPE_ETHDEV; 449 ev->op = RTE_EVENT_OP_NEW; 450 ev->sched_type = fq->ev.sched_type; 451 ev->queue_id = fq->ev.queue_id; 452 ev->priority = fq->ev.priority; 453 ev->impl_opaque = (uint8_t)DPAA_INVALID_MBUF_SEQN; 454 mbuf->seqn = DPAA_INVALID_MBUF_SEQN; 455 *bufs = mbuf; 456 457 return qman_cb_dqrr_consume; 458 } 459 460 enum qman_cb_dqrr_result 461 dpaa_rx_cb_atomic(void *event, 462 struct qman_portal *qm __always_unused, 463 struct qman_fq *fq, 464 const struct qm_dqrr_entry *dqrr, 465 void **bufs) 466 { 467 u8 index; 468 u32 ifid = ((struct dpaa_if *)fq->dpaa_intf)->ifid; 469 struct rte_mbuf *mbuf; 470 struct rte_event *ev = (struct rte_event *)event; 471 472 mbuf = dpaa_eth_fd_to_mbuf(&dqrr->fd, ifid); 473 ev->event_ptr = (void *)mbuf; 474 ev->flow_id = fq->ev.flow_id; 475 ev->sub_event_type = fq->ev.sub_event_type; 476 ev->event_type = RTE_EVENT_TYPE_ETHDEV; 477 ev->op = RTE_EVENT_OP_NEW; 478 ev->sched_type = fq->ev.sched_type; 479 ev->queue_id = fq->ev.queue_id; 480 ev->priority = fq->ev.priority; 481 482 /* Save active dqrr entries */ 483 index = DQRR_PTR2IDX(dqrr); 484 DPAA_PER_LCORE_DQRR_SIZE++; 485 DPAA_PER_LCORE_DQRR_HELD |= 1 << index; 486 DPAA_PER_LCORE_DQRR_MBUF(index) = mbuf; 487 ev->impl_opaque = index + 1; 488 mbuf->seqn = (uint32_t)index + 1; 489 *bufs = mbuf; 490 491 return qman_cb_dqrr_defer; 492 } 493 494 uint16_t dpaa_eth_queue_rx(void *q, 495 struct rte_mbuf **bufs, 496 uint16_t nb_bufs) 497 { 498 struct qman_fq *fq = q; 499 struct qm_dqrr_entry *dq; 500 uint32_t num_rx = 0, ifid = ((struct dpaa_if *)fq->dpaa_intf)->ifid; 501 int ret; 502 503 if (likely(fq->is_static)) 504 return dpaa_eth_queue_portal_rx(fq, bufs, nb_bufs); 505 506 if (unlikely(!RTE_PER_LCORE(dpaa_io))) { 507 ret = rte_dpaa_portal_init((void *)0); 508 if (ret) { 509 DPAA_PMD_ERR("Failure in affining portal"); 510 return 0; 511 } 512 } 513 514 ret = qman_set_vdq(fq, (nb_bufs > DPAA_MAX_DEQUEUE_NUM_FRAMES) ? 515 DPAA_MAX_DEQUEUE_NUM_FRAMES : nb_bufs); 516 if (ret) 517 return 0; 518 519 do { 520 dq = qman_dequeue(fq); 521 if (!dq) 522 continue; 523 bufs[num_rx++] = dpaa_eth_fd_to_mbuf(&dq->fd, ifid); 524 qman_dqrr_consume(fq, dq); 525 } while (fq->flags & QMAN_FQ_STATE_VDQCR); 526 527 return num_rx; 528 } 529 530 static void *dpaa_get_pktbuf(struct dpaa_bp_info *bp_info) 531 { 532 int ret; 533 uint64_t buf = 0; 534 struct bm_buffer bufs; 535 536 ret = bman_acquire(bp_info->bp, &bufs, 1, 0); 537 if (ret <= 0) { 538 DPAA_PMD_WARN("Failed to allocate buffers %d", ret); 539 return (void *)buf; 540 } 541 542 DPAA_DP_LOG(DEBUG, "got buffer 0x%lx from pool %d", 543 (uint64_t)bufs.addr, bufs.bpid); 544 545 buf = (uint64_t)DPAA_MEMPOOL_PTOV(bp_info, bufs.addr) 546 - bp_info->meta_data_size; 547 if (!buf) 548 goto out; 549 550 out: 551 return (void *)buf; 552 } 553 554 static struct rte_mbuf *dpaa_get_dmable_mbuf(struct rte_mbuf *mbuf, 555 struct dpaa_if *dpaa_intf) 556 { 557 struct rte_mbuf *dpaa_mbuf; 558 559 /* allocate pktbuffer on bpid for dpaa port */ 560 dpaa_mbuf = dpaa_get_pktbuf(dpaa_intf->bp_info); 561 if (!dpaa_mbuf) 562 return NULL; 563 564 memcpy((uint8_t *)(dpaa_mbuf->buf_addr) + RTE_PKTMBUF_HEADROOM, (void *) 565 ((uint8_t *)(mbuf->buf_addr) + mbuf->data_off), mbuf->pkt_len); 566 567 /* Copy only the required fields */ 568 dpaa_mbuf->data_off = RTE_PKTMBUF_HEADROOM; 569 dpaa_mbuf->pkt_len = mbuf->pkt_len; 570 dpaa_mbuf->ol_flags = mbuf->ol_flags; 571 dpaa_mbuf->packet_type = mbuf->packet_type; 572 dpaa_mbuf->tx_offload = mbuf->tx_offload; 573 rte_pktmbuf_free(mbuf); 574 return dpaa_mbuf; 575 } 576 577 int 578 dpaa_eth_mbuf_to_sg_fd(struct rte_mbuf *mbuf, 579 struct qm_fd *fd, 580 uint32_t bpid) 581 { 582 struct rte_mbuf *cur_seg = mbuf, *prev_seg = NULL; 583 struct dpaa_bp_info *bp_info = DPAA_BPID_TO_POOL_INFO(bpid); 584 struct rte_mbuf *temp, *mi; 585 struct qm_sg_entry *sg_temp, *sgt; 586 int i = 0; 587 588 DPAA_DP_LOG(DEBUG, "Creating SG FD to transmit"); 589 590 temp = rte_pktmbuf_alloc(bp_info->mp); 591 if (!temp) { 592 DPAA_PMD_ERR("Failure in allocation of mbuf"); 593 return -1; 594 } 595 if (temp->buf_len < ((mbuf->nb_segs * sizeof(struct qm_sg_entry)) 596 + temp->data_off)) { 597 DPAA_PMD_ERR("Insufficient space in mbuf for SG entries"); 598 return -1; 599 } 600 601 fd->cmd = 0; 602 fd->opaque_addr = 0; 603 604 if (mbuf->ol_flags & DPAA_TX_CKSUM_OFFLOAD_MASK) { 605 if (!mbuf->packet_type) { 606 struct rte_net_hdr_lens hdr_lens; 607 608 mbuf->packet_type = rte_net_get_ptype(mbuf, &hdr_lens, 609 RTE_PTYPE_L2_MASK | RTE_PTYPE_L3_MASK 610 | RTE_PTYPE_L4_MASK); 611 mbuf->l2_len = hdr_lens.l2_len; 612 mbuf->l3_len = hdr_lens.l3_len; 613 } 614 if (temp->data_off < DEFAULT_TX_ICEOF 615 + sizeof(struct dpaa_eth_parse_results_t)) 616 temp->data_off = DEFAULT_TX_ICEOF 617 + sizeof(struct dpaa_eth_parse_results_t); 618 dcbz_64(temp->buf_addr); 619 dpaa_checksum_offload(mbuf, fd, temp->buf_addr); 620 } 621 622 sgt = temp->buf_addr + temp->data_off; 623 fd->format = QM_FD_SG; 624 fd->addr = temp->buf_iova; 625 fd->offset = temp->data_off; 626 fd->bpid = bpid; 627 fd->length20 = mbuf->pkt_len; 628 629 while (i < DPAA_SGT_MAX_ENTRIES) { 630 sg_temp = &sgt[i++]; 631 sg_temp->opaque = 0; 632 sg_temp->val = 0; 633 sg_temp->addr = cur_seg->buf_iova; 634 sg_temp->offset = cur_seg->data_off; 635 sg_temp->length = cur_seg->data_len; 636 if (RTE_MBUF_DIRECT(cur_seg)) { 637 if (rte_mbuf_refcnt_read(cur_seg) > 1) { 638 /*If refcnt > 1, invalid bpid is set to ensure 639 * buffer is not freed by HW. 640 */ 641 sg_temp->bpid = 0xff; 642 rte_mbuf_refcnt_update(cur_seg, -1); 643 } else { 644 sg_temp->bpid = 645 DPAA_MEMPOOL_TO_BPID(cur_seg->pool); 646 } 647 cur_seg = cur_seg->next; 648 } else { 649 /* Get owner MBUF from indirect buffer */ 650 mi = rte_mbuf_from_indirect(cur_seg); 651 if (rte_mbuf_refcnt_read(mi) > 1) { 652 /*If refcnt > 1, invalid bpid is set to ensure 653 * owner buffer is not freed by HW. 654 */ 655 sg_temp->bpid = 0xff; 656 } else { 657 sg_temp->bpid = DPAA_MEMPOOL_TO_BPID(mi->pool); 658 rte_mbuf_refcnt_update(mi, 1); 659 } 660 prev_seg = cur_seg; 661 cur_seg = cur_seg->next; 662 prev_seg->next = NULL; 663 rte_pktmbuf_free(prev_seg); 664 } 665 if (cur_seg == NULL) { 666 sg_temp->final = 1; 667 cpu_to_hw_sg(sg_temp); 668 break; 669 } 670 cpu_to_hw_sg(sg_temp); 671 } 672 return 0; 673 } 674 675 /* Handle mbufs which are not segmented (non SG) */ 676 static inline void 677 tx_on_dpaa_pool_unsegmented(struct rte_mbuf *mbuf, 678 struct dpaa_bp_info *bp_info, 679 struct qm_fd *fd_arr) 680 { 681 struct rte_mbuf *mi = NULL; 682 683 if (RTE_MBUF_DIRECT(mbuf)) { 684 if (rte_mbuf_refcnt_read(mbuf) > 1) { 685 /* In case of direct mbuf and mbuf being cloned, 686 * BMAN should _not_ release buffer. 687 */ 688 DPAA_MBUF_TO_CONTIG_FD(mbuf, fd_arr, 0xff); 689 /* Buffer should be releasd by EAL */ 690 rte_mbuf_refcnt_update(mbuf, -1); 691 } else { 692 /* In case of direct mbuf and no cloning, mbuf can be 693 * released by BMAN. 694 */ 695 DPAA_MBUF_TO_CONTIG_FD(mbuf, fd_arr, bp_info->bpid); 696 } 697 } else { 698 /* This is data-containing core mbuf: 'mi' */ 699 mi = rte_mbuf_from_indirect(mbuf); 700 if (rte_mbuf_refcnt_read(mi) > 1) { 701 /* In case of indirect mbuf, and mbuf being cloned, 702 * BMAN should _not_ release it and let EAL release 703 * it through pktmbuf_free below. 704 */ 705 DPAA_MBUF_TO_CONTIG_FD(mbuf, fd_arr, 0xff); 706 } else { 707 /* In case of indirect mbuf, and no cloning, core mbuf 708 * should be released by BMAN. 709 * Increate refcnt of core mbuf so that when 710 * pktmbuf_free is called and mbuf is released, EAL 711 * doesn't try to release core mbuf which would have 712 * been released by BMAN. 713 */ 714 rte_mbuf_refcnt_update(mi, 1); 715 DPAA_MBUF_TO_CONTIG_FD(mbuf, fd_arr, bp_info->bpid); 716 } 717 rte_pktmbuf_free(mbuf); 718 } 719 720 if (mbuf->ol_flags & DPAA_TX_CKSUM_OFFLOAD_MASK) 721 dpaa_unsegmented_checksum(mbuf, fd_arr); 722 } 723 724 /* Handle all mbufs on dpaa BMAN managed pool */ 725 static inline uint16_t 726 tx_on_dpaa_pool(struct rte_mbuf *mbuf, 727 struct dpaa_bp_info *bp_info, 728 struct qm_fd *fd_arr) 729 { 730 DPAA_DP_LOG(DEBUG, "BMAN offloaded buffer, mbuf: %p", mbuf); 731 732 if (mbuf->nb_segs == 1) { 733 /* Case for non-segmented buffers */ 734 tx_on_dpaa_pool_unsegmented(mbuf, bp_info, fd_arr); 735 } else if (mbuf->nb_segs > 1 && 736 mbuf->nb_segs <= DPAA_SGT_MAX_ENTRIES) { 737 if (dpaa_eth_mbuf_to_sg_fd(mbuf, fd_arr, bp_info->bpid)) { 738 DPAA_PMD_DEBUG("Unable to create Scatter Gather FD"); 739 return 1; 740 } 741 } else { 742 DPAA_PMD_DEBUG("Number of Segments not supported"); 743 return 1; 744 } 745 746 return 0; 747 } 748 749 /* Handle all mbufs on an external pool (non-dpaa) */ 750 static inline uint16_t 751 tx_on_external_pool(struct qman_fq *txq, struct rte_mbuf *mbuf, 752 struct qm_fd *fd_arr) 753 { 754 struct dpaa_if *dpaa_intf = txq->dpaa_intf; 755 struct rte_mbuf *dmable_mbuf; 756 757 DPAA_DP_LOG(DEBUG, "Non-BMAN offloaded buffer." 758 "Allocating an offloaded buffer"); 759 dmable_mbuf = dpaa_get_dmable_mbuf(mbuf, dpaa_intf); 760 if (!dmable_mbuf) { 761 DPAA_DP_LOG(DEBUG, "no dpaa buffers."); 762 return 1; 763 } 764 765 DPAA_MBUF_TO_CONTIG_FD(dmable_mbuf, fd_arr, dpaa_intf->bp_info->bpid); 766 767 return 0; 768 } 769 770 uint16_t 771 dpaa_eth_queue_tx(void *q, struct rte_mbuf **bufs, uint16_t nb_bufs) 772 { 773 struct rte_mbuf *mbuf, *mi = NULL; 774 struct rte_mempool *mp; 775 struct dpaa_bp_info *bp_info; 776 struct qm_fd fd_arr[DPAA_TX_BURST_SIZE]; 777 uint32_t frames_to_send, loop, sent = 0; 778 uint16_t state; 779 int ret; 780 uint32_t seqn, index, flags[DPAA_TX_BURST_SIZE] = {0}; 781 782 if (unlikely(!RTE_PER_LCORE(dpaa_io))) { 783 ret = rte_dpaa_portal_init((void *)0); 784 if (ret) { 785 DPAA_PMD_ERR("Failure in affining portal"); 786 return 0; 787 } 788 } 789 790 DPAA_DP_LOG(DEBUG, "Transmitting %d buffers on queue: %p", nb_bufs, q); 791 792 while (nb_bufs) { 793 frames_to_send = (nb_bufs > DPAA_TX_BURST_SIZE) ? 794 DPAA_TX_BURST_SIZE : nb_bufs; 795 for (loop = 0; loop < frames_to_send; loop++) { 796 mbuf = *(bufs++); 797 if (likely(RTE_MBUF_DIRECT(mbuf))) { 798 mp = mbuf->pool; 799 bp_info = DPAA_MEMPOOL_TO_POOL_INFO(mp); 800 if (likely(mp->ops_index == 801 bp_info->dpaa_ops_index && 802 mbuf->nb_segs == 1 && 803 rte_mbuf_refcnt_read(mbuf) == 1)) { 804 DPAA_MBUF_TO_CONTIG_FD(mbuf, 805 &fd_arr[loop], bp_info->bpid); 806 if (mbuf->ol_flags & 807 DPAA_TX_CKSUM_OFFLOAD_MASK) 808 dpaa_unsegmented_checksum(mbuf, 809 &fd_arr[loop]); 810 continue; 811 } 812 } else { 813 mi = rte_mbuf_from_indirect(mbuf); 814 mp = mi->pool; 815 } 816 817 bp_info = DPAA_MEMPOOL_TO_POOL_INFO(mp); 818 if (likely(mp->ops_index == bp_info->dpaa_ops_index)) { 819 state = tx_on_dpaa_pool(mbuf, bp_info, 820 &fd_arr[loop]); 821 if (unlikely(state)) { 822 /* Set frames_to_send & nb_bufs so 823 * that packets are transmitted till 824 * previous frame. 825 */ 826 frames_to_send = loop; 827 nb_bufs = loop; 828 goto send_pkts; 829 } 830 } else { 831 state = tx_on_external_pool(q, mbuf, 832 &fd_arr[loop]); 833 if (unlikely(state)) { 834 /* Set frames_to_send & nb_bufs so 835 * that packets are transmitted till 836 * previous frame. 837 */ 838 frames_to_send = loop; 839 nb_bufs = loop; 840 goto send_pkts; 841 } 842 } 843 seqn = mbuf->seqn; 844 if (seqn != DPAA_INVALID_MBUF_SEQN) { 845 index = seqn - 1; 846 if (DPAA_PER_LCORE_DQRR_HELD & (1 << index)) { 847 flags[loop] = 848 ((index & QM_EQCR_DCA_IDXMASK) << 8); 849 flags[loop] |= QMAN_ENQUEUE_FLAG_DCA; 850 DPAA_PER_LCORE_DQRR_SIZE--; 851 DPAA_PER_LCORE_DQRR_HELD &= 852 ~(1 << index); 853 } 854 } 855 } 856 857 send_pkts: 858 loop = 0; 859 while (loop < frames_to_send) { 860 loop += qman_enqueue_multi(q, &fd_arr[loop], 861 &flags[loop], 862 frames_to_send - loop); 863 } 864 nb_bufs -= frames_to_send; 865 sent += frames_to_send; 866 } 867 868 DPAA_DP_LOG(DEBUG, "Transmitted %d buffers on queue: %p", sent, q); 869 870 return sent; 871 } 872 873 uint16_t dpaa_eth_tx_drop_all(void *q __rte_unused, 874 struct rte_mbuf **bufs __rte_unused, 875 uint16_t nb_bufs __rte_unused) 876 { 877 DPAA_DP_LOG(DEBUG, "Drop all packets"); 878 879 /* Drop all incoming packets. No need to free packets here 880 * because the rte_eth f/w frees up the packets through tx_buffer 881 * callback in case this functions returns count less than nb_bufs 882 */ 883 return 0; 884 } 885