1 /*- 2 * BSD LICENSE 3 * 4 * Copyright 2016 Freescale Semiconductor, Inc. All rights reserved. 5 * Copyright 2017 NXP. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Freescale Semiconductor, Inc nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 /* System headers */ 35 #include <stdio.h> 36 #include <inttypes.h> 37 #include <unistd.h> 38 #include <stdio.h> 39 #include <limits.h> 40 #include <sched.h> 41 #include <pthread.h> 42 43 #include <rte_config.h> 44 #include <rte_byteorder.h> 45 #include <rte_common.h> 46 #include <rte_interrupts.h> 47 #include <rte_log.h> 48 #include <rte_debug.h> 49 #include <rte_pci.h> 50 #include <rte_atomic.h> 51 #include <rte_branch_prediction.h> 52 #include <rte_memory.h> 53 #include <rte_memzone.h> 54 #include <rte_tailq.h> 55 #include <rte_eal.h> 56 #include <rte_alarm.h> 57 #include <rte_ether.h> 58 #include <rte_ethdev.h> 59 #include <rte_atomic.h> 60 #include <rte_malloc.h> 61 #include <rte_ring.h> 62 #include <rte_ip.h> 63 #include <rte_tcp.h> 64 #include <rte_udp.h> 65 66 #include "dpaa_ethdev.h" 67 #include "dpaa_rxtx.h" 68 #include <rte_dpaa_bus.h> 69 #include <dpaa_mempool.h> 70 71 #include <fsl_usd.h> 72 #include <fsl_qman.h> 73 #include <fsl_bman.h> 74 #include <of.h> 75 #include <netcfg.h> 76 77 #define DPAA_MBUF_TO_CONTIG_FD(_mbuf, _fd, _bpid) \ 78 do { \ 79 (_fd)->cmd = 0; \ 80 (_fd)->opaque_addr = 0; \ 81 (_fd)->opaque = QM_FD_CONTIG << DPAA_FD_FORMAT_SHIFT; \ 82 (_fd)->opaque |= ((_mbuf)->data_off) << DPAA_FD_OFFSET_SHIFT; \ 83 (_fd)->opaque |= (_mbuf)->pkt_len; \ 84 (_fd)->addr = (_mbuf)->buf_physaddr; \ 85 (_fd)->bpid = _bpid; \ 86 } while (0) 87 88 #if (defined RTE_LIBRTE_DPAA_DEBUG_DRIVER) 89 void dpaa_display_frame(const struct qm_fd *fd) 90 { 91 int ii; 92 char *ptr; 93 94 printf("%s::bpid %x addr %08x%08x, format %d off %d, len %d stat %x\n", 95 __func__, fd->bpid, fd->addr_hi, fd->addr_lo, fd->format, 96 fd->offset, fd->length20, fd->status); 97 98 ptr = (char *)rte_dpaa_mem_ptov(fd->addr); 99 ptr += fd->offset; 100 printf("%02x ", *ptr); 101 for (ii = 1; ii < fd->length20; ii++) { 102 printf("%02x ", *ptr); 103 if ((ii % 16) == 0) 104 printf("\n"); 105 ptr++; 106 } 107 printf("\n"); 108 } 109 #else 110 #define dpaa_display_frame(a) 111 #endif 112 113 static inline void dpaa_slow_parsing(struct rte_mbuf *m __rte_unused, 114 uint64_t prs __rte_unused) 115 { 116 DPAA_DP_LOG(DEBUG, "Slow parsing"); 117 /*TBD:XXX: to be implemented*/ 118 } 119 120 static inline void dpaa_eth_packet_info(struct rte_mbuf *m, 121 uint64_t fd_virt_addr) 122 { 123 struct annotations_t *annot = GET_ANNOTATIONS(fd_virt_addr); 124 uint64_t prs = *((uint64_t *)(&annot->parse)) & DPAA_PARSE_MASK; 125 126 DPAA_DP_LOG(DEBUG, " Parsing mbuf: %p with annotations: %p", m, annot); 127 128 switch (prs) { 129 case DPAA_PKT_TYPE_NONE: 130 m->packet_type = 0; 131 break; 132 case DPAA_PKT_TYPE_ETHER: 133 m->packet_type = RTE_PTYPE_L2_ETHER; 134 break; 135 case DPAA_PKT_TYPE_IPV4: 136 m->packet_type = RTE_PTYPE_L2_ETHER | 137 RTE_PTYPE_L3_IPV4; 138 break; 139 case DPAA_PKT_TYPE_IPV6: 140 m->packet_type = RTE_PTYPE_L2_ETHER | 141 RTE_PTYPE_L3_IPV6; 142 break; 143 case DPAA_PKT_TYPE_IPV4_FRAG: 144 case DPAA_PKT_TYPE_IPV4_FRAG_UDP: 145 case DPAA_PKT_TYPE_IPV4_FRAG_TCP: 146 case DPAA_PKT_TYPE_IPV4_FRAG_SCTP: 147 m->packet_type = RTE_PTYPE_L2_ETHER | 148 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_FRAG; 149 break; 150 case DPAA_PKT_TYPE_IPV6_FRAG: 151 case DPAA_PKT_TYPE_IPV6_FRAG_UDP: 152 case DPAA_PKT_TYPE_IPV6_FRAG_TCP: 153 case DPAA_PKT_TYPE_IPV6_FRAG_SCTP: 154 m->packet_type = RTE_PTYPE_L2_ETHER | 155 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_FRAG; 156 break; 157 case DPAA_PKT_TYPE_IPV4_EXT: 158 m->packet_type = RTE_PTYPE_L2_ETHER | 159 RTE_PTYPE_L3_IPV4_EXT; 160 break; 161 case DPAA_PKT_TYPE_IPV6_EXT: 162 m->packet_type = RTE_PTYPE_L2_ETHER | 163 RTE_PTYPE_L3_IPV6_EXT; 164 break; 165 case DPAA_PKT_TYPE_IPV4_TCP: 166 m->packet_type = RTE_PTYPE_L2_ETHER | 167 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_TCP; 168 break; 169 case DPAA_PKT_TYPE_IPV6_TCP: 170 m->packet_type = RTE_PTYPE_L2_ETHER | 171 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_TCP; 172 break; 173 case DPAA_PKT_TYPE_IPV4_UDP: 174 m->packet_type = RTE_PTYPE_L2_ETHER | 175 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_UDP; 176 break; 177 case DPAA_PKT_TYPE_IPV6_UDP: 178 m->packet_type = RTE_PTYPE_L2_ETHER | 179 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_UDP; 180 break; 181 case DPAA_PKT_TYPE_IPV4_EXT_UDP: 182 m->packet_type = RTE_PTYPE_L2_ETHER | 183 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_UDP; 184 break; 185 case DPAA_PKT_TYPE_IPV6_EXT_UDP: 186 m->packet_type = RTE_PTYPE_L2_ETHER | 187 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_UDP; 188 break; 189 case DPAA_PKT_TYPE_IPV4_EXT_TCP: 190 m->packet_type = RTE_PTYPE_L2_ETHER | 191 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_TCP; 192 break; 193 case DPAA_PKT_TYPE_IPV6_EXT_TCP: 194 m->packet_type = RTE_PTYPE_L2_ETHER | 195 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_TCP; 196 break; 197 case DPAA_PKT_TYPE_IPV4_SCTP: 198 m->packet_type = RTE_PTYPE_L2_ETHER | 199 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_SCTP; 200 break; 201 case DPAA_PKT_TYPE_IPV6_SCTP: 202 m->packet_type = RTE_PTYPE_L2_ETHER | 203 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_SCTP; 204 break; 205 /* More switch cases can be added */ 206 default: 207 dpaa_slow_parsing(m, prs); 208 } 209 210 m->tx_offload = annot->parse.ip_off[0]; 211 m->tx_offload |= (annot->parse.l4_off - annot->parse.ip_off[0]) 212 << DPAA_PKT_L3_LEN_SHIFT; 213 214 /* Set the hash values */ 215 m->hash.rss = (uint32_t)(rte_be_to_cpu_64(annot->hash)); 216 m->ol_flags = PKT_RX_RSS_HASH; 217 /* All packets with Bad checksum are dropped by interface (and 218 * corresponding notification issued to RX error queues). 219 */ 220 m->ol_flags |= PKT_RX_IP_CKSUM_GOOD; 221 222 /* Check if Vlan is present */ 223 if (prs & DPAA_PARSE_VLAN_MASK) 224 m->ol_flags |= PKT_RX_VLAN_PKT; 225 /* Packet received without stripping the vlan */ 226 } 227 228 static inline void dpaa_checksum(struct rte_mbuf *mbuf) 229 { 230 struct ether_hdr *eth_hdr = rte_pktmbuf_mtod(mbuf, struct ether_hdr *); 231 char *l3_hdr = (char *)eth_hdr + mbuf->l2_len; 232 struct ipv4_hdr *ipv4_hdr = (struct ipv4_hdr *)l3_hdr; 233 struct ipv6_hdr *ipv6_hdr = (struct ipv6_hdr *)l3_hdr; 234 235 DPAA_DP_LOG(DEBUG, "Calculating checksum for mbuf: %p", mbuf); 236 237 if (((mbuf->packet_type & RTE_PTYPE_L3_MASK) == RTE_PTYPE_L3_IPV4) || 238 ((mbuf->packet_type & RTE_PTYPE_L3_MASK) == 239 RTE_PTYPE_L3_IPV4_EXT)) { 240 ipv4_hdr = (struct ipv4_hdr *)l3_hdr; 241 ipv4_hdr->hdr_checksum = 0; 242 ipv4_hdr->hdr_checksum = rte_ipv4_cksum(ipv4_hdr); 243 } else if (((mbuf->packet_type & RTE_PTYPE_L3_MASK) == 244 RTE_PTYPE_L3_IPV6) || 245 ((mbuf->packet_type & RTE_PTYPE_L3_MASK) == 246 RTE_PTYPE_L3_IPV6_EXT)) 247 ipv6_hdr = (struct ipv6_hdr *)l3_hdr; 248 249 if ((mbuf->packet_type & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_TCP) { 250 struct tcp_hdr *tcp_hdr = (struct tcp_hdr *)(l3_hdr + 251 mbuf->l3_len); 252 tcp_hdr->cksum = 0; 253 if (eth_hdr->ether_type == htons(ETHER_TYPE_IPv4)) 254 tcp_hdr->cksum = rte_ipv4_udptcp_cksum(ipv4_hdr, 255 tcp_hdr); 256 else /* assume ethertype == ETHER_TYPE_IPv6 */ 257 tcp_hdr->cksum = rte_ipv6_udptcp_cksum(ipv6_hdr, 258 tcp_hdr); 259 } else if ((mbuf->packet_type & RTE_PTYPE_L4_MASK) == 260 RTE_PTYPE_L4_UDP) { 261 struct udp_hdr *udp_hdr = (struct udp_hdr *)(l3_hdr + 262 mbuf->l3_len); 263 udp_hdr->dgram_cksum = 0; 264 if (eth_hdr->ether_type == htons(ETHER_TYPE_IPv4)) 265 udp_hdr->dgram_cksum = rte_ipv4_udptcp_cksum(ipv4_hdr, 266 udp_hdr); 267 else /* assume ethertype == ETHER_TYPE_IPv6 */ 268 udp_hdr->dgram_cksum = rte_ipv6_udptcp_cksum(ipv6_hdr, 269 udp_hdr); 270 } 271 } 272 273 static inline void dpaa_checksum_offload(struct rte_mbuf *mbuf, 274 struct qm_fd *fd, char *prs_buf) 275 { 276 struct dpaa_eth_parse_results_t *prs; 277 278 DPAA_DP_LOG(DEBUG, " Offloading checksum for mbuf: %p", mbuf); 279 280 prs = GET_TX_PRS(prs_buf); 281 prs->l3r = 0; 282 prs->l4r = 0; 283 if (((mbuf->packet_type & RTE_PTYPE_L3_MASK) == RTE_PTYPE_L3_IPV4) || 284 ((mbuf->packet_type & RTE_PTYPE_L3_MASK) == 285 RTE_PTYPE_L3_IPV4_EXT)) 286 prs->l3r = DPAA_L3_PARSE_RESULT_IPV4; 287 else if (((mbuf->packet_type & RTE_PTYPE_L3_MASK) == 288 RTE_PTYPE_L3_IPV6) || 289 ((mbuf->packet_type & RTE_PTYPE_L3_MASK) == 290 RTE_PTYPE_L3_IPV6_EXT)) 291 prs->l3r = DPAA_L3_PARSE_RESULT_IPV6; 292 293 if ((mbuf->packet_type & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_TCP) 294 prs->l4r = DPAA_L4_PARSE_RESULT_TCP; 295 else if ((mbuf->packet_type & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_UDP) 296 prs->l4r = DPAA_L4_PARSE_RESULT_UDP; 297 298 prs->ip_off[0] = mbuf->l2_len; 299 prs->l4_off = mbuf->l3_len + mbuf->l2_len; 300 /* Enable L3 (and L4, if TCP or UDP) HW checksum*/ 301 fd->cmd = DPAA_FD_CMD_RPD | DPAA_FD_CMD_DTC; 302 } 303 304 struct rte_mbuf * 305 dpaa_eth_sg_to_mbuf(struct qm_fd *fd, uint32_t ifid) 306 { 307 struct dpaa_bp_info *bp_info = DPAA_BPID_TO_POOL_INFO(fd->bpid); 308 struct rte_mbuf *first_seg, *prev_seg, *cur_seg, *temp; 309 struct qm_sg_entry *sgt, *sg_temp; 310 void *vaddr, *sg_vaddr; 311 int i = 0; 312 uint8_t fd_offset = fd->offset; 313 314 DPAA_DP_LOG(DEBUG, "Received an SG frame"); 315 316 vaddr = rte_dpaa_mem_ptov(qm_fd_addr(fd)); 317 if (!vaddr) { 318 DPAA_PMD_ERR("unable to convert physical address"); 319 return NULL; 320 } 321 sgt = vaddr + fd_offset; 322 sg_temp = &sgt[i++]; 323 hw_sg_to_cpu(sg_temp); 324 temp = (struct rte_mbuf *)((char *)vaddr - bp_info->meta_data_size); 325 sg_vaddr = rte_dpaa_mem_ptov(qm_sg_entry_get64(sg_temp)); 326 327 first_seg = (struct rte_mbuf *)((char *)sg_vaddr - 328 bp_info->meta_data_size); 329 first_seg->data_off = sg_temp->offset; 330 first_seg->data_len = sg_temp->length; 331 first_seg->pkt_len = sg_temp->length; 332 rte_mbuf_refcnt_set(first_seg, 1); 333 334 first_seg->port = ifid; 335 first_seg->nb_segs = 1; 336 first_seg->ol_flags = 0; 337 prev_seg = first_seg; 338 while (i < DPAA_SGT_MAX_ENTRIES) { 339 sg_temp = &sgt[i++]; 340 hw_sg_to_cpu(sg_temp); 341 sg_vaddr = rte_dpaa_mem_ptov(qm_sg_entry_get64(sg_temp)); 342 cur_seg = (struct rte_mbuf *)((char *)sg_vaddr - 343 bp_info->meta_data_size); 344 cur_seg->data_off = sg_temp->offset; 345 cur_seg->data_len = sg_temp->length; 346 first_seg->pkt_len += sg_temp->length; 347 first_seg->nb_segs += 1; 348 rte_mbuf_refcnt_set(cur_seg, 1); 349 prev_seg->next = cur_seg; 350 if (sg_temp->final) { 351 cur_seg->next = NULL; 352 break; 353 } 354 prev_seg = cur_seg; 355 } 356 357 dpaa_eth_packet_info(first_seg, (uint64_t)vaddr); 358 rte_pktmbuf_free_seg(temp); 359 360 return first_seg; 361 } 362 363 static inline struct rte_mbuf *dpaa_eth_fd_to_mbuf(struct qm_fd *fd, 364 uint32_t ifid) 365 { 366 struct dpaa_bp_info *bp_info = DPAA_BPID_TO_POOL_INFO(fd->bpid); 367 struct rte_mbuf *mbuf; 368 void *ptr; 369 uint8_t format = 370 (fd->opaque & DPAA_FD_FORMAT_MASK) >> DPAA_FD_FORMAT_SHIFT; 371 uint16_t offset = 372 (fd->opaque & DPAA_FD_OFFSET_MASK) >> DPAA_FD_OFFSET_SHIFT; 373 uint32_t length = fd->opaque & DPAA_FD_LENGTH_MASK; 374 375 DPAA_DP_LOG(DEBUG, " FD--->MBUF"); 376 377 if (unlikely(format == qm_fd_sg)) 378 return dpaa_eth_sg_to_mbuf(fd, ifid); 379 380 /* Ignoring case when format != qm_fd_contig */ 381 dpaa_display_frame(fd); 382 ptr = rte_dpaa_mem_ptov(fd->addr); 383 /* Ignoring case when ptr would be NULL. That is only possible incase 384 * of a corrupted packet 385 */ 386 387 mbuf = (struct rte_mbuf *)((char *)ptr - bp_info->meta_data_size); 388 /* Prefetch the Parse results and packet data to L1 */ 389 rte_prefetch0((void *)((uint8_t *)ptr + DEFAULT_RX_ICEOF)); 390 rte_prefetch0((void *)((uint8_t *)ptr + offset)); 391 392 mbuf->data_off = offset; 393 mbuf->data_len = length; 394 mbuf->pkt_len = length; 395 396 mbuf->port = ifid; 397 mbuf->nb_segs = 1; 398 mbuf->ol_flags = 0; 399 mbuf->next = NULL; 400 rte_mbuf_refcnt_set(mbuf, 1); 401 dpaa_eth_packet_info(mbuf, (uint64_t)mbuf->buf_addr); 402 403 return mbuf; 404 } 405 406 uint16_t dpaa_eth_queue_rx(void *q, 407 struct rte_mbuf **bufs, 408 uint16_t nb_bufs) 409 { 410 struct qman_fq *fq = q; 411 struct qm_dqrr_entry *dq; 412 uint32_t num_rx = 0, ifid = ((struct dpaa_if *)fq->dpaa_intf)->ifid; 413 int ret; 414 415 ret = rte_dpaa_portal_init((void *)0); 416 if (ret) { 417 DPAA_PMD_ERR("Failure in affining portal"); 418 return 0; 419 } 420 421 ret = qman_set_vdq(fq, (nb_bufs > DPAA_MAX_DEQUEUE_NUM_FRAMES) ? 422 DPAA_MAX_DEQUEUE_NUM_FRAMES : nb_bufs); 423 if (ret) 424 return 0; 425 426 do { 427 dq = qman_dequeue(fq); 428 if (!dq) 429 continue; 430 bufs[num_rx++] = dpaa_eth_fd_to_mbuf(&dq->fd, ifid); 431 qman_dqrr_consume(fq, dq); 432 } while (fq->flags & QMAN_FQ_STATE_VDQCR); 433 434 return num_rx; 435 } 436 437 static void *dpaa_get_pktbuf(struct dpaa_bp_info *bp_info) 438 { 439 int ret; 440 uint64_t buf = 0; 441 struct bm_buffer bufs; 442 443 ret = bman_acquire(bp_info->bp, &bufs, 1, 0); 444 if (ret <= 0) { 445 DPAA_PMD_WARN("Failed to allocate buffers %d", ret); 446 return (void *)buf; 447 } 448 449 DPAA_DP_LOG(DEBUG, "got buffer 0x%lx from pool %d", 450 (uint64_t)bufs.addr, bufs.bpid); 451 452 buf = (uint64_t)rte_dpaa_mem_ptov(bufs.addr) - bp_info->meta_data_size; 453 if (!buf) 454 goto out; 455 456 out: 457 return (void *)buf; 458 } 459 460 static struct rte_mbuf *dpaa_get_dmable_mbuf(struct rte_mbuf *mbuf, 461 struct dpaa_if *dpaa_intf) 462 { 463 struct rte_mbuf *dpaa_mbuf; 464 465 /* allocate pktbuffer on bpid for dpaa port */ 466 dpaa_mbuf = dpaa_get_pktbuf(dpaa_intf->bp_info); 467 if (!dpaa_mbuf) 468 return NULL; 469 470 memcpy((uint8_t *)(dpaa_mbuf->buf_addr) + mbuf->data_off, (void *) 471 ((uint8_t *)(mbuf->buf_addr) + mbuf->data_off), mbuf->pkt_len); 472 473 /* Copy only the required fields */ 474 dpaa_mbuf->data_off = mbuf->data_off; 475 dpaa_mbuf->pkt_len = mbuf->pkt_len; 476 dpaa_mbuf->ol_flags = mbuf->ol_flags; 477 dpaa_mbuf->packet_type = mbuf->packet_type; 478 dpaa_mbuf->tx_offload = mbuf->tx_offload; 479 rte_pktmbuf_free(mbuf); 480 return dpaa_mbuf; 481 } 482 483 int 484 dpaa_eth_mbuf_to_sg_fd(struct rte_mbuf *mbuf, 485 struct qm_fd *fd, 486 uint32_t bpid) 487 { 488 struct rte_mbuf *cur_seg = mbuf, *prev_seg = NULL; 489 struct dpaa_bp_info *bp_info = DPAA_BPID_TO_POOL_INFO(bpid); 490 struct rte_mbuf *temp, *mi; 491 struct qm_sg_entry *sg_temp, *sgt; 492 int i = 0; 493 494 DPAA_DP_LOG(DEBUG, "Creating SG FD to transmit"); 495 496 temp = rte_pktmbuf_alloc(bp_info->mp); 497 if (!temp) { 498 DPAA_PMD_ERR("Failure in allocation of mbuf"); 499 return -1; 500 } 501 if (temp->buf_len < ((mbuf->nb_segs * sizeof(struct qm_sg_entry)) 502 + temp->data_off)) { 503 DPAA_PMD_ERR("Insufficient space in mbuf for SG entries"); 504 return -1; 505 } 506 507 fd->cmd = 0; 508 fd->opaque_addr = 0; 509 510 if (mbuf->ol_flags & DPAA_TX_CKSUM_OFFLOAD_MASK) { 511 if (temp->data_off < DEFAULT_TX_ICEOF 512 + sizeof(struct dpaa_eth_parse_results_t)) 513 temp->data_off = DEFAULT_TX_ICEOF 514 + sizeof(struct dpaa_eth_parse_results_t); 515 dcbz_64(temp->buf_addr); 516 dpaa_checksum_offload(mbuf, fd, temp->buf_addr); 517 } 518 519 sgt = temp->buf_addr + temp->data_off; 520 fd->format = QM_FD_SG; 521 fd->addr = temp->buf_physaddr; 522 fd->offset = temp->data_off; 523 fd->bpid = bpid; 524 fd->length20 = mbuf->pkt_len; 525 526 while (i < DPAA_SGT_MAX_ENTRIES) { 527 sg_temp = &sgt[i++]; 528 sg_temp->opaque = 0; 529 sg_temp->val = 0; 530 sg_temp->addr = cur_seg->buf_physaddr; 531 sg_temp->offset = cur_seg->data_off; 532 sg_temp->length = cur_seg->data_len; 533 if (RTE_MBUF_DIRECT(cur_seg)) { 534 if (rte_mbuf_refcnt_read(cur_seg) > 1) { 535 /*If refcnt > 1, invalid bpid is set to ensure 536 * buffer is not freed by HW. 537 */ 538 sg_temp->bpid = 0xff; 539 rte_mbuf_refcnt_update(cur_seg, -1); 540 } else { 541 sg_temp->bpid = 542 DPAA_MEMPOOL_TO_BPID(cur_seg->pool); 543 } 544 cur_seg = cur_seg->next; 545 } else { 546 /* Get owner MBUF from indirect buffer */ 547 mi = rte_mbuf_from_indirect(cur_seg); 548 if (rte_mbuf_refcnt_read(mi) > 1) { 549 /*If refcnt > 1, invalid bpid is set to ensure 550 * owner buffer is not freed by HW. 551 */ 552 sg_temp->bpid = 0xff; 553 } else { 554 sg_temp->bpid = DPAA_MEMPOOL_TO_BPID(mi->pool); 555 rte_mbuf_refcnt_update(mi, 1); 556 } 557 prev_seg = cur_seg; 558 cur_seg = cur_seg->next; 559 prev_seg->next = NULL; 560 rte_pktmbuf_free(prev_seg); 561 } 562 if (cur_seg == NULL) { 563 sg_temp->final = 1; 564 cpu_to_hw_sg(sg_temp); 565 break; 566 } 567 cpu_to_hw_sg(sg_temp); 568 } 569 return 0; 570 } 571 572 /* Handle mbufs which are not segmented (non SG) */ 573 static inline void 574 tx_on_dpaa_pool_unsegmented(struct rte_mbuf *mbuf, 575 struct dpaa_bp_info *bp_info, 576 struct qm_fd *fd_arr) 577 { 578 struct rte_mbuf *mi = NULL; 579 580 if (RTE_MBUF_DIRECT(mbuf)) { 581 if (rte_mbuf_refcnt_read(mbuf) > 1) { 582 /* In case of direct mbuf and mbuf being cloned, 583 * BMAN should _not_ release buffer. 584 */ 585 DPAA_MBUF_TO_CONTIG_FD(mbuf, fd_arr, 0xff); 586 /* Buffer should be releasd by EAL */ 587 rte_mbuf_refcnt_update(mbuf, -1); 588 } else { 589 /* In case of direct mbuf and no cloning, mbuf can be 590 * released by BMAN. 591 */ 592 DPAA_MBUF_TO_CONTIG_FD(mbuf, fd_arr, bp_info->bpid); 593 } 594 } else { 595 /* This is data-containing core mbuf: 'mi' */ 596 mi = rte_mbuf_from_indirect(mbuf); 597 if (rte_mbuf_refcnt_read(mi) > 1) { 598 /* In case of indirect mbuf, and mbuf being cloned, 599 * BMAN should _not_ release it and let EAL release 600 * it through pktmbuf_free below. 601 */ 602 DPAA_MBUF_TO_CONTIG_FD(mbuf, fd_arr, 0xff); 603 } else { 604 /* In case of indirect mbuf, and no cloning, core mbuf 605 * should be released by BMAN. 606 * Increate refcnt of core mbuf so that when 607 * pktmbuf_free is called and mbuf is released, EAL 608 * doesn't try to release core mbuf which would have 609 * been released by BMAN. 610 */ 611 rte_mbuf_refcnt_update(mi, 1); 612 DPAA_MBUF_TO_CONTIG_FD(mbuf, fd_arr, bp_info->bpid); 613 } 614 rte_pktmbuf_free(mbuf); 615 } 616 617 if (mbuf->ol_flags & DPAA_TX_CKSUM_OFFLOAD_MASK) { 618 if (mbuf->data_off < (DEFAULT_TX_ICEOF + 619 sizeof(struct dpaa_eth_parse_results_t))) { 620 DPAA_DP_LOG(DEBUG, "Checksum offload Err: " 621 "Not enough Headroom " 622 "space for correct Checksum offload." 623 "So Calculating checksum in Software."); 624 dpaa_checksum(mbuf); 625 } else { 626 dpaa_checksum_offload(mbuf, fd_arr, mbuf->buf_addr); 627 } 628 } 629 } 630 631 /* Handle all mbufs on dpaa BMAN managed pool */ 632 static inline uint16_t 633 tx_on_dpaa_pool(struct rte_mbuf *mbuf, 634 struct dpaa_bp_info *bp_info, 635 struct qm_fd *fd_arr) 636 { 637 DPAA_DP_LOG(DEBUG, "BMAN offloaded buffer, mbuf: %p", mbuf); 638 639 if (mbuf->nb_segs == 1) { 640 /* Case for non-segmented buffers */ 641 tx_on_dpaa_pool_unsegmented(mbuf, bp_info, fd_arr); 642 } else if (mbuf->nb_segs > 1 && 643 mbuf->nb_segs <= DPAA_SGT_MAX_ENTRIES) { 644 if (dpaa_eth_mbuf_to_sg_fd(mbuf, fd_arr, bp_info->bpid)) { 645 DPAA_PMD_DEBUG("Unable to create Scatter Gather FD"); 646 return 1; 647 } 648 } else { 649 DPAA_PMD_DEBUG("Number of Segments not supported"); 650 return 1; 651 } 652 653 return 0; 654 } 655 656 /* Handle all mbufs on an external pool (non-dpaa) */ 657 static inline uint16_t 658 tx_on_external_pool(struct qman_fq *txq, struct rte_mbuf *mbuf, 659 struct qm_fd *fd_arr) 660 { 661 struct dpaa_if *dpaa_intf = txq->dpaa_intf; 662 struct rte_mbuf *dmable_mbuf; 663 664 DPAA_DP_LOG(DEBUG, "Non-BMAN offloaded buffer." 665 "Allocating an offloaded buffer"); 666 dmable_mbuf = dpaa_get_dmable_mbuf(mbuf, dpaa_intf); 667 if (!dmable_mbuf) { 668 DPAA_DP_LOG(DEBUG, "no dpaa buffers."); 669 return 1; 670 } 671 672 DPAA_MBUF_TO_CONTIG_FD(mbuf, fd_arr, dpaa_intf->bp_info->bpid); 673 674 return 0; 675 } 676 677 uint16_t 678 dpaa_eth_queue_tx(void *q, struct rte_mbuf **bufs, uint16_t nb_bufs) 679 { 680 struct rte_mbuf *mbuf, *mi = NULL; 681 struct rte_mempool *mp; 682 struct dpaa_bp_info *bp_info; 683 struct qm_fd fd_arr[MAX_TX_RING_SLOTS]; 684 uint32_t frames_to_send, loop, i = 0; 685 uint16_t state; 686 int ret; 687 688 ret = rte_dpaa_portal_init((void *)0); 689 if (ret) { 690 DPAA_PMD_ERR("Failure in affining portal"); 691 return 0; 692 } 693 694 DPAA_DP_LOG(DEBUG, "Transmitting %d buffers on queue: %p", nb_bufs, q); 695 696 while (nb_bufs) { 697 frames_to_send = (nb_bufs >> 3) ? MAX_TX_RING_SLOTS : nb_bufs; 698 for (loop = 0; loop < frames_to_send; loop++, i++) { 699 mbuf = bufs[i]; 700 if (RTE_MBUF_DIRECT(mbuf)) { 701 mp = mbuf->pool; 702 } else { 703 mi = rte_mbuf_from_indirect(mbuf); 704 mp = mi->pool; 705 } 706 707 bp_info = DPAA_MEMPOOL_TO_POOL_INFO(mp); 708 if (likely(mp->ops_index == bp_info->dpaa_ops_index)) { 709 state = tx_on_dpaa_pool(mbuf, bp_info, 710 &fd_arr[loop]); 711 if (unlikely(state)) { 712 /* Set frames_to_send & nb_bufs so 713 * that packets are transmitted till 714 * previous frame. 715 */ 716 frames_to_send = loop; 717 nb_bufs = loop; 718 goto send_pkts; 719 } 720 } else { 721 state = tx_on_external_pool(q, mbuf, 722 &fd_arr[loop]); 723 if (unlikely(state)) { 724 /* Set frames_to_send & nb_bufs so 725 * that packets are transmitted till 726 * previous frame. 727 */ 728 frames_to_send = loop; 729 nb_bufs = loop; 730 goto send_pkts; 731 } 732 } 733 } 734 735 send_pkts: 736 loop = 0; 737 while (loop < frames_to_send) { 738 loop += qman_enqueue_multi(q, &fd_arr[loop], 739 frames_to_send - loop); 740 } 741 nb_bufs -= frames_to_send; 742 } 743 744 DPAA_DP_LOG(DEBUG, "Transmitted %d buffers on queue: %p", i, q); 745 746 return i; 747 } 748 749 uint16_t dpaa_eth_tx_drop_all(void *q __rte_unused, 750 struct rte_mbuf **bufs __rte_unused, 751 uint16_t nb_bufs __rte_unused) 752 { 753 DPAA_DP_LOG(DEBUG, "Drop all packets"); 754 755 /* Drop all incoming packets. No need to free packets here 756 * because the rte_eth f/w frees up the packets through tx_buffer 757 * callback in case this functions returns count less than nb_bufs 758 */ 759 return 0; 760 } 761