1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017 Marvell International Ltd. 3 * Copyright(c) 2017 Semihalf. 4 * All rights reserved. 5 */ 6 7 #include <rte_ethdev_driver.h> 8 #include <rte_kvargs.h> 9 #include <rte_log.h> 10 #include <rte_malloc.h> 11 #include <rte_bus_vdev.h> 12 13 #include <fcntl.h> 14 #include <linux/ethtool.h> 15 #include <linux/sockios.h> 16 #include <net/if.h> 17 #include <net/if_arp.h> 18 #include <sys/ioctl.h> 19 #include <sys/socket.h> 20 #include <sys/stat.h> 21 #include <sys/types.h> 22 23 #include <rte_mvep_common.h> 24 #include "mrvl_ethdev.h" 25 #include "mrvl_qos.h" 26 #include "mrvl_flow.h" 27 #include "mrvl_mtr.h" 28 #include "mrvl_tm.h" 29 30 /* bitmask with reserved hifs */ 31 #define MRVL_MUSDK_HIFS_RESERVED 0x0F 32 /* bitmask with reserved bpools */ 33 #define MRVL_MUSDK_BPOOLS_RESERVED 0x07 34 /* bitmask with reserved kernel RSS tables */ 35 #define MRVL_MUSDK_RSS_RESERVED 0x01 36 /* maximum number of available hifs */ 37 #define MRVL_MUSDK_HIFS_MAX 9 38 39 /* prefetch shift */ 40 #define MRVL_MUSDK_PREFETCH_SHIFT 2 41 42 /* TCAM has 25 entries reserved for uc/mc filter entries */ 43 #define MRVL_MAC_ADDRS_MAX 25 44 #define MRVL_MATCH_LEN 16 45 #define MRVL_PKT_EFFEC_OFFS (MRVL_PKT_OFFS + MV_MH_SIZE) 46 /* Maximum allowable packet size */ 47 #define MRVL_PKT_SIZE_MAX (10240 - MV_MH_SIZE) 48 49 #define MRVL_IFACE_NAME_ARG "iface" 50 #define MRVL_CFG_ARG "cfg" 51 52 #define MRVL_BURST_SIZE 64 53 54 #define MRVL_ARP_LENGTH 28 55 56 #define MRVL_COOKIE_ADDR_INVALID ~0ULL 57 #define MRVL_COOKIE_HIGH_ADDR_MASK 0xffffff0000000000 58 59 /** Port Rx offload capabilities */ 60 #define MRVL_RX_OFFLOADS (DEV_RX_OFFLOAD_VLAN_FILTER | \ 61 DEV_RX_OFFLOAD_JUMBO_FRAME | \ 62 DEV_RX_OFFLOAD_CHECKSUM) 63 64 /** Port Tx offloads capabilities */ 65 #define MRVL_TX_OFFLOADS (DEV_TX_OFFLOAD_IPV4_CKSUM | \ 66 DEV_TX_OFFLOAD_UDP_CKSUM | \ 67 DEV_TX_OFFLOAD_TCP_CKSUM | \ 68 DEV_TX_OFFLOAD_MULTI_SEGS) 69 70 static const char * const valid_args[] = { 71 MRVL_IFACE_NAME_ARG, 72 MRVL_CFG_ARG, 73 NULL 74 }; 75 76 static int used_hifs = MRVL_MUSDK_HIFS_RESERVED; 77 static struct pp2_hif *hifs[RTE_MAX_LCORE]; 78 static int used_bpools[PP2_NUM_PKT_PROC] = { 79 [0 ... PP2_NUM_PKT_PROC - 1] = MRVL_MUSDK_BPOOLS_RESERVED 80 }; 81 82 static struct pp2_bpool *mrvl_port_to_bpool_lookup[RTE_MAX_ETHPORTS]; 83 static int mrvl_port_bpool_size[PP2_NUM_PKT_PROC][PP2_BPOOL_NUM_POOLS][RTE_MAX_LCORE]; 84 static uint64_t cookie_addr_high = MRVL_COOKIE_ADDR_INVALID; 85 86 int mrvl_logtype; 87 88 struct mrvl_ifnames { 89 const char *names[PP2_NUM_ETH_PPIO * PP2_NUM_PKT_PROC]; 90 int idx; 91 }; 92 93 /* 94 * To use buffer harvesting based on loopback port shadow queue structure 95 * was introduced for buffers information bookkeeping. 96 * 97 * Before sending the packet, related buffer information (pp2_buff_inf) is 98 * stored in shadow queue. After packet is transmitted no longer used 99 * packet buffer is released back to it's original hardware pool, 100 * on condition it originated from interface. 101 * In case it was generated by application itself i.e: mbuf->port field is 102 * 0xff then its released to software mempool. 103 */ 104 struct mrvl_shadow_txq { 105 int head; /* write index - used when sending buffers */ 106 int tail; /* read index - used when releasing buffers */ 107 u16 size; /* queue occupied size */ 108 u16 num_to_release; /* number of descriptors sent, that can be 109 * released 110 */ 111 struct buff_release_entry ent[MRVL_PP2_TX_SHADOWQ_SIZE]; /* q entries */ 112 }; 113 114 struct mrvl_rxq { 115 struct mrvl_priv *priv; 116 struct rte_mempool *mp; 117 int queue_id; 118 int port_id; 119 int cksum_enabled; 120 uint64_t bytes_recv; 121 uint64_t drop_mac; 122 }; 123 124 struct mrvl_txq { 125 struct mrvl_priv *priv; 126 int queue_id; 127 int port_id; 128 uint64_t bytes_sent; 129 struct mrvl_shadow_txq shadow_txqs[RTE_MAX_LCORE]; 130 int tx_deferred_start; 131 }; 132 133 static int mrvl_lcore_first; 134 static int mrvl_lcore_last; 135 static int mrvl_dev_num; 136 137 static int mrvl_fill_bpool(struct mrvl_rxq *rxq, int num); 138 static inline void mrvl_free_sent_buffers(struct pp2_ppio *ppio, 139 struct pp2_hif *hif, unsigned int core_id, 140 struct mrvl_shadow_txq *sq, int qid, int force); 141 142 static uint16_t mrvl_tx_pkt_burst(void *txq, struct rte_mbuf **tx_pkts, 143 uint16_t nb_pkts); 144 static uint16_t mrvl_tx_sg_pkt_burst(void *txq, struct rte_mbuf **tx_pkts, 145 uint16_t nb_pkts); 146 147 148 #define MRVL_XSTATS_TBL_ENTRY(name) { \ 149 #name, offsetof(struct pp2_ppio_statistics, name), \ 150 sizeof(((struct pp2_ppio_statistics *)0)->name) \ 151 } 152 153 /* Table with xstats data */ 154 static struct { 155 const char *name; 156 unsigned int offset; 157 unsigned int size; 158 } mrvl_xstats_tbl[] = { 159 MRVL_XSTATS_TBL_ENTRY(rx_bytes), 160 MRVL_XSTATS_TBL_ENTRY(rx_packets), 161 MRVL_XSTATS_TBL_ENTRY(rx_unicast_packets), 162 MRVL_XSTATS_TBL_ENTRY(rx_errors), 163 MRVL_XSTATS_TBL_ENTRY(rx_fullq_dropped), 164 MRVL_XSTATS_TBL_ENTRY(rx_bm_dropped), 165 MRVL_XSTATS_TBL_ENTRY(rx_early_dropped), 166 MRVL_XSTATS_TBL_ENTRY(rx_fifo_dropped), 167 MRVL_XSTATS_TBL_ENTRY(rx_cls_dropped), 168 MRVL_XSTATS_TBL_ENTRY(tx_bytes), 169 MRVL_XSTATS_TBL_ENTRY(tx_packets), 170 MRVL_XSTATS_TBL_ENTRY(tx_unicast_packets), 171 MRVL_XSTATS_TBL_ENTRY(tx_errors) 172 }; 173 174 static inline void 175 mrvl_fill_shadowq(struct mrvl_shadow_txq *sq, struct rte_mbuf *buf) 176 { 177 sq->ent[sq->head].buff.cookie = (uint64_t)buf; 178 sq->ent[sq->head].buff.addr = buf ? 179 rte_mbuf_data_iova_default(buf) : 0; 180 181 sq->ent[sq->head].bpool = 182 (unlikely(!buf || buf->port >= RTE_MAX_ETHPORTS || 183 buf->refcnt > 1)) ? NULL : 184 mrvl_port_to_bpool_lookup[buf->port]; 185 186 sq->head = (sq->head + 1) & MRVL_PP2_TX_SHADOWQ_MASK; 187 sq->size++; 188 } 189 190 static inline void 191 mrvl_fill_desc(struct pp2_ppio_desc *desc, struct rte_mbuf *buf) 192 { 193 pp2_ppio_outq_desc_reset(desc); 194 pp2_ppio_outq_desc_set_phys_addr(desc, rte_pktmbuf_iova(buf)); 195 pp2_ppio_outq_desc_set_pkt_offset(desc, 0); 196 pp2_ppio_outq_desc_set_pkt_len(desc, rte_pktmbuf_data_len(buf)); 197 } 198 199 static inline int 200 mrvl_get_bpool_size(int pp2_id, int pool_id) 201 { 202 int i; 203 int size = 0; 204 205 for (i = mrvl_lcore_first; i <= mrvl_lcore_last; i++) 206 size += mrvl_port_bpool_size[pp2_id][pool_id][i]; 207 208 return size; 209 } 210 211 static inline int 212 mrvl_reserve_bit(int *bitmap, int max) 213 { 214 int n = sizeof(*bitmap) * 8 - __builtin_clz(*bitmap); 215 216 if (n >= max) 217 return -1; 218 219 *bitmap |= 1 << n; 220 221 return n; 222 } 223 224 static int 225 mrvl_init_hif(int core_id) 226 { 227 struct pp2_hif_params params; 228 char match[MRVL_MATCH_LEN]; 229 int ret; 230 231 ret = mrvl_reserve_bit(&used_hifs, MRVL_MUSDK_HIFS_MAX); 232 if (ret < 0) { 233 MRVL_LOG(ERR, "Failed to allocate hif %d", core_id); 234 return ret; 235 } 236 237 snprintf(match, sizeof(match), "hif-%d", ret); 238 memset(¶ms, 0, sizeof(params)); 239 params.match = match; 240 params.out_size = MRVL_PP2_AGGR_TXQD_MAX; 241 ret = pp2_hif_init(¶ms, &hifs[core_id]); 242 if (ret) { 243 MRVL_LOG(ERR, "Failed to initialize hif %d", core_id); 244 return ret; 245 } 246 247 return 0; 248 } 249 250 static inline struct pp2_hif* 251 mrvl_get_hif(struct mrvl_priv *priv, int core_id) 252 { 253 int ret; 254 255 if (likely(hifs[core_id] != NULL)) 256 return hifs[core_id]; 257 258 rte_spinlock_lock(&priv->lock); 259 260 ret = mrvl_init_hif(core_id); 261 if (ret < 0) { 262 MRVL_LOG(ERR, "Failed to allocate hif %d", core_id); 263 goto out; 264 } 265 266 if (core_id < mrvl_lcore_first) 267 mrvl_lcore_first = core_id; 268 269 if (core_id > mrvl_lcore_last) 270 mrvl_lcore_last = core_id; 271 out: 272 rte_spinlock_unlock(&priv->lock); 273 274 return hifs[core_id]; 275 } 276 277 /** 278 * Set tx burst function according to offload flag 279 * 280 * @param dev 281 * Pointer to Ethernet device structure. 282 */ 283 static void 284 mrvl_set_tx_function(struct rte_eth_dev *dev) 285 { 286 struct mrvl_priv *priv = dev->data->dev_private; 287 288 /* Use a simple Tx queue (no offloads, no multi segs) if possible */ 289 if (priv->multiseg) { 290 RTE_LOG(INFO, PMD, "Using multi-segment tx callback\n"); 291 dev->tx_pkt_burst = mrvl_tx_sg_pkt_burst; 292 } else { 293 RTE_LOG(INFO, PMD, "Using single-segment tx callback\n"); 294 dev->tx_pkt_burst = mrvl_tx_pkt_burst; 295 } 296 } 297 298 /** 299 * Configure rss based on dpdk rss configuration. 300 * 301 * @param priv 302 * Pointer to private structure. 303 * @param rss_conf 304 * Pointer to RSS configuration. 305 * 306 * @return 307 * 0 on success, negative error value otherwise. 308 */ 309 static int 310 mrvl_configure_rss(struct mrvl_priv *priv, struct rte_eth_rss_conf *rss_conf) 311 { 312 if (rss_conf->rss_key) 313 MRVL_LOG(WARNING, "Changing hash key is not supported"); 314 315 if (rss_conf->rss_hf == 0) { 316 priv->ppio_params.inqs_params.hash_type = PP2_PPIO_HASH_T_NONE; 317 } else if (rss_conf->rss_hf & ETH_RSS_IPV4) { 318 priv->ppio_params.inqs_params.hash_type = 319 PP2_PPIO_HASH_T_2_TUPLE; 320 } else if (rss_conf->rss_hf & ETH_RSS_NONFRAG_IPV4_TCP) { 321 priv->ppio_params.inqs_params.hash_type = 322 PP2_PPIO_HASH_T_5_TUPLE; 323 priv->rss_hf_tcp = 1; 324 } else if (rss_conf->rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) { 325 priv->ppio_params.inqs_params.hash_type = 326 PP2_PPIO_HASH_T_5_TUPLE; 327 priv->rss_hf_tcp = 0; 328 } else { 329 return -EINVAL; 330 } 331 332 return 0; 333 } 334 335 /** 336 * Ethernet device configuration. 337 * 338 * Prepare the driver for a given number of TX and RX queues and 339 * configure RSS. 340 * 341 * @param dev 342 * Pointer to Ethernet device structure. 343 * 344 * @return 345 * 0 on success, negative error value otherwise. 346 */ 347 static int 348 mrvl_dev_configure(struct rte_eth_dev *dev) 349 { 350 struct mrvl_priv *priv = dev->data->dev_private; 351 int ret; 352 353 if (priv->ppio) { 354 MRVL_LOG(INFO, "Device reconfiguration is not supported"); 355 return -EINVAL; 356 } 357 358 if (dev->data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_NONE && 359 dev->data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_RSS) { 360 MRVL_LOG(INFO, "Unsupported rx multi queue mode %d", 361 dev->data->dev_conf.rxmode.mq_mode); 362 return -EINVAL; 363 } 364 365 if (dev->data->dev_conf.rxmode.split_hdr_size) { 366 MRVL_LOG(INFO, "Split headers not supported"); 367 return -EINVAL; 368 } 369 370 if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) 371 dev->data->mtu = dev->data->dev_conf.rxmode.max_rx_pkt_len - 372 MRVL_PP2_ETH_HDRS_LEN; 373 374 if (dev->data->dev_conf.txmode.offloads & DEV_TX_OFFLOAD_MULTI_SEGS) 375 priv->multiseg = 1; 376 377 ret = mrvl_configure_rxqs(priv, dev->data->port_id, 378 dev->data->nb_rx_queues); 379 if (ret < 0) 380 return ret; 381 382 ret = mrvl_configure_txqs(priv, dev->data->port_id, 383 dev->data->nb_tx_queues); 384 if (ret < 0) 385 return ret; 386 387 priv->ppio_params.outqs_params.num_outqs = dev->data->nb_tx_queues; 388 priv->ppio_params.maintain_stats = 1; 389 priv->nb_rx_queues = dev->data->nb_rx_queues; 390 391 ret = mrvl_tm_init(dev); 392 if (ret < 0) 393 return ret; 394 395 if (dev->data->nb_rx_queues == 1 && 396 dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_RSS) { 397 MRVL_LOG(WARNING, "Disabling hash for 1 rx queue"); 398 priv->ppio_params.inqs_params.hash_type = PP2_PPIO_HASH_T_NONE; 399 400 return 0; 401 } 402 403 return mrvl_configure_rss(priv, 404 &dev->data->dev_conf.rx_adv_conf.rss_conf); 405 } 406 407 /** 408 * DPDK callback to change the MTU. 409 * 410 * Setting the MTU affects hardware MRU (packets larger than the MRU 411 * will be dropped). 412 * 413 * @param dev 414 * Pointer to Ethernet device structure. 415 * @param mtu 416 * New MTU. 417 * 418 * @return 419 * 0 on success, negative error value otherwise. 420 */ 421 static int 422 mrvl_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) 423 { 424 struct mrvl_priv *priv = dev->data->dev_private; 425 uint16_t mru; 426 uint16_t mbuf_data_size = 0; /* SW buffer size */ 427 int ret; 428 429 mru = MRVL_PP2_MTU_TO_MRU(mtu); 430 /* 431 * min_rx_buf_size is equal to mbuf data size 432 * if pmd didn't set it differently 433 */ 434 mbuf_data_size = dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM; 435 /* Prevent PMD from: 436 * - setting mru greater than the mbuf size resulting in 437 * hw and sw buffer size mismatch 438 * - setting mtu that requires the support of scattered packets 439 * when this feature has not been enabled/supported so far 440 * (TODO check scattered_rx flag here once scattered RX is supported). 441 */ 442 if (mru + MRVL_PKT_OFFS > mbuf_data_size) { 443 mru = mbuf_data_size - MRVL_PKT_OFFS; 444 mtu = MRVL_PP2_MRU_TO_MTU(mru); 445 MRVL_LOG(WARNING, "MTU too big, max MTU possible limitted " 446 "by current mbuf size: %u. Set MTU to %u, MRU to %u", 447 mbuf_data_size, mtu, mru); 448 } 449 450 if (mtu < ETHER_MIN_MTU || mru > MRVL_PKT_SIZE_MAX) { 451 MRVL_LOG(ERR, "Invalid MTU [%u] or MRU [%u]", mtu, mru); 452 return -EINVAL; 453 } 454 455 dev->data->mtu = mtu; 456 dev->data->dev_conf.rxmode.max_rx_pkt_len = mru - MV_MH_SIZE; 457 458 if (!priv->ppio) 459 return 0; 460 461 ret = pp2_ppio_set_mru(priv->ppio, mru); 462 if (ret) { 463 MRVL_LOG(ERR, "Failed to change MRU"); 464 return ret; 465 } 466 467 ret = pp2_ppio_set_mtu(priv->ppio, mtu); 468 if (ret) { 469 MRVL_LOG(ERR, "Failed to change MTU"); 470 return ret; 471 } 472 473 return 0; 474 } 475 476 /** 477 * DPDK callback to bring the link up. 478 * 479 * @param dev 480 * Pointer to Ethernet device structure. 481 * 482 * @return 483 * 0 on success, negative error value otherwise. 484 */ 485 static int 486 mrvl_dev_set_link_up(struct rte_eth_dev *dev) 487 { 488 struct mrvl_priv *priv = dev->data->dev_private; 489 int ret; 490 491 if (!priv->ppio) 492 return -EPERM; 493 494 ret = pp2_ppio_enable(priv->ppio); 495 if (ret) 496 return ret; 497 498 /* 499 * mtu/mru can be updated if pp2_ppio_enable() was called at least once 500 * as pp2_ppio_enable() changes port->t_mode from default 0 to 501 * PP2_TRAFFIC_INGRESS_EGRESS. 502 * 503 * Set mtu to default DPDK value here. 504 */ 505 ret = mrvl_mtu_set(dev, dev->data->mtu); 506 if (ret) 507 pp2_ppio_disable(priv->ppio); 508 509 return ret; 510 } 511 512 /** 513 * DPDK callback to bring the link down. 514 * 515 * @param dev 516 * Pointer to Ethernet device structure. 517 * 518 * @return 519 * 0 on success, negative error value otherwise. 520 */ 521 static int 522 mrvl_dev_set_link_down(struct rte_eth_dev *dev) 523 { 524 struct mrvl_priv *priv = dev->data->dev_private; 525 526 if (!priv->ppio) 527 return -EPERM; 528 529 return pp2_ppio_disable(priv->ppio); 530 } 531 532 /** 533 * DPDK callback to start tx queue. 534 * 535 * @param dev 536 * Pointer to Ethernet device structure. 537 * @param queue_id 538 * Transmit queue index. 539 * 540 * @return 541 * 0 on success, negative error value otherwise. 542 */ 543 static int 544 mrvl_tx_queue_start(struct rte_eth_dev *dev, uint16_t queue_id) 545 { 546 struct mrvl_priv *priv = dev->data->dev_private; 547 int ret; 548 549 if (!priv) 550 return -EPERM; 551 552 /* passing 1 enables given tx queue */ 553 ret = pp2_ppio_set_outq_state(priv->ppio, queue_id, 1); 554 if (ret) { 555 MRVL_LOG(ERR, "Failed to start txq %d", queue_id); 556 return ret; 557 } 558 559 dev->data->tx_queue_state[queue_id] = RTE_ETH_QUEUE_STATE_STARTED; 560 561 return 0; 562 } 563 564 /** 565 * DPDK callback to stop tx queue. 566 * 567 * @param dev 568 * Pointer to Ethernet device structure. 569 * @param queue_id 570 * Transmit queue index. 571 * 572 * @return 573 * 0 on success, negative error value otherwise. 574 */ 575 static int 576 mrvl_tx_queue_stop(struct rte_eth_dev *dev, uint16_t queue_id) 577 { 578 struct mrvl_priv *priv = dev->data->dev_private; 579 int ret; 580 581 if (!priv->ppio) 582 return -EPERM; 583 584 /* passing 0 disables given tx queue */ 585 ret = pp2_ppio_set_outq_state(priv->ppio, queue_id, 0); 586 if (ret) { 587 MRVL_LOG(ERR, "Failed to stop txq %d", queue_id); 588 return ret; 589 } 590 591 dev->data->tx_queue_state[queue_id] = RTE_ETH_QUEUE_STATE_STOPPED; 592 593 return 0; 594 } 595 596 /** 597 * DPDK callback to start the device. 598 * 599 * @param dev 600 * Pointer to Ethernet device structure. 601 * 602 * @return 603 * 0 on success, negative errno value on failure. 604 */ 605 static int 606 mrvl_dev_start(struct rte_eth_dev *dev) 607 { 608 struct mrvl_priv *priv = dev->data->dev_private; 609 char match[MRVL_MATCH_LEN]; 610 int ret = 0, i, def_init_size; 611 612 if (priv->ppio) 613 return mrvl_dev_set_link_up(dev); 614 615 snprintf(match, sizeof(match), "ppio-%d:%d", 616 priv->pp_id, priv->ppio_id); 617 priv->ppio_params.match = match; 618 619 /* 620 * Calculate the minimum bpool size for refill feature as follows: 621 * 2 default burst sizes multiply by number of rx queues. 622 * If the bpool size will be below this value, new buffers will 623 * be added to the pool. 624 */ 625 priv->bpool_min_size = priv->nb_rx_queues * MRVL_BURST_SIZE * 2; 626 627 /* In case initial bpool size configured in queues setup is 628 * smaller than minimum size add more buffers 629 */ 630 def_init_size = priv->bpool_min_size + MRVL_BURST_SIZE * 2; 631 if (priv->bpool_init_size < def_init_size) { 632 int buffs_to_add = def_init_size - priv->bpool_init_size; 633 634 priv->bpool_init_size += buffs_to_add; 635 ret = mrvl_fill_bpool(dev->data->rx_queues[0], buffs_to_add); 636 if (ret) 637 MRVL_LOG(ERR, "Failed to add buffers to bpool"); 638 } 639 640 /* 641 * Calculate the maximum bpool size for refill feature as follows: 642 * maximum number of descriptors in rx queue multiply by number 643 * of rx queues plus minimum bpool size. 644 * In case the bpool size will exceed this value, superfluous buffers 645 * will be removed 646 */ 647 priv->bpool_max_size = (priv->nb_rx_queues * MRVL_PP2_RXD_MAX) + 648 priv->bpool_min_size; 649 650 ret = pp2_ppio_init(&priv->ppio_params, &priv->ppio); 651 if (ret) { 652 MRVL_LOG(ERR, "Failed to init ppio"); 653 return ret; 654 } 655 656 /* 657 * In case there are some some stale uc/mc mac addresses flush them 658 * here. It cannot be done during mrvl_dev_close() as port information 659 * is already gone at that point (due to pp2_ppio_deinit() in 660 * mrvl_dev_stop()). 661 */ 662 if (!priv->uc_mc_flushed) { 663 ret = pp2_ppio_flush_mac_addrs(priv->ppio, 1, 1); 664 if (ret) { 665 MRVL_LOG(ERR, 666 "Failed to flush uc/mc filter list"); 667 goto out; 668 } 669 priv->uc_mc_flushed = 1; 670 } 671 672 if (!priv->vlan_flushed) { 673 ret = pp2_ppio_flush_vlan(priv->ppio); 674 if (ret) { 675 MRVL_LOG(ERR, "Failed to flush vlan list"); 676 /* 677 * TODO 678 * once pp2_ppio_flush_vlan() is supported jump to out 679 * goto out; 680 */ 681 } 682 priv->vlan_flushed = 1; 683 } 684 ret = mrvl_mtu_set(dev, dev->data->mtu); 685 if (ret) 686 MRVL_LOG(ERR, "Failed to set MTU to %d", dev->data->mtu); 687 688 /* For default QoS config, don't start classifier. */ 689 if (mrvl_qos_cfg && 690 mrvl_qos_cfg->port[dev->data->port_id].use_global_defaults == 0) { 691 ret = mrvl_start_qos_mapping(priv); 692 if (ret) { 693 MRVL_LOG(ERR, "Failed to setup QoS mapping"); 694 goto out; 695 } 696 } 697 698 ret = mrvl_dev_set_link_up(dev); 699 if (ret) { 700 MRVL_LOG(ERR, "Failed to set link up"); 701 goto out; 702 } 703 704 /* start tx queues */ 705 for (i = 0; i < dev->data->nb_tx_queues; i++) { 706 struct mrvl_txq *txq = dev->data->tx_queues[i]; 707 708 dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; 709 710 if (!txq->tx_deferred_start) 711 continue; 712 713 /* 714 * All txqs are started by default. Stop them 715 * so that tx_deferred_start works as expected. 716 */ 717 ret = mrvl_tx_queue_stop(dev, i); 718 if (ret) 719 goto out; 720 } 721 722 mrvl_flow_init(dev); 723 mrvl_mtr_init(dev); 724 mrvl_set_tx_function(dev); 725 726 return 0; 727 out: 728 MRVL_LOG(ERR, "Failed to start device"); 729 pp2_ppio_deinit(priv->ppio); 730 return ret; 731 } 732 733 /** 734 * Flush receive queues. 735 * 736 * @param dev 737 * Pointer to Ethernet device structure. 738 */ 739 static void 740 mrvl_flush_rx_queues(struct rte_eth_dev *dev) 741 { 742 int i; 743 744 MRVL_LOG(INFO, "Flushing rx queues"); 745 for (i = 0; i < dev->data->nb_rx_queues; i++) { 746 int ret, num; 747 748 do { 749 struct mrvl_rxq *q = dev->data->rx_queues[i]; 750 struct pp2_ppio_desc descs[MRVL_PP2_RXD_MAX]; 751 752 num = MRVL_PP2_RXD_MAX; 753 ret = pp2_ppio_recv(q->priv->ppio, 754 q->priv->rxq_map[q->queue_id].tc, 755 q->priv->rxq_map[q->queue_id].inq, 756 descs, (uint16_t *)&num); 757 } while (ret == 0 && num); 758 } 759 } 760 761 /** 762 * Flush transmit shadow queues. 763 * 764 * @param dev 765 * Pointer to Ethernet device structure. 766 */ 767 static void 768 mrvl_flush_tx_shadow_queues(struct rte_eth_dev *dev) 769 { 770 int i, j; 771 struct mrvl_txq *txq; 772 773 MRVL_LOG(INFO, "Flushing tx shadow queues"); 774 for (i = 0; i < dev->data->nb_tx_queues; i++) { 775 txq = (struct mrvl_txq *)dev->data->tx_queues[i]; 776 777 for (j = 0; j < RTE_MAX_LCORE; j++) { 778 struct mrvl_shadow_txq *sq; 779 780 if (!hifs[j]) 781 continue; 782 783 sq = &txq->shadow_txqs[j]; 784 mrvl_free_sent_buffers(txq->priv->ppio, 785 hifs[j], j, sq, txq->queue_id, 1); 786 while (sq->tail != sq->head) { 787 uint64_t addr = cookie_addr_high | 788 sq->ent[sq->tail].buff.cookie; 789 rte_pktmbuf_free( 790 (struct rte_mbuf *)addr); 791 sq->tail = (sq->tail + 1) & 792 MRVL_PP2_TX_SHADOWQ_MASK; 793 } 794 memset(sq, 0, sizeof(*sq)); 795 } 796 } 797 } 798 799 /** 800 * Flush hardware bpool (buffer-pool). 801 * 802 * @param dev 803 * Pointer to Ethernet device structure. 804 */ 805 static void 806 mrvl_flush_bpool(struct rte_eth_dev *dev) 807 { 808 struct mrvl_priv *priv = dev->data->dev_private; 809 struct pp2_hif *hif; 810 uint32_t num; 811 int ret; 812 unsigned int core_id = rte_lcore_id(); 813 814 if (core_id == LCORE_ID_ANY) 815 core_id = 0; 816 817 hif = mrvl_get_hif(priv, core_id); 818 819 ret = pp2_bpool_get_num_buffs(priv->bpool, &num); 820 if (ret) { 821 MRVL_LOG(ERR, "Failed to get bpool buffers number"); 822 return; 823 } 824 825 while (num--) { 826 struct pp2_buff_inf inf; 827 uint64_t addr; 828 829 ret = pp2_bpool_get_buff(hif, priv->bpool, &inf); 830 if (ret) 831 break; 832 833 addr = cookie_addr_high | inf.cookie; 834 rte_pktmbuf_free((struct rte_mbuf *)addr); 835 } 836 } 837 838 /** 839 * DPDK callback to stop the device. 840 * 841 * @param dev 842 * Pointer to Ethernet device structure. 843 */ 844 static void 845 mrvl_dev_stop(struct rte_eth_dev *dev) 846 { 847 mrvl_dev_set_link_down(dev); 848 } 849 850 /** 851 * DPDK callback to close the device. 852 * 853 * @param dev 854 * Pointer to Ethernet device structure. 855 */ 856 static void 857 mrvl_dev_close(struct rte_eth_dev *dev) 858 { 859 struct mrvl_priv *priv = dev->data->dev_private; 860 size_t i; 861 862 mrvl_flush_rx_queues(dev); 863 mrvl_flush_tx_shadow_queues(dev); 864 mrvl_flow_deinit(dev); 865 mrvl_mtr_deinit(dev); 866 867 for (i = 0; i < priv->ppio_params.inqs_params.num_tcs; ++i) { 868 struct pp2_ppio_tc_params *tc_params = 869 &priv->ppio_params.inqs_params.tcs_params[i]; 870 871 if (tc_params->inqs_params) { 872 rte_free(tc_params->inqs_params); 873 tc_params->inqs_params = NULL; 874 } 875 } 876 877 if (priv->cls_tbl) { 878 pp2_cls_tbl_deinit(priv->cls_tbl); 879 priv->cls_tbl = NULL; 880 } 881 882 if (priv->qos_tbl) { 883 pp2_cls_qos_tbl_deinit(priv->qos_tbl); 884 priv->qos_tbl = NULL; 885 } 886 887 mrvl_flush_bpool(dev); 888 mrvl_tm_deinit(dev); 889 890 if (priv->ppio) { 891 pp2_ppio_deinit(priv->ppio); 892 priv->ppio = NULL; 893 } 894 895 /* policer must be released after ppio deinitialization */ 896 if (priv->default_policer) { 897 pp2_cls_plcr_deinit(priv->default_policer); 898 priv->default_policer = NULL; 899 } 900 } 901 902 /** 903 * DPDK callback to retrieve physical link information. 904 * 905 * @param dev 906 * Pointer to Ethernet device structure. 907 * @param wait_to_complete 908 * Wait for request completion (ignored). 909 * 910 * @return 911 * 0 on success, negative error value otherwise. 912 */ 913 static int 914 mrvl_link_update(struct rte_eth_dev *dev, int wait_to_complete __rte_unused) 915 { 916 /* 917 * TODO 918 * once MUSDK provides necessary API use it here 919 */ 920 struct mrvl_priv *priv = dev->data->dev_private; 921 struct ethtool_cmd edata; 922 struct ifreq req; 923 int ret, fd, link_up; 924 925 if (!priv->ppio) 926 return -EPERM; 927 928 edata.cmd = ETHTOOL_GSET; 929 930 strcpy(req.ifr_name, dev->data->name); 931 req.ifr_data = (void *)&edata; 932 933 fd = socket(AF_INET, SOCK_DGRAM, 0); 934 if (fd == -1) 935 return -EFAULT; 936 937 ret = ioctl(fd, SIOCETHTOOL, &req); 938 if (ret == -1) { 939 close(fd); 940 return -EFAULT; 941 } 942 943 close(fd); 944 945 switch (ethtool_cmd_speed(&edata)) { 946 case SPEED_10: 947 dev->data->dev_link.link_speed = ETH_SPEED_NUM_10M; 948 break; 949 case SPEED_100: 950 dev->data->dev_link.link_speed = ETH_SPEED_NUM_100M; 951 break; 952 case SPEED_1000: 953 dev->data->dev_link.link_speed = ETH_SPEED_NUM_1G; 954 break; 955 case SPEED_10000: 956 dev->data->dev_link.link_speed = ETH_SPEED_NUM_10G; 957 break; 958 default: 959 dev->data->dev_link.link_speed = ETH_SPEED_NUM_NONE; 960 } 961 962 dev->data->dev_link.link_duplex = edata.duplex ? ETH_LINK_FULL_DUPLEX : 963 ETH_LINK_HALF_DUPLEX; 964 dev->data->dev_link.link_autoneg = edata.autoneg ? ETH_LINK_AUTONEG : 965 ETH_LINK_FIXED; 966 pp2_ppio_get_link_state(priv->ppio, &link_up); 967 dev->data->dev_link.link_status = link_up ? ETH_LINK_UP : ETH_LINK_DOWN; 968 969 return 0; 970 } 971 972 /** 973 * DPDK callback to enable promiscuous mode. 974 * 975 * @param dev 976 * Pointer to Ethernet device structure. 977 */ 978 static void 979 mrvl_promiscuous_enable(struct rte_eth_dev *dev) 980 { 981 struct mrvl_priv *priv = dev->data->dev_private; 982 int ret; 983 984 if (!priv->ppio) 985 return; 986 987 if (priv->isolated) 988 return; 989 990 ret = pp2_ppio_set_promisc(priv->ppio, 1); 991 if (ret) 992 MRVL_LOG(ERR, "Failed to enable promiscuous mode"); 993 } 994 995 /** 996 * DPDK callback to enable allmulti mode. 997 * 998 * @param dev 999 * Pointer to Ethernet device structure. 1000 */ 1001 static void 1002 mrvl_allmulticast_enable(struct rte_eth_dev *dev) 1003 { 1004 struct mrvl_priv *priv = dev->data->dev_private; 1005 int ret; 1006 1007 if (!priv->ppio) 1008 return; 1009 1010 if (priv->isolated) 1011 return; 1012 1013 ret = pp2_ppio_set_mc_promisc(priv->ppio, 1); 1014 if (ret) 1015 MRVL_LOG(ERR, "Failed enable all-multicast mode"); 1016 } 1017 1018 /** 1019 * DPDK callback to disable promiscuous mode. 1020 * 1021 * @param dev 1022 * Pointer to Ethernet device structure. 1023 */ 1024 static void 1025 mrvl_promiscuous_disable(struct rte_eth_dev *dev) 1026 { 1027 struct mrvl_priv *priv = dev->data->dev_private; 1028 int ret; 1029 1030 if (!priv->ppio) 1031 return; 1032 1033 ret = pp2_ppio_set_promisc(priv->ppio, 0); 1034 if (ret) 1035 MRVL_LOG(ERR, "Failed to disable promiscuous mode"); 1036 } 1037 1038 /** 1039 * DPDK callback to disable allmulticast mode. 1040 * 1041 * @param dev 1042 * Pointer to Ethernet device structure. 1043 */ 1044 static void 1045 mrvl_allmulticast_disable(struct rte_eth_dev *dev) 1046 { 1047 struct mrvl_priv *priv = dev->data->dev_private; 1048 int ret; 1049 1050 if (!priv->ppio) 1051 return; 1052 1053 ret = pp2_ppio_set_mc_promisc(priv->ppio, 0); 1054 if (ret) 1055 MRVL_LOG(ERR, "Failed to disable all-multicast mode"); 1056 } 1057 1058 /** 1059 * DPDK callback to remove a MAC address. 1060 * 1061 * @param dev 1062 * Pointer to Ethernet device structure. 1063 * @param index 1064 * MAC address index. 1065 */ 1066 static void 1067 mrvl_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index) 1068 { 1069 struct mrvl_priv *priv = dev->data->dev_private; 1070 char buf[ETHER_ADDR_FMT_SIZE]; 1071 int ret; 1072 1073 if (!priv->ppio) 1074 return; 1075 1076 if (priv->isolated) 1077 return; 1078 1079 ret = pp2_ppio_remove_mac_addr(priv->ppio, 1080 dev->data->mac_addrs[index].addr_bytes); 1081 if (ret) { 1082 ether_format_addr(buf, sizeof(buf), 1083 &dev->data->mac_addrs[index]); 1084 MRVL_LOG(ERR, "Failed to remove mac %s", buf); 1085 } 1086 } 1087 1088 /** 1089 * DPDK callback to add a MAC address. 1090 * 1091 * @param dev 1092 * Pointer to Ethernet device structure. 1093 * @param mac_addr 1094 * MAC address to register. 1095 * @param index 1096 * MAC address index. 1097 * @param vmdq 1098 * VMDq pool index to associate address with (unused). 1099 * 1100 * @return 1101 * 0 on success, negative error value otherwise. 1102 */ 1103 static int 1104 mrvl_mac_addr_add(struct rte_eth_dev *dev, struct ether_addr *mac_addr, 1105 uint32_t index, uint32_t vmdq __rte_unused) 1106 { 1107 struct mrvl_priv *priv = dev->data->dev_private; 1108 char buf[ETHER_ADDR_FMT_SIZE]; 1109 int ret; 1110 1111 if (priv->isolated) 1112 return -ENOTSUP; 1113 1114 if (index == 0) 1115 /* For setting index 0, mrvl_mac_addr_set() should be used.*/ 1116 return -1; 1117 1118 if (!priv->ppio) 1119 return 0; 1120 1121 /* 1122 * Maximum number of uc addresses can be tuned via kernel module mvpp2x 1123 * parameter uc_filter_max. Maximum number of mc addresses is then 1124 * MRVL_MAC_ADDRS_MAX - uc_filter_max. Currently it defaults to 4 and 1125 * 21 respectively. 1126 * 1127 * If more than uc_filter_max uc addresses were added to filter list 1128 * then NIC will switch to promiscuous mode automatically. 1129 * 1130 * If more than MRVL_MAC_ADDRS_MAX - uc_filter_max number mc addresses 1131 * were added to filter list then NIC will switch to all-multicast mode 1132 * automatically. 1133 */ 1134 ret = pp2_ppio_add_mac_addr(priv->ppio, mac_addr->addr_bytes); 1135 if (ret) { 1136 ether_format_addr(buf, sizeof(buf), mac_addr); 1137 MRVL_LOG(ERR, "Failed to add mac %s", buf); 1138 return -1; 1139 } 1140 1141 return 0; 1142 } 1143 1144 /** 1145 * DPDK callback to set the primary MAC address. 1146 * 1147 * @param dev 1148 * Pointer to Ethernet device structure. 1149 * @param mac_addr 1150 * MAC address to register. 1151 * 1152 * @return 1153 * 0 on success, negative error value otherwise. 1154 */ 1155 static int 1156 mrvl_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr) 1157 { 1158 struct mrvl_priv *priv = dev->data->dev_private; 1159 int ret; 1160 1161 if (!priv->ppio) 1162 return 0; 1163 1164 if (priv->isolated) 1165 return -ENOTSUP; 1166 1167 ret = pp2_ppio_set_mac_addr(priv->ppio, mac_addr->addr_bytes); 1168 if (ret) { 1169 char buf[ETHER_ADDR_FMT_SIZE]; 1170 ether_format_addr(buf, sizeof(buf), mac_addr); 1171 MRVL_LOG(ERR, "Failed to set mac to %s", buf); 1172 } 1173 1174 return ret; 1175 } 1176 1177 /** 1178 * DPDK callback to get device statistics. 1179 * 1180 * @param dev 1181 * Pointer to Ethernet device structure. 1182 * @param stats 1183 * Stats structure output buffer. 1184 * 1185 * @return 1186 * 0 on success, negative error value otherwise. 1187 */ 1188 static int 1189 mrvl_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) 1190 { 1191 struct mrvl_priv *priv = dev->data->dev_private; 1192 struct pp2_ppio_statistics ppio_stats; 1193 uint64_t drop_mac = 0; 1194 unsigned int i, idx, ret; 1195 1196 if (!priv->ppio) 1197 return -EPERM; 1198 1199 for (i = 0; i < dev->data->nb_rx_queues; i++) { 1200 struct mrvl_rxq *rxq = dev->data->rx_queues[i]; 1201 struct pp2_ppio_inq_statistics rx_stats; 1202 1203 if (!rxq) 1204 continue; 1205 1206 idx = rxq->queue_id; 1207 if (unlikely(idx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)) { 1208 MRVL_LOG(ERR, 1209 "rx queue %d stats out of range (0 - %d)", 1210 idx, RTE_ETHDEV_QUEUE_STAT_CNTRS - 1); 1211 continue; 1212 } 1213 1214 ret = pp2_ppio_inq_get_statistics(priv->ppio, 1215 priv->rxq_map[idx].tc, 1216 priv->rxq_map[idx].inq, 1217 &rx_stats, 0); 1218 if (unlikely(ret)) { 1219 MRVL_LOG(ERR, 1220 "Failed to update rx queue %d stats", idx); 1221 break; 1222 } 1223 1224 stats->q_ibytes[idx] = rxq->bytes_recv; 1225 stats->q_ipackets[idx] = rx_stats.enq_desc - rxq->drop_mac; 1226 stats->q_errors[idx] = rx_stats.drop_early + 1227 rx_stats.drop_fullq + 1228 rx_stats.drop_bm + 1229 rxq->drop_mac; 1230 stats->ibytes += rxq->bytes_recv; 1231 drop_mac += rxq->drop_mac; 1232 } 1233 1234 for (i = 0; i < dev->data->nb_tx_queues; i++) { 1235 struct mrvl_txq *txq = dev->data->tx_queues[i]; 1236 struct pp2_ppio_outq_statistics tx_stats; 1237 1238 if (!txq) 1239 continue; 1240 1241 idx = txq->queue_id; 1242 if (unlikely(idx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)) { 1243 MRVL_LOG(ERR, 1244 "tx queue %d stats out of range (0 - %d)", 1245 idx, RTE_ETHDEV_QUEUE_STAT_CNTRS - 1); 1246 } 1247 1248 ret = pp2_ppio_outq_get_statistics(priv->ppio, idx, 1249 &tx_stats, 0); 1250 if (unlikely(ret)) { 1251 MRVL_LOG(ERR, 1252 "Failed to update tx queue %d stats", idx); 1253 break; 1254 } 1255 1256 stats->q_opackets[idx] = tx_stats.deq_desc; 1257 stats->q_obytes[idx] = txq->bytes_sent; 1258 stats->obytes += txq->bytes_sent; 1259 } 1260 1261 ret = pp2_ppio_get_statistics(priv->ppio, &ppio_stats, 0); 1262 if (unlikely(ret)) { 1263 MRVL_LOG(ERR, "Failed to update port statistics"); 1264 return ret; 1265 } 1266 1267 stats->ipackets += ppio_stats.rx_packets - drop_mac; 1268 stats->opackets += ppio_stats.tx_packets; 1269 stats->imissed += ppio_stats.rx_fullq_dropped + 1270 ppio_stats.rx_bm_dropped + 1271 ppio_stats.rx_early_dropped + 1272 ppio_stats.rx_fifo_dropped + 1273 ppio_stats.rx_cls_dropped; 1274 stats->ierrors = drop_mac; 1275 1276 return 0; 1277 } 1278 1279 /** 1280 * DPDK callback to clear device statistics. 1281 * 1282 * @param dev 1283 * Pointer to Ethernet device structure. 1284 */ 1285 static void 1286 mrvl_stats_reset(struct rte_eth_dev *dev) 1287 { 1288 struct mrvl_priv *priv = dev->data->dev_private; 1289 int i; 1290 1291 if (!priv->ppio) 1292 return; 1293 1294 for (i = 0; i < dev->data->nb_rx_queues; i++) { 1295 struct mrvl_rxq *rxq = dev->data->rx_queues[i]; 1296 1297 pp2_ppio_inq_get_statistics(priv->ppio, priv->rxq_map[i].tc, 1298 priv->rxq_map[i].inq, NULL, 1); 1299 rxq->bytes_recv = 0; 1300 rxq->drop_mac = 0; 1301 } 1302 1303 for (i = 0; i < dev->data->nb_tx_queues; i++) { 1304 struct mrvl_txq *txq = dev->data->tx_queues[i]; 1305 1306 pp2_ppio_outq_get_statistics(priv->ppio, i, NULL, 1); 1307 txq->bytes_sent = 0; 1308 } 1309 1310 pp2_ppio_get_statistics(priv->ppio, NULL, 1); 1311 } 1312 1313 /** 1314 * DPDK callback to get extended statistics. 1315 * 1316 * @param dev 1317 * Pointer to Ethernet device structure. 1318 * @param stats 1319 * Pointer to xstats table. 1320 * @param n 1321 * Number of entries in xstats table. 1322 * @return 1323 * Negative value on error, number of read xstats otherwise. 1324 */ 1325 static int 1326 mrvl_xstats_get(struct rte_eth_dev *dev, 1327 struct rte_eth_xstat *stats, unsigned int n) 1328 { 1329 struct mrvl_priv *priv = dev->data->dev_private; 1330 struct pp2_ppio_statistics ppio_stats; 1331 unsigned int i; 1332 1333 if (!stats) 1334 return 0; 1335 1336 pp2_ppio_get_statistics(priv->ppio, &ppio_stats, 0); 1337 for (i = 0; i < n && i < RTE_DIM(mrvl_xstats_tbl); i++) { 1338 uint64_t val; 1339 1340 if (mrvl_xstats_tbl[i].size == sizeof(uint32_t)) 1341 val = *(uint32_t *)((uint8_t *)&ppio_stats + 1342 mrvl_xstats_tbl[i].offset); 1343 else if (mrvl_xstats_tbl[i].size == sizeof(uint64_t)) 1344 val = *(uint64_t *)((uint8_t *)&ppio_stats + 1345 mrvl_xstats_tbl[i].offset); 1346 else 1347 return -EINVAL; 1348 1349 stats[i].id = i; 1350 stats[i].value = val; 1351 } 1352 1353 return n; 1354 } 1355 1356 /** 1357 * DPDK callback to reset extended statistics. 1358 * 1359 * @param dev 1360 * Pointer to Ethernet device structure. 1361 */ 1362 static void 1363 mrvl_xstats_reset(struct rte_eth_dev *dev) 1364 { 1365 mrvl_stats_reset(dev); 1366 } 1367 1368 /** 1369 * DPDK callback to get extended statistics names. 1370 * 1371 * @param dev (unused) 1372 * Pointer to Ethernet device structure. 1373 * @param xstats_names 1374 * Pointer to xstats names table. 1375 * @param size 1376 * Size of the xstats names table. 1377 * @return 1378 * Number of read names. 1379 */ 1380 static int 1381 mrvl_xstats_get_names(struct rte_eth_dev *dev __rte_unused, 1382 struct rte_eth_xstat_name *xstats_names, 1383 unsigned int size) 1384 { 1385 unsigned int i; 1386 1387 if (!xstats_names) 1388 return RTE_DIM(mrvl_xstats_tbl); 1389 1390 for (i = 0; i < size && i < RTE_DIM(mrvl_xstats_tbl); i++) 1391 snprintf(xstats_names[i].name, RTE_ETH_XSTATS_NAME_SIZE, "%s", 1392 mrvl_xstats_tbl[i].name); 1393 1394 return size; 1395 } 1396 1397 /** 1398 * DPDK callback to get information about the device. 1399 * 1400 * @param dev 1401 * Pointer to Ethernet device structure (unused). 1402 * @param info 1403 * Info structure output buffer. 1404 */ 1405 static void 1406 mrvl_dev_infos_get(struct rte_eth_dev *dev __rte_unused, 1407 struct rte_eth_dev_info *info) 1408 { 1409 info->speed_capa = ETH_LINK_SPEED_10M | 1410 ETH_LINK_SPEED_100M | 1411 ETH_LINK_SPEED_1G | 1412 ETH_LINK_SPEED_10G; 1413 1414 info->max_rx_queues = MRVL_PP2_RXQ_MAX; 1415 info->max_tx_queues = MRVL_PP2_TXQ_MAX; 1416 info->max_mac_addrs = MRVL_MAC_ADDRS_MAX; 1417 1418 info->rx_desc_lim.nb_max = MRVL_PP2_RXD_MAX; 1419 info->rx_desc_lim.nb_min = MRVL_PP2_RXD_MIN; 1420 info->rx_desc_lim.nb_align = MRVL_PP2_RXD_ALIGN; 1421 1422 info->tx_desc_lim.nb_max = MRVL_PP2_TXD_MAX; 1423 info->tx_desc_lim.nb_min = MRVL_PP2_TXD_MIN; 1424 info->tx_desc_lim.nb_align = MRVL_PP2_TXD_ALIGN; 1425 1426 info->rx_offload_capa = MRVL_RX_OFFLOADS; 1427 info->rx_queue_offload_capa = MRVL_RX_OFFLOADS; 1428 1429 info->tx_offload_capa = MRVL_TX_OFFLOADS; 1430 info->tx_queue_offload_capa = MRVL_TX_OFFLOADS; 1431 1432 info->flow_type_rss_offloads = ETH_RSS_IPV4 | 1433 ETH_RSS_NONFRAG_IPV4_TCP | 1434 ETH_RSS_NONFRAG_IPV4_UDP; 1435 1436 /* By default packets are dropped if no descriptors are available */ 1437 info->default_rxconf.rx_drop_en = 1; 1438 1439 info->max_rx_pktlen = MRVL_PKT_SIZE_MAX; 1440 } 1441 1442 /** 1443 * Return supported packet types. 1444 * 1445 * @param dev 1446 * Pointer to Ethernet device structure (unused). 1447 * 1448 * @return 1449 * Const pointer to the table with supported packet types. 1450 */ 1451 static const uint32_t * 1452 mrvl_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused) 1453 { 1454 static const uint32_t ptypes[] = { 1455 RTE_PTYPE_L2_ETHER, 1456 RTE_PTYPE_L2_ETHER_VLAN, 1457 RTE_PTYPE_L2_ETHER_QINQ, 1458 RTE_PTYPE_L3_IPV4, 1459 RTE_PTYPE_L3_IPV4_EXT, 1460 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN, 1461 RTE_PTYPE_L3_IPV6, 1462 RTE_PTYPE_L3_IPV6_EXT, 1463 RTE_PTYPE_L2_ETHER_ARP, 1464 RTE_PTYPE_L4_TCP, 1465 RTE_PTYPE_L4_UDP 1466 }; 1467 1468 return ptypes; 1469 } 1470 1471 /** 1472 * DPDK callback to get information about specific receive queue. 1473 * 1474 * @param dev 1475 * Pointer to Ethernet device structure. 1476 * @param rx_queue_id 1477 * Receive queue index. 1478 * @param qinfo 1479 * Receive queue information structure. 1480 */ 1481 static void mrvl_rxq_info_get(struct rte_eth_dev *dev, uint16_t rx_queue_id, 1482 struct rte_eth_rxq_info *qinfo) 1483 { 1484 struct mrvl_rxq *q = dev->data->rx_queues[rx_queue_id]; 1485 struct mrvl_priv *priv = dev->data->dev_private; 1486 int inq = priv->rxq_map[rx_queue_id].inq; 1487 int tc = priv->rxq_map[rx_queue_id].tc; 1488 struct pp2_ppio_tc_params *tc_params = 1489 &priv->ppio_params.inqs_params.tcs_params[tc]; 1490 1491 qinfo->mp = q->mp; 1492 qinfo->nb_desc = tc_params->inqs_params[inq].size; 1493 } 1494 1495 /** 1496 * DPDK callback to get information about specific transmit queue. 1497 * 1498 * @param dev 1499 * Pointer to Ethernet device structure. 1500 * @param tx_queue_id 1501 * Transmit queue index. 1502 * @param qinfo 1503 * Transmit queue information structure. 1504 */ 1505 static void mrvl_txq_info_get(struct rte_eth_dev *dev, uint16_t tx_queue_id, 1506 struct rte_eth_txq_info *qinfo) 1507 { 1508 struct mrvl_priv *priv = dev->data->dev_private; 1509 struct mrvl_txq *txq = dev->data->tx_queues[tx_queue_id]; 1510 1511 qinfo->nb_desc = 1512 priv->ppio_params.outqs_params.outqs_params[tx_queue_id].size; 1513 qinfo->conf.tx_deferred_start = txq->tx_deferred_start; 1514 } 1515 1516 /** 1517 * DPDK callback to Configure a VLAN filter. 1518 * 1519 * @param dev 1520 * Pointer to Ethernet device structure. 1521 * @param vlan_id 1522 * VLAN ID to filter. 1523 * @param on 1524 * Toggle filter. 1525 * 1526 * @return 1527 * 0 on success, negative error value otherwise. 1528 */ 1529 static int 1530 mrvl_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) 1531 { 1532 struct mrvl_priv *priv = dev->data->dev_private; 1533 1534 if (!priv->ppio) 1535 return -EPERM; 1536 1537 if (priv->isolated) 1538 return -ENOTSUP; 1539 1540 return on ? pp2_ppio_add_vlan(priv->ppio, vlan_id) : 1541 pp2_ppio_remove_vlan(priv->ppio, vlan_id); 1542 } 1543 1544 /** 1545 * Release buffers to hardware bpool (buffer-pool) 1546 * 1547 * @param rxq 1548 * Receive queue pointer. 1549 * @param num 1550 * Number of buffers to release to bpool. 1551 * 1552 * @return 1553 * 0 on success, negative error value otherwise. 1554 */ 1555 static int 1556 mrvl_fill_bpool(struct mrvl_rxq *rxq, int num) 1557 { 1558 struct buff_release_entry entries[MRVL_PP2_RXD_MAX]; 1559 struct rte_mbuf *mbufs[MRVL_PP2_RXD_MAX]; 1560 int i, ret; 1561 unsigned int core_id; 1562 struct pp2_hif *hif; 1563 struct pp2_bpool *bpool; 1564 1565 core_id = rte_lcore_id(); 1566 if (core_id == LCORE_ID_ANY) 1567 core_id = 0; 1568 1569 hif = mrvl_get_hif(rxq->priv, core_id); 1570 if (!hif) 1571 return -1; 1572 1573 bpool = rxq->priv->bpool; 1574 1575 ret = rte_pktmbuf_alloc_bulk(rxq->mp, mbufs, num); 1576 if (ret) 1577 return ret; 1578 1579 if (cookie_addr_high == MRVL_COOKIE_ADDR_INVALID) 1580 cookie_addr_high = 1581 (uint64_t)mbufs[0] & MRVL_COOKIE_HIGH_ADDR_MASK; 1582 1583 for (i = 0; i < num; i++) { 1584 if (((uint64_t)mbufs[i] & MRVL_COOKIE_HIGH_ADDR_MASK) 1585 != cookie_addr_high) { 1586 MRVL_LOG(ERR, 1587 "mbuf virtual addr high 0x%lx out of range", 1588 (uint64_t)mbufs[i] >> 32); 1589 goto out; 1590 } 1591 1592 entries[i].buff.addr = 1593 rte_mbuf_data_iova_default(mbufs[i]); 1594 entries[i].buff.cookie = (uint64_t)mbufs[i]; 1595 entries[i].bpool = bpool; 1596 } 1597 1598 pp2_bpool_put_buffs(hif, entries, (uint16_t *)&i); 1599 mrvl_port_bpool_size[bpool->pp2_id][bpool->id][core_id] += i; 1600 1601 if (i != num) 1602 goto out; 1603 1604 return 0; 1605 out: 1606 for (; i < num; i++) 1607 rte_pktmbuf_free(mbufs[i]); 1608 1609 return -1; 1610 } 1611 1612 /** 1613 * DPDK callback to configure the receive queue. 1614 * 1615 * @param dev 1616 * Pointer to Ethernet device structure. 1617 * @param idx 1618 * RX queue index. 1619 * @param desc 1620 * Number of descriptors to configure in queue. 1621 * @param socket 1622 * NUMA socket on which memory must be allocated. 1623 * @param conf 1624 * Thresholds parameters. 1625 * @param mp 1626 * Memory pool for buffer allocations. 1627 * 1628 * @return 1629 * 0 on success, negative error value otherwise. 1630 */ 1631 static int 1632 mrvl_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, 1633 unsigned int socket, 1634 const struct rte_eth_rxconf *conf, 1635 struct rte_mempool *mp) 1636 { 1637 struct mrvl_priv *priv = dev->data->dev_private; 1638 struct mrvl_rxq *rxq; 1639 uint32_t frame_size, buf_size = rte_pktmbuf_data_room_size(mp); 1640 uint32_t max_rx_pkt_len = dev->data->dev_conf.rxmode.max_rx_pkt_len; 1641 int ret, tc, inq; 1642 uint64_t offloads; 1643 1644 offloads = conf->offloads | dev->data->dev_conf.rxmode.offloads; 1645 1646 if (priv->rxq_map[idx].tc == MRVL_UNKNOWN_TC) { 1647 /* 1648 * Unknown TC mapping, mapping will not have a correct queue. 1649 */ 1650 MRVL_LOG(ERR, "Unknown TC mapping for queue %hu eth%hhu", 1651 idx, priv->ppio_id); 1652 return -EFAULT; 1653 } 1654 1655 frame_size = buf_size - RTE_PKTMBUF_HEADROOM - MRVL_PKT_EFFEC_OFFS; 1656 if (frame_size < max_rx_pkt_len) { 1657 MRVL_LOG(WARNING, 1658 "Mbuf size must be increased to %u bytes to hold up " 1659 "to %u bytes of data.", 1660 buf_size + max_rx_pkt_len - frame_size, 1661 max_rx_pkt_len); 1662 dev->data->dev_conf.rxmode.max_rx_pkt_len = frame_size; 1663 MRVL_LOG(INFO, "Setting max rx pkt len to %u", 1664 dev->data->dev_conf.rxmode.max_rx_pkt_len); 1665 } 1666 1667 if (dev->data->rx_queues[idx]) { 1668 rte_free(dev->data->rx_queues[idx]); 1669 dev->data->rx_queues[idx] = NULL; 1670 } 1671 1672 rxq = rte_zmalloc_socket("rxq", sizeof(*rxq), 0, socket); 1673 if (!rxq) 1674 return -ENOMEM; 1675 1676 rxq->priv = priv; 1677 rxq->mp = mp; 1678 rxq->cksum_enabled = offloads & DEV_RX_OFFLOAD_IPV4_CKSUM; 1679 rxq->queue_id = idx; 1680 rxq->port_id = dev->data->port_id; 1681 mrvl_port_to_bpool_lookup[rxq->port_id] = priv->bpool; 1682 1683 tc = priv->rxq_map[rxq->queue_id].tc, 1684 inq = priv->rxq_map[rxq->queue_id].inq; 1685 priv->ppio_params.inqs_params.tcs_params[tc].inqs_params[inq].size = 1686 desc; 1687 1688 ret = mrvl_fill_bpool(rxq, desc); 1689 if (ret) { 1690 rte_free(rxq); 1691 return ret; 1692 } 1693 1694 priv->bpool_init_size += desc; 1695 1696 dev->data->rx_queues[idx] = rxq; 1697 1698 return 0; 1699 } 1700 1701 /** 1702 * DPDK callback to release the receive queue. 1703 * 1704 * @param rxq 1705 * Generic receive queue pointer. 1706 */ 1707 static void 1708 mrvl_rx_queue_release(void *rxq) 1709 { 1710 struct mrvl_rxq *q = rxq; 1711 struct pp2_ppio_tc_params *tc_params; 1712 int i, num, tc, inq; 1713 struct pp2_hif *hif; 1714 unsigned int core_id = rte_lcore_id(); 1715 1716 if (core_id == LCORE_ID_ANY) 1717 core_id = 0; 1718 1719 if (!q) 1720 return; 1721 1722 hif = mrvl_get_hif(q->priv, core_id); 1723 1724 if (!hif) 1725 return; 1726 1727 tc = q->priv->rxq_map[q->queue_id].tc; 1728 inq = q->priv->rxq_map[q->queue_id].inq; 1729 tc_params = &q->priv->ppio_params.inqs_params.tcs_params[tc]; 1730 num = tc_params->inqs_params[inq].size; 1731 for (i = 0; i < num; i++) { 1732 struct pp2_buff_inf inf; 1733 uint64_t addr; 1734 1735 pp2_bpool_get_buff(hif, q->priv->bpool, &inf); 1736 addr = cookie_addr_high | inf.cookie; 1737 rte_pktmbuf_free((struct rte_mbuf *)addr); 1738 } 1739 1740 rte_free(q); 1741 } 1742 1743 /** 1744 * DPDK callback to configure the transmit queue. 1745 * 1746 * @param dev 1747 * Pointer to Ethernet device structure. 1748 * @param idx 1749 * Transmit queue index. 1750 * @param desc 1751 * Number of descriptors to configure in the queue. 1752 * @param socket 1753 * NUMA socket on which memory must be allocated. 1754 * @param conf 1755 * Tx queue configuration parameters. 1756 * 1757 * @return 1758 * 0 on success, negative error value otherwise. 1759 */ 1760 static int 1761 mrvl_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, 1762 unsigned int socket, 1763 const struct rte_eth_txconf *conf) 1764 { 1765 struct mrvl_priv *priv = dev->data->dev_private; 1766 struct mrvl_txq *txq; 1767 1768 if (dev->data->tx_queues[idx]) { 1769 rte_free(dev->data->tx_queues[idx]); 1770 dev->data->tx_queues[idx] = NULL; 1771 } 1772 1773 txq = rte_zmalloc_socket("txq", sizeof(*txq), 0, socket); 1774 if (!txq) 1775 return -ENOMEM; 1776 1777 txq->priv = priv; 1778 txq->queue_id = idx; 1779 txq->port_id = dev->data->port_id; 1780 txq->tx_deferred_start = conf->tx_deferred_start; 1781 dev->data->tx_queues[idx] = txq; 1782 1783 priv->ppio_params.outqs_params.outqs_params[idx].size = desc; 1784 1785 return 0; 1786 } 1787 1788 /** 1789 * DPDK callback to release the transmit queue. 1790 * 1791 * @param txq 1792 * Generic transmit queue pointer. 1793 */ 1794 static void 1795 mrvl_tx_queue_release(void *txq) 1796 { 1797 struct mrvl_txq *q = txq; 1798 1799 if (!q) 1800 return; 1801 1802 rte_free(q); 1803 } 1804 1805 /** 1806 * DPDK callback to get flow control configuration. 1807 * 1808 * @param dev 1809 * Pointer to Ethernet device structure. 1810 * @param fc_conf 1811 * Pointer to the flow control configuration. 1812 * 1813 * @return 1814 * 0 on success, negative error value otherwise. 1815 */ 1816 static int 1817 mrvl_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) 1818 { 1819 struct mrvl_priv *priv = dev->data->dev_private; 1820 int ret, en; 1821 1822 if (!priv) 1823 return -EPERM; 1824 1825 ret = pp2_ppio_get_rx_pause(priv->ppio, &en); 1826 if (ret) { 1827 MRVL_LOG(ERR, "Failed to read rx pause state"); 1828 return ret; 1829 } 1830 1831 fc_conf->mode = en ? RTE_FC_RX_PAUSE : RTE_FC_NONE; 1832 1833 return 0; 1834 } 1835 1836 /** 1837 * DPDK callback to set flow control configuration. 1838 * 1839 * @param dev 1840 * Pointer to Ethernet device structure. 1841 * @param fc_conf 1842 * Pointer to the flow control configuration. 1843 * 1844 * @return 1845 * 0 on success, negative error value otherwise. 1846 */ 1847 static int 1848 mrvl_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) 1849 { 1850 struct mrvl_priv *priv = dev->data->dev_private; 1851 1852 if (!priv) 1853 return -EPERM; 1854 1855 if (fc_conf->high_water || 1856 fc_conf->low_water || 1857 fc_conf->pause_time || 1858 fc_conf->mac_ctrl_frame_fwd || 1859 fc_conf->autoneg) { 1860 MRVL_LOG(ERR, "Flowctrl parameter is not supported"); 1861 1862 return -EINVAL; 1863 } 1864 1865 if (fc_conf->mode == RTE_FC_NONE || 1866 fc_conf->mode == RTE_FC_RX_PAUSE) { 1867 int ret, en; 1868 1869 en = fc_conf->mode == RTE_FC_NONE ? 0 : 1; 1870 ret = pp2_ppio_set_rx_pause(priv->ppio, en); 1871 if (ret) 1872 MRVL_LOG(ERR, 1873 "Failed to change flowctrl on RX side"); 1874 1875 return ret; 1876 } 1877 1878 return 0; 1879 } 1880 1881 /** 1882 * Update RSS hash configuration 1883 * 1884 * @param dev 1885 * Pointer to Ethernet device structure. 1886 * @param rss_conf 1887 * Pointer to RSS configuration. 1888 * 1889 * @return 1890 * 0 on success, negative error value otherwise. 1891 */ 1892 static int 1893 mrvl_rss_hash_update(struct rte_eth_dev *dev, 1894 struct rte_eth_rss_conf *rss_conf) 1895 { 1896 struct mrvl_priv *priv = dev->data->dev_private; 1897 1898 if (priv->isolated) 1899 return -ENOTSUP; 1900 1901 return mrvl_configure_rss(priv, rss_conf); 1902 } 1903 1904 /** 1905 * DPDK callback to get RSS hash configuration. 1906 * 1907 * @param dev 1908 * Pointer to Ethernet device structure. 1909 * @rss_conf 1910 * Pointer to RSS configuration. 1911 * 1912 * @return 1913 * Always 0. 1914 */ 1915 static int 1916 mrvl_rss_hash_conf_get(struct rte_eth_dev *dev, 1917 struct rte_eth_rss_conf *rss_conf) 1918 { 1919 struct mrvl_priv *priv = dev->data->dev_private; 1920 enum pp2_ppio_hash_type hash_type = 1921 priv->ppio_params.inqs_params.hash_type; 1922 1923 rss_conf->rss_key = NULL; 1924 1925 if (hash_type == PP2_PPIO_HASH_T_NONE) 1926 rss_conf->rss_hf = 0; 1927 else if (hash_type == PP2_PPIO_HASH_T_2_TUPLE) 1928 rss_conf->rss_hf = ETH_RSS_IPV4; 1929 else if (hash_type == PP2_PPIO_HASH_T_5_TUPLE && priv->rss_hf_tcp) 1930 rss_conf->rss_hf = ETH_RSS_NONFRAG_IPV4_TCP; 1931 else if (hash_type == PP2_PPIO_HASH_T_5_TUPLE && !priv->rss_hf_tcp) 1932 rss_conf->rss_hf = ETH_RSS_NONFRAG_IPV4_UDP; 1933 1934 return 0; 1935 } 1936 1937 /** 1938 * DPDK callback to get rte_flow callbacks. 1939 * 1940 * @param dev 1941 * Pointer to the device structure. 1942 * @param filer_type 1943 * Flow filter type. 1944 * @param filter_op 1945 * Flow filter operation. 1946 * @param arg 1947 * Pointer to pass the flow ops. 1948 * 1949 * @return 1950 * 0 on success, negative error value otherwise. 1951 */ 1952 static int 1953 mrvl_eth_filter_ctrl(struct rte_eth_dev *dev __rte_unused, 1954 enum rte_filter_type filter_type, 1955 enum rte_filter_op filter_op, void *arg) 1956 { 1957 switch (filter_type) { 1958 case RTE_ETH_FILTER_GENERIC: 1959 if (filter_op != RTE_ETH_FILTER_GET) 1960 return -EINVAL; 1961 *(const void **)arg = &mrvl_flow_ops; 1962 return 0; 1963 default: 1964 MRVL_LOG(WARNING, "Filter type (%d) not supported", 1965 filter_type); 1966 return -EINVAL; 1967 } 1968 } 1969 1970 /** 1971 * DPDK callback to get rte_mtr callbacks. 1972 * 1973 * @param dev 1974 * Pointer to the device structure. 1975 * @param ops 1976 * Pointer to pass the mtr ops. 1977 * 1978 * @return 1979 * Always 0. 1980 */ 1981 static int 1982 mrvl_mtr_ops_get(struct rte_eth_dev *dev __rte_unused, void *ops) 1983 { 1984 *(const void **)ops = &mrvl_mtr_ops; 1985 1986 return 0; 1987 } 1988 1989 /** 1990 * DPDK callback to get rte_tm callbacks. 1991 * 1992 * @param dev 1993 * Pointer to the device structure. 1994 * @param ops 1995 * Pointer to pass the tm ops. 1996 * 1997 * @return 1998 * Always 0. 1999 */ 2000 static int 2001 mrvl_tm_ops_get(struct rte_eth_dev *dev __rte_unused, void *ops) 2002 { 2003 *(const void **)ops = &mrvl_tm_ops; 2004 2005 return 0; 2006 } 2007 2008 static const struct eth_dev_ops mrvl_ops = { 2009 .dev_configure = mrvl_dev_configure, 2010 .dev_start = mrvl_dev_start, 2011 .dev_stop = mrvl_dev_stop, 2012 .dev_set_link_up = mrvl_dev_set_link_up, 2013 .dev_set_link_down = mrvl_dev_set_link_down, 2014 .dev_close = mrvl_dev_close, 2015 .link_update = mrvl_link_update, 2016 .promiscuous_enable = mrvl_promiscuous_enable, 2017 .allmulticast_enable = mrvl_allmulticast_enable, 2018 .promiscuous_disable = mrvl_promiscuous_disable, 2019 .allmulticast_disable = mrvl_allmulticast_disable, 2020 .mac_addr_remove = mrvl_mac_addr_remove, 2021 .mac_addr_add = mrvl_mac_addr_add, 2022 .mac_addr_set = mrvl_mac_addr_set, 2023 .mtu_set = mrvl_mtu_set, 2024 .stats_get = mrvl_stats_get, 2025 .stats_reset = mrvl_stats_reset, 2026 .xstats_get = mrvl_xstats_get, 2027 .xstats_reset = mrvl_xstats_reset, 2028 .xstats_get_names = mrvl_xstats_get_names, 2029 .dev_infos_get = mrvl_dev_infos_get, 2030 .dev_supported_ptypes_get = mrvl_dev_supported_ptypes_get, 2031 .rxq_info_get = mrvl_rxq_info_get, 2032 .txq_info_get = mrvl_txq_info_get, 2033 .vlan_filter_set = mrvl_vlan_filter_set, 2034 .tx_queue_start = mrvl_tx_queue_start, 2035 .tx_queue_stop = mrvl_tx_queue_stop, 2036 .rx_queue_setup = mrvl_rx_queue_setup, 2037 .rx_queue_release = mrvl_rx_queue_release, 2038 .tx_queue_setup = mrvl_tx_queue_setup, 2039 .tx_queue_release = mrvl_tx_queue_release, 2040 .flow_ctrl_get = mrvl_flow_ctrl_get, 2041 .flow_ctrl_set = mrvl_flow_ctrl_set, 2042 .rss_hash_update = mrvl_rss_hash_update, 2043 .rss_hash_conf_get = mrvl_rss_hash_conf_get, 2044 .filter_ctrl = mrvl_eth_filter_ctrl, 2045 .mtr_ops_get = mrvl_mtr_ops_get, 2046 .tm_ops_get = mrvl_tm_ops_get, 2047 }; 2048 2049 /** 2050 * Return packet type information and l3/l4 offsets. 2051 * 2052 * @param desc 2053 * Pointer to the received packet descriptor. 2054 * @param l3_offset 2055 * l3 packet offset. 2056 * @param l4_offset 2057 * l4 packet offset. 2058 * 2059 * @return 2060 * Packet type information. 2061 */ 2062 static inline uint64_t 2063 mrvl_desc_to_packet_type_and_offset(struct pp2_ppio_desc *desc, 2064 uint8_t *l3_offset, uint8_t *l4_offset) 2065 { 2066 enum pp2_inq_l3_type l3_type; 2067 enum pp2_inq_l4_type l4_type; 2068 enum pp2_inq_vlan_tag vlan_tag; 2069 uint64_t packet_type; 2070 2071 pp2_ppio_inq_desc_get_l3_info(desc, &l3_type, l3_offset); 2072 pp2_ppio_inq_desc_get_l4_info(desc, &l4_type, l4_offset); 2073 pp2_ppio_inq_desc_get_vlan_tag(desc, &vlan_tag); 2074 2075 packet_type = RTE_PTYPE_L2_ETHER; 2076 2077 switch (vlan_tag) { 2078 case PP2_INQ_VLAN_TAG_SINGLE: 2079 packet_type |= RTE_PTYPE_L2_ETHER_VLAN; 2080 break; 2081 case PP2_INQ_VLAN_TAG_DOUBLE: 2082 case PP2_INQ_VLAN_TAG_TRIPLE: 2083 packet_type |= RTE_PTYPE_L2_ETHER_QINQ; 2084 break; 2085 default: 2086 break; 2087 } 2088 2089 switch (l3_type) { 2090 case PP2_INQ_L3_TYPE_IPV4_NO_OPTS: 2091 packet_type |= RTE_PTYPE_L3_IPV4; 2092 break; 2093 case PP2_INQ_L3_TYPE_IPV4_OK: 2094 packet_type |= RTE_PTYPE_L3_IPV4_EXT; 2095 break; 2096 case PP2_INQ_L3_TYPE_IPV4_TTL_ZERO: 2097 packet_type |= RTE_PTYPE_L3_IPV4_EXT_UNKNOWN; 2098 break; 2099 case PP2_INQ_L3_TYPE_IPV6_NO_EXT: 2100 packet_type |= RTE_PTYPE_L3_IPV6; 2101 break; 2102 case PP2_INQ_L3_TYPE_IPV6_EXT: 2103 packet_type |= RTE_PTYPE_L3_IPV6_EXT; 2104 break; 2105 case PP2_INQ_L3_TYPE_ARP: 2106 packet_type |= RTE_PTYPE_L2_ETHER_ARP; 2107 /* 2108 * In case of ARP l4_offset is set to wrong value. 2109 * Set it to proper one so that later on mbuf->l3_len can be 2110 * calculated subtracting l4_offset and l3_offset. 2111 */ 2112 *l4_offset = *l3_offset + MRVL_ARP_LENGTH; 2113 break; 2114 default: 2115 MRVL_LOG(DEBUG, "Failed to recognise l3 packet type"); 2116 break; 2117 } 2118 2119 switch (l4_type) { 2120 case PP2_INQ_L4_TYPE_TCP: 2121 packet_type |= RTE_PTYPE_L4_TCP; 2122 break; 2123 case PP2_INQ_L4_TYPE_UDP: 2124 packet_type |= RTE_PTYPE_L4_UDP; 2125 break; 2126 default: 2127 MRVL_LOG(DEBUG, "Failed to recognise l4 packet type"); 2128 break; 2129 } 2130 2131 return packet_type; 2132 } 2133 2134 /** 2135 * Get offload information from the received packet descriptor. 2136 * 2137 * @param desc 2138 * Pointer to the received packet descriptor. 2139 * 2140 * @return 2141 * Mbuf offload flags. 2142 */ 2143 static inline uint64_t 2144 mrvl_desc_to_ol_flags(struct pp2_ppio_desc *desc) 2145 { 2146 uint64_t flags; 2147 enum pp2_inq_desc_status status; 2148 2149 status = pp2_ppio_inq_desc_get_l3_pkt_error(desc); 2150 if (unlikely(status != PP2_DESC_ERR_OK)) 2151 flags = PKT_RX_IP_CKSUM_BAD; 2152 else 2153 flags = PKT_RX_IP_CKSUM_GOOD; 2154 2155 status = pp2_ppio_inq_desc_get_l4_pkt_error(desc); 2156 if (unlikely(status != PP2_DESC_ERR_OK)) 2157 flags |= PKT_RX_L4_CKSUM_BAD; 2158 else 2159 flags |= PKT_RX_L4_CKSUM_GOOD; 2160 2161 return flags; 2162 } 2163 2164 /** 2165 * DPDK callback for receive. 2166 * 2167 * @param rxq 2168 * Generic pointer to the receive queue. 2169 * @param rx_pkts 2170 * Array to store received packets. 2171 * @param nb_pkts 2172 * Maximum number of packets in array. 2173 * 2174 * @return 2175 * Number of packets successfully received. 2176 */ 2177 static uint16_t 2178 mrvl_rx_pkt_burst(void *rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) 2179 { 2180 struct mrvl_rxq *q = rxq; 2181 struct pp2_ppio_desc descs[nb_pkts]; 2182 struct pp2_bpool *bpool; 2183 int i, ret, rx_done = 0; 2184 int num; 2185 struct pp2_hif *hif; 2186 unsigned int core_id = rte_lcore_id(); 2187 2188 hif = mrvl_get_hif(q->priv, core_id); 2189 2190 if (unlikely(!q->priv->ppio || !hif)) 2191 return 0; 2192 2193 bpool = q->priv->bpool; 2194 2195 ret = pp2_ppio_recv(q->priv->ppio, q->priv->rxq_map[q->queue_id].tc, 2196 q->priv->rxq_map[q->queue_id].inq, descs, &nb_pkts); 2197 if (unlikely(ret < 0)) { 2198 MRVL_LOG(ERR, "Failed to receive packets"); 2199 return 0; 2200 } 2201 mrvl_port_bpool_size[bpool->pp2_id][bpool->id][core_id] -= nb_pkts; 2202 2203 for (i = 0; i < nb_pkts; i++) { 2204 struct rte_mbuf *mbuf; 2205 uint8_t l3_offset, l4_offset; 2206 enum pp2_inq_desc_status status; 2207 uint64_t addr; 2208 2209 if (likely(nb_pkts - i > MRVL_MUSDK_PREFETCH_SHIFT)) { 2210 struct pp2_ppio_desc *pref_desc; 2211 u64 pref_addr; 2212 2213 pref_desc = &descs[i + MRVL_MUSDK_PREFETCH_SHIFT]; 2214 pref_addr = cookie_addr_high | 2215 pp2_ppio_inq_desc_get_cookie(pref_desc); 2216 rte_mbuf_prefetch_part1((struct rte_mbuf *)(pref_addr)); 2217 rte_mbuf_prefetch_part2((struct rte_mbuf *)(pref_addr)); 2218 } 2219 2220 addr = cookie_addr_high | 2221 pp2_ppio_inq_desc_get_cookie(&descs[i]); 2222 mbuf = (struct rte_mbuf *)addr; 2223 rte_pktmbuf_reset(mbuf); 2224 2225 /* drop packet in case of mac, overrun or resource error */ 2226 status = pp2_ppio_inq_desc_get_l2_pkt_error(&descs[i]); 2227 if (unlikely(status != PP2_DESC_ERR_OK)) { 2228 struct pp2_buff_inf binf = { 2229 .addr = rte_mbuf_data_iova_default(mbuf), 2230 .cookie = (uint64_t)mbuf, 2231 }; 2232 2233 pp2_bpool_put_buff(hif, bpool, &binf); 2234 mrvl_port_bpool_size 2235 [bpool->pp2_id][bpool->id][core_id]++; 2236 q->drop_mac++; 2237 continue; 2238 } 2239 2240 mbuf->data_off += MRVL_PKT_EFFEC_OFFS; 2241 mbuf->pkt_len = pp2_ppio_inq_desc_get_pkt_len(&descs[i]); 2242 mbuf->data_len = mbuf->pkt_len; 2243 mbuf->port = q->port_id; 2244 mbuf->packet_type = 2245 mrvl_desc_to_packet_type_and_offset(&descs[i], 2246 &l3_offset, 2247 &l4_offset); 2248 mbuf->l2_len = l3_offset; 2249 mbuf->l3_len = l4_offset - l3_offset; 2250 2251 if (likely(q->cksum_enabled)) 2252 mbuf->ol_flags = mrvl_desc_to_ol_flags(&descs[i]); 2253 2254 rx_pkts[rx_done++] = mbuf; 2255 q->bytes_recv += mbuf->pkt_len; 2256 } 2257 2258 if (rte_spinlock_trylock(&q->priv->lock) == 1) { 2259 num = mrvl_get_bpool_size(bpool->pp2_id, bpool->id); 2260 2261 if (unlikely(num <= q->priv->bpool_min_size || 2262 (!rx_done && num < q->priv->bpool_init_size))) { 2263 ret = mrvl_fill_bpool(q, MRVL_BURST_SIZE); 2264 if (ret) 2265 MRVL_LOG(ERR, "Failed to fill bpool"); 2266 } else if (unlikely(num > q->priv->bpool_max_size)) { 2267 int i; 2268 int pkt_to_remove = num - q->priv->bpool_init_size; 2269 struct rte_mbuf *mbuf; 2270 struct pp2_buff_inf buff; 2271 2272 MRVL_LOG(DEBUG, 2273 "port-%d:%d: bpool %d oversize - remove %d buffers (pool size: %d -> %d)", 2274 bpool->pp2_id, q->priv->ppio->port_id, 2275 bpool->id, pkt_to_remove, num, 2276 q->priv->bpool_init_size); 2277 2278 for (i = 0; i < pkt_to_remove; i++) { 2279 ret = pp2_bpool_get_buff(hif, bpool, &buff); 2280 if (ret) 2281 break; 2282 mbuf = (struct rte_mbuf *) 2283 (cookie_addr_high | buff.cookie); 2284 rte_pktmbuf_free(mbuf); 2285 } 2286 mrvl_port_bpool_size 2287 [bpool->pp2_id][bpool->id][core_id] -= i; 2288 } 2289 rte_spinlock_unlock(&q->priv->lock); 2290 } 2291 2292 return rx_done; 2293 } 2294 2295 /** 2296 * Prepare offload information. 2297 * 2298 * @param ol_flags 2299 * Offload flags. 2300 * @param packet_type 2301 * Packet type bitfield. 2302 * @param l3_type 2303 * Pointer to the pp2_ouq_l3_type structure. 2304 * @param l4_type 2305 * Pointer to the pp2_outq_l4_type structure. 2306 * @param gen_l3_cksum 2307 * Will be set to 1 in case l3 checksum is computed. 2308 * @param l4_cksum 2309 * Will be set to 1 in case l4 checksum is computed. 2310 * 2311 * @return 2312 * 0 on success, negative error value otherwise. 2313 */ 2314 static inline int 2315 mrvl_prepare_proto_info(uint64_t ol_flags, uint32_t packet_type, 2316 enum pp2_outq_l3_type *l3_type, 2317 enum pp2_outq_l4_type *l4_type, 2318 int *gen_l3_cksum, 2319 int *gen_l4_cksum) 2320 { 2321 /* 2322 * Based on ol_flags prepare information 2323 * for pp2_ppio_outq_desc_set_proto_info() which setups descriptor 2324 * for offloading. 2325 */ 2326 if (ol_flags & PKT_TX_IPV4) { 2327 *l3_type = PP2_OUTQ_L3_TYPE_IPV4; 2328 *gen_l3_cksum = ol_flags & PKT_TX_IP_CKSUM ? 1 : 0; 2329 } else if (ol_flags & PKT_TX_IPV6) { 2330 *l3_type = PP2_OUTQ_L3_TYPE_IPV6; 2331 /* no checksum for ipv6 header */ 2332 *gen_l3_cksum = 0; 2333 } else { 2334 /* if something different then stop processing */ 2335 return -1; 2336 } 2337 2338 ol_flags &= PKT_TX_L4_MASK; 2339 if ((packet_type & RTE_PTYPE_L4_TCP) && 2340 ol_flags == PKT_TX_TCP_CKSUM) { 2341 *l4_type = PP2_OUTQ_L4_TYPE_TCP; 2342 *gen_l4_cksum = 1; 2343 } else if ((packet_type & RTE_PTYPE_L4_UDP) && 2344 ol_flags == PKT_TX_UDP_CKSUM) { 2345 *l4_type = PP2_OUTQ_L4_TYPE_UDP; 2346 *gen_l4_cksum = 1; 2347 } else { 2348 *l4_type = PP2_OUTQ_L4_TYPE_OTHER; 2349 /* no checksum for other type */ 2350 *gen_l4_cksum = 0; 2351 } 2352 2353 return 0; 2354 } 2355 2356 /** 2357 * Release already sent buffers to bpool (buffer-pool). 2358 * 2359 * @param ppio 2360 * Pointer to the port structure. 2361 * @param hif 2362 * Pointer to the MUSDK hardware interface. 2363 * @param sq 2364 * Pointer to the shadow queue. 2365 * @param qid 2366 * Queue id number. 2367 * @param force 2368 * Force releasing packets. 2369 */ 2370 static inline void 2371 mrvl_free_sent_buffers(struct pp2_ppio *ppio, struct pp2_hif *hif, 2372 unsigned int core_id, struct mrvl_shadow_txq *sq, 2373 int qid, int force) 2374 { 2375 struct buff_release_entry *entry; 2376 uint16_t nb_done = 0, num = 0, skip_bufs = 0; 2377 int i; 2378 2379 pp2_ppio_get_num_outq_done(ppio, hif, qid, &nb_done); 2380 2381 sq->num_to_release += nb_done; 2382 2383 if (likely(!force && 2384 sq->num_to_release < MRVL_PP2_BUF_RELEASE_BURST_SIZE)) 2385 return; 2386 2387 nb_done = sq->num_to_release; 2388 sq->num_to_release = 0; 2389 2390 for (i = 0; i < nb_done; i++) { 2391 entry = &sq->ent[sq->tail + num]; 2392 if (unlikely(!entry->buff.addr)) { 2393 MRVL_LOG(ERR, 2394 "Shadow memory @%d: cookie(%lx), pa(%lx)!", 2395 sq->tail, (u64)entry->buff.cookie, 2396 (u64)entry->buff.addr); 2397 skip_bufs = 1; 2398 goto skip; 2399 } 2400 2401 if (unlikely(!entry->bpool)) { 2402 struct rte_mbuf *mbuf; 2403 2404 mbuf = (struct rte_mbuf *) 2405 (cookie_addr_high | entry->buff.cookie); 2406 rte_pktmbuf_free(mbuf); 2407 skip_bufs = 1; 2408 goto skip; 2409 } 2410 2411 mrvl_port_bpool_size 2412 [entry->bpool->pp2_id][entry->bpool->id][core_id]++; 2413 num++; 2414 if (unlikely(sq->tail + num == MRVL_PP2_TX_SHADOWQ_SIZE)) 2415 goto skip; 2416 continue; 2417 skip: 2418 if (likely(num)) 2419 pp2_bpool_put_buffs(hif, &sq->ent[sq->tail], &num); 2420 num += skip_bufs; 2421 sq->tail = (sq->tail + num) & MRVL_PP2_TX_SHADOWQ_MASK; 2422 sq->size -= num; 2423 num = 0; 2424 skip_bufs = 0; 2425 } 2426 2427 if (likely(num)) { 2428 pp2_bpool_put_buffs(hif, &sq->ent[sq->tail], &num); 2429 sq->tail = (sq->tail + num) & MRVL_PP2_TX_SHADOWQ_MASK; 2430 sq->size -= num; 2431 } 2432 } 2433 2434 /** 2435 * DPDK callback for transmit. 2436 * 2437 * @param txq 2438 * Generic pointer transmit queue. 2439 * @param tx_pkts 2440 * Packets to transmit. 2441 * @param nb_pkts 2442 * Number of packets in array. 2443 * 2444 * @return 2445 * Number of packets successfully transmitted. 2446 */ 2447 static uint16_t 2448 mrvl_tx_pkt_burst(void *txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) 2449 { 2450 struct mrvl_txq *q = txq; 2451 struct mrvl_shadow_txq *sq; 2452 struct pp2_hif *hif; 2453 struct pp2_ppio_desc descs[nb_pkts]; 2454 unsigned int core_id = rte_lcore_id(); 2455 int i, ret, bytes_sent = 0; 2456 uint16_t num, sq_free_size; 2457 uint64_t addr; 2458 2459 hif = mrvl_get_hif(q->priv, core_id); 2460 sq = &q->shadow_txqs[core_id]; 2461 2462 if (unlikely(!q->priv->ppio || !hif)) 2463 return 0; 2464 2465 if (sq->size) 2466 mrvl_free_sent_buffers(q->priv->ppio, hif, core_id, 2467 sq, q->queue_id, 0); 2468 2469 sq_free_size = MRVL_PP2_TX_SHADOWQ_SIZE - sq->size - 1; 2470 if (unlikely(nb_pkts > sq_free_size)) { 2471 MRVL_LOG(DEBUG, 2472 "No room in shadow queue for %d packets! %d packets will be sent.", 2473 nb_pkts, sq_free_size); 2474 nb_pkts = sq_free_size; 2475 } 2476 2477 for (i = 0; i < nb_pkts; i++) { 2478 struct rte_mbuf *mbuf = tx_pkts[i]; 2479 int gen_l3_cksum, gen_l4_cksum; 2480 enum pp2_outq_l3_type l3_type; 2481 enum pp2_outq_l4_type l4_type; 2482 2483 if (likely(nb_pkts - i > MRVL_MUSDK_PREFETCH_SHIFT)) { 2484 struct rte_mbuf *pref_pkt_hdr; 2485 2486 pref_pkt_hdr = tx_pkts[i + MRVL_MUSDK_PREFETCH_SHIFT]; 2487 rte_mbuf_prefetch_part1(pref_pkt_hdr); 2488 rte_mbuf_prefetch_part2(pref_pkt_hdr); 2489 } 2490 2491 mrvl_fill_shadowq(sq, mbuf); 2492 mrvl_fill_desc(&descs[i], mbuf); 2493 2494 bytes_sent += rte_pktmbuf_pkt_len(mbuf); 2495 /* 2496 * in case unsupported ol_flags were passed 2497 * do not update descriptor offload information 2498 */ 2499 ret = mrvl_prepare_proto_info(mbuf->ol_flags, mbuf->packet_type, 2500 &l3_type, &l4_type, &gen_l3_cksum, 2501 &gen_l4_cksum); 2502 if (unlikely(ret)) 2503 continue; 2504 2505 pp2_ppio_outq_desc_set_proto_info(&descs[i], l3_type, l4_type, 2506 mbuf->l2_len, 2507 mbuf->l2_len + mbuf->l3_len, 2508 gen_l3_cksum, gen_l4_cksum); 2509 } 2510 2511 num = nb_pkts; 2512 pp2_ppio_send(q->priv->ppio, hif, q->queue_id, descs, &nb_pkts); 2513 /* number of packets that were not sent */ 2514 if (unlikely(num > nb_pkts)) { 2515 for (i = nb_pkts; i < num; i++) { 2516 sq->head = (MRVL_PP2_TX_SHADOWQ_SIZE + sq->head - 1) & 2517 MRVL_PP2_TX_SHADOWQ_MASK; 2518 addr = cookie_addr_high | sq->ent[sq->head].buff.cookie; 2519 bytes_sent -= 2520 rte_pktmbuf_pkt_len((struct rte_mbuf *)addr); 2521 } 2522 sq->size -= num - nb_pkts; 2523 } 2524 2525 q->bytes_sent += bytes_sent; 2526 2527 return nb_pkts; 2528 } 2529 2530 /** DPDK callback for S/G transmit. 2531 * 2532 * @param txq 2533 * Generic pointer transmit queue. 2534 * @param tx_pkts 2535 * Packets to transmit. 2536 * @param nb_pkts 2537 * Number of packets in array. 2538 * 2539 * @return 2540 * Number of packets successfully transmitted. 2541 */ 2542 static uint16_t 2543 mrvl_tx_sg_pkt_burst(void *txq, struct rte_mbuf **tx_pkts, 2544 uint16_t nb_pkts) 2545 { 2546 struct mrvl_txq *q = txq; 2547 struct mrvl_shadow_txq *sq; 2548 struct pp2_hif *hif; 2549 struct pp2_ppio_desc descs[nb_pkts * PP2_PPIO_DESC_NUM_FRAGS]; 2550 struct pp2_ppio_sg_pkts pkts; 2551 uint8_t frags[nb_pkts]; 2552 unsigned int core_id = rte_lcore_id(); 2553 int i, j, ret, bytes_sent = 0; 2554 int tail, tail_first; 2555 uint16_t num, sq_free_size; 2556 uint16_t nb_segs, total_descs = 0; 2557 uint64_t addr; 2558 2559 hif = mrvl_get_hif(q->priv, core_id); 2560 sq = &q->shadow_txqs[core_id]; 2561 pkts.frags = frags; 2562 pkts.num = 0; 2563 2564 if (unlikely(!q->priv->ppio || !hif)) 2565 return 0; 2566 2567 if (sq->size) 2568 mrvl_free_sent_buffers(q->priv->ppio, hif, core_id, 2569 sq, q->queue_id, 0); 2570 2571 /* Save shadow queue free size */ 2572 sq_free_size = MRVL_PP2_TX_SHADOWQ_SIZE - sq->size - 1; 2573 2574 tail = 0; 2575 for (i = 0; i < nb_pkts; i++) { 2576 struct rte_mbuf *mbuf = tx_pkts[i]; 2577 struct rte_mbuf *seg = NULL; 2578 int gen_l3_cksum, gen_l4_cksum; 2579 enum pp2_outq_l3_type l3_type; 2580 enum pp2_outq_l4_type l4_type; 2581 2582 nb_segs = mbuf->nb_segs; 2583 tail_first = tail; 2584 total_descs += nb_segs; 2585 2586 /* 2587 * Check if total_descs does not exceed 2588 * shadow queue free size 2589 */ 2590 if (unlikely(total_descs > sq_free_size)) { 2591 total_descs -= nb_segs; 2592 RTE_LOG(DEBUG, PMD, 2593 "No room in shadow queue for %d packets! " 2594 "%d packets will be sent.\n", 2595 nb_pkts, i); 2596 break; 2597 } 2598 2599 /* Check if nb_segs does not exceed the max nb of desc per 2600 * fragmented packet 2601 */ 2602 if (nb_segs > PP2_PPIO_DESC_NUM_FRAGS) { 2603 total_descs -= nb_segs; 2604 RTE_LOG(ERR, PMD, 2605 "Too many segments. Packet won't be sent.\n"); 2606 break; 2607 } 2608 2609 if (likely(nb_pkts - i > MRVL_MUSDK_PREFETCH_SHIFT)) { 2610 struct rte_mbuf *pref_pkt_hdr; 2611 2612 pref_pkt_hdr = tx_pkts[i + MRVL_MUSDK_PREFETCH_SHIFT]; 2613 rte_mbuf_prefetch_part1(pref_pkt_hdr); 2614 rte_mbuf_prefetch_part2(pref_pkt_hdr); 2615 } 2616 2617 pkts.frags[pkts.num] = nb_segs; 2618 pkts.num++; 2619 2620 seg = mbuf; 2621 for (j = 0; j < nb_segs - 1; j++) { 2622 /* For the subsequent segments, set shadow queue 2623 * buffer to NULL 2624 */ 2625 mrvl_fill_shadowq(sq, NULL); 2626 mrvl_fill_desc(&descs[tail], seg); 2627 2628 tail++; 2629 seg = seg->next; 2630 } 2631 /* Put first mbuf info in last shadow queue entry */ 2632 mrvl_fill_shadowq(sq, mbuf); 2633 /* Update descriptor with last segment */ 2634 mrvl_fill_desc(&descs[tail++], seg); 2635 2636 bytes_sent += rte_pktmbuf_pkt_len(mbuf); 2637 /* In case unsupported ol_flags were passed 2638 * do not update descriptor offload information 2639 */ 2640 ret = mrvl_prepare_proto_info(mbuf->ol_flags, mbuf->packet_type, 2641 &l3_type, &l4_type, &gen_l3_cksum, 2642 &gen_l4_cksum); 2643 if (unlikely(ret)) 2644 continue; 2645 2646 pp2_ppio_outq_desc_set_proto_info(&descs[tail_first], l3_type, 2647 l4_type, mbuf->l2_len, 2648 mbuf->l2_len + mbuf->l3_len, 2649 gen_l3_cksum, gen_l4_cksum); 2650 } 2651 2652 num = total_descs; 2653 pp2_ppio_send_sg(q->priv->ppio, hif, q->queue_id, descs, 2654 &total_descs, &pkts); 2655 /* number of packets that were not sent */ 2656 if (unlikely(num > total_descs)) { 2657 for (i = total_descs; i < num; i++) { 2658 sq->head = (MRVL_PP2_TX_SHADOWQ_SIZE + sq->head - 1) & 2659 MRVL_PP2_TX_SHADOWQ_MASK; 2660 2661 addr = sq->ent[sq->head].buff.cookie; 2662 if (addr) 2663 bytes_sent -= 2664 rte_pktmbuf_pkt_len((struct rte_mbuf *) 2665 (cookie_addr_high | addr)); 2666 } 2667 sq->size -= num - total_descs; 2668 nb_pkts = pkts.num; 2669 } 2670 2671 q->bytes_sent += bytes_sent; 2672 2673 return nb_pkts; 2674 } 2675 2676 /** 2677 * Initialize packet processor. 2678 * 2679 * @return 2680 * 0 on success, negative error value otherwise. 2681 */ 2682 static int 2683 mrvl_init_pp2(void) 2684 { 2685 struct pp2_init_params init_params; 2686 2687 memset(&init_params, 0, sizeof(init_params)); 2688 init_params.hif_reserved_map = MRVL_MUSDK_HIFS_RESERVED; 2689 init_params.bm_pool_reserved_map = MRVL_MUSDK_BPOOLS_RESERVED; 2690 init_params.rss_tbl_reserved_map = MRVL_MUSDK_RSS_RESERVED; 2691 2692 return pp2_init(&init_params); 2693 } 2694 2695 /** 2696 * Deinitialize packet processor. 2697 * 2698 * @return 2699 * 0 on success, negative error value otherwise. 2700 */ 2701 static void 2702 mrvl_deinit_pp2(void) 2703 { 2704 pp2_deinit(); 2705 } 2706 2707 /** 2708 * Create private device structure. 2709 * 2710 * @param dev_name 2711 * Pointer to the port name passed in the initialization parameters. 2712 * 2713 * @return 2714 * Pointer to the newly allocated private device structure. 2715 */ 2716 static struct mrvl_priv * 2717 mrvl_priv_create(const char *dev_name) 2718 { 2719 struct pp2_bpool_params bpool_params; 2720 char match[MRVL_MATCH_LEN]; 2721 struct mrvl_priv *priv; 2722 int ret, bpool_bit; 2723 2724 priv = rte_zmalloc_socket(dev_name, sizeof(*priv), 0, rte_socket_id()); 2725 if (!priv) 2726 return NULL; 2727 2728 ret = pp2_netdev_get_ppio_info((char *)(uintptr_t)dev_name, 2729 &priv->pp_id, &priv->ppio_id); 2730 if (ret) 2731 goto out_free_priv; 2732 2733 bpool_bit = mrvl_reserve_bit(&used_bpools[priv->pp_id], 2734 PP2_BPOOL_NUM_POOLS); 2735 if (bpool_bit < 0) 2736 goto out_free_priv; 2737 priv->bpool_bit = bpool_bit; 2738 2739 snprintf(match, sizeof(match), "pool-%d:%d", priv->pp_id, 2740 priv->bpool_bit); 2741 memset(&bpool_params, 0, sizeof(bpool_params)); 2742 bpool_params.match = match; 2743 bpool_params.buff_len = MRVL_PKT_SIZE_MAX + MRVL_PKT_EFFEC_OFFS; 2744 ret = pp2_bpool_init(&bpool_params, &priv->bpool); 2745 if (ret) 2746 goto out_clear_bpool_bit; 2747 2748 priv->ppio_params.type = PP2_PPIO_T_NIC; 2749 rte_spinlock_init(&priv->lock); 2750 2751 return priv; 2752 out_clear_bpool_bit: 2753 used_bpools[priv->pp_id] &= ~(1 << priv->bpool_bit); 2754 out_free_priv: 2755 rte_free(priv); 2756 return NULL; 2757 } 2758 2759 /** 2760 * Create device representing Ethernet port. 2761 * 2762 * @param name 2763 * Pointer to the port's name. 2764 * 2765 * @return 2766 * 0 on success, negative error value otherwise. 2767 */ 2768 static int 2769 mrvl_eth_dev_create(struct rte_vdev_device *vdev, const char *name) 2770 { 2771 int ret, fd = socket(AF_INET, SOCK_DGRAM, 0); 2772 struct rte_eth_dev *eth_dev; 2773 struct mrvl_priv *priv; 2774 struct ifreq req; 2775 2776 eth_dev = rte_eth_dev_allocate(name); 2777 if (!eth_dev) 2778 return -ENOMEM; 2779 2780 priv = mrvl_priv_create(name); 2781 if (!priv) { 2782 ret = -ENOMEM; 2783 goto out_free_dev; 2784 } 2785 2786 eth_dev->data->mac_addrs = 2787 rte_zmalloc("mac_addrs", 2788 ETHER_ADDR_LEN * MRVL_MAC_ADDRS_MAX, 0); 2789 if (!eth_dev->data->mac_addrs) { 2790 MRVL_LOG(ERR, "Failed to allocate space for eth addrs"); 2791 ret = -ENOMEM; 2792 goto out_free_priv; 2793 } 2794 2795 memset(&req, 0, sizeof(req)); 2796 strcpy(req.ifr_name, name); 2797 ret = ioctl(fd, SIOCGIFHWADDR, &req); 2798 if (ret) 2799 goto out_free_mac; 2800 2801 memcpy(eth_dev->data->mac_addrs[0].addr_bytes, 2802 req.ifr_addr.sa_data, ETHER_ADDR_LEN); 2803 2804 eth_dev->data->kdrv = RTE_KDRV_NONE; 2805 eth_dev->data->dev_private = priv; 2806 eth_dev->device = &vdev->device; 2807 eth_dev->rx_pkt_burst = mrvl_rx_pkt_burst; 2808 mrvl_set_tx_function(eth_dev); 2809 eth_dev->dev_ops = &mrvl_ops; 2810 2811 rte_eth_dev_probing_finish(eth_dev); 2812 return 0; 2813 out_free_mac: 2814 rte_free(eth_dev->data->mac_addrs); 2815 out_free_dev: 2816 rte_eth_dev_release_port(eth_dev); 2817 out_free_priv: 2818 rte_free(priv); 2819 2820 return ret; 2821 } 2822 2823 /** 2824 * Cleanup previously created device representing Ethernet port. 2825 * 2826 * @param name 2827 * Pointer to the port name. 2828 */ 2829 static void 2830 mrvl_eth_dev_destroy(const char *name) 2831 { 2832 struct rte_eth_dev *eth_dev; 2833 struct mrvl_priv *priv; 2834 2835 eth_dev = rte_eth_dev_allocated(name); 2836 if (!eth_dev) 2837 return; 2838 2839 priv = eth_dev->data->dev_private; 2840 pp2_bpool_deinit(priv->bpool); 2841 used_bpools[priv->pp_id] &= ~(1 << priv->bpool_bit); 2842 rte_free(priv); 2843 rte_free(eth_dev->data->mac_addrs); 2844 rte_eth_dev_release_port(eth_dev); 2845 } 2846 2847 /** 2848 * Callback used by rte_kvargs_process() during argument parsing. 2849 * 2850 * @param key 2851 * Pointer to the parsed key (unused). 2852 * @param value 2853 * Pointer to the parsed value. 2854 * @param extra_args 2855 * Pointer to the extra arguments which contains address of the 2856 * table of pointers to parsed interface names. 2857 * 2858 * @return 2859 * Always 0. 2860 */ 2861 static int 2862 mrvl_get_ifnames(const char *key __rte_unused, const char *value, 2863 void *extra_args) 2864 { 2865 struct mrvl_ifnames *ifnames = extra_args; 2866 2867 ifnames->names[ifnames->idx++] = value; 2868 2869 return 0; 2870 } 2871 2872 /** 2873 * Deinitialize per-lcore MUSDK hardware interfaces (hifs). 2874 */ 2875 static void 2876 mrvl_deinit_hifs(void) 2877 { 2878 int i; 2879 2880 for (i = mrvl_lcore_first; i <= mrvl_lcore_last; i++) { 2881 if (hifs[i]) 2882 pp2_hif_deinit(hifs[i]); 2883 } 2884 used_hifs = MRVL_MUSDK_HIFS_RESERVED; 2885 memset(hifs, 0, sizeof(hifs)); 2886 } 2887 2888 /** 2889 * DPDK callback to register the virtual device. 2890 * 2891 * @param vdev 2892 * Pointer to the virtual device. 2893 * 2894 * @return 2895 * 0 on success, negative error value otherwise. 2896 */ 2897 static int 2898 rte_pmd_mrvl_probe(struct rte_vdev_device *vdev) 2899 { 2900 struct rte_kvargs *kvlist; 2901 struct mrvl_ifnames ifnames; 2902 int ret = -EINVAL; 2903 uint32_t i, ifnum, cfgnum; 2904 const char *params; 2905 2906 params = rte_vdev_device_args(vdev); 2907 if (!params) 2908 return -EINVAL; 2909 2910 kvlist = rte_kvargs_parse(params, valid_args); 2911 if (!kvlist) 2912 return -EINVAL; 2913 2914 ifnum = rte_kvargs_count(kvlist, MRVL_IFACE_NAME_ARG); 2915 if (ifnum > RTE_DIM(ifnames.names)) 2916 goto out_free_kvlist; 2917 2918 ifnames.idx = 0; 2919 rte_kvargs_process(kvlist, MRVL_IFACE_NAME_ARG, 2920 mrvl_get_ifnames, &ifnames); 2921 2922 2923 /* 2924 * The below system initialization should be done only once, 2925 * on the first provided configuration file 2926 */ 2927 if (!mrvl_qos_cfg) { 2928 cfgnum = rte_kvargs_count(kvlist, MRVL_CFG_ARG); 2929 MRVL_LOG(INFO, "Parsing config file!"); 2930 if (cfgnum > 1) { 2931 MRVL_LOG(ERR, "Cannot handle more than one config file!"); 2932 goto out_free_kvlist; 2933 } else if (cfgnum == 1) { 2934 rte_kvargs_process(kvlist, MRVL_CFG_ARG, 2935 mrvl_get_qoscfg, &mrvl_qos_cfg); 2936 } 2937 } 2938 2939 if (mrvl_dev_num) 2940 goto init_devices; 2941 2942 MRVL_LOG(INFO, "Perform MUSDK initializations"); 2943 2944 ret = rte_mvep_init(MVEP_MOD_T_PP2, kvlist); 2945 if (ret) 2946 goto out_free_kvlist; 2947 2948 ret = mrvl_init_pp2(); 2949 if (ret) { 2950 MRVL_LOG(ERR, "Failed to init PP!"); 2951 rte_mvep_deinit(MVEP_MOD_T_PP2); 2952 goto out_free_kvlist; 2953 } 2954 2955 memset(mrvl_port_bpool_size, 0, sizeof(mrvl_port_bpool_size)); 2956 memset(mrvl_port_to_bpool_lookup, 0, sizeof(mrvl_port_to_bpool_lookup)); 2957 2958 mrvl_lcore_first = RTE_MAX_LCORE; 2959 mrvl_lcore_last = 0; 2960 2961 init_devices: 2962 for (i = 0; i < ifnum; i++) { 2963 MRVL_LOG(INFO, "Creating %s", ifnames.names[i]); 2964 ret = mrvl_eth_dev_create(vdev, ifnames.names[i]); 2965 if (ret) 2966 goto out_cleanup; 2967 } 2968 mrvl_dev_num += ifnum; 2969 2970 rte_kvargs_free(kvlist); 2971 2972 return 0; 2973 out_cleanup: 2974 for (; i > 0; i--) 2975 mrvl_eth_dev_destroy(ifnames.names[i]); 2976 2977 if (mrvl_dev_num == 0) { 2978 mrvl_deinit_pp2(); 2979 rte_mvep_deinit(MVEP_MOD_T_PP2); 2980 } 2981 out_free_kvlist: 2982 rte_kvargs_free(kvlist); 2983 2984 return ret; 2985 } 2986 2987 /** 2988 * DPDK callback to remove virtual device. 2989 * 2990 * @param vdev 2991 * Pointer to the removed virtual device. 2992 * 2993 * @return 2994 * 0 on success, negative error value otherwise. 2995 */ 2996 static int 2997 rte_pmd_mrvl_remove(struct rte_vdev_device *vdev) 2998 { 2999 int i; 3000 const char *name; 3001 3002 name = rte_vdev_device_name(vdev); 3003 if (!name) 3004 return -EINVAL; 3005 3006 MRVL_LOG(INFO, "Removing %s", name); 3007 3008 RTE_ETH_FOREACH_DEV(i) { /* FIXME: removing all devices! */ 3009 char ifname[RTE_ETH_NAME_MAX_LEN]; 3010 3011 rte_eth_dev_get_name_by_port(i, ifname); 3012 mrvl_eth_dev_destroy(ifname); 3013 mrvl_dev_num--; 3014 } 3015 3016 if (mrvl_dev_num == 0) { 3017 MRVL_LOG(INFO, "Perform MUSDK deinit"); 3018 mrvl_deinit_hifs(); 3019 mrvl_deinit_pp2(); 3020 rte_mvep_deinit(MVEP_MOD_T_PP2); 3021 } 3022 3023 return 0; 3024 } 3025 3026 static struct rte_vdev_driver pmd_mrvl_drv = { 3027 .probe = rte_pmd_mrvl_probe, 3028 .remove = rte_pmd_mrvl_remove, 3029 }; 3030 3031 RTE_PMD_REGISTER_VDEV(net_mvpp2, pmd_mrvl_drv); 3032 RTE_PMD_REGISTER_ALIAS(net_mvpp2, eth_mvpp2); 3033 3034 RTE_INIT(mrvl_init_log) 3035 { 3036 mrvl_logtype = rte_log_register("pmd.net.mvpp2"); 3037 if (mrvl_logtype >= 0) 3038 rte_log_set_level(mrvl_logtype, RTE_LOG_NOTICE); 3039 } 3040