1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017 Cavium, Inc 3 */ 4 5 #include <stdio.h> 6 #include <stdarg.h> 7 #include <stdbool.h> 8 #include <stdint.h> 9 #include <string.h> 10 #include <unistd.h> 11 12 #include <eventdev_pmd.h> 13 #include <rte_alarm.h> 14 #include <rte_branch_prediction.h> 15 #include <rte_bus_vdev.h> 16 #include <rte_cycles.h> 17 #include <rte_debug.h> 18 #include <rte_dev.h> 19 #include <rte_devargs.h> 20 #include <rte_kvargs.h> 21 #include <rte_malloc.h> 22 #include <rte_mbuf_pool_ops.h> 23 #include <rte_prefetch.h> 24 25 #include "octeontx_ethdev.h" 26 #include "octeontx_rxtx.h" 27 #include "octeontx_logs.h" 28 29 /* Useful in stopping/closing event device if no of 30 * eth ports are using it. 31 */ 32 uint16_t evdev_refcnt; 33 34 #define OCTEONTX_QLM_MODE_SGMII 7 35 #define OCTEONTX_QLM_MODE_XFI 12 36 37 struct evdev_priv_data { 38 OFFLOAD_FLAGS; /*Sequence should not be changed */ 39 } __rte_cache_aligned; 40 41 struct octeontx_vdev_init_params { 42 uint8_t nr_port; 43 }; 44 45 uint16_t 46 rte_octeontx_pchan_map[OCTEONTX_MAX_BGX_PORTS][OCTEONTX_MAX_LMAC_PER_BGX]; 47 48 enum octeontx_link_speed { 49 OCTEONTX_LINK_SPEED_SGMII, 50 OCTEONTX_LINK_SPEED_XAUI, 51 OCTEONTX_LINK_SPEED_RXAUI, 52 OCTEONTX_LINK_SPEED_10G_R, 53 OCTEONTX_LINK_SPEED_40G_R, 54 OCTEONTX_LINK_SPEED_RESERVE1, 55 OCTEONTX_LINK_SPEED_QSGMII, 56 OCTEONTX_LINK_SPEED_RESERVE2, 57 OCTEONTX_LINK_SPEED_UNKNOWN = 255 58 }; 59 60 RTE_LOG_REGISTER_SUFFIX(otx_net_logtype_mbox, mbox, NOTICE); 61 RTE_LOG_REGISTER_SUFFIX(otx_net_logtype_init, init, NOTICE); 62 RTE_LOG_REGISTER_SUFFIX(otx_net_logtype_driver, driver, NOTICE); 63 64 /* Parse integer from integer argument */ 65 static int 66 parse_integer_arg(const char *key __rte_unused, 67 const char *value, void *extra_args) 68 { 69 int *i = (int *)extra_args; 70 71 *i = atoi(value); 72 if (*i < 0) { 73 octeontx_log_err("argument has to be positive."); 74 return -1; 75 } 76 77 return 0; 78 } 79 80 static int 81 octeontx_parse_vdev_init_params(struct octeontx_vdev_init_params *params, 82 struct rte_vdev_device *dev) 83 { 84 struct rte_kvargs *kvlist = NULL; 85 int ret = 0; 86 87 static const char * const octeontx_vdev_valid_params[] = { 88 OCTEONTX_VDEV_NR_PORT_ARG, 89 NULL 90 }; 91 92 const char *input_args = rte_vdev_device_args(dev); 93 if (params == NULL) 94 return -EINVAL; 95 96 97 if (input_args) { 98 kvlist = rte_kvargs_parse(input_args, 99 octeontx_vdev_valid_params); 100 if (kvlist == NULL) 101 return -1; 102 103 ret = rte_kvargs_process(kvlist, 104 OCTEONTX_VDEV_NR_PORT_ARG, 105 &parse_integer_arg, 106 ¶ms->nr_port); 107 if (ret < 0) 108 goto free_kvlist; 109 } 110 111 free_kvlist: 112 rte_kvargs_free(kvlist); 113 return ret; 114 } 115 116 static int 117 octeontx_port_open(struct octeontx_nic *nic) 118 { 119 octeontx_mbox_bgx_port_conf_t bgx_port_conf; 120 octeontx_mbox_bgx_port_fifo_cfg_t fifo_cfg; 121 int res; 122 123 res = 0; 124 memset(&bgx_port_conf, 0x0, sizeof(bgx_port_conf)); 125 PMD_INIT_FUNC_TRACE(); 126 127 res = octeontx_bgx_port_open(nic->port_id, &bgx_port_conf); 128 if (res < 0) { 129 octeontx_log_err("failed to open port %d", res); 130 return res; 131 } 132 133 nic->node = bgx_port_conf.node; 134 nic->port_ena = bgx_port_conf.enable; 135 nic->base_ichan = bgx_port_conf.base_chan; 136 nic->base_ochan = bgx_port_conf.base_chan; 137 nic->num_ichans = bgx_port_conf.num_chans; 138 nic->num_ochans = bgx_port_conf.num_chans; 139 nic->bgx_mtu = bgx_port_conf.mtu; 140 nic->bpen = bgx_port_conf.bpen; 141 nic->fcs_strip = bgx_port_conf.fcs_strip; 142 nic->bcast_mode = bgx_port_conf.bcast_mode; 143 nic->mcast_mode = bgx_port_conf.mcast_mode; 144 nic->speed = bgx_port_conf.mode; 145 146 nic->duplex = RTE_ETH_LINK_FULL_DUPLEX; 147 memset(&fifo_cfg, 0x0, sizeof(fifo_cfg)); 148 149 res = octeontx_bgx_port_get_fifo_cfg(nic->port_id, &fifo_cfg); 150 if (res < 0) { 151 octeontx_log_err("failed to get port %d fifo cfg", res); 152 return res; 153 } 154 155 nic->fc.rx_fifosz = fifo_cfg.rx_fifosz; 156 157 memcpy(&nic->mac_addr[0], &bgx_port_conf.macaddr[0], 158 RTE_ETHER_ADDR_LEN); 159 160 octeontx_log_dbg("port opened %d", nic->port_id); 161 return res; 162 } 163 164 static void 165 octeontx_link_status_print(struct rte_eth_dev *eth_dev, 166 struct rte_eth_link *link) 167 { 168 if (link && link->link_status) 169 octeontx_log_info("Port %u: Link Up - speed %u Mbps - %s", 170 (eth_dev->data->port_id), 171 link->link_speed, 172 link->link_duplex == RTE_ETH_LINK_FULL_DUPLEX ? 173 "full-duplex" : "half-duplex"); 174 else 175 octeontx_log_info("Port %d: Link Down", 176 (int)(eth_dev->data->port_id)); 177 } 178 179 static inline uint32_t 180 octeontx_parse_link_speeds(uint32_t link_speeds) 181 { 182 uint32_t link_speed = OCTEONTX_LINK_SPEED_UNKNOWN; 183 184 if (link_speeds & RTE_ETH_LINK_SPEED_40G) 185 link_speed = OCTEONTX_LINK_SPEED_40G_R; 186 187 if (link_speeds & RTE_ETH_LINK_SPEED_10G) { 188 link_speed = OCTEONTX_LINK_SPEED_XAUI; 189 link_speed |= OCTEONTX_LINK_SPEED_RXAUI; 190 link_speed |= OCTEONTX_LINK_SPEED_10G_R; 191 } 192 193 if (link_speeds & RTE_ETH_LINK_SPEED_5G) 194 link_speed = OCTEONTX_LINK_SPEED_QSGMII; 195 196 if (link_speeds & RTE_ETH_LINK_SPEED_1G) 197 link_speed = OCTEONTX_LINK_SPEED_SGMII; 198 199 return link_speed; 200 } 201 202 static inline uint8_t 203 octeontx_parse_eth_link_duplex(uint32_t link_speeds) 204 { 205 if ((link_speeds & RTE_ETH_LINK_SPEED_10M_HD) || 206 (link_speeds & RTE_ETH_LINK_SPEED_100M_HD)) 207 return RTE_ETH_LINK_HALF_DUPLEX; 208 else 209 return RTE_ETH_LINK_FULL_DUPLEX; 210 } 211 212 static int 213 octeontx_apply_link_speed(struct rte_eth_dev *dev) 214 { 215 struct octeontx_nic *nic = octeontx_pmd_priv(dev); 216 struct rte_eth_conf *conf = &dev->data->dev_conf; 217 octeontx_mbox_bgx_port_change_mode_t cfg; 218 219 if (conf->link_speeds == RTE_ETH_LINK_SPEED_AUTONEG) 220 return 0; 221 222 cfg.speed = octeontx_parse_link_speeds(conf->link_speeds); 223 cfg.autoneg = (conf->link_speeds & RTE_ETH_LINK_SPEED_FIXED) ? 1 : 0; 224 cfg.duplex = octeontx_parse_eth_link_duplex(conf->link_speeds); 225 cfg.qlm_mode = ((conf->link_speeds & RTE_ETH_LINK_SPEED_1G) ? 226 OCTEONTX_QLM_MODE_SGMII : 227 (conf->link_speeds & RTE_ETH_LINK_SPEED_10G) ? 228 OCTEONTX_QLM_MODE_XFI : 0); 229 230 if (cfg.speed != OCTEONTX_LINK_SPEED_UNKNOWN && 231 (cfg.speed != nic->speed || cfg.duplex != nic->duplex)) { 232 nic->speed = cfg.speed; 233 nic->duplex = cfg.duplex; 234 return octeontx_bgx_port_change_mode(nic->port_id, &cfg); 235 } else { 236 return 0; 237 } 238 } 239 240 static void 241 octeontx_link_status_update(struct octeontx_nic *nic, 242 struct rte_eth_link *link) 243 { 244 memset(link, 0, sizeof(*link)); 245 246 link->link_status = nic->link_up ? RTE_ETH_LINK_UP : RTE_ETH_LINK_DOWN; 247 248 switch (nic->speed) { 249 case OCTEONTX_LINK_SPEED_SGMII: 250 link->link_speed = RTE_ETH_SPEED_NUM_1G; 251 break; 252 253 case OCTEONTX_LINK_SPEED_XAUI: 254 link->link_speed = RTE_ETH_SPEED_NUM_10G; 255 break; 256 257 case OCTEONTX_LINK_SPEED_RXAUI: 258 case OCTEONTX_LINK_SPEED_10G_R: 259 link->link_speed = RTE_ETH_SPEED_NUM_10G; 260 break; 261 case OCTEONTX_LINK_SPEED_QSGMII: 262 link->link_speed = RTE_ETH_SPEED_NUM_5G; 263 break; 264 case OCTEONTX_LINK_SPEED_40G_R: 265 link->link_speed = RTE_ETH_SPEED_NUM_40G; 266 break; 267 268 case OCTEONTX_LINK_SPEED_RESERVE1: 269 case OCTEONTX_LINK_SPEED_RESERVE2: 270 default: 271 link->link_speed = RTE_ETH_SPEED_NUM_NONE; 272 octeontx_log_err("incorrect link speed %d", nic->speed); 273 break; 274 } 275 276 link->link_duplex = RTE_ETH_LINK_FULL_DUPLEX; 277 link->link_autoneg = RTE_ETH_LINK_AUTONEG; 278 } 279 280 static void 281 octeontx_link_status_poll(void *arg) 282 { 283 struct octeontx_nic *nic = arg; 284 struct rte_eth_link link; 285 struct rte_eth_dev *dev; 286 int res; 287 288 PMD_INIT_FUNC_TRACE(); 289 290 dev = nic->dev; 291 292 res = octeontx_bgx_port_link_status(nic->port_id); 293 if (res < 0) { 294 octeontx_log_err("Failed to get port %d link status", 295 nic->port_id); 296 } else { 297 if (nic->link_up != (uint8_t)res) { 298 nic->link_up = (uint8_t)res; 299 octeontx_link_status_update(nic, &link); 300 octeontx_link_status_print(dev, &link); 301 rte_eth_linkstatus_set(dev, &link); 302 rte_eth_dev_callback_process(dev, 303 RTE_ETH_EVENT_INTR_LSC, 304 NULL); 305 } 306 } 307 308 res = rte_eal_alarm_set(OCCTX_INTR_POLL_INTERVAL_MS * 1000, 309 octeontx_link_status_poll, nic); 310 if (res < 0) 311 octeontx_log_err("Failed to restart alarm for port %d, err: %d", 312 nic->port_id, res); 313 } 314 315 static void 316 octeontx_port_close(struct octeontx_nic *nic) 317 { 318 PMD_INIT_FUNC_TRACE(); 319 320 rte_eal_alarm_cancel(octeontx_link_status_poll, nic); 321 octeontx_bgx_port_close(nic->port_id); 322 octeontx_log_dbg("port closed %d", nic->port_id); 323 } 324 325 static int 326 octeontx_port_start(struct octeontx_nic *nic) 327 { 328 PMD_INIT_FUNC_TRACE(); 329 330 return octeontx_bgx_port_start(nic->port_id); 331 } 332 333 static int 334 octeontx_port_stop(struct octeontx_nic *nic) 335 { 336 PMD_INIT_FUNC_TRACE(); 337 338 return octeontx_bgx_port_stop(nic->port_id); 339 } 340 341 static int 342 octeontx_port_promisc_set(struct octeontx_nic *nic, int en) 343 { 344 struct rte_eth_dev *dev; 345 int res; 346 347 res = 0; 348 PMD_INIT_FUNC_TRACE(); 349 dev = nic->dev; 350 351 res = octeontx_bgx_port_promisc_set(nic->port_id, en); 352 if (res < 0) { 353 octeontx_log_err("failed to set promiscuous mode %d", 354 nic->port_id); 355 return res; 356 } 357 358 /* Set proper flag for the mode */ 359 dev->data->promiscuous = (en != 0) ? 1 : 0; 360 361 octeontx_log_dbg("port %d : promiscuous mode %s", 362 nic->port_id, en ? "set" : "unset"); 363 364 return 0; 365 } 366 367 static int 368 octeontx_port_stats(struct octeontx_nic *nic, struct rte_eth_stats *stats) 369 { 370 octeontx_mbox_bgx_port_stats_t bgx_stats; 371 int res; 372 373 PMD_INIT_FUNC_TRACE(); 374 375 res = octeontx_bgx_port_stats(nic->port_id, &bgx_stats); 376 if (res < 0) { 377 octeontx_log_err("failed to get port stats %d", nic->port_id); 378 return res; 379 } 380 381 stats->ipackets = bgx_stats.rx_packets; 382 stats->ibytes = bgx_stats.rx_bytes; 383 stats->imissed = bgx_stats.rx_dropped; 384 stats->ierrors = bgx_stats.rx_errors; 385 stats->opackets = bgx_stats.tx_packets; 386 stats->obytes = bgx_stats.tx_bytes; 387 stats->oerrors = bgx_stats.tx_errors; 388 389 octeontx_log_dbg("port%d stats inpkts=%" PRIx64 " outpkts=%" PRIx64 "", 390 nic->port_id, stats->ipackets, stats->opackets); 391 392 return 0; 393 } 394 395 static int 396 octeontx_port_stats_clr(struct octeontx_nic *nic) 397 { 398 PMD_INIT_FUNC_TRACE(); 399 400 return octeontx_bgx_port_stats_clr(nic->port_id); 401 } 402 403 static inline void 404 devconf_set_default_sane_values(struct rte_event_dev_config *dev_conf, 405 struct rte_event_dev_info *info) 406 { 407 memset(dev_conf, 0, sizeof(struct rte_event_dev_config)); 408 dev_conf->dequeue_timeout_ns = info->min_dequeue_timeout_ns; 409 410 dev_conf->nb_event_ports = info->max_event_ports; 411 dev_conf->nb_event_queues = info->max_event_queues; 412 413 dev_conf->nb_event_queue_flows = info->max_event_queue_flows; 414 dev_conf->nb_event_port_dequeue_depth = 415 info->max_event_port_dequeue_depth; 416 dev_conf->nb_event_port_enqueue_depth = 417 info->max_event_port_enqueue_depth; 418 dev_conf->nb_event_port_enqueue_depth = 419 info->max_event_port_enqueue_depth; 420 dev_conf->nb_events_limit = 421 info->max_num_events; 422 } 423 424 static uint16_t 425 octeontx_tx_offload_flags(struct rte_eth_dev *eth_dev) 426 { 427 struct octeontx_nic *nic = octeontx_pmd_priv(eth_dev); 428 uint16_t flags = 0; 429 430 if (nic->tx_offloads & RTE_ETH_TX_OFFLOAD_OUTER_IPV4_CKSUM || 431 nic->tx_offloads & RTE_ETH_TX_OFFLOAD_OUTER_UDP_CKSUM) 432 flags |= OCCTX_TX_OFFLOAD_OL3_OL4_CSUM_F; 433 434 if (nic->tx_offloads & RTE_ETH_TX_OFFLOAD_IPV4_CKSUM || 435 nic->tx_offloads & RTE_ETH_TX_OFFLOAD_TCP_CKSUM || 436 nic->tx_offloads & RTE_ETH_TX_OFFLOAD_UDP_CKSUM || 437 nic->tx_offloads & RTE_ETH_TX_OFFLOAD_SCTP_CKSUM) 438 flags |= OCCTX_TX_OFFLOAD_L3_L4_CSUM_F; 439 440 if (!(nic->tx_offloads & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE)) 441 flags |= OCCTX_TX_OFFLOAD_MBUF_NOFF_F; 442 443 if (nic->tx_offloads & RTE_ETH_TX_OFFLOAD_MULTI_SEGS) 444 flags |= OCCTX_TX_MULTI_SEG_F; 445 446 return flags; 447 } 448 449 static uint16_t 450 octeontx_rx_offload_flags(struct rte_eth_dev *eth_dev) 451 { 452 struct octeontx_nic *nic = octeontx_pmd_priv(eth_dev); 453 uint16_t flags = 0; 454 455 if (nic->rx_offloads & (RTE_ETH_RX_OFFLOAD_TCP_CKSUM | 456 RTE_ETH_RX_OFFLOAD_UDP_CKSUM)) 457 flags |= OCCTX_RX_OFFLOAD_CSUM_F; 458 459 if (nic->rx_offloads & (RTE_ETH_RX_OFFLOAD_IPV4_CKSUM | 460 RTE_ETH_RX_OFFLOAD_OUTER_IPV4_CKSUM)) 461 flags |= OCCTX_RX_OFFLOAD_CSUM_F; 462 463 if (nic->rx_offloads & RTE_ETH_RX_OFFLOAD_SCATTER) { 464 flags |= OCCTX_RX_MULTI_SEG_F; 465 eth_dev->data->scattered_rx = 1; 466 /* If scatter mode is enabled, TX should also be in multi 467 * seg mode, else memory leak will occur 468 */ 469 nic->tx_offloads |= RTE_ETH_TX_OFFLOAD_MULTI_SEGS; 470 } 471 472 return flags; 473 } 474 475 static int 476 octeontx_dev_configure(struct rte_eth_dev *dev) 477 { 478 struct rte_eth_dev_data *data = dev->data; 479 struct rte_eth_conf *conf = &data->dev_conf; 480 struct rte_eth_rxmode *rxmode = &conf->rxmode; 481 struct rte_eth_txmode *txmode = &conf->txmode; 482 struct octeontx_nic *nic = octeontx_pmd_priv(dev); 483 int ret; 484 485 PMD_INIT_FUNC_TRACE(); 486 RTE_SET_USED(conf); 487 488 if (!rte_eal_has_hugepages()) { 489 octeontx_log_err("huge page is not configured"); 490 return -EINVAL; 491 } 492 493 if (txmode->mq_mode) { 494 octeontx_log_err("tx mq_mode DCB or VMDq not supported"); 495 return -EINVAL; 496 } 497 498 if (rxmode->mq_mode != RTE_ETH_MQ_RX_NONE && 499 rxmode->mq_mode != RTE_ETH_MQ_RX_RSS) { 500 octeontx_log_err("unsupported rx qmode %d", rxmode->mq_mode); 501 return -EINVAL; 502 } 503 504 if (!(txmode->offloads & RTE_ETH_TX_OFFLOAD_MT_LOCKFREE)) { 505 PMD_INIT_LOG(NOTICE, "cant disable lockfree tx"); 506 txmode->offloads |= RTE_ETH_TX_OFFLOAD_MT_LOCKFREE; 507 } 508 509 if (conf->dcb_capability_en) { 510 octeontx_log_err("DCB enable not supported"); 511 return -EINVAL; 512 } 513 514 if (conf->fdir_conf.mode != RTE_FDIR_MODE_NONE) { 515 octeontx_log_err("flow director not supported"); 516 return -EINVAL; 517 } 518 519 nic->num_tx_queues = dev->data->nb_tx_queues; 520 521 ret = octeontx_pko_channel_open(nic->pko_vfid * PKO_VF_NUM_DQ, 522 nic->num_tx_queues, 523 nic->base_ochan); 524 if (ret) { 525 octeontx_log_err("failed to open channel %d no-of-txq %d", 526 nic->base_ochan, nic->num_tx_queues); 527 return -EFAULT; 528 } 529 530 ret = octeontx_dev_vlan_offload_init(dev); 531 if (ret) { 532 octeontx_log_err("failed to initialize vlan offload"); 533 return -EFAULT; 534 } 535 536 nic->pki.classifier_enable = false; 537 nic->pki.hash_enable = true; 538 nic->pki.initialized = false; 539 540 nic->rx_offloads |= rxmode->offloads; 541 nic->tx_offloads |= txmode->offloads; 542 nic->rx_offload_flags |= octeontx_rx_offload_flags(dev); 543 nic->tx_offload_flags |= octeontx_tx_offload_flags(dev); 544 545 return 0; 546 } 547 548 static int 549 octeontx_dev_close(struct rte_eth_dev *dev) 550 { 551 struct octeontx_txq *txq = NULL; 552 struct octeontx_nic *nic = octeontx_pmd_priv(dev); 553 unsigned int i; 554 int ret; 555 556 PMD_INIT_FUNC_TRACE(); 557 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 558 return 0; 559 560 /* Stopping/closing event device once all eth ports are closed. */ 561 if (__atomic_sub_fetch(&evdev_refcnt, 1, __ATOMIC_ACQUIRE) == 0) { 562 rte_event_dev_stop(nic->evdev); 563 rte_event_dev_close(nic->evdev); 564 } 565 566 octeontx_dev_flow_ctrl_fini(dev); 567 568 octeontx_dev_vlan_offload_fini(dev); 569 570 ret = octeontx_pko_channel_close(nic->base_ochan); 571 if (ret < 0) { 572 octeontx_log_err("failed to close channel %d VF%d %d %d", 573 nic->base_ochan, nic->port_id, nic->num_tx_queues, 574 ret); 575 } 576 /* Free txq resources for this port */ 577 for (i = 0; i < nic->num_tx_queues; i++) { 578 txq = dev->data->tx_queues[i]; 579 if (!txq) 580 continue; 581 582 rte_free(txq); 583 } 584 585 octeontx_port_close(nic); 586 587 return 0; 588 } 589 590 static int 591 octeontx_dev_mtu_set(struct rte_eth_dev *eth_dev, uint16_t mtu) 592 { 593 uint32_t buffsz, frame_size = mtu + OCCTX_L2_OVERHEAD; 594 struct octeontx_nic *nic = octeontx_pmd_priv(eth_dev); 595 struct rte_eth_dev_data *data = eth_dev->data; 596 int rc = 0; 597 598 buffsz = data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM; 599 600 /* Refuse MTU that requires the support of scattered packets 601 * when this feature has not been enabled before. 602 */ 603 if (data->dev_started && frame_size > buffsz && 604 !(nic->rx_offloads & RTE_ETH_RX_OFFLOAD_SCATTER)) { 605 octeontx_log_err("Scatter mode is disabled"); 606 return -EINVAL; 607 } 608 609 /* Check <seg size> * <max_seg> >= max_frame */ 610 if ((nic->rx_offloads & RTE_ETH_RX_OFFLOAD_SCATTER) && 611 (frame_size > buffsz * OCCTX_RX_NB_SEG_MAX)) 612 return -EINVAL; 613 614 rc = octeontx_pko_send_mtu(nic->port_id, frame_size); 615 if (rc) 616 return rc; 617 618 rc = octeontx_bgx_port_mtu_set(nic->port_id, frame_size); 619 if (rc) 620 return rc; 621 622 octeontx_log_info("Received pkt beyond maxlen %d will be dropped", 623 frame_size); 624 625 return rc; 626 } 627 628 static int 629 octeontx_recheck_rx_offloads(struct octeontx_rxq *rxq) 630 { 631 struct rte_eth_dev *eth_dev = rxq->eth_dev; 632 struct octeontx_nic *nic = octeontx_pmd_priv(eth_dev); 633 struct rte_eth_dev_data *data = eth_dev->data; 634 struct rte_pktmbuf_pool_private *mbp_priv; 635 struct evdev_priv_data *evdev_priv; 636 struct rte_eventdev *dev; 637 uint32_t buffsz; 638 639 /* Get rx buffer size */ 640 mbp_priv = rte_mempool_get_priv(rxq->pool); 641 buffsz = mbp_priv->mbuf_data_room_size - RTE_PKTMBUF_HEADROOM; 642 643 /* Setup scatter mode if needed by jumbo */ 644 if (data->mtu > buffsz) { 645 nic->rx_offloads |= RTE_ETH_RX_OFFLOAD_SCATTER; 646 nic->rx_offload_flags |= octeontx_rx_offload_flags(eth_dev); 647 nic->tx_offload_flags |= octeontx_tx_offload_flags(eth_dev); 648 } 649 650 /* Sharing offload flags via eventdev priv region */ 651 dev = &rte_eventdevs[rxq->evdev]; 652 evdev_priv = dev->data->dev_private; 653 evdev_priv->rx_offload_flags = nic->rx_offload_flags; 654 evdev_priv->tx_offload_flags = nic->tx_offload_flags; 655 656 /* Setup MTU */ 657 nic->mtu = data->mtu; 658 659 return 0; 660 } 661 662 static int 663 octeontx_dev_start(struct rte_eth_dev *dev) 664 { 665 struct octeontx_nic *nic = octeontx_pmd_priv(dev); 666 struct octeontx_rxq *rxq; 667 int ret, i; 668 669 PMD_INIT_FUNC_TRACE(); 670 /* Rechecking if any new offload set to update 671 * rx/tx burst function pointer accordingly. 672 */ 673 for (i = 0; i < dev->data->nb_rx_queues; i++) { 674 rxq = dev->data->rx_queues[i]; 675 octeontx_recheck_rx_offloads(rxq); 676 } 677 678 /* Setting up the mtu */ 679 ret = octeontx_dev_mtu_set(dev, nic->mtu); 680 if (ret) { 681 octeontx_log_err("Failed to set default MTU size %d", ret); 682 goto error; 683 } 684 685 /* Apply new link configurations if changed */ 686 ret = octeontx_apply_link_speed(dev); 687 if (ret) { 688 octeontx_log_err("Failed to set link configuration: %d", ret); 689 goto error; 690 } 691 692 /* 693 * Tx start 694 */ 695 octeontx_set_tx_function(dev); 696 ret = octeontx_pko_channel_start(nic->base_ochan); 697 if (ret < 0) { 698 octeontx_log_err("fail to conf VF%d no. txq %d chan %d ret %d", 699 nic->port_id, nic->num_tx_queues, nic->base_ochan, 700 ret); 701 goto error; 702 } 703 704 /* 705 * Rx start 706 */ 707 dev->rx_pkt_burst = octeontx_recv_pkts; 708 ret = octeontx_pki_port_start(nic->port_id); 709 if (ret < 0) { 710 octeontx_log_err("fail to start Rx on port %d", nic->port_id); 711 goto channel_stop_error; 712 } 713 714 /* 715 * Start port 716 */ 717 ret = octeontx_port_start(nic); 718 if (ret < 0) { 719 octeontx_log_err("failed start port %d", ret); 720 goto pki_port_stop_error; 721 } 722 723 PMD_TX_LOG(DEBUG, "pko: start channel %d no.of txq %d port %d", 724 nic->base_ochan, nic->num_tx_queues, nic->port_id); 725 726 ret = rte_event_dev_start(nic->evdev); 727 if (ret < 0) { 728 octeontx_log_err("failed to start evdev: ret (%d)", ret); 729 goto pki_port_stop_error; 730 } 731 732 /* Success */ 733 return ret; 734 735 pki_port_stop_error: 736 octeontx_pki_port_stop(nic->port_id); 737 channel_stop_error: 738 octeontx_pko_channel_stop(nic->base_ochan); 739 error: 740 return ret; 741 } 742 743 static int 744 octeontx_dev_stop(struct rte_eth_dev *dev) 745 { 746 struct octeontx_nic *nic = octeontx_pmd_priv(dev); 747 int ret; 748 749 PMD_INIT_FUNC_TRACE(); 750 751 ret = octeontx_port_stop(nic); 752 if (ret < 0) { 753 octeontx_log_err("failed to req stop port %d res=%d", 754 nic->port_id, ret); 755 return ret; 756 } 757 758 ret = octeontx_pki_port_stop(nic->port_id); 759 if (ret < 0) { 760 octeontx_log_err("failed to stop pki port %d res=%d", 761 nic->port_id, ret); 762 return ret; 763 } 764 765 ret = octeontx_pko_channel_stop(nic->base_ochan); 766 if (ret < 0) { 767 octeontx_log_err("failed to stop channel %d VF%d %d %d", 768 nic->base_ochan, nic->port_id, nic->num_tx_queues, 769 ret); 770 return ret; 771 } 772 773 return 0; 774 } 775 776 static int 777 octeontx_dev_promisc_enable(struct rte_eth_dev *dev) 778 { 779 struct octeontx_nic *nic = octeontx_pmd_priv(dev); 780 781 PMD_INIT_FUNC_TRACE(); 782 return octeontx_port_promisc_set(nic, 1); 783 } 784 785 static int 786 octeontx_dev_promisc_disable(struct rte_eth_dev *dev) 787 { 788 struct octeontx_nic *nic = octeontx_pmd_priv(dev); 789 790 PMD_INIT_FUNC_TRACE(); 791 return octeontx_port_promisc_set(nic, 0); 792 } 793 794 static int 795 octeontx_port_link_status(struct octeontx_nic *nic) 796 { 797 int res; 798 799 PMD_INIT_FUNC_TRACE(); 800 res = octeontx_bgx_port_link_status(nic->port_id); 801 if (res < 0) { 802 octeontx_log_err("failed to get port %d link status", 803 nic->port_id); 804 return res; 805 } 806 807 if (nic->link_up != (uint8_t)res || nic->print_flag == -1) { 808 nic->link_up = (uint8_t)res; 809 nic->print_flag = 1; 810 } 811 octeontx_log_dbg("port %d link status %d", nic->port_id, nic->link_up); 812 813 return res; 814 } 815 816 /* 817 * Return 0 means link status changed, -1 means not changed 818 */ 819 static int 820 octeontx_dev_link_update(struct rte_eth_dev *dev, 821 int wait_to_complete __rte_unused) 822 { 823 struct octeontx_nic *nic = octeontx_pmd_priv(dev); 824 struct rte_eth_link link; 825 int res; 826 827 PMD_INIT_FUNC_TRACE(); 828 829 res = octeontx_port_link_status(nic); 830 if (res < 0) { 831 octeontx_log_err("failed to request link status %d", res); 832 return res; 833 } 834 835 octeontx_link_status_update(nic, &link); 836 if (nic->print_flag) { 837 octeontx_link_status_print(nic->dev, &link); 838 nic->print_flag = 0; 839 } 840 841 return rte_eth_linkstatus_set(dev, &link); 842 } 843 844 static int 845 octeontx_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) 846 { 847 struct octeontx_nic *nic = octeontx_pmd_priv(dev); 848 849 PMD_INIT_FUNC_TRACE(); 850 return octeontx_port_stats(nic, stats); 851 } 852 853 static int 854 octeontx_dev_stats_reset(struct rte_eth_dev *dev) 855 { 856 struct octeontx_nic *nic = octeontx_pmd_priv(dev); 857 858 PMD_INIT_FUNC_TRACE(); 859 return octeontx_port_stats_clr(nic); 860 } 861 862 static void 863 octeontx_dev_mac_addr_del(struct rte_eth_dev *dev, uint32_t index) 864 { 865 struct octeontx_nic *nic = octeontx_pmd_priv(dev); 866 int ret; 867 868 ret = octeontx_bgx_port_mac_del(nic->port_id, index); 869 if (ret != 0) 870 octeontx_log_err("failed to del MAC address filter on port %d", 871 nic->port_id); 872 } 873 874 static int 875 octeontx_dev_mac_addr_add(struct rte_eth_dev *dev, 876 struct rte_ether_addr *mac_addr, 877 uint32_t index, 878 __rte_unused uint32_t vmdq) 879 { 880 struct octeontx_nic *nic = octeontx_pmd_priv(dev); 881 int ret; 882 883 ret = octeontx_bgx_port_mac_add(nic->port_id, mac_addr->addr_bytes, 884 index); 885 if (ret < 0) { 886 octeontx_log_err("failed to add MAC address filter on port %d", 887 nic->port_id); 888 return ret; 889 } 890 891 return 0; 892 } 893 894 static int 895 octeontx_dev_default_mac_addr_set(struct rte_eth_dev *dev, 896 struct rte_ether_addr *addr) 897 { 898 struct octeontx_nic *nic = octeontx_pmd_priv(dev); 899 int ret; 900 901 ret = octeontx_bgx_port_mac_set(nic->port_id, addr->addr_bytes); 902 if (ret == 0) { 903 /* Update same mac address to BGX CAM table */ 904 ret = octeontx_bgx_port_mac_add(nic->port_id, addr->addr_bytes, 905 0); 906 } 907 if (ret < 0) { 908 octeontx_log_err("failed to set MAC address on port %d", 909 nic->port_id); 910 } 911 912 return ret; 913 } 914 915 static int 916 octeontx_dev_info(struct rte_eth_dev *dev, 917 struct rte_eth_dev_info *dev_info) 918 { 919 struct octeontx_nic *nic = octeontx_pmd_priv(dev); 920 921 /* Autonegotiation may be disabled */ 922 dev_info->speed_capa = RTE_ETH_LINK_SPEED_FIXED; 923 dev_info->speed_capa |= RTE_ETH_LINK_SPEED_10M | RTE_ETH_LINK_SPEED_100M | 924 RTE_ETH_LINK_SPEED_1G | RTE_ETH_LINK_SPEED_10G | 925 RTE_ETH_LINK_SPEED_40G; 926 927 /* Min/Max MTU supported */ 928 dev_info->min_rx_bufsize = OCCTX_MIN_FRS; 929 dev_info->max_rx_pktlen = OCCTX_MAX_FRS; 930 dev_info->max_mtu = dev_info->max_rx_pktlen - OCCTX_L2_OVERHEAD; 931 dev_info->min_mtu = dev_info->min_rx_bufsize - OCCTX_L2_OVERHEAD; 932 933 dev_info->max_mac_addrs = 934 octeontx_bgx_port_mac_entries_get(nic->port_id); 935 dev_info->max_rx_queues = 1; 936 dev_info->max_tx_queues = PKO_MAX_NUM_DQ; 937 dev_info->min_rx_bufsize = 0; 938 939 dev_info->default_rxconf = (struct rte_eth_rxconf) { 940 .rx_free_thresh = 0, 941 .rx_drop_en = 0, 942 .offloads = OCTEONTX_RX_OFFLOADS, 943 }; 944 945 dev_info->default_txconf = (struct rte_eth_txconf) { 946 .tx_free_thresh = 0, 947 .offloads = OCTEONTX_TX_OFFLOADS, 948 }; 949 950 dev_info->rx_offload_capa = OCTEONTX_RX_OFFLOADS; 951 dev_info->tx_offload_capa = OCTEONTX_TX_OFFLOADS; 952 dev_info->rx_queue_offload_capa = OCTEONTX_RX_OFFLOADS; 953 dev_info->tx_queue_offload_capa = OCTEONTX_TX_OFFLOADS; 954 955 return 0; 956 } 957 958 static void 959 octeontx_dq_info_getter(octeontx_dq_t *dq, void *out) 960 { 961 ((octeontx_dq_t *)out)->lmtline_va = dq->lmtline_va; 962 ((octeontx_dq_t *)out)->ioreg_va = dq->ioreg_va; 963 ((octeontx_dq_t *)out)->fc_status_va = dq->fc_status_va; 964 } 965 966 static int 967 octeontx_vf_start_tx_queue(struct rte_eth_dev *dev, struct octeontx_nic *nic, 968 uint16_t qidx) 969 { 970 struct octeontx_txq *txq; 971 int res; 972 973 PMD_INIT_FUNC_TRACE(); 974 975 if (dev->data->tx_queue_state[qidx] == RTE_ETH_QUEUE_STATE_STARTED) 976 return 0; 977 978 txq = dev->data->tx_queues[qidx]; 979 980 res = octeontx_pko_channel_query_dqs(nic->base_ochan, 981 &txq->dq, 982 sizeof(octeontx_dq_t), 983 txq->queue_id, 984 octeontx_dq_info_getter); 985 if (res < 0) { 986 res = -EFAULT; 987 goto close_port; 988 } 989 990 dev->data->tx_queue_state[qidx] = RTE_ETH_QUEUE_STATE_STARTED; 991 return res; 992 993 close_port: 994 (void)octeontx_port_stop(nic); 995 octeontx_pko_channel_stop(nic->base_ochan); 996 octeontx_pko_channel_close(nic->base_ochan); 997 dev->data->tx_queue_state[qidx] = RTE_ETH_QUEUE_STATE_STOPPED; 998 return res; 999 } 1000 1001 int 1002 octeontx_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t qidx) 1003 { 1004 struct octeontx_nic *nic = octeontx_pmd_priv(dev); 1005 1006 PMD_INIT_FUNC_TRACE(); 1007 qidx = qidx % PKO_VF_NUM_DQ; 1008 return octeontx_vf_start_tx_queue(dev, nic, qidx); 1009 } 1010 1011 static inline int 1012 octeontx_vf_stop_tx_queue(struct rte_eth_dev *dev, struct octeontx_nic *nic, 1013 uint16_t qidx) 1014 { 1015 int ret = 0; 1016 1017 RTE_SET_USED(nic); 1018 PMD_INIT_FUNC_TRACE(); 1019 1020 if (dev->data->tx_queue_state[qidx] == RTE_ETH_QUEUE_STATE_STOPPED) 1021 return 0; 1022 1023 dev->data->tx_queue_state[qidx] = RTE_ETH_QUEUE_STATE_STOPPED; 1024 return ret; 1025 } 1026 1027 int 1028 octeontx_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t qidx) 1029 { 1030 struct octeontx_nic *nic = octeontx_pmd_priv(dev); 1031 1032 PMD_INIT_FUNC_TRACE(); 1033 qidx = qidx % PKO_VF_NUM_DQ; 1034 1035 return octeontx_vf_stop_tx_queue(dev, nic, qidx); 1036 } 1037 1038 static void 1039 octeontx_dev_tx_queue_release(struct rte_eth_dev *dev, uint16_t qid) 1040 { 1041 int res; 1042 1043 PMD_INIT_FUNC_TRACE(); 1044 1045 if (dev->data->tx_queues[qid]) { 1046 res = octeontx_dev_tx_queue_stop(dev, qid); 1047 if (res < 0) 1048 octeontx_log_err("failed stop tx_queue(%d)\n", qid); 1049 1050 rte_free(dev->data->tx_queues[qid]); 1051 } 1052 } 1053 1054 static int 1055 octeontx_dev_tx_queue_setup(struct rte_eth_dev *dev, uint16_t qidx, 1056 uint16_t nb_desc, unsigned int socket_id, 1057 const struct rte_eth_txconf *tx_conf __rte_unused) 1058 { 1059 struct octeontx_nic *nic = octeontx_pmd_priv(dev); 1060 struct octeontx_txq *txq = NULL; 1061 uint16_t dq_num; 1062 int res = 0; 1063 1064 RTE_SET_USED(nb_desc); 1065 RTE_SET_USED(socket_id); 1066 1067 dq_num = (nic->pko_vfid * PKO_VF_NUM_DQ) + qidx; 1068 1069 /* Socket id check */ 1070 if (socket_id != (unsigned int)SOCKET_ID_ANY && 1071 socket_id != (unsigned int)nic->node) 1072 PMD_TX_LOG(INFO, "socket_id expected %d, configured %d", 1073 socket_id, nic->node); 1074 1075 /* Free memory prior to re-allocation if needed. */ 1076 if (dev->data->tx_queues[qidx] != NULL) { 1077 PMD_TX_LOG(DEBUG, "freeing memory prior to re-allocation %d", 1078 qidx); 1079 octeontx_dev_tx_queue_release(dev, qidx); 1080 dev->data->tx_queues[qidx] = NULL; 1081 } 1082 1083 /* Allocating tx queue data structure */ 1084 txq = rte_zmalloc_socket("ethdev TX queue", sizeof(struct octeontx_txq), 1085 RTE_CACHE_LINE_SIZE, nic->node); 1086 if (txq == NULL) { 1087 octeontx_log_err("failed to allocate txq=%d", qidx); 1088 res = -ENOMEM; 1089 goto err; 1090 } 1091 1092 txq->eth_dev = dev; 1093 txq->queue_id = dq_num; 1094 dev->data->tx_queues[qidx] = txq; 1095 dev->data->tx_queue_state[qidx] = RTE_ETH_QUEUE_STATE_STOPPED; 1096 1097 res = octeontx_pko_channel_query_dqs(nic->base_ochan, 1098 &txq->dq, 1099 sizeof(octeontx_dq_t), 1100 txq->queue_id, 1101 octeontx_dq_info_getter); 1102 if (res < 0) { 1103 res = -EFAULT; 1104 goto err; 1105 } 1106 1107 PMD_TX_LOG(DEBUG, "[%d]:[%d] txq=%p nb_desc=%d lmtline=%p ioreg_va=%p fc_status_va=%p", 1108 qidx, txq->queue_id, txq, nb_desc, txq->dq.lmtline_va, 1109 txq->dq.ioreg_va, 1110 txq->dq.fc_status_va); 1111 1112 return res; 1113 1114 err: 1115 rte_free(txq); 1116 1117 return res; 1118 } 1119 1120 static int 1121 octeontx_dev_rx_queue_setup(struct rte_eth_dev *dev, uint16_t qidx, 1122 uint16_t nb_desc, unsigned int socket_id, 1123 const struct rte_eth_rxconf *rx_conf, 1124 struct rte_mempool *mb_pool) 1125 { 1126 struct octeontx_nic *nic = octeontx_pmd_priv(dev); 1127 struct rte_mempool_ops *mp_ops = NULL; 1128 struct octeontx_rxq *rxq = NULL; 1129 pki_pktbuf_cfg_t pktbuf_conf; 1130 pki_hash_cfg_t pki_hash; 1131 pki_qos_cfg_t pki_qos; 1132 uintptr_t pool; 1133 int ret, port; 1134 uint16_t gaura; 1135 unsigned int ev_queues = (nic->ev_queues * nic->port_id) + qidx; 1136 unsigned int ev_ports = (nic->ev_ports * nic->port_id) + qidx; 1137 1138 RTE_SET_USED(nb_desc); 1139 1140 memset(&pktbuf_conf, 0, sizeof(pktbuf_conf)); 1141 memset(&pki_hash, 0, sizeof(pki_hash)); 1142 memset(&pki_qos, 0, sizeof(pki_qos)); 1143 1144 mp_ops = rte_mempool_get_ops(mb_pool->ops_index); 1145 if (strcmp(mp_ops->name, "octeontx_fpavf")) { 1146 octeontx_log_err("failed to find octeontx_fpavf mempool"); 1147 return -ENOTSUP; 1148 } 1149 1150 /* Handle forbidden configurations */ 1151 if (nic->pki.classifier_enable) { 1152 octeontx_log_err("cannot setup queue %d. " 1153 "Classifier option unsupported", qidx); 1154 return -EINVAL; 1155 } 1156 1157 port = nic->port_id; 1158 1159 /* Rx deferred start is not supported */ 1160 if (rx_conf->rx_deferred_start) { 1161 octeontx_log_err("rx deferred start not supported"); 1162 return -EINVAL; 1163 } 1164 1165 /* Verify queue index */ 1166 if (qidx >= dev->data->nb_rx_queues) { 1167 octeontx_log_err("QID %d not supported (0 - %d available)\n", 1168 qidx, (dev->data->nb_rx_queues - 1)); 1169 return -ENOTSUP; 1170 } 1171 1172 /* Socket id check */ 1173 if (socket_id != (unsigned int)SOCKET_ID_ANY && 1174 socket_id != (unsigned int)nic->node) 1175 PMD_RX_LOG(INFO, "socket_id expected %d, configured %d", 1176 socket_id, nic->node); 1177 1178 /* Allocating rx queue data structure */ 1179 rxq = rte_zmalloc_socket("ethdev RX queue", sizeof(struct octeontx_rxq), 1180 RTE_CACHE_LINE_SIZE, nic->node); 1181 if (rxq == NULL) { 1182 octeontx_log_err("failed to allocate rxq=%d", qidx); 1183 return -ENOMEM; 1184 } 1185 1186 if (!nic->pki.initialized) { 1187 pktbuf_conf.port_type = 0; 1188 pki_hash.port_type = 0; 1189 pki_qos.port_type = 0; 1190 1191 pktbuf_conf.mmask.f_wqe_skip = 1; 1192 pktbuf_conf.mmask.f_first_skip = 1; 1193 pktbuf_conf.mmask.f_later_skip = 1; 1194 pktbuf_conf.mmask.f_mbuff_size = 1; 1195 pktbuf_conf.mmask.f_cache_mode = 1; 1196 1197 pktbuf_conf.wqe_skip = OCTTX_PACKET_WQE_SKIP; 1198 pktbuf_conf.first_skip = OCTTX_PACKET_FIRST_SKIP(mb_pool); 1199 pktbuf_conf.later_skip = OCTTX_PACKET_LATER_SKIP; 1200 pktbuf_conf.mbuff_size = (mb_pool->elt_size - 1201 RTE_PKTMBUF_HEADROOM - 1202 rte_pktmbuf_priv_size(mb_pool) - 1203 sizeof(struct rte_mbuf)); 1204 1205 pktbuf_conf.cache_mode = PKI_OPC_MODE_STF2_STT; 1206 1207 ret = octeontx_pki_port_pktbuf_config(port, &pktbuf_conf); 1208 if (ret != 0) { 1209 octeontx_log_err("fail to configure pktbuf for port %d", 1210 port); 1211 rte_free(rxq); 1212 return ret; 1213 } 1214 PMD_RX_LOG(DEBUG, "Port %d Rx pktbuf configured:\n" 1215 "\tmbuf_size:\t0x%0x\n" 1216 "\twqe_skip:\t0x%0x\n" 1217 "\tfirst_skip:\t0x%0x\n" 1218 "\tlater_skip:\t0x%0x\n" 1219 "\tcache_mode:\t%s\n", 1220 port, 1221 pktbuf_conf.mbuff_size, 1222 pktbuf_conf.wqe_skip, 1223 pktbuf_conf.first_skip, 1224 pktbuf_conf.later_skip, 1225 (pktbuf_conf.cache_mode == 1226 PKI_OPC_MODE_STT) ? 1227 "STT" : 1228 (pktbuf_conf.cache_mode == 1229 PKI_OPC_MODE_STF) ? 1230 "STF" : 1231 (pktbuf_conf.cache_mode == 1232 PKI_OPC_MODE_STF1_STT) ? 1233 "STF1_STT" : "STF2_STT"); 1234 1235 if (nic->pki.hash_enable) { 1236 pki_hash.tag_dlc = 1; 1237 pki_hash.tag_slc = 1; 1238 pki_hash.tag_dlf = 1; 1239 pki_hash.tag_slf = 1; 1240 pki_hash.tag_prt = 1; 1241 octeontx_pki_port_hash_config(port, &pki_hash); 1242 } 1243 1244 pool = (uintptr_t)mb_pool->pool_id; 1245 1246 /* Get the gaura Id */ 1247 gaura = octeontx_fpa_bufpool_gaura(pool); 1248 1249 pki_qos.qpg_qos = PKI_QPG_QOS_NONE; 1250 pki_qos.num_entry = 1; 1251 pki_qos.drop_policy = 0; 1252 pki_qos.tag_type = 0L; 1253 pki_qos.qos_entry[0].port_add = 0; 1254 pki_qos.qos_entry[0].gaura = gaura; 1255 pki_qos.qos_entry[0].ggrp_ok = ev_queues; 1256 pki_qos.qos_entry[0].ggrp_bad = ev_queues; 1257 pki_qos.qos_entry[0].grptag_bad = 0; 1258 pki_qos.qos_entry[0].grptag_ok = 0; 1259 1260 ret = octeontx_pki_port_create_qos(port, &pki_qos); 1261 if (ret < 0) { 1262 octeontx_log_err("failed to create QOS port=%d, q=%d", 1263 port, qidx); 1264 rte_free(rxq); 1265 return ret; 1266 } 1267 nic->pki.initialized = true; 1268 } 1269 1270 rxq->port_id = nic->port_id; 1271 rxq->eth_dev = dev; 1272 rxq->queue_id = qidx; 1273 rxq->evdev = nic->evdev; 1274 rxq->ev_queues = ev_queues; 1275 rxq->ev_ports = ev_ports; 1276 rxq->pool = mb_pool; 1277 1278 octeontx_recheck_rx_offloads(rxq); 1279 dev->data->rx_queues[qidx] = rxq; 1280 dev->data->rx_queue_state[qidx] = RTE_ETH_QUEUE_STATE_STOPPED; 1281 1282 return 0; 1283 } 1284 1285 static void 1286 octeontx_dev_rx_queue_release(struct rte_eth_dev *dev, uint16_t qid) 1287 { 1288 rte_free(dev->data->rx_queues[qid]); 1289 } 1290 1291 static const uint32_t * 1292 octeontx_dev_supported_ptypes_get(struct rte_eth_dev *dev) 1293 { 1294 static const uint32_t ptypes[] = { 1295 RTE_PTYPE_L3_IPV4, 1296 RTE_PTYPE_L3_IPV4_EXT, 1297 RTE_PTYPE_L3_IPV6, 1298 RTE_PTYPE_L3_IPV6_EXT, 1299 RTE_PTYPE_L4_TCP, 1300 RTE_PTYPE_L4_UDP, 1301 RTE_PTYPE_L4_FRAG, 1302 RTE_PTYPE_UNKNOWN 1303 }; 1304 1305 if (dev->rx_pkt_burst == octeontx_recv_pkts) 1306 return ptypes; 1307 1308 return NULL; 1309 } 1310 1311 static int 1312 octeontx_pool_ops(struct rte_eth_dev *dev, const char *pool) 1313 { 1314 RTE_SET_USED(dev); 1315 1316 if (!strcmp(pool, "octeontx_fpavf")) 1317 return 0; 1318 1319 return -ENOTSUP; 1320 } 1321 1322 /* Initialize and register driver with DPDK Application */ 1323 static const struct eth_dev_ops octeontx_dev_ops = { 1324 .dev_configure = octeontx_dev_configure, 1325 .dev_infos_get = octeontx_dev_info, 1326 .dev_close = octeontx_dev_close, 1327 .dev_start = octeontx_dev_start, 1328 .dev_stop = octeontx_dev_stop, 1329 .promiscuous_enable = octeontx_dev_promisc_enable, 1330 .promiscuous_disable = octeontx_dev_promisc_disable, 1331 .link_update = octeontx_dev_link_update, 1332 .stats_get = octeontx_dev_stats_get, 1333 .stats_reset = octeontx_dev_stats_reset, 1334 .mac_addr_remove = octeontx_dev_mac_addr_del, 1335 .mac_addr_add = octeontx_dev_mac_addr_add, 1336 .mac_addr_set = octeontx_dev_default_mac_addr_set, 1337 .vlan_offload_set = octeontx_dev_vlan_offload_set, 1338 .vlan_filter_set = octeontx_dev_vlan_filter_set, 1339 .tx_queue_start = octeontx_dev_tx_queue_start, 1340 .tx_queue_stop = octeontx_dev_tx_queue_stop, 1341 .tx_queue_setup = octeontx_dev_tx_queue_setup, 1342 .tx_queue_release = octeontx_dev_tx_queue_release, 1343 .rx_queue_setup = octeontx_dev_rx_queue_setup, 1344 .rx_queue_release = octeontx_dev_rx_queue_release, 1345 .dev_set_link_up = octeontx_dev_set_link_up, 1346 .dev_set_link_down = octeontx_dev_set_link_down, 1347 .dev_supported_ptypes_get = octeontx_dev_supported_ptypes_get, 1348 .mtu_set = octeontx_dev_mtu_set, 1349 .pool_ops_supported = octeontx_pool_ops, 1350 .flow_ctrl_get = octeontx_dev_flow_ctrl_get, 1351 .flow_ctrl_set = octeontx_dev_flow_ctrl_set, 1352 }; 1353 1354 /* Create Ethdev interface per BGX LMAC ports */ 1355 static int 1356 octeontx_create(struct rte_vdev_device *dev, int port, uint8_t evdev, 1357 int socket_id) 1358 { 1359 int res; 1360 size_t pko_vfid; 1361 char octtx_name[OCTEONTX_MAX_NAME_LEN]; 1362 struct octeontx_nic *nic = NULL; 1363 struct rte_eth_dev *eth_dev = NULL; 1364 struct rte_eth_dev_data *data; 1365 const char *name = rte_vdev_device_name(dev); 1366 int max_entries; 1367 1368 PMD_INIT_FUNC_TRACE(); 1369 1370 sprintf(octtx_name, "%s_%d", name, port); 1371 if (rte_eal_process_type() != RTE_PROC_PRIMARY) { 1372 eth_dev = rte_eth_dev_attach_secondary(octtx_name); 1373 if (eth_dev == NULL) 1374 return -ENODEV; 1375 1376 eth_dev->dev_ops = &octeontx_dev_ops; 1377 eth_dev->device = &dev->device; 1378 octeontx_set_tx_function(eth_dev); 1379 eth_dev->rx_pkt_burst = octeontx_recv_pkts; 1380 rte_eth_dev_probing_finish(eth_dev); 1381 return 0; 1382 } 1383 1384 /* Reserve an ethdev entry */ 1385 eth_dev = rte_eth_dev_allocate(octtx_name); 1386 if (eth_dev == NULL) { 1387 octeontx_log_err("failed to allocate rte_eth_dev"); 1388 res = -ENOMEM; 1389 goto err; 1390 } 1391 data = eth_dev->data; 1392 1393 nic = rte_zmalloc_socket(octtx_name, sizeof(*nic), 0, socket_id); 1394 if (nic == NULL) { 1395 octeontx_log_err("failed to allocate nic structure"); 1396 res = -ENOMEM; 1397 goto err; 1398 } 1399 data->dev_private = nic; 1400 pko_vfid = octeontx_pko_get_vfid(); 1401 1402 if (pko_vfid == SIZE_MAX) { 1403 octeontx_log_err("failed to get pko vfid"); 1404 res = -ENODEV; 1405 goto err; 1406 } 1407 1408 nic->pko_vfid = pko_vfid; 1409 nic->port_id = port; 1410 nic->evdev = evdev; 1411 __atomic_add_fetch(&evdev_refcnt, 1, __ATOMIC_ACQUIRE); 1412 1413 res = octeontx_port_open(nic); 1414 if (res < 0) 1415 goto err; 1416 1417 /* Rx side port configuration */ 1418 res = octeontx_pki_port_open(port); 1419 if (res != 0) { 1420 octeontx_log_err("failed to open PKI port %d", port); 1421 res = -ENODEV; 1422 goto err; 1423 } 1424 1425 eth_dev->device = &dev->device; 1426 eth_dev->intr_handle = NULL; 1427 eth_dev->data->numa_node = dev->device.numa_node; 1428 1429 data->port_id = eth_dev->data->port_id; 1430 1431 nic->ev_queues = 1; 1432 nic->ev_ports = 1; 1433 nic->print_flag = -1; 1434 1435 data->dev_link.link_status = RTE_ETH_LINK_DOWN; 1436 data->dev_started = 0; 1437 data->promiscuous = 0; 1438 data->all_multicast = 0; 1439 data->scattered_rx = 0; 1440 1441 /* Get maximum number of supported MAC entries */ 1442 max_entries = octeontx_bgx_port_mac_entries_get(nic->port_id); 1443 if (max_entries < 0) { 1444 octeontx_log_err("Failed to get max entries for mac addr"); 1445 res = -ENOTSUP; 1446 goto err; 1447 } 1448 1449 data->mac_addrs = rte_zmalloc_socket(octtx_name, max_entries * 1450 RTE_ETHER_ADDR_LEN, 0, 1451 socket_id); 1452 if (data->mac_addrs == NULL) { 1453 octeontx_log_err("failed to allocate memory for mac_addrs"); 1454 res = -ENOMEM; 1455 goto err; 1456 } 1457 1458 eth_dev->dev_ops = &octeontx_dev_ops; 1459 1460 /* Finally save ethdev pointer to the NIC structure */ 1461 nic->dev = eth_dev; 1462 1463 if (nic->port_id != data->port_id) { 1464 octeontx_log_err("eth_dev->port_id (%d) is diff to orig (%d)", 1465 data->port_id, nic->port_id); 1466 res = -EINVAL; 1467 goto free_mac_addrs; 1468 } 1469 1470 res = rte_eal_alarm_set(OCCTX_INTR_POLL_INTERVAL_MS * 1000, 1471 octeontx_link_status_poll, nic); 1472 if (res) { 1473 octeontx_log_err("Failed to start link polling alarm"); 1474 goto err; 1475 } 1476 1477 /* Update port_id mac to eth_dev */ 1478 memcpy(data->mac_addrs, nic->mac_addr, RTE_ETHER_ADDR_LEN); 1479 1480 /* Update same mac address to BGX CAM table at index 0 */ 1481 octeontx_bgx_port_mac_add(nic->port_id, nic->mac_addr, 0); 1482 1483 res = octeontx_dev_flow_ctrl_init(eth_dev); 1484 if (res < 0) 1485 goto err; 1486 1487 PMD_INIT_LOG(DEBUG, "ethdev info: "); 1488 PMD_INIT_LOG(DEBUG, "port %d, port_ena %d ochan %d num_ochan %d tx_q %d", 1489 nic->port_id, nic->port_ena, 1490 nic->base_ochan, nic->num_ochans, 1491 nic->num_tx_queues); 1492 PMD_INIT_LOG(DEBUG, "speed %d mtu %d", nic->speed, nic->bgx_mtu); 1493 1494 rte_octeontx_pchan_map[(nic->base_ochan >> 8) & 0x7] 1495 [(nic->base_ochan >> 4) & 0xF] = data->port_id; 1496 1497 rte_eth_dev_probing_finish(eth_dev); 1498 return data->port_id; 1499 1500 free_mac_addrs: 1501 rte_free(data->mac_addrs); 1502 data->mac_addrs = NULL; 1503 err: 1504 if (nic) 1505 octeontx_port_close(nic); 1506 1507 rte_eth_dev_release_port(eth_dev); 1508 1509 return res; 1510 } 1511 1512 /* Un initialize octeontx device */ 1513 static int 1514 octeontx_remove(struct rte_vdev_device *dev) 1515 { 1516 char octtx_name[OCTEONTX_MAX_NAME_LEN]; 1517 struct rte_eth_dev *eth_dev = NULL; 1518 struct octeontx_nic *nic = NULL; 1519 int i; 1520 1521 if (dev == NULL) 1522 return -EINVAL; 1523 1524 for (i = 0; i < OCTEONTX_VDEV_DEFAULT_MAX_NR_PORT; i++) { 1525 sprintf(octtx_name, "eth_octeontx_%d", i); 1526 1527 eth_dev = rte_eth_dev_allocated(octtx_name); 1528 if (eth_dev == NULL) 1529 continue; /* port already released */ 1530 1531 if (rte_eal_process_type() != RTE_PROC_PRIMARY) { 1532 rte_eth_dev_release_port(eth_dev); 1533 continue; 1534 } 1535 1536 nic = octeontx_pmd_priv(eth_dev); 1537 rte_event_dev_stop(nic->evdev); 1538 PMD_INIT_LOG(INFO, "Closing octeontx device %s", octtx_name); 1539 octeontx_dev_close(eth_dev); 1540 rte_eth_dev_release_port(eth_dev); 1541 } 1542 1543 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 1544 return 0; 1545 1546 /* Free FC resource */ 1547 octeontx_pko_fc_free(); 1548 1549 return 0; 1550 } 1551 1552 /* Initialize octeontx device */ 1553 static int 1554 octeontx_probe(struct rte_vdev_device *dev) 1555 { 1556 const char *dev_name; 1557 static int probe_once; 1558 uint8_t socket_id, qlist; 1559 int tx_vfcnt, port_id, evdev, qnum, pnum, res, i; 1560 struct rte_event_dev_config dev_conf; 1561 const char *eventdev_name = "event_octeontx"; 1562 struct rte_event_dev_info info; 1563 struct rte_eth_dev *eth_dev; 1564 1565 struct octeontx_vdev_init_params init_params = { 1566 OCTEONTX_VDEV_DEFAULT_MAX_NR_PORT 1567 }; 1568 1569 dev_name = rte_vdev_device_name(dev); 1570 1571 if (rte_eal_process_type() == RTE_PROC_SECONDARY && 1572 strlen(rte_vdev_device_args(dev)) == 0) { 1573 eth_dev = rte_eth_dev_attach_secondary(dev_name); 1574 if (!eth_dev) { 1575 PMD_INIT_LOG(ERR, "Failed to probe %s", dev_name); 1576 return -1; 1577 } 1578 /* TODO: request info from primary to set up Rx and Tx */ 1579 eth_dev->dev_ops = &octeontx_dev_ops; 1580 eth_dev->device = &dev->device; 1581 rte_eth_dev_probing_finish(eth_dev); 1582 return 0; 1583 } 1584 1585 res = octeontx_parse_vdev_init_params(&init_params, dev); 1586 if (res < 0) 1587 return -EINVAL; 1588 1589 if (init_params.nr_port > OCTEONTX_VDEV_DEFAULT_MAX_NR_PORT) { 1590 octeontx_log_err("nr_port (%d) > max (%d)", init_params.nr_port, 1591 OCTEONTX_VDEV_DEFAULT_MAX_NR_PORT); 1592 return -ENOTSUP; 1593 } 1594 1595 PMD_INIT_LOG(DEBUG, "initializing %s pmd", dev_name); 1596 1597 socket_id = rte_socket_id(); 1598 1599 tx_vfcnt = octeontx_pko_vf_count(); 1600 1601 if (tx_vfcnt < init_params.nr_port) { 1602 octeontx_log_err("not enough PKO (%d) for port number (%d)", 1603 tx_vfcnt, init_params.nr_port); 1604 return -EINVAL; 1605 } 1606 evdev = rte_event_dev_get_dev_id(eventdev_name); 1607 if (evdev < 0) { 1608 octeontx_log_err("eventdev %s not found", eventdev_name); 1609 return -ENODEV; 1610 } 1611 1612 res = rte_event_dev_info_get(evdev, &info); 1613 if (res < 0) { 1614 octeontx_log_err("failed to eventdev info %d", res); 1615 return -EINVAL; 1616 } 1617 1618 PMD_INIT_LOG(DEBUG, "max_queue %d max_port %d", 1619 info.max_event_queues, info.max_event_ports); 1620 1621 if (octeontx_pko_init_fc(tx_vfcnt)) 1622 return -ENOMEM; 1623 1624 devconf_set_default_sane_values(&dev_conf, &info); 1625 res = rte_event_dev_configure(evdev, &dev_conf); 1626 if (res < 0) 1627 goto parse_error; 1628 1629 rte_event_dev_attr_get(evdev, RTE_EVENT_DEV_ATTR_PORT_COUNT, 1630 (uint32_t *)&pnum); 1631 rte_event_dev_attr_get(evdev, RTE_EVENT_DEV_ATTR_QUEUE_COUNT, 1632 (uint32_t *)&qnum); 1633 if (pnum < qnum) { 1634 octeontx_log_err("too few event ports (%d) for event_q(%d)", 1635 pnum, qnum); 1636 res = -EINVAL; 1637 goto parse_error; 1638 } 1639 1640 /* Enable all queues available */ 1641 for (i = 0; i < qnum; i++) { 1642 res = rte_event_queue_setup(evdev, i, NULL); 1643 if (res < 0) { 1644 octeontx_log_err("failed to setup event_q(%d): res %d", 1645 i, res); 1646 goto parse_error; 1647 } 1648 } 1649 1650 /* Enable all ports available */ 1651 for (i = 0; i < pnum; i++) { 1652 res = rte_event_port_setup(evdev, i, NULL); 1653 if (res < 0) { 1654 res = -ENODEV; 1655 octeontx_log_err("failed to setup ev port(%d) res=%d", 1656 i, res); 1657 goto parse_error; 1658 } 1659 } 1660 1661 __atomic_store_n(&evdev_refcnt, 0, __ATOMIC_RELEASE); 1662 /* 1663 * Do 1:1 links for ports & queues. All queues would be mapped to 1664 * one port. If there are more ports than queues, then some ports 1665 * won't be linked to any queue. 1666 */ 1667 for (i = 0; i < qnum; i++) { 1668 /* Link one queue to one event port */ 1669 qlist = i; 1670 res = rte_event_port_link(evdev, i, &qlist, NULL, 1); 1671 if (res < 0) { 1672 res = -ENODEV; 1673 octeontx_log_err("failed to link port (%d): res=%d", 1674 i, res); 1675 goto parse_error; 1676 } 1677 } 1678 1679 /* Create ethdev interface */ 1680 for (i = 0; i < init_params.nr_port; i++) { 1681 port_id = octeontx_create(dev, i, evdev, socket_id); 1682 if (port_id < 0) { 1683 octeontx_log_err("failed to create device %s", 1684 dev_name); 1685 res = -ENODEV; 1686 goto parse_error; 1687 } 1688 1689 PMD_INIT_LOG(INFO, "created ethdev %s for port %d", dev_name, 1690 port_id); 1691 } 1692 1693 if (probe_once) { 1694 octeontx_log_err("interface %s not supported", dev_name); 1695 octeontx_remove(dev); 1696 res = -ENOTSUP; 1697 goto parse_error; 1698 } 1699 rte_mbuf_set_platform_mempool_ops("octeontx_fpavf"); 1700 probe_once = 1; 1701 1702 return 0; 1703 1704 parse_error: 1705 octeontx_pko_fc_free(); 1706 return res; 1707 } 1708 1709 static struct rte_vdev_driver octeontx_pmd_drv = { 1710 .probe = octeontx_probe, 1711 .remove = octeontx_remove, 1712 }; 1713 1714 RTE_PMD_REGISTER_VDEV(OCTEONTX_PMD, octeontx_pmd_drv); 1715 RTE_PMD_REGISTER_ALIAS(OCTEONTX_PMD, eth_octeontx); 1716 RTE_PMD_REGISTER_PARAM_STRING(OCTEONTX_PMD, "nr_port=<int> "); 1717