1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(C) 2021 Marvell. 3 */ 4 5 #include <inttypes.h> 6 #include <ethdev_pci.h> 7 8 #include "otx_ep_common.h" 9 #include "otx_ep_vf.h" 10 #include "otx2_ep_vf.h" 11 #include "cnxk_ep_vf.h" 12 #include "otx_ep_rxtx.h" 13 #include "otx_ep_mbox.h" 14 15 #define OTX_EP_DEV(_eth_dev) \ 16 ((struct otx_ep_device *)(_eth_dev)->data->dev_private) 17 18 #define OTX_ISM_ENABLE "ism_enable" 19 20 static const struct rte_eth_desc_lim otx_ep_rx_desc_lim = { 21 .nb_max = OTX_EP_MAX_OQ_DESCRIPTORS, 22 .nb_min = OTX_EP_MIN_OQ_DESCRIPTORS, 23 .nb_align = OTX_EP_RXD_ALIGN, 24 }; 25 26 static const struct rte_eth_desc_lim otx_ep_tx_desc_lim = { 27 .nb_max = OTX_EP_MAX_IQ_DESCRIPTORS, 28 .nb_min = OTX_EP_MIN_IQ_DESCRIPTORS, 29 .nb_align = OTX_EP_TXD_ALIGN, 30 }; 31 32 static int 33 parse_flag(const char *key, const char *value, void *extra_args) 34 { 35 RTE_SET_USED(key); 36 37 *(uint8_t *)extra_args = atoi(value); 38 39 return 0; 40 } 41 42 static int 43 otx_ethdev_parse_devargs(struct rte_devargs *devargs, struct otx_ep_device *otx_epvf) 44 { 45 struct rte_kvargs *kvlist; 46 uint8_t ism_enable = 0; 47 48 if (devargs == NULL) 49 goto null_devargs; 50 51 kvlist = rte_kvargs_parse(devargs->args, NULL); 52 if (kvlist == NULL) 53 goto exit; 54 55 rte_kvargs_process(kvlist, OTX_ISM_ENABLE, &parse_flag, &ism_enable); 56 rte_kvargs_free(kvlist); 57 58 null_devargs: 59 otx_epvf->ism_ena = !!ism_enable; 60 61 return 0; 62 63 exit: 64 return -EINVAL; 65 } 66 67 static void 68 otx_ep_set_tx_func(struct rte_eth_dev *eth_dev) 69 { 70 struct otx_ep_device *otx_epvf = OTX_EP_DEV(eth_dev); 71 72 if (otx_epvf->chip_gen == OTX_EP_CN10XX || otx_epvf->chip_gen == OTX_EP_CN9XX) { 73 eth_dev->tx_pkt_burst = &cnxk_ep_xmit_pkts; 74 if (otx_epvf->tx_offloads & RTE_ETH_TX_OFFLOAD_MULTI_SEGS) 75 eth_dev->tx_pkt_burst = &cnxk_ep_xmit_pkts_mseg; 76 } else { 77 eth_dev->tx_pkt_burst = &otx_ep_xmit_pkts; 78 } 79 80 if (eth_dev->data->dev_started) 81 rte_eth_fp_ops[eth_dev->data->port_id].tx_pkt_burst = 82 eth_dev->tx_pkt_burst; 83 } 84 85 static void 86 otx_ep_set_rx_func(struct rte_eth_dev *eth_dev) 87 { 88 struct otx_ep_device *otx_epvf = OTX_EP_DEV(eth_dev); 89 90 if (otx_epvf->chip_gen == OTX_EP_CN10XX) { 91 eth_dev->rx_pkt_burst = &cnxk_ep_recv_pkts; 92 #ifdef RTE_ARCH_X86 93 eth_dev->rx_pkt_burst = &cnxk_ep_recv_pkts_sse; 94 #ifdef CC_AVX2_SUPPORT 95 if (rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_256 && 96 rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2) == 1) 97 eth_dev->rx_pkt_burst = &cnxk_ep_recv_pkts_avx; 98 #endif 99 #elif defined(RTE_ARCH_ARM64) 100 eth_dev->rx_pkt_burst = &cnxk_ep_recv_pkts_neon; 101 #endif 102 if (otx_epvf->rx_offloads & RTE_ETH_RX_OFFLOAD_SCATTER) 103 eth_dev->rx_pkt_burst = &cnxk_ep_recv_pkts_mseg; 104 } else if (otx_epvf->chip_gen == OTX_EP_CN9XX) { 105 eth_dev->rx_pkt_burst = &cn9k_ep_recv_pkts; 106 #ifdef RTE_ARCH_X86 107 eth_dev->rx_pkt_burst = &cn9k_ep_recv_pkts_sse; 108 #ifdef CC_AVX2_SUPPORT 109 if (rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_256 && 110 rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2) == 1) 111 eth_dev->rx_pkt_burst = &cn9k_ep_recv_pkts_avx; 112 #endif 113 #elif defined(RTE_ARCH_ARM64) 114 eth_dev->rx_pkt_burst = &cn9k_ep_recv_pkts_neon; 115 #endif 116 if (otx_epvf->rx_offloads & RTE_ETH_RX_OFFLOAD_SCATTER) 117 eth_dev->rx_pkt_burst = &cn9k_ep_recv_pkts_mseg; 118 } else { 119 eth_dev->rx_pkt_burst = &otx_ep_recv_pkts; 120 } 121 122 if (eth_dev->data->dev_started) 123 rte_eth_fp_ops[eth_dev->data->port_id].rx_pkt_burst = 124 eth_dev->rx_pkt_burst; 125 } 126 127 static int 128 otx_ep_dev_info_get(struct rte_eth_dev *eth_dev, 129 struct rte_eth_dev_info *devinfo) 130 { 131 struct otx_ep_device *otx_epvf; 132 int max_rx_pktlen; 133 134 otx_epvf = OTX_EP_DEV(eth_dev); 135 136 max_rx_pktlen = otx_ep_mbox_get_max_pkt_len(eth_dev); 137 if (!max_rx_pktlen) { 138 otx_ep_err("Failed to get Max Rx packet length"); 139 return -EINVAL; 140 } 141 142 devinfo->speed_capa = RTE_ETH_LINK_SPEED_10G; 143 devinfo->max_rx_queues = otx_epvf->max_rx_queues; 144 devinfo->max_tx_queues = otx_epvf->max_tx_queues; 145 146 devinfo->min_rx_bufsize = OTX_EP_MIN_RX_BUF_SIZE; 147 devinfo->max_rx_pktlen = max_rx_pktlen; 148 devinfo->max_mtu = devinfo->max_rx_pktlen - OTX_EP_ETH_OVERHEAD; 149 devinfo->min_mtu = RTE_ETHER_MIN_LEN; 150 devinfo->rx_offload_capa = RTE_ETH_RX_OFFLOAD_SCATTER; 151 devinfo->tx_offload_capa = RTE_ETH_TX_OFFLOAD_MULTI_SEGS; 152 153 devinfo->max_mac_addrs = OTX_EP_MAX_MAC_ADDRS; 154 155 devinfo->rx_desc_lim = otx_ep_rx_desc_lim; 156 devinfo->tx_desc_lim = otx_ep_tx_desc_lim; 157 158 devinfo->default_rxportconf.ring_size = OTX_EP_MIN_OQ_DESCRIPTORS; 159 devinfo->default_txportconf.ring_size = OTX_EP_MIN_IQ_DESCRIPTORS; 160 161 return 0; 162 } 163 164 static int 165 otx_ep_dev_link_update(struct rte_eth_dev *eth_dev, int wait_to_complete) 166 { 167 RTE_SET_USED(wait_to_complete); 168 169 if (!eth_dev->data->dev_started) 170 return 0; 171 struct rte_eth_link link; 172 int ret = 0; 173 174 memset(&link, 0, sizeof(link)); 175 ret = otx_ep_mbox_get_link_info(eth_dev, &link); 176 if (ret) 177 return -EINVAL; 178 otx_ep_dbg("link status resp link %d duplex %d autoneg %d link_speed %d", 179 link.link_status, link.link_duplex, link.link_autoneg, link.link_speed); 180 return rte_eth_linkstatus_set(eth_dev, &link); 181 } 182 183 static int 184 otx_ep_dev_mtu_set(struct rte_eth_dev *eth_dev, uint16_t mtu) 185 { 186 struct rte_eth_dev_info devinfo; 187 int32_t ret = 0; 188 189 /* Avoid what looks like a GCC optimisation bug on devinfo.max_mtu initialisation */ 190 memset(&devinfo, 0, sizeof(devinfo)); 191 if (otx_ep_dev_info_get(eth_dev, &devinfo)) { 192 otx_ep_err("Cannot set MTU to %u: failed to get device info", mtu); 193 return -EPERM; 194 } 195 196 /* Check if MTU is within the allowed range */ 197 if (mtu < devinfo.min_mtu) { 198 otx_ep_err("Invalid MTU %u: lower than minimum MTU %u", mtu, devinfo.min_mtu); 199 return -EINVAL; 200 } 201 202 if (mtu > devinfo.max_mtu) { 203 otx_ep_err("Invalid MTU %u; higher than maximum MTU %u", mtu, devinfo.max_mtu); 204 return -EINVAL; 205 } 206 207 ret = otx_ep_mbox_set_mtu(eth_dev, mtu); 208 if (ret) 209 return -EINVAL; 210 211 otx_ep_dbg("MTU is set to %u", mtu); 212 213 return 0; 214 } 215 216 static int 217 otx_ep_dev_set_default_mac_addr(struct rte_eth_dev *eth_dev, 218 struct rte_ether_addr *mac_addr) 219 { 220 int ret; 221 222 ret = otx_ep_mbox_set_mac_addr(eth_dev, mac_addr); 223 if (ret) 224 return -EINVAL; 225 otx_ep_dbg("Default MAC address " RTE_ETHER_ADDR_PRT_FMT "", 226 RTE_ETHER_ADDR_BYTES(mac_addr)); 227 rte_ether_addr_copy(mac_addr, eth_dev->data->mac_addrs); 228 return 0; 229 } 230 231 static int 232 otx_ep_dev_start(struct rte_eth_dev *eth_dev) 233 { 234 struct otx_ep_device *otx_epvf; 235 unsigned int q; 236 int ret; 237 238 otx_epvf = (struct otx_ep_device *)OTX_EP_DEV(eth_dev); 239 /* Enable IQ/OQ for this device */ 240 ret = otx_epvf->fn_list.enable_io_queues(otx_epvf); 241 if (ret) { 242 otx_ep_err("IOQ enable failed"); 243 return ret; 244 } 245 246 for (q = 0; q < otx_epvf->nb_rx_queues; q++) { 247 rte_write32(otx_epvf->droq[q]->nb_desc, 248 otx_epvf->droq[q]->pkts_credit_reg); 249 250 rte_wmb(); 251 otx_ep_info("OQ[%d] dbells [%d]", q, 252 rte_read32(otx_epvf->droq[q]->pkts_credit_reg)); 253 } 254 255 otx_ep_dev_link_update(eth_dev, 0); 256 257 otx_ep_set_tx_func(eth_dev); 258 otx_ep_set_rx_func(eth_dev); 259 260 otx_ep_info("dev started"); 261 262 for (q = 0; q < eth_dev->data->nb_rx_queues; q++) 263 eth_dev->data->rx_queue_state[q] = RTE_ETH_QUEUE_STATE_STARTED; 264 for (q = 0; q < eth_dev->data->nb_tx_queues; q++) 265 eth_dev->data->tx_queue_state[q] = RTE_ETH_QUEUE_STATE_STARTED; 266 267 return 0; 268 } 269 270 /* Stop device and disable input/output functions */ 271 static int 272 otx_ep_dev_stop(struct rte_eth_dev *eth_dev) 273 { 274 struct otx_ep_device *otx_epvf = OTX_EP_DEV(eth_dev); 275 uint16_t i; 276 277 otx_epvf->fn_list.disable_io_queues(otx_epvf); 278 279 for (i = 0; i < eth_dev->data->nb_rx_queues; i++) 280 eth_dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; 281 for (i = 0; i < eth_dev->data->nb_tx_queues; i++) 282 eth_dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; 283 284 return 0; 285 } 286 287 /* 288 * We only need 2 uint32_t locations per IOQ, but separate these so 289 * each IOQ has the variables on its own cache line. 290 */ 291 #define OTX_EP_ISM_BUFFER_SIZE (OTX_EP_MAX_IOQS_PER_VF * RTE_CACHE_LINE_SIZE) 292 static int 293 otx_ep_ism_setup(struct otx_ep_device *otx_epvf) 294 { 295 otx_epvf->ism_buffer_mz = 296 rte_eth_dma_zone_reserve(otx_epvf->eth_dev, "ism", 297 0, OTX_EP_ISM_BUFFER_SIZE, 298 OTX_EP_PCI_RING_ALIGN, 0); 299 300 /* Same DMA buffer is shared by OQ and IQ, clear it at start */ 301 memset(otx_epvf->ism_buffer_mz->addr, 0, OTX_EP_ISM_BUFFER_SIZE); 302 if (otx_epvf->ism_buffer_mz == NULL) { 303 otx_ep_err("Failed to allocate ISM buffer"); 304 return(-1); 305 } 306 otx_ep_dbg("ISM: virt: 0x%p, dma: 0x%" PRIX64, 307 (void *)otx_epvf->ism_buffer_mz->addr, 308 otx_epvf->ism_buffer_mz->iova); 309 310 return 0; 311 } 312 313 static int 314 otx_ep_chip_specific_setup(struct otx_ep_device *otx_epvf) 315 { 316 struct rte_pci_device *pdev = otx_epvf->pdev; 317 uint32_t dev_id = pdev->id.device_id; 318 int ret = 0; 319 320 switch (dev_id) { 321 case PCI_DEVID_OCTEONTX_EP_VF: 322 otx_epvf->chip_id = dev_id; 323 ret = otx_ep_vf_setup_device(otx_epvf); 324 break; 325 case PCI_DEVID_CN9K_EP_NET_VF: 326 case PCI_DEVID_CN98XX_EP_NET_VF: 327 case PCI_DEVID_CNF95N_EP_NET_VF: 328 case PCI_DEVID_CNF95O_EP_NET_VF: 329 otx_epvf->chip_id = dev_id; 330 ret = otx2_ep_vf_setup_device(otx_epvf); 331 break; 332 case PCI_DEVID_CN10KA_EP_NET_VF: 333 case PCI_DEVID_CN10KB_EP_NET_VF: 334 case PCI_DEVID_CNF10KA_EP_NET_VF: 335 case PCI_DEVID_CNF10KB_EP_NET_VF: 336 otx_epvf->chip_id = dev_id; 337 ret = cnxk_ep_vf_setup_device(otx_epvf); 338 break; 339 default: 340 otx_ep_err("Unsupported device"); 341 ret = -EINVAL; 342 } 343 344 if (!ret) 345 otx_ep_info("OTX_EP dev_id[%d]", dev_id); 346 else 347 return ret; 348 349 if (dev_id != PCI_DEVID_OCTEONTX_EP_VF) 350 ret = otx_ep_ism_setup(otx_epvf); 351 352 return ret; 353 } 354 355 /* OTX_EP VF device initialization */ 356 static int 357 otx_epdev_init(struct otx_ep_device *otx_epvf) 358 { 359 uint32_t ethdev_queues; 360 int ret = 0; 361 362 ret = otx_ep_chip_specific_setup(otx_epvf); 363 if (ret) { 364 otx_ep_err("Chip specific setup failed"); 365 goto setup_fail; 366 } 367 368 otx_epvf->eth_dev->tx_pkt_burst = &cnxk_ep_xmit_pkts; 369 otx_epvf->eth_dev->rx_pkt_burst = &otx_ep_recv_pkts; 370 if (otx_epvf->chip_id == PCI_DEVID_OCTEONTX_EP_VF) { 371 otx_epvf->eth_dev->tx_pkt_burst = &otx_ep_xmit_pkts; 372 otx_epvf->chip_gen = OTX_EP_CN8XX; 373 } else if (otx_epvf->chip_id == PCI_DEVID_CN9K_EP_NET_VF || 374 otx_epvf->chip_id == PCI_DEVID_CN98XX_EP_NET_VF || 375 otx_epvf->chip_id == PCI_DEVID_CNF95N_EP_NET_VF || 376 otx_epvf->chip_id == PCI_DEVID_CNF95O_EP_NET_VF) { 377 otx_epvf->eth_dev->rx_pkt_burst = &cn9k_ep_recv_pkts; 378 otx_epvf->chip_gen = OTX_EP_CN9XX; 379 } else if (otx_epvf->chip_id == PCI_DEVID_CN10KA_EP_NET_VF || 380 otx_epvf->chip_id == PCI_DEVID_CN10KB_EP_NET_VF || 381 otx_epvf->chip_id == PCI_DEVID_CNF10KA_EP_NET_VF || 382 otx_epvf->chip_id == PCI_DEVID_CNF10KB_EP_NET_VF) { 383 otx_epvf->eth_dev->rx_pkt_burst = &cnxk_ep_recv_pkts; 384 otx_epvf->chip_gen = OTX_EP_CN10XX; 385 } else { 386 otx_ep_err("Invalid chip_id"); 387 ret = -EINVAL; 388 goto setup_fail; 389 } 390 ethdev_queues = (uint32_t)(otx_epvf->sriov_info.rings_per_vf); 391 otx_epvf->max_rx_queues = ethdev_queues; 392 otx_epvf->max_tx_queues = ethdev_queues; 393 394 otx_ep_info("OTX_EP Device is Ready"); 395 396 setup_fail: 397 return ret; 398 } 399 400 static int 401 otx_ep_dev_configure(struct rte_eth_dev *eth_dev) 402 { 403 struct otx_ep_device *otx_epvf = OTX_EP_DEV(eth_dev); 404 struct rte_eth_dev_data *data = eth_dev->data; 405 struct rte_eth_rxmode *rxmode; 406 struct rte_eth_txmode *txmode; 407 struct rte_eth_conf *conf; 408 409 conf = &data->dev_conf; 410 rxmode = &conf->rxmode; 411 txmode = &conf->txmode; 412 if (eth_dev->data->nb_rx_queues > otx_epvf->max_rx_queues || 413 eth_dev->data->nb_tx_queues > otx_epvf->max_tx_queues) { 414 otx_ep_err("invalid num queues"); 415 return -EINVAL; 416 } 417 418 otx_epvf->fn_list.setup_device_regs(otx_epvf); 419 otx_epvf->fn_list.disable_io_queues(otx_epvf); 420 421 otx_ep_info("OTX_EP Device is configured with num_txq %d num_rxq %d", 422 eth_dev->data->nb_rx_queues, eth_dev->data->nb_tx_queues); 423 424 otx_epvf->rx_offloads = rxmode->offloads; 425 otx_epvf->tx_offloads = txmode->offloads; 426 427 return 0; 428 } 429 430 /** 431 * Setup our receive queue/ringbuffer. This is the 432 * queue the Octeon uses to send us packets and 433 * responses. We are given a memory pool for our 434 * packet buffers that are used to populate the receive 435 * queue. 436 * 437 * @param eth_dev 438 * Pointer to the structure rte_eth_dev 439 * @param q_no 440 * Queue number 441 * @param num_rx_descs 442 * Number of entries in the queue 443 * @param socket_id 444 * Where to allocate memory 445 * @param rx_conf 446 * Pointer to the struction rte_eth_rxconf 447 * @param mp 448 * Pointer to the packet pool 449 * 450 * @return 451 * - On success, return 0 452 * - On failure, return -1 453 */ 454 static int 455 otx_ep_rx_queue_setup(struct rte_eth_dev *eth_dev, uint16_t q_no, 456 uint16_t num_rx_descs, unsigned int socket_id, 457 const struct rte_eth_rxconf *rx_conf __rte_unused, 458 struct rte_mempool *mp) 459 { 460 struct otx_ep_device *otx_epvf = OTX_EP_DEV(eth_dev); 461 struct rte_pktmbuf_pool_private *mbp_priv; 462 uint16_t buf_size; 463 464 if (q_no >= otx_epvf->max_rx_queues) { 465 otx_ep_err("Invalid rx queue number %u", q_no); 466 return -EINVAL; 467 } 468 469 if (num_rx_descs & (num_rx_descs - 1)) { 470 otx_ep_err("Invalid rx desc number should be pow 2 %u", 471 num_rx_descs); 472 return -EINVAL; 473 } 474 if (num_rx_descs < (SDP_GBL_WMARK * 8)) { 475 otx_ep_err("Invalid rx desc number(%u) should at least be greater than 8xwmark %u", 476 num_rx_descs, (SDP_GBL_WMARK * 8)); 477 return -EINVAL; 478 } 479 480 otx_ep_dbg("setting up rx queue %u", q_no); 481 482 mbp_priv = rte_mempool_get_priv(mp); 483 buf_size = mbp_priv->mbuf_data_room_size - RTE_PKTMBUF_HEADROOM; 484 485 if (otx_ep_setup_oqs(otx_epvf, q_no, num_rx_descs, buf_size, mp, 486 socket_id)) { 487 otx_ep_err("droq allocation failed"); 488 return -1; 489 } 490 491 eth_dev->data->rx_queues[q_no] = otx_epvf->droq[q_no]; 492 493 return 0; 494 } 495 496 /** 497 * Release the receive queue/ringbuffer. Called by 498 * the upper layers. 499 * 500 * @param dev 501 * Pointer to Ethernet device structure. 502 * @param q_no 503 * Receive queue index. 504 * 505 * @return 506 * - nothing 507 */ 508 static void 509 otx_ep_rx_queue_release(struct rte_eth_dev *dev, uint16_t q_no) 510 { 511 struct otx_ep_droq *rq = dev->data->rx_queues[q_no]; 512 struct otx_ep_device *otx_epvf = rq->otx_ep_dev; 513 int q_id = rq->q_no; 514 515 if (otx_ep_delete_oqs(otx_epvf, q_id)) 516 otx_ep_err("Failed to delete OQ:%d", q_id); 517 } 518 519 /** 520 * Allocate and initialize SW ring. Initialize associated HW registers. 521 * 522 * @param eth_dev 523 * Pointer to structure rte_eth_dev 524 * 525 * @param q_no 526 * Queue number 527 * 528 * @param num_tx_descs 529 * Number of ringbuffer descriptors 530 * 531 * @param socket_id 532 * NUMA socket id, used for memory allocations 533 * 534 * @param tx_conf 535 * Pointer to the structure rte_eth_txconf 536 * 537 * @return 538 * - On success, return 0 539 * - On failure, return -errno value 540 */ 541 static int 542 otx_ep_tx_queue_setup(struct rte_eth_dev *eth_dev, uint16_t q_no, 543 uint16_t num_tx_descs, unsigned int socket_id, 544 const struct rte_eth_txconf *tx_conf __rte_unused) 545 { 546 struct otx_ep_device *otx_epvf = OTX_EP_DEV(eth_dev); 547 int retval; 548 549 if (q_no >= otx_epvf->max_tx_queues) { 550 otx_ep_err("Invalid tx queue number %u", q_no); 551 return -EINVAL; 552 } 553 if (num_tx_descs & (num_tx_descs - 1)) { 554 otx_ep_err("Invalid tx desc number should be pow 2 %u", 555 num_tx_descs); 556 return -EINVAL; 557 } 558 if (num_tx_descs < (SDP_GBL_WMARK * 8)) { 559 otx_ep_err("Invalid tx desc number(%u) should at least be greater than 8*wmark(%u)", 560 num_tx_descs, (SDP_GBL_WMARK * 8)); 561 return -EINVAL; 562 } 563 564 retval = otx_ep_setup_iqs(otx_epvf, q_no, num_tx_descs, socket_id); 565 566 if (retval) { 567 otx_ep_err("IQ(TxQ) creation failed."); 568 return retval; 569 } 570 571 eth_dev->data->tx_queues[q_no] = otx_epvf->instr_queue[q_no]; 572 otx_ep_dbg("tx queue[%d] setup", q_no); 573 return 0; 574 } 575 576 /** 577 * Release the transmit queue/ringbuffer. Called by 578 * the upper layers. 579 * 580 * @param dev 581 * Pointer to Ethernet device structure. 582 * @param q_no 583 * Transmit queue index. 584 * 585 * @return 586 * - nothing 587 */ 588 static void 589 otx_ep_tx_queue_release(struct rte_eth_dev *dev, uint16_t q_no) 590 { 591 struct otx_ep_instr_queue *tq = dev->data->tx_queues[q_no]; 592 593 otx_ep_delete_iqs(tq->otx_ep_dev, tq->q_no); 594 } 595 596 static int 597 otx_ep_dev_stats_reset(struct rte_eth_dev *dev) 598 { 599 struct otx_ep_device *otx_epvf = OTX_EP_DEV(dev); 600 uint32_t i; 601 602 for (i = 0; i < otx_epvf->nb_tx_queues; i++) 603 memset(&otx_epvf->instr_queue[i]->stats, 0, 604 sizeof(struct otx_ep_iq_stats)); 605 606 for (i = 0; i < otx_epvf->nb_rx_queues; i++) 607 memset(&otx_epvf->droq[i]->stats, 0, 608 sizeof(struct otx_ep_droq_stats)); 609 610 return 0; 611 } 612 613 static int 614 otx_ep_dev_stats_get(struct rte_eth_dev *eth_dev, 615 struct rte_eth_stats *stats) 616 { 617 struct otx_ep_device *otx_epvf = OTX_EP_DEV(eth_dev); 618 struct otx_ep_iq_stats *ostats; 619 struct otx_ep_droq_stats *istats; 620 uint32_t i; 621 622 memset(stats, 0, sizeof(struct rte_eth_stats)); 623 624 for (i = 0; i < otx_epvf->nb_tx_queues; i++) { 625 ostats = &otx_epvf->instr_queue[i]->stats; 626 stats->q_opackets[i] = ostats->tx_pkts; 627 stats->q_obytes[i] = ostats->tx_bytes; 628 stats->opackets += ostats->tx_pkts; 629 stats->obytes += ostats->tx_bytes; 630 stats->oerrors += ostats->instr_dropped; 631 } 632 for (i = 0; i < otx_epvf->nb_rx_queues; i++) { 633 istats = &otx_epvf->droq[i]->stats; 634 stats->q_ipackets[i] = istats->pkts_received; 635 stats->q_ibytes[i] = istats->bytes_received; 636 stats->q_errors[i] = istats->rx_err; 637 stats->ipackets += istats->pkts_received; 638 stats->ibytes += istats->bytes_received; 639 stats->imissed += istats->rx_alloc_failure; 640 stats->ierrors += istats->rx_err; 641 stats->rx_nombuf += istats->rx_alloc_failure; 642 } 643 return 0; 644 } 645 646 static int 647 otx_ep_dev_close(struct rte_eth_dev *eth_dev) 648 { 649 struct otx_ep_device *otx_epvf; 650 uint32_t num_queues, q_no; 651 652 if (rte_eal_process_type() != RTE_PROC_PRIMARY) { 653 eth_dev->dev_ops = NULL; 654 eth_dev->rx_pkt_burst = NULL; 655 eth_dev->tx_pkt_burst = NULL; 656 return 0; 657 } 658 659 otx_epvf = OTX_EP_DEV(eth_dev); 660 otx_ep_mbox_send_dev_exit(eth_dev); 661 otx_ep_mbox_uninit(eth_dev); 662 otx_epvf->fn_list.disable_io_queues(otx_epvf); 663 num_queues = otx_epvf->nb_rx_queues; 664 for (q_no = 0; q_no < num_queues; q_no++) { 665 if (otx_ep_delete_oqs(otx_epvf, q_no)) { 666 otx_ep_err("Failed to delete OQ:%d", q_no); 667 return -EINVAL; 668 } 669 } 670 otx_ep_dbg("Num OQs:%d freed", otx_epvf->nb_rx_queues); 671 672 num_queues = otx_epvf->nb_tx_queues; 673 for (q_no = 0; q_no < num_queues; q_no++) { 674 if (otx_ep_delete_iqs(otx_epvf, q_no)) { 675 otx_ep_err("Failed to delete IQ:%d", q_no); 676 return -EINVAL; 677 } 678 } 679 otx_ep_dbg("Num IQs:%d freed", otx_epvf->nb_tx_queues); 680 681 if (rte_eth_dma_zone_free(eth_dev, "ism", 0)) { 682 otx_ep_err("Failed to delete ISM buffer"); 683 return -EINVAL; 684 } 685 686 return 0; 687 } 688 689 static int 690 otx_ep_dev_get_mac_addr(struct rte_eth_dev *eth_dev, 691 struct rte_ether_addr *mac_addr) 692 { 693 int ret; 694 695 ret = otx_ep_mbox_get_mac_addr(eth_dev, mac_addr); 696 if (ret) 697 return -EINVAL; 698 otx_ep_dbg("Get MAC address " RTE_ETHER_ADDR_PRT_FMT, 699 RTE_ETHER_ADDR_BYTES(mac_addr)); 700 return 0; 701 } 702 703 /* Define our ethernet definitions */ 704 static const struct eth_dev_ops otx_ep_eth_dev_ops = { 705 .dev_configure = otx_ep_dev_configure, 706 .dev_start = otx_ep_dev_start, 707 .dev_stop = otx_ep_dev_stop, 708 .rx_queue_setup = otx_ep_rx_queue_setup, 709 .rx_queue_release = otx_ep_rx_queue_release, 710 .tx_queue_setup = otx_ep_tx_queue_setup, 711 .tx_queue_release = otx_ep_tx_queue_release, 712 .dev_infos_get = otx_ep_dev_info_get, 713 .stats_get = otx_ep_dev_stats_get, 714 .stats_reset = otx_ep_dev_stats_reset, 715 .link_update = otx_ep_dev_link_update, 716 .dev_close = otx_ep_dev_close, 717 .mtu_set = otx_ep_dev_mtu_set, 718 .mac_addr_set = otx_ep_dev_set_default_mac_addr, 719 }; 720 721 static int 722 otx_ep_eth_dev_uninit(struct rte_eth_dev *eth_dev) 723 { 724 if (rte_eal_process_type() != RTE_PROC_PRIMARY) { 725 eth_dev->dev_ops = NULL; 726 eth_dev->rx_pkt_burst = NULL; 727 eth_dev->tx_pkt_burst = NULL; 728 return 0; 729 } 730 731 otx_ep_mbox_uninit(eth_dev); 732 eth_dev->dev_ops = NULL; 733 eth_dev->rx_pkt_burst = NULL; 734 eth_dev->tx_pkt_burst = NULL; 735 736 return 0; 737 } 738 739 static void 740 otx_epdev_event_callback(const char *device_name, enum rte_dev_event_type type, 741 __rte_unused void *arg) 742 { 743 if (type == RTE_DEV_EVENT_REMOVE) 744 otx_ep_info("Octeon epdev: %s has been removed!", device_name); 745 746 /* Cease further execution when the device is removed; otherwise, 747 * accessing the device may lead to errors. 748 */ 749 RTE_VERIFY(type != RTE_DEV_EVENT_REMOVE); 750 } 751 752 static int otx_ep_eth_dev_query_set_vf_mac(struct rte_eth_dev *eth_dev, 753 struct rte_ether_addr *mac_addr) 754 { 755 int ret_val; 756 757 memset(mac_addr, 0, sizeof(struct rte_ether_addr)); 758 ret_val = otx_ep_dev_get_mac_addr(eth_dev, mac_addr); 759 if (!ret_val) { 760 if (!rte_is_valid_assigned_ether_addr(mac_addr)) { 761 otx_ep_dbg("PF doesn't have valid VF MAC addr" RTE_ETHER_ADDR_PRT_FMT, 762 RTE_ETHER_ADDR_BYTES(mac_addr)); 763 rte_eth_random_addr(mac_addr->addr_bytes); 764 otx_ep_dbg("Setting Random MAC address" RTE_ETHER_ADDR_PRT_FMT, 765 RTE_ETHER_ADDR_BYTES(mac_addr)); 766 ret_val = otx_ep_dev_set_default_mac_addr(eth_dev, mac_addr); 767 if (ret_val) { 768 otx_ep_err("Setting MAC address " RTE_ETHER_ADDR_PRT_FMT "fails", 769 RTE_ETHER_ADDR_BYTES(mac_addr)); 770 return ret_val; 771 } 772 } 773 otx_ep_dbg("Received valid MAC addr from PF" RTE_ETHER_ADDR_PRT_FMT, 774 RTE_ETHER_ADDR_BYTES(mac_addr)); 775 } else { 776 otx_ep_err("Getting MAC address from PF via Mbox fails with ret_val: %d", 777 ret_val); 778 return ret_val; 779 } 780 return 0; 781 } 782 783 static int 784 otx_ep_eth_dev_init(struct rte_eth_dev *eth_dev) 785 { 786 struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(eth_dev); 787 struct otx_ep_device *otx_epvf = OTX_EP_DEV(eth_dev); 788 struct rte_ether_addr vf_mac_addr; 789 int ret = 0; 790 791 /* Single process support */ 792 if (rte_eal_process_type() != RTE_PROC_PRIMARY) { 793 eth_dev->dev_ops = &otx_ep_eth_dev_ops; 794 otx_ep_set_tx_func(eth_dev); 795 otx_ep_set_rx_func(eth_dev); 796 return 0; 797 } 798 799 /* Parse devargs string */ 800 if (otx_ethdev_parse_devargs(eth_dev->device->devargs, otx_epvf)) { 801 otx_ep_err("Failed to parse devargs"); 802 return -EINVAL; 803 } 804 805 rte_eth_copy_pci_info(eth_dev, pdev); 806 otx_epvf->eth_dev = eth_dev; 807 otx_epvf->port_id = eth_dev->data->port_id; 808 eth_dev->dev_ops = &otx_ep_eth_dev_ops; 809 rte_spinlock_init(&otx_epvf->mbox_lock); 810 811 /* 812 * Initialize negotiated Mbox version to base version of VF Mbox 813 * This will address working legacy PF with latest VF. 814 */ 815 otx_epvf->mbox_neg_ver = OTX_EP_MBOX_VERSION_V1; 816 eth_dev->data->mac_addrs = rte_zmalloc("otx_ep", RTE_ETHER_ADDR_LEN, 0); 817 if (eth_dev->data->mac_addrs == NULL) { 818 otx_ep_err("MAC addresses memory allocation failed"); 819 eth_dev->dev_ops = NULL; 820 return -ENOMEM; 821 } 822 rte_eth_random_addr(vf_mac_addr.addr_bytes); 823 rte_ether_addr_copy(&vf_mac_addr, eth_dev->data->mac_addrs); 824 otx_epvf->hw_addr = pdev->mem_resource[0].addr; 825 otx_epvf->pdev = pdev; 826 827 if (rte_dev_event_callback_register(pdev->name, otx_epdev_event_callback, NULL)) { 828 otx_ep_err("Failed to register a device event callback"); 829 return -EINVAL; 830 } 831 832 if (otx_epdev_init(otx_epvf)) { 833 ret = -ENOMEM; 834 goto exit; 835 } 836 837 if (otx_epvf->chip_id == PCI_DEVID_CN9K_EP_NET_VF || 838 otx_epvf->chip_id == PCI_DEVID_CN98XX_EP_NET_VF || 839 otx_epvf->chip_id == PCI_DEVID_CNF95N_EP_NET_VF || 840 otx_epvf->chip_id == PCI_DEVID_CNF95O_EP_NET_VF || 841 otx_epvf->chip_id == PCI_DEVID_CN10KA_EP_NET_VF || 842 otx_epvf->chip_id == PCI_DEVID_CN10KB_EP_NET_VF || 843 otx_epvf->chip_id == PCI_DEVID_CNF10KA_EP_NET_VF || 844 otx_epvf->chip_id == PCI_DEVID_CNF10KB_EP_NET_VF) { 845 otx_epvf->pkind = SDP_OTX2_PKIND_FS0; 846 otx_ep_info("using pkind %d", otx_epvf->pkind); 847 } else if (otx_epvf->chip_id == PCI_DEVID_OCTEONTX_EP_VF) { 848 otx_epvf->pkind = SDP_PKIND; 849 otx_ep_info("Using pkind %d.", otx_epvf->pkind); 850 } else { 851 otx_ep_err("Invalid chip id"); 852 ret = -EINVAL; 853 goto exit; 854 } 855 856 if (otx_ep_mbox_init(eth_dev)) { 857 ret = -EINVAL; 858 goto exit; 859 } 860 861 if (otx_ep_eth_dev_query_set_vf_mac(eth_dev, 862 (struct rte_ether_addr *)&vf_mac_addr)) { 863 otx_ep_err("set mac addr failed"); 864 ret = -ENODEV; 865 goto exit; 866 } 867 rte_ether_addr_copy(&vf_mac_addr, eth_dev->data->mac_addrs); 868 869 exit: 870 rte_dev_event_callback_unregister(pdev->name, otx_epdev_event_callback, NULL); 871 872 return ret; 873 } 874 875 static int 876 otx_ep_eth_dev_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 877 struct rte_pci_device *pci_dev) 878 { 879 return rte_eth_dev_pci_generic_probe(pci_dev, 880 sizeof(struct otx_ep_device), 881 otx_ep_eth_dev_init); 882 } 883 884 static int 885 otx_ep_eth_dev_pci_remove(struct rte_pci_device *pci_dev) 886 { 887 return rte_eth_dev_pci_generic_remove(pci_dev, 888 otx_ep_eth_dev_uninit); 889 } 890 891 /* Set of PCI devices this driver supports */ 892 static const struct rte_pci_id pci_id_otx_ep_map[] = { 893 { RTE_PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_OCTEONTX_EP_VF) }, 894 { RTE_PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_CN9K_EP_NET_VF) }, 895 { RTE_PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_CN98XX_EP_NET_VF) }, 896 { RTE_PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_CNF95N_EP_NET_VF) }, 897 { RTE_PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_CNF95O_EP_NET_VF) }, 898 { RTE_PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_CN10KA_EP_NET_VF) }, 899 { RTE_PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_CN10KB_EP_NET_VF) }, 900 { RTE_PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_CNF10KA_EP_NET_VF) }, 901 { RTE_PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_CNF10KB_EP_NET_VF) }, 902 { .vendor_id = 0, /* sentinel */ } 903 }; 904 905 static struct rte_pci_driver rte_otx_ep_pmd = { 906 .id_table = pci_id_otx_ep_map, 907 .drv_flags = RTE_PCI_DRV_NEED_MAPPING, 908 .probe = otx_ep_eth_dev_pci_probe, 909 .remove = otx_ep_eth_dev_pci_remove, 910 }; 911 912 RTE_PMD_REGISTER_PCI(net_otx_ep, rte_otx_ep_pmd); 913 RTE_PMD_REGISTER_PCI_TABLE(net_otx_ep, pci_id_otx_ep_map); 914 RTE_PMD_REGISTER_KMOD_DEP(net_otx_ep, "* igb_uio | vfio-pci"); 915 RTE_LOG_REGISTER_DEFAULT(otx_net_ep_logtype, NOTICE); 916 RTE_PMD_REGISTER_PARAM_STRING(net_otx_ep, 917 OTX_ISM_ENABLE "=<0|1>"); 918