1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2008-2017 Cisco Systems, Inc. All rights reserved. 3 * Copyright 2007 Nuova Systems, Inc. All rights reserved. 4 */ 5 6 #include <stdio.h> 7 8 #include <sys/stat.h> 9 #include <sys/mman.h> 10 #include <fcntl.h> 11 12 #include <rte_pci.h> 13 #include <rte_bus_pci.h> 14 #include <rte_memzone.h> 15 #include <rte_malloc.h> 16 #include <rte_mbuf.h> 17 #include <rte_string_fns.h> 18 #include <rte_ethdev_driver.h> 19 20 #include "enic_compat.h" 21 #include "enic.h" 22 #include "wq_enet_desc.h" 23 #include "rq_enet_desc.h" 24 #include "cq_enet_desc.h" 25 #include "vnic_enet.h" 26 #include "vnic_dev.h" 27 #include "vnic_wq.h" 28 #include "vnic_rq.h" 29 #include "vnic_cq.h" 30 #include "vnic_intr.h" 31 #include "vnic_nic.h" 32 33 static inline int enic_is_sriov_vf(struct enic *enic) 34 { 35 return enic->pdev->id.device_id == PCI_DEVICE_ID_CISCO_VIC_ENET_VF; 36 } 37 38 static int is_zero_addr(uint8_t *addr) 39 { 40 return !(addr[0] | addr[1] | addr[2] | addr[3] | addr[4] | addr[5]); 41 } 42 43 static int is_mcast_addr(uint8_t *addr) 44 { 45 return addr[0] & 1; 46 } 47 48 static int is_eth_addr_valid(uint8_t *addr) 49 { 50 return !is_mcast_addr(addr) && !is_zero_addr(addr); 51 } 52 53 static void 54 enic_rxmbuf_queue_release(__rte_unused struct enic *enic, struct vnic_rq *rq) 55 { 56 uint16_t i; 57 58 if (!rq || !rq->mbuf_ring) { 59 dev_debug(enic, "Pointer to rq or mbuf_ring is NULL"); 60 return; 61 } 62 63 for (i = 0; i < rq->ring.desc_count; i++) { 64 if (rq->mbuf_ring[i]) { 65 rte_pktmbuf_free_seg(rq->mbuf_ring[i]); 66 rq->mbuf_ring[i] = NULL; 67 } 68 } 69 } 70 71 static void enic_free_wq_buf(struct rte_mbuf **buf) 72 { 73 struct rte_mbuf *mbuf = *buf; 74 75 rte_pktmbuf_free_seg(mbuf); 76 *buf = NULL; 77 } 78 79 static void enic_log_q_error(struct enic *enic) 80 { 81 unsigned int i; 82 u32 error_status; 83 84 for (i = 0; i < enic->wq_count; i++) { 85 error_status = vnic_wq_error_status(&enic->wq[i]); 86 if (error_status) 87 dev_err(enic, "WQ[%d] error_status %d\n", i, 88 error_status); 89 } 90 91 for (i = 0; i < enic_vnic_rq_count(enic); i++) { 92 if (!enic->rq[i].in_use) 93 continue; 94 error_status = vnic_rq_error_status(&enic->rq[i]); 95 if (error_status) 96 dev_err(enic, "RQ[%d] error_status %d\n", i, 97 error_status); 98 } 99 } 100 101 static void enic_clear_soft_stats(struct enic *enic) 102 { 103 struct enic_soft_stats *soft_stats = &enic->soft_stats; 104 rte_atomic64_clear(&soft_stats->rx_nombuf); 105 rte_atomic64_clear(&soft_stats->rx_packet_errors); 106 rte_atomic64_clear(&soft_stats->tx_oversized); 107 } 108 109 static void enic_init_soft_stats(struct enic *enic) 110 { 111 struct enic_soft_stats *soft_stats = &enic->soft_stats; 112 rte_atomic64_init(&soft_stats->rx_nombuf); 113 rte_atomic64_init(&soft_stats->rx_packet_errors); 114 rte_atomic64_init(&soft_stats->tx_oversized); 115 enic_clear_soft_stats(enic); 116 } 117 118 void enic_dev_stats_clear(struct enic *enic) 119 { 120 if (vnic_dev_stats_clear(enic->vdev)) 121 dev_err(enic, "Error in clearing stats\n"); 122 enic_clear_soft_stats(enic); 123 } 124 125 int enic_dev_stats_get(struct enic *enic, struct rte_eth_stats *r_stats) 126 { 127 struct vnic_stats *stats; 128 struct enic_soft_stats *soft_stats = &enic->soft_stats; 129 int64_t rx_truncated; 130 uint64_t rx_packet_errors; 131 int ret = vnic_dev_stats_dump(enic->vdev, &stats); 132 133 if (ret) { 134 dev_err(enic, "Error in getting stats\n"); 135 return ret; 136 } 137 138 /* The number of truncated packets can only be calculated by 139 * subtracting a hardware counter from error packets received by 140 * the driver. Note: this causes transient inaccuracies in the 141 * ipackets count. Also, the length of truncated packets are 142 * counted in ibytes even though truncated packets are dropped 143 * which can make ibytes be slightly higher than it should be. 144 */ 145 rx_packet_errors = rte_atomic64_read(&soft_stats->rx_packet_errors); 146 rx_truncated = rx_packet_errors - stats->rx.rx_errors; 147 148 r_stats->ipackets = stats->rx.rx_frames_ok - rx_truncated; 149 r_stats->opackets = stats->tx.tx_frames_ok; 150 151 r_stats->ibytes = stats->rx.rx_bytes_ok; 152 r_stats->obytes = stats->tx.tx_bytes_ok; 153 154 r_stats->ierrors = stats->rx.rx_errors + stats->rx.rx_drop; 155 r_stats->oerrors = stats->tx.tx_errors 156 + rte_atomic64_read(&soft_stats->tx_oversized); 157 158 r_stats->imissed = stats->rx.rx_no_bufs + rx_truncated; 159 160 r_stats->rx_nombuf = rte_atomic64_read(&soft_stats->rx_nombuf); 161 return 0; 162 } 163 164 int enic_del_mac_address(struct enic *enic, int mac_index) 165 { 166 struct rte_eth_dev *eth_dev = enic->rte_dev; 167 uint8_t *mac_addr = eth_dev->data->mac_addrs[mac_index].addr_bytes; 168 169 return vnic_dev_del_addr(enic->vdev, mac_addr); 170 } 171 172 int enic_set_mac_address(struct enic *enic, uint8_t *mac_addr) 173 { 174 int err; 175 176 if (!is_eth_addr_valid(mac_addr)) { 177 dev_err(enic, "invalid mac address\n"); 178 return -EINVAL; 179 } 180 181 err = vnic_dev_add_addr(enic->vdev, mac_addr); 182 if (err) 183 dev_err(enic, "add mac addr failed\n"); 184 return err; 185 } 186 187 static void 188 enic_free_rq_buf(struct rte_mbuf **mbuf) 189 { 190 if (*mbuf == NULL) 191 return; 192 193 rte_pktmbuf_free(*mbuf); 194 *mbuf = NULL; 195 } 196 197 void enic_init_vnic_resources(struct enic *enic) 198 { 199 unsigned int error_interrupt_enable = 1; 200 unsigned int error_interrupt_offset = 0; 201 unsigned int rxq_interrupt_enable = 0; 202 unsigned int rxq_interrupt_offset = ENICPMD_RXQ_INTR_OFFSET; 203 unsigned int index = 0; 204 unsigned int cq_idx; 205 struct vnic_rq *data_rq; 206 207 if (enic->rte_dev->data->dev_conf.intr_conf.rxq) 208 rxq_interrupt_enable = 1; 209 210 for (index = 0; index < enic->rq_count; index++) { 211 cq_idx = enic_cq_rq(enic, enic_rte_rq_idx_to_sop_idx(index)); 212 213 vnic_rq_init(&enic->rq[enic_rte_rq_idx_to_sop_idx(index)], 214 cq_idx, 215 error_interrupt_enable, 216 error_interrupt_offset); 217 218 data_rq = &enic->rq[enic_rte_rq_idx_to_data_idx(index)]; 219 if (data_rq->in_use) 220 vnic_rq_init(data_rq, 221 cq_idx, 222 error_interrupt_enable, 223 error_interrupt_offset); 224 225 vnic_cq_init(&enic->cq[cq_idx], 226 0 /* flow_control_enable */, 227 1 /* color_enable */, 228 0 /* cq_head */, 229 0 /* cq_tail */, 230 1 /* cq_tail_color */, 231 rxq_interrupt_enable, 232 1 /* cq_entry_enable */, 233 0 /* cq_message_enable */, 234 rxq_interrupt_offset, 235 0 /* cq_message_addr */); 236 if (rxq_interrupt_enable) 237 rxq_interrupt_offset++; 238 } 239 240 for (index = 0; index < enic->wq_count; index++) { 241 vnic_wq_init(&enic->wq[index], 242 enic_cq_wq(enic, index), 243 error_interrupt_enable, 244 error_interrupt_offset); 245 /* Compute unsupported ol flags for enic_prep_pkts() */ 246 enic->wq[index].tx_offload_notsup_mask = 247 PKT_TX_OFFLOAD_MASK ^ enic->tx_offload_mask; 248 249 cq_idx = enic_cq_wq(enic, index); 250 vnic_cq_init(&enic->cq[cq_idx], 251 0 /* flow_control_enable */, 252 1 /* color_enable */, 253 0 /* cq_head */, 254 0 /* cq_tail */, 255 1 /* cq_tail_color */, 256 0 /* interrupt_enable */, 257 0 /* cq_entry_enable */, 258 1 /* cq_message_enable */, 259 0 /* interrupt offset */, 260 (u64)enic->wq[index].cqmsg_rz->iova); 261 } 262 263 for (index = 0; index < enic->intr_count; index++) { 264 vnic_intr_init(&enic->intr[index], 265 enic->config.intr_timer_usec, 266 enic->config.intr_timer_type, 267 /*mask_on_assertion*/1); 268 } 269 } 270 271 272 static int 273 enic_alloc_rx_queue_mbufs(struct enic *enic, struct vnic_rq *rq) 274 { 275 struct rte_mbuf *mb; 276 struct rq_enet_desc *rqd = rq->ring.descs; 277 unsigned i; 278 dma_addr_t dma_addr; 279 uint32_t max_rx_pkt_len; 280 uint16_t rq_buf_len; 281 282 if (!rq->in_use) 283 return 0; 284 285 dev_debug(enic, "queue %u, allocating %u rx queue mbufs\n", rq->index, 286 rq->ring.desc_count); 287 288 /* 289 * If *not* using scatter and the mbuf size is greater than the 290 * requested max packet size (max_rx_pkt_len), then reduce the 291 * posted buffer size to max_rx_pkt_len. HW still receives packets 292 * larger than max_rx_pkt_len, but they will be truncated, which we 293 * drop in the rx handler. Not ideal, but better than returning 294 * large packets when the user is not expecting them. 295 */ 296 max_rx_pkt_len = enic->rte_dev->data->dev_conf.rxmode.max_rx_pkt_len; 297 rq_buf_len = rte_pktmbuf_data_room_size(rq->mp) - RTE_PKTMBUF_HEADROOM; 298 if (max_rx_pkt_len < rq_buf_len && !rq->data_queue_enable) 299 rq_buf_len = max_rx_pkt_len; 300 for (i = 0; i < rq->ring.desc_count; i++, rqd++) { 301 mb = rte_mbuf_raw_alloc(rq->mp); 302 if (mb == NULL) { 303 dev_err(enic, "RX mbuf alloc failed queue_id=%u\n", 304 (unsigned)rq->index); 305 return -ENOMEM; 306 } 307 308 mb->data_off = RTE_PKTMBUF_HEADROOM; 309 dma_addr = (dma_addr_t)(mb->buf_iova 310 + RTE_PKTMBUF_HEADROOM); 311 rq_enet_desc_enc(rqd, dma_addr, 312 (rq->is_sop ? RQ_ENET_TYPE_ONLY_SOP 313 : RQ_ENET_TYPE_NOT_SOP), 314 rq_buf_len); 315 rq->mbuf_ring[i] = mb; 316 } 317 /* 318 * Do not post the buffers to the NIC until we enable the RQ via 319 * enic_start_rq(). 320 */ 321 rq->need_initial_post = true; 322 /* Initialize fetch index while RQ is disabled */ 323 iowrite32(0, &rq->ctrl->fetch_index); 324 return 0; 325 } 326 327 /* 328 * Post the Rx buffers for the first time. enic_alloc_rx_queue_mbufs() has 329 * allocated the buffers and filled the RQ descriptor ring. Just need to push 330 * the post index to the NIC. 331 */ 332 static void 333 enic_initial_post_rx(struct enic *enic, struct vnic_rq *rq) 334 { 335 if (!rq->in_use || !rq->need_initial_post) 336 return; 337 338 /* make sure all prior writes are complete before doing the PIO write */ 339 rte_rmb(); 340 341 /* Post all but the last buffer to VIC. */ 342 rq->posted_index = rq->ring.desc_count - 1; 343 344 rq->rx_nb_hold = 0; 345 346 dev_debug(enic, "port=%u, qidx=%u, Write %u posted idx, %u sw held\n", 347 enic->port_id, rq->index, rq->posted_index, rq->rx_nb_hold); 348 iowrite32(rq->posted_index, &rq->ctrl->posted_index); 349 rte_rmb(); 350 rq->need_initial_post = false; 351 } 352 353 static void * 354 enic_alloc_consistent(void *priv, size_t size, 355 dma_addr_t *dma_handle, u8 *name) 356 { 357 void *vaddr; 358 const struct rte_memzone *rz; 359 *dma_handle = 0; 360 struct enic *enic = (struct enic *)priv; 361 struct enic_memzone_entry *mze; 362 363 rz = rte_memzone_reserve_aligned((const char *)name, size, 364 SOCKET_ID_ANY, RTE_MEMZONE_IOVA_CONTIG, ENIC_ALIGN); 365 if (!rz) { 366 pr_err("%s : Failed to allocate memory requested for %s\n", 367 __func__, name); 368 return NULL; 369 } 370 371 vaddr = rz->addr; 372 *dma_handle = (dma_addr_t)rz->iova; 373 374 mze = rte_malloc("enic memzone entry", 375 sizeof(struct enic_memzone_entry), 0); 376 377 if (!mze) { 378 pr_err("%s : Failed to allocate memory for memzone list\n", 379 __func__); 380 rte_memzone_free(rz); 381 return NULL; 382 } 383 384 mze->rz = rz; 385 386 rte_spinlock_lock(&enic->memzone_list_lock); 387 LIST_INSERT_HEAD(&enic->memzone_list, mze, entries); 388 rte_spinlock_unlock(&enic->memzone_list_lock); 389 390 return vaddr; 391 } 392 393 static void 394 enic_free_consistent(void *priv, 395 __rte_unused size_t size, 396 void *vaddr, 397 dma_addr_t dma_handle) 398 { 399 struct enic_memzone_entry *mze; 400 struct enic *enic = (struct enic *)priv; 401 402 rte_spinlock_lock(&enic->memzone_list_lock); 403 LIST_FOREACH(mze, &enic->memzone_list, entries) { 404 if (mze->rz->addr == vaddr && 405 mze->rz->iova == dma_handle) 406 break; 407 } 408 if (mze == NULL) { 409 rte_spinlock_unlock(&enic->memzone_list_lock); 410 dev_warning(enic, 411 "Tried to free memory, but couldn't find it in the memzone list\n"); 412 return; 413 } 414 LIST_REMOVE(mze, entries); 415 rte_spinlock_unlock(&enic->memzone_list_lock); 416 rte_memzone_free(mze->rz); 417 rte_free(mze); 418 } 419 420 int enic_link_update(struct enic *enic) 421 { 422 struct rte_eth_dev *eth_dev = enic->rte_dev; 423 struct rte_eth_link link; 424 425 memset(&link, 0, sizeof(link)); 426 link.link_status = enic_get_link_status(enic); 427 link.link_duplex = ETH_LINK_FULL_DUPLEX; 428 link.link_speed = vnic_dev_port_speed(enic->vdev); 429 430 return rte_eth_linkstatus_set(eth_dev, &link); 431 } 432 433 static void 434 enic_intr_handler(void *arg) 435 { 436 struct rte_eth_dev *dev = (struct rte_eth_dev *)arg; 437 struct enic *enic = pmd_priv(dev); 438 439 vnic_intr_return_all_credits(&enic->intr[ENICPMD_LSC_INTR_OFFSET]); 440 441 enic_link_update(enic); 442 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL); 443 enic_log_q_error(enic); 444 } 445 446 static int enic_rxq_intr_init(struct enic *enic) 447 { 448 struct rte_intr_handle *intr_handle; 449 uint32_t rxq_intr_count, i; 450 int err; 451 452 intr_handle = enic->rte_dev->intr_handle; 453 if (!enic->rte_dev->data->dev_conf.intr_conf.rxq) 454 return 0; 455 /* 456 * Rx queue interrupts only work when we have MSI-X interrupts, 457 * one per queue. Sharing one interrupt is technically 458 * possible with VIC, but it is not worth the complications it brings. 459 */ 460 if (!rte_intr_cap_multiple(intr_handle)) { 461 dev_err(enic, "Rx queue interrupts require MSI-X interrupts" 462 " (vfio-pci driver)\n"); 463 return -ENOTSUP; 464 } 465 rxq_intr_count = enic->intr_count - ENICPMD_RXQ_INTR_OFFSET; 466 err = rte_intr_efd_enable(intr_handle, rxq_intr_count); 467 if (err) { 468 dev_err(enic, "Failed to enable event fds for Rx queue" 469 " interrupts\n"); 470 return err; 471 } 472 intr_handle->intr_vec = rte_zmalloc("enic_intr_vec", 473 rxq_intr_count * sizeof(int), 0); 474 if (intr_handle->intr_vec == NULL) { 475 dev_err(enic, "Failed to allocate intr_vec\n"); 476 return -ENOMEM; 477 } 478 for (i = 0; i < rxq_intr_count; i++) 479 intr_handle->intr_vec[i] = i + ENICPMD_RXQ_INTR_OFFSET; 480 return 0; 481 } 482 483 static void enic_rxq_intr_deinit(struct enic *enic) 484 { 485 struct rte_intr_handle *intr_handle; 486 487 intr_handle = enic->rte_dev->intr_handle; 488 rte_intr_efd_disable(intr_handle); 489 if (intr_handle->intr_vec != NULL) { 490 rte_free(intr_handle->intr_vec); 491 intr_handle->intr_vec = NULL; 492 } 493 } 494 495 static void enic_prep_wq_for_simple_tx(struct enic *enic, uint16_t queue_idx) 496 { 497 struct wq_enet_desc *desc; 498 struct vnic_wq *wq; 499 unsigned int i; 500 501 /* 502 * Fill WQ descriptor fields that never change. Every descriptor is 503 * one packet, so set EOP. Also set CQ_ENTRY every ENIC_WQ_CQ_THRESH 504 * descriptors (i.e. request one completion update every 32 packets). 505 */ 506 wq = &enic->wq[queue_idx]; 507 desc = (struct wq_enet_desc *)wq->ring.descs; 508 for (i = 0; i < wq->ring.desc_count; i++, desc++) { 509 desc->header_length_flags = 1 << WQ_ENET_FLAGS_EOP_SHIFT; 510 if (i % ENIC_WQ_CQ_THRESH == ENIC_WQ_CQ_THRESH - 1) 511 desc->header_length_flags |= 512 (1 << WQ_ENET_FLAGS_CQ_ENTRY_SHIFT); 513 } 514 } 515 516 /* 517 * The 'strong' version is in enic_rxtx_vec_avx2.c. This weak version is used 518 * used when that file is not compiled. 519 */ 520 __rte_weak bool 521 enic_use_vector_rx_handler(__rte_unused struct enic *enic) 522 { 523 return false; 524 } 525 526 static void pick_rx_handler(struct enic *enic) 527 { 528 struct rte_eth_dev *eth_dev; 529 530 /* 531 * Preference order: 532 * 1. The vectorized handler if possible and requested. 533 * 2. The non-scatter, simplified handler if scatter Rx is not used. 534 * 3. The default handler as a fallback. 535 */ 536 eth_dev = enic->rte_dev; 537 if (enic_use_vector_rx_handler(enic)) 538 return; 539 if (enic->rq_count > 0 && enic->rq[0].data_queue_enable == 0) { 540 PMD_INIT_LOG(DEBUG, " use the non-scatter Rx handler"); 541 eth_dev->rx_pkt_burst = &enic_noscatter_recv_pkts; 542 } else { 543 PMD_INIT_LOG(DEBUG, " use the normal Rx handler"); 544 eth_dev->rx_pkt_burst = &enic_recv_pkts; 545 } 546 } 547 548 int enic_enable(struct enic *enic) 549 { 550 unsigned int index; 551 int err; 552 struct rte_eth_dev *eth_dev = enic->rte_dev; 553 uint64_t simple_tx_offloads; 554 uintptr_t p; 555 556 if (enic->enable_avx2_rx) { 557 struct rte_mbuf mb_def = { .buf_addr = 0 }; 558 559 /* 560 * mbuf_initializer contains const-after-init fields of 561 * receive mbufs (i.e. 64 bits of fields from rearm_data). 562 * It is currently used by the vectorized handler. 563 */ 564 mb_def.nb_segs = 1; 565 mb_def.data_off = RTE_PKTMBUF_HEADROOM; 566 mb_def.port = enic->port_id; 567 rte_mbuf_refcnt_set(&mb_def, 1); 568 rte_compiler_barrier(); 569 p = (uintptr_t)&mb_def.rearm_data; 570 enic->mbuf_initializer = *(uint64_t *)p; 571 } 572 573 eth_dev->data->dev_link.link_speed = vnic_dev_port_speed(enic->vdev); 574 eth_dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX; 575 576 /* vnic notification of link status has already been turned on in 577 * enic_dev_init() which is called during probe time. Here we are 578 * just turning on interrupt vector 0 if needed. 579 */ 580 if (eth_dev->data->dev_conf.intr_conf.lsc) 581 vnic_dev_notify_set(enic->vdev, 0); 582 583 err = enic_rxq_intr_init(enic); 584 if (err) 585 return err; 586 if (enic_clsf_init(enic)) 587 dev_warning(enic, "Init of hash table for clsf failed."\ 588 "Flow director feature will not work\n"); 589 590 for (index = 0; index < enic->rq_count; index++) { 591 err = enic_alloc_rx_queue_mbufs(enic, 592 &enic->rq[enic_rte_rq_idx_to_sop_idx(index)]); 593 if (err) { 594 dev_err(enic, "Failed to alloc sop RX queue mbufs\n"); 595 return err; 596 } 597 err = enic_alloc_rx_queue_mbufs(enic, 598 &enic->rq[enic_rte_rq_idx_to_data_idx(index)]); 599 if (err) { 600 /* release the allocated mbufs for the sop rq*/ 601 enic_rxmbuf_queue_release(enic, 602 &enic->rq[enic_rte_rq_idx_to_sop_idx(index)]); 603 604 dev_err(enic, "Failed to alloc data RX queue mbufs\n"); 605 return err; 606 } 607 } 608 609 /* 610 * Use the simple TX handler if possible. Only checksum offloads 611 * and vlan insertion are supported. 612 */ 613 simple_tx_offloads = enic->tx_offload_capa & 614 (DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM | 615 DEV_TX_OFFLOAD_VLAN_INSERT | 616 DEV_TX_OFFLOAD_IPV4_CKSUM | 617 DEV_TX_OFFLOAD_UDP_CKSUM | 618 DEV_TX_OFFLOAD_TCP_CKSUM); 619 if ((eth_dev->data->dev_conf.txmode.offloads & 620 ~simple_tx_offloads) == 0) { 621 PMD_INIT_LOG(DEBUG, " use the simple tx handler"); 622 eth_dev->tx_pkt_burst = &enic_simple_xmit_pkts; 623 for (index = 0; index < enic->wq_count; index++) 624 enic_prep_wq_for_simple_tx(enic, index); 625 } else { 626 PMD_INIT_LOG(DEBUG, " use the default tx handler"); 627 eth_dev->tx_pkt_burst = &enic_xmit_pkts; 628 } 629 630 pick_rx_handler(enic); 631 632 for (index = 0; index < enic->wq_count; index++) 633 enic_start_wq(enic, index); 634 for (index = 0; index < enic->rq_count; index++) 635 enic_start_rq(enic, index); 636 637 vnic_dev_add_addr(enic->vdev, enic->mac_addr); 638 639 vnic_dev_enable_wait(enic->vdev); 640 641 /* Register and enable error interrupt */ 642 rte_intr_callback_register(&(enic->pdev->intr_handle), 643 enic_intr_handler, (void *)enic->rte_dev); 644 645 rte_intr_enable(&(enic->pdev->intr_handle)); 646 /* Unmask LSC interrupt */ 647 vnic_intr_unmask(&enic->intr[ENICPMD_LSC_INTR_OFFSET]); 648 649 return 0; 650 } 651 652 int enic_alloc_intr_resources(struct enic *enic) 653 { 654 int err; 655 unsigned int i; 656 657 dev_info(enic, "vNIC resources used: "\ 658 "wq %d rq %d cq %d intr %d\n", 659 enic->wq_count, enic_vnic_rq_count(enic), 660 enic->cq_count, enic->intr_count); 661 662 for (i = 0; i < enic->intr_count; i++) { 663 err = vnic_intr_alloc(enic->vdev, &enic->intr[i], i); 664 if (err) { 665 enic_free_vnic_resources(enic); 666 return err; 667 } 668 } 669 return 0; 670 } 671 672 void enic_free_rq(void *rxq) 673 { 674 struct vnic_rq *rq_sop, *rq_data; 675 struct enic *enic; 676 677 if (rxq == NULL) 678 return; 679 680 rq_sop = (struct vnic_rq *)rxq; 681 enic = vnic_dev_priv(rq_sop->vdev); 682 rq_data = &enic->rq[rq_sop->data_queue_idx]; 683 684 if (rq_sop->free_mbufs) { 685 struct rte_mbuf **mb; 686 int i; 687 688 mb = rq_sop->free_mbufs; 689 for (i = ENIC_RX_BURST_MAX - rq_sop->num_free_mbufs; 690 i < ENIC_RX_BURST_MAX; i++) 691 rte_pktmbuf_free(mb[i]); 692 rte_free(rq_sop->free_mbufs); 693 rq_sop->free_mbufs = NULL; 694 rq_sop->num_free_mbufs = 0; 695 } 696 697 enic_rxmbuf_queue_release(enic, rq_sop); 698 if (rq_data->in_use) 699 enic_rxmbuf_queue_release(enic, rq_data); 700 701 rte_free(rq_sop->mbuf_ring); 702 if (rq_data->in_use) 703 rte_free(rq_data->mbuf_ring); 704 705 rq_sop->mbuf_ring = NULL; 706 rq_data->mbuf_ring = NULL; 707 708 vnic_rq_free(rq_sop); 709 if (rq_data->in_use) 710 vnic_rq_free(rq_data); 711 712 vnic_cq_free(&enic->cq[enic_sop_rq_idx_to_cq_idx(rq_sop->index)]); 713 714 rq_sop->in_use = 0; 715 rq_data->in_use = 0; 716 } 717 718 void enic_start_wq(struct enic *enic, uint16_t queue_idx) 719 { 720 struct rte_eth_dev *eth_dev = enic->rte_dev; 721 vnic_wq_enable(&enic->wq[queue_idx]); 722 eth_dev->data->tx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STARTED; 723 } 724 725 int enic_stop_wq(struct enic *enic, uint16_t queue_idx) 726 { 727 struct rte_eth_dev *eth_dev = enic->rte_dev; 728 int ret; 729 730 ret = vnic_wq_disable(&enic->wq[queue_idx]); 731 if (ret) 732 return ret; 733 734 eth_dev->data->tx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STOPPED; 735 return 0; 736 } 737 738 void enic_start_rq(struct enic *enic, uint16_t queue_idx) 739 { 740 struct vnic_rq *rq_sop; 741 struct vnic_rq *rq_data; 742 rq_sop = &enic->rq[enic_rte_rq_idx_to_sop_idx(queue_idx)]; 743 rq_data = &enic->rq[rq_sop->data_queue_idx]; 744 struct rte_eth_dev *eth_dev = enic->rte_dev; 745 746 if (rq_data->in_use) { 747 vnic_rq_enable(rq_data); 748 enic_initial_post_rx(enic, rq_data); 749 } 750 rte_mb(); 751 vnic_rq_enable(rq_sop); 752 enic_initial_post_rx(enic, rq_sop); 753 eth_dev->data->rx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STARTED; 754 } 755 756 int enic_stop_rq(struct enic *enic, uint16_t queue_idx) 757 { 758 int ret1 = 0, ret2 = 0; 759 struct rte_eth_dev *eth_dev = enic->rte_dev; 760 struct vnic_rq *rq_sop; 761 struct vnic_rq *rq_data; 762 rq_sop = &enic->rq[enic_rte_rq_idx_to_sop_idx(queue_idx)]; 763 rq_data = &enic->rq[rq_sop->data_queue_idx]; 764 765 ret2 = vnic_rq_disable(rq_sop); 766 rte_mb(); 767 if (rq_data->in_use) 768 ret1 = vnic_rq_disable(rq_data); 769 770 if (ret2) 771 return ret2; 772 else if (ret1) 773 return ret1; 774 775 eth_dev->data->rx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STOPPED; 776 return 0; 777 } 778 779 int enic_alloc_rq(struct enic *enic, uint16_t queue_idx, 780 unsigned int socket_id, struct rte_mempool *mp, 781 uint16_t nb_desc, uint16_t free_thresh) 782 { 783 int rc; 784 uint16_t sop_queue_idx = enic_rte_rq_idx_to_sop_idx(queue_idx); 785 uint16_t data_queue_idx = enic_rte_rq_idx_to_data_idx(queue_idx); 786 struct vnic_rq *rq_sop = &enic->rq[sop_queue_idx]; 787 struct vnic_rq *rq_data = &enic->rq[data_queue_idx]; 788 unsigned int mbuf_size, mbufs_per_pkt; 789 unsigned int nb_sop_desc, nb_data_desc; 790 uint16_t min_sop, max_sop, min_data, max_data; 791 uint32_t max_rx_pkt_len; 792 793 rq_sop->is_sop = 1; 794 rq_sop->data_queue_idx = data_queue_idx; 795 rq_data->is_sop = 0; 796 rq_data->data_queue_idx = 0; 797 rq_sop->socket_id = socket_id; 798 rq_sop->mp = mp; 799 rq_data->socket_id = socket_id; 800 rq_data->mp = mp; 801 rq_sop->in_use = 1; 802 rq_sop->rx_free_thresh = free_thresh; 803 rq_data->rx_free_thresh = free_thresh; 804 dev_debug(enic, "Set queue_id:%u free thresh:%u\n", queue_idx, 805 free_thresh); 806 807 mbuf_size = (uint16_t)(rte_pktmbuf_data_room_size(mp) - 808 RTE_PKTMBUF_HEADROOM); 809 /* max_rx_pkt_len includes the ethernet header and CRC. */ 810 max_rx_pkt_len = enic->rte_dev->data->dev_conf.rxmode.max_rx_pkt_len; 811 812 if (enic->rte_dev->data->dev_conf.rxmode.offloads & 813 DEV_RX_OFFLOAD_SCATTER) { 814 dev_info(enic, "Rq %u Scatter rx mode enabled\n", queue_idx); 815 /* ceil((max pkt len)/mbuf_size) */ 816 mbufs_per_pkt = (max_rx_pkt_len + mbuf_size - 1) / mbuf_size; 817 } else { 818 dev_info(enic, "Scatter rx mode disabled\n"); 819 mbufs_per_pkt = 1; 820 if (max_rx_pkt_len > mbuf_size) { 821 dev_warning(enic, "The maximum Rx packet size (%u) is" 822 " larger than the mbuf size (%u), and" 823 " scatter is disabled. Larger packets will" 824 " be truncated.\n", 825 max_rx_pkt_len, mbuf_size); 826 } 827 } 828 829 if (mbufs_per_pkt > 1) { 830 dev_info(enic, "Rq %u Scatter rx mode in use\n", queue_idx); 831 rq_sop->data_queue_enable = 1; 832 rq_data->in_use = 1; 833 /* 834 * HW does not directly support rxmode.max_rx_pkt_len. HW always 835 * receives packet sizes up to the "max" MTU. 836 * If not using scatter, we can achieve the effect of dropping 837 * larger packets by reducing the size of posted buffers. 838 * See enic_alloc_rx_queue_mbufs(). 839 */ 840 if (max_rx_pkt_len < 841 enic_mtu_to_max_rx_pktlen(enic->max_mtu)) { 842 dev_warning(enic, "rxmode.max_rx_pkt_len is ignored" 843 " when scatter rx mode is in use.\n"); 844 } 845 } else { 846 dev_info(enic, "Rq %u Scatter rx mode not being used\n", 847 queue_idx); 848 rq_sop->data_queue_enable = 0; 849 rq_data->in_use = 0; 850 } 851 852 /* number of descriptors have to be a multiple of 32 */ 853 nb_sop_desc = (nb_desc / mbufs_per_pkt) & ENIC_ALIGN_DESCS_MASK; 854 nb_data_desc = (nb_desc - nb_sop_desc) & ENIC_ALIGN_DESCS_MASK; 855 856 rq_sop->max_mbufs_per_pkt = mbufs_per_pkt; 857 rq_data->max_mbufs_per_pkt = mbufs_per_pkt; 858 859 if (mbufs_per_pkt > 1) { 860 min_sop = ENIC_RX_BURST_MAX; 861 max_sop = ((enic->config.rq_desc_count / 862 (mbufs_per_pkt - 1)) & ENIC_ALIGN_DESCS_MASK); 863 min_data = min_sop * (mbufs_per_pkt - 1); 864 max_data = enic->config.rq_desc_count; 865 } else { 866 min_sop = ENIC_RX_BURST_MAX; 867 max_sop = enic->config.rq_desc_count; 868 min_data = 0; 869 max_data = 0; 870 } 871 872 if (nb_desc < (min_sop + min_data)) { 873 dev_warning(enic, 874 "Number of rx descs too low, adjusting to minimum\n"); 875 nb_sop_desc = min_sop; 876 nb_data_desc = min_data; 877 } else if (nb_desc > (max_sop + max_data)) { 878 dev_warning(enic, 879 "Number of rx_descs too high, adjusting to maximum\n"); 880 nb_sop_desc = max_sop; 881 nb_data_desc = max_data; 882 } 883 if (mbufs_per_pkt > 1) { 884 dev_info(enic, "For max packet size %u and mbuf size %u valid" 885 " rx descriptor range is %u to %u\n", 886 max_rx_pkt_len, mbuf_size, min_sop + min_data, 887 max_sop + max_data); 888 } 889 dev_info(enic, "Using %d rx descriptors (sop %d, data %d)\n", 890 nb_sop_desc + nb_data_desc, nb_sop_desc, nb_data_desc); 891 892 /* Allocate sop queue resources */ 893 rc = vnic_rq_alloc(enic->vdev, rq_sop, sop_queue_idx, 894 nb_sop_desc, sizeof(struct rq_enet_desc)); 895 if (rc) { 896 dev_err(enic, "error in allocation of sop rq\n"); 897 goto err_exit; 898 } 899 nb_sop_desc = rq_sop->ring.desc_count; 900 901 if (rq_data->in_use) { 902 /* Allocate data queue resources */ 903 rc = vnic_rq_alloc(enic->vdev, rq_data, data_queue_idx, 904 nb_data_desc, 905 sizeof(struct rq_enet_desc)); 906 if (rc) { 907 dev_err(enic, "error in allocation of data rq\n"); 908 goto err_free_rq_sop; 909 } 910 nb_data_desc = rq_data->ring.desc_count; 911 } 912 rc = vnic_cq_alloc(enic->vdev, &enic->cq[queue_idx], queue_idx, 913 socket_id, nb_sop_desc + nb_data_desc, 914 sizeof(struct cq_enet_rq_desc)); 915 if (rc) { 916 dev_err(enic, "error in allocation of cq for rq\n"); 917 goto err_free_rq_data; 918 } 919 920 /* Allocate the mbuf rings */ 921 rq_sop->mbuf_ring = (struct rte_mbuf **) 922 rte_zmalloc_socket("rq->mbuf_ring", 923 sizeof(struct rte_mbuf *) * nb_sop_desc, 924 RTE_CACHE_LINE_SIZE, rq_sop->socket_id); 925 if (rq_sop->mbuf_ring == NULL) 926 goto err_free_cq; 927 928 if (rq_data->in_use) { 929 rq_data->mbuf_ring = (struct rte_mbuf **) 930 rte_zmalloc_socket("rq->mbuf_ring", 931 sizeof(struct rte_mbuf *) * nb_data_desc, 932 RTE_CACHE_LINE_SIZE, rq_sop->socket_id); 933 if (rq_data->mbuf_ring == NULL) 934 goto err_free_sop_mbuf; 935 } 936 937 rq_sop->free_mbufs = (struct rte_mbuf **) 938 rte_zmalloc_socket("rq->free_mbufs", 939 sizeof(struct rte_mbuf *) * 940 ENIC_RX_BURST_MAX, 941 RTE_CACHE_LINE_SIZE, rq_sop->socket_id); 942 if (rq_sop->free_mbufs == NULL) 943 goto err_free_data_mbuf; 944 rq_sop->num_free_mbufs = 0; 945 946 rq_sop->tot_nb_desc = nb_desc; /* squirl away for MTU update function */ 947 948 return 0; 949 950 err_free_data_mbuf: 951 rte_free(rq_data->mbuf_ring); 952 err_free_sop_mbuf: 953 rte_free(rq_sop->mbuf_ring); 954 err_free_cq: 955 /* cleanup on error */ 956 vnic_cq_free(&enic->cq[queue_idx]); 957 err_free_rq_data: 958 if (rq_data->in_use) 959 vnic_rq_free(rq_data); 960 err_free_rq_sop: 961 vnic_rq_free(rq_sop); 962 err_exit: 963 return -ENOMEM; 964 } 965 966 void enic_free_wq(void *txq) 967 { 968 struct vnic_wq *wq; 969 struct enic *enic; 970 971 if (txq == NULL) 972 return; 973 974 wq = (struct vnic_wq *)txq; 975 enic = vnic_dev_priv(wq->vdev); 976 rte_memzone_free(wq->cqmsg_rz); 977 vnic_wq_free(wq); 978 vnic_cq_free(&enic->cq[enic->rq_count + wq->index]); 979 } 980 981 int enic_alloc_wq(struct enic *enic, uint16_t queue_idx, 982 unsigned int socket_id, uint16_t nb_desc) 983 { 984 int err; 985 struct vnic_wq *wq = &enic->wq[queue_idx]; 986 unsigned int cq_index = enic_cq_wq(enic, queue_idx); 987 char name[NAME_MAX]; 988 static int instance; 989 990 wq->socket_id = socket_id; 991 /* 992 * rte_eth_tx_queue_setup() checks min, max, and alignment. So just 993 * print an info message for diagnostics. 994 */ 995 dev_info(enic, "TX Queues - effective number of descs:%d\n", nb_desc); 996 997 /* Allocate queue resources */ 998 err = vnic_wq_alloc(enic->vdev, &enic->wq[queue_idx], queue_idx, 999 nb_desc, 1000 sizeof(struct wq_enet_desc)); 1001 if (err) { 1002 dev_err(enic, "error in allocation of wq\n"); 1003 return err; 1004 } 1005 1006 err = vnic_cq_alloc(enic->vdev, &enic->cq[cq_index], cq_index, 1007 socket_id, nb_desc, 1008 sizeof(struct cq_enet_wq_desc)); 1009 if (err) { 1010 vnic_wq_free(wq); 1011 dev_err(enic, "error in allocation of cq for wq\n"); 1012 } 1013 1014 /* setup up CQ message */ 1015 snprintf((char *)name, sizeof(name), 1016 "vnic_cqmsg-%s-%d-%d", enic->bdf_name, queue_idx, 1017 instance++); 1018 1019 wq->cqmsg_rz = rte_memzone_reserve_aligned((const char *)name, 1020 sizeof(uint32_t), SOCKET_ID_ANY, 1021 RTE_MEMZONE_IOVA_CONTIG, ENIC_ALIGN); 1022 if (!wq->cqmsg_rz) 1023 return -ENOMEM; 1024 1025 return err; 1026 } 1027 1028 int enic_disable(struct enic *enic) 1029 { 1030 unsigned int i; 1031 int err; 1032 1033 for (i = 0; i < enic->intr_count; i++) { 1034 vnic_intr_mask(&enic->intr[i]); 1035 (void)vnic_intr_masked(&enic->intr[i]); /* flush write */ 1036 } 1037 enic_rxq_intr_deinit(enic); 1038 rte_intr_disable(&enic->pdev->intr_handle); 1039 rte_intr_callback_unregister(&enic->pdev->intr_handle, 1040 enic_intr_handler, 1041 (void *)enic->rte_dev); 1042 1043 vnic_dev_disable(enic->vdev); 1044 1045 enic_clsf_destroy(enic); 1046 1047 if (!enic_is_sriov_vf(enic)) 1048 vnic_dev_del_addr(enic->vdev, enic->mac_addr); 1049 1050 for (i = 0; i < enic->wq_count; i++) { 1051 err = vnic_wq_disable(&enic->wq[i]); 1052 if (err) 1053 return err; 1054 } 1055 for (i = 0; i < enic_vnic_rq_count(enic); i++) { 1056 if (enic->rq[i].in_use) { 1057 err = vnic_rq_disable(&enic->rq[i]); 1058 if (err) 1059 return err; 1060 } 1061 } 1062 1063 /* If we were using interrupts, set the interrupt vector to -1 1064 * to disable interrupts. We are not disabling link notifcations, 1065 * though, as we want the polling of link status to continue working. 1066 */ 1067 if (enic->rte_dev->data->dev_conf.intr_conf.lsc) 1068 vnic_dev_notify_set(enic->vdev, -1); 1069 1070 vnic_dev_set_reset_flag(enic->vdev, 1); 1071 1072 for (i = 0; i < enic->wq_count; i++) 1073 vnic_wq_clean(&enic->wq[i], enic_free_wq_buf); 1074 1075 for (i = 0; i < enic_vnic_rq_count(enic); i++) 1076 if (enic->rq[i].in_use) 1077 vnic_rq_clean(&enic->rq[i], enic_free_rq_buf); 1078 for (i = 0; i < enic->cq_count; i++) 1079 vnic_cq_clean(&enic->cq[i]); 1080 for (i = 0; i < enic->intr_count; i++) 1081 vnic_intr_clean(&enic->intr[i]); 1082 1083 return 0; 1084 } 1085 1086 static int enic_dev_wait(struct vnic_dev *vdev, 1087 int (*start)(struct vnic_dev *, int), 1088 int (*finished)(struct vnic_dev *, int *), 1089 int arg) 1090 { 1091 int done; 1092 int err; 1093 int i; 1094 1095 err = start(vdev, arg); 1096 if (err) 1097 return err; 1098 1099 /* Wait for func to complete...2 seconds max */ 1100 for (i = 0; i < 2000; i++) { 1101 err = finished(vdev, &done); 1102 if (err) 1103 return err; 1104 if (done) 1105 return 0; 1106 usleep(1000); 1107 } 1108 return -ETIMEDOUT; 1109 } 1110 1111 static int enic_dev_open(struct enic *enic) 1112 { 1113 int err; 1114 int flags = CMD_OPENF_IG_DESCCACHE; 1115 1116 err = enic_dev_wait(enic->vdev, vnic_dev_open, 1117 vnic_dev_open_done, flags); 1118 if (err) 1119 dev_err(enic_get_dev(enic), 1120 "vNIC device open failed, err %d\n", err); 1121 1122 return err; 1123 } 1124 1125 static int enic_set_rsskey(struct enic *enic, uint8_t *user_key) 1126 { 1127 dma_addr_t rss_key_buf_pa; 1128 union vnic_rss_key *rss_key_buf_va = NULL; 1129 int err, i; 1130 u8 name[NAME_MAX]; 1131 1132 RTE_ASSERT(user_key != NULL); 1133 snprintf((char *)name, NAME_MAX, "rss_key-%s", enic->bdf_name); 1134 rss_key_buf_va = enic_alloc_consistent(enic, sizeof(union vnic_rss_key), 1135 &rss_key_buf_pa, name); 1136 if (!rss_key_buf_va) 1137 return -ENOMEM; 1138 1139 for (i = 0; i < ENIC_RSS_HASH_KEY_SIZE; i++) 1140 rss_key_buf_va->key[i / 10].b[i % 10] = user_key[i]; 1141 1142 err = enic_set_rss_key(enic, 1143 rss_key_buf_pa, 1144 sizeof(union vnic_rss_key)); 1145 1146 /* Save for later queries */ 1147 if (!err) { 1148 rte_memcpy(&enic->rss_key, rss_key_buf_va, 1149 sizeof(union vnic_rss_key)); 1150 } 1151 enic_free_consistent(enic, sizeof(union vnic_rss_key), 1152 rss_key_buf_va, rss_key_buf_pa); 1153 1154 return err; 1155 } 1156 1157 int enic_set_rss_reta(struct enic *enic, union vnic_rss_cpu *rss_cpu) 1158 { 1159 dma_addr_t rss_cpu_buf_pa; 1160 union vnic_rss_cpu *rss_cpu_buf_va = NULL; 1161 int err; 1162 u8 name[NAME_MAX]; 1163 1164 snprintf((char *)name, NAME_MAX, "rss_cpu-%s", enic->bdf_name); 1165 rss_cpu_buf_va = enic_alloc_consistent(enic, sizeof(union vnic_rss_cpu), 1166 &rss_cpu_buf_pa, name); 1167 if (!rss_cpu_buf_va) 1168 return -ENOMEM; 1169 1170 rte_memcpy(rss_cpu_buf_va, rss_cpu, sizeof(union vnic_rss_cpu)); 1171 1172 err = enic_set_rss_cpu(enic, 1173 rss_cpu_buf_pa, 1174 sizeof(union vnic_rss_cpu)); 1175 1176 enic_free_consistent(enic, sizeof(union vnic_rss_cpu), 1177 rss_cpu_buf_va, rss_cpu_buf_pa); 1178 1179 /* Save for later queries */ 1180 if (!err) 1181 rte_memcpy(&enic->rss_cpu, rss_cpu, sizeof(union vnic_rss_cpu)); 1182 return err; 1183 } 1184 1185 static int enic_set_niccfg(struct enic *enic, u8 rss_default_cpu, 1186 u8 rss_hash_type, u8 rss_hash_bits, u8 rss_base_cpu, u8 rss_enable) 1187 { 1188 const u8 tso_ipid_split_en = 0; 1189 int err; 1190 1191 err = enic_set_nic_cfg(enic, 1192 rss_default_cpu, rss_hash_type, 1193 rss_hash_bits, rss_base_cpu, 1194 rss_enable, tso_ipid_split_en, 1195 enic->ig_vlan_strip_en); 1196 1197 return err; 1198 } 1199 1200 /* Initialize RSS with defaults, called from dev_configure */ 1201 int enic_init_rss_nic_cfg(struct enic *enic) 1202 { 1203 static uint8_t default_rss_key[] = { 1204 85, 67, 83, 97, 119, 101, 115, 111, 109, 101, 1205 80, 65, 76, 79, 117, 110, 105, 113, 117, 101, 1206 76, 73, 78, 85, 88, 114, 111, 99, 107, 115, 1207 69, 78, 73, 67, 105, 115, 99, 111, 111, 108, 1208 }; 1209 struct rte_eth_rss_conf rss_conf; 1210 union vnic_rss_cpu rss_cpu; 1211 int ret, i; 1212 1213 rss_conf = enic->rte_dev->data->dev_conf.rx_adv_conf.rss_conf; 1214 /* 1215 * If setting key for the first time, and the user gives us none, then 1216 * push the default key to NIC. 1217 */ 1218 if (rss_conf.rss_key == NULL) { 1219 rss_conf.rss_key = default_rss_key; 1220 rss_conf.rss_key_len = ENIC_RSS_HASH_KEY_SIZE; 1221 } 1222 ret = enic_set_rss_conf(enic, &rss_conf); 1223 if (ret) { 1224 dev_err(enic, "Failed to configure RSS\n"); 1225 return ret; 1226 } 1227 if (enic->rss_enable) { 1228 /* If enabling RSS, use the default reta */ 1229 for (i = 0; i < ENIC_RSS_RETA_SIZE; i++) { 1230 rss_cpu.cpu[i / 4].b[i % 4] = 1231 enic_rte_rq_idx_to_sop_idx(i % enic->rq_count); 1232 } 1233 ret = enic_set_rss_reta(enic, &rss_cpu); 1234 if (ret) 1235 dev_err(enic, "Failed to set RSS indirection table\n"); 1236 } 1237 return ret; 1238 } 1239 1240 int enic_setup_finish(struct enic *enic) 1241 { 1242 enic_init_soft_stats(enic); 1243 1244 /* Default conf */ 1245 vnic_dev_packet_filter(enic->vdev, 1246 1 /* directed */, 1247 1 /* multicast */, 1248 1 /* broadcast */, 1249 0 /* promisc */, 1250 1 /* allmulti */); 1251 1252 enic->promisc = 0; 1253 enic->allmulti = 1; 1254 1255 return 0; 1256 } 1257 1258 static int enic_rss_conf_valid(struct enic *enic, 1259 struct rte_eth_rss_conf *rss_conf) 1260 { 1261 /* RSS is disabled per VIC settings. Ignore rss_conf. */ 1262 if (enic->flow_type_rss_offloads == 0) 1263 return 0; 1264 if (rss_conf->rss_key != NULL && 1265 rss_conf->rss_key_len != ENIC_RSS_HASH_KEY_SIZE) { 1266 dev_err(enic, "Given rss_key is %d bytes, it must be %d\n", 1267 rss_conf->rss_key_len, ENIC_RSS_HASH_KEY_SIZE); 1268 return -EINVAL; 1269 } 1270 if (rss_conf->rss_hf != 0 && 1271 (rss_conf->rss_hf & enic->flow_type_rss_offloads) == 0) { 1272 dev_err(enic, "Given rss_hf contains none of the supported" 1273 " types\n"); 1274 return -EINVAL; 1275 } 1276 return 0; 1277 } 1278 1279 /* Set hash type and key according to rss_conf */ 1280 int enic_set_rss_conf(struct enic *enic, struct rte_eth_rss_conf *rss_conf) 1281 { 1282 struct rte_eth_dev *eth_dev; 1283 uint64_t rss_hf; 1284 u8 rss_hash_type; 1285 u8 rss_enable; 1286 int ret; 1287 1288 RTE_ASSERT(rss_conf != NULL); 1289 ret = enic_rss_conf_valid(enic, rss_conf); 1290 if (ret) { 1291 dev_err(enic, "RSS configuration (rss_conf) is invalid\n"); 1292 return ret; 1293 } 1294 1295 eth_dev = enic->rte_dev; 1296 rss_hash_type = 0; 1297 rss_hf = rss_conf->rss_hf & enic->flow_type_rss_offloads; 1298 if (enic->rq_count > 1 && 1299 (eth_dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) && 1300 rss_hf != 0) { 1301 rss_enable = 1; 1302 if (rss_hf & (ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | 1303 ETH_RSS_NONFRAG_IPV4_OTHER)) 1304 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_IPV4; 1305 if (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP) 1306 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV4; 1307 if (rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) { 1308 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_UDP_IPV4; 1309 if (enic->udp_rss_weak) { 1310 /* 1311 * 'TCP' is not a typo. The "weak" version of 1312 * UDP RSS requires both the TCP and UDP bits 1313 * be set. It does enable TCP RSS as well. 1314 */ 1315 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV4; 1316 } 1317 } 1318 if (rss_hf & (ETH_RSS_IPV6 | ETH_RSS_IPV6_EX | 1319 ETH_RSS_FRAG_IPV6 | ETH_RSS_NONFRAG_IPV6_OTHER)) 1320 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_IPV6; 1321 if (rss_hf & (ETH_RSS_NONFRAG_IPV6_TCP | ETH_RSS_IPV6_TCP_EX)) 1322 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV6; 1323 if (rss_hf & (ETH_RSS_NONFRAG_IPV6_UDP | ETH_RSS_IPV6_UDP_EX)) { 1324 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_UDP_IPV6; 1325 if (enic->udp_rss_weak) 1326 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV6; 1327 } 1328 } else { 1329 rss_enable = 0; 1330 rss_hf = 0; 1331 } 1332 1333 /* Set the hash key if provided */ 1334 if (rss_enable && rss_conf->rss_key) { 1335 ret = enic_set_rsskey(enic, rss_conf->rss_key); 1336 if (ret) { 1337 dev_err(enic, "Failed to set RSS key\n"); 1338 return ret; 1339 } 1340 } 1341 1342 ret = enic_set_niccfg(enic, ENIC_RSS_DEFAULT_CPU, rss_hash_type, 1343 ENIC_RSS_HASH_BITS, ENIC_RSS_BASE_CPU, 1344 rss_enable); 1345 if (!ret) { 1346 enic->rss_hf = rss_hf; 1347 enic->rss_hash_type = rss_hash_type; 1348 enic->rss_enable = rss_enable; 1349 } else { 1350 dev_err(enic, "Failed to update RSS configurations." 1351 " hash=0x%x\n", rss_hash_type); 1352 } 1353 return ret; 1354 } 1355 1356 int enic_set_vlan_strip(struct enic *enic) 1357 { 1358 /* 1359 * Unfortunately, VLAN strip on/off and RSS on/off are configured 1360 * together. So, re-do niccfg, preserving the current RSS settings. 1361 */ 1362 return enic_set_niccfg(enic, ENIC_RSS_DEFAULT_CPU, enic->rss_hash_type, 1363 ENIC_RSS_HASH_BITS, ENIC_RSS_BASE_CPU, 1364 enic->rss_enable); 1365 } 1366 1367 void enic_add_packet_filter(struct enic *enic) 1368 { 1369 /* Args -> directed, multicast, broadcast, promisc, allmulti */ 1370 vnic_dev_packet_filter(enic->vdev, 1, 1, 1, 1371 enic->promisc, enic->allmulti); 1372 } 1373 1374 int enic_get_link_status(struct enic *enic) 1375 { 1376 return vnic_dev_link_status(enic->vdev); 1377 } 1378 1379 static void enic_dev_deinit(struct enic *enic) 1380 { 1381 /* stop link status checking */ 1382 vnic_dev_notify_unset(enic->vdev); 1383 1384 /* mac_addrs is freed by rte_eth_dev_release_port() */ 1385 rte_free(enic->cq); 1386 rte_free(enic->intr); 1387 rte_free(enic->rq); 1388 rte_free(enic->wq); 1389 } 1390 1391 1392 int enic_set_vnic_res(struct enic *enic) 1393 { 1394 struct rte_eth_dev *eth_dev = enic->rte_dev; 1395 int rc = 0; 1396 unsigned int required_rq, required_wq, required_cq, required_intr; 1397 1398 /* Always use two vNIC RQs per eth_dev RQ, regardless of Rx scatter. */ 1399 required_rq = eth_dev->data->nb_rx_queues * 2; 1400 required_wq = eth_dev->data->nb_tx_queues; 1401 required_cq = eth_dev->data->nb_rx_queues + eth_dev->data->nb_tx_queues; 1402 required_intr = 1; /* 1 for LSC even if intr_conf.lsc is 0 */ 1403 if (eth_dev->data->dev_conf.intr_conf.rxq) { 1404 required_intr += eth_dev->data->nb_rx_queues; 1405 } 1406 1407 if (enic->conf_rq_count < required_rq) { 1408 dev_err(dev, "Not enough Receive queues. Requested:%u which uses %d RQs on VIC, Configured:%u\n", 1409 eth_dev->data->nb_rx_queues, 1410 required_rq, enic->conf_rq_count); 1411 rc = -EINVAL; 1412 } 1413 if (enic->conf_wq_count < required_wq) { 1414 dev_err(dev, "Not enough Transmit queues. Requested:%u, Configured:%u\n", 1415 eth_dev->data->nb_tx_queues, enic->conf_wq_count); 1416 rc = -EINVAL; 1417 } 1418 1419 if (enic->conf_cq_count < required_cq) { 1420 dev_err(dev, "Not enough Completion queues. Required:%u, Configured:%u\n", 1421 required_cq, enic->conf_cq_count); 1422 rc = -EINVAL; 1423 } 1424 if (enic->conf_intr_count < required_intr) { 1425 dev_err(dev, "Not enough Interrupts to support Rx queue" 1426 " interrupts. Required:%u, Configured:%u\n", 1427 required_intr, enic->conf_intr_count); 1428 rc = -EINVAL; 1429 } 1430 1431 if (rc == 0) { 1432 enic->rq_count = eth_dev->data->nb_rx_queues; 1433 enic->wq_count = eth_dev->data->nb_tx_queues; 1434 enic->cq_count = enic->rq_count + enic->wq_count; 1435 enic->intr_count = required_intr; 1436 } 1437 1438 return rc; 1439 } 1440 1441 /* Initialize the completion queue for an RQ */ 1442 static int 1443 enic_reinit_rq(struct enic *enic, unsigned int rq_idx) 1444 { 1445 struct vnic_rq *sop_rq, *data_rq; 1446 unsigned int cq_idx; 1447 int rc = 0; 1448 1449 sop_rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)]; 1450 data_rq = &enic->rq[enic_rte_rq_idx_to_data_idx(rq_idx)]; 1451 cq_idx = rq_idx; 1452 1453 vnic_cq_clean(&enic->cq[cq_idx]); 1454 vnic_cq_init(&enic->cq[cq_idx], 1455 0 /* flow_control_enable */, 1456 1 /* color_enable */, 1457 0 /* cq_head */, 1458 0 /* cq_tail */, 1459 1 /* cq_tail_color */, 1460 0 /* interrupt_enable */, 1461 1 /* cq_entry_enable */, 1462 0 /* cq_message_enable */, 1463 0 /* interrupt offset */, 1464 0 /* cq_message_addr */); 1465 1466 1467 vnic_rq_init_start(sop_rq, enic_cq_rq(enic, 1468 enic_rte_rq_idx_to_sop_idx(rq_idx)), 0, 1469 sop_rq->ring.desc_count - 1, 1, 0); 1470 if (data_rq->in_use) { 1471 vnic_rq_init_start(data_rq, 1472 enic_cq_rq(enic, 1473 enic_rte_rq_idx_to_data_idx(rq_idx)), 0, 1474 data_rq->ring.desc_count - 1, 1, 0); 1475 } 1476 1477 rc = enic_alloc_rx_queue_mbufs(enic, sop_rq); 1478 if (rc) 1479 return rc; 1480 1481 if (data_rq->in_use) { 1482 rc = enic_alloc_rx_queue_mbufs(enic, data_rq); 1483 if (rc) { 1484 enic_rxmbuf_queue_release(enic, sop_rq); 1485 return rc; 1486 } 1487 } 1488 1489 return 0; 1490 } 1491 1492 /* The Cisco NIC can send and receive packets up to a max packet size 1493 * determined by the NIC type and firmware. There is also an MTU 1494 * configured into the NIC via the CIMC/UCSM management interface 1495 * which can be overridden by this function (up to the max packet size). 1496 * Depending on the network setup, doing so may cause packet drops 1497 * and unexpected behavior. 1498 */ 1499 int enic_set_mtu(struct enic *enic, uint16_t new_mtu) 1500 { 1501 unsigned int rq_idx; 1502 struct vnic_rq *rq; 1503 int rc = 0; 1504 uint16_t old_mtu; /* previous setting */ 1505 uint16_t config_mtu; /* Value configured into NIC via CIMC/UCSM */ 1506 struct rte_eth_dev *eth_dev = enic->rte_dev; 1507 1508 old_mtu = eth_dev->data->mtu; 1509 config_mtu = enic->config.mtu; 1510 1511 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 1512 return -E_RTE_SECONDARY; 1513 1514 if (new_mtu > enic->max_mtu) { 1515 dev_err(enic, 1516 "MTU not updated: requested (%u) greater than max (%u)\n", 1517 new_mtu, enic->max_mtu); 1518 return -EINVAL; 1519 } 1520 if (new_mtu < ENIC_MIN_MTU) { 1521 dev_info(enic, 1522 "MTU not updated: requested (%u) less than min (%u)\n", 1523 new_mtu, ENIC_MIN_MTU); 1524 return -EINVAL; 1525 } 1526 if (new_mtu > config_mtu) 1527 dev_warning(enic, 1528 "MTU (%u) is greater than value configured in NIC (%u)\n", 1529 new_mtu, config_mtu); 1530 1531 /* Update the MTU and maximum packet length */ 1532 eth_dev->data->mtu = new_mtu; 1533 eth_dev->data->dev_conf.rxmode.max_rx_pkt_len = 1534 enic_mtu_to_max_rx_pktlen(new_mtu); 1535 1536 /* 1537 * If the device has not started (enic_enable), nothing to do. 1538 * Later, enic_enable() will set up RQs reflecting the new maximum 1539 * packet length. 1540 */ 1541 if (!eth_dev->data->dev_started) 1542 goto set_mtu_done; 1543 1544 /* 1545 * The device has started, re-do RQs on the fly. In the process, we 1546 * pick up the new maximum packet length. 1547 * 1548 * Some applications rely on the ability to change MTU without stopping 1549 * the device. So keep this behavior for now. 1550 */ 1551 rte_spinlock_lock(&enic->mtu_lock); 1552 1553 /* Stop traffic on all RQs */ 1554 for (rq_idx = 0; rq_idx < enic->rq_count * 2; rq_idx++) { 1555 rq = &enic->rq[rq_idx]; 1556 if (rq->is_sop && rq->in_use) { 1557 rc = enic_stop_rq(enic, 1558 enic_sop_rq_idx_to_rte_idx(rq_idx)); 1559 if (rc) { 1560 dev_err(enic, "Failed to stop Rq %u\n", rq_idx); 1561 goto set_mtu_done; 1562 } 1563 } 1564 } 1565 1566 /* replace Rx function with a no-op to avoid getting stale pkts */ 1567 eth_dev->rx_pkt_burst = enic_dummy_recv_pkts; 1568 rte_mb(); 1569 1570 /* Allow time for threads to exit the real Rx function. */ 1571 usleep(100000); 1572 1573 /* now it is safe to reconfigure the RQs */ 1574 1575 1576 /* free and reallocate RQs with the new MTU */ 1577 for (rq_idx = 0; rq_idx < enic->rq_count; rq_idx++) { 1578 rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)]; 1579 if (!rq->in_use) 1580 continue; 1581 1582 enic_free_rq(rq); 1583 rc = enic_alloc_rq(enic, rq_idx, rq->socket_id, rq->mp, 1584 rq->tot_nb_desc, rq->rx_free_thresh); 1585 if (rc) { 1586 dev_err(enic, 1587 "Fatal MTU alloc error- No traffic will pass\n"); 1588 goto set_mtu_done; 1589 } 1590 1591 rc = enic_reinit_rq(enic, rq_idx); 1592 if (rc) { 1593 dev_err(enic, 1594 "Fatal MTU RQ reinit- No traffic will pass\n"); 1595 goto set_mtu_done; 1596 } 1597 } 1598 1599 /* put back the real receive function */ 1600 rte_mb(); 1601 pick_rx_handler(enic); 1602 rte_mb(); 1603 1604 /* restart Rx traffic */ 1605 for (rq_idx = 0; rq_idx < enic->rq_count; rq_idx++) { 1606 rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)]; 1607 if (rq->is_sop && rq->in_use) 1608 enic_start_rq(enic, rq_idx); 1609 } 1610 1611 set_mtu_done: 1612 dev_info(enic, "MTU changed from %u to %u\n", old_mtu, new_mtu); 1613 rte_spinlock_unlock(&enic->mtu_lock); 1614 return rc; 1615 } 1616 1617 static int enic_dev_init(struct enic *enic) 1618 { 1619 int err; 1620 struct rte_eth_dev *eth_dev = enic->rte_dev; 1621 1622 vnic_dev_intr_coal_timer_info_default(enic->vdev); 1623 1624 /* Get vNIC configuration 1625 */ 1626 err = enic_get_vnic_config(enic); 1627 if (err) { 1628 dev_err(dev, "Get vNIC configuration failed, aborting\n"); 1629 return err; 1630 } 1631 1632 /* Get available resource counts */ 1633 enic_get_res_counts(enic); 1634 if (enic->conf_rq_count == 1) { 1635 dev_err(enic, "Running with only 1 RQ configured in the vNIC is not supported.\n"); 1636 dev_err(enic, "Please configure 2 RQs in the vNIC for each Rx queue used by DPDK.\n"); 1637 dev_err(enic, "See the ENIC PMD guide for more information.\n"); 1638 return -EINVAL; 1639 } 1640 /* Queue counts may be zeros. rte_zmalloc returns NULL in that case. */ 1641 enic->cq = rte_zmalloc("enic_vnic_cq", sizeof(struct vnic_cq) * 1642 enic->conf_cq_count, 8); 1643 enic->intr = rte_zmalloc("enic_vnic_intr", sizeof(struct vnic_intr) * 1644 enic->conf_intr_count, 8); 1645 enic->rq = rte_zmalloc("enic_vnic_rq", sizeof(struct vnic_rq) * 1646 enic->conf_rq_count, 8); 1647 enic->wq = rte_zmalloc("enic_vnic_wq", sizeof(struct vnic_wq) * 1648 enic->conf_wq_count, 8); 1649 if (enic->conf_cq_count > 0 && enic->cq == NULL) { 1650 dev_err(enic, "failed to allocate vnic_cq, aborting.\n"); 1651 return -1; 1652 } 1653 if (enic->conf_intr_count > 0 && enic->intr == NULL) { 1654 dev_err(enic, "failed to allocate vnic_intr, aborting.\n"); 1655 return -1; 1656 } 1657 if (enic->conf_rq_count > 0 && enic->rq == NULL) { 1658 dev_err(enic, "failed to allocate vnic_rq, aborting.\n"); 1659 return -1; 1660 } 1661 if (enic->conf_wq_count > 0 && enic->wq == NULL) { 1662 dev_err(enic, "failed to allocate vnic_wq, aborting.\n"); 1663 return -1; 1664 } 1665 1666 /* Get the supported filters */ 1667 enic_fdir_info(enic); 1668 1669 eth_dev->data->mac_addrs = rte_zmalloc("enic_mac_addr", 1670 sizeof(struct rte_ether_addr) * 1671 ENIC_UNICAST_PERFECT_FILTERS, 0); 1672 if (!eth_dev->data->mac_addrs) { 1673 dev_err(enic, "mac addr storage alloc failed, aborting.\n"); 1674 return -1; 1675 } 1676 rte_ether_addr_copy((struct rte_ether_addr *)enic->mac_addr, 1677 eth_dev->data->mac_addrs); 1678 1679 vnic_dev_set_reset_flag(enic->vdev, 0); 1680 1681 LIST_INIT(&enic->flows); 1682 rte_spinlock_init(&enic->flows_lock); 1683 enic->max_flow_counter = -1; 1684 1685 /* set up link status checking */ 1686 vnic_dev_notify_set(enic->vdev, -1); /* No Intr for notify */ 1687 1688 enic->overlay_offload = false; 1689 if (enic->disable_overlay && enic->vxlan) { 1690 /* 1691 * Explicitly disable overlay offload as the setting is 1692 * sticky, and resetting vNIC does not disable it. 1693 */ 1694 if (vnic_dev_overlay_offload_ctrl(enic->vdev, 1695 OVERLAY_FEATURE_VXLAN, 1696 OVERLAY_OFFLOAD_DISABLE)) { 1697 dev_err(enic, "failed to disable overlay offload\n"); 1698 } else { 1699 dev_info(enic, "Overlay offload is disabled\n"); 1700 } 1701 } 1702 if (!enic->disable_overlay && enic->vxlan && 1703 /* 'VXLAN feature' enables VXLAN, NVGRE, and GENEVE. */ 1704 vnic_dev_overlay_offload_ctrl(enic->vdev, 1705 OVERLAY_FEATURE_VXLAN, 1706 OVERLAY_OFFLOAD_ENABLE) == 0) { 1707 enic->tx_offload_capa |= 1708 DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM | 1709 DEV_TX_OFFLOAD_GENEVE_TNL_TSO | 1710 DEV_TX_OFFLOAD_VXLAN_TNL_TSO; 1711 enic->tx_offload_mask |= 1712 PKT_TX_OUTER_IPV6 | 1713 PKT_TX_OUTER_IPV4 | 1714 PKT_TX_OUTER_IP_CKSUM | 1715 PKT_TX_TUNNEL_MASK; 1716 enic->overlay_offload = true; 1717 dev_info(enic, "Overlay offload is enabled\n"); 1718 } 1719 /* 1720 * Reset the vxlan port if HW vxlan parsing is available. It 1721 * is always enabled regardless of overlay offload 1722 * enable/disable. 1723 */ 1724 if (enic->vxlan) { 1725 enic->vxlan_port = ENIC_DEFAULT_VXLAN_PORT; 1726 /* 1727 * Reset the vxlan port to the default, as the NIC firmware 1728 * does not reset it automatically and keeps the old setting. 1729 */ 1730 if (vnic_dev_overlay_offload_cfg(enic->vdev, 1731 OVERLAY_CFG_VXLAN_PORT_UPDATE, 1732 ENIC_DEFAULT_VXLAN_PORT)) { 1733 dev_err(enic, "failed to update vxlan port\n"); 1734 return -EINVAL; 1735 } 1736 } 1737 1738 return 0; 1739 1740 } 1741 1742 int enic_probe(struct enic *enic) 1743 { 1744 struct rte_pci_device *pdev = enic->pdev; 1745 int err = -1; 1746 1747 dev_debug(enic, " Initializing ENIC PMD\n"); 1748 1749 /* if this is a secondary process the hardware is already initialized */ 1750 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 1751 return 0; 1752 1753 enic->bar0.vaddr = (void *)pdev->mem_resource[0].addr; 1754 enic->bar0.len = pdev->mem_resource[0].len; 1755 1756 /* Register vNIC device */ 1757 enic->vdev = vnic_dev_register(NULL, enic, enic->pdev, &enic->bar0, 1); 1758 if (!enic->vdev) { 1759 dev_err(enic, "vNIC registration failed, aborting\n"); 1760 goto err_out; 1761 } 1762 1763 LIST_INIT(&enic->memzone_list); 1764 rte_spinlock_init(&enic->memzone_list_lock); 1765 1766 vnic_register_cbacks(enic->vdev, 1767 enic_alloc_consistent, 1768 enic_free_consistent); 1769 1770 /* 1771 * Allocate the consistent memory for stats and counters upfront so 1772 * both primary and secondary processes can access them. 1773 */ 1774 err = vnic_dev_alloc_stats_mem(enic->vdev); 1775 if (err) { 1776 dev_err(enic, "Failed to allocate cmd memory, aborting\n"); 1777 goto err_out_unregister; 1778 } 1779 err = vnic_dev_alloc_counter_mem(enic->vdev); 1780 if (err) { 1781 dev_err(enic, "Failed to allocate counter memory, aborting\n"); 1782 goto err_out_unregister; 1783 } 1784 1785 /* Issue device open to get device in known state */ 1786 err = enic_dev_open(enic); 1787 if (err) { 1788 dev_err(enic, "vNIC dev open failed, aborting\n"); 1789 goto err_out_unregister; 1790 } 1791 1792 /* Set ingress vlan rewrite mode before vnic initialization */ 1793 dev_debug(enic, "Set ig_vlan_rewrite_mode=%u\n", 1794 enic->ig_vlan_rewrite_mode); 1795 err = vnic_dev_set_ig_vlan_rewrite_mode(enic->vdev, 1796 enic->ig_vlan_rewrite_mode); 1797 if (err) { 1798 dev_err(enic, 1799 "Failed to set ingress vlan rewrite mode, aborting.\n"); 1800 goto err_out_dev_close; 1801 } 1802 1803 /* Issue device init to initialize the vnic-to-switch link. 1804 * We'll start with carrier off and wait for link UP 1805 * notification later to turn on carrier. We don't need 1806 * to wait here for the vnic-to-switch link initialization 1807 * to complete; link UP notification is the indication that 1808 * the process is complete. 1809 */ 1810 1811 err = vnic_dev_init(enic->vdev, 0); 1812 if (err) { 1813 dev_err(enic, "vNIC dev init failed, aborting\n"); 1814 goto err_out_dev_close; 1815 } 1816 1817 err = enic_dev_init(enic); 1818 if (err) { 1819 dev_err(enic, "Device initialization failed, aborting\n"); 1820 goto err_out_dev_close; 1821 } 1822 1823 return 0; 1824 1825 err_out_dev_close: 1826 vnic_dev_close(enic->vdev); 1827 err_out_unregister: 1828 vnic_dev_unregister(enic->vdev); 1829 err_out: 1830 return err; 1831 } 1832 1833 void enic_remove(struct enic *enic) 1834 { 1835 enic_dev_deinit(enic); 1836 vnic_dev_close(enic->vdev); 1837 vnic_dev_unregister(enic->vdev); 1838 } 1839