1 /* 2 * Copyright 2008-2014 Cisco Systems, Inc. All rights reserved. 3 * Copyright 2007 Nuova Systems, Inc. All rights reserved. 4 * 5 * Copyright (c) 2014, Cisco Systems, Inc. 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in 17 * the documentation and/or other materials provided with the 18 * distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 23 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 24 * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 25 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 26 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 27 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 28 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 29 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 30 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 31 * POSSIBILITY OF SUCH DAMAGE. 32 * 33 */ 34 35 #include <stdio.h> 36 37 #include <sys/stat.h> 38 #include <sys/mman.h> 39 #include <fcntl.h> 40 #include <libgen.h> 41 42 #include <rte_pci.h> 43 #include <rte_memzone.h> 44 #include <rte_malloc.h> 45 #include <rte_mbuf.h> 46 #include <rte_string_fns.h> 47 #include <rte_ethdev.h> 48 49 #include "enic_compat.h" 50 #include "enic.h" 51 #include "wq_enet_desc.h" 52 #include "rq_enet_desc.h" 53 #include "cq_enet_desc.h" 54 #include "vnic_enet.h" 55 #include "vnic_dev.h" 56 #include "vnic_wq.h" 57 #include "vnic_rq.h" 58 #include "vnic_cq.h" 59 #include "vnic_intr.h" 60 #include "vnic_nic.h" 61 #include "enic_vnic_wq.h" 62 63 static inline struct rte_mbuf * 64 rte_rxmbuf_alloc(struct rte_mempool *mp) 65 { 66 struct rte_mbuf *m; 67 68 m = __rte_mbuf_raw_alloc(mp); 69 __rte_mbuf_sanity_check_raw(m, 0); 70 return m; 71 } 72 73 74 static inline int enic_is_sriov_vf(struct enic *enic) 75 { 76 return enic->pdev->id.device_id == PCI_DEVICE_ID_CISCO_VIC_ENET_VF; 77 } 78 79 static int is_zero_addr(uint8_t *addr) 80 { 81 return !(addr[0] | addr[1] | addr[2] | addr[3] | addr[4] | addr[5]); 82 } 83 84 static int is_mcast_addr(uint8_t *addr) 85 { 86 return addr[0] & 1; 87 } 88 89 static int is_eth_addr_valid(uint8_t *addr) 90 { 91 return !is_mcast_addr(addr) && !is_zero_addr(addr); 92 } 93 94 static void 95 enic_rxmbuf_queue_release(struct enic *enic, struct vnic_rq *rq) 96 { 97 uint16_t i; 98 99 if (!rq || !rq->mbuf_ring) { 100 dev_debug(enic, "Pointer to rq or mbuf_ring is NULL"); 101 return; 102 } 103 104 for (i = 0; i < enic->config.rq_desc_count; i++) { 105 if (rq->mbuf_ring[i]) { 106 rte_pktmbuf_free_seg(rq->mbuf_ring[i]); 107 rq->mbuf_ring[i] = NULL; 108 } 109 } 110 } 111 112 113 void enic_set_hdr_split_size(struct enic *enic, u16 split_hdr_size) 114 { 115 vnic_set_hdr_split_size(enic->vdev, split_hdr_size); 116 } 117 118 static void enic_free_wq_buf(__rte_unused struct vnic_wq *wq, struct vnic_wq_buf *buf) 119 { 120 struct rte_mbuf *mbuf = (struct rte_mbuf *)buf->os_buf; 121 122 rte_mempool_put(mbuf->pool, mbuf); 123 buf->os_buf = NULL; 124 } 125 126 static void enic_wq_free_buf(struct vnic_wq *wq, 127 __rte_unused struct cq_desc *cq_desc, 128 struct vnic_wq_buf *buf, 129 __rte_unused void *opaque) 130 { 131 enic_free_wq_buf(wq, buf); 132 } 133 134 static int enic_wq_service(struct vnic_dev *vdev, struct cq_desc *cq_desc, 135 __rte_unused u8 type, u16 q_number, u16 completed_index, void *opaque) 136 { 137 struct enic *enic = vnic_dev_priv(vdev); 138 139 vnic_wq_service(&enic->wq[q_number], cq_desc, 140 completed_index, enic_wq_free_buf, 141 opaque); 142 143 return 0; 144 } 145 146 static void enic_log_q_error(struct enic *enic) 147 { 148 unsigned int i; 149 u32 error_status; 150 151 for (i = 0; i < enic->wq_count; i++) { 152 error_status = vnic_wq_error_status(&enic->wq[i]); 153 if (error_status) 154 dev_err(enic, "WQ[%d] error_status %d\n", i, 155 error_status); 156 } 157 158 for (i = 0; i < enic->rq_count; i++) { 159 error_status = vnic_rq_error_status(&enic->rq[i]); 160 if (error_status) 161 dev_err(enic, "RQ[%d] error_status %d\n", i, 162 error_status); 163 } 164 } 165 166 unsigned int enic_cleanup_wq(struct enic *enic, struct vnic_wq *wq) 167 { 168 unsigned int cq = enic_cq_wq(enic, wq->index); 169 170 /* Return the work done */ 171 return vnic_cq_service(&enic->cq[cq], 172 -1 /*wq_work_to_do*/, enic_wq_service, NULL); 173 } 174 175 void enic_post_wq_index(struct vnic_wq *wq) 176 { 177 enic_vnic_post_wq_index(wq); 178 } 179 180 void enic_send_pkt(struct enic *enic, struct vnic_wq *wq, 181 struct rte_mbuf *tx_pkt, unsigned short len, 182 uint8_t sop, uint8_t eop, uint8_t cq_entry, 183 uint16_t ol_flags, uint16_t vlan_tag) 184 { 185 struct wq_enet_desc *desc = vnic_wq_next_desc(wq); 186 uint16_t mss = 0; 187 uint8_t vlan_tag_insert = 0; 188 uint64_t bus_addr = (dma_addr_t) 189 (tx_pkt->buf_physaddr + tx_pkt->data_off); 190 191 if (sop) { 192 if (ol_flags & PKT_TX_VLAN_PKT) 193 vlan_tag_insert = 1; 194 195 if (enic->hw_ip_checksum) { 196 if (ol_flags & PKT_TX_IP_CKSUM) 197 mss |= ENIC_CALC_IP_CKSUM; 198 199 if (ol_flags & PKT_TX_TCP_UDP_CKSUM) 200 mss |= ENIC_CALC_TCP_UDP_CKSUM; 201 } 202 } 203 204 wq_enet_desc_enc(desc, 205 bus_addr, 206 len, 207 mss, 208 0 /* header_length */, 209 0 /* offload_mode WQ_ENET_OFFLOAD_MODE_CSUM */, 210 eop, 211 cq_entry, 212 0 /* fcoe_encap */, 213 vlan_tag_insert, 214 vlan_tag, 215 0 /* loopback */); 216 217 enic_vnic_post_wq(wq, (void *)tx_pkt, bus_addr, len, 218 sop, 219 1 /*desc_skip_cnt*/, 220 cq_entry, 221 0 /*compressed send*/, 222 0 /*wrid*/); 223 } 224 225 void enic_dev_stats_clear(struct enic *enic) 226 { 227 if (vnic_dev_stats_clear(enic->vdev)) 228 dev_err(enic, "Error in clearing stats\n"); 229 } 230 231 void enic_dev_stats_get(struct enic *enic, struct rte_eth_stats *r_stats) 232 { 233 struct vnic_stats *stats; 234 235 if (vnic_dev_stats_dump(enic->vdev, &stats)) { 236 dev_err(enic, "Error in getting stats\n"); 237 return; 238 } 239 240 r_stats->ipackets = stats->rx.rx_frames_ok; 241 r_stats->opackets = stats->tx.tx_frames_ok; 242 243 r_stats->ibytes = stats->rx.rx_bytes_ok; 244 r_stats->obytes = stats->tx.tx_bytes_ok; 245 246 r_stats->ierrors = stats->rx.rx_errors; 247 r_stats->oerrors = stats->tx.tx_errors; 248 249 r_stats->imissed = stats->rx.rx_drop; 250 251 r_stats->imcasts = stats->rx.rx_multicast_frames_ok; 252 r_stats->rx_nombuf = stats->rx.rx_no_bufs; 253 } 254 255 void enic_del_mac_address(struct enic *enic) 256 { 257 if (vnic_dev_del_addr(enic->vdev, enic->mac_addr)) 258 dev_err(enic, "del mac addr failed\n"); 259 } 260 261 void enic_set_mac_address(struct enic *enic, uint8_t *mac_addr) 262 { 263 int err; 264 265 if (!is_eth_addr_valid(mac_addr)) { 266 dev_err(enic, "invalid mac address\n"); 267 return; 268 } 269 270 err = vnic_dev_del_addr(enic->vdev, mac_addr); 271 if (err) { 272 dev_err(enic, "del mac addr failed\n"); 273 return; 274 } 275 276 ether_addr_copy((struct ether_addr *)mac_addr, 277 (struct ether_addr *)enic->mac_addr); 278 279 err = vnic_dev_add_addr(enic->vdev, mac_addr); 280 if (err) { 281 dev_err(enic, "add mac addr failed\n"); 282 return; 283 } 284 } 285 286 static void 287 enic_free_rq_buf(struct rte_mbuf **mbuf) 288 { 289 if (*mbuf == NULL) 290 return; 291 292 rte_pktmbuf_free(*mbuf); 293 mbuf = NULL; 294 } 295 296 void enic_init_vnic_resources(struct enic *enic) 297 { 298 unsigned int error_interrupt_enable = 1; 299 unsigned int error_interrupt_offset = 0; 300 unsigned int index = 0; 301 302 for (index = 0; index < enic->rq_count; index++) { 303 vnic_rq_init(&enic->rq[index], 304 enic_cq_rq(enic, index), 305 error_interrupt_enable, 306 error_interrupt_offset); 307 } 308 309 for (index = 0; index < enic->wq_count; index++) { 310 vnic_wq_init(&enic->wq[index], 311 enic_cq_wq(enic, index), 312 error_interrupt_enable, 313 error_interrupt_offset); 314 } 315 316 vnic_dev_stats_clear(enic->vdev); 317 318 for (index = 0; index < enic->cq_count; index++) { 319 vnic_cq_init(&enic->cq[index], 320 0 /* flow_control_enable */, 321 1 /* color_enable */, 322 0 /* cq_head */, 323 0 /* cq_tail */, 324 1 /* cq_tail_color */, 325 0 /* interrupt_enable */, 326 1 /* cq_entry_enable */, 327 0 /* cq_message_enable */, 328 0 /* interrupt offset */, 329 0 /* cq_message_addr */); 330 } 331 332 vnic_intr_init(&enic->intr, 333 enic->config.intr_timer_usec, 334 enic->config.intr_timer_type, 335 /*mask_on_assertion*/1); 336 } 337 338 339 static int 340 enic_alloc_rx_queue_mbufs(struct enic *enic, struct vnic_rq *rq) 341 { 342 struct rte_mbuf *mb; 343 struct rq_enet_desc *rqd = rq->ring.descs; 344 unsigned i; 345 dma_addr_t dma_addr; 346 347 dev_debug(enic, "queue %u, allocating %u rx queue mbufs\n", rq->index, 348 rq->ring.desc_count); 349 350 for (i = 0; i < rq->ring.desc_count; i++, rqd++) { 351 mb = rte_rxmbuf_alloc(rq->mp); 352 if (mb == NULL) { 353 dev_err(enic, "RX mbuf alloc failed queue_id=%u\n", 354 (unsigned)rq->index); 355 return -ENOMEM; 356 } 357 358 dma_addr = (dma_addr_t)(mb->buf_physaddr + mb->data_off); 359 360 rq_enet_desc_enc(rqd, dma_addr, RQ_ENET_TYPE_ONLY_SOP, 361 mb->buf_len); 362 rq->mbuf_ring[i] = mb; 363 } 364 365 /* make sure all prior writes are complete before doing the PIO write */ 366 rte_rmb(); 367 368 /* Post all but the last 2 cache lines' worth of descriptors */ 369 rq->posted_index = rq->ring.desc_count - (2 * RTE_CACHE_LINE_SIZE 370 / sizeof(struct rq_enet_desc)); 371 rq->rx_nb_hold = 0; 372 373 dev_debug(enic, "port=%u, qidx=%u, Write %u posted idx, %u sw held\n", 374 enic->port_id, rq->index, rq->posted_index, rq->rx_nb_hold); 375 iowrite32(rq->posted_index, &rq->ctrl->posted_index); 376 rte_rmb(); 377 378 return 0; 379 380 } 381 382 static void * 383 enic_alloc_consistent(__rte_unused void *priv, size_t size, 384 dma_addr_t *dma_handle, u8 *name) 385 { 386 void *vaddr; 387 const struct rte_memzone *rz; 388 *dma_handle = 0; 389 390 rz = rte_memzone_reserve_aligned((const char *)name, 391 size, SOCKET_ID_ANY, 0, ENIC_ALIGN); 392 if (!rz) { 393 pr_err("%s : Failed to allocate memory requested for %s\n", 394 __func__, name); 395 return NULL; 396 } 397 398 vaddr = rz->addr; 399 *dma_handle = (dma_addr_t)rz->phys_addr; 400 401 return vaddr; 402 } 403 404 static void 405 enic_free_consistent(__rte_unused struct rte_pci_device *hwdev, 406 __rte_unused size_t size, 407 __rte_unused void *vaddr, 408 __rte_unused dma_addr_t dma_handle) 409 { 410 /* Nothing to be done */ 411 } 412 413 static void 414 enic_intr_handler(__rte_unused struct rte_intr_handle *handle, 415 void *arg) 416 { 417 struct enic *enic = pmd_priv((struct rte_eth_dev *)arg); 418 419 vnic_intr_return_all_credits(&enic->intr); 420 421 enic_log_q_error(enic); 422 } 423 424 int enic_enable(struct enic *enic) 425 { 426 unsigned int index; 427 int err; 428 struct rte_eth_dev *eth_dev = enic->rte_dev; 429 430 eth_dev->data->dev_link.link_speed = vnic_dev_port_speed(enic->vdev); 431 eth_dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX; 432 vnic_dev_notify_set(enic->vdev, -1); /* No Intr for notify */ 433 434 if (enic_clsf_init(enic)) 435 dev_warning(enic, "Init of hash table for clsf failed."\ 436 "Flow director feature will not work\n"); 437 438 for (index = 0; index < enic->rq_count; index++) { 439 err = enic_alloc_rx_queue_mbufs(enic, &enic->rq[index]); 440 if (err) { 441 dev_err(enic, "Failed to alloc RX queue mbufs\n"); 442 return err; 443 } 444 } 445 446 for (index = 0; index < enic->wq_count; index++) 447 vnic_wq_enable(&enic->wq[index]); 448 for (index = 0; index < enic->rq_count; index++) 449 vnic_rq_enable(&enic->rq[index]); 450 451 vnic_dev_enable_wait(enic->vdev); 452 453 /* Register and enable error interrupt */ 454 rte_intr_callback_register(&(enic->pdev->intr_handle), 455 enic_intr_handler, (void *)enic->rte_dev); 456 457 rte_intr_enable(&(enic->pdev->intr_handle)); 458 vnic_intr_unmask(&enic->intr); 459 460 return 0; 461 } 462 463 int enic_alloc_intr_resources(struct enic *enic) 464 { 465 int err; 466 467 dev_info(enic, "vNIC resources used: "\ 468 "wq %d rq %d cq %d intr %d\n", 469 enic->wq_count, enic->rq_count, 470 enic->cq_count, enic->intr_count); 471 472 err = vnic_intr_alloc(enic->vdev, &enic->intr, 0); 473 if (err) 474 enic_free_vnic_resources(enic); 475 476 return err; 477 } 478 479 void enic_free_rq(void *rxq) 480 { 481 struct vnic_rq *rq = (struct vnic_rq *)rxq; 482 struct enic *enic = vnic_dev_priv(rq->vdev); 483 484 enic_rxmbuf_queue_release(enic, rq); 485 rte_free(rq->mbuf_ring); 486 rq->mbuf_ring = NULL; 487 vnic_rq_free(rq); 488 vnic_cq_free(&enic->cq[rq->index]); 489 } 490 491 void enic_start_wq(struct enic *enic, uint16_t queue_idx) 492 { 493 vnic_wq_enable(&enic->wq[queue_idx]); 494 } 495 496 int enic_stop_wq(struct enic *enic, uint16_t queue_idx) 497 { 498 return vnic_wq_disable(&enic->wq[queue_idx]); 499 } 500 501 void enic_start_rq(struct enic *enic, uint16_t queue_idx) 502 { 503 vnic_rq_enable(&enic->rq[queue_idx]); 504 } 505 506 int enic_stop_rq(struct enic *enic, uint16_t queue_idx) 507 { 508 return vnic_rq_disable(&enic->rq[queue_idx]); 509 } 510 511 int enic_alloc_rq(struct enic *enic, uint16_t queue_idx, 512 unsigned int socket_id, struct rte_mempool *mp, 513 uint16_t nb_desc) 514 { 515 int rc; 516 struct vnic_rq *rq = &enic->rq[queue_idx]; 517 518 rq->socket_id = socket_id; 519 rq->mp = mp; 520 521 if (nb_desc) { 522 if (nb_desc > enic->config.rq_desc_count) { 523 dev_warning(enic, 524 "RQ %d - number of rx desc in cmd line (%d)"\ 525 "is greater than that in the UCSM/CIMC adapter"\ 526 "policy. Applying the value in the adapter "\ 527 "policy (%d).\n", 528 queue_idx, nb_desc, enic->config.rq_desc_count); 529 nb_desc = enic->config.rq_desc_count; 530 } 531 dev_info(enic, "RX Queues - effective number of descs:%d\n", 532 nb_desc); 533 } 534 535 /* Allocate queue resources */ 536 rc = vnic_rq_alloc(enic->vdev, rq, queue_idx, 537 nb_desc, sizeof(struct rq_enet_desc)); 538 if (rc) { 539 dev_err(enic, "error in allocation of rq\n"); 540 goto err_exit; 541 } 542 543 rc = vnic_cq_alloc(enic->vdev, &enic->cq[queue_idx], queue_idx, 544 socket_id, nb_desc, 545 sizeof(struct cq_enet_rq_desc)); 546 if (rc) { 547 dev_err(enic, "error in allocation of cq for rq\n"); 548 goto err_free_rq_exit; 549 } 550 551 /* Allocate the mbuf ring */ 552 rq->mbuf_ring = (struct rte_mbuf **)rte_zmalloc_socket("rq->mbuf_ring", 553 sizeof(struct rte_mbuf *) * nb_desc, 554 RTE_CACHE_LINE_SIZE, rq->socket_id); 555 556 if (rq->mbuf_ring != NULL) 557 return 0; 558 559 /* cleanup on error */ 560 vnic_cq_free(&enic->cq[queue_idx]); 561 err_free_rq_exit: 562 vnic_rq_free(rq); 563 err_exit: 564 return -ENOMEM; 565 } 566 567 void enic_free_wq(void *txq) 568 { 569 struct vnic_wq *wq = (struct vnic_wq *)txq; 570 struct enic *enic = vnic_dev_priv(wq->vdev); 571 572 vnic_wq_free(wq); 573 vnic_cq_free(&enic->cq[enic->rq_count + wq->index]); 574 } 575 576 int enic_alloc_wq(struct enic *enic, uint16_t queue_idx, 577 unsigned int socket_id, uint16_t nb_desc) 578 { 579 int err; 580 struct vnic_wq *wq = &enic->wq[queue_idx]; 581 unsigned int cq_index = enic_cq_wq(enic, queue_idx); 582 583 wq->socket_id = socket_id; 584 if (nb_desc) { 585 if (nb_desc > enic->config.wq_desc_count) { 586 dev_warning(enic, 587 "WQ %d - number of tx desc in cmd line (%d)"\ 588 "is greater than that in the UCSM/CIMC adapter"\ 589 "policy. Applying the value in the adapter "\ 590 "policy (%d)\n", 591 queue_idx, nb_desc, enic->config.wq_desc_count); 592 } else if (nb_desc != enic->config.wq_desc_count) { 593 enic->config.wq_desc_count = nb_desc; 594 dev_info(enic, 595 "TX Queues - effective number of descs:%d\n", 596 nb_desc); 597 } 598 } 599 600 /* Allocate queue resources */ 601 err = vnic_wq_alloc(enic->vdev, &enic->wq[queue_idx], queue_idx, 602 enic->config.wq_desc_count, 603 sizeof(struct wq_enet_desc)); 604 if (err) { 605 dev_err(enic, "error in allocation of wq\n"); 606 return err; 607 } 608 609 err = vnic_cq_alloc(enic->vdev, &enic->cq[cq_index], cq_index, 610 socket_id, enic->config.wq_desc_count, 611 sizeof(struct cq_enet_wq_desc)); 612 if (err) { 613 vnic_wq_free(wq); 614 dev_err(enic, "error in allocation of cq for wq\n"); 615 } 616 617 return err; 618 } 619 620 int enic_disable(struct enic *enic) 621 { 622 unsigned int i; 623 int err; 624 625 vnic_intr_mask(&enic->intr); 626 (void)vnic_intr_masked(&enic->intr); /* flush write */ 627 628 vnic_dev_disable(enic->vdev); 629 630 enic_clsf_destroy(enic); 631 632 if (!enic_is_sriov_vf(enic)) 633 vnic_dev_del_addr(enic->vdev, enic->mac_addr); 634 635 for (i = 0; i < enic->wq_count; i++) { 636 err = vnic_wq_disable(&enic->wq[i]); 637 if (err) 638 return err; 639 } 640 for (i = 0; i < enic->rq_count; i++) { 641 err = vnic_rq_disable(&enic->rq[i]); 642 if (err) 643 return err; 644 } 645 646 vnic_dev_set_reset_flag(enic->vdev, 1); 647 vnic_dev_notify_unset(enic->vdev); 648 649 for (i = 0; i < enic->wq_count; i++) 650 vnic_wq_clean(&enic->wq[i], enic_free_wq_buf); 651 652 for (i = 0; i < enic->rq_count; i++) 653 vnic_rq_clean(&enic->rq[i], enic_free_rq_buf); 654 for (i = 0; i < enic->cq_count; i++) 655 vnic_cq_clean(&enic->cq[i]); 656 vnic_intr_clean(&enic->intr); 657 658 return 0; 659 } 660 661 static int enic_dev_wait(struct vnic_dev *vdev, 662 int (*start)(struct vnic_dev *, int), 663 int (*finished)(struct vnic_dev *, int *), 664 int arg) 665 { 666 int done; 667 int err; 668 int i; 669 670 err = start(vdev, arg); 671 if (err) 672 return err; 673 674 /* Wait for func to complete...2 seconds max */ 675 for (i = 0; i < 2000; i++) { 676 err = finished(vdev, &done); 677 if (err) 678 return err; 679 if (done) 680 return 0; 681 usleep(1000); 682 } 683 return -ETIMEDOUT; 684 } 685 686 static int enic_dev_open(struct enic *enic) 687 { 688 int err; 689 690 err = enic_dev_wait(enic->vdev, vnic_dev_open, 691 vnic_dev_open_done, 0); 692 if (err) 693 dev_err(enic_get_dev(enic), 694 "vNIC device open failed, err %d\n", err); 695 696 return err; 697 } 698 699 static int enic_set_rsskey(struct enic *enic) 700 { 701 dma_addr_t rss_key_buf_pa; 702 union vnic_rss_key *rss_key_buf_va = NULL; 703 static union vnic_rss_key rss_key = { 704 .key = { 705 [0] = {.b = {85, 67, 83, 97, 119, 101, 115, 111, 109, 101}}, 706 [1] = {.b = {80, 65, 76, 79, 117, 110, 105, 113, 117, 101}}, 707 [2] = {.b = {76, 73, 78, 85, 88, 114, 111, 99, 107, 115}}, 708 [3] = {.b = {69, 78, 73, 67, 105, 115, 99, 111, 111, 108}}, 709 } 710 }; 711 int err; 712 u8 name[NAME_MAX]; 713 714 snprintf((char *)name, NAME_MAX, "rss_key-%s", enic->bdf_name); 715 rss_key_buf_va = enic_alloc_consistent(enic, sizeof(union vnic_rss_key), 716 &rss_key_buf_pa, name); 717 if (!rss_key_buf_va) 718 return -ENOMEM; 719 720 rte_memcpy(rss_key_buf_va, &rss_key, sizeof(union vnic_rss_key)); 721 722 err = enic_set_rss_key(enic, 723 rss_key_buf_pa, 724 sizeof(union vnic_rss_key)); 725 726 enic_free_consistent(enic->pdev, sizeof(union vnic_rss_key), 727 rss_key_buf_va, rss_key_buf_pa); 728 729 return err; 730 } 731 732 static int enic_set_rsscpu(struct enic *enic, u8 rss_hash_bits) 733 { 734 dma_addr_t rss_cpu_buf_pa; 735 union vnic_rss_cpu *rss_cpu_buf_va = NULL; 736 int i; 737 int err; 738 u8 name[NAME_MAX]; 739 740 snprintf((char *)name, NAME_MAX, "rss_cpu-%s", enic->bdf_name); 741 rss_cpu_buf_va = enic_alloc_consistent(enic, sizeof(union vnic_rss_cpu), 742 &rss_cpu_buf_pa, name); 743 if (!rss_cpu_buf_va) 744 return -ENOMEM; 745 746 for (i = 0; i < (1 << rss_hash_bits); i++) 747 (*rss_cpu_buf_va).cpu[i/4].b[i%4] = i % enic->rq_count; 748 749 err = enic_set_rss_cpu(enic, 750 rss_cpu_buf_pa, 751 sizeof(union vnic_rss_cpu)); 752 753 enic_free_consistent(enic->pdev, sizeof(union vnic_rss_cpu), 754 rss_cpu_buf_va, rss_cpu_buf_pa); 755 756 return err; 757 } 758 759 static int enic_set_niccfg(struct enic *enic, u8 rss_default_cpu, 760 u8 rss_hash_type, u8 rss_hash_bits, u8 rss_base_cpu, u8 rss_enable) 761 { 762 const u8 tso_ipid_split_en = 0; 763 int err; 764 765 /* Enable VLAN tag stripping */ 766 767 err = enic_set_nic_cfg(enic, 768 rss_default_cpu, rss_hash_type, 769 rss_hash_bits, rss_base_cpu, 770 rss_enable, tso_ipid_split_en, 771 enic->ig_vlan_strip_en); 772 773 return err; 774 } 775 776 int enic_set_rss_nic_cfg(struct enic *enic) 777 { 778 const u8 rss_default_cpu = 0; 779 const u8 rss_hash_type = NIC_CFG_RSS_HASH_TYPE_IPV4 | 780 NIC_CFG_RSS_HASH_TYPE_TCP_IPV4 | 781 NIC_CFG_RSS_HASH_TYPE_IPV6 | 782 NIC_CFG_RSS_HASH_TYPE_TCP_IPV6; 783 const u8 rss_hash_bits = 7; 784 const u8 rss_base_cpu = 0; 785 u8 rss_enable = ENIC_SETTING(enic, RSS) && (enic->rq_count > 1); 786 787 if (rss_enable) { 788 if (!enic_set_rsskey(enic)) { 789 if (enic_set_rsscpu(enic, rss_hash_bits)) { 790 rss_enable = 0; 791 dev_warning(enic, "RSS disabled, "\ 792 "Failed to set RSS cpu indirection table."); 793 } 794 } else { 795 rss_enable = 0; 796 dev_warning(enic, 797 "RSS disabled, Failed to set RSS key.\n"); 798 } 799 } 800 801 return enic_set_niccfg(enic, rss_default_cpu, rss_hash_type, 802 rss_hash_bits, rss_base_cpu, rss_enable); 803 } 804 805 int enic_setup_finish(struct enic *enic) 806 { 807 int ret; 808 809 ret = enic_set_rss_nic_cfg(enic); 810 if (ret) { 811 dev_err(enic, "Failed to config nic, aborting.\n"); 812 return -1; 813 } 814 815 vnic_dev_add_addr(enic->vdev, enic->mac_addr); 816 817 /* Default conf */ 818 vnic_dev_packet_filter(enic->vdev, 819 1 /* directed */, 820 1 /* multicast */, 821 1 /* broadcast */, 822 0 /* promisc */, 823 1 /* allmulti */); 824 825 enic->promisc = 0; 826 enic->allmulti = 1; 827 828 return 0; 829 } 830 831 void enic_add_packet_filter(struct enic *enic) 832 { 833 /* Args -> directed, multicast, broadcast, promisc, allmulti */ 834 vnic_dev_packet_filter(enic->vdev, 1, 1, 1, 835 enic->promisc, enic->allmulti); 836 } 837 838 int enic_get_link_status(struct enic *enic) 839 { 840 return vnic_dev_link_status(enic->vdev); 841 } 842 843 static void enic_dev_deinit(struct enic *enic) 844 { 845 struct rte_eth_dev *eth_dev = enic->rte_dev; 846 847 rte_free(eth_dev->data->mac_addrs); 848 } 849 850 851 int enic_set_vnic_res(struct enic *enic) 852 { 853 struct rte_eth_dev *eth_dev = enic->rte_dev; 854 855 if ((enic->rq_count < eth_dev->data->nb_rx_queues) || 856 (enic->wq_count < eth_dev->data->nb_tx_queues)) { 857 dev_err(dev, "Not enough resources configured, aborting\n"); 858 return -1; 859 } 860 861 enic->rq_count = eth_dev->data->nb_rx_queues; 862 enic->wq_count = eth_dev->data->nb_tx_queues; 863 if (enic->cq_count < (enic->rq_count + enic->wq_count)) { 864 dev_err(dev, "Not enough resources configured, aborting\n"); 865 return -1; 866 } 867 868 enic->cq_count = enic->rq_count + enic->wq_count; 869 return 0; 870 } 871 872 static int enic_dev_init(struct enic *enic) 873 { 874 int err; 875 struct rte_eth_dev *eth_dev = enic->rte_dev; 876 877 vnic_dev_intr_coal_timer_info_default(enic->vdev); 878 879 /* Get vNIC configuration 880 */ 881 err = enic_get_vnic_config(enic); 882 if (err) { 883 dev_err(dev, "Get vNIC configuration failed, aborting\n"); 884 return err; 885 } 886 887 eth_dev->data->mac_addrs = rte_zmalloc("enic_mac_addr", ETH_ALEN, 0); 888 if (!eth_dev->data->mac_addrs) { 889 dev_err(enic, "mac addr storage alloc failed, aborting.\n"); 890 return -1; 891 } 892 ether_addr_copy((struct ether_addr *) enic->mac_addr, 893 ð_dev->data->mac_addrs[0]); 894 895 896 /* Get available resource counts 897 */ 898 enic_get_res_counts(enic); 899 900 vnic_dev_set_reset_flag(enic->vdev, 0); 901 902 return 0; 903 904 } 905 906 int enic_probe(struct enic *enic) 907 { 908 struct rte_pci_device *pdev = enic->pdev; 909 int err = -1; 910 911 dev_debug(enic, " Initializing ENIC PMD version %s\n", DRV_VERSION); 912 913 enic->bar0.vaddr = (void *)pdev->mem_resource[0].addr; 914 enic->bar0.len = pdev->mem_resource[0].len; 915 916 /* Register vNIC device */ 917 enic->vdev = vnic_dev_register(NULL, enic, enic->pdev, &enic->bar0, 1); 918 if (!enic->vdev) { 919 dev_err(enic, "vNIC registration failed, aborting\n"); 920 goto err_out; 921 } 922 923 vnic_register_cbacks(enic->vdev, 924 enic_alloc_consistent, 925 enic_free_consistent); 926 927 /* Issue device open to get device in known state */ 928 err = enic_dev_open(enic); 929 if (err) { 930 dev_err(enic, "vNIC dev open failed, aborting\n"); 931 goto err_out_unregister; 932 } 933 934 /* Set ingress vlan rewrite mode before vnic initialization */ 935 err = vnic_dev_set_ig_vlan_rewrite_mode(enic->vdev, 936 IG_VLAN_REWRITE_MODE_PASS_THRU); 937 if (err) { 938 dev_err(enic, 939 "Failed to set ingress vlan rewrite mode, aborting.\n"); 940 goto err_out_dev_close; 941 } 942 943 /* Issue device init to initialize the vnic-to-switch link. 944 * We'll start with carrier off and wait for link UP 945 * notification later to turn on carrier. We don't need 946 * to wait here for the vnic-to-switch link initialization 947 * to complete; link UP notification is the indication that 948 * the process is complete. 949 */ 950 951 err = vnic_dev_init(enic->vdev, 0); 952 if (err) { 953 dev_err(enic, "vNIC dev init failed, aborting\n"); 954 goto err_out_dev_close; 955 } 956 957 err = enic_dev_init(enic); 958 if (err) { 959 dev_err(enic, "Device initialization failed, aborting\n"); 960 goto err_out_dev_close; 961 } 962 963 return 0; 964 965 err_out_dev_close: 966 vnic_dev_close(enic->vdev); 967 err_out_unregister: 968 vnic_dev_unregister(enic->vdev); 969 err_out: 970 return err; 971 } 972 973 void enic_remove(struct enic *enic) 974 { 975 enic_dev_deinit(enic); 976 vnic_dev_close(enic->vdev); 977 vnic_dev_unregister(enic->vdev); 978 } 979