1 /* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0) 2 * Copyright(c) 2018-2019 Pensando Systems, Inc. All rights reserved. 3 */ 4 5 #include <rte_malloc.h> 6 #include <rte_ethdev_driver.h> 7 8 #include "ionic.h" 9 #include "ionic_logs.h" 10 #include "ionic_lif.h" 11 #include "ionic_ethdev.h" 12 #include "ionic_rx_filter.h" 13 #include "ionic_rxtx.h" 14 15 static int ionic_lif_addr_add(struct ionic_lif *lif, const uint8_t *addr); 16 static int ionic_lif_addr_del(struct ionic_lif *lif, const uint8_t *addr); 17 18 int 19 ionic_qcq_enable(struct ionic_qcq *qcq) 20 { 21 struct ionic_queue *q = &qcq->q; 22 struct ionic_lif *lif = q->lif; 23 struct ionic_dev *idev = &lif->adapter->idev; 24 struct ionic_admin_ctx ctx = { 25 .pending_work = true, 26 .cmd.q_control = { 27 .opcode = IONIC_CMD_Q_CONTROL, 28 .type = q->type, 29 .index = q->index, 30 .oper = IONIC_Q_ENABLE, 31 }, 32 }; 33 34 if (qcq->flags & IONIC_QCQ_F_INTR) { 35 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index, 36 IONIC_INTR_MASK_CLEAR); 37 } 38 39 return ionic_adminq_post_wait(lif, &ctx); 40 } 41 42 int 43 ionic_qcq_disable(struct ionic_qcq *qcq) 44 { 45 struct ionic_queue *q = &qcq->q; 46 struct ionic_lif *lif = q->lif; 47 struct ionic_dev *idev = &lif->adapter->idev; 48 struct ionic_admin_ctx ctx = { 49 .pending_work = true, 50 .cmd.q_control = { 51 .opcode = IONIC_CMD_Q_CONTROL, 52 .type = q->type, 53 .index = q->index, 54 .oper = IONIC_Q_DISABLE, 55 }, 56 }; 57 58 if (qcq->flags & IONIC_QCQ_F_INTR) { 59 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index, 60 IONIC_INTR_MASK_SET); 61 } 62 63 return ionic_adminq_post_wait(lif, &ctx); 64 } 65 66 void 67 ionic_lif_stop(struct ionic_lif *lif) 68 { 69 uint32_t i; 70 71 IONIC_PRINT_CALL(); 72 73 lif->state &= ~IONIC_LIF_F_UP; 74 75 for (i = 0; i < lif->nrxqcqs; i++) { 76 struct ionic_qcq *rxq = lif->rxqcqs[i]; 77 if (rxq->flags & IONIC_QCQ_F_INITED) 78 (void)ionic_dev_rx_queue_stop(lif->eth_dev, i); 79 } 80 81 for (i = 0; i < lif->ntxqcqs; i++) { 82 struct ionic_qcq *txq = lif->txqcqs[i]; 83 if (txq->flags & IONIC_QCQ_F_INITED) 84 (void)ionic_dev_tx_queue_stop(lif->eth_dev, i); 85 } 86 } 87 88 void 89 ionic_lif_reset(struct ionic_lif *lif) 90 { 91 struct ionic_dev *idev = &lif->adapter->idev; 92 int err; 93 94 IONIC_PRINT_CALL(); 95 96 ionic_dev_cmd_lif_reset(idev); 97 err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT); 98 if (err) 99 IONIC_PRINT(WARNING, "Failed to reset %s", lif->name); 100 } 101 102 static void 103 ionic_lif_get_abs_stats(const struct ionic_lif *lif, struct rte_eth_stats *stats) 104 { 105 struct ionic_lif_stats *ls = &lif->info->stats; 106 uint32_t i; 107 uint32_t num_rx_q_counters = RTE_MIN(lif->nrxqcqs, (uint32_t) 108 RTE_ETHDEV_QUEUE_STAT_CNTRS); 109 uint32_t num_tx_q_counters = RTE_MIN(lif->ntxqcqs, (uint32_t) 110 RTE_ETHDEV_QUEUE_STAT_CNTRS); 111 112 memset(stats, 0, sizeof(*stats)); 113 114 if (ls == NULL) { 115 IONIC_PRINT(DEBUG, "Stats on port %u not yet initialized", 116 lif->port_id); 117 return; 118 } 119 120 /* RX */ 121 122 stats->ipackets = ls->rx_ucast_packets + 123 ls->rx_mcast_packets + 124 ls->rx_bcast_packets; 125 126 stats->ibytes = ls->rx_ucast_bytes + 127 ls->rx_mcast_bytes + 128 ls->rx_bcast_bytes; 129 130 for (i = 0; i < lif->nrxqcqs; i++) { 131 struct ionic_rx_stats *rx_stats = &lif->rxqcqs[i]->stats.rx; 132 stats->imissed += 133 rx_stats->no_cb_arg + 134 rx_stats->bad_cq_status + 135 rx_stats->no_room + 136 rx_stats->bad_len; 137 } 138 139 stats->imissed += 140 ls->rx_ucast_drop_packets + 141 ls->rx_mcast_drop_packets + 142 ls->rx_bcast_drop_packets; 143 144 stats->imissed += 145 ls->rx_queue_empty + 146 ls->rx_dma_error + 147 ls->rx_queue_disabled + 148 ls->rx_desc_fetch_error + 149 ls->rx_desc_data_error; 150 151 for (i = 0; i < num_rx_q_counters; i++) { 152 struct ionic_rx_stats *rx_stats = &lif->rxqcqs[i]->stats.rx; 153 stats->q_ipackets[i] = rx_stats->packets; 154 stats->q_ibytes[i] = rx_stats->bytes; 155 stats->q_errors[i] = 156 rx_stats->no_cb_arg + 157 rx_stats->bad_cq_status + 158 rx_stats->no_room + 159 rx_stats->bad_len; 160 } 161 162 /* TX */ 163 164 stats->opackets = ls->tx_ucast_packets + 165 ls->tx_mcast_packets + 166 ls->tx_bcast_packets; 167 168 stats->obytes = ls->tx_ucast_bytes + 169 ls->tx_mcast_bytes + 170 ls->tx_bcast_bytes; 171 172 for (i = 0; i < lif->ntxqcqs; i++) { 173 struct ionic_tx_stats *tx_stats = &lif->txqcqs[i]->stats.tx; 174 stats->oerrors += tx_stats->drop; 175 } 176 177 stats->oerrors += 178 ls->tx_ucast_drop_packets + 179 ls->tx_mcast_drop_packets + 180 ls->tx_bcast_drop_packets; 181 182 stats->oerrors += 183 ls->tx_dma_error + 184 ls->tx_queue_disabled + 185 ls->tx_desc_fetch_error + 186 ls->tx_desc_data_error; 187 188 for (i = 0; i < num_tx_q_counters; i++) { 189 struct ionic_tx_stats *tx_stats = &lif->txqcqs[i]->stats.tx; 190 stats->q_opackets[i] = tx_stats->packets; 191 stats->q_obytes[i] = tx_stats->bytes; 192 } 193 } 194 195 void 196 ionic_lif_get_stats(const struct ionic_lif *lif, 197 struct rte_eth_stats *stats) 198 { 199 ionic_lif_get_abs_stats(lif, stats); 200 201 stats->ipackets -= lif->stats_base.ipackets; 202 stats->opackets -= lif->stats_base.opackets; 203 stats->ibytes -= lif->stats_base.ibytes; 204 stats->obytes -= lif->stats_base.obytes; 205 stats->imissed -= lif->stats_base.imissed; 206 stats->ierrors -= lif->stats_base.ierrors; 207 stats->oerrors -= lif->stats_base.oerrors; 208 stats->rx_nombuf -= lif->stats_base.rx_nombuf; 209 } 210 211 void 212 ionic_lif_reset_stats(struct ionic_lif *lif) 213 { 214 uint32_t i; 215 216 for (i = 0; i < lif->nrxqcqs; i++) { 217 memset(&lif->rxqcqs[i]->stats.rx, 0, 218 sizeof(struct ionic_rx_stats)); 219 memset(&lif->txqcqs[i]->stats.tx, 0, 220 sizeof(struct ionic_tx_stats)); 221 } 222 223 ionic_lif_get_abs_stats(lif, &lif->stats_base); 224 } 225 226 void 227 ionic_lif_get_hw_stats(struct ionic_lif *lif, struct ionic_lif_stats *stats) 228 { 229 uint16_t i, count = sizeof(struct ionic_lif_stats) / sizeof(uint64_t); 230 uint64_t *stats64 = (uint64_t *)stats; 231 uint64_t *lif_stats64 = (uint64_t *)&lif->info->stats; 232 uint64_t *lif_stats64_base = (uint64_t *)&lif->lif_stats_base; 233 234 for (i = 0; i < count; i++) 235 stats64[i] = lif_stats64[i] - lif_stats64_base[i]; 236 } 237 238 void 239 ionic_lif_reset_hw_stats(struct ionic_lif *lif) 240 { 241 uint16_t i, count = sizeof(struct ionic_lif_stats) / sizeof(uint64_t); 242 uint64_t *lif_stats64 = (uint64_t *)&lif->info->stats; 243 uint64_t *lif_stats64_base = (uint64_t *)&lif->lif_stats_base; 244 245 for (i = 0; i < count; i++) 246 lif_stats64_base[i] = lif_stats64[i]; 247 } 248 249 static int 250 ionic_lif_addr_add(struct ionic_lif *lif, const uint8_t *addr) 251 { 252 struct ionic_admin_ctx ctx = { 253 .pending_work = true, 254 .cmd.rx_filter_add = { 255 .opcode = IONIC_CMD_RX_FILTER_ADD, 256 .match = IONIC_RX_FILTER_MATCH_MAC, 257 }, 258 }; 259 int err; 260 261 memcpy(ctx.cmd.rx_filter_add.mac.addr, addr, RTE_ETHER_ADDR_LEN); 262 263 err = ionic_adminq_post_wait(lif, &ctx); 264 if (err) 265 return err; 266 267 IONIC_PRINT(INFO, "rx_filter add (id %d)", 268 ctx.comp.rx_filter_add.filter_id); 269 270 return ionic_rx_filter_save(lif, 0, IONIC_RXQ_INDEX_ANY, &ctx); 271 } 272 273 static int 274 ionic_lif_addr_del(struct ionic_lif *lif, const uint8_t *addr) 275 { 276 struct ionic_admin_ctx ctx = { 277 .pending_work = true, 278 .cmd.rx_filter_del = { 279 .opcode = IONIC_CMD_RX_FILTER_DEL, 280 }, 281 }; 282 struct ionic_rx_filter *f; 283 int err; 284 285 IONIC_PRINT_CALL(); 286 287 rte_spinlock_lock(&lif->rx_filters.lock); 288 289 f = ionic_rx_filter_by_addr(lif, addr); 290 if (!f) { 291 rte_spinlock_unlock(&lif->rx_filters.lock); 292 return -ENOENT; 293 } 294 295 ctx.cmd.rx_filter_del.filter_id = f->filter_id; 296 ionic_rx_filter_free(f); 297 298 rte_spinlock_unlock(&lif->rx_filters.lock); 299 300 err = ionic_adminq_post_wait(lif, &ctx); 301 if (err) 302 return err; 303 304 IONIC_PRINT(INFO, "rx_filter del (id %d)", 305 ctx.cmd.rx_filter_del.filter_id); 306 307 return 0; 308 } 309 310 int 311 ionic_dev_add_mac(struct rte_eth_dev *eth_dev, 312 struct rte_ether_addr *mac_addr, 313 uint32_t index __rte_unused, uint32_t pool __rte_unused) 314 { 315 struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev); 316 317 IONIC_PRINT_CALL(); 318 319 return ionic_lif_addr_add(lif, (const uint8_t *)mac_addr); 320 } 321 322 void 323 ionic_dev_remove_mac(struct rte_eth_dev *eth_dev, uint32_t index) 324 { 325 struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev); 326 struct ionic_adapter *adapter = lif->adapter; 327 struct rte_ether_addr *mac_addr; 328 329 IONIC_PRINT_CALL(); 330 331 if (index >= adapter->max_mac_addrs) { 332 IONIC_PRINT(WARNING, 333 "Index %u is above MAC filter limit %u", 334 index, adapter->max_mac_addrs); 335 return; 336 } 337 338 mac_addr = ð_dev->data->mac_addrs[index]; 339 340 if (!rte_is_valid_assigned_ether_addr(mac_addr)) 341 return; 342 343 ionic_lif_addr_del(lif, (const uint8_t *)mac_addr); 344 } 345 346 int 347 ionic_dev_set_mac(struct rte_eth_dev *eth_dev, struct rte_ether_addr *mac_addr) 348 { 349 struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev); 350 351 IONIC_PRINT_CALL(); 352 353 if (mac_addr == NULL) { 354 IONIC_PRINT(NOTICE, "New mac is null"); 355 return -1; 356 } 357 358 if (!rte_is_zero_ether_addr((struct rte_ether_addr *)lif->mac_addr)) { 359 IONIC_PRINT(INFO, "Deleting mac addr %pM", 360 lif->mac_addr); 361 ionic_lif_addr_del(lif, lif->mac_addr); 362 memset(lif->mac_addr, 0, RTE_ETHER_ADDR_LEN); 363 } 364 365 IONIC_PRINT(INFO, "Updating mac addr"); 366 367 rte_ether_addr_copy(mac_addr, (struct rte_ether_addr *)lif->mac_addr); 368 369 return ionic_lif_addr_add(lif, (const uint8_t *)mac_addr); 370 } 371 372 static int 373 ionic_vlan_rx_add_vid(struct ionic_lif *lif, uint16_t vid) 374 { 375 struct ionic_admin_ctx ctx = { 376 .pending_work = true, 377 .cmd.rx_filter_add = { 378 .opcode = IONIC_CMD_RX_FILTER_ADD, 379 .match = IONIC_RX_FILTER_MATCH_VLAN, 380 .vlan.vlan = vid, 381 }, 382 }; 383 int err; 384 385 err = ionic_adminq_post_wait(lif, &ctx); 386 if (err) 387 return err; 388 389 IONIC_PRINT(INFO, "rx_filter add VLAN %d (id %d)", vid, 390 ctx.comp.rx_filter_add.filter_id); 391 392 return ionic_rx_filter_save(lif, 0, IONIC_RXQ_INDEX_ANY, &ctx); 393 } 394 395 static int 396 ionic_vlan_rx_kill_vid(struct ionic_lif *lif, uint16_t vid) 397 { 398 struct ionic_admin_ctx ctx = { 399 .pending_work = true, 400 .cmd.rx_filter_del = { 401 .opcode = IONIC_CMD_RX_FILTER_DEL, 402 }, 403 }; 404 struct ionic_rx_filter *f; 405 int err; 406 407 IONIC_PRINT_CALL(); 408 409 rte_spinlock_lock(&lif->rx_filters.lock); 410 411 f = ionic_rx_filter_by_vlan(lif, vid); 412 if (!f) { 413 rte_spinlock_unlock(&lif->rx_filters.lock); 414 return -ENOENT; 415 } 416 417 ctx.cmd.rx_filter_del.filter_id = f->filter_id; 418 ionic_rx_filter_free(f); 419 rte_spinlock_unlock(&lif->rx_filters.lock); 420 421 err = ionic_adminq_post_wait(lif, &ctx); 422 if (err) 423 return err; 424 425 IONIC_PRINT(INFO, "rx_filter del VLAN %d (id %d)", vid, 426 ctx.cmd.rx_filter_del.filter_id); 427 428 return 0; 429 } 430 431 int 432 ionic_dev_vlan_filter_set(struct rte_eth_dev *eth_dev, uint16_t vlan_id, 433 int on) 434 { 435 struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev); 436 int err; 437 438 if (on) 439 err = ionic_vlan_rx_add_vid(lif, vlan_id); 440 else 441 err = ionic_vlan_rx_kill_vid(lif, vlan_id); 442 443 return err; 444 } 445 446 static void 447 ionic_lif_rx_mode(struct ionic_lif *lif, uint32_t rx_mode) 448 { 449 struct ionic_admin_ctx ctx = { 450 .pending_work = true, 451 .cmd.rx_mode_set = { 452 .opcode = IONIC_CMD_RX_MODE_SET, 453 .rx_mode = rx_mode, 454 }, 455 }; 456 int err; 457 458 if (rx_mode & IONIC_RX_MODE_F_UNICAST) 459 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_UNICAST"); 460 if (rx_mode & IONIC_RX_MODE_F_MULTICAST) 461 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_MULTICAST"); 462 if (rx_mode & IONIC_RX_MODE_F_BROADCAST) 463 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_BROADCAST"); 464 if (rx_mode & IONIC_RX_MODE_F_PROMISC) 465 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_PROMISC"); 466 if (rx_mode & IONIC_RX_MODE_F_ALLMULTI) 467 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_ALLMULTI"); 468 469 err = ionic_adminq_post_wait(lif, &ctx); 470 if (err) 471 IONIC_PRINT(ERR, "Failure setting RX mode"); 472 } 473 474 static void 475 ionic_set_rx_mode(struct ionic_lif *lif, uint32_t rx_mode) 476 { 477 if (lif->rx_mode != rx_mode) { 478 lif->rx_mode = rx_mode; 479 ionic_lif_rx_mode(lif, rx_mode); 480 } 481 } 482 483 int 484 ionic_dev_promiscuous_enable(struct rte_eth_dev *eth_dev) 485 { 486 struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev); 487 uint32_t rx_mode = lif->rx_mode; 488 489 IONIC_PRINT_CALL(); 490 491 rx_mode |= IONIC_RX_MODE_F_PROMISC; 492 493 ionic_set_rx_mode(lif, rx_mode); 494 495 return 0; 496 } 497 498 int 499 ionic_dev_promiscuous_disable(struct rte_eth_dev *eth_dev) 500 { 501 struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev); 502 uint32_t rx_mode = lif->rx_mode; 503 504 rx_mode &= ~IONIC_RX_MODE_F_PROMISC; 505 506 ionic_set_rx_mode(lif, rx_mode); 507 508 return 0; 509 } 510 511 int 512 ionic_dev_allmulticast_enable(struct rte_eth_dev *eth_dev) 513 { 514 struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev); 515 uint32_t rx_mode = lif->rx_mode; 516 517 rx_mode |= IONIC_RX_MODE_F_ALLMULTI; 518 519 ionic_set_rx_mode(lif, rx_mode); 520 521 return 0; 522 } 523 524 int 525 ionic_dev_allmulticast_disable(struct rte_eth_dev *eth_dev) 526 { 527 struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev); 528 uint32_t rx_mode = lif->rx_mode; 529 530 rx_mode &= ~IONIC_RX_MODE_F_ALLMULTI; 531 532 ionic_set_rx_mode(lif, rx_mode); 533 534 return 0; 535 } 536 537 int 538 ionic_lif_change_mtu(struct ionic_lif *lif, int new_mtu) 539 { 540 struct ionic_admin_ctx ctx = { 541 .pending_work = true, 542 .cmd.lif_setattr = { 543 .opcode = IONIC_CMD_LIF_SETATTR, 544 .attr = IONIC_LIF_ATTR_MTU, 545 .mtu = new_mtu, 546 }, 547 }; 548 int err; 549 550 err = ionic_adminq_post_wait(lif, &ctx); 551 if (err) 552 return err; 553 554 return 0; 555 } 556 557 int 558 ionic_intr_alloc(struct ionic_lif *lif, struct ionic_intr_info *intr) 559 { 560 struct ionic_adapter *adapter = lif->adapter; 561 struct ionic_dev *idev = &adapter->idev; 562 unsigned long index; 563 564 /* 565 * Note: interrupt handler is called for index = 0 only 566 * (we use interrupts for the notifyq only anyway, 567 * which has index = 0) 568 */ 569 570 for (index = 0; index < adapter->nintrs; index++) 571 if (!adapter->intrs[index]) 572 break; 573 574 if (index == adapter->nintrs) 575 return -ENOSPC; 576 577 adapter->intrs[index] = true; 578 579 ionic_intr_init(idev, intr, index); 580 581 return 0; 582 } 583 584 void 585 ionic_intr_free(struct ionic_lif *lif, struct ionic_intr_info *intr) 586 { 587 if (intr->index != IONIC_INTR_INDEX_NOT_ASSIGNED) 588 lif->adapter->intrs[intr->index] = false; 589 } 590 591 static int 592 ionic_qcq_alloc(struct ionic_lif *lif, uint8_t type, 593 uint32_t index, 594 const char *base, uint32_t flags, 595 uint32_t num_descs, 596 uint32_t desc_size, 597 uint32_t cq_desc_size, 598 uint32_t sg_desc_size, 599 struct ionic_qcq **qcq) 600 { 601 struct ionic_dev *idev = &lif->adapter->idev; 602 struct ionic_qcq *new; 603 uint32_t q_size, cq_size, sg_size, total_size; 604 void *q_base, *cq_base, *sg_base; 605 rte_iova_t q_base_pa = 0; 606 rte_iova_t cq_base_pa = 0; 607 rte_iova_t sg_base_pa = 0; 608 uint32_t socket_id = rte_socket_id(); 609 int err; 610 611 *qcq = NULL; 612 613 q_size = num_descs * desc_size; 614 cq_size = num_descs * cq_desc_size; 615 sg_size = num_descs * sg_desc_size; 616 617 total_size = RTE_ALIGN(q_size, PAGE_SIZE) + 618 RTE_ALIGN(cq_size, PAGE_SIZE); 619 /* 620 * Note: aligning q_size/cq_size is not enough due to cq_base address 621 * aligning as q_base could be not aligned to the page. 622 * Adding PAGE_SIZE. 623 */ 624 total_size += PAGE_SIZE; 625 626 if (flags & IONIC_QCQ_F_SG) { 627 total_size += RTE_ALIGN(sg_size, PAGE_SIZE); 628 total_size += PAGE_SIZE; 629 } 630 631 new = rte_zmalloc("ionic", sizeof(*new), 0); 632 if (!new) { 633 IONIC_PRINT(ERR, "Cannot allocate queue structure"); 634 return -ENOMEM; 635 } 636 637 new->lif = lif; 638 new->flags = flags; 639 640 new->q.info = rte_zmalloc("ionic", sizeof(*new->q.info) * num_descs, 0); 641 if (!new->q.info) { 642 IONIC_PRINT(ERR, "Cannot allocate queue info"); 643 return -ENOMEM; 644 } 645 646 new->q.type = type; 647 648 err = ionic_q_init(lif, idev, &new->q, index, num_descs, 649 desc_size, sg_desc_size); 650 if (err) { 651 IONIC_PRINT(ERR, "Queue initialization failed"); 652 return err; 653 } 654 655 if (flags & IONIC_QCQ_F_INTR) { 656 err = ionic_intr_alloc(lif, &new->intr); 657 if (err) 658 return err; 659 660 ionic_intr_mask_assert(idev->intr_ctrl, new->intr.index, 661 IONIC_INTR_MASK_SET); 662 } else { 663 new->intr.index = IONIC_INTR_INDEX_NOT_ASSIGNED; 664 } 665 666 err = ionic_cq_init(lif, &new->cq, &new->intr, 667 num_descs, cq_desc_size); 668 if (err) { 669 IONIC_PRINT(ERR, "Completion queue initialization failed"); 670 goto err_out_free_intr; 671 } 672 673 new->base_z = rte_eth_dma_zone_reserve(lif->eth_dev, 674 base /* name */, index /* queue_idx */, 675 total_size, IONIC_ALIGN, socket_id); 676 677 if (!new->base_z) { 678 IONIC_PRINT(ERR, "Cannot reserve queue DMA memory"); 679 err = -ENOMEM; 680 goto err_out_free_intr; 681 } 682 683 new->base = new->base_z->addr; 684 new->base_pa = new->base_z->iova; 685 new->total_size = total_size; 686 687 q_base = new->base; 688 q_base_pa = new->base_pa; 689 690 cq_base = (void *)RTE_ALIGN((uintptr_t)q_base + q_size, PAGE_SIZE); 691 cq_base_pa = RTE_ALIGN(q_base_pa + q_size, PAGE_SIZE); 692 693 if (flags & IONIC_QCQ_F_SG) { 694 sg_base = (void *)RTE_ALIGN((uintptr_t)cq_base + cq_size, 695 PAGE_SIZE); 696 sg_base_pa = RTE_ALIGN(cq_base_pa + cq_size, PAGE_SIZE); 697 ionic_q_sg_map(&new->q, sg_base, sg_base_pa); 698 } 699 700 IONIC_PRINT(DEBUG, "Q-Base-PA = %#jx CQ-Base-PA = %#jx " 701 "SG-base-PA = %#jx", 702 q_base_pa, cq_base_pa, sg_base_pa); 703 704 ionic_q_map(&new->q, q_base, q_base_pa); 705 ionic_cq_map(&new->cq, cq_base, cq_base_pa); 706 ionic_cq_bind(&new->cq, &new->q); 707 708 *qcq = new; 709 710 return 0; 711 712 err_out_free_intr: 713 if (flags & IONIC_QCQ_F_INTR) 714 ionic_intr_free(lif, &new->intr); 715 716 return err; 717 } 718 719 void 720 ionic_qcq_free(struct ionic_qcq *qcq) 721 { 722 if (qcq->base_z) { 723 qcq->base = NULL; 724 qcq->base_pa = 0; 725 rte_memzone_free(qcq->base_z); 726 qcq->base_z = NULL; 727 } 728 729 if (qcq->q.info) { 730 rte_free(qcq->q.info); 731 qcq->q.info = NULL; 732 } 733 734 rte_free(qcq); 735 } 736 737 int 738 ionic_rx_qcq_alloc(struct ionic_lif *lif, uint32_t index, uint16_t nrxq_descs, 739 struct ionic_qcq **qcq) 740 { 741 uint32_t flags; 742 int err = -ENOMEM; 743 744 flags = IONIC_QCQ_F_SG; 745 err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, index, "rx", flags, 746 nrxq_descs, 747 sizeof(struct ionic_rxq_desc), 748 sizeof(struct ionic_rxq_comp), 749 sizeof(struct ionic_rxq_sg_desc), 750 &lif->rxqcqs[index]); 751 if (err) 752 return err; 753 754 *qcq = lif->rxqcqs[index]; 755 756 return 0; 757 } 758 759 int 760 ionic_tx_qcq_alloc(struct ionic_lif *lif, uint32_t index, uint16_t ntxq_descs, 761 struct ionic_qcq **qcq) 762 { 763 uint32_t flags; 764 int err = -ENOMEM; 765 766 flags = IONIC_QCQ_F_SG; 767 err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, index, "tx", flags, 768 ntxq_descs, 769 sizeof(struct ionic_txq_desc), 770 sizeof(struct ionic_txq_comp), 771 sizeof(struct ionic_txq_sg_desc), 772 &lif->txqcqs[index]); 773 if (err) 774 return err; 775 776 *qcq = lif->txqcqs[index]; 777 778 return 0; 779 } 780 781 static int 782 ionic_admin_qcq_alloc(struct ionic_lif *lif) 783 { 784 uint32_t flags; 785 int err = -ENOMEM; 786 787 flags = 0; 788 err = ionic_qcq_alloc(lif, IONIC_QTYPE_ADMINQ, 0, "admin", flags, 789 IONIC_ADMINQ_LENGTH, 790 sizeof(struct ionic_admin_cmd), 791 sizeof(struct ionic_admin_comp), 792 0, 793 &lif->adminqcq); 794 if (err) 795 return err; 796 797 return 0; 798 } 799 800 static int 801 ionic_notify_qcq_alloc(struct ionic_lif *lif) 802 { 803 uint32_t flags; 804 int err = -ENOMEM; 805 806 flags = IONIC_QCQ_F_NOTIFYQ | IONIC_QCQ_F_INTR; 807 808 err = ionic_qcq_alloc(lif, IONIC_QTYPE_NOTIFYQ, 0, "notify", 809 flags, 810 IONIC_NOTIFYQ_LENGTH, 811 sizeof(struct ionic_notifyq_cmd), 812 sizeof(union ionic_notifyq_comp), 813 0, 814 &lif->notifyqcq); 815 if (err) 816 return err; 817 818 return 0; 819 } 820 821 static void * 822 ionic_bus_map_dbpage(struct ionic_adapter *adapter, int page_num) 823 { 824 char *vaddr = adapter->bars[IONIC_PCI_BAR_DBELL].vaddr; 825 826 if (adapter->num_bars <= IONIC_PCI_BAR_DBELL) 827 return NULL; 828 829 return (void *)&vaddr[page_num << PAGE_SHIFT]; 830 } 831 832 int 833 ionic_lif_alloc(struct ionic_lif *lif) 834 { 835 struct ionic_adapter *adapter = lif->adapter; 836 uint32_t socket_id = rte_socket_id(); 837 int err; 838 839 /* 840 * lif->name was zeroed on allocation. 841 * Copy (sizeof() - 1) bytes to ensure that it is NULL terminated. 842 */ 843 memcpy(lif->name, lif->eth_dev->data->name, sizeof(lif->name) - 1); 844 845 IONIC_PRINT(DEBUG, "LIF: %s", lif->name); 846 847 IONIC_PRINT(DEBUG, "Allocating Lif Info"); 848 849 rte_spinlock_init(&lif->adminq_lock); 850 rte_spinlock_init(&lif->adminq_service_lock); 851 852 lif->kern_dbpage = ionic_bus_map_dbpage(adapter, 0); 853 if (!lif->kern_dbpage) { 854 IONIC_PRINT(ERR, "Cannot map dbpage, aborting"); 855 return -ENOMEM; 856 } 857 858 lif->txqcqs = rte_zmalloc("ionic", sizeof(*lif->txqcqs) * 859 adapter->max_ntxqs_per_lif, 0); 860 861 if (!lif->txqcqs) { 862 IONIC_PRINT(ERR, "Cannot allocate tx queues array"); 863 return -ENOMEM; 864 } 865 866 lif->rxqcqs = rte_zmalloc("ionic", sizeof(*lif->rxqcqs) * 867 adapter->max_nrxqs_per_lif, 0); 868 869 if (!lif->rxqcqs) { 870 IONIC_PRINT(ERR, "Cannot allocate rx queues array"); 871 return -ENOMEM; 872 } 873 874 IONIC_PRINT(DEBUG, "Allocating Notify Queue"); 875 876 err = ionic_notify_qcq_alloc(lif); 877 if (err) { 878 IONIC_PRINT(ERR, "Cannot allocate notify queue"); 879 return err; 880 } 881 882 IONIC_PRINT(DEBUG, "Allocating Admin Queue"); 883 884 err = ionic_admin_qcq_alloc(lif); 885 if (err) { 886 IONIC_PRINT(ERR, "Cannot allocate admin queue"); 887 return err; 888 } 889 890 IONIC_PRINT(DEBUG, "Allocating Lif Info"); 891 892 lif->info_sz = RTE_ALIGN(sizeof(*lif->info), PAGE_SIZE); 893 894 lif->info_z = rte_eth_dma_zone_reserve(lif->eth_dev, 895 "lif_info", 0 /* queue_idx*/, 896 lif->info_sz, IONIC_ALIGN, socket_id); 897 if (!lif->info_z) { 898 IONIC_PRINT(ERR, "Cannot allocate lif info memory"); 899 return -ENOMEM; 900 } 901 902 lif->info = lif->info_z->addr; 903 lif->info_pa = lif->info_z->iova; 904 905 return 0; 906 } 907 908 void 909 ionic_lif_free(struct ionic_lif *lif) 910 { 911 if (lif->notifyqcq) { 912 ionic_qcq_free(lif->notifyqcq); 913 lif->notifyqcq = NULL; 914 } 915 916 if (lif->adminqcq) { 917 ionic_qcq_free(lif->adminqcq); 918 lif->adminqcq = NULL; 919 } 920 921 if (lif->txqcqs) { 922 rte_free(lif->txqcqs); 923 lif->txqcqs = NULL; 924 } 925 926 if (lif->rxqcqs) { 927 rte_free(lif->rxqcqs); 928 lif->rxqcqs = NULL; 929 } 930 931 if (lif->info) { 932 rte_memzone_free(lif->info_z); 933 lif->info = NULL; 934 } 935 } 936 937 void 938 ionic_lif_free_queues(struct ionic_lif *lif) 939 { 940 uint32_t i; 941 942 for (i = 0; i < lif->ntxqcqs; i++) { 943 ionic_dev_tx_queue_release(lif->eth_dev->data->tx_queues[i]); 944 lif->eth_dev->data->tx_queues[i] = NULL; 945 } 946 for (i = 0; i < lif->nrxqcqs; i++) { 947 ionic_dev_rx_queue_release(lif->eth_dev->data->rx_queues[i]); 948 lif->eth_dev->data->rx_queues[i] = NULL; 949 } 950 } 951 952 int 953 ionic_lif_rss_config(struct ionic_lif *lif, 954 const uint16_t types, const uint8_t *key, const uint32_t *indir) 955 { 956 struct ionic_admin_ctx ctx = { 957 .pending_work = true, 958 .cmd.lif_setattr = { 959 .opcode = IONIC_CMD_LIF_SETATTR, 960 .attr = IONIC_LIF_ATTR_RSS, 961 .rss.types = types, 962 .rss.addr = lif->rss_ind_tbl_pa, 963 }, 964 }; 965 unsigned int i; 966 967 IONIC_PRINT_CALL(); 968 969 lif->rss_types = types; 970 971 if (key) 972 memcpy(lif->rss_hash_key, key, IONIC_RSS_HASH_KEY_SIZE); 973 974 if (indir) 975 for (i = 0; i < lif->adapter->ident.lif.eth.rss_ind_tbl_sz; i++) 976 lif->rss_ind_tbl[i] = indir[i]; 977 978 memcpy(ctx.cmd.lif_setattr.rss.key, lif->rss_hash_key, 979 IONIC_RSS_HASH_KEY_SIZE); 980 981 return ionic_adminq_post_wait(lif, &ctx); 982 } 983 984 static int 985 ionic_lif_rss_setup(struct ionic_lif *lif) 986 { 987 static const uint8_t toeplitz_symmetric_key[] = { 988 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 989 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 990 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 991 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 992 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 993 }; 994 uint32_t i; 995 uint16_t tbl_sz = lif->adapter->ident.lif.eth.rss_ind_tbl_sz; 996 997 IONIC_PRINT_CALL(); 998 999 if (!lif->rss_ind_tbl_z) { 1000 lif->rss_ind_tbl_z = rte_eth_dma_zone_reserve(lif->eth_dev, 1001 "rss_ind_tbl", 0 /* queue_idx */, 1002 sizeof(*lif->rss_ind_tbl) * tbl_sz, 1003 IONIC_ALIGN, rte_socket_id()); 1004 if (!lif->rss_ind_tbl_z) { 1005 IONIC_PRINT(ERR, "OOM"); 1006 return -ENOMEM; 1007 } 1008 1009 lif->rss_ind_tbl = lif->rss_ind_tbl_z->addr; 1010 lif->rss_ind_tbl_pa = lif->rss_ind_tbl_z->iova; 1011 } 1012 1013 if (lif->rss_ind_tbl_nrxqcqs != lif->nrxqcqs) { 1014 lif->rss_ind_tbl_nrxqcqs = lif->nrxqcqs; 1015 1016 /* Fill indirection table with 'default' values */ 1017 for (i = 0; i < tbl_sz; i++) 1018 lif->rss_ind_tbl[i] = i % lif->nrxqcqs; 1019 } 1020 1021 return ionic_lif_rss_config(lif, IONIC_RSS_OFFLOAD_ALL, 1022 toeplitz_symmetric_key, NULL); 1023 } 1024 1025 static void 1026 ionic_lif_rss_teardown(struct ionic_lif *lif) 1027 { 1028 if (!lif->rss_ind_tbl) 1029 return; 1030 1031 if (lif->rss_ind_tbl_z) { 1032 /* Disable RSS on the NIC */ 1033 ionic_lif_rss_config(lif, 0x0, NULL, NULL); 1034 1035 lif->rss_ind_tbl = NULL; 1036 lif->rss_ind_tbl_pa = 0; 1037 rte_memzone_free(lif->rss_ind_tbl_z); 1038 lif->rss_ind_tbl_z = NULL; 1039 } 1040 } 1041 1042 static void 1043 ionic_lif_qcq_deinit(struct ionic_lif *lif, struct ionic_qcq *qcq) 1044 { 1045 struct ionic_dev *idev = &lif->adapter->idev; 1046 1047 if (!(qcq->flags & IONIC_QCQ_F_INITED)) 1048 return; 1049 1050 if (qcq->flags & IONIC_QCQ_F_INTR) 1051 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index, 1052 IONIC_INTR_MASK_SET); 1053 1054 qcq->flags &= ~IONIC_QCQ_F_INITED; 1055 } 1056 1057 void 1058 ionic_lif_txq_deinit(struct ionic_qcq *qcq) 1059 { 1060 ionic_lif_qcq_deinit(qcq->lif, qcq); 1061 } 1062 1063 void 1064 ionic_lif_rxq_deinit(struct ionic_qcq *qcq) 1065 { 1066 ionic_lif_qcq_deinit(qcq->lif, qcq); 1067 } 1068 1069 bool 1070 ionic_adminq_service(struct ionic_cq *cq, uint32_t cq_desc_index, 1071 void *cb_arg __rte_unused) 1072 { 1073 struct ionic_admin_comp *cq_desc_base = cq->base; 1074 struct ionic_admin_comp *cq_desc = &cq_desc_base[cq_desc_index]; 1075 1076 if (!color_match(cq_desc->color, cq->done_color)) 1077 return false; 1078 1079 ionic_q_service(cq->bound_q, cq_desc_index, cq_desc->comp_index, NULL); 1080 1081 return true; 1082 } 1083 1084 /* This acts like ionic_napi */ 1085 int 1086 ionic_qcq_service(struct ionic_qcq *qcq, int budget, ionic_cq_cb cb, 1087 void *cb_arg) 1088 { 1089 struct ionic_cq *cq = &qcq->cq; 1090 uint32_t work_done; 1091 1092 work_done = ionic_cq_service(cq, budget, cb, cb_arg); 1093 1094 return work_done; 1095 } 1096 1097 static void 1098 ionic_link_status_check(struct ionic_lif *lif) 1099 { 1100 struct ionic_adapter *adapter = lif->adapter; 1101 bool link_up; 1102 1103 lif->state &= ~IONIC_LIF_F_LINK_CHECK_NEEDED; 1104 1105 if (!lif->info) 1106 return; 1107 1108 link_up = (lif->info->status.link_status == IONIC_PORT_OPER_STATUS_UP); 1109 1110 if ((link_up && adapter->link_up) || 1111 (!link_up && !adapter->link_up)) 1112 return; 1113 1114 if (link_up) { 1115 adapter->link_speed = lif->info->status.link_speed; 1116 IONIC_PRINT(DEBUG, "Link up - %d Gbps", 1117 adapter->link_speed); 1118 } else { 1119 IONIC_PRINT(DEBUG, "Link down"); 1120 } 1121 1122 adapter->link_up = link_up; 1123 ionic_dev_link_update(lif->eth_dev, 0); 1124 } 1125 1126 static void 1127 ionic_lif_handle_fw_down(struct ionic_lif *lif) 1128 { 1129 if (lif->state & IONIC_LIF_F_FW_RESET) 1130 return; 1131 1132 lif->state |= IONIC_LIF_F_FW_RESET; 1133 1134 if (lif->state & IONIC_LIF_F_UP) { 1135 IONIC_PRINT(NOTICE, 1136 "Surprise FW stop, stopping %s\n", lif->name); 1137 ionic_lif_stop(lif); 1138 } 1139 1140 IONIC_PRINT(NOTICE, "FW down, %s stopped", lif->name); 1141 } 1142 1143 static bool 1144 ionic_notifyq_cb(struct ionic_cq *cq, uint32_t cq_desc_index, void *cb_arg) 1145 { 1146 union ionic_notifyq_comp *cq_desc_base = cq->base; 1147 union ionic_notifyq_comp *cq_desc = &cq_desc_base[cq_desc_index]; 1148 struct ionic_lif *lif = cb_arg; 1149 1150 IONIC_PRINT(DEBUG, "Notifyq callback eid = %jd ecode = %d", 1151 cq_desc->event.eid, cq_desc->event.ecode); 1152 1153 /* Have we run out of new completions to process? */ 1154 if (!(cq_desc->event.eid > lif->last_eid)) 1155 return false; 1156 1157 lif->last_eid = cq_desc->event.eid; 1158 1159 switch (cq_desc->event.ecode) { 1160 case IONIC_EVENT_LINK_CHANGE: 1161 IONIC_PRINT(DEBUG, 1162 "Notifyq IONIC_EVENT_LINK_CHANGE %s " 1163 "eid=%jd link_status=%d link_speed=%d", 1164 lif->name, 1165 cq_desc->event.eid, 1166 cq_desc->link_change.link_status, 1167 cq_desc->link_change.link_speed); 1168 1169 lif->state |= IONIC_LIF_F_LINK_CHECK_NEEDED; 1170 break; 1171 1172 case IONIC_EVENT_RESET: 1173 IONIC_PRINT(NOTICE, 1174 "Notifyq IONIC_EVENT_RESET %s " 1175 "eid=%jd, reset_code=%d state=%d", 1176 lif->name, 1177 cq_desc->event.eid, 1178 cq_desc->reset.reset_code, 1179 cq_desc->reset.state); 1180 ionic_lif_handle_fw_down(lif); 1181 break; 1182 1183 default: 1184 IONIC_PRINT(WARNING, "Notifyq bad event ecode=%d eid=%jd", 1185 cq_desc->event.ecode, cq_desc->event.eid); 1186 break; 1187 } 1188 1189 return true; 1190 } 1191 1192 int 1193 ionic_notifyq_handler(struct ionic_lif *lif, int budget) 1194 { 1195 struct ionic_dev *idev = &lif->adapter->idev; 1196 struct ionic_qcq *qcq = lif->notifyqcq; 1197 uint32_t work_done; 1198 1199 if (!(qcq->flags & IONIC_QCQ_F_INITED)) { 1200 IONIC_PRINT(DEBUG, "Notifyq not yet initialized"); 1201 return -1; 1202 } 1203 1204 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index, 1205 IONIC_INTR_MASK_SET); 1206 1207 work_done = ionic_qcq_service(qcq, budget, ionic_notifyq_cb, lif); 1208 1209 if (lif->state & IONIC_LIF_F_LINK_CHECK_NEEDED) 1210 ionic_link_status_check(lif); 1211 1212 ionic_intr_credits(idev->intr_ctrl, qcq->intr.index, 1213 work_done, IONIC_INTR_CRED_RESET_COALESCE); 1214 1215 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index, 1216 IONIC_INTR_MASK_CLEAR); 1217 1218 return 0; 1219 } 1220 1221 static int 1222 ionic_lif_adminq_init(struct ionic_lif *lif) 1223 { 1224 struct ionic_dev *idev = &lif->adapter->idev; 1225 struct ionic_qcq *qcq = lif->adminqcq; 1226 struct ionic_queue *q = &qcq->q; 1227 struct ionic_q_init_comp comp; 1228 int err; 1229 1230 ionic_dev_cmd_adminq_init(idev, qcq, qcq->intr.index); 1231 err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT); 1232 if (err) 1233 return err; 1234 1235 ionic_dev_cmd_comp(idev, &comp); 1236 1237 q->hw_type = comp.hw_type; 1238 q->hw_index = comp.hw_index; 1239 q->db = ionic_db_map(lif, q); 1240 1241 IONIC_PRINT(DEBUG, "adminq->hw_type %d", q->hw_type); 1242 IONIC_PRINT(DEBUG, "adminq->hw_index %d", q->hw_index); 1243 IONIC_PRINT(DEBUG, "adminq->db %p", q->db); 1244 1245 if (qcq->flags & IONIC_QCQ_F_INTR) 1246 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index, 1247 IONIC_INTR_MASK_CLEAR); 1248 1249 qcq->flags |= IONIC_QCQ_F_INITED; 1250 1251 return 0; 1252 } 1253 1254 static int 1255 ionic_lif_notifyq_init(struct ionic_lif *lif) 1256 { 1257 struct ionic_dev *idev = &lif->adapter->idev; 1258 struct ionic_qcq *qcq = lif->notifyqcq; 1259 struct ionic_queue *q = &qcq->q; 1260 int err; 1261 1262 struct ionic_admin_ctx ctx = { 1263 .pending_work = true, 1264 .cmd.q_init = { 1265 .opcode = IONIC_CMD_Q_INIT, 1266 .type = q->type, 1267 .index = q->index, 1268 .flags = (IONIC_QINIT_F_IRQ | IONIC_QINIT_F_ENA), 1269 .intr_index = qcq->intr.index, 1270 .ring_size = rte_log2_u32(q->num_descs), 1271 .ring_base = q->base_pa, 1272 } 1273 }; 1274 1275 IONIC_PRINT(DEBUG, "notifyq_init.index %d", 1276 ctx.cmd.q_init.index); 1277 IONIC_PRINT(DEBUG, "notifyq_init.ring_base 0x%" PRIx64 "", 1278 ctx.cmd.q_init.ring_base); 1279 IONIC_PRINT(DEBUG, "notifyq_init.ring_size %d", 1280 ctx.cmd.q_init.ring_size); 1281 IONIC_PRINT(DEBUG, "notifyq_init.ver %u", ctx.cmd.q_init.ver); 1282 1283 err = ionic_adminq_post_wait(lif, &ctx); 1284 if (err) 1285 return err; 1286 1287 q->hw_type = ctx.comp.q_init.hw_type; 1288 q->hw_index = ctx.comp.q_init.hw_index; 1289 q->db = NULL; 1290 1291 IONIC_PRINT(DEBUG, "notifyq->hw_type %d", q->hw_type); 1292 IONIC_PRINT(DEBUG, "notifyq->hw_index %d", q->hw_index); 1293 IONIC_PRINT(DEBUG, "notifyq->db %p", q->db); 1294 1295 if (qcq->flags & IONIC_QCQ_F_INTR) 1296 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index, 1297 IONIC_INTR_MASK_CLEAR); 1298 1299 qcq->flags |= IONIC_QCQ_F_INITED; 1300 1301 return 0; 1302 } 1303 1304 int 1305 ionic_lif_set_features(struct ionic_lif *lif) 1306 { 1307 struct ionic_admin_ctx ctx = { 1308 .pending_work = true, 1309 .cmd.lif_setattr = { 1310 .opcode = IONIC_CMD_LIF_SETATTR, 1311 .attr = IONIC_LIF_ATTR_FEATURES, 1312 .features = lif->features, 1313 }, 1314 }; 1315 int err; 1316 1317 err = ionic_adminq_post_wait(lif, &ctx); 1318 if (err) 1319 return err; 1320 1321 lif->hw_features = (ctx.cmd.lif_setattr.features & 1322 ctx.comp.lif_setattr.features); 1323 1324 if (lif->hw_features & IONIC_ETH_HW_VLAN_TX_TAG) 1325 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_VLAN_TX_TAG"); 1326 if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_STRIP) 1327 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_VLAN_RX_STRIP"); 1328 if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_FILTER) 1329 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_VLAN_RX_FILTER"); 1330 if (lif->hw_features & IONIC_ETH_HW_RX_HASH) 1331 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_RX_HASH"); 1332 if (lif->hw_features & IONIC_ETH_HW_TX_SG) 1333 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TX_SG"); 1334 if (lif->hw_features & IONIC_ETH_HW_RX_SG) 1335 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_RX_SG"); 1336 if (lif->hw_features & IONIC_ETH_HW_TX_CSUM) 1337 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TX_CSUM"); 1338 if (lif->hw_features & IONIC_ETH_HW_RX_CSUM) 1339 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_RX_CSUM"); 1340 if (lif->hw_features & IONIC_ETH_HW_TSO) 1341 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO"); 1342 if (lif->hw_features & IONIC_ETH_HW_TSO_IPV6) 1343 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_IPV6"); 1344 if (lif->hw_features & IONIC_ETH_HW_TSO_ECN) 1345 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_ECN"); 1346 if (lif->hw_features & IONIC_ETH_HW_TSO_GRE) 1347 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_GRE"); 1348 if (lif->hw_features & IONIC_ETH_HW_TSO_GRE_CSUM) 1349 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_GRE_CSUM"); 1350 if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP4) 1351 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_IPXIP4"); 1352 if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP6) 1353 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_IPXIP6"); 1354 if (lif->hw_features & IONIC_ETH_HW_TSO_UDP) 1355 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_UDP"); 1356 if (lif->hw_features & IONIC_ETH_HW_TSO_UDP_CSUM) 1357 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_UDP_CSUM"); 1358 1359 return 0; 1360 } 1361 1362 int 1363 ionic_lif_txq_init(struct ionic_qcq *qcq) 1364 { 1365 struct ionic_queue *q = &qcq->q; 1366 struct ionic_lif *lif = qcq->lif; 1367 struct ionic_cq *cq = &qcq->cq; 1368 struct ionic_admin_ctx ctx = { 1369 .pending_work = true, 1370 .cmd.q_init = { 1371 .opcode = IONIC_CMD_Q_INIT, 1372 .type = q->type, 1373 .index = q->index, 1374 .flags = IONIC_QINIT_F_SG | IONIC_QINIT_F_ENA, 1375 .intr_index = cq->bound_intr->index, 1376 .ring_size = rte_log2_u32(q->num_descs), 1377 .ring_base = q->base_pa, 1378 .cq_ring_base = cq->base_pa, 1379 .sg_ring_base = q->sg_base_pa, 1380 }, 1381 }; 1382 int err; 1383 1384 IONIC_PRINT(DEBUG, "txq_init.index %d", ctx.cmd.q_init.index); 1385 IONIC_PRINT(DEBUG, "txq_init.ring_base 0x%" PRIx64 "", 1386 ctx.cmd.q_init.ring_base); 1387 IONIC_PRINT(DEBUG, "txq_init.ring_size %d", 1388 ctx.cmd.q_init.ring_size); 1389 IONIC_PRINT(DEBUG, "txq_init.ver %u", ctx.cmd.q_init.ver); 1390 1391 err = ionic_adminq_post_wait(qcq->lif, &ctx); 1392 if (err) 1393 return err; 1394 1395 q->hw_type = ctx.comp.q_init.hw_type; 1396 q->hw_index = ctx.comp.q_init.hw_index; 1397 q->db = ionic_db_map(lif, q); 1398 1399 IONIC_PRINT(DEBUG, "txq->hw_type %d", q->hw_type); 1400 IONIC_PRINT(DEBUG, "txq->hw_index %d", q->hw_index); 1401 IONIC_PRINT(DEBUG, "txq->db %p", q->db); 1402 1403 qcq->flags |= IONIC_QCQ_F_INITED; 1404 1405 return 0; 1406 } 1407 1408 int 1409 ionic_lif_rxq_init(struct ionic_qcq *qcq) 1410 { 1411 struct ionic_queue *q = &qcq->q; 1412 struct ionic_lif *lif = qcq->lif; 1413 struct ionic_cq *cq = &qcq->cq; 1414 struct ionic_admin_ctx ctx = { 1415 .pending_work = true, 1416 .cmd.q_init = { 1417 .opcode = IONIC_CMD_Q_INIT, 1418 .type = q->type, 1419 .index = q->index, 1420 .flags = IONIC_QINIT_F_SG | IONIC_QINIT_F_ENA, 1421 .intr_index = cq->bound_intr->index, 1422 .ring_size = rte_log2_u32(q->num_descs), 1423 .ring_base = q->base_pa, 1424 .cq_ring_base = cq->base_pa, 1425 .sg_ring_base = q->sg_base_pa, 1426 }, 1427 }; 1428 int err; 1429 1430 IONIC_PRINT(DEBUG, "rxq_init.index %d", ctx.cmd.q_init.index); 1431 IONIC_PRINT(DEBUG, "rxq_init.ring_base 0x%" PRIx64 "", 1432 ctx.cmd.q_init.ring_base); 1433 IONIC_PRINT(DEBUG, "rxq_init.ring_size %d", 1434 ctx.cmd.q_init.ring_size); 1435 IONIC_PRINT(DEBUG, "rxq_init.ver %u", ctx.cmd.q_init.ver); 1436 1437 err = ionic_adminq_post_wait(qcq->lif, &ctx); 1438 if (err) 1439 return err; 1440 1441 q->hw_type = ctx.comp.q_init.hw_type; 1442 q->hw_index = ctx.comp.q_init.hw_index; 1443 q->db = ionic_db_map(lif, q); 1444 1445 qcq->flags |= IONIC_QCQ_F_INITED; 1446 1447 IONIC_PRINT(DEBUG, "rxq->hw_type %d", q->hw_type); 1448 IONIC_PRINT(DEBUG, "rxq->hw_index %d", q->hw_index); 1449 IONIC_PRINT(DEBUG, "rxq->db %p", q->db); 1450 1451 return 0; 1452 } 1453 1454 static int 1455 ionic_station_set(struct ionic_lif *lif) 1456 { 1457 struct ionic_admin_ctx ctx = { 1458 .pending_work = true, 1459 .cmd.lif_getattr = { 1460 .opcode = IONIC_CMD_LIF_GETATTR, 1461 .attr = IONIC_LIF_ATTR_MAC, 1462 }, 1463 }; 1464 int err; 1465 1466 IONIC_PRINT_CALL(); 1467 1468 err = ionic_adminq_post_wait(lif, &ctx); 1469 if (err) 1470 return err; 1471 1472 memcpy(lif->mac_addr, ctx.comp.lif_getattr.mac, RTE_ETHER_ADDR_LEN); 1473 1474 return 0; 1475 } 1476 1477 static void 1478 ionic_lif_set_name(struct ionic_lif *lif) 1479 { 1480 struct ionic_admin_ctx ctx = { 1481 .pending_work = true, 1482 .cmd.lif_setattr = { 1483 .opcode = IONIC_CMD_LIF_SETATTR, 1484 .attr = IONIC_LIF_ATTR_NAME, 1485 }, 1486 }; 1487 1488 memcpy(ctx.cmd.lif_setattr.name, lif->name, 1489 sizeof(ctx.cmd.lif_setattr.name) - 1); 1490 1491 ionic_adminq_post_wait(lif, &ctx); 1492 } 1493 1494 int 1495 ionic_lif_init(struct ionic_lif *lif) 1496 { 1497 struct ionic_dev *idev = &lif->adapter->idev; 1498 struct ionic_q_init_comp comp; 1499 int err; 1500 1501 memset(&lif->stats_base, 0, sizeof(lif->stats_base)); 1502 1503 ionic_dev_cmd_lif_init(idev, lif->info_pa); 1504 err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT); 1505 ionic_dev_cmd_comp(idev, &comp); 1506 if (err) 1507 return err; 1508 1509 lif->hw_index = comp.hw_index; 1510 1511 err = ionic_lif_adminq_init(lif); 1512 if (err) 1513 return err; 1514 1515 err = ionic_lif_notifyq_init(lif); 1516 if (err) 1517 goto err_out_adminq_deinit; 1518 1519 /* 1520 * Configure initial feature set 1521 * This will be updated later by the dev_configure() step 1522 */ 1523 lif->features = IONIC_ETH_HW_RX_HASH | IONIC_ETH_HW_VLAN_RX_FILTER; 1524 1525 err = ionic_lif_set_features(lif); 1526 if (err) 1527 goto err_out_notifyq_deinit; 1528 1529 err = ionic_rx_filters_init(lif); 1530 if (err) 1531 goto err_out_notifyq_deinit; 1532 1533 err = ionic_station_set(lif); 1534 if (err) 1535 goto err_out_rx_filter_deinit; 1536 1537 ionic_lif_set_name(lif); 1538 1539 lif->state |= IONIC_LIF_F_INITED; 1540 1541 return 0; 1542 1543 err_out_rx_filter_deinit: 1544 ionic_rx_filters_deinit(lif); 1545 1546 err_out_notifyq_deinit: 1547 ionic_lif_qcq_deinit(lif, lif->notifyqcq); 1548 1549 err_out_adminq_deinit: 1550 ionic_lif_qcq_deinit(lif, lif->adminqcq); 1551 1552 return err; 1553 } 1554 1555 void 1556 ionic_lif_deinit(struct ionic_lif *lif) 1557 { 1558 if (!(lif->state & IONIC_LIF_F_INITED)) 1559 return; 1560 1561 ionic_rx_filters_deinit(lif); 1562 ionic_lif_rss_teardown(lif); 1563 ionic_lif_qcq_deinit(lif, lif->notifyqcq); 1564 ionic_lif_qcq_deinit(lif, lif->adminqcq); 1565 1566 lif->state &= ~IONIC_LIF_F_INITED; 1567 } 1568 1569 void 1570 ionic_lif_configure_vlan_offload(struct ionic_lif *lif, int mask) 1571 { 1572 struct rte_eth_dev *eth_dev = lif->eth_dev; 1573 struct rte_eth_rxmode *rxmode = ð_dev->data->dev_conf.rxmode; 1574 1575 /* 1576 * IONIC_ETH_HW_VLAN_RX_FILTER cannot be turned off, so 1577 * set DEV_RX_OFFLOAD_VLAN_FILTER and ignore ETH_VLAN_FILTER_MASK 1578 */ 1579 rxmode->offloads |= DEV_RX_OFFLOAD_VLAN_FILTER; 1580 1581 if (mask & ETH_VLAN_STRIP_MASK) { 1582 if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_STRIP) 1583 lif->features |= IONIC_ETH_HW_VLAN_RX_STRIP; 1584 else 1585 lif->features &= ~IONIC_ETH_HW_VLAN_RX_STRIP; 1586 } 1587 } 1588 1589 void 1590 ionic_lif_configure(struct ionic_lif *lif) 1591 { 1592 struct rte_eth_rxmode *rxmode = &lif->eth_dev->data->dev_conf.rxmode; 1593 struct rte_eth_txmode *txmode = &lif->eth_dev->data->dev_conf.txmode; 1594 struct ionic_identity *ident = &lif->adapter->ident; 1595 uint32_t ntxqs_per_lif = 1596 ident->lif.eth.config.queue_count[IONIC_QTYPE_TXQ]; 1597 uint32_t nrxqs_per_lif = 1598 ident->lif.eth.config.queue_count[IONIC_QTYPE_RXQ]; 1599 uint32_t nrxqs = lif->eth_dev->data->nb_rx_queues; 1600 uint32_t ntxqs = lif->eth_dev->data->nb_tx_queues; 1601 1602 lif->port_id = lif->eth_dev->data->port_id; 1603 1604 IONIC_PRINT(DEBUG, "Configuring LIF on port %u", 1605 lif->port_id); 1606 1607 if (nrxqs > 0) 1608 nrxqs_per_lif = RTE_MIN(nrxqs_per_lif, nrxqs); 1609 1610 if (ntxqs > 0) 1611 ntxqs_per_lif = RTE_MIN(ntxqs_per_lif, ntxqs); 1612 1613 lif->nrxqcqs = nrxqs_per_lif; 1614 lif->ntxqcqs = ntxqs_per_lif; 1615 1616 /* Update the LIF configuration based on the eth_dev */ 1617 1618 /* 1619 * NB: While it is true that RSS_HASH is always enabled on ionic, 1620 * setting this flag unconditionally causes problems in DTS. 1621 * rxmode->offloads |= DEV_RX_OFFLOAD_RSS_HASH; 1622 */ 1623 1624 /* RX per-port */ 1625 1626 if (rxmode->offloads & DEV_RX_OFFLOAD_IPV4_CKSUM || 1627 rxmode->offloads & DEV_RX_OFFLOAD_UDP_CKSUM || 1628 rxmode->offloads & DEV_RX_OFFLOAD_TCP_CKSUM) 1629 lif->features |= IONIC_ETH_HW_RX_CSUM; 1630 else 1631 lif->features &= ~IONIC_ETH_HW_RX_CSUM; 1632 1633 if (rxmode->offloads & DEV_RX_OFFLOAD_SCATTER) { 1634 lif->features |= IONIC_ETH_HW_RX_SG; 1635 lif->eth_dev->data->scattered_rx = 1; 1636 } else { 1637 lif->features &= ~IONIC_ETH_HW_RX_SG; 1638 lif->eth_dev->data->scattered_rx = 0; 1639 } 1640 1641 /* Covers VLAN_STRIP */ 1642 ionic_lif_configure_vlan_offload(lif, ETH_VLAN_STRIP_MASK); 1643 1644 /* TX per-port */ 1645 1646 if (txmode->offloads & DEV_TX_OFFLOAD_IPV4_CKSUM || 1647 txmode->offloads & DEV_TX_OFFLOAD_UDP_CKSUM || 1648 txmode->offloads & DEV_TX_OFFLOAD_TCP_CKSUM || 1649 txmode->offloads & DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM || 1650 txmode->offloads & DEV_TX_OFFLOAD_OUTER_UDP_CKSUM) 1651 lif->features |= IONIC_ETH_HW_TX_CSUM; 1652 else 1653 lif->features &= ~IONIC_ETH_HW_TX_CSUM; 1654 1655 if (txmode->offloads & DEV_TX_OFFLOAD_VLAN_INSERT) 1656 lif->features |= IONIC_ETH_HW_VLAN_TX_TAG; 1657 else 1658 lif->features &= ~IONIC_ETH_HW_VLAN_TX_TAG; 1659 1660 if (txmode->offloads & DEV_TX_OFFLOAD_MULTI_SEGS) 1661 lif->features |= IONIC_ETH_HW_TX_SG; 1662 else 1663 lif->features &= ~IONIC_ETH_HW_TX_SG; 1664 1665 if (txmode->offloads & DEV_TX_OFFLOAD_TCP_TSO) { 1666 lif->features |= IONIC_ETH_HW_TSO; 1667 lif->features |= IONIC_ETH_HW_TSO_IPV6; 1668 lif->features |= IONIC_ETH_HW_TSO_ECN; 1669 } else { 1670 lif->features &= ~IONIC_ETH_HW_TSO; 1671 lif->features &= ~IONIC_ETH_HW_TSO_IPV6; 1672 lif->features &= ~IONIC_ETH_HW_TSO_ECN; 1673 } 1674 } 1675 1676 int 1677 ionic_lif_start(struct ionic_lif *lif) 1678 { 1679 uint32_t rx_mode; 1680 uint32_t i; 1681 int err; 1682 1683 err = ionic_lif_rss_setup(lif); 1684 if (err) 1685 return err; 1686 1687 if (!lif->rx_mode) { 1688 IONIC_PRINT(DEBUG, "Setting RX mode on %s", 1689 lif->name); 1690 1691 rx_mode = IONIC_RX_MODE_F_UNICAST; 1692 rx_mode |= IONIC_RX_MODE_F_MULTICAST; 1693 rx_mode |= IONIC_RX_MODE_F_BROADCAST; 1694 1695 ionic_set_rx_mode(lif, rx_mode); 1696 } 1697 1698 IONIC_PRINT(DEBUG, "Starting %u RX queues and %u TX queues " 1699 "on port %u", 1700 lif->nrxqcqs, lif->ntxqcqs, lif->port_id); 1701 1702 for (i = 0; i < lif->nrxqcqs; i++) { 1703 struct ionic_qcq *rxq = lif->rxqcqs[i]; 1704 if (!(rxq->flags & IONIC_QCQ_F_DEFERRED)) { 1705 err = ionic_dev_rx_queue_start(lif->eth_dev, i); 1706 1707 if (err) 1708 return err; 1709 } 1710 } 1711 1712 for (i = 0; i < lif->ntxqcqs; i++) { 1713 struct ionic_qcq *txq = lif->txqcqs[i]; 1714 if (!(txq->flags & IONIC_QCQ_F_DEFERRED)) { 1715 err = ionic_dev_tx_queue_start(lif->eth_dev, i); 1716 1717 if (err) 1718 return err; 1719 } 1720 } 1721 1722 /* Carrier ON here */ 1723 lif->state |= IONIC_LIF_F_UP; 1724 1725 ionic_link_status_check(lif); 1726 1727 return 0; 1728 } 1729 1730 int 1731 ionic_lif_identify(struct ionic_adapter *adapter) 1732 { 1733 struct ionic_dev *idev = &adapter->idev; 1734 struct ionic_identity *ident = &adapter->ident; 1735 int err; 1736 unsigned int i; 1737 unsigned int lif_words = sizeof(ident->lif.words) / 1738 sizeof(ident->lif.words[0]); 1739 unsigned int cmd_words = sizeof(idev->dev_cmd->data) / 1740 sizeof(idev->dev_cmd->data[0]); 1741 unsigned int nwords; 1742 1743 ionic_dev_cmd_lif_identify(idev, IONIC_LIF_TYPE_CLASSIC, 1744 IONIC_IDENTITY_VERSION_1); 1745 err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT); 1746 if (err) 1747 return (err); 1748 1749 nwords = RTE_MIN(lif_words, cmd_words); 1750 for (i = 0; i < nwords; i++) 1751 ident->lif.words[i] = ioread32(&idev->dev_cmd->data[i]); 1752 1753 IONIC_PRINT(INFO, "capabilities 0x%" PRIx64 " ", 1754 ident->lif.capabilities); 1755 1756 IONIC_PRINT(INFO, "eth.max_ucast_filters 0x%" PRIx32 " ", 1757 ident->lif.eth.max_ucast_filters); 1758 IONIC_PRINT(INFO, "eth.max_mcast_filters 0x%" PRIx32 " ", 1759 ident->lif.eth.max_mcast_filters); 1760 1761 IONIC_PRINT(INFO, "eth.features 0x%" PRIx64 " ", 1762 ident->lif.eth.config.features); 1763 IONIC_PRINT(INFO, "eth.queue_count[IONIC_QTYPE_ADMINQ] 0x%" PRIx32 " ", 1764 ident->lif.eth.config.queue_count[IONIC_QTYPE_ADMINQ]); 1765 IONIC_PRINT(INFO, "eth.queue_count[IONIC_QTYPE_NOTIFYQ] 0x%" PRIx32 " ", 1766 ident->lif.eth.config.queue_count[IONIC_QTYPE_NOTIFYQ]); 1767 IONIC_PRINT(INFO, "eth.queue_count[IONIC_QTYPE_RXQ] 0x%" PRIx32 " ", 1768 ident->lif.eth.config.queue_count[IONIC_QTYPE_RXQ]); 1769 IONIC_PRINT(INFO, "eth.queue_count[IONIC_QTYPE_TXQ] 0x%" PRIx32 " ", 1770 ident->lif.eth.config.queue_count[IONIC_QTYPE_TXQ]); 1771 1772 return 0; 1773 } 1774 1775 int 1776 ionic_lifs_size(struct ionic_adapter *adapter) 1777 { 1778 struct ionic_identity *ident = &adapter->ident; 1779 uint32_t nintrs, dev_nintrs = ident->dev.nintrs; 1780 1781 adapter->max_ntxqs_per_lif = 1782 ident->lif.eth.config.queue_count[IONIC_QTYPE_TXQ]; 1783 adapter->max_nrxqs_per_lif = 1784 ident->lif.eth.config.queue_count[IONIC_QTYPE_RXQ]; 1785 1786 nintrs = 1 /* notifyq */; 1787 1788 if (nintrs > dev_nintrs) { 1789 IONIC_PRINT(ERR, 1790 "At most %d intr supported, minimum req'd is %u", 1791 dev_nintrs, nintrs); 1792 return -ENOSPC; 1793 } 1794 1795 adapter->nintrs = nintrs; 1796 1797 return 0; 1798 } 1799