1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) 2016-2017 Solarflare Communications Inc. 5 * All rights reserved. 6 * 7 * This software was jointly developed between OKTET Labs (under contract 8 * for Solarflare) and Solarflare Communications, Inc. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions are met: 12 * 13 * 1. Redistributions of source code must retain the above copyright notice, 14 * this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright notice, 16 * this list of conditions and the following disclaimer in the documentation 17 * and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 20 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 21 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 23 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 24 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 25 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 26 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 27 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 28 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 29 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #include <rte_mempool.h> 33 34 #include "efx.h" 35 36 #include "sfc.h" 37 #include "sfc_debug.h" 38 #include "sfc_log.h" 39 #include "sfc_ev.h" 40 #include "sfc_rx.h" 41 #include "sfc_kvargs.h" 42 #include "sfc_tweak.h" 43 44 /* 45 * Maximum number of Rx queue flush attempt in the case of failure or 46 * flush timeout 47 */ 48 #define SFC_RX_QFLUSH_ATTEMPTS (3) 49 50 /* 51 * Time to wait between event queue polling attempts when waiting for Rx 52 * queue flush done or failed events. 53 */ 54 #define SFC_RX_QFLUSH_POLL_WAIT_MS (1) 55 56 /* 57 * Maximum number of event queue polling attempts when waiting for Rx queue 58 * flush done or failed events. It defines Rx queue flush attempt timeout 59 * together with SFC_RX_QFLUSH_POLL_WAIT_MS. 60 */ 61 #define SFC_RX_QFLUSH_POLL_ATTEMPTS (2000) 62 63 void 64 sfc_rx_qflush_done(struct sfc_rxq *rxq) 65 { 66 rxq->state |= SFC_RXQ_FLUSHED; 67 rxq->state &= ~SFC_RXQ_FLUSHING; 68 } 69 70 void 71 sfc_rx_qflush_failed(struct sfc_rxq *rxq) 72 { 73 rxq->state |= SFC_RXQ_FLUSH_FAILED; 74 rxq->state &= ~SFC_RXQ_FLUSHING; 75 } 76 77 static void 78 sfc_efx_rx_qrefill(struct sfc_efx_rxq *rxq) 79 { 80 unsigned int free_space; 81 unsigned int bulks; 82 void *objs[SFC_RX_REFILL_BULK]; 83 efsys_dma_addr_t addr[RTE_DIM(objs)]; 84 unsigned int added = rxq->added; 85 unsigned int id; 86 unsigned int i; 87 struct sfc_efx_rx_sw_desc *rxd; 88 struct rte_mbuf *m; 89 uint16_t port_id = rxq->dp.dpq.port_id; 90 91 free_space = EFX_RXQ_LIMIT(rxq->ptr_mask + 1) - 92 (added - rxq->completed); 93 94 if (free_space < rxq->refill_threshold) 95 return; 96 97 bulks = free_space / RTE_DIM(objs); 98 /* refill_threshold guarantees that bulks is positive */ 99 SFC_ASSERT(bulks > 0); 100 101 id = added & rxq->ptr_mask; 102 do { 103 if (unlikely(rte_mempool_get_bulk(rxq->refill_mb_pool, objs, 104 RTE_DIM(objs)) < 0)) { 105 /* 106 * It is hardly a safe way to increment counter 107 * from different contexts, but all PMDs do it. 108 */ 109 rxq->evq->sa->eth_dev->data->rx_mbuf_alloc_failed += 110 RTE_DIM(objs); 111 /* Return if we have posted nothing yet */ 112 if (added == rxq->added) 113 return; 114 /* Push posted */ 115 break; 116 } 117 118 for (i = 0; i < RTE_DIM(objs); 119 ++i, id = (id + 1) & rxq->ptr_mask) { 120 m = objs[i]; 121 122 rxd = &rxq->sw_desc[id]; 123 rxd->mbuf = m; 124 125 SFC_ASSERT(rte_mbuf_refcnt_read(m) == 1); 126 m->data_off = RTE_PKTMBUF_HEADROOM; 127 SFC_ASSERT(m->next == NULL); 128 SFC_ASSERT(m->nb_segs == 1); 129 m->port = port_id; 130 131 addr[i] = rte_pktmbuf_mtophys(m); 132 } 133 134 efx_rx_qpost(rxq->common, addr, rxq->buf_size, 135 RTE_DIM(objs), rxq->completed, added); 136 added += RTE_DIM(objs); 137 } while (--bulks > 0); 138 139 SFC_ASSERT(added != rxq->added); 140 rxq->added = added; 141 efx_rx_qpush(rxq->common, added, &rxq->pushed); 142 } 143 144 static uint64_t 145 sfc_efx_rx_desc_flags_to_offload_flags(const unsigned int desc_flags) 146 { 147 uint64_t mbuf_flags = 0; 148 149 switch (desc_flags & (EFX_PKT_IPV4 | EFX_CKSUM_IPV4)) { 150 case (EFX_PKT_IPV4 | EFX_CKSUM_IPV4): 151 mbuf_flags |= PKT_RX_IP_CKSUM_GOOD; 152 break; 153 case EFX_PKT_IPV4: 154 mbuf_flags |= PKT_RX_IP_CKSUM_BAD; 155 break; 156 default: 157 RTE_BUILD_BUG_ON(PKT_RX_IP_CKSUM_UNKNOWN != 0); 158 SFC_ASSERT((mbuf_flags & PKT_RX_IP_CKSUM_MASK) == 159 PKT_RX_IP_CKSUM_UNKNOWN); 160 break; 161 } 162 163 switch ((desc_flags & 164 (EFX_PKT_TCP | EFX_PKT_UDP | EFX_CKSUM_TCPUDP))) { 165 case (EFX_PKT_TCP | EFX_CKSUM_TCPUDP): 166 case (EFX_PKT_UDP | EFX_CKSUM_TCPUDP): 167 mbuf_flags |= PKT_RX_L4_CKSUM_GOOD; 168 break; 169 case EFX_PKT_TCP: 170 case EFX_PKT_UDP: 171 mbuf_flags |= PKT_RX_L4_CKSUM_BAD; 172 break; 173 default: 174 RTE_BUILD_BUG_ON(PKT_RX_L4_CKSUM_UNKNOWN != 0); 175 SFC_ASSERT((mbuf_flags & PKT_RX_L4_CKSUM_MASK) == 176 PKT_RX_L4_CKSUM_UNKNOWN); 177 break; 178 } 179 180 return mbuf_flags; 181 } 182 183 static uint32_t 184 sfc_efx_rx_desc_flags_to_packet_type(const unsigned int desc_flags) 185 { 186 return RTE_PTYPE_L2_ETHER | 187 ((desc_flags & EFX_PKT_IPV4) ? 188 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN : 0) | 189 ((desc_flags & EFX_PKT_IPV6) ? 190 RTE_PTYPE_L3_IPV6_EXT_UNKNOWN : 0) | 191 ((desc_flags & EFX_PKT_TCP) ? RTE_PTYPE_L4_TCP : 0) | 192 ((desc_flags & EFX_PKT_UDP) ? RTE_PTYPE_L4_UDP : 0); 193 } 194 195 static const uint32_t * 196 sfc_efx_supported_ptypes_get(void) 197 { 198 static const uint32_t ptypes[] = { 199 RTE_PTYPE_L2_ETHER, 200 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN, 201 RTE_PTYPE_L3_IPV6_EXT_UNKNOWN, 202 RTE_PTYPE_L4_TCP, 203 RTE_PTYPE_L4_UDP, 204 RTE_PTYPE_UNKNOWN 205 }; 206 207 return ptypes; 208 } 209 210 static void 211 sfc_efx_rx_set_rss_hash(struct sfc_efx_rxq *rxq, unsigned int flags, 212 struct rte_mbuf *m) 213 { 214 #if EFSYS_OPT_RX_SCALE 215 uint8_t *mbuf_data; 216 217 218 if ((rxq->flags & SFC_EFX_RXQ_FLAG_RSS_HASH) == 0) 219 return; 220 221 mbuf_data = rte_pktmbuf_mtod(m, uint8_t *); 222 223 if (flags & (EFX_PKT_IPV4 | EFX_PKT_IPV6)) { 224 m->hash.rss = efx_pseudo_hdr_hash_get(rxq->common, 225 EFX_RX_HASHALG_TOEPLITZ, 226 mbuf_data); 227 228 m->ol_flags |= PKT_RX_RSS_HASH; 229 } 230 #endif 231 } 232 233 static uint16_t 234 sfc_efx_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) 235 { 236 struct sfc_dp_rxq *dp_rxq = rx_queue; 237 struct sfc_efx_rxq *rxq = sfc_efx_rxq_by_dp_rxq(dp_rxq); 238 unsigned int completed; 239 unsigned int prefix_size = rxq->prefix_size; 240 unsigned int done_pkts = 0; 241 boolean_t discard_next = B_FALSE; 242 struct rte_mbuf *scatter_pkt = NULL; 243 244 if (unlikely((rxq->flags & SFC_EFX_RXQ_FLAG_RUNNING) == 0)) 245 return 0; 246 247 sfc_ev_qpoll(rxq->evq); 248 249 completed = rxq->completed; 250 while (completed != rxq->pending && done_pkts < nb_pkts) { 251 unsigned int id; 252 struct sfc_efx_rx_sw_desc *rxd; 253 struct rte_mbuf *m; 254 unsigned int seg_len; 255 unsigned int desc_flags; 256 257 id = completed++ & rxq->ptr_mask; 258 rxd = &rxq->sw_desc[id]; 259 m = rxd->mbuf; 260 desc_flags = rxd->flags; 261 262 if (discard_next) 263 goto discard; 264 265 if (desc_flags & (EFX_ADDR_MISMATCH | EFX_DISCARD)) 266 goto discard; 267 268 if (desc_flags & EFX_PKT_PREFIX_LEN) { 269 uint16_t tmp_size; 270 int rc __rte_unused; 271 272 rc = efx_pseudo_hdr_pkt_length_get(rxq->common, 273 rte_pktmbuf_mtod(m, uint8_t *), &tmp_size); 274 SFC_ASSERT(rc == 0); 275 seg_len = tmp_size; 276 } else { 277 seg_len = rxd->size - prefix_size; 278 } 279 280 rte_pktmbuf_data_len(m) = seg_len; 281 rte_pktmbuf_pkt_len(m) = seg_len; 282 283 if (scatter_pkt != NULL) { 284 if (rte_pktmbuf_chain(scatter_pkt, m) != 0) { 285 rte_pktmbuf_free(scatter_pkt); 286 goto discard; 287 } 288 /* The packet to deliver */ 289 m = scatter_pkt; 290 } 291 292 if (desc_flags & EFX_PKT_CONT) { 293 /* The packet is scattered, more fragments to come */ 294 scatter_pkt = m; 295 /* Further fragments have no prefix */ 296 prefix_size = 0; 297 continue; 298 } 299 300 /* Scattered packet is done */ 301 scatter_pkt = NULL; 302 /* The first fragment of the packet has prefix */ 303 prefix_size = rxq->prefix_size; 304 305 m->ol_flags = 306 sfc_efx_rx_desc_flags_to_offload_flags(desc_flags); 307 m->packet_type = 308 sfc_efx_rx_desc_flags_to_packet_type(desc_flags); 309 310 /* 311 * Extract RSS hash from the packet prefix and 312 * set the corresponding field (if needed and possible) 313 */ 314 sfc_efx_rx_set_rss_hash(rxq, desc_flags, m); 315 316 m->data_off += prefix_size; 317 318 *rx_pkts++ = m; 319 done_pkts++; 320 continue; 321 322 discard: 323 discard_next = ((desc_flags & EFX_PKT_CONT) != 0); 324 rte_mempool_put(rxq->refill_mb_pool, m); 325 rxd->mbuf = NULL; 326 } 327 328 /* pending is only moved when entire packet is received */ 329 SFC_ASSERT(scatter_pkt == NULL); 330 331 rxq->completed = completed; 332 333 sfc_efx_rx_qrefill(rxq); 334 335 return done_pkts; 336 } 337 338 static sfc_dp_rx_qdesc_npending_t sfc_efx_rx_qdesc_npending; 339 static unsigned int 340 sfc_efx_rx_qdesc_npending(struct sfc_dp_rxq *dp_rxq) 341 { 342 struct sfc_efx_rxq *rxq = sfc_efx_rxq_by_dp_rxq(dp_rxq); 343 344 if ((rxq->flags & SFC_EFX_RXQ_FLAG_RUNNING) == 0) 345 return 0; 346 347 sfc_ev_qpoll(rxq->evq); 348 349 return rxq->pending - rxq->completed; 350 } 351 352 struct sfc_rxq * 353 sfc_rxq_by_dp_rxq(const struct sfc_dp_rxq *dp_rxq) 354 { 355 const struct sfc_dp_queue *dpq = &dp_rxq->dpq; 356 struct rte_eth_dev *eth_dev; 357 struct sfc_adapter *sa; 358 struct sfc_rxq *rxq; 359 360 SFC_ASSERT(rte_eth_dev_is_valid_port(dpq->port_id)); 361 eth_dev = &rte_eth_devices[dpq->port_id]; 362 363 sa = eth_dev->data->dev_private; 364 365 SFC_ASSERT(dpq->queue_id < sa->rxq_count); 366 rxq = sa->rxq_info[dpq->queue_id].rxq; 367 368 SFC_ASSERT(rxq != NULL); 369 return rxq; 370 } 371 372 static sfc_dp_rx_qcreate_t sfc_efx_rx_qcreate; 373 static int 374 sfc_efx_rx_qcreate(uint16_t port_id, uint16_t queue_id, 375 const struct rte_pci_addr *pci_addr, int socket_id, 376 const struct sfc_dp_rx_qcreate_info *info, 377 struct sfc_dp_rxq **dp_rxqp) 378 { 379 struct sfc_efx_rxq *rxq; 380 int rc; 381 382 rc = ENOMEM; 383 rxq = rte_zmalloc_socket("sfc-efx-rxq", sizeof(*rxq), 384 RTE_CACHE_LINE_SIZE, socket_id); 385 if (rxq == NULL) 386 goto fail_rxq_alloc; 387 388 sfc_dp_queue_init(&rxq->dp.dpq, port_id, queue_id, pci_addr); 389 390 rc = ENOMEM; 391 rxq->sw_desc = rte_calloc_socket("sfc-efx-rxq-sw_desc", 392 info->rxq_entries, 393 sizeof(*rxq->sw_desc), 394 RTE_CACHE_LINE_SIZE, socket_id); 395 if (rxq->sw_desc == NULL) 396 goto fail_desc_alloc; 397 398 /* efx datapath is bound to efx control path */ 399 rxq->evq = sfc_rxq_by_dp_rxq(&rxq->dp)->evq; 400 if (info->flags & SFC_RXQ_FLAG_RSS_HASH) 401 rxq->flags |= SFC_EFX_RXQ_FLAG_RSS_HASH; 402 rxq->ptr_mask = info->rxq_entries - 1; 403 rxq->batch_max = info->batch_max; 404 rxq->prefix_size = info->prefix_size; 405 rxq->refill_threshold = info->refill_threshold; 406 rxq->buf_size = info->buf_size; 407 rxq->refill_mb_pool = info->refill_mb_pool; 408 409 *dp_rxqp = &rxq->dp; 410 return 0; 411 412 fail_desc_alloc: 413 rte_free(rxq); 414 415 fail_rxq_alloc: 416 return rc; 417 } 418 419 static sfc_dp_rx_qdestroy_t sfc_efx_rx_qdestroy; 420 static void 421 sfc_efx_rx_qdestroy(struct sfc_dp_rxq *dp_rxq) 422 { 423 struct sfc_efx_rxq *rxq = sfc_efx_rxq_by_dp_rxq(dp_rxq); 424 425 rte_free(rxq->sw_desc); 426 rte_free(rxq); 427 } 428 429 static sfc_dp_rx_qstart_t sfc_efx_rx_qstart; 430 static int 431 sfc_efx_rx_qstart(struct sfc_dp_rxq *dp_rxq, 432 __rte_unused unsigned int evq_read_ptr) 433 { 434 /* libefx-based datapath is specific to libefx-based PMD */ 435 struct sfc_efx_rxq *rxq = sfc_efx_rxq_by_dp_rxq(dp_rxq); 436 struct sfc_rxq *crxq = sfc_rxq_by_dp_rxq(dp_rxq); 437 438 rxq->common = crxq->common; 439 440 rxq->pending = rxq->completed = rxq->added = rxq->pushed = 0; 441 442 sfc_efx_rx_qrefill(rxq); 443 444 rxq->flags |= (SFC_EFX_RXQ_FLAG_STARTED | SFC_EFX_RXQ_FLAG_RUNNING); 445 446 return 0; 447 } 448 449 static sfc_dp_rx_qstop_t sfc_efx_rx_qstop; 450 static void 451 sfc_efx_rx_qstop(struct sfc_dp_rxq *dp_rxq, 452 __rte_unused unsigned int *evq_read_ptr) 453 { 454 struct sfc_efx_rxq *rxq = sfc_efx_rxq_by_dp_rxq(dp_rxq); 455 456 rxq->flags &= ~SFC_EFX_RXQ_FLAG_RUNNING; 457 458 /* libefx-based datapath is bound to libefx-based PMD and uses 459 * event queue structure directly. So, there is no necessity to 460 * return EvQ read pointer. 461 */ 462 } 463 464 static sfc_dp_rx_qpurge_t sfc_efx_rx_qpurge; 465 static void 466 sfc_efx_rx_qpurge(struct sfc_dp_rxq *dp_rxq) 467 { 468 struct sfc_efx_rxq *rxq = sfc_efx_rxq_by_dp_rxq(dp_rxq); 469 unsigned int i; 470 struct sfc_efx_rx_sw_desc *rxd; 471 472 for (i = rxq->completed; i != rxq->added; ++i) { 473 rxd = &rxq->sw_desc[i & rxq->ptr_mask]; 474 rte_mempool_put(rxq->refill_mb_pool, rxd->mbuf); 475 rxd->mbuf = NULL; 476 /* Packed stream relies on 0 in inactive SW desc. 477 * Rx queue stop is not performance critical, so 478 * there is no harm to do it always. 479 */ 480 rxd->flags = 0; 481 rxd->size = 0; 482 } 483 484 rxq->flags &= ~SFC_EFX_RXQ_FLAG_STARTED; 485 } 486 487 struct sfc_dp_rx sfc_efx_rx = { 488 .dp = { 489 .name = SFC_KVARG_DATAPATH_EFX, 490 .type = SFC_DP_RX, 491 .hw_fw_caps = 0, 492 }, 493 .features = SFC_DP_RX_FEAT_SCATTER, 494 .qcreate = sfc_efx_rx_qcreate, 495 .qdestroy = sfc_efx_rx_qdestroy, 496 .qstart = sfc_efx_rx_qstart, 497 .qstop = sfc_efx_rx_qstop, 498 .qpurge = sfc_efx_rx_qpurge, 499 .supported_ptypes_get = sfc_efx_supported_ptypes_get, 500 .qdesc_npending = sfc_efx_rx_qdesc_npending, 501 .pkt_burst = sfc_efx_recv_pkts, 502 }; 503 504 unsigned int 505 sfc_rx_qdesc_npending(struct sfc_adapter *sa, unsigned int sw_index) 506 { 507 struct sfc_rxq *rxq; 508 509 SFC_ASSERT(sw_index < sa->rxq_count); 510 rxq = sa->rxq_info[sw_index].rxq; 511 512 if (rxq == NULL || (rxq->state & SFC_RXQ_STARTED) == 0) 513 return 0; 514 515 return sa->dp_rx->qdesc_npending(rxq->dp); 516 } 517 518 int 519 sfc_rx_qdesc_done(struct sfc_dp_rxq *dp_rxq, unsigned int offset) 520 { 521 struct sfc_rxq *rxq = sfc_rxq_by_dp_rxq(dp_rxq); 522 523 return offset < rxq->evq->sa->dp_rx->qdesc_npending(dp_rxq); 524 } 525 526 static void 527 sfc_rx_qflush(struct sfc_adapter *sa, unsigned int sw_index) 528 { 529 struct sfc_rxq *rxq; 530 unsigned int retry_count; 531 unsigned int wait_count; 532 int rc; 533 534 rxq = sa->rxq_info[sw_index].rxq; 535 SFC_ASSERT(rxq->state & SFC_RXQ_STARTED); 536 537 /* 538 * Retry Rx queue flushing in the case of flush failed or 539 * timeout. In the worst case it can delay for 6 seconds. 540 */ 541 for (retry_count = 0; 542 ((rxq->state & SFC_RXQ_FLUSHED) == 0) && 543 (retry_count < SFC_RX_QFLUSH_ATTEMPTS); 544 ++retry_count) { 545 rc = efx_rx_qflush(rxq->common); 546 if (rc != 0) { 547 rxq->state |= (rc == EALREADY) ? 548 SFC_RXQ_FLUSHED : SFC_RXQ_FLUSH_FAILED; 549 break; 550 } 551 rxq->state &= ~SFC_RXQ_FLUSH_FAILED; 552 rxq->state |= SFC_RXQ_FLUSHING; 553 554 /* 555 * Wait for Rx queue flush done or failed event at least 556 * SFC_RX_QFLUSH_POLL_WAIT_MS milliseconds and not more 557 * than 2 seconds (SFC_RX_QFLUSH_POLL_WAIT_MS multiplied 558 * by SFC_RX_QFLUSH_POLL_ATTEMPTS). 559 */ 560 wait_count = 0; 561 do { 562 rte_delay_ms(SFC_RX_QFLUSH_POLL_WAIT_MS); 563 sfc_ev_qpoll(rxq->evq); 564 } while ((rxq->state & SFC_RXQ_FLUSHING) && 565 (wait_count++ < SFC_RX_QFLUSH_POLL_ATTEMPTS)); 566 567 if (rxq->state & SFC_RXQ_FLUSHING) 568 sfc_err(sa, "RxQ %u flush timed out", sw_index); 569 570 if (rxq->state & SFC_RXQ_FLUSH_FAILED) 571 sfc_err(sa, "RxQ %u flush failed", sw_index); 572 573 if (rxq->state & SFC_RXQ_FLUSHED) 574 sfc_info(sa, "RxQ %u flushed", sw_index); 575 } 576 577 sa->dp_rx->qpurge(rxq->dp); 578 } 579 580 static int 581 sfc_rx_default_rxq_set_filter(struct sfc_adapter *sa, struct sfc_rxq *rxq) 582 { 583 boolean_t rss = (sa->rss_channels > 0) ? B_TRUE : B_FALSE; 584 struct sfc_port *port = &sa->port; 585 int rc; 586 587 /* 588 * If promiscuous or all-multicast mode has been requested, setting 589 * filter for the default Rx queue might fail, in particular, while 590 * running over PCI function which is not a member of corresponding 591 * privilege groups; if this occurs, few iterations will be made to 592 * repeat this step without promiscuous and all-multicast flags set 593 */ 594 retry: 595 rc = efx_mac_filter_default_rxq_set(sa->nic, rxq->common, rss); 596 if (rc == 0) 597 return 0; 598 else if (rc != EOPNOTSUPP) 599 return rc; 600 601 if (port->promisc) { 602 sfc_warn(sa, "promiscuous mode has been requested, " 603 "but the HW rejects it"); 604 sfc_warn(sa, "promiscuous mode will be disabled"); 605 606 port->promisc = B_FALSE; 607 rc = sfc_set_rx_mode(sa); 608 if (rc != 0) 609 return rc; 610 611 goto retry; 612 } 613 614 if (port->allmulti) { 615 sfc_warn(sa, "all-multicast mode has been requested, " 616 "but the HW rejects it"); 617 sfc_warn(sa, "all-multicast mode will be disabled"); 618 619 port->allmulti = B_FALSE; 620 rc = sfc_set_rx_mode(sa); 621 if (rc != 0) 622 return rc; 623 624 goto retry; 625 } 626 627 return rc; 628 } 629 630 int 631 sfc_rx_qstart(struct sfc_adapter *sa, unsigned int sw_index) 632 { 633 struct sfc_rxq_info *rxq_info; 634 struct sfc_rxq *rxq; 635 struct sfc_evq *evq; 636 int rc; 637 638 sfc_log_init(sa, "sw_index=%u", sw_index); 639 640 SFC_ASSERT(sw_index < sa->rxq_count); 641 642 rxq_info = &sa->rxq_info[sw_index]; 643 rxq = rxq_info->rxq; 644 SFC_ASSERT(rxq->state == SFC_RXQ_INITIALIZED); 645 646 evq = rxq->evq; 647 648 rc = sfc_ev_qstart(evq, sfc_evq_index_by_rxq_sw_index(sa, sw_index)); 649 if (rc != 0) 650 goto fail_ev_qstart; 651 652 rc = efx_rx_qcreate(sa->nic, rxq->hw_index, 0, rxq_info->type, 653 &rxq->mem, rxq_info->entries, 654 0 /* not used on EF10 */, evq->common, 655 &rxq->common); 656 if (rc != 0) 657 goto fail_rx_qcreate; 658 659 efx_rx_qenable(rxq->common); 660 661 rc = sa->dp_rx->qstart(rxq->dp, evq->read_ptr); 662 if (rc != 0) 663 goto fail_dp_qstart; 664 665 rxq->state |= SFC_RXQ_STARTED; 666 667 if (sw_index == 0) { 668 rc = sfc_rx_default_rxq_set_filter(sa, rxq); 669 if (rc != 0) 670 goto fail_mac_filter_default_rxq_set; 671 } 672 673 /* It seems to be used by DPDK for debug purposes only ('rte_ether') */ 674 sa->eth_dev->data->rx_queue_state[sw_index] = 675 RTE_ETH_QUEUE_STATE_STARTED; 676 677 return 0; 678 679 fail_mac_filter_default_rxq_set: 680 sa->dp_rx->qstop(rxq->dp, &rxq->evq->read_ptr); 681 682 fail_dp_qstart: 683 sfc_rx_qflush(sa, sw_index); 684 685 fail_rx_qcreate: 686 sfc_ev_qstop(evq); 687 688 fail_ev_qstart: 689 return rc; 690 } 691 692 void 693 sfc_rx_qstop(struct sfc_adapter *sa, unsigned int sw_index) 694 { 695 struct sfc_rxq_info *rxq_info; 696 struct sfc_rxq *rxq; 697 698 sfc_log_init(sa, "sw_index=%u", sw_index); 699 700 SFC_ASSERT(sw_index < sa->rxq_count); 701 702 rxq_info = &sa->rxq_info[sw_index]; 703 rxq = rxq_info->rxq; 704 705 if (rxq->state == SFC_RXQ_INITIALIZED) 706 return; 707 SFC_ASSERT(rxq->state & SFC_RXQ_STARTED); 708 709 /* It seems to be used by DPDK for debug purposes only ('rte_ether') */ 710 sa->eth_dev->data->rx_queue_state[sw_index] = 711 RTE_ETH_QUEUE_STATE_STOPPED; 712 713 sa->dp_rx->qstop(rxq->dp, &rxq->evq->read_ptr); 714 715 if (sw_index == 0) 716 efx_mac_filter_default_rxq_clear(sa->nic); 717 718 sfc_rx_qflush(sa, sw_index); 719 720 rxq->state = SFC_RXQ_INITIALIZED; 721 722 efx_rx_qdestroy(rxq->common); 723 724 sfc_ev_qstop(rxq->evq); 725 } 726 727 static int 728 sfc_rx_qcheck_conf(struct sfc_adapter *sa, uint16_t nb_rx_desc, 729 const struct rte_eth_rxconf *rx_conf) 730 { 731 const uint16_t rx_free_thresh_max = EFX_RXQ_LIMIT(nb_rx_desc); 732 int rc = 0; 733 734 if (rx_conf->rx_thresh.pthresh != 0 || 735 rx_conf->rx_thresh.hthresh != 0 || 736 rx_conf->rx_thresh.wthresh != 0) { 737 sfc_err(sa, 738 "RxQ prefetch/host/writeback thresholds are not supported"); 739 rc = EINVAL; 740 } 741 742 if (rx_conf->rx_free_thresh > rx_free_thresh_max) { 743 sfc_err(sa, 744 "RxQ free threshold too large: %u vs maximum %u", 745 rx_conf->rx_free_thresh, rx_free_thresh_max); 746 rc = EINVAL; 747 } 748 749 if (rx_conf->rx_drop_en == 0) { 750 sfc_err(sa, "RxQ drop disable is not supported"); 751 rc = EINVAL; 752 } 753 754 return rc; 755 } 756 757 static unsigned int 758 sfc_rx_mbuf_data_alignment(struct rte_mempool *mb_pool) 759 { 760 uint32_t data_off; 761 uint32_t order; 762 763 /* The mbuf object itself is always cache line aligned */ 764 order = rte_bsf32(RTE_CACHE_LINE_SIZE); 765 766 /* Data offset from mbuf object start */ 767 data_off = sizeof(struct rte_mbuf) + rte_pktmbuf_priv_size(mb_pool) + 768 RTE_PKTMBUF_HEADROOM; 769 770 order = MIN(order, rte_bsf32(data_off)); 771 772 return 1u << (order - 1); 773 } 774 775 static uint16_t 776 sfc_rx_mb_pool_buf_size(struct sfc_adapter *sa, struct rte_mempool *mb_pool) 777 { 778 const efx_nic_cfg_t *encp = efx_nic_cfg_get(sa->nic); 779 const uint32_t nic_align_start = MAX(1, encp->enc_rx_buf_align_start); 780 const uint32_t nic_align_end = MAX(1, encp->enc_rx_buf_align_end); 781 uint16_t buf_size; 782 unsigned int buf_aligned; 783 unsigned int start_alignment; 784 unsigned int end_padding_alignment; 785 786 /* Below it is assumed that both alignments are power of 2 */ 787 SFC_ASSERT(rte_is_power_of_2(nic_align_start)); 788 SFC_ASSERT(rte_is_power_of_2(nic_align_end)); 789 790 /* 791 * mbuf is always cache line aligned, double-check 792 * that it meets rx buffer start alignment requirements. 793 */ 794 795 /* Start from mbuf pool data room size */ 796 buf_size = rte_pktmbuf_data_room_size(mb_pool); 797 798 /* Remove headroom */ 799 if (buf_size <= RTE_PKTMBUF_HEADROOM) { 800 sfc_err(sa, 801 "RxQ mbuf pool %s object data room size %u is smaller than headroom %u", 802 mb_pool->name, buf_size, RTE_PKTMBUF_HEADROOM); 803 return 0; 804 } 805 buf_size -= RTE_PKTMBUF_HEADROOM; 806 807 /* Calculate guaranteed data start alignment */ 808 buf_aligned = sfc_rx_mbuf_data_alignment(mb_pool); 809 810 /* Reserve space for start alignment */ 811 if (buf_aligned < nic_align_start) { 812 start_alignment = nic_align_start - buf_aligned; 813 if (buf_size <= start_alignment) { 814 sfc_err(sa, 815 "RxQ mbuf pool %s object data room size %u is insufficient for headroom %u and buffer start alignment %u required by NIC", 816 mb_pool->name, 817 rte_pktmbuf_data_room_size(mb_pool), 818 RTE_PKTMBUF_HEADROOM, start_alignment); 819 return 0; 820 } 821 buf_aligned = nic_align_start; 822 buf_size -= start_alignment; 823 } else { 824 start_alignment = 0; 825 } 826 827 /* Make sure that end padding does not write beyond the buffer */ 828 if (buf_aligned < nic_align_end) { 829 /* 830 * Estimate space which can be lost. If guarnteed buffer 831 * size is odd, lost space is (nic_align_end - 1). More 832 * accurate formula is below. 833 */ 834 end_padding_alignment = nic_align_end - 835 MIN(buf_aligned, 1u << (rte_bsf32(buf_size) - 1)); 836 if (buf_size <= end_padding_alignment) { 837 sfc_err(sa, 838 "RxQ mbuf pool %s object data room size %u is insufficient for headroom %u, buffer start alignment %u and end padding alignment %u required by NIC", 839 mb_pool->name, 840 rte_pktmbuf_data_room_size(mb_pool), 841 RTE_PKTMBUF_HEADROOM, start_alignment, 842 end_padding_alignment); 843 return 0; 844 } 845 buf_size -= end_padding_alignment; 846 } else { 847 /* 848 * Start is aligned the same or better than end, 849 * just align length. 850 */ 851 buf_size = P2ALIGN(buf_size, nic_align_end); 852 } 853 854 return buf_size; 855 } 856 857 int 858 sfc_rx_qinit(struct sfc_adapter *sa, unsigned int sw_index, 859 uint16_t nb_rx_desc, unsigned int socket_id, 860 const struct rte_eth_rxconf *rx_conf, 861 struct rte_mempool *mb_pool) 862 { 863 const efx_nic_cfg_t *encp = efx_nic_cfg_get(sa->nic); 864 int rc; 865 uint16_t buf_size; 866 struct sfc_rxq_info *rxq_info; 867 struct sfc_evq *evq; 868 struct sfc_rxq *rxq; 869 struct sfc_dp_rx_qcreate_info info; 870 871 rc = sfc_rx_qcheck_conf(sa, nb_rx_desc, rx_conf); 872 if (rc != 0) 873 goto fail_bad_conf; 874 875 buf_size = sfc_rx_mb_pool_buf_size(sa, mb_pool); 876 if (buf_size == 0) { 877 sfc_err(sa, "RxQ %u mbuf pool object size is too small", 878 sw_index); 879 rc = EINVAL; 880 goto fail_bad_conf; 881 } 882 883 if ((buf_size < sa->port.pdu + encp->enc_rx_prefix_size) && 884 !sa->eth_dev->data->dev_conf.rxmode.enable_scatter) { 885 sfc_err(sa, "Rx scatter is disabled and RxQ %u mbuf pool " 886 "object size is too small", sw_index); 887 sfc_err(sa, "RxQ %u calculated Rx buffer size is %u vs " 888 "PDU size %u plus Rx prefix %u bytes", 889 sw_index, buf_size, (unsigned int)sa->port.pdu, 890 encp->enc_rx_prefix_size); 891 rc = EINVAL; 892 goto fail_bad_conf; 893 } 894 895 SFC_ASSERT(sw_index < sa->rxq_count); 896 rxq_info = &sa->rxq_info[sw_index]; 897 898 SFC_ASSERT(nb_rx_desc <= rxq_info->max_entries); 899 rxq_info->entries = nb_rx_desc; 900 rxq_info->type = 901 sa->eth_dev->data->dev_conf.rxmode.enable_scatter ? 902 EFX_RXQ_TYPE_SCATTER : EFX_RXQ_TYPE_DEFAULT; 903 904 rc = sfc_ev_qinit(sa, SFC_EVQ_TYPE_RX, sw_index, 905 rxq_info->entries, socket_id, &evq); 906 if (rc != 0) 907 goto fail_ev_qinit; 908 909 rc = ENOMEM; 910 rxq = rte_zmalloc_socket("sfc-rxq", sizeof(*rxq), RTE_CACHE_LINE_SIZE, 911 socket_id); 912 if (rxq == NULL) 913 goto fail_rxq_alloc; 914 915 rxq_info->rxq = rxq; 916 917 rxq->evq = evq; 918 rxq->hw_index = sw_index; 919 rxq->refill_threshold = 920 RTE_MAX(rx_conf->rx_free_thresh, SFC_RX_REFILL_BULK); 921 rxq->refill_mb_pool = mb_pool; 922 923 rc = sfc_dma_alloc(sa, "rxq", sw_index, EFX_RXQ_SIZE(rxq_info->entries), 924 socket_id, &rxq->mem); 925 if (rc != 0) 926 goto fail_dma_alloc; 927 928 memset(&info, 0, sizeof(info)); 929 info.refill_mb_pool = rxq->refill_mb_pool; 930 info.refill_threshold = rxq->refill_threshold; 931 info.buf_size = buf_size; 932 info.batch_max = encp->enc_rx_batch_max; 933 info.prefix_size = encp->enc_rx_prefix_size; 934 935 #if EFSYS_OPT_RX_SCALE 936 if (sa->hash_support == EFX_RX_HASH_AVAILABLE && sa->rss_channels > 0) 937 info.flags |= SFC_RXQ_FLAG_RSS_HASH; 938 #endif 939 940 info.rxq_entries = rxq_info->entries; 941 info.rxq_hw_ring = rxq->mem.esm_base; 942 info.evq_entries = rxq_info->entries; 943 info.evq_hw_ring = evq->mem.esm_base; 944 info.hw_index = rxq->hw_index; 945 info.mem_bar = sa->mem_bar.esb_base; 946 947 rc = sa->dp_rx->qcreate(sa->eth_dev->data->port_id, sw_index, 948 &RTE_ETH_DEV_TO_PCI(sa->eth_dev)->addr, 949 socket_id, &info, &rxq->dp); 950 if (rc != 0) 951 goto fail_dp_rx_qcreate; 952 953 evq->dp_rxq = rxq->dp; 954 955 rxq->state = SFC_RXQ_INITIALIZED; 956 957 rxq_info->deferred_start = (rx_conf->rx_deferred_start != 0); 958 959 return 0; 960 961 fail_dp_rx_qcreate: 962 sfc_dma_free(sa, &rxq->mem); 963 964 fail_dma_alloc: 965 rxq_info->rxq = NULL; 966 rte_free(rxq); 967 968 fail_rxq_alloc: 969 sfc_ev_qfini(evq); 970 971 fail_ev_qinit: 972 rxq_info->entries = 0; 973 974 fail_bad_conf: 975 sfc_log_init(sa, "failed %d", rc); 976 return rc; 977 } 978 979 void 980 sfc_rx_qfini(struct sfc_adapter *sa, unsigned int sw_index) 981 { 982 struct sfc_rxq_info *rxq_info; 983 struct sfc_rxq *rxq; 984 985 SFC_ASSERT(sw_index < sa->rxq_count); 986 987 rxq_info = &sa->rxq_info[sw_index]; 988 989 rxq = rxq_info->rxq; 990 SFC_ASSERT(rxq->state == SFC_RXQ_INITIALIZED); 991 992 sa->dp_rx->qdestroy(rxq->dp); 993 rxq->dp = NULL; 994 995 rxq_info->rxq = NULL; 996 rxq_info->entries = 0; 997 998 sfc_dma_free(sa, &rxq->mem); 999 1000 sfc_ev_qfini(rxq->evq); 1001 rxq->evq = NULL; 1002 1003 rte_free(rxq); 1004 } 1005 1006 #if EFSYS_OPT_RX_SCALE 1007 efx_rx_hash_type_t 1008 sfc_rte_to_efx_hash_type(uint64_t rss_hf) 1009 { 1010 efx_rx_hash_type_t efx_hash_types = 0; 1011 1012 if ((rss_hf & (ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | 1013 ETH_RSS_NONFRAG_IPV4_OTHER)) != 0) 1014 efx_hash_types |= EFX_RX_HASH_IPV4; 1015 1016 if ((rss_hf & ETH_RSS_NONFRAG_IPV4_TCP) != 0) 1017 efx_hash_types |= EFX_RX_HASH_TCPIPV4; 1018 1019 if ((rss_hf & (ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | 1020 ETH_RSS_NONFRAG_IPV6_OTHER | ETH_RSS_IPV6_EX)) != 0) 1021 efx_hash_types |= EFX_RX_HASH_IPV6; 1022 1023 if ((rss_hf & (ETH_RSS_NONFRAG_IPV6_TCP | ETH_RSS_IPV6_TCP_EX)) != 0) 1024 efx_hash_types |= EFX_RX_HASH_TCPIPV6; 1025 1026 return efx_hash_types; 1027 } 1028 1029 uint64_t 1030 sfc_efx_to_rte_hash_type(efx_rx_hash_type_t efx_hash_types) 1031 { 1032 uint64_t rss_hf = 0; 1033 1034 if ((efx_hash_types & EFX_RX_HASH_IPV4) != 0) 1035 rss_hf |= (ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | 1036 ETH_RSS_NONFRAG_IPV4_OTHER); 1037 1038 if ((efx_hash_types & EFX_RX_HASH_TCPIPV4) != 0) 1039 rss_hf |= ETH_RSS_NONFRAG_IPV4_TCP; 1040 1041 if ((efx_hash_types & EFX_RX_HASH_IPV6) != 0) 1042 rss_hf |= (ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | 1043 ETH_RSS_NONFRAG_IPV6_OTHER | ETH_RSS_IPV6_EX); 1044 1045 if ((efx_hash_types & EFX_RX_HASH_TCPIPV6) != 0) 1046 rss_hf |= (ETH_RSS_NONFRAG_IPV6_TCP | ETH_RSS_IPV6_TCP_EX); 1047 1048 return rss_hf; 1049 } 1050 #endif 1051 1052 static int 1053 sfc_rx_rss_config(struct sfc_adapter *sa) 1054 { 1055 int rc = 0; 1056 1057 #if EFSYS_OPT_RX_SCALE 1058 if (sa->rss_channels > 0) { 1059 rc = efx_rx_scale_mode_set(sa->nic, EFX_RX_HASHALG_TOEPLITZ, 1060 sa->rss_hash_types, B_TRUE); 1061 if (rc != 0) 1062 goto finish; 1063 1064 rc = efx_rx_scale_key_set(sa->nic, sa->rss_key, 1065 sizeof(sa->rss_key)); 1066 if (rc != 0) 1067 goto finish; 1068 1069 rc = efx_rx_scale_tbl_set(sa->nic, sa->rss_tbl, 1070 sizeof(sa->rss_tbl)); 1071 } 1072 1073 finish: 1074 #endif 1075 return rc; 1076 } 1077 1078 int 1079 sfc_rx_start(struct sfc_adapter *sa) 1080 { 1081 unsigned int sw_index; 1082 int rc; 1083 1084 sfc_log_init(sa, "rxq_count=%u", sa->rxq_count); 1085 1086 rc = efx_rx_init(sa->nic); 1087 if (rc != 0) 1088 goto fail_rx_init; 1089 1090 rc = sfc_rx_rss_config(sa); 1091 if (rc != 0) 1092 goto fail_rss_config; 1093 1094 for (sw_index = 0; sw_index < sa->rxq_count; ++sw_index) { 1095 if ((!sa->rxq_info[sw_index].deferred_start || 1096 sa->rxq_info[sw_index].deferred_started)) { 1097 rc = sfc_rx_qstart(sa, sw_index); 1098 if (rc != 0) 1099 goto fail_rx_qstart; 1100 } 1101 } 1102 1103 return 0; 1104 1105 fail_rx_qstart: 1106 while (sw_index-- > 0) 1107 sfc_rx_qstop(sa, sw_index); 1108 1109 fail_rss_config: 1110 efx_rx_fini(sa->nic); 1111 1112 fail_rx_init: 1113 sfc_log_init(sa, "failed %d", rc); 1114 return rc; 1115 } 1116 1117 void 1118 sfc_rx_stop(struct sfc_adapter *sa) 1119 { 1120 unsigned int sw_index; 1121 1122 sfc_log_init(sa, "rxq_count=%u", sa->rxq_count); 1123 1124 sw_index = sa->rxq_count; 1125 while (sw_index-- > 0) { 1126 if (sa->rxq_info[sw_index].rxq != NULL) 1127 sfc_rx_qstop(sa, sw_index); 1128 } 1129 1130 efx_rx_fini(sa->nic); 1131 } 1132 1133 static int 1134 sfc_rx_qinit_info(struct sfc_adapter *sa, unsigned int sw_index) 1135 { 1136 struct sfc_rxq_info *rxq_info = &sa->rxq_info[sw_index]; 1137 unsigned int max_entries; 1138 1139 max_entries = EFX_RXQ_MAXNDESCS; 1140 SFC_ASSERT(rte_is_power_of_2(max_entries)); 1141 1142 rxq_info->max_entries = max_entries; 1143 1144 return 0; 1145 } 1146 1147 static int 1148 sfc_rx_check_mode(struct sfc_adapter *sa, struct rte_eth_rxmode *rxmode) 1149 { 1150 int rc = 0; 1151 1152 switch (rxmode->mq_mode) { 1153 case ETH_MQ_RX_NONE: 1154 /* No special checks are required */ 1155 break; 1156 #if EFSYS_OPT_RX_SCALE 1157 case ETH_MQ_RX_RSS: 1158 if (sa->rss_support == EFX_RX_SCALE_UNAVAILABLE) { 1159 sfc_err(sa, "RSS is not available"); 1160 rc = EINVAL; 1161 } 1162 break; 1163 #endif 1164 default: 1165 sfc_err(sa, "Rx multi-queue mode %u not supported", 1166 rxmode->mq_mode); 1167 rc = EINVAL; 1168 } 1169 1170 if (rxmode->header_split) { 1171 sfc_err(sa, "Header split on Rx not supported"); 1172 rc = EINVAL; 1173 } 1174 1175 if (rxmode->hw_vlan_filter) { 1176 sfc_err(sa, "HW VLAN filtering not supported"); 1177 rc = EINVAL; 1178 } 1179 1180 if (rxmode->hw_vlan_strip) { 1181 sfc_err(sa, "HW VLAN stripping not supported"); 1182 rc = EINVAL; 1183 } 1184 1185 if (rxmode->hw_vlan_extend) { 1186 sfc_err(sa, 1187 "Q-in-Q HW VLAN stripping not supported"); 1188 rc = EINVAL; 1189 } 1190 1191 if (!rxmode->hw_strip_crc) { 1192 sfc_warn(sa, 1193 "FCS stripping control not supported - always stripped"); 1194 rxmode->hw_strip_crc = 1; 1195 } 1196 1197 if (rxmode->enable_scatter && 1198 (~sa->dp_rx->features & SFC_DP_RX_FEAT_SCATTER)) { 1199 sfc_err(sa, "Rx scatter not supported by %s datapath", 1200 sa->dp_rx->dp.name); 1201 rc = EINVAL; 1202 } 1203 1204 if (rxmode->enable_lro) { 1205 sfc_err(sa, "LRO not supported"); 1206 rc = EINVAL; 1207 } 1208 1209 return rc; 1210 } 1211 1212 /** 1213 * Destroy excess queues that are no longer needed after reconfiguration 1214 * or complete close. 1215 */ 1216 static void 1217 sfc_rx_fini_queues(struct sfc_adapter *sa, unsigned int nb_rx_queues) 1218 { 1219 int sw_index; 1220 1221 SFC_ASSERT(nb_rx_queues <= sa->rxq_count); 1222 1223 sw_index = sa->rxq_count; 1224 while (--sw_index >= (int)nb_rx_queues) { 1225 if (sa->rxq_info[sw_index].rxq != NULL) 1226 sfc_rx_qfini(sa, sw_index); 1227 } 1228 1229 sa->rxq_count = nb_rx_queues; 1230 } 1231 1232 /** 1233 * Initialize Rx subsystem. 1234 * 1235 * Called at device (re)configuration stage when number of receive queues is 1236 * specified together with other device level receive configuration. 1237 * 1238 * It should be used to allocate NUMA-unaware resources. 1239 */ 1240 int 1241 sfc_rx_configure(struct sfc_adapter *sa) 1242 { 1243 struct rte_eth_conf *dev_conf = &sa->eth_dev->data->dev_conf; 1244 const unsigned int nb_rx_queues = sa->eth_dev->data->nb_rx_queues; 1245 unsigned int sw_index; 1246 int rc; 1247 1248 sfc_log_init(sa, "nb_rx_queues=%u (old %u)", 1249 nb_rx_queues, sa->rxq_count); 1250 1251 rc = sfc_rx_check_mode(sa, &dev_conf->rxmode); 1252 if (rc != 0) 1253 goto fail_check_mode; 1254 1255 if (nb_rx_queues == sa->rxq_count) 1256 goto done; 1257 1258 if (sa->rxq_info == NULL) { 1259 rc = ENOMEM; 1260 sa->rxq_info = rte_calloc_socket("sfc-rxqs", nb_rx_queues, 1261 sizeof(sa->rxq_info[0]), 0, 1262 sa->socket_id); 1263 if (sa->rxq_info == NULL) 1264 goto fail_rxqs_alloc; 1265 } else { 1266 struct sfc_rxq_info *new_rxq_info; 1267 1268 if (nb_rx_queues < sa->rxq_count) 1269 sfc_rx_fini_queues(sa, nb_rx_queues); 1270 1271 rc = ENOMEM; 1272 new_rxq_info = 1273 rte_realloc(sa->rxq_info, 1274 nb_rx_queues * sizeof(sa->rxq_info[0]), 0); 1275 if (new_rxq_info == NULL && nb_rx_queues > 0) 1276 goto fail_rxqs_realloc; 1277 1278 sa->rxq_info = new_rxq_info; 1279 if (nb_rx_queues > sa->rxq_count) 1280 memset(&sa->rxq_info[sa->rxq_count], 0, 1281 (nb_rx_queues - sa->rxq_count) * 1282 sizeof(sa->rxq_info[0])); 1283 } 1284 1285 while (sa->rxq_count < nb_rx_queues) { 1286 rc = sfc_rx_qinit_info(sa, sa->rxq_count); 1287 if (rc != 0) 1288 goto fail_rx_qinit_info; 1289 1290 sa->rxq_count++; 1291 } 1292 1293 #if EFSYS_OPT_RX_SCALE 1294 sa->rss_channels = (dev_conf->rxmode.mq_mode == ETH_MQ_RX_RSS) ? 1295 MIN(sa->rxq_count, EFX_MAXRSS) : 0; 1296 1297 if (sa->rss_channels > 0) { 1298 for (sw_index = 0; sw_index < EFX_RSS_TBL_SIZE; ++sw_index) 1299 sa->rss_tbl[sw_index] = sw_index % sa->rss_channels; 1300 } 1301 #endif 1302 1303 done: 1304 return 0; 1305 1306 fail_rx_qinit_info: 1307 fail_rxqs_realloc: 1308 fail_rxqs_alloc: 1309 sfc_rx_close(sa); 1310 1311 fail_check_mode: 1312 sfc_log_init(sa, "failed %d", rc); 1313 return rc; 1314 } 1315 1316 /** 1317 * Shutdown Rx subsystem. 1318 * 1319 * Called at device close stage, for example, before device shutdown. 1320 */ 1321 void 1322 sfc_rx_close(struct sfc_adapter *sa) 1323 { 1324 sfc_rx_fini_queues(sa, 0); 1325 1326 sa->rss_channels = 0; 1327 1328 rte_free(sa->rxq_info); 1329 sa->rxq_info = NULL; 1330 } 1331