1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2015 Intel Corporation 3 */ 4 5 #include <sys/queue.h> 6 7 #include <stdio.h> 8 #include <stdlib.h> 9 #include <string.h> 10 #include <errno.h> 11 #include <stdint.h> 12 #include <stdarg.h> 13 #include <unistd.h> 14 #include <inttypes.h> 15 16 #include <rte_byteorder.h> 17 #include <rte_common.h> 18 #include <rte_cycles.h> 19 #include <rte_log.h> 20 #include <rte_debug.h> 21 #include <rte_interrupts.h> 22 #include <rte_pci.h> 23 #include <rte_memory.h> 24 #include <rte_memzone.h> 25 #include <rte_launch.h> 26 #include <rte_eal.h> 27 #include <rte_per_lcore.h> 28 #include <rte_lcore.h> 29 #include <rte_atomic.h> 30 #include <rte_branch_prediction.h> 31 #include <rte_mempool.h> 32 #include <rte_malloc.h> 33 #include <rte_mbuf.h> 34 #include <rte_ether.h> 35 #include <rte_ethdev_driver.h> 36 #include <rte_prefetch.h> 37 #include <rte_ip.h> 38 #include <rte_udp.h> 39 #include <rte_tcp.h> 40 #include <rte_sctp.h> 41 #include <rte_string_fns.h> 42 #include <rte_errno.h> 43 #include <rte_net.h> 44 45 #include "base/vmxnet3_defs.h" 46 #include "vmxnet3_ring.h" 47 48 #include "vmxnet3_logs.h" 49 #include "vmxnet3_ethdev.h" 50 51 #define VMXNET3_TX_OFFLOAD_MASK ( \ 52 PKT_TX_VLAN_PKT | \ 53 PKT_TX_L4_MASK | \ 54 PKT_TX_TCP_SEG) 55 56 #define VMXNET3_TX_OFFLOAD_NOTSUP_MASK \ 57 (PKT_TX_OFFLOAD_MASK ^ VMXNET3_TX_OFFLOAD_MASK) 58 59 static const uint32_t rxprod_reg[2] = {VMXNET3_REG_RXPROD, VMXNET3_REG_RXPROD2}; 60 61 static int vmxnet3_post_rx_bufs(vmxnet3_rx_queue_t*, uint8_t); 62 static void vmxnet3_tq_tx_complete(vmxnet3_tx_queue_t *); 63 #ifdef RTE_LIBRTE_VMXNET3_DEBUG_DRIVER_NOT_USED 64 static void vmxnet3_rxq_dump(struct vmxnet3_rx_queue *); 65 static void vmxnet3_txq_dump(struct vmxnet3_tx_queue *); 66 #endif 67 68 #ifdef RTE_LIBRTE_VMXNET3_DEBUG_DRIVER_NOT_USED 69 static void 70 vmxnet3_rxq_dump(struct vmxnet3_rx_queue *rxq) 71 { 72 uint32_t avail = 0; 73 74 if (rxq == NULL) 75 return; 76 77 PMD_RX_LOG(DEBUG, 78 "RXQ: cmd0 base : %p cmd1 base : %p comp ring base : %p.", 79 rxq->cmd_ring[0].base, rxq->cmd_ring[1].base, rxq->comp_ring.base); 80 PMD_RX_LOG(DEBUG, 81 "RXQ: cmd0 basePA : 0x%lx cmd1 basePA : 0x%lx comp ring basePA : 0x%lx.", 82 (unsigned long)rxq->cmd_ring[0].basePA, 83 (unsigned long)rxq->cmd_ring[1].basePA, 84 (unsigned long)rxq->comp_ring.basePA); 85 86 avail = vmxnet3_cmd_ring_desc_avail(&rxq->cmd_ring[0]); 87 PMD_RX_LOG(DEBUG, 88 "RXQ:cmd0: size=%u; free=%u; next2proc=%u; queued=%u", 89 (uint32_t)rxq->cmd_ring[0].size, avail, 90 rxq->comp_ring.next2proc, 91 rxq->cmd_ring[0].size - avail); 92 93 avail = vmxnet3_cmd_ring_desc_avail(&rxq->cmd_ring[1]); 94 PMD_RX_LOG(DEBUG, "RXQ:cmd1 size=%u; free=%u; next2proc=%u; queued=%u", 95 (uint32_t)rxq->cmd_ring[1].size, avail, rxq->comp_ring.next2proc, 96 rxq->cmd_ring[1].size - avail); 97 98 } 99 100 static void 101 vmxnet3_txq_dump(struct vmxnet3_tx_queue *txq) 102 { 103 uint32_t avail = 0; 104 105 if (txq == NULL) 106 return; 107 108 PMD_TX_LOG(DEBUG, "TXQ: cmd base : %p comp ring base : %p data ring base : %p.", 109 txq->cmd_ring.base, txq->comp_ring.base, txq->data_ring.base); 110 PMD_TX_LOG(DEBUG, "TXQ: cmd basePA : 0x%lx comp ring basePA : 0x%lx data ring basePA : 0x%lx.", 111 (unsigned long)txq->cmd_ring.basePA, 112 (unsigned long)txq->comp_ring.basePA, 113 (unsigned long)txq->data_ring.basePA); 114 115 avail = vmxnet3_cmd_ring_desc_avail(&txq->cmd_ring); 116 PMD_TX_LOG(DEBUG, "TXQ: size=%u; free=%u; next2proc=%u; queued=%u", 117 (uint32_t)txq->cmd_ring.size, avail, 118 txq->comp_ring.next2proc, txq->cmd_ring.size - avail); 119 } 120 #endif 121 122 static void 123 vmxnet3_tx_cmd_ring_release_mbufs(vmxnet3_cmd_ring_t *ring) 124 { 125 while (ring->next2comp != ring->next2fill) { 126 /* No need to worry about desc ownership, device is quiesced by now. */ 127 vmxnet3_buf_info_t *buf_info = ring->buf_info + ring->next2comp; 128 129 if (buf_info->m) { 130 rte_pktmbuf_free(buf_info->m); 131 buf_info->m = NULL; 132 buf_info->bufPA = 0; 133 buf_info->len = 0; 134 } 135 vmxnet3_cmd_ring_adv_next2comp(ring); 136 } 137 } 138 139 static void 140 vmxnet3_rx_cmd_ring_release_mbufs(vmxnet3_cmd_ring_t *ring) 141 { 142 uint32_t i; 143 144 for (i = 0; i < ring->size; i++) { 145 /* No need to worry about desc ownership, device is quiesced by now. */ 146 vmxnet3_buf_info_t *buf_info = &ring->buf_info[i]; 147 148 if (buf_info->m) { 149 rte_pktmbuf_free_seg(buf_info->m); 150 buf_info->m = NULL; 151 buf_info->bufPA = 0; 152 buf_info->len = 0; 153 } 154 vmxnet3_cmd_ring_adv_next2comp(ring); 155 } 156 } 157 158 static void 159 vmxnet3_cmd_ring_release(vmxnet3_cmd_ring_t *ring) 160 { 161 rte_free(ring->buf_info); 162 ring->buf_info = NULL; 163 } 164 165 void 166 vmxnet3_dev_tx_queue_release(void *txq) 167 { 168 vmxnet3_tx_queue_t *tq = txq; 169 170 if (tq != NULL) { 171 /* Release mbufs */ 172 vmxnet3_tx_cmd_ring_release_mbufs(&tq->cmd_ring); 173 /* Release the cmd_ring */ 174 vmxnet3_cmd_ring_release(&tq->cmd_ring); 175 /* Release the memzone */ 176 rte_memzone_free(tq->mz); 177 /* Release the queue */ 178 rte_free(tq); 179 } 180 } 181 182 void 183 vmxnet3_dev_rx_queue_release(void *rxq) 184 { 185 int i; 186 vmxnet3_rx_queue_t *rq = rxq; 187 188 if (rq != NULL) { 189 /* Release mbufs */ 190 for (i = 0; i < VMXNET3_RX_CMDRING_SIZE; i++) 191 vmxnet3_rx_cmd_ring_release_mbufs(&rq->cmd_ring[i]); 192 193 /* Release both the cmd_rings */ 194 for (i = 0; i < VMXNET3_RX_CMDRING_SIZE; i++) 195 vmxnet3_cmd_ring_release(&rq->cmd_ring[i]); 196 197 /* Release the memzone */ 198 rte_memzone_free(rq->mz); 199 200 /* Release the queue */ 201 rte_free(rq); 202 } 203 } 204 205 static void 206 vmxnet3_dev_tx_queue_reset(void *txq) 207 { 208 vmxnet3_tx_queue_t *tq = txq; 209 struct vmxnet3_cmd_ring *ring = &tq->cmd_ring; 210 struct vmxnet3_comp_ring *comp_ring = &tq->comp_ring; 211 struct vmxnet3_data_ring *data_ring = &tq->data_ring; 212 int size; 213 214 if (tq != NULL) { 215 /* Release the cmd_ring mbufs */ 216 vmxnet3_tx_cmd_ring_release_mbufs(&tq->cmd_ring); 217 } 218 219 /* Tx vmxnet rings structure initialization*/ 220 ring->next2fill = 0; 221 ring->next2comp = 0; 222 ring->gen = VMXNET3_INIT_GEN; 223 comp_ring->next2proc = 0; 224 comp_ring->gen = VMXNET3_INIT_GEN; 225 226 size = sizeof(struct Vmxnet3_TxDesc) * ring->size; 227 size += sizeof(struct Vmxnet3_TxCompDesc) * comp_ring->size; 228 size += tq->txdata_desc_size * data_ring->size; 229 230 memset(ring->base, 0, size); 231 } 232 233 static void 234 vmxnet3_dev_rx_queue_reset(void *rxq) 235 { 236 int i; 237 vmxnet3_rx_queue_t *rq = rxq; 238 struct vmxnet3_hw *hw = rq->hw; 239 struct vmxnet3_cmd_ring *ring0, *ring1; 240 struct vmxnet3_comp_ring *comp_ring; 241 struct vmxnet3_rx_data_ring *data_ring = &rq->data_ring; 242 int size; 243 244 /* Release both the cmd_rings mbufs */ 245 for (i = 0; i < VMXNET3_RX_CMDRING_SIZE; i++) 246 vmxnet3_rx_cmd_ring_release_mbufs(&rq->cmd_ring[i]); 247 248 ring0 = &rq->cmd_ring[0]; 249 ring1 = &rq->cmd_ring[1]; 250 comp_ring = &rq->comp_ring; 251 252 /* Rx vmxnet rings structure initialization */ 253 ring0->next2fill = 0; 254 ring1->next2fill = 0; 255 ring0->next2comp = 0; 256 ring1->next2comp = 0; 257 ring0->gen = VMXNET3_INIT_GEN; 258 ring1->gen = VMXNET3_INIT_GEN; 259 comp_ring->next2proc = 0; 260 comp_ring->gen = VMXNET3_INIT_GEN; 261 262 size = sizeof(struct Vmxnet3_RxDesc) * (ring0->size + ring1->size); 263 size += sizeof(struct Vmxnet3_RxCompDesc) * comp_ring->size; 264 if (VMXNET3_VERSION_GE_3(hw) && rq->data_desc_size) 265 size += rq->data_desc_size * data_ring->size; 266 267 memset(ring0->base, 0, size); 268 } 269 270 void 271 vmxnet3_dev_clear_queues(struct rte_eth_dev *dev) 272 { 273 unsigned i; 274 275 PMD_INIT_FUNC_TRACE(); 276 277 for (i = 0; i < dev->data->nb_tx_queues; i++) { 278 struct vmxnet3_tx_queue *txq = dev->data->tx_queues[i]; 279 280 if (txq != NULL) { 281 txq->stopped = TRUE; 282 vmxnet3_dev_tx_queue_reset(txq); 283 } 284 } 285 286 for (i = 0; i < dev->data->nb_rx_queues; i++) { 287 struct vmxnet3_rx_queue *rxq = dev->data->rx_queues[i]; 288 289 if (rxq != NULL) { 290 rxq->stopped = TRUE; 291 vmxnet3_dev_rx_queue_reset(rxq); 292 } 293 } 294 } 295 296 static int 297 vmxnet3_unmap_pkt(uint16_t eop_idx, vmxnet3_tx_queue_t *txq) 298 { 299 int completed = 0; 300 struct rte_mbuf *mbuf; 301 302 /* Release cmd_ring descriptor and free mbuf */ 303 RTE_ASSERT(txq->cmd_ring.base[eop_idx].txd.eop == 1); 304 305 mbuf = txq->cmd_ring.buf_info[eop_idx].m; 306 if (mbuf == NULL) 307 rte_panic("EOP desc does not point to a valid mbuf"); 308 rte_pktmbuf_free(mbuf); 309 310 txq->cmd_ring.buf_info[eop_idx].m = NULL; 311 312 while (txq->cmd_ring.next2comp != eop_idx) { 313 /* no out-of-order completion */ 314 RTE_ASSERT(txq->cmd_ring.base[txq->cmd_ring.next2comp].txd.cq == 0); 315 vmxnet3_cmd_ring_adv_next2comp(&txq->cmd_ring); 316 completed++; 317 } 318 319 /* Mark the txd for which tcd was generated as completed */ 320 vmxnet3_cmd_ring_adv_next2comp(&txq->cmd_ring); 321 322 return completed + 1; 323 } 324 325 static void 326 vmxnet3_tq_tx_complete(vmxnet3_tx_queue_t *txq) 327 { 328 int completed = 0; 329 vmxnet3_comp_ring_t *comp_ring = &txq->comp_ring; 330 struct Vmxnet3_TxCompDesc *tcd = (struct Vmxnet3_TxCompDesc *) 331 (comp_ring->base + comp_ring->next2proc); 332 333 while (tcd->gen == comp_ring->gen) { 334 completed += vmxnet3_unmap_pkt(tcd->txdIdx, txq); 335 336 vmxnet3_comp_ring_adv_next2proc(comp_ring); 337 tcd = (struct Vmxnet3_TxCompDesc *)(comp_ring->base + 338 comp_ring->next2proc); 339 } 340 341 PMD_TX_LOG(DEBUG, "Processed %d tx comps & command descs.", completed); 342 } 343 344 uint16_t 345 vmxnet3_prep_pkts(__rte_unused void *tx_queue, struct rte_mbuf **tx_pkts, 346 uint16_t nb_pkts) 347 { 348 int32_t ret; 349 uint32_t i; 350 uint64_t ol_flags; 351 struct rte_mbuf *m; 352 353 for (i = 0; i != nb_pkts; i++) { 354 m = tx_pkts[i]; 355 ol_flags = m->ol_flags; 356 357 /* Non-TSO packet cannot occupy more than 358 * VMXNET3_MAX_TXD_PER_PKT TX descriptors. 359 */ 360 if ((ol_flags & PKT_TX_TCP_SEG) == 0 && 361 m->nb_segs > VMXNET3_MAX_TXD_PER_PKT) { 362 rte_errno = -EINVAL; 363 return i; 364 } 365 366 /* check that only supported TX offloads are requested. */ 367 if ((ol_flags & VMXNET3_TX_OFFLOAD_NOTSUP_MASK) != 0 || 368 (ol_flags & PKT_TX_L4_MASK) == 369 PKT_TX_SCTP_CKSUM) { 370 rte_errno = -ENOTSUP; 371 return i; 372 } 373 374 #ifdef RTE_LIBRTE_ETHDEV_DEBUG 375 ret = rte_validate_tx_offload(m); 376 if (ret != 0) { 377 rte_errno = ret; 378 return i; 379 } 380 #endif 381 ret = rte_net_intel_cksum_prepare(m); 382 if (ret != 0) { 383 rte_errno = ret; 384 return i; 385 } 386 } 387 388 return i; 389 } 390 391 uint16_t 392 vmxnet3_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 393 uint16_t nb_pkts) 394 { 395 uint16_t nb_tx; 396 vmxnet3_tx_queue_t *txq = tx_queue; 397 struct vmxnet3_hw *hw = txq->hw; 398 Vmxnet3_TxQueueCtrl *txq_ctrl = &txq->shared->ctrl; 399 uint32_t deferred = rte_le_to_cpu_32(txq_ctrl->txNumDeferred); 400 401 if (unlikely(txq->stopped)) { 402 PMD_TX_LOG(DEBUG, "Tx queue is stopped."); 403 return 0; 404 } 405 406 /* Free up the comp_descriptors aggressively */ 407 vmxnet3_tq_tx_complete(txq); 408 409 nb_tx = 0; 410 while (nb_tx < nb_pkts) { 411 Vmxnet3_GenericDesc *gdesc; 412 vmxnet3_buf_info_t *tbi; 413 uint32_t first2fill, avail, dw2; 414 struct rte_mbuf *txm = tx_pkts[nb_tx]; 415 struct rte_mbuf *m_seg = txm; 416 int copy_size = 0; 417 bool tso = (txm->ol_flags & PKT_TX_TCP_SEG) != 0; 418 /* # of descriptors needed for a packet. */ 419 unsigned count = txm->nb_segs; 420 421 avail = vmxnet3_cmd_ring_desc_avail(&txq->cmd_ring); 422 if (count > avail) { 423 /* Is command ring full? */ 424 if (unlikely(avail == 0)) { 425 PMD_TX_LOG(DEBUG, "No free ring descriptors"); 426 txq->stats.tx_ring_full++; 427 txq->stats.drop_total += (nb_pkts - nb_tx); 428 break; 429 } 430 431 /* Command ring is not full but cannot handle the 432 * multi-segmented packet. Let's try the next packet 433 * in this case. 434 */ 435 PMD_TX_LOG(DEBUG, "Running out of ring descriptors " 436 "(avail %d needed %d)", avail, count); 437 txq->stats.drop_total++; 438 if (tso) 439 txq->stats.drop_tso++; 440 rte_pktmbuf_free(txm); 441 nb_tx++; 442 continue; 443 } 444 445 /* Drop non-TSO packet that is excessively fragmented */ 446 if (unlikely(!tso && count > VMXNET3_MAX_TXD_PER_PKT)) { 447 PMD_TX_LOG(ERR, "Non-TSO packet cannot occupy more than %d tx " 448 "descriptors. Packet dropped.", VMXNET3_MAX_TXD_PER_PKT); 449 txq->stats.drop_too_many_segs++; 450 txq->stats.drop_total++; 451 rte_pktmbuf_free(txm); 452 nb_tx++; 453 continue; 454 } 455 456 if (txm->nb_segs == 1 && 457 rte_pktmbuf_pkt_len(txm) <= txq->txdata_desc_size) { 458 struct Vmxnet3_TxDataDesc *tdd; 459 460 tdd = (struct Vmxnet3_TxDataDesc *) 461 ((uint8 *)txq->data_ring.base + 462 txq->cmd_ring.next2fill * 463 txq->txdata_desc_size); 464 copy_size = rte_pktmbuf_pkt_len(txm); 465 rte_memcpy(tdd->data, rte_pktmbuf_mtod(txm, char *), copy_size); 466 } 467 468 /* use the previous gen bit for the SOP desc */ 469 dw2 = (txq->cmd_ring.gen ^ 0x1) << VMXNET3_TXD_GEN_SHIFT; 470 first2fill = txq->cmd_ring.next2fill; 471 do { 472 /* Remember the transmit buffer for cleanup */ 473 tbi = txq->cmd_ring.buf_info + txq->cmd_ring.next2fill; 474 475 /* NB: the following assumes that VMXNET3 maximum 476 * transmit buffer size (16K) is greater than 477 * maximum size of mbuf segment size. 478 */ 479 gdesc = txq->cmd_ring.base + txq->cmd_ring.next2fill; 480 if (copy_size) { 481 uint64 offset = 482 (uint64)txq->cmd_ring.next2fill * 483 txq->txdata_desc_size; 484 gdesc->txd.addr = 485 rte_cpu_to_le_64(txq->data_ring.basePA + 486 offset); 487 } else { 488 gdesc->txd.addr = rte_mbuf_data_iova(m_seg); 489 } 490 491 gdesc->dword[2] = dw2 | m_seg->data_len; 492 gdesc->dword[3] = 0; 493 494 /* move to the next2fill descriptor */ 495 vmxnet3_cmd_ring_adv_next2fill(&txq->cmd_ring); 496 497 /* use the right gen for non-SOP desc */ 498 dw2 = txq->cmd_ring.gen << VMXNET3_TXD_GEN_SHIFT; 499 } while ((m_seg = m_seg->next) != NULL); 500 501 /* set the last buf_info for the pkt */ 502 tbi->m = txm; 503 /* Update the EOP descriptor */ 504 gdesc->dword[3] |= VMXNET3_TXD_EOP | VMXNET3_TXD_CQ; 505 506 /* Add VLAN tag if present */ 507 gdesc = txq->cmd_ring.base + first2fill; 508 if (txm->ol_flags & PKT_TX_VLAN_PKT) { 509 gdesc->txd.ti = 1; 510 gdesc->txd.tci = txm->vlan_tci; 511 } 512 513 if (tso) { 514 uint16_t mss = txm->tso_segsz; 515 516 RTE_ASSERT(mss > 0); 517 518 gdesc->txd.hlen = txm->l2_len + txm->l3_len + txm->l4_len; 519 gdesc->txd.om = VMXNET3_OM_TSO; 520 gdesc->txd.msscof = mss; 521 522 deferred += (rte_pktmbuf_pkt_len(txm) - gdesc->txd.hlen + mss - 1) / mss; 523 } else if (txm->ol_flags & PKT_TX_L4_MASK) { 524 gdesc->txd.om = VMXNET3_OM_CSUM; 525 gdesc->txd.hlen = txm->l2_len + txm->l3_len; 526 527 switch (txm->ol_flags & PKT_TX_L4_MASK) { 528 case PKT_TX_TCP_CKSUM: 529 gdesc->txd.msscof = gdesc->txd.hlen + offsetof(struct tcp_hdr, cksum); 530 break; 531 case PKT_TX_UDP_CKSUM: 532 gdesc->txd.msscof = gdesc->txd.hlen + offsetof(struct udp_hdr, dgram_cksum); 533 break; 534 default: 535 PMD_TX_LOG(WARNING, "requested cksum offload not supported %#llx", 536 txm->ol_flags & PKT_TX_L4_MASK); 537 abort(); 538 } 539 deferred++; 540 } else { 541 gdesc->txd.hlen = 0; 542 gdesc->txd.om = VMXNET3_OM_NONE; 543 gdesc->txd.msscof = 0; 544 deferred++; 545 } 546 547 /* flip the GEN bit on the SOP */ 548 rte_compiler_barrier(); 549 gdesc->dword[2] ^= VMXNET3_TXD_GEN; 550 551 txq_ctrl->txNumDeferred = rte_cpu_to_le_32(deferred); 552 nb_tx++; 553 } 554 555 PMD_TX_LOG(DEBUG, "vmxnet3 txThreshold: %u", rte_le_to_cpu_32(txq_ctrl->txThreshold)); 556 557 if (deferred >= rte_le_to_cpu_32(txq_ctrl->txThreshold)) { 558 txq_ctrl->txNumDeferred = 0; 559 /* Notify vSwitch that packets are available. */ 560 VMXNET3_WRITE_BAR0_REG(hw, (VMXNET3_REG_TXPROD + txq->queue_id * VMXNET3_REG_ALIGN), 561 txq->cmd_ring.next2fill); 562 } 563 564 return nb_tx; 565 } 566 567 static inline void 568 vmxnet3_renew_desc(vmxnet3_rx_queue_t *rxq, uint8_t ring_id, 569 struct rte_mbuf *mbuf) 570 { 571 uint32_t val; 572 struct vmxnet3_cmd_ring *ring = &rxq->cmd_ring[ring_id]; 573 struct Vmxnet3_RxDesc *rxd = 574 (struct Vmxnet3_RxDesc *)(ring->base + ring->next2fill); 575 vmxnet3_buf_info_t *buf_info = &ring->buf_info[ring->next2fill]; 576 577 if (ring_id == 0) { 578 /* Usually: One HEAD type buf per packet 579 * val = (ring->next2fill % rxq->hw->bufs_per_pkt) ? 580 * VMXNET3_RXD_BTYPE_BODY : VMXNET3_RXD_BTYPE_HEAD; 581 */ 582 583 /* We use single packet buffer so all heads here */ 584 val = VMXNET3_RXD_BTYPE_HEAD; 585 } else { 586 /* All BODY type buffers for 2nd ring */ 587 val = VMXNET3_RXD_BTYPE_BODY; 588 } 589 590 /* 591 * Load mbuf pointer into buf_info[ring_size] 592 * buf_info structure is equivalent to cookie for virtio-virtqueue 593 */ 594 buf_info->m = mbuf; 595 buf_info->len = (uint16_t)(mbuf->buf_len - RTE_PKTMBUF_HEADROOM); 596 buf_info->bufPA = rte_mbuf_data_iova_default(mbuf); 597 598 /* Load Rx Descriptor with the buffer's GPA */ 599 rxd->addr = buf_info->bufPA; 600 601 /* After this point rxd->addr MUST not be NULL */ 602 rxd->btype = val; 603 rxd->len = buf_info->len; 604 /* Flip gen bit at the end to change ownership */ 605 rxd->gen = ring->gen; 606 607 vmxnet3_cmd_ring_adv_next2fill(ring); 608 } 609 /* 610 * Allocates mbufs and clusters. Post rx descriptors with buffer details 611 * so that device can receive packets in those buffers. 612 * Ring layout: 613 * Among the two rings, 1st ring contains buffers of type 0 and type 1. 614 * bufs_per_pkt is set such that for non-LRO cases all the buffers required 615 * by a frame will fit in 1st ring (1st buf of type0 and rest of type1). 616 * 2nd ring contains buffers of type 1 alone. Second ring mostly be used 617 * only for LRO. 618 */ 619 static int 620 vmxnet3_post_rx_bufs(vmxnet3_rx_queue_t *rxq, uint8_t ring_id) 621 { 622 int err = 0; 623 uint32_t i = 0; 624 struct vmxnet3_cmd_ring *ring = &rxq->cmd_ring[ring_id]; 625 626 while (vmxnet3_cmd_ring_desc_avail(ring) > 0) { 627 struct rte_mbuf *mbuf; 628 629 /* Allocate blank mbuf for the current Rx Descriptor */ 630 mbuf = rte_mbuf_raw_alloc(rxq->mp); 631 if (unlikely(mbuf == NULL)) { 632 PMD_RX_LOG(ERR, "Error allocating mbuf"); 633 rxq->stats.rx_buf_alloc_failure++; 634 err = ENOMEM; 635 break; 636 } 637 638 vmxnet3_renew_desc(rxq, ring_id, mbuf); 639 i++; 640 } 641 642 /* Return error only if no buffers are posted at present */ 643 if (vmxnet3_cmd_ring_desc_avail(ring) >= (ring->size - 1)) 644 return -err; 645 else 646 return i; 647 } 648 649 650 /* Receive side checksum and other offloads */ 651 static void 652 vmxnet3_rx_offload(const Vmxnet3_RxCompDesc *rcd, struct rte_mbuf *rxm) 653 { 654 /* Check for RSS */ 655 if (rcd->rssType != VMXNET3_RCD_RSS_TYPE_NONE) { 656 rxm->ol_flags |= PKT_RX_RSS_HASH; 657 rxm->hash.rss = rcd->rssHash; 658 } 659 660 /* Check packet type, checksum errors, etc. Only support IPv4 for now. */ 661 if (rcd->v4) { 662 struct ether_hdr *eth = rte_pktmbuf_mtod(rxm, struct ether_hdr *); 663 struct ipv4_hdr *ip = (struct ipv4_hdr *)(eth + 1); 664 665 if (((ip->version_ihl & 0xf) << 2) > (int)sizeof(struct ipv4_hdr)) 666 rxm->packet_type = RTE_PTYPE_L3_IPV4_EXT; 667 else 668 rxm->packet_type = RTE_PTYPE_L3_IPV4; 669 670 if (!rcd->cnc) { 671 if (!rcd->ipc) 672 rxm->ol_flags |= PKT_RX_IP_CKSUM_BAD; 673 674 if ((rcd->tcp || rcd->udp) && !rcd->tuc) 675 rxm->ol_flags |= PKT_RX_L4_CKSUM_BAD; 676 } 677 } else { 678 rxm->packet_type = RTE_PTYPE_UNKNOWN; 679 } 680 } 681 682 /* 683 * Process the Rx Completion Ring of given vmxnet3_rx_queue 684 * for nb_pkts burst and return the number of packets received 685 */ 686 uint16_t 687 vmxnet3_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) 688 { 689 uint16_t nb_rx; 690 uint32_t nb_rxd, idx; 691 uint8_t ring_idx; 692 vmxnet3_rx_queue_t *rxq; 693 Vmxnet3_RxCompDesc *rcd; 694 vmxnet3_buf_info_t *rbi; 695 Vmxnet3_RxDesc *rxd; 696 struct rte_mbuf *rxm = NULL; 697 struct vmxnet3_hw *hw; 698 699 nb_rx = 0; 700 ring_idx = 0; 701 nb_rxd = 0; 702 idx = 0; 703 704 rxq = rx_queue; 705 hw = rxq->hw; 706 707 rcd = &rxq->comp_ring.base[rxq->comp_ring.next2proc].rcd; 708 709 if (unlikely(rxq->stopped)) { 710 PMD_RX_LOG(DEBUG, "Rx queue is stopped."); 711 return 0; 712 } 713 714 while (rcd->gen == rxq->comp_ring.gen) { 715 struct rte_mbuf *newm; 716 717 if (nb_rx >= nb_pkts) 718 break; 719 720 newm = rte_mbuf_raw_alloc(rxq->mp); 721 if (unlikely(newm == NULL)) { 722 PMD_RX_LOG(ERR, "Error allocating mbuf"); 723 rxq->stats.rx_buf_alloc_failure++; 724 break; 725 } 726 727 idx = rcd->rxdIdx; 728 ring_idx = vmxnet3_get_ring_idx(hw, rcd->rqID); 729 rxd = (Vmxnet3_RxDesc *)rxq->cmd_ring[ring_idx].base + idx; 730 RTE_SET_USED(rxd); /* used only for assert when enabled */ 731 rbi = rxq->cmd_ring[ring_idx].buf_info + idx; 732 733 PMD_RX_LOG(DEBUG, "rxd idx: %d ring idx: %d.", idx, ring_idx); 734 735 RTE_ASSERT(rcd->len <= rxd->len); 736 RTE_ASSERT(rbi->m); 737 738 /* Get the packet buffer pointer from buf_info */ 739 rxm = rbi->m; 740 741 /* Clear descriptor associated buf_info to be reused */ 742 rbi->m = NULL; 743 rbi->bufPA = 0; 744 745 /* Update the index that we received a packet */ 746 rxq->cmd_ring[ring_idx].next2comp = idx; 747 748 /* For RCD with EOP set, check if there is frame error */ 749 if (unlikely(rcd->eop && rcd->err)) { 750 rxq->stats.drop_total++; 751 rxq->stats.drop_err++; 752 753 if (!rcd->fcs) { 754 rxq->stats.drop_fcs++; 755 PMD_RX_LOG(ERR, "Recv packet dropped due to frame err."); 756 } 757 PMD_RX_LOG(ERR, "Error in received packet rcd#:%d rxd:%d", 758 (int)(rcd - (struct Vmxnet3_RxCompDesc *) 759 rxq->comp_ring.base), rcd->rxdIdx); 760 rte_pktmbuf_free_seg(rxm); 761 if (rxq->start_seg) { 762 struct rte_mbuf *start = rxq->start_seg; 763 764 rxq->start_seg = NULL; 765 rte_pktmbuf_free(start); 766 } 767 goto rcd_done; 768 } 769 770 /* Initialize newly received packet buffer */ 771 rxm->port = rxq->port_id; 772 rxm->nb_segs = 1; 773 rxm->next = NULL; 774 rxm->pkt_len = (uint16_t)rcd->len; 775 rxm->data_len = (uint16_t)rcd->len; 776 rxm->data_off = RTE_PKTMBUF_HEADROOM; 777 rxm->ol_flags = 0; 778 rxm->vlan_tci = 0; 779 780 /* 781 * If this is the first buffer of the received packet, 782 * set the pointer to the first mbuf of the packet 783 * Otherwise, update the total length and the number of segments 784 * of the current scattered packet, and update the pointer to 785 * the last mbuf of the current packet. 786 */ 787 if (rcd->sop) { 788 RTE_ASSERT(rxd->btype == VMXNET3_RXD_BTYPE_HEAD); 789 790 if (unlikely(rcd->len == 0)) { 791 RTE_ASSERT(rcd->eop); 792 793 PMD_RX_LOG(DEBUG, 794 "Rx buf was skipped. rxring[%d][%d])", 795 ring_idx, idx); 796 rte_pktmbuf_free_seg(rxm); 797 goto rcd_done; 798 } 799 800 if (vmxnet3_rx_data_ring(hw, rcd->rqID)) { 801 uint8_t *rdd = rxq->data_ring.base + 802 idx * rxq->data_desc_size; 803 804 RTE_ASSERT(VMXNET3_VERSION_GE_3(hw)); 805 rte_memcpy(rte_pktmbuf_mtod(rxm, char *), 806 rdd, rcd->len); 807 } 808 809 rxq->start_seg = rxm; 810 vmxnet3_rx_offload(rcd, rxm); 811 } else { 812 struct rte_mbuf *start = rxq->start_seg; 813 814 RTE_ASSERT(rxd->btype == VMXNET3_RXD_BTYPE_BODY); 815 816 start->pkt_len += rxm->data_len; 817 start->nb_segs++; 818 819 rxq->last_seg->next = rxm; 820 } 821 rxq->last_seg = rxm; 822 823 if (rcd->eop) { 824 struct rte_mbuf *start = rxq->start_seg; 825 826 /* Check for hardware stripped VLAN tag */ 827 if (rcd->ts) { 828 start->ol_flags |= (PKT_RX_VLAN | 829 PKT_RX_VLAN_STRIPPED); 830 start->vlan_tci = rte_le_to_cpu_16((uint16_t)rcd->tci); 831 } 832 833 rx_pkts[nb_rx++] = start; 834 rxq->start_seg = NULL; 835 } 836 837 rcd_done: 838 rxq->cmd_ring[ring_idx].next2comp = idx; 839 VMXNET3_INC_RING_IDX_ONLY(rxq->cmd_ring[ring_idx].next2comp, 840 rxq->cmd_ring[ring_idx].size); 841 842 /* It's time to renew descriptors */ 843 vmxnet3_renew_desc(rxq, ring_idx, newm); 844 if (unlikely(rxq->shared->ctrl.updateRxProd)) { 845 VMXNET3_WRITE_BAR0_REG(hw, rxprod_reg[ring_idx] + (rxq->queue_id * VMXNET3_REG_ALIGN), 846 rxq->cmd_ring[ring_idx].next2fill); 847 } 848 849 /* Advance to the next descriptor in comp_ring */ 850 vmxnet3_comp_ring_adv_next2proc(&rxq->comp_ring); 851 852 rcd = &rxq->comp_ring.base[rxq->comp_ring.next2proc].rcd; 853 nb_rxd++; 854 if (nb_rxd > rxq->cmd_ring[0].size) { 855 PMD_RX_LOG(ERR, "Used up quota of receiving packets," 856 " relinquish control."); 857 break; 858 } 859 } 860 861 if (unlikely(nb_rxd == 0)) { 862 uint32_t avail; 863 for (ring_idx = 0; ring_idx < VMXNET3_RX_CMDRING_SIZE; ring_idx++) { 864 avail = vmxnet3_cmd_ring_desc_avail(&rxq->cmd_ring[ring_idx]); 865 if (unlikely(avail > 0)) { 866 /* try to alloc new buf and renew descriptors */ 867 vmxnet3_post_rx_bufs(rxq, ring_idx); 868 } 869 } 870 if (unlikely(rxq->shared->ctrl.updateRxProd)) { 871 for (ring_idx = 0; ring_idx < VMXNET3_RX_CMDRING_SIZE; ring_idx++) { 872 VMXNET3_WRITE_BAR0_REG(hw, rxprod_reg[ring_idx] + (rxq->queue_id * VMXNET3_REG_ALIGN), 873 rxq->cmd_ring[ring_idx].next2fill); 874 } 875 } 876 } 877 878 return nb_rx; 879 } 880 881 int 882 vmxnet3_dev_tx_queue_setup(struct rte_eth_dev *dev, 883 uint16_t queue_idx, 884 uint16_t nb_desc, 885 unsigned int socket_id, 886 const struct rte_eth_txconf *tx_conf) 887 { 888 struct vmxnet3_hw *hw = dev->data->dev_private; 889 const struct rte_memzone *mz; 890 struct vmxnet3_tx_queue *txq; 891 struct vmxnet3_cmd_ring *ring; 892 struct vmxnet3_comp_ring *comp_ring; 893 struct vmxnet3_data_ring *data_ring; 894 int size; 895 896 PMD_INIT_FUNC_TRACE(); 897 898 if ((tx_conf->txq_flags & ETH_TXQ_FLAGS_NOXSUMSCTP) != 899 ETH_TXQ_FLAGS_NOXSUMSCTP) { 900 PMD_INIT_LOG(ERR, "SCTP checksum offload not supported"); 901 return -EINVAL; 902 } 903 904 txq = rte_zmalloc("ethdev_tx_queue", sizeof(struct vmxnet3_tx_queue), 905 RTE_CACHE_LINE_SIZE); 906 if (txq == NULL) { 907 PMD_INIT_LOG(ERR, "Can not allocate tx queue structure"); 908 return -ENOMEM; 909 } 910 911 txq->queue_id = queue_idx; 912 txq->port_id = dev->data->port_id; 913 txq->shared = &hw->tqd_start[queue_idx]; 914 txq->hw = hw; 915 txq->qid = queue_idx; 916 txq->stopped = TRUE; 917 txq->txdata_desc_size = hw->txdata_desc_size; 918 919 ring = &txq->cmd_ring; 920 comp_ring = &txq->comp_ring; 921 data_ring = &txq->data_ring; 922 923 /* Tx vmxnet ring length should be between 512-4096 */ 924 if (nb_desc < VMXNET3_DEF_TX_RING_SIZE) { 925 PMD_INIT_LOG(ERR, "VMXNET3 Tx Ring Size Min: %u", 926 VMXNET3_DEF_TX_RING_SIZE); 927 return -EINVAL; 928 } else if (nb_desc > VMXNET3_TX_RING_MAX_SIZE) { 929 PMD_INIT_LOG(ERR, "VMXNET3 Tx Ring Size Max: %u", 930 VMXNET3_TX_RING_MAX_SIZE); 931 return -EINVAL; 932 } else { 933 ring->size = nb_desc; 934 ring->size &= ~VMXNET3_RING_SIZE_MASK; 935 } 936 comp_ring->size = data_ring->size = ring->size; 937 938 /* Tx vmxnet rings structure initialization*/ 939 ring->next2fill = 0; 940 ring->next2comp = 0; 941 ring->gen = VMXNET3_INIT_GEN; 942 comp_ring->next2proc = 0; 943 comp_ring->gen = VMXNET3_INIT_GEN; 944 945 size = sizeof(struct Vmxnet3_TxDesc) * ring->size; 946 size += sizeof(struct Vmxnet3_TxCompDesc) * comp_ring->size; 947 size += txq->txdata_desc_size * data_ring->size; 948 949 mz = rte_eth_dma_zone_reserve(dev, "txdesc", queue_idx, size, 950 VMXNET3_RING_BA_ALIGN, socket_id); 951 if (mz == NULL) { 952 PMD_INIT_LOG(ERR, "ERROR: Creating queue descriptors zone"); 953 return -ENOMEM; 954 } 955 txq->mz = mz; 956 memset(mz->addr, 0, mz->len); 957 958 /* cmd_ring initialization */ 959 ring->base = mz->addr; 960 ring->basePA = mz->iova; 961 962 /* comp_ring initialization */ 963 comp_ring->base = ring->base + ring->size; 964 comp_ring->basePA = ring->basePA + 965 (sizeof(struct Vmxnet3_TxDesc) * ring->size); 966 967 /* data_ring initialization */ 968 data_ring->base = (Vmxnet3_TxDataDesc *)(comp_ring->base + comp_ring->size); 969 data_ring->basePA = comp_ring->basePA + 970 (sizeof(struct Vmxnet3_TxCompDesc) * comp_ring->size); 971 972 /* cmd_ring0 buf_info allocation */ 973 ring->buf_info = rte_zmalloc("tx_ring_buf_info", 974 ring->size * sizeof(vmxnet3_buf_info_t), RTE_CACHE_LINE_SIZE); 975 if (ring->buf_info == NULL) { 976 PMD_INIT_LOG(ERR, "ERROR: Creating tx_buf_info structure"); 977 return -ENOMEM; 978 } 979 980 /* Update the data portion with txq */ 981 dev->data->tx_queues[queue_idx] = txq; 982 983 return 0; 984 } 985 986 int 987 vmxnet3_dev_rx_queue_setup(struct rte_eth_dev *dev, 988 uint16_t queue_idx, 989 uint16_t nb_desc, 990 unsigned int socket_id, 991 __rte_unused const struct rte_eth_rxconf *rx_conf, 992 struct rte_mempool *mp) 993 { 994 const struct rte_memzone *mz; 995 struct vmxnet3_rx_queue *rxq; 996 struct vmxnet3_hw *hw = dev->data->dev_private; 997 struct vmxnet3_cmd_ring *ring0, *ring1, *ring; 998 struct vmxnet3_comp_ring *comp_ring; 999 struct vmxnet3_rx_data_ring *data_ring; 1000 int size; 1001 uint8_t i; 1002 char mem_name[32]; 1003 1004 PMD_INIT_FUNC_TRACE(); 1005 1006 rxq = rte_zmalloc("ethdev_rx_queue", sizeof(struct vmxnet3_rx_queue), 1007 RTE_CACHE_LINE_SIZE); 1008 if (rxq == NULL) { 1009 PMD_INIT_LOG(ERR, "Can not allocate rx queue structure"); 1010 return -ENOMEM; 1011 } 1012 1013 rxq->mp = mp; 1014 rxq->queue_id = queue_idx; 1015 rxq->port_id = dev->data->port_id; 1016 rxq->shared = &hw->rqd_start[queue_idx]; 1017 rxq->hw = hw; 1018 rxq->qid1 = queue_idx; 1019 rxq->qid2 = queue_idx + hw->num_rx_queues; 1020 rxq->data_ring_qid = queue_idx + 2 * hw->num_rx_queues; 1021 rxq->data_desc_size = hw->rxdata_desc_size; 1022 rxq->stopped = TRUE; 1023 1024 ring0 = &rxq->cmd_ring[0]; 1025 ring1 = &rxq->cmd_ring[1]; 1026 comp_ring = &rxq->comp_ring; 1027 data_ring = &rxq->data_ring; 1028 1029 /* Rx vmxnet rings length should be between 256-4096 */ 1030 if (nb_desc < VMXNET3_DEF_RX_RING_SIZE) { 1031 PMD_INIT_LOG(ERR, "VMXNET3 Rx Ring Size Min: 256"); 1032 return -EINVAL; 1033 } else if (nb_desc > VMXNET3_RX_RING_MAX_SIZE) { 1034 PMD_INIT_LOG(ERR, "VMXNET3 Rx Ring Size Max: 4096"); 1035 return -EINVAL; 1036 } else { 1037 ring0->size = nb_desc; 1038 ring0->size &= ~VMXNET3_RING_SIZE_MASK; 1039 ring1->size = ring0->size; 1040 } 1041 1042 comp_ring->size = ring0->size + ring1->size; 1043 data_ring->size = ring0->size; 1044 1045 /* Rx vmxnet rings structure initialization */ 1046 ring0->next2fill = 0; 1047 ring1->next2fill = 0; 1048 ring0->next2comp = 0; 1049 ring1->next2comp = 0; 1050 ring0->gen = VMXNET3_INIT_GEN; 1051 ring1->gen = VMXNET3_INIT_GEN; 1052 comp_ring->next2proc = 0; 1053 comp_ring->gen = VMXNET3_INIT_GEN; 1054 1055 size = sizeof(struct Vmxnet3_RxDesc) * (ring0->size + ring1->size); 1056 size += sizeof(struct Vmxnet3_RxCompDesc) * comp_ring->size; 1057 if (VMXNET3_VERSION_GE_3(hw) && rxq->data_desc_size) 1058 size += rxq->data_desc_size * data_ring->size; 1059 1060 mz = rte_eth_dma_zone_reserve(dev, "rxdesc", queue_idx, size, 1061 VMXNET3_RING_BA_ALIGN, socket_id); 1062 if (mz == NULL) { 1063 PMD_INIT_LOG(ERR, "ERROR: Creating queue descriptors zone"); 1064 return -ENOMEM; 1065 } 1066 rxq->mz = mz; 1067 memset(mz->addr, 0, mz->len); 1068 1069 /* cmd_ring0 initialization */ 1070 ring0->base = mz->addr; 1071 ring0->basePA = mz->iova; 1072 1073 /* cmd_ring1 initialization */ 1074 ring1->base = ring0->base + ring0->size; 1075 ring1->basePA = ring0->basePA + sizeof(struct Vmxnet3_RxDesc) * ring0->size; 1076 1077 /* comp_ring initialization */ 1078 comp_ring->base = ring1->base + ring1->size; 1079 comp_ring->basePA = ring1->basePA + sizeof(struct Vmxnet3_RxDesc) * 1080 ring1->size; 1081 1082 /* data_ring initialization */ 1083 if (VMXNET3_VERSION_GE_3(hw) && rxq->data_desc_size) { 1084 data_ring->base = 1085 (uint8_t *)(comp_ring->base + comp_ring->size); 1086 data_ring->basePA = comp_ring->basePA + 1087 sizeof(struct Vmxnet3_RxCompDesc) * comp_ring->size; 1088 } 1089 1090 /* cmd_ring0-cmd_ring1 buf_info allocation */ 1091 for (i = 0; i < VMXNET3_RX_CMDRING_SIZE; i++) { 1092 1093 ring = &rxq->cmd_ring[i]; 1094 ring->rid = i; 1095 snprintf(mem_name, sizeof(mem_name), "rx_ring_%d_buf_info", i); 1096 1097 ring->buf_info = rte_zmalloc(mem_name, 1098 ring->size * sizeof(vmxnet3_buf_info_t), 1099 RTE_CACHE_LINE_SIZE); 1100 if (ring->buf_info == NULL) { 1101 PMD_INIT_LOG(ERR, "ERROR: Creating rx_buf_info structure"); 1102 return -ENOMEM; 1103 } 1104 } 1105 1106 /* Update the data portion with rxq */ 1107 dev->data->rx_queues[queue_idx] = rxq; 1108 1109 return 0; 1110 } 1111 1112 /* 1113 * Initializes Receive Unit 1114 * Load mbufs in rx queue in advance 1115 */ 1116 int 1117 vmxnet3_dev_rxtx_init(struct rte_eth_dev *dev) 1118 { 1119 struct vmxnet3_hw *hw = dev->data->dev_private; 1120 1121 int i, ret; 1122 uint8_t j; 1123 1124 PMD_INIT_FUNC_TRACE(); 1125 1126 for (i = 0; i < hw->num_rx_queues; i++) { 1127 vmxnet3_rx_queue_t *rxq = dev->data->rx_queues[i]; 1128 1129 for (j = 0; j < VMXNET3_RX_CMDRING_SIZE; j++) { 1130 /* Passing 0 as alloc_num will allocate full ring */ 1131 ret = vmxnet3_post_rx_bufs(rxq, j); 1132 if (ret <= 0) { 1133 PMD_INIT_LOG(ERR, 1134 "ERROR: Posting Rxq: %d buffers ring: %d", 1135 i, j); 1136 return -ret; 1137 } 1138 /* 1139 * Updating device with the index:next2fill to fill the 1140 * mbufs for coming packets. 1141 */ 1142 if (unlikely(rxq->shared->ctrl.updateRxProd)) { 1143 VMXNET3_WRITE_BAR0_REG(hw, rxprod_reg[j] + (rxq->queue_id * VMXNET3_REG_ALIGN), 1144 rxq->cmd_ring[j].next2fill); 1145 } 1146 } 1147 rxq->stopped = FALSE; 1148 rxq->start_seg = NULL; 1149 } 1150 1151 for (i = 0; i < dev->data->nb_tx_queues; i++) { 1152 struct vmxnet3_tx_queue *txq = dev->data->tx_queues[i]; 1153 1154 txq->stopped = FALSE; 1155 } 1156 1157 return 0; 1158 } 1159 1160 static uint8_t rss_intel_key[40] = { 1161 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2, 1162 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0, 1163 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4, 1164 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C, 1165 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA, 1166 }; 1167 1168 /* 1169 * Configure RSS feature 1170 */ 1171 int 1172 vmxnet3_rss_configure(struct rte_eth_dev *dev) 1173 { 1174 struct vmxnet3_hw *hw = dev->data->dev_private; 1175 struct VMXNET3_RSSConf *dev_rss_conf; 1176 struct rte_eth_rss_conf *port_rss_conf; 1177 uint64_t rss_hf; 1178 uint8_t i, j; 1179 1180 PMD_INIT_FUNC_TRACE(); 1181 1182 dev_rss_conf = hw->rss_conf; 1183 port_rss_conf = &dev->data->dev_conf.rx_adv_conf.rss_conf; 1184 1185 /* loading hashFunc */ 1186 dev_rss_conf->hashFunc = VMXNET3_RSS_HASH_FUNC_TOEPLITZ; 1187 /* loading hashKeySize */ 1188 dev_rss_conf->hashKeySize = VMXNET3_RSS_MAX_KEY_SIZE; 1189 /* loading indTableSize: Must not exceed VMXNET3_RSS_MAX_IND_TABLE_SIZE (128)*/ 1190 dev_rss_conf->indTableSize = (uint16_t)(hw->num_rx_queues * 4); 1191 1192 if (port_rss_conf->rss_key == NULL) { 1193 /* Default hash key */ 1194 port_rss_conf->rss_key = rss_intel_key; 1195 } 1196 1197 /* loading hashKey */ 1198 memcpy(&dev_rss_conf->hashKey[0], port_rss_conf->rss_key, 1199 dev_rss_conf->hashKeySize); 1200 1201 /* loading indTable */ 1202 for (i = 0, j = 0; i < dev_rss_conf->indTableSize; i++, j++) { 1203 if (j == dev->data->nb_rx_queues) 1204 j = 0; 1205 dev_rss_conf->indTable[i] = j; 1206 } 1207 1208 /* loading hashType */ 1209 dev_rss_conf->hashType = 0; 1210 rss_hf = port_rss_conf->rss_hf & VMXNET3_RSS_OFFLOAD_ALL; 1211 if (rss_hf & ETH_RSS_IPV4) 1212 dev_rss_conf->hashType |= VMXNET3_RSS_HASH_TYPE_IPV4; 1213 if (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP) 1214 dev_rss_conf->hashType |= VMXNET3_RSS_HASH_TYPE_TCP_IPV4; 1215 if (rss_hf & ETH_RSS_IPV6) 1216 dev_rss_conf->hashType |= VMXNET3_RSS_HASH_TYPE_IPV6; 1217 if (rss_hf & ETH_RSS_NONFRAG_IPV6_TCP) 1218 dev_rss_conf->hashType |= VMXNET3_RSS_HASH_TYPE_TCP_IPV6; 1219 1220 return VMXNET3_SUCCESS; 1221 } 1222