1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(C) 2022 Intel Corporation 3 */ 4 5 #include "gve_ethdev.h" 6 #include "base/gve_adminq.h" 7 8 #define GVE_PKT_CONT_BIT_IS_SET(x) (GVE_RXF_PKT_CONT & (x)) 9 10 static inline void 11 gve_rx_refill(struct gve_rx_queue *rxq) 12 { 13 uint16_t mask = rxq->nb_rx_desc - 1; 14 uint16_t idx = rxq->next_avail & mask; 15 uint32_t next_avail = rxq->next_avail; 16 uint16_t nb_alloc, i; 17 struct rte_mbuf *nmb; 18 int diag; 19 20 /* wrap around */ 21 nb_alloc = rxq->nb_rx_desc - idx; 22 if (nb_alloc <= rxq->nb_avail) { 23 diag = rte_pktmbuf_alloc_bulk(rxq->mpool, &rxq->sw_ring[idx], nb_alloc); 24 if (diag < 0) { 25 rxq->stats.no_mbufs_bulk++; 26 for (i = 0; i < nb_alloc; i++) { 27 nmb = rte_pktmbuf_alloc(rxq->mpool); 28 if (!nmb) 29 break; 30 rxq->sw_ring[idx + i] = nmb; 31 } 32 if (i != nb_alloc) { 33 rxq->stats.no_mbufs += nb_alloc - i; 34 nb_alloc = i; 35 } 36 } 37 rxq->nb_avail -= nb_alloc; 38 next_avail += nb_alloc; 39 40 /* queue page list mode doesn't need real refill. */ 41 if (rxq->is_gqi_qpl) { 42 idx += nb_alloc; 43 } else { 44 for (i = 0; i < nb_alloc; i++) { 45 nmb = rxq->sw_ring[idx]; 46 rxq->rx_data_ring[idx].addr = 47 rte_cpu_to_be_64(rte_mbuf_data_iova(nmb)); 48 idx++; 49 } 50 } 51 if (idx == rxq->nb_rx_desc) 52 idx = 0; 53 } 54 55 if (rxq->nb_avail > 0) { 56 nb_alloc = rxq->nb_avail; 57 if (rxq->nb_rx_desc < idx + rxq->nb_avail) 58 nb_alloc = rxq->nb_rx_desc - idx; 59 diag = rte_pktmbuf_alloc_bulk(rxq->mpool, &rxq->sw_ring[idx], nb_alloc); 60 if (diag < 0) { 61 rxq->stats.no_mbufs_bulk++; 62 for (i = 0; i < nb_alloc; i++) { 63 nmb = rte_pktmbuf_alloc(rxq->mpool); 64 if (!nmb) 65 break; 66 rxq->sw_ring[idx + i] = nmb; 67 } 68 if (i != nb_alloc) { 69 rxq->stats.no_mbufs += nb_alloc - i; 70 nb_alloc = i; 71 } 72 } 73 rxq->nb_avail -= nb_alloc; 74 next_avail += nb_alloc; 75 76 if (!rxq->is_gqi_qpl) { 77 for (i = 0; i < nb_alloc; i++) { 78 nmb = rxq->sw_ring[idx]; 79 rxq->rx_data_ring[idx].addr = 80 rte_cpu_to_be_64(rte_mbuf_data_iova(nmb)); 81 idx++; 82 } 83 } 84 } 85 86 if (next_avail != rxq->next_avail) { 87 rte_write32(rte_cpu_to_be_32(next_avail), rxq->qrx_tail); 88 rxq->next_avail = next_avail; 89 } 90 } 91 92 /* 93 * This method processes a single rte_mbuf and handles packet segmentation 94 * In QPL mode it copies data from the mbuf to the gve_rx_queue. 95 */ 96 static void 97 gve_rx_mbuf(struct gve_rx_queue *rxq, struct rte_mbuf *rxe, uint16_t len, 98 uint16_t rx_id) 99 { 100 uint16_t padding = 0; 101 uint64_t addr; 102 103 rxe->data_len = len; 104 if (!rxq->ctx.mbuf_head) { 105 rxq->ctx.mbuf_head = rxe; 106 rxq->ctx.mbuf_tail = rxe; 107 rxe->nb_segs = 1; 108 rxe->pkt_len = len; 109 rxe->data_len = len; 110 rxe->port = rxq->port_id; 111 rxe->ol_flags = 0; 112 padding = GVE_RX_PAD; 113 } else { 114 rxq->ctx.mbuf_head->pkt_len += len; 115 rxq->ctx.mbuf_head->nb_segs += 1; 116 rxq->ctx.mbuf_tail->next = rxe; 117 rxq->ctx.mbuf_tail = rxe; 118 } 119 if (rxq->is_gqi_qpl) { 120 addr = (uint64_t)(rxq->qpl->mz->addr) + rx_id * PAGE_SIZE + padding; 121 rte_memcpy((void *)((size_t)rxe->buf_addr + rxe->data_off), 122 (void *)(size_t)addr, len); 123 } 124 } 125 126 /* 127 * This method processes a single packet fragment associated with the 128 * passed packet descriptor. 129 * This methods returns whether the fragment is the last fragment 130 * of a packet. 131 */ 132 static bool 133 gve_rx(struct gve_rx_queue *rxq, volatile struct gve_rx_desc *rxd, uint16_t rx_id) 134 { 135 bool is_last_frag = !GVE_PKT_CONT_BIT_IS_SET(rxd->flags_seq); 136 uint16_t frag_size = rte_be_to_cpu_16(rxd->len); 137 struct gve_rx_ctx *ctx = &rxq->ctx; 138 bool is_first_frag = ctx->total_frags == 0; 139 struct rte_mbuf *rxe; 140 141 if (ctx->drop_pkt) 142 goto finish_frag; 143 144 if (rxd->flags_seq & GVE_RXF_ERR) { 145 ctx->drop_pkt = true; 146 rxq->stats.errors++; 147 goto finish_frag; 148 } 149 150 if (is_first_frag) 151 frag_size -= GVE_RX_PAD; 152 153 rxe = rxq->sw_ring[rx_id]; 154 gve_rx_mbuf(rxq, rxe, frag_size, rx_id); 155 rxq->stats.bytes += frag_size; 156 157 if (is_first_frag) { 158 if (rxd->flags_seq & GVE_RXF_TCP) 159 rxe->packet_type |= RTE_PTYPE_L4_TCP; 160 if (rxd->flags_seq & GVE_RXF_UDP) 161 rxe->packet_type |= RTE_PTYPE_L4_UDP; 162 if (rxd->flags_seq & GVE_RXF_IPV4) 163 rxe->packet_type |= RTE_PTYPE_L3_IPV4; 164 if (rxd->flags_seq & GVE_RXF_IPV6) 165 rxe->packet_type |= RTE_PTYPE_L3_IPV6; 166 167 if (gve_needs_rss(rxd->flags_seq)) { 168 rxe->ol_flags |= RTE_MBUF_F_RX_RSS_HASH; 169 rxe->hash.rss = rte_be_to_cpu_32(rxd->rss_hash); 170 } 171 } 172 173 finish_frag: 174 ctx->total_frags++; 175 return is_last_frag; 176 } 177 178 static void 179 gve_rx_ctx_clear(struct gve_rx_ctx *ctx) 180 { 181 ctx->mbuf_head = NULL; 182 ctx->mbuf_tail = NULL; 183 ctx->drop_pkt = false; 184 ctx->total_frags = 0; 185 } 186 187 uint16_t 188 gve_rx_burst(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) 189 { 190 volatile struct gve_rx_desc *rxr, *rxd; 191 struct gve_rx_queue *rxq = rx_queue; 192 struct gve_rx_ctx *ctx = &rxq->ctx; 193 uint16_t rx_id = rxq->rx_tail; 194 uint16_t nb_rx; 195 196 rxr = rxq->rx_desc_ring; 197 nb_rx = 0; 198 199 while (nb_rx < nb_pkts) { 200 rxd = &rxr[rx_id]; 201 if (GVE_SEQNO(rxd->flags_seq) != rxq->expected_seqno) 202 break; 203 204 if (gve_rx(rxq, rxd, rx_id)) { 205 if (!ctx->drop_pkt) 206 rx_pkts[nb_rx++] = ctx->mbuf_head; 207 rxq->nb_avail += ctx->total_frags; 208 gve_rx_ctx_clear(ctx); 209 } 210 211 rx_id++; 212 if (rx_id == rxq->nb_rx_desc) 213 rx_id = 0; 214 215 rxq->expected_seqno = gve_next_seqno(rxq->expected_seqno); 216 } 217 218 rxq->rx_tail = rx_id; 219 220 if (rxq->nb_avail > rxq->free_thresh) 221 gve_rx_refill(rxq); 222 223 if (nb_rx) 224 rxq->stats.packets += nb_rx; 225 226 return nb_rx; 227 } 228 229 static inline void 230 gve_reset_rxq(struct gve_rx_queue *rxq) 231 { 232 struct rte_mbuf **sw_ring; 233 uint32_t size, i; 234 235 if (rxq == NULL) { 236 PMD_DRV_LOG(ERR, "pointer to rxq is NULL"); 237 return; 238 } 239 240 size = rxq->nb_rx_desc * sizeof(struct gve_rx_desc); 241 for (i = 0; i < size; i++) 242 ((volatile char *)rxq->rx_desc_ring)[i] = 0; 243 244 size = rxq->nb_rx_desc * sizeof(union gve_rx_data_slot); 245 for (i = 0; i < size; i++) 246 ((volatile char *)rxq->rx_data_ring)[i] = 0; 247 248 sw_ring = rxq->sw_ring; 249 for (i = 0; i < rxq->nb_rx_desc; i++) 250 sw_ring[i] = NULL; 251 252 rxq->rx_tail = 0; 253 rxq->next_avail = 0; 254 rxq->nb_avail = rxq->nb_rx_desc; 255 rxq->expected_seqno = 1; 256 } 257 258 static inline void 259 gve_release_rxq_mbufs(struct gve_rx_queue *rxq) 260 { 261 uint16_t i; 262 263 for (i = 0; i < rxq->nb_rx_desc; i++) { 264 if (rxq->sw_ring[i]) { 265 rte_pktmbuf_free_seg(rxq->sw_ring[i]); 266 rxq->sw_ring[i] = NULL; 267 } 268 } 269 270 rxq->nb_avail = rxq->nb_rx_desc; 271 } 272 273 void 274 gve_rx_queue_release(struct rte_eth_dev *dev, uint16_t qid) 275 { 276 struct gve_rx_queue *q = dev->data->rx_queues[qid]; 277 278 if (!q) 279 return; 280 281 if (q->is_gqi_qpl) { 282 gve_teardown_queue_page_list(q->hw, q->qpl); 283 q->qpl = NULL; 284 } 285 286 gve_release_rxq_mbufs(q); 287 rte_free(q->sw_ring); 288 rte_memzone_free(q->data_mz); 289 rte_memzone_free(q->mz); 290 rte_memzone_free(q->qres_mz); 291 q->qres = NULL; 292 rte_free(q); 293 } 294 295 int 296 gve_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id, 297 uint16_t nb_desc, unsigned int socket_id, 298 const struct rte_eth_rxconf *conf, struct rte_mempool *pool) 299 { 300 struct gve_priv *hw = dev->data->dev_private; 301 const struct rte_memzone *mz; 302 struct gve_rx_queue *rxq; 303 uint16_t free_thresh; 304 uint32_t mbuf_len; 305 int err = 0; 306 307 /* Ring size is required to be a power of two. */ 308 if (!rte_is_power_of_2(nb_desc)) { 309 PMD_DRV_LOG(ERR, "Invalid ring size %u. GVE ring size must be a power of 2.", 310 nb_desc); 311 return -EINVAL; 312 } 313 314 /* Free memory if needed. */ 315 if (dev->data->rx_queues[queue_id]) { 316 gve_rx_queue_release(dev, queue_id); 317 dev->data->rx_queues[queue_id] = NULL; 318 } 319 320 /* Allocate the RX queue data structure. */ 321 rxq = rte_zmalloc_socket("gve rxq", 322 sizeof(struct gve_rx_queue), 323 RTE_CACHE_LINE_SIZE, 324 socket_id); 325 if (!rxq) { 326 PMD_DRV_LOG(ERR, "Failed to allocate memory for rx queue structure"); 327 err = -ENOMEM; 328 goto err_rxq; 329 } 330 331 free_thresh = conf->rx_free_thresh ? conf->rx_free_thresh : GVE_DEFAULT_RX_FREE_THRESH; 332 if (free_thresh >= nb_desc) { 333 PMD_DRV_LOG(ERR, "rx_free_thresh (%u) must be less than nb_desc (%u) minus 3.", 334 free_thresh, rxq->nb_rx_desc); 335 err = -EINVAL; 336 goto err_rxq; 337 } 338 339 rxq->nb_rx_desc = nb_desc; 340 rxq->free_thresh = free_thresh; 341 rxq->queue_id = queue_id; 342 rxq->port_id = dev->data->port_id; 343 rxq->ntfy_id = hw->num_ntfy_blks / 2 + queue_id; 344 rxq->is_gqi_qpl = hw->queue_format == GVE_GQI_QPL_FORMAT; 345 rxq->mpool = pool; 346 rxq->hw = hw; 347 rxq->ntfy_addr = &hw->db_bar2[rte_be_to_cpu_32(hw->irq_dbs[rxq->ntfy_id].id)]; 348 349 mbuf_len = 350 rte_pktmbuf_data_room_size(rxq->mpool) - RTE_PKTMBUF_HEADROOM; 351 rxq->rx_buf_len = 352 RTE_MIN((uint16_t)GVE_RX_MAX_BUF_SIZE_GQI, 353 RTE_ALIGN_FLOOR(mbuf_len, GVE_RX_BUF_ALIGN_GQI)); 354 355 /* Allocate software ring */ 356 rxq->sw_ring = rte_zmalloc_socket("gve rx sw ring", sizeof(struct rte_mbuf *) * nb_desc, 357 RTE_CACHE_LINE_SIZE, socket_id); 358 if (!rxq->sw_ring) { 359 PMD_DRV_LOG(ERR, "Failed to allocate memory for SW RX ring"); 360 err = -ENOMEM; 361 goto err_rxq; 362 } 363 364 mz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_id, 365 nb_desc * sizeof(struct gve_rx_desc), 366 PAGE_SIZE, socket_id); 367 if (mz == NULL) { 368 PMD_DRV_LOG(ERR, "Failed to reserve DMA memory for RX"); 369 err = -ENOMEM; 370 goto err_sw_ring; 371 } 372 rxq->rx_desc_ring = (struct gve_rx_desc *)mz->addr; 373 rxq->rx_ring_phys_addr = mz->iova; 374 rxq->mz = mz; 375 376 mz = rte_eth_dma_zone_reserve(dev, "gve rx data ring", queue_id, 377 sizeof(union gve_rx_data_slot) * nb_desc, 378 PAGE_SIZE, socket_id); 379 if (mz == NULL) { 380 PMD_DRV_LOG(ERR, "Failed to allocate memory for RX data ring"); 381 err = -ENOMEM; 382 goto err_rx_ring; 383 } 384 rxq->rx_data_ring = (union gve_rx_data_slot *)mz->addr; 385 rxq->data_mz = mz; 386 387 /* Allocate and register QPL for the queue. */ 388 if (rxq->is_gqi_qpl) { 389 rxq->qpl = gve_setup_queue_page_list(hw, queue_id, true, 390 nb_desc); 391 if (!rxq->qpl) { 392 err = -ENOMEM; 393 PMD_DRV_LOG(ERR, 394 "Failed to alloc rx qpl for queue %hu.", 395 queue_id); 396 goto err_data_ring; 397 } 398 } 399 400 mz = rte_eth_dma_zone_reserve(dev, "rxq_res", queue_id, 401 sizeof(struct gve_queue_resources), 402 PAGE_SIZE, socket_id); 403 if (mz == NULL) { 404 PMD_DRV_LOG(ERR, "Failed to reserve DMA memory for RX resource"); 405 err = -ENOMEM; 406 goto err_qpl; 407 } 408 rxq->qres = (struct gve_queue_resources *)mz->addr; 409 rxq->qres_mz = mz; 410 411 gve_reset_rxq(rxq); 412 413 dev->data->rx_queues[queue_id] = rxq; 414 415 return 0; 416 err_qpl: 417 if (rxq->is_gqi_qpl) { 418 gve_teardown_queue_page_list(hw, rxq->qpl); 419 rxq->qpl = NULL; 420 } 421 err_data_ring: 422 rte_memzone_free(rxq->data_mz); 423 err_rx_ring: 424 rte_memzone_free(rxq->mz); 425 err_sw_ring: 426 rte_free(rxq->sw_ring); 427 err_rxq: 428 rte_free(rxq); 429 return err; 430 } 431 432 static int 433 gve_rxq_mbufs_alloc(struct gve_rx_queue *rxq) 434 { 435 struct rte_mbuf *nmb; 436 uint16_t i; 437 int diag; 438 439 diag = rte_pktmbuf_alloc_bulk(rxq->mpool, &rxq->sw_ring[0], rxq->nb_rx_desc); 440 if (diag < 0) { 441 for (i = 0; i < rxq->nb_rx_desc - 1; i++) { 442 nmb = rte_pktmbuf_alloc(rxq->mpool); 443 if (!nmb) 444 break; 445 rxq->sw_ring[i] = nmb; 446 } 447 if (i < rxq->nb_rx_desc - 1) 448 return -ENOMEM; 449 } 450 rxq->nb_avail = 0; 451 rxq->next_avail = rxq->nb_rx_desc - 1; 452 453 for (i = 0; i < rxq->nb_rx_desc; i++) { 454 if (rxq->is_gqi_qpl) { 455 rxq->rx_data_ring[i].addr = rte_cpu_to_be_64(i * PAGE_SIZE); 456 } else { 457 if (i == rxq->nb_rx_desc - 1) 458 break; 459 nmb = rxq->sw_ring[i]; 460 rxq->rx_data_ring[i].addr = rte_cpu_to_be_64(rte_mbuf_data_iova(nmb)); 461 } 462 } 463 464 rte_write32(rte_cpu_to_be_32(rxq->next_avail), rxq->qrx_tail); 465 466 return 0; 467 } 468 469 int 470 gve_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id) 471 { 472 struct gve_priv *hw = dev->data->dev_private; 473 struct gve_rx_queue *rxq; 474 int ret; 475 476 if (rx_queue_id >= dev->data->nb_rx_queues) 477 return -EINVAL; 478 479 rxq = dev->data->rx_queues[rx_queue_id]; 480 481 rxq->qrx_tail = &hw->db_bar2[rte_be_to_cpu_32(rxq->qres->db_index)]; 482 483 rte_write32(rte_cpu_to_be_32(GVE_IRQ_MASK), rxq->ntfy_addr); 484 485 ret = gve_rxq_mbufs_alloc(rxq); 486 if (ret != 0) { 487 PMD_DRV_LOG(ERR, "Failed to alloc Rx queue mbuf"); 488 return ret; 489 } 490 491 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED; 492 493 return 0; 494 } 495 496 int 497 gve_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id) 498 { 499 struct gve_rx_queue *rxq; 500 501 if (rx_queue_id >= dev->data->nb_rx_queues) 502 return -EINVAL; 503 504 rxq = dev->data->rx_queues[rx_queue_id]; 505 gve_release_rxq_mbufs(rxq); 506 gve_reset_rxq(rxq); 507 508 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED; 509 510 return 0; 511 } 512 513 void 514 gve_stop_rx_queues(struct rte_eth_dev *dev) 515 { 516 struct gve_priv *hw = dev->data->dev_private; 517 uint16_t i; 518 int err; 519 520 if (!gve_is_gqi(hw)) 521 return gve_stop_rx_queues_dqo(dev); 522 523 err = gve_adminq_destroy_rx_queues(hw, dev->data->nb_rx_queues); 524 if (err != 0) 525 PMD_DRV_LOG(WARNING, "failed to destroy rxqs"); 526 527 for (i = 0; i < dev->data->nb_rx_queues; i++) 528 if (gve_rx_queue_stop(dev, i) != 0) 529 PMD_DRV_LOG(WARNING, "Fail to stop Rx queue %d", i); 530 } 531 532 void 533 gve_set_rx_function(struct rte_eth_dev *dev) 534 { 535 dev->rx_pkt_burst = gve_rx_burst; 536 } 537