1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(C) 2021 Marvell. 3 */ 4 5 #include <unistd.h> 6 #include <assert.h> 7 #include <rte_eal.h> 8 #include <rte_mempool.h> 9 #include <rte_mbuf.h> 10 #include <rte_io.h> 11 #include <rte_net.h> 12 #include <ethdev_pci.h> 13 14 #include "cnxk_ep_rx.h" 15 #include "otx_ep_common.h" 16 #include "otx_ep_vf.h" 17 #include "otx_ep_rxtx.h" 18 19 static void 20 otx_ep_dmazone_free(const struct rte_memzone *mz) 21 { 22 const struct rte_memzone *mz_tmp; 23 int ret = 0; 24 25 if (mz == NULL) { 26 otx_ep_err("Memzone: NULL"); 27 return; 28 } 29 30 mz_tmp = rte_memzone_lookup(mz->name); 31 if (mz_tmp == NULL) { 32 otx_ep_err("Memzone %s Not Found", mz->name); 33 return; 34 } 35 36 ret = rte_memzone_free(mz); 37 if (ret) 38 otx_ep_err("Memzone free failed : ret = %d", ret); 39 } 40 41 /* Free IQ resources */ 42 int 43 otx_ep_delete_iqs(struct otx_ep_device *otx_ep, uint32_t iq_no) 44 { 45 struct otx_ep_instr_queue *iq; 46 uint32_t i; 47 48 iq = otx_ep->instr_queue[iq_no]; 49 if (iq == NULL) { 50 otx_ep_err("Invalid IQ[%d]", iq_no); 51 return -EINVAL; 52 } 53 54 if (iq->req_list) { 55 for (i = 0; i < iq->nb_desc; i++) 56 rte_free(iq->req_list[i].finfo.g.sg); 57 rte_free(iq->req_list); 58 } 59 60 iq->req_list = NULL; 61 62 if (iq->iq_mz) { 63 otx_ep_dmazone_free(iq->iq_mz); 64 iq->iq_mz = NULL; 65 } 66 67 rte_free(otx_ep->instr_queue[iq_no]); 68 otx_ep->instr_queue[iq_no] = NULL; 69 70 otx_ep->nb_tx_queues--; 71 72 otx_ep_info("IQ[%d] is deleted", iq_no); 73 74 return 0; 75 } 76 77 /* IQ initialization */ 78 static int 79 otx_ep_init_instr_queue(struct otx_ep_device *otx_ep, int iq_no, int num_descs, 80 unsigned int socket_id) 81 { 82 const struct otx_ep_config *conf; 83 struct otx_ep_instr_queue *iq; 84 struct otx_ep_sg_entry *sg; 85 uint32_t i, q_size; 86 int ret; 87 88 conf = otx_ep->conf; 89 iq = otx_ep->instr_queue[iq_no]; 90 q_size = conf->iq.instr_type * num_descs; 91 92 /* IQ memory creation for Instruction submission to OCTEON 9 */ 93 iq->iq_mz = rte_eth_dma_zone_reserve(otx_ep->eth_dev, 94 "instr_queue", iq_no, q_size, 95 OTX_EP_PCI_RING_ALIGN, 96 socket_id); 97 if (iq->iq_mz == NULL) { 98 otx_ep_err("IQ[%d] memzone alloc failed", iq_no); 99 goto iq_init_fail; 100 } 101 102 iq->base_addr_dma = iq->iq_mz->iova; 103 iq->base_addr = (uint8_t *)iq->iq_mz->addr; 104 105 if (num_descs & (num_descs - 1)) { 106 otx_ep_err("IQ[%d] descs not in power of 2", iq_no); 107 goto iq_init_fail; 108 } 109 110 iq->nb_desc = num_descs; 111 112 /* Create a IQ request list to hold requests that have been 113 * posted to OCTEON 9. This list will be used for freeing the IQ 114 * data buffer(s) later once the OCTEON 9 fetched the requests. 115 */ 116 iq->req_list = rte_zmalloc_socket("request_list", 117 (iq->nb_desc * OTX_EP_IQREQ_LIST_SIZE), 118 RTE_CACHE_LINE_SIZE, 119 rte_socket_id()); 120 if (iq->req_list == NULL) { 121 otx_ep_err("IQ[%d] req_list alloc failed", iq_no); 122 goto iq_init_fail; 123 } 124 125 for (i = 0; i < iq->nb_desc; i++) { 126 sg = rte_zmalloc_socket("sg_entry", (OTX_EP_MAX_SG_LISTS * OTX_EP_SG_ENTRY_SIZE), 127 OTX_EP_SG_ALIGN, rte_socket_id()); 128 if (sg == NULL) { 129 otx_ep_err("IQ[%d] sg_entries alloc failed", iq_no); 130 goto iq_init_fail; 131 } 132 133 iq->req_list[i].finfo.g.num_sg = OTX_EP_MAX_SG_LISTS; 134 iq->req_list[i].finfo.g.sg = sg; 135 } 136 137 otx_ep_info("IQ[%d]: base: %p basedma: %lx count: %d", 138 iq_no, iq->base_addr, (unsigned long)iq->base_addr_dma, 139 iq->nb_desc); 140 141 iq->mbuf_list = rte_zmalloc_socket("mbuf_list", (iq->nb_desc * sizeof(struct rte_mbuf *)), 142 RTE_CACHE_LINE_SIZE, rte_socket_id()); 143 if (!iq->mbuf_list) { 144 otx_ep_err("IQ[%d] mbuf_list alloc failed", iq_no); 145 goto iq_init_fail; 146 } 147 148 iq->otx_ep_dev = otx_ep; 149 iq->q_no = iq_no; 150 iq->fill_cnt = 0; 151 iq->host_write_index = 0; 152 iq->otx_read_index = 0; 153 iq->flush_index = 0; 154 iq->instr_pending = 0; 155 156 otx_ep->io_qmask.iq |= (1ull << iq_no); 157 158 /* Set 32B/64B mode for each input queue */ 159 if (conf->iq.instr_type == 64) 160 otx_ep->io_qmask.iq64B |= (1ull << iq_no); 161 162 iq->iqcmd_64B = (conf->iq.instr_type == 64); 163 iq->ism_ena = otx_ep->ism_ena; 164 165 /* Set up IQ registers */ 166 ret = otx_ep->fn_list.setup_iq_regs(otx_ep, iq_no); 167 if (ret) 168 return ret; 169 170 return 0; 171 172 iq_init_fail: 173 return -ENOMEM; 174 } 175 176 int 177 otx_ep_setup_iqs(struct otx_ep_device *otx_ep, uint32_t iq_no, int num_descs, 178 unsigned int socket_id) 179 { 180 struct otx_ep_instr_queue *iq; 181 182 iq = (struct otx_ep_instr_queue *)rte_zmalloc("otx_ep_IQ", sizeof(*iq), 183 RTE_CACHE_LINE_SIZE); 184 if (iq == NULL) 185 return -ENOMEM; 186 187 otx_ep->instr_queue[iq_no] = iq; 188 189 if (otx_ep_init_instr_queue(otx_ep, iq_no, num_descs, socket_id)) { 190 otx_ep_err("IQ init is failed"); 191 goto delete_IQ; 192 } 193 otx_ep->nb_tx_queues++; 194 195 otx_ep_info("IQ[%d] is created.", iq_no); 196 197 return 0; 198 199 delete_IQ: 200 otx_ep_delete_iqs(otx_ep, iq_no); 201 return -ENOMEM; 202 } 203 204 static void 205 otx_ep_droq_reset_indices(struct otx_ep_droq *droq) 206 { 207 droq->read_idx = 0; 208 droq->write_idx = 0; 209 droq->refill_idx = 0; 210 droq->refill_count = 0; 211 droq->last_pkt_count = 0; 212 droq->pkts_pending = 0; 213 } 214 215 static void 216 otx_ep_droq_destroy_ring_buffers(struct otx_ep_droq *droq) 217 { 218 uint32_t idx; 219 220 for (idx = 0; idx < droq->nb_desc; idx++) { 221 if (droq->recv_buf_list[idx]) { 222 rte_pktmbuf_free(droq->recv_buf_list[idx]); 223 droq->recv_buf_list[idx] = NULL; 224 } 225 } 226 227 otx_ep_droq_reset_indices(droq); 228 } 229 230 /* Free OQs resources */ 231 int 232 otx_ep_delete_oqs(struct otx_ep_device *otx_ep, uint32_t oq_no) 233 { 234 struct otx_ep_droq *droq; 235 236 droq = otx_ep->droq[oq_no]; 237 if (droq == NULL) { 238 otx_ep_err("Invalid droq[%d]", oq_no); 239 return -EINVAL; 240 } 241 242 otx_ep_droq_destroy_ring_buffers(droq); 243 rte_free(droq->recv_buf_list); 244 droq->recv_buf_list = NULL; 245 246 if (droq->desc_ring_mz) { 247 otx_ep_dmazone_free(droq->desc_ring_mz); 248 droq->desc_ring_mz = NULL; 249 } 250 251 memset(droq, 0, OTX_EP_DROQ_SIZE); 252 253 rte_free(otx_ep->droq[oq_no]); 254 otx_ep->droq[oq_no] = NULL; 255 256 otx_ep->nb_rx_queues--; 257 258 otx_ep_info("OQ[%d] is deleted", oq_no); 259 return 0; 260 } 261 262 static int 263 otx_ep_droq_setup_ring_buffers(struct otx_ep_droq *droq) 264 { 265 struct otx_ep_droq_desc *desc_ring = droq->desc_ring; 266 struct otx_ep_droq_info *info; 267 struct rte_mbuf *buf; 268 uint32_t idx; 269 270 for (idx = 0; idx < droq->nb_desc; idx++) { 271 buf = rte_pktmbuf_alloc(droq->mpool); 272 if (buf == NULL) { 273 otx_ep_err("OQ buffer alloc failed"); 274 droq->stats.rx_alloc_failure++; 275 return -ENOMEM; 276 } 277 278 droq->recv_buf_list[idx] = buf; 279 info = rte_pktmbuf_mtod(buf, struct otx_ep_droq_info *); 280 memset(info, 0, sizeof(*info)); 281 desc_ring[idx].buffer_ptr = rte_mbuf_data_iova_default(buf); 282 } 283 284 otx_ep_droq_reset_indices(droq); 285 286 return 0; 287 } 288 289 static inline uint64_t 290 otx_ep_set_rearm_data(struct otx_ep_device *otx_ep) 291 { 292 uint16_t port_id = otx_ep->port_id; 293 struct rte_mbuf mb_def; 294 uint64_t *tmp; 295 296 RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_off) % 8 != 0); 297 RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, refcnt) - offsetof(struct rte_mbuf, data_off) != 298 2); 299 RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, nb_segs) - offsetof(struct rte_mbuf, data_off) != 300 4); 301 RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, port) - offsetof(struct rte_mbuf, data_off) != 302 6); 303 mb_def.nb_segs = 1; 304 mb_def.data_off = RTE_PKTMBUF_HEADROOM + OTX_EP_INFO_SIZE; 305 mb_def.port = port_id; 306 rte_mbuf_refcnt_set(&mb_def, 1); 307 308 /* Prevent compiler reordering: rearm_data covers previous fields */ 309 rte_compiler_barrier(); 310 tmp = (uint64_t *)&mb_def.rearm_data; 311 312 return *tmp; 313 } 314 315 /* OQ initialization */ 316 static int 317 otx_ep_init_droq(struct otx_ep_device *otx_ep, uint32_t q_no, 318 uint32_t num_descs, uint32_t desc_size, 319 struct rte_mempool *mpool, unsigned int socket_id) 320 { 321 const struct otx_ep_config *conf = otx_ep->conf; 322 uint32_t c_refill_threshold; 323 struct otx_ep_droq *droq; 324 uint32_t desc_ring_size; 325 int ret; 326 327 otx_ep_info("OQ[%d] Init start", q_no); 328 329 droq = otx_ep->droq[q_no]; 330 droq->otx_ep_dev = otx_ep; 331 droq->q_no = q_no; 332 droq->mpool = mpool; 333 334 droq->nb_desc = num_descs; 335 droq->buffer_size = desc_size; 336 c_refill_threshold = RTE_MAX(conf->oq.refill_threshold, 337 droq->nb_desc / 2); 338 339 /* OQ desc_ring set up */ 340 desc_ring_size = droq->nb_desc * OTX_EP_DROQ_DESC_SIZE; 341 droq->desc_ring_mz = rte_eth_dma_zone_reserve(otx_ep->eth_dev, "droq", 342 q_no, desc_ring_size, 343 OTX_EP_PCI_RING_ALIGN, 344 socket_id); 345 346 if (droq->desc_ring_mz == NULL) { 347 otx_ep_err("OQ:%d desc_ring allocation failed", q_no); 348 goto init_droq_fail; 349 } 350 351 droq->desc_ring_dma = droq->desc_ring_mz->iova; 352 droq->desc_ring = (struct otx_ep_droq_desc *)droq->desc_ring_mz->addr; 353 354 otx_ep_dbg("OQ[%d]: desc_ring: virt: 0x%p, dma: %lx", 355 q_no, droq->desc_ring, (unsigned long)droq->desc_ring_dma); 356 otx_ep_dbg("OQ[%d]: num_desc: %d", q_no, droq->nb_desc); 357 358 /* OQ buf_list set up */ 359 droq->recv_buf_list = rte_zmalloc_socket("recv_buf_list", 360 (droq->nb_desc * sizeof(struct rte_mbuf *)), 361 RTE_CACHE_LINE_SIZE, socket_id); 362 if (droq->recv_buf_list == NULL) { 363 otx_ep_err("OQ recv_buf_list alloc failed"); 364 goto init_droq_fail; 365 } 366 367 if (otx_ep_droq_setup_ring_buffers(droq)) 368 goto init_droq_fail; 369 370 droq->refill_threshold = c_refill_threshold; 371 droq->rearm_data = otx_ep_set_rearm_data(otx_ep); 372 droq->ism_ena = otx_ep->ism_ena; 373 374 /* Set up OQ registers */ 375 ret = otx_ep->fn_list.setup_oq_regs(otx_ep, q_no); 376 if (ret) 377 return ret; 378 379 otx_ep->io_qmask.oq |= (1ull << q_no); 380 381 return 0; 382 383 init_droq_fail: 384 return -ENOMEM; 385 } 386 387 /* OQ configuration and setup */ 388 int 389 otx_ep_setup_oqs(struct otx_ep_device *otx_ep, int oq_no, int num_descs, 390 int desc_size, struct rte_mempool *mpool, 391 unsigned int socket_id) 392 { 393 struct otx_ep_droq *droq; 394 395 /* Allocate new droq. */ 396 droq = (struct otx_ep_droq *)rte_zmalloc("otx_ep_OQ", 397 sizeof(*droq), RTE_CACHE_LINE_SIZE); 398 if (droq == NULL) { 399 otx_ep_err("Droq[%d] Creation Failed", oq_no); 400 return -ENOMEM; 401 } 402 otx_ep->droq[oq_no] = droq; 403 404 if (otx_ep_init_droq(otx_ep, oq_no, num_descs, desc_size, mpool, 405 socket_id)) { 406 otx_ep_err("Droq[%d] Initialization failed", oq_no); 407 goto delete_OQ; 408 } 409 otx_ep_info("OQ[%d] is created.", oq_no); 410 411 otx_ep->nb_rx_queues++; 412 413 return 0; 414 415 delete_OQ: 416 otx_ep_delete_oqs(otx_ep, oq_no); 417 return -ENOMEM; 418 } 419 420 static inline void 421 otx_ep_iqreq_delete(struct otx_ep_instr_queue *iq, uint32_t idx) 422 { 423 struct rte_mbuf *mbuf; 424 uint32_t reqtype; 425 426 mbuf = iq->req_list[idx].finfo.mbuf; 427 reqtype = iq->req_list[idx].reqtype; 428 429 switch (reqtype) { 430 case OTX_EP_REQTYPE_NORESP_NET: 431 case OTX_EP_REQTYPE_NORESP_GATHER: 432 /* This will take care of multiple segments also */ 433 rte_pktmbuf_free(mbuf); 434 otx_ep_dbg("IQ buffer freed at idx[%d]", idx); 435 break; 436 437 case OTX_EP_REQTYPE_NONE: 438 default: 439 otx_ep_info("This iqreq mode is not supported:%d", reqtype); 440 } 441 442 /* Reset the request list at this index */ 443 iq->req_list[idx].finfo.mbuf = NULL; 444 iq->req_list[idx].reqtype = 0; 445 } 446 447 static inline void 448 otx_ep_iqreq_add(struct otx_ep_instr_queue *iq, struct rte_mbuf *mbuf, 449 uint32_t reqtype, int index) 450 { 451 iq->req_list[index].finfo.mbuf = mbuf; 452 iq->req_list[index].reqtype = reqtype; 453 } 454 455 static uint32_t 456 otx_vf_update_read_index(struct otx_ep_instr_queue *iq) 457 { 458 uint32_t val; 459 460 /* 461 * Batch subtractions from the HW counter to reduce PCIe traffic 462 * This adds an extra local variable, but almost halves the 463 * number of PCIe writes. 464 */ 465 val = *iq->inst_cnt_ism; 466 iq->inst_cnt += val - iq->inst_cnt_prev; 467 iq->inst_cnt_prev = val; 468 469 if (val > (uint32_t)(1 << 31)) { 470 /* 471 * Only subtract the packet count in the HW counter 472 * when count above halfway to saturation. 473 */ 474 rte_write32(val, iq->inst_cnt_reg); 475 rte_mb(); 476 477 rte_write64(OTX2_SDP_REQUEST_ISM, iq->inst_cnt_reg); 478 while (rte_atomic_load_explicit(iq->inst_cnt_ism, 479 rte_memory_order_relaxed) >= val) { 480 rte_write64(OTX2_SDP_REQUEST_ISM, iq->inst_cnt_reg); 481 rte_mb(); 482 } 483 484 iq->inst_cnt_prev = 0; 485 } 486 rte_write64(OTX2_SDP_REQUEST_ISM, iq->inst_cnt_reg); 487 488 /* Modulo of the new index with the IQ size will give us 489 * the new index. 490 */ 491 return iq->inst_cnt & (iq->nb_desc - 1); 492 } 493 494 static void 495 otx_ep_flush_iq(struct otx_ep_instr_queue *iq) 496 { 497 uint32_t instr_processed = 0; 498 499 iq->otx_read_index = otx_vf_update_read_index(iq); 500 while (iq->flush_index != iq->otx_read_index) { 501 /* Free the IQ data buffer to the pool */ 502 otx_ep_iqreq_delete(iq, iq->flush_index); 503 iq->flush_index = 504 otx_ep_incr_index(iq->flush_index, 1, iq->nb_desc); 505 506 instr_processed++; 507 } 508 509 iq->stats.instr_processed = instr_processed; 510 iq->instr_pending -= instr_processed; 511 } 512 513 static inline void 514 otx_ep_ring_doorbell(struct otx_ep_device *otx_ep __rte_unused, 515 struct otx_ep_instr_queue *iq) 516 { 517 rte_wmb(); 518 rte_write64(iq->fill_cnt, iq->doorbell_reg); 519 iq->fill_cnt = 0; 520 } 521 522 static inline int 523 post_iqcmd(struct otx_ep_instr_queue *iq, uint8_t *iqcmd) 524 { 525 uint8_t *iqptr, cmdsize; 526 527 /* This ensures that the read index does not wrap around to 528 * the same position if queue gets full before OCTEON 9 could 529 * fetch any instr. 530 */ 531 if (iq->instr_pending > (iq->nb_desc - 1)) 532 return OTX_EP_IQ_SEND_FAILED; 533 534 /* Copy cmd into iq */ 535 cmdsize = 64; 536 iqptr = iq->base_addr + (iq->host_write_index << 6); 537 538 rte_memcpy(iqptr, iqcmd, cmdsize); 539 540 /* Increment the host write index */ 541 iq->host_write_index = 542 otx_ep_incr_index(iq->host_write_index, 1, iq->nb_desc); 543 544 iq->fill_cnt++; 545 546 /* Flush the command into memory. We need to be sure the data 547 * is in memory before indicating that the instruction is 548 * pending. 549 */ 550 iq->instr_pending++; 551 /* OTX_EP_IQ_SEND_SUCCESS */ 552 return 0; 553 } 554 555 556 static int 557 otx_ep_send_data(struct otx_ep_device *otx_ep, struct otx_ep_instr_queue *iq, 558 void *cmd, int dbell) 559 { 560 uint32_t ret; 561 562 /* Submit IQ command */ 563 ret = post_iqcmd(iq, cmd); 564 565 if (ret == OTX_EP_IQ_SEND_SUCCESS) { 566 if (dbell) 567 otx_ep_ring_doorbell(otx_ep, iq); 568 iq->stats.instr_posted++; 569 570 } else { 571 iq->stats.instr_dropped++; 572 if (iq->fill_cnt) 573 otx_ep_ring_doorbell(otx_ep, iq); 574 } 575 return ret; 576 } 577 578 static inline void 579 set_sg_size(struct otx_ep_sg_entry *sg_entry, uint16_t size, uint32_t pos) 580 { 581 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN 582 sg_entry->u.size[pos] = size; 583 #elif RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN 584 sg_entry->u.size[(OTX_EP_NUM_SG_PTRS - 1) - pos] = size; 585 #endif 586 } 587 588 static inline int 589 prepare_xmit_gather_list(struct otx_ep_instr_queue *iq, struct rte_mbuf *m, uint64_t *dptr, 590 union otx_ep_instr_ih *ih) 591 { 592 uint16_t j = 0, frags, num_sg, mask = OTX_EP_NUM_SG_PTRS - 1; 593 struct otx_ep_buf_free_info *finfo; 594 uint32_t pkt_len; 595 int rc = -1; 596 597 pkt_len = rte_pktmbuf_pkt_len(m); 598 frags = m->nb_segs; 599 num_sg = (frags + mask) / OTX_EP_NUM_SG_PTRS; 600 601 if (unlikely(pkt_len > OTX_EP_MAX_PKT_SZ && num_sg > OTX_EP_MAX_SG_LISTS)) { 602 otx_ep_err("Failed to xmit the pkt, pkt_len is higher or pkt has more segments"); 603 goto exit; 604 } 605 606 finfo = &iq->req_list[iq->host_write_index].finfo; 607 *dptr = rte_mem_virt2iova(finfo->g.sg); 608 ih->u64 |= ((1ULL << 62) | ((uint64_t)frags << 48) | (pkt_len + ih->s.fsz)); 609 610 while (frags--) { 611 finfo->g.sg[(j >> 2)].ptr[(j & mask)] = rte_mbuf_data_iova(m); 612 set_sg_size(&finfo->g.sg[(j >> 2)], m->data_len, (j & mask)); 613 j++; 614 m = m->next; 615 } 616 617 return 0; 618 619 exit: 620 return rc; 621 } 622 623 /* Enqueue requests/packets to OTX_EP IQ queue. 624 * returns number of requests enqueued successfully 625 */ 626 uint16_t 627 otx_ep_xmit_pkts(void *tx_queue, struct rte_mbuf **pkts, uint16_t nb_pkts) 628 { 629 struct otx_ep_instr_queue *iq = (struct otx_ep_instr_queue *)tx_queue; 630 struct otx_ep_device *otx_ep = iq->otx_ep_dev; 631 struct otx_ep_instr_64B iqcmd; 632 int dbell, index, count = 0; 633 uint32_t iqreq_type; 634 uint32_t pkt_len, i; 635 struct rte_mbuf *m; 636 637 iqcmd.ih.u64 = 0; 638 iqcmd.pki_ih3.u64 = 0; 639 iqcmd.irh.u64 = 0; 640 641 /* ih invars */ 642 iqcmd.ih.s.fsz = OTX_EP_FSZ; 643 iqcmd.ih.s.pkind = otx_ep->pkind; /* The SDK decided PKIND value */ 644 645 /* pki ih3 invars */ 646 iqcmd.pki_ih3.s.w = 1; 647 iqcmd.pki_ih3.s.utt = 1; 648 iqcmd.pki_ih3.s.tagtype = ORDERED_TAG; 649 /* sl will be sizeof(pki_ih3) */ 650 iqcmd.pki_ih3.s.sl = OTX_EP_FSZ + OTX_CUST_DATA_LEN; 651 652 /* irh invars */ 653 iqcmd.irh.s.opcode = OTX_EP_NW_PKT_OP; 654 655 for (i = 0; i < nb_pkts; i++) { 656 m = pkts[i]; 657 if (m->nb_segs == 1) { 658 pkt_len = rte_pktmbuf_data_len(m); 659 iqcmd.ih.s.tlen = pkt_len + iqcmd.ih.s.fsz; 660 iqcmd.dptr = rte_mbuf_data_iova(m); /*dptr*/ 661 iqcmd.ih.s.gather = 0; 662 iqcmd.ih.s.gsz = 0; 663 iqreq_type = OTX_EP_REQTYPE_NORESP_NET; 664 } else { 665 if (!(otx_ep->tx_offloads & RTE_ETH_TX_OFFLOAD_MULTI_SEGS)) 666 goto xmit_fail; 667 668 if (unlikely(prepare_xmit_gather_list(iq, m, &iqcmd.dptr, &iqcmd.ih) < 0)) 669 goto xmit_fail; 670 671 pkt_len = rte_pktmbuf_pkt_len(m); 672 iqreq_type = OTX_EP_REQTYPE_NORESP_GATHER; 673 } 674 675 iqcmd.irh.u64 = rte_bswap64(iqcmd.irh.u64); 676 677 #ifdef OTX_EP_IO_DEBUG 678 otx_ep_dbg("After swapping"); 679 otx_ep_dbg("Word0 [dptr]: 0x%016lx", 680 (unsigned long)iqcmd.dptr); 681 otx_ep_dbg("Word1 [ihtx]: 0x%016lx", (unsigned long)iqcmd.ih); 682 otx_ep_dbg("Word2 [pki_ih3]: 0x%016lx", 683 (unsigned long)iqcmd.pki_ih3); 684 otx_ep_dbg("Word3 [rptr]: 0x%016lx", 685 (unsigned long)iqcmd.rptr); 686 otx_ep_dbg("Word4 [irh]: 0x%016lx", (unsigned long)iqcmd.irh); 687 otx_ep_dbg("Word5 [exhdr[0]]: 0x%016lx", 688 (unsigned long)iqcmd.exhdr[0]); 689 rte_pktmbuf_dump(stdout, m, rte_pktmbuf_pkt_len(m)); 690 #endif 691 dbell = (i == (unsigned int)(nb_pkts - 1)) ? 1 : 0; 692 index = iq->host_write_index; 693 if (otx_ep_send_data(otx_ep, iq, &iqcmd, dbell)) 694 goto xmit_fail; 695 otx_ep_iqreq_add(iq, m, iqreq_type, index); 696 iq->stats.tx_pkts++; 697 iq->stats.tx_bytes += pkt_len; 698 count++; 699 } 700 701 xmit_fail: 702 if (iq->instr_pending >= OTX_EP_MAX_INSTR) 703 otx_ep_flush_iq(iq); 704 705 /* Return no# of instructions posted successfully. */ 706 return count; 707 } 708 709 static uint32_t 710 otx_ep_droq_refill(struct otx_ep_droq *droq) 711 { 712 struct otx_ep_droq_desc *desc_ring = droq->desc_ring; 713 struct otx_ep_droq_info *info; 714 struct rte_mbuf *buf = NULL; 715 uint32_t desc_refilled = 0; 716 717 while (droq->refill_count && (desc_refilled < droq->nb_desc)) { 718 buf = rte_pktmbuf_alloc(droq->mpool); 719 /* If a buffer could not be allocated, no point in 720 * continuing 721 */ 722 if (unlikely(!buf)) { 723 droq->stats.rx_alloc_failure++; 724 break; 725 } 726 info = rte_pktmbuf_mtod(buf, struct otx_ep_droq_info *); 727 info->length = 0; 728 729 droq->recv_buf_list[droq->refill_idx] = buf; 730 desc_ring[droq->refill_idx].buffer_ptr = 731 rte_mbuf_data_iova_default(buf); 732 droq->refill_idx = otx_ep_incr_index(droq->refill_idx, 1, 733 droq->nb_desc); 734 735 desc_refilled++; 736 droq->refill_count--; 737 } 738 739 return desc_refilled; 740 } 741 742 static struct rte_mbuf * 743 otx_ep_droq_read_packet(struct otx_ep_device *otx_ep, struct otx_ep_droq *droq, int next_fetch) 744 { 745 volatile struct otx_ep_droq_info *info; 746 struct rte_mbuf *mbuf_next = NULL; 747 struct rte_mbuf *mbuf = NULL; 748 uint64_t total_pkt_len; 749 uint32_t pkt_len = 0; 750 int next_idx; 751 752 mbuf = droq->recv_buf_list[droq->read_idx]; 753 info = rte_pktmbuf_mtod(mbuf, struct otx_ep_droq_info *); 754 755 /* make sure info is available */ 756 rte_rmb(); 757 if (unlikely(!info->length)) { 758 int retry = OTX_EP_MAX_DELAYED_PKT_RETRIES; 759 /* otx_ep_dbg("OCTEON DROQ[%d]: read_idx: %d; Data not ready " 760 * "yet, Retry; pending=%lu", droq->q_no, droq->read_idx, 761 * droq->pkts_pending); 762 */ 763 droq->stats.pkts_delayed_data++; 764 while (retry && !info->length) { 765 retry--; 766 rte_delay_us_block(50); 767 } 768 if (!retry && !info->length) { 769 otx_ep_err("OCTEON DROQ[%d]: read_idx: %d; Retry failed !!", 770 droq->q_no, droq->read_idx); 771 /* May be zero length packet; drop it */ 772 assert(0); 773 } 774 } 775 776 if (next_fetch) { 777 next_idx = otx_ep_incr_index(droq->read_idx, 1, droq->nb_desc); 778 mbuf_next = droq->recv_buf_list[next_idx]; 779 rte_prefetch0(rte_pktmbuf_mtod(mbuf_next, void *)); 780 } 781 782 info->length = rte_bswap16(info->length >> 48); 783 /* Deduce the actual data size */ 784 total_pkt_len = info->length + OTX_EP_INFO_SIZE; 785 if (total_pkt_len <= droq->buffer_size) { 786 mbuf->data_off += OTX_EP_INFO_SIZE; 787 pkt_len = (uint32_t)info->length; 788 mbuf->pkt_len = pkt_len; 789 mbuf->data_len = pkt_len; 790 mbuf->port = otx_ep->port_id; 791 droq->recv_buf_list[droq->read_idx] = NULL; 792 droq->read_idx = otx_ep_incr_index(droq->read_idx, 1, droq->nb_desc); 793 droq->refill_count++; 794 } else { 795 struct rte_mbuf *first_buf = NULL; 796 struct rte_mbuf *last_buf = NULL; 797 798 /* csr read helps to flush pending dma */ 799 droq->sent_reg_val = rte_read32(droq->pkts_sent_reg); 800 rte_rmb(); 801 802 while (pkt_len < total_pkt_len) { 803 int cpy_len = 0; 804 805 cpy_len = ((pkt_len + droq->buffer_size) > total_pkt_len) 806 ? ((uint32_t)total_pkt_len - pkt_len) 807 : droq->buffer_size; 808 809 mbuf = droq->recv_buf_list[droq->read_idx]; 810 droq->recv_buf_list[droq->read_idx] = NULL; 811 812 if (likely(mbuf)) { 813 /* Note the first seg */ 814 if (!pkt_len) 815 first_buf = mbuf; 816 817 mbuf->port = otx_ep->port_id; 818 if (!pkt_len) { 819 mbuf->data_off += OTX_EP_INFO_SIZE; 820 mbuf->pkt_len = cpy_len - OTX_EP_INFO_SIZE; 821 mbuf->data_len = cpy_len - OTX_EP_INFO_SIZE; 822 } else { 823 mbuf->pkt_len = cpy_len; 824 mbuf->data_len = cpy_len; 825 } 826 827 if (pkt_len) { 828 first_buf->nb_segs++; 829 first_buf->pkt_len += mbuf->pkt_len; 830 } 831 832 if (last_buf) 833 last_buf->next = mbuf; 834 835 last_buf = mbuf; 836 } else { 837 otx_ep_err("no buf"); 838 assert(0); 839 } 840 841 pkt_len += cpy_len; 842 droq->read_idx = otx_ep_incr_index(droq->read_idx, 1, droq->nb_desc); 843 droq->refill_count++; 844 } 845 mbuf = first_buf; 846 } 847 848 return mbuf; 849 } 850 851 static inline uint32_t 852 otx_ep_check_droq_pkts(struct otx_ep_droq *droq) 853 { 854 uint32_t new_pkts; 855 uint32_t val; 856 857 /* 858 * Batch subtractions from the HW counter to reduce PCIe traffic 859 * This adds an extra local variable, but almost halves the 860 * number of PCIe writes. 861 */ 862 val = *droq->pkts_sent_ism; 863 new_pkts = val - droq->pkts_sent_prev; 864 droq->pkts_sent_prev = val; 865 866 if (val > (uint32_t)(1 << 31)) { 867 /* 868 * Only subtract the packet count in the HW counter 869 * when count above halfway to saturation. 870 */ 871 rte_write32(val, droq->pkts_sent_reg); 872 rte_mb(); 873 874 rte_write64(OTX2_SDP_REQUEST_ISM, droq->pkts_sent_reg); 875 while (rte_atomic_load_explicit(droq->pkts_sent_ism, 876 rte_memory_order_relaxed) >= val) { 877 rte_write64(OTX2_SDP_REQUEST_ISM, droq->pkts_sent_reg); 878 rte_mb(); 879 } 880 881 droq->pkts_sent_prev = 0; 882 } 883 rte_write64(OTX2_SDP_REQUEST_ISM, droq->pkts_sent_reg); 884 droq->pkts_pending += new_pkts; 885 886 return new_pkts; 887 } 888 889 static inline int32_t __rte_hot 890 otx_ep_rx_pkts_to_process(struct otx_ep_droq *droq, uint16_t nb_pkts) 891 { 892 if (unlikely(droq->pkts_pending < nb_pkts)) 893 otx_ep_check_droq_pkts(droq); 894 895 return RTE_MIN(nb_pkts, droq->pkts_pending); 896 } 897 898 /* Check for response arrival from OCTEON 9 899 * returns number of requests completed 900 */ 901 uint16_t 902 otx_ep_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) 903 { 904 struct otx_ep_droq *droq = rx_queue; 905 struct otx_ep_device *otx_ep; 906 struct rte_mbuf *oq_pkt; 907 uint16_t pkts, new_pkts; 908 uint32_t valid_pkts = 0; 909 int next_fetch; 910 911 otx_ep = droq->otx_ep_dev; 912 new_pkts = otx_ep_rx_pkts_to_process(droq, nb_pkts); 913 914 for (pkts = 0; pkts < new_pkts; pkts++) { 915 /* Push the received pkt to application */ 916 next_fetch = (pkts == new_pkts - 1) ? 0 : 1; 917 oq_pkt = otx_ep_droq_read_packet(otx_ep, droq, next_fetch); 918 if (!oq_pkt) { 919 RTE_LOG_DP_LINE(ERR, OTX_NET_EP, 920 "DROQ read pkt failed pending %" PRIu64 921 "last_pkt_count %" PRIu64 "new_pkts %d.", 922 droq->pkts_pending, droq->last_pkt_count, 923 new_pkts); 924 droq->stats.rx_err++; 925 continue; 926 } else { 927 rx_pkts[valid_pkts] = oq_pkt; 928 valid_pkts++; 929 /* Stats */ 930 droq->stats.pkts_received++; 931 droq->stats.bytes_received += oq_pkt->pkt_len; 932 } 933 } 934 droq->pkts_pending -= pkts; 935 936 /* Refill DROQ buffers */ 937 if (droq->refill_count >= DROQ_REFILL_THRESHOLD) { 938 int desc_refilled = otx_ep_droq_refill(droq); 939 940 /* Flush the droq descriptor data to memory to be sure 941 * that when we update the credits the data in memory is 942 * accurate. 943 */ 944 rte_io_wmb(); 945 rte_write32(desc_refilled, droq->pkts_credit_reg); 946 } else { 947 /* 948 * SDP output goes into DROP state when output doorbell count 949 * goes below drop count. When door bell count is written with 950 * a value greater than drop count SDP output should come out 951 * of DROP state. Due to a race condition this is not happening. 952 * Writing doorbell register with 0 again may make SDP output 953 * come out of this state. 954 */ 955 956 rte_write32(0, droq->pkts_credit_reg); 957 } 958 return valid_pkts; 959 } 960