1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(C) 2021 Marvell. 3 */ 4 5 #include <unistd.h> 6 #include <assert.h> 7 #include <rte_eal.h> 8 #include <rte_mempool.h> 9 #include <rte_mbuf.h> 10 #include <rte_io.h> 11 #include <rte_net.h> 12 #include <ethdev_pci.h> 13 14 #include "cnxk_ep_rx.h" 15 #include "otx_ep_common.h" 16 #include "otx_ep_vf.h" 17 #include "otx_ep_rxtx.h" 18 19 static void 20 otx_ep_dmazone_free(const struct rte_memzone *mz) 21 { 22 const struct rte_memzone *mz_tmp; 23 int ret = 0; 24 25 if (mz == NULL) { 26 otx_ep_err("Memzone: NULL\n"); 27 return; 28 } 29 30 mz_tmp = rte_memzone_lookup(mz->name); 31 if (mz_tmp == NULL) { 32 otx_ep_err("Memzone %s Not Found\n", mz->name); 33 return; 34 } 35 36 ret = rte_memzone_free(mz); 37 if (ret) 38 otx_ep_err("Memzone free failed : ret = %d\n", ret); 39 } 40 41 /* Free IQ resources */ 42 int 43 otx_ep_delete_iqs(struct otx_ep_device *otx_ep, uint32_t iq_no) 44 { 45 struct otx_ep_instr_queue *iq; 46 uint32_t i; 47 48 iq = otx_ep->instr_queue[iq_no]; 49 if (iq == NULL) { 50 otx_ep_err("Invalid IQ[%d]\n", iq_no); 51 return -EINVAL; 52 } 53 54 if (iq->req_list) { 55 for (i = 0; i < iq->nb_desc; i++) 56 rte_free(iq->req_list[i].finfo.g.sg); 57 rte_free(iq->req_list); 58 } 59 60 iq->req_list = NULL; 61 62 if (iq->iq_mz) { 63 otx_ep_dmazone_free(iq->iq_mz); 64 iq->iq_mz = NULL; 65 } 66 67 rte_free(otx_ep->instr_queue[iq_no]); 68 otx_ep->instr_queue[iq_no] = NULL; 69 70 otx_ep->nb_tx_queues--; 71 72 otx_ep_info("IQ[%d] is deleted\n", iq_no); 73 74 return 0; 75 } 76 77 /* IQ initialization */ 78 static int 79 otx_ep_init_instr_queue(struct otx_ep_device *otx_ep, int iq_no, int num_descs, 80 unsigned int socket_id) 81 { 82 const struct otx_ep_config *conf; 83 struct otx_ep_instr_queue *iq; 84 struct otx_ep_sg_entry *sg; 85 uint32_t i, q_size; 86 int ret; 87 88 conf = otx_ep->conf; 89 iq = otx_ep->instr_queue[iq_no]; 90 q_size = conf->iq.instr_type * num_descs; 91 92 /* IQ memory creation for Instruction submission to OCTEON 9 */ 93 iq->iq_mz = rte_eth_dma_zone_reserve(otx_ep->eth_dev, 94 "instr_queue", iq_no, q_size, 95 OTX_EP_PCI_RING_ALIGN, 96 socket_id); 97 if (iq->iq_mz == NULL) { 98 otx_ep_err("IQ[%d] memzone alloc failed\n", iq_no); 99 goto iq_init_fail; 100 } 101 102 iq->base_addr_dma = iq->iq_mz->iova; 103 iq->base_addr = (uint8_t *)iq->iq_mz->addr; 104 105 if (num_descs & (num_descs - 1)) { 106 otx_ep_err("IQ[%d] descs not in power of 2\n", iq_no); 107 goto iq_init_fail; 108 } 109 110 iq->nb_desc = num_descs; 111 112 /* Create a IQ request list to hold requests that have been 113 * posted to OCTEON 9. This list will be used for freeing the IQ 114 * data buffer(s) later once the OCTEON 9 fetched the requests. 115 */ 116 iq->req_list = rte_zmalloc_socket("request_list", 117 (iq->nb_desc * OTX_EP_IQREQ_LIST_SIZE), 118 RTE_CACHE_LINE_SIZE, 119 rte_socket_id()); 120 if (iq->req_list == NULL) { 121 otx_ep_err("IQ[%d] req_list alloc failed\n", iq_no); 122 goto iq_init_fail; 123 } 124 125 for (i = 0; i < iq->nb_desc; i++) { 126 sg = rte_zmalloc_socket("sg_entry", (OTX_EP_MAX_SG_LISTS * OTX_EP_SG_ENTRY_SIZE), 127 OTX_EP_SG_ALIGN, rte_socket_id()); 128 if (sg == NULL) { 129 otx_ep_err("IQ[%d] sg_entries alloc failed\n", iq_no); 130 goto iq_init_fail; 131 } 132 133 iq->req_list[i].finfo.g.num_sg = OTX_EP_MAX_SG_LISTS; 134 iq->req_list[i].finfo.g.sg = sg; 135 } 136 137 otx_ep_info("IQ[%d]: base: %p basedma: %lx count: %d\n", 138 iq_no, iq->base_addr, (unsigned long)iq->base_addr_dma, 139 iq->nb_desc); 140 141 iq->mbuf_list = rte_zmalloc_socket("mbuf_list", (iq->nb_desc * sizeof(struct rte_mbuf *)), 142 RTE_CACHE_LINE_SIZE, rte_socket_id()); 143 if (!iq->mbuf_list) { 144 otx_ep_err("IQ[%d] mbuf_list alloc failed\n", iq_no); 145 goto iq_init_fail; 146 } 147 148 iq->otx_ep_dev = otx_ep; 149 iq->q_no = iq_no; 150 iq->fill_cnt = 0; 151 iq->host_write_index = 0; 152 iq->otx_read_index = 0; 153 iq->flush_index = 0; 154 iq->instr_pending = 0; 155 156 otx_ep->io_qmask.iq |= (1ull << iq_no); 157 158 /* Set 32B/64B mode for each input queue */ 159 if (conf->iq.instr_type == 64) 160 otx_ep->io_qmask.iq64B |= (1ull << iq_no); 161 162 iq->iqcmd_64B = (conf->iq.instr_type == 64); 163 iq->ism_ena = otx_ep->ism_ena; 164 165 /* Set up IQ registers */ 166 ret = otx_ep->fn_list.setup_iq_regs(otx_ep, iq_no); 167 if (ret) 168 return ret; 169 170 return 0; 171 172 iq_init_fail: 173 return -ENOMEM; 174 } 175 176 int 177 otx_ep_setup_iqs(struct otx_ep_device *otx_ep, uint32_t iq_no, int num_descs, 178 unsigned int socket_id) 179 { 180 struct otx_ep_instr_queue *iq; 181 182 iq = (struct otx_ep_instr_queue *)rte_zmalloc("otx_ep_IQ", sizeof(*iq), 183 RTE_CACHE_LINE_SIZE); 184 if (iq == NULL) 185 return -ENOMEM; 186 187 otx_ep->instr_queue[iq_no] = iq; 188 189 if (otx_ep_init_instr_queue(otx_ep, iq_no, num_descs, socket_id)) { 190 otx_ep_err("IQ init is failed\n"); 191 goto delete_IQ; 192 } 193 otx_ep->nb_tx_queues++; 194 195 otx_ep_info("IQ[%d] is created.\n", iq_no); 196 197 return 0; 198 199 delete_IQ: 200 otx_ep_delete_iqs(otx_ep, iq_no); 201 return -ENOMEM; 202 } 203 204 static void 205 otx_ep_droq_reset_indices(struct otx_ep_droq *droq) 206 { 207 droq->read_idx = 0; 208 droq->write_idx = 0; 209 droq->refill_idx = 0; 210 droq->refill_count = 0; 211 droq->last_pkt_count = 0; 212 droq->pkts_pending = 0; 213 } 214 215 static void 216 otx_ep_droq_destroy_ring_buffers(struct otx_ep_droq *droq) 217 { 218 uint32_t idx; 219 220 for (idx = 0; idx < droq->nb_desc; idx++) { 221 if (droq->recv_buf_list[idx]) { 222 rte_pktmbuf_free(droq->recv_buf_list[idx]); 223 droq->recv_buf_list[idx] = NULL; 224 } 225 } 226 227 otx_ep_droq_reset_indices(droq); 228 } 229 230 /* Free OQs resources */ 231 int 232 otx_ep_delete_oqs(struct otx_ep_device *otx_ep, uint32_t oq_no) 233 { 234 struct otx_ep_droq *droq; 235 236 droq = otx_ep->droq[oq_no]; 237 if (droq == NULL) { 238 otx_ep_err("Invalid droq[%d]\n", oq_no); 239 return -EINVAL; 240 } 241 242 otx_ep_droq_destroy_ring_buffers(droq); 243 rte_free(droq->recv_buf_list); 244 droq->recv_buf_list = NULL; 245 246 if (droq->desc_ring_mz) { 247 otx_ep_dmazone_free(droq->desc_ring_mz); 248 droq->desc_ring_mz = NULL; 249 } 250 251 memset(droq, 0, OTX_EP_DROQ_SIZE); 252 253 rte_free(otx_ep->droq[oq_no]); 254 otx_ep->droq[oq_no] = NULL; 255 256 otx_ep->nb_rx_queues--; 257 258 otx_ep_info("OQ[%d] is deleted\n", oq_no); 259 return 0; 260 } 261 262 static int 263 otx_ep_droq_setup_ring_buffers(struct otx_ep_droq *droq) 264 { 265 struct otx_ep_droq_desc *desc_ring = droq->desc_ring; 266 struct otx_ep_droq_info *info; 267 struct rte_mbuf *buf; 268 uint32_t idx; 269 270 for (idx = 0; idx < droq->nb_desc; idx++) { 271 buf = rte_pktmbuf_alloc(droq->mpool); 272 if (buf == NULL) { 273 otx_ep_err("OQ buffer alloc failed\n"); 274 droq->stats.rx_alloc_failure++; 275 return -ENOMEM; 276 } 277 278 droq->recv_buf_list[idx] = buf; 279 info = rte_pktmbuf_mtod(buf, struct otx_ep_droq_info *); 280 memset(info, 0, sizeof(*info)); 281 desc_ring[idx].buffer_ptr = rte_mbuf_data_iova_default(buf); 282 } 283 284 otx_ep_droq_reset_indices(droq); 285 286 return 0; 287 } 288 289 static inline uint64_t 290 otx_ep_set_rearm_data(struct otx_ep_device *otx_ep) 291 { 292 uint16_t port_id = otx_ep->port_id; 293 struct rte_mbuf mb_def; 294 uint64_t *tmp; 295 296 RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_off) % 8 != 0); 297 RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, refcnt) - offsetof(struct rte_mbuf, data_off) != 298 2); 299 RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, nb_segs) - offsetof(struct rte_mbuf, data_off) != 300 4); 301 RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, port) - offsetof(struct rte_mbuf, data_off) != 302 6); 303 mb_def.nb_segs = 1; 304 mb_def.data_off = RTE_PKTMBUF_HEADROOM + OTX_EP_INFO_SIZE; 305 mb_def.port = port_id; 306 rte_mbuf_refcnt_set(&mb_def, 1); 307 308 /* Prevent compiler reordering: rearm_data covers previous fields */ 309 rte_compiler_barrier(); 310 tmp = (uint64_t *)&mb_def.rearm_data; 311 312 return *tmp; 313 } 314 315 /* OQ initialization */ 316 static int 317 otx_ep_init_droq(struct otx_ep_device *otx_ep, uint32_t q_no, 318 uint32_t num_descs, uint32_t desc_size, 319 struct rte_mempool *mpool, unsigned int socket_id) 320 { 321 const struct otx_ep_config *conf = otx_ep->conf; 322 uint32_t c_refill_threshold; 323 struct otx_ep_droq *droq; 324 uint32_t desc_ring_size; 325 int ret; 326 327 otx_ep_info("OQ[%d] Init start\n", q_no); 328 329 droq = otx_ep->droq[q_no]; 330 droq->otx_ep_dev = otx_ep; 331 droq->q_no = q_no; 332 droq->mpool = mpool; 333 334 droq->nb_desc = num_descs; 335 droq->buffer_size = desc_size; 336 c_refill_threshold = RTE_MAX(conf->oq.refill_threshold, 337 droq->nb_desc / 2); 338 339 /* OQ desc_ring set up */ 340 desc_ring_size = droq->nb_desc * OTX_EP_DROQ_DESC_SIZE; 341 droq->desc_ring_mz = rte_eth_dma_zone_reserve(otx_ep->eth_dev, "droq", 342 q_no, desc_ring_size, 343 OTX_EP_PCI_RING_ALIGN, 344 socket_id); 345 346 if (droq->desc_ring_mz == NULL) { 347 otx_ep_err("OQ:%d desc_ring allocation failed\n", q_no); 348 goto init_droq_fail; 349 } 350 351 droq->desc_ring_dma = droq->desc_ring_mz->iova; 352 droq->desc_ring = (struct otx_ep_droq_desc *)droq->desc_ring_mz->addr; 353 354 otx_ep_dbg("OQ[%d]: desc_ring: virt: 0x%p, dma: %lx\n", 355 q_no, droq->desc_ring, (unsigned long)droq->desc_ring_dma); 356 otx_ep_dbg("OQ[%d]: num_desc: %d\n", q_no, droq->nb_desc); 357 358 /* OQ buf_list set up */ 359 droq->recv_buf_list = rte_zmalloc_socket("recv_buf_list", 360 (droq->nb_desc * sizeof(struct rte_mbuf *)), 361 RTE_CACHE_LINE_SIZE, socket_id); 362 if (droq->recv_buf_list == NULL) { 363 otx_ep_err("OQ recv_buf_list alloc failed\n"); 364 goto init_droq_fail; 365 } 366 367 if (otx_ep_droq_setup_ring_buffers(droq)) 368 goto init_droq_fail; 369 370 droq->refill_threshold = c_refill_threshold; 371 droq->rearm_data = otx_ep_set_rearm_data(otx_ep); 372 droq->ism_ena = otx_ep->ism_ena; 373 374 /* Set up OQ registers */ 375 ret = otx_ep->fn_list.setup_oq_regs(otx_ep, q_no); 376 if (ret) 377 return ret; 378 379 otx_ep->io_qmask.oq |= (1ull << q_no); 380 381 return 0; 382 383 init_droq_fail: 384 return -ENOMEM; 385 } 386 387 /* OQ configuration and setup */ 388 int 389 otx_ep_setup_oqs(struct otx_ep_device *otx_ep, int oq_no, int num_descs, 390 int desc_size, struct rte_mempool *mpool, 391 unsigned int socket_id) 392 { 393 struct otx_ep_droq *droq; 394 395 /* Allocate new droq. */ 396 droq = (struct otx_ep_droq *)rte_zmalloc("otx_ep_OQ", 397 sizeof(*droq), RTE_CACHE_LINE_SIZE); 398 if (droq == NULL) { 399 otx_ep_err("Droq[%d] Creation Failed\n", oq_no); 400 return -ENOMEM; 401 } 402 otx_ep->droq[oq_no] = droq; 403 404 if (otx_ep_init_droq(otx_ep, oq_no, num_descs, desc_size, mpool, 405 socket_id)) { 406 otx_ep_err("Droq[%d] Initialization failed\n", oq_no); 407 goto delete_OQ; 408 } 409 otx_ep_info("OQ[%d] is created.\n", oq_no); 410 411 otx_ep->nb_rx_queues++; 412 413 return 0; 414 415 delete_OQ: 416 otx_ep_delete_oqs(otx_ep, oq_no); 417 return -ENOMEM; 418 } 419 420 static inline void 421 otx_ep_iqreq_delete(struct otx_ep_instr_queue *iq, uint32_t idx) 422 { 423 struct rte_mbuf *mbuf; 424 uint32_t reqtype; 425 426 mbuf = iq->req_list[idx].finfo.mbuf; 427 reqtype = iq->req_list[idx].reqtype; 428 429 switch (reqtype) { 430 case OTX_EP_REQTYPE_NORESP_NET: 431 case OTX_EP_REQTYPE_NORESP_GATHER: 432 /* This will take care of multiple segments also */ 433 rte_pktmbuf_free(mbuf); 434 otx_ep_dbg("IQ buffer freed at idx[%d]\n", idx); 435 break; 436 437 case OTX_EP_REQTYPE_NONE: 438 default: 439 otx_ep_info("This iqreq mode is not supported:%d\n", reqtype); 440 } 441 442 /* Reset the request list at this index */ 443 iq->req_list[idx].finfo.mbuf = NULL; 444 iq->req_list[idx].reqtype = 0; 445 } 446 447 static inline void 448 otx_ep_iqreq_add(struct otx_ep_instr_queue *iq, struct rte_mbuf *mbuf, 449 uint32_t reqtype, int index) 450 { 451 iq->req_list[index].finfo.mbuf = mbuf; 452 iq->req_list[index].reqtype = reqtype; 453 } 454 455 static uint32_t 456 otx_vf_update_read_index(struct otx_ep_instr_queue *iq) 457 { 458 uint32_t val; 459 460 /* 461 * Batch subtractions from the HW counter to reduce PCIe traffic 462 * This adds an extra local variable, but almost halves the 463 * number of PCIe writes. 464 */ 465 val = *iq->inst_cnt_ism; 466 iq->inst_cnt += val - iq->inst_cnt_prev; 467 iq->inst_cnt_prev = val; 468 469 if (val > (uint32_t)(1 << 31)) { 470 /* 471 * Only subtract the packet count in the HW counter 472 * when count above halfway to saturation. 473 */ 474 rte_write32(val, iq->inst_cnt_reg); 475 rte_mb(); 476 477 rte_write64(OTX2_SDP_REQUEST_ISM, iq->inst_cnt_reg); 478 while (__atomic_load_n(iq->inst_cnt_ism, __ATOMIC_RELAXED) >= val) { 479 rte_write64(OTX2_SDP_REQUEST_ISM, iq->inst_cnt_reg); 480 rte_mb(); 481 } 482 483 iq->inst_cnt_prev = 0; 484 } 485 rte_write64(OTX2_SDP_REQUEST_ISM, iq->inst_cnt_reg); 486 487 /* Modulo of the new index with the IQ size will give us 488 * the new index. 489 */ 490 return iq->inst_cnt & (iq->nb_desc - 1); 491 } 492 493 static void 494 otx_ep_flush_iq(struct otx_ep_instr_queue *iq) 495 { 496 uint32_t instr_processed = 0; 497 498 iq->otx_read_index = otx_vf_update_read_index(iq); 499 while (iq->flush_index != iq->otx_read_index) { 500 /* Free the IQ data buffer to the pool */ 501 otx_ep_iqreq_delete(iq, iq->flush_index); 502 iq->flush_index = 503 otx_ep_incr_index(iq->flush_index, 1, iq->nb_desc); 504 505 instr_processed++; 506 } 507 508 iq->stats.instr_processed = instr_processed; 509 iq->instr_pending -= instr_processed; 510 } 511 512 static inline void 513 otx_ep_ring_doorbell(struct otx_ep_device *otx_ep __rte_unused, 514 struct otx_ep_instr_queue *iq) 515 { 516 rte_wmb(); 517 rte_write64(iq->fill_cnt, iq->doorbell_reg); 518 iq->fill_cnt = 0; 519 } 520 521 static inline int 522 post_iqcmd(struct otx_ep_instr_queue *iq, uint8_t *iqcmd) 523 { 524 uint8_t *iqptr, cmdsize; 525 526 /* This ensures that the read index does not wrap around to 527 * the same position if queue gets full before OCTEON 9 could 528 * fetch any instr. 529 */ 530 if (iq->instr_pending > (iq->nb_desc - 1)) 531 return OTX_EP_IQ_SEND_FAILED; 532 533 /* Copy cmd into iq */ 534 cmdsize = 64; 535 iqptr = iq->base_addr + (iq->host_write_index << 6); 536 537 rte_memcpy(iqptr, iqcmd, cmdsize); 538 539 /* Increment the host write index */ 540 iq->host_write_index = 541 otx_ep_incr_index(iq->host_write_index, 1, iq->nb_desc); 542 543 iq->fill_cnt++; 544 545 /* Flush the command into memory. We need to be sure the data 546 * is in memory before indicating that the instruction is 547 * pending. 548 */ 549 iq->instr_pending++; 550 /* OTX_EP_IQ_SEND_SUCCESS */ 551 return 0; 552 } 553 554 555 static int 556 otx_ep_send_data(struct otx_ep_device *otx_ep, struct otx_ep_instr_queue *iq, 557 void *cmd, int dbell) 558 { 559 uint32_t ret; 560 561 /* Submit IQ command */ 562 ret = post_iqcmd(iq, cmd); 563 564 if (ret == OTX_EP_IQ_SEND_SUCCESS) { 565 if (dbell) 566 otx_ep_ring_doorbell(otx_ep, iq); 567 iq->stats.instr_posted++; 568 569 } else { 570 iq->stats.instr_dropped++; 571 if (iq->fill_cnt) 572 otx_ep_ring_doorbell(otx_ep, iq); 573 } 574 return ret; 575 } 576 577 static inline void 578 set_sg_size(struct otx_ep_sg_entry *sg_entry, uint16_t size, uint32_t pos) 579 { 580 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN 581 sg_entry->u.size[pos] = size; 582 #elif RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN 583 sg_entry->u.size[(OTX_EP_NUM_SG_PTRS - 1) - pos] = size; 584 #endif 585 } 586 587 static inline int 588 prepare_xmit_gather_list(struct otx_ep_instr_queue *iq, struct rte_mbuf *m, uint64_t *dptr, 589 union otx_ep_instr_ih *ih) 590 { 591 uint16_t j = 0, frags, num_sg, mask = OTX_EP_NUM_SG_PTRS - 1; 592 struct otx_ep_buf_free_info *finfo; 593 uint32_t pkt_len; 594 int rc = -1; 595 596 pkt_len = rte_pktmbuf_pkt_len(m); 597 frags = m->nb_segs; 598 num_sg = (frags + mask) / OTX_EP_NUM_SG_PTRS; 599 600 if (unlikely(pkt_len > OTX_EP_MAX_PKT_SZ && num_sg > OTX_EP_MAX_SG_LISTS)) { 601 otx_ep_err("Failed to xmit the pkt, pkt_len is higher or pkt has more segments\n"); 602 goto exit; 603 } 604 605 finfo = &iq->req_list[iq->host_write_index].finfo; 606 *dptr = rte_mem_virt2iova(finfo->g.sg); 607 ih->u64 |= ((1ULL << 62) | ((uint64_t)frags << 48) | (pkt_len + ih->s.fsz)); 608 609 while (frags--) { 610 finfo->g.sg[(j >> 2)].ptr[(j & mask)] = rte_mbuf_data_iova(m); 611 set_sg_size(&finfo->g.sg[(j >> 2)], m->data_len, (j & mask)); 612 j++; 613 m = m->next; 614 } 615 616 return 0; 617 618 exit: 619 return rc; 620 } 621 622 /* Enqueue requests/packets to OTX_EP IQ queue. 623 * returns number of requests enqueued successfully 624 */ 625 uint16_t 626 otx_ep_xmit_pkts(void *tx_queue, struct rte_mbuf **pkts, uint16_t nb_pkts) 627 { 628 struct otx_ep_instr_queue *iq = (struct otx_ep_instr_queue *)tx_queue; 629 struct otx_ep_device *otx_ep = iq->otx_ep_dev; 630 struct otx_ep_instr_64B iqcmd; 631 int dbell, index, count = 0; 632 uint32_t iqreq_type; 633 uint32_t pkt_len, i; 634 struct rte_mbuf *m; 635 636 iqcmd.ih.u64 = 0; 637 iqcmd.pki_ih3.u64 = 0; 638 iqcmd.irh.u64 = 0; 639 640 /* ih invars */ 641 iqcmd.ih.s.fsz = OTX_EP_FSZ; 642 iqcmd.ih.s.pkind = otx_ep->pkind; /* The SDK decided PKIND value */ 643 644 /* pki ih3 invars */ 645 iqcmd.pki_ih3.s.w = 1; 646 iqcmd.pki_ih3.s.utt = 1; 647 iqcmd.pki_ih3.s.tagtype = ORDERED_TAG; 648 /* sl will be sizeof(pki_ih3) */ 649 iqcmd.pki_ih3.s.sl = OTX_EP_FSZ + OTX_CUST_DATA_LEN; 650 651 /* irh invars */ 652 iqcmd.irh.s.opcode = OTX_EP_NW_PKT_OP; 653 654 for (i = 0; i < nb_pkts; i++) { 655 m = pkts[i]; 656 if (m->nb_segs == 1) { 657 pkt_len = rte_pktmbuf_data_len(m); 658 iqcmd.ih.s.tlen = pkt_len + iqcmd.ih.s.fsz; 659 iqcmd.dptr = rte_mbuf_data_iova(m); /*dptr*/ 660 iqcmd.ih.s.gather = 0; 661 iqcmd.ih.s.gsz = 0; 662 iqreq_type = OTX_EP_REQTYPE_NORESP_NET; 663 } else { 664 if (!(otx_ep->tx_offloads & RTE_ETH_TX_OFFLOAD_MULTI_SEGS)) 665 goto xmit_fail; 666 667 if (unlikely(prepare_xmit_gather_list(iq, m, &iqcmd.dptr, &iqcmd.ih) < 0)) 668 goto xmit_fail; 669 670 pkt_len = rte_pktmbuf_pkt_len(m); 671 iqreq_type = OTX_EP_REQTYPE_NORESP_GATHER; 672 } 673 674 iqcmd.irh.u64 = rte_bswap64(iqcmd.irh.u64); 675 676 #ifdef OTX_EP_IO_DEBUG 677 otx_ep_dbg("After swapping\n"); 678 otx_ep_dbg("Word0 [dptr]: 0x%016lx\n", 679 (unsigned long)iqcmd.dptr); 680 otx_ep_dbg("Word1 [ihtx]: 0x%016lx\n", (unsigned long)iqcmd.ih); 681 otx_ep_dbg("Word2 [pki_ih3]: 0x%016lx\n", 682 (unsigned long)iqcmd.pki_ih3); 683 otx_ep_dbg("Word3 [rptr]: 0x%016lx\n", 684 (unsigned long)iqcmd.rptr); 685 otx_ep_dbg("Word4 [irh]: 0x%016lx\n", (unsigned long)iqcmd.irh); 686 otx_ep_dbg("Word5 [exhdr[0]]: 0x%016lx\n", 687 (unsigned long)iqcmd.exhdr[0]); 688 rte_pktmbuf_dump(stdout, m, rte_pktmbuf_pkt_len(m)); 689 #endif 690 dbell = (i == (unsigned int)(nb_pkts - 1)) ? 1 : 0; 691 index = iq->host_write_index; 692 if (otx_ep_send_data(otx_ep, iq, &iqcmd, dbell)) 693 goto xmit_fail; 694 otx_ep_iqreq_add(iq, m, iqreq_type, index); 695 iq->stats.tx_pkts++; 696 iq->stats.tx_bytes += pkt_len; 697 count++; 698 } 699 700 xmit_fail: 701 if (iq->instr_pending >= OTX_EP_MAX_INSTR) 702 otx_ep_flush_iq(iq); 703 704 /* Return no# of instructions posted successfully. */ 705 return count; 706 } 707 708 static uint32_t 709 otx_ep_droq_refill(struct otx_ep_droq *droq) 710 { 711 struct otx_ep_droq_desc *desc_ring = droq->desc_ring; 712 struct otx_ep_droq_info *info; 713 struct rte_mbuf *buf = NULL; 714 uint32_t desc_refilled = 0; 715 716 while (droq->refill_count && (desc_refilled < droq->nb_desc)) { 717 buf = rte_pktmbuf_alloc(droq->mpool); 718 /* If a buffer could not be allocated, no point in 719 * continuing 720 */ 721 if (unlikely(!buf)) { 722 droq->stats.rx_alloc_failure++; 723 break; 724 } 725 info = rte_pktmbuf_mtod(buf, struct otx_ep_droq_info *); 726 info->length = 0; 727 728 droq->recv_buf_list[droq->refill_idx] = buf; 729 desc_ring[droq->refill_idx].buffer_ptr = 730 rte_mbuf_data_iova_default(buf); 731 droq->refill_idx = otx_ep_incr_index(droq->refill_idx, 1, 732 droq->nb_desc); 733 734 desc_refilled++; 735 droq->refill_count--; 736 } 737 738 return desc_refilled; 739 } 740 741 static struct rte_mbuf * 742 otx_ep_droq_read_packet(struct otx_ep_device *otx_ep, struct otx_ep_droq *droq, int next_fetch) 743 { 744 volatile struct otx_ep_droq_info *info; 745 struct rte_mbuf *mbuf_next = NULL; 746 struct rte_mbuf *mbuf = NULL; 747 uint64_t total_pkt_len; 748 uint32_t pkt_len = 0; 749 int next_idx; 750 751 mbuf = droq->recv_buf_list[droq->read_idx]; 752 info = rte_pktmbuf_mtod(mbuf, struct otx_ep_droq_info *); 753 754 /* make sure info is available */ 755 rte_rmb(); 756 if (unlikely(!info->length)) { 757 int retry = OTX_EP_MAX_DELAYED_PKT_RETRIES; 758 /* otx_ep_dbg("OCTEON DROQ[%d]: read_idx: %d; Data not ready " 759 * "yet, Retry; pending=%lu\n", droq->q_no, droq->read_idx, 760 * droq->pkts_pending); 761 */ 762 droq->stats.pkts_delayed_data++; 763 while (retry && !info->length) { 764 retry--; 765 rte_delay_us_block(50); 766 } 767 if (!retry && !info->length) { 768 otx_ep_err("OCTEON DROQ[%d]: read_idx: %d; Retry failed !!\n", 769 droq->q_no, droq->read_idx); 770 /* May be zero length packet; drop it */ 771 assert(0); 772 } 773 } 774 775 if (next_fetch) { 776 next_idx = otx_ep_incr_index(droq->read_idx, 1, droq->nb_desc); 777 mbuf_next = droq->recv_buf_list[next_idx]; 778 rte_prefetch0(rte_pktmbuf_mtod(mbuf_next, void *)); 779 } 780 781 info->length = rte_bswap16(info->length >> 48); 782 /* Deduce the actual data size */ 783 total_pkt_len = info->length + OTX_EP_INFO_SIZE; 784 if (total_pkt_len <= droq->buffer_size) { 785 mbuf->data_off += OTX_EP_INFO_SIZE; 786 pkt_len = (uint32_t)info->length; 787 mbuf->pkt_len = pkt_len; 788 mbuf->data_len = pkt_len; 789 mbuf->port = otx_ep->port_id; 790 droq->recv_buf_list[droq->read_idx] = NULL; 791 droq->read_idx = otx_ep_incr_index(droq->read_idx, 1, droq->nb_desc); 792 droq->refill_count++; 793 } else { 794 struct rte_mbuf *first_buf = NULL; 795 struct rte_mbuf *last_buf = NULL; 796 797 /* csr read helps to flush pending dma */ 798 droq->sent_reg_val = rte_read32(droq->pkts_sent_reg); 799 rte_rmb(); 800 801 while (pkt_len < total_pkt_len) { 802 int cpy_len = 0; 803 804 cpy_len = ((pkt_len + droq->buffer_size) > total_pkt_len) 805 ? ((uint32_t)total_pkt_len - pkt_len) 806 : droq->buffer_size; 807 808 mbuf = droq->recv_buf_list[droq->read_idx]; 809 droq->recv_buf_list[droq->read_idx] = NULL; 810 811 if (likely(mbuf)) { 812 /* Note the first seg */ 813 if (!pkt_len) 814 first_buf = mbuf; 815 816 mbuf->port = otx_ep->port_id; 817 if (!pkt_len) { 818 mbuf->data_off += OTX_EP_INFO_SIZE; 819 mbuf->pkt_len = cpy_len - OTX_EP_INFO_SIZE; 820 mbuf->data_len = cpy_len - OTX_EP_INFO_SIZE; 821 } else { 822 mbuf->pkt_len = cpy_len; 823 mbuf->data_len = cpy_len; 824 } 825 826 if (pkt_len) { 827 first_buf->nb_segs++; 828 first_buf->pkt_len += mbuf->pkt_len; 829 } 830 831 if (last_buf) 832 last_buf->next = mbuf; 833 834 last_buf = mbuf; 835 } else { 836 otx_ep_err("no buf\n"); 837 assert(0); 838 } 839 840 pkt_len += cpy_len; 841 droq->read_idx = otx_ep_incr_index(droq->read_idx, 1, droq->nb_desc); 842 droq->refill_count++; 843 } 844 mbuf = first_buf; 845 } 846 847 return mbuf; 848 } 849 850 static inline uint32_t 851 otx_ep_check_droq_pkts(struct otx_ep_droq *droq) 852 { 853 uint32_t new_pkts; 854 uint32_t val; 855 856 /* 857 * Batch subtractions from the HW counter to reduce PCIe traffic 858 * This adds an extra local variable, but almost halves the 859 * number of PCIe writes. 860 */ 861 val = *droq->pkts_sent_ism; 862 new_pkts = val - droq->pkts_sent_prev; 863 droq->pkts_sent_prev = val; 864 865 if (val > (uint32_t)(1 << 31)) { 866 /* 867 * Only subtract the packet count in the HW counter 868 * when count above halfway to saturation. 869 */ 870 rte_write32(val, droq->pkts_sent_reg); 871 rte_mb(); 872 873 rte_write64(OTX2_SDP_REQUEST_ISM, droq->pkts_sent_reg); 874 while (__atomic_load_n(droq->pkts_sent_ism, __ATOMIC_RELAXED) >= val) { 875 rte_write64(OTX2_SDP_REQUEST_ISM, droq->pkts_sent_reg); 876 rte_mb(); 877 } 878 879 droq->pkts_sent_prev = 0; 880 } 881 rte_write64(OTX2_SDP_REQUEST_ISM, droq->pkts_sent_reg); 882 droq->pkts_pending += new_pkts; 883 884 return new_pkts; 885 } 886 887 static inline int32_t __rte_hot 888 otx_ep_rx_pkts_to_process(struct otx_ep_droq *droq, uint16_t nb_pkts) 889 { 890 if (unlikely(droq->pkts_pending < nb_pkts)) 891 otx_ep_check_droq_pkts(droq); 892 893 return RTE_MIN(nb_pkts, droq->pkts_pending); 894 } 895 896 /* Check for response arrival from OCTEON 9 897 * returns number of requests completed 898 */ 899 uint16_t 900 otx_ep_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) 901 { 902 struct otx_ep_droq *droq = rx_queue; 903 struct otx_ep_device *otx_ep; 904 struct rte_mbuf *oq_pkt; 905 uint16_t pkts, new_pkts; 906 uint32_t valid_pkts = 0; 907 int next_fetch; 908 909 otx_ep = droq->otx_ep_dev; 910 new_pkts = otx_ep_rx_pkts_to_process(droq, nb_pkts); 911 912 for (pkts = 0; pkts < new_pkts; pkts++) { 913 /* Push the received pkt to application */ 914 next_fetch = (pkts == new_pkts - 1) ? 0 : 1; 915 oq_pkt = otx_ep_droq_read_packet(otx_ep, droq, next_fetch); 916 if (!oq_pkt) { 917 RTE_LOG_DP(ERR, OTX_NET_EP, 918 "DROQ read pkt failed pending %" PRIu64 919 "last_pkt_count %" PRIu64 "new_pkts %d.\n", 920 droq->pkts_pending, droq->last_pkt_count, 921 new_pkts); 922 droq->stats.rx_err++; 923 continue; 924 } else { 925 rx_pkts[valid_pkts] = oq_pkt; 926 valid_pkts++; 927 /* Stats */ 928 droq->stats.pkts_received++; 929 droq->stats.bytes_received += oq_pkt->pkt_len; 930 } 931 } 932 droq->pkts_pending -= pkts; 933 934 /* Refill DROQ buffers */ 935 if (droq->refill_count >= DROQ_REFILL_THRESHOLD) { 936 int desc_refilled = otx_ep_droq_refill(droq); 937 938 /* Flush the droq descriptor data to memory to be sure 939 * that when we update the credits the data in memory is 940 * accurate. 941 */ 942 rte_io_wmb(); 943 rte_write32(desc_refilled, droq->pkts_credit_reg); 944 } else { 945 /* 946 * SDP output goes into DROP state when output doorbell count 947 * goes below drop count. When door bell count is written with 948 * a value greater than drop count SDP output should come out 949 * of DROP state. Due to a race condition this is not happening. 950 * Writing doorbell register with 0 again may make SDP output 951 * come out of this state. 952 */ 953 954 rte_write32(0, droq->pkts_credit_reg); 955 } 956 return valid_pkts; 957 } 958