1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2018-2019 NXP 3 */ 4 5 #include "pfe_logs.h" 6 #include "pfe_mod.h" 7 #include <sys/ioctl.h> 8 #include <sys/epoll.h> 9 #include <sys/eventfd.h> 10 11 static int 12 pfe_hif_alloc_descr(struct pfe_hif *hif) 13 { 14 void *addr; 15 int err = 0; 16 17 PMD_INIT_FUNC_TRACE(); 18 19 addr = rte_zmalloc(NULL, HIF_RX_DESC_NT * sizeof(struct hif_desc) + 20 HIF_TX_DESC_NT * sizeof(struct hif_desc), RTE_CACHE_LINE_SIZE); 21 if (!addr) { 22 PFE_PMD_ERR("Could not allocate buffer descriptors!"); 23 err = -ENOMEM; 24 goto err0; 25 } 26 27 hif->descr_baseaddr_p = pfe_mem_vtop((uintptr_t)addr); 28 hif->descr_baseaddr_v = addr; 29 hif->rx_ring_size = HIF_RX_DESC_NT; 30 hif->tx_ring_size = HIF_TX_DESC_NT; 31 32 return 0; 33 34 err0: 35 return err; 36 } 37 38 static void 39 pfe_hif_free_descr(struct pfe_hif *hif) 40 { 41 PMD_INIT_FUNC_TRACE(); 42 43 rte_free(hif->descr_baseaddr_v); 44 } 45 46 /* pfe_hif_release_buffers */ 47 static void 48 pfe_hif_release_buffers(struct pfe_hif *hif) 49 { 50 struct hif_desc *desc; 51 uint32_t i = 0; 52 struct rte_mbuf *mbuf; 53 struct rte_pktmbuf_pool_private *mb_priv; 54 55 hif->rx_base = hif->descr_baseaddr_v; 56 57 /*Free Rx buffers */ 58 desc = hif->rx_base; 59 mb_priv = rte_mempool_get_priv(hif->shm->pool); 60 for (i = 0; i < hif->rx_ring_size; i++) { 61 if (readl(&desc->data)) { 62 if (i < hif->shm->rx_buf_pool_cnt && 63 !hif->shm->rx_buf_pool[i]) { 64 mbuf = hif->rx_buf_vaddr[i] + PFE_PKT_HEADER_SZ 65 - sizeof(struct rte_mbuf) 66 - RTE_PKTMBUF_HEADROOM 67 - mb_priv->mbuf_priv_size; 68 hif->shm->rx_buf_pool[i] = mbuf; 69 } 70 } 71 writel(0, &desc->data); 72 writel(0, &desc->status); 73 writel(0, &desc->ctrl); 74 desc++; 75 } 76 } 77 78 /* 79 * pfe_hif_init_buffers 80 * This function initializes the HIF Rx/Tx ring descriptors and 81 * initialize Rx queue with buffers. 82 */ 83 int 84 pfe_hif_init_buffers(struct pfe_hif *hif) 85 { 86 struct hif_desc *desc, *first_desc_p; 87 uint32_t i = 0; 88 89 PMD_INIT_FUNC_TRACE(); 90 91 /* Check enough Rx buffers available in the shared memory */ 92 if (hif->shm->rx_buf_pool_cnt < hif->rx_ring_size) 93 return -ENOMEM; 94 95 hif->rx_base = hif->descr_baseaddr_v; 96 memset(hif->rx_base, 0, hif->rx_ring_size * sizeof(struct hif_desc)); 97 98 /*Initialize Rx descriptors */ 99 desc = hif->rx_base; 100 first_desc_p = (struct hif_desc *)hif->descr_baseaddr_p; 101 102 for (i = 0; i < hif->rx_ring_size; i++) { 103 /* Initialize Rx buffers from the shared memory */ 104 struct rte_mbuf *mbuf = 105 (struct rte_mbuf *)hif->shm->rx_buf_pool[i]; 106 107 /* PFE mbuf structure is as follow: 108 * ----------------------------------------------------------+ 109 * | mbuf | priv | headroom (annotation + PFE data) | data | 110 * ----------------------------------------------------------+ 111 * 112 * As we are expecting additional information like parse 113 * results, eth id, queue id from PFE block along with data. 114 * so we have to provide additional memory for each packet to 115 * HIF rx rings so that PFE block can write its headers. 116 * so, we are giving the data pointor to HIF rings whose 117 * calculation is as below: 118 * mbuf->data_pointor - Required_header_size 119 * 120 * We are utilizing the HEADROOM area to receive the PFE 121 * block headers. On packet reception, HIF driver will use 122 * PFE headers information based on which it will decide 123 * the clients and fill the parse results. 124 * after that application can use/overwrite the HEADROOM area. 125 */ 126 hif->rx_buf_vaddr[i] = 127 (void *)((size_t)mbuf->buf_addr + mbuf->data_off - 128 PFE_PKT_HEADER_SZ); 129 hif->rx_buf_addr[i] = 130 (void *)(size_t)(rte_pktmbuf_iova(mbuf) - 131 PFE_PKT_HEADER_SZ); 132 hif->rx_buf_len[i] = mbuf->buf_len - RTE_PKTMBUF_HEADROOM; 133 134 hif->shm->rx_buf_pool[i] = NULL; 135 136 writel(DDR_PHYS_TO_PFE(hif->rx_buf_addr[i]), 137 &desc->data); 138 writel(0, &desc->status); 139 140 /* 141 * Ensure everything else is written to DDR before 142 * writing bd->ctrl 143 */ 144 rte_wmb(); 145 146 writel((BD_CTRL_PKT_INT_EN | BD_CTRL_LIFM 147 | BD_CTRL_DIR | BD_CTRL_DESC_EN 148 | BD_BUF_LEN(hif->rx_buf_len[i])), &desc->ctrl); 149 150 /* Chain descriptors */ 151 writel((u32)DDR_PHYS_TO_PFE(first_desc_p + i + 1), &desc->next); 152 desc++; 153 } 154 155 /* Overwrite last descriptor to chain it to first one*/ 156 desc--; 157 writel((u32)DDR_PHYS_TO_PFE(first_desc_p), &desc->next); 158 159 hif->rxtoclean_index = 0; 160 161 /*Initialize Rx buffer descriptor ring base address */ 162 writel(DDR_PHYS_TO_PFE(hif->descr_baseaddr_p), HIF_RX_BDP_ADDR); 163 164 hif->tx_base = hif->rx_base + hif->rx_ring_size; 165 first_desc_p = (struct hif_desc *)hif->descr_baseaddr_p + 166 hif->rx_ring_size; 167 memset(hif->tx_base, 0, hif->tx_ring_size * sizeof(struct hif_desc)); 168 169 /*Initialize tx descriptors */ 170 desc = hif->tx_base; 171 172 for (i = 0; i < hif->tx_ring_size; i++) { 173 /* Chain descriptors */ 174 writel((u32)DDR_PHYS_TO_PFE(first_desc_p + i + 1), &desc->next); 175 writel(0, &desc->ctrl); 176 desc++; 177 } 178 179 /* Overwrite last descriptor to chain it to first one */ 180 desc--; 181 writel((u32)DDR_PHYS_TO_PFE(first_desc_p), &desc->next); 182 hif->txavail = hif->tx_ring_size; 183 hif->txtosend = 0; 184 hif->txtoclean = 0; 185 hif->txtoflush = 0; 186 187 /*Initialize Tx buffer descriptor ring base address */ 188 writel((u32)DDR_PHYS_TO_PFE(first_desc_p), HIF_TX_BDP_ADDR); 189 190 return 0; 191 } 192 193 /* 194 * pfe_hif_client_register 195 * 196 * This function used to register a client driver with the HIF driver. 197 * 198 * Return value: 199 * 0 - on Successful registration 200 */ 201 static int 202 pfe_hif_client_register(struct pfe_hif *hif, u32 client_id, 203 struct hif_client_shm *client_shm) 204 { 205 struct hif_client *client = &hif->client[client_id]; 206 u32 i, cnt; 207 struct rx_queue_desc *rx_qbase; 208 struct tx_queue_desc *tx_qbase; 209 struct hif_rx_queue *rx_queue; 210 struct hif_tx_queue *tx_queue; 211 int err = 0; 212 213 PMD_INIT_FUNC_TRACE(); 214 215 rte_spinlock_lock(&hif->tx_lock); 216 217 if (test_bit(client_id, &hif->shm->g_client_status[0])) { 218 PFE_PMD_ERR("client %d already registered", client_id); 219 err = -1; 220 goto unlock; 221 } 222 223 memset(client, 0, sizeof(struct hif_client)); 224 225 /* Initialize client Rx queues baseaddr, size */ 226 227 cnt = CLIENT_CTRL_RX_Q_CNT(client_shm->ctrl); 228 /* Check if client is requesting for more queues than supported */ 229 if (cnt > HIF_CLIENT_QUEUES_MAX) 230 cnt = HIF_CLIENT_QUEUES_MAX; 231 232 client->rx_qn = cnt; 233 rx_qbase = (struct rx_queue_desc *)client_shm->rx_qbase; 234 for (i = 0; i < cnt; i++) { 235 rx_queue = &client->rx_q[i]; 236 rx_queue->base = rx_qbase + i * client_shm->rx_qsize; 237 rx_queue->size = client_shm->rx_qsize; 238 rx_queue->write_idx = 0; 239 } 240 241 /* Initialize client Tx queues baseaddr, size */ 242 cnt = CLIENT_CTRL_TX_Q_CNT(client_shm->ctrl); 243 244 /* Check if client is requesting for more queues than supported */ 245 if (cnt > HIF_CLIENT_QUEUES_MAX) 246 cnt = HIF_CLIENT_QUEUES_MAX; 247 248 client->tx_qn = cnt; 249 tx_qbase = (struct tx_queue_desc *)client_shm->tx_qbase; 250 for (i = 0; i < cnt; i++) { 251 tx_queue = &client->tx_q[i]; 252 tx_queue->base = tx_qbase + i * client_shm->tx_qsize; 253 tx_queue->size = client_shm->tx_qsize; 254 tx_queue->ack_idx = 0; 255 } 256 257 set_bit(client_id, &hif->shm->g_client_status[0]); 258 259 unlock: 260 rte_spinlock_unlock(&hif->tx_lock); 261 262 return err; 263 } 264 265 /* 266 * pfe_hif_client_unregister 267 * 268 * This function used to unregister a client from the HIF driver. 269 * 270 */ 271 static void 272 pfe_hif_client_unregister(struct pfe_hif *hif, u32 client_id) 273 { 274 PMD_INIT_FUNC_TRACE(); 275 276 /* 277 * Mark client as no longer available (which prevents further packet 278 * receive for this client) 279 */ 280 rte_spinlock_lock(&hif->tx_lock); 281 282 if (!test_bit(client_id, &hif->shm->g_client_status[0])) { 283 PFE_PMD_ERR("client %d not registered", client_id); 284 285 rte_spinlock_unlock(&hif->tx_lock); 286 return; 287 } 288 289 clear_bit(client_id, &hif->shm->g_client_status[0]); 290 291 rte_spinlock_unlock(&hif->tx_lock); 292 } 293 294 /* 295 * client_put_rxpacket- 296 */ 297 static struct rte_mbuf * 298 client_put_rxpacket(struct hif_rx_queue *queue, 299 void *pkt, u32 len, 300 u32 flags, u32 client_ctrl, 301 struct rte_mempool *pool, 302 u32 *rem_len) 303 { 304 struct rx_queue_desc *desc = queue->base + queue->write_idx; 305 struct rte_mbuf *mbuf = NULL; 306 307 308 if (readl(&desc->ctrl) & CL_DESC_OWN) { 309 mbuf = rte_cpu_to_le_64(rte_pktmbuf_alloc(pool)); 310 if (unlikely(!mbuf)) { 311 PFE_PMD_WARN("Buffer allocation failure\n"); 312 return NULL; 313 } 314 315 desc->data = pkt; 316 desc->client_ctrl = client_ctrl; 317 /* 318 * Ensure everything else is written to DDR before 319 * writing bd->ctrl 320 */ 321 rte_wmb(); 322 writel(CL_DESC_BUF_LEN(len) | flags, &desc->ctrl); 323 queue->write_idx = (queue->write_idx + 1) 324 & (queue->size - 1); 325 326 *rem_len = mbuf->buf_len; 327 } 328 329 return mbuf; 330 } 331 332 /* 333 * pfe_hif_rx_process- 334 * This function does pfe hif rx queue processing. 335 * Dequeue packet from Rx queue and send it to corresponding client queue 336 */ 337 int 338 pfe_hif_rx_process(struct pfe *pfe, int budget) 339 { 340 struct hif_desc *desc; 341 struct hif_hdr *pkt_hdr; 342 struct __hif_hdr hif_hdr; 343 void *free_buf; 344 int rtc, len, rx_processed = 0; 345 struct __hif_desc local_desc; 346 int flags = 0, wait_for_last = 0, retry = 0; 347 unsigned int buf_size = 0; 348 struct rte_mbuf *mbuf = NULL; 349 struct pfe_hif *hif = &pfe->hif; 350 351 rte_spinlock_lock(&hif->lock); 352 353 rtc = hif->rxtoclean_index; 354 355 while (rx_processed < budget) { 356 desc = hif->rx_base + rtc; 357 358 __memcpy12(&local_desc, desc); 359 360 /* ACK pending Rx interrupt */ 361 if (local_desc.ctrl & BD_CTRL_DESC_EN) { 362 if (unlikely(wait_for_last)) 363 continue; 364 else 365 break; 366 } 367 368 len = BD_BUF_LEN(local_desc.ctrl); 369 pkt_hdr = (struct hif_hdr *)hif->rx_buf_vaddr[rtc]; 370 371 /* Track last HIF header received */ 372 if (!hif->started) { 373 hif->started = 1; 374 375 __memcpy8(&hif_hdr, pkt_hdr); 376 377 hif->qno = hif_hdr.hdr.q_num; 378 hif->client_id = hif_hdr.hdr.client_id; 379 hif->client_ctrl = (hif_hdr.hdr.client_ctrl1 << 16) | 380 hif_hdr.hdr.client_ctrl; 381 flags = CL_DESC_FIRST; 382 383 } else { 384 flags = 0; 385 } 386 387 if (local_desc.ctrl & BD_CTRL_LIFM) { 388 flags |= CL_DESC_LAST; 389 wait_for_last = 0; 390 } else { 391 wait_for_last = 1; 392 } 393 394 /* Check for valid client id and still registered */ 395 if (hif->client_id >= HIF_CLIENTS_MAX || 396 !(test_bit(hif->client_id, 397 &hif->shm->g_client_status[0]))) { 398 PFE_PMD_INFO("packet with invalid client id %d qnum %d", 399 hif->client_id, hif->qno); 400 401 free_buf = hif->rx_buf_addr[rtc]; 402 403 goto pkt_drop; 404 } 405 406 /* Check to valid queue number */ 407 if (hif->client[hif->client_id].rx_qn <= hif->qno) { 408 PFE_DP_LOG(DEBUG, "packet with invalid queue: %d", 409 hif->qno); 410 hif->qno = 0; 411 } 412 413 retry: 414 mbuf = 415 client_put_rxpacket(&hif->client[hif->client_id].rx_q[hif->qno], 416 (void *)pkt_hdr, len, flags, 417 hif->client_ctrl, hif->shm->pool, 418 &buf_size); 419 420 if (unlikely(!mbuf)) { 421 if (!retry) { 422 pfe_tx_do_cleanup(pfe); 423 retry = 1; 424 goto retry; 425 } 426 rx_processed = budget; 427 428 if (flags & CL_DESC_FIRST) 429 hif->started = 0; 430 431 PFE_DP_LOG(DEBUG, "No buffers"); 432 break; 433 } 434 435 retry = 0; 436 437 free_buf = (void *)(size_t)rte_pktmbuf_iova(mbuf); 438 free_buf = free_buf - PFE_PKT_HEADER_SZ; 439 440 /*Fill free buffer in the descriptor */ 441 hif->rx_buf_addr[rtc] = free_buf; 442 hif->rx_buf_vaddr[rtc] = (void *)((size_t)mbuf->buf_addr + 443 mbuf->data_off - PFE_PKT_HEADER_SZ); 444 hif->rx_buf_len[rtc] = buf_size - RTE_PKTMBUF_HEADROOM; 445 446 pkt_drop: 447 writel(DDR_PHYS_TO_PFE(free_buf), &desc->data); 448 /* 449 * Ensure everything else is written to DDR before 450 * writing bd->ctrl 451 */ 452 rte_wmb(); 453 writel((BD_CTRL_PKT_INT_EN | BD_CTRL_LIFM | BD_CTRL_DIR | 454 BD_CTRL_DESC_EN | BD_BUF_LEN(hif->rx_buf_len[rtc])), 455 &desc->ctrl); 456 457 rtc = (rtc + 1) & (hif->rx_ring_size - 1); 458 459 if (local_desc.ctrl & BD_CTRL_LIFM) { 460 if (!(hif->client_ctrl & HIF_CTRL_RX_CONTINUED)) 461 rx_processed++; 462 463 hif->started = 0; 464 } 465 } 466 467 468 hif->rxtoclean_index = rtc; 469 rte_spinlock_unlock(&hif->lock); 470 471 /* we made some progress, re-start rx dma in case it stopped */ 472 hif_rx_dma_start(); 473 474 return rx_processed; 475 } 476 477 /* 478 * client_ack_txpacket- 479 * This function ack the Tx packet in the give client Tx queue by resetting 480 * ownership bit in the descriptor. 481 */ 482 static int 483 client_ack_txpacket(struct pfe_hif *hif, unsigned int client_id, 484 unsigned int q_no) 485 { 486 struct hif_tx_queue *queue = &hif->client[client_id].tx_q[q_no]; 487 struct tx_queue_desc *desc = queue->base + queue->ack_idx; 488 489 if (readl(&desc->ctrl) & CL_DESC_OWN) { 490 writel((readl(&desc->ctrl) & ~CL_DESC_OWN), &desc->ctrl); 491 queue->ack_idx = (queue->ack_idx + 1) & (queue->size - 1); 492 493 return 0; 494 495 } else { 496 /*This should not happen */ 497 PFE_PMD_ERR("%d %d %d %d %d %p %d", 498 hif->txtosend, hif->txtoclean, hif->txavail, 499 client_id, q_no, queue, queue->ack_idx); 500 return 1; 501 } 502 } 503 504 static void 505 __hif_tx_done_process(struct pfe *pfe, int count) 506 { 507 struct hif_desc *desc; 508 struct hif_desc_sw *desc_sw; 509 unsigned int ttc, tx_avl; 510 int pkts_done[HIF_CLIENTS_MAX] = {0, 0}; 511 struct pfe_hif *hif = &pfe->hif; 512 513 ttc = hif->txtoclean; 514 tx_avl = hif->txavail; 515 516 while ((tx_avl < hif->tx_ring_size) && count--) { 517 desc = hif->tx_base + ttc; 518 519 if (readl(&desc->ctrl) & BD_CTRL_DESC_EN) 520 break; 521 522 desc_sw = &hif->tx_sw_queue[ttc]; 523 524 if (desc_sw->client_id > HIF_CLIENTS_MAX) 525 PFE_PMD_ERR("Invalid cl id %d", desc_sw->client_id); 526 527 pkts_done[desc_sw->client_id]++; 528 529 client_ack_txpacket(hif, desc_sw->client_id, desc_sw->q_no); 530 531 ttc = (ttc + 1) & (hif->tx_ring_size - 1); 532 tx_avl++; 533 } 534 535 if (pkts_done[0]) 536 hif_lib_indicate_client(pfe->hif_client[0], EVENT_TXDONE_IND, 537 0); 538 if (pkts_done[1]) 539 hif_lib_indicate_client(pfe->hif_client[1], EVENT_TXDONE_IND, 540 0); 541 hif->txtoclean = ttc; 542 hif->txavail = tx_avl; 543 } 544 545 static inline void 546 hif_tx_done_process(struct pfe *pfe, int count) 547 { 548 struct pfe_hif *hif = &pfe->hif; 549 rte_spinlock_lock(&hif->tx_lock); 550 __hif_tx_done_process(pfe, count); 551 rte_spinlock_unlock(&hif->tx_lock); 552 } 553 554 void 555 pfe_tx_do_cleanup(struct pfe *pfe) 556 { 557 hif_tx_done_process(pfe, HIF_TX_DESC_NT); 558 } 559 560 /* 561 * __hif_xmit_pkt - 562 * This function puts one packet in the HIF Tx queue 563 */ 564 void 565 hif_xmit_pkt(struct pfe_hif *hif, unsigned int client_id, unsigned int 566 q_no, void *data, u32 len, unsigned int flags) 567 { 568 struct hif_desc *desc; 569 struct hif_desc_sw *desc_sw; 570 571 desc = hif->tx_base + hif->txtosend; 572 desc_sw = &hif->tx_sw_queue[hif->txtosend]; 573 574 desc_sw->len = len; 575 desc_sw->client_id = client_id; 576 desc_sw->q_no = q_no; 577 desc_sw->flags = flags; 578 579 writel((u32)DDR_PHYS_TO_PFE(data), &desc->data); 580 581 hif->txtosend = (hif->txtosend + 1) & (hif->tx_ring_size - 1); 582 hif->txavail--; 583 584 if ((!((flags & HIF_DATA_VALID) && (flags & 585 HIF_LAST_BUFFER)))) 586 goto skip_tx; 587 588 /* 589 * Ensure everything else is written to DDR before 590 * writing bd->ctrl 591 */ 592 rte_wmb(); 593 594 do { 595 desc_sw = &hif->tx_sw_queue[hif->txtoflush]; 596 desc = hif->tx_base + hif->txtoflush; 597 598 if (desc_sw->flags & HIF_LAST_BUFFER) { 599 writel((BD_CTRL_LIFM | 600 BD_CTRL_BRFETCH_DISABLE | BD_CTRL_RTFETCH_DISABLE 601 | BD_CTRL_PARSE_DISABLE | BD_CTRL_DESC_EN | 602 BD_BUF_LEN(desc_sw->len)), 603 &desc->ctrl); 604 } else { 605 writel((BD_CTRL_DESC_EN | 606 BD_BUF_LEN(desc_sw->len)), &desc->ctrl); 607 } 608 hif->txtoflush = (hif->txtoflush + 1) & (hif->tx_ring_size - 1); 609 } 610 while (hif->txtoflush != hif->txtosend) 611 ; 612 613 skip_tx: 614 return; 615 } 616 617 void 618 hif_process_client_req(struct pfe_hif *hif, int req, 619 int data1, __rte_unused int data2) 620 { 621 unsigned int client_id = data1; 622 623 if (client_id >= HIF_CLIENTS_MAX) { 624 PFE_PMD_ERR("client id %d out of bounds", client_id); 625 return; 626 } 627 628 switch (req) { 629 case REQUEST_CL_REGISTER: 630 /* Request for register a client */ 631 PFE_PMD_INFO("register client_id %d", client_id); 632 pfe_hif_client_register(hif, client_id, (struct 633 hif_client_shm *)&hif->shm->client[client_id]); 634 break; 635 636 case REQUEST_CL_UNREGISTER: 637 PFE_PMD_INFO("unregister client_id %d", client_id); 638 639 /* Request for unregister a client */ 640 pfe_hif_client_unregister(hif, client_id); 641 642 break; 643 644 default: 645 PFE_PMD_ERR("unsupported request %d", req); 646 break; 647 } 648 649 /* 650 * Process client Tx queues 651 * Currently we don't have checking for tx pending 652 */ 653 } 654 655 #if defined(LS1012A_PFE_RESET_WA) 656 static void 657 pfe_hif_disable_rx_desc(struct pfe_hif *hif) 658 { 659 u32 ii; 660 struct hif_desc *desc = hif->rx_base; 661 662 /*Mark all descriptors as LAST_BD */ 663 for (ii = 0; ii < hif->rx_ring_size; ii++) { 664 desc->ctrl |= BD_CTRL_LAST_BD; 665 desc++; 666 } 667 } 668 669 struct class_rx_hdr_t { 670 u32 next_ptr; /* ptr to the start of the first DDR buffer */ 671 u16 length; /* total packet length */ 672 u16 phyno; /* input physical port number */ 673 u32 status; /* gemac status bits */ 674 u32 status2; /* reserved for software usage */ 675 }; 676 677 /* STATUS_BAD_FRAME_ERR is set for all errors (including checksums if enabled) 678 * except overflow 679 */ 680 #define STATUS_BAD_FRAME_ERR BIT(16) 681 #define STATUS_LENGTH_ERR BIT(17) 682 #define STATUS_CRC_ERR BIT(18) 683 #define STATUS_TOO_SHORT_ERR BIT(19) 684 #define STATUS_TOO_LONG_ERR BIT(20) 685 #define STATUS_CODE_ERR BIT(21) 686 #define STATUS_MC_HASH_MATCH BIT(22) 687 #define STATUS_CUMULATIVE_ARC_HIT BIT(23) 688 #define STATUS_UNICAST_HASH_MATCH BIT(24) 689 #define STATUS_IP_CHECKSUM_CORRECT BIT(25) 690 #define STATUS_TCP_CHECKSUM_CORRECT BIT(26) 691 #define STATUS_UDP_CHECKSUM_CORRECT BIT(27) 692 #define STATUS_OVERFLOW_ERR BIT(28) /* GPI error */ 693 #define MIN_PKT_SIZE 64 694 #define DUMMY_PKT_COUNT 128 695 696 static inline void 697 copy_to_lmem(u32 *dst, u32 *src, int len) 698 { 699 int i; 700 701 for (i = 0; i < len; i += sizeof(u32)) { 702 *dst = htonl(*src); 703 dst++; src++; 704 } 705 } 706 #if defined(RTE_TOOLCHAIN_GCC) 707 __attribute__ ((optimize(1))) 708 #endif 709 static void 710 send_dummy_pkt_to_hif(void) 711 { 712 void *lmem_ptr, *ddr_ptr, *lmem_virt_addr; 713 u64 physaddr; 714 struct class_rx_hdr_t local_hdr; 715 static u32 dummy_pkt[] = { 716 0x33221100, 0x2b785544, 0xd73093cb, 0x01000608, 717 0x04060008, 0x2b780200, 0xd73093cb, 0x0a01a8c0, 718 0x33221100, 0xa8c05544, 0x00000301, 0x00000000, 719 0x00000000, 0x00000000, 0x00000000, 0xbe86c51f }; 720 721 ddr_ptr = (void *)(size_t)readl(BMU2_BASE_ADDR + BMU_ALLOC_CTRL); 722 if (!ddr_ptr) 723 return; 724 725 lmem_ptr = (void *)(size_t)readl(BMU1_BASE_ADDR + BMU_ALLOC_CTRL); 726 if (!lmem_ptr) 727 return; 728 729 PFE_PMD_INFO("Sending a dummy pkt to HIF %p %p", ddr_ptr, lmem_ptr); 730 physaddr = DDR_VIRT_TO_PFE(ddr_ptr); 731 732 lmem_virt_addr = (void *)CBUS_PFE_TO_VIRT((unsigned long)lmem_ptr); 733 734 local_hdr.phyno = htons(0); /* RX_PHY_0 */ 735 local_hdr.length = htons(MIN_PKT_SIZE); 736 737 local_hdr.next_ptr = htonl((u32)physaddr); 738 /*Mark checksum is correct */ 739 local_hdr.status = htonl((STATUS_IP_CHECKSUM_CORRECT | 740 STATUS_UDP_CHECKSUM_CORRECT | 741 STATUS_TCP_CHECKSUM_CORRECT | 742 STATUS_UNICAST_HASH_MATCH | 743 STATUS_CUMULATIVE_ARC_HIT)); 744 copy_to_lmem((u32 *)lmem_virt_addr, (u32 *)&local_hdr, 745 sizeof(local_hdr)); 746 747 copy_to_lmem((u32 *)(lmem_virt_addr + LMEM_HDR_SIZE), (u32 *)dummy_pkt, 748 0x40); 749 750 writel((unsigned long)lmem_ptr, CLASS_INQ_PKTPTR); 751 } 752 753 void 754 pfe_hif_rx_idle(struct pfe_hif *hif) 755 { 756 int hif_stop_loop = DUMMY_PKT_COUNT; 757 u32 rx_status; 758 759 pfe_hif_disable_rx_desc(hif); 760 PFE_PMD_INFO("Bringing hif to idle state..."); 761 writel(0, HIF_INT_ENABLE); 762 /*If HIF Rx BDP is busy send a dummy packet */ 763 do { 764 rx_status = readl(HIF_RX_STATUS); 765 if (rx_status & BDP_CSR_RX_DMA_ACTV) 766 send_dummy_pkt_to_hif(); 767 768 sleep(1); 769 } while (--hif_stop_loop); 770 771 if (readl(HIF_RX_STATUS) & BDP_CSR_RX_DMA_ACTV) 772 PFE_PMD_ERR("Failed\n"); 773 else 774 PFE_PMD_INFO("Done\n"); 775 } 776 #endif 777 778 /* 779 * pfe_hif_init 780 * This function initializes the baseaddresses and irq, etc. 781 */ 782 int 783 pfe_hif_init(struct pfe *pfe) 784 { 785 struct pfe_hif *hif = &pfe->hif; 786 int err; 787 788 PMD_INIT_FUNC_TRACE(); 789 790 #if defined(LS1012A_PFE_RESET_WA) 791 pfe_hif_rx_idle(hif); 792 #endif 793 794 err = pfe_hif_alloc_descr(hif); 795 if (err) 796 goto err0; 797 798 rte_spinlock_init(&hif->tx_lock); 799 rte_spinlock_init(&hif->lock); 800 801 gpi_enable(HGPI_BASE_ADDR); 802 if (getenv("PFE_INTR_SUPPORT")) { 803 struct epoll_event epoll_ev; 804 int event_fd = -1, epoll_fd, pfe_cdev_fd; 805 806 pfe_cdev_fd = open(PFE_CDEV_PATH, O_RDWR); 807 if (pfe_cdev_fd < 0) { 808 PFE_PMD_WARN("Unable to open PFE device file (%s).\n", 809 PFE_CDEV_PATH); 810 pfe->cdev_fd = PFE_CDEV_INVALID_FD; 811 return -1; 812 } 813 pfe->cdev_fd = pfe_cdev_fd; 814 815 event_fd = eventfd(0, EFD_NONBLOCK); 816 /* hif interrupt enable */ 817 err = ioctl(pfe->cdev_fd, PFE_CDEV_HIF_INTR_EN, &event_fd); 818 if (err) { 819 PFE_PMD_ERR("\nioctl failed for intr enable err: %d\n", 820 errno); 821 goto err0; 822 } 823 epoll_fd = epoll_create(1); 824 epoll_ev.events = EPOLLIN | EPOLLPRI | EPOLLET; 825 epoll_ev.data.fd = event_fd; 826 err = epoll_ctl(epoll_fd, EPOLL_CTL_ADD, event_fd, &epoll_ev); 827 if (err < 0) { 828 PFE_PMD_ERR("epoll_ctl failed with err = %d\n", errno); 829 goto err0; 830 } 831 pfe->hif.epoll_fd = epoll_fd; 832 } 833 return 0; 834 err0: 835 return err; 836 } 837 838 /* pfe_hif_exit- */ 839 void 840 pfe_hif_exit(struct pfe *pfe) 841 { 842 struct pfe_hif *hif = &pfe->hif; 843 844 PMD_INIT_FUNC_TRACE(); 845 846 rte_spinlock_lock(&hif->lock); 847 hif->shm->g_client_status[0] = 0; 848 /* Make sure all clients are disabled*/ 849 hif->shm->g_client_status[1] = 0; 850 851 rte_spinlock_unlock(&hif->lock); 852 853 if (hif->setuped) { 854 #if defined(LS1012A_PFE_RESET_WA) 855 pfe_hif_rx_idle(hif); 856 #endif 857 /*Disable Rx/Tx */ 858 hif_rx_disable(); 859 hif_tx_disable(); 860 861 pfe_hif_release_buffers(hif); 862 pfe_hif_shm_clean(hif->shm); 863 864 pfe_hif_free_descr(hif); 865 pfe->hif.setuped = 0; 866 } 867 gpi_disable(HGPI_BASE_ADDR); 868 } 869