1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2021-2024 NXP 3 */ 4 5 #include <bus_dpaa_driver.h> 6 #include <rte_dmadev_pmd.h> 7 #include <rte_kvargs.h> 8 9 #include "dpaa_qdma.h" 10 #include "dpaa_qdma_logs.h" 11 12 static uint32_t s_sg_max_entry_sz = 2000; 13 static bool s_hw_err_check; 14 15 #define DPAA_DMA_ERROR_CHECK "dpaa_dma_err_check" 16 17 static inline void 18 qdma_desc_addr_set64(struct fsl_qdma_comp_cmd_desc *ccdf, u64 addr) 19 { 20 ccdf->addr_hi = upper_32_bits(addr); 21 ccdf->addr_lo = rte_cpu_to_le_32(lower_32_bits(addr)); 22 } 23 24 static inline void 25 qdma_desc_sge_addr_set64(struct fsl_qdma_comp_sg_desc *sge, u64 addr) 26 { 27 sge->addr_hi = upper_32_bits(addr); 28 sge->addr_lo = rte_cpu_to_le_32(lower_32_bits(addr)); 29 } 30 31 static inline int 32 qdma_ccdf_get_queue(struct fsl_qdma_comp_cmd_desc *ccdf, 33 uint8_t *queue_idx) 34 { 35 uint64_t addr = ((uint64_t)ccdf->addr_hi) << 32 | ccdf->addr_lo; 36 37 if (addr && queue_idx) 38 *queue_idx = ccdf->queue; 39 if (addr) { 40 ccdf->addr_hi = 0; 41 ccdf->addr_lo = 0; 42 return true; 43 } 44 45 return false; 46 } 47 48 static inline int 49 ilog2(int x) 50 { 51 int log = 0; 52 53 x >>= 1; 54 55 while (x) { 56 log++; 57 x >>= 1; 58 } 59 return log; 60 } 61 62 static inline int 63 ilog2_qsize(uint32_t q_size) 64 { 65 return (ilog2(q_size) - ilog2(64)); 66 } 67 68 static inline int 69 ilog2_qthld(uint32_t q_thld) 70 { 71 return (ilog2(q_thld) - ilog2(16)); 72 } 73 74 static inline int 75 fsl_qdma_queue_bd_in_hw(struct fsl_qdma_queue *fsl_queue) 76 { 77 struct rte_dma_stats *stats = &fsl_queue->stats; 78 79 return (stats->submitted - stats->completed); 80 } 81 82 static u32 83 qdma_readl(void *addr) 84 { 85 return QDMA_IN(addr); 86 } 87 88 static void 89 qdma_writel(u32 val, void *addr) 90 { 91 QDMA_OUT(addr, val); 92 } 93 94 static u32 95 qdma_readl_be(void *addr) 96 { 97 return QDMA_IN_BE(addr); 98 } 99 100 static void 101 qdma_writel_be(u32 val, void *addr) 102 { 103 QDMA_OUT_BE(addr, val); 104 } 105 106 static void * 107 dma_pool_alloc(char *nm, int size, int aligned, dma_addr_t *phy_addr) 108 { 109 void *virt_addr; 110 111 virt_addr = rte_zmalloc(nm, size, aligned); 112 if (!virt_addr) 113 return NULL; 114 115 *phy_addr = rte_mem_virt2iova(virt_addr); 116 117 return virt_addr; 118 } 119 120 /* 121 * Pre-request command descriptor and compound S/G for enqueue. 122 */ 123 static int 124 fsl_qdma_pre_comp_sd_desc(struct fsl_qdma_queue *queue) 125 { 126 struct fsl_qdma_engine *fsl_qdma = queue->engine; 127 struct fsl_qdma_sdf *sdf; 128 struct fsl_qdma_ddf *ddf; 129 struct fsl_qdma_comp_cmd_desc *ccdf; 130 uint16_t i, j; 131 struct fsl_qdma_cmpd_ft *ft; 132 133 for (i = 0; i < queue->n_cq; i++) { 134 dma_addr_t phy_ft = 0; 135 136 queue->ft[i] = dma_pool_alloc(NULL, 137 sizeof(struct fsl_qdma_cmpd_ft), 138 RTE_CACHE_LINE_SIZE, &phy_ft); 139 if (!queue->ft[i]) 140 goto fail; 141 if (((uint64_t)queue->ft[i]) & 142 (RTE_CACHE_LINE_SIZE - 1)) { 143 DPAA_QDMA_ERR("FD[%d] addr(%p) not cache aligned", 144 i, queue->ft[i]); 145 rte_free(queue->ft[i]); 146 queue->ft[i] = NULL; 147 goto fail; 148 } 149 if (((uint64_t)(&queue->ft[i]->desc_ssge[0])) & 150 (RTE_CACHE_LINE_SIZE - 1)) { 151 DPAA_QDMA_ERR("FD[%d] SGE addr(%p) not cache aligned", 152 i, &queue->ft[i]->desc_ssge[0]); 153 rte_free(queue->ft[i]); 154 queue->ft[i] = NULL; 155 goto fail; 156 } 157 queue->ft[i]->phy_ssge = phy_ft + 158 offsetof(struct fsl_qdma_cmpd_ft, desc_ssge); 159 queue->ft[i]->phy_dsge = phy_ft + 160 offsetof(struct fsl_qdma_cmpd_ft, desc_dsge); 161 queue->ft[i]->phy_df = phy_ft + 162 offsetof(struct fsl_qdma_cmpd_ft, df); 163 164 ft = queue->ft[i]; 165 sdf = &ft->df.sdf; 166 ddf = &ft->df.ddf; 167 /* Compound Command Descriptor(Frame List Table) */ 168 qdma_desc_sge_addr_set64(&ft->desc_buf, ft->phy_df); 169 /* It must be 32 as Compound S/G Descriptor */ 170 ft->desc_buf.length = sizeof(struct fsl_qdma_df); 171 172 /* Descriptor Buffer */ 173 sdf->srttype = FSL_QDMA_CMD_RWTTYPE; 174 #ifdef RTE_DMA_DPAA_ERRATA_ERR050265 175 sdf->prefetch = 1; 176 #endif 177 ddf->dwttype = FSL_QDMA_CMD_RWTTYPE; 178 ddf->lwc = FSL_QDMA_CMD_LWC; 179 180 ccdf = &queue->cq[i]; 181 qdma_desc_addr_set64(ccdf, phy_ft); 182 ccdf->format = FSL_QDMA_COMP_SG_FORMAT; 183 if (!fsl_qdma->is_silent) 184 ccdf->ser = 1; 185 ccdf->queue = queue->queue_id; 186 } 187 queue->ci = 0; 188 189 return 0; 190 191 fail: 192 for (j = 0; j < i; j++) 193 rte_free(queue->ft[j]); 194 195 return -ENOMEM; 196 } 197 198 static int 199 fsl_qdma_alloc_queue_resources(struct fsl_qdma_engine *fsl_qdma, 200 int queue_id, int block_id) 201 { 202 struct fsl_qdma_queue *cmd_queue; 203 uint32_t queue_size; 204 char nm[RTE_MEMZONE_NAMESIZE]; 205 206 cmd_queue = &fsl_qdma->cmd_queues[block_id][queue_id]; 207 cmd_queue->engine = fsl_qdma; 208 209 queue_size = sizeof(struct fsl_qdma_comp_cmd_desc) * 210 QDMA_QUEUE_SIZE; 211 212 sprintf(nm, "Command queue_%d_%d", 213 block_id, queue_id); 214 cmd_queue->cq = dma_pool_alloc(nm, queue_size, 215 queue_size, &cmd_queue->bus_addr); 216 if (!cmd_queue->cq) { 217 DPAA_QDMA_ERR("%s alloc failed!", nm); 218 return -ENOMEM; 219 } 220 221 cmd_queue->block_vir = fsl_qdma->block_base + 222 FSL_QDMA_BLOCK_BASE_OFFSET(fsl_qdma, block_id); 223 cmd_queue->n_cq = QDMA_QUEUE_SIZE; 224 cmd_queue->queue_id = queue_id; 225 cmd_queue->block_id = block_id; 226 cmd_queue->pending_start = 0; 227 cmd_queue->pending_num = 0; 228 cmd_queue->complete_start = 0; 229 230 sprintf(nm, "Compound Table_%d_%d", 231 block_id, queue_id); 232 cmd_queue->ft = rte_zmalloc(nm, 233 sizeof(void *) * QDMA_QUEUE_SIZE, 0); 234 if (!cmd_queue->ft) { 235 DPAA_QDMA_ERR("%s zmalloc failed!", nm); 236 rte_free(cmd_queue->cq); 237 return -ENOMEM; 238 } 239 sprintf(nm, "Pending_desc_%d_%d", 240 block_id, queue_id); 241 cmd_queue->pending_desc = rte_zmalloc(nm, 242 sizeof(struct fsl_qdma_desc) * FSL_QDMA_MAX_DESC_NUM, 0); 243 if (!cmd_queue->pending_desc) { 244 DPAA_QDMA_ERR("%s zmalloc failed!", nm); 245 rte_free(cmd_queue->ft); 246 rte_free(cmd_queue->cq); 247 return -ENOMEM; 248 } 249 sprintf(nm, "complete-burst_ring_%d_%d", 250 block_id, queue_id); 251 cmd_queue->complete_burst = rte_ring_create(nm, 252 QDMA_QUEUE_SIZE * 2, 0, 253 RING_F_SP_ENQ | RING_F_SC_DEQ); 254 if (!cmd_queue->complete_burst) { 255 DPAA_QDMA_ERR("%s create failed!", nm); 256 rte_free(cmd_queue->pending_desc); 257 rte_free(cmd_queue->ft); 258 rte_free(cmd_queue->cq); 259 return -ENOMEM; 260 } 261 sprintf(nm, "complete-desc_ring_%d_%d", 262 block_id, queue_id); 263 cmd_queue->complete_desc = rte_ring_create(nm, 264 FSL_QDMA_MAX_DESC_NUM * 2, 0, 265 RING_F_SP_ENQ | RING_F_SC_DEQ); 266 if (!cmd_queue->complete_desc) { 267 DPAA_QDMA_ERR("%s create failed!", nm); 268 rte_ring_free(cmd_queue->complete_burst); 269 rte_free(cmd_queue->pending_desc); 270 rte_free(cmd_queue->ft); 271 rte_free(cmd_queue->cq); 272 return -ENOMEM; 273 } 274 sprintf(nm, "complete-pool-desc_ring_%d_%d", 275 block_id, queue_id); 276 cmd_queue->complete_pool = rte_ring_create(nm, 277 FSL_QDMA_MAX_DESC_NUM * 2, 0, 278 RING_F_SP_ENQ | RING_F_SC_DEQ); 279 if (!cmd_queue->complete_pool) { 280 DPAA_QDMA_ERR("%s create failed!", nm); 281 rte_ring_free(cmd_queue->complete_desc); 282 rte_ring_free(cmd_queue->complete_burst); 283 rte_free(cmd_queue->pending_desc); 284 rte_free(cmd_queue->ft); 285 rte_free(cmd_queue->cq); 286 return -ENOMEM; 287 } 288 289 memset(&cmd_queue->stats, 0, sizeof(struct rte_dma_stats)); 290 cmd_queue->pending_max = FSL_QDMA_MAX_DESC_NUM; 291 292 return 0; 293 } 294 295 static void 296 fsl_qdma_free_cmdq_res(struct fsl_qdma_queue *queue) 297 { 298 rte_free(queue->ft); 299 rte_free(queue->cq); 300 rte_free(queue->pending_desc); 301 rte_ring_free(queue->complete_burst); 302 rte_ring_free(queue->complete_desc); 303 rte_ring_free(queue->complete_pool); 304 } 305 306 static void 307 fsl_qdma_free_stq_res(struct fsl_qdma_status_queue *queue) 308 { 309 rte_free(queue->cq); 310 } 311 312 static int 313 fsl_qdma_prep_status_queue(struct fsl_qdma_engine *fsl_qdma, 314 uint32_t block_id) 315 { 316 struct fsl_qdma_status_queue *status; 317 uint32_t status_size; 318 319 status = &fsl_qdma->stat_queues[block_id]; 320 status->engine = fsl_qdma; 321 322 status_size = QDMA_STATUS_SIZE * 323 sizeof(struct fsl_qdma_comp_cmd_desc); 324 325 status->cq = dma_pool_alloc(NULL, status_size, 326 status_size, &status->bus_addr); 327 328 if (!status->cq) 329 return -ENOMEM; 330 331 memset(status->cq, 0x0, status_size); 332 status->n_cq = QDMA_STATUS_SIZE; 333 status->complete = 0; 334 status->block_id = block_id; 335 status->block_vir = fsl_qdma->block_base + 336 FSL_QDMA_BLOCK_BASE_OFFSET(fsl_qdma, block_id); 337 338 return 0; 339 } 340 341 static int 342 fsl_qdma_halt(struct fsl_qdma_engine *fsl_qdma) 343 { 344 void *ctrl = fsl_qdma->ctrl_base; 345 void *block; 346 int i, count = RETRIES; 347 unsigned int j; 348 u32 reg; 349 350 /* Disable the command queue and wait for idle state. */ 351 reg = qdma_readl(ctrl + FSL_QDMA_DMR); 352 reg |= FSL_QDMA_DMR_DQD; 353 qdma_writel(reg, ctrl + FSL_QDMA_DMR); 354 for (j = 0; j < fsl_qdma->num_blocks; j++) { 355 block = fsl_qdma->block_base + 356 FSL_QDMA_BLOCK_BASE_OFFSET(fsl_qdma, j); 357 for (i = 0; i < FSL_QDMA_QUEUE_NUM_MAX; i++) 358 qdma_writel(0, block + FSL_QDMA_BCQMR(i)); 359 } 360 while (true) { 361 reg = qdma_readl(ctrl + FSL_QDMA_DSR); 362 if (!(reg & FSL_QDMA_DSR_DB)) 363 break; 364 if (count-- < 0) 365 return -EBUSY; 366 rte_delay_us(100); 367 } 368 369 for (j = 0; j < fsl_qdma->num_blocks; j++) { 370 block = fsl_qdma->block_base + 371 FSL_QDMA_BLOCK_BASE_OFFSET(fsl_qdma, j); 372 373 /* Disable status queue. */ 374 qdma_writel(0, block + FSL_QDMA_BSQMR); 375 376 /* 377 * clear the command queue interrupt detect register for 378 * all queues. 379 */ 380 qdma_writel(0xffffffff, block + FSL_QDMA_BCQIDR(0)); 381 } 382 383 return 0; 384 } 385 386 static void 387 fsl_qdma_data_validation(struct fsl_qdma_desc *desc[], 388 uint8_t num, struct fsl_qdma_queue *fsl_queue) 389 { 390 uint32_t i, j; 391 uint8_t *v_src, *v_dst; 392 char err_msg[512]; 393 int offset; 394 395 396 offset = sprintf(err_msg, "Fatal TC%d/queue%d: ", 397 fsl_queue->block_id, 398 fsl_queue->queue_id); 399 for (i = 0; i < num; i++) { 400 v_src = rte_mem_iova2virt(desc[i]->src); 401 v_dst = rte_mem_iova2virt(desc[i]->dst); 402 for (j = 0; j < desc[i]->len; j++) { 403 if (v_src[j] != v_dst[j]) { 404 sprintf(&err_msg[offset], 405 "job[%"PRIu64"]:src(%p)[%d](%d)!=dst(%p)[%d](%d)", 406 desc[i]->flag, v_src, j, v_src[j], 407 v_dst, j, v_dst[j]); 408 DPAA_QDMA_ERR("%s, stop validating!", 409 err_msg); 410 return; 411 } 412 } 413 } 414 } 415 416 static int 417 fsl_qdma_reg_init(struct fsl_qdma_engine *fsl_qdma) 418 { 419 struct fsl_qdma_queue *temp; 420 struct fsl_qdma_status_queue *temp_stat; 421 void *ctrl = fsl_qdma->ctrl_base; 422 void *block; 423 u32 i, j; 424 u32 reg; 425 int ret, val; 426 427 /* Try to halt the qDMA engine first. */ 428 ret = fsl_qdma_halt(fsl_qdma); 429 if (ret) { 430 DPAA_QDMA_ERR("DMA halt failed!"); 431 return ret; 432 } 433 434 for (j = 0; j < fsl_qdma->num_blocks; j++) { 435 block = fsl_qdma->block_base + 436 FSL_QDMA_BLOCK_BASE_OFFSET(fsl_qdma, j); 437 for (i = 0; i < QDMA_QUEUES; i++) { 438 temp = &fsl_qdma->cmd_queues[j][i]; 439 /* 440 * Initialize Command Queue registers to 441 * point to the first 442 * command descriptor in memory. 443 * Dequeue Pointer Address Registers 444 * Enqueue Pointer Address Registers 445 */ 446 447 qdma_writel(lower_32_bits(temp->bus_addr), 448 block + FSL_QDMA_BCQDPA_SADDR(i)); 449 qdma_writel(upper_32_bits(temp->bus_addr), 450 block + FSL_QDMA_BCQEDPA_SADDR(i)); 451 qdma_writel(lower_32_bits(temp->bus_addr), 452 block + FSL_QDMA_BCQEPA_SADDR(i)); 453 qdma_writel(upper_32_bits(temp->bus_addr), 454 block + FSL_QDMA_BCQEEPA_SADDR(i)); 455 456 /* Initialize the queue mode. */ 457 reg = FSL_QDMA_BCQMR_EN; 458 reg |= FSL_QDMA_BCQMR_CD_THLD(ilog2_qthld(temp->n_cq)); 459 reg |= FSL_QDMA_BCQMR_CQ_SIZE(ilog2_qsize(temp->n_cq)); 460 temp->le_cqmr = reg; 461 qdma_writel(reg, block + FSL_QDMA_BCQMR(i)); 462 } 463 464 /* 465 * Workaround for erratum: ERR010812. 466 * We must enable XOFF to avoid the enqueue rejection occurs. 467 * Setting SQCCMR ENTER_WM to 0x20. 468 */ 469 470 qdma_writel(FSL_QDMA_SQCCMR_ENTER_WM, 471 block + FSL_QDMA_SQCCMR); 472 473 /* 474 * Initialize status queue registers to point to the first 475 * command descriptor in memory. 476 * Dequeue Pointer Address Registers 477 * Enqueue Pointer Address Registers 478 */ 479 480 temp_stat = &fsl_qdma->stat_queues[j]; 481 qdma_writel(upper_32_bits(temp_stat->bus_addr), 482 block + FSL_QDMA_SQEEPAR); 483 qdma_writel(lower_32_bits(temp_stat->bus_addr), 484 block + FSL_QDMA_SQEPAR); 485 qdma_writel(upper_32_bits(temp_stat->bus_addr), 486 block + FSL_QDMA_SQEDPAR); 487 qdma_writel(lower_32_bits(temp_stat->bus_addr), 488 block + FSL_QDMA_SQDPAR); 489 /* Desiable status queue interrupt. */ 490 491 qdma_writel(0x0, block + FSL_QDMA_BCQIER(0)); 492 qdma_writel(0x0, block + FSL_QDMA_BSQICR); 493 qdma_writel(0x0, block + FSL_QDMA_CQIER); 494 495 /* Initialize the status queue mode. */ 496 reg = FSL_QDMA_BSQMR_EN; 497 val = ilog2_qsize(temp_stat->n_cq); 498 reg |= FSL_QDMA_BSQMR_CQ_SIZE(val); 499 qdma_writel(reg, block + FSL_QDMA_BSQMR); 500 } 501 502 reg = qdma_readl(ctrl + FSL_QDMA_DMR); 503 reg &= ~FSL_QDMA_DMR_DQD; 504 qdma_writel(reg, ctrl + FSL_QDMA_DMR); 505 506 return 0; 507 } 508 509 static uint16_t 510 dpaa_qdma_block_dequeue(struct fsl_qdma_engine *fsl_qdma, 511 uint8_t block_id) 512 { 513 struct fsl_qdma_status_queue *stat_queue; 514 struct fsl_qdma_queue *cmd_queue; 515 struct fsl_qdma_comp_cmd_desc *cq; 516 uint16_t start, count = 0; 517 uint8_t qid = 0; 518 uint32_t reg; 519 int ret; 520 uint8_t *block; 521 uint16_t *dq_complete; 522 struct fsl_qdma_desc *desc[FSL_QDMA_SG_MAX_ENTRY]; 523 524 stat_queue = &fsl_qdma->stat_queues[block_id]; 525 cq = stat_queue->cq; 526 start = stat_queue->complete; 527 528 block = fsl_qdma->block_base + 529 FSL_QDMA_BLOCK_BASE_OFFSET(fsl_qdma, block_id); 530 531 do { 532 reg = qdma_readl_be(block + FSL_QDMA_BSQSR); 533 if (reg & FSL_QDMA_BSQSR_QE_BE) 534 break; 535 536 qdma_writel_be(FSL_QDMA_BSQMR_DI, block + FSL_QDMA_BSQMR); 537 ret = qdma_ccdf_get_queue(&cq[start], &qid); 538 if (ret == true) { 539 cmd_queue = &fsl_qdma->cmd_queues[block_id][qid]; 540 541 ret = rte_ring_dequeue(cmd_queue->complete_burst, 542 (void **)&dq_complete); 543 if (ret) { 544 DPAA_QDMA_ERR("DQ desc number failed!"); 545 break; 546 } 547 548 ret = rte_ring_dequeue_bulk(cmd_queue->complete_desc, 549 (void **)desc, *dq_complete, NULL); 550 if (ret != (*dq_complete)) { 551 DPAA_QDMA_ERR("DQ %d descs failed!(%d)", 552 *dq_complete, ret); 553 break; 554 } 555 556 fsl_qdma_data_validation(desc, *dq_complete, cmd_queue); 557 558 ret = rte_ring_enqueue_bulk(cmd_queue->complete_pool, 559 (void **)desc, (*dq_complete), NULL); 560 if (ret != (*dq_complete)) { 561 DPAA_QDMA_ERR("Failed desc eq %d!=%d to %s", 562 ret, *dq_complete, 563 cmd_queue->complete_pool->name); 564 break; 565 } 566 567 cmd_queue->complete_start = 568 (cmd_queue->complete_start + (*dq_complete)) & 569 (cmd_queue->pending_max - 1); 570 cmd_queue->stats.completed++; 571 572 start++; 573 if (unlikely(start == stat_queue->n_cq)) 574 start = 0; 575 count++; 576 } else { 577 DPAA_QDMA_ERR("Block%d not empty but dq-queue failed!", 578 block_id); 579 break; 580 } 581 } while (1); 582 stat_queue->complete = start; 583 584 return count; 585 } 586 587 static int 588 fsl_qdma_enqueue_desc_to_ring(struct fsl_qdma_queue *fsl_queue, 589 uint16_t num) 590 { 591 struct fsl_qdma_engine *fsl_qdma = fsl_queue->engine; 592 uint16_t i, idx, start, dq; 593 int ret, dq_cnt; 594 595 if (fsl_qdma->is_silent) 596 return 0; 597 598 fsl_queue->desc_in_hw[fsl_queue->ci] = num; 599 eq_again: 600 ret = rte_ring_enqueue(fsl_queue->complete_burst, 601 &fsl_queue->desc_in_hw[fsl_queue->ci]); 602 if (ret) { 603 DPAA_QDMA_DP_DEBUG("%s: Queue is full, try dequeue first", 604 __func__); 605 DPAA_QDMA_DP_DEBUG("%s: submitted:%"PRIu64", completed:%"PRIu64"", 606 __func__, fsl_queue->stats.submitted, 607 fsl_queue->stats.completed); 608 dq_cnt = 0; 609 dq_again: 610 dq = dpaa_qdma_block_dequeue(fsl_queue->engine, 611 fsl_queue->block_id); 612 dq_cnt++; 613 if (dq > 0) { 614 goto eq_again; 615 } else { 616 if (dq_cnt < 100) 617 goto dq_again; 618 DPAA_QDMA_ERR("%s: Dq block%d failed!", 619 __func__, fsl_queue->block_id); 620 } 621 return ret; 622 } 623 start = fsl_queue->pending_start; 624 for (i = 0; i < num; i++) { 625 idx = (start + i) & (fsl_queue->pending_max - 1); 626 ret = rte_ring_enqueue(fsl_queue->complete_desc, 627 &fsl_queue->pending_desc[idx]); 628 if (ret) { 629 DPAA_QDMA_ERR("Descriptors eq failed!"); 630 return ret; 631 } 632 } 633 634 return 0; 635 } 636 637 static int 638 fsl_qdma_enqueue_overflow(struct fsl_qdma_queue *fsl_queue) 639 { 640 int overflow = 0; 641 uint32_t reg; 642 uint16_t blk_drain, check_num, drain_num; 643 uint8_t *block = fsl_queue->block_vir; 644 const struct rte_dma_stats *st = &fsl_queue->stats; 645 struct fsl_qdma_engine *fsl_qdma = fsl_queue->engine; 646 647 check_num = 0; 648 overflow_check: 649 if (fsl_qdma->is_silent || unlikely(s_hw_err_check)) { 650 reg = qdma_readl_be(block + 651 FSL_QDMA_BCQSR(fsl_queue->queue_id)); 652 overflow = (reg & FSL_QDMA_BCQSR_QF_XOFF_BE) ? 653 1 : 0; 654 } else { 655 overflow = (fsl_qdma_queue_bd_in_hw(fsl_queue) >= 656 QDMA_QUEUE_CR_WM) ? 1 : 0; 657 } 658 659 if (likely(!overflow)) { 660 return 0; 661 } else if (fsl_qdma->is_silent) { 662 check_num++; 663 if (check_num >= 10000) { 664 DPAA_QDMA_WARN("Waiting for HW complete in silent mode"); 665 check_num = 0; 666 } 667 goto overflow_check; 668 } 669 670 DPAA_QDMA_DP_DEBUG("TC%d/Q%d submitted(%"PRIu64")-completed(%"PRIu64") >= %d", 671 fsl_queue->block_id, fsl_queue->queue_id, 672 st->submitted, st->completed, QDMA_QUEUE_CR_WM); 673 drain_num = 0; 674 675 drain_again: 676 blk_drain = dpaa_qdma_block_dequeue(fsl_qdma, 677 fsl_queue->block_id); 678 if (!blk_drain) { 679 drain_num++; 680 if (drain_num >= 10000) { 681 DPAA_QDMA_WARN("TC%d failed drain, Q%d's %"PRIu64" bd in HW.", 682 fsl_queue->block_id, fsl_queue->queue_id, 683 st->submitted - st->completed); 684 drain_num = 0; 685 } 686 goto drain_again; 687 } 688 check_num++; 689 if (check_num >= 1000) { 690 DPAA_QDMA_WARN("TC%d failed check, Q%d's %"PRIu64" bd in HW.", 691 fsl_queue->block_id, fsl_queue->queue_id, 692 st->submitted - st->completed); 693 check_num = 0; 694 } 695 goto overflow_check; 696 697 return 0; 698 } 699 700 static int 701 fsl_qdma_enqueue_desc_single(struct fsl_qdma_queue *fsl_queue, 702 dma_addr_t dst, dma_addr_t src, size_t len) 703 { 704 uint8_t *block = fsl_queue->block_vir; 705 struct fsl_qdma_comp_sg_desc *csgf_src, *csgf_dest; 706 struct fsl_qdma_cmpd_ft *ft; 707 int ret; 708 #ifdef RTE_DMA_DPAA_ERRATA_ERR050757 709 struct fsl_qdma_sdf *sdf; 710 #endif 711 712 ret = fsl_qdma_enqueue_overflow(fsl_queue); 713 if (unlikely(ret)) 714 return ret; 715 716 ft = fsl_queue->ft[fsl_queue->ci]; 717 718 #ifdef RTE_DMA_DPAA_ERRATA_ERR050757 719 sdf = &ft->df.sdf; 720 sdf->srttype = FSL_QDMA_CMD_RWTTYPE; 721 #ifdef RTE_DMA_DPAA_ERRATA_ERR050265 722 sdf->prefetch = 1; 723 #endif 724 if (len > FSL_QDMA_CMD_SS_ERR050757_LEN) { 725 sdf->ssen = 1; 726 sdf->sss = FSL_QDMA_CMD_SS_ERR050757_LEN; 727 sdf->ssd = FSL_QDMA_CMD_SS_ERR050757_LEN; 728 } else { 729 sdf->ssen = 0; 730 sdf->sss = 0; 731 sdf->ssd = 0; 732 } 733 #endif 734 csgf_src = &ft->desc_sbuf; 735 csgf_dest = &ft->desc_dbuf; 736 qdma_desc_sge_addr_set64(csgf_src, src); 737 csgf_src->length = len; 738 csgf_src->extion = 0; 739 qdma_desc_sge_addr_set64(csgf_dest, dst); 740 csgf_dest->length = len; 741 csgf_dest->extion = 0; 742 /* This entry is the last entry. */ 743 csgf_dest->final = 1; 744 745 ret = fsl_qdma_enqueue_desc_to_ring(fsl_queue, 1); 746 if (ret) 747 return ret; 748 fsl_queue->ci = (fsl_queue->ci + 1) & (fsl_queue->n_cq - 1); 749 750 qdma_writel(fsl_queue->le_cqmr | FSL_QDMA_BCQMR_EI, 751 block + FSL_QDMA_BCQMR(fsl_queue->queue_id)); 752 fsl_queue->stats.submitted++; 753 754 return 0; 755 } 756 757 static int 758 fsl_qdma_enqueue_desc_sg(struct fsl_qdma_queue *fsl_queue) 759 { 760 uint8_t *block = fsl_queue->block_vir; 761 struct fsl_qdma_comp_sg_desc *csgf_src, *csgf_dest; 762 struct fsl_qdma_cmpd_ft *ft; 763 uint32_t total_len; 764 uint16_t start, idx, num, i, next_idx; 765 int ret; 766 #ifdef RTE_DMA_DPAA_ERRATA_ERR050757 767 struct fsl_qdma_sdf *sdf; 768 #endif 769 770 eq_sg: 771 total_len = 0; 772 start = fsl_queue->pending_start; 773 if (fsl_queue->pending_desc[start].len > s_sg_max_entry_sz || 774 fsl_queue->pending_num == 1) { 775 ret = fsl_qdma_enqueue_desc_single(fsl_queue, 776 fsl_queue->pending_desc[start].dst, 777 fsl_queue->pending_desc[start].src, 778 fsl_queue->pending_desc[start].len); 779 if (!ret) { 780 fsl_queue->pending_start = 781 (start + 1) & (fsl_queue->pending_max - 1); 782 fsl_queue->pending_num--; 783 } 784 if (fsl_queue->pending_num > 0) 785 goto eq_sg; 786 787 return ret; 788 } 789 790 ret = fsl_qdma_enqueue_overflow(fsl_queue); 791 if (unlikely(ret)) 792 return ret; 793 794 if (fsl_queue->pending_num > FSL_QDMA_SG_MAX_ENTRY) 795 num = FSL_QDMA_SG_MAX_ENTRY; 796 else 797 num = fsl_queue->pending_num; 798 799 ft = fsl_queue->ft[fsl_queue->ci]; 800 csgf_src = &ft->desc_sbuf; 801 csgf_dest = &ft->desc_dbuf; 802 803 qdma_desc_sge_addr_set64(csgf_src, ft->phy_ssge); 804 csgf_src->extion = 1; 805 qdma_desc_sge_addr_set64(csgf_dest, ft->phy_dsge); 806 csgf_dest->extion = 1; 807 /* This entry is the last entry. */ 808 csgf_dest->final = 1; 809 for (i = 0; i < num; i++) { 810 idx = (start + i) & (fsl_queue->pending_max - 1); 811 qdma_desc_sge_addr_set64(&ft->desc_ssge[i], 812 fsl_queue->pending_desc[idx].src); 813 ft->desc_ssge[i].length = fsl_queue->pending_desc[idx].len; 814 ft->desc_ssge[i].final = 0; 815 qdma_desc_sge_addr_set64(&ft->desc_dsge[i], 816 fsl_queue->pending_desc[idx].dst); 817 ft->desc_dsge[i].length = fsl_queue->pending_desc[idx].len; 818 ft->desc_dsge[i].final = 0; 819 total_len += fsl_queue->pending_desc[idx].len; 820 if ((i + 1) != num) { 821 next_idx = (idx + 1) & (fsl_queue->pending_max - 1); 822 if (fsl_queue->pending_desc[next_idx].len > 823 s_sg_max_entry_sz) { 824 num = i + 1; 825 break; 826 } 827 } 828 } 829 830 ft->desc_ssge[num - 1].final = 1; 831 ft->desc_dsge[num - 1].final = 1; 832 csgf_src->length = total_len; 833 csgf_dest->length = total_len; 834 #ifdef RTE_DMA_DPAA_ERRATA_ERR050757 835 sdf = &ft->df.sdf; 836 sdf->srttype = FSL_QDMA_CMD_RWTTYPE; 837 #ifdef RTE_DMA_DPAA_ERRATA_ERR050265 838 sdf->prefetch = 1; 839 #endif 840 if (total_len > FSL_QDMA_CMD_SS_ERR050757_LEN) { 841 sdf->ssen = 1; 842 sdf->sss = FSL_QDMA_CMD_SS_ERR050757_LEN; 843 sdf->ssd = FSL_QDMA_CMD_SS_ERR050757_LEN; 844 } else { 845 sdf->ssen = 0; 846 sdf->sss = 0; 847 sdf->ssd = 0; 848 } 849 #endif 850 ret = fsl_qdma_enqueue_desc_to_ring(fsl_queue, num); 851 if (ret) 852 return ret; 853 854 fsl_queue->ci = (fsl_queue->ci + 1) & (fsl_queue->n_cq - 1); 855 856 qdma_writel(fsl_queue->le_cqmr | FSL_QDMA_BCQMR_EI, 857 block + FSL_QDMA_BCQMR(fsl_queue->queue_id)); 858 fsl_queue->stats.submitted++; 859 860 fsl_queue->pending_start = 861 (start + num) & (fsl_queue->pending_max - 1); 862 fsl_queue->pending_num -= num; 863 if (fsl_queue->pending_num > 0) 864 goto eq_sg; 865 866 return 0; 867 } 868 869 static int 870 fsl_qdma_enqueue_desc(struct fsl_qdma_queue *fsl_queue) 871 { 872 uint16_t start = fsl_queue->pending_start; 873 int ret; 874 875 if (fsl_queue->pending_num == 1) { 876 ret = fsl_qdma_enqueue_desc_single(fsl_queue, 877 fsl_queue->pending_desc[start].dst, 878 fsl_queue->pending_desc[start].src, 879 fsl_queue->pending_desc[start].len); 880 if (!ret) { 881 fsl_queue->pending_start = 882 (start + 1) & (fsl_queue->pending_max - 1); 883 fsl_queue->pending_num = 0; 884 } 885 return ret; 886 } 887 888 return fsl_qdma_enqueue_desc_sg(fsl_queue); 889 } 890 891 static int 892 dpaa_qdma_info_get(const struct rte_dma_dev *dev, 893 struct rte_dma_info *dev_info, __rte_unused uint32_t info_sz) 894 { 895 struct fsl_qdma_engine *fsl_qdma = dev->data->dev_private; 896 897 dev_info->dev_capa = RTE_DMA_CAPA_MEM_TO_MEM | 898 RTE_DMA_CAPA_SILENT | RTE_DMA_CAPA_OPS_COPY | 899 RTE_DMA_CAPA_OPS_COPY_SG; 900 dev_info->dev_capa |= DPAA_QDMA_FLAGS_INDEX; 901 dev_info->max_vchans = fsl_qdma->n_queues; 902 dev_info->max_desc = FSL_QDMA_MAX_DESC_NUM; 903 dev_info->min_desc = QDMA_QUEUE_SIZE; 904 dev_info->max_sges = FSL_QDMA_SG_MAX_ENTRY; 905 906 return 0; 907 } 908 909 static int 910 dpaa_get_channel(struct fsl_qdma_engine *fsl_qdma, 911 uint16_t vchan) 912 { 913 int ret, i, j, found = 0; 914 struct fsl_qdma_queue *fsl_queue = fsl_qdma->chan[vchan]; 915 916 if (fsl_queue) { 917 found = 1; 918 goto queue_found; 919 } 920 921 for (i = 0; i < QDMA_BLOCKS; i++) { 922 for (j = 0; j < QDMA_QUEUES; j++) { 923 fsl_queue = &fsl_qdma->cmd_queues[i][j]; 924 925 if (fsl_queue->channel_id == vchan) { 926 found = 1; 927 fsl_qdma->chan[vchan] = fsl_queue; 928 goto queue_found; 929 } 930 } 931 } 932 933 queue_found: 934 if (!found) 935 return -ENXIO; 936 937 if (fsl_queue->used) 938 return 0; 939 940 ret = fsl_qdma_pre_comp_sd_desc(fsl_queue); 941 if (ret) 942 return ret; 943 944 fsl_queue->used = 1; 945 fsl_qdma->block_queues[fsl_queue->block_id]++; 946 947 return 0; 948 } 949 950 static int 951 dpaa_qdma_configure(struct rte_dma_dev *dmadev, 952 const struct rte_dma_conf *dev_conf, 953 __rte_unused uint32_t conf_sz) 954 { 955 struct fsl_qdma_engine *fsl_qdma = dmadev->data->dev_private; 956 957 fsl_qdma->is_silent = dev_conf->enable_silent; 958 return 0; 959 } 960 961 static int 962 dpaa_qdma_start(__rte_unused struct rte_dma_dev *dev) 963 { 964 return 0; 965 } 966 967 static int 968 dpaa_qdma_close(__rte_unused struct rte_dma_dev *dev) 969 { 970 return 0; 971 } 972 973 static int 974 dpaa_qdma_queue_setup(struct rte_dma_dev *dmadev, 975 uint16_t vchan, 976 __rte_unused const struct rte_dma_vchan_conf *conf, 977 __rte_unused uint32_t conf_sz) 978 { 979 struct fsl_qdma_engine *fsl_qdma = dmadev->data->dev_private; 980 981 return dpaa_get_channel(fsl_qdma, vchan); 982 } 983 984 static int 985 dpaa_qdma_submit(void *dev_private, uint16_t vchan) 986 { 987 struct fsl_qdma_engine *fsl_qdma = dev_private; 988 struct fsl_qdma_queue *fsl_queue = fsl_qdma->chan[vchan]; 989 990 if (!fsl_queue->pending_num) 991 return 0; 992 993 return fsl_qdma_enqueue_desc(fsl_queue); 994 } 995 996 static int 997 dpaa_qdma_enqueue(void *dev_private, uint16_t vchan, 998 rte_iova_t src, rte_iova_t dst, 999 uint32_t length, uint64_t flags) 1000 { 1001 struct fsl_qdma_engine *fsl_qdma = dev_private; 1002 struct fsl_qdma_queue *fsl_queue = fsl_qdma->chan[vchan]; 1003 uint16_t start = fsl_queue->pending_start; 1004 uint8_t pending = fsl_queue->pending_num; 1005 uint16_t idx; 1006 int ret; 1007 1008 if (pending >= fsl_queue->pending_max) { 1009 DPAA_QDMA_ERR("Too many pending jobs(%d) on queue%d", 1010 pending, vchan); 1011 return -ENOSPC; 1012 } 1013 idx = (start + pending) & (fsl_queue->pending_max - 1); 1014 1015 fsl_queue->pending_desc[idx].src = src; 1016 fsl_queue->pending_desc[idx].dst = dst; 1017 fsl_queue->pending_desc[idx].flag = 1018 DPAA_QDMA_IDX_FROM_FLAG(flags); 1019 fsl_queue->pending_desc[idx].len = length; 1020 fsl_queue->pending_num++; 1021 1022 if (!(flags & RTE_DMA_OP_FLAG_SUBMIT)) 1023 return idx; 1024 1025 ret = fsl_qdma_enqueue_desc(fsl_queue); 1026 if (!ret) 1027 return fsl_queue->pending_start; 1028 1029 return ret; 1030 } 1031 1032 static int 1033 dpaa_qdma_copy_sg(void *dev_private, 1034 uint16_t vchan, 1035 const struct rte_dma_sge *src, 1036 const struct rte_dma_sge *dst, 1037 uint16_t nb_src, uint16_t nb_dst, 1038 uint64_t flags) 1039 { 1040 int ret; 1041 uint16_t i, start, idx; 1042 struct fsl_qdma_engine *fsl_qdma = dev_private; 1043 struct fsl_qdma_queue *fsl_queue = fsl_qdma->chan[vchan]; 1044 const uint16_t *idx_addr = NULL; 1045 1046 if (unlikely(nb_src != nb_dst)) { 1047 DPAA_QDMA_ERR("%s: nb_src(%d) != nb_dst(%d) on queue%d", 1048 __func__, nb_src, nb_dst, vchan); 1049 return -EINVAL; 1050 } 1051 1052 if ((fsl_queue->pending_num + nb_src) > FSL_QDMA_SG_MAX_ENTRY) { 1053 DPAA_QDMA_ERR("Too many pending jobs on queue%d", 1054 vchan); 1055 return -ENOSPC; 1056 } 1057 start = fsl_queue->pending_start + fsl_queue->pending_num; 1058 start = start & (fsl_queue->pending_max - 1); 1059 idx = start; 1060 1061 idx_addr = DPAA_QDMA_IDXADDR_FROM_SG_FLAG(flags); 1062 1063 for (i = 0; i < nb_src; i++) { 1064 if (unlikely(src[i].length != dst[i].length)) { 1065 DPAA_QDMA_ERR("src.len(%d) != dst.len(%d)", 1066 src[i].length, dst[i].length); 1067 return -EINVAL; 1068 } 1069 idx = (start + i) & (fsl_queue->pending_max - 1); 1070 fsl_queue->pending_desc[idx].src = src[i].addr; 1071 fsl_queue->pending_desc[idx].dst = dst[i].addr; 1072 fsl_queue->pending_desc[idx].len = dst[i].length; 1073 fsl_queue->pending_desc[idx].flag = idx_addr[i]; 1074 } 1075 fsl_queue->pending_num += nb_src; 1076 1077 if (!(flags & RTE_DMA_OP_FLAG_SUBMIT)) 1078 return idx; 1079 1080 ret = fsl_qdma_enqueue_desc(fsl_queue); 1081 if (!ret) 1082 return fsl_queue->pending_start; 1083 1084 return ret; 1085 } 1086 1087 static int 1088 dpaa_qdma_err_handle(struct fsl_qdma_err_reg *reg) 1089 { 1090 struct fsl_qdma_err_reg local; 1091 size_t i, offset = 0; 1092 char err_msg[512]; 1093 1094 local.dedr_be = rte_read32(®->dedr_be); 1095 if (!local.dedr_be) 1096 return 0; 1097 offset = sprintf(err_msg, "ERR detected:"); 1098 if (local.dedr.ere) { 1099 offset += sprintf(&err_msg[offset], 1100 " ere(Enqueue rejection error)"); 1101 } 1102 if (local.dedr.dde) { 1103 offset += sprintf(&err_msg[offset], 1104 " dde(Destination descriptor error)"); 1105 } 1106 if (local.dedr.sde) { 1107 offset += sprintf(&err_msg[offset], 1108 " sde(Source descriptor error)"); 1109 } 1110 if (local.dedr.cde) { 1111 offset += sprintf(&err_msg[offset], 1112 " cde(Command descriptor error)"); 1113 } 1114 if (local.dedr.wte) { 1115 offset += sprintf(&err_msg[offset], 1116 " wte(Write transaction error)"); 1117 } 1118 if (local.dedr.rte) { 1119 offset += sprintf(&err_msg[offset], 1120 " rte(Read transaction error)"); 1121 } 1122 if (local.dedr.me) { 1123 offset += sprintf(&err_msg[offset], 1124 " me(Multiple errors of the same type)"); 1125 } 1126 DPAA_QDMA_ERR("%s", err_msg); 1127 for (i = 0; i < FSL_QDMA_DECCD_ERR_NUM; i++) { 1128 local.deccd_le[FSL_QDMA_DECCD_ERR_NUM - 1 - i] = 1129 QDMA_IN(®->deccd_le[i]); 1130 } 1131 local.deccqidr_be = rte_read32(®->deccqidr_be); 1132 local.decbr = rte_read32(®->decbr); 1133 1134 offset = sprintf(err_msg, "ERR command:"); 1135 offset += sprintf(&err_msg[offset], 1136 " status: %02x, ser: %d, offset:%d, fmt: %02x", 1137 local.err_cmd.status, local.err_cmd.ser, 1138 local.err_cmd.offset, local.err_cmd.format); 1139 offset += sprintf(&err_msg[offset], 1140 " address: 0x%"PRIx64", queue: %d, dd: %02x", 1141 (uint64_t)local.err_cmd.addr_hi << 32 | 1142 local.err_cmd.addr_lo, 1143 local.err_cmd.queue, local.err_cmd.dd); 1144 DPAA_QDMA_ERR("%s", err_msg); 1145 DPAA_QDMA_ERR("ERR command block: %d, queue: %d", 1146 local.deccqidr.block, local.deccqidr.queue); 1147 1148 rte_write32(local.dedr_be, ®->dedr_be); 1149 1150 return -EIO; 1151 } 1152 1153 static uint16_t 1154 dpaa_qdma_dequeue_status(void *dev_private, uint16_t vchan, 1155 const uint16_t nb_cpls, uint16_t *last_idx, 1156 enum rte_dma_status_code *st) 1157 { 1158 struct fsl_qdma_engine *fsl_qdma = dev_private; 1159 int err; 1160 struct fsl_qdma_queue *fsl_queue = fsl_qdma->chan[vchan]; 1161 void *status = fsl_qdma->status_base; 1162 struct fsl_qdma_desc *desc_complete[nb_cpls]; 1163 uint16_t i, dq_num; 1164 1165 if (unlikely(fsl_qdma->is_silent)) { 1166 DPAA_QDMA_WARN("Can't dq in silent mode"); 1167 1168 return 0; 1169 } 1170 1171 dq_num = dpaa_qdma_block_dequeue(fsl_qdma, 1172 fsl_queue->block_id); 1173 DPAA_QDMA_DP_DEBUG("%s: block dq(%d)", 1174 __func__, dq_num); 1175 1176 dq_num = rte_ring_dequeue_burst(fsl_queue->complete_pool, 1177 (void **)desc_complete, nb_cpls, NULL); 1178 for (i = 0; i < dq_num; i++) 1179 last_idx[i] = desc_complete[i]->flag; 1180 1181 if (st) { 1182 for (i = 0; i < dq_num; i++) 1183 st[i] = RTE_DMA_STATUS_SUCCESSFUL; 1184 } 1185 1186 if (s_hw_err_check) { 1187 err = dpaa_qdma_err_handle(status + 1188 FSL_QDMA_ERR_REG_STATUS_OFFSET); 1189 if (err) 1190 fsl_queue->stats.errors++; 1191 } 1192 1193 return dq_num; 1194 } 1195 1196 static uint16_t 1197 dpaa_qdma_dequeue(void *dev_private, 1198 uint16_t vchan, const uint16_t nb_cpls, 1199 uint16_t *last_idx, bool *has_error) 1200 { 1201 struct fsl_qdma_engine *fsl_qdma = dev_private; 1202 int err; 1203 struct fsl_qdma_queue *fsl_queue = fsl_qdma->chan[vchan]; 1204 void *status = fsl_qdma->status_base; 1205 struct fsl_qdma_desc *desc_complete[nb_cpls]; 1206 uint16_t i, dq_num; 1207 1208 if (unlikely(fsl_qdma->is_silent)) { 1209 DPAA_QDMA_WARN("Can't dq in silent mode"); 1210 1211 return 0; 1212 } 1213 1214 *has_error = false; 1215 dq_num = dpaa_qdma_block_dequeue(fsl_qdma, 1216 fsl_queue->block_id); 1217 DPAA_QDMA_DP_DEBUG("%s: block dq(%d)", 1218 __func__, dq_num); 1219 1220 dq_num = rte_ring_dequeue_burst(fsl_queue->complete_pool, 1221 (void **)desc_complete, nb_cpls, NULL); 1222 for (i = 0; i < dq_num; i++) 1223 last_idx[i] = desc_complete[i]->flag; 1224 1225 if (s_hw_err_check) { 1226 err = dpaa_qdma_err_handle(status + 1227 FSL_QDMA_ERR_REG_STATUS_OFFSET); 1228 if (err) { 1229 if (has_error) 1230 *has_error = true; 1231 fsl_queue->stats.errors++; 1232 } 1233 } 1234 1235 return dq_num; 1236 } 1237 1238 static int 1239 dpaa_qdma_stats_get(const struct rte_dma_dev *dmadev, 1240 uint16_t vchan, struct rte_dma_stats *rte_stats, uint32_t size) 1241 { 1242 struct fsl_qdma_engine *fsl_qdma = dmadev->data->dev_private; 1243 struct fsl_qdma_queue *fsl_queue = fsl_qdma->chan[vchan]; 1244 struct rte_dma_stats *stats = &fsl_queue->stats; 1245 1246 if (size < sizeof(rte_stats)) 1247 return -EINVAL; 1248 if (rte_stats == NULL) 1249 return -EINVAL; 1250 1251 *rte_stats = *stats; 1252 1253 return 0; 1254 } 1255 1256 static int 1257 dpaa_qdma_stats_reset(struct rte_dma_dev *dmadev, uint16_t vchan) 1258 { 1259 struct fsl_qdma_engine *fsl_qdma = dmadev->data->dev_private; 1260 struct fsl_qdma_queue *fsl_queue = fsl_qdma->chan[vchan]; 1261 1262 memset(&fsl_queue->stats, 0, sizeof(struct rte_dma_stats)); 1263 1264 return 0; 1265 } 1266 1267 static uint16_t 1268 dpaa_qdma_burst_capacity(const void *dev_private, uint16_t vchan) 1269 { 1270 const struct fsl_qdma_engine *fsl_qdma = dev_private; 1271 struct fsl_qdma_queue *fsl_queue = fsl_qdma->chan[vchan]; 1272 1273 return fsl_queue->pending_max - fsl_queue->pending_num; 1274 } 1275 1276 static struct rte_dma_dev_ops dpaa_qdma_ops = { 1277 .dev_info_get = dpaa_qdma_info_get, 1278 .dev_configure = dpaa_qdma_configure, 1279 .dev_start = dpaa_qdma_start, 1280 .dev_close = dpaa_qdma_close, 1281 .vchan_setup = dpaa_qdma_queue_setup, 1282 .stats_get = dpaa_qdma_stats_get, 1283 .stats_reset = dpaa_qdma_stats_reset, 1284 }; 1285 1286 static int 1287 check_devargs_handler(__rte_unused const char *key, const char *value, 1288 __rte_unused void *opaque) 1289 { 1290 if (strcmp(value, "1")) 1291 return -1; 1292 1293 return 0; 1294 } 1295 1296 static int 1297 dpaa_get_devargs(struct rte_devargs *devargs, const char *key) 1298 { 1299 struct rte_kvargs *kvlist; 1300 1301 if (!devargs) 1302 return 0; 1303 1304 kvlist = rte_kvargs_parse(devargs->args, NULL); 1305 if (!kvlist) 1306 return 0; 1307 1308 if (!rte_kvargs_count(kvlist, key)) { 1309 rte_kvargs_free(kvlist); 1310 return 0; 1311 } 1312 1313 if (rte_kvargs_process(kvlist, key, 1314 check_devargs_handler, NULL) < 0) { 1315 rte_kvargs_free(kvlist); 1316 return 0; 1317 } 1318 rte_kvargs_free(kvlist); 1319 1320 return 1; 1321 } 1322 1323 static int 1324 dpaa_qdma_init(struct rte_dma_dev *dmadev) 1325 { 1326 struct fsl_qdma_engine *fsl_qdma = dmadev->data->dev_private; 1327 uint64_t phys_addr; 1328 int ccsr_qdma_fd; 1329 int regs_size; 1330 int ret; 1331 uint32_t i, j, k; 1332 1333 if (dpaa_get_devargs(dmadev->device->devargs, DPAA_DMA_ERROR_CHECK)) { 1334 s_hw_err_check = true; 1335 DPAA_QDMA_INFO("Enable DMA error checks"); 1336 } 1337 1338 fsl_qdma->n_queues = QDMA_QUEUES * QDMA_BLOCKS; 1339 fsl_qdma->num_blocks = QDMA_BLOCKS; 1340 fsl_qdma->block_offset = QDMA_BLOCK_OFFSET; 1341 1342 ccsr_qdma_fd = open("/dev/mem", O_RDWR); 1343 if (unlikely(ccsr_qdma_fd < 0)) { 1344 DPAA_QDMA_ERR("Can not open /dev/mem for qdma CCSR map"); 1345 return ccsr_qdma_fd; 1346 } 1347 1348 regs_size = fsl_qdma->block_offset * fsl_qdma->num_blocks; 1349 regs_size += (QDMA_CTRL_REGION_SIZE + QDMA_STATUS_REGION_SIZE); 1350 phys_addr = QDMA_CCSR_BASE; 1351 fsl_qdma->reg_base = mmap(NULL, regs_size, 1352 PROT_READ | PROT_WRITE, MAP_SHARED, 1353 ccsr_qdma_fd, phys_addr); 1354 1355 close(ccsr_qdma_fd); 1356 if (fsl_qdma->reg_base == MAP_FAILED) { 1357 DPAA_QDMA_ERR("Map qdma reg: Phys(0x%"PRIx64"), size(%d)", 1358 phys_addr, regs_size); 1359 return -ENOMEM; 1360 } 1361 1362 fsl_qdma->ctrl_base = 1363 fsl_qdma->reg_base + QDMA_CTRL_REGION_OFFSET; 1364 fsl_qdma->status_base = 1365 fsl_qdma->reg_base + QDMA_STATUS_REGION_OFFSET; 1366 fsl_qdma->block_base = 1367 fsl_qdma->status_base + QDMA_STATUS_REGION_SIZE; 1368 1369 for (i = 0; i < QDMA_BLOCKS; i++) { 1370 ret = fsl_qdma_prep_status_queue(fsl_qdma, i); 1371 if (ret) 1372 goto mem_free; 1373 } 1374 1375 k = 0; 1376 for (i = 0; i < QDMA_QUEUES; i++) { 1377 for (j = 0; j < QDMA_BLOCKS; j++) { 1378 ret = fsl_qdma_alloc_queue_resources(fsl_qdma, i, j); 1379 if (ret) 1380 goto mem_free; 1381 fsl_qdma->cmd_queues[j][i].channel_id = k; 1382 k++; 1383 } 1384 } 1385 1386 ret = fsl_qdma_reg_init(fsl_qdma); 1387 if (ret) { 1388 DPAA_QDMA_ERR("Can't Initialize the qDMA engine."); 1389 goto mem_free; 1390 } 1391 1392 return 0; 1393 1394 mem_free: 1395 for (i = 0; i < fsl_qdma->num_blocks; i++) 1396 fsl_qdma_free_stq_res(&fsl_qdma->stat_queues[i]); 1397 1398 for (i = 0; i < fsl_qdma->num_blocks; i++) { 1399 for (j = 0; j < QDMA_QUEUES; j++) 1400 fsl_qdma_free_cmdq_res(&fsl_qdma->cmd_queues[i][j]); 1401 } 1402 1403 munmap(fsl_qdma->ctrl_base, regs_size); 1404 1405 return ret; 1406 } 1407 1408 static int 1409 dpaa_qdma_probe(__rte_unused struct rte_dpaa_driver *dpaa_drv, 1410 struct rte_dpaa_device *dpaa_dev) 1411 { 1412 struct rte_dma_dev *dmadev; 1413 int ret; 1414 1415 dmadev = rte_dma_pmd_allocate(dpaa_dev->device.name, 1416 rte_socket_id(), 1417 sizeof(struct fsl_qdma_engine)); 1418 if (!dmadev) { 1419 DPAA_QDMA_ERR("Unable to allocate dmadevice"); 1420 return -EINVAL; 1421 } 1422 1423 dpaa_dev->dmadev = dmadev; 1424 dmadev->dev_ops = &dpaa_qdma_ops; 1425 dmadev->device = &dpaa_dev->device; 1426 dmadev->fp_obj->dev_private = dmadev->data->dev_private; 1427 dmadev->fp_obj->copy = dpaa_qdma_enqueue; 1428 dmadev->fp_obj->copy_sg = dpaa_qdma_copy_sg; 1429 dmadev->fp_obj->submit = dpaa_qdma_submit; 1430 dmadev->fp_obj->completed = dpaa_qdma_dequeue; 1431 dmadev->fp_obj->completed_status = dpaa_qdma_dequeue_status; 1432 dmadev->fp_obj->burst_capacity = dpaa_qdma_burst_capacity; 1433 1434 /* Invoke PMD device initialization function */ 1435 ret = dpaa_qdma_init(dmadev); 1436 if (ret) { 1437 (void)rte_dma_pmd_release(dpaa_dev->device.name); 1438 return ret; 1439 } 1440 1441 dmadev->state = RTE_DMA_DEV_READY; 1442 return 0; 1443 } 1444 1445 static int 1446 dpaa_qdma_remove(struct rte_dpaa_device *dpaa_dev) 1447 { 1448 struct rte_dma_dev *dmadev = dpaa_dev->dmadev; 1449 struct fsl_qdma_engine *fsl_qdma = dmadev->data->dev_private; 1450 uint32_t i, j, regs_size; 1451 1452 regs_size = fsl_qdma->block_offset * fsl_qdma->num_blocks; 1453 regs_size += (QDMA_CTRL_REGION_SIZE + QDMA_STATUS_REGION_SIZE); 1454 1455 for (i = 0; i < QDMA_BLOCKS; i++) 1456 fsl_qdma_free_stq_res(&fsl_qdma->stat_queues[i]); 1457 1458 for (i = 0; i < QDMA_BLOCKS; i++) { 1459 for (j = 0; j < QDMA_QUEUES; j++) 1460 fsl_qdma_free_cmdq_res(&fsl_qdma->cmd_queues[i][j]); 1461 } 1462 1463 munmap(fsl_qdma->ctrl_base, regs_size); 1464 1465 (void)rte_dma_pmd_release(dpaa_dev->device.name); 1466 1467 return 0; 1468 } 1469 1470 static struct rte_dpaa_driver rte_dpaa_qdma_pmd; 1471 1472 static struct rte_dpaa_driver rte_dpaa_qdma_pmd = { 1473 .drv_type = FSL_DPAA_QDMA, 1474 .probe = dpaa_qdma_probe, 1475 .remove = dpaa_qdma_remove, 1476 }; 1477 1478 RTE_PMD_REGISTER_DPAA(dpaa_qdma, rte_dpaa_qdma_pmd); 1479 RTE_PMD_REGISTER_PARAM_STRING(dpaa_qdma, DPAA_DMA_ERROR_CHECK "=<int>"); 1480 RTE_LOG_REGISTER_DEFAULT(dpaa_qdma_logtype, INFO); 1481