1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2018-2024 NXP 3 */ 4 5 #include <rte_eal.h> 6 #include <bus_fslmc_driver.h> 7 #include <rte_dmadev.h> 8 #include <rte_dmadev_pmd.h> 9 #include <rte_kvargs.h> 10 11 #include <mc/fsl_dpdmai.h> 12 13 #include <rte_pmd_dpaax_qdma.h> 14 #include "dpaa2_hw_dpio.h" 15 #include "dpaa2_qdma.h" 16 #include "dpaa2_qdma_logs.h" 17 18 #define DPAA2_QDMA_FLE_PRE_POPULATE "fle_pre_populate" 19 #define DPAA2_QDMA_DESC_DEBUG "desc_debug" 20 #define DPAA2_QDMA_USING_SHORT_FD "short_fd" 21 22 static uint32_t dpaa2_coherent_no_alloc_cache; 23 static uint32_t dpaa2_coherent_alloc_cache; 24 25 static struct fsl_mc_io s_proc_mc_reg; 26 27 static int 28 check_devargs_handler(__rte_unused const char *key, const char *value, 29 __rte_unused void *opaque) 30 { 31 if (strcmp(value, "1")) 32 return -1; 33 34 return 0; 35 } 36 37 static int 38 dpaa2_qdma_get_devargs(struct rte_devargs *devargs, const char *key) 39 { 40 struct rte_kvargs *kvlist; 41 42 if (!devargs) 43 return 0; 44 45 kvlist = rte_kvargs_parse(devargs->args, NULL); 46 if (!kvlist) 47 return 0; 48 49 if (!rte_kvargs_count(kvlist, key)) { 50 rte_kvargs_free(kvlist); 51 return 0; 52 } 53 54 if (rte_kvargs_process(kvlist, key, 55 check_devargs_handler, NULL) < 0) { 56 rte_kvargs_free(kvlist); 57 return 0; 58 } 59 rte_kvargs_free(kvlist); 60 61 return 1; 62 } 63 64 static inline int 65 qdma_cntx_idx_ring_eq(struct qdma_cntx_idx_ring *ring, 66 const uint16_t *elem, uint16_t nb, 67 uint16_t *free_space) 68 { 69 uint16_t i; 70 71 if (unlikely(nb > ring->free_space)) 72 return 0; 73 74 for (i = 0; i < nb; i++) { 75 ring->cntx_idx_ring[ring->tail] = elem[i]; 76 ring->tail = (ring->tail + 1) & 77 (DPAA2_QDMA_MAX_DESC - 1); 78 } 79 ring->free_space -= nb; 80 ring->nb_in_ring += nb; 81 82 if (free_space) 83 *free_space = ring->free_space; 84 85 return nb; 86 } 87 88 static inline int 89 qdma_cntx_idx_ring_dq(struct qdma_cntx_idx_ring *ring, 90 uint16_t *elem, uint16_t max) 91 { 92 int ret = ring->nb_in_ring > max ? max : ring->nb_in_ring; 93 94 if (!ret) 95 return 0; 96 97 if ((ring->start + ret) < DPAA2_QDMA_MAX_DESC) { 98 rte_memcpy(elem, 99 &ring->cntx_idx_ring[ring->start], 100 ret * sizeof(uint16_t)); 101 ring->start += ret; 102 } else { 103 rte_memcpy(elem, 104 &ring->cntx_idx_ring[ring->start], 105 (DPAA2_QDMA_MAX_DESC - ring->start) * 106 sizeof(uint16_t)); 107 rte_memcpy(&elem[DPAA2_QDMA_MAX_DESC - ring->start], 108 &ring->cntx_idx_ring[0], 109 (ret - DPAA2_QDMA_MAX_DESC + ring->start) * 110 sizeof(uint16_t)); 111 ring->start = (ring->start + ret) & (DPAA2_QDMA_MAX_DESC - 1); 112 } 113 ring->free_space += ret; 114 ring->nb_in_ring -= ret; 115 116 return ret; 117 } 118 119 static int 120 dpaa2_qdma_multi_eq(struct qdma_virt_queue *qdma_vq) 121 { 122 struct dpaa2_dpdmai_dev *dpdmai_dev = qdma_vq->dpdmai_dev; 123 uint16_t txq_id = dpdmai_dev->tx_queue[qdma_vq->vq_id].fqid; 124 struct qbman_eq_desc eqdesc; 125 struct qbman_swp *swp; 126 uint32_t num_to_send = 0; 127 uint16_t num_tx = 0; 128 uint32_t enqueue_loop, loop; 129 int ret; 130 struct qbman_fd *fd = qdma_vq->fd; 131 uint16_t nb_fds = qdma_vq->fd_idx, idx, dst_idx; 132 133 if (unlikely(!DPAA2_PER_LCORE_DPIO)) { 134 ret = dpaa2_affine_qbman_swp(); 135 if (ret) { 136 DPAA2_QDMA_ERR("Failed to allocate IO portal, tid: %d", 137 rte_gettid()); 138 return -EIO; 139 } 140 } 141 swp = DPAA2_PER_LCORE_PORTAL; 142 143 /* Prepare enqueue descriptor */ 144 qbman_eq_desc_clear(&eqdesc); 145 qbman_eq_desc_set_fq(&eqdesc, txq_id); 146 qbman_eq_desc_set_no_orp(&eqdesc, 0); 147 qbman_eq_desc_set_response(&eqdesc, 0, 0); 148 149 while (nb_fds > 0) { 150 num_to_send = (nb_fds > dpaa2_eqcr_size) ? 151 dpaa2_eqcr_size : nb_fds; 152 153 /* Enqueue the packet to the QBMAN */ 154 enqueue_loop = 0; 155 loop = num_to_send; 156 157 while (enqueue_loop < loop) { 158 ret = qbman_swp_enqueue_multiple(swp, 159 &eqdesc, 160 &fd[num_tx + enqueue_loop], 161 NULL, 162 loop - enqueue_loop); 163 if (likely(ret >= 0)) 164 enqueue_loop += ret; 165 } 166 num_tx += num_to_send; 167 nb_fds -= loop; 168 } 169 170 qdma_vq->num_enqueues += num_tx; 171 if (unlikely(num_tx != qdma_vq->fd_idx)) { 172 dst_idx = 0; 173 for (idx = num_tx; idx < qdma_vq->fd_idx; idx++) { 174 rte_memcpy(&qdma_vq->fd[dst_idx], 175 &qdma_vq->fd[idx], 176 sizeof(struct qbman_fd)); 177 dst_idx++; 178 } 179 } 180 qdma_vq->fd_idx -= num_tx; 181 182 return num_tx; 183 } 184 185 static void 186 fle_sdd_pre_populate(struct qdma_cntx_fle_sdd *fle_sdd, 187 struct dpaa2_qdma_rbp *rbp, uint64_t src, uint64_t dest, 188 uint32_t fmt) 189 { 190 struct qbman_fle *fle = fle_sdd->fle; 191 struct qdma_sdd *sdd = fle_sdd->sdd; 192 uint64_t sdd_iova = DPAA2_VADDR_TO_IOVA(sdd); 193 194 /* first frame list to source descriptor */ 195 DPAA2_SET_FLE_ADDR(&fle[DPAA2_QDMA_SDD_FLE], sdd_iova); 196 DPAA2_SET_FLE_LEN(&fle[DPAA2_QDMA_SDD_FLE], 197 DPAA2_QDMA_MAX_SDD * (sizeof(struct qdma_sdd))); 198 199 /* source and destination descriptor */ 200 if (rbp && rbp->enable) { 201 /* source */ 202 sdd[DPAA2_QDMA_SRC_SDD].read_cmd.portid = 203 rbp->sportid; 204 sdd[DPAA2_QDMA_SRC_SDD].rbpcmd_simple.pfid = 205 rbp->spfid; 206 sdd[DPAA2_QDMA_SRC_SDD].rbpcmd_simple.vfid = 207 rbp->svfid; 208 sdd[DPAA2_QDMA_SRC_SDD].rbpcmd_simple.vfa = 209 rbp->svfa; 210 211 if (rbp->srbp) { 212 sdd[DPAA2_QDMA_SRC_SDD].read_cmd.rbp = 213 rbp->srbp; 214 sdd[DPAA2_QDMA_SRC_SDD].read_cmd.rdtype = 215 DPAA2_RBP_MEM_RW; 216 } else { 217 sdd[DPAA2_QDMA_SRC_SDD].read_cmd.rdtype = 218 dpaa2_coherent_no_alloc_cache; 219 } 220 /* destination */ 221 sdd[DPAA2_QDMA_DST_SDD].write_cmd.portid = 222 rbp->dportid; 223 sdd[DPAA2_QDMA_DST_SDD].rbpcmd_simple.pfid = 224 rbp->dpfid; 225 sdd[DPAA2_QDMA_DST_SDD].rbpcmd_simple.vfid = 226 rbp->dvfid; 227 sdd[DPAA2_QDMA_DST_SDD].rbpcmd_simple.vfa = 228 rbp->dvfa; 229 230 if (rbp->drbp) { 231 sdd[DPAA2_QDMA_DST_SDD].write_cmd.rbp = 232 rbp->drbp; 233 sdd[DPAA2_QDMA_DST_SDD].write_cmd.wrttype = 234 DPAA2_RBP_MEM_RW; 235 } else { 236 sdd[DPAA2_QDMA_DST_SDD].write_cmd.wrttype = 237 dpaa2_coherent_alloc_cache; 238 } 239 } else { 240 sdd[DPAA2_QDMA_SRC_SDD].read_cmd.rdtype = 241 dpaa2_coherent_no_alloc_cache; 242 sdd[DPAA2_QDMA_DST_SDD].write_cmd.wrttype = 243 dpaa2_coherent_alloc_cache; 244 } 245 /* source frame list to source buffer */ 246 DPAA2_SET_FLE_ADDR(&fle[DPAA2_QDMA_SRC_FLE], src); 247 /** IOMMU is always on for either VA or PA mode, 248 * so Bypass Memory Translation should be disabled. 249 * 250 * DPAA2_SET_FLE_BMT(&fle[DPAA2_QDMA_SRC_FLE]); 251 * DPAA2_SET_FLE_BMT(&fle[DPAA2_QDMA_DST_FLE]); 252 */ 253 fle[DPAA2_QDMA_SRC_FLE].word4.fmt = fmt; 254 255 /* destination frame list to destination buffer */ 256 DPAA2_SET_FLE_ADDR(&fle[DPAA2_QDMA_DST_FLE], dest); 257 fle[DPAA2_QDMA_DST_FLE].word4.fmt = fmt; 258 259 /* Final bit: 1, for last frame list */ 260 DPAA2_SET_FLE_FIN(&fle[DPAA2_QDMA_DST_FLE]); 261 } 262 263 static void 264 sg_entry_pre_populate(struct qdma_cntx_sg *sg_cntx) 265 { 266 uint16_t i; 267 struct qdma_sg_entry *src_sge = sg_cntx->sg_src_entry; 268 struct qdma_sg_entry *dst_sge = sg_cntx->sg_dst_entry; 269 270 for (i = 0; i < RTE_DPAAX_QDMA_JOB_SUBMIT_MAX; i++) { 271 /* source SG */ 272 src_sge[i].ctrl.sl = QDMA_SG_SL_LONG; 273 src_sge[i].ctrl.fmt = QDMA_SG_FMT_SDB; 274 /** IOMMU is always on for either VA or PA mode, 275 * so Bypass Memory Translation should be disabled. 276 */ 277 src_sge[i].ctrl.bmt = QDMA_SG_BMT_DISABLE; 278 /* destination SG */ 279 dst_sge[i].ctrl.sl = QDMA_SG_SL_LONG; 280 dst_sge[i].ctrl.fmt = QDMA_SG_FMT_SDB; 281 /** IOMMU is always on for either VA or PA mode, 282 * so Bypass Memory Translation should be disabled. 283 */ 284 dst_sge[i].ctrl.bmt = QDMA_SG_BMT_DISABLE; 285 } 286 } 287 288 static void 289 fle_sdd_sg_pre_populate(struct qdma_cntx_sg *sg_cntx, 290 struct qdma_virt_queue *qdma_vq) 291 { 292 struct qdma_sg_entry *src_sge = sg_cntx->sg_src_entry; 293 struct qdma_sg_entry *dst_sge = sg_cntx->sg_dst_entry; 294 rte_iova_t src_sge_iova, dst_sge_iova; 295 struct dpaa2_qdma_rbp *rbp = &qdma_vq->rbp; 296 297 memset(sg_cntx, 0, sizeof(struct qdma_cntx_sg)); 298 299 src_sge_iova = DPAA2_VADDR_TO_IOVA(src_sge); 300 dst_sge_iova = DPAA2_VADDR_TO_IOVA(dst_sge); 301 302 sg_entry_pre_populate(sg_cntx); 303 fle_sdd_pre_populate(&sg_cntx->fle_sdd, 304 rbp, src_sge_iova, dst_sge_iova, 305 QBMAN_FLE_WORD4_FMT_SGE); 306 } 307 308 static inline uint32_t 309 sg_entry_post_populate(const struct rte_dma_sge *src, 310 const struct rte_dma_sge *dst, struct qdma_cntx_sg *sg_cntx, 311 uint16_t nb_sge) 312 { 313 uint16_t i; 314 uint32_t total_len = 0; 315 struct qdma_sg_entry *src_sge = sg_cntx->sg_src_entry; 316 struct qdma_sg_entry *dst_sge = sg_cntx->sg_dst_entry; 317 318 for (i = 0; i < (nb_sge - 1); i++) { 319 if (unlikely(src[i].length != dst[i].length)) 320 return -ENOTSUP; 321 src_sge->addr_lo = (uint32_t)src[i].addr; 322 src_sge->addr_hi = (src[i].addr >> 32); 323 src_sge->data_len.data_len_sl0 = src[i].length; 324 325 dst_sge->addr_lo = (uint32_t)dst[i].addr; 326 dst_sge->addr_hi = (dst[i].addr >> 32); 327 dst_sge->data_len.data_len_sl0 = dst[i].length; 328 total_len += dst[i].length; 329 330 src_sge->ctrl.f = 0; 331 dst_sge->ctrl.f = 0; 332 src_sge++; 333 dst_sge++; 334 } 335 336 if (unlikely(src[i].length != dst[i].length)) 337 return -ENOTSUP; 338 339 src_sge->addr_lo = (uint32_t)src[i].addr; 340 src_sge->addr_hi = (src[i].addr >> 32); 341 src_sge->data_len.data_len_sl0 = src[i].length; 342 343 dst_sge->addr_lo = (uint32_t)dst[i].addr; 344 dst_sge->addr_hi = (dst[i].addr >> 32); 345 dst_sge->data_len.data_len_sl0 = dst[i].length; 346 347 total_len += dst[i].length; 348 sg_cntx->job_nb = nb_sge; 349 350 src_sge->ctrl.f = QDMA_SG_F; 351 dst_sge->ctrl.f = QDMA_SG_F; 352 353 return total_len; 354 } 355 356 static inline void 357 sg_fle_post_populate(struct qbman_fle fle[], 358 size_t len) 359 { 360 DPAA2_SET_FLE_LEN(&fle[DPAA2_QDMA_SRC_FLE], len); 361 DPAA2_SET_FLE_LEN(&fle[DPAA2_QDMA_DST_FLE], len); 362 } 363 364 static inline uint32_t 365 sg_entry_populate(const struct rte_dma_sge *src, 366 const struct rte_dma_sge *dst, struct qdma_cntx_sg *sg_cntx, 367 uint16_t nb_sge) 368 { 369 uint16_t i; 370 uint32_t total_len = 0; 371 struct qdma_sg_entry *src_sge = sg_cntx->sg_src_entry; 372 struct qdma_sg_entry *dst_sge = sg_cntx->sg_dst_entry; 373 374 for (i = 0; i < nb_sge; i++) { 375 if (unlikely(src[i].length != dst[i].length)) 376 return -ENOTSUP; 377 378 src_sge->addr_lo = (uint32_t)src[i].addr; 379 src_sge->addr_hi = (src[i].addr >> 32); 380 src_sge->data_len.data_len_sl0 = src[i].length; 381 src_sge->ctrl.sl = QDMA_SG_SL_LONG; 382 src_sge->ctrl.fmt = QDMA_SG_FMT_SDB; 383 /** IOMMU is always on for either VA or PA mode, 384 * so Bypass Memory Translation should be disabled. 385 */ 386 src_sge->ctrl.bmt = QDMA_SG_BMT_DISABLE; 387 dst_sge->addr_lo = (uint32_t)dst[i].addr; 388 dst_sge->addr_hi = (dst[i].addr >> 32); 389 dst_sge->data_len.data_len_sl0 = dst[i].length; 390 dst_sge->ctrl.sl = QDMA_SG_SL_LONG; 391 dst_sge->ctrl.fmt = QDMA_SG_FMT_SDB; 392 /** IOMMU is always on for either VA or PA mode, 393 * so Bypass Memory Translation should be disabled. 394 */ 395 dst_sge->ctrl.bmt = QDMA_SG_BMT_DISABLE; 396 total_len += src[i].length; 397 398 if (i == (nb_sge - 1)) { 399 src_sge->ctrl.f = QDMA_SG_F; 400 dst_sge->ctrl.f = QDMA_SG_F; 401 } else { 402 src_sge->ctrl.f = 0; 403 dst_sge->ctrl.f = 0; 404 } 405 src_sge++; 406 dst_sge++; 407 } 408 409 sg_cntx->job_nb = nb_sge; 410 411 return total_len; 412 } 413 414 static inline void 415 fle_populate(struct qbman_fle fle[], 416 struct qdma_sdd sdd[], uint64_t sdd_iova, 417 struct dpaa2_qdma_rbp *rbp, 418 uint64_t src_iova, uint64_t dst_iova, size_t len, 419 uint32_t fmt) 420 { 421 /* first frame list to source descriptor */ 422 DPAA2_SET_FLE_ADDR(&fle[DPAA2_QDMA_SDD_FLE], sdd_iova); 423 DPAA2_SET_FLE_LEN(&fle[DPAA2_QDMA_SDD_FLE], 424 (DPAA2_QDMA_MAX_SDD * (sizeof(struct qdma_sdd)))); 425 426 /* source and destination descriptor */ 427 if (rbp && rbp->enable) { 428 /* source */ 429 sdd[DPAA2_QDMA_SRC_SDD].read_cmd.portid = 430 rbp->sportid; 431 sdd[DPAA2_QDMA_SRC_SDD].rbpcmd_simple.pfid = 432 rbp->spfid; 433 sdd[DPAA2_QDMA_SRC_SDD].rbpcmd_simple.vfid = 434 rbp->svfid; 435 sdd[DPAA2_QDMA_SRC_SDD].rbpcmd_simple.vfa = 436 rbp->svfa; 437 438 if (rbp->srbp) { 439 sdd[DPAA2_QDMA_SRC_SDD].read_cmd.rbp = 440 rbp->srbp; 441 sdd[DPAA2_QDMA_SRC_SDD].read_cmd.rdtype = 442 DPAA2_RBP_MEM_RW; 443 } else { 444 sdd[DPAA2_QDMA_SRC_SDD].read_cmd.rdtype = 445 dpaa2_coherent_no_alloc_cache; 446 } 447 /* destination */ 448 sdd[DPAA2_QDMA_DST_SDD].write_cmd.portid = 449 rbp->dportid; 450 sdd[DPAA2_QDMA_DST_SDD].rbpcmd_simple.pfid = 451 rbp->dpfid; 452 sdd[DPAA2_QDMA_DST_SDD].rbpcmd_simple.vfid = 453 rbp->dvfid; 454 sdd[DPAA2_QDMA_DST_SDD].rbpcmd_simple.vfa = 455 rbp->dvfa; 456 457 if (rbp->drbp) { 458 sdd[DPAA2_QDMA_DST_SDD].write_cmd.rbp = 459 rbp->drbp; 460 sdd[DPAA2_QDMA_DST_SDD].write_cmd.wrttype = 461 DPAA2_RBP_MEM_RW; 462 } else { 463 sdd[DPAA2_QDMA_DST_SDD].write_cmd.wrttype = 464 dpaa2_coherent_alloc_cache; 465 } 466 467 } else { 468 sdd[DPAA2_QDMA_SRC_SDD].read_cmd.rdtype = 469 dpaa2_coherent_no_alloc_cache; 470 sdd[DPAA2_QDMA_DST_SDD].write_cmd.wrttype = 471 dpaa2_coherent_alloc_cache; 472 } 473 /* source frame list to source buffer */ 474 DPAA2_SET_FLE_ADDR(&fle[DPAA2_QDMA_SRC_FLE], src_iova); 475 /** IOMMU is always on for either VA or PA mode, 476 * so Bypass Memory Translation should be disabled. 477 * DPAA2_SET_FLE_BMT(&fle[DPAA2_QDMA_SRC_FLE]); 478 * DPAA2_SET_FLE_BMT(&fle[DPAA2_QDMA_DST_FLE]); 479 */ 480 fle[DPAA2_QDMA_SRC_FLE].word4.fmt = fmt; 481 DPAA2_SET_FLE_LEN(&fle[DPAA2_QDMA_SRC_FLE], len); 482 483 /* destination frame list to destination buffer */ 484 DPAA2_SET_FLE_ADDR(&fle[DPAA2_QDMA_DST_FLE], dst_iova); 485 fle[DPAA2_QDMA_DST_FLE].word4.fmt = fmt; 486 DPAA2_SET_FLE_LEN(&fle[DPAA2_QDMA_DST_FLE], len); 487 488 /* Final bit: 1, for last frame list */ 489 DPAA2_SET_FLE_FIN(&fle[DPAA2_QDMA_DST_FLE]); 490 } 491 492 static inline void 493 fle_post_populate(struct qbman_fle fle[], 494 uint64_t src, uint64_t dest, size_t len) 495 { 496 DPAA2_SET_FLE_ADDR(&fle[DPAA2_QDMA_SRC_FLE], src); 497 DPAA2_SET_FLE_LEN(&fle[DPAA2_QDMA_SRC_FLE], len); 498 499 DPAA2_SET_FLE_ADDR(&fle[DPAA2_QDMA_DST_FLE], dest); 500 DPAA2_SET_FLE_LEN(&fle[DPAA2_QDMA_DST_FLE], len); 501 } 502 503 static inline int 504 dpaa2_qdma_submit(void *dev_private, uint16_t vchan) 505 { 506 struct dpaa2_dpdmai_dev *dpdmai_dev = dev_private; 507 struct qdma_device *qdma_dev = dpdmai_dev->qdma_dev; 508 struct qdma_virt_queue *qdma_vq = &qdma_dev->vqs[vchan]; 509 uint16_t expected = qdma_vq->fd_idx; 510 int ret; 511 512 ret = dpaa2_qdma_multi_eq(qdma_vq); 513 if (likely(ret == expected)) 514 return 0; 515 516 return -EBUSY; 517 } 518 519 static inline void 520 dpaa2_qdma_fle_dump(const struct qbman_fle *fle) 521 { 522 DPAA2_QDMA_INFO("addr:0x%08x-0x%08x, len:%d, frc:0x%08x, bpid:%d", 523 fle->addr_hi, fle->addr_lo, fle->length, fle->frc, 524 fle->word4.bpid); 525 DPAA2_QDMA_INFO("ivp:%d, bmt:%d, off:%d, fmt:%d, sl:%d, f:%d", 526 fle->word4.ivp, fle->word4.bmt, fle->word4.offset, 527 fle->word4.fmt, fle->word4.sl, fle->word4.f); 528 } 529 530 static inline void 531 dpaa2_qdma_sdd_dump(const struct qdma_sdd *sdd) 532 { 533 DPAA2_QDMA_INFO("stride:%d, rbpcmd:0x%08x, cmd:0x%08x", 534 sdd->stride, sdd->rbpcmd, sdd->cmd); 535 } 536 537 static inline void 538 dpaa2_qdma_sge_dump(const struct qdma_sg_entry *sge) 539 { 540 DPAA2_QDMA_INFO("addr 0x%08x-0x%08x, len:0x%08x, ctl:0x%08x", 541 sge->addr_hi, sge->addr_lo, sge->data_len.data_len_sl0, 542 sge->ctrl_fields); 543 } 544 545 static void 546 dpaa2_qdma_long_fmt_dump(const struct qbman_fle *fle) 547 { 548 int i; 549 const struct qdma_cntx_fle_sdd *fle_sdd; 550 const struct qdma_sdd *sdd; 551 const struct qdma_cntx_sg *cntx_sg = NULL; 552 553 fle_sdd = container_of(fle, const struct qdma_cntx_fle_sdd, fle[0]); 554 sdd = fle_sdd->sdd; 555 556 for (i = 0; i < DPAA2_QDMA_MAX_FLE; i++) { 557 DPAA2_QDMA_INFO("fle[%d] info:", i); 558 dpaa2_qdma_fle_dump(&fle[i]); 559 } 560 561 if (fle[DPAA2_QDMA_SRC_FLE].word4.fmt != 562 fle[DPAA2_QDMA_DST_FLE].word4.fmt) { 563 DPAA2_QDMA_ERR("fle[%d].fmt(%d) != fle[%d].fmt(%d)", 564 DPAA2_QDMA_SRC_FLE, 565 fle[DPAA2_QDMA_SRC_FLE].word4.fmt, 566 DPAA2_QDMA_DST_FLE, 567 fle[DPAA2_QDMA_DST_FLE].word4.fmt); 568 569 return; 570 } else if (fle[DPAA2_QDMA_SRC_FLE].word4.fmt == 571 QBMAN_FLE_WORD4_FMT_SGE) { 572 cntx_sg = container_of(fle_sdd, const struct qdma_cntx_sg, 573 fle_sdd); 574 } else if (fle[DPAA2_QDMA_SRC_FLE].word4.fmt != 575 QBMAN_FLE_WORD4_FMT_SBF) { 576 DPAA2_QDMA_ERR("Unsupported fle format:%d", 577 fle[DPAA2_QDMA_SRC_FLE].word4.fmt); 578 return; 579 } 580 581 for (i = 0; i < DPAA2_QDMA_MAX_SDD; i++) { 582 DPAA2_QDMA_INFO("sdd[%d] info:", i); 583 dpaa2_qdma_sdd_dump(&sdd[i]); 584 } 585 586 if (cntx_sg) { 587 DPAA2_QDMA_INFO("long format/SG format, job number:%d", 588 cntx_sg->job_nb); 589 if (!cntx_sg->job_nb || 590 cntx_sg->job_nb > RTE_DPAAX_QDMA_JOB_SUBMIT_MAX) { 591 DPAA2_QDMA_ERR("Invalid SG job number:%d", 592 cntx_sg->job_nb); 593 return; 594 } 595 for (i = 0; i < cntx_sg->job_nb; i++) { 596 DPAA2_QDMA_INFO("sg[%d] src info:", i); 597 dpaa2_qdma_sge_dump(&cntx_sg->sg_src_entry[i]); 598 DPAA2_QDMA_INFO("sg[%d] dst info:", i); 599 dpaa2_qdma_sge_dump(&cntx_sg->sg_dst_entry[i]); 600 DPAA2_QDMA_INFO("cntx_idx[%d]:%d", i, 601 cntx_sg->cntx_idx[i]); 602 } 603 } else { 604 DPAA2_QDMA_INFO("long format/Single buffer cntx"); 605 } 606 } 607 608 static int 609 dpaa2_qdma_copy_sg(void *dev_private, 610 uint16_t vchan, 611 const struct rte_dma_sge *src, 612 const struct rte_dma_sge *dst, 613 uint16_t nb_src, uint16_t nb_dst, 614 uint64_t flags) 615 { 616 struct dpaa2_dpdmai_dev *dpdmai_dev = dev_private; 617 struct qdma_device *qdma_dev = dpdmai_dev->qdma_dev; 618 struct qdma_virt_queue *qdma_vq = &qdma_dev->vqs[vchan]; 619 int ret = 0, expected, i; 620 uint32_t len; 621 struct qbman_fd *fd = &qdma_vq->fd[qdma_vq->fd_idx]; 622 struct qdma_cntx_sg *cntx_sg = NULL; 623 rte_iova_t cntx_iova, fle_iova, sdd_iova; 624 rte_iova_t src_sge_iova, dst_sge_iova; 625 struct qbman_fle *fle; 626 struct qdma_sdd *sdd; 627 const uint16_t *idx_addr = NULL; 628 629 if (unlikely(nb_src != nb_dst)) { 630 DPAA2_QDMA_ERR("SG entry src num(%d) != dst num(%d)", 631 nb_src, nb_dst); 632 return -ENOTSUP; 633 } 634 635 if (unlikely(!nb_src)) { 636 DPAA2_QDMA_ERR("No SG entry specified"); 637 return -EINVAL; 638 } 639 640 if (unlikely(nb_src > RTE_DPAAX_QDMA_JOB_SUBMIT_MAX)) { 641 DPAA2_QDMA_ERR("SG entry number(%d) > MAX(%d)", 642 nb_src, RTE_DPAAX_QDMA_JOB_SUBMIT_MAX); 643 return -EINVAL; 644 } 645 646 memset(fd, 0, sizeof(struct qbman_fd)); 647 648 if (qdma_dev->is_silent) { 649 cntx_sg = qdma_vq->cntx_sg[qdma_vq->silent_idx]; 650 } else { 651 ret = rte_mempool_get(qdma_vq->fle_pool, 652 (void **)&cntx_sg); 653 if (ret) 654 return ret; 655 DPAA2_SET_FD_FRC(fd, QDMA_SER_CTX); 656 idx_addr = DPAA2_QDMA_IDXADDR_FROM_SG_FLAG(flags); 657 for (i = 0; i < nb_src; i++) 658 cntx_sg->cntx_idx[i] = idx_addr[i]; 659 } 660 661 cntx_iova = (uint64_t)cntx_sg - qdma_vq->fle_iova2va_offset; 662 663 fle = cntx_sg->fle_sdd.fle; 664 fle_iova = cntx_iova + 665 offsetof(struct qdma_cntx_sg, fle_sdd) + 666 offsetof(struct qdma_cntx_fle_sdd, fle); 667 668 dpaa2_qdma_fd_set_addr(fd, fle_iova); 669 DPAA2_SET_FD_COMPOUND_FMT(fd); 670 DPAA2_SET_FD_FLC(fd, (uint64_t)cntx_sg); 671 672 if (qdma_vq->fle_pre_populate) { 673 if (unlikely(!fle[DPAA2_QDMA_SRC_FLE].length)) { 674 fle_sdd_sg_pre_populate(cntx_sg, qdma_vq); 675 if (!qdma_dev->is_silent && cntx_sg && idx_addr) { 676 for (i = 0; i < nb_src; i++) 677 cntx_sg->cntx_idx[i] = idx_addr[i]; 678 } 679 } 680 681 len = sg_entry_post_populate(src, dst, 682 cntx_sg, nb_src); 683 sg_fle_post_populate(fle, len); 684 } else { 685 sdd = cntx_sg->fle_sdd.sdd; 686 sdd_iova = cntx_iova + 687 offsetof(struct qdma_cntx_sg, fle_sdd) + 688 offsetof(struct qdma_cntx_fle_sdd, sdd); 689 src_sge_iova = cntx_iova + 690 offsetof(struct qdma_cntx_sg, sg_src_entry); 691 dst_sge_iova = cntx_iova + 692 offsetof(struct qdma_cntx_sg, sg_dst_entry); 693 len = sg_entry_populate(src, dst, cntx_sg, nb_src); 694 695 fle_populate(fle, sdd, sdd_iova, 696 &qdma_vq->rbp, src_sge_iova, dst_sge_iova, len, 697 QBMAN_FLE_WORD4_FMT_SGE); 698 } 699 700 if (unlikely(qdma_vq->flags & DPAA2_QDMA_DESC_DEBUG_FLAG)) 701 dpaa2_qdma_long_fmt_dump(cntx_sg->fle_sdd.fle); 702 703 dpaa2_qdma_fd_save_att(fd, 0, DPAA2_QDMA_FD_SG); 704 qdma_vq->fd_idx++; 705 qdma_vq->silent_idx = 706 (qdma_vq->silent_idx + 1) & (DPAA2_QDMA_MAX_DESC - 1); 707 708 if (flags & RTE_DMA_OP_FLAG_SUBMIT) { 709 expected = qdma_vq->fd_idx; 710 ret = dpaa2_qdma_multi_eq(qdma_vq); 711 if (likely(ret == expected)) { 712 qdma_vq->copy_num += nb_src; 713 return (qdma_vq->copy_num - 1) & UINT16_MAX; 714 } 715 } else { 716 qdma_vq->copy_num += nb_src; 717 return (qdma_vq->copy_num - 1) & UINT16_MAX; 718 } 719 720 return ret; 721 } 722 723 static inline void 724 qdma_populate_fd_pci(uint64_t src, uint64_t dest, 725 uint32_t len, struct qbman_fd *fd, 726 struct dpaa2_qdma_rbp *rbp, int ser) 727 { 728 fd->simple_pci.saddr_lo = lower_32_bits(src); 729 fd->simple_pci.saddr_hi = upper_32_bits(src); 730 731 fd->simple_pci.len_sl = len; 732 733 fd->simple_pci.bmt = DPAA2_QDMA_BMT_DISABLE; 734 fd->simple_pci.fmt = DPAA2_QDMA_FD_SHORT_FORMAT; 735 fd->simple_pci.sl = 1; 736 fd->simple_pci.ser = ser; 737 if (ser) 738 fd->simple.frc |= QDMA_SER_CTX; 739 740 fd->simple_pci.sportid = rbp->sportid; 741 742 fd->simple_pci.svfid = rbp->svfid; 743 fd->simple_pci.spfid = rbp->spfid; 744 fd->simple_pci.svfa = rbp->svfa; 745 fd->simple_pci.dvfid = rbp->dvfid; 746 fd->simple_pci.dpfid = rbp->dpfid; 747 fd->simple_pci.dvfa = rbp->dvfa; 748 749 fd->simple_pci.srbp = rbp->srbp; 750 if (rbp->srbp) 751 fd->simple_pci.rdttype = 0; 752 else 753 fd->simple_pci.rdttype = dpaa2_coherent_alloc_cache; 754 755 /*dest is pcie memory */ 756 fd->simple_pci.dportid = rbp->dportid; 757 fd->simple_pci.drbp = rbp->drbp; 758 if (rbp->drbp) 759 fd->simple_pci.wrttype = 0; 760 else 761 fd->simple_pci.wrttype = dpaa2_coherent_no_alloc_cache; 762 763 fd->simple_pci.daddr_lo = lower_32_bits(dest); 764 fd->simple_pci.daddr_hi = upper_32_bits(dest); 765 } 766 767 static inline void 768 qdma_populate_fd_ddr(uint64_t src, uint64_t dest, 769 uint32_t len, struct qbman_fd *fd, int ser) 770 { 771 fd->simple_ddr.saddr_lo = lower_32_bits(src); 772 fd->simple_ddr.saddr_hi = upper_32_bits(src); 773 774 fd->simple_ddr.len = len; 775 776 fd->simple_ddr.bmt = DPAA2_QDMA_BMT_DISABLE; 777 fd->simple_ddr.fmt = DPAA2_QDMA_FD_SHORT_FORMAT; 778 fd->simple_ddr.sl = 1; 779 fd->simple_ddr.ser = ser; 780 if (ser) 781 fd->simple.frc |= QDMA_SER_CTX; 782 /** 783 * src If RBP=0 {NS,RDTTYPE[3:0]}: 0_1011 784 * Coherent copy of cacheable memory, 785 * lookup in downstream cache, no allocate 786 * on miss. 787 */ 788 fd->simple_ddr.rns = 0; 789 fd->simple_ddr.rdttype = dpaa2_coherent_alloc_cache; 790 /** 791 * dest If RBP=0 {NS,WRTTYPE[3:0]}: 0_0111 792 * Coherent write of cacheable memory, 793 * lookup in downstream cache, no allocate on miss 794 */ 795 fd->simple_ddr.wns = 0; 796 fd->simple_ddr.wrttype = dpaa2_coherent_no_alloc_cache; 797 798 fd->simple_ddr.daddr_lo = lower_32_bits(dest); 799 fd->simple_ddr.daddr_hi = upper_32_bits(dest); 800 } 801 802 static int 803 dpaa2_qdma_short_copy(struct qdma_virt_queue *qdma_vq, 804 rte_iova_t src, rte_iova_t dst, uint32_t length, 805 int is_silent, uint64_t flags) 806 { 807 int ret = 0, expected; 808 struct qbman_fd *fd = &qdma_vq->fd[qdma_vq->fd_idx]; 809 810 memset(fd, 0, sizeof(struct qbman_fd)); 811 812 if (qdma_vq->rbp.drbp || qdma_vq->rbp.srbp) { 813 /** PCIe EP*/ 814 qdma_populate_fd_pci(src, 815 dst, length, 816 fd, &qdma_vq->rbp, 817 is_silent ? 0 : 1); 818 } else { 819 /** DDR or PCIe RC*/ 820 qdma_populate_fd_ddr(src, 821 dst, length, 822 fd, is_silent ? 0 : 1); 823 } 824 dpaa2_qdma_fd_save_att(fd, DPAA2_QDMA_IDX_FROM_FLAG(flags), 825 DPAA2_QDMA_FD_SHORT); 826 qdma_vq->fd_idx++; 827 828 if (flags & RTE_DMA_OP_FLAG_SUBMIT) { 829 expected = qdma_vq->fd_idx; 830 ret = dpaa2_qdma_multi_eq(qdma_vq); 831 if (likely(ret == expected)) { 832 qdma_vq->copy_num++; 833 return (qdma_vq->copy_num - 1) & UINT16_MAX; 834 } 835 } else { 836 qdma_vq->copy_num++; 837 return (qdma_vq->copy_num - 1) & UINT16_MAX; 838 } 839 840 return ret; 841 } 842 843 static int 844 dpaa2_qdma_long_copy(struct qdma_virt_queue *qdma_vq, 845 rte_iova_t src, rte_iova_t dst, uint32_t length, 846 int is_silent, uint64_t flags) 847 { 848 int ret = 0, expected; 849 struct qbman_fd *fd = &qdma_vq->fd[qdma_vq->fd_idx]; 850 struct qdma_cntx_fle_sdd *fle_sdd = NULL; 851 rte_iova_t fle_iova, sdd_iova; 852 struct qbman_fle *fle; 853 struct qdma_sdd *sdd; 854 855 memset(fd, 0, sizeof(struct qbman_fd)); 856 857 if (is_silent) { 858 fle_sdd = qdma_vq->cntx_fle_sdd[qdma_vq->silent_idx]; 859 } else { 860 ret = rte_mempool_get(qdma_vq->fle_pool, 861 (void **)&fle_sdd); 862 if (ret) 863 return ret; 864 DPAA2_SET_FD_FRC(fd, QDMA_SER_CTX); 865 } 866 867 fle = fle_sdd->fle; 868 fle_iova = (uint64_t)fle - qdma_vq->fle_iova2va_offset; 869 870 dpaa2_qdma_fd_set_addr(fd, fle_iova); 871 DPAA2_SET_FD_COMPOUND_FMT(fd); 872 DPAA2_SET_FD_FLC(fd, (uint64_t)fle); 873 874 if (qdma_vq->fle_pre_populate) { 875 if (unlikely(!fle[DPAA2_QDMA_SRC_FLE].length)) { 876 fle_sdd_pre_populate(fle_sdd, 877 &qdma_vq->rbp, 878 0, 0, QBMAN_FLE_WORD4_FMT_SBF); 879 } 880 881 fle_post_populate(fle, src, dst, length); 882 } else { 883 sdd = fle_sdd->sdd; 884 sdd_iova = (uint64_t)sdd - qdma_vq->fle_iova2va_offset; 885 fle_populate(fle, sdd, sdd_iova, &qdma_vq->rbp, 886 src, dst, length, 887 QBMAN_FLE_WORD4_FMT_SBF); 888 } 889 890 if (unlikely(qdma_vq->flags & DPAA2_QDMA_DESC_DEBUG_FLAG)) 891 dpaa2_qdma_long_fmt_dump(fle); 892 893 dpaa2_qdma_fd_save_att(fd, DPAA2_QDMA_IDX_FROM_FLAG(flags), 894 DPAA2_QDMA_FD_LONG); 895 qdma_vq->fd_idx++; 896 qdma_vq->silent_idx = 897 (qdma_vq->silent_idx + 1) & (DPAA2_QDMA_MAX_DESC - 1); 898 899 if (flags & RTE_DMA_OP_FLAG_SUBMIT) { 900 expected = qdma_vq->fd_idx; 901 ret = dpaa2_qdma_multi_eq(qdma_vq); 902 if (likely(ret == expected)) { 903 qdma_vq->copy_num++; 904 return (qdma_vq->copy_num - 1) & UINT16_MAX; 905 } 906 } else { 907 qdma_vq->copy_num++; 908 return (qdma_vq->copy_num - 1) & UINT16_MAX; 909 } 910 911 return ret; 912 } 913 914 static int 915 dpaa2_qdma_copy(void *dev_private, uint16_t vchan, 916 rte_iova_t src, rte_iova_t dst, 917 uint32_t length, uint64_t flags) 918 { 919 struct dpaa2_dpdmai_dev *dpdmai_dev = dev_private; 920 struct qdma_device *qdma_dev = dpdmai_dev->qdma_dev; 921 struct qdma_virt_queue *qdma_vq = &qdma_dev->vqs[vchan]; 922 923 if (qdma_vq->using_short_fd) 924 return dpaa2_qdma_short_copy(qdma_vq, src, dst, 925 length, qdma_dev->is_silent, flags); 926 else 927 return dpaa2_qdma_long_copy(qdma_vq, src, dst, 928 length, qdma_dev->is_silent, flags); 929 } 930 931 static inline int 932 dpaa2_qdma_dq_fd(const struct qbman_fd *fd, 933 struct qdma_virt_queue *qdma_vq, 934 uint16_t *free_space, uint16_t *fle_elem_nb) 935 { 936 uint16_t idx, att; 937 enum dpaa2_qdma_fd_type type; 938 int ret; 939 struct qdma_cntx_sg *cntx_sg; 940 struct qdma_cntx_fle_sdd *fle_sdd; 941 942 att = dpaa2_qdma_fd_get_att(fd); 943 type = DPAA2_QDMA_FD_ATT_TYPE(att); 944 if (type == DPAA2_QDMA_FD_SHORT) { 945 idx = DPAA2_QDMA_FD_ATT_CNTX(att); 946 ret = qdma_cntx_idx_ring_eq(qdma_vq->ring_cntx_idx, 947 &idx, 1, free_space); 948 if (unlikely(ret != 1)) 949 return -ENOSPC; 950 951 return 0; 952 } 953 if (type == DPAA2_QDMA_FD_LONG) { 954 idx = DPAA2_QDMA_FD_ATT_CNTX(att); 955 fle_sdd = (void *)(uintptr_t)DPAA2_GET_FD_FLC(fd); 956 qdma_vq->fle_elem[*fle_elem_nb] = fle_sdd; 957 (*fle_elem_nb)++; 958 ret = qdma_cntx_idx_ring_eq(qdma_vq->ring_cntx_idx, 959 &idx, 1, free_space); 960 if (unlikely(ret != 1)) 961 return -ENOSPC; 962 963 return 0; 964 } 965 if (type == DPAA2_QDMA_FD_SG) { 966 fle_sdd = (void *)(uintptr_t)DPAA2_GET_FD_FLC(fd); 967 qdma_vq->fle_elem[*fle_elem_nb] = fle_sdd; 968 (*fle_elem_nb)++; 969 cntx_sg = container_of(fle_sdd, 970 struct qdma_cntx_sg, fle_sdd); 971 ret = qdma_cntx_idx_ring_eq(qdma_vq->ring_cntx_idx, 972 cntx_sg->cntx_idx, 973 cntx_sg->job_nb, free_space); 974 if (unlikely(ret < cntx_sg->job_nb)) 975 return -ENOSPC; 976 977 return 0; 978 } 979 980 DPAA2_QDMA_ERR("Invalid FD type, ATT=0x%04x", 981 fd->simple_ddr.rsv1_att); 982 return -EIO; 983 } 984 985 static uint16_t 986 dpaa2_qdma_dequeue(void *dev_private, 987 uint16_t vchan, const uint16_t nb_cpls, 988 uint16_t *cntx_idx, bool *has_error) 989 { 990 struct dpaa2_dpdmai_dev *dpdmai_dev = dev_private; 991 struct qdma_device *qdma_dev = dpdmai_dev->qdma_dev; 992 struct qdma_virt_queue *qdma_vq = &qdma_dev->vqs[vchan]; 993 994 struct dpaa2_queue *rxq; 995 struct qbman_result *dq_storage, *dq_storage1 = NULL; 996 struct qbman_pull_desc pulldesc; 997 struct qbman_swp *swp; 998 struct queue_storage_info_t *q_storage; 999 uint32_t fqid; 1000 uint8_t status, pending; 1001 uint8_t num_rx = 0; 1002 const struct qbman_fd *fd; 1003 int ret, pull_size; 1004 uint16_t free_space = 0, fle_elem_nb = 0; 1005 1006 if (unlikely(qdma_dev->is_silent)) 1007 return 0; 1008 1009 if (unlikely(!DPAA2_PER_LCORE_DPIO)) { 1010 ret = dpaa2_affine_qbman_swp(); 1011 if (ret) { 1012 DPAA2_QDMA_ERR("Allocate portal err, tid(%d)", 1013 rte_gettid()); 1014 if (has_error) 1015 *has_error = true; 1016 return 0; 1017 } 1018 } 1019 swp = DPAA2_PER_LCORE_PORTAL; 1020 1021 pull_size = (nb_cpls > dpaa2_dqrr_size) ? 1022 dpaa2_dqrr_size : nb_cpls; 1023 rxq = &(dpdmai_dev->rx_queue[qdma_vq->vq_id]); 1024 fqid = rxq->fqid; 1025 q_storage = rxq->q_storage[0]; 1026 1027 if (unlikely(!q_storage->active_dqs)) { 1028 q_storage->toggle = 0; 1029 dq_storage = q_storage->dq_storage[q_storage->toggle]; 1030 q_storage->last_num_pkts = pull_size; 1031 qbman_pull_desc_clear(&pulldesc); 1032 qbman_pull_desc_set_numframes(&pulldesc, 1033 q_storage->last_num_pkts); 1034 qbman_pull_desc_set_fq(&pulldesc, fqid); 1035 qbman_pull_desc_set_storage(&pulldesc, dq_storage, 1036 DPAA2_VADDR_TO_IOVA(dq_storage), 1); 1037 if (check_swp_active_dqs(DPAA2_PER_LCORE_DPIO->index)) { 1038 while (!qbman_check_command_complete( 1039 get_swp_active_dqs( 1040 DPAA2_PER_LCORE_DPIO->index))) 1041 ; 1042 clear_swp_active_dqs(DPAA2_PER_LCORE_DPIO->index); 1043 } 1044 while (1) { 1045 if (qbman_swp_pull(swp, &pulldesc)) { 1046 DPAA2_QDMA_DP_WARN("QBMAN busy"); 1047 /* Portal was busy, try again */ 1048 continue; 1049 } 1050 break; 1051 } 1052 q_storage->active_dqs = dq_storage; 1053 q_storage->active_dpio_id = DPAA2_PER_LCORE_DPIO->index; 1054 set_swp_active_dqs(DPAA2_PER_LCORE_DPIO->index, 1055 dq_storage); 1056 } 1057 1058 dq_storage = q_storage->active_dqs; 1059 rte_prefetch0((void *)(size_t)(dq_storage)); 1060 rte_prefetch0((void *)(size_t)(dq_storage + 1)); 1061 1062 /* Prepare next pull descriptor. This will give space for the 1063 * prefething done on DQRR entries 1064 */ 1065 q_storage->toggle ^= 1; 1066 dq_storage1 = q_storage->dq_storage[q_storage->toggle]; 1067 qbman_pull_desc_clear(&pulldesc); 1068 qbman_pull_desc_set_numframes(&pulldesc, pull_size); 1069 qbman_pull_desc_set_fq(&pulldesc, fqid); 1070 qbman_pull_desc_set_storage(&pulldesc, dq_storage1, 1071 DPAA2_VADDR_TO_IOVA(dq_storage1), 1); 1072 1073 /* Check if the previous issued command is completed. 1074 * Also seems like the SWP is shared between the Ethernet Driver 1075 * and the SEC driver. 1076 */ 1077 while (!qbman_check_command_complete(dq_storage)) 1078 ; 1079 if (dq_storage == get_swp_active_dqs(q_storage->active_dpio_id)) 1080 clear_swp_active_dqs(q_storage->active_dpio_id); 1081 1082 pending = 1; 1083 1084 do { 1085 /* Loop until the dq_storage is updated with 1086 * new token by QBMAN 1087 */ 1088 while (!qbman_check_new_result(dq_storage)) 1089 ; 1090 rte_prefetch0((void *)((size_t)(dq_storage + 2))); 1091 /* Check whether Last Pull command is Expired and 1092 * setting Condition for Loop termination 1093 */ 1094 if (qbman_result_DQ_is_pull_complete(dq_storage)) { 1095 pending = 0; 1096 /* Check for valid frame. */ 1097 status = qbman_result_DQ_flags(dq_storage); 1098 if (unlikely((status & QBMAN_DQ_STAT_VALIDFRAME) == 0)) 1099 continue; 1100 } 1101 fd = qbman_result_DQ_fd(dq_storage); 1102 ret = dpaa2_qdma_dq_fd(fd, qdma_vq, &free_space, &fle_elem_nb); 1103 if (ret || free_space < RTE_DPAAX_QDMA_JOB_SUBMIT_MAX) 1104 pending = 0; 1105 1106 dq_storage++; 1107 } while (pending); 1108 1109 if (check_swp_active_dqs(DPAA2_PER_LCORE_DPIO->index)) { 1110 while (!qbman_check_command_complete( 1111 get_swp_active_dqs(DPAA2_PER_LCORE_DPIO->index))) 1112 ; 1113 clear_swp_active_dqs(DPAA2_PER_LCORE_DPIO->index); 1114 } 1115 /* issue a volatile dequeue command for next pull */ 1116 while (1) { 1117 if (qbman_swp_pull(swp, &pulldesc)) { 1118 DPAA2_QDMA_DP_WARN("QBMAN is busy (2)"); 1119 continue; 1120 } 1121 break; 1122 } 1123 1124 q_storage->active_dqs = dq_storage1; 1125 q_storage->active_dpio_id = DPAA2_PER_LCORE_DPIO->index; 1126 set_swp_active_dqs(DPAA2_PER_LCORE_DPIO->index, dq_storage1); 1127 1128 if (fle_elem_nb > 0) { 1129 rte_mempool_put_bulk(qdma_vq->fle_pool, 1130 qdma_vq->fle_elem, fle_elem_nb); 1131 } 1132 1133 num_rx = qdma_cntx_idx_ring_dq(qdma_vq->ring_cntx_idx, 1134 cntx_idx, nb_cpls); 1135 1136 if (has_error) 1137 *has_error = false; 1138 1139 return num_rx; 1140 } 1141 1142 static int 1143 dpaa2_qdma_info_get(const struct rte_dma_dev *dev, 1144 struct rte_dma_info *dev_info, 1145 uint32_t info_sz __rte_unused) 1146 { 1147 struct dpaa2_dpdmai_dev *dpdmai_dev = dev->data->dev_private; 1148 1149 dev_info->dev_capa = RTE_DMA_CAPA_MEM_TO_MEM | 1150 RTE_DMA_CAPA_MEM_TO_DEV | 1151 RTE_DMA_CAPA_DEV_TO_DEV | 1152 RTE_DMA_CAPA_DEV_TO_MEM | 1153 RTE_DMA_CAPA_SILENT | 1154 RTE_DMA_CAPA_OPS_COPY | 1155 RTE_DMA_CAPA_OPS_COPY_SG; 1156 dev_info->dev_capa |= RTE_DMA_CAPA_DPAAX_QDMA_FLAGS_INDEX; 1157 dev_info->max_vchans = dpdmai_dev->num_queues; 1158 dev_info->max_desc = DPAA2_QDMA_MAX_DESC; 1159 dev_info->min_desc = DPAA2_QDMA_MIN_DESC; 1160 dev_info->max_sges = RTE_DPAAX_QDMA_JOB_SUBMIT_MAX; 1161 dev_info->dev_name = dev->device->name; 1162 if (dpdmai_dev->qdma_dev) 1163 dev_info->nb_vchans = dpdmai_dev->qdma_dev->num_vqs; 1164 1165 return 0; 1166 } 1167 1168 static int 1169 dpaa2_qdma_configure(struct rte_dma_dev *dev, 1170 const struct rte_dma_conf *dev_conf, 1171 uint32_t conf_sz) 1172 { 1173 struct dpaa2_dpdmai_dev *dpdmai_dev = dev->data->dev_private; 1174 struct qdma_device *qdma_dev = dpdmai_dev->qdma_dev; 1175 uint16_t i; 1176 struct dpdmai_rx_queue_cfg rx_queue_cfg; 1177 struct dpdmai_rx_queue_attr rx_attr; 1178 struct dpdmai_tx_queue_attr tx_attr; 1179 struct dpaa2_queue *rxq; 1180 int ret = 0; 1181 1182 DPAA2_QDMA_FUNC_TRACE(); 1183 1184 RTE_SET_USED(conf_sz); 1185 1186 if (dev_conf->nb_vchans > dpdmai_dev->num_queues) { 1187 DPAA2_QDMA_ERR("%s config queues(%d) > hw queues(%d)", 1188 dev->data->dev_name, dev_conf->nb_vchans, 1189 dpdmai_dev->num_queues); 1190 1191 return -ENOTSUP; 1192 } 1193 1194 if (qdma_dev->vqs) { 1195 DPAA2_QDMA_DEBUG("%s: queues de-config(%d)/re-config(%d)", 1196 dev->data->dev_name, 1197 qdma_dev->num_vqs, dev_conf->nb_vchans); 1198 for (i = 0; i < qdma_dev->num_vqs; i++) { 1199 if ((qdma_dev->vqs[i].num_enqueues != 1200 qdma_dev->vqs[i].num_dequeues) && 1201 !qdma_dev->is_silent) { 1202 DPAA2_QDMA_ERR("VQ(%d) %"PRIu64" jobs in dma.", 1203 i, qdma_dev->vqs[i].num_enqueues - 1204 qdma_dev->vqs[i].num_dequeues); 1205 return -EBUSY; 1206 } 1207 } 1208 for (i = 0; i < qdma_dev->num_vqs; i++) { 1209 if (qdma_dev->vqs[i].fle_pool) { 1210 rte_mempool_free(qdma_dev->vqs[i].fle_pool); 1211 qdma_dev->vqs[i].fle_pool = NULL; 1212 } 1213 if (qdma_dev->vqs[i].ring_cntx_idx) { 1214 rte_free(qdma_dev->vqs[i].ring_cntx_idx); 1215 qdma_dev->vqs[i].ring_cntx_idx = NULL; 1216 } 1217 rxq = &dpdmai_dev->rx_queue[i]; 1218 dpaa2_queue_storage_free(rxq, 1); 1219 } 1220 rte_free(qdma_dev->vqs); 1221 qdma_dev->vqs = NULL; 1222 qdma_dev->num_vqs = 0; 1223 } 1224 1225 /* Set up Rx Queues */ 1226 for (i = 0; i < dev_conf->nb_vchans; i++) { 1227 memset(&rx_queue_cfg, 0, sizeof(struct dpdmai_rx_queue_cfg)); 1228 rxq = &dpdmai_dev->rx_queue[i]; 1229 ret = dpdmai_set_rx_queue(&s_proc_mc_reg, 1230 CMD_PRI_LOW, 1231 dpdmai_dev->token, 1232 i, 0, &rx_queue_cfg); 1233 if (ret) { 1234 DPAA2_QDMA_ERR("%s RXQ%d set failed(%d)", 1235 dev->data->dev_name, i, ret); 1236 return ret; 1237 } 1238 } 1239 1240 /* Get Rx and Tx queues FQID's */ 1241 for (i = 0; i < dev_conf->nb_vchans; i++) { 1242 ret = dpdmai_get_rx_queue(&s_proc_mc_reg, CMD_PRI_LOW, 1243 dpdmai_dev->token, i, 0, &rx_attr); 1244 if (ret) { 1245 DPAA2_QDMA_ERR("Get DPDMAI%d-RXQ%d failed(%d)", 1246 dpdmai_dev->dpdmai_id, i, ret); 1247 return ret; 1248 } 1249 dpdmai_dev->rx_queue[i].fqid = rx_attr.fqid; 1250 1251 ret = dpdmai_get_tx_queue(&s_proc_mc_reg, CMD_PRI_LOW, 1252 dpdmai_dev->token, i, 0, &tx_attr); 1253 if (ret) { 1254 DPAA2_QDMA_ERR("Get DPDMAI%d-TXQ%d failed(%d)", 1255 dpdmai_dev->dpdmai_id, i, ret); 1256 return ret; 1257 } 1258 dpdmai_dev->tx_queue[i].fqid = tx_attr.fqid; 1259 } 1260 1261 /* Allocate Virtual Queues */ 1262 qdma_dev->vqs = rte_zmalloc(NULL, 1263 (sizeof(struct qdma_virt_queue) * dev_conf->nb_vchans), 1264 RTE_CACHE_LINE_SIZE); 1265 if (!qdma_dev->vqs) { 1266 DPAA2_QDMA_ERR("%s: VQs(%d) alloc failed.", 1267 dev->data->dev_name, dev_conf->nb_vchans); 1268 return -ENOMEM; 1269 } 1270 for (i = 0; i < dev_conf->nb_vchans; i++) { 1271 qdma_dev->vqs[i].vq_id = i; 1272 rxq = &dpdmai_dev->rx_queue[i]; 1273 /* Allocate DQ storage for the DPDMAI Rx queues */ 1274 ret = dpaa2_queue_storage_alloc(rxq, 1); 1275 if (ret) 1276 goto alloc_failed; 1277 } 1278 1279 qdma_dev->num_vqs = dev_conf->nb_vchans; 1280 qdma_dev->is_silent = dev_conf->enable_silent; 1281 1282 return 0; 1283 1284 alloc_failed: 1285 for (i = 0; i < dev_conf->nb_vchans; i++) { 1286 rxq = &dpdmai_dev->rx_queue[i]; 1287 dpaa2_queue_storage_free(rxq, 1); 1288 } 1289 1290 rte_free(qdma_dev->vqs); 1291 qdma_dev->vqs = NULL; 1292 qdma_dev->num_vqs = 0; 1293 1294 return ret; 1295 } 1296 1297 static int 1298 dpaa2_qdma_vchan_rbp_set(struct qdma_virt_queue *vq, 1299 const struct rte_dma_vchan_conf *conf) 1300 { 1301 if (conf->direction == RTE_DMA_DIR_MEM_TO_DEV || 1302 conf->direction == RTE_DMA_DIR_DEV_TO_DEV) { 1303 if (conf->dst_port.port_type != RTE_DMA_PORT_PCIE) 1304 return -EINVAL; 1305 vq->rbp.enable = 1; 1306 vq->rbp.dportid = conf->dst_port.pcie.coreid; 1307 vq->rbp.dpfid = conf->dst_port.pcie.pfid; 1308 if (conf->dst_port.pcie.vfen) { 1309 vq->rbp.dvfa = 1; 1310 vq->rbp.dvfid = conf->dst_port.pcie.vfid; 1311 } 1312 vq->rbp.drbp = 1; 1313 } 1314 if (conf->direction == RTE_DMA_DIR_DEV_TO_MEM || 1315 conf->direction == RTE_DMA_DIR_DEV_TO_DEV) { 1316 if (conf->src_port.port_type != RTE_DMA_PORT_PCIE) 1317 return -EINVAL; 1318 vq->rbp.enable = 1; 1319 vq->rbp.sportid = conf->src_port.pcie.coreid; 1320 vq->rbp.spfid = conf->src_port.pcie.pfid; 1321 if (conf->src_port.pcie.vfen) { 1322 vq->rbp.svfa = 1; 1323 vq->rbp.dvfid = conf->src_port.pcie.vfid; 1324 } 1325 vq->rbp.srbp = 1; 1326 } 1327 1328 return 0; 1329 } 1330 1331 static int 1332 dpaa2_qdma_vchan_setup(struct rte_dma_dev *dev, uint16_t vchan, 1333 const struct rte_dma_vchan_conf *conf, 1334 uint32_t conf_sz) 1335 { 1336 struct dpaa2_dpdmai_dev *dpdmai_dev = dev->data->dev_private; 1337 struct qdma_device *qdma_dev = dpdmai_dev->qdma_dev; 1338 uint32_t pool_size; 1339 char pool_name[64]; 1340 int ret; 1341 uint64_t iova, va; 1342 1343 DPAA2_QDMA_FUNC_TRACE(); 1344 1345 RTE_SET_USED(conf_sz); 1346 1347 ret = dpaa2_qdma_vchan_rbp_set(&qdma_dev->vqs[vchan], conf); 1348 if (ret) 1349 return ret; 1350 1351 if (dpaa2_qdma_get_devargs(dev->device->devargs, DPAA2_QDMA_FLE_PRE_POPULATE)) 1352 qdma_dev->vqs[vchan].fle_pre_populate = 1; 1353 else 1354 qdma_dev->vqs[vchan].fle_pre_populate = 0; 1355 1356 if (dpaa2_qdma_get_devargs(dev->device->devargs, DPAA2_QDMA_DESC_DEBUG)) 1357 qdma_dev->vqs[vchan].flags |= DPAA2_QDMA_DESC_DEBUG_FLAG; 1358 else 1359 qdma_dev->vqs[vchan].flags &= (~DPAA2_QDMA_DESC_DEBUG_FLAG); 1360 1361 if (dpaa2_qdma_get_devargs(dev->device->devargs, DPAA2_QDMA_USING_SHORT_FD)) 1362 qdma_dev->vqs[vchan].using_short_fd = 1; 1363 else 1364 qdma_dev->vqs[vchan].using_short_fd = 0; 1365 1366 snprintf(pool_name, sizeof(pool_name), 1367 "qdma_fle_pool_dev%d_qid%d", dpdmai_dev->dpdmai_id, vchan); 1368 pool_size = sizeof(struct qdma_cntx_sg); 1369 qdma_dev->vqs[vchan].fle_pool = rte_mempool_create(pool_name, 1370 DPAA2_QDMA_MAX_DESC * 2, pool_size, 1371 512, 0, NULL, NULL, NULL, NULL, 1372 SOCKET_ID_ANY, 0); 1373 if (!qdma_dev->vqs[vchan].fle_pool) { 1374 DPAA2_QDMA_ERR("%s create failed", pool_name); 1375 return -ENOMEM; 1376 } 1377 iova = qdma_dev->vqs[vchan].fle_pool->mz->iova; 1378 va = qdma_dev->vqs[vchan].fle_pool->mz->addr_64; 1379 qdma_dev->vqs[vchan].fle_iova2va_offset = va - iova; 1380 1381 if (qdma_dev->is_silent) { 1382 ret = rte_mempool_get_bulk(qdma_dev->vqs[vchan].fle_pool, 1383 (void **)qdma_dev->vqs[vchan].cntx_sg, 1384 DPAA2_QDMA_MAX_DESC); 1385 if (ret) { 1386 DPAA2_QDMA_ERR("sg cntx get from %s for silent mode", 1387 pool_name); 1388 return ret; 1389 } 1390 ret = rte_mempool_get_bulk(qdma_dev->vqs[vchan].fle_pool, 1391 (void **)qdma_dev->vqs[vchan].cntx_fle_sdd, 1392 DPAA2_QDMA_MAX_DESC); 1393 if (ret) { 1394 DPAA2_QDMA_ERR("long cntx get from %s for silent mode", 1395 pool_name); 1396 return ret; 1397 } 1398 } else { 1399 qdma_dev->vqs[vchan].ring_cntx_idx = rte_malloc(NULL, 1400 sizeof(struct qdma_cntx_idx_ring), 1401 RTE_CACHE_LINE_SIZE); 1402 if (!qdma_dev->vqs[vchan].ring_cntx_idx) { 1403 DPAA2_QDMA_ERR("DQ response ring alloc failed."); 1404 return -ENOMEM; 1405 } 1406 qdma_dev->vqs[vchan].ring_cntx_idx->start = 0; 1407 qdma_dev->vqs[vchan].ring_cntx_idx->tail = 0; 1408 qdma_dev->vqs[vchan].ring_cntx_idx->free_space = 1409 QDMA_CNTX_IDX_RING_MAX_FREE; 1410 qdma_dev->vqs[vchan].ring_cntx_idx->nb_in_ring = 0; 1411 qdma_dev->vqs[vchan].fle_elem = rte_malloc(NULL, 1412 sizeof(void *) * DPAA2_QDMA_MAX_DESC, 1413 RTE_CACHE_LINE_SIZE); 1414 } 1415 1416 qdma_dev->vqs[vchan].dpdmai_dev = dpdmai_dev; 1417 qdma_dev->vqs[vchan].nb_desc = conf->nb_desc; 1418 1419 return 0; 1420 } 1421 1422 static int 1423 dpaa2_qdma_start(struct rte_dma_dev *dev) 1424 { 1425 struct dpaa2_dpdmai_dev *dpdmai_dev = dev->data->dev_private; 1426 int ret; 1427 1428 DPAA2_QDMA_FUNC_TRACE(); 1429 1430 /* Enable the device */ 1431 ret = dpdmai_enable(&s_proc_mc_reg, CMD_PRI_LOW, 1432 dpdmai_dev->token); 1433 if (ret) { 1434 DPAA2_QDMA_ERR("Enabling device failed with err: %d", ret); 1435 return ret; 1436 } 1437 1438 return 0; 1439 } 1440 1441 static int 1442 dpaa2_qdma_stop(struct rte_dma_dev *dev) 1443 { 1444 struct dpaa2_dpdmai_dev *dpdmai_dev = dev->data->dev_private; 1445 int ret; 1446 1447 DPAA2_QDMA_FUNC_TRACE(); 1448 1449 /* Disable the device */ 1450 ret = dpdmai_disable(&s_proc_mc_reg, CMD_PRI_LOW, 1451 dpdmai_dev->token); 1452 if (ret) { 1453 DPAA2_QDMA_ERR("Disable device failed with err: %d", ret); 1454 return ret; 1455 } 1456 1457 return 0; 1458 } 1459 1460 static int 1461 dpaa2_qdma_close(struct rte_dma_dev *dev) 1462 { 1463 struct dpaa2_dpdmai_dev *dpdmai_dev = dev->data->dev_private; 1464 struct qdma_device *qdma_dev = dpdmai_dev->qdma_dev; 1465 struct dpaa2_queue *rxq; 1466 int i; 1467 1468 DPAA2_QDMA_FUNC_TRACE(); 1469 1470 if (!qdma_dev) 1471 return 0; 1472 1473 /* In case there are pending jobs on any VQ, return -EBUSY */ 1474 for (i = 0; i < qdma_dev->num_vqs; i++) { 1475 if ((qdma_dev->vqs[i].num_enqueues != 1476 qdma_dev->vqs[i].num_dequeues) && 1477 !qdma_dev->is_silent) { 1478 DPAA2_QDMA_ERR("VQ(%d) pending: eq(%"PRIu64") != dq(%"PRId64")", 1479 i, qdma_dev->vqs[i].num_enqueues, 1480 qdma_dev->vqs[i].num_dequeues); 1481 return -EBUSY; 1482 } 1483 } 1484 1485 /* Free RXQ storages */ 1486 for (i = 0; i < qdma_dev->num_vqs; i++) { 1487 rxq = &dpdmai_dev->rx_queue[i]; 1488 dpaa2_queue_storage_free(rxq, 1); 1489 } 1490 1491 if (qdma_dev->vqs) { 1492 /* Free RXQ fle pool */ 1493 for (i = 0; i < qdma_dev->num_vqs; i++) { 1494 if (qdma_dev->vqs[i].fle_pool) { 1495 rte_mempool_free(qdma_dev->vqs[i].fle_pool); 1496 qdma_dev->vqs[i].fle_pool = NULL; 1497 } 1498 if (qdma_dev->vqs[i].ring_cntx_idx) { 1499 rte_free(qdma_dev->vqs[i].ring_cntx_idx); 1500 qdma_dev->vqs[i].ring_cntx_idx = NULL; 1501 } 1502 } 1503 rte_free(qdma_dev->vqs); 1504 qdma_dev->vqs = NULL; 1505 } 1506 1507 /* Reset QDMA device structure */ 1508 qdma_dev->num_vqs = 0; 1509 1510 return 0; 1511 } 1512 1513 static int 1514 dpaa2_qdma_stats_get(const struct rte_dma_dev *dmadev, 1515 uint16_t vchan, struct rte_dma_stats *rte_stats, uint32_t size) 1516 { 1517 struct dpaa2_dpdmai_dev *dpdmai_dev = dmadev->data->dev_private; 1518 struct qdma_device *qdma_dev = dpdmai_dev->qdma_dev; 1519 struct qdma_virt_queue *qdma_vq = &qdma_dev->vqs[vchan]; 1520 struct rte_dma_stats *stats = &qdma_vq->stats; 1521 1522 RTE_SET_USED(size); 1523 1524 /* TODO - directly use stats */ 1525 stats->submitted = qdma_vq->num_enqueues; 1526 stats->completed = qdma_vq->num_dequeues; 1527 *rte_stats = *stats; 1528 1529 return 0; 1530 } 1531 1532 static int 1533 dpaa2_qdma_stats_reset(struct rte_dma_dev *dmadev, uint16_t vchan) 1534 { 1535 struct dpaa2_dpdmai_dev *dpdmai_dev = dmadev->data->dev_private; 1536 struct qdma_device *qdma_dev = dpdmai_dev->qdma_dev; 1537 struct qdma_virt_queue *qdma_vq = &qdma_dev->vqs[vchan]; 1538 1539 qdma_vq->num_enqueues = 0; 1540 qdma_vq->num_dequeues = 0; 1541 1542 return 0; 1543 } 1544 1545 static uint16_t 1546 dpaa2_qdma_burst_capacity(const void *dev_private, uint16_t vchan) 1547 { 1548 const struct dpaa2_dpdmai_dev *dpdmai_dev = dev_private; 1549 struct qdma_device *qdma_dev = dpdmai_dev->qdma_dev; 1550 struct qdma_virt_queue *qdma_vq = &qdma_dev->vqs[vchan]; 1551 1552 return qdma_vq->nb_desc - qdma_vq->num_valid_jobs; 1553 } 1554 1555 static struct rte_dma_dev_ops dpaa2_qdma_ops = { 1556 .dev_info_get = dpaa2_qdma_info_get, 1557 .dev_configure = dpaa2_qdma_configure, 1558 .dev_start = dpaa2_qdma_start, 1559 .dev_stop = dpaa2_qdma_stop, 1560 .dev_close = dpaa2_qdma_close, 1561 .vchan_setup = dpaa2_qdma_vchan_setup, 1562 .stats_get = dpaa2_qdma_stats_get, 1563 .stats_reset = dpaa2_qdma_stats_reset, 1564 }; 1565 1566 static int 1567 dpaa2_dpdmai_dev_uninit(struct rte_dma_dev *dev) 1568 { 1569 struct dpaa2_dpdmai_dev *dpdmai_dev = dev->data->dev_private; 1570 struct qdma_device *qdma_dev = dpdmai_dev->qdma_dev; 1571 int ret; 1572 1573 DPAA2_QDMA_FUNC_TRACE(); 1574 1575 if (rte_eal_process_type() == RTE_PROC_SECONDARY) { 1576 DPAA2_QDMA_DEBUG("Un-attach DMA(%d) in the 2nd proess.", 1577 dpdmai_dev->dpdmai_id); 1578 return 0; 1579 } 1580 1581 /* Close the device at underlying layer*/ 1582 ret = dpdmai_close(&s_proc_mc_reg, CMD_PRI_LOW, 1583 dpdmai_dev->token); 1584 if (ret) { 1585 DPAA2_QDMA_ERR("dpdmai(%d) close failed(%d)", 1586 dpdmai_dev->dpdmai_id, ret); 1587 1588 return ret; 1589 } 1590 1591 if (qdma_dev) { 1592 rte_free(qdma_dev); 1593 dpdmai_dev->qdma_dev = NULL; 1594 } 1595 1596 return ret; 1597 } 1598 1599 static int 1600 dpaa2_dpdmai_dev_init(struct rte_dma_dev *dev, uint32_t dpdmai_id) 1601 { 1602 struct dpaa2_dpdmai_dev *dpdmai_dev = dev->data->dev_private; 1603 struct dpdmai_attr attr; 1604 int ret, err; 1605 1606 DPAA2_QDMA_FUNC_TRACE(); 1607 1608 if (!dpaa2_coherent_no_alloc_cache) { 1609 if (dpaa2_svr_family == SVR_LX2160A) { 1610 dpaa2_coherent_no_alloc_cache = 1611 DPAA2_LX2_COHERENT_NO_ALLOCATE_CACHE; 1612 dpaa2_coherent_alloc_cache = 1613 DPAA2_LX2_COHERENT_ALLOCATE_CACHE; 1614 } else { 1615 dpaa2_coherent_no_alloc_cache = 1616 DPAA2_COHERENT_NO_ALLOCATE_CACHE; 1617 dpaa2_coherent_alloc_cache = 1618 DPAA2_COHERENT_ALLOCATE_CACHE; 1619 } 1620 } 1621 1622 if (!s_proc_mc_reg.regs) 1623 s_proc_mc_reg.regs = dpaa2_get_mcp_ptr(MC_PORTAL_INDEX); 1624 1625 if (rte_eal_process_type() == RTE_PROC_SECONDARY) { 1626 DPAA2_QDMA_DEBUG("Attach DMA(%d) in the 2nd proess.", 1627 dpdmai_id); 1628 if (dpdmai_id != dpdmai_dev->dpdmai_id) { 1629 DPAA2_QDMA_ERR("Fatal: Attach DMA(%d) to DMA(%d)", 1630 dpdmai_id, dpdmai_dev->dpdmai_id); 1631 return -EINVAL; 1632 } 1633 if (!dpdmai_dev->qdma_dev) { 1634 DPAA2_QDMA_ERR("Fatal: DMA(%d) qdma_dev NOT allocated", 1635 dpdmai_id); 1636 return -ENOMEM; 1637 } 1638 if (dpdmai_dev->qdma_dev->num_vqs) { 1639 DPAA2_QDMA_WARN("DMA(%d) %d vqs were configured", 1640 dpdmai_id, dpdmai_dev->qdma_dev->num_vqs); 1641 } 1642 1643 return 0; 1644 } 1645 1646 /* Open DPDMAI device */ 1647 dpdmai_dev->dpdmai_id = dpdmai_id; 1648 1649 if (dpdmai_dev->qdma_dev) { 1650 rte_free(dpdmai_dev->qdma_dev); 1651 dpdmai_dev->qdma_dev = NULL; 1652 } 1653 dpdmai_dev->qdma_dev = rte_zmalloc(NULL, 1654 sizeof(struct qdma_device), RTE_CACHE_LINE_SIZE); 1655 if (!dpdmai_dev->qdma_dev) { 1656 DPAA2_QDMA_ERR("DMA(%d) alloc memory failed", 1657 dpdmai_id); 1658 return -ENOMEM; 1659 } 1660 ret = dpdmai_open(&s_proc_mc_reg, CMD_PRI_LOW, 1661 dpdmai_dev->dpdmai_id, &dpdmai_dev->token); 1662 if (ret) { 1663 DPAA2_QDMA_ERR("%s: dma(%d) open failed(%d)", 1664 __func__, dpdmai_dev->dpdmai_id, ret); 1665 return ret; 1666 } 1667 1668 /* Get DPDMAI attributes */ 1669 ret = dpdmai_get_attributes(&s_proc_mc_reg, CMD_PRI_LOW, 1670 dpdmai_dev->token, &attr); 1671 if (ret) { 1672 DPAA2_QDMA_ERR("%s: dma(%d) get attributes failed(%d)", 1673 __func__, dpdmai_dev->dpdmai_id, ret); 1674 err = dpdmai_close(&s_proc_mc_reg, CMD_PRI_LOW, 1675 dpdmai_dev->token); 1676 if (err) { 1677 DPAA2_QDMA_ERR("dpdmai(%d) close failed(%d)", 1678 dpdmai_dev->dpdmai_id, err); 1679 } 1680 return ret; 1681 } 1682 dpdmai_dev->num_queues = attr.num_of_queues; 1683 1684 DPAA2_QDMA_DEBUG("DMA(%d) is initialized.", dpdmai_id); 1685 1686 return 0; 1687 } 1688 1689 static int 1690 dpaa2_qdma_probe(struct rte_dpaa2_driver *dpaa2_drv, 1691 struct rte_dpaa2_device *dpaa2_dev) 1692 { 1693 struct rte_dma_dev *dmadev; 1694 int ret; 1695 1696 DPAA2_QDMA_FUNC_TRACE(); 1697 1698 RTE_SET_USED(dpaa2_drv); 1699 1700 dmadev = rte_dma_pmd_allocate(dpaa2_dev->device.name, 1701 rte_socket_id(), 1702 sizeof(struct dpaa2_dpdmai_dev)); 1703 if (!dmadev) { 1704 DPAA2_QDMA_ERR("Unable to allocate dmadevice"); 1705 return -EINVAL; 1706 } 1707 1708 dpaa2_dev->dmadev = dmadev; 1709 dmadev->dev_ops = &dpaa2_qdma_ops; 1710 dmadev->device = &dpaa2_dev->device; 1711 dmadev->fp_obj->dev_private = dmadev->data->dev_private; 1712 dmadev->fp_obj->copy = dpaa2_qdma_copy; 1713 dmadev->fp_obj->copy_sg = dpaa2_qdma_copy_sg; 1714 dmadev->fp_obj->submit = dpaa2_qdma_submit; 1715 dmadev->fp_obj->completed = dpaa2_qdma_dequeue; 1716 dmadev->fp_obj->burst_capacity = dpaa2_qdma_burst_capacity; 1717 1718 /* Invoke PMD device initialization function */ 1719 ret = dpaa2_dpdmai_dev_init(dmadev, dpaa2_dev->object_id); 1720 if (ret) { 1721 rte_dma_pmd_release(dpaa2_dev->device.name); 1722 return ret; 1723 } 1724 1725 dmadev->state = RTE_DMA_DEV_READY; 1726 return 0; 1727 } 1728 1729 static int 1730 dpaa2_qdma_remove(struct rte_dpaa2_device *dpaa2_dev) 1731 { 1732 struct rte_dma_dev *dmadev = dpaa2_dev->dmadev; 1733 int ret; 1734 1735 DPAA2_QDMA_FUNC_TRACE(); 1736 1737 dpaa2_dpdmai_dev_uninit(dmadev); 1738 1739 ret = rte_dma_pmd_release(dpaa2_dev->device.name); 1740 if (ret) 1741 DPAA2_QDMA_ERR("Device cleanup failed"); 1742 1743 return 0; 1744 } 1745 1746 static struct rte_dpaa2_driver rte_dpaa2_qdma_pmd; 1747 1748 static struct rte_dpaa2_driver rte_dpaa2_qdma_pmd = { 1749 .drv_flags = RTE_DPAA2_DRV_IOVA_AS_VA, 1750 .drv_type = DPAA2_QDMA, 1751 .probe = dpaa2_qdma_probe, 1752 .remove = dpaa2_qdma_remove, 1753 }; 1754 1755 RTE_PMD_REGISTER_DPAA2(dpaa2_qdma, rte_dpaa2_qdma_pmd); 1756 RTE_PMD_REGISTER_PARAM_STRING(dpaa2_qdma, 1757 DPAA2_QDMA_FLE_PRE_POPULATE "=<int>" 1758 DPAA2_QDMA_DESC_DEBUG"=<int>" 1759 DPAA2_QDMA_USING_SHORT_FD"=<int>"); 1760 RTE_LOG_REGISTER_DEFAULT(dpaa2_qdma_logtype, INFO); 1761