1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2021 HiSilicon Limited 3 * Copyright(c) 2021 Intel Corporation 4 * Copyright(c) 2021 Marvell International Ltd 5 * Copyright(c) 2021 SmartShare Systems 6 */ 7 8 #ifndef RTE_DMADEV_H 9 #define RTE_DMADEV_H 10 11 /** 12 * @file rte_dmadev.h 13 * 14 * DMA (Direct Memory Access) device API. 15 * 16 * The DMA framework is built on the following model: 17 * 18 * --------------- --------------- --------------- 19 * | virtual DMA | | virtual DMA | | virtual DMA | 20 * | channel | | channel | | channel | 21 * --------------- --------------- --------------- 22 * | | | 23 * ------------------ | 24 * | | 25 * ------------ ------------ 26 * | dmadev | | dmadev | 27 * ------------ ------------ 28 * | | 29 * ------------------ ------------------ 30 * | HW DMA channel | | HW DMA channel | 31 * ------------------ ------------------ 32 * | | 33 * -------------------------------- 34 * | 35 * --------------------- 36 * | HW DMA Controller | 37 * --------------------- 38 * 39 * The DMA controller could have multiple HW-DMA-channels (aka. HW-DMA-queues), 40 * each HW-DMA-channel should be represented by a dmadev. 41 * 42 * The dmadev could create multiple virtual DMA channels, each virtual DMA 43 * channel represents a different transfer context. The DMA operation request 44 * must be submitted to the virtual DMA channel. e.g. Application could create 45 * virtual DMA channel 0 for memory-to-memory transfer scenario, and create 46 * virtual DMA channel 1 for memory-to-device transfer scenario. 47 * 48 * This framework uses 'int16_t dev_id' as the device identifier of a dmadev, 49 * and 'uint16_t vchan' as the virtual DMA channel identifier in one dmadev. 50 * 51 * The functions exported by the dmadev API to setup a device designated by its 52 * device identifier must be invoked in the following order: 53 * - rte_dma_configure() 54 * - rte_dma_vchan_setup() 55 * - rte_dma_start() 56 * 57 * Then, the application can invoke dataplane functions to process jobs. 58 * 59 * If the application wants to change the configuration (i.e. invoke 60 * rte_dma_configure() or rte_dma_vchan_setup()), it must invoke 61 * rte_dma_stop() first to stop the device and then do the reconfiguration 62 * before invoking rte_dma_start() again. The dataplane functions should not 63 * be invoked when the device is stopped. 64 * 65 * Finally, an application can close a dmadev by invoking the rte_dma_close() 66 * function. 67 * 68 * The dataplane APIs include two parts: 69 * The first part is the submission of operation requests: 70 * - rte_dma_copy() 71 * - rte_dma_copy_sg() 72 * - rte_dma_fill() 73 * - rte_dma_submit() 74 * 75 * These APIs could work with different virtual DMA channels which have 76 * different contexts. 77 * 78 * The first three APIs are used to submit the operation request to the virtual 79 * DMA channel, if the submission is successful, a positive 80 * ring_idx <= UINT16_MAX is returned, otherwise a negative number is returned. 81 * 82 * The last API is used to issue doorbell to hardware, and also there are flags 83 * (@see RTE_DMA_OP_FLAG_SUBMIT) parameter of the first three APIs could do the 84 * same work. 85 * @note When enqueuing a set of jobs to the device, having a separate submit 86 * outside a loop makes for clearer code than having a check for the last 87 * iteration inside the loop to set a special submit flag. However, for cases 88 * where one item alone is to be submitted or there is a small set of jobs to 89 * be submitted sequentially, having a submit flag provides a lower-overhead 90 * way of doing the submission while still keeping the code clean. 91 * 92 * The second part is to obtain the result of requests: 93 * - rte_dma_completed() 94 * - return the number of operation requests completed successfully. 95 * - rte_dma_completed_status() 96 * - return the number of operation requests completed. 97 * 98 * @note If the dmadev works in silent mode (@see RTE_DMA_CAPA_SILENT), 99 * application does not invoke the above two completed APIs. 100 * 101 * About the ring_idx which enqueue APIs (e.g. rte_dma_copy(), rte_dma_fill()) 102 * return, the rules are as follows: 103 * - ring_idx for each virtual DMA channel are independent. 104 * - For a virtual DMA channel, the ring_idx is monotonically incremented, 105 * when it reach UINT16_MAX, it wraps back to zero. 106 * - This ring_idx can be used by applications to track per-operation 107 * metadata in an application-defined circular ring. 108 * - The initial ring_idx of a virtual DMA channel is zero, after the 109 * device is stopped, the ring_idx needs to be reset to zero. 110 * 111 * One example: 112 * - step-1: start one dmadev 113 * - step-2: enqueue a copy operation, the ring_idx return is 0 114 * - step-3: enqueue a copy operation again, the ring_idx return is 1 115 * - ... 116 * - step-101: stop the dmadev 117 * - step-102: start the dmadev 118 * - step-103: enqueue a copy operation, the ring_idx return is 0 119 * - ... 120 * - step-x+0: enqueue a fill operation, the ring_idx return is 65535 121 * - step-x+1: enqueue a copy operation, the ring_idx return is 0 122 * - ... 123 * 124 * The DMA operation address used in enqueue APIs (i.e. rte_dma_copy(), 125 * rte_dma_copy_sg(), rte_dma_fill()) is defined as rte_iova_t type. 126 * 127 * The dmadev supports two types of address: memory address and device address. 128 * 129 * - memory address: the source and destination address of the memory-to-memory 130 * transfer type, or the source address of the memory-to-device transfer type, 131 * or the destination address of the device-to-memory transfer type. 132 * @note If the device support SVA (@see RTE_DMA_CAPA_SVA), the memory address 133 * can be any VA address, otherwise it must be an IOVA address. 134 * 135 * - device address: the source and destination address of the device-to-device 136 * transfer type, or the source address of the device-to-memory transfer type, 137 * or the destination address of the memory-to-device transfer type. 138 * 139 * About MT-safe, all the functions of the dmadev API implemented by a PMD are 140 * lock-free functions which assume to not be invoked in parallel on different 141 * logical cores to work on the same target dmadev object. 142 * @note Different virtual DMA channels on the same dmadev *DO NOT* support 143 * parallel invocation because these virtual DMA channels share the same 144 * HW-DMA-channel. 145 */ 146 147 #include <stdint.h> 148 149 #include <rte_bitops.h> 150 #include <rte_common.h> 151 152 #ifdef __cplusplus 153 extern "C" { 154 #endif 155 156 /** Maximum number of devices if rte_dma_dev_max() is not called. */ 157 #define RTE_DMADEV_DEFAULT_MAX 64 158 159 /** 160 * Configure the maximum number of dmadevs. 161 * @note This function can be invoked before the primary process rte_eal_init() 162 * to change the maximum number of dmadevs. If not invoked, the maximum number 163 * of dmadevs is @see RTE_DMADEV_DEFAULT_MAX 164 * 165 * @param dev_max 166 * maximum number of dmadevs. 167 * 168 * @return 169 * 0 on success. Otherwise negative value is returned. 170 */ 171 int rte_dma_dev_max(size_t dev_max); 172 173 /** 174 * Get the device identifier for the named DMA device. 175 * 176 * @param name 177 * DMA device name. 178 * 179 * @return 180 * Returns DMA device identifier on success. 181 * - <0: Failure to find named DMA device. 182 */ 183 int rte_dma_get_dev_id_by_name(const char *name); 184 185 /** 186 * Check whether the dev_id is valid. 187 * 188 * @param dev_id 189 * DMA device index. 190 * 191 * @return 192 * - If the device index is valid (true) or not (false). 193 */ 194 bool rte_dma_is_valid(int16_t dev_id); 195 196 /** 197 * Get the total number of DMA devices that have been successfully 198 * initialised. 199 * 200 * @return 201 * The total number of usable DMA devices. 202 */ 203 uint16_t rte_dma_count_avail(void); 204 205 /** 206 * Iterates over valid dmadev instances. 207 * 208 * @param start_dev_id 209 * The id of the next possible dmadev. 210 * @return 211 * Next valid dmadev, UINT16_MAX if there is none. 212 */ 213 int16_t rte_dma_next_dev(int16_t start_dev_id); 214 215 /** Utility macro to iterate over all available dmadevs */ 216 #define RTE_DMA_FOREACH_DEV(p) \ 217 for (p = rte_dma_next_dev(0); \ 218 p != -1; \ 219 p = rte_dma_next_dev(p + 1)) 220 221 222 /**@{@name DMA capability 223 * @see struct rte_dma_info::dev_capa 224 */ 225 /** Support memory-to-memory transfer */ 226 #define RTE_DMA_CAPA_MEM_TO_MEM RTE_BIT64(0) 227 /** Support memory-to-device transfer. */ 228 #define RTE_DMA_CAPA_MEM_TO_DEV RTE_BIT64(1) 229 /** Support device-to-memory transfer. */ 230 #define RTE_DMA_CAPA_DEV_TO_MEM RTE_BIT64(2) 231 /** Support device-to-device transfer. */ 232 #define RTE_DMA_CAPA_DEV_TO_DEV RTE_BIT64(3) 233 /** Support SVA which could use VA as DMA address. 234 * If device support SVA then application could pass any VA address like memory 235 * from rte_malloc(), rte_memzone(), malloc, stack memory. 236 * If device don't support SVA, then application should pass IOVA address which 237 * from rte_malloc(), rte_memzone(). 238 */ 239 #define RTE_DMA_CAPA_SVA RTE_BIT64(4) 240 /** Support work in silent mode. 241 * In this mode, application don't required to invoke rte_dma_completed*() 242 * API. 243 * @see struct rte_dma_conf::silent_mode 244 */ 245 #define RTE_DMA_CAPA_SILENT RTE_BIT64(5) 246 /** Supports error handling 247 * 248 * With this bit set, invalid input addresses will be reported as operation failures 249 * to the user but other operations can continue. 250 * Without this bit set, invalid data is not handled by either HW or driver, so user 251 * must ensure that all memory addresses are valid and accessible by HW. 252 */ 253 #define RTE_DMA_CAPA_HANDLES_ERRORS RTE_BIT64(6) 254 /** Support auto free for source buffer once mem to dev transfer is completed. 255 * 256 * @note Even though the DMA driver has this capability, it may not support all 257 * mempool drivers. If the mempool is not supported by the DMA driver, 258 * rte_dma_vchan_setup() will fail. 259 */ 260 #define RTE_DMA_CAPA_M2D_AUTO_FREE RTE_BIT64(7) 261 262 /** Support copy operation. 263 * This capability start with index of 32, so that it could leave gap between 264 * normal capability and ops capability. 265 */ 266 #define RTE_DMA_CAPA_OPS_COPY RTE_BIT64(32) 267 /** Support scatter-gather list copy operation. */ 268 #define RTE_DMA_CAPA_OPS_COPY_SG RTE_BIT64(33) 269 /** Support fill operation. */ 270 #define RTE_DMA_CAPA_OPS_FILL RTE_BIT64(34) 271 /**@}*/ 272 273 /** 274 * A structure used to retrieve the information of a DMA device. 275 * 276 * @see rte_dma_info_get 277 */ 278 struct rte_dma_info { 279 const char *dev_name; /**< Unique device name. */ 280 /** Device capabilities (RTE_DMA_CAPA_*). */ 281 uint64_t dev_capa; 282 /** Maximum number of virtual DMA channels supported. */ 283 uint16_t max_vchans; 284 /** Maximum allowed number of virtual DMA channel descriptors. */ 285 uint16_t max_desc; 286 /** Minimum allowed number of virtual DMA channel descriptors. */ 287 uint16_t min_desc; 288 /** Maximum number of source or destination scatter-gather entry 289 * supported. 290 * If the device does not support COPY_SG capability, this value can be 291 * zero. 292 * If the device supports COPY_SG capability, then rte_dma_copy_sg() 293 * parameter nb_src/nb_dst should not exceed this value. 294 */ 295 uint16_t max_sges; 296 /** NUMA node connection, -1 if unknown. */ 297 int16_t numa_node; 298 /** Number of virtual DMA channel configured. */ 299 uint16_t nb_vchans; 300 }; 301 302 /** 303 * Retrieve information of a DMA device. 304 * 305 * @param dev_id 306 * The identifier of the device. 307 * @param[out] dev_info 308 * A pointer to a structure of type *rte_dma_info* to be filled with the 309 * information of the device. 310 * 311 * @return 312 * 0 on success. Otherwise negative value is returned. 313 */ 314 int rte_dma_info_get(int16_t dev_id, struct rte_dma_info *dev_info); 315 316 /** 317 * A structure used to configure a DMA device. 318 * 319 * @see rte_dma_configure 320 */ 321 struct rte_dma_conf { 322 /** The number of virtual DMA channels to set up for the DMA device. 323 * This value cannot be greater than the field 'max_vchans' of struct 324 * rte_dma_info which get from rte_dma_info_get(). 325 */ 326 uint16_t nb_vchans; 327 /** Indicates whether to enable silent mode. 328 * false-default mode, true-silent mode. 329 * This value can be set to true only when the SILENT capability is 330 * supported. 331 * 332 * @see RTE_DMA_CAPA_SILENT 333 */ 334 bool enable_silent; 335 }; 336 337 /** 338 * Configure a DMA device. 339 * 340 * This function must be invoked first before any other function in the 341 * API. This function can also be re-invoked when a device is in the 342 * stopped state. 343 * 344 * @param dev_id 345 * The identifier of the device to configure. 346 * @param dev_conf 347 * The DMA device configuration structure encapsulated into rte_dma_conf 348 * object. 349 * 350 * @return 351 * 0 on success. Otherwise negative value is returned. 352 */ 353 int rte_dma_configure(int16_t dev_id, const struct rte_dma_conf *dev_conf); 354 355 /** 356 * Start a DMA device. 357 * 358 * The device start step is the last one and consists of setting the DMA 359 * to start accepting jobs. 360 * 361 * @param dev_id 362 * The identifier of the device. 363 * 364 * @return 365 * 0 on success. Otherwise negative value is returned. 366 */ 367 int rte_dma_start(int16_t dev_id); 368 369 /** 370 * Stop a DMA device. 371 * 372 * The device can be restarted with a call to rte_dma_start(). 373 * 374 * @param dev_id 375 * The identifier of the device. 376 * 377 * @return 378 * 0 on success. Otherwise negative value is returned. 379 */ 380 int rte_dma_stop(int16_t dev_id); 381 382 /** 383 * Close a DMA device. 384 * 385 * The device cannot be restarted after this call. 386 * 387 * @param dev_id 388 * The identifier of the device. 389 * 390 * @return 391 * 0 on success. Otherwise negative value is returned. 392 */ 393 int rte_dma_close(int16_t dev_id); 394 395 /** 396 * DMA transfer direction defines. 397 * 398 * @see struct rte_dma_vchan_conf::direction 399 */ 400 enum rte_dma_direction { 401 /** DMA transfer direction - from memory to memory. 402 * 403 * @see struct rte_dma_vchan_conf::direction 404 */ 405 RTE_DMA_DIR_MEM_TO_MEM, 406 /** DMA transfer direction - from memory to device. 407 * In a typical scenario, the SoCs are installed on host servers as 408 * iNICs through the PCIe interface. In this case, the SoCs works in 409 * EP(endpoint) mode, it could initiate a DMA move request from memory 410 * (which is SoCs memory) to device (which is host memory). 411 * 412 * @see struct rte_dma_vchan_conf::direction 413 */ 414 RTE_DMA_DIR_MEM_TO_DEV, 415 /** DMA transfer direction - from device to memory. 416 * In a typical scenario, the SoCs are installed on host servers as 417 * iNICs through the PCIe interface. In this case, the SoCs works in 418 * EP(endpoint) mode, it could initiate a DMA move request from device 419 * (which is host memory) to memory (which is SoCs memory). 420 * 421 * @see struct rte_dma_vchan_conf::direction 422 */ 423 RTE_DMA_DIR_DEV_TO_MEM, 424 /** DMA transfer direction - from device to device. 425 * In a typical scenario, the SoCs are installed on host servers as 426 * iNICs through the PCIe interface. In this case, the SoCs works in 427 * EP(endpoint) mode, it could initiate a DMA move request from device 428 * (which is host memory) to the device (which is another host memory). 429 * 430 * @see struct rte_dma_vchan_conf::direction 431 */ 432 RTE_DMA_DIR_DEV_TO_DEV, 433 }; 434 435 /** 436 * DMA access port type defines. 437 * 438 * @see struct rte_dma_port_param::port_type 439 */ 440 enum rte_dma_port_type { 441 RTE_DMA_PORT_NONE, 442 RTE_DMA_PORT_PCIE, /**< The DMA access port is PCIe. */ 443 }; 444 445 /** 446 * A structure used to descript DMA access port parameters. 447 * 448 * @see struct rte_dma_vchan_conf::src_port 449 * @see struct rte_dma_vchan_conf::dst_port 450 */ 451 struct rte_dma_port_param { 452 /** The device access port type. 453 * 454 * @see enum rte_dma_port_type 455 */ 456 enum rte_dma_port_type port_type; 457 union { 458 /** PCIe access port parameters. 459 * 460 * The following model shows SoC's PCIe module connects to 461 * multiple PCIe hosts and multiple endpoints. The PCIe module 462 * has an integrated DMA controller. 463 * 464 * If the DMA wants to access the memory of host A, it can be 465 * initiated by PF1 in core0, or by VF0 of PF0 in core0. 466 * 467 * \code{.unparsed} 468 * System Bus 469 * | ----------PCIe module---------- 470 * | Bus 471 * | Interface 472 * | ----- ------------------ 473 * | | | | PCIe Core0 | 474 * | | | | | ----------- 475 * | | | | PF-0 -- VF-0 | | Host A | 476 * | | |--------| |- VF-1 |--------| Root | 477 * | | | | PF-1 | | Complex | 478 * | | | | PF-2 | ----------- 479 * | | | ------------------ 480 * | | | 481 * | | | ------------------ 482 * | | | | PCIe Core1 | 483 * | | | | | ----------- 484 * | | | | PF-0 -- VF-0 | | Host B | 485 * |-----| |--------| PF-1 -- VF-0 |--------| Root | 486 * | | | | |- VF-1 | | Complex | 487 * | | | | PF-2 | ----------- 488 * | | | ------------------ 489 * | | | 490 * | | | ------------------ 491 * | |DMA| | | ------ 492 * | | | | |--------| EP | 493 * | | |--------| PCIe Core2 | ------ 494 * | | | | | ------ 495 * | | | | |--------| EP | 496 * | | | | | ------ 497 * | ----- ------------------ 498 * 499 * \endcode 500 * 501 * @note If some fields can not be supported by the 502 * hardware/driver, then the driver ignores those fields. 503 * Please check driver-specific documentation for limitations 504 * and capabilities. 505 */ 506 __extension__ 507 struct { 508 uint64_t coreid : 4; /**< PCIe core id used. */ 509 uint64_t pfid : 8; /**< PF id used. */ 510 uint64_t vfen : 1; /**< VF enable bit. */ 511 uint64_t vfid : 16; /**< VF id used. */ 512 /** The pasid filed in TLP packet. */ 513 uint64_t pasid : 20; 514 /** The attributes filed in TLP packet. */ 515 uint64_t attr : 3; 516 /** The processing hint filed in TLP packet. */ 517 uint64_t ph : 2; 518 /** The steering tag filed in TLP packet. */ 519 uint64_t st : 16; 520 } pcie; 521 }; 522 uint64_t reserved[2]; /**< Reserved for future fields. */ 523 }; 524 525 /** 526 * A structure used for offload auto free params. 527 */ 528 struct rte_dma_auto_free_param { 529 union { 530 struct { 531 /** 532 * Mempool from which buffer is allocated. Mempool info 533 * is used for freeing buffer by hardware. 534 * 535 * @note If the mempool is not supported by the DMA device, 536 * rte_dma_vchan_setup() will fail. 537 */ 538 struct rte_mempool *pool; 539 } m2d; 540 }; 541 /** Reserved for future fields. */ 542 uint64_t reserved[2]; 543 }; 544 545 /** 546 * A structure used to configure a virtual DMA channel. 547 * 548 * @see rte_dma_vchan_setup 549 */ 550 struct rte_dma_vchan_conf { 551 /** Transfer direction 552 * 553 * @see enum rte_dma_direction 554 */ 555 enum rte_dma_direction direction; 556 /** Number of descriptor for the virtual DMA channel */ 557 uint16_t nb_desc; 558 /** 1) Used to describes the device access port parameter in the 559 * device-to-memory transfer scenario. 560 * 2) Used to describes the source device access port parameter in the 561 * device-to-device transfer scenario. 562 * 563 * @see struct rte_dma_port_param 564 */ 565 struct rte_dma_port_param src_port; 566 /** 1) Used to describes the device access port parameter in the 567 * memory-to-device transfer scenario. 568 * 2) Used to describes the destination device access port parameter in 569 * the device-to-device transfer scenario. 570 * 571 * @see struct rte_dma_port_param 572 */ 573 struct rte_dma_port_param dst_port; 574 /** Buffer params to auto free buffer by hardware. To free the buffer 575 * by hardware, RTE_DMA_OP_FLAG_AUTO_FREE must be set while calling 576 * rte_dma_copy and rte_dma_copy_sg(). 577 * 578 * @see RTE_DMA_OP_FLAG_AUTO_FREE 579 * @see struct rte_dma_auto_free_param 580 */ 581 struct rte_dma_auto_free_param auto_free; 582 }; 583 584 /** 585 * Allocate and set up a virtual DMA channel. 586 * 587 * @param dev_id 588 * The identifier of the device. 589 * @param vchan 590 * The identifier of virtual DMA channel. The value must be in the range 591 * [0, nb_vchans - 1] previously supplied to rte_dma_configure(). 592 * @param conf 593 * The virtual DMA channel configuration structure encapsulated into 594 * rte_dma_vchan_conf object. 595 * 596 * @return 597 * 0 on success. Otherwise negative value is returned. 598 */ 599 int rte_dma_vchan_setup(int16_t dev_id, uint16_t vchan, 600 const struct rte_dma_vchan_conf *conf); 601 602 /** 603 * A structure used to retrieve statistics. 604 * 605 * @see rte_dma_stats_get 606 */ 607 struct rte_dma_stats { 608 /** Count of operations which were submitted to hardware. */ 609 uint64_t submitted; 610 /** Count of operations which were completed, including successful and 611 * failed completions. 612 */ 613 uint64_t completed; 614 /** Count of operations which failed to complete. */ 615 uint64_t errors; 616 }; 617 618 /** 619 * Special ID, which is used to represent all virtual DMA channels. 620 * 621 * @see rte_dma_stats_get 622 * @see rte_dma_stats_reset 623 */ 624 #define RTE_DMA_ALL_VCHAN 0xFFFFu 625 626 /** 627 * Retrieve basic statistics of a or all virtual DMA channel(s). 628 * 629 * @param dev_id 630 * The identifier of the device. 631 * @param vchan 632 * The identifier of virtual DMA channel. 633 * If equal RTE_DMA_ALL_VCHAN means all channels. 634 * @param[out] stats 635 * The basic statistics structure encapsulated into rte_dma_stats 636 * object. 637 * 638 * @return 639 * 0 on success. Otherwise negative value is returned. 640 */ 641 int rte_dma_stats_get(int16_t dev_id, uint16_t vchan, 642 struct rte_dma_stats *stats); 643 644 /** 645 * Reset basic statistics of a or all virtual DMA channel(s). 646 * 647 * @param dev_id 648 * The identifier of the device. 649 * @param vchan 650 * The identifier of virtual DMA channel. 651 * If equal RTE_DMA_ALL_VCHAN means all channels. 652 * 653 * @return 654 * 0 on success. Otherwise negative value is returned. 655 */ 656 int rte_dma_stats_reset(int16_t dev_id, uint16_t vchan); 657 658 /** 659 * device vchannel status 660 * 661 * Enum with the options for the channel status, either idle, active or halted due to error 662 * @see rte_dma_vchan_status 663 */ 664 enum rte_dma_vchan_status { 665 RTE_DMA_VCHAN_IDLE, /**< not processing, awaiting ops */ 666 RTE_DMA_VCHAN_ACTIVE, /**< currently processing jobs */ 667 RTE_DMA_VCHAN_HALTED_ERROR, /**< not processing due to error, cannot accept new ops */ 668 }; 669 670 /** 671 * Determine if all jobs have completed on a device channel. 672 * This function is primarily designed for testing use, as it allows a process to check if 673 * all jobs are completed, without actually gathering completions from those jobs. 674 * 675 * @param dev_id 676 * The identifier of the device. 677 * @param vchan 678 * The identifier of virtual DMA channel. 679 * @param[out] status 680 * The vchan status 681 * @return 682 * 0 - call completed successfully 683 * < 0 - error code indicating there was a problem calling the API 684 */ 685 int 686 rte_dma_vchan_status(int16_t dev_id, uint16_t vchan, enum rte_dma_vchan_status *status); 687 688 /** 689 * Dump DMA device info. 690 * 691 * @param dev_id 692 * The identifier of the device. 693 * @param f 694 * The file to write the output to. 695 * 696 * @return 697 * 0 on success. Otherwise negative value is returned. 698 */ 699 int rte_dma_dump(int16_t dev_id, FILE *f); 700 701 /** 702 * DMA transfer result status code defines. 703 * 704 * @see rte_dma_completed_status 705 */ 706 enum rte_dma_status_code { 707 /** The operation completed successfully. */ 708 RTE_DMA_STATUS_SUCCESSFUL, 709 /** The operation failed to complete due abort by user. 710 * This is mainly used when processing dev_stop, user could modify the 711 * descriptors (e.g. change one bit to tell hardware abort this job), 712 * it allows outstanding requests to be complete as much as possible, 713 * so reduce the time to stop the device. 714 */ 715 RTE_DMA_STATUS_USER_ABORT, 716 /** The operation failed to complete due to following scenarios: 717 * The jobs in a particular batch are not attempted because they 718 * appeared after a fence where a previous job failed. In some HW 719 * implementation it's possible for jobs from later batches would be 720 * completed, though, so report the status from the not attempted jobs 721 * before reporting those newer completed jobs. 722 */ 723 RTE_DMA_STATUS_NOT_ATTEMPTED, 724 /** The operation failed to complete due invalid source address. */ 725 RTE_DMA_STATUS_INVALID_SRC_ADDR, 726 /** The operation failed to complete due invalid destination address. */ 727 RTE_DMA_STATUS_INVALID_DST_ADDR, 728 /** The operation failed to complete due invalid source or destination 729 * address, cover the case that only knows the address error, but not 730 * sure which address error. 731 */ 732 RTE_DMA_STATUS_INVALID_ADDR, 733 /** The operation failed to complete due invalid length. */ 734 RTE_DMA_STATUS_INVALID_LENGTH, 735 /** The operation failed to complete due invalid opcode. 736 * The DMA descriptor could have multiple format, which are 737 * distinguished by the opcode field. 738 */ 739 RTE_DMA_STATUS_INVALID_OPCODE, 740 /** The operation failed to complete due bus read error. */ 741 RTE_DMA_STATUS_BUS_READ_ERROR, 742 /** The operation failed to complete due bus write error. */ 743 RTE_DMA_STATUS_BUS_WRITE_ERROR, 744 /** The operation failed to complete due bus error, cover the case that 745 * only knows the bus error, but not sure which direction error. 746 */ 747 RTE_DMA_STATUS_BUS_ERROR, 748 /** The operation failed to complete due data poison. */ 749 RTE_DMA_STATUS_DATA_POISION, 750 /** The operation failed to complete due descriptor read error. */ 751 RTE_DMA_STATUS_DESCRIPTOR_READ_ERROR, 752 /** The operation failed to complete due device link error. 753 * Used to indicates that the link error in the memory-to-device/ 754 * device-to-memory/device-to-device transfer scenario. 755 */ 756 RTE_DMA_STATUS_DEV_LINK_ERROR, 757 /** The operation failed to complete due lookup page fault. */ 758 RTE_DMA_STATUS_PAGE_FAULT, 759 /** The operation failed to complete due unknown reason. 760 * The initial value is 256, which reserves space for future errors. 761 */ 762 RTE_DMA_STATUS_ERROR_UNKNOWN = 0x100, 763 }; 764 765 /** 766 * A structure used to hold scatter-gather DMA operation request entry. 767 * 768 * @see rte_dma_copy_sg 769 */ 770 struct rte_dma_sge { 771 rte_iova_t addr; /**< The DMA operation address. */ 772 uint32_t length; /**< The DMA operation length. */ 773 }; 774 775 #include "rte_dmadev_core.h" 776 #include "rte_dmadev_trace_fp.h" 777 778 /**@{@name DMA operation flag 779 * @see rte_dma_copy() 780 * @see rte_dma_copy_sg() 781 * @see rte_dma_fill() 782 */ 783 /** Fence flag. 784 * It means the operation with this flag must be processed only after all 785 * previous operations are completed. 786 * If the specify DMA HW works in-order (it means it has default fence between 787 * operations), this flag could be NOP. 788 */ 789 #define RTE_DMA_OP_FLAG_FENCE RTE_BIT64(0) 790 /** Submit flag. 791 * It means the operation with this flag must issue doorbell to hardware after 792 * enqueued jobs. 793 */ 794 #define RTE_DMA_OP_FLAG_SUBMIT RTE_BIT64(1) 795 /** Write data to low level cache hint. 796 * Used for performance optimization, this is just a hint, and there is no 797 * capability bit for this, driver should not return error if this flag was set. 798 */ 799 #define RTE_DMA_OP_FLAG_LLC RTE_BIT64(2) 800 /** Auto free buffer flag. 801 * Operation with this flag must issue command to hardware to free the DMA 802 * buffer after DMA transfer is completed. 803 * 804 * @see struct rte_dma_vchan_conf::auto_free 805 */ 806 #define RTE_DMA_OP_FLAG_AUTO_FREE RTE_BIT64(3) 807 /**@}*/ 808 809 /** 810 * Enqueue a copy operation onto the virtual DMA channel. 811 * 812 * This queues up a copy operation to be performed by hardware, if the 'flags' 813 * parameter contains RTE_DMA_OP_FLAG_SUBMIT then trigger doorbell to begin 814 * this operation, otherwise do not trigger doorbell. 815 * 816 * @param dev_id 817 * The identifier of the device. 818 * @param vchan 819 * The identifier of virtual DMA channel. 820 * @param src 821 * The address of the source buffer. 822 * @param dst 823 * The address of the destination buffer. 824 * @param length 825 * The length of the data to be copied. 826 * @param flags 827 * An flags for this operation. 828 * @see RTE_DMA_OP_FLAG_* 829 * 830 * @return 831 * - 0..UINT16_MAX: index of enqueued job. 832 * - -ENOSPC: if no space left to enqueue. 833 * - other values < 0 on failure. 834 */ 835 static inline int 836 rte_dma_copy(int16_t dev_id, uint16_t vchan, rte_iova_t src, rte_iova_t dst, 837 uint32_t length, uint64_t flags) 838 { 839 struct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id]; 840 int ret; 841 842 #ifdef RTE_DMADEV_DEBUG 843 if (!rte_dma_is_valid(dev_id) || length == 0) 844 return -EINVAL; 845 if (*obj->copy == NULL) 846 return -ENOTSUP; 847 #endif 848 849 ret = (*obj->copy)(obj->dev_private, vchan, src, dst, length, flags); 850 rte_dma_trace_copy(dev_id, vchan, src, dst, length, flags, ret); 851 852 return ret; 853 } 854 855 /** 856 * Enqueue a scatter-gather list copy operation onto the virtual DMA channel. 857 * 858 * This queues up a scatter-gather list copy operation to be performed by 859 * hardware, if the 'flags' parameter contains RTE_DMA_OP_FLAG_SUBMIT then 860 * trigger doorbell to begin this operation, otherwise do not trigger doorbell. 861 * 862 * @param dev_id 863 * The identifier of the device. 864 * @param vchan 865 * The identifier of virtual DMA channel. 866 * @param src 867 * The pointer of source scatter-gather entry array. 868 * @param dst 869 * The pointer of destination scatter-gather entry array. 870 * @param nb_src 871 * The number of source scatter-gather entry. 872 * @see struct rte_dma_info::max_sges 873 * @param nb_dst 874 * The number of destination scatter-gather entry. 875 * @see struct rte_dma_info::max_sges 876 * @param flags 877 * An flags for this operation. 878 * @see RTE_DMA_OP_FLAG_* 879 * 880 * @return 881 * - 0..UINT16_MAX: index of enqueued job. 882 * - -ENOSPC: if no space left to enqueue. 883 * - other values < 0 on failure. 884 */ 885 static inline int 886 rte_dma_copy_sg(int16_t dev_id, uint16_t vchan, struct rte_dma_sge *src, 887 struct rte_dma_sge *dst, uint16_t nb_src, uint16_t nb_dst, 888 uint64_t flags) 889 { 890 struct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id]; 891 int ret; 892 893 #ifdef RTE_DMADEV_DEBUG 894 if (!rte_dma_is_valid(dev_id) || src == NULL || dst == NULL || 895 nb_src == 0 || nb_dst == 0) 896 return -EINVAL; 897 if (*obj->copy_sg == NULL) 898 return -ENOTSUP; 899 #endif 900 901 ret = (*obj->copy_sg)(obj->dev_private, vchan, src, dst, nb_src, 902 nb_dst, flags); 903 rte_dma_trace_copy_sg(dev_id, vchan, src, dst, nb_src, nb_dst, flags, 904 ret); 905 906 return ret; 907 } 908 909 /** 910 * Enqueue a fill operation onto the virtual DMA channel. 911 * 912 * This queues up a fill operation to be performed by hardware, if the 'flags' 913 * parameter contains RTE_DMA_OP_FLAG_SUBMIT then trigger doorbell to begin 914 * this operation, otherwise do not trigger doorbell. 915 * 916 * @param dev_id 917 * The identifier of the device. 918 * @param vchan 919 * The identifier of virtual DMA channel. 920 * @param pattern 921 * The pattern to populate the destination buffer with. 922 * @param dst 923 * The address of the destination buffer. 924 * @param length 925 * The length of the destination buffer. 926 * @param flags 927 * An flags for this operation. 928 * @see RTE_DMA_OP_FLAG_* 929 * 930 * @return 931 * - 0..UINT16_MAX: index of enqueued job. 932 * - -ENOSPC: if no space left to enqueue. 933 * - other values < 0 on failure. 934 */ 935 static inline int 936 rte_dma_fill(int16_t dev_id, uint16_t vchan, uint64_t pattern, 937 rte_iova_t dst, uint32_t length, uint64_t flags) 938 { 939 struct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id]; 940 int ret; 941 942 #ifdef RTE_DMADEV_DEBUG 943 if (!rte_dma_is_valid(dev_id) || length == 0) 944 return -EINVAL; 945 if (*obj->fill == NULL) 946 return -ENOTSUP; 947 #endif 948 949 ret = (*obj->fill)(obj->dev_private, vchan, pattern, dst, length, 950 flags); 951 rte_dma_trace_fill(dev_id, vchan, pattern, dst, length, flags, ret); 952 953 return ret; 954 } 955 956 /** 957 * Trigger hardware to begin performing enqueued operations. 958 * 959 * Writes the "doorbell" to the hardware to trigger it 960 * to begin the operations previously enqueued by rte_dma_copy/fill(). 961 * 962 * @param dev_id 963 * The identifier of the device. 964 * @param vchan 965 * The identifier of virtual DMA channel. 966 * 967 * @return 968 * 0 on success. Otherwise negative value is returned. 969 */ 970 static inline int 971 rte_dma_submit(int16_t dev_id, uint16_t vchan) 972 { 973 struct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id]; 974 int ret; 975 976 #ifdef RTE_DMADEV_DEBUG 977 if (!rte_dma_is_valid(dev_id)) 978 return -EINVAL; 979 if (*obj->submit == NULL) 980 return -ENOTSUP; 981 #endif 982 983 ret = (*obj->submit)(obj->dev_private, vchan); 984 rte_dma_trace_submit(dev_id, vchan, ret); 985 986 return ret; 987 } 988 989 /** 990 * Return the number of operations that have been successfully completed. 991 * Once an operation has been reported as completed, the results of that 992 * operation will be visible to all cores on the system. 993 * 994 * @param dev_id 995 * The identifier of the device. 996 * @param vchan 997 * The identifier of virtual DMA channel. 998 * @param nb_cpls 999 * The maximum number of completed operations that can be processed. 1000 * @param[out] last_idx 1001 * The last completed operation's ring_idx. 1002 * If not required, NULL can be passed in. 1003 * @param[out] has_error 1004 * Indicates if there are transfer error. 1005 * If not required, NULL can be passed in. 1006 * 1007 * @return 1008 * The number of operations that successfully completed. This return value 1009 * must be less than or equal to the value of nb_cpls. 1010 */ 1011 static inline uint16_t 1012 rte_dma_completed(int16_t dev_id, uint16_t vchan, const uint16_t nb_cpls, 1013 uint16_t *last_idx, bool *has_error) 1014 { 1015 struct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id]; 1016 uint16_t idx, ret; 1017 bool err; 1018 1019 #ifdef RTE_DMADEV_DEBUG 1020 if (!rte_dma_is_valid(dev_id) || nb_cpls == 0) 1021 return 0; 1022 if (*obj->completed == NULL) 1023 return 0; 1024 #endif 1025 1026 /* Ensure the pointer values are non-null to simplify drivers. 1027 * In most cases these should be compile time evaluated, since this is 1028 * an inline function. 1029 * - If NULL is explicitly passed as parameter, then compiler knows the 1030 * value is NULL 1031 * - If address of local variable is passed as parameter, then compiler 1032 * can know it's non-NULL. 1033 */ 1034 if (last_idx == NULL) 1035 last_idx = &idx; 1036 if (has_error == NULL) 1037 has_error = &err; 1038 1039 *has_error = false; 1040 ret = (*obj->completed)(obj->dev_private, vchan, nb_cpls, last_idx, 1041 has_error); 1042 rte_dma_trace_completed(dev_id, vchan, nb_cpls, last_idx, has_error, 1043 ret); 1044 1045 return ret; 1046 } 1047 1048 /** 1049 * Return the number of operations that have been completed, and the operations 1050 * result may succeed or fail. 1051 * Once an operation has been reported as completed successfully, the results of that 1052 * operation will be visible to all cores on the system. 1053 * 1054 * @param dev_id 1055 * The identifier of the device. 1056 * @param vchan 1057 * The identifier of virtual DMA channel. 1058 * @param nb_cpls 1059 * Indicates the size of status array. 1060 * @param[out] last_idx 1061 * The last completed operation's ring_idx. 1062 * If not required, NULL can be passed in. 1063 * @param[out] status 1064 * This is a pointer to an array of length 'nb_cpls' that holds the completion 1065 * status code of each operation. 1066 * @see enum rte_dma_status_code 1067 * 1068 * @return 1069 * The number of operations that completed. This return value must be less 1070 * than or equal to the value of nb_cpls. 1071 * If this number is greater than zero (assuming n), then n values in the 1072 * status array are also set. 1073 */ 1074 static inline uint16_t 1075 rte_dma_completed_status(int16_t dev_id, uint16_t vchan, 1076 const uint16_t nb_cpls, uint16_t *last_idx, 1077 enum rte_dma_status_code *status) 1078 { 1079 struct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id]; 1080 uint16_t idx, ret; 1081 1082 #ifdef RTE_DMADEV_DEBUG 1083 if (!rte_dma_is_valid(dev_id) || nb_cpls == 0 || status == NULL) 1084 return 0; 1085 if (*obj->completed_status == NULL) 1086 return 0; 1087 #endif 1088 1089 if (last_idx == NULL) 1090 last_idx = &idx; 1091 1092 ret = (*obj->completed_status)(obj->dev_private, vchan, nb_cpls, 1093 last_idx, status); 1094 rte_dma_trace_completed_status(dev_id, vchan, nb_cpls, last_idx, status, 1095 ret); 1096 1097 return ret; 1098 } 1099 1100 /** 1101 * Check remaining capacity in descriptor ring for the current burst. 1102 * 1103 * @param dev_id 1104 * The identifier of the device. 1105 * @param vchan 1106 * The identifier of virtual DMA channel. 1107 * 1108 * @return 1109 * - Remaining space in the descriptor ring for the current burst. 1110 * - 0 on error 1111 */ 1112 static inline uint16_t 1113 rte_dma_burst_capacity(int16_t dev_id, uint16_t vchan) 1114 { 1115 struct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id]; 1116 uint16_t ret; 1117 1118 #ifdef RTE_DMADEV_DEBUG 1119 if (!rte_dma_is_valid(dev_id)) 1120 return 0; 1121 if (*obj->burst_capacity == NULL) 1122 return 0; 1123 #endif 1124 ret = (*obj->burst_capacity)(obj->dev_private, vchan); 1125 rte_dma_trace_burst_capacity(dev_id, vchan, ret); 1126 1127 return ret; 1128 } 1129 1130 #ifdef __cplusplus 1131 } 1132 #endif 1133 1134 #endif /* RTE_DMADEV_H */ 1135