1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2021 HiSilicon Limited 3 * Copyright(c) 2021 Intel Corporation 4 * Copyright(c) 2021 Marvell International Ltd 5 * Copyright(c) 2021 SmartShare Systems 6 */ 7 8 #ifndef RTE_DMADEV_H 9 #define RTE_DMADEV_H 10 11 /** 12 * @file rte_dmadev.h 13 * 14 * DMA (Direct Memory Access) device API. 15 * 16 * The DMA framework is built on the following model: 17 * 18 * --------------- --------------- --------------- 19 * | virtual DMA | | virtual DMA | | virtual DMA | 20 * | channel | | channel | | channel | 21 * --------------- --------------- --------------- 22 * | | | 23 * ------------------ | 24 * | | 25 * ------------ ------------ 26 * | dmadev | | dmadev | 27 * ------------ ------------ 28 * | | 29 * ------------------ ------------------ 30 * | HW DMA channel | | HW DMA channel | 31 * ------------------ ------------------ 32 * | | 33 * -------------------------------- 34 * | 35 * --------------------- 36 * | HW DMA Controller | 37 * --------------------- 38 * 39 * The DMA controller could have multiple HW-DMA-channels (aka. HW-DMA-queues), 40 * each HW-DMA-channel should be represented by a dmadev. 41 * 42 * The dmadev could create multiple virtual DMA channels, each virtual DMA 43 * channel represents a different transfer context. The DMA operation request 44 * must be submitted to the virtual DMA channel. e.g. Application could create 45 * virtual DMA channel 0 for memory-to-memory transfer scenario, and create 46 * virtual DMA channel 1 for memory-to-device transfer scenario. 47 * 48 * This framework uses 'int16_t dev_id' as the device identifier of a dmadev, 49 * and 'uint16_t vchan' as the virtual DMA channel identifier in one dmadev. 50 * 51 * The functions exported by the dmadev API to setup a device designated by its 52 * device identifier must be invoked in the following order: 53 * - rte_dma_configure() 54 * - rte_dma_vchan_setup() 55 * - rte_dma_start() 56 * 57 * Then, the application can invoke dataplane functions to process jobs. 58 * 59 * If the application wants to change the configuration (i.e. invoke 60 * rte_dma_configure() or rte_dma_vchan_setup()), it must invoke 61 * rte_dma_stop() first to stop the device and then do the reconfiguration 62 * before invoking rte_dma_start() again. The dataplane functions should not 63 * be invoked when the device is stopped. 64 * 65 * Finally, an application can close a dmadev by invoking the rte_dma_close() 66 * function. 67 * 68 * The dataplane APIs include two parts: 69 * The first part is the submission of operation requests: 70 * - rte_dma_copy() 71 * - rte_dma_copy_sg() 72 * - rte_dma_fill() 73 * - rte_dma_submit() 74 * 75 * These APIs could work with different virtual DMA channels which have 76 * different contexts. 77 * 78 * The first three APIs are used to submit the operation request to the virtual 79 * DMA channel, if the submission is successful, a positive 80 * ring_idx <= UINT16_MAX is returned, otherwise a negative number is returned. 81 * 82 * The last API is used to issue doorbell to hardware, and also there are flags 83 * (@see RTE_DMA_OP_FLAG_SUBMIT) parameter of the first three APIs could do the 84 * same work. 85 * @note When enqueuing a set of jobs to the device, having a separate submit 86 * outside a loop makes for clearer code than having a check for the last 87 * iteration inside the loop to set a special submit flag. However, for cases 88 * where one item alone is to be submitted or there is a small set of jobs to 89 * be submitted sequentially, having a submit flag provides a lower-overhead 90 * way of doing the submission while still keeping the code clean. 91 * 92 * The second part is to obtain the result of requests: 93 * - rte_dma_completed() 94 * - return the number of operation requests completed successfully. 95 * - rte_dma_completed_status() 96 * - return the number of operation requests completed. 97 * 98 * @note If the dmadev works in silent mode (@see RTE_DMA_CAPA_SILENT), 99 * application does not invoke the above two completed APIs. 100 * 101 * About the ring_idx which enqueue APIs (e.g. rte_dma_copy(), rte_dma_fill()) 102 * return, the rules are as follows: 103 * - ring_idx for each virtual DMA channel are independent. 104 * - For a virtual DMA channel, the ring_idx is monotonically incremented, 105 * when it reach UINT16_MAX, it wraps back to zero. 106 * - This ring_idx can be used by applications to track per-operation 107 * metadata in an application-defined circular ring. 108 * - The initial ring_idx of a virtual DMA channel is zero, after the 109 * device is stopped, the ring_idx needs to be reset to zero. 110 * 111 * One example: 112 * - step-1: start one dmadev 113 * - step-2: enqueue a copy operation, the ring_idx return is 0 114 * - step-3: enqueue a copy operation again, the ring_idx return is 1 115 * - ... 116 * - step-101: stop the dmadev 117 * - step-102: start the dmadev 118 * - step-103: enqueue a copy operation, the ring_idx return is 0 119 * - ... 120 * - step-x+0: enqueue a fill operation, the ring_idx return is 65535 121 * - step-x+1: enqueue a copy operation, the ring_idx return is 0 122 * - ... 123 * 124 * The DMA operation address used in enqueue APIs (i.e. rte_dma_copy(), 125 * rte_dma_copy_sg(), rte_dma_fill()) is defined as rte_iova_t type. 126 * 127 * The dmadev supports two types of address: memory address and device address. 128 * 129 * - memory address: the source and destination address of the memory-to-memory 130 * transfer type, or the source address of the memory-to-device transfer type, 131 * or the destination address of the device-to-memory transfer type. 132 * @note If the device support SVA (@see RTE_DMA_CAPA_SVA), the memory address 133 * can be any VA address, otherwise it must be an IOVA address. 134 * 135 * - device address: the source and destination address of the device-to-device 136 * transfer type, or the source address of the device-to-memory transfer type, 137 * or the destination address of the memory-to-device transfer type. 138 * 139 * About MT-safe, all the functions of the dmadev API implemented by a PMD are 140 * lock-free functions which assume to not be invoked in parallel on different 141 * logical cores to work on the same target dmadev object. 142 * @note Different virtual DMA channels on the same dmadev *DO NOT* support 143 * parallel invocation because these virtual DMA channels share the same 144 * HW-DMA-channel. 145 */ 146 147 #include <stdint.h> 148 149 #include <rte_bitops.h> 150 #include <rte_common.h> 151 152 #ifdef __cplusplus 153 extern "C" { 154 #endif 155 156 /** Maximum number of devices if rte_dma_dev_max() is not called. */ 157 #define RTE_DMADEV_DEFAULT_MAX 64 158 159 /** 160 * Configure the maximum number of dmadevs. 161 * @note This function can be invoked before the primary process rte_eal_init() 162 * to change the maximum number of dmadevs. If not invoked, the maximum number 163 * of dmadevs is @see RTE_DMADEV_DEFAULT_MAX 164 * 165 * @param dev_max 166 * maximum number of dmadevs. 167 * 168 * @return 169 * 0 on success. Otherwise negative value is returned. 170 */ 171 int rte_dma_dev_max(size_t dev_max); 172 173 /** 174 * Get the device identifier for the named DMA device. 175 * 176 * @param name 177 * DMA device name. 178 * 179 * @return 180 * Returns DMA device identifier on success. 181 * - <0: Failure to find named DMA device. 182 */ 183 int rte_dma_get_dev_id_by_name(const char *name); 184 185 /** 186 * Check whether the dev_id is valid. 187 * 188 * @param dev_id 189 * DMA device index. 190 * 191 * @return 192 * - If the device index is valid (true) or not (false). 193 */ 194 bool rte_dma_is_valid(int16_t dev_id); 195 196 /** 197 * Get the total number of DMA devices that have been successfully 198 * initialised. 199 * 200 * @return 201 * The total number of usable DMA devices. 202 */ 203 uint16_t rte_dma_count_avail(void); 204 205 /** 206 * Iterates over valid dmadev instances. 207 * 208 * @param start_dev_id 209 * The id of the next possible dmadev. 210 * @return 211 * Next valid dmadev, UINT16_MAX if there is none. 212 */ 213 int16_t rte_dma_next_dev(int16_t start_dev_id); 214 215 /** Utility macro to iterate over all available dmadevs */ 216 #define RTE_DMA_FOREACH_DEV(p) \ 217 for (p = rte_dma_next_dev(0); \ 218 p != -1; \ 219 p = rte_dma_next_dev(p + 1)) 220 221 222 /**@{@name DMA capability 223 * @see struct rte_dma_info::dev_capa 224 */ 225 /** Support memory-to-memory transfer */ 226 #define RTE_DMA_CAPA_MEM_TO_MEM RTE_BIT64(0) 227 /** Support memory-to-device transfer. */ 228 #define RTE_DMA_CAPA_MEM_TO_DEV RTE_BIT64(1) 229 /** Support device-to-memory transfer. */ 230 #define RTE_DMA_CAPA_DEV_TO_MEM RTE_BIT64(2) 231 /** Support device-to-device transfer. */ 232 #define RTE_DMA_CAPA_DEV_TO_DEV RTE_BIT64(3) 233 /** Support SVA which could use VA as DMA address. 234 * If device support SVA then application could pass any VA address like memory 235 * from rte_malloc(), rte_memzone(), malloc, stack memory. 236 * If device don't support SVA, then application should pass IOVA address which 237 * from rte_malloc(), rte_memzone(). 238 */ 239 #define RTE_DMA_CAPA_SVA RTE_BIT64(4) 240 /** Support work in silent mode. 241 * In this mode, application don't required to invoke rte_dma_completed*() 242 * API. 243 * @see struct rte_dma_conf::silent_mode 244 */ 245 #define RTE_DMA_CAPA_SILENT RTE_BIT64(5) 246 /** Supports error handling 247 * 248 * With this bit set, invalid input addresses will be reported as operation failures 249 * to the user but other operations can continue. 250 * Without this bit set, invalid data is not handled by either HW or driver, so user 251 * must ensure that all memory addresses are valid and accessible by HW. 252 */ 253 #define RTE_DMA_CAPA_HANDLES_ERRORS RTE_BIT64(6) 254 /** Support auto free for source buffer once mem to dev transfer is completed. 255 * 256 * @note Even though the DMA driver has this capability, it may not support all 257 * mempool drivers. If the mempool is not supported by the DMA driver, 258 * rte_dma_vchan_setup() will fail. 259 */ 260 #define RTE_DMA_CAPA_M2D_AUTO_FREE RTE_BIT64(7) 261 /** Support strict priority scheduling. 262 * 263 * Application could assign fixed priority to the DMA device using 'priority' 264 * field in struct rte_dma_conf. Number of supported priority levels will be 265 * known from 'nb_priorities' field in struct rte_dma_info. 266 */ 267 #define RTE_DMA_CAPA_PRI_POLICY_SP RTE_BIT64(8) 268 269 /** Support copy operation. 270 * This capability start with index of 32, so that it could leave gap between 271 * normal capability and ops capability. 272 */ 273 #define RTE_DMA_CAPA_OPS_COPY RTE_BIT64(32) 274 /** Support scatter-gather list copy operation. */ 275 #define RTE_DMA_CAPA_OPS_COPY_SG RTE_BIT64(33) 276 /** Support fill operation. */ 277 #define RTE_DMA_CAPA_OPS_FILL RTE_BIT64(34) 278 /**@}*/ 279 280 /** 281 * A structure used to retrieve the information of a DMA device. 282 * 283 * @see rte_dma_info_get 284 */ 285 struct rte_dma_info { 286 const char *dev_name; /**< Unique device name. */ 287 /** Device capabilities (RTE_DMA_CAPA_*). */ 288 uint64_t dev_capa; 289 /** Maximum number of virtual DMA channels supported. */ 290 uint16_t max_vchans; 291 /** Maximum allowed number of virtual DMA channel descriptors. */ 292 uint16_t max_desc; 293 /** Minimum allowed number of virtual DMA channel descriptors. */ 294 uint16_t min_desc; 295 /** Maximum number of source or destination scatter-gather entry 296 * supported. 297 * If the device does not support COPY_SG capability, this value can be 298 * zero. 299 * If the device supports COPY_SG capability, then rte_dma_copy_sg() 300 * parameter nb_src/nb_dst should not exceed this value. 301 */ 302 uint16_t max_sges; 303 /** NUMA node connection, -1 if unknown. */ 304 int16_t numa_node; 305 /** Number of virtual DMA channel configured. */ 306 uint16_t nb_vchans; 307 /** Number of priority levels (must be > 1) if priority scheduling is supported, 308 * 0 otherwise. 309 */ 310 uint16_t nb_priorities; 311 }; 312 313 /** 314 * Retrieve information of a DMA device. 315 * 316 * @param dev_id 317 * The identifier of the device. 318 * @param[out] dev_info 319 * A pointer to a structure of type *rte_dma_info* to be filled with the 320 * information of the device. 321 * 322 * @return 323 * 0 on success. Otherwise negative value is returned. 324 */ 325 int rte_dma_info_get(int16_t dev_id, struct rte_dma_info *dev_info); 326 327 /** 328 * A structure used to configure a DMA device. 329 * 330 * @see rte_dma_configure 331 */ 332 struct rte_dma_conf { 333 /** The number of virtual DMA channels to set up for the DMA device. 334 * This value cannot be greater than the field 'max_vchans' of struct 335 * rte_dma_info which get from rte_dma_info_get(). 336 */ 337 uint16_t nb_vchans; 338 /** Indicates whether to enable silent mode. 339 * false-default mode, true-silent mode. 340 * This value can be set to true only when the SILENT capability is 341 * supported. 342 * 343 * @see RTE_DMA_CAPA_SILENT 344 */ 345 bool enable_silent; 346 /* The priority of the DMA device. 347 * This value should be lower than the field 'nb_priorities' of struct 348 * rte_dma_info which get from rte_dma_info_get(). If the DMA device 349 * does not support priority scheduling, this value should be zero. 350 * 351 * Lowest value indicates higher priority and vice-versa. 352 */ 353 uint16_t priority; 354 }; 355 356 /** 357 * Configure a DMA device. 358 * 359 * This function must be invoked first before any other function in the 360 * API. This function can also be re-invoked when a device is in the 361 * stopped state. 362 * 363 * @param dev_id 364 * The identifier of the device to configure. 365 * @param dev_conf 366 * The DMA device configuration structure encapsulated into rte_dma_conf 367 * object. 368 * 369 * @return 370 * 0 on success. Otherwise negative value is returned. 371 */ 372 int rte_dma_configure(int16_t dev_id, const struct rte_dma_conf *dev_conf); 373 374 /** 375 * Start a DMA device. 376 * 377 * The device start step is the last one and consists of setting the DMA 378 * to start accepting jobs. 379 * 380 * @param dev_id 381 * The identifier of the device. 382 * 383 * @return 384 * 0 on success. Otherwise negative value is returned. 385 */ 386 int rte_dma_start(int16_t dev_id); 387 388 /** 389 * Stop a DMA device. 390 * 391 * The device can be restarted with a call to rte_dma_start(). 392 * 393 * @param dev_id 394 * The identifier of the device. 395 * 396 * @return 397 * 0 on success. Otherwise negative value is returned. 398 */ 399 int rte_dma_stop(int16_t dev_id); 400 401 /** 402 * Close a DMA device. 403 * 404 * The device cannot be restarted after this call. 405 * 406 * @param dev_id 407 * The identifier of the device. 408 * 409 * @return 410 * 0 on success. Otherwise negative value is returned. 411 */ 412 int rte_dma_close(int16_t dev_id); 413 414 /** 415 * DMA transfer direction defines. 416 * 417 * @see struct rte_dma_vchan_conf::direction 418 */ 419 enum rte_dma_direction { 420 /** DMA transfer direction - from memory to memory. 421 * 422 * @see struct rte_dma_vchan_conf::direction 423 */ 424 RTE_DMA_DIR_MEM_TO_MEM, 425 /** DMA transfer direction - from memory to device. 426 * In a typical scenario, the SoCs are installed on host servers as 427 * iNICs through the PCIe interface. In this case, the SoCs works in 428 * EP(endpoint) mode, it could initiate a DMA move request from memory 429 * (which is SoCs memory) to device (which is host memory). 430 * 431 * @see struct rte_dma_vchan_conf::direction 432 */ 433 RTE_DMA_DIR_MEM_TO_DEV, 434 /** DMA transfer direction - from device to memory. 435 * In a typical scenario, the SoCs are installed on host servers as 436 * iNICs through the PCIe interface. In this case, the SoCs works in 437 * EP(endpoint) mode, it could initiate a DMA move request from device 438 * (which is host memory) to memory (which is SoCs memory). 439 * 440 * @see struct rte_dma_vchan_conf::direction 441 */ 442 RTE_DMA_DIR_DEV_TO_MEM, 443 /** DMA transfer direction - from device to device. 444 * In a typical scenario, the SoCs are installed on host servers as 445 * iNICs through the PCIe interface. In this case, the SoCs works in 446 * EP(endpoint) mode, it could initiate a DMA move request from device 447 * (which is host memory) to the device (which is another host memory). 448 * 449 * @see struct rte_dma_vchan_conf::direction 450 */ 451 RTE_DMA_DIR_DEV_TO_DEV, 452 }; 453 454 /** 455 * DMA access port type defines. 456 * 457 * @see struct rte_dma_port_param::port_type 458 */ 459 enum rte_dma_port_type { 460 RTE_DMA_PORT_NONE, 461 RTE_DMA_PORT_PCIE, /**< The DMA access port is PCIe. */ 462 }; 463 464 /** 465 * A structure used to descript DMA access port parameters. 466 * 467 * @see struct rte_dma_vchan_conf::src_port 468 * @see struct rte_dma_vchan_conf::dst_port 469 */ 470 struct rte_dma_port_param { 471 /** The device access port type. 472 * 473 * @see enum rte_dma_port_type 474 */ 475 enum rte_dma_port_type port_type; 476 union { 477 /** PCIe access port parameters. 478 * 479 * The following model shows SoC's PCIe module connects to 480 * multiple PCIe hosts and multiple endpoints. The PCIe module 481 * has an integrated DMA controller. 482 * 483 * If the DMA wants to access the memory of host A, it can be 484 * initiated by PF1 in core0, or by VF0 of PF0 in core0. 485 * 486 * \code{.unparsed} 487 * System Bus 488 * | ----------PCIe module---------- 489 * | Bus 490 * | Interface 491 * | ----- ------------------ 492 * | | | | PCIe Core0 | 493 * | | | | | ----------- 494 * | | | | PF-0 -- VF-0 | | Host A | 495 * | | |--------| |- VF-1 |--------| Root | 496 * | | | | PF-1 | | Complex | 497 * | | | | PF-2 | ----------- 498 * | | | ------------------ 499 * | | | 500 * | | | ------------------ 501 * | | | | PCIe Core1 | 502 * | | | | | ----------- 503 * | | | | PF-0 -- VF-0 | | Host B | 504 * |-----| |--------| PF-1 -- VF-0 |--------| Root | 505 * | | | | |- VF-1 | | Complex | 506 * | | | | PF-2 | ----------- 507 * | | | ------------------ 508 * | | | 509 * | | | ------------------ 510 * | |DMA| | | ------ 511 * | | | | |--------| EP | 512 * | | |--------| PCIe Core2 | ------ 513 * | | | | | ------ 514 * | | | | |--------| EP | 515 * | | | | | ------ 516 * | ----- ------------------ 517 * 518 * \endcode 519 * 520 * @note If some fields can not be supported by the 521 * hardware/driver, then the driver ignores those fields. 522 * Please check driver-specific documentation for limitations 523 * and capabilities. 524 */ 525 __extension__ 526 struct { 527 uint64_t coreid : 4; /**< PCIe core id used. */ 528 uint64_t pfid : 8; /**< PF id used. */ 529 uint64_t vfen : 1; /**< VF enable bit. */ 530 uint64_t vfid : 16; /**< VF id used. */ 531 /** The pasid filed in TLP packet. */ 532 uint64_t pasid : 20; 533 /** The attributes filed in TLP packet. */ 534 uint64_t attr : 3; 535 /** The processing hint filed in TLP packet. */ 536 uint64_t ph : 2; 537 /** The steering tag filed in TLP packet. */ 538 uint64_t st : 16; 539 } pcie; 540 }; 541 uint64_t reserved[2]; /**< Reserved for future fields. */ 542 }; 543 544 /** 545 * A structure used for offload auto free params. 546 */ 547 struct rte_dma_auto_free_param { 548 union { 549 struct { 550 /** 551 * Mempool from which buffer is allocated. Mempool info 552 * is used for freeing buffer by hardware. 553 * 554 * @note If the mempool is not supported by the DMA device, 555 * rte_dma_vchan_setup() will fail. 556 */ 557 struct rte_mempool *pool; 558 } m2d; 559 }; 560 /** Reserved for future fields. */ 561 uint64_t reserved[2]; 562 }; 563 564 /** 565 * A structure used to configure a virtual DMA channel. 566 * 567 * @see rte_dma_vchan_setup 568 */ 569 struct rte_dma_vchan_conf { 570 /** Transfer direction 571 * 572 * @see enum rte_dma_direction 573 */ 574 enum rte_dma_direction direction; 575 /** Number of descriptor for the virtual DMA channel */ 576 uint16_t nb_desc; 577 /** 1) Used to describes the device access port parameter in the 578 * device-to-memory transfer scenario. 579 * 2) Used to describes the source device access port parameter in the 580 * device-to-device transfer scenario. 581 * 582 * @see struct rte_dma_port_param 583 */ 584 struct rte_dma_port_param src_port; 585 /** 1) Used to describes the device access port parameter in the 586 * memory-to-device transfer scenario. 587 * 2) Used to describes the destination device access port parameter in 588 * the device-to-device transfer scenario. 589 * 590 * @see struct rte_dma_port_param 591 */ 592 struct rte_dma_port_param dst_port; 593 /** Buffer params to auto free buffer by hardware. To free the buffer 594 * by hardware, RTE_DMA_OP_FLAG_AUTO_FREE must be set while calling 595 * rte_dma_copy and rte_dma_copy_sg(). 596 * 597 * @see RTE_DMA_OP_FLAG_AUTO_FREE 598 * @see struct rte_dma_auto_free_param 599 */ 600 struct rte_dma_auto_free_param auto_free; 601 }; 602 603 /** 604 * Allocate and set up a virtual DMA channel. 605 * 606 * @param dev_id 607 * The identifier of the device. 608 * @param vchan 609 * The identifier of virtual DMA channel. The value must be in the range 610 * [0, nb_vchans - 1] previously supplied to rte_dma_configure(). 611 * @param conf 612 * The virtual DMA channel configuration structure encapsulated into 613 * rte_dma_vchan_conf object. 614 * 615 * @return 616 * 0 on success. Otherwise negative value is returned. 617 */ 618 int rte_dma_vchan_setup(int16_t dev_id, uint16_t vchan, 619 const struct rte_dma_vchan_conf *conf); 620 621 /** 622 * A structure used to retrieve statistics. 623 * 624 * @see rte_dma_stats_get 625 */ 626 struct rte_dma_stats { 627 /** Count of operations which were submitted to hardware. */ 628 uint64_t submitted; 629 /** Count of operations which were completed, including successful and 630 * failed completions. 631 */ 632 uint64_t completed; 633 /** Count of operations which failed to complete. */ 634 uint64_t errors; 635 }; 636 637 /** 638 * Special ID, which is used to represent all virtual DMA channels. 639 * 640 * @see rte_dma_stats_get 641 * @see rte_dma_stats_reset 642 */ 643 #define RTE_DMA_ALL_VCHAN 0xFFFFu 644 645 /** 646 * Retrieve basic statistics of a or all virtual DMA channel(s). 647 * 648 * @param dev_id 649 * The identifier of the device. 650 * @param vchan 651 * The identifier of virtual DMA channel. 652 * If equal RTE_DMA_ALL_VCHAN means all channels. 653 * @param[out] stats 654 * The basic statistics structure encapsulated into rte_dma_stats 655 * object. 656 * 657 * @return 658 * 0 on success. Otherwise negative value is returned. 659 */ 660 int rte_dma_stats_get(int16_t dev_id, uint16_t vchan, 661 struct rte_dma_stats *stats); 662 663 /** 664 * Reset basic statistics of a or all virtual DMA channel(s). 665 * 666 * @param dev_id 667 * The identifier of the device. 668 * @param vchan 669 * The identifier of virtual DMA channel. 670 * If equal RTE_DMA_ALL_VCHAN means all channels. 671 * 672 * @return 673 * 0 on success. Otherwise negative value is returned. 674 */ 675 int rte_dma_stats_reset(int16_t dev_id, uint16_t vchan); 676 677 /** 678 * device vchannel status 679 * 680 * Enum with the options for the channel status, either idle, active or halted due to error 681 * @see rte_dma_vchan_status 682 */ 683 enum rte_dma_vchan_status { 684 RTE_DMA_VCHAN_IDLE, /**< not processing, awaiting ops */ 685 RTE_DMA_VCHAN_ACTIVE, /**< currently processing jobs */ 686 RTE_DMA_VCHAN_HALTED_ERROR, /**< not processing due to error, cannot accept new ops */ 687 }; 688 689 /** 690 * Determine if all jobs have completed on a device channel. 691 * This function is primarily designed for testing use, as it allows a process to check if 692 * all jobs are completed, without actually gathering completions from those jobs. 693 * 694 * @param dev_id 695 * The identifier of the device. 696 * @param vchan 697 * The identifier of virtual DMA channel. 698 * @param[out] status 699 * The vchan status 700 * @return 701 * 0 - call completed successfully 702 * < 0 - error code indicating there was a problem calling the API 703 */ 704 int 705 rte_dma_vchan_status(int16_t dev_id, uint16_t vchan, enum rte_dma_vchan_status *status); 706 707 /** 708 * Dump DMA device info. 709 * 710 * @param dev_id 711 * The identifier of the device. 712 * @param f 713 * The file to write the output to. 714 * 715 * @return 716 * 0 on success. Otherwise negative value is returned. 717 */ 718 int rte_dma_dump(int16_t dev_id, FILE *f); 719 720 /** 721 * DMA transfer result status code defines. 722 * 723 * @see rte_dma_completed_status 724 */ 725 enum rte_dma_status_code { 726 /** The operation completed successfully. */ 727 RTE_DMA_STATUS_SUCCESSFUL, 728 /** The operation failed to complete due abort by user. 729 * This is mainly used when processing dev_stop, user could modify the 730 * descriptors (e.g. change one bit to tell hardware abort this job), 731 * it allows outstanding requests to be complete as much as possible, 732 * so reduce the time to stop the device. 733 */ 734 RTE_DMA_STATUS_USER_ABORT, 735 /** The operation failed to complete due to following scenarios: 736 * The jobs in a particular batch are not attempted because they 737 * appeared after a fence where a previous job failed. In some HW 738 * implementation it's possible for jobs from later batches would be 739 * completed, though, so report the status from the not attempted jobs 740 * before reporting those newer completed jobs. 741 */ 742 RTE_DMA_STATUS_NOT_ATTEMPTED, 743 /** The operation failed to complete due invalid source address. */ 744 RTE_DMA_STATUS_INVALID_SRC_ADDR, 745 /** The operation failed to complete due invalid destination address. */ 746 RTE_DMA_STATUS_INVALID_DST_ADDR, 747 /** The operation failed to complete due invalid source or destination 748 * address, cover the case that only knows the address error, but not 749 * sure which address error. 750 */ 751 RTE_DMA_STATUS_INVALID_ADDR, 752 /** The operation failed to complete due invalid length. */ 753 RTE_DMA_STATUS_INVALID_LENGTH, 754 /** The operation failed to complete due invalid opcode. 755 * The DMA descriptor could have multiple format, which are 756 * distinguished by the opcode field. 757 */ 758 RTE_DMA_STATUS_INVALID_OPCODE, 759 /** The operation failed to complete due bus read error. */ 760 RTE_DMA_STATUS_BUS_READ_ERROR, 761 /** The operation failed to complete due bus write error. */ 762 RTE_DMA_STATUS_BUS_WRITE_ERROR, 763 /** The operation failed to complete due bus error, cover the case that 764 * only knows the bus error, but not sure which direction error. 765 */ 766 RTE_DMA_STATUS_BUS_ERROR, 767 /** The operation failed to complete due data poison. */ 768 RTE_DMA_STATUS_DATA_POISION, 769 /** The operation failed to complete due descriptor read error. */ 770 RTE_DMA_STATUS_DESCRIPTOR_READ_ERROR, 771 /** The operation failed to complete due device link error. 772 * Used to indicates that the link error in the memory-to-device/ 773 * device-to-memory/device-to-device transfer scenario. 774 */ 775 RTE_DMA_STATUS_DEV_LINK_ERROR, 776 /** The operation failed to complete due lookup page fault. */ 777 RTE_DMA_STATUS_PAGE_FAULT, 778 /** The operation failed to complete due unknown reason. 779 * The initial value is 256, which reserves space for future errors. 780 */ 781 RTE_DMA_STATUS_ERROR_UNKNOWN = 0x100, 782 }; 783 784 /** 785 * A structure used to hold scatter-gather DMA operation request entry. 786 * 787 * @see rte_dma_copy_sg 788 */ 789 struct rte_dma_sge { 790 rte_iova_t addr; /**< The DMA operation address. */ 791 uint32_t length; /**< The DMA operation length. */ 792 }; 793 794 #ifdef __cplusplus 795 } 796 #endif 797 798 #include "rte_dmadev_core.h" 799 #include "rte_dmadev_trace_fp.h" 800 801 #ifdef __cplusplus 802 extern "C" { 803 #endif 804 805 /**@{@name DMA operation flag 806 * @see rte_dma_copy() 807 * @see rte_dma_copy_sg() 808 * @see rte_dma_fill() 809 */ 810 /** Fence flag. 811 * It means the operation with this flag must be processed only after all 812 * previous operations are completed. 813 * If the specify DMA HW works in-order (it means it has default fence between 814 * operations), this flag could be NOP. 815 */ 816 #define RTE_DMA_OP_FLAG_FENCE RTE_BIT64(0) 817 /** Submit flag. 818 * It means the operation with this flag must issue doorbell to hardware after 819 * enqueued jobs. 820 */ 821 #define RTE_DMA_OP_FLAG_SUBMIT RTE_BIT64(1) 822 /** Write data to low level cache hint. 823 * Used for performance optimization, this is just a hint, and there is no 824 * capability bit for this, driver should not return error if this flag was set. 825 */ 826 #define RTE_DMA_OP_FLAG_LLC RTE_BIT64(2) 827 /** Auto free buffer flag. 828 * Operation with this flag must issue command to hardware to free the DMA 829 * buffer after DMA transfer is completed. 830 * 831 * @see struct rte_dma_vchan_conf::auto_free 832 */ 833 #define RTE_DMA_OP_FLAG_AUTO_FREE RTE_BIT64(3) 834 /**@}*/ 835 836 /** 837 * Enqueue a copy operation onto the virtual DMA channel. 838 * 839 * This queues up a copy operation to be performed by hardware, if the 'flags' 840 * parameter contains RTE_DMA_OP_FLAG_SUBMIT then trigger doorbell to begin 841 * this operation, otherwise do not trigger doorbell. 842 * 843 * @param dev_id 844 * The identifier of the device. 845 * @param vchan 846 * The identifier of virtual DMA channel. 847 * @param src 848 * The address of the source buffer. 849 * @param dst 850 * The address of the destination buffer. 851 * @param length 852 * The length of the data to be copied. 853 * @param flags 854 * An flags for this operation. 855 * @see RTE_DMA_OP_FLAG_* 856 * 857 * @return 858 * - 0..UINT16_MAX: index of enqueued job. 859 * - -ENOSPC: if no space left to enqueue. 860 * - other values < 0 on failure. 861 */ 862 static inline int 863 rte_dma_copy(int16_t dev_id, uint16_t vchan, rte_iova_t src, rte_iova_t dst, 864 uint32_t length, uint64_t flags) 865 { 866 struct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id]; 867 int ret; 868 869 #ifdef RTE_DMADEV_DEBUG 870 if (!rte_dma_is_valid(dev_id) || length == 0) 871 return -EINVAL; 872 if (*obj->copy == NULL) 873 return -ENOTSUP; 874 #endif 875 876 ret = (*obj->copy)(obj->dev_private, vchan, src, dst, length, flags); 877 rte_dma_trace_copy(dev_id, vchan, src, dst, length, flags, ret); 878 879 return ret; 880 } 881 882 /** 883 * Enqueue a scatter-gather list copy operation onto the virtual DMA channel. 884 * 885 * This queues up a scatter-gather list copy operation to be performed by 886 * hardware, if the 'flags' parameter contains RTE_DMA_OP_FLAG_SUBMIT then 887 * trigger doorbell to begin this operation, otherwise do not trigger doorbell. 888 * 889 * @param dev_id 890 * The identifier of the device. 891 * @param vchan 892 * The identifier of virtual DMA channel. 893 * @param src 894 * The pointer of source scatter-gather entry array. 895 * @param dst 896 * The pointer of destination scatter-gather entry array. 897 * @param nb_src 898 * The number of source scatter-gather entry. 899 * @see struct rte_dma_info::max_sges 900 * @param nb_dst 901 * The number of destination scatter-gather entry. 902 * @see struct rte_dma_info::max_sges 903 * @param flags 904 * An flags for this operation. 905 * @see RTE_DMA_OP_FLAG_* 906 * 907 * @return 908 * - 0..UINT16_MAX: index of enqueued job. 909 * - -ENOSPC: if no space left to enqueue. 910 * - other values < 0 on failure. 911 */ 912 static inline int 913 rte_dma_copy_sg(int16_t dev_id, uint16_t vchan, struct rte_dma_sge *src, 914 struct rte_dma_sge *dst, uint16_t nb_src, uint16_t nb_dst, 915 uint64_t flags) 916 { 917 struct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id]; 918 int ret; 919 920 #ifdef RTE_DMADEV_DEBUG 921 if (!rte_dma_is_valid(dev_id) || src == NULL || dst == NULL || 922 nb_src == 0 || nb_dst == 0) 923 return -EINVAL; 924 if (*obj->copy_sg == NULL) 925 return -ENOTSUP; 926 #endif 927 928 ret = (*obj->copy_sg)(obj->dev_private, vchan, src, dst, nb_src, 929 nb_dst, flags); 930 rte_dma_trace_copy_sg(dev_id, vchan, src, dst, nb_src, nb_dst, flags, 931 ret); 932 933 return ret; 934 } 935 936 /** 937 * Enqueue a fill operation onto the virtual DMA channel. 938 * 939 * This queues up a fill operation to be performed by hardware, if the 'flags' 940 * parameter contains RTE_DMA_OP_FLAG_SUBMIT then trigger doorbell to begin 941 * this operation, otherwise do not trigger doorbell. 942 * 943 * @param dev_id 944 * The identifier of the device. 945 * @param vchan 946 * The identifier of virtual DMA channel. 947 * @param pattern 948 * The pattern to populate the destination buffer with. 949 * @param dst 950 * The address of the destination buffer. 951 * @param length 952 * The length of the destination buffer. 953 * @param flags 954 * An flags for this operation. 955 * @see RTE_DMA_OP_FLAG_* 956 * 957 * @return 958 * - 0..UINT16_MAX: index of enqueued job. 959 * - -ENOSPC: if no space left to enqueue. 960 * - other values < 0 on failure. 961 */ 962 static inline int 963 rte_dma_fill(int16_t dev_id, uint16_t vchan, uint64_t pattern, 964 rte_iova_t dst, uint32_t length, uint64_t flags) 965 { 966 struct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id]; 967 int ret; 968 969 #ifdef RTE_DMADEV_DEBUG 970 if (!rte_dma_is_valid(dev_id) || length == 0) 971 return -EINVAL; 972 if (*obj->fill == NULL) 973 return -ENOTSUP; 974 #endif 975 976 ret = (*obj->fill)(obj->dev_private, vchan, pattern, dst, length, 977 flags); 978 rte_dma_trace_fill(dev_id, vchan, pattern, dst, length, flags, ret); 979 980 return ret; 981 } 982 983 /** 984 * Trigger hardware to begin performing enqueued operations. 985 * 986 * Writes the "doorbell" to the hardware to trigger it 987 * to begin the operations previously enqueued by rte_dma_copy/fill(). 988 * 989 * @param dev_id 990 * The identifier of the device. 991 * @param vchan 992 * The identifier of virtual DMA channel. 993 * 994 * @return 995 * 0 on success. Otherwise negative value is returned. 996 */ 997 static inline int 998 rte_dma_submit(int16_t dev_id, uint16_t vchan) 999 { 1000 struct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id]; 1001 int ret; 1002 1003 #ifdef RTE_DMADEV_DEBUG 1004 if (!rte_dma_is_valid(dev_id)) 1005 return -EINVAL; 1006 if (*obj->submit == NULL) 1007 return -ENOTSUP; 1008 #endif 1009 1010 ret = (*obj->submit)(obj->dev_private, vchan); 1011 rte_dma_trace_submit(dev_id, vchan, ret); 1012 1013 return ret; 1014 } 1015 1016 /** 1017 * Return the number of operations that have been successfully completed. 1018 * Once an operation has been reported as completed, the results of that 1019 * operation will be visible to all cores on the system. 1020 * 1021 * @param dev_id 1022 * The identifier of the device. 1023 * @param vchan 1024 * The identifier of virtual DMA channel. 1025 * @param nb_cpls 1026 * The maximum number of completed operations that can be processed. 1027 * @param[out] last_idx 1028 * The last completed operation's ring_idx. 1029 * If not required, NULL can be passed in. 1030 * @param[out] has_error 1031 * Indicates if there are transfer error. 1032 * If not required, NULL can be passed in. 1033 * 1034 * @return 1035 * The number of operations that successfully completed. This return value 1036 * must be less than or equal to the value of nb_cpls. 1037 */ 1038 static inline uint16_t 1039 rte_dma_completed(int16_t dev_id, uint16_t vchan, const uint16_t nb_cpls, 1040 uint16_t *last_idx, bool *has_error) 1041 { 1042 struct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id]; 1043 uint16_t idx, ret; 1044 bool err; 1045 1046 #ifdef RTE_DMADEV_DEBUG 1047 if (!rte_dma_is_valid(dev_id) || nb_cpls == 0) 1048 return 0; 1049 if (*obj->completed == NULL) 1050 return 0; 1051 #endif 1052 1053 /* Ensure the pointer values are non-null to simplify drivers. 1054 * In most cases these should be compile time evaluated, since this is 1055 * an inline function. 1056 * - If NULL is explicitly passed as parameter, then compiler knows the 1057 * value is NULL 1058 * - If address of local variable is passed as parameter, then compiler 1059 * can know it's non-NULL. 1060 */ 1061 if (last_idx == NULL) 1062 last_idx = &idx; 1063 if (has_error == NULL) 1064 has_error = &err; 1065 1066 *has_error = false; 1067 ret = (*obj->completed)(obj->dev_private, vchan, nb_cpls, last_idx, 1068 has_error); 1069 rte_dma_trace_completed(dev_id, vchan, nb_cpls, last_idx, has_error, 1070 ret); 1071 1072 return ret; 1073 } 1074 1075 /** 1076 * Return the number of operations that have been completed, and the operations 1077 * result may succeed or fail. 1078 * Once an operation has been reported as completed successfully, the results of that 1079 * operation will be visible to all cores on the system. 1080 * 1081 * @param dev_id 1082 * The identifier of the device. 1083 * @param vchan 1084 * The identifier of virtual DMA channel. 1085 * @param nb_cpls 1086 * Indicates the size of status array. 1087 * @param[out] last_idx 1088 * The last completed operation's ring_idx. 1089 * If not required, NULL can be passed in. 1090 * @param[out] status 1091 * This is a pointer to an array of length 'nb_cpls' that holds the completion 1092 * status code of each operation. 1093 * @see enum rte_dma_status_code 1094 * 1095 * @return 1096 * The number of operations that completed. This return value must be less 1097 * than or equal to the value of nb_cpls. 1098 * If this number is greater than zero (assuming n), then n values in the 1099 * status array are also set. 1100 */ 1101 static inline uint16_t 1102 rte_dma_completed_status(int16_t dev_id, uint16_t vchan, 1103 const uint16_t nb_cpls, uint16_t *last_idx, 1104 enum rte_dma_status_code *status) 1105 { 1106 struct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id]; 1107 uint16_t idx, ret; 1108 1109 #ifdef RTE_DMADEV_DEBUG 1110 if (!rte_dma_is_valid(dev_id) || nb_cpls == 0 || status == NULL) 1111 return 0; 1112 if (*obj->completed_status == NULL) 1113 return 0; 1114 #endif 1115 1116 if (last_idx == NULL) 1117 last_idx = &idx; 1118 1119 ret = (*obj->completed_status)(obj->dev_private, vchan, nb_cpls, 1120 last_idx, status); 1121 rte_dma_trace_completed_status(dev_id, vchan, nb_cpls, last_idx, status, 1122 ret); 1123 1124 return ret; 1125 } 1126 1127 /** 1128 * Check remaining capacity in descriptor ring for the current burst. 1129 * 1130 * @param dev_id 1131 * The identifier of the device. 1132 * @param vchan 1133 * The identifier of virtual DMA channel. 1134 * 1135 * @return 1136 * - Remaining space in the descriptor ring for the current burst. 1137 * - 0 on error 1138 */ 1139 static inline uint16_t 1140 rte_dma_burst_capacity(int16_t dev_id, uint16_t vchan) 1141 { 1142 struct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id]; 1143 uint16_t ret; 1144 1145 #ifdef RTE_DMADEV_DEBUG 1146 if (!rte_dma_is_valid(dev_id)) 1147 return 0; 1148 if (*obj->burst_capacity == NULL) 1149 return 0; 1150 #endif 1151 ret = (*obj->burst_capacity)(obj->dev_private, vchan); 1152 rte_dma_trace_burst_capacity(dev_id, vchan, ret); 1153 1154 return ret; 1155 } 1156 1157 #ifdef __cplusplus 1158 } 1159 #endif 1160 1161 #endif /* RTE_DMADEV_H */ 1162