1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) 2021 NVIDIA Corporation & Affiliates 3 */ 4 5 #ifndef RTE_GPUDEV_H 6 #define RTE_GPUDEV_H 7 8 #include <stddef.h> 9 #include <stdint.h> 10 #include <stdbool.h> 11 12 #include <rte_mbuf.h> 13 #include <rte_bitops.h> 14 #include <rte_compat.h> 15 16 /** 17 * @file 18 * Generic library to interact with GPU computing device. 19 * 20 * The API is not thread-safe. 21 * Device management must be done by a single thread. 22 * 23 * @warning 24 * @b EXPERIMENTAL: this API may change without prior notice. 25 */ 26 27 #ifdef __cplusplus 28 extern "C" { 29 #endif 30 31 /** Maximum number of devices if rte_gpu_init() is not called. */ 32 #define RTE_GPU_DEFAULT_MAX 32 33 34 /** Empty device ID. */ 35 #define RTE_GPU_ID_NONE -1 36 /** Catch-all device ID. */ 37 #define RTE_GPU_ID_ANY INT16_MIN 38 39 /** Catch-all callback data. */ 40 #define RTE_GPU_CALLBACK_ANY_DATA ((void *)-1) 41 42 /** Access variable as volatile. */ 43 #define RTE_GPU_VOLATILE(x) (*(volatile typeof(x) *)&(x)) 44 45 /** Max number of packets per communication list. */ 46 #define RTE_GPU_COMM_LIST_PKTS_MAX 1024 47 48 /** Store device info. */ 49 struct rte_gpu_info { 50 /** Unique identifier name. */ 51 const char *name; 52 /** Opaque handler of the device context. */ 53 uint64_t context; 54 /** Device ID. */ 55 int16_t dev_id; 56 /** ID of the parent device, RTE_GPU_ID_NONE if no parent */ 57 int16_t parent; 58 /** Total processors available on device. */ 59 uint32_t processor_count; 60 /** Total memory available on device. */ 61 size_t total_memory; 62 /** GPU memory page size. */ 63 size_t page_size; 64 /** Local NUMA memory ID. -1 if unknown. */ 65 int16_t numa_node; 66 }; 67 68 /** Flags passed in notification callback. */ 69 enum rte_gpu_event { 70 /** Device is just initialized. */ 71 RTE_GPU_EVENT_NEW, 72 /** Device is going to be released. */ 73 RTE_GPU_EVENT_DEL, 74 }; 75 76 /** Prototype of event callback function. */ 77 typedef void (rte_gpu_callback_t)(int16_t dev_id, 78 enum rte_gpu_event event, void *user_data); 79 80 /** Memory where communication flag is allocated. */ 81 enum rte_gpu_comm_flag_type { 82 /** Allocate flag on CPU memory visible from device. */ 83 RTE_GPU_COMM_FLAG_CPU = 0, 84 }; 85 86 /** Communication flag to coordinate CPU with the device. */ 87 struct rte_gpu_comm_flag { 88 /** Device that will use the device flag. */ 89 uint16_t dev_id; 90 /** Pointer to flag memory area. */ 91 uint32_t *ptr; 92 /** Type of memory used to allocate the flag. */ 93 enum rte_gpu_comm_flag_type mtype; 94 }; 95 96 /** List of packets shared among CPU and device. */ 97 struct rte_gpu_comm_pkt { 98 /** Address of the packet in memory (e.g. mbuf->buf_addr). */ 99 uintptr_t addr; 100 /** Size in byte of the packet. */ 101 size_t size; 102 }; 103 104 /** Possible status for the list of packets shared among CPU and device. */ 105 enum rte_gpu_comm_list_status { 106 /** Packet list can be filled with new mbufs, no one is using it. */ 107 RTE_GPU_COMM_LIST_FREE = 0, 108 /** Packet list has been filled with new mbufs and it's ready to be used .*/ 109 RTE_GPU_COMM_LIST_READY, 110 /** Packet list has been processed, it's ready to be freed. */ 111 RTE_GPU_COMM_LIST_DONE, 112 /** Some error occurred during packet list processing. */ 113 RTE_GPU_COMM_LIST_ERROR, 114 }; 115 116 /** 117 * Communication list holding a number of lists of packets 118 * each having a status flag. 119 */ 120 struct rte_gpu_comm_list { 121 /** Device that will use the communication list. */ 122 uint16_t dev_id; 123 /** List of mbufs populated by the CPU with a set of mbufs. */ 124 struct rte_mbuf **mbufs; 125 /** List of packets populated by the CPU with a set of mbufs info. */ 126 struct rte_gpu_comm_pkt *pkt_list; 127 /** Number of packets in the list. */ 128 uint32_t num_pkts; 129 /** Status of the list. CPU pointer. */ 130 enum rte_gpu_comm_list_status *status_h; 131 /** Status of the list. GPU pointer. */ 132 enum rte_gpu_comm_list_status *status_d; 133 }; 134 135 /** 136 * @warning 137 * @b EXPERIMENTAL: this API may change without prior notice. 138 * 139 * Initialize the device array before probing devices. 140 * If not called, the maximum of probed devices is RTE_GPU_DEFAULT_MAX. 141 * 142 * @param dev_max 143 * Maximum number of devices. 144 * 145 * @return 146 * 0 on success, -rte_errno otherwise: 147 * - ENOMEM if out of memory 148 * - EINVAL if 0 size 149 * - EBUSY if already initialized 150 */ 151 __rte_experimental 152 int rte_gpu_init(size_t dev_max); 153 154 /** 155 * @warning 156 * @b EXPERIMENTAL: this API may change without prior notice. 157 * 158 * Return the number of GPU detected and associated to DPDK. 159 * 160 * @return 161 * The number of available computing devices. 162 */ 163 __rte_experimental 164 uint16_t rte_gpu_count_avail(void); 165 166 /** 167 * @warning 168 * @b EXPERIMENTAL: this API may change without prior notice. 169 * 170 * Check if the device is valid and initialized in DPDK. 171 * 172 * @param dev_id 173 * The input device ID. 174 * 175 * @return 176 * - True if dev_id is a valid and initialized computing device. 177 * - False otherwise. 178 */ 179 __rte_experimental 180 bool rte_gpu_is_valid(int16_t dev_id); 181 182 /** 183 * @warning 184 * @b EXPERIMENTAL: this API may change without prior notice. 185 * 186 * Create a virtual device representing a context in the parent device. 187 * 188 * @param name 189 * Unique string to identify the device. 190 * @param parent 191 * Device ID of the parent. 192 * @param child_context 193 * Opaque context handler. 194 * 195 * @return 196 * Device ID of the new created child, -rte_errno otherwise: 197 * - EINVAL if empty name 198 * - ENAMETOOLONG if long name 199 * - EEXIST if existing device name 200 * - ENODEV if invalid parent 201 * - EPERM if secondary process 202 * - ENOENT if too many devices 203 * - ENOMEM if out of space 204 */ 205 __rte_experimental 206 int16_t rte_gpu_add_child(const char *name, 207 int16_t parent, uint64_t child_context); 208 209 /** 210 * @warning 211 * @b EXPERIMENTAL: this API may change without prior notice. 212 * 213 * Get the ID of the next valid GPU initialized in DPDK. 214 * 215 * @param dev_id 216 * The initial device ID to start the research. 217 * @param parent 218 * The device ID of the parent. 219 * RTE_GPU_ID_NONE means no parent. 220 * RTE_GPU_ID_ANY means no or any parent. 221 * 222 * @return 223 * Next device ID corresponding to a valid and initialized computing device, 224 * RTE_GPU_ID_NONE if there is none. 225 */ 226 __rte_experimental 227 int16_t rte_gpu_find_next(int16_t dev_id, int16_t parent); 228 229 /** 230 * @warning 231 * @b EXPERIMENTAL: this API may change without prior notice. 232 * 233 * Macro to iterate over all valid GPU devices. 234 * 235 * @param dev_id 236 * The ID of the next possible valid device, usually 0 to iterate all. 237 */ 238 #define RTE_GPU_FOREACH(dev_id) \ 239 RTE_GPU_FOREACH_CHILD(dev_id, RTE_GPU_ID_ANY) 240 241 /** 242 * @warning 243 * @b EXPERIMENTAL: this API may change without prior notice. 244 * 245 * Macro to iterate over all valid computing devices having no parent. 246 * 247 * @param dev_id 248 * The ID of the next possible valid device, usually 0 to iterate all. 249 */ 250 #define RTE_GPU_FOREACH_PARENT(dev_id) \ 251 RTE_GPU_FOREACH_CHILD(dev_id, RTE_GPU_ID_NONE) 252 253 /** 254 * @warning 255 * @b EXPERIMENTAL: this API may change without prior notice. 256 * 257 * Macro to iterate over all valid children of a computing device parent. 258 * 259 * @param dev_id 260 * The ID of the next possible valid device, usually 0 to iterate all. 261 * @param parent 262 * The device ID of the parent. 263 */ 264 #define RTE_GPU_FOREACH_CHILD(dev_id, parent) \ 265 for (dev_id = rte_gpu_find_next(0, parent); \ 266 dev_id >= 0; \ 267 dev_id = rte_gpu_find_next(dev_id + 1, parent)) 268 269 /** 270 * @warning 271 * @b EXPERIMENTAL: this API may change without prior notice. 272 * 273 * Close device or child context. 274 * All resources are released. 275 * 276 * @param dev_id 277 * Device ID to close. 278 * 279 * @return 280 * 0 on success, -rte_errno otherwise: 281 * - ENODEV if invalid dev_id 282 * - EPERM if driver error 283 */ 284 __rte_experimental 285 int rte_gpu_close(int16_t dev_id); 286 287 /** 288 * @warning 289 * @b EXPERIMENTAL: this API may change without prior notice. 290 * 291 * Register a function as event callback. 292 * A function may be registered multiple times for different events. 293 * 294 * @param dev_id 295 * Device ID to get notified about. 296 * RTE_GPU_ID_ANY means all devices. 297 * @param event 298 * Device event to be registered for. 299 * @param function 300 * Callback function to be called on event. 301 * @param user_data 302 * Optional parameter passed in the callback. 303 * 304 * @return 305 * 0 on success, -rte_errno otherwise: 306 * - ENODEV if invalid dev_id 307 * - EINVAL if NULL function 308 * - ENOMEM if out of memory 309 */ 310 __rte_experimental 311 int rte_gpu_callback_register(int16_t dev_id, enum rte_gpu_event event, 312 rte_gpu_callback_t *function, void *user_data); 313 314 /** 315 * @warning 316 * @b EXPERIMENTAL: this API may change without prior notice. 317 * 318 * Unregister for an event. 319 * 320 * @param dev_id 321 * Device ID to be silenced. 322 * RTE_GPU_ID_ANY means all devices. 323 * @param event 324 * Registered event. 325 * @param function 326 * Registered function. 327 * @param user_data 328 * Optional parameter as registered. 329 * RTE_GPU_CALLBACK_ANY_DATA is a catch-all. 330 * 331 * @return 332 * 0 on success, -rte_errno otherwise: 333 * - ENODEV if invalid dev_id 334 * - EINVAL if NULL function 335 */ 336 __rte_experimental 337 int rte_gpu_callback_unregister(int16_t dev_id, enum rte_gpu_event event, 338 rte_gpu_callback_t *function, void *user_data); 339 340 /** 341 * @warning 342 * @b EXPERIMENTAL: this API may change without prior notice. 343 * 344 * Return device specific info. 345 * 346 * @param dev_id 347 * Device ID to get info. 348 * @param info 349 * Memory structure to fill with the info. 350 * 351 * @return 352 * 0 on success, -rte_errno otherwise: 353 * - ENODEV if invalid dev_id 354 * - EINVAL if NULL info 355 * - EPERM if driver error 356 */ 357 __rte_experimental 358 int rte_gpu_info_get(int16_t dev_id, struct rte_gpu_info *info); 359 360 /** 361 * @warning 362 * @b EXPERIMENTAL: this API may change without prior notice. 363 * 364 * Allocate a chunk of memory in the device. 365 * 366 * @param dev_id 367 * Device ID requiring allocated memory. 368 * @param size 369 * Number of bytes to allocate. 370 * Requesting 0 will do nothing. 371 * @param align 372 * If 0, the return is a pointer that is suitably aligned 373 * for any kind of variable (in the same manner as malloc()). 374 * Otherwise, the return is a pointer that is a multiple of *align*. 375 * In this case, it must obviously be a power of two. 376 * 377 * @return 378 * A pointer to the allocated memory, otherwise NULL and rte_errno is set: 379 * - ENODEV if invalid dev_id 380 * - EINVAL if align is not a power of two 381 * - ENOTSUP if operation not supported by the driver 382 * - E2BIG if size is higher than limit 383 * - ENOMEM if out of space 384 * - EPERM if driver error 385 */ 386 __rte_experimental 387 void *rte_gpu_mem_alloc(int16_t dev_id, size_t size, unsigned int align) 388 __rte_alloc_size(2); 389 390 /** 391 * @warning 392 * @b EXPERIMENTAL: this API may change without prior notice. 393 * 394 * Deallocate a chunk of memory allocated with rte_gpu_mem_alloc(). 395 * 396 * @param dev_id 397 * Reference device ID. 398 * @param ptr 399 * Pointer to the memory area to be deallocated. 400 * NULL is a no-op accepted value. 401 * 402 * @return 403 * 0 on success, -rte_errno otherwise: 404 * - ENODEV if invalid dev_id 405 * - ENOTSUP if operation not supported by the driver 406 * - EPERM if driver error 407 */ 408 __rte_experimental 409 int rte_gpu_mem_free(int16_t dev_id, void *ptr); 410 411 /** 412 * @warning 413 * @b EXPERIMENTAL: this API may change without prior notice. 414 * 415 * Register a chunk of memory on the CPU usable by the device. 416 * 417 * @param dev_id 418 * Device ID requiring allocated memory. 419 * @param size 420 * Number of bytes to allocate. 421 * Requesting 0 will do nothing. 422 * @param ptr 423 * Pointer to the memory area to be registered. 424 * NULL is a no-op accepted value. 425 426 * @return 427 * A pointer to the allocated memory, otherwise NULL and rte_errno is set: 428 * - ENODEV if invalid dev_id 429 * - EINVAL if reserved flags 430 * - ENOTSUP if operation not supported by the driver 431 * - E2BIG if size is higher than limit 432 * - ENOMEM if out of space 433 * - EPERM if driver error 434 */ 435 __rte_experimental 436 int rte_gpu_mem_register(int16_t dev_id, size_t size, void *ptr); 437 438 /** 439 * @warning 440 * @b EXPERIMENTAL: this API may change without prior notice. 441 * 442 * Deregister a chunk of memory previously registered with rte_gpu_mem_register() 443 * 444 * @param dev_id 445 * Reference device ID. 446 * @param ptr 447 * Pointer to the memory area to be unregistered. 448 * NULL is a no-op accepted value. 449 * 450 * @return 451 * 0 on success, -rte_errno otherwise: 452 * - ENODEV if invalid dev_id 453 * - ENOTSUP if operation not supported by the driver 454 * - EPERM if driver error 455 */ 456 __rte_experimental 457 int rte_gpu_mem_unregister(int16_t dev_id, void *ptr); 458 459 /** 460 * @warning 461 * @b EXPERIMENTAL: this API may change without prior notice. 462 * 463 * Map a chunk of GPU memory to make it accessible from the CPU 464 * using the memory pointer returned by the function. 465 * GPU memory has to be allocated via rte_gpu_mem_alloc(). 466 * 467 * @param dev_id 468 * Device ID requiring mapped memory. 469 * @param size 470 * Number of bytes to map. 471 * Requesting 0 will do nothing. 472 * @param ptr 473 * Pointer to the GPU memory area to be mapped. 474 * NULL is a no-op accepted value. 475 476 * @return 477 * A pointer to the mapped GPU memory usable by the CPU, otherwise NULL and rte_errno is set: 478 * - ENODEV if invalid dev_id 479 * - ENOTSUP if operation not supported by the driver 480 * - E2BIG if size is higher than limit 481 * - ENOMEM if out of space 482 * - EPERM if driver error 483 */ 484 __rte_experimental 485 void *rte_gpu_mem_cpu_map(int16_t dev_id, size_t size, void *ptr); 486 487 /** 488 * @warning 489 * @b EXPERIMENTAL: this API may change without prior notice. 490 * 491 * Unmap a chunk of GPU memory previously mapped with rte_gpu_mem_cpu_map() 492 * 493 * @param dev_id 494 * Reference device ID. 495 * @param ptr 496 * Pointer to the GPU memory area to be unmapped. 497 * NULL is a no-op accepted value. 498 * 499 * @return 500 * 0 on success, -rte_errno otherwise: 501 * - ENODEV if invalid dev_id 502 * - ENOTSUP if operation not supported by the driver 503 * - EPERM if driver error 504 */ 505 __rte_experimental 506 int rte_gpu_mem_cpu_unmap(int16_t dev_id, void *ptr); 507 508 /** 509 * @warning 510 * @b EXPERIMENTAL: this API may change without prior notice. 511 * 512 * Enforce a GPU write memory barrier. 513 * 514 * @param dev_id 515 * Reference device ID. 516 * 517 * @return 518 * 0 on success, -rte_errno otherwise: 519 * - ENODEV if invalid dev_id 520 * - ENOTSUP if operation not supported by the driver 521 * - EPERM if driver error 522 */ 523 __rte_experimental 524 int rte_gpu_wmb(int16_t dev_id); 525 526 /** 527 * @warning 528 * @b EXPERIMENTAL: this API may change without prior notice. 529 * 530 * Create a communication flag that can be shared 531 * between CPU threads and device workload to exchange some status info 532 * (e.g. work is done, processing can start, etc..). 533 * 534 * @param dev_id 535 * Reference device ID. 536 * @param devflag 537 * Pointer to the memory area of the devflag structure. 538 * @param mtype 539 * Type of memory to allocate the communication flag. 540 * 541 * @return 542 * 0 on success, -rte_errno otherwise: 543 * - ENODEV if invalid dev_id 544 * - EINVAL if invalid inputs 545 * - ENOTSUP if operation not supported by the driver 546 * - ENOMEM if out of space 547 * - EPERM if driver error 548 */ 549 __rte_experimental 550 int rte_gpu_comm_create_flag(uint16_t dev_id, 551 struct rte_gpu_comm_flag *devflag, 552 enum rte_gpu_comm_flag_type mtype); 553 554 /** 555 * @warning 556 * @b EXPERIMENTAL: this API may change without prior notice. 557 * 558 * Deallocate a communication flag. 559 * 560 * @param devflag 561 * Pointer to the memory area of the devflag structure. 562 * 563 * @return 564 * 0 on success, -rte_errno otherwise: 565 * - ENODEV if invalid dev_id 566 * - EINVAL if NULL devflag 567 * - ENOTSUP if operation not supported by the driver 568 * - EPERM if driver error 569 */ 570 __rte_experimental 571 int rte_gpu_comm_destroy_flag(struct rte_gpu_comm_flag *devflag); 572 573 /** 574 * @warning 575 * @b EXPERIMENTAL: this API may change without prior notice. 576 * 577 * Set the value of a communication flag as the input value. 578 * Flag memory area is treated as volatile. 579 * The flag must have been allocated with RTE_GPU_COMM_FLAG_CPU. 580 * 581 * @param devflag 582 * Pointer to the memory area of the devflag structure. 583 * @param val 584 * Value to set in the flag. 585 * 586 * @return 587 * 0 on success, -rte_errno otherwise: 588 * - EINVAL if invalid input params 589 */ 590 __rte_experimental 591 int rte_gpu_comm_set_flag(struct rte_gpu_comm_flag *devflag, 592 uint32_t val); 593 594 /** 595 * @warning 596 * @b EXPERIMENTAL: this API may change without prior notice. 597 * 598 * Get the value of the communication flag. 599 * Flag memory area is treated as volatile. 600 * The flag must have been allocated with RTE_GPU_COMM_FLAG_CPU. 601 * 602 * @param devflag 603 * Pointer to the memory area of the devflag structure. 604 * @param val 605 * Flag output value. 606 * 607 * @return 608 * 0 on success, -rte_errno otherwise: 609 * - EINVAL if invalid input params 610 */ 611 __rte_experimental 612 int rte_gpu_comm_get_flag_value(struct rte_gpu_comm_flag *devflag, 613 uint32_t *val); 614 615 /** 616 * @warning 617 * @b EXPERIMENTAL: this API may change without prior notice. 618 * 619 * Create a communication list that can be used to share packets 620 * between CPU and device. 621 * Each element of the list contains: 622 * - a packet list of RTE_GPU_COMM_LIST_PKTS_MAX elements 623 * - number of packets in the list 624 * - a status flag to communicate if the packet list is FREE, 625 * READY to be processed, DONE with processing. 626 * 627 * The list is allocated in CPU-visible memory. 628 * At creation time, every list is in FREE state. 629 * 630 * @param dev_id 631 * Reference device ID. 632 * @param num_comm_items 633 * Number of items in the communication list. 634 * 635 * @return 636 * A pointer to the allocated list, otherwise NULL and rte_errno is set: 637 * - EINVAL if invalid input params 638 */ 639 __rte_experimental 640 struct rte_gpu_comm_list *rte_gpu_comm_create_list(uint16_t dev_id, 641 uint32_t num_comm_items); 642 643 /** 644 * @warning 645 * @b EXPERIMENTAL: this API may change without prior notice. 646 * 647 * Destroy a communication list. 648 * 649 * @param comm_list 650 * Communication list to be destroyed. 651 * @param num_comm_items 652 * Number of items in the communication list. 653 * 654 * @return 655 * 0 on success, -rte_errno otherwise: 656 * - EINVAL if invalid input params 657 */ 658 __rte_experimental 659 int rte_gpu_comm_destroy_list(struct rte_gpu_comm_list *comm_list, 660 uint32_t num_comm_items); 661 662 /** 663 * @warning 664 * @b EXPERIMENTAL: this API may change without prior notice. 665 * 666 * Populate the packets list of the communication item 667 * with info from a list of mbufs. 668 * Status flag of that packet list is set to READY. 669 * 670 * @param comm_list_item 671 * Communication list item to fill. 672 * @param mbufs 673 * List of mbufs. 674 * @param num_mbufs 675 * Number of mbufs. 676 * 677 * @return 678 * 0 on success, -rte_errno otherwise: 679 * - EINVAL if invalid input params 680 * - ENOTSUP if mbufs are chained (multiple segments) 681 */ 682 __rte_experimental 683 int rte_gpu_comm_populate_list_pkts(struct rte_gpu_comm_list *comm_list_item, 684 struct rte_mbuf **mbufs, uint32_t num_mbufs); 685 686 /** 687 * @warning 688 * @b EXPERIMENTAL: this API may change without prior notice. 689 * 690 * Set status flag value of a communication list item. 691 * 692 * @param comm_list_item 693 * Communication list item to query. 694 * @param status 695 * Status value to set. 696 * 697 * @return 698 * 0 on success, -rte_errno otherwise: 699 * - EINVAL if invalid input params 700 */ 701 __rte_experimental 702 int rte_gpu_comm_set_status(struct rte_gpu_comm_list *comm_list_item, 703 enum rte_gpu_comm_list_status status); 704 705 /** 706 * @warning 707 * @b EXPERIMENTAL: this API may change without prior notice. 708 * 709 * Get status flag value of a communication list item. 710 * 711 * @param comm_list_item 712 * Communication list item to query. 713 * Input parameter. 714 * @param status 715 * Communication list item status flag value. 716 * Output parameter. 717 * 718 * @return 719 * 0 on success, -rte_errno otherwise: 720 * - EINVAL if invalid input params 721 */ 722 __rte_experimental 723 int rte_gpu_comm_get_status(struct rte_gpu_comm_list *comm_list_item, 724 enum rte_gpu_comm_list_status *status); 725 726 /** 727 * @warning 728 * @b EXPERIMENTAL: this API may change without prior notice. 729 * 730 * Reset a communication list item to the original state. 731 * The status flag set to FREE and mbufs are returned to the pool. 732 * 733 * @param comm_list_item 734 * Communication list item to reset. 735 * 736 * @return 737 * 0 on success, -rte_errno otherwise: 738 * - EINVAL if invalid input params 739 */ 740 __rte_experimental 741 int rte_gpu_comm_cleanup_list(struct rte_gpu_comm_list *comm_list_item); 742 743 #ifdef __cplusplus 744 } 745 #endif 746 747 #endif /* RTE_GPUDEV_H */ 748