1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) 2021 NVIDIA Corporation & Affiliates 3 */ 4 5 #include <stdlib.h> 6 7 #include <rte_eal.h> 8 #include <rte_tailq.h> 9 #include <rte_string_fns.h> 10 #include <rte_memzone.h> 11 #include <rte_malloc.h> 12 #include <rte_errno.h> 13 #include <rte_log.h> 14 15 #include "rte_gpudev.h" 16 #include "gpudev_driver.h" 17 18 /* Logging */ 19 RTE_LOG_REGISTER_DEFAULT(gpu_logtype, NOTICE); 20 #define RTE_LOGTYPE_GPUDEV gpu_logtype 21 22 #define GPU_LOG(level, ...) \ 23 RTE_LOG_LINE(level, GPUDEV, "" __VA_ARGS__) 24 25 /* Set any driver error as EPERM */ 26 #define GPU_DRV_RET(function) \ 27 ((function != 0) ? -(rte_errno = EPERM) : (rte_errno = 0)) 28 29 /* Array of devices */ 30 static struct rte_gpu *gpus; 31 /* Number of currently valid devices */ 32 static int16_t gpu_max; 33 /* Number of currently valid devices */ 34 static int16_t gpu_count; 35 36 /* Shared memory between processes. */ 37 static const char *GPU_MEMZONE = "rte_gpu_shared"; 38 static struct { 39 __extension__ struct rte_gpu_mpshared gpus[0]; 40 } *gpu_shared_mem; 41 42 /* Event callback object */ 43 struct rte_gpu_callback { 44 TAILQ_ENTRY(rte_gpu_callback) next; 45 rte_gpu_callback_t *function; 46 void *user_data; 47 enum rte_gpu_event event; 48 }; 49 static rte_rwlock_t gpu_callback_lock = RTE_RWLOCK_INITIALIZER; 50 static void gpu_free_callbacks(struct rte_gpu *dev); 51 52 int 53 rte_gpu_init(size_t dev_max) 54 { 55 if (dev_max == 0 || dev_max > INT16_MAX) { 56 GPU_LOG(ERR, "invalid array size"); 57 rte_errno = EINVAL; 58 return -rte_errno; 59 } 60 61 /* No lock, it must be called before or during first probing. */ 62 if (gpus != NULL) { 63 GPU_LOG(ERR, "already initialized"); 64 rte_errno = EBUSY; 65 return -rte_errno; 66 } 67 68 gpus = calloc(dev_max, sizeof(struct rte_gpu)); 69 if (gpus == NULL) { 70 GPU_LOG(ERR, "cannot initialize library"); 71 rte_errno = ENOMEM; 72 return -rte_errno; 73 } 74 75 gpu_max = dev_max; 76 return 0; 77 } 78 79 uint16_t 80 rte_gpu_count_avail(void) 81 { 82 return gpu_count; 83 } 84 85 bool 86 rte_gpu_is_valid(int16_t dev_id) 87 { 88 if (dev_id >= 0 && dev_id < gpu_max && 89 gpus[dev_id].process_state == RTE_GPU_STATE_INITIALIZED) 90 return true; 91 return false; 92 } 93 94 static bool 95 gpu_match_parent(int16_t dev_id, int16_t parent) 96 { 97 if (parent == RTE_GPU_ID_ANY) 98 return true; 99 return gpus[dev_id].mpshared->info.parent == parent; 100 } 101 102 int16_t 103 rte_gpu_find_next(int16_t dev_id, int16_t parent) 104 { 105 if (dev_id < 0) 106 dev_id = 0; 107 while (dev_id < gpu_max && 108 (gpus[dev_id].process_state == RTE_GPU_STATE_UNUSED || 109 !gpu_match_parent(dev_id, parent))) 110 dev_id++; 111 112 if (dev_id >= gpu_max) 113 return RTE_GPU_ID_NONE; 114 return dev_id; 115 } 116 117 static int16_t 118 gpu_find_free_id(void) 119 { 120 int16_t dev_id; 121 122 for (dev_id = 0; dev_id < gpu_max; dev_id++) { 123 if (gpus[dev_id].process_state == RTE_GPU_STATE_UNUSED) 124 return dev_id; 125 } 126 return RTE_GPU_ID_NONE; 127 } 128 129 static struct rte_gpu * 130 gpu_get_by_id(int16_t dev_id) 131 { 132 if (!rte_gpu_is_valid(dev_id)) 133 return NULL; 134 return &gpus[dev_id]; 135 } 136 137 struct rte_gpu * 138 rte_gpu_get_by_name(const char *name) 139 { 140 int16_t dev_id; 141 struct rte_gpu *dev; 142 143 if (name == NULL) { 144 rte_errno = EINVAL; 145 return NULL; 146 } 147 148 RTE_GPU_FOREACH(dev_id) { 149 dev = &gpus[dev_id]; 150 if (strncmp(name, dev->mpshared->name, RTE_DEV_NAME_MAX_LEN) == 0) 151 return dev; 152 } 153 return NULL; 154 } 155 156 static int 157 gpu_shared_mem_init(void) 158 { 159 const struct rte_memzone *memzone; 160 161 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 162 memzone = rte_memzone_reserve(GPU_MEMZONE, 163 sizeof(*gpu_shared_mem) + 164 sizeof(*gpu_shared_mem->gpus) * gpu_max, 165 SOCKET_ID_ANY, 0); 166 } else { 167 memzone = rte_memzone_lookup(GPU_MEMZONE); 168 } 169 if (memzone == NULL) { 170 GPU_LOG(ERR, "cannot initialize shared memory"); 171 rte_errno = ENOMEM; 172 return -rte_errno; 173 } 174 175 gpu_shared_mem = memzone->addr; 176 return 0; 177 } 178 179 struct rte_gpu * 180 rte_gpu_allocate(const char *name) 181 { 182 int16_t dev_id; 183 struct rte_gpu *dev; 184 185 if (rte_eal_process_type() != RTE_PROC_PRIMARY) { 186 GPU_LOG(ERR, "only primary process can allocate device"); 187 rte_errno = EPERM; 188 return NULL; 189 } 190 if (name == NULL) { 191 GPU_LOG(ERR, "allocate device without a name"); 192 rte_errno = EINVAL; 193 return NULL; 194 } 195 196 /* implicit initialization of library before adding first device */ 197 if (gpus == NULL && rte_gpu_init(RTE_GPU_DEFAULT_MAX) < 0) 198 return NULL; 199 200 /* initialize shared memory before adding first device */ 201 if (gpu_shared_mem == NULL && gpu_shared_mem_init() < 0) 202 return NULL; 203 204 if (rte_gpu_get_by_name(name) != NULL) { 205 GPU_LOG(ERR, "device with name %s already exists", name); 206 rte_errno = EEXIST; 207 return NULL; 208 } 209 dev_id = gpu_find_free_id(); 210 if (dev_id == RTE_GPU_ID_NONE) { 211 GPU_LOG(ERR, "reached maximum number of devices"); 212 rte_errno = ENOENT; 213 return NULL; 214 } 215 216 dev = &gpus[dev_id]; 217 memset(dev, 0, sizeof(*dev)); 218 219 dev->mpshared = &gpu_shared_mem->gpus[dev_id]; 220 memset(dev->mpshared, 0, sizeof(*dev->mpshared)); 221 222 if (rte_strscpy(dev->mpshared->name, name, RTE_DEV_NAME_MAX_LEN) < 0) { 223 GPU_LOG(ERR, "device name too long: %s", name); 224 rte_errno = ENAMETOOLONG; 225 return NULL; 226 } 227 dev->mpshared->info.name = dev->mpshared->name; 228 dev->mpshared->info.dev_id = dev_id; 229 dev->mpshared->info.numa_node = -1; 230 dev->mpshared->info.parent = RTE_GPU_ID_NONE; 231 TAILQ_INIT(&dev->callbacks); 232 rte_atomic_fetch_add_explicit(&dev->mpshared->process_refcnt, 1, rte_memory_order_relaxed); 233 234 gpu_count++; 235 GPU_LOG(DEBUG, "new device %s (id %d) of total %d", 236 name, dev_id, gpu_count); 237 return dev; 238 } 239 240 struct rte_gpu * 241 rte_gpu_attach(const char *name) 242 { 243 int16_t dev_id; 244 struct rte_gpu *dev; 245 struct rte_gpu_mpshared *shared_dev; 246 247 if (rte_eal_process_type() != RTE_PROC_SECONDARY) { 248 GPU_LOG(ERR, "only secondary process can attach device"); 249 rte_errno = EPERM; 250 return NULL; 251 } 252 if (name == NULL) { 253 GPU_LOG(ERR, "attach device without a name"); 254 rte_errno = EINVAL; 255 return NULL; 256 } 257 258 /* implicit initialization of library before adding first device */ 259 if (gpus == NULL && rte_gpu_init(RTE_GPU_DEFAULT_MAX) < 0) 260 return NULL; 261 262 /* initialize shared memory before adding first device */ 263 if (gpu_shared_mem == NULL && gpu_shared_mem_init() < 0) 264 return NULL; 265 266 for (dev_id = 0; dev_id < gpu_max; dev_id++) { 267 shared_dev = &gpu_shared_mem->gpus[dev_id]; 268 if (strncmp(name, shared_dev->name, RTE_DEV_NAME_MAX_LEN) == 0) 269 break; 270 } 271 if (dev_id >= gpu_max) { 272 GPU_LOG(ERR, "device with name %s not found", name); 273 rte_errno = ENOENT; 274 return NULL; 275 } 276 dev = &gpus[dev_id]; 277 memset(dev, 0, sizeof(*dev)); 278 279 TAILQ_INIT(&dev->callbacks); 280 dev->mpshared = shared_dev; 281 rte_atomic_fetch_add_explicit(&dev->mpshared->process_refcnt, 1, rte_memory_order_relaxed); 282 283 gpu_count++; 284 GPU_LOG(DEBUG, "attached device %s (id %d) of total %d", 285 name, dev_id, gpu_count); 286 return dev; 287 } 288 289 int16_t 290 rte_gpu_add_child(const char *name, int16_t parent, uint64_t child_context) 291 { 292 struct rte_gpu *dev; 293 294 if (!rte_gpu_is_valid(parent)) { 295 GPU_LOG(ERR, "add child to invalid parent ID %d", parent); 296 rte_errno = ENODEV; 297 return -rte_errno; 298 } 299 300 dev = rte_gpu_allocate(name); 301 if (dev == NULL) 302 return -rte_errno; 303 304 dev->mpshared->info.parent = parent; 305 dev->mpshared->info.context = child_context; 306 307 rte_gpu_complete_new(dev); 308 return dev->mpshared->info.dev_id; 309 } 310 311 void 312 rte_gpu_complete_new(struct rte_gpu *dev) 313 { 314 if (dev == NULL) 315 return; 316 317 dev->process_state = RTE_GPU_STATE_INITIALIZED; 318 rte_gpu_notify(dev, RTE_GPU_EVENT_NEW); 319 } 320 321 int 322 rte_gpu_release(struct rte_gpu *dev) 323 { 324 int16_t dev_id, child; 325 326 if (dev == NULL) { 327 rte_errno = ENODEV; 328 return -rte_errno; 329 } 330 dev_id = dev->mpshared->info.dev_id; 331 RTE_GPU_FOREACH_CHILD(child, dev_id) { 332 GPU_LOG(ERR, "cannot release device %d with child %d", 333 dev_id, child); 334 rte_errno = EBUSY; 335 return -rte_errno; 336 } 337 338 GPU_LOG(DEBUG, "free device %s (id %d)", 339 dev->mpshared->info.name, dev->mpshared->info.dev_id); 340 rte_gpu_notify(dev, RTE_GPU_EVENT_DEL); 341 342 gpu_free_callbacks(dev); 343 dev->process_state = RTE_GPU_STATE_UNUSED; 344 rte_atomic_fetch_sub_explicit(&dev->mpshared->process_refcnt, 1, rte_memory_order_relaxed); 345 gpu_count--; 346 347 return 0; 348 } 349 350 int 351 rte_gpu_close(int16_t dev_id) 352 { 353 int firsterr, binerr; 354 int *lasterr = &firsterr; 355 struct rte_gpu *dev; 356 357 dev = gpu_get_by_id(dev_id); 358 if (dev == NULL) { 359 GPU_LOG(ERR, "close invalid device ID %d", dev_id); 360 rte_errno = ENODEV; 361 return -rte_errno; 362 } 363 364 if (dev->ops.dev_close != NULL) { 365 *lasterr = GPU_DRV_RET(dev->ops.dev_close(dev)); 366 if (*lasterr != 0) 367 lasterr = &binerr; 368 } 369 370 *lasterr = rte_gpu_release(dev); 371 372 rte_errno = -firsterr; 373 return firsterr; 374 } 375 376 int 377 rte_gpu_callback_register(int16_t dev_id, enum rte_gpu_event event, 378 rte_gpu_callback_t *function, void *user_data) 379 { 380 int16_t next_dev, last_dev; 381 struct rte_gpu_callback_list *callbacks; 382 struct rte_gpu_callback *callback; 383 384 if (!rte_gpu_is_valid(dev_id) && dev_id != RTE_GPU_ID_ANY) { 385 GPU_LOG(ERR, "register callback of invalid ID %d", dev_id); 386 rte_errno = ENODEV; 387 return -rte_errno; 388 } 389 if (function == NULL) { 390 GPU_LOG(ERR, "cannot register callback without function"); 391 rte_errno = EINVAL; 392 return -rte_errno; 393 } 394 395 if (dev_id == RTE_GPU_ID_ANY) { 396 next_dev = 0; 397 last_dev = gpu_max - 1; 398 } else { 399 next_dev = last_dev = dev_id; 400 } 401 402 rte_rwlock_write_lock(&gpu_callback_lock); 403 do { 404 callbacks = &gpus[next_dev].callbacks; 405 406 /* check if not already registered */ 407 TAILQ_FOREACH(callback, callbacks, next) { 408 if (callback->event == event && 409 callback->function == function && 410 callback->user_data == user_data) { 411 GPU_LOG(INFO, "callback already registered"); 412 rte_rwlock_write_unlock(&gpu_callback_lock); 413 return 0; 414 } 415 } 416 417 callback = malloc(sizeof(*callback)); 418 if (callback == NULL) { 419 GPU_LOG(ERR, "cannot allocate callback"); 420 rte_rwlock_write_unlock(&gpu_callback_lock); 421 rte_errno = ENOMEM; 422 return -rte_errno; 423 } 424 callback->function = function; 425 callback->user_data = user_data; 426 callback->event = event; 427 TAILQ_INSERT_TAIL(callbacks, callback, next); 428 429 } while (++next_dev <= last_dev); 430 rte_rwlock_write_unlock(&gpu_callback_lock); 431 432 return 0; 433 } 434 435 int 436 rte_gpu_callback_unregister(int16_t dev_id, enum rte_gpu_event event, 437 rte_gpu_callback_t *function, void *user_data) 438 { 439 int16_t next_dev, last_dev; 440 struct rte_gpu_callback_list *callbacks; 441 struct rte_gpu_callback *callback, *nextcb; 442 443 if (!rte_gpu_is_valid(dev_id) && dev_id != RTE_GPU_ID_ANY) { 444 GPU_LOG(ERR, "unregister callback of invalid ID %d", dev_id); 445 rte_errno = ENODEV; 446 return -rte_errno; 447 } 448 if (function == NULL) { 449 GPU_LOG(ERR, "cannot unregister callback without function"); 450 rte_errno = EINVAL; 451 return -rte_errno; 452 } 453 454 if (dev_id == RTE_GPU_ID_ANY) { 455 next_dev = 0; 456 last_dev = gpu_max - 1; 457 } else { 458 next_dev = last_dev = dev_id; 459 } 460 461 rte_rwlock_write_lock(&gpu_callback_lock); 462 do { 463 callbacks = &gpus[next_dev].callbacks; 464 RTE_TAILQ_FOREACH_SAFE(callback, callbacks, next, nextcb) { 465 if (callback->event != event || 466 callback->function != function || 467 (callback->user_data != user_data && 468 user_data != (void *)-1)) 469 continue; 470 TAILQ_REMOVE(callbacks, callback, next); 471 free(callback); 472 } 473 } while (++next_dev <= last_dev); 474 rte_rwlock_write_unlock(&gpu_callback_lock); 475 476 return 0; 477 } 478 479 static void 480 gpu_free_callbacks(struct rte_gpu *dev) 481 { 482 struct rte_gpu_callback_list *callbacks; 483 struct rte_gpu_callback *callback, *nextcb; 484 485 callbacks = &dev->callbacks; 486 rte_rwlock_write_lock(&gpu_callback_lock); 487 RTE_TAILQ_FOREACH_SAFE(callback, callbacks, next, nextcb) { 488 TAILQ_REMOVE(callbacks, callback, next); 489 free(callback); 490 } 491 rte_rwlock_write_unlock(&gpu_callback_lock); 492 } 493 494 void 495 rte_gpu_notify(struct rte_gpu *dev, enum rte_gpu_event event) 496 { 497 int16_t dev_id; 498 struct rte_gpu_callback *callback; 499 500 dev_id = dev->mpshared->info.dev_id; 501 rte_rwlock_read_lock(&gpu_callback_lock); 502 TAILQ_FOREACH(callback, &dev->callbacks, next) { 503 if (callback->event != event || callback->function == NULL) 504 continue; 505 callback->function(dev_id, event, callback->user_data); 506 } 507 rte_rwlock_read_unlock(&gpu_callback_lock); 508 } 509 510 int 511 rte_gpu_info_get(int16_t dev_id, struct rte_gpu_info *info) 512 { 513 struct rte_gpu *dev; 514 515 dev = gpu_get_by_id(dev_id); 516 if (dev == NULL) { 517 GPU_LOG(ERR, "query invalid device ID %d", dev_id); 518 rte_errno = ENODEV; 519 return -rte_errno; 520 } 521 if (info == NULL) { 522 GPU_LOG(ERR, "query without storage"); 523 rte_errno = EINVAL; 524 return -rte_errno; 525 } 526 527 if (dev->ops.dev_info_get == NULL) { 528 *info = dev->mpshared->info; 529 return 0; 530 } 531 return GPU_DRV_RET(dev->ops.dev_info_get(dev, info)); 532 } 533 534 void * 535 rte_gpu_mem_alloc(int16_t dev_id, size_t size, unsigned int align) 536 { 537 struct rte_gpu *dev; 538 void *ptr; 539 int ret; 540 541 dev = gpu_get_by_id(dev_id); 542 if (dev == NULL) { 543 GPU_LOG(ERR, "alloc mem for invalid device ID %d", dev_id); 544 rte_errno = ENODEV; 545 return NULL; 546 } 547 548 if (dev->ops.mem_alloc == NULL) { 549 GPU_LOG(ERR, "mem allocation not supported"); 550 rte_errno = ENOTSUP; 551 return NULL; 552 } 553 554 if (size == 0) /* dry-run */ 555 return NULL; 556 557 if (align && !rte_is_power_of_2(align)) { 558 GPU_LOG(ERR, "requested alignment is not a power of two %u", align); 559 rte_errno = EINVAL; 560 return NULL; 561 } 562 563 ret = dev->ops.mem_alloc(dev, size, align, &ptr); 564 565 switch (ret) { 566 case 0: 567 return ptr; 568 case -ENOMEM: 569 case -E2BIG: 570 rte_errno = -ret; 571 return NULL; 572 default: 573 rte_errno = -EPERM; 574 return NULL; 575 } 576 } 577 578 int 579 rte_gpu_mem_free(int16_t dev_id, void *ptr) 580 { 581 struct rte_gpu *dev; 582 583 dev = gpu_get_by_id(dev_id); 584 if (dev == NULL) { 585 GPU_LOG(ERR, "free mem for invalid device ID %d", dev_id); 586 rte_errno = ENODEV; 587 return -rte_errno; 588 } 589 590 if (dev->ops.mem_free == NULL) { 591 rte_errno = ENOTSUP; 592 return -rte_errno; 593 } 594 595 if (ptr == NULL) /* dry-run */ 596 return 0; 597 598 return GPU_DRV_RET(dev->ops.mem_free(dev, ptr)); 599 } 600 601 int 602 rte_gpu_mem_register(int16_t dev_id, size_t size, void *ptr) 603 { 604 struct rte_gpu *dev; 605 606 dev = gpu_get_by_id(dev_id); 607 if (dev == NULL) { 608 GPU_LOG(ERR, "alloc mem for invalid device ID %d", dev_id); 609 rte_errno = ENODEV; 610 return -rte_errno; 611 } 612 613 if (dev->ops.mem_register == NULL) { 614 GPU_LOG(ERR, "mem registration not supported"); 615 rte_errno = ENOTSUP; 616 return -rte_errno; 617 } 618 619 if (ptr == NULL || size == 0) /* dry-run */ 620 return 0; 621 622 return GPU_DRV_RET(dev->ops.mem_register(dev, size, ptr)); 623 } 624 625 int 626 rte_gpu_mem_unregister(int16_t dev_id, void *ptr) 627 { 628 struct rte_gpu *dev; 629 630 dev = gpu_get_by_id(dev_id); 631 if (dev == NULL) { 632 GPU_LOG(ERR, "unregister mem for invalid device ID %d", dev_id); 633 rte_errno = ENODEV; 634 return -rte_errno; 635 } 636 637 if (dev->ops.mem_unregister == NULL) { 638 rte_errno = ENOTSUP; 639 return -rte_errno; 640 } 641 642 if (ptr == NULL) /* dry-run */ 643 return 0; 644 645 return GPU_DRV_RET(dev->ops.mem_unregister(dev, ptr)); 646 } 647 648 void * 649 rte_gpu_mem_cpu_map(int16_t dev_id, size_t size, void *ptr) 650 { 651 struct rte_gpu *dev; 652 void *ptr_out; 653 int ret; 654 655 dev = gpu_get_by_id(dev_id); 656 if (dev == NULL) { 657 GPU_LOG(ERR, "mem CPU map for invalid device ID %d", dev_id); 658 rte_errno = ENODEV; 659 return NULL; 660 } 661 662 if (dev->ops.mem_cpu_map == NULL) { 663 GPU_LOG(ERR, "mem CPU map not supported"); 664 rte_errno = ENOTSUP; 665 return NULL; 666 } 667 668 if (ptr == NULL || size == 0) /* dry-run */ 669 return NULL; 670 671 ret = GPU_DRV_RET(dev->ops.mem_cpu_map(dev, size, ptr, &ptr_out)); 672 673 switch (ret) { 674 case 0: 675 return ptr_out; 676 case -ENOMEM: 677 case -E2BIG: 678 rte_errno = -ret; 679 return NULL; 680 default: 681 rte_errno = -EPERM; 682 return NULL; 683 } 684 } 685 686 int 687 rte_gpu_mem_cpu_unmap(int16_t dev_id, void *ptr) 688 { 689 struct rte_gpu *dev; 690 691 dev = gpu_get_by_id(dev_id); 692 if (dev == NULL) { 693 GPU_LOG(ERR, "cpu_unmap mem for invalid device ID %d", dev_id); 694 rte_errno = ENODEV; 695 return -rte_errno; 696 } 697 698 if (dev->ops.mem_cpu_unmap == NULL) { 699 rte_errno = ENOTSUP; 700 return -rte_errno; 701 } 702 703 if (ptr == NULL) /* dry-run */ 704 return 0; 705 706 return GPU_DRV_RET(dev->ops.mem_cpu_unmap(dev, ptr)); 707 } 708 709 int 710 rte_gpu_wmb(int16_t dev_id) 711 { 712 struct rte_gpu *dev; 713 714 dev = gpu_get_by_id(dev_id); 715 if (dev == NULL) { 716 GPU_LOG(ERR, "memory barrier for invalid device ID %d", dev_id); 717 rte_errno = ENODEV; 718 return -rte_errno; 719 } 720 721 if (dev->ops.wmb == NULL) { 722 rte_errno = ENOTSUP; 723 return -rte_errno; 724 } 725 return GPU_DRV_RET(dev->ops.wmb(dev)); 726 } 727 728 int 729 rte_gpu_comm_create_flag(uint16_t dev_id, struct rte_gpu_comm_flag *devflag, 730 enum rte_gpu_comm_flag_type mtype) 731 { 732 size_t flag_size; 733 int ret; 734 735 if (devflag == NULL) { 736 rte_errno = EINVAL; 737 return -rte_errno; 738 } 739 if (mtype != RTE_GPU_COMM_FLAG_CPU) { 740 rte_errno = EINVAL; 741 return -rte_errno; 742 } 743 744 flag_size = sizeof(uint32_t); 745 746 devflag->ptr = rte_zmalloc(NULL, flag_size, 0); 747 if (devflag->ptr == NULL) { 748 rte_errno = ENOMEM; 749 return -rte_errno; 750 } 751 752 ret = rte_gpu_mem_register(dev_id, flag_size, devflag->ptr); 753 if (ret < 0) { 754 rte_errno = ENOMEM; 755 return -rte_errno; 756 } 757 758 devflag->mtype = mtype; 759 devflag->dev_id = dev_id; 760 761 return 0; 762 } 763 764 int 765 rte_gpu_comm_destroy_flag(struct rte_gpu_comm_flag *devflag) 766 { 767 int ret; 768 769 if (devflag == NULL) { 770 rte_errno = EINVAL; 771 return -rte_errno; 772 } 773 774 ret = rte_gpu_mem_unregister(devflag->dev_id, devflag->ptr); 775 if (ret < 0) { 776 rte_errno = EINVAL; 777 return -1; 778 } 779 780 rte_free(devflag->ptr); 781 782 return 0; 783 } 784 785 int 786 rte_gpu_comm_set_flag(struct rte_gpu_comm_flag *devflag, uint32_t val) 787 { 788 if (devflag == NULL) { 789 rte_errno = EINVAL; 790 return -rte_errno; 791 } 792 793 if (devflag->mtype != RTE_GPU_COMM_FLAG_CPU) { 794 rte_errno = EINVAL; 795 return -rte_errno; 796 } 797 798 RTE_GPU_VOLATILE(*devflag->ptr) = val; 799 800 return 0; 801 } 802 803 int 804 rte_gpu_comm_get_flag_value(struct rte_gpu_comm_flag *devflag, uint32_t *val) 805 { 806 if (devflag == NULL) { 807 rte_errno = EINVAL; 808 return -rte_errno; 809 } 810 if (devflag->mtype != RTE_GPU_COMM_FLAG_CPU) { 811 rte_errno = EINVAL; 812 return -rte_errno; 813 } 814 815 *val = RTE_GPU_VOLATILE(*devflag->ptr); 816 817 return 0; 818 } 819 820 struct rte_gpu_comm_list * 821 rte_gpu_comm_create_list(uint16_t dev_id, 822 uint32_t num_comm_items) 823 { 824 struct rte_gpu_comm_list *comm_list; 825 uint32_t idx_l; 826 int ret; 827 struct rte_gpu *dev; 828 struct rte_gpu_info info; 829 830 if (num_comm_items == 0) { 831 rte_errno = EINVAL; 832 return NULL; 833 } 834 835 dev = gpu_get_by_id(dev_id); 836 if (dev == NULL) { 837 GPU_LOG(ERR, "memory barrier for invalid device ID %d", dev_id); 838 rte_errno = ENODEV; 839 return NULL; 840 } 841 842 ret = rte_gpu_info_get(dev_id, &info); 843 if (ret < 0) { 844 rte_errno = ENODEV; 845 return NULL; 846 } 847 848 comm_list = rte_zmalloc(NULL, 849 sizeof(struct rte_gpu_comm_list) * num_comm_items, 0); 850 if (comm_list == NULL) { 851 rte_errno = ENOMEM; 852 return NULL; 853 } 854 855 ret = rte_gpu_mem_register(dev_id, 856 sizeof(struct rte_gpu_comm_list) * num_comm_items, comm_list); 857 if (ret < 0) { 858 rte_errno = ENOMEM; 859 return NULL; 860 } 861 862 /* 863 * Use GPU memory CPU map feature if enabled in the driver 864 * to allocate the status flags of the list. 865 * Allocating this flag in GPU memory will reduce 866 * the latency when GPU workload is polling this flag. 867 */ 868 comm_list[0].status_d = rte_gpu_mem_alloc(dev_id, 869 sizeof(enum rte_gpu_comm_list_status) * num_comm_items, 870 info.page_size); 871 if (ret < 0) { 872 rte_errno = ENOMEM; 873 return NULL; 874 } 875 876 comm_list[0].status_h = rte_gpu_mem_cpu_map(dev_id, 877 sizeof(enum rte_gpu_comm_list_status) * num_comm_items, 878 comm_list[0].status_d); 879 if (comm_list[0].status_h == NULL) { 880 /* 881 * If CPU mapping is not supported by driver 882 * use regular CPU registered memory. 883 */ 884 comm_list[0].status_h = rte_zmalloc(NULL, 885 sizeof(enum rte_gpu_comm_list_status) * num_comm_items, 0); 886 if (comm_list[0].status_h == NULL) { 887 rte_errno = ENOMEM; 888 return NULL; 889 } 890 891 ret = rte_gpu_mem_register(dev_id, 892 sizeof(enum rte_gpu_comm_list_status) * num_comm_items, 893 comm_list[0].status_h); 894 if (ret < 0) { 895 rte_errno = ENOMEM; 896 return NULL; 897 } 898 899 comm_list[0].status_d = comm_list[0].status_h; 900 } 901 902 for (idx_l = 0; idx_l < num_comm_items; idx_l++) { 903 comm_list[idx_l].pkt_list = rte_zmalloc(NULL, 904 sizeof(struct rte_gpu_comm_pkt) * RTE_GPU_COMM_LIST_PKTS_MAX, 0); 905 if (comm_list[idx_l].pkt_list == NULL) { 906 rte_errno = ENOMEM; 907 return NULL; 908 } 909 910 ret = rte_gpu_mem_register(dev_id, 911 sizeof(struct rte_gpu_comm_pkt) * RTE_GPU_COMM_LIST_PKTS_MAX, 912 comm_list[idx_l].pkt_list); 913 if (ret < 0) { 914 rte_errno = ENOMEM; 915 return NULL; 916 } 917 918 comm_list[idx_l].num_pkts = 0; 919 comm_list[idx_l].dev_id = dev_id; 920 921 comm_list[idx_l].mbufs = rte_zmalloc(NULL, 922 sizeof(struct rte_mbuf *) * RTE_GPU_COMM_LIST_PKTS_MAX, 0); 923 if (comm_list[idx_l].mbufs == NULL) { 924 rte_errno = ENOMEM; 925 return NULL; 926 } 927 928 if (idx_l > 0) { 929 comm_list[idx_l].status_h = &(comm_list[0].status_h[idx_l]); 930 comm_list[idx_l].status_d = &(comm_list[0].status_d[idx_l]); 931 932 ret = rte_gpu_comm_set_status(&comm_list[idx_l], RTE_GPU_COMM_LIST_FREE); 933 if (ret < 0) { 934 rte_errno = ENOMEM; 935 return NULL; 936 } 937 } 938 } 939 940 return comm_list; 941 } 942 943 int 944 rte_gpu_comm_destroy_list(struct rte_gpu_comm_list *comm_list, 945 uint32_t num_comm_items) 946 { 947 uint32_t idx_l; 948 int ret; 949 uint16_t dev_id; 950 951 if (comm_list == NULL) { 952 rte_errno = EINVAL; 953 return -rte_errno; 954 } 955 956 dev_id = comm_list[0].dev_id; 957 958 for (idx_l = 0; idx_l < num_comm_items; idx_l++) { 959 ret = rte_gpu_mem_unregister(dev_id, comm_list[idx_l].pkt_list); 960 if (ret < 0) { 961 rte_errno = EINVAL; 962 return -1; 963 } 964 965 rte_free(comm_list[idx_l].pkt_list); 966 rte_free(comm_list[idx_l].mbufs); 967 } 968 969 ret = rte_gpu_mem_unregister(dev_id, comm_list); 970 if (ret < 0) { 971 rte_errno = EINVAL; 972 return -1; 973 } 974 975 ret = rte_gpu_mem_cpu_unmap(dev_id, comm_list[0].status_d); 976 if (ret == 0) { 977 rte_gpu_mem_free(dev_id, comm_list[0].status_d); 978 } else { 979 rte_gpu_mem_unregister(dev_id, comm_list[0].status_h); 980 rte_free(comm_list[0].status_h); 981 } 982 983 rte_free(comm_list); 984 985 return 0; 986 } 987 988 int 989 rte_gpu_comm_populate_list_pkts(struct rte_gpu_comm_list *comm_list_item, 990 struct rte_mbuf **mbufs, uint32_t num_mbufs) 991 { 992 uint32_t idx; 993 int ret; 994 995 if (comm_list_item == NULL || comm_list_item->pkt_list == NULL || 996 mbufs == NULL || num_mbufs > RTE_GPU_COMM_LIST_PKTS_MAX) { 997 rte_errno = EINVAL; 998 return -rte_errno; 999 } 1000 1001 for (idx = 0; idx < num_mbufs; idx++) { 1002 /* support only unchained mbufs */ 1003 if (unlikely((mbufs[idx]->nb_segs > 1) || 1004 (mbufs[idx]->next != NULL) || 1005 (mbufs[idx]->data_len != mbufs[idx]->pkt_len))) { 1006 rte_errno = ENOTSUP; 1007 return -rte_errno; 1008 } 1009 comm_list_item->pkt_list[idx].addr = 1010 rte_pktmbuf_mtod_offset(mbufs[idx], uintptr_t, 0); 1011 comm_list_item->pkt_list[idx].size = mbufs[idx]->pkt_len; 1012 comm_list_item->mbufs[idx] = mbufs[idx]; 1013 } 1014 1015 RTE_GPU_VOLATILE(comm_list_item->num_pkts) = num_mbufs; 1016 rte_gpu_wmb(comm_list_item->dev_id); 1017 ret = rte_gpu_comm_set_status(comm_list_item, RTE_GPU_COMM_LIST_READY); 1018 if (ret < 0) { 1019 rte_errno = EINVAL; 1020 return -rte_errno; 1021 } 1022 1023 return 0; 1024 } 1025 1026 int 1027 rte_gpu_comm_set_status(struct rte_gpu_comm_list *comm_list_item, 1028 enum rte_gpu_comm_list_status status) 1029 { 1030 if (comm_list_item == NULL) { 1031 rte_errno = EINVAL; 1032 return -rte_errno; 1033 } 1034 1035 RTE_GPU_VOLATILE(comm_list_item->status_h[0]) = status; 1036 1037 return 0; 1038 } 1039 1040 int 1041 rte_gpu_comm_get_status(struct rte_gpu_comm_list *comm_list_item, 1042 enum rte_gpu_comm_list_status *status) 1043 { 1044 if (comm_list_item == NULL || status == NULL) { 1045 rte_errno = EINVAL; 1046 return -rte_errno; 1047 } 1048 1049 *status = RTE_GPU_VOLATILE(comm_list_item->status_h[0]); 1050 1051 return 0; 1052 } 1053 1054 int 1055 rte_gpu_comm_cleanup_list(struct rte_gpu_comm_list *comm_list_item) 1056 { 1057 uint32_t idx = 0; 1058 enum rte_gpu_comm_list_status status; 1059 int ret; 1060 1061 if (comm_list_item == NULL) { 1062 rte_errno = EINVAL; 1063 return -rte_errno; 1064 } 1065 1066 ret = rte_gpu_comm_get_status(comm_list_item, &status); 1067 if (ret < 0) { 1068 rte_errno = EINVAL; 1069 return -rte_errno; 1070 } 1071 1072 if (status == RTE_GPU_COMM_LIST_READY) { 1073 GPU_LOG(ERR, "packet list is still in progress"); 1074 rte_errno = EINVAL; 1075 return -rte_errno; 1076 } 1077 1078 for (idx = 0; idx < RTE_GPU_COMM_LIST_PKTS_MAX; idx++) { 1079 if (comm_list_item->pkt_list[idx].addr == 0) 1080 break; 1081 1082 comm_list_item->pkt_list[idx].addr = 0; 1083 comm_list_item->pkt_list[idx].size = 0; 1084 comm_list_item->mbufs[idx] = NULL; 1085 } 1086 1087 ret = rte_gpu_comm_set_status(comm_list_item, RTE_GPU_COMM_LIST_FREE); 1088 if (ret < 0) { 1089 rte_errno = EINVAL; 1090 return -rte_errno; 1091 } 1092 RTE_GPU_VOLATILE(comm_list_item->num_pkts) = 0; 1093 rte_mb(); 1094 1095 return 0; 1096 } 1097