1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) 2021 NVIDIA Corporation & Affiliates 3 */ 4 5 #include <stdlib.h> 6 7 #include <rte_eal.h> 8 #include <rte_tailq.h> 9 #include <rte_string_fns.h> 10 #include <rte_memzone.h> 11 #include <rte_malloc.h> 12 #include <rte_errno.h> 13 #include <rte_log.h> 14 15 #include "rte_gpudev.h" 16 #include "gpudev_driver.h" 17 18 /* Logging */ 19 RTE_LOG_REGISTER_DEFAULT(gpu_logtype, NOTICE); 20 #define GPU_LOG(level, ...) \ 21 rte_log(RTE_LOG_ ## level, gpu_logtype, RTE_FMT("gpu: " \ 22 RTE_FMT_HEAD(__VA_ARGS__, ) "\n", RTE_FMT_TAIL(__VA_ARGS__, ))) 23 24 /* Set any driver error as EPERM */ 25 #define GPU_DRV_RET(function) \ 26 ((function != 0) ? -(rte_errno = EPERM) : (rte_errno = 0)) 27 28 /* Array of devices */ 29 static struct rte_gpu *gpus; 30 /* Number of currently valid devices */ 31 static int16_t gpu_max; 32 /* Number of currently valid devices */ 33 static int16_t gpu_count; 34 35 /* Shared memory between processes. */ 36 static const char *GPU_MEMZONE = "rte_gpu_shared"; 37 static struct { 38 __extension__ struct rte_gpu_mpshared gpus[0]; 39 } *gpu_shared_mem; 40 41 /* Event callback object */ 42 struct rte_gpu_callback { 43 TAILQ_ENTRY(rte_gpu_callback) next; 44 rte_gpu_callback_t *function; 45 void *user_data; 46 enum rte_gpu_event event; 47 }; 48 static rte_rwlock_t gpu_callback_lock = RTE_RWLOCK_INITIALIZER; 49 static void gpu_free_callbacks(struct rte_gpu *dev); 50 51 int 52 rte_gpu_init(size_t dev_max) 53 { 54 if (dev_max == 0 || dev_max > INT16_MAX) { 55 GPU_LOG(ERR, "invalid array size"); 56 rte_errno = EINVAL; 57 return -rte_errno; 58 } 59 60 /* No lock, it must be called before or during first probing. */ 61 if (gpus != NULL) { 62 GPU_LOG(ERR, "already initialized"); 63 rte_errno = EBUSY; 64 return -rte_errno; 65 } 66 67 gpus = calloc(dev_max, sizeof(struct rte_gpu)); 68 if (gpus == NULL) { 69 GPU_LOG(ERR, "cannot initialize library"); 70 rte_errno = ENOMEM; 71 return -rte_errno; 72 } 73 74 gpu_max = dev_max; 75 return 0; 76 } 77 78 uint16_t 79 rte_gpu_count_avail(void) 80 { 81 return gpu_count; 82 } 83 84 bool 85 rte_gpu_is_valid(int16_t dev_id) 86 { 87 if (dev_id >= 0 && dev_id < gpu_max && 88 gpus[dev_id].process_state == RTE_GPU_STATE_INITIALIZED) 89 return true; 90 return false; 91 } 92 93 static bool 94 gpu_match_parent(int16_t dev_id, int16_t parent) 95 { 96 if (parent == RTE_GPU_ID_ANY) 97 return true; 98 return gpus[dev_id].mpshared->info.parent == parent; 99 } 100 101 int16_t 102 rte_gpu_find_next(int16_t dev_id, int16_t parent) 103 { 104 if (dev_id < 0) 105 dev_id = 0; 106 while (dev_id < gpu_max && 107 (gpus[dev_id].process_state == RTE_GPU_STATE_UNUSED || 108 !gpu_match_parent(dev_id, parent))) 109 dev_id++; 110 111 if (dev_id >= gpu_max) 112 return RTE_GPU_ID_NONE; 113 return dev_id; 114 } 115 116 static int16_t 117 gpu_find_free_id(void) 118 { 119 int16_t dev_id; 120 121 for (dev_id = 0; dev_id < gpu_max; dev_id++) { 122 if (gpus[dev_id].process_state == RTE_GPU_STATE_UNUSED) 123 return dev_id; 124 } 125 return RTE_GPU_ID_NONE; 126 } 127 128 static struct rte_gpu * 129 gpu_get_by_id(int16_t dev_id) 130 { 131 if (!rte_gpu_is_valid(dev_id)) 132 return NULL; 133 return &gpus[dev_id]; 134 } 135 136 struct rte_gpu * 137 rte_gpu_get_by_name(const char *name) 138 { 139 int16_t dev_id; 140 struct rte_gpu *dev; 141 142 if (name == NULL) { 143 rte_errno = EINVAL; 144 return NULL; 145 } 146 147 RTE_GPU_FOREACH(dev_id) { 148 dev = &gpus[dev_id]; 149 if (strncmp(name, dev->mpshared->name, RTE_DEV_NAME_MAX_LEN) == 0) 150 return dev; 151 } 152 return NULL; 153 } 154 155 static int 156 gpu_shared_mem_init(void) 157 { 158 const struct rte_memzone *memzone; 159 160 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 161 memzone = rte_memzone_reserve(GPU_MEMZONE, 162 sizeof(*gpu_shared_mem) + 163 sizeof(*gpu_shared_mem->gpus) * gpu_max, 164 SOCKET_ID_ANY, 0); 165 } else { 166 memzone = rte_memzone_lookup(GPU_MEMZONE); 167 } 168 if (memzone == NULL) { 169 GPU_LOG(ERR, "cannot initialize shared memory"); 170 rte_errno = ENOMEM; 171 return -rte_errno; 172 } 173 174 gpu_shared_mem = memzone->addr; 175 return 0; 176 } 177 178 struct rte_gpu * 179 rte_gpu_allocate(const char *name) 180 { 181 int16_t dev_id; 182 struct rte_gpu *dev; 183 184 if (rte_eal_process_type() != RTE_PROC_PRIMARY) { 185 GPU_LOG(ERR, "only primary process can allocate device"); 186 rte_errno = EPERM; 187 return NULL; 188 } 189 if (name == NULL) { 190 GPU_LOG(ERR, "allocate device without a name"); 191 rte_errno = EINVAL; 192 return NULL; 193 } 194 195 /* implicit initialization of library before adding first device */ 196 if (gpus == NULL && rte_gpu_init(RTE_GPU_DEFAULT_MAX) < 0) 197 return NULL; 198 199 /* initialize shared memory before adding first device */ 200 if (gpu_shared_mem == NULL && gpu_shared_mem_init() < 0) 201 return NULL; 202 203 if (rte_gpu_get_by_name(name) != NULL) { 204 GPU_LOG(ERR, "device with name %s already exists", name); 205 rte_errno = EEXIST; 206 return NULL; 207 } 208 dev_id = gpu_find_free_id(); 209 if (dev_id == RTE_GPU_ID_NONE) { 210 GPU_LOG(ERR, "reached maximum number of devices"); 211 rte_errno = ENOENT; 212 return NULL; 213 } 214 215 dev = &gpus[dev_id]; 216 memset(dev, 0, sizeof(*dev)); 217 218 dev->mpshared = &gpu_shared_mem->gpus[dev_id]; 219 memset(dev->mpshared, 0, sizeof(*dev->mpshared)); 220 221 if (rte_strscpy(dev->mpshared->name, name, RTE_DEV_NAME_MAX_LEN) < 0) { 222 GPU_LOG(ERR, "device name too long: %s", name); 223 rte_errno = ENAMETOOLONG; 224 return NULL; 225 } 226 dev->mpshared->info.name = dev->mpshared->name; 227 dev->mpshared->info.dev_id = dev_id; 228 dev->mpshared->info.numa_node = -1; 229 dev->mpshared->info.parent = RTE_GPU_ID_NONE; 230 TAILQ_INIT(&dev->callbacks); 231 rte_atomic_fetch_add_explicit(&dev->mpshared->process_refcnt, 1, rte_memory_order_relaxed); 232 233 gpu_count++; 234 GPU_LOG(DEBUG, "new device %s (id %d) of total %d", 235 name, dev_id, gpu_count); 236 return dev; 237 } 238 239 struct rte_gpu * 240 rte_gpu_attach(const char *name) 241 { 242 int16_t dev_id; 243 struct rte_gpu *dev; 244 struct rte_gpu_mpshared *shared_dev; 245 246 if (rte_eal_process_type() != RTE_PROC_SECONDARY) { 247 GPU_LOG(ERR, "only secondary process can attach device"); 248 rte_errno = EPERM; 249 return NULL; 250 } 251 if (name == NULL) { 252 GPU_LOG(ERR, "attach device without a name"); 253 rte_errno = EINVAL; 254 return NULL; 255 } 256 257 /* implicit initialization of library before adding first device */ 258 if (gpus == NULL && rte_gpu_init(RTE_GPU_DEFAULT_MAX) < 0) 259 return NULL; 260 261 /* initialize shared memory before adding first device */ 262 if (gpu_shared_mem == NULL && gpu_shared_mem_init() < 0) 263 return NULL; 264 265 for (dev_id = 0; dev_id < gpu_max; dev_id++) { 266 shared_dev = &gpu_shared_mem->gpus[dev_id]; 267 if (strncmp(name, shared_dev->name, RTE_DEV_NAME_MAX_LEN) == 0) 268 break; 269 } 270 if (dev_id >= gpu_max) { 271 GPU_LOG(ERR, "device with name %s not found", name); 272 rte_errno = ENOENT; 273 return NULL; 274 } 275 dev = &gpus[dev_id]; 276 memset(dev, 0, sizeof(*dev)); 277 278 TAILQ_INIT(&dev->callbacks); 279 dev->mpshared = shared_dev; 280 rte_atomic_fetch_add_explicit(&dev->mpshared->process_refcnt, 1, rte_memory_order_relaxed); 281 282 gpu_count++; 283 GPU_LOG(DEBUG, "attached device %s (id %d) of total %d", 284 name, dev_id, gpu_count); 285 return dev; 286 } 287 288 int16_t 289 rte_gpu_add_child(const char *name, int16_t parent, uint64_t child_context) 290 { 291 struct rte_gpu *dev; 292 293 if (!rte_gpu_is_valid(parent)) { 294 GPU_LOG(ERR, "add child to invalid parent ID %d", parent); 295 rte_errno = ENODEV; 296 return -rte_errno; 297 } 298 299 dev = rte_gpu_allocate(name); 300 if (dev == NULL) 301 return -rte_errno; 302 303 dev->mpshared->info.parent = parent; 304 dev->mpshared->info.context = child_context; 305 306 rte_gpu_complete_new(dev); 307 return dev->mpshared->info.dev_id; 308 } 309 310 void 311 rte_gpu_complete_new(struct rte_gpu *dev) 312 { 313 if (dev == NULL) 314 return; 315 316 dev->process_state = RTE_GPU_STATE_INITIALIZED; 317 rte_gpu_notify(dev, RTE_GPU_EVENT_NEW); 318 } 319 320 int 321 rte_gpu_release(struct rte_gpu *dev) 322 { 323 int16_t dev_id, child; 324 325 if (dev == NULL) { 326 rte_errno = ENODEV; 327 return -rte_errno; 328 } 329 dev_id = dev->mpshared->info.dev_id; 330 RTE_GPU_FOREACH_CHILD(child, dev_id) { 331 GPU_LOG(ERR, "cannot release device %d with child %d", 332 dev_id, child); 333 rte_errno = EBUSY; 334 return -rte_errno; 335 } 336 337 GPU_LOG(DEBUG, "free device %s (id %d)", 338 dev->mpshared->info.name, dev->mpshared->info.dev_id); 339 rte_gpu_notify(dev, RTE_GPU_EVENT_DEL); 340 341 gpu_free_callbacks(dev); 342 dev->process_state = RTE_GPU_STATE_UNUSED; 343 rte_atomic_fetch_sub_explicit(&dev->mpshared->process_refcnt, 1, rte_memory_order_relaxed); 344 gpu_count--; 345 346 return 0; 347 } 348 349 int 350 rte_gpu_close(int16_t dev_id) 351 { 352 int firsterr, binerr; 353 int *lasterr = &firsterr; 354 struct rte_gpu *dev; 355 356 dev = gpu_get_by_id(dev_id); 357 if (dev == NULL) { 358 GPU_LOG(ERR, "close invalid device ID %d", dev_id); 359 rte_errno = ENODEV; 360 return -rte_errno; 361 } 362 363 if (dev->ops.dev_close != NULL) { 364 *lasterr = GPU_DRV_RET(dev->ops.dev_close(dev)); 365 if (*lasterr != 0) 366 lasterr = &binerr; 367 } 368 369 *lasterr = rte_gpu_release(dev); 370 371 rte_errno = -firsterr; 372 return firsterr; 373 } 374 375 int 376 rte_gpu_callback_register(int16_t dev_id, enum rte_gpu_event event, 377 rte_gpu_callback_t *function, void *user_data) 378 { 379 int16_t next_dev, last_dev; 380 struct rte_gpu_callback_list *callbacks; 381 struct rte_gpu_callback *callback; 382 383 if (!rte_gpu_is_valid(dev_id) && dev_id != RTE_GPU_ID_ANY) { 384 GPU_LOG(ERR, "register callback of invalid ID %d", dev_id); 385 rte_errno = ENODEV; 386 return -rte_errno; 387 } 388 if (function == NULL) { 389 GPU_LOG(ERR, "cannot register callback without function"); 390 rte_errno = EINVAL; 391 return -rte_errno; 392 } 393 394 if (dev_id == RTE_GPU_ID_ANY) { 395 next_dev = 0; 396 last_dev = gpu_max - 1; 397 } else { 398 next_dev = last_dev = dev_id; 399 } 400 401 rte_rwlock_write_lock(&gpu_callback_lock); 402 do { 403 callbacks = &gpus[next_dev].callbacks; 404 405 /* check if not already registered */ 406 TAILQ_FOREACH(callback, callbacks, next) { 407 if (callback->event == event && 408 callback->function == function && 409 callback->user_data == user_data) { 410 GPU_LOG(INFO, "callback already registered"); 411 rte_rwlock_write_unlock(&gpu_callback_lock); 412 return 0; 413 } 414 } 415 416 callback = malloc(sizeof(*callback)); 417 if (callback == NULL) { 418 GPU_LOG(ERR, "cannot allocate callback"); 419 rte_rwlock_write_unlock(&gpu_callback_lock); 420 rte_errno = ENOMEM; 421 return -rte_errno; 422 } 423 callback->function = function; 424 callback->user_data = user_data; 425 callback->event = event; 426 TAILQ_INSERT_TAIL(callbacks, callback, next); 427 428 } while (++next_dev <= last_dev); 429 rte_rwlock_write_unlock(&gpu_callback_lock); 430 431 return 0; 432 } 433 434 int 435 rte_gpu_callback_unregister(int16_t dev_id, enum rte_gpu_event event, 436 rte_gpu_callback_t *function, void *user_data) 437 { 438 int16_t next_dev, last_dev; 439 struct rte_gpu_callback_list *callbacks; 440 struct rte_gpu_callback *callback, *nextcb; 441 442 if (!rte_gpu_is_valid(dev_id) && dev_id != RTE_GPU_ID_ANY) { 443 GPU_LOG(ERR, "unregister callback of invalid ID %d", dev_id); 444 rte_errno = ENODEV; 445 return -rte_errno; 446 } 447 if (function == NULL) { 448 GPU_LOG(ERR, "cannot unregister callback without function"); 449 rte_errno = EINVAL; 450 return -rte_errno; 451 } 452 453 if (dev_id == RTE_GPU_ID_ANY) { 454 next_dev = 0; 455 last_dev = gpu_max - 1; 456 } else { 457 next_dev = last_dev = dev_id; 458 } 459 460 rte_rwlock_write_lock(&gpu_callback_lock); 461 do { 462 callbacks = &gpus[next_dev].callbacks; 463 RTE_TAILQ_FOREACH_SAFE(callback, callbacks, next, nextcb) { 464 if (callback->event != event || 465 callback->function != function || 466 (callback->user_data != user_data && 467 user_data != (void *)-1)) 468 continue; 469 TAILQ_REMOVE(callbacks, callback, next); 470 free(callback); 471 } 472 } while (++next_dev <= last_dev); 473 rte_rwlock_write_unlock(&gpu_callback_lock); 474 475 return 0; 476 } 477 478 static void 479 gpu_free_callbacks(struct rte_gpu *dev) 480 { 481 struct rte_gpu_callback_list *callbacks; 482 struct rte_gpu_callback *callback, *nextcb; 483 484 callbacks = &dev->callbacks; 485 rte_rwlock_write_lock(&gpu_callback_lock); 486 RTE_TAILQ_FOREACH_SAFE(callback, callbacks, next, nextcb) { 487 TAILQ_REMOVE(callbacks, callback, next); 488 free(callback); 489 } 490 rte_rwlock_write_unlock(&gpu_callback_lock); 491 } 492 493 void 494 rte_gpu_notify(struct rte_gpu *dev, enum rte_gpu_event event) 495 { 496 int16_t dev_id; 497 struct rte_gpu_callback *callback; 498 499 dev_id = dev->mpshared->info.dev_id; 500 rte_rwlock_read_lock(&gpu_callback_lock); 501 TAILQ_FOREACH(callback, &dev->callbacks, next) { 502 if (callback->event != event || callback->function == NULL) 503 continue; 504 callback->function(dev_id, event, callback->user_data); 505 } 506 rte_rwlock_read_unlock(&gpu_callback_lock); 507 } 508 509 int 510 rte_gpu_info_get(int16_t dev_id, struct rte_gpu_info *info) 511 { 512 struct rte_gpu *dev; 513 514 dev = gpu_get_by_id(dev_id); 515 if (dev == NULL) { 516 GPU_LOG(ERR, "query invalid device ID %d", dev_id); 517 rte_errno = ENODEV; 518 return -rte_errno; 519 } 520 if (info == NULL) { 521 GPU_LOG(ERR, "query without storage"); 522 rte_errno = EINVAL; 523 return -rte_errno; 524 } 525 526 if (dev->ops.dev_info_get == NULL) { 527 *info = dev->mpshared->info; 528 return 0; 529 } 530 return GPU_DRV_RET(dev->ops.dev_info_get(dev, info)); 531 } 532 533 void * 534 rte_gpu_mem_alloc(int16_t dev_id, size_t size, unsigned int align) 535 { 536 struct rte_gpu *dev; 537 void *ptr; 538 int ret; 539 540 dev = gpu_get_by_id(dev_id); 541 if (dev == NULL) { 542 GPU_LOG(ERR, "alloc mem for invalid device ID %d", dev_id); 543 rte_errno = ENODEV; 544 return NULL; 545 } 546 547 if (dev->ops.mem_alloc == NULL) { 548 GPU_LOG(ERR, "mem allocation not supported"); 549 rte_errno = ENOTSUP; 550 return NULL; 551 } 552 553 if (size == 0) /* dry-run */ 554 return NULL; 555 556 if (align && !rte_is_power_of_2(align)) { 557 GPU_LOG(ERR, "requested alignment is not a power of two %u", align); 558 rte_errno = EINVAL; 559 return NULL; 560 } 561 562 ret = dev->ops.mem_alloc(dev, size, align, &ptr); 563 564 switch (ret) { 565 case 0: 566 return ptr; 567 case -ENOMEM: 568 case -E2BIG: 569 rte_errno = -ret; 570 return NULL; 571 default: 572 rte_errno = -EPERM; 573 return NULL; 574 } 575 } 576 577 int 578 rte_gpu_mem_free(int16_t dev_id, void *ptr) 579 { 580 struct rte_gpu *dev; 581 582 dev = gpu_get_by_id(dev_id); 583 if (dev == NULL) { 584 GPU_LOG(ERR, "free mem for invalid device ID %d", dev_id); 585 rte_errno = ENODEV; 586 return -rte_errno; 587 } 588 589 if (dev->ops.mem_free == NULL) { 590 rte_errno = ENOTSUP; 591 return -rte_errno; 592 } 593 594 if (ptr == NULL) /* dry-run */ 595 return 0; 596 597 return GPU_DRV_RET(dev->ops.mem_free(dev, ptr)); 598 } 599 600 int 601 rte_gpu_mem_register(int16_t dev_id, size_t size, void *ptr) 602 { 603 struct rte_gpu *dev; 604 605 dev = gpu_get_by_id(dev_id); 606 if (dev == NULL) { 607 GPU_LOG(ERR, "alloc mem for invalid device ID %d", dev_id); 608 rte_errno = ENODEV; 609 return -rte_errno; 610 } 611 612 if (dev->ops.mem_register == NULL) { 613 GPU_LOG(ERR, "mem registration not supported"); 614 rte_errno = ENOTSUP; 615 return -rte_errno; 616 } 617 618 if (ptr == NULL || size == 0) /* dry-run */ 619 return 0; 620 621 return GPU_DRV_RET(dev->ops.mem_register(dev, size, ptr)); 622 } 623 624 int 625 rte_gpu_mem_unregister(int16_t dev_id, void *ptr) 626 { 627 struct rte_gpu *dev; 628 629 dev = gpu_get_by_id(dev_id); 630 if (dev == NULL) { 631 GPU_LOG(ERR, "unregister mem for invalid device ID %d", dev_id); 632 rte_errno = ENODEV; 633 return -rte_errno; 634 } 635 636 if (dev->ops.mem_unregister == NULL) { 637 rte_errno = ENOTSUP; 638 return -rte_errno; 639 } 640 641 if (ptr == NULL) /* dry-run */ 642 return 0; 643 644 return GPU_DRV_RET(dev->ops.mem_unregister(dev, ptr)); 645 } 646 647 void * 648 rte_gpu_mem_cpu_map(int16_t dev_id, size_t size, void *ptr) 649 { 650 struct rte_gpu *dev; 651 void *ptr_out; 652 int ret; 653 654 dev = gpu_get_by_id(dev_id); 655 if (dev == NULL) { 656 GPU_LOG(ERR, "mem CPU map for invalid device ID %d", dev_id); 657 rte_errno = ENODEV; 658 return NULL; 659 } 660 661 if (dev->ops.mem_cpu_map == NULL) { 662 GPU_LOG(ERR, "mem CPU map not supported"); 663 rte_errno = ENOTSUP; 664 return NULL; 665 } 666 667 if (ptr == NULL || size == 0) /* dry-run */ 668 return NULL; 669 670 ret = GPU_DRV_RET(dev->ops.mem_cpu_map(dev, size, ptr, &ptr_out)); 671 672 switch (ret) { 673 case 0: 674 return ptr_out; 675 case -ENOMEM: 676 case -E2BIG: 677 rte_errno = -ret; 678 return NULL; 679 default: 680 rte_errno = -EPERM; 681 return NULL; 682 } 683 } 684 685 int 686 rte_gpu_mem_cpu_unmap(int16_t dev_id, void *ptr) 687 { 688 struct rte_gpu *dev; 689 690 dev = gpu_get_by_id(dev_id); 691 if (dev == NULL) { 692 GPU_LOG(ERR, "cpu_unmap mem for invalid device ID %d", dev_id); 693 rte_errno = ENODEV; 694 return -rte_errno; 695 } 696 697 if (dev->ops.mem_cpu_unmap == NULL) { 698 rte_errno = ENOTSUP; 699 return -rte_errno; 700 } 701 702 if (ptr == NULL) /* dry-run */ 703 return 0; 704 705 return GPU_DRV_RET(dev->ops.mem_cpu_unmap(dev, ptr)); 706 } 707 708 int 709 rte_gpu_wmb(int16_t dev_id) 710 { 711 struct rte_gpu *dev; 712 713 dev = gpu_get_by_id(dev_id); 714 if (dev == NULL) { 715 GPU_LOG(ERR, "memory barrier for invalid device ID %d", dev_id); 716 rte_errno = ENODEV; 717 return -rte_errno; 718 } 719 720 if (dev->ops.wmb == NULL) { 721 rte_errno = ENOTSUP; 722 return -rte_errno; 723 } 724 return GPU_DRV_RET(dev->ops.wmb(dev)); 725 } 726 727 int 728 rte_gpu_comm_create_flag(uint16_t dev_id, struct rte_gpu_comm_flag *devflag, 729 enum rte_gpu_comm_flag_type mtype) 730 { 731 size_t flag_size; 732 int ret; 733 734 if (devflag == NULL) { 735 rte_errno = EINVAL; 736 return -rte_errno; 737 } 738 if (mtype != RTE_GPU_COMM_FLAG_CPU) { 739 rte_errno = EINVAL; 740 return -rte_errno; 741 } 742 743 flag_size = sizeof(uint32_t); 744 745 devflag->ptr = rte_zmalloc(NULL, flag_size, 0); 746 if (devflag->ptr == NULL) { 747 rte_errno = ENOMEM; 748 return -rte_errno; 749 } 750 751 ret = rte_gpu_mem_register(dev_id, flag_size, devflag->ptr); 752 if (ret < 0) { 753 rte_errno = ENOMEM; 754 return -rte_errno; 755 } 756 757 devflag->mtype = mtype; 758 devflag->dev_id = dev_id; 759 760 return 0; 761 } 762 763 int 764 rte_gpu_comm_destroy_flag(struct rte_gpu_comm_flag *devflag) 765 { 766 int ret; 767 768 if (devflag == NULL) { 769 rte_errno = EINVAL; 770 return -rte_errno; 771 } 772 773 ret = rte_gpu_mem_unregister(devflag->dev_id, devflag->ptr); 774 if (ret < 0) { 775 rte_errno = EINVAL; 776 return -1; 777 } 778 779 rte_free(devflag->ptr); 780 781 return 0; 782 } 783 784 int 785 rte_gpu_comm_set_flag(struct rte_gpu_comm_flag *devflag, uint32_t val) 786 { 787 if (devflag == NULL) { 788 rte_errno = EINVAL; 789 return -rte_errno; 790 } 791 792 if (devflag->mtype != RTE_GPU_COMM_FLAG_CPU) { 793 rte_errno = EINVAL; 794 return -rte_errno; 795 } 796 797 RTE_GPU_VOLATILE(*devflag->ptr) = val; 798 799 return 0; 800 } 801 802 int 803 rte_gpu_comm_get_flag_value(struct rte_gpu_comm_flag *devflag, uint32_t *val) 804 { 805 if (devflag == NULL) { 806 rte_errno = EINVAL; 807 return -rte_errno; 808 } 809 if (devflag->mtype != RTE_GPU_COMM_FLAG_CPU) { 810 rte_errno = EINVAL; 811 return -rte_errno; 812 } 813 814 *val = RTE_GPU_VOLATILE(*devflag->ptr); 815 816 return 0; 817 } 818 819 struct rte_gpu_comm_list * 820 rte_gpu_comm_create_list(uint16_t dev_id, 821 uint32_t num_comm_items) 822 { 823 struct rte_gpu_comm_list *comm_list; 824 uint32_t idx_l; 825 int ret; 826 struct rte_gpu *dev; 827 struct rte_gpu_info info; 828 829 if (num_comm_items == 0) { 830 rte_errno = EINVAL; 831 return NULL; 832 } 833 834 dev = gpu_get_by_id(dev_id); 835 if (dev == NULL) { 836 GPU_LOG(ERR, "memory barrier for invalid device ID %d", dev_id); 837 rte_errno = ENODEV; 838 return NULL; 839 } 840 841 ret = rte_gpu_info_get(dev_id, &info); 842 if (ret < 0) { 843 rte_errno = ENODEV; 844 return NULL; 845 } 846 847 comm_list = rte_zmalloc(NULL, 848 sizeof(struct rte_gpu_comm_list) * num_comm_items, 0); 849 if (comm_list == NULL) { 850 rte_errno = ENOMEM; 851 return NULL; 852 } 853 854 ret = rte_gpu_mem_register(dev_id, 855 sizeof(struct rte_gpu_comm_list) * num_comm_items, comm_list); 856 if (ret < 0) { 857 rte_errno = ENOMEM; 858 return NULL; 859 } 860 861 /* 862 * Use GPU memory CPU map feature if enabled in the driver 863 * to allocate the status flags of the list. 864 * Allocating this flag in GPU memory will reduce 865 * the latency when GPU workload is polling this flag. 866 */ 867 comm_list[0].status_d = rte_gpu_mem_alloc(dev_id, 868 sizeof(enum rte_gpu_comm_list_status) * num_comm_items, 869 info.page_size); 870 if (ret < 0) { 871 rte_errno = ENOMEM; 872 return NULL; 873 } 874 875 comm_list[0].status_h = rte_gpu_mem_cpu_map(dev_id, 876 sizeof(enum rte_gpu_comm_list_status) * num_comm_items, 877 comm_list[0].status_d); 878 if (comm_list[0].status_h == NULL) { 879 /* 880 * If CPU mapping is not supported by driver 881 * use regular CPU registered memory. 882 */ 883 comm_list[0].status_h = rte_zmalloc(NULL, 884 sizeof(enum rte_gpu_comm_list_status) * num_comm_items, 0); 885 if (comm_list[0].status_h == NULL) { 886 rte_errno = ENOMEM; 887 return NULL; 888 } 889 890 ret = rte_gpu_mem_register(dev_id, 891 sizeof(enum rte_gpu_comm_list_status) * num_comm_items, 892 comm_list[0].status_h); 893 if (ret < 0) { 894 rte_errno = ENOMEM; 895 return NULL; 896 } 897 898 comm_list[0].status_d = comm_list[0].status_h; 899 } 900 901 for (idx_l = 0; idx_l < num_comm_items; idx_l++) { 902 comm_list[idx_l].pkt_list = rte_zmalloc(NULL, 903 sizeof(struct rte_gpu_comm_pkt) * RTE_GPU_COMM_LIST_PKTS_MAX, 0); 904 if (comm_list[idx_l].pkt_list == NULL) { 905 rte_errno = ENOMEM; 906 return NULL; 907 } 908 909 ret = rte_gpu_mem_register(dev_id, 910 sizeof(struct rte_gpu_comm_pkt) * RTE_GPU_COMM_LIST_PKTS_MAX, 911 comm_list[idx_l].pkt_list); 912 if (ret < 0) { 913 rte_errno = ENOMEM; 914 return NULL; 915 } 916 917 comm_list[idx_l].num_pkts = 0; 918 comm_list[idx_l].dev_id = dev_id; 919 920 comm_list[idx_l].mbufs = rte_zmalloc(NULL, 921 sizeof(struct rte_mbuf *) * RTE_GPU_COMM_LIST_PKTS_MAX, 0); 922 if (comm_list[idx_l].mbufs == NULL) { 923 rte_errno = ENOMEM; 924 return NULL; 925 } 926 927 if (idx_l > 0) { 928 comm_list[idx_l].status_h = &(comm_list[0].status_h[idx_l]); 929 comm_list[idx_l].status_d = &(comm_list[0].status_d[idx_l]); 930 931 ret = rte_gpu_comm_set_status(&comm_list[idx_l], RTE_GPU_COMM_LIST_FREE); 932 if (ret < 0) { 933 rte_errno = ENOMEM; 934 return NULL; 935 } 936 } 937 } 938 939 return comm_list; 940 } 941 942 int 943 rte_gpu_comm_destroy_list(struct rte_gpu_comm_list *comm_list, 944 uint32_t num_comm_items) 945 { 946 uint32_t idx_l; 947 int ret; 948 uint16_t dev_id; 949 950 if (comm_list == NULL) { 951 rte_errno = EINVAL; 952 return -rte_errno; 953 } 954 955 dev_id = comm_list[0].dev_id; 956 957 for (idx_l = 0; idx_l < num_comm_items; idx_l++) { 958 ret = rte_gpu_mem_unregister(dev_id, comm_list[idx_l].pkt_list); 959 if (ret < 0) { 960 rte_errno = EINVAL; 961 return -1; 962 } 963 964 rte_free(comm_list[idx_l].pkt_list); 965 rte_free(comm_list[idx_l].mbufs); 966 } 967 968 ret = rte_gpu_mem_unregister(dev_id, comm_list); 969 if (ret < 0) { 970 rte_errno = EINVAL; 971 return -1; 972 } 973 974 ret = rte_gpu_mem_cpu_unmap(dev_id, comm_list[0].status_d); 975 if (ret == 0) { 976 rte_gpu_mem_free(dev_id, comm_list[0].status_d); 977 } else { 978 rte_gpu_mem_unregister(dev_id, comm_list[0].status_h); 979 rte_free(comm_list[0].status_h); 980 } 981 982 rte_free(comm_list); 983 984 return 0; 985 } 986 987 int 988 rte_gpu_comm_populate_list_pkts(struct rte_gpu_comm_list *comm_list_item, 989 struct rte_mbuf **mbufs, uint32_t num_mbufs) 990 { 991 uint32_t idx; 992 int ret; 993 994 if (comm_list_item == NULL || comm_list_item->pkt_list == NULL || 995 mbufs == NULL || num_mbufs > RTE_GPU_COMM_LIST_PKTS_MAX) { 996 rte_errno = EINVAL; 997 return -rte_errno; 998 } 999 1000 for (idx = 0; idx < num_mbufs; idx++) { 1001 /* support only unchained mbufs */ 1002 if (unlikely((mbufs[idx]->nb_segs > 1) || 1003 (mbufs[idx]->next != NULL) || 1004 (mbufs[idx]->data_len != mbufs[idx]->pkt_len))) { 1005 rte_errno = ENOTSUP; 1006 return -rte_errno; 1007 } 1008 comm_list_item->pkt_list[idx].addr = 1009 rte_pktmbuf_mtod_offset(mbufs[idx], uintptr_t, 0); 1010 comm_list_item->pkt_list[idx].size = mbufs[idx]->pkt_len; 1011 comm_list_item->mbufs[idx] = mbufs[idx]; 1012 } 1013 1014 RTE_GPU_VOLATILE(comm_list_item->num_pkts) = num_mbufs; 1015 rte_gpu_wmb(comm_list_item->dev_id); 1016 ret = rte_gpu_comm_set_status(comm_list_item, RTE_GPU_COMM_LIST_READY); 1017 if (ret < 0) { 1018 rte_errno = EINVAL; 1019 return -rte_errno; 1020 } 1021 1022 return 0; 1023 } 1024 1025 int 1026 rte_gpu_comm_set_status(struct rte_gpu_comm_list *comm_list_item, 1027 enum rte_gpu_comm_list_status status) 1028 { 1029 if (comm_list_item == NULL) { 1030 rte_errno = EINVAL; 1031 return -rte_errno; 1032 } 1033 1034 RTE_GPU_VOLATILE(comm_list_item->status_h[0]) = status; 1035 1036 return 0; 1037 } 1038 1039 int 1040 rte_gpu_comm_get_status(struct rte_gpu_comm_list *comm_list_item, 1041 enum rte_gpu_comm_list_status *status) 1042 { 1043 if (comm_list_item == NULL || status == NULL) { 1044 rte_errno = EINVAL; 1045 return -rte_errno; 1046 } 1047 1048 *status = RTE_GPU_VOLATILE(comm_list_item->status_h[0]); 1049 1050 return 0; 1051 } 1052 1053 int 1054 rte_gpu_comm_cleanup_list(struct rte_gpu_comm_list *comm_list_item) 1055 { 1056 uint32_t idx = 0; 1057 enum rte_gpu_comm_list_status status; 1058 int ret; 1059 1060 if (comm_list_item == NULL) { 1061 rte_errno = EINVAL; 1062 return -rte_errno; 1063 } 1064 1065 ret = rte_gpu_comm_get_status(comm_list_item, &status); 1066 if (ret < 0) { 1067 rte_errno = EINVAL; 1068 return -rte_errno; 1069 } 1070 1071 if (status == RTE_GPU_COMM_LIST_READY) { 1072 GPU_LOG(ERR, "packet list is still in progress"); 1073 rte_errno = EINVAL; 1074 return -rte_errno; 1075 } 1076 1077 for (idx = 0; idx < RTE_GPU_COMM_LIST_PKTS_MAX; idx++) { 1078 if (comm_list_item->pkt_list[idx].addr == 0) 1079 break; 1080 1081 comm_list_item->pkt_list[idx].addr = 0; 1082 comm_list_item->pkt_list[idx].size = 0; 1083 comm_list_item->mbufs[idx] = NULL; 1084 } 1085 1086 ret = rte_gpu_comm_set_status(comm_list_item, RTE_GPU_COMM_LIST_FREE); 1087 if (ret < 0) { 1088 rte_errno = EINVAL; 1089 return -rte_errno; 1090 } 1091 RTE_GPU_VOLATILE(comm_list_item->num_pkts) = 0; 1092 rte_mb(); 1093 1094 return 0; 1095 } 1096