1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) 2021 NVIDIA Corporation & Affiliates 3 */ 4 5 #include <stdlib.h> 6 7 #include <rte_eal.h> 8 #include <rte_tailq.h> 9 #include <rte_string_fns.h> 10 #include <rte_memzone.h> 11 #include <rte_malloc.h> 12 #include <rte_errno.h> 13 #include <rte_log.h> 14 15 #include "rte_gpudev.h" 16 #include "gpudev_driver.h" 17 18 /* Logging */ 19 RTE_LOG_REGISTER_DEFAULT(gpu_logtype, NOTICE); 20 #define GPU_LOG(level, ...) \ 21 rte_log(RTE_LOG_ ## level, gpu_logtype, RTE_FMT("gpu: " \ 22 RTE_FMT_HEAD(__VA_ARGS__, ) "\n", RTE_FMT_TAIL(__VA_ARGS__, ))) 23 24 /* Set any driver error as EPERM */ 25 #define GPU_DRV_RET(function) \ 26 ((function != 0) ? -(rte_errno = EPERM) : (rte_errno = 0)) 27 28 /* Array of devices */ 29 static struct rte_gpu *gpus; 30 /* Number of currently valid devices */ 31 static int16_t gpu_max; 32 /* Number of currently valid devices */ 33 static int16_t gpu_count; 34 35 /* Shared memory between processes. */ 36 static const char *GPU_MEMZONE = "rte_gpu_shared"; 37 static struct { 38 __extension__ struct rte_gpu_mpshared gpus[0]; 39 } *gpu_shared_mem; 40 41 /* Event callback object */ 42 struct rte_gpu_callback { 43 TAILQ_ENTRY(rte_gpu_callback) next; 44 rte_gpu_callback_t *function; 45 void *user_data; 46 enum rte_gpu_event event; 47 }; 48 static rte_rwlock_t gpu_callback_lock = RTE_RWLOCK_INITIALIZER; 49 static void gpu_free_callbacks(struct rte_gpu *dev); 50 51 int 52 rte_gpu_init(size_t dev_max) 53 { 54 if (dev_max == 0 || dev_max > INT16_MAX) { 55 GPU_LOG(ERR, "invalid array size"); 56 rte_errno = EINVAL; 57 return -rte_errno; 58 } 59 60 /* No lock, it must be called before or during first probing. */ 61 if (gpus != NULL) { 62 GPU_LOG(ERR, "already initialized"); 63 rte_errno = EBUSY; 64 return -rte_errno; 65 } 66 67 gpus = calloc(dev_max, sizeof(struct rte_gpu)); 68 if (gpus == NULL) { 69 GPU_LOG(ERR, "cannot initialize library"); 70 rte_errno = ENOMEM; 71 return -rte_errno; 72 } 73 74 gpu_max = dev_max; 75 return 0; 76 } 77 78 uint16_t 79 rte_gpu_count_avail(void) 80 { 81 return gpu_count; 82 } 83 84 bool 85 rte_gpu_is_valid(int16_t dev_id) 86 { 87 if (dev_id >= 0 && dev_id < gpu_max && 88 gpus[dev_id].process_state == RTE_GPU_STATE_INITIALIZED) 89 return true; 90 return false; 91 } 92 93 static bool 94 gpu_match_parent(int16_t dev_id, int16_t parent) 95 { 96 if (parent == RTE_GPU_ID_ANY) 97 return true; 98 return gpus[dev_id].mpshared->info.parent == parent; 99 } 100 101 int16_t 102 rte_gpu_find_next(int16_t dev_id, int16_t parent) 103 { 104 if (dev_id < 0) 105 dev_id = 0; 106 while (dev_id < gpu_max && 107 (gpus[dev_id].process_state == RTE_GPU_STATE_UNUSED || 108 !gpu_match_parent(dev_id, parent))) 109 dev_id++; 110 111 if (dev_id >= gpu_max) 112 return RTE_GPU_ID_NONE; 113 return dev_id; 114 } 115 116 static int16_t 117 gpu_find_free_id(void) 118 { 119 int16_t dev_id; 120 121 for (dev_id = 0; dev_id < gpu_max; dev_id++) { 122 if (gpus[dev_id].process_state == RTE_GPU_STATE_UNUSED) 123 return dev_id; 124 } 125 return RTE_GPU_ID_NONE; 126 } 127 128 static struct rte_gpu * 129 gpu_get_by_id(int16_t dev_id) 130 { 131 if (!rte_gpu_is_valid(dev_id)) 132 return NULL; 133 return &gpus[dev_id]; 134 } 135 136 struct rte_gpu * 137 rte_gpu_get_by_name(const char *name) 138 { 139 int16_t dev_id; 140 struct rte_gpu *dev; 141 142 if (name == NULL) { 143 rte_errno = EINVAL; 144 return NULL; 145 } 146 147 RTE_GPU_FOREACH(dev_id) { 148 dev = &gpus[dev_id]; 149 if (strncmp(name, dev->mpshared->name, RTE_DEV_NAME_MAX_LEN) == 0) 150 return dev; 151 } 152 return NULL; 153 } 154 155 static int 156 gpu_shared_mem_init(void) 157 { 158 const struct rte_memzone *memzone; 159 160 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 161 memzone = rte_memzone_reserve(GPU_MEMZONE, 162 sizeof(*gpu_shared_mem) + 163 sizeof(*gpu_shared_mem->gpus) * gpu_max, 164 SOCKET_ID_ANY, 0); 165 } else { 166 memzone = rte_memzone_lookup(GPU_MEMZONE); 167 } 168 if (memzone == NULL) { 169 GPU_LOG(ERR, "cannot initialize shared memory"); 170 rte_errno = ENOMEM; 171 return -rte_errno; 172 } 173 174 gpu_shared_mem = memzone->addr; 175 return 0; 176 } 177 178 struct rte_gpu * 179 rte_gpu_allocate(const char *name) 180 { 181 int16_t dev_id; 182 struct rte_gpu *dev; 183 184 if (rte_eal_process_type() != RTE_PROC_PRIMARY) { 185 GPU_LOG(ERR, "only primary process can allocate device"); 186 rte_errno = EPERM; 187 return NULL; 188 } 189 if (name == NULL) { 190 GPU_LOG(ERR, "allocate device without a name"); 191 rte_errno = EINVAL; 192 return NULL; 193 } 194 195 /* implicit initialization of library before adding first device */ 196 if (gpus == NULL && rte_gpu_init(RTE_GPU_DEFAULT_MAX) < 0) 197 return NULL; 198 199 /* initialize shared memory before adding first device */ 200 if (gpu_shared_mem == NULL && gpu_shared_mem_init() < 0) 201 return NULL; 202 203 if (rte_gpu_get_by_name(name) != NULL) { 204 GPU_LOG(ERR, "device with name %s already exists", name); 205 rte_errno = EEXIST; 206 return NULL; 207 } 208 dev_id = gpu_find_free_id(); 209 if (dev_id == RTE_GPU_ID_NONE) { 210 GPU_LOG(ERR, "reached maximum number of devices"); 211 rte_errno = ENOENT; 212 return NULL; 213 } 214 215 dev = &gpus[dev_id]; 216 memset(dev, 0, sizeof(*dev)); 217 218 dev->mpshared = &gpu_shared_mem->gpus[dev_id]; 219 memset(dev->mpshared, 0, sizeof(*dev->mpshared)); 220 221 if (rte_strscpy(dev->mpshared->name, name, RTE_DEV_NAME_MAX_LEN) < 0) { 222 GPU_LOG(ERR, "device name too long: %s", name); 223 rte_errno = ENAMETOOLONG; 224 return NULL; 225 } 226 dev->mpshared->info.name = dev->mpshared->name; 227 dev->mpshared->info.dev_id = dev_id; 228 dev->mpshared->info.numa_node = -1; 229 dev->mpshared->info.parent = RTE_GPU_ID_NONE; 230 TAILQ_INIT(&dev->callbacks); 231 __atomic_fetch_add(&dev->mpshared->process_refcnt, 1, __ATOMIC_RELAXED); 232 233 gpu_count++; 234 GPU_LOG(DEBUG, "new device %s (id %d) of total %d", 235 name, dev_id, gpu_count); 236 return dev; 237 } 238 239 struct rte_gpu * 240 rte_gpu_attach(const char *name) 241 { 242 int16_t dev_id; 243 struct rte_gpu *dev; 244 struct rte_gpu_mpshared *shared_dev; 245 246 if (rte_eal_process_type() != RTE_PROC_SECONDARY) { 247 GPU_LOG(ERR, "only secondary process can attach device"); 248 rte_errno = EPERM; 249 return NULL; 250 } 251 if (name == NULL) { 252 GPU_LOG(ERR, "attach device without a name"); 253 rte_errno = EINVAL; 254 return NULL; 255 } 256 257 /* implicit initialization of library before adding first device */ 258 if (gpus == NULL && rte_gpu_init(RTE_GPU_DEFAULT_MAX) < 0) 259 return NULL; 260 261 /* initialize shared memory before adding first device */ 262 if (gpu_shared_mem == NULL && gpu_shared_mem_init() < 0) 263 return NULL; 264 265 for (dev_id = 0; dev_id < gpu_max; dev_id++) { 266 shared_dev = &gpu_shared_mem->gpus[dev_id]; 267 if (strncmp(name, shared_dev->name, RTE_DEV_NAME_MAX_LEN) == 0) 268 break; 269 } 270 if (dev_id >= gpu_max) { 271 GPU_LOG(ERR, "device with name %s not found", name); 272 rte_errno = ENOENT; 273 return NULL; 274 } 275 dev = &gpus[dev_id]; 276 memset(dev, 0, sizeof(*dev)); 277 278 TAILQ_INIT(&dev->callbacks); 279 dev->mpshared = shared_dev; 280 __atomic_fetch_add(&dev->mpshared->process_refcnt, 1, __ATOMIC_RELAXED); 281 282 gpu_count++; 283 GPU_LOG(DEBUG, "attached device %s (id %d) of total %d", 284 name, dev_id, gpu_count); 285 return dev; 286 } 287 288 int16_t 289 rte_gpu_add_child(const char *name, int16_t parent, uint64_t child_context) 290 { 291 struct rte_gpu *dev; 292 293 if (!rte_gpu_is_valid(parent)) { 294 GPU_LOG(ERR, "add child to invalid parent ID %d", parent); 295 rte_errno = ENODEV; 296 return -rte_errno; 297 } 298 299 dev = rte_gpu_allocate(name); 300 if (dev == NULL) 301 return -rte_errno; 302 303 dev->mpshared->info.parent = parent; 304 dev->mpshared->info.context = child_context; 305 306 rte_gpu_complete_new(dev); 307 return dev->mpshared->info.dev_id; 308 } 309 310 void 311 rte_gpu_complete_new(struct rte_gpu *dev) 312 { 313 if (dev == NULL) 314 return; 315 316 dev->process_state = RTE_GPU_STATE_INITIALIZED; 317 rte_gpu_notify(dev, RTE_GPU_EVENT_NEW); 318 } 319 320 int 321 rte_gpu_release(struct rte_gpu *dev) 322 { 323 int16_t dev_id, child; 324 325 if (dev == NULL) { 326 rte_errno = ENODEV; 327 return -rte_errno; 328 } 329 dev_id = dev->mpshared->info.dev_id; 330 RTE_GPU_FOREACH_CHILD(child, dev_id) { 331 GPU_LOG(ERR, "cannot release device %d with child %d", 332 dev_id, child); 333 rte_errno = EBUSY; 334 return -rte_errno; 335 } 336 337 GPU_LOG(DEBUG, "free device %s (id %d)", 338 dev->mpshared->info.name, dev->mpshared->info.dev_id); 339 rte_gpu_notify(dev, RTE_GPU_EVENT_DEL); 340 341 gpu_free_callbacks(dev); 342 dev->process_state = RTE_GPU_STATE_UNUSED; 343 __atomic_fetch_sub(&dev->mpshared->process_refcnt, 1, __ATOMIC_RELAXED); 344 gpu_count--; 345 346 return 0; 347 } 348 349 int 350 rte_gpu_close(int16_t dev_id) 351 { 352 int firsterr, binerr; 353 int *lasterr = &firsterr; 354 struct rte_gpu *dev; 355 356 dev = gpu_get_by_id(dev_id); 357 if (dev == NULL) { 358 GPU_LOG(ERR, "close invalid device ID %d", dev_id); 359 rte_errno = ENODEV; 360 return -rte_errno; 361 } 362 363 if (dev->ops.dev_close != NULL) { 364 *lasterr = GPU_DRV_RET(dev->ops.dev_close(dev)); 365 if (*lasterr != 0) 366 lasterr = &binerr; 367 } 368 369 *lasterr = rte_gpu_release(dev); 370 371 rte_errno = -firsterr; 372 return firsterr; 373 } 374 375 int 376 rte_gpu_callback_register(int16_t dev_id, enum rte_gpu_event event, 377 rte_gpu_callback_t *function, void *user_data) 378 { 379 int16_t next_dev, last_dev; 380 struct rte_gpu_callback_list *callbacks; 381 struct rte_gpu_callback *callback; 382 383 if (!rte_gpu_is_valid(dev_id) && dev_id != RTE_GPU_ID_ANY) { 384 GPU_LOG(ERR, "register callback of invalid ID %d", dev_id); 385 rte_errno = ENODEV; 386 return -rte_errno; 387 } 388 if (function == NULL) { 389 GPU_LOG(ERR, "cannot register callback without function"); 390 rte_errno = EINVAL; 391 return -rte_errno; 392 } 393 394 if (dev_id == RTE_GPU_ID_ANY) { 395 next_dev = 0; 396 last_dev = gpu_max - 1; 397 } else { 398 next_dev = last_dev = dev_id; 399 } 400 401 rte_rwlock_write_lock(&gpu_callback_lock); 402 do { 403 callbacks = &gpus[next_dev].callbacks; 404 405 /* check if not already registered */ 406 TAILQ_FOREACH(callback, callbacks, next) { 407 if (callback->event == event && 408 callback->function == function && 409 callback->user_data == user_data) { 410 GPU_LOG(INFO, "callback already registered"); 411 return 0; 412 } 413 } 414 415 callback = malloc(sizeof(*callback)); 416 if (callback == NULL) { 417 GPU_LOG(ERR, "cannot allocate callback"); 418 return -ENOMEM; 419 } 420 callback->function = function; 421 callback->user_data = user_data; 422 callback->event = event; 423 TAILQ_INSERT_TAIL(callbacks, callback, next); 424 425 } while (++next_dev <= last_dev); 426 rte_rwlock_write_unlock(&gpu_callback_lock); 427 428 return 0; 429 } 430 431 int 432 rte_gpu_callback_unregister(int16_t dev_id, enum rte_gpu_event event, 433 rte_gpu_callback_t *function, void *user_data) 434 { 435 int16_t next_dev, last_dev; 436 struct rte_gpu_callback_list *callbacks; 437 struct rte_gpu_callback *callback, *nextcb; 438 439 if (!rte_gpu_is_valid(dev_id) && dev_id != RTE_GPU_ID_ANY) { 440 GPU_LOG(ERR, "unregister callback of invalid ID %d", dev_id); 441 rte_errno = ENODEV; 442 return -rte_errno; 443 } 444 if (function == NULL) { 445 GPU_LOG(ERR, "cannot unregister callback without function"); 446 rte_errno = EINVAL; 447 return -rte_errno; 448 } 449 450 if (dev_id == RTE_GPU_ID_ANY) { 451 next_dev = 0; 452 last_dev = gpu_max - 1; 453 } else { 454 next_dev = last_dev = dev_id; 455 } 456 457 rte_rwlock_write_lock(&gpu_callback_lock); 458 do { 459 callbacks = &gpus[next_dev].callbacks; 460 RTE_TAILQ_FOREACH_SAFE(callback, callbacks, next, nextcb) { 461 if (callback->event != event || 462 callback->function != function || 463 (callback->user_data != user_data && 464 user_data != (void *)-1)) 465 continue; 466 TAILQ_REMOVE(callbacks, callback, next); 467 free(callback); 468 } 469 } while (++next_dev <= last_dev); 470 rte_rwlock_write_unlock(&gpu_callback_lock); 471 472 return 0; 473 } 474 475 static void 476 gpu_free_callbacks(struct rte_gpu *dev) 477 { 478 struct rte_gpu_callback_list *callbacks; 479 struct rte_gpu_callback *callback, *nextcb; 480 481 callbacks = &dev->callbacks; 482 rte_rwlock_write_lock(&gpu_callback_lock); 483 RTE_TAILQ_FOREACH_SAFE(callback, callbacks, next, nextcb) { 484 TAILQ_REMOVE(callbacks, callback, next); 485 free(callback); 486 } 487 rte_rwlock_write_unlock(&gpu_callback_lock); 488 } 489 490 void 491 rte_gpu_notify(struct rte_gpu *dev, enum rte_gpu_event event) 492 { 493 int16_t dev_id; 494 struct rte_gpu_callback *callback; 495 496 dev_id = dev->mpshared->info.dev_id; 497 rte_rwlock_read_lock(&gpu_callback_lock); 498 TAILQ_FOREACH(callback, &dev->callbacks, next) { 499 if (callback->event != event || callback->function == NULL) 500 continue; 501 callback->function(dev_id, event, callback->user_data); 502 } 503 rte_rwlock_read_unlock(&gpu_callback_lock); 504 } 505 506 int 507 rte_gpu_info_get(int16_t dev_id, struct rte_gpu_info *info) 508 { 509 struct rte_gpu *dev; 510 511 dev = gpu_get_by_id(dev_id); 512 if (dev == NULL) { 513 GPU_LOG(ERR, "query invalid device ID %d", dev_id); 514 rte_errno = ENODEV; 515 return -rte_errno; 516 } 517 if (info == NULL) { 518 GPU_LOG(ERR, "query without storage"); 519 rte_errno = EINVAL; 520 return -rte_errno; 521 } 522 523 if (dev->ops.dev_info_get == NULL) { 524 *info = dev->mpshared->info; 525 return 0; 526 } 527 return GPU_DRV_RET(dev->ops.dev_info_get(dev, info)); 528 } 529 530 void * 531 rte_gpu_mem_alloc(int16_t dev_id, size_t size, unsigned int align) 532 { 533 struct rte_gpu *dev; 534 void *ptr; 535 int ret; 536 537 dev = gpu_get_by_id(dev_id); 538 if (dev == NULL) { 539 GPU_LOG(ERR, "alloc mem for invalid device ID %d", dev_id); 540 rte_errno = ENODEV; 541 return NULL; 542 } 543 544 if (dev->ops.mem_alloc == NULL) { 545 GPU_LOG(ERR, "mem allocation not supported"); 546 rte_errno = ENOTSUP; 547 return NULL; 548 } 549 550 if (size == 0) /* dry-run */ 551 return NULL; 552 553 if (align && !rte_is_power_of_2(align)) { 554 GPU_LOG(ERR, "requested alignment is not a power of two %u", align); 555 rte_errno = EINVAL; 556 return NULL; 557 } 558 559 ret = dev->ops.mem_alloc(dev, size, align, &ptr); 560 561 switch (ret) { 562 case 0: 563 return ptr; 564 case -ENOMEM: 565 case -E2BIG: 566 rte_errno = -ret; 567 return NULL; 568 default: 569 rte_errno = -EPERM; 570 return NULL; 571 } 572 } 573 574 int 575 rte_gpu_mem_free(int16_t dev_id, void *ptr) 576 { 577 struct rte_gpu *dev; 578 579 dev = gpu_get_by_id(dev_id); 580 if (dev == NULL) { 581 GPU_LOG(ERR, "free mem for invalid device ID %d", dev_id); 582 rte_errno = ENODEV; 583 return -rte_errno; 584 } 585 586 if (dev->ops.mem_free == NULL) { 587 rte_errno = ENOTSUP; 588 return -rte_errno; 589 } 590 591 if (ptr == NULL) /* dry-run */ 592 return 0; 593 594 return GPU_DRV_RET(dev->ops.mem_free(dev, ptr)); 595 } 596 597 int 598 rte_gpu_mem_register(int16_t dev_id, size_t size, void *ptr) 599 { 600 struct rte_gpu *dev; 601 602 dev = gpu_get_by_id(dev_id); 603 if (dev == NULL) { 604 GPU_LOG(ERR, "alloc mem for invalid device ID %d", dev_id); 605 rte_errno = ENODEV; 606 return -rte_errno; 607 } 608 609 if (dev->ops.mem_register == NULL) { 610 GPU_LOG(ERR, "mem registration not supported"); 611 rte_errno = ENOTSUP; 612 return -rte_errno; 613 } 614 615 if (ptr == NULL || size == 0) /* dry-run */ 616 return 0; 617 618 return GPU_DRV_RET(dev->ops.mem_register(dev, size, ptr)); 619 } 620 621 int 622 rte_gpu_mem_unregister(int16_t dev_id, void *ptr) 623 { 624 struct rte_gpu *dev; 625 626 dev = gpu_get_by_id(dev_id); 627 if (dev == NULL) { 628 GPU_LOG(ERR, "unregister mem for invalid device ID %d", dev_id); 629 rte_errno = ENODEV; 630 return -rte_errno; 631 } 632 633 if (dev->ops.mem_unregister == NULL) { 634 rte_errno = ENOTSUP; 635 return -rte_errno; 636 } 637 638 if (ptr == NULL) /* dry-run */ 639 return 0; 640 641 return GPU_DRV_RET(dev->ops.mem_unregister(dev, ptr)); 642 } 643 644 void * 645 rte_gpu_mem_cpu_map(int16_t dev_id, size_t size, void *ptr) 646 { 647 struct rte_gpu *dev; 648 void *ptr_out; 649 int ret; 650 651 dev = gpu_get_by_id(dev_id); 652 if (dev == NULL) { 653 GPU_LOG(ERR, "mem CPU map for invalid device ID %d", dev_id); 654 rte_errno = ENODEV; 655 return NULL; 656 } 657 658 if (dev->ops.mem_cpu_map == NULL) { 659 GPU_LOG(ERR, "mem CPU map not supported"); 660 rte_errno = ENOTSUP; 661 return NULL; 662 } 663 664 if (ptr == NULL || size == 0) /* dry-run */ 665 return NULL; 666 667 ret = GPU_DRV_RET(dev->ops.mem_cpu_map(dev, size, ptr, &ptr_out)); 668 669 switch (ret) { 670 case 0: 671 return ptr_out; 672 case -ENOMEM: 673 case -E2BIG: 674 rte_errno = -ret; 675 return NULL; 676 default: 677 rte_errno = -EPERM; 678 return NULL; 679 } 680 } 681 682 int 683 rte_gpu_mem_cpu_unmap(int16_t dev_id, void *ptr) 684 { 685 struct rte_gpu *dev; 686 687 dev = gpu_get_by_id(dev_id); 688 if (dev == NULL) { 689 GPU_LOG(ERR, "cpu_unmap mem for invalid device ID %d", dev_id); 690 rte_errno = ENODEV; 691 return -rte_errno; 692 } 693 694 if (dev->ops.mem_cpu_unmap == NULL) { 695 rte_errno = ENOTSUP; 696 return -rte_errno; 697 } 698 699 if (ptr == NULL) /* dry-run */ 700 return 0; 701 702 return GPU_DRV_RET(dev->ops.mem_cpu_unmap(dev, ptr)); 703 } 704 705 int 706 rte_gpu_wmb(int16_t dev_id) 707 { 708 struct rte_gpu *dev; 709 710 dev = gpu_get_by_id(dev_id); 711 if (dev == NULL) { 712 GPU_LOG(ERR, "memory barrier for invalid device ID %d", dev_id); 713 rte_errno = ENODEV; 714 return -rte_errno; 715 } 716 717 if (dev->ops.wmb == NULL) { 718 rte_errno = ENOTSUP; 719 return -rte_errno; 720 } 721 return GPU_DRV_RET(dev->ops.wmb(dev)); 722 } 723 724 int 725 rte_gpu_comm_create_flag(uint16_t dev_id, struct rte_gpu_comm_flag *devflag, 726 enum rte_gpu_comm_flag_type mtype) 727 { 728 size_t flag_size; 729 int ret; 730 731 if (devflag == NULL) { 732 rte_errno = EINVAL; 733 return -rte_errno; 734 } 735 if (mtype != RTE_GPU_COMM_FLAG_CPU) { 736 rte_errno = EINVAL; 737 return -rte_errno; 738 } 739 740 flag_size = sizeof(uint32_t); 741 742 devflag->ptr = rte_zmalloc(NULL, flag_size, 0); 743 if (devflag->ptr == NULL) { 744 rte_errno = ENOMEM; 745 return -rte_errno; 746 } 747 748 ret = rte_gpu_mem_register(dev_id, flag_size, devflag->ptr); 749 if (ret < 0) { 750 rte_errno = ENOMEM; 751 return -rte_errno; 752 } 753 754 devflag->mtype = mtype; 755 devflag->dev_id = dev_id; 756 757 return 0; 758 } 759 760 int 761 rte_gpu_comm_destroy_flag(struct rte_gpu_comm_flag *devflag) 762 { 763 int ret; 764 765 if (devflag == NULL) { 766 rte_errno = EINVAL; 767 return -rte_errno; 768 } 769 770 ret = rte_gpu_mem_unregister(devflag->dev_id, devflag->ptr); 771 if (ret < 0) { 772 rte_errno = EINVAL; 773 return -1; 774 } 775 776 rte_free(devflag->ptr); 777 778 return 0; 779 } 780 781 int 782 rte_gpu_comm_set_flag(struct rte_gpu_comm_flag *devflag, uint32_t val) 783 { 784 if (devflag == NULL) { 785 rte_errno = EINVAL; 786 return -rte_errno; 787 } 788 789 if (devflag->mtype != RTE_GPU_COMM_FLAG_CPU) { 790 rte_errno = EINVAL; 791 return -rte_errno; 792 } 793 794 RTE_GPU_VOLATILE(*devflag->ptr) = val; 795 796 return 0; 797 } 798 799 int 800 rte_gpu_comm_get_flag_value(struct rte_gpu_comm_flag *devflag, uint32_t *val) 801 { 802 if (devflag == NULL) { 803 rte_errno = EINVAL; 804 return -rte_errno; 805 } 806 if (devflag->mtype != RTE_GPU_COMM_FLAG_CPU) { 807 rte_errno = EINVAL; 808 return -rte_errno; 809 } 810 811 *val = RTE_GPU_VOLATILE(*devflag->ptr); 812 813 return 0; 814 } 815 816 struct rte_gpu_comm_list * 817 rte_gpu_comm_create_list(uint16_t dev_id, 818 uint32_t num_comm_items) 819 { 820 struct rte_gpu_comm_list *comm_list; 821 uint32_t idx_l; 822 int ret; 823 struct rte_gpu *dev; 824 struct rte_gpu_info info; 825 826 if (num_comm_items == 0) { 827 rte_errno = EINVAL; 828 return NULL; 829 } 830 831 dev = gpu_get_by_id(dev_id); 832 if (dev == NULL) { 833 GPU_LOG(ERR, "memory barrier for invalid device ID %d", dev_id); 834 rte_errno = ENODEV; 835 return NULL; 836 } 837 838 ret = rte_gpu_info_get(dev_id, &info); 839 if (ret < 0) { 840 rte_errno = ENODEV; 841 return NULL; 842 } 843 844 comm_list = rte_zmalloc(NULL, 845 sizeof(struct rte_gpu_comm_list) * num_comm_items, 0); 846 if (comm_list == NULL) { 847 rte_errno = ENOMEM; 848 return NULL; 849 } 850 851 ret = rte_gpu_mem_register(dev_id, 852 sizeof(struct rte_gpu_comm_list) * num_comm_items, comm_list); 853 if (ret < 0) { 854 rte_errno = ENOMEM; 855 return NULL; 856 } 857 858 /* 859 * Use GPU memory CPU map feature if enabled in the driver 860 * to allocate the status flags of the list. 861 * Allocating this flag in GPU memory will reduce 862 * the latency when GPU workload is polling this flag. 863 */ 864 comm_list[0].status_d = rte_gpu_mem_alloc(dev_id, 865 sizeof(enum rte_gpu_comm_list_status) * num_comm_items, 866 info.page_size); 867 if (ret < 0) { 868 rte_errno = ENOMEM; 869 return NULL; 870 } 871 872 comm_list[0].status_h = rte_gpu_mem_cpu_map(dev_id, 873 sizeof(enum rte_gpu_comm_list_status) * num_comm_items, 874 comm_list[0].status_d); 875 if (comm_list[0].status_h == NULL) { 876 /* 877 * If CPU mapping is not supported by driver 878 * use regular CPU registered memory. 879 */ 880 comm_list[0].status_h = rte_zmalloc(NULL, 881 sizeof(enum rte_gpu_comm_list_status) * num_comm_items, 0); 882 if (comm_list[0].status_h == NULL) { 883 rte_errno = ENOMEM; 884 return NULL; 885 } 886 887 ret = rte_gpu_mem_register(dev_id, 888 sizeof(enum rte_gpu_comm_list_status) * num_comm_items, 889 comm_list[0].status_h); 890 if (ret < 0) { 891 rte_errno = ENOMEM; 892 return NULL; 893 } 894 895 comm_list[0].status_d = comm_list[0].status_h; 896 } 897 898 for (idx_l = 0; idx_l < num_comm_items; idx_l++) { 899 comm_list[idx_l].pkt_list = rte_zmalloc(NULL, 900 sizeof(struct rte_gpu_comm_pkt) * RTE_GPU_COMM_LIST_PKTS_MAX, 0); 901 if (comm_list[idx_l].pkt_list == NULL) { 902 rte_errno = ENOMEM; 903 return NULL; 904 } 905 906 ret = rte_gpu_mem_register(dev_id, 907 sizeof(struct rte_gpu_comm_pkt) * RTE_GPU_COMM_LIST_PKTS_MAX, 908 comm_list[idx_l].pkt_list); 909 if (ret < 0) { 910 rte_errno = ENOMEM; 911 return NULL; 912 } 913 914 comm_list[idx_l].num_pkts = 0; 915 comm_list[idx_l].dev_id = dev_id; 916 917 comm_list[idx_l].mbufs = rte_zmalloc(NULL, 918 sizeof(struct rte_mbuf *) * RTE_GPU_COMM_LIST_PKTS_MAX, 0); 919 if (comm_list[idx_l].mbufs == NULL) { 920 rte_errno = ENOMEM; 921 return NULL; 922 } 923 924 if (idx_l > 0) { 925 comm_list[idx_l].status_h = &(comm_list[0].status_h[idx_l]); 926 comm_list[idx_l].status_d = &(comm_list[0].status_d[idx_l]); 927 928 ret = rte_gpu_comm_set_status(&comm_list[idx_l], RTE_GPU_COMM_LIST_FREE); 929 if (ret < 0) { 930 rte_errno = ENOMEM; 931 return NULL; 932 } 933 } 934 } 935 936 return comm_list; 937 } 938 939 int 940 rte_gpu_comm_destroy_list(struct rte_gpu_comm_list *comm_list, 941 uint32_t num_comm_items) 942 { 943 uint32_t idx_l; 944 int ret; 945 uint16_t dev_id; 946 947 if (comm_list == NULL) { 948 rte_errno = EINVAL; 949 return -rte_errno; 950 } 951 952 dev_id = comm_list[0].dev_id; 953 954 for (idx_l = 0; idx_l < num_comm_items; idx_l++) { 955 ret = rte_gpu_mem_unregister(dev_id, comm_list[idx_l].pkt_list); 956 if (ret < 0) { 957 rte_errno = EINVAL; 958 return -1; 959 } 960 961 rte_free(comm_list[idx_l].pkt_list); 962 rte_free(comm_list[idx_l].mbufs); 963 } 964 965 ret = rte_gpu_mem_unregister(dev_id, comm_list); 966 if (ret < 0) { 967 rte_errno = EINVAL; 968 return -1; 969 } 970 971 ret = rte_gpu_mem_cpu_unmap(dev_id, comm_list[0].status_d); 972 if (ret == 0) { 973 rte_gpu_mem_free(dev_id, comm_list[0].status_d); 974 } else { 975 rte_gpu_mem_unregister(dev_id, comm_list[0].status_h); 976 rte_free(comm_list[0].status_h); 977 } 978 979 rte_free(comm_list); 980 981 return 0; 982 } 983 984 int 985 rte_gpu_comm_populate_list_pkts(struct rte_gpu_comm_list *comm_list_item, 986 struct rte_mbuf **mbufs, uint32_t num_mbufs) 987 { 988 uint32_t idx; 989 int ret; 990 991 if (comm_list_item == NULL || comm_list_item->pkt_list == NULL || 992 mbufs == NULL || num_mbufs > RTE_GPU_COMM_LIST_PKTS_MAX) { 993 rte_errno = EINVAL; 994 return -rte_errno; 995 } 996 997 for (idx = 0; idx < num_mbufs; idx++) { 998 /* support only unchained mbufs */ 999 if (unlikely((mbufs[idx]->nb_segs > 1) || 1000 (mbufs[idx]->next != NULL) || 1001 (mbufs[idx]->data_len != mbufs[idx]->pkt_len))) { 1002 rte_errno = ENOTSUP; 1003 return -rte_errno; 1004 } 1005 comm_list_item->pkt_list[idx].addr = 1006 rte_pktmbuf_mtod_offset(mbufs[idx], uintptr_t, 0); 1007 comm_list_item->pkt_list[idx].size = mbufs[idx]->pkt_len; 1008 comm_list_item->mbufs[idx] = mbufs[idx]; 1009 } 1010 1011 RTE_GPU_VOLATILE(comm_list_item->num_pkts) = num_mbufs; 1012 rte_gpu_wmb(comm_list_item->dev_id); 1013 ret = rte_gpu_comm_set_status(comm_list_item, RTE_GPU_COMM_LIST_READY); 1014 if (ret < 0) { 1015 rte_errno = EINVAL; 1016 return -rte_errno; 1017 } 1018 1019 return 0; 1020 } 1021 1022 int 1023 rte_gpu_comm_set_status(struct rte_gpu_comm_list *comm_list_item, 1024 enum rte_gpu_comm_list_status status) 1025 { 1026 if (comm_list_item == NULL) { 1027 rte_errno = EINVAL; 1028 return -rte_errno; 1029 } 1030 1031 RTE_GPU_VOLATILE(comm_list_item->status_h[0]) = status; 1032 1033 return 0; 1034 } 1035 1036 int 1037 rte_gpu_comm_get_status(struct rte_gpu_comm_list *comm_list_item, 1038 enum rte_gpu_comm_list_status *status) 1039 { 1040 if (comm_list_item == NULL || status == NULL) { 1041 rte_errno = EINVAL; 1042 return -rte_errno; 1043 } 1044 1045 *status = RTE_GPU_VOLATILE(comm_list_item->status_h[0]); 1046 1047 return 0; 1048 } 1049 1050 int 1051 rte_gpu_comm_cleanup_list(struct rte_gpu_comm_list *comm_list_item) 1052 { 1053 uint32_t idx = 0; 1054 enum rte_gpu_comm_list_status status; 1055 int ret; 1056 1057 if (comm_list_item == NULL) { 1058 rte_errno = EINVAL; 1059 return -rte_errno; 1060 } 1061 1062 ret = rte_gpu_comm_get_status(comm_list_item, &status); 1063 if (ret < 0) { 1064 rte_errno = EINVAL; 1065 return -rte_errno; 1066 } 1067 1068 if (status == RTE_GPU_COMM_LIST_READY) { 1069 GPU_LOG(ERR, "packet list is still in progress"); 1070 rte_errno = EINVAL; 1071 return -rte_errno; 1072 } 1073 1074 for (idx = 0; idx < RTE_GPU_COMM_LIST_PKTS_MAX; idx++) { 1075 if (comm_list_item->pkt_list[idx].addr == 0) 1076 break; 1077 1078 comm_list_item->pkt_list[idx].addr = 0; 1079 comm_list_item->pkt_list[idx].size = 0; 1080 comm_list_item->mbufs[idx] = NULL; 1081 } 1082 1083 ret = rte_gpu_comm_set_status(comm_list_item, RTE_GPU_COMM_LIST_FREE); 1084 if (ret < 0) { 1085 rte_errno = EINVAL; 1086 return -rte_errno; 1087 } 1088 RTE_GPU_VOLATILE(comm_list_item->num_pkts) = 0; 1089 rte_mb(); 1090 1091 return 0; 1092 } 1093