1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) 2021 NVIDIA Corporation & Affiliates 3 */ 4 5 #include <rte_eal.h> 6 #include <rte_tailq.h> 7 #include <rte_string_fns.h> 8 #include <rte_memzone.h> 9 #include <rte_malloc.h> 10 #include <rte_errno.h> 11 #include <rte_log.h> 12 13 #include "rte_gpudev.h" 14 #include "gpudev_driver.h" 15 16 /* Logging */ 17 RTE_LOG_REGISTER_DEFAULT(gpu_logtype, NOTICE); 18 #define GPU_LOG(level, ...) \ 19 rte_log(RTE_LOG_ ## level, gpu_logtype, RTE_FMT("gpu: " \ 20 RTE_FMT_HEAD(__VA_ARGS__, ) "\n", RTE_FMT_TAIL(__VA_ARGS__, ))) 21 22 /* Set any driver error as EPERM */ 23 #define GPU_DRV_RET(function) \ 24 ((function != 0) ? -(rte_errno = EPERM) : (rte_errno = 0)) 25 26 /* Array of devices */ 27 static struct rte_gpu *gpus; 28 /* Number of currently valid devices */ 29 static int16_t gpu_max; 30 /* Number of currently valid devices */ 31 static int16_t gpu_count; 32 33 /* Shared memory between processes. */ 34 static const char *GPU_MEMZONE = "rte_gpu_shared"; 35 static struct { 36 __extension__ struct rte_gpu_mpshared gpus[0]; 37 } *gpu_shared_mem; 38 39 /* Event callback object */ 40 struct rte_gpu_callback { 41 TAILQ_ENTRY(rte_gpu_callback) next; 42 rte_gpu_callback_t *function; 43 void *user_data; 44 enum rte_gpu_event event; 45 }; 46 static rte_rwlock_t gpu_callback_lock = RTE_RWLOCK_INITIALIZER; 47 static void gpu_free_callbacks(struct rte_gpu *dev); 48 49 int 50 rte_gpu_init(size_t dev_max) 51 { 52 if (dev_max == 0 || dev_max > INT16_MAX) { 53 GPU_LOG(ERR, "invalid array size"); 54 rte_errno = EINVAL; 55 return -rte_errno; 56 } 57 58 /* No lock, it must be called before or during first probing. */ 59 if (gpus != NULL) { 60 GPU_LOG(ERR, "already initialized"); 61 rte_errno = EBUSY; 62 return -rte_errno; 63 } 64 65 gpus = calloc(dev_max, sizeof(struct rte_gpu)); 66 if (gpus == NULL) { 67 GPU_LOG(ERR, "cannot initialize library"); 68 rte_errno = ENOMEM; 69 return -rte_errno; 70 } 71 72 gpu_max = dev_max; 73 return 0; 74 } 75 76 uint16_t 77 rte_gpu_count_avail(void) 78 { 79 return gpu_count; 80 } 81 82 bool 83 rte_gpu_is_valid(int16_t dev_id) 84 { 85 if (dev_id >= 0 && dev_id < gpu_max && 86 gpus[dev_id].process_state == RTE_GPU_STATE_INITIALIZED) 87 return true; 88 return false; 89 } 90 91 static bool 92 gpu_match_parent(int16_t dev_id, int16_t parent) 93 { 94 if (parent == RTE_GPU_ID_ANY) 95 return true; 96 return gpus[dev_id].mpshared->info.parent == parent; 97 } 98 99 int16_t 100 rte_gpu_find_next(int16_t dev_id, int16_t parent) 101 { 102 if (dev_id < 0) 103 dev_id = 0; 104 while (dev_id < gpu_max && 105 (gpus[dev_id].process_state == RTE_GPU_STATE_UNUSED || 106 !gpu_match_parent(dev_id, parent))) 107 dev_id++; 108 109 if (dev_id >= gpu_max) 110 return RTE_GPU_ID_NONE; 111 return dev_id; 112 } 113 114 static int16_t 115 gpu_find_free_id(void) 116 { 117 int16_t dev_id; 118 119 for (dev_id = 0; dev_id < gpu_max; dev_id++) { 120 if (gpus[dev_id].process_state == RTE_GPU_STATE_UNUSED) 121 return dev_id; 122 } 123 return RTE_GPU_ID_NONE; 124 } 125 126 static struct rte_gpu * 127 gpu_get_by_id(int16_t dev_id) 128 { 129 if (!rte_gpu_is_valid(dev_id)) 130 return NULL; 131 return &gpus[dev_id]; 132 } 133 134 struct rte_gpu * 135 rte_gpu_get_by_name(const char *name) 136 { 137 int16_t dev_id; 138 struct rte_gpu *dev; 139 140 if (name == NULL) { 141 rte_errno = EINVAL; 142 return NULL; 143 } 144 145 RTE_GPU_FOREACH(dev_id) { 146 dev = &gpus[dev_id]; 147 if (strncmp(name, dev->mpshared->name, RTE_DEV_NAME_MAX_LEN) == 0) 148 return dev; 149 } 150 return NULL; 151 } 152 153 static int 154 gpu_shared_mem_init(void) 155 { 156 const struct rte_memzone *memzone; 157 158 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 159 memzone = rte_memzone_reserve(GPU_MEMZONE, 160 sizeof(*gpu_shared_mem) + 161 sizeof(*gpu_shared_mem->gpus) * gpu_max, 162 SOCKET_ID_ANY, 0); 163 } else { 164 memzone = rte_memzone_lookup(GPU_MEMZONE); 165 } 166 if (memzone == NULL) { 167 GPU_LOG(ERR, "cannot initialize shared memory"); 168 rte_errno = ENOMEM; 169 return -rte_errno; 170 } 171 172 gpu_shared_mem = memzone->addr; 173 return 0; 174 } 175 176 struct rte_gpu * 177 rte_gpu_allocate(const char *name) 178 { 179 int16_t dev_id; 180 struct rte_gpu *dev; 181 182 if (rte_eal_process_type() != RTE_PROC_PRIMARY) { 183 GPU_LOG(ERR, "only primary process can allocate device"); 184 rte_errno = EPERM; 185 return NULL; 186 } 187 if (name == NULL) { 188 GPU_LOG(ERR, "allocate device without a name"); 189 rte_errno = EINVAL; 190 return NULL; 191 } 192 193 /* implicit initialization of library before adding first device */ 194 if (gpus == NULL && rte_gpu_init(RTE_GPU_DEFAULT_MAX) < 0) 195 return NULL; 196 197 /* initialize shared memory before adding first device */ 198 if (gpu_shared_mem == NULL && gpu_shared_mem_init() < 0) 199 return NULL; 200 201 if (rte_gpu_get_by_name(name) != NULL) { 202 GPU_LOG(ERR, "device with name %s already exists", name); 203 rte_errno = EEXIST; 204 return NULL; 205 } 206 dev_id = gpu_find_free_id(); 207 if (dev_id == RTE_GPU_ID_NONE) { 208 GPU_LOG(ERR, "reached maximum number of devices"); 209 rte_errno = ENOENT; 210 return NULL; 211 } 212 213 dev = &gpus[dev_id]; 214 memset(dev, 0, sizeof(*dev)); 215 216 dev->mpshared = &gpu_shared_mem->gpus[dev_id]; 217 memset(dev->mpshared, 0, sizeof(*dev->mpshared)); 218 219 if (rte_strscpy(dev->mpshared->name, name, RTE_DEV_NAME_MAX_LEN) < 0) { 220 GPU_LOG(ERR, "device name too long: %s", name); 221 rte_errno = ENAMETOOLONG; 222 return NULL; 223 } 224 dev->mpshared->info.name = dev->mpshared->name; 225 dev->mpshared->info.dev_id = dev_id; 226 dev->mpshared->info.numa_node = -1; 227 dev->mpshared->info.parent = RTE_GPU_ID_NONE; 228 TAILQ_INIT(&dev->callbacks); 229 __atomic_fetch_add(&dev->mpshared->process_refcnt, 1, __ATOMIC_RELAXED); 230 231 gpu_count++; 232 GPU_LOG(DEBUG, "new device %s (id %d) of total %d", 233 name, dev_id, gpu_count); 234 return dev; 235 } 236 237 struct rte_gpu * 238 rte_gpu_attach(const char *name) 239 { 240 int16_t dev_id; 241 struct rte_gpu *dev; 242 struct rte_gpu_mpshared *shared_dev; 243 244 if (rte_eal_process_type() != RTE_PROC_SECONDARY) { 245 GPU_LOG(ERR, "only secondary process can attach device"); 246 rte_errno = EPERM; 247 return NULL; 248 } 249 if (name == NULL) { 250 GPU_LOG(ERR, "attach device without a name"); 251 rte_errno = EINVAL; 252 return NULL; 253 } 254 255 /* implicit initialization of library before adding first device */ 256 if (gpus == NULL && rte_gpu_init(RTE_GPU_DEFAULT_MAX) < 0) 257 return NULL; 258 259 /* initialize shared memory before adding first device */ 260 if (gpu_shared_mem == NULL && gpu_shared_mem_init() < 0) 261 return NULL; 262 263 for (dev_id = 0; dev_id < gpu_max; dev_id++) { 264 shared_dev = &gpu_shared_mem->gpus[dev_id]; 265 if (strncmp(name, shared_dev->name, RTE_DEV_NAME_MAX_LEN) == 0) 266 break; 267 } 268 if (dev_id >= gpu_max) { 269 GPU_LOG(ERR, "device with name %s not found", name); 270 rte_errno = ENOENT; 271 return NULL; 272 } 273 dev = &gpus[dev_id]; 274 memset(dev, 0, sizeof(*dev)); 275 276 TAILQ_INIT(&dev->callbacks); 277 dev->mpshared = shared_dev; 278 __atomic_fetch_add(&dev->mpshared->process_refcnt, 1, __ATOMIC_RELAXED); 279 280 gpu_count++; 281 GPU_LOG(DEBUG, "attached device %s (id %d) of total %d", 282 name, dev_id, gpu_count); 283 return dev; 284 } 285 286 int16_t 287 rte_gpu_add_child(const char *name, int16_t parent, uint64_t child_context) 288 { 289 struct rte_gpu *dev; 290 291 if (!rte_gpu_is_valid(parent)) { 292 GPU_LOG(ERR, "add child to invalid parent ID %d", parent); 293 rte_errno = ENODEV; 294 return -rte_errno; 295 } 296 297 dev = rte_gpu_allocate(name); 298 if (dev == NULL) 299 return -rte_errno; 300 301 dev->mpshared->info.parent = parent; 302 dev->mpshared->info.context = child_context; 303 304 rte_gpu_complete_new(dev); 305 return dev->mpshared->info.dev_id; 306 } 307 308 void 309 rte_gpu_complete_new(struct rte_gpu *dev) 310 { 311 if (dev == NULL) 312 return; 313 314 dev->process_state = RTE_GPU_STATE_INITIALIZED; 315 rte_gpu_notify(dev, RTE_GPU_EVENT_NEW); 316 } 317 318 int 319 rte_gpu_release(struct rte_gpu *dev) 320 { 321 int16_t dev_id, child; 322 323 if (dev == NULL) { 324 rte_errno = ENODEV; 325 return -rte_errno; 326 } 327 dev_id = dev->mpshared->info.dev_id; 328 RTE_GPU_FOREACH_CHILD(child, dev_id) { 329 GPU_LOG(ERR, "cannot release device %d with child %d", 330 dev_id, child); 331 rte_errno = EBUSY; 332 return -rte_errno; 333 } 334 335 GPU_LOG(DEBUG, "free device %s (id %d)", 336 dev->mpshared->info.name, dev->mpshared->info.dev_id); 337 rte_gpu_notify(dev, RTE_GPU_EVENT_DEL); 338 339 gpu_free_callbacks(dev); 340 dev->process_state = RTE_GPU_STATE_UNUSED; 341 __atomic_fetch_sub(&dev->mpshared->process_refcnt, 1, __ATOMIC_RELAXED); 342 gpu_count--; 343 344 return 0; 345 } 346 347 int 348 rte_gpu_close(int16_t dev_id) 349 { 350 int firsterr, binerr; 351 int *lasterr = &firsterr; 352 struct rte_gpu *dev; 353 354 dev = gpu_get_by_id(dev_id); 355 if (dev == NULL) { 356 GPU_LOG(ERR, "close invalid device ID %d", dev_id); 357 rte_errno = ENODEV; 358 return -rte_errno; 359 } 360 361 if (dev->ops.dev_close != NULL) { 362 *lasterr = GPU_DRV_RET(dev->ops.dev_close(dev)); 363 if (*lasterr != 0) 364 lasterr = &binerr; 365 } 366 367 *lasterr = rte_gpu_release(dev); 368 369 rte_errno = -firsterr; 370 return firsterr; 371 } 372 373 int 374 rte_gpu_callback_register(int16_t dev_id, enum rte_gpu_event event, 375 rte_gpu_callback_t *function, void *user_data) 376 { 377 int16_t next_dev, last_dev; 378 struct rte_gpu_callback_list *callbacks; 379 struct rte_gpu_callback *callback; 380 381 if (!rte_gpu_is_valid(dev_id) && dev_id != RTE_GPU_ID_ANY) { 382 GPU_LOG(ERR, "register callback of invalid ID %d", dev_id); 383 rte_errno = ENODEV; 384 return -rte_errno; 385 } 386 if (function == NULL) { 387 GPU_LOG(ERR, "cannot register callback without function"); 388 rte_errno = EINVAL; 389 return -rte_errno; 390 } 391 392 if (dev_id == RTE_GPU_ID_ANY) { 393 next_dev = 0; 394 last_dev = gpu_max - 1; 395 } else { 396 next_dev = last_dev = dev_id; 397 } 398 399 rte_rwlock_write_lock(&gpu_callback_lock); 400 do { 401 callbacks = &gpus[next_dev].callbacks; 402 403 /* check if not already registered */ 404 TAILQ_FOREACH(callback, callbacks, next) { 405 if (callback->event == event && 406 callback->function == function && 407 callback->user_data == user_data) { 408 GPU_LOG(INFO, "callback already registered"); 409 return 0; 410 } 411 } 412 413 callback = malloc(sizeof(*callback)); 414 if (callback == NULL) { 415 GPU_LOG(ERR, "cannot allocate callback"); 416 return -ENOMEM; 417 } 418 callback->function = function; 419 callback->user_data = user_data; 420 callback->event = event; 421 TAILQ_INSERT_TAIL(callbacks, callback, next); 422 423 } while (++next_dev <= last_dev); 424 rte_rwlock_write_unlock(&gpu_callback_lock); 425 426 return 0; 427 } 428 429 int 430 rte_gpu_callback_unregister(int16_t dev_id, enum rte_gpu_event event, 431 rte_gpu_callback_t *function, void *user_data) 432 { 433 int16_t next_dev, last_dev; 434 struct rte_gpu_callback_list *callbacks; 435 struct rte_gpu_callback *callback, *nextcb; 436 437 if (!rte_gpu_is_valid(dev_id) && dev_id != RTE_GPU_ID_ANY) { 438 GPU_LOG(ERR, "unregister callback of invalid ID %d", dev_id); 439 rte_errno = ENODEV; 440 return -rte_errno; 441 } 442 if (function == NULL) { 443 GPU_LOG(ERR, "cannot unregister callback without function"); 444 rte_errno = EINVAL; 445 return -rte_errno; 446 } 447 448 if (dev_id == RTE_GPU_ID_ANY) { 449 next_dev = 0; 450 last_dev = gpu_max - 1; 451 } else { 452 next_dev = last_dev = dev_id; 453 } 454 455 rte_rwlock_write_lock(&gpu_callback_lock); 456 do { 457 callbacks = &gpus[next_dev].callbacks; 458 RTE_TAILQ_FOREACH_SAFE(callback, callbacks, next, nextcb) { 459 if (callback->event != event || 460 callback->function != function || 461 (callback->user_data != user_data && 462 user_data != (void *)-1)) 463 continue; 464 TAILQ_REMOVE(callbacks, callback, next); 465 free(callback); 466 } 467 } while (++next_dev <= last_dev); 468 rte_rwlock_write_unlock(&gpu_callback_lock); 469 470 return 0; 471 } 472 473 static void 474 gpu_free_callbacks(struct rte_gpu *dev) 475 { 476 struct rte_gpu_callback_list *callbacks; 477 struct rte_gpu_callback *callback, *nextcb; 478 479 callbacks = &dev->callbacks; 480 rte_rwlock_write_lock(&gpu_callback_lock); 481 RTE_TAILQ_FOREACH_SAFE(callback, callbacks, next, nextcb) { 482 TAILQ_REMOVE(callbacks, callback, next); 483 free(callback); 484 } 485 rte_rwlock_write_unlock(&gpu_callback_lock); 486 } 487 488 void 489 rte_gpu_notify(struct rte_gpu *dev, enum rte_gpu_event event) 490 { 491 int16_t dev_id; 492 struct rte_gpu_callback *callback; 493 494 dev_id = dev->mpshared->info.dev_id; 495 rte_rwlock_read_lock(&gpu_callback_lock); 496 TAILQ_FOREACH(callback, &dev->callbacks, next) { 497 if (callback->event != event || callback->function == NULL) 498 continue; 499 callback->function(dev_id, event, callback->user_data); 500 } 501 rte_rwlock_read_unlock(&gpu_callback_lock); 502 } 503 504 int 505 rte_gpu_info_get(int16_t dev_id, struct rte_gpu_info *info) 506 { 507 struct rte_gpu *dev; 508 509 dev = gpu_get_by_id(dev_id); 510 if (dev == NULL) { 511 GPU_LOG(ERR, "query invalid device ID %d", dev_id); 512 rte_errno = ENODEV; 513 return -rte_errno; 514 } 515 if (info == NULL) { 516 GPU_LOG(ERR, "query without storage"); 517 rte_errno = EINVAL; 518 return -rte_errno; 519 } 520 521 if (dev->ops.dev_info_get == NULL) { 522 *info = dev->mpshared->info; 523 return 0; 524 } 525 return GPU_DRV_RET(dev->ops.dev_info_get(dev, info)); 526 } 527 528 void * 529 rte_gpu_mem_alloc(int16_t dev_id, size_t size, unsigned int align) 530 { 531 struct rte_gpu *dev; 532 void *ptr; 533 int ret; 534 535 dev = gpu_get_by_id(dev_id); 536 if (dev == NULL) { 537 GPU_LOG(ERR, "alloc mem for invalid device ID %d", dev_id); 538 rte_errno = ENODEV; 539 return NULL; 540 } 541 542 if (dev->ops.mem_alloc == NULL) { 543 GPU_LOG(ERR, "mem allocation not supported"); 544 rte_errno = ENOTSUP; 545 return NULL; 546 } 547 548 if (size == 0) /* dry-run */ 549 return NULL; 550 551 if (align && !rte_is_power_of_2(align)) { 552 GPU_LOG(ERR, "requested alignment is not a power of two %u", align); 553 rte_errno = EINVAL; 554 return NULL; 555 } 556 557 ret = dev->ops.mem_alloc(dev, size, align, &ptr); 558 559 switch (ret) { 560 case 0: 561 return ptr; 562 case -ENOMEM: 563 case -E2BIG: 564 rte_errno = -ret; 565 return NULL; 566 default: 567 rte_errno = -EPERM; 568 return NULL; 569 } 570 } 571 572 int 573 rte_gpu_mem_free(int16_t dev_id, void *ptr) 574 { 575 struct rte_gpu *dev; 576 577 dev = gpu_get_by_id(dev_id); 578 if (dev == NULL) { 579 GPU_LOG(ERR, "free mem for invalid device ID %d", dev_id); 580 rte_errno = ENODEV; 581 return -rte_errno; 582 } 583 584 if (dev->ops.mem_free == NULL) { 585 rte_errno = ENOTSUP; 586 return -rte_errno; 587 } 588 589 if (ptr == NULL) /* dry-run */ 590 return 0; 591 592 return GPU_DRV_RET(dev->ops.mem_free(dev, ptr)); 593 } 594 595 int 596 rte_gpu_mem_register(int16_t dev_id, size_t size, void *ptr) 597 { 598 struct rte_gpu *dev; 599 600 dev = gpu_get_by_id(dev_id); 601 if (dev == NULL) { 602 GPU_LOG(ERR, "alloc mem for invalid device ID %d", dev_id); 603 rte_errno = ENODEV; 604 return -rte_errno; 605 } 606 607 if (dev->ops.mem_register == NULL) { 608 GPU_LOG(ERR, "mem registration not supported"); 609 rte_errno = ENOTSUP; 610 return -rte_errno; 611 } 612 613 if (ptr == NULL || size == 0) /* dry-run */ 614 return 0; 615 616 return GPU_DRV_RET(dev->ops.mem_register(dev, size, ptr)); 617 } 618 619 int 620 rte_gpu_mem_unregister(int16_t dev_id, void *ptr) 621 { 622 struct rte_gpu *dev; 623 624 dev = gpu_get_by_id(dev_id); 625 if (dev == NULL) { 626 GPU_LOG(ERR, "unregister mem for invalid device ID %d", dev_id); 627 rte_errno = ENODEV; 628 return -rte_errno; 629 } 630 631 if (dev->ops.mem_unregister == NULL) { 632 rte_errno = ENOTSUP; 633 return -rte_errno; 634 } 635 636 if (ptr == NULL) /* dry-run */ 637 return 0; 638 639 return GPU_DRV_RET(dev->ops.mem_unregister(dev, ptr)); 640 } 641 642 void * 643 rte_gpu_mem_cpu_map(int16_t dev_id, size_t size, void *ptr) 644 { 645 struct rte_gpu *dev; 646 void *ptr_out; 647 int ret; 648 649 dev = gpu_get_by_id(dev_id); 650 if (dev == NULL) { 651 GPU_LOG(ERR, "mem CPU map for invalid device ID %d", dev_id); 652 rte_errno = ENODEV; 653 return NULL; 654 } 655 656 if (dev->ops.mem_cpu_map == NULL) { 657 GPU_LOG(ERR, "mem CPU map not supported"); 658 rte_errno = ENOTSUP; 659 return NULL; 660 } 661 662 if (ptr == NULL || size == 0) /* dry-run */ 663 return NULL; 664 665 ret = GPU_DRV_RET(dev->ops.mem_cpu_map(dev, size, ptr, &ptr_out)); 666 667 switch (ret) { 668 case 0: 669 return ptr_out; 670 case -ENOMEM: 671 case -E2BIG: 672 rte_errno = -ret; 673 return NULL; 674 default: 675 rte_errno = -EPERM; 676 return NULL; 677 } 678 } 679 680 int 681 rte_gpu_mem_cpu_unmap(int16_t dev_id, void *ptr) 682 { 683 struct rte_gpu *dev; 684 685 dev = gpu_get_by_id(dev_id); 686 if (dev == NULL) { 687 GPU_LOG(ERR, "cpu_unmap mem for invalid device ID %d", dev_id); 688 rte_errno = ENODEV; 689 return -rte_errno; 690 } 691 692 if (dev->ops.mem_cpu_unmap == NULL) { 693 rte_errno = ENOTSUP; 694 return -rte_errno; 695 } 696 697 if (ptr == NULL) /* dry-run */ 698 return 0; 699 700 return GPU_DRV_RET(dev->ops.mem_cpu_unmap(dev, ptr)); 701 } 702 703 int 704 rte_gpu_wmb(int16_t dev_id) 705 { 706 struct rte_gpu *dev; 707 708 dev = gpu_get_by_id(dev_id); 709 if (dev == NULL) { 710 GPU_LOG(ERR, "memory barrier for invalid device ID %d", dev_id); 711 rte_errno = ENODEV; 712 return -rte_errno; 713 } 714 715 if (dev->ops.wmb == NULL) { 716 rte_errno = ENOTSUP; 717 return -rte_errno; 718 } 719 return GPU_DRV_RET(dev->ops.wmb(dev)); 720 } 721 722 int 723 rte_gpu_comm_create_flag(uint16_t dev_id, struct rte_gpu_comm_flag *devflag, 724 enum rte_gpu_comm_flag_type mtype) 725 { 726 size_t flag_size; 727 int ret; 728 729 if (devflag == NULL) { 730 rte_errno = EINVAL; 731 return -rte_errno; 732 } 733 if (mtype != RTE_GPU_COMM_FLAG_CPU) { 734 rte_errno = EINVAL; 735 return -rte_errno; 736 } 737 738 flag_size = sizeof(uint32_t); 739 740 devflag->ptr = rte_zmalloc(NULL, flag_size, 0); 741 if (devflag->ptr == NULL) { 742 rte_errno = ENOMEM; 743 return -rte_errno; 744 } 745 746 ret = rte_gpu_mem_register(dev_id, flag_size, devflag->ptr); 747 if (ret < 0) { 748 rte_errno = ENOMEM; 749 return -rte_errno; 750 } 751 752 devflag->mtype = mtype; 753 devflag->dev_id = dev_id; 754 755 return 0; 756 } 757 758 int 759 rte_gpu_comm_destroy_flag(struct rte_gpu_comm_flag *devflag) 760 { 761 int ret; 762 763 if (devflag == NULL) { 764 rte_errno = EINVAL; 765 return -rte_errno; 766 } 767 768 ret = rte_gpu_mem_unregister(devflag->dev_id, devflag->ptr); 769 if (ret < 0) { 770 rte_errno = EINVAL; 771 return -1; 772 } 773 774 rte_free(devflag->ptr); 775 776 return 0; 777 } 778 779 int 780 rte_gpu_comm_set_flag(struct rte_gpu_comm_flag *devflag, uint32_t val) 781 { 782 if (devflag == NULL) { 783 rte_errno = EINVAL; 784 return -rte_errno; 785 } 786 787 if (devflag->mtype != RTE_GPU_COMM_FLAG_CPU) { 788 rte_errno = EINVAL; 789 return -rte_errno; 790 } 791 792 RTE_GPU_VOLATILE(*devflag->ptr) = val; 793 794 return 0; 795 } 796 797 int 798 rte_gpu_comm_get_flag_value(struct rte_gpu_comm_flag *devflag, uint32_t *val) 799 { 800 if (devflag == NULL) { 801 rte_errno = EINVAL; 802 return -rte_errno; 803 } 804 if (devflag->mtype != RTE_GPU_COMM_FLAG_CPU) { 805 rte_errno = EINVAL; 806 return -rte_errno; 807 } 808 809 *val = RTE_GPU_VOLATILE(*devflag->ptr); 810 811 return 0; 812 } 813 814 struct rte_gpu_comm_list * 815 rte_gpu_comm_create_list(uint16_t dev_id, 816 uint32_t num_comm_items) 817 { 818 struct rte_gpu_comm_list *comm_list; 819 uint32_t idx_l; 820 int ret; 821 struct rte_gpu *dev; 822 struct rte_gpu_info info; 823 824 if (num_comm_items == 0) { 825 rte_errno = EINVAL; 826 return NULL; 827 } 828 829 dev = gpu_get_by_id(dev_id); 830 if (dev == NULL) { 831 GPU_LOG(ERR, "memory barrier for invalid device ID %d", dev_id); 832 rte_errno = ENODEV; 833 return NULL; 834 } 835 836 ret = rte_gpu_info_get(dev_id, &info); 837 if (ret < 0) { 838 rte_errno = ENODEV; 839 return NULL; 840 } 841 842 comm_list = rte_zmalloc(NULL, 843 sizeof(struct rte_gpu_comm_list) * num_comm_items, 0); 844 if (comm_list == NULL) { 845 rte_errno = ENOMEM; 846 return NULL; 847 } 848 849 ret = rte_gpu_mem_register(dev_id, 850 sizeof(struct rte_gpu_comm_list) * num_comm_items, comm_list); 851 if (ret < 0) { 852 rte_errno = ENOMEM; 853 return NULL; 854 } 855 856 /* 857 * Use GPU memory CPU map feature if enabled in the driver 858 * to allocate the status flags of the list. 859 * Allocating this flag in GPU memory will reduce 860 * the latency when GPU workload is polling this flag. 861 */ 862 comm_list[0].status_d = rte_gpu_mem_alloc(dev_id, 863 sizeof(enum rte_gpu_comm_list_status) * num_comm_items, 864 info.page_size); 865 if (ret < 0) { 866 rte_errno = ENOMEM; 867 return NULL; 868 } 869 870 comm_list[0].status_h = rte_gpu_mem_cpu_map(dev_id, 871 sizeof(enum rte_gpu_comm_list_status) * num_comm_items, 872 comm_list[0].status_d); 873 if (comm_list[0].status_h == NULL) { 874 /* 875 * If CPU mapping is not supported by driver 876 * use regular CPU registered memory. 877 */ 878 comm_list[0].status_h = rte_zmalloc(NULL, 879 sizeof(enum rte_gpu_comm_list_status) * num_comm_items, 0); 880 if (comm_list[0].status_h == NULL) { 881 rte_errno = ENOMEM; 882 return NULL; 883 } 884 885 ret = rte_gpu_mem_register(dev_id, 886 sizeof(enum rte_gpu_comm_list_status) * num_comm_items, 887 comm_list[0].status_h); 888 if (ret < 0) { 889 rte_errno = ENOMEM; 890 return NULL; 891 } 892 893 comm_list[0].status_d = comm_list[0].status_h; 894 } 895 896 for (idx_l = 0; idx_l < num_comm_items; idx_l++) { 897 comm_list[idx_l].pkt_list = rte_zmalloc(NULL, 898 sizeof(struct rte_gpu_comm_pkt) * RTE_GPU_COMM_LIST_PKTS_MAX, 0); 899 if (comm_list[idx_l].pkt_list == NULL) { 900 rte_errno = ENOMEM; 901 return NULL; 902 } 903 904 ret = rte_gpu_mem_register(dev_id, 905 sizeof(struct rte_gpu_comm_pkt) * RTE_GPU_COMM_LIST_PKTS_MAX, 906 comm_list[idx_l].pkt_list); 907 if (ret < 0) { 908 rte_errno = ENOMEM; 909 return NULL; 910 } 911 912 comm_list[idx_l].num_pkts = 0; 913 comm_list[idx_l].dev_id = dev_id; 914 915 comm_list[idx_l].mbufs = rte_zmalloc(NULL, 916 sizeof(struct rte_mbuf *) * RTE_GPU_COMM_LIST_PKTS_MAX, 0); 917 if (comm_list[idx_l].mbufs == NULL) { 918 rte_errno = ENOMEM; 919 return NULL; 920 } 921 922 if (idx_l > 0) { 923 comm_list[idx_l].status_h = &(comm_list[0].status_h[idx_l]); 924 comm_list[idx_l].status_d = &(comm_list[0].status_d[idx_l]); 925 926 ret = rte_gpu_comm_set_status(&comm_list[idx_l], RTE_GPU_COMM_LIST_FREE); 927 if (ret < 0) { 928 rte_errno = ENOMEM; 929 return NULL; 930 } 931 } 932 } 933 934 return comm_list; 935 } 936 937 int 938 rte_gpu_comm_destroy_list(struct rte_gpu_comm_list *comm_list, 939 uint32_t num_comm_items) 940 { 941 uint32_t idx_l; 942 int ret; 943 uint16_t dev_id; 944 945 if (comm_list == NULL) { 946 rte_errno = EINVAL; 947 return -rte_errno; 948 } 949 950 dev_id = comm_list[0].dev_id; 951 952 for (idx_l = 0; idx_l < num_comm_items; idx_l++) { 953 ret = rte_gpu_mem_unregister(dev_id, comm_list[idx_l].pkt_list); 954 if (ret < 0) { 955 rte_errno = EINVAL; 956 return -1; 957 } 958 959 rte_free(comm_list[idx_l].pkt_list); 960 rte_free(comm_list[idx_l].mbufs); 961 } 962 963 ret = rte_gpu_mem_unregister(dev_id, comm_list); 964 if (ret < 0) { 965 rte_errno = EINVAL; 966 return -1; 967 } 968 969 ret = rte_gpu_mem_cpu_unmap(dev_id, comm_list[0].status_d); 970 if (ret == 0) { 971 rte_gpu_mem_free(dev_id, comm_list[0].status_d); 972 } else { 973 rte_gpu_mem_unregister(dev_id, comm_list[0].status_h); 974 rte_free(comm_list[0].status_h); 975 } 976 977 rte_free(comm_list); 978 979 return 0; 980 } 981 982 int 983 rte_gpu_comm_populate_list_pkts(struct rte_gpu_comm_list *comm_list_item, 984 struct rte_mbuf **mbufs, uint32_t num_mbufs) 985 { 986 uint32_t idx; 987 int ret; 988 989 if (comm_list_item == NULL || comm_list_item->pkt_list == NULL || 990 mbufs == NULL || num_mbufs > RTE_GPU_COMM_LIST_PKTS_MAX) { 991 rte_errno = EINVAL; 992 return -rte_errno; 993 } 994 995 for (idx = 0; idx < num_mbufs; idx++) { 996 /* support only unchained mbufs */ 997 if (unlikely((mbufs[idx]->nb_segs > 1) || 998 (mbufs[idx]->next != NULL) || 999 (mbufs[idx]->data_len != mbufs[idx]->pkt_len))) { 1000 rte_errno = ENOTSUP; 1001 return -rte_errno; 1002 } 1003 comm_list_item->pkt_list[idx].addr = 1004 rte_pktmbuf_mtod_offset(mbufs[idx], uintptr_t, 0); 1005 comm_list_item->pkt_list[idx].size = mbufs[idx]->pkt_len; 1006 comm_list_item->mbufs[idx] = mbufs[idx]; 1007 } 1008 1009 RTE_GPU_VOLATILE(comm_list_item->num_pkts) = num_mbufs; 1010 rte_gpu_wmb(comm_list_item->dev_id); 1011 ret = rte_gpu_comm_set_status(comm_list_item, RTE_GPU_COMM_LIST_READY); 1012 if (ret < 0) { 1013 rte_errno = EINVAL; 1014 return -rte_errno; 1015 } 1016 1017 return 0; 1018 } 1019 1020 int 1021 rte_gpu_comm_set_status(struct rte_gpu_comm_list *comm_list_item, 1022 enum rte_gpu_comm_list_status status) 1023 { 1024 if (comm_list_item == NULL) { 1025 rte_errno = EINVAL; 1026 return -rte_errno; 1027 } 1028 1029 RTE_GPU_VOLATILE(comm_list_item->status_h[0]) = status; 1030 1031 return 0; 1032 } 1033 1034 int 1035 rte_gpu_comm_get_status(struct rte_gpu_comm_list *comm_list_item, 1036 enum rte_gpu_comm_list_status *status) 1037 { 1038 if (comm_list_item == NULL || status == NULL) { 1039 rte_errno = EINVAL; 1040 return -rte_errno; 1041 } 1042 1043 *status = RTE_GPU_VOLATILE(comm_list_item->status_h[0]); 1044 1045 return 0; 1046 } 1047 1048 int 1049 rte_gpu_comm_cleanup_list(struct rte_gpu_comm_list *comm_list_item) 1050 { 1051 uint32_t idx = 0; 1052 enum rte_gpu_comm_list_status status; 1053 int ret; 1054 1055 if (comm_list_item == NULL) { 1056 rte_errno = EINVAL; 1057 return -rte_errno; 1058 } 1059 1060 ret = rte_gpu_comm_get_status(comm_list_item, &status); 1061 if (ret < 0) { 1062 rte_errno = EINVAL; 1063 return -rte_errno; 1064 } 1065 1066 if (status == RTE_GPU_COMM_LIST_READY) { 1067 GPU_LOG(ERR, "packet list is still in progress"); 1068 rte_errno = EINVAL; 1069 return -rte_errno; 1070 } 1071 1072 for (idx = 0; idx < RTE_GPU_COMM_LIST_PKTS_MAX; idx++) { 1073 if (comm_list_item->pkt_list[idx].addr == 0) 1074 break; 1075 1076 comm_list_item->pkt_list[idx].addr = 0; 1077 comm_list_item->pkt_list[idx].size = 0; 1078 comm_list_item->mbufs[idx] = NULL; 1079 } 1080 1081 ret = rte_gpu_comm_set_status(comm_list_item, RTE_GPU_COMM_LIST_FREE); 1082 if (ret < 0) { 1083 rte_errno = EINVAL; 1084 return -rte_errno; 1085 } 1086 RTE_GPU_VOLATILE(comm_list_item->num_pkts) = 0; 1087 rte_mb(); 1088 1089 return 0; 1090 } 1091