1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright (c) 2021 NVIDIA Corporation & Affiliates
3 */
4
5 #include <stdlib.h>
6
7 #include <rte_eal.h>
8 #include <rte_tailq.h>
9 #include <rte_string_fns.h>
10 #include <rte_memzone.h>
11 #include <rte_malloc.h>
12 #include <rte_errno.h>
13 #include <rte_log.h>
14
15 #include "rte_gpudev.h"
16 #include "gpudev_driver.h"
17
18 /* Logging */
19 RTE_LOG_REGISTER_DEFAULT(gpu_logtype, NOTICE);
20 #define RTE_LOGTYPE_GPUDEV gpu_logtype
21
22 #define GPU_LOG(level, ...) \
23 RTE_LOG_LINE(level, GPUDEV, "" __VA_ARGS__)
24
25 /* Set any driver error as EPERM */
26 #define GPU_DRV_RET(function) \
27 ((function != 0) ? -(rte_errno = EPERM) : (rte_errno = 0))
28
29 /* Array of devices */
30 static struct rte_gpu *gpus;
31 /* Number of currently valid devices */
32 static int16_t gpu_max;
33 /* Number of currently valid devices */
34 static int16_t gpu_count;
35
36 /* Shared memory between processes. */
37 static const char *GPU_MEMZONE = "rte_gpu_shared";
38 static struct {
39 __extension__ struct rte_gpu_mpshared gpus[0];
40 } *gpu_shared_mem;
41
42 /* Event callback object */
43 struct rte_gpu_callback {
44 TAILQ_ENTRY(rte_gpu_callback) next;
45 rte_gpu_callback_t *function;
46 void *user_data;
47 enum rte_gpu_event event;
48 };
49 static rte_rwlock_t gpu_callback_lock = RTE_RWLOCK_INITIALIZER;
50 static void gpu_free_callbacks(struct rte_gpu *dev);
51
52 int
rte_gpu_init(size_t dev_max)53 rte_gpu_init(size_t dev_max)
54 {
55 if (dev_max == 0 || dev_max > INT16_MAX) {
56 GPU_LOG(ERR, "invalid array size");
57 rte_errno = EINVAL;
58 return -rte_errno;
59 }
60
61 /* No lock, it must be called before or during first probing. */
62 if (gpus != NULL) {
63 GPU_LOG(ERR, "already initialized");
64 rte_errno = EBUSY;
65 return -rte_errno;
66 }
67
68 gpus = calloc(dev_max, sizeof(struct rte_gpu));
69 if (gpus == NULL) {
70 GPU_LOG(ERR, "cannot initialize library");
71 rte_errno = ENOMEM;
72 return -rte_errno;
73 }
74
75 gpu_max = dev_max;
76 return 0;
77 }
78
79 uint16_t
rte_gpu_count_avail(void)80 rte_gpu_count_avail(void)
81 {
82 return gpu_count;
83 }
84
85 bool
rte_gpu_is_valid(int16_t dev_id)86 rte_gpu_is_valid(int16_t dev_id)
87 {
88 if (dev_id >= 0 && dev_id < gpu_max &&
89 gpus[dev_id].process_state == RTE_GPU_STATE_INITIALIZED)
90 return true;
91 return false;
92 }
93
94 static bool
gpu_match_parent(int16_t dev_id,int16_t parent)95 gpu_match_parent(int16_t dev_id, int16_t parent)
96 {
97 if (parent == RTE_GPU_ID_ANY)
98 return true;
99 return gpus[dev_id].mpshared->info.parent == parent;
100 }
101
102 int16_t
rte_gpu_find_next(int16_t dev_id,int16_t parent)103 rte_gpu_find_next(int16_t dev_id, int16_t parent)
104 {
105 if (dev_id < 0)
106 dev_id = 0;
107 while (dev_id < gpu_max &&
108 (gpus[dev_id].process_state == RTE_GPU_STATE_UNUSED ||
109 !gpu_match_parent(dev_id, parent)))
110 dev_id++;
111
112 if (dev_id >= gpu_max)
113 return RTE_GPU_ID_NONE;
114 return dev_id;
115 }
116
117 static int16_t
gpu_find_free_id(void)118 gpu_find_free_id(void)
119 {
120 int16_t dev_id;
121
122 for (dev_id = 0; dev_id < gpu_max; dev_id++) {
123 if (gpus[dev_id].process_state == RTE_GPU_STATE_UNUSED)
124 return dev_id;
125 }
126 return RTE_GPU_ID_NONE;
127 }
128
129 static struct rte_gpu *
gpu_get_by_id(int16_t dev_id)130 gpu_get_by_id(int16_t dev_id)
131 {
132 if (!rte_gpu_is_valid(dev_id))
133 return NULL;
134 return &gpus[dev_id];
135 }
136
137 struct rte_gpu *
rte_gpu_get_by_name(const char * name)138 rte_gpu_get_by_name(const char *name)
139 {
140 int16_t dev_id;
141 struct rte_gpu *dev;
142
143 if (name == NULL) {
144 rte_errno = EINVAL;
145 return NULL;
146 }
147
148 RTE_GPU_FOREACH(dev_id) {
149 dev = &gpus[dev_id];
150 if (strncmp(name, dev->mpshared->name, RTE_DEV_NAME_MAX_LEN) == 0)
151 return dev;
152 }
153 return NULL;
154 }
155
156 static int
gpu_shared_mem_init(void)157 gpu_shared_mem_init(void)
158 {
159 const struct rte_memzone *memzone;
160
161 if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
162 memzone = rte_memzone_reserve(GPU_MEMZONE,
163 sizeof(*gpu_shared_mem) +
164 sizeof(*gpu_shared_mem->gpus) * gpu_max,
165 SOCKET_ID_ANY, 0);
166 } else {
167 memzone = rte_memzone_lookup(GPU_MEMZONE);
168 }
169 if (memzone == NULL) {
170 GPU_LOG(ERR, "cannot initialize shared memory");
171 rte_errno = ENOMEM;
172 return -rte_errno;
173 }
174
175 gpu_shared_mem = memzone->addr;
176 return 0;
177 }
178
179 struct rte_gpu *
rte_gpu_allocate(const char * name)180 rte_gpu_allocate(const char *name)
181 {
182 int16_t dev_id;
183 struct rte_gpu *dev;
184
185 if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
186 GPU_LOG(ERR, "only primary process can allocate device");
187 rte_errno = EPERM;
188 return NULL;
189 }
190 if (name == NULL) {
191 GPU_LOG(ERR, "allocate device without a name");
192 rte_errno = EINVAL;
193 return NULL;
194 }
195
196 /* implicit initialization of library before adding first device */
197 if (gpus == NULL && rte_gpu_init(RTE_GPU_DEFAULT_MAX) < 0)
198 return NULL;
199
200 /* initialize shared memory before adding first device */
201 if (gpu_shared_mem == NULL && gpu_shared_mem_init() < 0)
202 return NULL;
203
204 if (rte_gpu_get_by_name(name) != NULL) {
205 GPU_LOG(ERR, "device with name %s already exists", name);
206 rte_errno = EEXIST;
207 return NULL;
208 }
209 dev_id = gpu_find_free_id();
210 if (dev_id == RTE_GPU_ID_NONE) {
211 GPU_LOG(ERR, "reached maximum number of devices");
212 rte_errno = ENOENT;
213 return NULL;
214 }
215
216 dev = &gpus[dev_id];
217 memset(dev, 0, sizeof(*dev));
218
219 dev->mpshared = &gpu_shared_mem->gpus[dev_id];
220 memset(dev->mpshared, 0, sizeof(*dev->mpshared));
221
222 if (rte_strscpy(dev->mpshared->name, name, RTE_DEV_NAME_MAX_LEN) < 0) {
223 GPU_LOG(ERR, "device name too long: %s", name);
224 rte_errno = ENAMETOOLONG;
225 return NULL;
226 }
227 dev->mpshared->info.name = dev->mpshared->name;
228 dev->mpshared->info.dev_id = dev_id;
229 dev->mpshared->info.numa_node = -1;
230 dev->mpshared->info.parent = RTE_GPU_ID_NONE;
231 TAILQ_INIT(&dev->callbacks);
232 rte_atomic_fetch_add_explicit(&dev->mpshared->process_refcnt, 1, rte_memory_order_relaxed);
233
234 gpu_count++;
235 GPU_LOG(DEBUG, "new device %s (id %d) of total %d",
236 name, dev_id, gpu_count);
237 return dev;
238 }
239
240 struct rte_gpu *
rte_gpu_attach(const char * name)241 rte_gpu_attach(const char *name)
242 {
243 int16_t dev_id;
244 struct rte_gpu *dev;
245 struct rte_gpu_mpshared *shared_dev;
246
247 if (rte_eal_process_type() != RTE_PROC_SECONDARY) {
248 GPU_LOG(ERR, "only secondary process can attach device");
249 rte_errno = EPERM;
250 return NULL;
251 }
252 if (name == NULL) {
253 GPU_LOG(ERR, "attach device without a name");
254 rte_errno = EINVAL;
255 return NULL;
256 }
257
258 /* implicit initialization of library before adding first device */
259 if (gpus == NULL && rte_gpu_init(RTE_GPU_DEFAULT_MAX) < 0)
260 return NULL;
261
262 /* initialize shared memory before adding first device */
263 if (gpu_shared_mem == NULL && gpu_shared_mem_init() < 0)
264 return NULL;
265
266 for (dev_id = 0; dev_id < gpu_max; dev_id++) {
267 shared_dev = &gpu_shared_mem->gpus[dev_id];
268 if (strncmp(name, shared_dev->name, RTE_DEV_NAME_MAX_LEN) == 0)
269 break;
270 }
271 if (dev_id >= gpu_max) {
272 GPU_LOG(ERR, "device with name %s not found", name);
273 rte_errno = ENOENT;
274 return NULL;
275 }
276 dev = &gpus[dev_id];
277 memset(dev, 0, sizeof(*dev));
278
279 TAILQ_INIT(&dev->callbacks);
280 dev->mpshared = shared_dev;
281 rte_atomic_fetch_add_explicit(&dev->mpshared->process_refcnt, 1, rte_memory_order_relaxed);
282
283 gpu_count++;
284 GPU_LOG(DEBUG, "attached device %s (id %d) of total %d",
285 name, dev_id, gpu_count);
286 return dev;
287 }
288
289 int16_t
rte_gpu_add_child(const char * name,int16_t parent,uint64_t child_context)290 rte_gpu_add_child(const char *name, int16_t parent, uint64_t child_context)
291 {
292 struct rte_gpu *dev;
293
294 if (!rte_gpu_is_valid(parent)) {
295 GPU_LOG(ERR, "add child to invalid parent ID %d", parent);
296 rte_errno = ENODEV;
297 return -rte_errno;
298 }
299
300 dev = rte_gpu_allocate(name);
301 if (dev == NULL)
302 return -rte_errno;
303
304 dev->mpshared->info.parent = parent;
305 dev->mpshared->info.context = child_context;
306
307 rte_gpu_complete_new(dev);
308 return dev->mpshared->info.dev_id;
309 }
310
311 void
rte_gpu_complete_new(struct rte_gpu * dev)312 rte_gpu_complete_new(struct rte_gpu *dev)
313 {
314 if (dev == NULL)
315 return;
316
317 dev->process_state = RTE_GPU_STATE_INITIALIZED;
318 rte_gpu_notify(dev, RTE_GPU_EVENT_NEW);
319 }
320
321 int
rte_gpu_release(struct rte_gpu * dev)322 rte_gpu_release(struct rte_gpu *dev)
323 {
324 int16_t dev_id, child;
325
326 if (dev == NULL) {
327 rte_errno = ENODEV;
328 return -rte_errno;
329 }
330 dev_id = dev->mpshared->info.dev_id;
331 RTE_GPU_FOREACH_CHILD(child, dev_id) {
332 GPU_LOG(ERR, "cannot release device %d with child %d",
333 dev_id, child);
334 rte_errno = EBUSY;
335 return -rte_errno;
336 }
337
338 GPU_LOG(DEBUG, "free device %s (id %d)",
339 dev->mpshared->info.name, dev->mpshared->info.dev_id);
340 rte_gpu_notify(dev, RTE_GPU_EVENT_DEL);
341
342 gpu_free_callbacks(dev);
343 dev->process_state = RTE_GPU_STATE_UNUSED;
344 rte_atomic_fetch_sub_explicit(&dev->mpshared->process_refcnt, 1, rte_memory_order_relaxed);
345 gpu_count--;
346
347 return 0;
348 }
349
350 int
rte_gpu_close(int16_t dev_id)351 rte_gpu_close(int16_t dev_id)
352 {
353 int firsterr, binerr;
354 int *lasterr = &firsterr;
355 struct rte_gpu *dev;
356
357 dev = gpu_get_by_id(dev_id);
358 if (dev == NULL) {
359 GPU_LOG(ERR, "close invalid device ID %d", dev_id);
360 rte_errno = ENODEV;
361 return -rte_errno;
362 }
363
364 if (dev->ops.dev_close != NULL) {
365 *lasterr = GPU_DRV_RET(dev->ops.dev_close(dev));
366 if (*lasterr != 0)
367 lasterr = &binerr;
368 }
369
370 *lasterr = rte_gpu_release(dev);
371
372 rte_errno = -firsterr;
373 return firsterr;
374 }
375
376 int
rte_gpu_callback_register(int16_t dev_id,enum rte_gpu_event event,rte_gpu_callback_t * function,void * user_data)377 rte_gpu_callback_register(int16_t dev_id, enum rte_gpu_event event,
378 rte_gpu_callback_t *function, void *user_data)
379 {
380 int16_t next_dev, last_dev;
381 struct rte_gpu_callback_list *callbacks;
382 struct rte_gpu_callback *callback;
383
384 if (!rte_gpu_is_valid(dev_id) && dev_id != RTE_GPU_ID_ANY) {
385 GPU_LOG(ERR, "register callback of invalid ID %d", dev_id);
386 rte_errno = ENODEV;
387 return -rte_errno;
388 }
389 if (function == NULL) {
390 GPU_LOG(ERR, "cannot register callback without function");
391 rte_errno = EINVAL;
392 return -rte_errno;
393 }
394
395 if (dev_id == RTE_GPU_ID_ANY) {
396 next_dev = 0;
397 last_dev = gpu_max - 1;
398 } else {
399 next_dev = last_dev = dev_id;
400 }
401
402 rte_rwlock_write_lock(&gpu_callback_lock);
403 do {
404 callbacks = &gpus[next_dev].callbacks;
405
406 /* check if not already registered */
407 TAILQ_FOREACH(callback, callbacks, next) {
408 if (callback->event == event &&
409 callback->function == function &&
410 callback->user_data == user_data) {
411 GPU_LOG(INFO, "callback already registered");
412 rte_rwlock_write_unlock(&gpu_callback_lock);
413 return 0;
414 }
415 }
416
417 callback = malloc(sizeof(*callback));
418 if (callback == NULL) {
419 GPU_LOG(ERR, "cannot allocate callback");
420 rte_rwlock_write_unlock(&gpu_callback_lock);
421 rte_errno = ENOMEM;
422 return -rte_errno;
423 }
424 callback->function = function;
425 callback->user_data = user_data;
426 callback->event = event;
427 TAILQ_INSERT_TAIL(callbacks, callback, next);
428
429 } while (++next_dev <= last_dev);
430 rte_rwlock_write_unlock(&gpu_callback_lock);
431
432 return 0;
433 }
434
435 int
rte_gpu_callback_unregister(int16_t dev_id,enum rte_gpu_event event,rte_gpu_callback_t * function,void * user_data)436 rte_gpu_callback_unregister(int16_t dev_id, enum rte_gpu_event event,
437 rte_gpu_callback_t *function, void *user_data)
438 {
439 int16_t next_dev, last_dev;
440 struct rte_gpu_callback_list *callbacks;
441 struct rte_gpu_callback *callback, *nextcb;
442
443 if (!rte_gpu_is_valid(dev_id) && dev_id != RTE_GPU_ID_ANY) {
444 GPU_LOG(ERR, "unregister callback of invalid ID %d", dev_id);
445 rte_errno = ENODEV;
446 return -rte_errno;
447 }
448 if (function == NULL) {
449 GPU_LOG(ERR, "cannot unregister callback without function");
450 rte_errno = EINVAL;
451 return -rte_errno;
452 }
453
454 if (dev_id == RTE_GPU_ID_ANY) {
455 next_dev = 0;
456 last_dev = gpu_max - 1;
457 } else {
458 next_dev = last_dev = dev_id;
459 }
460
461 rte_rwlock_write_lock(&gpu_callback_lock);
462 do {
463 callbacks = &gpus[next_dev].callbacks;
464 RTE_TAILQ_FOREACH_SAFE(callback, callbacks, next, nextcb) {
465 if (callback->event != event ||
466 callback->function != function ||
467 (callback->user_data != user_data &&
468 user_data != (void *)-1))
469 continue;
470 TAILQ_REMOVE(callbacks, callback, next);
471 free(callback);
472 }
473 } while (++next_dev <= last_dev);
474 rte_rwlock_write_unlock(&gpu_callback_lock);
475
476 return 0;
477 }
478
479 static void
gpu_free_callbacks(struct rte_gpu * dev)480 gpu_free_callbacks(struct rte_gpu *dev)
481 {
482 struct rte_gpu_callback_list *callbacks;
483 struct rte_gpu_callback *callback, *nextcb;
484
485 callbacks = &dev->callbacks;
486 rte_rwlock_write_lock(&gpu_callback_lock);
487 RTE_TAILQ_FOREACH_SAFE(callback, callbacks, next, nextcb) {
488 TAILQ_REMOVE(callbacks, callback, next);
489 free(callback);
490 }
491 rte_rwlock_write_unlock(&gpu_callback_lock);
492 }
493
494 void
rte_gpu_notify(struct rte_gpu * dev,enum rte_gpu_event event)495 rte_gpu_notify(struct rte_gpu *dev, enum rte_gpu_event event)
496 {
497 int16_t dev_id;
498 struct rte_gpu_callback *callback;
499
500 dev_id = dev->mpshared->info.dev_id;
501 rte_rwlock_read_lock(&gpu_callback_lock);
502 TAILQ_FOREACH(callback, &dev->callbacks, next) {
503 if (callback->event != event || callback->function == NULL)
504 continue;
505 callback->function(dev_id, event, callback->user_data);
506 }
507 rte_rwlock_read_unlock(&gpu_callback_lock);
508 }
509
510 int
rte_gpu_info_get(int16_t dev_id,struct rte_gpu_info * info)511 rte_gpu_info_get(int16_t dev_id, struct rte_gpu_info *info)
512 {
513 struct rte_gpu *dev;
514
515 dev = gpu_get_by_id(dev_id);
516 if (dev == NULL) {
517 GPU_LOG(ERR, "query invalid device ID %d", dev_id);
518 rte_errno = ENODEV;
519 return -rte_errno;
520 }
521 if (info == NULL) {
522 GPU_LOG(ERR, "query without storage");
523 rte_errno = EINVAL;
524 return -rte_errno;
525 }
526
527 if (dev->ops.dev_info_get == NULL) {
528 *info = dev->mpshared->info;
529 return 0;
530 }
531 return GPU_DRV_RET(dev->ops.dev_info_get(dev, info));
532 }
533
534 void *
rte_gpu_mem_alloc(int16_t dev_id,size_t size,unsigned int align)535 rte_gpu_mem_alloc(int16_t dev_id, size_t size, unsigned int align)
536 {
537 struct rte_gpu *dev;
538 void *ptr;
539 int ret;
540
541 dev = gpu_get_by_id(dev_id);
542 if (dev == NULL) {
543 GPU_LOG(ERR, "alloc mem for invalid device ID %d", dev_id);
544 rte_errno = ENODEV;
545 return NULL;
546 }
547
548 if (dev->ops.mem_alloc == NULL) {
549 GPU_LOG(ERR, "mem allocation not supported");
550 rte_errno = ENOTSUP;
551 return NULL;
552 }
553
554 if (size == 0) /* dry-run */
555 return NULL;
556
557 if (align && !rte_is_power_of_2(align)) {
558 GPU_LOG(ERR, "requested alignment is not a power of two %u", align);
559 rte_errno = EINVAL;
560 return NULL;
561 }
562
563 ret = dev->ops.mem_alloc(dev, size, align, &ptr);
564
565 switch (ret) {
566 case 0:
567 return ptr;
568 case -ENOMEM:
569 case -E2BIG:
570 rte_errno = -ret;
571 return NULL;
572 default:
573 rte_errno = -EPERM;
574 return NULL;
575 }
576 }
577
578 int
rte_gpu_mem_free(int16_t dev_id,void * ptr)579 rte_gpu_mem_free(int16_t dev_id, void *ptr)
580 {
581 struct rte_gpu *dev;
582
583 dev = gpu_get_by_id(dev_id);
584 if (dev == NULL) {
585 GPU_LOG(ERR, "free mem for invalid device ID %d", dev_id);
586 rte_errno = ENODEV;
587 return -rte_errno;
588 }
589
590 if (dev->ops.mem_free == NULL) {
591 rte_errno = ENOTSUP;
592 return -rte_errno;
593 }
594
595 if (ptr == NULL) /* dry-run */
596 return 0;
597
598 return GPU_DRV_RET(dev->ops.mem_free(dev, ptr));
599 }
600
601 int
rte_gpu_mem_register(int16_t dev_id,size_t size,void * ptr)602 rte_gpu_mem_register(int16_t dev_id, size_t size, void *ptr)
603 {
604 struct rte_gpu *dev;
605
606 dev = gpu_get_by_id(dev_id);
607 if (dev == NULL) {
608 GPU_LOG(ERR, "alloc mem for invalid device ID %d", dev_id);
609 rte_errno = ENODEV;
610 return -rte_errno;
611 }
612
613 if (dev->ops.mem_register == NULL) {
614 GPU_LOG(ERR, "mem registration not supported");
615 rte_errno = ENOTSUP;
616 return -rte_errno;
617 }
618
619 if (ptr == NULL || size == 0) /* dry-run */
620 return 0;
621
622 return GPU_DRV_RET(dev->ops.mem_register(dev, size, ptr));
623 }
624
625 int
rte_gpu_mem_unregister(int16_t dev_id,void * ptr)626 rte_gpu_mem_unregister(int16_t dev_id, void *ptr)
627 {
628 struct rte_gpu *dev;
629
630 dev = gpu_get_by_id(dev_id);
631 if (dev == NULL) {
632 GPU_LOG(ERR, "unregister mem for invalid device ID %d", dev_id);
633 rte_errno = ENODEV;
634 return -rte_errno;
635 }
636
637 if (dev->ops.mem_unregister == NULL) {
638 rte_errno = ENOTSUP;
639 return -rte_errno;
640 }
641
642 if (ptr == NULL) /* dry-run */
643 return 0;
644
645 return GPU_DRV_RET(dev->ops.mem_unregister(dev, ptr));
646 }
647
648 void *
rte_gpu_mem_cpu_map(int16_t dev_id,size_t size,void * ptr)649 rte_gpu_mem_cpu_map(int16_t dev_id, size_t size, void *ptr)
650 {
651 struct rte_gpu *dev;
652 void *ptr_out;
653 int ret;
654
655 dev = gpu_get_by_id(dev_id);
656 if (dev == NULL) {
657 GPU_LOG(ERR, "mem CPU map for invalid device ID %d", dev_id);
658 rte_errno = ENODEV;
659 return NULL;
660 }
661
662 if (dev->ops.mem_cpu_map == NULL) {
663 GPU_LOG(ERR, "mem CPU map not supported");
664 rte_errno = ENOTSUP;
665 return NULL;
666 }
667
668 if (ptr == NULL || size == 0) /* dry-run */
669 return NULL;
670
671 ret = GPU_DRV_RET(dev->ops.mem_cpu_map(dev, size, ptr, &ptr_out));
672
673 switch (ret) {
674 case 0:
675 return ptr_out;
676 case -ENOMEM:
677 case -E2BIG:
678 rte_errno = -ret;
679 return NULL;
680 default:
681 rte_errno = -EPERM;
682 return NULL;
683 }
684 }
685
686 int
rte_gpu_mem_cpu_unmap(int16_t dev_id,void * ptr)687 rte_gpu_mem_cpu_unmap(int16_t dev_id, void *ptr)
688 {
689 struct rte_gpu *dev;
690
691 dev = gpu_get_by_id(dev_id);
692 if (dev == NULL) {
693 GPU_LOG(ERR, "cpu_unmap mem for invalid device ID %d", dev_id);
694 rte_errno = ENODEV;
695 return -rte_errno;
696 }
697
698 if (dev->ops.mem_cpu_unmap == NULL) {
699 rte_errno = ENOTSUP;
700 return -rte_errno;
701 }
702
703 if (ptr == NULL) /* dry-run */
704 return 0;
705
706 return GPU_DRV_RET(dev->ops.mem_cpu_unmap(dev, ptr));
707 }
708
709 int
rte_gpu_wmb(int16_t dev_id)710 rte_gpu_wmb(int16_t dev_id)
711 {
712 struct rte_gpu *dev;
713
714 dev = gpu_get_by_id(dev_id);
715 if (dev == NULL) {
716 GPU_LOG(ERR, "memory barrier for invalid device ID %d", dev_id);
717 rte_errno = ENODEV;
718 return -rte_errno;
719 }
720
721 if (dev->ops.wmb == NULL) {
722 rte_errno = ENOTSUP;
723 return -rte_errno;
724 }
725 return GPU_DRV_RET(dev->ops.wmb(dev));
726 }
727
728 int
rte_gpu_comm_create_flag(uint16_t dev_id,struct rte_gpu_comm_flag * devflag,enum rte_gpu_comm_flag_type mtype)729 rte_gpu_comm_create_flag(uint16_t dev_id, struct rte_gpu_comm_flag *devflag,
730 enum rte_gpu_comm_flag_type mtype)
731 {
732 size_t flag_size;
733 int ret;
734
735 if (devflag == NULL) {
736 rte_errno = EINVAL;
737 return -rte_errno;
738 }
739 if (mtype != RTE_GPU_COMM_FLAG_CPU) {
740 rte_errno = EINVAL;
741 return -rte_errno;
742 }
743
744 flag_size = sizeof(uint32_t);
745
746 devflag->ptr = rte_zmalloc(NULL, flag_size, 0);
747 if (devflag->ptr == NULL) {
748 rte_errno = ENOMEM;
749 return -rte_errno;
750 }
751
752 ret = rte_gpu_mem_register(dev_id, flag_size, devflag->ptr);
753 if (ret < 0) {
754 rte_errno = ENOMEM;
755 return -rte_errno;
756 }
757
758 devflag->mtype = mtype;
759 devflag->dev_id = dev_id;
760
761 return 0;
762 }
763
764 int
rte_gpu_comm_destroy_flag(struct rte_gpu_comm_flag * devflag)765 rte_gpu_comm_destroy_flag(struct rte_gpu_comm_flag *devflag)
766 {
767 int ret;
768
769 if (devflag == NULL) {
770 rte_errno = EINVAL;
771 return -rte_errno;
772 }
773
774 ret = rte_gpu_mem_unregister(devflag->dev_id, devflag->ptr);
775 if (ret < 0) {
776 rte_errno = EINVAL;
777 return -1;
778 }
779
780 rte_free(devflag->ptr);
781
782 return 0;
783 }
784
785 int
rte_gpu_comm_set_flag(struct rte_gpu_comm_flag * devflag,uint32_t val)786 rte_gpu_comm_set_flag(struct rte_gpu_comm_flag *devflag, uint32_t val)
787 {
788 if (devflag == NULL) {
789 rte_errno = EINVAL;
790 return -rte_errno;
791 }
792
793 if (devflag->mtype != RTE_GPU_COMM_FLAG_CPU) {
794 rte_errno = EINVAL;
795 return -rte_errno;
796 }
797
798 RTE_GPU_VOLATILE(*devflag->ptr) = val;
799
800 return 0;
801 }
802
803 int
rte_gpu_comm_get_flag_value(struct rte_gpu_comm_flag * devflag,uint32_t * val)804 rte_gpu_comm_get_flag_value(struct rte_gpu_comm_flag *devflag, uint32_t *val)
805 {
806 if (devflag == NULL) {
807 rte_errno = EINVAL;
808 return -rte_errno;
809 }
810 if (devflag->mtype != RTE_GPU_COMM_FLAG_CPU) {
811 rte_errno = EINVAL;
812 return -rte_errno;
813 }
814
815 *val = RTE_GPU_VOLATILE(*devflag->ptr);
816
817 return 0;
818 }
819
820 struct rte_gpu_comm_list *
rte_gpu_comm_create_list(uint16_t dev_id,uint32_t num_comm_items)821 rte_gpu_comm_create_list(uint16_t dev_id,
822 uint32_t num_comm_items)
823 {
824 struct rte_gpu_comm_list *comm_list;
825 uint32_t idx_l;
826 int ret;
827 struct rte_gpu *dev;
828 struct rte_gpu_info info;
829
830 if (num_comm_items == 0) {
831 rte_errno = EINVAL;
832 return NULL;
833 }
834
835 dev = gpu_get_by_id(dev_id);
836 if (dev == NULL) {
837 GPU_LOG(ERR, "memory barrier for invalid device ID %d", dev_id);
838 rte_errno = ENODEV;
839 return NULL;
840 }
841
842 ret = rte_gpu_info_get(dev_id, &info);
843 if (ret < 0) {
844 rte_errno = ENODEV;
845 return NULL;
846 }
847
848 comm_list = rte_zmalloc(NULL,
849 sizeof(struct rte_gpu_comm_list) * num_comm_items, 0);
850 if (comm_list == NULL) {
851 rte_errno = ENOMEM;
852 return NULL;
853 }
854
855 ret = rte_gpu_mem_register(dev_id,
856 sizeof(struct rte_gpu_comm_list) * num_comm_items, comm_list);
857 if (ret < 0) {
858 rte_errno = ENOMEM;
859 return NULL;
860 }
861
862 /*
863 * Use GPU memory CPU map feature if enabled in the driver
864 * to allocate the status flags of the list.
865 * Allocating this flag in GPU memory will reduce
866 * the latency when GPU workload is polling this flag.
867 */
868 comm_list[0].status_d = rte_gpu_mem_alloc(dev_id,
869 sizeof(enum rte_gpu_comm_list_status) * num_comm_items,
870 info.page_size);
871 if (ret < 0) {
872 rte_errno = ENOMEM;
873 return NULL;
874 }
875
876 comm_list[0].status_h = rte_gpu_mem_cpu_map(dev_id,
877 sizeof(enum rte_gpu_comm_list_status) * num_comm_items,
878 comm_list[0].status_d);
879 if (comm_list[0].status_h == NULL) {
880 /*
881 * If CPU mapping is not supported by driver
882 * use regular CPU registered memory.
883 */
884 comm_list[0].status_h = rte_zmalloc(NULL,
885 sizeof(enum rte_gpu_comm_list_status) * num_comm_items, 0);
886 if (comm_list[0].status_h == NULL) {
887 rte_errno = ENOMEM;
888 return NULL;
889 }
890
891 ret = rte_gpu_mem_register(dev_id,
892 sizeof(enum rte_gpu_comm_list_status) * num_comm_items,
893 comm_list[0].status_h);
894 if (ret < 0) {
895 rte_errno = ENOMEM;
896 return NULL;
897 }
898
899 comm_list[0].status_d = comm_list[0].status_h;
900 }
901
902 for (idx_l = 0; idx_l < num_comm_items; idx_l++) {
903 comm_list[idx_l].pkt_list = rte_zmalloc(NULL,
904 sizeof(struct rte_gpu_comm_pkt) * RTE_GPU_COMM_LIST_PKTS_MAX, 0);
905 if (comm_list[idx_l].pkt_list == NULL) {
906 rte_errno = ENOMEM;
907 return NULL;
908 }
909
910 ret = rte_gpu_mem_register(dev_id,
911 sizeof(struct rte_gpu_comm_pkt) * RTE_GPU_COMM_LIST_PKTS_MAX,
912 comm_list[idx_l].pkt_list);
913 if (ret < 0) {
914 rte_errno = ENOMEM;
915 return NULL;
916 }
917
918 comm_list[idx_l].num_pkts = 0;
919 comm_list[idx_l].dev_id = dev_id;
920
921 comm_list[idx_l].mbufs = rte_zmalloc(NULL,
922 sizeof(struct rte_mbuf *) * RTE_GPU_COMM_LIST_PKTS_MAX, 0);
923 if (comm_list[idx_l].mbufs == NULL) {
924 rte_errno = ENOMEM;
925 return NULL;
926 }
927
928 if (idx_l > 0) {
929 comm_list[idx_l].status_h = &(comm_list[0].status_h[idx_l]);
930 comm_list[idx_l].status_d = &(comm_list[0].status_d[idx_l]);
931
932 ret = rte_gpu_comm_set_status(&comm_list[idx_l], RTE_GPU_COMM_LIST_FREE);
933 if (ret < 0) {
934 rte_errno = ENOMEM;
935 return NULL;
936 }
937 }
938 }
939
940 return comm_list;
941 }
942
943 int
rte_gpu_comm_destroy_list(struct rte_gpu_comm_list * comm_list,uint32_t num_comm_items)944 rte_gpu_comm_destroy_list(struct rte_gpu_comm_list *comm_list,
945 uint32_t num_comm_items)
946 {
947 uint32_t idx_l;
948 int ret;
949 uint16_t dev_id;
950
951 if (comm_list == NULL) {
952 rte_errno = EINVAL;
953 return -rte_errno;
954 }
955
956 dev_id = comm_list[0].dev_id;
957
958 for (idx_l = 0; idx_l < num_comm_items; idx_l++) {
959 ret = rte_gpu_mem_unregister(dev_id, comm_list[idx_l].pkt_list);
960 if (ret < 0) {
961 rte_errno = EINVAL;
962 return -1;
963 }
964
965 rte_free(comm_list[idx_l].pkt_list);
966 rte_free(comm_list[idx_l].mbufs);
967 }
968
969 ret = rte_gpu_mem_unregister(dev_id, comm_list);
970 if (ret < 0) {
971 rte_errno = EINVAL;
972 return -1;
973 }
974
975 ret = rte_gpu_mem_cpu_unmap(dev_id, comm_list[0].status_d);
976 if (ret == 0) {
977 rte_gpu_mem_free(dev_id, comm_list[0].status_d);
978 } else {
979 rte_gpu_mem_unregister(dev_id, comm_list[0].status_h);
980 rte_free(comm_list[0].status_h);
981 }
982
983 rte_free(comm_list);
984
985 return 0;
986 }
987
988 int
rte_gpu_comm_populate_list_pkts(struct rte_gpu_comm_list * comm_list_item,struct rte_mbuf ** mbufs,uint32_t num_mbufs)989 rte_gpu_comm_populate_list_pkts(struct rte_gpu_comm_list *comm_list_item,
990 struct rte_mbuf **mbufs, uint32_t num_mbufs)
991 {
992 uint32_t idx;
993 int ret;
994
995 if (comm_list_item == NULL || comm_list_item->pkt_list == NULL ||
996 mbufs == NULL || num_mbufs > RTE_GPU_COMM_LIST_PKTS_MAX) {
997 rte_errno = EINVAL;
998 return -rte_errno;
999 }
1000
1001 for (idx = 0; idx < num_mbufs; idx++) {
1002 /* support only unchained mbufs */
1003 if (unlikely((mbufs[idx]->nb_segs > 1) ||
1004 (mbufs[idx]->next != NULL) ||
1005 (mbufs[idx]->data_len != mbufs[idx]->pkt_len))) {
1006 rte_errno = ENOTSUP;
1007 return -rte_errno;
1008 }
1009 comm_list_item->pkt_list[idx].addr =
1010 rte_pktmbuf_mtod_offset(mbufs[idx], uintptr_t, 0);
1011 comm_list_item->pkt_list[idx].size = mbufs[idx]->pkt_len;
1012 comm_list_item->mbufs[idx] = mbufs[idx];
1013 }
1014
1015 RTE_GPU_VOLATILE(comm_list_item->num_pkts) = num_mbufs;
1016 rte_gpu_wmb(comm_list_item->dev_id);
1017 ret = rte_gpu_comm_set_status(comm_list_item, RTE_GPU_COMM_LIST_READY);
1018 if (ret < 0) {
1019 rte_errno = EINVAL;
1020 return -rte_errno;
1021 }
1022
1023 return 0;
1024 }
1025
1026 int
rte_gpu_comm_set_status(struct rte_gpu_comm_list * comm_list_item,enum rte_gpu_comm_list_status status)1027 rte_gpu_comm_set_status(struct rte_gpu_comm_list *comm_list_item,
1028 enum rte_gpu_comm_list_status status)
1029 {
1030 if (comm_list_item == NULL) {
1031 rte_errno = EINVAL;
1032 return -rte_errno;
1033 }
1034
1035 RTE_GPU_VOLATILE(comm_list_item->status_h[0]) = status;
1036
1037 return 0;
1038 }
1039
1040 int
rte_gpu_comm_get_status(struct rte_gpu_comm_list * comm_list_item,enum rte_gpu_comm_list_status * status)1041 rte_gpu_comm_get_status(struct rte_gpu_comm_list *comm_list_item,
1042 enum rte_gpu_comm_list_status *status)
1043 {
1044 if (comm_list_item == NULL || status == NULL) {
1045 rte_errno = EINVAL;
1046 return -rte_errno;
1047 }
1048
1049 *status = RTE_GPU_VOLATILE(comm_list_item->status_h[0]);
1050
1051 return 0;
1052 }
1053
1054 int
rte_gpu_comm_cleanup_list(struct rte_gpu_comm_list * comm_list_item)1055 rte_gpu_comm_cleanup_list(struct rte_gpu_comm_list *comm_list_item)
1056 {
1057 uint32_t idx = 0;
1058 enum rte_gpu_comm_list_status status;
1059 int ret;
1060
1061 if (comm_list_item == NULL) {
1062 rte_errno = EINVAL;
1063 return -rte_errno;
1064 }
1065
1066 ret = rte_gpu_comm_get_status(comm_list_item, &status);
1067 if (ret < 0) {
1068 rte_errno = EINVAL;
1069 return -rte_errno;
1070 }
1071
1072 if (status == RTE_GPU_COMM_LIST_READY) {
1073 GPU_LOG(ERR, "packet list is still in progress");
1074 rte_errno = EINVAL;
1075 return -rte_errno;
1076 }
1077
1078 for (idx = 0; idx < RTE_GPU_COMM_LIST_PKTS_MAX; idx++) {
1079 if (comm_list_item->pkt_list[idx].addr == 0)
1080 break;
1081
1082 comm_list_item->pkt_list[idx].addr = 0;
1083 comm_list_item->pkt_list[idx].size = 0;
1084 comm_list_item->mbufs[idx] = NULL;
1085 }
1086
1087 ret = rte_gpu_comm_set_status(comm_list_item, RTE_GPU_COMM_LIST_FREE);
1088 if (ret < 0) {
1089 rte_errno = EINVAL;
1090 return -rte_errno;
1091 }
1092 RTE_GPU_VOLATILE(comm_list_item->num_pkts) = 0;
1093 rte_mb();
1094
1095 return 0;
1096 }
1097