1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(C) 2023 Intel Corporation
3 */
4
5 #include "graph_private.h"
6 #include "rte_graph_model_mcore_dispatch.h"
7
8 int
graph_sched_wq_create(struct graph * _graph,struct graph * _parent_graph,struct rte_graph_param * prm)9 graph_sched_wq_create(struct graph *_graph, struct graph *_parent_graph,
10 struct rte_graph_param *prm)
11 {
12 struct rte_graph *parent_graph = _parent_graph->graph;
13 struct rte_graph *graph = _graph->graph;
14 unsigned int wq_size;
15 unsigned int flags = RING_F_SC_DEQ;
16
17 wq_size = RTE_GRAPH_SCHED_WQ_SIZE(graph->nb_nodes);
18 wq_size = rte_align32pow2(wq_size + 1);
19
20 if (prm->dispatch.wq_size_max > 0)
21 wq_size = wq_size <= (prm->dispatch.wq_size_max) ? wq_size :
22 prm->dispatch.wq_size_max;
23
24 if (!rte_is_power_of_2(wq_size))
25 flags |= RING_F_EXACT_SZ;
26
27 graph->dispatch.wq = rte_ring_create(graph->name, wq_size, graph->socket,
28 flags);
29 if (graph->dispatch.wq == NULL)
30 SET_ERR_JMP(EIO, fail, "Failed to allocate graph WQ");
31
32 if (prm->dispatch.mp_capacity > 0)
33 wq_size = (wq_size <= prm->dispatch.mp_capacity) ? wq_size :
34 prm->dispatch.mp_capacity;
35
36 graph->dispatch.mp = rte_mempool_create(graph->name, wq_size,
37 sizeof(struct graph_mcore_dispatch_wq_node),
38 0, 0, NULL, NULL, NULL, NULL,
39 graph->socket, MEMPOOL_F_SP_PUT);
40 if (graph->dispatch.mp == NULL)
41 SET_ERR_JMP(EIO, fail_mp,
42 "Failed to allocate graph WQ schedule entry");
43
44 graph->dispatch.lcore_id = _graph->lcore_id;
45
46 if (parent_graph->dispatch.rq == NULL) {
47 parent_graph->dispatch.rq = &parent_graph->dispatch.rq_head;
48 SLIST_INIT(parent_graph->dispatch.rq);
49 }
50
51 graph->dispatch.rq = parent_graph->dispatch.rq;
52 SLIST_INSERT_HEAD(graph->dispatch.rq, graph, next);
53
54 return 0;
55
56 fail_mp:
57 rte_ring_free(graph->dispatch.wq);
58 graph->dispatch.wq = NULL;
59 fail:
60 return -rte_errno;
61 }
62
63 void
graph_sched_wq_destroy(struct graph * _graph)64 graph_sched_wq_destroy(struct graph *_graph)
65 {
66 struct rte_graph *graph = _graph->graph;
67
68 if (graph == NULL)
69 return;
70
71 rte_ring_free(graph->dispatch.wq);
72 graph->dispatch.wq = NULL;
73
74 rte_mempool_free(graph->dispatch.mp);
75 graph->dispatch.mp = NULL;
76 }
77
78 static __rte_always_inline bool
__graph_sched_node_enqueue(struct rte_node * node,struct rte_graph * graph)79 __graph_sched_node_enqueue(struct rte_node *node, struct rte_graph *graph)
80 {
81 struct graph_mcore_dispatch_wq_node *wq_node;
82 uint16_t off = 0;
83 uint16_t size;
84
85 submit_again:
86 if (rte_mempool_get(graph->dispatch.mp, (void **)&wq_node) < 0)
87 goto fallback;
88
89 size = RTE_MIN(node->idx, RTE_DIM(wq_node->objs));
90 wq_node->node_off = node->off;
91 wq_node->nb_objs = size;
92 rte_memcpy(wq_node->objs, &node->objs[off], size * sizeof(void *));
93
94 while (rte_ring_mp_enqueue_bulk_elem(graph->dispatch.wq, (void *)&wq_node,
95 sizeof(wq_node), 1, NULL) == 0)
96 rte_pause();
97
98 off += size;
99 node->dispatch.total_sched_objs += size;
100 node->idx -= size;
101 if (node->idx > 0)
102 goto submit_again;
103
104 return true;
105
106 fallback:
107 if (off != 0)
108 memmove(&node->objs[0], &node->objs[off],
109 node->idx * sizeof(void *));
110
111 node->dispatch.total_sched_fail += node->idx;
112
113 return false;
114 }
115
116 bool __rte_noinline
__rte_graph_mcore_dispatch_sched_node_enqueue(struct rte_node * node,struct rte_graph_rq_head * rq)117 __rte_graph_mcore_dispatch_sched_node_enqueue(struct rte_node *node,
118 struct rte_graph_rq_head *rq)
119 {
120 const unsigned int lcore_id = node->dispatch.lcore_id;
121 struct rte_graph *graph;
122
123 SLIST_FOREACH(graph, rq, next)
124 if (graph->dispatch.lcore_id == lcore_id)
125 break;
126
127 return graph != NULL ? __graph_sched_node_enqueue(node, graph) : false;
128 }
129
130 void
__rte_graph_mcore_dispatch_sched_wq_process(struct rte_graph * graph)131 __rte_graph_mcore_dispatch_sched_wq_process(struct rte_graph *graph)
132 {
133 #define WQ_SZ 32
134 struct graph_mcore_dispatch_wq_node *wq_node;
135 struct rte_mempool *mp = graph->dispatch.mp;
136 struct rte_ring *wq = graph->dispatch.wq;
137 uint16_t idx, free_space;
138 struct rte_node *node;
139 unsigned int i, n;
140 struct graph_mcore_dispatch_wq_node *wq_nodes[WQ_SZ];
141
142 n = rte_ring_sc_dequeue_burst_elem(wq, wq_nodes, sizeof(wq_nodes[0]),
143 RTE_DIM(wq_nodes), NULL);
144 if (n == 0)
145 return;
146
147 for (i = 0; i < n; i++) {
148 wq_node = wq_nodes[i];
149 node = RTE_PTR_ADD(graph, wq_node->node_off);
150 RTE_ASSERT(node->fence == RTE_GRAPH_FENCE);
151 idx = node->idx;
152 free_space = node->size - idx;
153
154 if (unlikely(free_space < wq_node->nb_objs))
155 __rte_node_stream_alloc_size(graph, node, node->size + wq_node->nb_objs);
156
157 memmove(&node->objs[idx], wq_node->objs, wq_node->nb_objs * sizeof(void *));
158 node->idx = idx + wq_node->nb_objs;
159
160 __rte_node_process(graph, node);
161
162 wq_node->nb_objs = 0;
163 node->idx = 0;
164 }
165
166 rte_mempool_put_bulk(mp, (void **)wq_nodes, n);
167 }
168
169 int
rte_graph_model_mcore_dispatch_node_lcore_affinity_set(const char * name,unsigned int lcore_id)170 rte_graph_model_mcore_dispatch_node_lcore_affinity_set(const char *name, unsigned int lcore_id)
171 {
172 struct node *node;
173 int ret = -EINVAL;
174
175 if (lcore_id >= RTE_MAX_LCORE)
176 return ret;
177
178 graph_spinlock_lock();
179
180 STAILQ_FOREACH(node, node_list_head_get(), next) {
181 if (strncmp(node->name, name, RTE_NODE_NAMESIZE) == 0) {
182 node->lcore_id = lcore_id;
183 ret = 0;
184 break;
185 }
186 }
187
188 graph_spinlock_unlock();
189
190 return ret;
191 }
192