1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(C) 2023 Intel Corporation 3 */ 4 5 #include "graph_private.h" 6 #include "rte_graph_model_mcore_dispatch.h" 7 8 int 9 graph_sched_wq_create(struct graph *_graph, struct graph *_parent_graph, 10 struct rte_graph_param *prm) 11 { 12 struct rte_graph *parent_graph = _parent_graph->graph; 13 struct rte_graph *graph = _graph->graph; 14 unsigned int wq_size; 15 unsigned int flags = RING_F_SC_DEQ; 16 17 wq_size = RTE_GRAPH_SCHED_WQ_SIZE(graph->nb_nodes); 18 wq_size = rte_align32pow2(wq_size + 1); 19 20 if (prm->dispatch.wq_size_max > 0) 21 wq_size = wq_size <= (prm->dispatch.wq_size_max) ? wq_size : 22 prm->dispatch.wq_size_max; 23 24 if (!rte_is_power_of_2(wq_size)) 25 flags |= RING_F_EXACT_SZ; 26 27 graph->dispatch.wq = rte_ring_create(graph->name, wq_size, graph->socket, 28 flags); 29 if (graph->dispatch.wq == NULL) 30 SET_ERR_JMP(EIO, fail, "Failed to allocate graph WQ"); 31 32 if (prm->dispatch.mp_capacity > 0) 33 wq_size = (wq_size <= prm->dispatch.mp_capacity) ? wq_size : 34 prm->dispatch.mp_capacity; 35 36 graph->dispatch.mp = rte_mempool_create(graph->name, wq_size, 37 sizeof(struct graph_mcore_dispatch_wq_node), 38 0, 0, NULL, NULL, NULL, NULL, 39 graph->socket, MEMPOOL_F_SP_PUT); 40 if (graph->dispatch.mp == NULL) 41 SET_ERR_JMP(EIO, fail_mp, 42 "Failed to allocate graph WQ schedule entry"); 43 44 graph->dispatch.lcore_id = _graph->lcore_id; 45 46 if (parent_graph->dispatch.rq == NULL) { 47 parent_graph->dispatch.rq = &parent_graph->dispatch.rq_head; 48 SLIST_INIT(parent_graph->dispatch.rq); 49 } 50 51 graph->dispatch.rq = parent_graph->dispatch.rq; 52 SLIST_INSERT_HEAD(graph->dispatch.rq, graph, next); 53 54 return 0; 55 56 fail_mp: 57 rte_ring_free(graph->dispatch.wq); 58 graph->dispatch.wq = NULL; 59 fail: 60 return -rte_errno; 61 } 62 63 void 64 graph_sched_wq_destroy(struct graph *_graph) 65 { 66 struct rte_graph *graph = _graph->graph; 67 68 if (graph == NULL) 69 return; 70 71 rte_ring_free(graph->dispatch.wq); 72 graph->dispatch.wq = NULL; 73 74 rte_mempool_free(graph->dispatch.mp); 75 graph->dispatch.mp = NULL; 76 } 77 78 static __rte_always_inline bool 79 __graph_sched_node_enqueue(struct rte_node *node, struct rte_graph *graph) 80 { 81 struct graph_mcore_dispatch_wq_node *wq_node; 82 uint16_t off = 0; 83 uint16_t size; 84 85 submit_again: 86 if (rte_mempool_get(graph->dispatch.mp, (void **)&wq_node) < 0) 87 goto fallback; 88 89 size = RTE_MIN(node->idx, RTE_DIM(wq_node->objs)); 90 wq_node->node_off = node->off; 91 wq_node->nb_objs = size; 92 rte_memcpy(wq_node->objs, &node->objs[off], size * sizeof(void *)); 93 94 while (rte_ring_mp_enqueue_bulk_elem(graph->dispatch.wq, (void *)&wq_node, 95 sizeof(wq_node), 1, NULL) == 0) 96 rte_pause(); 97 98 off += size; 99 node->dispatch.total_sched_objs += size; 100 node->idx -= size; 101 if (node->idx > 0) 102 goto submit_again; 103 104 return true; 105 106 fallback: 107 if (off != 0) 108 memmove(&node->objs[0], &node->objs[off], 109 node->idx * sizeof(void *)); 110 111 node->dispatch.total_sched_fail += node->idx; 112 113 return false; 114 } 115 116 bool __rte_noinline 117 __rte_graph_mcore_dispatch_sched_node_enqueue(struct rte_node *node, 118 struct rte_graph_rq_head *rq) 119 { 120 const unsigned int lcore_id = node->dispatch.lcore_id; 121 struct rte_graph *graph; 122 123 SLIST_FOREACH(graph, rq, next) 124 if (graph->dispatch.lcore_id == lcore_id) 125 break; 126 127 return graph != NULL ? __graph_sched_node_enqueue(node, graph) : false; 128 } 129 130 void 131 __rte_graph_mcore_dispatch_sched_wq_process(struct rte_graph *graph) 132 { 133 #define WQ_SZ 32 134 struct graph_mcore_dispatch_wq_node *wq_node; 135 struct rte_mempool *mp = graph->dispatch.mp; 136 struct rte_ring *wq = graph->dispatch.wq; 137 uint16_t idx, free_space; 138 struct rte_node *node; 139 unsigned int i, n; 140 struct graph_mcore_dispatch_wq_node *wq_nodes[WQ_SZ]; 141 142 n = rte_ring_sc_dequeue_burst_elem(wq, wq_nodes, sizeof(wq_nodes[0]), 143 RTE_DIM(wq_nodes), NULL); 144 if (n == 0) 145 return; 146 147 for (i = 0; i < n; i++) { 148 wq_node = wq_nodes[i]; 149 node = RTE_PTR_ADD(graph, wq_node->node_off); 150 RTE_ASSERT(node->fence == RTE_GRAPH_FENCE); 151 idx = node->idx; 152 free_space = node->size - idx; 153 154 if (unlikely(free_space < wq_node->nb_objs)) 155 __rte_node_stream_alloc_size(graph, node, node->size + wq_node->nb_objs); 156 157 memmove(&node->objs[idx], wq_node->objs, wq_node->nb_objs * sizeof(void *)); 158 node->idx = idx + wq_node->nb_objs; 159 160 __rte_node_process(graph, node); 161 162 wq_node->nb_objs = 0; 163 node->idx = 0; 164 } 165 166 rte_mempool_put_bulk(mp, (void **)wq_nodes, n); 167 } 168 169 int 170 rte_graph_model_mcore_dispatch_node_lcore_affinity_set(const char *name, unsigned int lcore_id) 171 { 172 struct node *node; 173 int ret = -EINVAL; 174 175 if (lcore_id >= RTE_MAX_LCORE) 176 return ret; 177 178 graph_spinlock_lock(); 179 180 STAILQ_FOREACH(node, node_list_head_get(), next) { 181 if (strncmp(node->name, name, RTE_NODE_NAMESIZE) == 0) { 182 node->lcore_id = lcore_id; 183 ret = 0; 184 break; 185 } 186 } 187 188 graph_spinlock_unlock(); 189 190 return ret; 191 } 192