1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright (c) 2022 NVIDIA Corporation & Affiliates
3 */
4 #include <string.h>
5 #include <unistd.h>
6 #include <sys/eventfd.h>
7
8 #include <rte_malloc.h>
9 #include <rte_errno.h>
10 #include <rte_io.h>
11 #include <rte_alarm.h>
12 #include <rte_tailq.h>
13 #include <rte_ring_elem.h>
14 #include <rte_ring_peek.h>
15
16 #include <mlx5_common.h>
17
18 #include "mlx5_vdpa_utils.h"
19 #include "mlx5_vdpa.h"
20
21 static inline uint32_t
mlx5_vdpa_c_thrd_ring_dequeue_bulk(struct rte_ring * r,void ** obj,uint32_t n,uint32_t * avail)22 mlx5_vdpa_c_thrd_ring_dequeue_bulk(struct rte_ring *r,
23 void **obj, uint32_t n, uint32_t *avail)
24 {
25 uint32_t m;
26
27 m = rte_ring_dequeue_bulk_elem_start(r, obj,
28 sizeof(struct mlx5_vdpa_task), n, avail);
29 n = (m == n) ? n : 0;
30 rte_ring_dequeue_elem_finish(r, n);
31 return n;
32 }
33
34 static inline uint32_t
mlx5_vdpa_c_thrd_ring_enqueue_bulk(struct rte_ring * r,void * const * obj,uint32_t n,uint32_t * free)35 mlx5_vdpa_c_thrd_ring_enqueue_bulk(struct rte_ring *r,
36 void * const *obj, uint32_t n, uint32_t *free)
37 {
38 uint32_t m;
39
40 m = rte_ring_enqueue_bulk_elem_start(r, n, free);
41 n = (m == n) ? n : 0;
42 rte_ring_enqueue_elem_finish(r, obj,
43 sizeof(struct mlx5_vdpa_task), n);
44 return n;
45 }
46
47 bool
mlx5_vdpa_task_add(struct mlx5_vdpa_priv * priv,uint32_t thrd_idx,enum mlx5_vdpa_task_type task_type,RTE_ATOMIC (uint32_t)* remaining_cnt,RTE_ATOMIC (uint32_t)* err_cnt,void ** task_data,uint32_t num)48 mlx5_vdpa_task_add(struct mlx5_vdpa_priv *priv,
49 uint32_t thrd_idx,
50 enum mlx5_vdpa_task_type task_type,
51 RTE_ATOMIC(uint32_t) *remaining_cnt, RTE_ATOMIC(uint32_t) *err_cnt,
52 void **task_data, uint32_t num)
53 {
54 struct rte_ring *rng = conf_thread_mng.cthrd[thrd_idx].rng;
55 struct mlx5_vdpa_task task[MLX5_VDPA_TASKS_PER_DEV];
56 uint32_t *data = (uint32_t *)task_data;
57 uint32_t i;
58
59 MLX5_ASSERT(num <= MLX5_VDPA_TASKS_PER_DEV);
60 for (i = 0 ; i < num; i++) {
61 task[i].priv = priv;
62 /* To be added later. */
63 task[i].type = task_type;
64 task[i].remaining_cnt = remaining_cnt;
65 task[i].err_cnt = err_cnt;
66 if (data)
67 task[i].idx = data[i];
68 }
69 if (!mlx5_vdpa_c_thrd_ring_enqueue_bulk(rng, (void **)&task, num, NULL))
70 return -1;
71 for (i = 0 ; i < num; i++)
72 if (task[i].remaining_cnt)
73 rte_atomic_fetch_add_explicit(task[i].remaining_cnt, 1,
74 rte_memory_order_relaxed);
75 /* wake up conf thread. */
76 pthread_mutex_lock(&conf_thread_mng.cthrd_lock);
77 pthread_cond_signal(&conf_thread_mng.cthrd[thrd_idx].c_cond);
78 pthread_mutex_unlock(&conf_thread_mng.cthrd_lock);
79 return 0;
80 }
81
82 bool
mlx5_vdpa_c_thread_wait_bulk_tasks_done(RTE_ATOMIC (uint32_t)* remaining_cnt,RTE_ATOMIC (uint32_t)* err_cnt,uint32_t sleep_time)83 mlx5_vdpa_c_thread_wait_bulk_tasks_done(RTE_ATOMIC(uint32_t) *remaining_cnt,
84 RTE_ATOMIC(uint32_t) *err_cnt, uint32_t sleep_time)
85 {
86 /* Check and wait all tasks done. */
87 while (rte_atomic_load_explicit(remaining_cnt,
88 rte_memory_order_relaxed) != 0) {
89 rte_delay_us_sleep(sleep_time);
90 }
91 if (rte_atomic_load_explicit(err_cnt,
92 rte_memory_order_relaxed)) {
93 DRV_LOG(ERR, "Tasks done with error.");
94 return true;
95 }
96 return false;
97 }
98
99 static uint32_t
mlx5_vdpa_c_thread_handle(void * arg)100 mlx5_vdpa_c_thread_handle(void *arg)
101 {
102 struct mlx5_vdpa_conf_thread_mng *multhrd = arg;
103 struct mlx5_vdpa_virtq *virtq;
104 struct mlx5_vdpa_priv *priv;
105 struct mlx5_vdpa_task task;
106 struct rte_ring *rng;
107 uint64_t features;
108 uint32_t thrd_idx;
109 uint32_t task_num;
110 int ret;
111
112 for (thrd_idx = 0; thrd_idx < multhrd->max_thrds;
113 thrd_idx++)
114 if (rte_thread_equal(multhrd->cthrd[thrd_idx].tid, rte_thread_self()))
115 break;
116 if (thrd_idx >= multhrd->max_thrds)
117 return 1;
118 rng = multhrd->cthrd[thrd_idx].rng;
119 while (1) {
120 task_num = mlx5_vdpa_c_thrd_ring_dequeue_bulk(rng,
121 (void **)&task, 1, NULL);
122 if (!task_num) {
123 /* No task and condition wait. */
124 pthread_mutex_lock(&multhrd->cthrd_lock);
125 pthread_cond_wait(
126 &multhrd->cthrd[thrd_idx].c_cond,
127 &multhrd->cthrd_lock);
128 pthread_mutex_unlock(&multhrd->cthrd_lock);
129 continue;
130 }
131 priv = task.priv;
132 if (priv == NULL)
133 continue;
134 switch (task.type) {
135 case MLX5_VDPA_TASK_REG_MR:
136 ret = mlx5_vdpa_register_mr(priv, task.idx);
137 if (ret) {
138 DRV_LOG(ERR,
139 "Failed to register mr %d.", task.idx);
140 rte_atomic_fetch_add_explicit(task.err_cnt, 1,
141 rte_memory_order_relaxed);
142 }
143 break;
144 case MLX5_VDPA_TASK_SETUP_VIRTQ:
145 virtq = &priv->virtqs[task.idx];
146 pthread_mutex_lock(&virtq->virtq_lock);
147 ret = mlx5_vdpa_virtq_setup(priv,
148 task.idx, false);
149 if (ret) {
150 DRV_LOG(ERR,
151 "Failed to setup virtq %d.", task.idx);
152 rte_atomic_fetch_add_explicit(
153 task.err_cnt, 1, rte_memory_order_relaxed);
154 }
155 virtq->enable = 1;
156 pthread_mutex_unlock(&virtq->virtq_lock);
157 break;
158 case MLX5_VDPA_TASK_STOP_VIRTQ:
159 virtq = &priv->virtqs[task.idx];
160 pthread_mutex_lock(&virtq->virtq_lock);
161 ret = mlx5_vdpa_virtq_stop(priv,
162 task.idx);
163 if (ret) {
164 DRV_LOG(ERR,
165 "Failed to stop virtq %d.",
166 task.idx);
167 rte_atomic_fetch_add_explicit(
168 task.err_cnt, 1,
169 rte_memory_order_relaxed);
170 pthread_mutex_unlock(&virtq->virtq_lock);
171 break;
172 }
173 ret = rte_vhost_get_negotiated_features(
174 priv->vid, &features);
175 if (ret) {
176 DRV_LOG(ERR,
177 "Failed to get negotiated features virtq %d.",
178 task.idx);
179 rte_atomic_fetch_add_explicit(
180 task.err_cnt, 1,
181 rte_memory_order_relaxed);
182 pthread_mutex_unlock(&virtq->virtq_lock);
183 break;
184 }
185 if (RTE_VHOST_NEED_LOG(features))
186 rte_vhost_log_used_vring(
187 priv->vid, task.idx, 0,
188 MLX5_VDPA_USED_RING_LEN(virtq->vq_size));
189 pthread_mutex_unlock(&virtq->virtq_lock);
190 break;
191 case MLX5_VDPA_TASK_DEV_CLOSE_NOWAIT:
192 pthread_mutex_lock(&priv->steer_update_lock);
193 mlx5_vdpa_steer_unset(priv);
194 pthread_mutex_unlock(&priv->steer_update_lock);
195 mlx5_vdpa_virtqs_release(priv, false);
196 mlx5_vdpa_drain_cq(priv);
197 if (priv->lm_mr.addr)
198 mlx5_os_wrapped_mkey_destroy(
199 &priv->lm_mr);
200 if (!priv->connected)
201 mlx5_vdpa_dev_cache_clean(priv);
202 priv->vid = 0;
203 rte_atomic_store_explicit(
204 &priv->dev_close_progress, 0,
205 rte_memory_order_relaxed);
206 break;
207 case MLX5_VDPA_TASK_PREPARE_VIRTQ:
208 ret = mlx5_vdpa_virtq_single_resource_prepare(
209 priv, task.idx);
210 if (ret) {
211 DRV_LOG(ERR,
212 "Failed to prepare virtq %d.",
213 task.idx);
214 rte_atomic_fetch_add_explicit(
215 task.err_cnt, 1,
216 rte_memory_order_relaxed);
217 }
218 break;
219 default:
220 DRV_LOG(ERR, "Invalid vdpa task type %d.",
221 task.type);
222 break;
223 }
224 if (task.remaining_cnt)
225 rte_atomic_fetch_sub_explicit(task.remaining_cnt,
226 1, rte_memory_order_relaxed);
227 }
228 return 0;
229 }
230
231 static void
mlx5_vdpa_c_thread_destroy(uint32_t thrd_idx,bool need_unlock)232 mlx5_vdpa_c_thread_destroy(uint32_t thrd_idx, bool need_unlock)
233 {
234 pthread_t *tid = (pthread_t *)&conf_thread_mng.cthrd[thrd_idx].tid.opaque_id;
235 if (*tid != 0) {
236 pthread_cancel(*tid);
237 rte_thread_join(conf_thread_mng.cthrd[thrd_idx].tid, NULL);
238 *tid = 0;
239 if (need_unlock)
240 pthread_mutex_init(&conf_thread_mng.cthrd_lock, NULL);
241 }
242 if (conf_thread_mng.cthrd[thrd_idx].rng) {
243 rte_ring_free(conf_thread_mng.cthrd[thrd_idx].rng);
244 conf_thread_mng.cthrd[thrd_idx].rng = NULL;
245 }
246 }
247
248 static int
mlx5_vdpa_c_thread_create(void)249 mlx5_vdpa_c_thread_create(void)
250 {
251 uint32_t thrd_idx;
252 uint32_t ring_num;
253 char name[RTE_RING_NAMESIZE];
254 int ret;
255
256 pthread_mutex_lock(&conf_thread_mng.cthrd_lock);
257 ring_num = MLX5_VDPA_MAX_TASKS_PER_THRD / conf_thread_mng.max_thrds;
258 if (!ring_num) {
259 DRV_LOG(ERR, "Invalid ring number for thread.");
260 goto c_thread_err;
261 }
262 for (thrd_idx = 0; thrd_idx < conf_thread_mng.max_thrds;
263 thrd_idx++) {
264 snprintf(name, sizeof(name), "vDPA-mthread-ring-%d",
265 thrd_idx);
266 conf_thread_mng.cthrd[thrd_idx].rng = rte_ring_create_elem(name,
267 sizeof(struct mlx5_vdpa_task), ring_num,
268 rte_socket_id(),
269 RING_F_MP_HTS_ENQ | RING_F_MC_HTS_DEQ |
270 RING_F_EXACT_SZ);
271 if (!conf_thread_mng.cthrd[thrd_idx].rng) {
272 DRV_LOG(ERR,
273 "Failed to create vdpa multi-threads %d ring.",
274 thrd_idx);
275 goto c_thread_err;
276 }
277 snprintf(name, RTE_THREAD_INTERNAL_NAME_SIZE, "vmlx5-c%d", thrd_idx);
278 ret = rte_thread_create_internal_control(&conf_thread_mng.cthrd[thrd_idx].tid,
279 name,
280 mlx5_vdpa_c_thread_handle, &conf_thread_mng);
281 if (ret) {
282 DRV_LOG(ERR, "Failed to create vdpa multi-threads %d.",
283 thrd_idx);
284 goto c_thread_err;
285 }
286 pthread_cond_init(&conf_thread_mng.cthrd[thrd_idx].c_cond,
287 NULL);
288 }
289 pthread_mutex_unlock(&conf_thread_mng.cthrd_lock);
290 return 0;
291 c_thread_err:
292 for (thrd_idx = 0; thrd_idx < conf_thread_mng.max_thrds;
293 thrd_idx++)
294 mlx5_vdpa_c_thread_destroy(thrd_idx, false);
295 pthread_mutex_unlock(&conf_thread_mng.cthrd_lock);
296 return -1;
297 }
298
299 int
mlx5_vdpa_mult_threads_create(void)300 mlx5_vdpa_mult_threads_create(void)
301 {
302 pthread_mutex_init(&conf_thread_mng.cthrd_lock, NULL);
303 if (mlx5_vdpa_c_thread_create()) {
304 DRV_LOG(ERR, "Cannot create vDPA configuration threads.");
305 mlx5_vdpa_mult_threads_destroy(false);
306 return -1;
307 }
308 return 0;
309 }
310
311 void
mlx5_vdpa_mult_threads_destroy(bool need_unlock)312 mlx5_vdpa_mult_threads_destroy(bool need_unlock)
313 {
314 uint32_t thrd_idx;
315
316 if (!conf_thread_mng.initializer_priv)
317 return;
318 for (thrd_idx = 0; thrd_idx < conf_thread_mng.max_thrds;
319 thrd_idx++)
320 mlx5_vdpa_c_thread_destroy(thrd_idx, need_unlock);
321 pthread_mutex_destroy(&conf_thread_mng.cthrd_lock);
322 memset(&conf_thread_mng, 0, sizeof(struct mlx5_vdpa_conf_thread_mng));
323 }
324