1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2019 6WIND S.A.
3 * Copyright 2019 Mellanox Technologies, Ltd
4 */
5
6 #include <stdio.h>
7 #include <stdlib.h>
8 #include <time.h>
9 #include <unistd.h>
10
11 #include <rte_eal.h>
12 #include <ethdev_driver.h>
13 #include <rte_string_fns.h>
14
15 #include "mlx4.h"
16 #include "mlx4_rxtx.h"
17 #include "mlx4_utils.h"
18
19 /**
20 * Initialize IPC message.
21 *
22 * @param[in] dev
23 * Pointer to Ethernet structure.
24 * @param[out] msg
25 * Pointer to message to fill in.
26 * @param[in] type
27 * Message type.
28 */
29 static inline void
mp_init_msg(struct rte_eth_dev * dev,struct rte_mp_msg * msg,enum mlx4_mp_req_type type)30 mp_init_msg(struct rte_eth_dev *dev, struct rte_mp_msg *msg,
31 enum mlx4_mp_req_type type)
32 {
33 struct mlx4_mp_param *param = (struct mlx4_mp_param *)msg->param;
34
35 memset(msg, 0, sizeof(*msg));
36 strlcpy(msg->name, MLX4_MP_NAME, sizeof(msg->name));
37 msg->len_param = sizeof(*param);
38 param->type = type;
39 param->port_id = dev->data->port_id;
40 }
41
42 /**
43 * IPC message handler of primary process.
44 *
45 * @param[in] dev
46 * Pointer to Ethernet structure.
47 * @param[in] peer
48 * Pointer to the peer socket path.
49 *
50 * @return
51 * 0 on success, negative errno value otherwise and rte_errno is set.
52 */
53 static int
mp_primary_handle(const struct rte_mp_msg * mp_msg,const void * peer)54 mp_primary_handle(const struct rte_mp_msg *mp_msg, const void *peer)
55 {
56 struct rte_mp_msg mp_res;
57 struct mlx4_mp_param *res = (struct mlx4_mp_param *)mp_res.param;
58 const struct mlx4_mp_param *param =
59 (const struct mlx4_mp_param *)mp_msg->param;
60 struct rte_eth_dev *dev;
61 struct mlx4_priv *priv;
62 struct mlx4_mr_cache entry;
63 uint32_t lkey;
64 int ret;
65
66 MLX4_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY);
67 if (!rte_eth_dev_is_valid_port(param->port_id)) {
68 rte_errno = ENODEV;
69 ERROR("port %u invalid port ID", param->port_id);
70 return -rte_errno;
71 }
72 dev = &rte_eth_devices[param->port_id];
73 priv = dev->data->dev_private;
74 switch (param->type) {
75 case MLX4_MP_REQ_CREATE_MR:
76 mp_init_msg(dev, &mp_res, param->type);
77 lkey = mlx4_mr_create_primary(dev, &entry, param->args.addr);
78 if (lkey == UINT32_MAX)
79 res->result = -rte_errno;
80 ret = rte_mp_reply(&mp_res, peer);
81 break;
82 case MLX4_MP_REQ_VERBS_CMD_FD:
83 mp_init_msg(dev, &mp_res, param->type);
84 mp_res.num_fds = 1;
85 mp_res.fds[0] = priv->ctx->cmd_fd;
86 res->result = 0;
87 ret = rte_mp_reply(&mp_res, peer);
88 break;
89 default:
90 rte_errno = EINVAL;
91 ERROR("port %u invalid mp request type", dev->data->port_id);
92 return -rte_errno;
93 }
94 return ret;
95 }
96
97 /**
98 * IPC message handler of a secondary process.
99 *
100 * @param[in] dev
101 * Pointer to Ethernet structure.
102 * @param[in] peer
103 * Pointer to the peer socket path.
104 *
105 * @return
106 * 0 on success, a negative errno value otherwise and rte_errno is set.
107 */
108 static int
mp_secondary_handle(const struct rte_mp_msg * mp_msg,const void * peer)109 mp_secondary_handle(const struct rte_mp_msg *mp_msg, const void *peer)
110 {
111 struct rte_mp_msg mp_res;
112 struct mlx4_mp_param *res = (struct mlx4_mp_param *)mp_res.param;
113 const struct mlx4_mp_param *param =
114 (const struct mlx4_mp_param *)mp_msg->param;
115 struct rte_eth_dev *dev;
116 #ifdef HAVE_IBV_MLX4_UAR_MMAP_OFFSET
117 struct mlx4_proc_priv *ppriv;
118 #endif
119 int ret;
120
121 MLX4_ASSERT(rte_eal_process_type() == RTE_PROC_SECONDARY);
122 if (!rte_eth_dev_is_valid_port(param->port_id)) {
123 rte_errno = ENODEV;
124 ERROR("port %u invalid port ID", param->port_id);
125 return -rte_errno;
126 }
127 dev = &rte_eth_devices[param->port_id];
128 switch (param->type) {
129 case MLX4_MP_REQ_START_RXTX:
130 INFO("port %u starting datapath", dev->data->port_id);
131 dev->tx_pkt_burst = mlx4_tx_burst;
132 dev->rx_pkt_burst = mlx4_rx_burst;
133 #ifdef HAVE_IBV_MLX4_UAR_MMAP_OFFSET
134 ppriv = (struct mlx4_proc_priv *)dev->process_private;
135 if (ppriv->uar_table_sz != dev->data->nb_tx_queues) {
136 mlx4_tx_uar_uninit_secondary(dev);
137 mlx4_proc_priv_uninit(dev);
138 ret = mlx4_proc_priv_init(dev);
139 if (ret) {
140 close(mp_msg->fds[0]);
141 return -rte_errno;
142 }
143 ret = mlx4_tx_uar_init_secondary(dev, mp_msg->fds[0]);
144 if (ret) {
145 close(mp_msg->fds[0]);
146 mlx4_proc_priv_uninit(dev);
147 return -rte_errno;
148 }
149 }
150 #endif
151 close(mp_msg->fds[0]);
152 rte_mb();
153 mp_init_msg(dev, &mp_res, param->type);
154 res->result = 0;
155 ret = rte_mp_reply(&mp_res, peer);
156 break;
157 case MLX4_MP_REQ_STOP_RXTX:
158 INFO("port %u stopping datapath", dev->data->port_id);
159 dev->tx_pkt_burst = rte_eth_pkt_burst_dummy;
160 dev->rx_pkt_burst = rte_eth_pkt_burst_dummy;
161 rte_mb();
162 mp_init_msg(dev, &mp_res, param->type);
163 res->result = 0;
164 ret = rte_mp_reply(&mp_res, peer);
165 break;
166 default:
167 rte_errno = EINVAL;
168 ERROR("port %u invalid mp request type", dev->data->port_id);
169 return -rte_errno;
170 }
171 return ret;
172 }
173
174 /**
175 * Broadcast request of stopping/starting data-path to secondary processes.
176 *
177 * @param[in] dev
178 * Pointer to Ethernet structure.
179 * @param[in] type
180 * Request type.
181 */
182 static void
mp_req_on_rxtx(struct rte_eth_dev * dev,enum mlx4_mp_req_type type)183 mp_req_on_rxtx(struct rte_eth_dev *dev, enum mlx4_mp_req_type type)
184 {
185 struct rte_mp_msg mp_req;
186 struct rte_mp_msg *mp_res;
187 struct rte_mp_reply mp_rep;
188 struct mlx4_mp_param *res __rte_unused;
189 struct timespec ts = {.tv_sec = MLX4_MP_REQ_TIMEOUT_SEC, .tv_nsec = 0};
190 struct mlx4_priv *priv;
191 int ret;
192 int i;
193
194 MLX4_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY);
195 if (!mlx4_shared_data->secondary_cnt)
196 return;
197 if (type != MLX4_MP_REQ_START_RXTX && type != MLX4_MP_REQ_STOP_RXTX) {
198 ERROR("port %u unknown request (req_type %d)",
199 dev->data->port_id, type);
200 return;
201 }
202 mp_init_msg(dev, &mp_req, type);
203 if (type == MLX4_MP_REQ_START_RXTX) {
204 priv = dev->data->dev_private;
205 mp_req.num_fds = 1;
206 mp_req.fds[0] = priv->ctx->cmd_fd;
207 }
208 ret = rte_mp_request_sync(&mp_req, &mp_rep, &ts);
209 if (ret) {
210 if (rte_errno != ENOTSUP)
211 ERROR("port %u failed to request stop/start Rx/Tx (%d)",
212 dev->data->port_id, type);
213 goto exit;
214 }
215 if (mp_rep.nb_sent != mp_rep.nb_received) {
216 ERROR("port %u not all secondaries responded (req_type %d)",
217 dev->data->port_id, type);
218 goto exit;
219 }
220 for (i = 0; i < mp_rep.nb_received; i++) {
221 mp_res = &mp_rep.msgs[i];
222 res = (struct mlx4_mp_param *)mp_res->param;
223 if (res->result) {
224 ERROR("port %u request failed on secondary #%d",
225 dev->data->port_id, i);
226 goto exit;
227 }
228 }
229 exit:
230 free(mp_rep.msgs);
231 }
232
233 /**
234 * Broadcast request of starting data-path to secondary processes. The request
235 * is synchronous.
236 *
237 * @param[in] dev
238 * Pointer to Ethernet structure.
239 */
240 void
mlx4_mp_req_start_rxtx(struct rte_eth_dev * dev)241 mlx4_mp_req_start_rxtx(struct rte_eth_dev *dev)
242 {
243 mp_req_on_rxtx(dev, MLX4_MP_REQ_START_RXTX);
244 }
245
246 /**
247 * Broadcast request of stopping data-path to secondary processes. The request
248 * is synchronous.
249 *
250 * @param[in] dev
251 * Pointer to Ethernet structure.
252 */
253 void
mlx4_mp_req_stop_rxtx(struct rte_eth_dev * dev)254 mlx4_mp_req_stop_rxtx(struct rte_eth_dev *dev)
255 {
256 mp_req_on_rxtx(dev, MLX4_MP_REQ_STOP_RXTX);
257 }
258
259 /**
260 * Request Memory Region creation to the primary process.
261 *
262 * @param[in] dev
263 * Pointer to Ethernet structure.
264 * @param addr
265 * Target virtual address to register.
266 *
267 * @return
268 * 0 on success, a negative errno value otherwise and rte_errno is set.
269 */
270 int
mlx4_mp_req_mr_create(struct rte_eth_dev * dev,uintptr_t addr)271 mlx4_mp_req_mr_create(struct rte_eth_dev *dev, uintptr_t addr)
272 {
273 struct rte_mp_msg mp_req;
274 struct rte_mp_msg *mp_res;
275 struct rte_mp_reply mp_rep;
276 struct mlx4_mp_param *req = (struct mlx4_mp_param *)mp_req.param;
277 struct mlx4_mp_param *res;
278 struct timespec ts = {.tv_sec = MLX4_MP_REQ_TIMEOUT_SEC, .tv_nsec = 0};
279 int ret;
280
281 MLX4_ASSERT(rte_eal_process_type() == RTE_PROC_SECONDARY);
282 mp_init_msg(dev, &mp_req, MLX4_MP_REQ_CREATE_MR);
283 req->args.addr = addr;
284 ret = rte_mp_request_sync(&mp_req, &mp_rep, &ts);
285 if (ret) {
286 ERROR("port %u request to primary process failed",
287 dev->data->port_id);
288 return -rte_errno;
289 }
290 MLX4_ASSERT(mp_rep.nb_received == 1);
291 mp_res = &mp_rep.msgs[0];
292 res = (struct mlx4_mp_param *)mp_res->param;
293 ret = res->result;
294 if (ret)
295 rte_errno = -ret;
296 free(mp_rep.msgs);
297 return ret;
298 }
299
300 /**
301 * IPC message handler of primary process.
302 *
303 * @param[in] dev
304 * Pointer to Ethernet structure.
305 *
306 * @return
307 * fd on success, a negative errno value otherwise and rte_errno is set.
308 */
309 int
mlx4_mp_req_verbs_cmd_fd(struct rte_eth_dev * dev)310 mlx4_mp_req_verbs_cmd_fd(struct rte_eth_dev *dev)
311 {
312 struct rte_mp_msg mp_req;
313 struct rte_mp_msg *mp_res;
314 struct rte_mp_reply mp_rep;
315 struct mlx4_mp_param *res;
316 struct timespec ts = {.tv_sec = MLX4_MP_REQ_TIMEOUT_SEC, .tv_nsec = 0};
317 int ret;
318
319 MLX4_ASSERT(rte_eal_process_type() == RTE_PROC_SECONDARY);
320 mp_init_msg(dev, &mp_req, MLX4_MP_REQ_VERBS_CMD_FD);
321 ret = rte_mp_request_sync(&mp_req, &mp_rep, &ts);
322 if (ret) {
323 ERROR("port %u request to primary process failed",
324 dev->data->port_id);
325 return -rte_errno;
326 }
327 MLX4_ASSERT(mp_rep.nb_received == 1);
328 mp_res = &mp_rep.msgs[0];
329 res = (struct mlx4_mp_param *)mp_res->param;
330 if (res->result) {
331 rte_errno = -res->result;
332 ERROR("port %u failed to get command FD from primary process",
333 dev->data->port_id);
334 ret = -rte_errno;
335 goto exit;
336 }
337 MLX4_ASSERT(mp_res->num_fds == 1);
338 ret = mp_res->fds[0];
339 DEBUG("port %u command FD from primary is %d",
340 dev->data->port_id, ret);
341 exit:
342 free(mp_rep.msgs);
343 return ret;
344 }
345
346 /**
347 * Initialize by primary process.
348 */
349 int
mlx4_mp_init_primary(void)350 mlx4_mp_init_primary(void)
351 {
352 int ret;
353
354 MLX4_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY);
355
356 /* primary is allowed to not support IPC */
357 ret = rte_mp_action_register(MLX4_MP_NAME, mp_primary_handle);
358 if (ret && rte_errno != ENOTSUP)
359 return -1;
360 return 0;
361 }
362
363 /**
364 * Un-initialize by primary process.
365 */
366 void
mlx4_mp_uninit_primary(void)367 mlx4_mp_uninit_primary(void)
368 {
369 MLX4_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY);
370 rte_mp_action_unregister(MLX4_MP_NAME);
371 }
372
373 /**
374 * Initialize by secondary process.
375 */
376 int
mlx4_mp_init_secondary(void)377 mlx4_mp_init_secondary(void)
378 {
379 MLX4_ASSERT(rte_eal_process_type() == RTE_PROC_SECONDARY);
380 return rte_mp_action_register(MLX4_MP_NAME, mp_secondary_handle);
381 }
382
383 /**
384 * Un-initialize by secondary process.
385 */
386 void
mlx4_mp_uninit_secondary(void)387 mlx4_mp_uninit_secondary(void)
388 {
389 MLX4_ASSERT(rte_eal_process_type() == RTE_PROC_SECONDARY);
390 rte_mp_action_unregister(MLX4_MP_NAME);
391 }
392