1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2019 6WIND S.A. 3 * Copyright 2019 Mellanox Technologies, Ltd 4 */ 5 6 #include <stdio.h> 7 #include <time.h> 8 9 #include <rte_eal.h> 10 #include <ethdev_driver.h> 11 #include <rte_string_fns.h> 12 13 #include <mlx5_common_mp.h> 14 #include <mlx5_common_mr.h> 15 #include <mlx5_malloc.h> 16 17 #include "mlx5.h" 18 #include "mlx5_rxtx.h" 19 #include "mlx5_rx.h" 20 #include "mlx5_tx.h" 21 #include "mlx5_utils.h" 22 23 /** 24 * Handle a port-agnostic message. 25 * 26 * @return 27 * 0 on success, 1 when message is not port-agnostic, (-1) on error. 28 */ 29 static int 30 mlx5_mp_os_handle_port_agnostic(const struct rte_mp_msg *mp_msg, 31 const void *peer) 32 { 33 struct rte_mp_msg mp_res; 34 struct mlx5_mp_param *res = (struct mlx5_mp_param *)mp_res.param; 35 const struct mlx5_mp_param *param = 36 (const struct mlx5_mp_param *)mp_msg->param; 37 const struct mlx5_mp_arg_mempool_reg *mpr; 38 struct mlx5_mp_id mp_id; 39 40 switch (param->type) { 41 case MLX5_MP_REQ_MEMPOOL_REGISTER: 42 mlx5_mp_id_init(&mp_id, param->port_id); 43 mp_init_msg(&mp_id, &mp_res, param->type); 44 mpr = ¶m->args.mempool_reg; 45 res->result = mlx5_mr_mempool_register(mpr->share_cache, 46 mpr->pd, mpr->mempool, 47 NULL); 48 return rte_mp_reply(&mp_res, peer); 49 case MLX5_MP_REQ_MEMPOOL_UNREGISTER: 50 mlx5_mp_id_init(&mp_id, param->port_id); 51 mp_init_msg(&mp_id, &mp_res, param->type); 52 mpr = ¶m->args.mempool_reg; 53 res->result = mlx5_mr_mempool_unregister(mpr->share_cache, 54 mpr->mempool, NULL); 55 return rte_mp_reply(&mp_res, peer); 56 default: 57 return 1; 58 } 59 return -1; 60 } 61 62 int 63 mlx5_mp_os_primary_handle(const struct rte_mp_msg *mp_msg, const void *peer) 64 { 65 struct rte_mp_msg mp_res; 66 struct mlx5_mp_param *res = (struct mlx5_mp_param *)mp_res.param; 67 const struct mlx5_mp_param *param = 68 (const struct mlx5_mp_param *)mp_msg->param; 69 struct rte_eth_dev *dev; 70 struct mlx5_priv *priv; 71 struct mr_cache_entry entry; 72 uint32_t lkey; 73 int ret; 74 75 MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY); 76 /* Port-agnostic messages. */ 77 ret = mlx5_mp_os_handle_port_agnostic(mp_msg, peer); 78 if (ret <= 0) 79 return ret; 80 /* Port-specific messages. */ 81 if (!rte_eth_dev_is_valid_port(param->port_id)) { 82 rte_errno = ENODEV; 83 DRV_LOG(ERR, "port %u invalid port ID", param->port_id); 84 return -rte_errno; 85 } 86 dev = &rte_eth_devices[param->port_id]; 87 priv = dev->data->dev_private; 88 switch (param->type) { 89 case MLX5_MP_REQ_CREATE_MR: 90 mp_init_msg(&priv->mp_id, &mp_res, param->type); 91 lkey = mlx5_mr_create_primary(priv->sh->pd, 92 &priv->sh->share_cache, 93 &entry, param->args.addr, 94 priv->config.mr_ext_memseg_en); 95 if (lkey == UINT32_MAX) 96 res->result = -rte_errno; 97 ret = rte_mp_reply(&mp_res, peer); 98 break; 99 case MLX5_MP_REQ_VERBS_CMD_FD: 100 mp_init_msg(&priv->mp_id, &mp_res, param->type); 101 mp_res.num_fds = 1; 102 mp_res.fds[0] = ((struct ibv_context *)priv->sh->ctx)->cmd_fd; 103 res->result = 0; 104 ret = rte_mp_reply(&mp_res, peer); 105 break; 106 case MLX5_MP_REQ_QUEUE_STATE_MODIFY: 107 mp_init_msg(&priv->mp_id, &mp_res, param->type); 108 res->result = mlx5_queue_state_modify_primary 109 (dev, ¶m->args.state_modify); 110 ret = rte_mp_reply(&mp_res, peer); 111 break; 112 case MLX5_MP_REQ_QUEUE_RX_STOP: 113 mp_init_msg(&priv->mp_id, &mp_res, param->type); 114 res->result = mlx5_rx_queue_stop_primary 115 (dev, param->args.queue_id.queue_id); 116 ret = rte_mp_reply(&mp_res, peer); 117 break; 118 case MLX5_MP_REQ_QUEUE_RX_START: 119 mp_init_msg(&priv->mp_id, &mp_res, param->type); 120 res->result = mlx5_rx_queue_start_primary 121 (dev, param->args.queue_id.queue_id); 122 ret = rte_mp_reply(&mp_res, peer); 123 break; 124 case MLX5_MP_REQ_QUEUE_TX_STOP: 125 mp_init_msg(&priv->mp_id, &mp_res, param->type); 126 res->result = mlx5_tx_queue_stop_primary 127 (dev, param->args.queue_id.queue_id); 128 ret = rte_mp_reply(&mp_res, peer); 129 break; 130 case MLX5_MP_REQ_QUEUE_TX_START: 131 mp_init_msg(&priv->mp_id, &mp_res, param->type); 132 res->result = mlx5_tx_queue_start_primary 133 (dev, param->args.queue_id.queue_id); 134 ret = rte_mp_reply(&mp_res, peer); 135 break; 136 default: 137 rte_errno = EINVAL; 138 DRV_LOG(ERR, "port %u invalid mp request type", 139 dev->data->port_id); 140 return -rte_errno; 141 } 142 return ret; 143 } 144 145 /** 146 * IPC message handler of a secondary process. 147 * 148 * @param[in] dev 149 * Pointer to Ethernet structure. 150 * @param[in] peer 151 * Pointer to the peer socket path. 152 * 153 * @return 154 * 0 on success, a negative errno value otherwise and rte_errno is set. 155 */ 156 int 157 mlx5_mp_os_secondary_handle(const struct rte_mp_msg *mp_msg, const void *peer) 158 { 159 struct rte_mp_msg mp_res; 160 struct mlx5_mp_param *res = (struct mlx5_mp_param *)mp_res.param; 161 const struct mlx5_mp_param *param = 162 (const struct mlx5_mp_param *)mp_msg->param; 163 struct rte_eth_dev *dev; 164 struct mlx5_proc_priv *ppriv; 165 struct mlx5_priv *priv; 166 int ret; 167 168 MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_SECONDARY); 169 if (!rte_eth_dev_is_valid_port(param->port_id)) { 170 rte_errno = ENODEV; 171 DRV_LOG(ERR, "port %u invalid port ID", param->port_id); 172 return -rte_errno; 173 } 174 dev = &rte_eth_devices[param->port_id]; 175 priv = dev->data->dev_private; 176 switch (param->type) { 177 case MLX5_MP_REQ_START_RXTX: 178 DRV_LOG(INFO, "port %u starting datapath", dev->data->port_id); 179 dev->rx_pkt_burst = mlx5_select_rx_function(dev); 180 dev->tx_pkt_burst = mlx5_select_tx_function(dev); 181 ppriv = (struct mlx5_proc_priv *)dev->process_private; 182 /* If Tx queue number changes, re-initialize UAR. */ 183 if (ppriv->uar_table_sz != priv->txqs_n) { 184 mlx5_tx_uar_uninit_secondary(dev); 185 mlx5_proc_priv_uninit(dev); 186 ret = mlx5_proc_priv_init(dev); 187 if (ret) 188 return -rte_errno; 189 ret = mlx5_tx_uar_init_secondary(dev, mp_msg->fds[0]); 190 if (ret) { 191 mlx5_proc_priv_uninit(dev); 192 return -rte_errno; 193 } 194 } 195 rte_mb(); 196 mp_init_msg(&priv->mp_id, &mp_res, param->type); 197 res->result = 0; 198 ret = rte_mp_reply(&mp_res, peer); 199 break; 200 case MLX5_MP_REQ_STOP_RXTX: 201 DRV_LOG(INFO, "port %u stopping datapath", dev->data->port_id); 202 dev->rx_pkt_burst = removed_rx_burst; 203 dev->tx_pkt_burst = removed_tx_burst; 204 rte_mb(); 205 mp_init_msg(&priv->mp_id, &mp_res, param->type); 206 res->result = 0; 207 ret = rte_mp_reply(&mp_res, peer); 208 break; 209 default: 210 rte_errno = EINVAL; 211 DRV_LOG(ERR, "port %u invalid mp request type", 212 dev->data->port_id); 213 return -rte_errno; 214 } 215 return ret; 216 } 217 218 /** 219 * Broadcast request of stopping/starting data-path to secondary processes. 220 * 221 * @param[in] dev 222 * Pointer to Ethernet structure. 223 * @param[in] type 224 * Request type. 225 */ 226 static void 227 mp_req_on_rxtx(struct rte_eth_dev *dev, enum mlx5_mp_req_type type) 228 { 229 struct rte_mp_msg mp_req; 230 struct rte_mp_msg *mp_res; 231 struct rte_mp_reply mp_rep; 232 struct mlx5_mp_param *res; 233 struct timespec ts = {.tv_sec = MLX5_MP_REQ_TIMEOUT_SEC, .tv_nsec = 0}; 234 struct mlx5_priv *priv = dev->data->dev_private; 235 int ret; 236 int i; 237 238 MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY); 239 if (!mlx5_shared_data->secondary_cnt) 240 return; 241 if (type != MLX5_MP_REQ_START_RXTX && type != MLX5_MP_REQ_STOP_RXTX) { 242 DRV_LOG(ERR, "port %u unknown request (req_type %d)", 243 dev->data->port_id, type); 244 return; 245 } 246 mp_init_msg(&priv->mp_id, &mp_req, type); 247 if (type == MLX5_MP_REQ_START_RXTX) { 248 mp_req.num_fds = 1; 249 mp_req.fds[0] = ((struct ibv_context *)priv->sh->ctx)->cmd_fd; 250 } 251 ret = rte_mp_request_sync(&mp_req, &mp_rep, &ts); 252 if (ret) { 253 if (rte_errno != ENOTSUP) 254 DRV_LOG(ERR, "port %u failed to request stop/start Rx/Tx (%d)", 255 dev->data->port_id, type); 256 goto exit; 257 } 258 if (mp_rep.nb_sent != mp_rep.nb_received) { 259 DRV_LOG(ERR, 260 "port %u not all secondaries responded (req_type %d)", 261 dev->data->port_id, type); 262 goto exit; 263 } 264 for (i = 0; i < mp_rep.nb_received; i++) { 265 mp_res = &mp_rep.msgs[i]; 266 res = (struct mlx5_mp_param *)mp_res->param; 267 if (res->result) { 268 DRV_LOG(ERR, "port %u request failed on secondary #%d", 269 dev->data->port_id, i); 270 goto exit; 271 } 272 } 273 exit: 274 mlx5_free(mp_rep.msgs); 275 } 276 277 /** 278 * Broadcast request of starting data-path to secondary processes. The request 279 * is synchronous. 280 * 281 * @param[in] dev 282 * Pointer to Ethernet structure. 283 */ 284 void 285 mlx5_mp_os_req_start_rxtx(struct rte_eth_dev *dev) 286 { 287 mp_req_on_rxtx(dev, MLX5_MP_REQ_START_RXTX); 288 } 289 290 /** 291 * Broadcast request of stopping data-path to secondary processes. The request 292 * is synchronous. 293 * 294 * @param[in] dev 295 * Pointer to Ethernet structure. 296 */ 297 void 298 mlx5_mp_os_req_stop_rxtx(struct rte_eth_dev *dev) 299 { 300 mp_req_on_rxtx(dev, MLX5_MP_REQ_STOP_RXTX); 301 } 302 303 /** 304 * Request Verbs Rx/Tx queue stop or start to the primary process. 305 * 306 * @param[in] dev 307 * Pointer to Ethernet structure. 308 * @param queue_id 309 * Queue ID to control. 310 * @param req_type 311 * request type 312 * MLX5_MP_REQ_QUEUE_RX_START - start Rx queue 313 * MLX5_MP_REQ_QUEUE_TX_START - stop Tx queue 314 * MLX5_MP_REQ_QUEUE_RX_STOP - stop Rx queue 315 * MLX5_MP_REQ_QUEUE_TX_STOP - stop Tx queue 316 * @return 317 * 0 on success, a negative errno value otherwise and 318 * rte_errno is set. 319 */ 320 int 321 mlx5_mp_os_req_queue_control(struct rte_eth_dev *dev, uint16_t queue_id, 322 enum mlx5_mp_req_type req_type) 323 { 324 struct rte_mp_msg mp_req; 325 struct rte_mp_msg *mp_res; 326 struct rte_mp_reply mp_rep; 327 struct mlx5_mp_param *req = (struct mlx5_mp_param *)mp_req.param; 328 struct mlx5_mp_param *res; 329 struct timespec ts = {.tv_sec = MLX5_MP_REQ_TIMEOUT_SEC, .tv_nsec = 0}; 330 struct mlx5_priv *priv; 331 int ret; 332 333 MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_SECONDARY); 334 priv = dev->data->dev_private; 335 mp_init_msg(&priv->mp_id, &mp_req, req_type); 336 req->args.queue_id.queue_id = queue_id; 337 ret = rte_mp_request_sync(&mp_req, &mp_rep, &ts); 338 if (ret) { 339 DRV_LOG(ERR, "port %u request to primary process failed", 340 dev->data->port_id); 341 return -rte_errno; 342 } 343 MLX5_ASSERT(mp_rep.nb_received == 1); 344 mp_res = &mp_rep.msgs[0]; 345 res = (struct mlx5_mp_param *)mp_res->param; 346 ret = res->result; 347 free(mp_rep.msgs); 348 return ret; 349 } 350