1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2022 Microsoft Corporation 3 */ 4 5 #include <rte_malloc.h> 6 #include <ethdev_driver.h> 7 #include <rte_log.h> 8 #include <stdlib.h> 9 10 #include <infiniband/verbs.h> 11 12 #include "mana.h" 13 14 extern struct mana_shared_data *mana_shared_data; 15 16 /* 17 * Process MR request from secondary process. 18 */ 19 static int 20 mana_mp_mr_create(struct mana_priv *priv, uintptr_t addr, uint32_t len) 21 { 22 struct ibv_mr *ibv_mr; 23 int ret; 24 struct mana_mr_cache *mr; 25 26 ibv_mr = ibv_reg_mr(priv->ib_pd, (void *)addr, len, 27 IBV_ACCESS_LOCAL_WRITE); 28 29 if (!ibv_mr) 30 return -errno; 31 32 DP_LOG(DEBUG, "MR (2nd) lkey %u addr %p len %zu", 33 ibv_mr->lkey, ibv_mr->addr, ibv_mr->length); 34 35 mr = rte_calloc("MANA MR", 1, sizeof(*mr), 0); 36 if (!mr) { 37 DRV_LOG(ERR, "(2nd) Failed to allocate MR"); 38 ret = -ENOMEM; 39 goto fail_alloc; 40 } 41 mr->lkey = ibv_mr->lkey; 42 mr->addr = (uintptr_t)ibv_mr->addr; 43 mr->len = ibv_mr->length; 44 mr->verb_obj = ibv_mr; 45 46 rte_spinlock_lock(&priv->mr_btree_lock); 47 ret = mana_mr_btree_insert(&priv->mr_btree, mr); 48 rte_spinlock_unlock(&priv->mr_btree_lock); 49 if (ret) { 50 DRV_LOG(ERR, "(2nd) Failed to add to global MR btree"); 51 goto fail_btree; 52 } 53 54 return 0; 55 56 fail_btree: 57 rte_free(mr); 58 59 fail_alloc: 60 ibv_dereg_mr(ibv_mr); 61 62 return ret; 63 } 64 65 static void 66 mp_init_msg(struct rte_mp_msg *msg, enum mana_mp_req_type type, int port_id) 67 { 68 struct mana_mp_param *param; 69 70 strlcpy(msg->name, MANA_MP_NAME, sizeof(msg->name)); 71 msg->len_param = sizeof(*param); 72 73 param = (struct mana_mp_param *)msg->param; 74 param->type = type; 75 param->port_id = port_id; 76 } 77 78 static int 79 mana_mp_primary_handle(const struct rte_mp_msg *mp_msg, const void *peer) 80 { 81 struct rte_eth_dev *dev; 82 const struct mana_mp_param *param = 83 (const struct mana_mp_param *)mp_msg->param; 84 struct rte_mp_msg mp_res = { 0 }; 85 struct mana_mp_param *res = (struct mana_mp_param *)mp_res.param; 86 int ret; 87 struct mana_priv *priv; 88 89 if (!rte_eth_dev_is_valid_port(param->port_id)) { 90 DRV_LOG(ERR, "MP handle port ID %u invalid", param->port_id); 91 return -ENODEV; 92 } 93 94 dev = &rte_eth_devices[param->port_id]; 95 priv = dev->data->dev_private; 96 97 mp_init_msg(&mp_res, param->type, param->port_id); 98 99 switch (param->type) { 100 case MANA_MP_REQ_CREATE_MR: 101 ret = mana_mp_mr_create(priv, param->addr, param->len); 102 res->result = ret; 103 ret = rte_mp_reply(&mp_res, peer); 104 break; 105 106 case MANA_MP_REQ_VERBS_CMD_FD: 107 mp_res.num_fds = 1; 108 mp_res.fds[0] = priv->ib_ctx->cmd_fd; 109 res->result = 0; 110 ret = rte_mp_reply(&mp_res, peer); 111 break; 112 113 default: 114 DRV_LOG(ERR, "Port %u unknown primary MP type %u", 115 param->port_id, param->type); 116 ret = -EINVAL; 117 } 118 119 return ret; 120 } 121 122 static int 123 mana_mp_secondary_handle(const struct rte_mp_msg *mp_msg, const void *peer) 124 { 125 struct rte_mp_msg mp_res = { 0 }; 126 struct mana_mp_param *res = (struct mana_mp_param *)mp_res.param; 127 const struct mana_mp_param *param = 128 (const struct mana_mp_param *)mp_msg->param; 129 struct rte_eth_dev *dev; 130 int ret; 131 132 if (!rte_eth_dev_is_valid_port(param->port_id)) { 133 DRV_LOG(ERR, "MP handle port ID %u invalid", param->port_id); 134 return -ENODEV; 135 } 136 137 dev = &rte_eth_devices[param->port_id]; 138 139 mp_init_msg(&mp_res, param->type, param->port_id); 140 141 switch (param->type) { 142 case MANA_MP_REQ_START_RXTX: 143 DRV_LOG(INFO, "Port %u starting datapath", dev->data->port_id); 144 145 dev->tx_pkt_burst = mana_tx_burst; 146 dev->rx_pkt_burst = mana_rx_burst; 147 148 rte_mb(); 149 150 res->result = 0; 151 ret = rte_mp_reply(&mp_res, peer); 152 break; 153 154 case MANA_MP_REQ_STOP_RXTX: 155 DRV_LOG(INFO, "Port %u stopping datapath", dev->data->port_id); 156 157 dev->tx_pkt_burst = mana_tx_burst_removed; 158 dev->rx_pkt_burst = mana_rx_burst_removed; 159 160 rte_mb(); 161 162 res->result = 0; 163 ret = rte_mp_reply(&mp_res, peer); 164 break; 165 166 default: 167 DRV_LOG(ERR, "Port %u unknown secondary MP type %u", 168 param->port_id, param->type); 169 ret = -EINVAL; 170 } 171 172 return ret; 173 } 174 175 int 176 mana_mp_init_primary(void) 177 { 178 int ret; 179 180 ret = rte_mp_action_register(MANA_MP_NAME, mana_mp_primary_handle); 181 if (ret && rte_errno != ENOTSUP) { 182 DRV_LOG(ERR, "Failed to register primary handler %d %d", 183 ret, rte_errno); 184 return -1; 185 } 186 187 return 0; 188 } 189 190 void 191 mana_mp_uninit_primary(void) 192 { 193 rte_mp_action_unregister(MANA_MP_NAME); 194 } 195 196 int 197 mana_mp_init_secondary(void) 198 { 199 return rte_mp_action_register(MANA_MP_NAME, mana_mp_secondary_handle); 200 } 201 202 void 203 mana_mp_uninit_secondary(void) 204 { 205 rte_mp_action_unregister(MANA_MP_NAME); 206 } 207 208 int 209 mana_mp_req_verbs_cmd_fd(struct rte_eth_dev *dev) 210 { 211 struct rte_mp_msg mp_req = { 0 }; 212 struct rte_mp_msg *mp_res; 213 struct rte_mp_reply mp_rep; 214 struct mana_mp_param *res; 215 struct timespec ts = {.tv_sec = MANA_MP_REQ_TIMEOUT_SEC, .tv_nsec = 0}; 216 int ret; 217 218 mp_init_msg(&mp_req, MANA_MP_REQ_VERBS_CMD_FD, dev->data->port_id); 219 220 ret = rte_mp_request_sync(&mp_req, &mp_rep, &ts); 221 if (ret) { 222 DRV_LOG(ERR, "port %u request to primary process failed", 223 dev->data->port_id); 224 return ret; 225 } 226 227 if (mp_rep.nb_received != 1) { 228 DRV_LOG(ERR, "primary replied %u messages", mp_rep.nb_received); 229 ret = -EPROTO; 230 goto exit; 231 } 232 233 mp_res = &mp_rep.msgs[0]; 234 res = (struct mana_mp_param *)mp_res->param; 235 if (res->result) { 236 DRV_LOG(ERR, "failed to get CMD FD, port %u", 237 dev->data->port_id); 238 ret = res->result; 239 goto exit; 240 } 241 242 if (mp_res->num_fds != 1) { 243 DRV_LOG(ERR, "got FDs %d unexpected", mp_res->num_fds); 244 ret = -EPROTO; 245 goto exit; 246 } 247 248 ret = mp_res->fds[0]; 249 DRV_LOG(ERR, "port %u command FD from primary is %d", 250 dev->data->port_id, ret); 251 exit: 252 free(mp_rep.msgs); 253 return ret; 254 } 255 256 /* 257 * Request the primary process to register a MR. 258 */ 259 int 260 mana_mp_req_mr_create(struct mana_priv *priv, uintptr_t addr, uint32_t len) 261 { 262 struct rte_mp_msg mp_req = {0}; 263 struct rte_mp_msg *mp_res; 264 struct rte_mp_reply mp_rep; 265 struct mana_mp_param *req = (struct mana_mp_param *)mp_req.param; 266 struct mana_mp_param *res; 267 struct timespec ts = {.tv_sec = MANA_MP_REQ_TIMEOUT_SEC, .tv_nsec = 0}; 268 int ret; 269 270 mp_init_msg(&mp_req, MANA_MP_REQ_CREATE_MR, priv->port_id); 271 req->addr = addr; 272 req->len = len; 273 274 ret = rte_mp_request_sync(&mp_req, &mp_rep, &ts); 275 if (ret) { 276 DRV_LOG(ERR, "Port %u request to primary failed", 277 req->port_id); 278 return ret; 279 } 280 281 if (mp_rep.nb_received != 1) 282 return -EPROTO; 283 284 mp_res = &mp_rep.msgs[0]; 285 res = (struct mana_mp_param *)mp_res->param; 286 ret = res->result; 287 288 free(mp_rep.msgs); 289 290 return ret; 291 } 292 293 void 294 mana_mp_req_on_rxtx(struct rte_eth_dev *dev, enum mana_mp_req_type type) 295 { 296 struct rte_mp_msg mp_req = { 0 }; 297 struct rte_mp_msg *mp_res; 298 struct rte_mp_reply mp_rep; 299 struct mana_mp_param *res; 300 struct timespec ts = {.tv_sec = MANA_MP_REQ_TIMEOUT_SEC, .tv_nsec = 0}; 301 int i, ret; 302 303 if (type != MANA_MP_REQ_START_RXTX && type != MANA_MP_REQ_STOP_RXTX) { 304 DRV_LOG(ERR, "port %u unknown request (req_type %d)", 305 dev->data->port_id, type); 306 return; 307 } 308 309 if (!mana_shared_data->secondary_cnt) 310 return; 311 312 mp_init_msg(&mp_req, type, dev->data->port_id); 313 314 ret = rte_mp_request_sync(&mp_req, &mp_rep, &ts); 315 if (ret) { 316 if (rte_errno != ENOTSUP) 317 DRV_LOG(ERR, "port %u failed to request Rx/Tx (%d)", 318 dev->data->port_id, type); 319 goto exit; 320 } 321 if (mp_rep.nb_sent != mp_rep.nb_received) { 322 DRV_LOG(ERR, "port %u not all secondaries responded (%d)", 323 dev->data->port_id, type); 324 goto exit; 325 } 326 for (i = 0; i < mp_rep.nb_received; i++) { 327 mp_res = &mp_rep.msgs[i]; 328 res = (struct mana_mp_param *)mp_res->param; 329 if (res->result) { 330 DRV_LOG(ERR, "port %u request failed on secondary %d", 331 dev->data->port_id, i); 332 goto exit; 333 } 334 } 335 exit: 336 free(mp_rep.msgs); 337 } 338