1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2022 Microsoft Corporation 3 */ 4 5 #include <rte_malloc.h> 6 #include <ethdev_driver.h> 7 #include <rte_log.h> 8 9 #include <infiniband/verbs.h> 10 11 #include "mana.h" 12 13 extern struct mana_shared_data *mana_shared_data; 14 15 /* 16 * Process MR request from secondary process. 17 */ 18 static int 19 mana_mp_mr_create(struct mana_priv *priv, uintptr_t addr, uint32_t len) 20 { 21 struct ibv_mr *ibv_mr; 22 int ret; 23 struct mana_mr_cache *mr; 24 25 ibv_mr = ibv_reg_mr(priv->ib_pd, (void *)addr, len, 26 IBV_ACCESS_LOCAL_WRITE); 27 28 if (!ibv_mr) 29 return -errno; 30 31 DRV_LOG(DEBUG, "MR (2nd) lkey %u addr %p len %zu", 32 ibv_mr->lkey, ibv_mr->addr, ibv_mr->length); 33 34 mr = rte_calloc("MANA MR", 1, sizeof(*mr), 0); 35 if (!mr) { 36 DRV_LOG(ERR, "(2nd) Failed to allocate MR"); 37 ret = -ENOMEM; 38 goto fail_alloc; 39 } 40 mr->lkey = ibv_mr->lkey; 41 mr->addr = (uintptr_t)ibv_mr->addr; 42 mr->len = ibv_mr->length; 43 mr->verb_obj = ibv_mr; 44 45 rte_spinlock_lock(&priv->mr_btree_lock); 46 ret = mana_mr_btree_insert(&priv->mr_btree, mr); 47 rte_spinlock_unlock(&priv->mr_btree_lock); 48 if (ret) { 49 DRV_LOG(ERR, "(2nd) Failed to add to global MR btree"); 50 goto fail_btree; 51 } 52 53 return 0; 54 55 fail_btree: 56 rte_free(mr); 57 58 fail_alloc: 59 ibv_dereg_mr(ibv_mr); 60 61 return ret; 62 } 63 64 static void 65 mp_init_msg(struct rte_mp_msg *msg, enum mana_mp_req_type type, int port_id) 66 { 67 struct mana_mp_param *param; 68 69 strlcpy(msg->name, MANA_MP_NAME, sizeof(msg->name)); 70 msg->len_param = sizeof(*param); 71 72 param = (struct mana_mp_param *)msg->param; 73 param->type = type; 74 param->port_id = port_id; 75 } 76 77 static int 78 mana_mp_primary_handle(const struct rte_mp_msg *mp_msg, const void *peer) 79 { 80 struct rte_eth_dev *dev; 81 const struct mana_mp_param *param = 82 (const struct mana_mp_param *)mp_msg->param; 83 struct rte_mp_msg mp_res = { 0 }; 84 struct mana_mp_param *res = (struct mana_mp_param *)mp_res.param; 85 int ret; 86 struct mana_priv *priv; 87 88 if (!rte_eth_dev_is_valid_port(param->port_id)) { 89 DRV_LOG(ERR, "MP handle port ID %u invalid", param->port_id); 90 return -ENODEV; 91 } 92 93 dev = &rte_eth_devices[param->port_id]; 94 priv = dev->data->dev_private; 95 96 mp_init_msg(&mp_res, param->type, param->port_id); 97 98 switch (param->type) { 99 case MANA_MP_REQ_CREATE_MR: 100 ret = mana_mp_mr_create(priv, param->addr, param->len); 101 res->result = ret; 102 ret = rte_mp_reply(&mp_res, peer); 103 break; 104 105 case MANA_MP_REQ_VERBS_CMD_FD: 106 mp_res.num_fds = 1; 107 mp_res.fds[0] = priv->ib_ctx->cmd_fd; 108 res->result = 0; 109 ret = rte_mp_reply(&mp_res, peer); 110 break; 111 112 default: 113 DRV_LOG(ERR, "Port %u unknown primary MP type %u", 114 param->port_id, param->type); 115 ret = -EINVAL; 116 } 117 118 return ret; 119 } 120 121 static int 122 mana_mp_secondary_handle(const struct rte_mp_msg *mp_msg, const void *peer) 123 { 124 struct rte_mp_msg mp_res = { 0 }; 125 struct mana_mp_param *res = (struct mana_mp_param *)mp_res.param; 126 const struct mana_mp_param *param = 127 (const struct mana_mp_param *)mp_msg->param; 128 struct rte_eth_dev *dev; 129 int ret; 130 131 if (!rte_eth_dev_is_valid_port(param->port_id)) { 132 DRV_LOG(ERR, "MP handle port ID %u invalid", param->port_id); 133 return -ENODEV; 134 } 135 136 dev = &rte_eth_devices[param->port_id]; 137 138 mp_init_msg(&mp_res, param->type, param->port_id); 139 140 switch (param->type) { 141 case MANA_MP_REQ_START_RXTX: 142 DRV_LOG(INFO, "Port %u starting datapath", dev->data->port_id); 143 144 dev->tx_pkt_burst = mana_tx_burst; 145 dev->rx_pkt_burst = mana_rx_burst; 146 147 rte_mb(); 148 149 res->result = 0; 150 ret = rte_mp_reply(&mp_res, peer); 151 break; 152 153 case MANA_MP_REQ_STOP_RXTX: 154 DRV_LOG(INFO, "Port %u stopping datapath", dev->data->port_id); 155 156 dev->tx_pkt_burst = mana_tx_burst_removed; 157 dev->rx_pkt_burst = mana_rx_burst_removed; 158 159 rte_mb(); 160 161 res->result = 0; 162 ret = rte_mp_reply(&mp_res, peer); 163 break; 164 165 default: 166 DRV_LOG(ERR, "Port %u unknown secondary MP type %u", 167 param->port_id, param->type); 168 ret = -EINVAL; 169 } 170 171 return ret; 172 } 173 174 int 175 mana_mp_init_primary(void) 176 { 177 int ret; 178 179 ret = rte_mp_action_register(MANA_MP_NAME, mana_mp_primary_handle); 180 if (ret && rte_errno != ENOTSUP) { 181 DRV_LOG(ERR, "Failed to register primary handler %d %d", 182 ret, rte_errno); 183 return -1; 184 } 185 186 return 0; 187 } 188 189 void 190 mana_mp_uninit_primary(void) 191 { 192 rte_mp_action_unregister(MANA_MP_NAME); 193 } 194 195 int 196 mana_mp_init_secondary(void) 197 { 198 return rte_mp_action_register(MANA_MP_NAME, mana_mp_secondary_handle); 199 } 200 201 void 202 mana_mp_uninit_secondary(void) 203 { 204 rte_mp_action_unregister(MANA_MP_NAME); 205 } 206 207 int 208 mana_mp_req_verbs_cmd_fd(struct rte_eth_dev *dev) 209 { 210 struct rte_mp_msg mp_req = { 0 }; 211 struct rte_mp_msg *mp_res; 212 struct rte_mp_reply mp_rep; 213 struct mana_mp_param *res; 214 struct timespec ts = {.tv_sec = MANA_MP_REQ_TIMEOUT_SEC, .tv_nsec = 0}; 215 int ret; 216 217 mp_init_msg(&mp_req, MANA_MP_REQ_VERBS_CMD_FD, dev->data->port_id); 218 219 ret = rte_mp_request_sync(&mp_req, &mp_rep, &ts); 220 if (ret) { 221 DRV_LOG(ERR, "port %u request to primary process failed", 222 dev->data->port_id); 223 return ret; 224 } 225 226 if (mp_rep.nb_received != 1) { 227 DRV_LOG(ERR, "primary replied %u messages", mp_rep.nb_received); 228 ret = -EPROTO; 229 goto exit; 230 } 231 232 mp_res = &mp_rep.msgs[0]; 233 res = (struct mana_mp_param *)mp_res->param; 234 if (res->result) { 235 DRV_LOG(ERR, "failed to get CMD FD, port %u", 236 dev->data->port_id); 237 ret = res->result; 238 goto exit; 239 } 240 241 if (mp_res->num_fds != 1) { 242 DRV_LOG(ERR, "got FDs %d unexpected", mp_res->num_fds); 243 ret = -EPROTO; 244 goto exit; 245 } 246 247 ret = mp_res->fds[0]; 248 DRV_LOG(ERR, "port %u command FD from primary is %d", 249 dev->data->port_id, ret); 250 exit: 251 free(mp_rep.msgs); 252 return ret; 253 } 254 255 /* 256 * Request the primary process to register a MR. 257 */ 258 int 259 mana_mp_req_mr_create(struct mana_priv *priv, uintptr_t addr, uint32_t len) 260 { 261 struct rte_mp_msg mp_req = {0}; 262 struct rte_mp_msg *mp_res; 263 struct rte_mp_reply mp_rep; 264 struct mana_mp_param *req = (struct mana_mp_param *)mp_req.param; 265 struct mana_mp_param *res; 266 struct timespec ts = {.tv_sec = MANA_MP_REQ_TIMEOUT_SEC, .tv_nsec = 0}; 267 int ret; 268 269 mp_init_msg(&mp_req, MANA_MP_REQ_CREATE_MR, priv->port_id); 270 req->addr = addr; 271 req->len = len; 272 273 ret = rte_mp_request_sync(&mp_req, &mp_rep, &ts); 274 if (ret) { 275 DRV_LOG(ERR, "Port %u request to primary failed", 276 req->port_id); 277 return ret; 278 } 279 280 if (mp_rep.nb_received != 1) 281 return -EPROTO; 282 283 mp_res = &mp_rep.msgs[0]; 284 res = (struct mana_mp_param *)mp_res->param; 285 ret = res->result; 286 287 free(mp_rep.msgs); 288 289 return ret; 290 } 291 292 void 293 mana_mp_req_on_rxtx(struct rte_eth_dev *dev, enum mana_mp_req_type type) 294 { 295 struct rte_mp_msg mp_req = { 0 }; 296 struct rte_mp_msg *mp_res; 297 struct rte_mp_reply mp_rep; 298 struct mana_mp_param *res; 299 struct timespec ts = {.tv_sec = MANA_MP_REQ_TIMEOUT_SEC, .tv_nsec = 0}; 300 int i, ret; 301 302 if (type != MANA_MP_REQ_START_RXTX && type != MANA_MP_REQ_STOP_RXTX) { 303 DRV_LOG(ERR, "port %u unknown request (req_type %d)", 304 dev->data->port_id, type); 305 return; 306 } 307 308 if (!mana_shared_data->secondary_cnt) 309 return; 310 311 mp_init_msg(&mp_req, type, dev->data->port_id); 312 313 ret = rte_mp_request_sync(&mp_req, &mp_rep, &ts); 314 if (ret) { 315 if (rte_errno != ENOTSUP) 316 DRV_LOG(ERR, "port %u failed to request Rx/Tx (%d)", 317 dev->data->port_id, type); 318 goto exit; 319 } 320 if (mp_rep.nb_sent != mp_rep.nb_received) { 321 DRV_LOG(ERR, "port %u not all secondaries responded (%d)", 322 dev->data->port_id, type); 323 goto exit; 324 } 325 for (i = 0; i < mp_rep.nb_received; i++) { 326 mp_res = &mp_rep.msgs[i]; 327 res = (struct mana_mp_param *)mp_res->param; 328 if (res->result) { 329 DRV_LOG(ERR, "port %u request failed on secondary %d", 330 dev->data->port_id, i); 331 goto exit; 332 } 333 } 334 exit: 335 free(mp_rep.msgs); 336 } 337