1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2020 Intel Corporation. All rights reserved. 3 * Copyright (c) 2020, 2021 Mellanox Technologies LTD. All rights reserved. 4 * Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 5 */ 6 7 #include <rdma/rdma_cma.h> 8 #include <infiniband/mlx5dv.h> 9 10 #include "spdk/stdinc.h" 11 #include "spdk/string.h" 12 #include "spdk/likely.h" 13 14 #include "spdk_internal/rdma_provider.h" 15 #include "spdk/log.h" 16 #include "spdk/util.h" 17 18 struct spdk_rdma_mlx5_dv_qp { 19 struct spdk_rdma_provider_qp common; 20 struct ibv_qp_ex *qpex; 21 }; 22 23 static int 24 rdma_mlx5_dv_init_qpair(struct spdk_rdma_mlx5_dv_qp *mlx5_qp) 25 { 26 struct ibv_qp_attr qp_attr; 27 int qp_attr_mask, rc; 28 29 qp_attr.qp_state = IBV_QPS_INIT; 30 rc = rdma_init_qp_attr(mlx5_qp->common.cm_id, &qp_attr, &qp_attr_mask); 31 if (rc) { 32 SPDK_ERRLOG("Failed to init attr IBV_QPS_INIT, errno %s (%d)\n", spdk_strerror(errno), errno); 33 return rc; 34 } 35 36 rc = ibv_modify_qp(mlx5_qp->common.qp, &qp_attr, qp_attr_mask); 37 if (rc) { 38 SPDK_ERRLOG("ibv_modify_qp(IBV_QPS_INIT) failed, rc %d\n", rc); 39 return rc; 40 } 41 42 qp_attr.qp_state = IBV_QPS_RTR; 43 rc = rdma_init_qp_attr(mlx5_qp->common.cm_id, &qp_attr, &qp_attr_mask); 44 if (rc) { 45 SPDK_ERRLOG("Failed to init attr IBV_QPS_RTR, errno %s (%d)\n", spdk_strerror(errno), errno); 46 return rc; 47 } 48 49 rc = ibv_modify_qp(mlx5_qp->common.qp, &qp_attr, qp_attr_mask); 50 if (rc) { 51 SPDK_ERRLOG("ibv_modify_qp(IBV_QPS_RTR) failed, rc %d\n", rc); 52 return rc; 53 } 54 55 qp_attr.qp_state = IBV_QPS_RTS; 56 rc = rdma_init_qp_attr(mlx5_qp->common.cm_id, &qp_attr, &qp_attr_mask); 57 if (rc) { 58 SPDK_ERRLOG("Failed to init attr IBV_QPS_RTR, errno %s (%d)\n", spdk_strerror(errno), errno); 59 return rc; 60 } 61 62 rc = ibv_modify_qp(mlx5_qp->common.qp, &qp_attr, qp_attr_mask); 63 if (rc) { 64 SPDK_ERRLOG("ibv_modify_qp(IBV_QPS_RTS) failed, rc %d\n", rc); 65 } 66 67 return rc; 68 } 69 70 struct spdk_rdma_provider_qp * 71 spdk_rdma_provider_qp_create(struct rdma_cm_id *cm_id, 72 struct spdk_rdma_provider_qp_init_attr *qp_attr) 73 { 74 assert(cm_id); 75 assert(qp_attr); 76 77 struct ibv_qp *qp; 78 struct spdk_rdma_mlx5_dv_qp *mlx5_qp; 79 struct ibv_qp_init_attr_ex dv_qp_attr = { 80 .qp_context = qp_attr->qp_context, 81 .send_cq = qp_attr->send_cq, 82 .recv_cq = qp_attr->recv_cq, 83 .srq = qp_attr->srq, 84 .cap = qp_attr->cap, 85 .qp_type = IBV_QPT_RC, 86 .comp_mask = IBV_QP_INIT_ATTR_PD | IBV_QP_INIT_ATTR_SEND_OPS_FLAGS, 87 .pd = qp_attr->pd ? qp_attr->pd : cm_id->pd 88 }; 89 90 assert(dv_qp_attr.pd); 91 92 mlx5_qp = calloc(1, sizeof(*mlx5_qp)); 93 if (!mlx5_qp) { 94 SPDK_ERRLOG("qp memory allocation failed\n"); 95 return NULL; 96 } 97 98 if (qp_attr->stats) { 99 mlx5_qp->common.stats = qp_attr->stats; 100 mlx5_qp->common.shared_stats = true; 101 } else { 102 mlx5_qp->common.stats = calloc(1, sizeof(*mlx5_qp->common.stats)); 103 if (!mlx5_qp->common.stats) { 104 SPDK_ERRLOG("qp statistics memory allocation failed\n"); 105 free(mlx5_qp); 106 return NULL; 107 } 108 } 109 110 qp = mlx5dv_create_qp(cm_id->verbs, &dv_qp_attr, NULL); 111 112 if (!qp) { 113 SPDK_ERRLOG("Failed to create qpair, errno %s (%d)\n", spdk_strerror(errno), errno); 114 free(mlx5_qp); 115 return NULL; 116 } 117 118 mlx5_qp->common.qp = qp; 119 mlx5_qp->common.cm_id = cm_id; 120 mlx5_qp->qpex = ibv_qp_to_qp_ex(qp); 121 122 if (!mlx5_qp->qpex) { 123 spdk_rdma_provider_qp_destroy(&mlx5_qp->common); 124 return NULL; 125 } 126 127 qp_attr->cap = dv_qp_attr.cap; 128 129 return &mlx5_qp->common; 130 } 131 132 int 133 spdk_rdma_provider_qp_accept(struct spdk_rdma_provider_qp *spdk_rdma_qp, 134 struct rdma_conn_param *conn_param) 135 { 136 struct spdk_rdma_mlx5_dv_qp *mlx5_qp; 137 138 assert(spdk_rdma_qp != NULL); 139 assert(spdk_rdma_qp->cm_id != NULL); 140 141 mlx5_qp = SPDK_CONTAINEROF(spdk_rdma_qp, struct spdk_rdma_mlx5_dv_qp, common); 142 143 /* NVMEoF target must move qpair to RTS state */ 144 if (rdma_mlx5_dv_init_qpair(mlx5_qp) != 0) { 145 SPDK_ERRLOG("Failed to initialize qpair\n"); 146 /* Set errno to be compliant with rdma_accept behaviour */ 147 errno = ECONNABORTED; 148 return -1; 149 } 150 151 return rdma_accept(spdk_rdma_qp->cm_id, conn_param); 152 } 153 154 int 155 spdk_rdma_provider_qp_complete_connect(struct spdk_rdma_provider_qp *spdk_rdma_qp) 156 { 157 struct spdk_rdma_mlx5_dv_qp *mlx5_qp; 158 int rc; 159 160 assert(spdk_rdma_qp); 161 162 mlx5_qp = SPDK_CONTAINEROF(spdk_rdma_qp, struct spdk_rdma_mlx5_dv_qp, common); 163 164 rc = rdma_mlx5_dv_init_qpair(mlx5_qp); 165 if (rc) { 166 SPDK_ERRLOG("Failed to initialize qpair\n"); 167 return rc; 168 } 169 170 rc = rdma_establish(mlx5_qp->common.cm_id); 171 if (rc) { 172 SPDK_ERRLOG("rdma_establish failed, errno %s (%d)\n", spdk_strerror(errno), errno); 173 } 174 175 return rc; 176 } 177 178 void 179 spdk_rdma_provider_qp_destroy(struct spdk_rdma_provider_qp *spdk_rdma_qp) 180 { 181 struct spdk_rdma_mlx5_dv_qp *mlx5_qp; 182 int rc; 183 184 assert(spdk_rdma_qp != NULL); 185 186 mlx5_qp = SPDK_CONTAINEROF(spdk_rdma_qp, struct spdk_rdma_mlx5_dv_qp, common); 187 188 if (spdk_rdma_qp->send_wrs.first != NULL) { 189 SPDK_WARNLOG("Destroying qpair with queued Work Requests\n"); 190 } 191 192 if (!mlx5_qp->common.shared_stats) { 193 free(mlx5_qp->common.stats); 194 } 195 196 if (mlx5_qp->common.qp) { 197 rc = ibv_destroy_qp(mlx5_qp->common.qp); 198 if (rc) { 199 SPDK_ERRLOG("Failed to destroy ibv qp %p, rc %d\n", mlx5_qp->common.qp, rc); 200 } 201 } 202 203 free(mlx5_qp); 204 } 205 206 int 207 spdk_rdma_provider_qp_disconnect(struct spdk_rdma_provider_qp *spdk_rdma_qp) 208 { 209 int rc = 0; 210 211 assert(spdk_rdma_qp != NULL); 212 213 if (spdk_rdma_qp->qp) { 214 struct ibv_qp_attr qp_attr = {.qp_state = IBV_QPS_ERR}; 215 216 rc = ibv_modify_qp(spdk_rdma_qp->qp, &qp_attr, IBV_QP_STATE); 217 if (rc) { 218 SPDK_ERRLOG("Failed to modify ibv qp %p state to ERR, rc %d\n", spdk_rdma_qp->qp, rc); 219 return rc; 220 } 221 } 222 223 if (spdk_rdma_qp->cm_id) { 224 rc = rdma_disconnect(spdk_rdma_qp->cm_id); 225 if (rc) { 226 SPDK_ERRLOG("rdma_disconnect failed, errno %s (%d)\n", spdk_strerror(errno), errno); 227 } 228 } 229 230 return rc; 231 } 232 233 bool 234 spdk_rdma_provider_qp_queue_send_wrs(struct spdk_rdma_provider_qp *spdk_rdma_qp, 235 struct ibv_send_wr *first) 236 { 237 struct ibv_send_wr *tmp; 238 struct spdk_rdma_mlx5_dv_qp *mlx5_qp; 239 bool is_first; 240 241 assert(spdk_rdma_qp); 242 assert(first); 243 244 is_first = spdk_rdma_qp->send_wrs.first == NULL; 245 mlx5_qp = SPDK_CONTAINEROF(spdk_rdma_qp, struct spdk_rdma_mlx5_dv_qp, common); 246 247 if (is_first) { 248 ibv_wr_start(mlx5_qp->qpex); 249 spdk_rdma_qp->send_wrs.first = first; 250 } else { 251 spdk_rdma_qp->send_wrs.last->next = first; 252 } 253 254 for (tmp = first; tmp != NULL; tmp = tmp->next) { 255 mlx5_qp->qpex->wr_id = tmp->wr_id; 256 mlx5_qp->qpex->wr_flags = tmp->send_flags; 257 258 switch (tmp->opcode) { 259 case IBV_WR_SEND: 260 ibv_wr_send(mlx5_qp->qpex); 261 break; 262 case IBV_WR_SEND_WITH_INV: 263 ibv_wr_send_inv(mlx5_qp->qpex, tmp->invalidate_rkey); 264 break; 265 case IBV_WR_RDMA_READ: 266 ibv_wr_rdma_read(mlx5_qp->qpex, tmp->wr.rdma.rkey, tmp->wr.rdma.remote_addr); 267 break; 268 case IBV_WR_RDMA_WRITE: 269 ibv_wr_rdma_write(mlx5_qp->qpex, tmp->wr.rdma.rkey, tmp->wr.rdma.remote_addr); 270 break; 271 default: 272 SPDK_ERRLOG("Unexpected opcode %d\n", tmp->opcode); 273 assert(0); 274 } 275 276 ibv_wr_set_sge_list(mlx5_qp->qpex, tmp->num_sge, tmp->sg_list); 277 278 spdk_rdma_qp->send_wrs.last = tmp; 279 spdk_rdma_qp->stats->send.num_submitted_wrs++; 280 } 281 282 return is_first; 283 } 284 285 int 286 spdk_rdma_provider_qp_flush_send_wrs(struct spdk_rdma_provider_qp *spdk_rdma_qp, 287 struct ibv_send_wr **bad_wr) 288 { 289 struct spdk_rdma_mlx5_dv_qp *mlx5_qp; 290 int rc; 291 292 assert(bad_wr); 293 assert(spdk_rdma_qp); 294 295 mlx5_qp = SPDK_CONTAINEROF(spdk_rdma_qp, struct spdk_rdma_mlx5_dv_qp, common); 296 297 if (spdk_unlikely(spdk_rdma_qp->send_wrs.first == NULL)) { 298 return 0; 299 } 300 301 rc = ibv_wr_complete(mlx5_qp->qpex); 302 303 if (spdk_unlikely(rc)) { 304 /* If ibv_wr_complete reports an error that means that no WRs are posted to NIC */ 305 *bad_wr = spdk_rdma_qp->send_wrs.first; 306 } 307 308 spdk_rdma_qp->send_wrs.first = NULL; 309 spdk_rdma_qp->stats->send.doorbell_updates++; 310 311 return rc; 312 } 313