1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2021 Intel Corporation. All rights reserved. 3 * Copyright (c) 2020, 2021 Mellanox Technologies LTD. All rights reserved. 4 * Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 5 */ 6 7 #include <rdma/rdma_cma.h> 8 9 #include "spdk/log.h" 10 #include "spdk/string.h" 11 #include "spdk/likely.h" 12 13 #include "spdk_internal/rdma_provider.h" 14 15 struct spdk_rdma_provider_srq * 16 spdk_rdma_provider_srq_create(struct spdk_rdma_provider_srq_init_attr *init_attr) 17 { 18 assert(init_attr); 19 assert(init_attr->pd); 20 21 struct spdk_rdma_provider_srq *rdma_srq = calloc(1, sizeof(*rdma_srq)); 22 23 if (!rdma_srq) { 24 SPDK_ERRLOG("Can't allocate memory for SRQ handle\n"); 25 return NULL; 26 } 27 28 if (init_attr->stats) { 29 rdma_srq->stats = init_attr->stats; 30 rdma_srq->shared_stats = true; 31 } else { 32 rdma_srq->stats = calloc(1, sizeof(*rdma_srq->stats)); 33 if (!rdma_srq->stats) { 34 SPDK_ERRLOG("SRQ statistics memory allocation failed"); 35 free(rdma_srq); 36 return NULL; 37 } 38 } 39 40 rdma_srq->srq = ibv_create_srq(init_attr->pd, &init_attr->srq_init_attr); 41 if (!rdma_srq->srq) { 42 if (!init_attr->stats) { 43 free(rdma_srq->stats); 44 } 45 SPDK_ERRLOG("Unable to create SRQ, errno %d (%s)\n", errno, spdk_strerror(errno)); 46 free(rdma_srq); 47 return NULL; 48 } 49 50 return rdma_srq; 51 } 52 53 int 54 spdk_rdma_provider_srq_destroy(struct spdk_rdma_provider_srq *rdma_srq) 55 { 56 int rc; 57 58 if (!rdma_srq) { 59 return 0; 60 } 61 62 assert(rdma_srq->srq); 63 64 if (rdma_srq->recv_wrs.first != NULL) { 65 SPDK_WARNLOG("Destroying RDMA SRQ with queued recv WRs\n"); 66 } 67 68 rc = ibv_destroy_srq(rdma_srq->srq); 69 if (rc) { 70 SPDK_ERRLOG("SRQ destroy failed with %d\n", rc); 71 } 72 73 if (!rdma_srq->shared_stats) { 74 free(rdma_srq->stats); 75 } 76 77 free(rdma_srq); 78 79 return rc; 80 } 81 82 static inline bool 83 rdma_queue_recv_wrs(struct spdk_rdma_provider_recv_wr_list *recv_wrs, struct ibv_recv_wr *first, 84 struct spdk_rdma_provider_wr_stats *recv_stats) 85 { 86 struct ibv_recv_wr *last; 87 88 recv_stats->num_submitted_wrs++; 89 last = first; 90 while (last->next != NULL) { 91 last = last->next; 92 recv_stats->num_submitted_wrs++; 93 } 94 95 if (recv_wrs->first == NULL) { 96 recv_wrs->first = first; 97 recv_wrs->last = last; 98 return true; 99 } else { 100 recv_wrs->last->next = first; 101 recv_wrs->last = last; 102 return false; 103 } 104 } 105 106 bool 107 spdk_rdma_provider_srq_queue_recv_wrs(struct spdk_rdma_provider_srq *rdma_srq, 108 struct ibv_recv_wr *first) 109 { 110 assert(rdma_srq); 111 assert(first); 112 113 return rdma_queue_recv_wrs(&rdma_srq->recv_wrs, first, rdma_srq->stats); 114 } 115 116 int 117 spdk_rdma_provider_srq_flush_recv_wrs(struct spdk_rdma_provider_srq *rdma_srq, 118 struct ibv_recv_wr **bad_wr) 119 { 120 int rc; 121 122 if (spdk_unlikely(rdma_srq->recv_wrs.first == NULL)) { 123 return 0; 124 } 125 126 rc = ibv_post_srq_recv(rdma_srq->srq, rdma_srq->recv_wrs.first, bad_wr); 127 128 rdma_srq->recv_wrs.first = NULL; 129 rdma_srq->stats->doorbell_updates++; 130 131 return rc; 132 } 133 134 bool 135 spdk_rdma_provider_qp_queue_recv_wrs(struct spdk_rdma_provider_qp *spdk_rdma_qp, 136 struct ibv_recv_wr *first) 137 { 138 assert(spdk_rdma_qp); 139 assert(first); 140 141 return rdma_queue_recv_wrs(&spdk_rdma_qp->recv_wrs, first, &spdk_rdma_qp->stats->recv); 142 } 143 144 int 145 spdk_rdma_provider_qp_flush_recv_wrs(struct spdk_rdma_provider_qp *spdk_rdma_qp, 146 struct ibv_recv_wr **bad_wr) 147 { 148 int rc; 149 150 if (spdk_unlikely(spdk_rdma_qp->recv_wrs.first == NULL)) { 151 return 0; 152 } 153 154 rc = ibv_post_recv(spdk_rdma_qp->qp, spdk_rdma_qp->recv_wrs.first, bad_wr); 155 156 spdk_rdma_qp->recv_wrs.first = NULL; 157 spdk_rdma_qp->stats->recv.doorbell_updates++; 158 159 return rc; 160 } 161