1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright (C) 2021 Intel Corporation. All rights reserved.
3 * Copyright (c) 2020, 2021 Mellanox Technologies LTD. All rights reserved.
4 * Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
5 */
6
7 #include <rdma/rdma_cma.h>
8
9 #include "spdk/log.h"
10 #include "spdk/string.h"
11 #include "spdk/likely.h"
12
13 #include "spdk_internal/rdma_provider.h"
14
15 struct spdk_rdma_provider_srq *
spdk_rdma_provider_srq_create(struct spdk_rdma_provider_srq_init_attr * init_attr)16 spdk_rdma_provider_srq_create(struct spdk_rdma_provider_srq_init_attr *init_attr)
17 {
18 assert(init_attr);
19 assert(init_attr->pd);
20
21 struct spdk_rdma_provider_srq *rdma_srq = calloc(1, sizeof(*rdma_srq));
22
23 if (!rdma_srq) {
24 SPDK_ERRLOG("Can't allocate memory for SRQ handle\n");
25 return NULL;
26 }
27
28 if (init_attr->stats) {
29 rdma_srq->stats = init_attr->stats;
30 rdma_srq->shared_stats = true;
31 } else {
32 rdma_srq->stats = calloc(1, sizeof(*rdma_srq->stats));
33 if (!rdma_srq->stats) {
34 SPDK_ERRLOG("SRQ statistics memory allocation failed");
35 free(rdma_srq);
36 return NULL;
37 }
38 }
39
40 rdma_srq->srq = ibv_create_srq(init_attr->pd, &init_attr->srq_init_attr);
41 if (!rdma_srq->srq) {
42 if (!init_attr->stats) {
43 free(rdma_srq->stats);
44 }
45 SPDK_ERRLOG("Unable to create SRQ, errno %d (%s)\n", errno, spdk_strerror(errno));
46 free(rdma_srq);
47 return NULL;
48 }
49
50 return rdma_srq;
51 }
52
53 int
spdk_rdma_provider_srq_destroy(struct spdk_rdma_provider_srq * rdma_srq)54 spdk_rdma_provider_srq_destroy(struct spdk_rdma_provider_srq *rdma_srq)
55 {
56 int rc;
57
58 if (!rdma_srq) {
59 return 0;
60 }
61
62 assert(rdma_srq->srq);
63
64 if (rdma_srq->recv_wrs.first != NULL) {
65 SPDK_WARNLOG("Destroying RDMA SRQ with queued recv WRs\n");
66 }
67
68 rc = ibv_destroy_srq(rdma_srq->srq);
69 if (rc) {
70 SPDK_ERRLOG("SRQ destroy failed with %d\n", rc);
71 }
72
73 if (!rdma_srq->shared_stats) {
74 free(rdma_srq->stats);
75 }
76
77 free(rdma_srq);
78
79 return rc;
80 }
81
82 static inline bool
rdma_queue_recv_wrs(struct spdk_rdma_provider_recv_wr_list * recv_wrs,struct ibv_recv_wr * first,struct spdk_rdma_provider_wr_stats * recv_stats)83 rdma_queue_recv_wrs(struct spdk_rdma_provider_recv_wr_list *recv_wrs, struct ibv_recv_wr *first,
84 struct spdk_rdma_provider_wr_stats *recv_stats)
85 {
86 struct ibv_recv_wr *last;
87
88 recv_stats->num_submitted_wrs++;
89 last = first;
90 while (last->next != NULL) {
91 last = last->next;
92 recv_stats->num_submitted_wrs++;
93 }
94
95 if (recv_wrs->first == NULL) {
96 recv_wrs->first = first;
97 recv_wrs->last = last;
98 return true;
99 } else {
100 recv_wrs->last->next = first;
101 recv_wrs->last = last;
102 return false;
103 }
104 }
105
106 bool
spdk_rdma_provider_srq_queue_recv_wrs(struct spdk_rdma_provider_srq * rdma_srq,struct ibv_recv_wr * first)107 spdk_rdma_provider_srq_queue_recv_wrs(struct spdk_rdma_provider_srq *rdma_srq,
108 struct ibv_recv_wr *first)
109 {
110 assert(rdma_srq);
111 assert(first);
112
113 return rdma_queue_recv_wrs(&rdma_srq->recv_wrs, first, rdma_srq->stats);
114 }
115
116 int
spdk_rdma_provider_srq_flush_recv_wrs(struct spdk_rdma_provider_srq * rdma_srq,struct ibv_recv_wr ** bad_wr)117 spdk_rdma_provider_srq_flush_recv_wrs(struct spdk_rdma_provider_srq *rdma_srq,
118 struct ibv_recv_wr **bad_wr)
119 {
120 int rc;
121
122 if (spdk_unlikely(rdma_srq->recv_wrs.first == NULL)) {
123 return 0;
124 }
125
126 rc = ibv_post_srq_recv(rdma_srq->srq, rdma_srq->recv_wrs.first, bad_wr);
127
128 rdma_srq->recv_wrs.first = NULL;
129 rdma_srq->stats->doorbell_updates++;
130
131 return rc;
132 }
133
134 bool
spdk_rdma_provider_qp_queue_recv_wrs(struct spdk_rdma_provider_qp * spdk_rdma_qp,struct ibv_recv_wr * first)135 spdk_rdma_provider_qp_queue_recv_wrs(struct spdk_rdma_provider_qp *spdk_rdma_qp,
136 struct ibv_recv_wr *first)
137 {
138 assert(spdk_rdma_qp);
139 assert(first);
140
141 return rdma_queue_recv_wrs(&spdk_rdma_qp->recv_wrs, first, &spdk_rdma_qp->stats->recv);
142 }
143
144 int
spdk_rdma_provider_qp_flush_recv_wrs(struct spdk_rdma_provider_qp * spdk_rdma_qp,struct ibv_recv_wr ** bad_wr)145 spdk_rdma_provider_qp_flush_recv_wrs(struct spdk_rdma_provider_qp *spdk_rdma_qp,
146 struct ibv_recv_wr **bad_wr)
147 {
148 int rc;
149
150 if (spdk_unlikely(spdk_rdma_qp->recv_wrs.first == NULL)) {
151 return 0;
152 }
153
154 rc = ibv_post_recv(spdk_rdma_qp->qp, spdk_rdma_qp->recv_wrs.first, bad_wr);
155
156 spdk_rdma_qp->recv_wrs.first = NULL;
157 spdk_rdma_qp->stats->recv.doorbell_updates++;
158
159 return rc;
160 }
161