1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. All rights reserved. 5 * Copyright (c) 2019 Mellanox Technologies LTD. All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "spdk/stdinc.h" 35 36 #include <infiniband/verbs.h> 37 #include <rdma/rdma_cma.h> 38 #include <rdma/rdma_verbs.h> 39 40 #include "nvmf_internal.h" 41 #include "transport.h" 42 43 #include "spdk/config.h" 44 #include "spdk/thread.h" 45 #include "spdk/nvmf.h" 46 #include "spdk/nvmf_spec.h" 47 #include "spdk/string.h" 48 #include "spdk/trace.h" 49 #include "spdk/util.h" 50 51 #include "spdk_internal/assert.h" 52 #include "spdk_internal/log.h" 53 54 struct spdk_nvme_rdma_hooks g_nvmf_hooks = {}; 55 56 /* 57 RDMA Connection Resource Defaults 58 */ 59 #define NVMF_DEFAULT_TX_SGE SPDK_NVMF_MAX_SGL_ENTRIES 60 #define NVMF_DEFAULT_RSP_SGE 1 61 #define NVMF_DEFAULT_RX_SGE 2 62 63 /* The RDMA completion queue size */ 64 #define DEFAULT_NVMF_RDMA_CQ_SIZE 4096 65 #define MAX_WR_PER_QP(queue_depth) (queue_depth * 3 + 2) 66 67 /* Timeout for destroying defunct rqpairs */ 68 #define NVMF_RDMA_QPAIR_DESTROY_TIMEOUT_US 4000000 69 70 /* The maximum number of buffers per request */ 71 #define NVMF_REQ_MAX_BUFFERS (SPDK_NVMF_MAX_SGL_ENTRIES * 2) 72 73 static int g_spdk_nvmf_ibv_query_mask = 74 IBV_QP_STATE | 75 IBV_QP_PKEY_INDEX | 76 IBV_QP_PORT | 77 IBV_QP_ACCESS_FLAGS | 78 IBV_QP_AV | 79 IBV_QP_PATH_MTU | 80 IBV_QP_DEST_QPN | 81 IBV_QP_RQ_PSN | 82 IBV_QP_MAX_DEST_RD_ATOMIC | 83 IBV_QP_MIN_RNR_TIMER | 84 IBV_QP_SQ_PSN | 85 IBV_QP_TIMEOUT | 86 IBV_QP_RETRY_CNT | 87 IBV_QP_RNR_RETRY | 88 IBV_QP_MAX_QP_RD_ATOMIC; 89 90 enum spdk_nvmf_rdma_request_state { 91 /* The request is not currently in use */ 92 RDMA_REQUEST_STATE_FREE = 0, 93 94 /* Initial state when request first received */ 95 RDMA_REQUEST_STATE_NEW, 96 97 /* The request is queued until a data buffer is available. */ 98 RDMA_REQUEST_STATE_NEED_BUFFER, 99 100 /* The request is waiting on RDMA queue depth availability 101 * to transfer data from the host to the controller. 102 */ 103 RDMA_REQUEST_STATE_DATA_TRANSFER_TO_CONTROLLER_PENDING, 104 105 /* The request is currently transferring data from the host to the controller. */ 106 RDMA_REQUEST_STATE_TRANSFERRING_HOST_TO_CONTROLLER, 107 108 /* The request is ready to execute at the block device */ 109 RDMA_REQUEST_STATE_READY_TO_EXECUTE, 110 111 /* The request is currently executing at the block device */ 112 RDMA_REQUEST_STATE_EXECUTING, 113 114 /* The request finished executing at the block device */ 115 RDMA_REQUEST_STATE_EXECUTED, 116 117 /* The request is waiting on RDMA queue depth availability 118 * to transfer data from the controller to the host. 119 */ 120 RDMA_REQUEST_STATE_DATA_TRANSFER_TO_HOST_PENDING, 121 122 /* The request is ready to send a completion */ 123 RDMA_REQUEST_STATE_READY_TO_COMPLETE, 124 125 /* The request is currently transferring data from the controller to the host. */ 126 RDMA_REQUEST_STATE_TRANSFERRING_CONTROLLER_TO_HOST, 127 128 /* The request currently has an outstanding completion without an 129 * associated data transfer. 130 */ 131 RDMA_REQUEST_STATE_COMPLETING, 132 133 /* The request completed and can be marked free. */ 134 RDMA_REQUEST_STATE_COMPLETED, 135 136 /* Terminator */ 137 RDMA_REQUEST_NUM_STATES, 138 }; 139 140 #define OBJECT_NVMF_RDMA_IO 0x40 141 142 #define TRACE_GROUP_NVMF_RDMA 0x4 143 #define TRACE_RDMA_REQUEST_STATE_NEW SPDK_TPOINT_ID(TRACE_GROUP_NVMF_RDMA, 0x0) 144 #define TRACE_RDMA_REQUEST_STATE_NEED_BUFFER SPDK_TPOINT_ID(TRACE_GROUP_NVMF_RDMA, 0x1) 145 #define TRACE_RDMA_REQUEST_STATE_DATA_TRANSFER_TO_CONTROLLER_PENDING SPDK_TPOINT_ID(TRACE_GROUP_NVMF_RDMA, 0x2) 146 #define TRACE_RDMA_REQUEST_STATE_TRANSFERRING_HOST_TO_CONTROLLER SPDK_TPOINT_ID(TRACE_GROUP_NVMF_RDMA, 0x3) 147 #define TRACE_RDMA_REQUEST_STATE_READY_TO_EXECUTE SPDK_TPOINT_ID(TRACE_GROUP_NVMF_RDMA, 0x4) 148 #define TRACE_RDMA_REQUEST_STATE_EXECUTING SPDK_TPOINT_ID(TRACE_GROUP_NVMF_RDMA, 0x5) 149 #define TRACE_RDMA_REQUEST_STATE_EXECUTED SPDK_TPOINT_ID(TRACE_GROUP_NVMF_RDMA, 0x6) 150 #define TRACE_RDMA_REQUEST_STATE_DATA_TRANSFER_TO_HOST_PENDING SPDK_TPOINT_ID(TRACE_GROUP_NVMF_RDMA, 0x7) 151 #define TRACE_RDMA_REQUEST_STATE_READY_TO_COMPLETE SPDK_TPOINT_ID(TRACE_GROUP_NVMF_RDMA, 0x8) 152 #define TRACE_RDMA_REQUEST_STATE_TRANSFERRING_CONTROLLER_TO_HOST SPDK_TPOINT_ID(TRACE_GROUP_NVMF_RDMA, 0x9) 153 #define TRACE_RDMA_REQUEST_STATE_COMPLETING SPDK_TPOINT_ID(TRACE_GROUP_NVMF_RDMA, 0xA) 154 #define TRACE_RDMA_REQUEST_STATE_COMPLETED SPDK_TPOINT_ID(TRACE_GROUP_NVMF_RDMA, 0xB) 155 #define TRACE_RDMA_QP_CREATE SPDK_TPOINT_ID(TRACE_GROUP_NVMF_RDMA, 0xC) 156 #define TRACE_RDMA_IBV_ASYNC_EVENT SPDK_TPOINT_ID(TRACE_GROUP_NVMF_RDMA, 0xD) 157 #define TRACE_RDMA_CM_ASYNC_EVENT SPDK_TPOINT_ID(TRACE_GROUP_NVMF_RDMA, 0xE) 158 #define TRACE_RDMA_QP_STATE_CHANGE SPDK_TPOINT_ID(TRACE_GROUP_NVMF_RDMA, 0xF) 159 #define TRACE_RDMA_QP_DISCONNECT SPDK_TPOINT_ID(TRACE_GROUP_NVMF_RDMA, 0x10) 160 #define TRACE_RDMA_QP_DESTROY SPDK_TPOINT_ID(TRACE_GROUP_NVMF_RDMA, 0x11) 161 162 SPDK_TRACE_REGISTER_FN(nvmf_trace, "nvmf_rdma", TRACE_GROUP_NVMF_RDMA) 163 { 164 spdk_trace_register_object(OBJECT_NVMF_RDMA_IO, 'r'); 165 spdk_trace_register_description("RDMA_REQ_NEW", TRACE_RDMA_REQUEST_STATE_NEW, 166 OWNER_NONE, OBJECT_NVMF_RDMA_IO, 1, 1, "cmid: "); 167 spdk_trace_register_description("RDMA_REQ_NEED_BUFFER", TRACE_RDMA_REQUEST_STATE_NEED_BUFFER, 168 OWNER_NONE, OBJECT_NVMF_RDMA_IO, 0, 1, "cmid: "); 169 spdk_trace_register_description("RDMA_REQ_TX_PENDING_C2H", 170 TRACE_RDMA_REQUEST_STATE_DATA_TRANSFER_TO_HOST_PENDING, 171 OWNER_NONE, OBJECT_NVMF_RDMA_IO, 0, 1, "cmid: "); 172 spdk_trace_register_description("RDMA_REQ_TX_PENDING_H2C", 173 TRACE_RDMA_REQUEST_STATE_DATA_TRANSFER_TO_CONTROLLER_PENDING, 174 OWNER_NONE, OBJECT_NVMF_RDMA_IO, 0, 1, "cmid: "); 175 spdk_trace_register_description("RDMA_REQ_TX_H2C", 176 TRACE_RDMA_REQUEST_STATE_TRANSFERRING_HOST_TO_CONTROLLER, 177 OWNER_NONE, OBJECT_NVMF_RDMA_IO, 0, 1, "cmid: "); 178 spdk_trace_register_description("RDMA_REQ_RDY_TO_EXECUTE", 179 TRACE_RDMA_REQUEST_STATE_READY_TO_EXECUTE, 180 OWNER_NONE, OBJECT_NVMF_RDMA_IO, 0, 1, "cmid: "); 181 spdk_trace_register_description("RDMA_REQ_EXECUTING", 182 TRACE_RDMA_REQUEST_STATE_EXECUTING, 183 OWNER_NONE, OBJECT_NVMF_RDMA_IO, 0, 1, "cmid: "); 184 spdk_trace_register_description("RDMA_REQ_EXECUTED", 185 TRACE_RDMA_REQUEST_STATE_EXECUTED, 186 OWNER_NONE, OBJECT_NVMF_RDMA_IO, 0, 1, "cmid: "); 187 spdk_trace_register_description("RDMA_REQ_RDY_TO_COMPL", 188 TRACE_RDMA_REQUEST_STATE_READY_TO_COMPLETE, 189 OWNER_NONE, OBJECT_NVMF_RDMA_IO, 0, 1, "cmid: "); 190 spdk_trace_register_description("RDMA_REQ_COMPLETING_C2H", 191 TRACE_RDMA_REQUEST_STATE_TRANSFERRING_CONTROLLER_TO_HOST, 192 OWNER_NONE, OBJECT_NVMF_RDMA_IO, 0, 1, "cmid: "); 193 spdk_trace_register_description("RDMA_REQ_COMPLETING", 194 TRACE_RDMA_REQUEST_STATE_COMPLETING, 195 OWNER_NONE, OBJECT_NVMF_RDMA_IO, 0, 1, "cmid: "); 196 spdk_trace_register_description("RDMA_REQ_COMPLETED", 197 TRACE_RDMA_REQUEST_STATE_COMPLETED, 198 OWNER_NONE, OBJECT_NVMF_RDMA_IO, 0, 1, "cmid: "); 199 200 spdk_trace_register_description("RDMA_QP_CREATE", TRACE_RDMA_QP_CREATE, 201 OWNER_NONE, OBJECT_NONE, 0, 0, ""); 202 spdk_trace_register_description("RDMA_IBV_ASYNC_EVENT", TRACE_RDMA_IBV_ASYNC_EVENT, 203 OWNER_NONE, OBJECT_NONE, 0, 0, "type: "); 204 spdk_trace_register_description("RDMA_CM_ASYNC_EVENT", TRACE_RDMA_CM_ASYNC_EVENT, 205 OWNER_NONE, OBJECT_NONE, 0, 0, "type: "); 206 spdk_trace_register_description("RDMA_QP_STATE_CHANGE", TRACE_RDMA_QP_STATE_CHANGE, 207 OWNER_NONE, OBJECT_NONE, 0, 1, "state: "); 208 spdk_trace_register_description("RDMA_QP_DISCONNECT", TRACE_RDMA_QP_DISCONNECT, 209 OWNER_NONE, OBJECT_NONE, 0, 0, ""); 210 spdk_trace_register_description("RDMA_QP_DESTROY", TRACE_RDMA_QP_DESTROY, 211 OWNER_NONE, OBJECT_NONE, 0, 0, ""); 212 } 213 214 enum spdk_nvmf_rdma_wr_type { 215 RDMA_WR_TYPE_RECV, 216 RDMA_WR_TYPE_SEND, 217 RDMA_WR_TYPE_DATA, 218 }; 219 220 struct spdk_nvmf_rdma_wr { 221 enum spdk_nvmf_rdma_wr_type type; 222 }; 223 224 /* This structure holds commands as they are received off the wire. 225 * It must be dynamically paired with a full request object 226 * (spdk_nvmf_rdma_request) to service a request. It is separate 227 * from the request because RDMA does not appear to order 228 * completions, so occasionally we'll get a new incoming 229 * command when there aren't any free request objects. 230 */ 231 struct spdk_nvmf_rdma_recv { 232 struct ibv_recv_wr wr; 233 struct ibv_sge sgl[NVMF_DEFAULT_RX_SGE]; 234 235 struct spdk_nvmf_rdma_qpair *qpair; 236 237 /* In-capsule data buffer */ 238 uint8_t *buf; 239 240 struct spdk_nvmf_rdma_wr rdma_wr; 241 uint64_t receive_tsc; 242 243 STAILQ_ENTRY(spdk_nvmf_rdma_recv) link; 244 }; 245 246 struct spdk_nvmf_rdma_request_data { 247 struct spdk_nvmf_rdma_wr rdma_wr; 248 struct ibv_send_wr wr; 249 struct ibv_sge sgl[SPDK_NVMF_MAX_SGL_ENTRIES]; 250 }; 251 252 struct spdk_nvmf_rdma_request { 253 struct spdk_nvmf_request req; 254 255 enum spdk_nvmf_rdma_request_state state; 256 257 struct spdk_nvmf_rdma_recv *recv; 258 259 struct { 260 struct spdk_nvmf_rdma_wr rdma_wr; 261 struct ibv_send_wr wr; 262 struct ibv_sge sgl[NVMF_DEFAULT_RSP_SGE]; 263 } rsp; 264 265 struct spdk_nvmf_rdma_request_data data; 266 267 uint32_t iovpos; 268 269 uint32_t num_outstanding_data_wr; 270 uint64_t receive_tsc; 271 272 struct spdk_dif_ctx dif_ctx; 273 bool dif_insert_or_strip; 274 uint32_t elba_length; 275 uint32_t orig_length; 276 277 STAILQ_ENTRY(spdk_nvmf_rdma_request) state_link; 278 }; 279 280 enum spdk_nvmf_rdma_qpair_disconnect_flags { 281 RDMA_QP_DISCONNECTING = 1, 282 RDMA_QP_RECV_DRAINED = 1 << 1, 283 RDMA_QP_SEND_DRAINED = 1 << 2 284 }; 285 286 struct spdk_nvmf_rdma_resource_opts { 287 struct spdk_nvmf_rdma_qpair *qpair; 288 /* qp points either to an ibv_qp object or an ibv_srq object depending on the value of shared. */ 289 void *qp; 290 struct ibv_pd *pd; 291 uint32_t max_queue_depth; 292 uint32_t in_capsule_data_size; 293 bool shared; 294 }; 295 296 struct spdk_nvmf_send_wr_list { 297 struct ibv_send_wr *first; 298 struct ibv_send_wr *last; 299 }; 300 301 struct spdk_nvmf_recv_wr_list { 302 struct ibv_recv_wr *first; 303 struct ibv_recv_wr *last; 304 }; 305 306 struct spdk_nvmf_rdma_resources { 307 /* Array of size "max_queue_depth" containing RDMA requests. */ 308 struct spdk_nvmf_rdma_request *reqs; 309 310 /* Array of size "max_queue_depth" containing RDMA recvs. */ 311 struct spdk_nvmf_rdma_recv *recvs; 312 313 /* Array of size "max_queue_depth" containing 64 byte capsules 314 * used for receive. 315 */ 316 union nvmf_h2c_msg *cmds; 317 struct ibv_mr *cmds_mr; 318 319 /* Array of size "max_queue_depth" containing 16 byte completions 320 * to be sent back to the user. 321 */ 322 union nvmf_c2h_msg *cpls; 323 struct ibv_mr *cpls_mr; 324 325 /* Array of size "max_queue_depth * InCapsuleDataSize" containing 326 * buffers to be used for in capsule data. 327 */ 328 void *bufs; 329 struct ibv_mr *bufs_mr; 330 331 /* The list of pending recvs to transfer */ 332 struct spdk_nvmf_recv_wr_list recvs_to_post; 333 334 /* Receives that are waiting for a request object */ 335 STAILQ_HEAD(, spdk_nvmf_rdma_recv) incoming_queue; 336 337 /* Queue to track free requests */ 338 STAILQ_HEAD(, spdk_nvmf_rdma_request) free_queue; 339 }; 340 341 struct spdk_nvmf_rdma_qpair { 342 struct spdk_nvmf_qpair qpair; 343 344 struct spdk_nvmf_rdma_port *port; 345 struct spdk_nvmf_rdma_poller *poller; 346 347 struct rdma_cm_id *cm_id; 348 struct ibv_srq *srq; 349 struct rdma_cm_id *listen_id; 350 351 /* The maximum number of I/O outstanding on this connection at one time */ 352 uint16_t max_queue_depth; 353 354 /* The maximum number of active RDMA READ and ATOMIC operations at one time */ 355 uint16_t max_read_depth; 356 357 /* The maximum number of RDMA SEND operations at one time */ 358 uint32_t max_send_depth; 359 360 /* The current number of outstanding WRs from this qpair's 361 * recv queue. Should not exceed device->attr.max_queue_depth. 362 */ 363 uint16_t current_recv_depth; 364 365 /* The current number of active RDMA READ operations */ 366 uint16_t current_read_depth; 367 368 /* The current number of posted WRs from this qpair's 369 * send queue. Should not exceed max_send_depth. 370 */ 371 uint32_t current_send_depth; 372 373 /* The maximum number of SGEs per WR on the send queue */ 374 uint32_t max_send_sge; 375 376 /* The maximum number of SGEs per WR on the recv queue */ 377 uint32_t max_recv_sge; 378 379 /* The list of pending send requests for a transfer */ 380 struct spdk_nvmf_send_wr_list sends_to_post; 381 382 struct spdk_nvmf_rdma_resources *resources; 383 384 STAILQ_HEAD(, spdk_nvmf_rdma_request) pending_rdma_read_queue; 385 386 STAILQ_HEAD(, spdk_nvmf_rdma_request) pending_rdma_write_queue; 387 388 /* Number of requests not in the free state */ 389 uint32_t qd; 390 391 TAILQ_ENTRY(spdk_nvmf_rdma_qpair) link; 392 393 STAILQ_ENTRY(spdk_nvmf_rdma_qpair) recv_link; 394 395 STAILQ_ENTRY(spdk_nvmf_rdma_qpair) send_link; 396 397 /* IBV queue pair attributes: they are used to manage 398 * qp state and recover from errors. 399 */ 400 enum ibv_qp_state ibv_state; 401 402 uint32_t disconnect_flags; 403 404 /* Poller registered in case the qpair doesn't properly 405 * complete the qpair destruct process and becomes defunct. 406 */ 407 408 struct spdk_poller *destruct_poller; 409 410 /* There are several ways a disconnect can start on a qpair 411 * and they are not all mutually exclusive. It is important 412 * that we only initialize one of these paths. 413 */ 414 bool disconnect_started; 415 /* Lets us know that we have received the last_wqe event. */ 416 bool last_wqe_reached; 417 }; 418 419 struct spdk_nvmf_rdma_poller_stat { 420 uint64_t completions; 421 uint64_t polls; 422 uint64_t requests; 423 uint64_t request_latency; 424 uint64_t pending_free_request; 425 uint64_t pending_rdma_read; 426 uint64_t pending_rdma_write; 427 }; 428 429 struct spdk_nvmf_rdma_poller { 430 struct spdk_nvmf_rdma_device *device; 431 struct spdk_nvmf_rdma_poll_group *group; 432 433 int num_cqe; 434 int required_num_wr; 435 struct ibv_cq *cq; 436 437 /* The maximum number of I/O outstanding on the shared receive queue at one time */ 438 uint16_t max_srq_depth; 439 440 /* Shared receive queue */ 441 struct ibv_srq *srq; 442 443 struct spdk_nvmf_rdma_resources *resources; 444 struct spdk_nvmf_rdma_poller_stat stat; 445 446 TAILQ_HEAD(, spdk_nvmf_rdma_qpair) qpairs; 447 448 STAILQ_HEAD(, spdk_nvmf_rdma_qpair) qpairs_pending_recv; 449 450 STAILQ_HEAD(, spdk_nvmf_rdma_qpair) qpairs_pending_send; 451 452 TAILQ_ENTRY(spdk_nvmf_rdma_poller) link; 453 }; 454 455 struct spdk_nvmf_rdma_poll_group_stat { 456 uint64_t pending_data_buffer; 457 }; 458 459 struct spdk_nvmf_rdma_poll_group { 460 struct spdk_nvmf_transport_poll_group group; 461 struct spdk_nvmf_rdma_poll_group_stat stat; 462 TAILQ_HEAD(, spdk_nvmf_rdma_poller) pollers; 463 TAILQ_ENTRY(spdk_nvmf_rdma_poll_group) link; 464 /* 465 * buffers which are split across multiple RDMA 466 * memory regions cannot be used by this transport. 467 */ 468 STAILQ_HEAD(, spdk_nvmf_transport_pg_cache_buf) retired_bufs; 469 }; 470 471 struct spdk_nvmf_rdma_conn_sched { 472 struct spdk_nvmf_rdma_poll_group *next_admin_pg; 473 struct spdk_nvmf_rdma_poll_group *next_io_pg; 474 }; 475 476 /* Assuming rdma_cm uses just one protection domain per ibv_context. */ 477 struct spdk_nvmf_rdma_device { 478 struct ibv_device_attr attr; 479 struct ibv_context *context; 480 481 struct spdk_mem_map *map; 482 struct ibv_pd *pd; 483 484 int num_srq; 485 486 TAILQ_ENTRY(spdk_nvmf_rdma_device) link; 487 }; 488 489 struct spdk_nvmf_rdma_port { 490 struct spdk_nvme_transport_id trid; 491 struct rdma_cm_id *id; 492 struct spdk_nvmf_rdma_device *device; 493 uint32_t ref; 494 TAILQ_ENTRY(spdk_nvmf_rdma_port) link; 495 }; 496 497 struct spdk_nvmf_rdma_transport { 498 struct spdk_nvmf_transport transport; 499 500 struct spdk_nvmf_rdma_conn_sched conn_sched; 501 502 struct rdma_event_channel *event_channel; 503 504 struct spdk_mempool *data_wr_pool; 505 506 pthread_mutex_t lock; 507 508 /* fields used to poll RDMA/IB events */ 509 nfds_t npoll_fds; 510 struct pollfd *poll_fds; 511 512 TAILQ_HEAD(, spdk_nvmf_rdma_device) devices; 513 TAILQ_HEAD(, spdk_nvmf_rdma_port) ports; 514 TAILQ_HEAD(, spdk_nvmf_rdma_poll_group) poll_groups; 515 }; 516 517 static inline void 518 spdk_nvmf_rdma_start_disconnect(struct spdk_nvmf_rdma_qpair *rqpair); 519 520 static inline int 521 spdk_nvmf_rdma_check_ibv_state(enum ibv_qp_state state) 522 { 523 switch (state) { 524 case IBV_QPS_RESET: 525 case IBV_QPS_INIT: 526 case IBV_QPS_RTR: 527 case IBV_QPS_RTS: 528 case IBV_QPS_SQD: 529 case IBV_QPS_SQE: 530 case IBV_QPS_ERR: 531 return 0; 532 default: 533 return -1; 534 } 535 } 536 537 static inline enum spdk_nvme_media_error_status_code 538 spdk_nvmf_rdma_dif_error_to_compl_status(uint8_t err_type) { 539 enum spdk_nvme_media_error_status_code result; 540 switch (err_type) 541 { 542 case SPDK_DIF_REFTAG_ERROR: 543 result = SPDK_NVME_SC_REFERENCE_TAG_CHECK_ERROR; 544 break; 545 case SPDK_DIF_APPTAG_ERROR: 546 result = SPDK_NVME_SC_APPLICATION_TAG_CHECK_ERROR; 547 break; 548 case SPDK_DIF_GUARD_ERROR: 549 result = SPDK_NVME_SC_GUARD_CHECK_ERROR; 550 break; 551 default: 552 SPDK_UNREACHABLE(); 553 } 554 555 return result; 556 } 557 558 static enum ibv_qp_state 559 spdk_nvmf_rdma_update_ibv_state(struct spdk_nvmf_rdma_qpair *rqpair) { 560 enum ibv_qp_state old_state, new_state; 561 struct ibv_qp_attr qp_attr; 562 struct ibv_qp_init_attr init_attr; 563 int rc; 564 565 old_state = rqpair->ibv_state; 566 rc = ibv_query_qp(rqpair->cm_id->qp, &qp_attr, 567 g_spdk_nvmf_ibv_query_mask, &init_attr); 568 569 if (rc) 570 { 571 SPDK_ERRLOG("Failed to get updated RDMA queue pair state!\n"); 572 return IBV_QPS_ERR + 1; 573 } 574 575 new_state = qp_attr.qp_state; 576 rqpair->ibv_state = new_state; 577 qp_attr.ah_attr.port_num = qp_attr.port_num; 578 579 rc = spdk_nvmf_rdma_check_ibv_state(new_state); 580 if (rc) 581 { 582 SPDK_ERRLOG("QP#%d: bad state updated: %u, maybe hardware issue\n", rqpair->qpair.qid, new_state); 583 /* 584 * IBV_QPS_UNKNOWN undefined if lib version smaller than libibverbs-1.1.8 585 * IBV_QPS_UNKNOWN is the enum element after IBV_QPS_ERR 586 */ 587 return IBV_QPS_ERR + 1; 588 } 589 590 if (old_state != new_state) 591 { 592 spdk_trace_record(TRACE_RDMA_QP_STATE_CHANGE, 0, 0, 593 (uintptr_t)rqpair->cm_id, new_state); 594 } 595 return new_state; 596 } 597 598 static const char *str_ibv_qp_state[] = { 599 "IBV_QPS_RESET", 600 "IBV_QPS_INIT", 601 "IBV_QPS_RTR", 602 "IBV_QPS_RTS", 603 "IBV_QPS_SQD", 604 "IBV_QPS_SQE", 605 "IBV_QPS_ERR", 606 "IBV_QPS_UNKNOWN" 607 }; 608 609 static int 610 spdk_nvmf_rdma_set_ibv_state(struct spdk_nvmf_rdma_qpair *rqpair, 611 enum ibv_qp_state new_state) 612 { 613 struct ibv_qp_attr qp_attr; 614 struct ibv_qp_init_attr init_attr; 615 int rc; 616 enum ibv_qp_state state; 617 static int attr_mask_rc[] = { 618 [IBV_QPS_RESET] = IBV_QP_STATE, 619 [IBV_QPS_INIT] = (IBV_QP_STATE | 620 IBV_QP_PKEY_INDEX | 621 IBV_QP_PORT | 622 IBV_QP_ACCESS_FLAGS), 623 [IBV_QPS_RTR] = (IBV_QP_STATE | 624 IBV_QP_AV | 625 IBV_QP_PATH_MTU | 626 IBV_QP_DEST_QPN | 627 IBV_QP_RQ_PSN | 628 IBV_QP_MAX_DEST_RD_ATOMIC | 629 IBV_QP_MIN_RNR_TIMER), 630 [IBV_QPS_RTS] = (IBV_QP_STATE | 631 IBV_QP_SQ_PSN | 632 IBV_QP_TIMEOUT | 633 IBV_QP_RETRY_CNT | 634 IBV_QP_RNR_RETRY | 635 IBV_QP_MAX_QP_RD_ATOMIC), 636 [IBV_QPS_SQD] = IBV_QP_STATE, 637 [IBV_QPS_SQE] = IBV_QP_STATE, 638 [IBV_QPS_ERR] = IBV_QP_STATE, 639 }; 640 641 rc = spdk_nvmf_rdma_check_ibv_state(new_state); 642 if (rc) { 643 SPDK_ERRLOG("QP#%d: bad state requested: %u\n", 644 rqpair->qpair.qid, new_state); 645 return rc; 646 } 647 648 rc = ibv_query_qp(rqpair->cm_id->qp, &qp_attr, 649 g_spdk_nvmf_ibv_query_mask, &init_attr); 650 651 if (rc) { 652 SPDK_ERRLOG("Failed to get updated RDMA queue pair state!\n"); 653 assert(false); 654 } 655 656 qp_attr.cur_qp_state = rqpair->ibv_state; 657 qp_attr.qp_state = new_state; 658 659 rc = ibv_modify_qp(rqpair->cm_id->qp, &qp_attr, 660 attr_mask_rc[new_state]); 661 662 if (rc) { 663 SPDK_ERRLOG("QP#%d: failed to set state to: %s, %d (%s)\n", 664 rqpair->qpair.qid, str_ibv_qp_state[new_state], errno, strerror(errno)); 665 return rc; 666 } 667 668 state = spdk_nvmf_rdma_update_ibv_state(rqpair); 669 670 if (state != new_state) { 671 SPDK_ERRLOG("QP#%d: expected state: %s, actual state: %s\n", 672 rqpair->qpair.qid, str_ibv_qp_state[new_state], 673 str_ibv_qp_state[state]); 674 return -1; 675 } 676 SPDK_DEBUGLOG(SPDK_LOG_RDMA, "IBV QP#%u changed to: %s\n", rqpair->qpair.qid, 677 str_ibv_qp_state[state]); 678 return 0; 679 } 680 681 static void 682 nvmf_rdma_request_free_data(struct spdk_nvmf_rdma_request *rdma_req, 683 struct spdk_nvmf_rdma_transport *rtransport) 684 { 685 struct spdk_nvmf_rdma_request_data *data_wr; 686 struct ibv_send_wr *next_send_wr; 687 uint64_t req_wrid; 688 689 rdma_req->num_outstanding_data_wr = 0; 690 data_wr = &rdma_req->data; 691 req_wrid = data_wr->wr.wr_id; 692 while (data_wr && data_wr->wr.wr_id == req_wrid) { 693 memset(data_wr->sgl, 0, sizeof(data_wr->wr.sg_list[0]) * data_wr->wr.num_sge); 694 data_wr->wr.num_sge = 0; 695 next_send_wr = data_wr->wr.next; 696 if (data_wr != &rdma_req->data) { 697 spdk_mempool_put(rtransport->data_wr_pool, data_wr); 698 } 699 data_wr = (!next_send_wr || next_send_wr == &rdma_req->rsp.wr) ? NULL : 700 SPDK_CONTAINEROF(next_send_wr, struct spdk_nvmf_rdma_request_data, wr); 701 } 702 } 703 704 static void 705 nvmf_rdma_dump_request(struct spdk_nvmf_rdma_request *req) 706 { 707 SPDK_ERRLOG("\t\tRequest Data From Pool: %d\n", req->req.data_from_pool); 708 if (req->req.cmd) { 709 SPDK_ERRLOG("\t\tRequest opcode: %d\n", req->req.cmd->nvmf_cmd.opcode); 710 } 711 if (req->recv) { 712 SPDK_ERRLOG("\t\tRequest recv wr_id%lu\n", req->recv->wr.wr_id); 713 } 714 } 715 716 static void 717 nvmf_rdma_dump_qpair_contents(struct spdk_nvmf_rdma_qpair *rqpair) 718 { 719 int i; 720 721 SPDK_ERRLOG("Dumping contents of queue pair (QID %d)\n", rqpair->qpair.qid); 722 for (i = 0; i < rqpair->max_queue_depth; i++) { 723 if (rqpair->resources->reqs[i].state != RDMA_REQUEST_STATE_FREE) { 724 nvmf_rdma_dump_request(&rqpair->resources->reqs[i]); 725 } 726 } 727 } 728 729 static void 730 nvmf_rdma_resources_destroy(struct spdk_nvmf_rdma_resources *resources) 731 { 732 if (resources->cmds_mr) { 733 ibv_dereg_mr(resources->cmds_mr); 734 } 735 736 if (resources->cpls_mr) { 737 ibv_dereg_mr(resources->cpls_mr); 738 } 739 740 if (resources->bufs_mr) { 741 ibv_dereg_mr(resources->bufs_mr); 742 } 743 744 spdk_free(resources->cmds); 745 spdk_free(resources->cpls); 746 spdk_free(resources->bufs); 747 free(resources->reqs); 748 free(resources->recvs); 749 free(resources); 750 } 751 752 753 static struct spdk_nvmf_rdma_resources * 754 nvmf_rdma_resources_create(struct spdk_nvmf_rdma_resource_opts *opts) 755 { 756 struct spdk_nvmf_rdma_resources *resources; 757 struct spdk_nvmf_rdma_request *rdma_req; 758 struct spdk_nvmf_rdma_recv *rdma_recv; 759 struct ibv_qp *qp; 760 struct ibv_srq *srq; 761 uint32_t i; 762 int rc; 763 764 resources = calloc(1, sizeof(struct spdk_nvmf_rdma_resources)); 765 if (!resources) { 766 SPDK_ERRLOG("Unable to allocate resources for receive queue.\n"); 767 return NULL; 768 } 769 770 resources->reqs = calloc(opts->max_queue_depth, sizeof(*resources->reqs)); 771 resources->recvs = calloc(opts->max_queue_depth, sizeof(*resources->recvs)); 772 resources->cmds = spdk_zmalloc(opts->max_queue_depth * sizeof(*resources->cmds), 773 0x1000, NULL, SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA); 774 resources->cpls = spdk_zmalloc(opts->max_queue_depth * sizeof(*resources->cpls), 775 0x1000, NULL, SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA); 776 777 if (opts->in_capsule_data_size > 0) { 778 resources->bufs = spdk_zmalloc(opts->max_queue_depth * opts->in_capsule_data_size, 779 0x1000, NULL, SPDK_ENV_LCORE_ID_ANY, 780 SPDK_MALLOC_DMA); 781 } 782 783 if (!resources->reqs || !resources->recvs || !resources->cmds || 784 !resources->cpls || (opts->in_capsule_data_size && !resources->bufs)) { 785 SPDK_ERRLOG("Unable to allocate sufficient memory for RDMA queue.\n"); 786 goto cleanup; 787 } 788 789 resources->cmds_mr = ibv_reg_mr(opts->pd, resources->cmds, 790 opts->max_queue_depth * sizeof(*resources->cmds), 791 IBV_ACCESS_LOCAL_WRITE); 792 resources->cpls_mr = ibv_reg_mr(opts->pd, resources->cpls, 793 opts->max_queue_depth * sizeof(*resources->cpls), 794 0); 795 796 if (opts->in_capsule_data_size) { 797 resources->bufs_mr = ibv_reg_mr(opts->pd, resources->bufs, 798 opts->max_queue_depth * 799 opts->in_capsule_data_size, 800 IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE); 801 } 802 803 if (!resources->cmds_mr || !resources->cpls_mr || 804 (opts->in_capsule_data_size && 805 !resources->bufs_mr)) { 806 goto cleanup; 807 } 808 SPDK_DEBUGLOG(SPDK_LOG_RDMA, "Command Array: %p Length: %lx LKey: %x\n", 809 resources->cmds, opts->max_queue_depth * sizeof(*resources->cmds), 810 resources->cmds_mr->lkey); 811 SPDK_DEBUGLOG(SPDK_LOG_RDMA, "Completion Array: %p Length: %lx LKey: %x\n", 812 resources->cpls, opts->max_queue_depth * sizeof(*resources->cpls), 813 resources->cpls_mr->lkey); 814 if (resources->bufs && resources->bufs_mr) { 815 SPDK_DEBUGLOG(SPDK_LOG_RDMA, "In Capsule Data Array: %p Length: %x LKey: %x\n", 816 resources->bufs, opts->max_queue_depth * 817 opts->in_capsule_data_size, resources->bufs_mr->lkey); 818 } 819 820 /* Initialize queues */ 821 STAILQ_INIT(&resources->incoming_queue); 822 STAILQ_INIT(&resources->free_queue); 823 824 for (i = 0; i < opts->max_queue_depth; i++) { 825 struct ibv_recv_wr *bad_wr = NULL; 826 827 rdma_recv = &resources->recvs[i]; 828 rdma_recv->qpair = opts->qpair; 829 830 /* Set up memory to receive commands */ 831 if (resources->bufs) { 832 rdma_recv->buf = (void *)((uintptr_t)resources->bufs + (i * 833 opts->in_capsule_data_size)); 834 } 835 836 rdma_recv->rdma_wr.type = RDMA_WR_TYPE_RECV; 837 838 rdma_recv->sgl[0].addr = (uintptr_t)&resources->cmds[i]; 839 rdma_recv->sgl[0].length = sizeof(resources->cmds[i]); 840 rdma_recv->sgl[0].lkey = resources->cmds_mr->lkey; 841 rdma_recv->wr.num_sge = 1; 842 843 if (rdma_recv->buf && resources->bufs_mr) { 844 rdma_recv->sgl[1].addr = (uintptr_t)rdma_recv->buf; 845 rdma_recv->sgl[1].length = opts->in_capsule_data_size; 846 rdma_recv->sgl[1].lkey = resources->bufs_mr->lkey; 847 rdma_recv->wr.num_sge++; 848 } 849 850 rdma_recv->wr.wr_id = (uintptr_t)&rdma_recv->rdma_wr; 851 rdma_recv->wr.sg_list = rdma_recv->sgl; 852 if (opts->shared) { 853 srq = (struct ibv_srq *)opts->qp; 854 rc = ibv_post_srq_recv(srq, &rdma_recv->wr, &bad_wr); 855 } else { 856 qp = (struct ibv_qp *)opts->qp; 857 rc = ibv_post_recv(qp, &rdma_recv->wr, &bad_wr); 858 } 859 if (rc) { 860 goto cleanup; 861 } 862 } 863 864 for (i = 0; i < opts->max_queue_depth; i++) { 865 rdma_req = &resources->reqs[i]; 866 867 if (opts->qpair != NULL) { 868 rdma_req->req.qpair = &opts->qpair->qpair; 869 } else { 870 rdma_req->req.qpair = NULL; 871 } 872 rdma_req->req.cmd = NULL; 873 874 /* Set up memory to send responses */ 875 rdma_req->req.rsp = &resources->cpls[i]; 876 877 rdma_req->rsp.sgl[0].addr = (uintptr_t)&resources->cpls[i]; 878 rdma_req->rsp.sgl[0].length = sizeof(resources->cpls[i]); 879 rdma_req->rsp.sgl[0].lkey = resources->cpls_mr->lkey; 880 881 rdma_req->rsp.rdma_wr.type = RDMA_WR_TYPE_SEND; 882 rdma_req->rsp.wr.wr_id = (uintptr_t)&rdma_req->rsp.rdma_wr; 883 rdma_req->rsp.wr.next = NULL; 884 rdma_req->rsp.wr.opcode = IBV_WR_SEND; 885 rdma_req->rsp.wr.send_flags = IBV_SEND_SIGNALED; 886 rdma_req->rsp.wr.sg_list = rdma_req->rsp.sgl; 887 rdma_req->rsp.wr.num_sge = SPDK_COUNTOF(rdma_req->rsp.sgl); 888 889 /* Set up memory for data buffers */ 890 rdma_req->data.rdma_wr.type = RDMA_WR_TYPE_DATA; 891 rdma_req->data.wr.wr_id = (uintptr_t)&rdma_req->data.rdma_wr; 892 rdma_req->data.wr.next = NULL; 893 rdma_req->data.wr.send_flags = IBV_SEND_SIGNALED; 894 rdma_req->data.wr.sg_list = rdma_req->data.sgl; 895 rdma_req->data.wr.num_sge = SPDK_COUNTOF(rdma_req->data.sgl); 896 897 /* Initialize request state to FREE */ 898 rdma_req->state = RDMA_REQUEST_STATE_FREE; 899 STAILQ_INSERT_TAIL(&resources->free_queue, rdma_req, state_link); 900 } 901 902 return resources; 903 904 cleanup: 905 nvmf_rdma_resources_destroy(resources); 906 return NULL; 907 } 908 909 static void 910 spdk_nvmf_rdma_qpair_destroy(struct spdk_nvmf_rdma_qpair *rqpair) 911 { 912 struct spdk_nvmf_rdma_recv *rdma_recv, *recv_tmp; 913 struct ibv_recv_wr *bad_recv_wr = NULL; 914 int rc; 915 916 spdk_trace_record(TRACE_RDMA_QP_DESTROY, 0, 0, (uintptr_t)rqpair->cm_id, 0); 917 918 spdk_poller_unregister(&rqpair->destruct_poller); 919 920 if (rqpair->qd != 0) { 921 if (rqpair->srq == NULL) { 922 nvmf_rdma_dump_qpair_contents(rqpair); 923 } 924 SPDK_WARNLOG("Destroying qpair when queue depth is %d\n", rqpair->qd); 925 } 926 927 if (rqpair->poller) { 928 TAILQ_REMOVE(&rqpair->poller->qpairs, rqpair, link); 929 930 if (rqpair->srq != NULL && rqpair->resources != NULL) { 931 /* Drop all received but unprocessed commands for this queue and return them to SRQ */ 932 STAILQ_FOREACH_SAFE(rdma_recv, &rqpair->resources->incoming_queue, link, recv_tmp) { 933 if (rqpair == rdma_recv->qpair) { 934 STAILQ_REMOVE_HEAD(&rqpair->resources->incoming_queue, link); 935 rc = ibv_post_srq_recv(rqpair->srq, &rdma_recv->wr, &bad_recv_wr); 936 if (rc) { 937 SPDK_ERRLOG("Unable to re-post rx descriptor\n"); 938 } 939 } 940 } 941 } 942 } 943 944 if (rqpair->cm_id) { 945 if (rqpair->cm_id->qp != NULL) { 946 rdma_destroy_qp(rqpair->cm_id); 947 } 948 rdma_destroy_id(rqpair->cm_id); 949 950 if (rqpair->poller != NULL && rqpair->srq == NULL) { 951 rqpair->poller->required_num_wr -= MAX_WR_PER_QP(rqpair->max_queue_depth); 952 } 953 } 954 955 if (rqpair->srq == NULL && rqpair->resources != NULL) { 956 nvmf_rdma_resources_destroy(rqpair->resources); 957 } 958 959 free(rqpair); 960 } 961 962 static int 963 nvmf_rdma_resize_cq(struct spdk_nvmf_rdma_qpair *rqpair, struct spdk_nvmf_rdma_device *device) 964 { 965 struct spdk_nvmf_rdma_poller *rpoller; 966 int rc, num_cqe, required_num_wr; 967 968 /* Enlarge CQ size dynamically */ 969 rpoller = rqpair->poller; 970 required_num_wr = rpoller->required_num_wr + MAX_WR_PER_QP(rqpair->max_queue_depth); 971 num_cqe = rpoller->num_cqe; 972 if (num_cqe < required_num_wr) { 973 num_cqe = spdk_max(num_cqe * 2, required_num_wr); 974 num_cqe = spdk_min(num_cqe, device->attr.max_cqe); 975 } 976 977 if (rpoller->num_cqe != num_cqe) { 978 if (required_num_wr > device->attr.max_cqe) { 979 SPDK_ERRLOG("RDMA CQE requirement (%d) exceeds device max_cqe limitation (%d)\n", 980 required_num_wr, device->attr.max_cqe); 981 return -1; 982 } 983 984 SPDK_DEBUGLOG(SPDK_LOG_RDMA, "Resize RDMA CQ from %d to %d\n", rpoller->num_cqe, num_cqe); 985 rc = ibv_resize_cq(rpoller->cq, num_cqe); 986 if (rc) { 987 SPDK_ERRLOG("RDMA CQ resize failed: errno %d: %s\n", errno, spdk_strerror(errno)); 988 return -1; 989 } 990 991 rpoller->num_cqe = num_cqe; 992 } 993 994 rpoller->required_num_wr = required_num_wr; 995 return 0; 996 } 997 998 static int 999 spdk_nvmf_rdma_qpair_initialize(struct spdk_nvmf_qpair *qpair) 1000 { 1001 struct spdk_nvmf_rdma_qpair *rqpair; 1002 int rc; 1003 struct spdk_nvmf_rdma_transport *rtransport; 1004 struct spdk_nvmf_transport *transport; 1005 struct spdk_nvmf_rdma_resource_opts opts; 1006 struct spdk_nvmf_rdma_device *device; 1007 struct ibv_qp_init_attr ibv_init_attr; 1008 1009 rqpair = SPDK_CONTAINEROF(qpair, struct spdk_nvmf_rdma_qpair, qpair); 1010 device = rqpair->port->device; 1011 1012 memset(&ibv_init_attr, 0, sizeof(struct ibv_qp_init_attr)); 1013 ibv_init_attr.qp_context = rqpair; 1014 ibv_init_attr.qp_type = IBV_QPT_RC; 1015 ibv_init_attr.send_cq = rqpair->poller->cq; 1016 ibv_init_attr.recv_cq = rqpair->poller->cq; 1017 1018 if (rqpair->srq) { 1019 ibv_init_attr.srq = rqpair->srq; 1020 } else { 1021 ibv_init_attr.cap.max_recv_wr = rqpair->max_queue_depth + 1022 1; /* RECV operations + dummy drain WR */ 1023 } 1024 1025 ibv_init_attr.cap.max_send_wr = rqpair->max_queue_depth * 1026 2 + 1; /* SEND, READ, and WRITE operations + dummy drain WR */ 1027 ibv_init_attr.cap.max_send_sge = spdk_min(device->attr.max_sge, NVMF_DEFAULT_TX_SGE); 1028 ibv_init_attr.cap.max_recv_sge = spdk_min(device->attr.max_sge, NVMF_DEFAULT_RX_SGE); 1029 1030 if (rqpair->srq == NULL && nvmf_rdma_resize_cq(rqpair, device) < 0) { 1031 SPDK_ERRLOG("Failed to resize the completion queue. Cannot initialize qpair.\n"); 1032 goto error; 1033 } 1034 1035 rc = rdma_create_qp(rqpair->cm_id, rqpair->port->device->pd, &ibv_init_attr); 1036 if (rc) { 1037 SPDK_ERRLOG("rdma_create_qp failed: errno %d: %s\n", errno, spdk_strerror(errno)); 1038 goto error; 1039 } 1040 1041 rqpair->max_send_depth = spdk_min((uint32_t)(rqpair->max_queue_depth * 2 + 1), 1042 ibv_init_attr.cap.max_send_wr); 1043 rqpair->max_send_sge = spdk_min(NVMF_DEFAULT_TX_SGE, ibv_init_attr.cap.max_send_sge); 1044 rqpair->max_recv_sge = spdk_min(NVMF_DEFAULT_RX_SGE, ibv_init_attr.cap.max_recv_sge); 1045 spdk_trace_record(TRACE_RDMA_QP_CREATE, 0, 0, (uintptr_t)rqpair->cm_id, 0); 1046 SPDK_DEBUGLOG(SPDK_LOG_RDMA, "New RDMA Connection: %p\n", qpair); 1047 1048 rqpair->sends_to_post.first = NULL; 1049 rqpair->sends_to_post.last = NULL; 1050 1051 if (rqpair->poller->srq == NULL) { 1052 rtransport = SPDK_CONTAINEROF(qpair->transport, struct spdk_nvmf_rdma_transport, transport); 1053 transport = &rtransport->transport; 1054 1055 opts.qp = rqpair->cm_id->qp; 1056 opts.pd = rqpair->cm_id->pd; 1057 opts.qpair = rqpair; 1058 opts.shared = false; 1059 opts.max_queue_depth = rqpair->max_queue_depth; 1060 opts.in_capsule_data_size = transport->opts.in_capsule_data_size; 1061 1062 rqpair->resources = nvmf_rdma_resources_create(&opts); 1063 1064 if (!rqpair->resources) { 1065 SPDK_ERRLOG("Unable to allocate resources for receive queue.\n"); 1066 rdma_destroy_qp(rqpair->cm_id); 1067 goto error; 1068 } 1069 } else { 1070 rqpair->resources = rqpair->poller->resources; 1071 } 1072 1073 rqpair->current_recv_depth = 0; 1074 STAILQ_INIT(&rqpair->pending_rdma_read_queue); 1075 STAILQ_INIT(&rqpair->pending_rdma_write_queue); 1076 1077 return 0; 1078 1079 error: 1080 rdma_destroy_id(rqpair->cm_id); 1081 rqpair->cm_id = NULL; 1082 return -1; 1083 } 1084 1085 /* Append the given recv wr structure to the resource structs outstanding recvs list. */ 1086 /* This function accepts either a single wr or the first wr in a linked list. */ 1087 static void 1088 nvmf_rdma_qpair_queue_recv_wrs(struct spdk_nvmf_rdma_qpair *rqpair, struct ibv_recv_wr *first) 1089 { 1090 struct ibv_recv_wr *last; 1091 1092 last = first; 1093 while (last->next != NULL) { 1094 last = last->next; 1095 } 1096 1097 if (rqpair->resources->recvs_to_post.first == NULL) { 1098 rqpair->resources->recvs_to_post.first = first; 1099 rqpair->resources->recvs_to_post.last = last; 1100 if (rqpair->srq == NULL) { 1101 STAILQ_INSERT_TAIL(&rqpair->poller->qpairs_pending_recv, rqpair, recv_link); 1102 } 1103 } else { 1104 rqpair->resources->recvs_to_post.last->next = first; 1105 rqpair->resources->recvs_to_post.last = last; 1106 } 1107 } 1108 1109 /* Append the given send wr structure to the qpair's outstanding sends list. */ 1110 /* This function accepts either a single wr or the first wr in a linked list. */ 1111 static void 1112 nvmf_rdma_qpair_queue_send_wrs(struct spdk_nvmf_rdma_qpair *rqpair, struct ibv_send_wr *first) 1113 { 1114 struct ibv_send_wr *last; 1115 1116 last = first; 1117 while (last->next != NULL) { 1118 last = last->next; 1119 } 1120 1121 if (rqpair->sends_to_post.first == NULL) { 1122 rqpair->sends_to_post.first = first; 1123 rqpair->sends_to_post.last = last; 1124 STAILQ_INSERT_TAIL(&rqpair->poller->qpairs_pending_send, rqpair, send_link); 1125 } else { 1126 rqpair->sends_to_post.last->next = first; 1127 rqpair->sends_to_post.last = last; 1128 } 1129 } 1130 1131 static int 1132 request_transfer_in(struct spdk_nvmf_request *req) 1133 { 1134 struct spdk_nvmf_rdma_request *rdma_req; 1135 struct spdk_nvmf_qpair *qpair; 1136 struct spdk_nvmf_rdma_qpair *rqpair; 1137 1138 qpair = req->qpair; 1139 rdma_req = SPDK_CONTAINEROF(req, struct spdk_nvmf_rdma_request, req); 1140 rqpair = SPDK_CONTAINEROF(qpair, struct spdk_nvmf_rdma_qpair, qpair); 1141 1142 assert(req->xfer == SPDK_NVME_DATA_HOST_TO_CONTROLLER); 1143 assert(rdma_req != NULL); 1144 1145 nvmf_rdma_qpair_queue_send_wrs(rqpair, &rdma_req->data.wr); 1146 rqpair->current_read_depth += rdma_req->num_outstanding_data_wr; 1147 rqpair->current_send_depth += rdma_req->num_outstanding_data_wr; 1148 return 0; 1149 } 1150 1151 static int 1152 request_transfer_out(struct spdk_nvmf_request *req, int *data_posted) 1153 { 1154 int num_outstanding_data_wr = 0; 1155 struct spdk_nvmf_rdma_request *rdma_req; 1156 struct spdk_nvmf_qpair *qpair; 1157 struct spdk_nvmf_rdma_qpair *rqpair; 1158 struct spdk_nvme_cpl *rsp; 1159 struct ibv_send_wr *first = NULL; 1160 1161 *data_posted = 0; 1162 qpair = req->qpair; 1163 rsp = &req->rsp->nvme_cpl; 1164 rdma_req = SPDK_CONTAINEROF(req, struct spdk_nvmf_rdma_request, req); 1165 rqpair = SPDK_CONTAINEROF(qpair, struct spdk_nvmf_rdma_qpair, qpair); 1166 1167 /* Advance our sq_head pointer */ 1168 if (qpair->sq_head == qpair->sq_head_max) { 1169 qpair->sq_head = 0; 1170 } else { 1171 qpair->sq_head++; 1172 } 1173 rsp->sqhd = qpair->sq_head; 1174 1175 /* queue the capsule for the recv buffer */ 1176 assert(rdma_req->recv != NULL); 1177 1178 nvmf_rdma_qpair_queue_recv_wrs(rqpair, &rdma_req->recv->wr); 1179 1180 rdma_req->recv = NULL; 1181 assert(rqpair->current_recv_depth > 0); 1182 rqpair->current_recv_depth--; 1183 1184 /* Build the response which consists of optional 1185 * RDMA WRITEs to transfer data, plus an RDMA SEND 1186 * containing the response. 1187 */ 1188 first = &rdma_req->rsp.wr; 1189 1190 if (rsp->status.sc == SPDK_NVME_SC_SUCCESS && 1191 req->xfer == SPDK_NVME_DATA_CONTROLLER_TO_HOST) { 1192 first = &rdma_req->data.wr; 1193 *data_posted = 1; 1194 num_outstanding_data_wr = rdma_req->num_outstanding_data_wr; 1195 } 1196 nvmf_rdma_qpair_queue_send_wrs(rqpair, first); 1197 /* +1 for the rsp wr */ 1198 rqpair->current_send_depth += num_outstanding_data_wr + 1; 1199 1200 return 0; 1201 } 1202 1203 static int 1204 spdk_nvmf_rdma_event_accept(struct rdma_cm_id *id, struct spdk_nvmf_rdma_qpair *rqpair) 1205 { 1206 struct spdk_nvmf_rdma_accept_private_data accept_data; 1207 struct rdma_conn_param ctrlr_event_data = {}; 1208 int rc; 1209 1210 accept_data.recfmt = 0; 1211 accept_data.crqsize = rqpair->max_queue_depth; 1212 1213 ctrlr_event_data.private_data = &accept_data; 1214 ctrlr_event_data.private_data_len = sizeof(accept_data); 1215 if (id->ps == RDMA_PS_TCP) { 1216 ctrlr_event_data.responder_resources = 0; /* We accept 0 reads from the host */ 1217 ctrlr_event_data.initiator_depth = rqpair->max_read_depth; 1218 } 1219 1220 /* Configure infinite retries for the initiator side qpair. 1221 * When using a shared receive queue on the target side, 1222 * we need to pass this value to the initiator to prevent the 1223 * initiator side NIC from completing SEND requests back to the 1224 * initiator with status rnr_retry_count_exceeded. */ 1225 if (rqpair->srq != NULL) { 1226 ctrlr_event_data.rnr_retry_count = 0x7; 1227 } 1228 1229 rc = rdma_accept(id, &ctrlr_event_data); 1230 if (rc) { 1231 SPDK_ERRLOG("Error %d on rdma_accept\n", errno); 1232 } else { 1233 SPDK_DEBUGLOG(SPDK_LOG_RDMA, "Sent back the accept\n"); 1234 } 1235 1236 return rc; 1237 } 1238 1239 static void 1240 spdk_nvmf_rdma_event_reject(struct rdma_cm_id *id, enum spdk_nvmf_rdma_transport_error error) 1241 { 1242 struct spdk_nvmf_rdma_reject_private_data rej_data; 1243 1244 rej_data.recfmt = 0; 1245 rej_data.sts = error; 1246 1247 rdma_reject(id, &rej_data, sizeof(rej_data)); 1248 } 1249 1250 static int 1251 nvmf_rdma_connect(struct spdk_nvmf_transport *transport, struct rdma_cm_event *event, 1252 new_qpair_fn cb_fn) 1253 { 1254 struct spdk_nvmf_rdma_transport *rtransport; 1255 struct spdk_nvmf_rdma_qpair *rqpair = NULL; 1256 struct spdk_nvmf_rdma_port *port; 1257 struct rdma_conn_param *rdma_param = NULL; 1258 const struct spdk_nvmf_rdma_request_private_data *private_data = NULL; 1259 uint16_t max_queue_depth; 1260 uint16_t max_read_depth; 1261 1262 rtransport = SPDK_CONTAINEROF(transport, struct spdk_nvmf_rdma_transport, transport); 1263 1264 assert(event->id != NULL); /* Impossible. Can't even reject the connection. */ 1265 assert(event->id->verbs != NULL); /* Impossible. No way to handle this. */ 1266 1267 rdma_param = &event->param.conn; 1268 if (rdma_param->private_data == NULL || 1269 rdma_param->private_data_len < sizeof(struct spdk_nvmf_rdma_request_private_data)) { 1270 SPDK_ERRLOG("connect request: no private data provided\n"); 1271 spdk_nvmf_rdma_event_reject(event->id, SPDK_NVMF_RDMA_ERROR_INVALID_PRIVATE_DATA_LENGTH); 1272 return -1; 1273 } 1274 1275 private_data = rdma_param->private_data; 1276 if (private_data->recfmt != 0) { 1277 SPDK_ERRLOG("Received RDMA private data with RECFMT != 0\n"); 1278 spdk_nvmf_rdma_event_reject(event->id, SPDK_NVMF_RDMA_ERROR_INVALID_RECFMT); 1279 return -1; 1280 } 1281 1282 SPDK_DEBUGLOG(SPDK_LOG_RDMA, "Connect Recv on fabric intf name %s, dev_name %s\n", 1283 event->id->verbs->device->name, event->id->verbs->device->dev_name); 1284 1285 port = event->listen_id->context; 1286 SPDK_DEBUGLOG(SPDK_LOG_RDMA, "Listen Id was %p with verbs %p. ListenAddr: %p\n", 1287 event->listen_id, event->listen_id->verbs, port); 1288 1289 /* Figure out the supported queue depth. This is a multi-step process 1290 * that takes into account hardware maximums, host provided values, 1291 * and our target's internal memory limits */ 1292 1293 SPDK_DEBUGLOG(SPDK_LOG_RDMA, "Calculating Queue Depth\n"); 1294 1295 /* Start with the maximum queue depth allowed by the target */ 1296 max_queue_depth = rtransport->transport.opts.max_queue_depth; 1297 max_read_depth = rtransport->transport.opts.max_queue_depth; 1298 SPDK_DEBUGLOG(SPDK_LOG_RDMA, "Target Max Queue Depth: %d\n", 1299 rtransport->transport.opts.max_queue_depth); 1300 1301 /* Next check the local NIC's hardware limitations */ 1302 SPDK_DEBUGLOG(SPDK_LOG_RDMA, 1303 "Local NIC Max Send/Recv Queue Depth: %d Max Read/Write Queue Depth: %d\n", 1304 port->device->attr.max_qp_wr, port->device->attr.max_qp_rd_atom); 1305 max_queue_depth = spdk_min(max_queue_depth, port->device->attr.max_qp_wr); 1306 max_read_depth = spdk_min(max_read_depth, port->device->attr.max_qp_init_rd_atom); 1307 1308 /* Next check the remote NIC's hardware limitations */ 1309 SPDK_DEBUGLOG(SPDK_LOG_RDMA, 1310 "Host (Initiator) NIC Max Incoming RDMA R/W operations: %d Max Outgoing RDMA R/W operations: %d\n", 1311 rdma_param->initiator_depth, rdma_param->responder_resources); 1312 if (rdma_param->initiator_depth > 0) { 1313 max_read_depth = spdk_min(max_read_depth, rdma_param->initiator_depth); 1314 } 1315 1316 /* Finally check for the host software requested values, which are 1317 * optional. */ 1318 if (rdma_param->private_data != NULL && 1319 rdma_param->private_data_len >= sizeof(struct spdk_nvmf_rdma_request_private_data)) { 1320 SPDK_DEBUGLOG(SPDK_LOG_RDMA, "Host Receive Queue Size: %d\n", private_data->hrqsize); 1321 SPDK_DEBUGLOG(SPDK_LOG_RDMA, "Host Send Queue Size: %d\n", private_data->hsqsize); 1322 max_queue_depth = spdk_min(max_queue_depth, private_data->hrqsize); 1323 max_queue_depth = spdk_min(max_queue_depth, private_data->hsqsize + 1); 1324 } 1325 1326 SPDK_DEBUGLOG(SPDK_LOG_RDMA, "Final Negotiated Queue Depth: %d R/W Depth: %d\n", 1327 max_queue_depth, max_read_depth); 1328 1329 rqpair = calloc(1, sizeof(struct spdk_nvmf_rdma_qpair)); 1330 if (rqpair == NULL) { 1331 SPDK_ERRLOG("Could not allocate new connection.\n"); 1332 spdk_nvmf_rdma_event_reject(event->id, SPDK_NVMF_RDMA_ERROR_NO_RESOURCES); 1333 return -1; 1334 } 1335 1336 rqpair->port = port; 1337 rqpair->max_queue_depth = max_queue_depth; 1338 rqpair->max_read_depth = max_read_depth; 1339 rqpair->cm_id = event->id; 1340 rqpair->listen_id = event->listen_id; 1341 rqpair->qpair.transport = transport; 1342 /* use qid from the private data to determine the qpair type 1343 qid will be set to the appropriate value when the controller is created */ 1344 rqpair->qpair.qid = private_data->qid; 1345 1346 event->id->context = &rqpair->qpair; 1347 1348 cb_fn(&rqpair->qpair); 1349 1350 return 0; 1351 } 1352 1353 static int 1354 spdk_nvmf_rdma_mem_notify(void *cb_ctx, struct spdk_mem_map *map, 1355 enum spdk_mem_map_notify_action action, 1356 void *vaddr, size_t size) 1357 { 1358 struct ibv_pd *pd = cb_ctx; 1359 struct ibv_mr *mr; 1360 int rc; 1361 1362 switch (action) { 1363 case SPDK_MEM_MAP_NOTIFY_REGISTER: 1364 if (!g_nvmf_hooks.get_rkey) { 1365 mr = ibv_reg_mr(pd, vaddr, size, 1366 IBV_ACCESS_LOCAL_WRITE | 1367 IBV_ACCESS_REMOTE_READ | 1368 IBV_ACCESS_REMOTE_WRITE); 1369 if (mr == NULL) { 1370 SPDK_ERRLOG("ibv_reg_mr() failed\n"); 1371 return -1; 1372 } else { 1373 rc = spdk_mem_map_set_translation(map, (uint64_t)vaddr, size, (uint64_t)mr); 1374 } 1375 } else { 1376 rc = spdk_mem_map_set_translation(map, (uint64_t)vaddr, size, 1377 g_nvmf_hooks.get_rkey(pd, vaddr, size)); 1378 } 1379 break; 1380 case SPDK_MEM_MAP_NOTIFY_UNREGISTER: 1381 if (!g_nvmf_hooks.get_rkey) { 1382 mr = (struct ibv_mr *)spdk_mem_map_translate(map, (uint64_t)vaddr, NULL); 1383 if (mr) { 1384 ibv_dereg_mr(mr); 1385 } 1386 } 1387 rc = spdk_mem_map_clear_translation(map, (uint64_t)vaddr, size); 1388 break; 1389 default: 1390 SPDK_UNREACHABLE(); 1391 } 1392 1393 return rc; 1394 } 1395 1396 static int 1397 spdk_nvmf_rdma_check_contiguous_entries(uint64_t addr_1, uint64_t addr_2) 1398 { 1399 /* Two contiguous mappings will point to the same address which is the start of the RDMA MR. */ 1400 return addr_1 == addr_2; 1401 } 1402 1403 typedef enum spdk_nvme_data_transfer spdk_nvme_data_transfer_t; 1404 1405 static spdk_nvme_data_transfer_t 1406 spdk_nvmf_rdma_request_get_xfer(struct spdk_nvmf_rdma_request *rdma_req) 1407 { 1408 enum spdk_nvme_data_transfer xfer; 1409 struct spdk_nvme_cmd *cmd = &rdma_req->req.cmd->nvme_cmd; 1410 struct spdk_nvme_sgl_descriptor *sgl = &cmd->dptr.sgl1; 1411 1412 #ifdef SPDK_CONFIG_RDMA_SEND_WITH_INVAL 1413 rdma_req->rsp.wr.opcode = IBV_WR_SEND; 1414 rdma_req->rsp.wr.imm_data = 0; 1415 #endif 1416 1417 /* Figure out data transfer direction */ 1418 if (cmd->opc == SPDK_NVME_OPC_FABRIC) { 1419 xfer = spdk_nvme_opc_get_data_transfer(rdma_req->req.cmd->nvmf_cmd.fctype); 1420 } else { 1421 xfer = spdk_nvme_opc_get_data_transfer(cmd->opc); 1422 1423 /* Some admin commands are special cases */ 1424 if ((rdma_req->req.qpair->qid == 0) && 1425 ((cmd->opc == SPDK_NVME_OPC_GET_FEATURES) || 1426 (cmd->opc == SPDK_NVME_OPC_SET_FEATURES))) { 1427 switch (cmd->cdw10 & 0xff) { 1428 case SPDK_NVME_FEAT_LBA_RANGE_TYPE: 1429 case SPDK_NVME_FEAT_AUTONOMOUS_POWER_STATE_TRANSITION: 1430 case SPDK_NVME_FEAT_HOST_IDENTIFIER: 1431 break; 1432 default: 1433 xfer = SPDK_NVME_DATA_NONE; 1434 } 1435 } 1436 } 1437 1438 if (xfer == SPDK_NVME_DATA_NONE) { 1439 return xfer; 1440 } 1441 1442 /* Even for commands that may transfer data, they could have specified 0 length. 1443 * We want those to show up with xfer SPDK_NVME_DATA_NONE. 1444 */ 1445 switch (sgl->generic.type) { 1446 case SPDK_NVME_SGL_TYPE_DATA_BLOCK: 1447 case SPDK_NVME_SGL_TYPE_BIT_BUCKET: 1448 case SPDK_NVME_SGL_TYPE_SEGMENT: 1449 case SPDK_NVME_SGL_TYPE_LAST_SEGMENT: 1450 case SPDK_NVME_SGL_TYPE_TRANSPORT_DATA_BLOCK: 1451 if (sgl->unkeyed.length == 0) { 1452 xfer = SPDK_NVME_DATA_NONE; 1453 } 1454 break; 1455 case SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK: 1456 if (sgl->keyed.length == 0) { 1457 xfer = SPDK_NVME_DATA_NONE; 1458 } 1459 break; 1460 } 1461 1462 return xfer; 1463 } 1464 1465 static int 1466 nvmf_request_alloc_wrs(struct spdk_nvmf_rdma_transport *rtransport, 1467 struct spdk_nvmf_rdma_request *rdma_req, 1468 uint32_t num_sgl_descriptors) 1469 { 1470 struct spdk_nvmf_rdma_request_data *work_requests[SPDK_NVMF_MAX_SGL_ENTRIES]; 1471 struct spdk_nvmf_rdma_request_data *current_data_wr; 1472 uint32_t i; 1473 1474 if (spdk_mempool_get_bulk(rtransport->data_wr_pool, (void **)work_requests, num_sgl_descriptors)) { 1475 return -ENOMEM; 1476 } 1477 1478 current_data_wr = &rdma_req->data; 1479 1480 for (i = 0; i < num_sgl_descriptors; i++) { 1481 if (rdma_req->req.xfer == SPDK_NVME_DATA_CONTROLLER_TO_HOST) { 1482 current_data_wr->wr.opcode = IBV_WR_RDMA_WRITE; 1483 current_data_wr->wr.send_flags = 0; 1484 } else if (rdma_req->req.xfer == SPDK_NVME_DATA_HOST_TO_CONTROLLER) { 1485 current_data_wr->wr.opcode = IBV_WR_RDMA_READ; 1486 current_data_wr->wr.send_flags = IBV_SEND_SIGNALED; 1487 } else { 1488 assert(false); 1489 } 1490 work_requests[i]->wr.sg_list = work_requests[i]->sgl; 1491 work_requests[i]->wr.wr_id = rdma_req->data.wr.wr_id; 1492 current_data_wr->wr.next = &work_requests[i]->wr; 1493 current_data_wr = work_requests[i]; 1494 } 1495 1496 if (rdma_req->req.xfer == SPDK_NVME_DATA_CONTROLLER_TO_HOST) { 1497 current_data_wr->wr.opcode = IBV_WR_RDMA_WRITE; 1498 current_data_wr->wr.next = &rdma_req->rsp.wr; 1499 current_data_wr->wr.send_flags = 0; 1500 } else if (rdma_req->req.xfer == SPDK_NVME_DATA_HOST_TO_CONTROLLER) { 1501 current_data_wr->wr.opcode = IBV_WR_RDMA_READ; 1502 current_data_wr->wr.next = NULL; 1503 current_data_wr->wr.send_flags = IBV_SEND_SIGNALED; 1504 } 1505 return 0; 1506 } 1507 1508 /* This function is used in the rare case that we have a buffer split over multiple memory regions. */ 1509 static int 1510 nvmf_rdma_replace_buffer(struct spdk_nvmf_rdma_poll_group *rgroup, void **buf) 1511 { 1512 struct spdk_nvmf_transport_poll_group *group = &rgroup->group; 1513 struct spdk_nvmf_transport *transport = group->transport; 1514 struct spdk_nvmf_transport_pg_cache_buf *old_buf; 1515 void *new_buf; 1516 1517 if (!(STAILQ_EMPTY(&group->buf_cache))) { 1518 group->buf_cache_count--; 1519 new_buf = STAILQ_FIRST(&group->buf_cache); 1520 STAILQ_REMOVE_HEAD(&group->buf_cache, link); 1521 assert(*buf != NULL); 1522 } else { 1523 new_buf = spdk_mempool_get(transport->data_buf_pool); 1524 } 1525 1526 if (*buf == NULL) { 1527 return -ENOMEM; 1528 } 1529 1530 old_buf = *buf; 1531 STAILQ_INSERT_HEAD(&rgroup->retired_bufs, old_buf, link); 1532 *buf = new_buf; 1533 return 0; 1534 } 1535 1536 static bool 1537 nvmf_rdma_get_lkey(struct spdk_nvmf_rdma_device *device, struct iovec *iov, 1538 uint32_t *_lkey) 1539 { 1540 uint64_t translation_len; 1541 uint32_t lkey; 1542 1543 translation_len = iov->iov_len; 1544 1545 if (!g_nvmf_hooks.get_rkey) { 1546 lkey = ((struct ibv_mr *)spdk_mem_map_translate(device->map, 1547 (uint64_t)iov->iov_base, &translation_len))->lkey; 1548 } else { 1549 lkey = spdk_mem_map_translate(device->map, 1550 (uint64_t)iov->iov_base, &translation_len); 1551 } 1552 1553 if (spdk_unlikely(translation_len < iov->iov_len)) { 1554 return false; 1555 } 1556 1557 *_lkey = lkey; 1558 return true; 1559 } 1560 1561 static bool 1562 nvmf_rdma_fill_wr_sge(struct spdk_nvmf_rdma_device *device, 1563 struct iovec *iov, struct ibv_send_wr *wr, 1564 uint32_t *_remaining_data_block, uint32_t *_offset, 1565 const struct spdk_dif_ctx *dif_ctx) 1566 { 1567 struct ibv_sge *sg_ele = &wr->sg_list[wr->num_sge]; 1568 uint32_t lkey = 0; 1569 uint32_t remaining, data_block_size, md_size, sge_len; 1570 1571 if (spdk_unlikely(!nvmf_rdma_get_lkey(device, iov, &lkey))) { 1572 /* This is a very rare case that can occur when using DPDK version < 19.05 */ 1573 SPDK_ERRLOG("Data buffer split over multiple RDMA Memory Regions. Removing it from circulation.\n"); 1574 return false; 1575 } 1576 1577 if (spdk_likely(!dif_ctx)) { 1578 sg_ele->lkey = lkey; 1579 sg_ele->addr = (uintptr_t)(iov->iov_base); 1580 sg_ele->length = iov->iov_len; 1581 wr->num_sge++; 1582 } else { 1583 remaining = iov->iov_len - *_offset; 1584 data_block_size = dif_ctx->block_size - dif_ctx->md_size; 1585 md_size = dif_ctx->md_size; 1586 1587 while (remaining && wr->num_sge < SPDK_NVMF_MAX_SGL_ENTRIES) { 1588 sg_ele->lkey = lkey; 1589 sg_ele->addr = (uintptr_t)((char *)iov->iov_base + *_offset); 1590 sge_len = spdk_min(remaining, *_remaining_data_block); 1591 sg_ele->length = sge_len; 1592 remaining -= sge_len; 1593 *_remaining_data_block -= sge_len; 1594 *_offset += sge_len; 1595 1596 sg_ele++; 1597 wr->num_sge++; 1598 1599 if (*_remaining_data_block == 0) { 1600 /* skip metadata */ 1601 *_offset += md_size; 1602 /* Metadata that do not fit this IO buffer will be included in the next IO buffer */ 1603 remaining -= spdk_min(remaining, md_size); 1604 *_remaining_data_block = data_block_size; 1605 } 1606 1607 if (remaining == 0) { 1608 /* By subtracting the size of the last IOV from the offset, we ensure that we skip 1609 the remaining metadata bits at the beginning of the next buffer */ 1610 *_offset -= iov->iov_len; 1611 } 1612 } 1613 } 1614 1615 return true; 1616 } 1617 1618 static int 1619 nvmf_rdma_fill_wr_sgl(struct spdk_nvmf_rdma_poll_group *rgroup, 1620 struct spdk_nvmf_rdma_device *device, 1621 struct spdk_nvmf_rdma_request *rdma_req, 1622 struct ibv_send_wr *wr, 1623 uint32_t length) 1624 { 1625 struct spdk_nvmf_request *req = &rdma_req->req; 1626 struct spdk_dif_ctx *dif_ctx = NULL; 1627 uint32_t remaining_data_block = 0; 1628 uint32_t offset = 0; 1629 1630 if (spdk_unlikely(rdma_req->dif_insert_or_strip)) { 1631 dif_ctx = &rdma_req->dif_ctx; 1632 remaining_data_block = dif_ctx->block_size - dif_ctx->md_size; 1633 } 1634 1635 wr->num_sge = 0; 1636 1637 while (length && wr->num_sge < SPDK_NVMF_MAX_SGL_ENTRIES) { 1638 while (spdk_unlikely(!nvmf_rdma_fill_wr_sge(device, &req->iov[rdma_req->iovpos], wr, 1639 &remaining_data_block, &offset, dif_ctx))) { 1640 if (nvmf_rdma_replace_buffer(rgroup, &req->buffers[rdma_req->iovpos]) == -ENOMEM) { 1641 return -ENOMEM; 1642 } 1643 req->iov[rdma_req->iovpos].iov_base = (void *)((uintptr_t)(req->buffers[rdma_req->iovpos] + 1644 NVMF_DATA_BUFFER_MASK) & 1645 ~NVMF_DATA_BUFFER_MASK); 1646 } 1647 1648 length -= req->iov[rdma_req->iovpos].iov_len; 1649 rdma_req->iovpos++; 1650 } 1651 1652 if (length) { 1653 SPDK_ERRLOG("Not enough SG entries to hold data buffer\n"); 1654 return -EINVAL; 1655 } 1656 1657 return 0; 1658 } 1659 1660 static int 1661 spdk_nvmf_rdma_request_fill_iovs(struct spdk_nvmf_rdma_transport *rtransport, 1662 struct spdk_nvmf_rdma_device *device, 1663 struct spdk_nvmf_rdma_request *rdma_req, 1664 uint32_t length) 1665 { 1666 struct spdk_nvmf_rdma_qpair *rqpair; 1667 struct spdk_nvmf_rdma_poll_group *rgroup; 1668 struct spdk_nvmf_request *req = &rdma_req->req; 1669 struct ibv_send_wr *wr = &rdma_req->data.wr; 1670 int rc; 1671 1672 rqpair = SPDK_CONTAINEROF(req->qpair, struct spdk_nvmf_rdma_qpair, qpair); 1673 rgroup = rqpair->poller->group; 1674 1675 rc = spdk_nvmf_request_get_buffers(req, &rgroup->group, &rtransport->transport, 1676 length); 1677 if (rc != 0) { 1678 return rc; 1679 } 1680 1681 assert(req->iovcnt <= rqpair->max_send_sge); 1682 1683 rdma_req->iovpos = 0; 1684 1685 rc = nvmf_rdma_fill_wr_sgl(rgroup, device, rdma_req, wr, length); 1686 if (rc != 0) { 1687 goto err_exit; 1688 } 1689 1690 return rc; 1691 1692 err_exit: 1693 spdk_nvmf_request_free_buffers(req, &rgroup->group, &rtransport->transport); 1694 memset(wr->sg_list, 0, sizeof(wr->sg_list[0]) * wr->num_sge); 1695 wr->num_sge = 0; 1696 req->iovcnt = 0; 1697 return rc; 1698 } 1699 1700 static int 1701 nvmf_rdma_request_fill_iovs_multi_sgl(struct spdk_nvmf_rdma_transport *rtransport, 1702 struct spdk_nvmf_rdma_device *device, 1703 struct spdk_nvmf_rdma_request *rdma_req) 1704 { 1705 struct spdk_nvmf_rdma_qpair *rqpair; 1706 struct spdk_nvmf_rdma_poll_group *rgroup; 1707 struct ibv_send_wr *current_wr; 1708 struct spdk_nvmf_request *req = &rdma_req->req; 1709 struct spdk_nvme_sgl_descriptor *inline_segment, *desc; 1710 uint32_t num_sgl_descriptors; 1711 uint32_t lengths[SPDK_NVMF_MAX_SGL_ENTRIES]; 1712 uint32_t i; 1713 int rc; 1714 1715 rqpair = SPDK_CONTAINEROF(rdma_req->req.qpair, struct spdk_nvmf_rdma_qpair, qpair); 1716 rgroup = rqpair->poller->group; 1717 1718 inline_segment = &req->cmd->nvme_cmd.dptr.sgl1; 1719 assert(inline_segment->generic.type == SPDK_NVME_SGL_TYPE_LAST_SEGMENT); 1720 assert(inline_segment->unkeyed.subtype == SPDK_NVME_SGL_SUBTYPE_OFFSET); 1721 1722 num_sgl_descriptors = inline_segment->unkeyed.length / sizeof(struct spdk_nvme_sgl_descriptor); 1723 assert(num_sgl_descriptors <= SPDK_NVMF_MAX_SGL_ENTRIES); 1724 1725 if (nvmf_request_alloc_wrs(rtransport, rdma_req, num_sgl_descriptors - 1) != 0) { 1726 return -ENOMEM; 1727 } 1728 1729 desc = (struct spdk_nvme_sgl_descriptor *)rdma_req->recv->buf + inline_segment->address; 1730 for (i = 0; i < num_sgl_descriptors; i++) { 1731 lengths[i] = desc->keyed.length; 1732 desc++; 1733 } 1734 1735 rc = spdk_nvmf_request_get_buffers_multi(req, &rgroup->group, &rtransport->transport, 1736 lengths, num_sgl_descriptors); 1737 if (rc != 0) { 1738 nvmf_rdma_request_free_data(rdma_req, rtransport); 1739 return rc; 1740 } 1741 1742 /* The first WR must always be the embedded data WR. This is how we unwind them later. */ 1743 current_wr = &rdma_req->data.wr; 1744 assert(current_wr != NULL); 1745 1746 req->length = 0; 1747 rdma_req->iovpos = 0; 1748 desc = (struct spdk_nvme_sgl_descriptor *)rdma_req->recv->buf + inline_segment->address; 1749 for (i = 0; i < num_sgl_descriptors; i++) { 1750 /* The descriptors must be keyed data block descriptors with an address, not an offset. */ 1751 if (spdk_unlikely(desc->generic.type != SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK || 1752 desc->keyed.subtype != SPDK_NVME_SGL_SUBTYPE_ADDRESS)) { 1753 rc = -EINVAL; 1754 goto err_exit; 1755 } 1756 1757 current_wr->num_sge = 0; 1758 1759 rc = nvmf_rdma_fill_wr_sgl(rgroup, device, rdma_req, current_wr, desc->keyed.length); 1760 if (rc != 0) { 1761 rc = -ENOMEM; 1762 goto err_exit; 1763 } 1764 1765 req->length += desc->keyed.length; 1766 current_wr->wr.rdma.rkey = desc->keyed.key; 1767 current_wr->wr.rdma.remote_addr = desc->address; 1768 current_wr = current_wr->next; 1769 desc++; 1770 } 1771 1772 #ifdef SPDK_CONFIG_RDMA_SEND_WITH_INVAL 1773 /* Go back to the last descriptor in the list. */ 1774 desc--; 1775 if ((device->attr.device_cap_flags & IBV_DEVICE_MEM_MGT_EXTENSIONS) != 0) { 1776 if (desc->keyed.subtype == SPDK_NVME_SGL_SUBTYPE_INVALIDATE_KEY) { 1777 rdma_req->rsp.wr.opcode = IBV_WR_SEND_WITH_INV; 1778 rdma_req->rsp.wr.imm_data = desc->keyed.key; 1779 } 1780 } 1781 #endif 1782 1783 rdma_req->num_outstanding_data_wr = num_sgl_descriptors; 1784 1785 return 0; 1786 1787 err_exit: 1788 spdk_nvmf_request_free_buffers(req, &rgroup->group, &rtransport->transport); 1789 nvmf_rdma_request_free_data(rdma_req, rtransport); 1790 return rc; 1791 } 1792 1793 static int 1794 spdk_nvmf_rdma_request_parse_sgl(struct spdk_nvmf_rdma_transport *rtransport, 1795 struct spdk_nvmf_rdma_device *device, 1796 struct spdk_nvmf_rdma_request *rdma_req) 1797 { 1798 struct spdk_nvmf_request *req = &rdma_req->req; 1799 struct spdk_nvme_cmd *cmd; 1800 struct spdk_nvme_cpl *rsp; 1801 struct spdk_nvme_sgl_descriptor *sgl; 1802 struct ibv_send_wr *wr; 1803 int rc; 1804 uint32_t length; 1805 1806 cmd = &req->cmd->nvme_cmd; 1807 rsp = &req->rsp->nvme_cpl; 1808 sgl = &cmd->dptr.sgl1; 1809 1810 if (sgl->generic.type == SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK && 1811 (sgl->keyed.subtype == SPDK_NVME_SGL_SUBTYPE_ADDRESS || 1812 sgl->keyed.subtype == SPDK_NVME_SGL_SUBTYPE_INVALIDATE_KEY)) { 1813 1814 length = sgl->keyed.length; 1815 if (length > rtransport->transport.opts.max_io_size) { 1816 SPDK_ERRLOG("SGL length 0x%x exceeds max io size 0x%x\n", 1817 length, rtransport->transport.opts.max_io_size); 1818 rsp->status.sc = SPDK_NVME_SC_DATA_SGL_LENGTH_INVALID; 1819 return -1; 1820 } 1821 #ifdef SPDK_CONFIG_RDMA_SEND_WITH_INVAL 1822 if ((device->attr.device_cap_flags & IBV_DEVICE_MEM_MGT_EXTENSIONS) != 0) { 1823 if (sgl->keyed.subtype == SPDK_NVME_SGL_SUBTYPE_INVALIDATE_KEY) { 1824 rdma_req->rsp.wr.opcode = IBV_WR_SEND_WITH_INV; 1825 rdma_req->rsp.wr.imm_data = sgl->keyed.key; 1826 } 1827 } 1828 #endif 1829 1830 /* fill request length and populate iovs */ 1831 req->length = length; 1832 1833 if (spdk_unlikely(rdma_req->dif_insert_or_strip)) { 1834 rdma_req->orig_length = length; 1835 length = spdk_dif_get_length_with_md(length, &rdma_req->dif_ctx); 1836 rdma_req->elba_length = length; 1837 } 1838 1839 if (spdk_nvmf_rdma_request_fill_iovs(rtransport, device, rdma_req, length) < 0) { 1840 /* No available buffers. Queue this request up. */ 1841 SPDK_DEBUGLOG(SPDK_LOG_RDMA, "No available large data buffers. Queueing request %p\n", rdma_req); 1842 return 0; 1843 } 1844 1845 /* backward compatible */ 1846 req->data = req->iov[0].iov_base; 1847 1848 /* rdma wr specifics */ 1849 wr = &rdma_req->data.wr; 1850 wr->wr.rdma.rkey = sgl->keyed.key; 1851 wr->wr.rdma.remote_addr = sgl->address; 1852 if (req->xfer == SPDK_NVME_DATA_CONTROLLER_TO_HOST) { 1853 wr->opcode = IBV_WR_RDMA_WRITE; 1854 wr->next = &rdma_req->rsp.wr; 1855 wr->send_flags &= ~IBV_SEND_SIGNALED; 1856 } else if (req->xfer == SPDK_NVME_DATA_HOST_TO_CONTROLLER) { 1857 wr->opcode = IBV_WR_RDMA_READ; 1858 wr->next = NULL; 1859 wr->send_flags |= IBV_SEND_SIGNALED; 1860 } 1861 1862 /* set the number of outstanding data WRs for this request. */ 1863 rdma_req->num_outstanding_data_wr = 1; 1864 1865 SPDK_DEBUGLOG(SPDK_LOG_RDMA, "Request %p took %d buffer/s from central pool\n", rdma_req, 1866 req->iovcnt); 1867 1868 return 0; 1869 } else if (sgl->generic.type == SPDK_NVME_SGL_TYPE_DATA_BLOCK && 1870 sgl->unkeyed.subtype == SPDK_NVME_SGL_SUBTYPE_OFFSET) { 1871 uint64_t offset = sgl->address; 1872 uint32_t max_len = rtransport->transport.opts.in_capsule_data_size; 1873 1874 SPDK_DEBUGLOG(SPDK_LOG_NVMF, "In-capsule data: offset 0x%" PRIx64 ", length 0x%x\n", 1875 offset, sgl->unkeyed.length); 1876 1877 if (offset > max_len) { 1878 SPDK_ERRLOG("In-capsule offset 0x%" PRIx64 " exceeds capsule length 0x%x\n", 1879 offset, max_len); 1880 rsp->status.sc = SPDK_NVME_SC_INVALID_SGL_OFFSET; 1881 return -1; 1882 } 1883 max_len -= (uint32_t)offset; 1884 1885 if (sgl->unkeyed.length > max_len) { 1886 SPDK_ERRLOG("In-capsule data length 0x%x exceeds capsule length 0x%x\n", 1887 sgl->unkeyed.length, max_len); 1888 rsp->status.sc = SPDK_NVME_SC_DATA_SGL_LENGTH_INVALID; 1889 return -1; 1890 } 1891 1892 rdma_req->num_outstanding_data_wr = 0; 1893 req->data = rdma_req->recv->buf + offset; 1894 req->data_from_pool = false; 1895 req->length = sgl->unkeyed.length; 1896 1897 req->iov[0].iov_base = req->data; 1898 req->iov[0].iov_len = req->length; 1899 req->iovcnt = 1; 1900 1901 return 0; 1902 } else if (sgl->generic.type == SPDK_NVME_SGL_TYPE_LAST_SEGMENT && 1903 sgl->unkeyed.subtype == SPDK_NVME_SGL_SUBTYPE_OFFSET) { 1904 1905 rc = nvmf_rdma_request_fill_iovs_multi_sgl(rtransport, device, rdma_req); 1906 if (rc == -ENOMEM) { 1907 SPDK_DEBUGLOG(SPDK_LOG_RDMA, "No available large data buffers. Queueing request %p\n", rdma_req); 1908 return 0; 1909 } else if (rc == -EINVAL) { 1910 SPDK_ERRLOG("Multi SGL element request length exceeds the max I/O size\n"); 1911 return -1; 1912 } 1913 1914 /* backward compatible */ 1915 req->data = req->iov[0].iov_base; 1916 1917 SPDK_DEBUGLOG(SPDK_LOG_RDMA, "Request %p took %d buffer/s from central pool\n", rdma_req, 1918 req->iovcnt); 1919 1920 return 0; 1921 } 1922 1923 SPDK_ERRLOG("Invalid NVMf I/O Command SGL: Type 0x%x, Subtype 0x%x\n", 1924 sgl->generic.type, sgl->generic.subtype); 1925 rsp->status.sc = SPDK_NVME_SC_SGL_DESCRIPTOR_TYPE_INVALID; 1926 return -1; 1927 } 1928 1929 static void 1930 nvmf_rdma_request_free(struct spdk_nvmf_rdma_request *rdma_req, 1931 struct spdk_nvmf_rdma_transport *rtransport) 1932 { 1933 struct spdk_nvmf_rdma_qpair *rqpair; 1934 struct spdk_nvmf_rdma_poll_group *rgroup; 1935 1936 rqpair = SPDK_CONTAINEROF(rdma_req->req.qpair, struct spdk_nvmf_rdma_qpair, qpair); 1937 if (rdma_req->req.data_from_pool) { 1938 rgroup = rqpair->poller->group; 1939 1940 spdk_nvmf_request_free_buffers(&rdma_req->req, &rgroup->group, &rtransport->transport); 1941 } 1942 nvmf_rdma_request_free_data(rdma_req, rtransport); 1943 rdma_req->req.length = 0; 1944 rdma_req->req.iovcnt = 0; 1945 rdma_req->req.data = NULL; 1946 rdma_req->rsp.wr.next = NULL; 1947 rdma_req->data.wr.next = NULL; 1948 rdma_req->dif_insert_or_strip = false; 1949 rdma_req->elba_length = 0; 1950 rdma_req->orig_length = 0; 1951 memset(&rdma_req->dif_ctx, 0, sizeof(rdma_req->dif_ctx)); 1952 rqpair->qd--; 1953 1954 STAILQ_INSERT_HEAD(&rqpair->resources->free_queue, rdma_req, state_link); 1955 rdma_req->state = RDMA_REQUEST_STATE_FREE; 1956 } 1957 1958 static bool 1959 spdk_nvmf_rdma_request_process(struct spdk_nvmf_rdma_transport *rtransport, 1960 struct spdk_nvmf_rdma_request *rdma_req) 1961 { 1962 struct spdk_nvmf_rdma_qpair *rqpair; 1963 struct spdk_nvmf_rdma_device *device; 1964 struct spdk_nvmf_rdma_poll_group *rgroup; 1965 struct spdk_nvme_cpl *rsp = &rdma_req->req.rsp->nvme_cpl; 1966 int rc; 1967 struct spdk_nvmf_rdma_recv *rdma_recv; 1968 enum spdk_nvmf_rdma_request_state prev_state; 1969 bool progress = false; 1970 int data_posted; 1971 uint32_t num_blocks; 1972 1973 rqpair = SPDK_CONTAINEROF(rdma_req->req.qpair, struct spdk_nvmf_rdma_qpair, qpair); 1974 device = rqpair->port->device; 1975 rgroup = rqpair->poller->group; 1976 1977 assert(rdma_req->state != RDMA_REQUEST_STATE_FREE); 1978 1979 /* If the queue pair is in an error state, force the request to the completed state 1980 * to release resources. */ 1981 if (rqpair->ibv_state == IBV_QPS_ERR || rqpair->qpair.state != SPDK_NVMF_QPAIR_ACTIVE) { 1982 if (rdma_req->state == RDMA_REQUEST_STATE_NEED_BUFFER) { 1983 STAILQ_REMOVE(&rgroup->group.pending_buf_queue, &rdma_req->req, spdk_nvmf_request, buf_link); 1984 } else if (rdma_req->state == RDMA_REQUEST_STATE_DATA_TRANSFER_TO_CONTROLLER_PENDING) { 1985 STAILQ_REMOVE(&rqpair->pending_rdma_read_queue, rdma_req, spdk_nvmf_rdma_request, state_link); 1986 } else if (rdma_req->state == RDMA_REQUEST_STATE_DATA_TRANSFER_TO_HOST_PENDING) { 1987 STAILQ_REMOVE(&rqpair->pending_rdma_write_queue, rdma_req, spdk_nvmf_rdma_request, state_link); 1988 } 1989 rdma_req->state = RDMA_REQUEST_STATE_COMPLETED; 1990 } 1991 1992 /* The loop here is to allow for several back-to-back state changes. */ 1993 do { 1994 prev_state = rdma_req->state; 1995 1996 SPDK_DEBUGLOG(SPDK_LOG_RDMA, "Request %p entering state %d\n", rdma_req, prev_state); 1997 1998 switch (rdma_req->state) { 1999 case RDMA_REQUEST_STATE_FREE: 2000 /* Some external code must kick a request into RDMA_REQUEST_STATE_NEW 2001 * to escape this state. */ 2002 break; 2003 case RDMA_REQUEST_STATE_NEW: 2004 spdk_trace_record(TRACE_RDMA_REQUEST_STATE_NEW, 0, 0, 2005 (uintptr_t)rdma_req, (uintptr_t)rqpair->cm_id); 2006 rdma_recv = rdma_req->recv; 2007 2008 /* The first element of the SGL is the NVMe command */ 2009 rdma_req->req.cmd = (union nvmf_h2c_msg *)rdma_recv->sgl[0].addr; 2010 memset(rdma_req->req.rsp, 0, sizeof(*rdma_req->req.rsp)); 2011 2012 if (rqpair->ibv_state == IBV_QPS_ERR || rqpair->qpair.state != SPDK_NVMF_QPAIR_ACTIVE) { 2013 rdma_req->state = RDMA_REQUEST_STATE_COMPLETED; 2014 break; 2015 } 2016 2017 if (spdk_unlikely(spdk_nvmf_request_get_dif_ctx(&rdma_req->req, &rdma_req->dif_ctx))) { 2018 rdma_req->dif_insert_or_strip = true; 2019 } 2020 2021 /* The next state transition depends on the data transfer needs of this request. */ 2022 rdma_req->req.xfer = spdk_nvmf_rdma_request_get_xfer(rdma_req); 2023 2024 /* If no data to transfer, ready to execute. */ 2025 if (rdma_req->req.xfer == SPDK_NVME_DATA_NONE) { 2026 rdma_req->state = RDMA_REQUEST_STATE_READY_TO_EXECUTE; 2027 break; 2028 } 2029 2030 rdma_req->state = RDMA_REQUEST_STATE_NEED_BUFFER; 2031 STAILQ_INSERT_TAIL(&rgroup->group.pending_buf_queue, &rdma_req->req, buf_link); 2032 break; 2033 case RDMA_REQUEST_STATE_NEED_BUFFER: 2034 spdk_trace_record(TRACE_RDMA_REQUEST_STATE_NEED_BUFFER, 0, 0, 2035 (uintptr_t)rdma_req, (uintptr_t)rqpair->cm_id); 2036 2037 assert(rdma_req->req.xfer != SPDK_NVME_DATA_NONE); 2038 2039 if (&rdma_req->req != STAILQ_FIRST(&rgroup->group.pending_buf_queue)) { 2040 /* This request needs to wait in line to obtain a buffer */ 2041 break; 2042 } 2043 2044 /* Try to get a data buffer */ 2045 rc = spdk_nvmf_rdma_request_parse_sgl(rtransport, device, rdma_req); 2046 if (rc < 0) { 2047 STAILQ_REMOVE_HEAD(&rgroup->group.pending_buf_queue, buf_link); 2048 rsp->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; 2049 rdma_req->state = RDMA_REQUEST_STATE_READY_TO_COMPLETE; 2050 break; 2051 } 2052 2053 if (!rdma_req->req.data) { 2054 /* No buffers available. */ 2055 rgroup->stat.pending_data_buffer++; 2056 break; 2057 } 2058 2059 STAILQ_REMOVE_HEAD(&rgroup->group.pending_buf_queue, buf_link); 2060 2061 /* If data is transferring from host to controller and the data didn't 2062 * arrive using in capsule data, we need to do a transfer from the host. 2063 */ 2064 if (rdma_req->req.xfer == SPDK_NVME_DATA_HOST_TO_CONTROLLER && 2065 rdma_req->req.data_from_pool) { 2066 STAILQ_INSERT_TAIL(&rqpair->pending_rdma_read_queue, rdma_req, state_link); 2067 rdma_req->state = RDMA_REQUEST_STATE_DATA_TRANSFER_TO_CONTROLLER_PENDING; 2068 break; 2069 } 2070 2071 rdma_req->state = RDMA_REQUEST_STATE_READY_TO_EXECUTE; 2072 break; 2073 case RDMA_REQUEST_STATE_DATA_TRANSFER_TO_CONTROLLER_PENDING: 2074 spdk_trace_record(TRACE_RDMA_REQUEST_STATE_DATA_TRANSFER_TO_CONTROLLER_PENDING, 0, 0, 2075 (uintptr_t)rdma_req, (uintptr_t)rqpair->cm_id); 2076 2077 if (rdma_req != STAILQ_FIRST(&rqpair->pending_rdma_read_queue)) { 2078 /* This request needs to wait in line to perform RDMA */ 2079 break; 2080 } 2081 if (rqpair->current_send_depth + rdma_req->num_outstanding_data_wr > rqpair->max_send_depth 2082 || rqpair->current_read_depth + rdma_req->num_outstanding_data_wr > rqpair->max_read_depth) { 2083 /* We can only have so many WRs outstanding. we have to wait until some finish. */ 2084 rqpair->poller->stat.pending_rdma_read++; 2085 break; 2086 } 2087 2088 /* We have already verified that this request is the head of the queue. */ 2089 STAILQ_REMOVE_HEAD(&rqpair->pending_rdma_read_queue, state_link); 2090 2091 rc = request_transfer_in(&rdma_req->req); 2092 if (!rc) { 2093 rdma_req->state = RDMA_REQUEST_STATE_TRANSFERRING_HOST_TO_CONTROLLER; 2094 } else { 2095 rsp->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; 2096 rdma_req->state = RDMA_REQUEST_STATE_READY_TO_COMPLETE; 2097 } 2098 break; 2099 case RDMA_REQUEST_STATE_TRANSFERRING_HOST_TO_CONTROLLER: 2100 spdk_trace_record(TRACE_RDMA_REQUEST_STATE_TRANSFERRING_HOST_TO_CONTROLLER, 0, 0, 2101 (uintptr_t)rdma_req, (uintptr_t)rqpair->cm_id); 2102 /* Some external code must kick a request into RDMA_REQUEST_STATE_READY_TO_EXECUTE 2103 * to escape this state. */ 2104 break; 2105 case RDMA_REQUEST_STATE_READY_TO_EXECUTE: 2106 spdk_trace_record(TRACE_RDMA_REQUEST_STATE_READY_TO_EXECUTE, 0, 0, 2107 (uintptr_t)rdma_req, (uintptr_t)rqpair->cm_id); 2108 2109 if (spdk_unlikely(rdma_req->dif_insert_or_strip)) { 2110 if (rdma_req->req.xfer == SPDK_NVME_DATA_HOST_TO_CONTROLLER) { 2111 /* generate DIF for write operation */ 2112 num_blocks = SPDK_CEIL_DIV(rdma_req->elba_length, rdma_req->dif_ctx.block_size); 2113 assert(num_blocks > 0); 2114 2115 rc = spdk_dif_generate(rdma_req->req.iov, rdma_req->req.iovcnt, 2116 num_blocks, &rdma_req->dif_ctx); 2117 if (rc != 0) { 2118 SPDK_ERRLOG("DIF generation failed\n"); 2119 rdma_req->state = RDMA_REQUEST_STATE_COMPLETED; 2120 spdk_nvmf_rdma_start_disconnect(rqpair); 2121 break; 2122 } 2123 } 2124 2125 assert(rdma_req->elba_length >= rdma_req->req.length); 2126 /* set extended length before IO operation */ 2127 rdma_req->req.length = rdma_req->elba_length; 2128 } 2129 2130 rdma_req->state = RDMA_REQUEST_STATE_EXECUTING; 2131 spdk_nvmf_request_exec(&rdma_req->req); 2132 break; 2133 case RDMA_REQUEST_STATE_EXECUTING: 2134 spdk_trace_record(TRACE_RDMA_REQUEST_STATE_EXECUTING, 0, 0, 2135 (uintptr_t)rdma_req, (uintptr_t)rqpair->cm_id); 2136 /* Some external code must kick a request into RDMA_REQUEST_STATE_EXECUTED 2137 * to escape this state. */ 2138 break; 2139 case RDMA_REQUEST_STATE_EXECUTED: 2140 spdk_trace_record(TRACE_RDMA_REQUEST_STATE_EXECUTED, 0, 0, 2141 (uintptr_t)rdma_req, (uintptr_t)rqpair->cm_id); 2142 if (rdma_req->req.xfer == SPDK_NVME_DATA_CONTROLLER_TO_HOST) { 2143 STAILQ_INSERT_TAIL(&rqpair->pending_rdma_write_queue, rdma_req, state_link); 2144 rdma_req->state = RDMA_REQUEST_STATE_DATA_TRANSFER_TO_HOST_PENDING; 2145 } else { 2146 rdma_req->state = RDMA_REQUEST_STATE_READY_TO_COMPLETE; 2147 } 2148 if (spdk_unlikely(rdma_req->dif_insert_or_strip)) { 2149 /* restore the original length */ 2150 rdma_req->req.length = rdma_req->orig_length; 2151 2152 if (rdma_req->req.xfer == SPDK_NVME_DATA_CONTROLLER_TO_HOST) { 2153 struct spdk_dif_error error_blk; 2154 2155 num_blocks = SPDK_CEIL_DIV(rdma_req->elba_length, rdma_req->dif_ctx.block_size); 2156 2157 rc = spdk_dif_verify(rdma_req->req.iov, rdma_req->req.iovcnt, num_blocks, &rdma_req->dif_ctx, 2158 &error_blk); 2159 if (rc) { 2160 struct spdk_nvme_cpl *rsp = &rdma_req->req.rsp->nvme_cpl; 2161 2162 SPDK_ERRLOG("DIF error detected. type=%d, offset=%" PRIu32 "\n", error_blk.err_type, 2163 error_blk.err_offset); 2164 rsp->status.sct = SPDK_NVME_SCT_MEDIA_ERROR; 2165 rsp->status.sc = spdk_nvmf_rdma_dif_error_to_compl_status(error_blk.err_type); 2166 rdma_req->state = RDMA_REQUEST_STATE_READY_TO_COMPLETE; 2167 STAILQ_REMOVE(&rqpair->pending_rdma_write_queue, rdma_req, spdk_nvmf_rdma_request, state_link); 2168 } 2169 } 2170 } 2171 break; 2172 case RDMA_REQUEST_STATE_DATA_TRANSFER_TO_HOST_PENDING: 2173 spdk_trace_record(TRACE_RDMA_REQUEST_STATE_DATA_TRANSFER_TO_HOST_PENDING, 0, 0, 2174 (uintptr_t)rdma_req, (uintptr_t)rqpair->cm_id); 2175 2176 if (rdma_req != STAILQ_FIRST(&rqpair->pending_rdma_write_queue)) { 2177 /* This request needs to wait in line to perform RDMA */ 2178 break; 2179 } 2180 if ((rqpair->current_send_depth + rdma_req->num_outstanding_data_wr + 1) > 2181 rqpair->max_send_depth) { 2182 /* We can only have so many WRs outstanding. we have to wait until some finish. 2183 * +1 since each request has an additional wr in the resp. */ 2184 rqpair->poller->stat.pending_rdma_write++; 2185 break; 2186 } 2187 2188 /* We have already verified that this request is the head of the queue. */ 2189 STAILQ_REMOVE_HEAD(&rqpair->pending_rdma_write_queue, state_link); 2190 2191 /* The data transfer will be kicked off from 2192 * RDMA_REQUEST_STATE_READY_TO_COMPLETE state. 2193 */ 2194 rdma_req->state = RDMA_REQUEST_STATE_READY_TO_COMPLETE; 2195 break; 2196 case RDMA_REQUEST_STATE_READY_TO_COMPLETE: 2197 spdk_trace_record(TRACE_RDMA_REQUEST_STATE_READY_TO_COMPLETE, 0, 0, 2198 (uintptr_t)rdma_req, (uintptr_t)rqpair->cm_id); 2199 rc = request_transfer_out(&rdma_req->req, &data_posted); 2200 assert(rc == 0); /* No good way to handle this currently */ 2201 if (rc) { 2202 rdma_req->state = RDMA_REQUEST_STATE_COMPLETED; 2203 } else { 2204 rdma_req->state = data_posted ? RDMA_REQUEST_STATE_TRANSFERRING_CONTROLLER_TO_HOST : 2205 RDMA_REQUEST_STATE_COMPLETING; 2206 } 2207 break; 2208 case RDMA_REQUEST_STATE_TRANSFERRING_CONTROLLER_TO_HOST: 2209 spdk_trace_record(TRACE_RDMA_REQUEST_STATE_TRANSFERRING_CONTROLLER_TO_HOST, 0, 0, 2210 (uintptr_t)rdma_req, (uintptr_t)rqpair->cm_id); 2211 /* Some external code must kick a request into RDMA_REQUEST_STATE_COMPLETED 2212 * to escape this state. */ 2213 break; 2214 case RDMA_REQUEST_STATE_COMPLETING: 2215 spdk_trace_record(TRACE_RDMA_REQUEST_STATE_COMPLETING, 0, 0, 2216 (uintptr_t)rdma_req, (uintptr_t)rqpair->cm_id); 2217 /* Some external code must kick a request into RDMA_REQUEST_STATE_COMPLETED 2218 * to escape this state. */ 2219 break; 2220 case RDMA_REQUEST_STATE_COMPLETED: 2221 spdk_trace_record(TRACE_RDMA_REQUEST_STATE_COMPLETED, 0, 0, 2222 (uintptr_t)rdma_req, (uintptr_t)rqpair->cm_id); 2223 2224 rqpair->poller->stat.request_latency += spdk_get_ticks() - rdma_req->receive_tsc; 2225 nvmf_rdma_request_free(rdma_req, rtransport); 2226 break; 2227 case RDMA_REQUEST_NUM_STATES: 2228 default: 2229 assert(0); 2230 break; 2231 } 2232 2233 if (rdma_req->state != prev_state) { 2234 progress = true; 2235 } 2236 } while (rdma_req->state != prev_state); 2237 2238 return progress; 2239 } 2240 2241 /* Public API callbacks begin here */ 2242 2243 #define SPDK_NVMF_RDMA_DEFAULT_MAX_QUEUE_DEPTH 128 2244 #define SPDK_NVMF_RDMA_DEFAULT_AQ_DEPTH 128 2245 #define SPDK_NVMF_RDMA_DEFAULT_SRQ_DEPTH 4096 2246 #define SPDK_NVMF_RDMA_DEFAULT_MAX_QPAIRS_PER_CTRLR 128 2247 #define SPDK_NVMF_RDMA_DEFAULT_IN_CAPSULE_DATA_SIZE 4096 2248 #define SPDK_NVMF_RDMA_DEFAULT_MAX_IO_SIZE 131072 2249 #define SPDK_NVMF_RDMA_MIN_IO_BUFFER_SIZE (SPDK_NVMF_RDMA_DEFAULT_MAX_IO_SIZE / SPDK_NVMF_MAX_SGL_ENTRIES) 2250 #define SPDK_NVMF_RDMA_DEFAULT_NUM_SHARED_BUFFERS 4095 2251 #define SPDK_NVMF_RDMA_DEFAULT_BUFFER_CACHE_SIZE 32 2252 #define SPDK_NVMF_RDMA_DEFAULT_NO_SRQ false 2253 #define SPDK_NVMF_RDMA_DIF_INSERT_OR_STRIP false 2254 2255 static void 2256 spdk_nvmf_rdma_opts_init(struct spdk_nvmf_transport_opts *opts) 2257 { 2258 opts->max_queue_depth = SPDK_NVMF_RDMA_DEFAULT_MAX_QUEUE_DEPTH; 2259 opts->max_qpairs_per_ctrlr = SPDK_NVMF_RDMA_DEFAULT_MAX_QPAIRS_PER_CTRLR; 2260 opts->in_capsule_data_size = SPDK_NVMF_RDMA_DEFAULT_IN_CAPSULE_DATA_SIZE; 2261 opts->max_io_size = SPDK_NVMF_RDMA_DEFAULT_MAX_IO_SIZE; 2262 opts->io_unit_size = SPDK_NVMF_RDMA_MIN_IO_BUFFER_SIZE; 2263 opts->max_aq_depth = SPDK_NVMF_RDMA_DEFAULT_AQ_DEPTH; 2264 opts->num_shared_buffers = SPDK_NVMF_RDMA_DEFAULT_NUM_SHARED_BUFFERS; 2265 opts->buf_cache_size = SPDK_NVMF_RDMA_DEFAULT_BUFFER_CACHE_SIZE; 2266 opts->max_srq_depth = SPDK_NVMF_RDMA_DEFAULT_SRQ_DEPTH; 2267 opts->no_srq = SPDK_NVMF_RDMA_DEFAULT_NO_SRQ; 2268 opts->dif_insert_or_strip = SPDK_NVMF_RDMA_DIF_INSERT_OR_STRIP; 2269 } 2270 2271 const struct spdk_mem_map_ops g_nvmf_rdma_map_ops = { 2272 .notify_cb = spdk_nvmf_rdma_mem_notify, 2273 .are_contiguous = spdk_nvmf_rdma_check_contiguous_entries 2274 }; 2275 2276 static int spdk_nvmf_rdma_destroy(struct spdk_nvmf_transport *transport); 2277 2278 static struct spdk_nvmf_transport * 2279 spdk_nvmf_rdma_create(struct spdk_nvmf_transport_opts *opts) 2280 { 2281 int rc; 2282 struct spdk_nvmf_rdma_transport *rtransport; 2283 struct spdk_nvmf_rdma_device *device, *tmp; 2284 struct ibv_context **contexts; 2285 uint32_t i; 2286 int flag; 2287 uint32_t sge_count; 2288 uint32_t min_shared_buffers; 2289 int max_device_sge = SPDK_NVMF_MAX_SGL_ENTRIES; 2290 pthread_mutexattr_t attr; 2291 2292 rtransport = calloc(1, sizeof(*rtransport)); 2293 if (!rtransport) { 2294 return NULL; 2295 } 2296 2297 if (pthread_mutexattr_init(&attr)) { 2298 SPDK_ERRLOG("pthread_mutexattr_init() failed\n"); 2299 free(rtransport); 2300 return NULL; 2301 } 2302 2303 if (pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE)) { 2304 SPDK_ERRLOG("pthread_mutexattr_settype() failed\n"); 2305 pthread_mutexattr_destroy(&attr); 2306 free(rtransport); 2307 return NULL; 2308 } 2309 2310 if (pthread_mutex_init(&rtransport->lock, &attr)) { 2311 SPDK_ERRLOG("pthread_mutex_init() failed\n"); 2312 pthread_mutexattr_destroy(&attr); 2313 free(rtransport); 2314 return NULL; 2315 } 2316 2317 pthread_mutexattr_destroy(&attr); 2318 2319 TAILQ_INIT(&rtransport->devices); 2320 TAILQ_INIT(&rtransport->ports); 2321 TAILQ_INIT(&rtransport->poll_groups); 2322 2323 rtransport->transport.ops = &spdk_nvmf_transport_rdma; 2324 2325 SPDK_INFOLOG(SPDK_LOG_RDMA, "*** RDMA Transport Init ***\n" 2326 " Transport opts: max_ioq_depth=%d, max_io_size=%d,\n" 2327 " max_qpairs_per_ctrlr=%d, io_unit_size=%d,\n" 2328 " in_capsule_data_size=%d, max_aq_depth=%d,\n" 2329 " num_shared_buffers=%d, max_srq_depth=%d, no_srq=%d\n", 2330 opts->max_queue_depth, 2331 opts->max_io_size, 2332 opts->max_qpairs_per_ctrlr, 2333 opts->io_unit_size, 2334 opts->in_capsule_data_size, 2335 opts->max_aq_depth, 2336 opts->num_shared_buffers, 2337 opts->max_srq_depth, 2338 opts->no_srq); 2339 2340 /* I/O unit size cannot be larger than max I/O size */ 2341 if (opts->io_unit_size > opts->max_io_size) { 2342 opts->io_unit_size = opts->max_io_size; 2343 } 2344 2345 if (opts->num_shared_buffers < (SPDK_NVMF_MAX_SGL_ENTRIES * 2)) { 2346 SPDK_ERRLOG("The number of shared data buffers (%d) is less than" 2347 "the minimum number required to guarantee that forward progress can be made (%d)\n", 2348 opts->num_shared_buffers, (SPDK_NVMF_MAX_SGL_ENTRIES * 2)); 2349 spdk_nvmf_rdma_destroy(&rtransport->transport); 2350 return NULL; 2351 } 2352 2353 min_shared_buffers = spdk_thread_get_count() * opts->buf_cache_size; 2354 if (min_shared_buffers > opts->num_shared_buffers) { 2355 SPDK_ERRLOG("There are not enough buffers to satisfy" 2356 "per-poll group caches for each thread. (%" PRIu32 ")" 2357 "supplied. (%" PRIu32 ") required\n", opts->num_shared_buffers, min_shared_buffers); 2358 SPDK_ERRLOG("Please specify a larger number of shared buffers\n"); 2359 spdk_nvmf_rdma_destroy(&rtransport->transport); 2360 return NULL; 2361 } 2362 2363 sge_count = opts->max_io_size / opts->io_unit_size; 2364 if (sge_count > NVMF_DEFAULT_TX_SGE) { 2365 SPDK_ERRLOG("Unsupported IO Unit size specified, %d bytes\n", opts->io_unit_size); 2366 spdk_nvmf_rdma_destroy(&rtransport->transport); 2367 return NULL; 2368 } 2369 2370 rtransport->event_channel = rdma_create_event_channel(); 2371 if (rtransport->event_channel == NULL) { 2372 SPDK_ERRLOG("rdma_create_event_channel() failed, %s\n", spdk_strerror(errno)); 2373 spdk_nvmf_rdma_destroy(&rtransport->transport); 2374 return NULL; 2375 } 2376 2377 flag = fcntl(rtransport->event_channel->fd, F_GETFL); 2378 if (fcntl(rtransport->event_channel->fd, F_SETFL, flag | O_NONBLOCK) < 0) { 2379 SPDK_ERRLOG("fcntl can't set nonblocking mode for socket, fd: %d (%s)\n", 2380 rtransport->event_channel->fd, spdk_strerror(errno)); 2381 spdk_nvmf_rdma_destroy(&rtransport->transport); 2382 return NULL; 2383 } 2384 2385 rtransport->data_wr_pool = spdk_mempool_create("spdk_nvmf_rdma_wr_data", 2386 opts->max_queue_depth * SPDK_NVMF_MAX_SGL_ENTRIES, 2387 sizeof(struct spdk_nvmf_rdma_request_data), 2388 SPDK_MEMPOOL_DEFAULT_CACHE_SIZE, 2389 SPDK_ENV_SOCKET_ID_ANY); 2390 if (!rtransport->data_wr_pool) { 2391 SPDK_ERRLOG("Unable to allocate work request pool for poll group\n"); 2392 spdk_nvmf_rdma_destroy(&rtransport->transport); 2393 return NULL; 2394 } 2395 2396 contexts = rdma_get_devices(NULL); 2397 if (contexts == NULL) { 2398 SPDK_ERRLOG("rdma_get_devices() failed: %s (%d)\n", spdk_strerror(errno), errno); 2399 spdk_nvmf_rdma_destroy(&rtransport->transport); 2400 return NULL; 2401 } 2402 2403 i = 0; 2404 rc = 0; 2405 while (contexts[i] != NULL) { 2406 device = calloc(1, sizeof(*device)); 2407 if (!device) { 2408 SPDK_ERRLOG("Unable to allocate memory for RDMA devices.\n"); 2409 rc = -ENOMEM; 2410 break; 2411 } 2412 device->context = contexts[i]; 2413 rc = ibv_query_device(device->context, &device->attr); 2414 if (rc < 0) { 2415 SPDK_ERRLOG("Failed to query RDMA device attributes.\n"); 2416 free(device); 2417 break; 2418 2419 } 2420 2421 max_device_sge = spdk_min(max_device_sge, device->attr.max_sge); 2422 2423 #ifdef SPDK_CONFIG_RDMA_SEND_WITH_INVAL 2424 if ((device->attr.device_cap_flags & IBV_DEVICE_MEM_MGT_EXTENSIONS) == 0) { 2425 SPDK_WARNLOG("The libibverbs on this system supports SEND_WITH_INVALIDATE,"); 2426 SPDK_WARNLOG("but the device with vendor ID %u does not.\n", device->attr.vendor_id); 2427 } 2428 2429 /** 2430 * The vendor ID is assigned by the IEEE and an ID of 0 implies Soft-RoCE. 2431 * The Soft-RoCE RXE driver does not currently support send with invalidate, 2432 * but incorrectly reports that it does. There are changes making their way 2433 * through the kernel now that will enable this feature. When they are merged, 2434 * we can conditionally enable this feature. 2435 * 2436 * TODO: enable this for versions of the kernel rxe driver that support it. 2437 */ 2438 if (device->attr.vendor_id == 0) { 2439 device->attr.device_cap_flags &= ~(IBV_DEVICE_MEM_MGT_EXTENSIONS); 2440 } 2441 #endif 2442 2443 /* set up device context async ev fd as NON_BLOCKING */ 2444 flag = fcntl(device->context->async_fd, F_GETFL); 2445 rc = fcntl(device->context->async_fd, F_SETFL, flag | O_NONBLOCK); 2446 if (rc < 0) { 2447 SPDK_ERRLOG("Failed to set context async fd to NONBLOCK.\n"); 2448 free(device); 2449 break; 2450 } 2451 2452 TAILQ_INSERT_TAIL(&rtransport->devices, device, link); 2453 i++; 2454 2455 if (g_nvmf_hooks.get_ibv_pd) { 2456 device->pd = g_nvmf_hooks.get_ibv_pd(NULL, device->context); 2457 } else { 2458 device->pd = ibv_alloc_pd(device->context); 2459 } 2460 2461 if (!device->pd) { 2462 SPDK_ERRLOG("Unable to allocate protection domain.\n"); 2463 rc = -ENOMEM; 2464 break; 2465 } 2466 2467 assert(device->map == NULL); 2468 2469 device->map = spdk_mem_map_alloc(0, &g_nvmf_rdma_map_ops, device->pd); 2470 if (!device->map) { 2471 SPDK_ERRLOG("Unable to allocate memory map for listen address\n"); 2472 rc = -ENOMEM; 2473 break; 2474 } 2475 2476 assert(device->map != NULL); 2477 assert(device->pd != NULL); 2478 } 2479 rdma_free_devices(contexts); 2480 2481 if (opts->io_unit_size * max_device_sge < opts->max_io_size) { 2482 /* divide and round up. */ 2483 opts->io_unit_size = (opts->max_io_size + max_device_sge - 1) / max_device_sge; 2484 2485 /* round up to the nearest 4k. */ 2486 opts->io_unit_size = (opts->io_unit_size + NVMF_DATA_BUFFER_ALIGNMENT - 1) & ~NVMF_DATA_BUFFER_MASK; 2487 2488 opts->io_unit_size = spdk_max(opts->io_unit_size, SPDK_NVMF_RDMA_MIN_IO_BUFFER_SIZE); 2489 SPDK_NOTICELOG("Adjusting the io unit size to fit the device's maximum I/O size. New I/O unit size %u\n", 2490 opts->io_unit_size); 2491 } 2492 2493 if (rc < 0) { 2494 spdk_nvmf_rdma_destroy(&rtransport->transport); 2495 return NULL; 2496 } 2497 2498 /* Set up poll descriptor array to monitor events from RDMA and IB 2499 * in a single poll syscall 2500 */ 2501 rtransport->npoll_fds = i + 1; 2502 i = 0; 2503 rtransport->poll_fds = calloc(rtransport->npoll_fds, sizeof(struct pollfd)); 2504 if (rtransport->poll_fds == NULL) { 2505 SPDK_ERRLOG("poll_fds allocation failed\n"); 2506 spdk_nvmf_rdma_destroy(&rtransport->transport); 2507 return NULL; 2508 } 2509 2510 rtransport->poll_fds[i].fd = rtransport->event_channel->fd; 2511 rtransport->poll_fds[i++].events = POLLIN; 2512 2513 TAILQ_FOREACH_SAFE(device, &rtransport->devices, link, tmp) { 2514 rtransport->poll_fds[i].fd = device->context->async_fd; 2515 rtransport->poll_fds[i++].events = POLLIN; 2516 } 2517 2518 return &rtransport->transport; 2519 } 2520 2521 static int 2522 spdk_nvmf_rdma_destroy(struct spdk_nvmf_transport *transport) 2523 { 2524 struct spdk_nvmf_rdma_transport *rtransport; 2525 struct spdk_nvmf_rdma_port *port, *port_tmp; 2526 struct spdk_nvmf_rdma_device *device, *device_tmp; 2527 2528 rtransport = SPDK_CONTAINEROF(transport, struct spdk_nvmf_rdma_transport, transport); 2529 2530 TAILQ_FOREACH_SAFE(port, &rtransport->ports, link, port_tmp) { 2531 TAILQ_REMOVE(&rtransport->ports, port, link); 2532 rdma_destroy_id(port->id); 2533 free(port); 2534 } 2535 2536 if (rtransport->poll_fds != NULL) { 2537 free(rtransport->poll_fds); 2538 } 2539 2540 if (rtransport->event_channel != NULL) { 2541 rdma_destroy_event_channel(rtransport->event_channel); 2542 } 2543 2544 TAILQ_FOREACH_SAFE(device, &rtransport->devices, link, device_tmp) { 2545 TAILQ_REMOVE(&rtransport->devices, device, link); 2546 if (device->map) { 2547 spdk_mem_map_free(&device->map); 2548 } 2549 if (device->pd) { 2550 if (!g_nvmf_hooks.get_ibv_pd) { 2551 ibv_dealloc_pd(device->pd); 2552 } 2553 } 2554 free(device); 2555 } 2556 2557 if (rtransport->data_wr_pool != NULL) { 2558 if (spdk_mempool_count(rtransport->data_wr_pool) != 2559 (transport->opts.max_queue_depth * SPDK_NVMF_MAX_SGL_ENTRIES)) { 2560 SPDK_ERRLOG("transport wr pool count is %zu but should be %u\n", 2561 spdk_mempool_count(rtransport->data_wr_pool), 2562 transport->opts.max_queue_depth * SPDK_NVMF_MAX_SGL_ENTRIES); 2563 } 2564 } 2565 2566 spdk_mempool_free(rtransport->data_wr_pool); 2567 2568 pthread_mutex_destroy(&rtransport->lock); 2569 free(rtransport); 2570 2571 return 0; 2572 } 2573 2574 static int 2575 spdk_nvmf_rdma_trid_from_cm_id(struct rdma_cm_id *id, 2576 struct spdk_nvme_transport_id *trid, 2577 bool peer); 2578 2579 static int 2580 spdk_nvmf_rdma_listen(struct spdk_nvmf_transport *transport, 2581 const struct spdk_nvme_transport_id *trid) 2582 { 2583 struct spdk_nvmf_rdma_transport *rtransport; 2584 struct spdk_nvmf_rdma_device *device; 2585 struct spdk_nvmf_rdma_port *port_tmp, *port; 2586 struct addrinfo *res; 2587 struct addrinfo hints; 2588 int family; 2589 int rc; 2590 2591 rtransport = SPDK_CONTAINEROF(transport, struct spdk_nvmf_rdma_transport, transport); 2592 2593 port = calloc(1, sizeof(*port)); 2594 if (!port) { 2595 return -ENOMEM; 2596 } 2597 2598 /* Selectively copy the trid. Things like NQN don't matter here - that 2599 * mapping is enforced elsewhere. 2600 */ 2601 port->trid.trtype = SPDK_NVME_TRANSPORT_RDMA; 2602 port->trid.adrfam = trid->adrfam; 2603 snprintf(port->trid.traddr, sizeof(port->trid.traddr), "%s", trid->traddr); 2604 snprintf(port->trid.trsvcid, sizeof(port->trid.trsvcid), "%s", trid->trsvcid); 2605 2606 switch (port->trid.adrfam) { 2607 case SPDK_NVMF_ADRFAM_IPV4: 2608 family = AF_INET; 2609 break; 2610 case SPDK_NVMF_ADRFAM_IPV6: 2611 family = AF_INET6; 2612 break; 2613 default: 2614 SPDK_ERRLOG("Unhandled ADRFAM %d\n", port->trid.adrfam); 2615 free(port); 2616 return -EINVAL; 2617 } 2618 2619 memset(&hints, 0, sizeof(hints)); 2620 hints.ai_family = family; 2621 hints.ai_flags = AI_NUMERICSERV; 2622 hints.ai_socktype = SOCK_STREAM; 2623 hints.ai_protocol = 0; 2624 2625 rc = getaddrinfo(port->trid.traddr, port->trid.trsvcid, &hints, &res); 2626 if (rc) { 2627 SPDK_ERRLOG("getaddrinfo failed: %s (%d)\n", gai_strerror(rc), rc); 2628 free(port); 2629 return -EINVAL; 2630 } 2631 2632 pthread_mutex_lock(&rtransport->lock); 2633 assert(rtransport->event_channel != NULL); 2634 TAILQ_FOREACH(port_tmp, &rtransport->ports, link) { 2635 if (spdk_nvme_transport_id_compare(&port_tmp->trid, &port->trid) == 0) { 2636 port_tmp->ref++; 2637 freeaddrinfo(res); 2638 free(port); 2639 /* Already listening at this address */ 2640 pthread_mutex_unlock(&rtransport->lock); 2641 return 0; 2642 } 2643 } 2644 2645 rc = rdma_create_id(rtransport->event_channel, &port->id, port, RDMA_PS_TCP); 2646 if (rc < 0) { 2647 SPDK_ERRLOG("rdma_create_id() failed\n"); 2648 freeaddrinfo(res); 2649 free(port); 2650 pthread_mutex_unlock(&rtransport->lock); 2651 return rc; 2652 } 2653 2654 rc = rdma_bind_addr(port->id, res->ai_addr); 2655 freeaddrinfo(res); 2656 2657 if (rc < 0) { 2658 SPDK_ERRLOG("rdma_bind_addr() failed\n"); 2659 rdma_destroy_id(port->id); 2660 free(port); 2661 pthread_mutex_unlock(&rtransport->lock); 2662 return rc; 2663 } 2664 2665 if (!port->id->verbs) { 2666 SPDK_ERRLOG("ibv_context is null\n"); 2667 rdma_destroy_id(port->id); 2668 free(port); 2669 pthread_mutex_unlock(&rtransport->lock); 2670 return -1; 2671 } 2672 2673 rc = rdma_listen(port->id, 10); /* 10 = backlog */ 2674 if (rc < 0) { 2675 SPDK_ERRLOG("rdma_listen() failed\n"); 2676 rdma_destroy_id(port->id); 2677 free(port); 2678 pthread_mutex_unlock(&rtransport->lock); 2679 return rc; 2680 } 2681 2682 TAILQ_FOREACH(device, &rtransport->devices, link) { 2683 if (device->context == port->id->verbs) { 2684 port->device = device; 2685 break; 2686 } 2687 } 2688 if (!port->device) { 2689 SPDK_ERRLOG("Accepted a connection with verbs %p, but unable to find a corresponding device.\n", 2690 port->id->verbs); 2691 rdma_destroy_id(port->id); 2692 free(port); 2693 pthread_mutex_unlock(&rtransport->lock); 2694 return -EINVAL; 2695 } 2696 2697 SPDK_INFOLOG(SPDK_LOG_RDMA, "*** NVMf Target Listening on %s port %d ***\n", 2698 port->trid.traddr, ntohs(rdma_get_src_port(port->id))); 2699 2700 port->ref = 1; 2701 2702 TAILQ_INSERT_TAIL(&rtransport->ports, port, link); 2703 pthread_mutex_unlock(&rtransport->lock); 2704 2705 return 0; 2706 } 2707 2708 static int 2709 spdk_nvmf_rdma_stop_listen(struct spdk_nvmf_transport *transport, 2710 const struct spdk_nvme_transport_id *_trid) 2711 { 2712 struct spdk_nvmf_rdma_transport *rtransport; 2713 struct spdk_nvmf_rdma_port *port, *tmp; 2714 struct spdk_nvme_transport_id trid = {}; 2715 2716 rtransport = SPDK_CONTAINEROF(transport, struct spdk_nvmf_rdma_transport, transport); 2717 2718 /* Selectively copy the trid. Things like NQN don't matter here - that 2719 * mapping is enforced elsewhere. 2720 */ 2721 trid.trtype = SPDK_NVME_TRANSPORT_RDMA; 2722 trid.adrfam = _trid->adrfam; 2723 snprintf(trid.traddr, sizeof(port->trid.traddr), "%s", _trid->traddr); 2724 snprintf(trid.trsvcid, sizeof(port->trid.trsvcid), "%s", _trid->trsvcid); 2725 2726 pthread_mutex_lock(&rtransport->lock); 2727 TAILQ_FOREACH_SAFE(port, &rtransport->ports, link, tmp) { 2728 if (spdk_nvme_transport_id_compare(&port->trid, &trid) == 0) { 2729 assert(port->ref > 0); 2730 port->ref--; 2731 if (port->ref == 0) { 2732 TAILQ_REMOVE(&rtransport->ports, port, link); 2733 rdma_destroy_id(port->id); 2734 free(port); 2735 } 2736 break; 2737 } 2738 } 2739 2740 pthread_mutex_unlock(&rtransport->lock); 2741 return 0; 2742 } 2743 2744 static void 2745 spdk_nvmf_rdma_qpair_process_pending(struct spdk_nvmf_rdma_transport *rtransport, 2746 struct spdk_nvmf_rdma_qpair *rqpair, bool drain) 2747 { 2748 struct spdk_nvmf_request *req, *tmp; 2749 struct spdk_nvmf_rdma_request *rdma_req, *req_tmp; 2750 struct spdk_nvmf_rdma_resources *resources; 2751 2752 /* We process I/O in the data transfer pending queue at the highest priority. RDMA reads first */ 2753 STAILQ_FOREACH_SAFE(rdma_req, &rqpair->pending_rdma_read_queue, state_link, req_tmp) { 2754 if (spdk_nvmf_rdma_request_process(rtransport, rdma_req) == false && drain == false) { 2755 break; 2756 } 2757 } 2758 2759 /* Then RDMA writes since reads have stronger restrictions than writes */ 2760 STAILQ_FOREACH_SAFE(rdma_req, &rqpair->pending_rdma_write_queue, state_link, req_tmp) { 2761 if (spdk_nvmf_rdma_request_process(rtransport, rdma_req) == false && drain == false) { 2762 break; 2763 } 2764 } 2765 2766 /* The second highest priority is I/O waiting on memory buffers. */ 2767 STAILQ_FOREACH_SAFE(req, &rqpair->poller->group->group.pending_buf_queue, buf_link, tmp) { 2768 rdma_req = SPDK_CONTAINEROF(req, struct spdk_nvmf_rdma_request, req); 2769 if (spdk_nvmf_rdma_request_process(rtransport, rdma_req) == false && drain == false) { 2770 break; 2771 } 2772 } 2773 2774 resources = rqpair->resources; 2775 while (!STAILQ_EMPTY(&resources->free_queue) && !STAILQ_EMPTY(&resources->incoming_queue)) { 2776 rdma_req = STAILQ_FIRST(&resources->free_queue); 2777 STAILQ_REMOVE_HEAD(&resources->free_queue, state_link); 2778 rdma_req->recv = STAILQ_FIRST(&resources->incoming_queue); 2779 STAILQ_REMOVE_HEAD(&resources->incoming_queue, link); 2780 2781 if (rqpair->srq != NULL) { 2782 rdma_req->req.qpair = &rdma_req->recv->qpair->qpair; 2783 rdma_req->recv->qpair->qd++; 2784 } else { 2785 rqpair->qd++; 2786 } 2787 2788 rdma_req->receive_tsc = rdma_req->recv->receive_tsc; 2789 rdma_req->state = RDMA_REQUEST_STATE_NEW; 2790 if (spdk_nvmf_rdma_request_process(rtransport, rdma_req) == false) { 2791 break; 2792 } 2793 } 2794 if (!STAILQ_EMPTY(&resources->incoming_queue) && STAILQ_EMPTY(&resources->free_queue)) { 2795 rqpair->poller->stat.pending_free_request++; 2796 } 2797 } 2798 2799 static void 2800 _nvmf_rdma_qpair_disconnect(void *ctx) 2801 { 2802 struct spdk_nvmf_qpair *qpair = ctx; 2803 2804 spdk_nvmf_qpair_disconnect(qpair, NULL, NULL); 2805 } 2806 2807 static void 2808 _nvmf_rdma_try_disconnect(void *ctx) 2809 { 2810 struct spdk_nvmf_qpair *qpair = ctx; 2811 struct spdk_nvmf_poll_group *group; 2812 2813 /* Read the group out of the qpair. This is normally set and accessed only from 2814 * the thread that created the group. Here, we're not on that thread necessarily. 2815 * The data member qpair->group begins it's life as NULL and then is assigned to 2816 * a pointer and never changes. So fortunately reading this and checking for 2817 * non-NULL is thread safe in the x86_64 memory model. */ 2818 group = qpair->group; 2819 2820 if (group == NULL) { 2821 /* The qpair hasn't been assigned to a group yet, so we can't 2822 * process a disconnect. Send a message to ourself and try again. */ 2823 spdk_thread_send_msg(spdk_get_thread(), _nvmf_rdma_try_disconnect, qpair); 2824 return; 2825 } 2826 2827 spdk_thread_send_msg(group->thread, _nvmf_rdma_qpair_disconnect, qpair); 2828 } 2829 2830 static inline void 2831 spdk_nvmf_rdma_start_disconnect(struct spdk_nvmf_rdma_qpair *rqpair) 2832 { 2833 if (__sync_bool_compare_and_swap(&rqpair->disconnect_started, false, true)) { 2834 _nvmf_rdma_try_disconnect(&rqpair->qpair); 2835 } 2836 } 2837 2838 static void nvmf_rdma_destroy_drained_qpair(void *ctx) 2839 { 2840 struct spdk_nvmf_rdma_qpair *rqpair = ctx; 2841 struct spdk_nvmf_rdma_transport *rtransport = SPDK_CONTAINEROF(rqpair->qpair.transport, 2842 struct spdk_nvmf_rdma_transport, transport); 2843 2844 /* In non SRQ path, we will reach rqpair->max_queue_depth. In SRQ path, we will get the last_wqe event. */ 2845 if (rqpair->current_send_depth != 0) { 2846 return; 2847 } 2848 2849 if (rqpair->srq == NULL && rqpair->current_recv_depth != rqpair->max_queue_depth) { 2850 return; 2851 } 2852 2853 if (rqpair->srq != NULL && rqpair->last_wqe_reached == false) { 2854 return; 2855 } 2856 2857 spdk_nvmf_rdma_qpair_process_pending(rtransport, rqpair, true); 2858 2859 /* Qpair will be destroyed after nvmf layer closes this qpair */ 2860 if (rqpair->qpair.state != SPDK_NVMF_QPAIR_ERROR) { 2861 return; 2862 } 2863 2864 spdk_nvmf_rdma_qpair_destroy(rqpair); 2865 } 2866 2867 2868 static int 2869 nvmf_rdma_disconnect(struct rdma_cm_event *evt) 2870 { 2871 struct spdk_nvmf_qpair *qpair; 2872 struct spdk_nvmf_rdma_qpair *rqpair; 2873 2874 if (evt->id == NULL) { 2875 SPDK_ERRLOG("disconnect request: missing cm_id\n"); 2876 return -1; 2877 } 2878 2879 qpair = evt->id->context; 2880 if (qpair == NULL) { 2881 SPDK_ERRLOG("disconnect request: no active connection\n"); 2882 return -1; 2883 } 2884 2885 rqpair = SPDK_CONTAINEROF(qpair, struct spdk_nvmf_rdma_qpair, qpair); 2886 2887 spdk_trace_record(TRACE_RDMA_QP_DISCONNECT, 0, 0, (uintptr_t)rqpair->cm_id, 0); 2888 2889 spdk_nvmf_rdma_update_ibv_state(rqpair); 2890 2891 spdk_nvmf_rdma_start_disconnect(rqpair); 2892 2893 return 0; 2894 } 2895 2896 #ifdef DEBUG 2897 static const char *CM_EVENT_STR[] = { 2898 "RDMA_CM_EVENT_ADDR_RESOLVED", 2899 "RDMA_CM_EVENT_ADDR_ERROR", 2900 "RDMA_CM_EVENT_ROUTE_RESOLVED", 2901 "RDMA_CM_EVENT_ROUTE_ERROR", 2902 "RDMA_CM_EVENT_CONNECT_REQUEST", 2903 "RDMA_CM_EVENT_CONNECT_RESPONSE", 2904 "RDMA_CM_EVENT_CONNECT_ERROR", 2905 "RDMA_CM_EVENT_UNREACHABLE", 2906 "RDMA_CM_EVENT_REJECTED", 2907 "RDMA_CM_EVENT_ESTABLISHED", 2908 "RDMA_CM_EVENT_DISCONNECTED", 2909 "RDMA_CM_EVENT_DEVICE_REMOVAL", 2910 "RDMA_CM_EVENT_MULTICAST_JOIN", 2911 "RDMA_CM_EVENT_MULTICAST_ERROR", 2912 "RDMA_CM_EVENT_ADDR_CHANGE", 2913 "RDMA_CM_EVENT_TIMEWAIT_EXIT" 2914 }; 2915 #endif /* DEBUG */ 2916 2917 static void 2918 nvmf_rdma_handle_last_wqe_reached(void *ctx) 2919 { 2920 struct spdk_nvmf_rdma_qpair *rqpair = ctx; 2921 rqpair->last_wqe_reached = true; 2922 2923 nvmf_rdma_destroy_drained_qpair(rqpair); 2924 } 2925 2926 static void 2927 spdk_nvmf_process_cm_event(struct spdk_nvmf_transport *transport, new_qpair_fn cb_fn) 2928 { 2929 struct spdk_nvmf_rdma_transport *rtransport; 2930 struct rdma_cm_event *event; 2931 int rc; 2932 2933 rtransport = SPDK_CONTAINEROF(transport, struct spdk_nvmf_rdma_transport, transport); 2934 2935 if (rtransport->event_channel == NULL) { 2936 return; 2937 } 2938 2939 while (1) { 2940 rc = rdma_get_cm_event(rtransport->event_channel, &event); 2941 if (rc == 0) { 2942 SPDK_DEBUGLOG(SPDK_LOG_RDMA, "Acceptor Event: %s\n", CM_EVENT_STR[event->event]); 2943 2944 spdk_trace_record(TRACE_RDMA_CM_ASYNC_EVENT, 0, 0, 0, event->event); 2945 2946 switch (event->event) { 2947 case RDMA_CM_EVENT_ADDR_RESOLVED: 2948 case RDMA_CM_EVENT_ADDR_ERROR: 2949 case RDMA_CM_EVENT_ROUTE_RESOLVED: 2950 case RDMA_CM_EVENT_ROUTE_ERROR: 2951 /* No action required. The target never attempts to resolve routes. */ 2952 break; 2953 case RDMA_CM_EVENT_CONNECT_REQUEST: 2954 rc = nvmf_rdma_connect(transport, event, cb_fn); 2955 if (rc < 0) { 2956 SPDK_ERRLOG("Unable to process connect event. rc: %d\n", rc); 2957 break; 2958 } 2959 break; 2960 case RDMA_CM_EVENT_CONNECT_RESPONSE: 2961 /* The target never initiates a new connection. So this will not occur. */ 2962 break; 2963 case RDMA_CM_EVENT_CONNECT_ERROR: 2964 /* Can this happen? The docs say it can, but not sure what causes it. */ 2965 break; 2966 case RDMA_CM_EVENT_UNREACHABLE: 2967 case RDMA_CM_EVENT_REJECTED: 2968 /* These only occur on the client side. */ 2969 break; 2970 case RDMA_CM_EVENT_ESTABLISHED: 2971 /* TODO: Should we be waiting for this event anywhere? */ 2972 break; 2973 case RDMA_CM_EVENT_DISCONNECTED: 2974 case RDMA_CM_EVENT_DEVICE_REMOVAL: 2975 rc = nvmf_rdma_disconnect(event); 2976 if (rc < 0) { 2977 SPDK_ERRLOG("Unable to process disconnect event. rc: %d\n", rc); 2978 break; 2979 } 2980 break; 2981 case RDMA_CM_EVENT_MULTICAST_JOIN: 2982 case RDMA_CM_EVENT_MULTICAST_ERROR: 2983 /* Multicast is not used */ 2984 break; 2985 case RDMA_CM_EVENT_ADDR_CHANGE: 2986 /* Not utilizing this event */ 2987 break; 2988 case RDMA_CM_EVENT_TIMEWAIT_EXIT: 2989 /* For now, do nothing. The target never re-uses queue pairs. */ 2990 break; 2991 default: 2992 SPDK_ERRLOG("Unexpected Acceptor Event [%d]\n", event->event); 2993 break; 2994 } 2995 2996 rdma_ack_cm_event(event); 2997 } else { 2998 if (errno != EAGAIN && errno != EWOULDBLOCK) { 2999 SPDK_ERRLOG("Acceptor Event Error: %s\n", spdk_strerror(errno)); 3000 } 3001 break; 3002 } 3003 } 3004 } 3005 3006 static void 3007 spdk_nvmf_process_ib_event(struct spdk_nvmf_rdma_device *device) 3008 { 3009 int rc; 3010 struct spdk_nvmf_rdma_qpair *rqpair = NULL; 3011 struct ibv_async_event event; 3012 enum ibv_qp_state state; 3013 3014 rc = ibv_get_async_event(device->context, &event); 3015 3016 if (rc) { 3017 SPDK_ERRLOG("Failed to get async_event (%d): %s\n", 3018 errno, spdk_strerror(errno)); 3019 return; 3020 } 3021 3022 switch (event.event_type) { 3023 case IBV_EVENT_QP_FATAL: 3024 rqpair = event.element.qp->qp_context; 3025 SPDK_ERRLOG("Fatal event received for rqpair %p\n", rqpair); 3026 spdk_trace_record(TRACE_RDMA_IBV_ASYNC_EVENT, 0, 0, 3027 (uintptr_t)rqpair->cm_id, event.event_type); 3028 spdk_nvmf_rdma_update_ibv_state(rqpair); 3029 spdk_nvmf_rdma_start_disconnect(rqpair); 3030 break; 3031 case IBV_EVENT_QP_LAST_WQE_REACHED: 3032 /* This event only occurs for shared receive queues. */ 3033 rqpair = event.element.qp->qp_context; 3034 SPDK_DEBUGLOG(SPDK_LOG_RDMA, "Last WQE reached event received for rqpair %p\n", rqpair); 3035 /* This must be handled on the polling thread if it exists. Otherwise the timeout will catch it. */ 3036 if (rqpair->qpair.group) { 3037 spdk_thread_send_msg(rqpair->qpair.group->thread, nvmf_rdma_handle_last_wqe_reached, rqpair); 3038 } else { 3039 SPDK_ERRLOG("Unable to destroy the qpair %p since it does not have a poll group.\n", rqpair); 3040 rqpair->last_wqe_reached = true; 3041 } 3042 3043 break; 3044 case IBV_EVENT_SQ_DRAINED: 3045 /* This event occurs frequently in both error and non-error states. 3046 * Check if the qpair is in an error state before sending a message. 3047 * Note that we're not on the correct thread to access the qpair, but 3048 * the operations that the below calls make all happen to be thread 3049 * safe. */ 3050 rqpair = event.element.qp->qp_context; 3051 SPDK_DEBUGLOG(SPDK_LOG_RDMA, "Last sq drained event received for rqpair %p\n", rqpair); 3052 spdk_trace_record(TRACE_RDMA_IBV_ASYNC_EVENT, 0, 0, 3053 (uintptr_t)rqpair->cm_id, event.event_type); 3054 state = spdk_nvmf_rdma_update_ibv_state(rqpair); 3055 if (state == IBV_QPS_ERR) { 3056 spdk_nvmf_rdma_start_disconnect(rqpair); 3057 } 3058 break; 3059 case IBV_EVENT_QP_REQ_ERR: 3060 case IBV_EVENT_QP_ACCESS_ERR: 3061 case IBV_EVENT_COMM_EST: 3062 case IBV_EVENT_PATH_MIG: 3063 case IBV_EVENT_PATH_MIG_ERR: 3064 SPDK_NOTICELOG("Async event: %s\n", 3065 ibv_event_type_str(event.event_type)); 3066 rqpair = event.element.qp->qp_context; 3067 spdk_trace_record(TRACE_RDMA_IBV_ASYNC_EVENT, 0, 0, 3068 (uintptr_t)rqpair->cm_id, event.event_type); 3069 spdk_nvmf_rdma_update_ibv_state(rqpair); 3070 break; 3071 case IBV_EVENT_CQ_ERR: 3072 case IBV_EVENT_DEVICE_FATAL: 3073 case IBV_EVENT_PORT_ACTIVE: 3074 case IBV_EVENT_PORT_ERR: 3075 case IBV_EVENT_LID_CHANGE: 3076 case IBV_EVENT_PKEY_CHANGE: 3077 case IBV_EVENT_SM_CHANGE: 3078 case IBV_EVENT_SRQ_ERR: 3079 case IBV_EVENT_SRQ_LIMIT_REACHED: 3080 case IBV_EVENT_CLIENT_REREGISTER: 3081 case IBV_EVENT_GID_CHANGE: 3082 default: 3083 SPDK_NOTICELOG("Async event: %s\n", 3084 ibv_event_type_str(event.event_type)); 3085 spdk_trace_record(TRACE_RDMA_IBV_ASYNC_EVENT, 0, 0, 0, event.event_type); 3086 break; 3087 } 3088 ibv_ack_async_event(&event); 3089 } 3090 3091 static void 3092 spdk_nvmf_rdma_accept(struct spdk_nvmf_transport *transport, new_qpair_fn cb_fn) 3093 { 3094 int nfds, i = 0; 3095 struct spdk_nvmf_rdma_transport *rtransport; 3096 struct spdk_nvmf_rdma_device *device, *tmp; 3097 3098 rtransport = SPDK_CONTAINEROF(transport, struct spdk_nvmf_rdma_transport, transport); 3099 nfds = poll(rtransport->poll_fds, rtransport->npoll_fds, 0); 3100 3101 if (nfds <= 0) { 3102 return; 3103 } 3104 3105 /* The first poll descriptor is RDMA CM event */ 3106 if (rtransport->poll_fds[i++].revents & POLLIN) { 3107 spdk_nvmf_process_cm_event(transport, cb_fn); 3108 nfds--; 3109 } 3110 3111 if (nfds == 0) { 3112 return; 3113 } 3114 3115 /* Second and subsequent poll descriptors are IB async events */ 3116 TAILQ_FOREACH_SAFE(device, &rtransport->devices, link, tmp) { 3117 if (rtransport->poll_fds[i++].revents & POLLIN) { 3118 spdk_nvmf_process_ib_event(device); 3119 nfds--; 3120 } 3121 } 3122 /* check all flagged fd's have been served */ 3123 assert(nfds == 0); 3124 } 3125 3126 static void 3127 spdk_nvmf_rdma_discover(struct spdk_nvmf_transport *transport, 3128 struct spdk_nvme_transport_id *trid, 3129 struct spdk_nvmf_discovery_log_page_entry *entry) 3130 { 3131 entry->trtype = SPDK_NVMF_TRTYPE_RDMA; 3132 entry->adrfam = trid->adrfam; 3133 entry->treq.secure_channel = SPDK_NVMF_TREQ_SECURE_CHANNEL_NOT_REQUIRED; 3134 3135 spdk_strcpy_pad(entry->trsvcid, trid->trsvcid, sizeof(entry->trsvcid), ' '); 3136 spdk_strcpy_pad(entry->traddr, trid->traddr, sizeof(entry->traddr), ' '); 3137 3138 entry->tsas.rdma.rdma_qptype = SPDK_NVMF_RDMA_QPTYPE_RELIABLE_CONNECTED; 3139 entry->tsas.rdma.rdma_prtype = SPDK_NVMF_RDMA_PRTYPE_NONE; 3140 entry->tsas.rdma.rdma_cms = SPDK_NVMF_RDMA_CMS_RDMA_CM; 3141 } 3142 3143 static void 3144 spdk_nvmf_rdma_poll_group_destroy(struct spdk_nvmf_transport_poll_group *group); 3145 3146 static struct spdk_nvmf_transport_poll_group * 3147 spdk_nvmf_rdma_poll_group_create(struct spdk_nvmf_transport *transport) 3148 { 3149 struct spdk_nvmf_rdma_transport *rtransport; 3150 struct spdk_nvmf_rdma_poll_group *rgroup; 3151 struct spdk_nvmf_rdma_poller *poller; 3152 struct spdk_nvmf_rdma_device *device; 3153 struct ibv_srq_init_attr srq_init_attr; 3154 struct spdk_nvmf_rdma_resource_opts opts; 3155 int num_cqe; 3156 3157 rtransport = SPDK_CONTAINEROF(transport, struct spdk_nvmf_rdma_transport, transport); 3158 3159 rgroup = calloc(1, sizeof(*rgroup)); 3160 if (!rgroup) { 3161 return NULL; 3162 } 3163 3164 TAILQ_INIT(&rgroup->pollers); 3165 STAILQ_INIT(&rgroup->retired_bufs); 3166 3167 pthread_mutex_lock(&rtransport->lock); 3168 TAILQ_FOREACH(device, &rtransport->devices, link) { 3169 poller = calloc(1, sizeof(*poller)); 3170 if (!poller) { 3171 SPDK_ERRLOG("Unable to allocate memory for new RDMA poller\n"); 3172 spdk_nvmf_rdma_poll_group_destroy(&rgroup->group); 3173 pthread_mutex_unlock(&rtransport->lock); 3174 return NULL; 3175 } 3176 3177 poller->device = device; 3178 poller->group = rgroup; 3179 3180 TAILQ_INIT(&poller->qpairs); 3181 STAILQ_INIT(&poller->qpairs_pending_send); 3182 STAILQ_INIT(&poller->qpairs_pending_recv); 3183 3184 TAILQ_INSERT_TAIL(&rgroup->pollers, poller, link); 3185 if (transport->opts.no_srq == false && device->num_srq < device->attr.max_srq) { 3186 poller->max_srq_depth = transport->opts.max_srq_depth; 3187 3188 device->num_srq++; 3189 memset(&srq_init_attr, 0, sizeof(struct ibv_srq_init_attr)); 3190 srq_init_attr.attr.max_wr = poller->max_srq_depth; 3191 srq_init_attr.attr.max_sge = spdk_min(device->attr.max_sge, NVMF_DEFAULT_RX_SGE); 3192 poller->srq = ibv_create_srq(device->pd, &srq_init_attr); 3193 if (!poller->srq) { 3194 SPDK_ERRLOG("Unable to create shared receive queue, errno %d\n", errno); 3195 spdk_nvmf_rdma_poll_group_destroy(&rgroup->group); 3196 pthread_mutex_unlock(&rtransport->lock); 3197 return NULL; 3198 } 3199 3200 opts.qp = poller->srq; 3201 opts.pd = device->pd; 3202 opts.qpair = NULL; 3203 opts.shared = true; 3204 opts.max_queue_depth = poller->max_srq_depth; 3205 opts.in_capsule_data_size = transport->opts.in_capsule_data_size; 3206 3207 poller->resources = nvmf_rdma_resources_create(&opts); 3208 if (!poller->resources) { 3209 SPDK_ERRLOG("Unable to allocate resources for shared receive queue.\n"); 3210 spdk_nvmf_rdma_poll_group_destroy(&rgroup->group); 3211 pthread_mutex_unlock(&rtransport->lock); 3212 return NULL; 3213 } 3214 } 3215 3216 /* 3217 * When using an srq, we can limit the completion queue at startup. 3218 * The following formula represents the calculation: 3219 * num_cqe = num_recv + num_data_wr + num_send_wr. 3220 * where num_recv=num_data_wr=and num_send_wr=poller->max_srq_depth 3221 */ 3222 if (poller->srq) { 3223 num_cqe = poller->max_srq_depth * 3; 3224 } else { 3225 num_cqe = DEFAULT_NVMF_RDMA_CQ_SIZE; 3226 } 3227 3228 poller->cq = ibv_create_cq(device->context, num_cqe, poller, NULL, 0); 3229 if (!poller->cq) { 3230 SPDK_ERRLOG("Unable to create completion queue\n"); 3231 spdk_nvmf_rdma_poll_group_destroy(&rgroup->group); 3232 pthread_mutex_unlock(&rtransport->lock); 3233 return NULL; 3234 } 3235 poller->num_cqe = num_cqe; 3236 } 3237 3238 TAILQ_INSERT_TAIL(&rtransport->poll_groups, rgroup, link); 3239 if (rtransport->conn_sched.next_admin_pg == NULL) { 3240 rtransport->conn_sched.next_admin_pg = rgroup; 3241 rtransport->conn_sched.next_io_pg = rgroup; 3242 } 3243 3244 pthread_mutex_unlock(&rtransport->lock); 3245 return &rgroup->group; 3246 } 3247 3248 static struct spdk_nvmf_transport_poll_group * 3249 spdk_nvmf_rdma_get_optimal_poll_group(struct spdk_nvmf_qpair *qpair) 3250 { 3251 struct spdk_nvmf_rdma_transport *rtransport; 3252 struct spdk_nvmf_rdma_poll_group **pg; 3253 struct spdk_nvmf_transport_poll_group *result; 3254 3255 rtransport = SPDK_CONTAINEROF(qpair->transport, struct spdk_nvmf_rdma_transport, transport); 3256 3257 pthread_mutex_lock(&rtransport->lock); 3258 3259 if (TAILQ_EMPTY(&rtransport->poll_groups)) { 3260 pthread_mutex_unlock(&rtransport->lock); 3261 return NULL; 3262 } 3263 3264 if (qpair->qid == 0) { 3265 pg = &rtransport->conn_sched.next_admin_pg; 3266 } else { 3267 pg = &rtransport->conn_sched.next_io_pg; 3268 } 3269 3270 assert(*pg != NULL); 3271 3272 result = &(*pg)->group; 3273 3274 *pg = TAILQ_NEXT(*pg, link); 3275 if (*pg == NULL) { 3276 *pg = TAILQ_FIRST(&rtransport->poll_groups); 3277 } 3278 3279 pthread_mutex_unlock(&rtransport->lock); 3280 3281 return result; 3282 } 3283 3284 static void 3285 spdk_nvmf_rdma_poll_group_destroy(struct spdk_nvmf_transport_poll_group *group) 3286 { 3287 struct spdk_nvmf_rdma_poll_group *rgroup, *next_rgroup; 3288 struct spdk_nvmf_rdma_poller *poller, *tmp; 3289 struct spdk_nvmf_rdma_qpair *qpair, *tmp_qpair; 3290 struct spdk_nvmf_transport_pg_cache_buf *buf, *tmp_buf; 3291 struct spdk_nvmf_rdma_transport *rtransport; 3292 3293 rgroup = SPDK_CONTAINEROF(group, struct spdk_nvmf_rdma_poll_group, group); 3294 rtransport = SPDK_CONTAINEROF(rgroup->group.transport, struct spdk_nvmf_rdma_transport, transport); 3295 3296 if (!rgroup) { 3297 return; 3298 } 3299 3300 /* free all retired buffers back to the transport so we don't short the mempool. */ 3301 STAILQ_FOREACH_SAFE(buf, &rgroup->retired_bufs, link, tmp_buf) { 3302 STAILQ_REMOVE(&rgroup->retired_bufs, buf, spdk_nvmf_transport_pg_cache_buf, link); 3303 assert(group->transport != NULL); 3304 spdk_mempool_put(group->transport->data_buf_pool, buf); 3305 } 3306 3307 TAILQ_FOREACH_SAFE(poller, &rgroup->pollers, link, tmp) { 3308 TAILQ_REMOVE(&rgroup->pollers, poller, link); 3309 3310 TAILQ_FOREACH_SAFE(qpair, &poller->qpairs, link, tmp_qpair) { 3311 spdk_nvmf_rdma_qpair_destroy(qpair); 3312 } 3313 3314 if (poller->srq) { 3315 nvmf_rdma_resources_destroy(poller->resources); 3316 ibv_destroy_srq(poller->srq); 3317 SPDK_DEBUGLOG(SPDK_LOG_RDMA, "Destroyed RDMA shared queue %p\n", poller->srq); 3318 } 3319 3320 if (poller->cq) { 3321 ibv_destroy_cq(poller->cq); 3322 } 3323 3324 free(poller); 3325 } 3326 3327 pthread_mutex_lock(&rtransport->lock); 3328 next_rgroup = TAILQ_NEXT(rgroup, link); 3329 TAILQ_REMOVE(&rtransport->poll_groups, rgroup, link); 3330 if (next_rgroup == NULL) { 3331 next_rgroup = TAILQ_FIRST(&rtransport->poll_groups); 3332 } 3333 if (rtransport->conn_sched.next_admin_pg == rgroup) { 3334 rtransport->conn_sched.next_admin_pg = next_rgroup; 3335 } 3336 if (rtransport->conn_sched.next_io_pg == rgroup) { 3337 rtransport->conn_sched.next_io_pg = next_rgroup; 3338 } 3339 pthread_mutex_unlock(&rtransport->lock); 3340 3341 free(rgroup); 3342 } 3343 3344 static void 3345 spdk_nvmf_rdma_qpair_reject_connection(struct spdk_nvmf_rdma_qpair *rqpair) 3346 { 3347 if (rqpair->cm_id != NULL) { 3348 spdk_nvmf_rdma_event_reject(rqpair->cm_id, SPDK_NVMF_RDMA_ERROR_NO_RESOURCES); 3349 } 3350 spdk_nvmf_rdma_qpair_destroy(rqpair); 3351 } 3352 3353 static int 3354 spdk_nvmf_rdma_poll_group_add(struct spdk_nvmf_transport_poll_group *group, 3355 struct spdk_nvmf_qpair *qpair) 3356 { 3357 struct spdk_nvmf_rdma_poll_group *rgroup; 3358 struct spdk_nvmf_rdma_qpair *rqpair; 3359 struct spdk_nvmf_rdma_device *device; 3360 struct spdk_nvmf_rdma_poller *poller; 3361 int rc; 3362 3363 rgroup = SPDK_CONTAINEROF(group, struct spdk_nvmf_rdma_poll_group, group); 3364 rqpair = SPDK_CONTAINEROF(qpair, struct spdk_nvmf_rdma_qpair, qpair); 3365 3366 device = rqpair->port->device; 3367 3368 TAILQ_FOREACH(poller, &rgroup->pollers, link) { 3369 if (poller->device == device) { 3370 break; 3371 } 3372 } 3373 3374 if (!poller) { 3375 SPDK_ERRLOG("No poller found for device.\n"); 3376 return -1; 3377 } 3378 3379 TAILQ_INSERT_TAIL(&poller->qpairs, rqpair, link); 3380 rqpair->poller = poller; 3381 rqpair->srq = rqpair->poller->srq; 3382 3383 rc = spdk_nvmf_rdma_qpair_initialize(qpair); 3384 if (rc < 0) { 3385 SPDK_ERRLOG("Failed to initialize nvmf_rdma_qpair with qpair=%p\n", qpair); 3386 return -1; 3387 } 3388 3389 rc = spdk_nvmf_rdma_event_accept(rqpair->cm_id, rqpair); 3390 if (rc) { 3391 /* Try to reject, but we probably can't */ 3392 spdk_nvmf_rdma_qpair_reject_connection(rqpair); 3393 return -1; 3394 } 3395 3396 spdk_nvmf_rdma_update_ibv_state(rqpair); 3397 3398 return 0; 3399 } 3400 3401 static int 3402 spdk_nvmf_rdma_request_free(struct spdk_nvmf_request *req) 3403 { 3404 struct spdk_nvmf_rdma_request *rdma_req = SPDK_CONTAINEROF(req, struct spdk_nvmf_rdma_request, req); 3405 struct spdk_nvmf_rdma_transport *rtransport = SPDK_CONTAINEROF(req->qpair->transport, 3406 struct spdk_nvmf_rdma_transport, transport); 3407 3408 nvmf_rdma_request_free(rdma_req, rtransport); 3409 return 0; 3410 } 3411 3412 static int 3413 spdk_nvmf_rdma_request_complete(struct spdk_nvmf_request *req) 3414 { 3415 struct spdk_nvmf_rdma_transport *rtransport = SPDK_CONTAINEROF(req->qpair->transport, 3416 struct spdk_nvmf_rdma_transport, transport); 3417 struct spdk_nvmf_rdma_request *rdma_req = SPDK_CONTAINEROF(req, 3418 struct spdk_nvmf_rdma_request, req); 3419 struct spdk_nvmf_rdma_qpair *rqpair = SPDK_CONTAINEROF(rdma_req->req.qpair, 3420 struct spdk_nvmf_rdma_qpair, qpair); 3421 3422 if (rqpair->ibv_state != IBV_QPS_ERR) { 3423 /* The connection is alive, so process the request as normal */ 3424 rdma_req->state = RDMA_REQUEST_STATE_EXECUTED; 3425 } else { 3426 /* The connection is dead. Move the request directly to the completed state. */ 3427 rdma_req->state = RDMA_REQUEST_STATE_COMPLETED; 3428 } 3429 3430 spdk_nvmf_rdma_request_process(rtransport, rdma_req); 3431 3432 return 0; 3433 } 3434 3435 static int 3436 spdk_nvmf_rdma_destroy_defunct_qpair(void *ctx) 3437 { 3438 struct spdk_nvmf_rdma_qpair *rqpair = ctx; 3439 struct spdk_nvmf_rdma_transport *rtransport = SPDK_CONTAINEROF(rqpair->qpair.transport, 3440 struct spdk_nvmf_rdma_transport, transport); 3441 3442 SPDK_INFOLOG(SPDK_LOG_RDMA, "QP#%d hasn't been drained as expected, manually destroy it\n", 3443 rqpair->qpair.qid); 3444 3445 spdk_nvmf_rdma_qpair_process_pending(rtransport, rqpair, true); 3446 spdk_nvmf_rdma_qpair_destroy(rqpair); 3447 3448 return 0; 3449 } 3450 3451 static void 3452 spdk_nvmf_rdma_close_qpair(struct spdk_nvmf_qpair *qpair) 3453 { 3454 struct spdk_nvmf_rdma_qpair *rqpair = SPDK_CONTAINEROF(qpair, struct spdk_nvmf_rdma_qpair, qpair); 3455 3456 if (rqpair->disconnect_flags & RDMA_QP_DISCONNECTING) { 3457 return; 3458 } 3459 3460 rqpair->disconnect_flags |= RDMA_QP_DISCONNECTING; 3461 3462 /* This happens only when the qpair is disconnected before 3463 * it is added to the poll group. Since there is no poll group, 3464 * the RDMA qp has not been initialized yet and the RDMA CM 3465 * event has not yet been acknowledged, so we need to reject it. 3466 */ 3467 if (rqpair->qpair.state == SPDK_NVMF_QPAIR_UNINITIALIZED) { 3468 spdk_nvmf_rdma_qpair_reject_connection(rqpair); 3469 return; 3470 } 3471 3472 if (rqpair->ibv_state != IBV_QPS_ERR) { 3473 spdk_nvmf_rdma_set_ibv_state(rqpair, IBV_QPS_ERR); 3474 } 3475 3476 rqpair->destruct_poller = spdk_poller_register(spdk_nvmf_rdma_destroy_defunct_qpair, (void *)rqpair, 3477 NVMF_RDMA_QPAIR_DESTROY_TIMEOUT_US); 3478 } 3479 3480 static struct spdk_nvmf_rdma_qpair * 3481 get_rdma_qpair_from_wc(struct spdk_nvmf_rdma_poller *rpoller, struct ibv_wc *wc) 3482 { 3483 struct spdk_nvmf_rdma_qpair *rqpair; 3484 /* @todo: improve QP search */ 3485 TAILQ_FOREACH(rqpair, &rpoller->qpairs, link) { 3486 if (wc->qp_num == rqpair->cm_id->qp->qp_num) { 3487 return rqpair; 3488 } 3489 } 3490 SPDK_ERRLOG("Didn't find QP with qp_num %u\n", wc->qp_num); 3491 return NULL; 3492 } 3493 3494 #ifdef DEBUG 3495 static int 3496 spdk_nvmf_rdma_req_is_completing(struct spdk_nvmf_rdma_request *rdma_req) 3497 { 3498 return rdma_req->state == RDMA_REQUEST_STATE_TRANSFERRING_CONTROLLER_TO_HOST || 3499 rdma_req->state == RDMA_REQUEST_STATE_COMPLETING; 3500 } 3501 #endif 3502 3503 static void 3504 _poller_reset_failed_recvs(struct spdk_nvmf_rdma_poller *rpoller, struct ibv_recv_wr *bad_recv_wr, 3505 int rc) 3506 { 3507 struct spdk_nvmf_rdma_recv *rdma_recv; 3508 struct spdk_nvmf_rdma_wr *bad_rdma_wr; 3509 3510 SPDK_ERRLOG("Failed to post a recv for the poller %p with errno %d\n", rpoller, -rc); 3511 while (bad_recv_wr != NULL) { 3512 bad_rdma_wr = (struct spdk_nvmf_rdma_wr *)bad_recv_wr->wr_id; 3513 rdma_recv = SPDK_CONTAINEROF(bad_rdma_wr, struct spdk_nvmf_rdma_recv, rdma_wr); 3514 3515 rdma_recv->qpair->current_recv_depth++; 3516 bad_recv_wr = bad_recv_wr->next; 3517 SPDK_ERRLOG("Failed to post a recv for the qpair %p with errno %d\n", rdma_recv->qpair, -rc); 3518 spdk_nvmf_rdma_start_disconnect(rdma_recv->qpair); 3519 } 3520 } 3521 3522 static void 3523 _qp_reset_failed_recvs(struct spdk_nvmf_rdma_qpair *rqpair, struct ibv_recv_wr *bad_recv_wr, int rc) 3524 { 3525 SPDK_ERRLOG("Failed to post a recv for the qpair %p with errno %d\n", rqpair, -rc); 3526 while (bad_recv_wr != NULL) { 3527 bad_recv_wr = bad_recv_wr->next; 3528 rqpair->current_recv_depth++; 3529 } 3530 spdk_nvmf_rdma_start_disconnect(rqpair); 3531 } 3532 3533 static void 3534 _poller_submit_recvs(struct spdk_nvmf_rdma_transport *rtransport, 3535 struct spdk_nvmf_rdma_poller *rpoller) 3536 { 3537 struct spdk_nvmf_rdma_qpair *rqpair; 3538 struct ibv_recv_wr *bad_recv_wr; 3539 int rc; 3540 3541 if (rpoller->srq) { 3542 if (rpoller->resources->recvs_to_post.first != NULL) { 3543 rc = ibv_post_srq_recv(rpoller->srq, rpoller->resources->recvs_to_post.first, &bad_recv_wr); 3544 if (rc) { 3545 _poller_reset_failed_recvs(rpoller, bad_recv_wr, rc); 3546 } 3547 rpoller->resources->recvs_to_post.first = NULL; 3548 rpoller->resources->recvs_to_post.last = NULL; 3549 } 3550 } else { 3551 while (!STAILQ_EMPTY(&rpoller->qpairs_pending_recv)) { 3552 rqpair = STAILQ_FIRST(&rpoller->qpairs_pending_recv); 3553 assert(rqpair->resources->recvs_to_post.first != NULL); 3554 rc = ibv_post_recv(rqpair->cm_id->qp, rqpair->resources->recvs_to_post.first, &bad_recv_wr); 3555 if (rc) { 3556 _qp_reset_failed_recvs(rqpair, bad_recv_wr, rc); 3557 } 3558 rqpair->resources->recvs_to_post.first = NULL; 3559 rqpair->resources->recvs_to_post.last = NULL; 3560 STAILQ_REMOVE_HEAD(&rpoller->qpairs_pending_recv, recv_link); 3561 } 3562 } 3563 } 3564 3565 static void 3566 _qp_reset_failed_sends(struct spdk_nvmf_rdma_transport *rtransport, 3567 struct spdk_nvmf_rdma_qpair *rqpair, struct ibv_send_wr *bad_wr, int rc) 3568 { 3569 struct spdk_nvmf_rdma_wr *bad_rdma_wr; 3570 struct spdk_nvmf_rdma_request *prev_rdma_req = NULL, *cur_rdma_req = NULL; 3571 3572 SPDK_ERRLOG("Failed to post a send for the qpair %p with errno %d\n", rqpair, -rc); 3573 for (; bad_wr != NULL; bad_wr = bad_wr->next) { 3574 bad_rdma_wr = (struct spdk_nvmf_rdma_wr *)bad_wr->wr_id; 3575 assert(rqpair->current_send_depth > 0); 3576 rqpair->current_send_depth--; 3577 switch (bad_rdma_wr->type) { 3578 case RDMA_WR_TYPE_DATA: 3579 cur_rdma_req = SPDK_CONTAINEROF(bad_rdma_wr, struct spdk_nvmf_rdma_request, data.rdma_wr); 3580 if (bad_wr->opcode == IBV_WR_RDMA_READ) { 3581 assert(rqpair->current_read_depth > 0); 3582 rqpair->current_read_depth--; 3583 } 3584 break; 3585 case RDMA_WR_TYPE_SEND: 3586 cur_rdma_req = SPDK_CONTAINEROF(bad_rdma_wr, struct spdk_nvmf_rdma_request, rsp.rdma_wr); 3587 break; 3588 default: 3589 SPDK_ERRLOG("Found a RECV in the list of pending SEND requests for qpair %p\n", rqpair); 3590 prev_rdma_req = cur_rdma_req; 3591 continue; 3592 } 3593 3594 if (prev_rdma_req == cur_rdma_req) { 3595 /* this request was handled by an earlier wr. i.e. we were performing an nvme read. */ 3596 /* We only have to check against prev_wr since each requests wrs are contiguous in this list. */ 3597 continue; 3598 } 3599 3600 switch (cur_rdma_req->state) { 3601 case RDMA_REQUEST_STATE_TRANSFERRING_HOST_TO_CONTROLLER: 3602 cur_rdma_req->req.rsp->nvme_cpl.status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; 3603 cur_rdma_req->state = RDMA_REQUEST_STATE_READY_TO_COMPLETE; 3604 break; 3605 case RDMA_REQUEST_STATE_TRANSFERRING_CONTROLLER_TO_HOST: 3606 case RDMA_REQUEST_STATE_COMPLETING: 3607 cur_rdma_req->state = RDMA_REQUEST_STATE_COMPLETED; 3608 break; 3609 default: 3610 SPDK_ERRLOG("Found a request in a bad state %d when draining pending SEND requests for qpair %p\n", 3611 cur_rdma_req->state, rqpair); 3612 continue; 3613 } 3614 3615 spdk_nvmf_rdma_request_process(rtransport, cur_rdma_req); 3616 prev_rdma_req = cur_rdma_req; 3617 } 3618 3619 if (rqpair->qpair.state == SPDK_NVMF_QPAIR_ACTIVE) { 3620 /* Disconnect the connection. */ 3621 spdk_nvmf_rdma_start_disconnect(rqpair); 3622 } 3623 3624 } 3625 3626 static void 3627 _poller_submit_sends(struct spdk_nvmf_rdma_transport *rtransport, 3628 struct spdk_nvmf_rdma_poller *rpoller) 3629 { 3630 struct spdk_nvmf_rdma_qpair *rqpair; 3631 struct ibv_send_wr *bad_wr = NULL; 3632 int rc; 3633 3634 while (!STAILQ_EMPTY(&rpoller->qpairs_pending_send)) { 3635 rqpair = STAILQ_FIRST(&rpoller->qpairs_pending_send); 3636 assert(rqpair->sends_to_post.first != NULL); 3637 rc = ibv_post_send(rqpair->cm_id->qp, rqpair->sends_to_post.first, &bad_wr); 3638 3639 /* bad wr always points to the first wr that failed. */ 3640 if (rc) { 3641 _qp_reset_failed_sends(rtransport, rqpair, bad_wr, rc); 3642 } 3643 rqpair->sends_to_post.first = NULL; 3644 rqpair->sends_to_post.last = NULL; 3645 STAILQ_REMOVE_HEAD(&rpoller->qpairs_pending_send, send_link); 3646 } 3647 } 3648 3649 static int 3650 spdk_nvmf_rdma_poller_poll(struct spdk_nvmf_rdma_transport *rtransport, 3651 struct spdk_nvmf_rdma_poller *rpoller) 3652 { 3653 struct ibv_wc wc[32]; 3654 struct spdk_nvmf_rdma_wr *rdma_wr; 3655 struct spdk_nvmf_rdma_request *rdma_req; 3656 struct spdk_nvmf_rdma_recv *rdma_recv; 3657 struct spdk_nvmf_rdma_qpair *rqpair; 3658 int reaped, i; 3659 int count = 0; 3660 bool error = false; 3661 uint64_t poll_tsc = spdk_get_ticks(); 3662 3663 /* Poll for completing operations. */ 3664 reaped = ibv_poll_cq(rpoller->cq, 32, wc); 3665 if (reaped < 0) { 3666 SPDK_ERRLOG("Error polling CQ! (%d): %s\n", 3667 errno, spdk_strerror(errno)); 3668 return -1; 3669 } 3670 3671 rpoller->stat.polls++; 3672 rpoller->stat.completions += reaped; 3673 3674 for (i = 0; i < reaped; i++) { 3675 3676 rdma_wr = (struct spdk_nvmf_rdma_wr *)wc[i].wr_id; 3677 3678 switch (rdma_wr->type) { 3679 case RDMA_WR_TYPE_SEND: 3680 rdma_req = SPDK_CONTAINEROF(rdma_wr, struct spdk_nvmf_rdma_request, rsp.rdma_wr); 3681 rqpair = SPDK_CONTAINEROF(rdma_req->req.qpair, struct spdk_nvmf_rdma_qpair, qpair); 3682 3683 if (!wc[i].status) { 3684 count++; 3685 assert(wc[i].opcode == IBV_WC_SEND); 3686 assert(spdk_nvmf_rdma_req_is_completing(rdma_req)); 3687 } else { 3688 SPDK_ERRLOG("data=%p length=%u\n", rdma_req->req.data, rdma_req->req.length); 3689 } 3690 3691 rdma_req->state = RDMA_REQUEST_STATE_COMPLETED; 3692 /* +1 for the response wr */ 3693 rqpair->current_send_depth -= rdma_req->num_outstanding_data_wr + 1; 3694 rdma_req->num_outstanding_data_wr = 0; 3695 3696 spdk_nvmf_rdma_request_process(rtransport, rdma_req); 3697 break; 3698 case RDMA_WR_TYPE_RECV: 3699 /* rdma_recv->qpair will be invalid if using an SRQ. In that case we have to get the qpair from the wc. */ 3700 rdma_recv = SPDK_CONTAINEROF(rdma_wr, struct spdk_nvmf_rdma_recv, rdma_wr); 3701 if (rpoller->srq != NULL) { 3702 rdma_recv->qpair = get_rdma_qpair_from_wc(rpoller, &wc[i]); 3703 /* It is possible that there are still some completions for destroyed QP 3704 * associated with SRQ. We just ignore these late completions and re-post 3705 * receive WRs back to SRQ. 3706 */ 3707 if (spdk_unlikely(NULL == rdma_recv->qpair)) { 3708 struct ibv_recv_wr *bad_wr; 3709 int rc; 3710 3711 rdma_recv->wr.next = NULL; 3712 rc = ibv_post_srq_recv(rpoller->srq, 3713 &rdma_recv->wr, 3714 &bad_wr); 3715 if (rc) { 3716 SPDK_ERRLOG("Failed to re-post recv WR to SRQ, err %d\n", rc); 3717 } 3718 continue; 3719 } 3720 } 3721 rqpair = rdma_recv->qpair; 3722 3723 assert(rqpair != NULL); 3724 if (!wc[i].status) { 3725 assert(wc[i].opcode == IBV_WC_RECV); 3726 if (rqpair->current_recv_depth >= rqpair->max_queue_depth) { 3727 spdk_nvmf_rdma_start_disconnect(rqpair); 3728 break; 3729 } 3730 } 3731 3732 rdma_recv->wr.next = NULL; 3733 rqpair->current_recv_depth++; 3734 rdma_recv->receive_tsc = poll_tsc; 3735 rpoller->stat.requests++; 3736 STAILQ_INSERT_TAIL(&rqpair->resources->incoming_queue, rdma_recv, link); 3737 break; 3738 case RDMA_WR_TYPE_DATA: 3739 rdma_req = SPDK_CONTAINEROF(rdma_wr, struct spdk_nvmf_rdma_request, data.rdma_wr); 3740 rqpair = SPDK_CONTAINEROF(rdma_req->req.qpair, struct spdk_nvmf_rdma_qpair, qpair); 3741 3742 assert(rdma_req->num_outstanding_data_wr > 0); 3743 3744 rqpair->current_send_depth--; 3745 rdma_req->num_outstanding_data_wr--; 3746 if (!wc[i].status) { 3747 assert(wc[i].opcode == IBV_WC_RDMA_READ); 3748 rqpair->current_read_depth--; 3749 /* wait for all outstanding reads associated with the same rdma_req to complete before proceeding. */ 3750 if (rdma_req->num_outstanding_data_wr == 0) { 3751 rdma_req->state = RDMA_REQUEST_STATE_READY_TO_EXECUTE; 3752 spdk_nvmf_rdma_request_process(rtransport, rdma_req); 3753 } 3754 } else { 3755 /* If the data transfer fails still force the queue into the error state, 3756 * if we were performing an RDMA_READ, we need to force the request into a 3757 * completed state since it wasn't linked to a send. However, in the RDMA_WRITE 3758 * case, we should wait for the SEND to complete. */ 3759 SPDK_ERRLOG("data=%p length=%u\n", rdma_req->req.data, rdma_req->req.length); 3760 if (rdma_req->data.wr.opcode == IBV_WR_RDMA_READ) { 3761 rqpair->current_read_depth--; 3762 if (rdma_req->num_outstanding_data_wr == 0) { 3763 rdma_req->state = RDMA_REQUEST_STATE_COMPLETED; 3764 } 3765 } 3766 } 3767 break; 3768 default: 3769 SPDK_ERRLOG("Received an unknown opcode on the CQ: %d\n", wc[i].opcode); 3770 continue; 3771 } 3772 3773 /* Handle error conditions */ 3774 if (wc[i].status) { 3775 SPDK_DEBUGLOG(SPDK_LOG_RDMA, "CQ error on CQ %p, Request 0x%lu (%d): %s\n", 3776 rpoller->cq, wc[i].wr_id, wc[i].status, ibv_wc_status_str(wc[i].status)); 3777 3778 error = true; 3779 3780 if (rqpair->qpair.state == SPDK_NVMF_QPAIR_ACTIVE) { 3781 /* Disconnect the connection. */ 3782 spdk_nvmf_rdma_start_disconnect(rqpair); 3783 } else { 3784 nvmf_rdma_destroy_drained_qpair(rqpair); 3785 } 3786 continue; 3787 } 3788 3789 spdk_nvmf_rdma_qpair_process_pending(rtransport, rqpair, false); 3790 3791 if (rqpair->qpair.state != SPDK_NVMF_QPAIR_ACTIVE) { 3792 nvmf_rdma_destroy_drained_qpair(rqpair); 3793 } 3794 } 3795 3796 if (error == true) { 3797 return -1; 3798 } 3799 3800 /* submit outstanding work requests. */ 3801 _poller_submit_recvs(rtransport, rpoller); 3802 _poller_submit_sends(rtransport, rpoller); 3803 3804 return count; 3805 } 3806 3807 static int 3808 spdk_nvmf_rdma_poll_group_poll(struct spdk_nvmf_transport_poll_group *group) 3809 { 3810 struct spdk_nvmf_rdma_transport *rtransport; 3811 struct spdk_nvmf_rdma_poll_group *rgroup; 3812 struct spdk_nvmf_rdma_poller *rpoller; 3813 int count, rc; 3814 3815 rtransport = SPDK_CONTAINEROF(group->transport, struct spdk_nvmf_rdma_transport, transport); 3816 rgroup = SPDK_CONTAINEROF(group, struct spdk_nvmf_rdma_poll_group, group); 3817 3818 count = 0; 3819 TAILQ_FOREACH(rpoller, &rgroup->pollers, link) { 3820 rc = spdk_nvmf_rdma_poller_poll(rtransport, rpoller); 3821 if (rc < 0) { 3822 return rc; 3823 } 3824 count += rc; 3825 } 3826 3827 return count; 3828 } 3829 3830 static int 3831 spdk_nvmf_rdma_trid_from_cm_id(struct rdma_cm_id *id, 3832 struct spdk_nvme_transport_id *trid, 3833 bool peer) 3834 { 3835 struct sockaddr *saddr; 3836 uint16_t port; 3837 3838 trid->trtype = SPDK_NVME_TRANSPORT_RDMA; 3839 3840 if (peer) { 3841 saddr = rdma_get_peer_addr(id); 3842 } else { 3843 saddr = rdma_get_local_addr(id); 3844 } 3845 switch (saddr->sa_family) { 3846 case AF_INET: { 3847 struct sockaddr_in *saddr_in = (struct sockaddr_in *)saddr; 3848 3849 trid->adrfam = SPDK_NVMF_ADRFAM_IPV4; 3850 inet_ntop(AF_INET, &saddr_in->sin_addr, 3851 trid->traddr, sizeof(trid->traddr)); 3852 if (peer) { 3853 port = ntohs(rdma_get_dst_port(id)); 3854 } else { 3855 port = ntohs(rdma_get_src_port(id)); 3856 } 3857 snprintf(trid->trsvcid, sizeof(trid->trsvcid), "%u", port); 3858 break; 3859 } 3860 case AF_INET6: { 3861 struct sockaddr_in6 *saddr_in = (struct sockaddr_in6 *)saddr; 3862 trid->adrfam = SPDK_NVMF_ADRFAM_IPV6; 3863 inet_ntop(AF_INET6, &saddr_in->sin6_addr, 3864 trid->traddr, sizeof(trid->traddr)); 3865 if (peer) { 3866 port = ntohs(rdma_get_dst_port(id)); 3867 } else { 3868 port = ntohs(rdma_get_src_port(id)); 3869 } 3870 snprintf(trid->trsvcid, sizeof(trid->trsvcid), "%u", port); 3871 break; 3872 } 3873 default: 3874 return -1; 3875 3876 } 3877 3878 return 0; 3879 } 3880 3881 static int 3882 spdk_nvmf_rdma_qpair_get_peer_trid(struct spdk_nvmf_qpair *qpair, 3883 struct spdk_nvme_transport_id *trid) 3884 { 3885 struct spdk_nvmf_rdma_qpair *rqpair; 3886 3887 rqpair = SPDK_CONTAINEROF(qpair, struct spdk_nvmf_rdma_qpair, qpair); 3888 3889 return spdk_nvmf_rdma_trid_from_cm_id(rqpair->cm_id, trid, true); 3890 } 3891 3892 static int 3893 spdk_nvmf_rdma_qpair_get_local_trid(struct spdk_nvmf_qpair *qpair, 3894 struct spdk_nvme_transport_id *trid) 3895 { 3896 struct spdk_nvmf_rdma_qpair *rqpair; 3897 3898 rqpair = SPDK_CONTAINEROF(qpair, struct spdk_nvmf_rdma_qpair, qpair); 3899 3900 return spdk_nvmf_rdma_trid_from_cm_id(rqpair->cm_id, trid, false); 3901 } 3902 3903 static int 3904 spdk_nvmf_rdma_qpair_get_listen_trid(struct spdk_nvmf_qpair *qpair, 3905 struct spdk_nvme_transport_id *trid) 3906 { 3907 struct spdk_nvmf_rdma_qpair *rqpair; 3908 3909 rqpair = SPDK_CONTAINEROF(qpair, struct spdk_nvmf_rdma_qpair, qpair); 3910 3911 return spdk_nvmf_rdma_trid_from_cm_id(rqpair->listen_id, trid, false); 3912 } 3913 3914 void 3915 spdk_nvmf_rdma_init_hooks(struct spdk_nvme_rdma_hooks *hooks) 3916 { 3917 g_nvmf_hooks = *hooks; 3918 } 3919 3920 static int 3921 spdk_nvmf_rdma_poll_group_get_stat(struct spdk_nvmf_tgt *tgt, 3922 struct spdk_nvmf_transport_poll_group_stat **stat) 3923 { 3924 struct spdk_io_channel *ch; 3925 struct spdk_nvmf_poll_group *group; 3926 struct spdk_nvmf_transport_poll_group *tgroup; 3927 struct spdk_nvmf_rdma_poll_group *rgroup; 3928 struct spdk_nvmf_rdma_poller *rpoller; 3929 struct spdk_nvmf_rdma_device_stat *device_stat; 3930 uint64_t num_devices = 0; 3931 3932 if (tgt == NULL || stat == NULL) { 3933 return -EINVAL; 3934 } 3935 3936 ch = spdk_get_io_channel(tgt); 3937 group = spdk_io_channel_get_ctx(ch);; 3938 spdk_put_io_channel(ch); 3939 TAILQ_FOREACH(tgroup, &group->tgroups, link) { 3940 if (SPDK_NVME_TRANSPORT_RDMA == tgroup->transport->ops->type) { 3941 *stat = calloc(1, sizeof(struct spdk_nvmf_transport_poll_group_stat)); 3942 if (!*stat) { 3943 SPDK_ERRLOG("Failed to allocate memory for NVMf RDMA statistics\n"); 3944 return -ENOMEM; 3945 } 3946 (*stat)->trtype = SPDK_NVME_TRANSPORT_RDMA; 3947 3948 rgroup = SPDK_CONTAINEROF(tgroup, struct spdk_nvmf_rdma_poll_group, group); 3949 /* Count devices to allocate enough memory */ 3950 TAILQ_FOREACH(rpoller, &rgroup->pollers, link) { 3951 ++num_devices; 3952 } 3953 (*stat)->rdma.devices = calloc(num_devices, sizeof(struct spdk_nvmf_rdma_device_stat)); 3954 if (!(*stat)->rdma.devices) { 3955 SPDK_ERRLOG("Failed to allocate NVMf RDMA devices statistics\n"); 3956 free(*stat); 3957 return -ENOMEM; 3958 } 3959 3960 (*stat)->rdma.pending_data_buffer = rgroup->stat.pending_data_buffer; 3961 (*stat)->rdma.num_devices = num_devices; 3962 num_devices = 0; 3963 TAILQ_FOREACH(rpoller, &rgroup->pollers, link) { 3964 device_stat = &(*stat)->rdma.devices[num_devices++]; 3965 device_stat->name = ibv_get_device_name(rpoller->device->context->device); 3966 device_stat->polls = rpoller->stat.polls; 3967 device_stat->completions = rpoller->stat.completions; 3968 device_stat->requests = rpoller->stat.requests; 3969 device_stat->request_latency = rpoller->stat.request_latency; 3970 device_stat->pending_free_request = rpoller->stat.pending_free_request; 3971 device_stat->pending_rdma_read = rpoller->stat.pending_rdma_read; 3972 device_stat->pending_rdma_write = rpoller->stat.pending_rdma_write; 3973 } 3974 return 0; 3975 } 3976 } 3977 return -ENOENT; 3978 } 3979 3980 static void 3981 spdk_nvmf_rdma_poll_group_free_stat(struct spdk_nvmf_transport_poll_group_stat *stat) 3982 { 3983 if (stat) { 3984 free(stat->rdma.devices); 3985 } 3986 free(stat); 3987 } 3988 3989 const struct spdk_nvmf_transport_ops spdk_nvmf_transport_rdma = { 3990 .type = SPDK_NVME_TRANSPORT_RDMA, 3991 .opts_init = spdk_nvmf_rdma_opts_init, 3992 .create = spdk_nvmf_rdma_create, 3993 .destroy = spdk_nvmf_rdma_destroy, 3994 3995 .listen = spdk_nvmf_rdma_listen, 3996 .stop_listen = spdk_nvmf_rdma_stop_listen, 3997 .accept = spdk_nvmf_rdma_accept, 3998 3999 .listener_discover = spdk_nvmf_rdma_discover, 4000 4001 .poll_group_create = spdk_nvmf_rdma_poll_group_create, 4002 .get_optimal_poll_group = spdk_nvmf_rdma_get_optimal_poll_group, 4003 .poll_group_destroy = spdk_nvmf_rdma_poll_group_destroy, 4004 .poll_group_add = spdk_nvmf_rdma_poll_group_add, 4005 .poll_group_poll = spdk_nvmf_rdma_poll_group_poll, 4006 4007 .req_free = spdk_nvmf_rdma_request_free, 4008 .req_complete = spdk_nvmf_rdma_request_complete, 4009 4010 .qpair_fini = spdk_nvmf_rdma_close_qpair, 4011 .qpair_get_peer_trid = spdk_nvmf_rdma_qpair_get_peer_trid, 4012 .qpair_get_local_trid = spdk_nvmf_rdma_qpair_get_local_trid, 4013 .qpair_get_listen_trid = spdk_nvmf_rdma_qpair_get_listen_trid, 4014 4015 .poll_group_get_stat = spdk_nvmf_rdma_poll_group_get_stat, 4016 .poll_group_free_stat = spdk_nvmf_rdma_poll_group_free_stat, 4017 }; 4018 4019 SPDK_LOG_REGISTER_COMPONENT("rdma", SPDK_LOG_RDMA) 4020