1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. All rights reserved. 5 * Copyright (c) 2019 Mellanox Technologies LTD. All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "spdk/stdinc.h" 35 36 #include <infiniband/verbs.h> 37 #include <rdma/rdma_cma.h> 38 #include <rdma/rdma_verbs.h> 39 40 #include "nvmf_internal.h" 41 #include "transport.h" 42 43 #include "spdk/config.h" 44 #include "spdk/thread.h" 45 #include "spdk/nvmf.h" 46 #include "spdk/nvmf_spec.h" 47 #include "spdk/string.h" 48 #include "spdk/trace.h" 49 #include "spdk/util.h" 50 51 #include "spdk_internal/assert.h" 52 #include "spdk_internal/log.h" 53 54 struct spdk_nvme_rdma_hooks g_nvmf_hooks = {}; 55 56 /* 57 RDMA Connection Resource Defaults 58 */ 59 #define NVMF_DEFAULT_TX_SGE SPDK_NVMF_MAX_SGL_ENTRIES 60 #define NVMF_DEFAULT_RSP_SGE 1 61 #define NVMF_DEFAULT_RX_SGE 2 62 63 /* The RDMA completion queue size */ 64 #define DEFAULT_NVMF_RDMA_CQ_SIZE 4096 65 #define MAX_WR_PER_QP(queue_depth) (queue_depth * 3 + 2) 66 67 /* Timeout for destroying defunct rqpairs */ 68 #define NVMF_RDMA_QPAIR_DESTROY_TIMEOUT_US 4000000 69 70 /* The maximum number of buffers per request */ 71 #define NVMF_REQ_MAX_BUFFERS (SPDK_NVMF_MAX_SGL_ENTRIES * 2) 72 73 static int g_spdk_nvmf_ibv_query_mask = 74 IBV_QP_STATE | 75 IBV_QP_PKEY_INDEX | 76 IBV_QP_PORT | 77 IBV_QP_ACCESS_FLAGS | 78 IBV_QP_AV | 79 IBV_QP_PATH_MTU | 80 IBV_QP_DEST_QPN | 81 IBV_QP_RQ_PSN | 82 IBV_QP_MAX_DEST_RD_ATOMIC | 83 IBV_QP_MIN_RNR_TIMER | 84 IBV_QP_SQ_PSN | 85 IBV_QP_TIMEOUT | 86 IBV_QP_RETRY_CNT | 87 IBV_QP_RNR_RETRY | 88 IBV_QP_MAX_QP_RD_ATOMIC; 89 90 enum spdk_nvmf_rdma_request_state { 91 /* The request is not currently in use */ 92 RDMA_REQUEST_STATE_FREE = 0, 93 94 /* Initial state when request first received */ 95 RDMA_REQUEST_STATE_NEW, 96 97 /* The request is queued until a data buffer is available. */ 98 RDMA_REQUEST_STATE_NEED_BUFFER, 99 100 /* The request is waiting on RDMA queue depth availability 101 * to transfer data from the host to the controller. 102 */ 103 RDMA_REQUEST_STATE_DATA_TRANSFER_TO_CONTROLLER_PENDING, 104 105 /* The request is currently transferring data from the host to the controller. */ 106 RDMA_REQUEST_STATE_TRANSFERRING_HOST_TO_CONTROLLER, 107 108 /* The request is ready to execute at the block device */ 109 RDMA_REQUEST_STATE_READY_TO_EXECUTE, 110 111 /* The request is currently executing at the block device */ 112 RDMA_REQUEST_STATE_EXECUTING, 113 114 /* The request finished executing at the block device */ 115 RDMA_REQUEST_STATE_EXECUTED, 116 117 /* The request is waiting on RDMA queue depth availability 118 * to transfer data from the controller to the host. 119 */ 120 RDMA_REQUEST_STATE_DATA_TRANSFER_TO_HOST_PENDING, 121 122 /* The request is ready to send a completion */ 123 RDMA_REQUEST_STATE_READY_TO_COMPLETE, 124 125 /* The request is currently transferring data from the controller to the host. */ 126 RDMA_REQUEST_STATE_TRANSFERRING_CONTROLLER_TO_HOST, 127 128 /* The request currently has an outstanding completion without an 129 * associated data transfer. 130 */ 131 RDMA_REQUEST_STATE_COMPLETING, 132 133 /* The request completed and can be marked free. */ 134 RDMA_REQUEST_STATE_COMPLETED, 135 136 /* Terminator */ 137 RDMA_REQUEST_NUM_STATES, 138 }; 139 140 #define OBJECT_NVMF_RDMA_IO 0x40 141 142 #define TRACE_GROUP_NVMF_RDMA 0x4 143 #define TRACE_RDMA_REQUEST_STATE_NEW SPDK_TPOINT_ID(TRACE_GROUP_NVMF_RDMA, 0x0) 144 #define TRACE_RDMA_REQUEST_STATE_NEED_BUFFER SPDK_TPOINT_ID(TRACE_GROUP_NVMF_RDMA, 0x1) 145 #define TRACE_RDMA_REQUEST_STATE_DATA_TRANSFER_TO_CONTROLLER_PENDING SPDK_TPOINT_ID(TRACE_GROUP_NVMF_RDMA, 0x2) 146 #define TRACE_RDMA_REQUEST_STATE_TRANSFERRING_HOST_TO_CONTROLLER SPDK_TPOINT_ID(TRACE_GROUP_NVMF_RDMA, 0x3) 147 #define TRACE_RDMA_REQUEST_STATE_READY_TO_EXECUTE SPDK_TPOINT_ID(TRACE_GROUP_NVMF_RDMA, 0x4) 148 #define TRACE_RDMA_REQUEST_STATE_EXECUTING SPDK_TPOINT_ID(TRACE_GROUP_NVMF_RDMA, 0x5) 149 #define TRACE_RDMA_REQUEST_STATE_EXECUTED SPDK_TPOINT_ID(TRACE_GROUP_NVMF_RDMA, 0x6) 150 #define TRACE_RDMA_REQUEST_STATE_DATA_TRANSFER_TO_HOST_PENDING SPDK_TPOINT_ID(TRACE_GROUP_NVMF_RDMA, 0x7) 151 #define TRACE_RDMA_REQUEST_STATE_READY_TO_COMPLETE SPDK_TPOINT_ID(TRACE_GROUP_NVMF_RDMA, 0x8) 152 #define TRACE_RDMA_REQUEST_STATE_TRANSFERRING_CONTROLLER_TO_HOST SPDK_TPOINT_ID(TRACE_GROUP_NVMF_RDMA, 0x9) 153 #define TRACE_RDMA_REQUEST_STATE_COMPLETING SPDK_TPOINT_ID(TRACE_GROUP_NVMF_RDMA, 0xA) 154 #define TRACE_RDMA_REQUEST_STATE_COMPLETED SPDK_TPOINT_ID(TRACE_GROUP_NVMF_RDMA, 0xB) 155 #define TRACE_RDMA_QP_CREATE SPDK_TPOINT_ID(TRACE_GROUP_NVMF_RDMA, 0xC) 156 #define TRACE_RDMA_IBV_ASYNC_EVENT SPDK_TPOINT_ID(TRACE_GROUP_NVMF_RDMA, 0xD) 157 #define TRACE_RDMA_CM_ASYNC_EVENT SPDK_TPOINT_ID(TRACE_GROUP_NVMF_RDMA, 0xE) 158 #define TRACE_RDMA_QP_STATE_CHANGE SPDK_TPOINT_ID(TRACE_GROUP_NVMF_RDMA, 0xF) 159 #define TRACE_RDMA_QP_DISCONNECT SPDK_TPOINT_ID(TRACE_GROUP_NVMF_RDMA, 0x10) 160 #define TRACE_RDMA_QP_DESTROY SPDK_TPOINT_ID(TRACE_GROUP_NVMF_RDMA, 0x11) 161 162 SPDK_TRACE_REGISTER_FN(nvmf_trace, "nvmf_rdma", TRACE_GROUP_NVMF_RDMA) 163 { 164 spdk_trace_register_object(OBJECT_NVMF_RDMA_IO, 'r'); 165 spdk_trace_register_description("RDMA_REQ_NEW", TRACE_RDMA_REQUEST_STATE_NEW, 166 OWNER_NONE, OBJECT_NVMF_RDMA_IO, 1, 1, "cmid: "); 167 spdk_trace_register_description("RDMA_REQ_NEED_BUFFER", TRACE_RDMA_REQUEST_STATE_NEED_BUFFER, 168 OWNER_NONE, OBJECT_NVMF_RDMA_IO, 0, 1, "cmid: "); 169 spdk_trace_register_description("RDMA_REQ_TX_PENDING_C2H", 170 TRACE_RDMA_REQUEST_STATE_DATA_TRANSFER_TO_HOST_PENDING, 171 OWNER_NONE, OBJECT_NVMF_RDMA_IO, 0, 1, "cmid: "); 172 spdk_trace_register_description("RDMA_REQ_TX_PENDING_H2C", 173 TRACE_RDMA_REQUEST_STATE_DATA_TRANSFER_TO_CONTROLLER_PENDING, 174 OWNER_NONE, OBJECT_NVMF_RDMA_IO, 0, 1, "cmid: "); 175 spdk_trace_register_description("RDMA_REQ_TX_H2C", 176 TRACE_RDMA_REQUEST_STATE_TRANSFERRING_HOST_TO_CONTROLLER, 177 OWNER_NONE, OBJECT_NVMF_RDMA_IO, 0, 1, "cmid: "); 178 spdk_trace_register_description("RDMA_REQ_RDY_TO_EXECUTE", 179 TRACE_RDMA_REQUEST_STATE_READY_TO_EXECUTE, 180 OWNER_NONE, OBJECT_NVMF_RDMA_IO, 0, 1, "cmid: "); 181 spdk_trace_register_description("RDMA_REQ_EXECUTING", 182 TRACE_RDMA_REQUEST_STATE_EXECUTING, 183 OWNER_NONE, OBJECT_NVMF_RDMA_IO, 0, 1, "cmid: "); 184 spdk_trace_register_description("RDMA_REQ_EXECUTED", 185 TRACE_RDMA_REQUEST_STATE_EXECUTED, 186 OWNER_NONE, OBJECT_NVMF_RDMA_IO, 0, 1, "cmid: "); 187 spdk_trace_register_description("RDMA_REQ_RDY_TO_COMPL", 188 TRACE_RDMA_REQUEST_STATE_READY_TO_COMPLETE, 189 OWNER_NONE, OBJECT_NVMF_RDMA_IO, 0, 1, "cmid: "); 190 spdk_trace_register_description("RDMA_REQ_COMPLETING_C2H", 191 TRACE_RDMA_REQUEST_STATE_TRANSFERRING_CONTROLLER_TO_HOST, 192 OWNER_NONE, OBJECT_NVMF_RDMA_IO, 0, 1, "cmid: "); 193 spdk_trace_register_description("RDMA_REQ_COMPLETING", 194 TRACE_RDMA_REQUEST_STATE_COMPLETING, 195 OWNER_NONE, OBJECT_NVMF_RDMA_IO, 0, 1, "cmid: "); 196 spdk_trace_register_description("RDMA_REQ_COMPLETED", 197 TRACE_RDMA_REQUEST_STATE_COMPLETED, 198 OWNER_NONE, OBJECT_NVMF_RDMA_IO, 0, 1, "cmid: "); 199 200 spdk_trace_register_description("RDMA_QP_CREATE", TRACE_RDMA_QP_CREATE, 201 OWNER_NONE, OBJECT_NONE, 0, 0, ""); 202 spdk_trace_register_description("RDMA_IBV_ASYNC_EVENT", TRACE_RDMA_IBV_ASYNC_EVENT, 203 OWNER_NONE, OBJECT_NONE, 0, 0, "type: "); 204 spdk_trace_register_description("RDMA_CM_ASYNC_EVENT", TRACE_RDMA_CM_ASYNC_EVENT, 205 OWNER_NONE, OBJECT_NONE, 0, 0, "type: "); 206 spdk_trace_register_description("RDMA_QP_STATE_CHANGE", TRACE_RDMA_QP_STATE_CHANGE, 207 OWNER_NONE, OBJECT_NONE, 0, 1, "state: "); 208 spdk_trace_register_description("RDMA_QP_DISCONNECT", TRACE_RDMA_QP_DISCONNECT, 209 OWNER_NONE, OBJECT_NONE, 0, 0, ""); 210 spdk_trace_register_description("RDMA_QP_DESTROY", TRACE_RDMA_QP_DESTROY, 211 OWNER_NONE, OBJECT_NONE, 0, 0, ""); 212 } 213 214 enum spdk_nvmf_rdma_wr_type { 215 RDMA_WR_TYPE_RECV, 216 RDMA_WR_TYPE_SEND, 217 RDMA_WR_TYPE_DATA, 218 }; 219 220 struct spdk_nvmf_rdma_wr { 221 enum spdk_nvmf_rdma_wr_type type; 222 }; 223 224 /* This structure holds commands as they are received off the wire. 225 * It must be dynamically paired with a full request object 226 * (spdk_nvmf_rdma_request) to service a request. It is separate 227 * from the request because RDMA does not appear to order 228 * completions, so occasionally we'll get a new incoming 229 * command when there aren't any free request objects. 230 */ 231 struct spdk_nvmf_rdma_recv { 232 struct ibv_recv_wr wr; 233 struct ibv_sge sgl[NVMF_DEFAULT_RX_SGE]; 234 235 struct spdk_nvmf_rdma_qpair *qpair; 236 237 /* In-capsule data buffer */ 238 uint8_t *buf; 239 240 struct spdk_nvmf_rdma_wr rdma_wr; 241 uint64_t receive_tsc; 242 243 STAILQ_ENTRY(spdk_nvmf_rdma_recv) link; 244 }; 245 246 struct spdk_nvmf_rdma_request_data { 247 struct spdk_nvmf_rdma_wr rdma_wr; 248 struct ibv_send_wr wr; 249 struct ibv_sge sgl[SPDK_NVMF_MAX_SGL_ENTRIES]; 250 }; 251 252 struct spdk_nvmf_rdma_request { 253 struct spdk_nvmf_request req; 254 255 enum spdk_nvmf_rdma_request_state state; 256 257 struct spdk_nvmf_rdma_recv *recv; 258 259 struct { 260 struct spdk_nvmf_rdma_wr rdma_wr; 261 struct ibv_send_wr wr; 262 struct ibv_sge sgl[NVMF_DEFAULT_RSP_SGE]; 263 } rsp; 264 265 struct spdk_nvmf_rdma_request_data data; 266 267 uint32_t iovpos; 268 269 uint32_t num_outstanding_data_wr; 270 uint64_t receive_tsc; 271 272 STAILQ_ENTRY(spdk_nvmf_rdma_request) state_link; 273 }; 274 275 enum spdk_nvmf_rdma_qpair_disconnect_flags { 276 RDMA_QP_DISCONNECTING = 1, 277 RDMA_QP_RECV_DRAINED = 1 << 1, 278 RDMA_QP_SEND_DRAINED = 1 << 2 279 }; 280 281 struct spdk_nvmf_rdma_resource_opts { 282 struct spdk_nvmf_rdma_qpair *qpair; 283 /* qp points either to an ibv_qp object or an ibv_srq object depending on the value of shared. */ 284 void *qp; 285 struct ibv_pd *pd; 286 uint32_t max_queue_depth; 287 uint32_t in_capsule_data_size; 288 bool shared; 289 }; 290 291 struct spdk_nvmf_send_wr_list { 292 struct ibv_send_wr *first; 293 struct ibv_send_wr *last; 294 }; 295 296 struct spdk_nvmf_recv_wr_list { 297 struct ibv_recv_wr *first; 298 struct ibv_recv_wr *last; 299 }; 300 301 struct spdk_nvmf_rdma_resources { 302 /* Array of size "max_queue_depth" containing RDMA requests. */ 303 struct spdk_nvmf_rdma_request *reqs; 304 305 /* Array of size "max_queue_depth" containing RDMA recvs. */ 306 struct spdk_nvmf_rdma_recv *recvs; 307 308 /* Array of size "max_queue_depth" containing 64 byte capsules 309 * used for receive. 310 */ 311 union nvmf_h2c_msg *cmds; 312 struct ibv_mr *cmds_mr; 313 314 /* Array of size "max_queue_depth" containing 16 byte completions 315 * to be sent back to the user. 316 */ 317 union nvmf_c2h_msg *cpls; 318 struct ibv_mr *cpls_mr; 319 320 /* Array of size "max_queue_depth * InCapsuleDataSize" containing 321 * buffers to be used for in capsule data. 322 */ 323 void *bufs; 324 struct ibv_mr *bufs_mr; 325 326 /* The list of pending recvs to transfer */ 327 struct spdk_nvmf_recv_wr_list recvs_to_post; 328 329 /* Receives that are waiting for a request object */ 330 STAILQ_HEAD(, spdk_nvmf_rdma_recv) incoming_queue; 331 332 /* Queue to track free requests */ 333 STAILQ_HEAD(, spdk_nvmf_rdma_request) free_queue; 334 }; 335 336 struct spdk_nvmf_rdma_qpair { 337 struct spdk_nvmf_qpair qpair; 338 339 struct spdk_nvmf_rdma_port *port; 340 struct spdk_nvmf_rdma_poller *poller; 341 342 struct rdma_cm_id *cm_id; 343 struct ibv_srq *srq; 344 struct rdma_cm_id *listen_id; 345 346 /* The maximum number of I/O outstanding on this connection at one time */ 347 uint16_t max_queue_depth; 348 349 /* The maximum number of active RDMA READ and ATOMIC operations at one time */ 350 uint16_t max_read_depth; 351 352 /* The maximum number of RDMA SEND operations at one time */ 353 uint32_t max_send_depth; 354 355 /* The current number of outstanding WRs from this qpair's 356 * recv queue. Should not exceed device->attr.max_queue_depth. 357 */ 358 uint16_t current_recv_depth; 359 360 /* The current number of active RDMA READ operations */ 361 uint16_t current_read_depth; 362 363 /* The current number of posted WRs from this qpair's 364 * send queue. Should not exceed max_send_depth. 365 */ 366 uint32_t current_send_depth; 367 368 /* The maximum number of SGEs per WR on the send queue */ 369 uint32_t max_send_sge; 370 371 /* The maximum number of SGEs per WR on the recv queue */ 372 uint32_t max_recv_sge; 373 374 /* The list of pending send requests for a transfer */ 375 struct spdk_nvmf_send_wr_list sends_to_post; 376 377 struct spdk_nvmf_rdma_resources *resources; 378 379 STAILQ_HEAD(, spdk_nvmf_rdma_request) pending_rdma_read_queue; 380 381 STAILQ_HEAD(, spdk_nvmf_rdma_request) pending_rdma_write_queue; 382 383 /* Number of requests not in the free state */ 384 uint32_t qd; 385 386 TAILQ_ENTRY(spdk_nvmf_rdma_qpair) link; 387 388 STAILQ_ENTRY(spdk_nvmf_rdma_qpair) recv_link; 389 390 STAILQ_ENTRY(spdk_nvmf_rdma_qpair) send_link; 391 392 /* IBV queue pair attributes: they are used to manage 393 * qp state and recover from errors. 394 */ 395 enum ibv_qp_state ibv_state; 396 397 uint32_t disconnect_flags; 398 399 /* Poller registered in case the qpair doesn't properly 400 * complete the qpair destruct process and becomes defunct. 401 */ 402 403 struct spdk_poller *destruct_poller; 404 405 /* There are several ways a disconnect can start on a qpair 406 * and they are not all mutually exclusive. It is important 407 * that we only initialize one of these paths. 408 */ 409 bool disconnect_started; 410 /* Lets us know that we have received the last_wqe event. */ 411 bool last_wqe_reached; 412 }; 413 414 struct spdk_nvmf_rdma_poller_stat { 415 uint64_t completions; 416 uint64_t polls; 417 uint64_t requests; 418 uint64_t request_latency; 419 uint64_t pending_free_request; 420 uint64_t pending_rdma_read; 421 uint64_t pending_rdma_write; 422 }; 423 424 struct spdk_nvmf_rdma_poller { 425 struct spdk_nvmf_rdma_device *device; 426 struct spdk_nvmf_rdma_poll_group *group; 427 428 int num_cqe; 429 int required_num_wr; 430 struct ibv_cq *cq; 431 432 /* The maximum number of I/O outstanding on the shared receive queue at one time */ 433 uint16_t max_srq_depth; 434 435 /* Shared receive queue */ 436 struct ibv_srq *srq; 437 438 struct spdk_nvmf_rdma_resources *resources; 439 struct spdk_nvmf_rdma_poller_stat stat; 440 441 TAILQ_HEAD(, spdk_nvmf_rdma_qpair) qpairs; 442 443 STAILQ_HEAD(, spdk_nvmf_rdma_qpair) qpairs_pending_recv; 444 445 STAILQ_HEAD(, spdk_nvmf_rdma_qpair) qpairs_pending_send; 446 447 TAILQ_ENTRY(spdk_nvmf_rdma_poller) link; 448 }; 449 450 struct spdk_nvmf_rdma_poll_group_stat { 451 uint64_t pending_data_buffer; 452 }; 453 454 struct spdk_nvmf_rdma_poll_group { 455 struct spdk_nvmf_transport_poll_group group; 456 struct spdk_nvmf_rdma_poll_group_stat stat; 457 TAILQ_HEAD(, spdk_nvmf_rdma_poller) pollers; 458 TAILQ_ENTRY(spdk_nvmf_rdma_poll_group) link; 459 /* 460 * buffers which are split across multiple RDMA 461 * memory regions cannot be used by this transport. 462 */ 463 STAILQ_HEAD(, spdk_nvmf_transport_pg_cache_buf) retired_bufs; 464 }; 465 466 struct spdk_nvmf_rdma_conn_sched { 467 struct spdk_nvmf_rdma_poll_group *next_admin_pg; 468 struct spdk_nvmf_rdma_poll_group *next_io_pg; 469 }; 470 471 /* Assuming rdma_cm uses just one protection domain per ibv_context. */ 472 struct spdk_nvmf_rdma_device { 473 struct ibv_device_attr attr; 474 struct ibv_context *context; 475 476 struct spdk_mem_map *map; 477 struct ibv_pd *pd; 478 479 int num_srq; 480 481 TAILQ_ENTRY(spdk_nvmf_rdma_device) link; 482 }; 483 484 struct spdk_nvmf_rdma_port { 485 struct spdk_nvme_transport_id trid; 486 struct rdma_cm_id *id; 487 struct spdk_nvmf_rdma_device *device; 488 uint32_t ref; 489 TAILQ_ENTRY(spdk_nvmf_rdma_port) link; 490 }; 491 492 struct spdk_nvmf_rdma_transport { 493 struct spdk_nvmf_transport transport; 494 495 struct spdk_nvmf_rdma_conn_sched conn_sched; 496 497 struct rdma_event_channel *event_channel; 498 499 struct spdk_mempool *data_wr_pool; 500 501 pthread_mutex_t lock; 502 503 /* fields used to poll RDMA/IB events */ 504 nfds_t npoll_fds; 505 struct pollfd *poll_fds; 506 507 TAILQ_HEAD(, spdk_nvmf_rdma_device) devices; 508 TAILQ_HEAD(, spdk_nvmf_rdma_port) ports; 509 TAILQ_HEAD(, spdk_nvmf_rdma_poll_group) poll_groups; 510 }; 511 512 static inline void 513 spdk_nvmf_rdma_start_disconnect(struct spdk_nvmf_rdma_qpair *rqpair); 514 515 static inline int 516 spdk_nvmf_rdma_check_ibv_state(enum ibv_qp_state state) 517 { 518 switch (state) { 519 case IBV_QPS_RESET: 520 case IBV_QPS_INIT: 521 case IBV_QPS_RTR: 522 case IBV_QPS_RTS: 523 case IBV_QPS_SQD: 524 case IBV_QPS_SQE: 525 case IBV_QPS_ERR: 526 return 0; 527 default: 528 return -1; 529 } 530 } 531 532 static inline enum spdk_nvme_media_error_status_code 533 spdk_nvmf_rdma_dif_error_to_compl_status(uint8_t err_type) { 534 enum spdk_nvme_media_error_status_code result; 535 switch (err_type) 536 { 537 case SPDK_DIF_REFTAG_ERROR: 538 result = SPDK_NVME_SC_REFERENCE_TAG_CHECK_ERROR; 539 break; 540 case SPDK_DIF_APPTAG_ERROR: 541 result = SPDK_NVME_SC_APPLICATION_TAG_CHECK_ERROR; 542 break; 543 case SPDK_DIF_GUARD_ERROR: 544 result = SPDK_NVME_SC_GUARD_CHECK_ERROR; 545 break; 546 default: 547 SPDK_UNREACHABLE(); 548 } 549 550 return result; 551 } 552 553 static enum ibv_qp_state 554 spdk_nvmf_rdma_update_ibv_state(struct spdk_nvmf_rdma_qpair *rqpair) { 555 enum ibv_qp_state old_state, new_state; 556 struct ibv_qp_attr qp_attr; 557 struct ibv_qp_init_attr init_attr; 558 int rc; 559 560 old_state = rqpair->ibv_state; 561 rc = ibv_query_qp(rqpair->cm_id->qp, &qp_attr, 562 g_spdk_nvmf_ibv_query_mask, &init_attr); 563 564 if (rc) 565 { 566 SPDK_ERRLOG("Failed to get updated RDMA queue pair state!\n"); 567 return IBV_QPS_ERR + 1; 568 } 569 570 new_state = qp_attr.qp_state; 571 rqpair->ibv_state = new_state; 572 qp_attr.ah_attr.port_num = qp_attr.port_num; 573 574 rc = spdk_nvmf_rdma_check_ibv_state(new_state); 575 if (rc) 576 { 577 SPDK_ERRLOG("QP#%d: bad state updated: %u, maybe hardware issue\n", rqpair->qpair.qid, new_state); 578 /* 579 * IBV_QPS_UNKNOWN undefined if lib version smaller than libibverbs-1.1.8 580 * IBV_QPS_UNKNOWN is the enum element after IBV_QPS_ERR 581 */ 582 return IBV_QPS_ERR + 1; 583 } 584 585 if (old_state != new_state) 586 { 587 spdk_trace_record(TRACE_RDMA_QP_STATE_CHANGE, 0, 0, 588 (uintptr_t)rqpair->cm_id, new_state); 589 } 590 return new_state; 591 } 592 593 static const char *str_ibv_qp_state[] = { 594 "IBV_QPS_RESET", 595 "IBV_QPS_INIT", 596 "IBV_QPS_RTR", 597 "IBV_QPS_RTS", 598 "IBV_QPS_SQD", 599 "IBV_QPS_SQE", 600 "IBV_QPS_ERR", 601 "IBV_QPS_UNKNOWN" 602 }; 603 604 static int 605 spdk_nvmf_rdma_set_ibv_state(struct spdk_nvmf_rdma_qpair *rqpair, 606 enum ibv_qp_state new_state) 607 { 608 struct ibv_qp_attr qp_attr; 609 struct ibv_qp_init_attr init_attr; 610 int rc; 611 enum ibv_qp_state state; 612 static int attr_mask_rc[] = { 613 [IBV_QPS_RESET] = IBV_QP_STATE, 614 [IBV_QPS_INIT] = (IBV_QP_STATE | 615 IBV_QP_PKEY_INDEX | 616 IBV_QP_PORT | 617 IBV_QP_ACCESS_FLAGS), 618 [IBV_QPS_RTR] = (IBV_QP_STATE | 619 IBV_QP_AV | 620 IBV_QP_PATH_MTU | 621 IBV_QP_DEST_QPN | 622 IBV_QP_RQ_PSN | 623 IBV_QP_MAX_DEST_RD_ATOMIC | 624 IBV_QP_MIN_RNR_TIMER), 625 [IBV_QPS_RTS] = (IBV_QP_STATE | 626 IBV_QP_SQ_PSN | 627 IBV_QP_TIMEOUT | 628 IBV_QP_RETRY_CNT | 629 IBV_QP_RNR_RETRY | 630 IBV_QP_MAX_QP_RD_ATOMIC), 631 [IBV_QPS_SQD] = IBV_QP_STATE, 632 [IBV_QPS_SQE] = IBV_QP_STATE, 633 [IBV_QPS_ERR] = IBV_QP_STATE, 634 }; 635 636 rc = spdk_nvmf_rdma_check_ibv_state(new_state); 637 if (rc) { 638 SPDK_ERRLOG("QP#%d: bad state requested: %u\n", 639 rqpair->qpair.qid, new_state); 640 return rc; 641 } 642 643 rc = ibv_query_qp(rqpair->cm_id->qp, &qp_attr, 644 g_spdk_nvmf_ibv_query_mask, &init_attr); 645 646 if (rc) { 647 SPDK_ERRLOG("Failed to get updated RDMA queue pair state!\n"); 648 assert(false); 649 } 650 651 qp_attr.cur_qp_state = rqpair->ibv_state; 652 qp_attr.qp_state = new_state; 653 654 rc = ibv_modify_qp(rqpair->cm_id->qp, &qp_attr, 655 attr_mask_rc[new_state]); 656 657 if (rc) { 658 SPDK_ERRLOG("QP#%d: failed to set state to: %s, %d (%s)\n", 659 rqpair->qpair.qid, str_ibv_qp_state[new_state], errno, strerror(errno)); 660 return rc; 661 } 662 663 state = spdk_nvmf_rdma_update_ibv_state(rqpair); 664 665 if (state != new_state) { 666 SPDK_ERRLOG("QP#%d: expected state: %s, actual state: %s\n", 667 rqpair->qpair.qid, str_ibv_qp_state[new_state], 668 str_ibv_qp_state[state]); 669 return -1; 670 } 671 SPDK_DEBUGLOG(SPDK_LOG_RDMA, "IBV QP#%u changed to: %s\n", rqpair->qpair.qid, 672 str_ibv_qp_state[state]); 673 return 0; 674 } 675 676 static void 677 nvmf_rdma_request_free_data(struct spdk_nvmf_rdma_request *rdma_req, 678 struct spdk_nvmf_rdma_transport *rtransport) 679 { 680 struct spdk_nvmf_rdma_request_data *data_wr; 681 struct ibv_send_wr *next_send_wr; 682 uint64_t req_wrid; 683 684 rdma_req->num_outstanding_data_wr = 0; 685 data_wr = &rdma_req->data; 686 req_wrid = data_wr->wr.wr_id; 687 while (data_wr && data_wr->wr.wr_id == req_wrid) { 688 memset(data_wr->sgl, 0, sizeof(data_wr->wr.sg_list[0]) * data_wr->wr.num_sge); 689 data_wr->wr.num_sge = 0; 690 next_send_wr = data_wr->wr.next; 691 if (data_wr != &rdma_req->data) { 692 spdk_mempool_put(rtransport->data_wr_pool, data_wr); 693 } 694 data_wr = (!next_send_wr || next_send_wr == &rdma_req->rsp.wr) ? NULL : 695 SPDK_CONTAINEROF(next_send_wr, struct spdk_nvmf_rdma_request_data, wr); 696 } 697 } 698 699 static void 700 nvmf_rdma_dump_request(struct spdk_nvmf_rdma_request *req) 701 { 702 SPDK_ERRLOG("\t\tRequest Data From Pool: %d\n", req->req.data_from_pool); 703 if (req->req.cmd) { 704 SPDK_ERRLOG("\t\tRequest opcode: %d\n", req->req.cmd->nvmf_cmd.opcode); 705 } 706 if (req->recv) { 707 SPDK_ERRLOG("\t\tRequest recv wr_id%lu\n", req->recv->wr.wr_id); 708 } 709 } 710 711 static void 712 nvmf_rdma_dump_qpair_contents(struct spdk_nvmf_rdma_qpair *rqpair) 713 { 714 int i; 715 716 SPDK_ERRLOG("Dumping contents of queue pair (QID %d)\n", rqpair->qpair.qid); 717 for (i = 0; i < rqpair->max_queue_depth; i++) { 718 if (rqpair->resources->reqs[i].state != RDMA_REQUEST_STATE_FREE) { 719 nvmf_rdma_dump_request(&rqpair->resources->reqs[i]); 720 } 721 } 722 } 723 724 static void 725 nvmf_rdma_resources_destroy(struct spdk_nvmf_rdma_resources *resources) 726 { 727 if (resources->cmds_mr) { 728 ibv_dereg_mr(resources->cmds_mr); 729 } 730 731 if (resources->cpls_mr) { 732 ibv_dereg_mr(resources->cpls_mr); 733 } 734 735 if (resources->bufs_mr) { 736 ibv_dereg_mr(resources->bufs_mr); 737 } 738 739 spdk_free(resources->cmds); 740 spdk_free(resources->cpls); 741 spdk_free(resources->bufs); 742 free(resources->reqs); 743 free(resources->recvs); 744 free(resources); 745 } 746 747 748 static struct spdk_nvmf_rdma_resources * 749 nvmf_rdma_resources_create(struct spdk_nvmf_rdma_resource_opts *opts) 750 { 751 struct spdk_nvmf_rdma_resources *resources; 752 struct spdk_nvmf_rdma_request *rdma_req; 753 struct spdk_nvmf_rdma_recv *rdma_recv; 754 struct ibv_qp *qp; 755 struct ibv_srq *srq; 756 uint32_t i; 757 int rc; 758 759 resources = calloc(1, sizeof(struct spdk_nvmf_rdma_resources)); 760 if (!resources) { 761 SPDK_ERRLOG("Unable to allocate resources for receive queue.\n"); 762 return NULL; 763 } 764 765 resources->reqs = calloc(opts->max_queue_depth, sizeof(*resources->reqs)); 766 resources->recvs = calloc(opts->max_queue_depth, sizeof(*resources->recvs)); 767 resources->cmds = spdk_zmalloc(opts->max_queue_depth * sizeof(*resources->cmds), 768 0x1000, NULL, SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA); 769 resources->cpls = spdk_zmalloc(opts->max_queue_depth * sizeof(*resources->cpls), 770 0x1000, NULL, SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA); 771 772 if (opts->in_capsule_data_size > 0) { 773 resources->bufs = spdk_zmalloc(opts->max_queue_depth * opts->in_capsule_data_size, 774 0x1000, NULL, SPDK_ENV_LCORE_ID_ANY, 775 SPDK_MALLOC_DMA); 776 } 777 778 if (!resources->reqs || !resources->recvs || !resources->cmds || 779 !resources->cpls || (opts->in_capsule_data_size && !resources->bufs)) { 780 SPDK_ERRLOG("Unable to allocate sufficient memory for RDMA queue.\n"); 781 goto cleanup; 782 } 783 784 resources->cmds_mr = ibv_reg_mr(opts->pd, resources->cmds, 785 opts->max_queue_depth * sizeof(*resources->cmds), 786 IBV_ACCESS_LOCAL_WRITE); 787 resources->cpls_mr = ibv_reg_mr(opts->pd, resources->cpls, 788 opts->max_queue_depth * sizeof(*resources->cpls), 789 0); 790 791 if (opts->in_capsule_data_size) { 792 resources->bufs_mr = ibv_reg_mr(opts->pd, resources->bufs, 793 opts->max_queue_depth * 794 opts->in_capsule_data_size, 795 IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE); 796 } 797 798 if (!resources->cmds_mr || !resources->cpls_mr || 799 (opts->in_capsule_data_size && 800 !resources->bufs_mr)) { 801 goto cleanup; 802 } 803 SPDK_DEBUGLOG(SPDK_LOG_RDMA, "Command Array: %p Length: %lx LKey: %x\n", 804 resources->cmds, opts->max_queue_depth * sizeof(*resources->cmds), 805 resources->cmds_mr->lkey); 806 SPDK_DEBUGLOG(SPDK_LOG_RDMA, "Completion Array: %p Length: %lx LKey: %x\n", 807 resources->cpls, opts->max_queue_depth * sizeof(*resources->cpls), 808 resources->cpls_mr->lkey); 809 if (resources->bufs && resources->bufs_mr) { 810 SPDK_DEBUGLOG(SPDK_LOG_RDMA, "In Capsule Data Array: %p Length: %x LKey: %x\n", 811 resources->bufs, opts->max_queue_depth * 812 opts->in_capsule_data_size, resources->bufs_mr->lkey); 813 } 814 815 /* Initialize queues */ 816 STAILQ_INIT(&resources->incoming_queue); 817 STAILQ_INIT(&resources->free_queue); 818 819 for (i = 0; i < opts->max_queue_depth; i++) { 820 struct ibv_recv_wr *bad_wr = NULL; 821 822 rdma_recv = &resources->recvs[i]; 823 rdma_recv->qpair = opts->qpair; 824 825 /* Set up memory to receive commands */ 826 if (resources->bufs) { 827 rdma_recv->buf = (void *)((uintptr_t)resources->bufs + (i * 828 opts->in_capsule_data_size)); 829 } 830 831 rdma_recv->rdma_wr.type = RDMA_WR_TYPE_RECV; 832 833 rdma_recv->sgl[0].addr = (uintptr_t)&resources->cmds[i]; 834 rdma_recv->sgl[0].length = sizeof(resources->cmds[i]); 835 rdma_recv->sgl[0].lkey = resources->cmds_mr->lkey; 836 rdma_recv->wr.num_sge = 1; 837 838 if (rdma_recv->buf && resources->bufs_mr) { 839 rdma_recv->sgl[1].addr = (uintptr_t)rdma_recv->buf; 840 rdma_recv->sgl[1].length = opts->in_capsule_data_size; 841 rdma_recv->sgl[1].lkey = resources->bufs_mr->lkey; 842 rdma_recv->wr.num_sge++; 843 } 844 845 rdma_recv->wr.wr_id = (uintptr_t)&rdma_recv->rdma_wr; 846 rdma_recv->wr.sg_list = rdma_recv->sgl; 847 if (opts->shared) { 848 srq = (struct ibv_srq *)opts->qp; 849 rc = ibv_post_srq_recv(srq, &rdma_recv->wr, &bad_wr); 850 } else { 851 qp = (struct ibv_qp *)opts->qp; 852 rc = ibv_post_recv(qp, &rdma_recv->wr, &bad_wr); 853 } 854 if (rc) { 855 goto cleanup; 856 } 857 } 858 859 for (i = 0; i < opts->max_queue_depth; i++) { 860 rdma_req = &resources->reqs[i]; 861 862 if (opts->qpair != NULL) { 863 rdma_req->req.qpair = &opts->qpair->qpair; 864 } else { 865 rdma_req->req.qpair = NULL; 866 } 867 rdma_req->req.cmd = NULL; 868 869 /* Set up memory to send responses */ 870 rdma_req->req.rsp = &resources->cpls[i]; 871 872 rdma_req->rsp.sgl[0].addr = (uintptr_t)&resources->cpls[i]; 873 rdma_req->rsp.sgl[0].length = sizeof(resources->cpls[i]); 874 rdma_req->rsp.sgl[0].lkey = resources->cpls_mr->lkey; 875 876 rdma_req->rsp.rdma_wr.type = RDMA_WR_TYPE_SEND; 877 rdma_req->rsp.wr.wr_id = (uintptr_t)&rdma_req->rsp.rdma_wr; 878 rdma_req->rsp.wr.next = NULL; 879 rdma_req->rsp.wr.opcode = IBV_WR_SEND; 880 rdma_req->rsp.wr.send_flags = IBV_SEND_SIGNALED; 881 rdma_req->rsp.wr.sg_list = rdma_req->rsp.sgl; 882 rdma_req->rsp.wr.num_sge = SPDK_COUNTOF(rdma_req->rsp.sgl); 883 884 /* Set up memory for data buffers */ 885 rdma_req->data.rdma_wr.type = RDMA_WR_TYPE_DATA; 886 rdma_req->data.wr.wr_id = (uintptr_t)&rdma_req->data.rdma_wr; 887 rdma_req->data.wr.next = NULL; 888 rdma_req->data.wr.send_flags = IBV_SEND_SIGNALED; 889 rdma_req->data.wr.sg_list = rdma_req->data.sgl; 890 rdma_req->data.wr.num_sge = SPDK_COUNTOF(rdma_req->data.sgl); 891 892 /* Initialize request state to FREE */ 893 rdma_req->state = RDMA_REQUEST_STATE_FREE; 894 STAILQ_INSERT_TAIL(&resources->free_queue, rdma_req, state_link); 895 } 896 897 return resources; 898 899 cleanup: 900 nvmf_rdma_resources_destroy(resources); 901 return NULL; 902 } 903 904 static void 905 spdk_nvmf_rdma_qpair_destroy(struct spdk_nvmf_rdma_qpair *rqpair) 906 { 907 struct spdk_nvmf_rdma_recv *rdma_recv, *recv_tmp; 908 struct ibv_recv_wr *bad_recv_wr = NULL; 909 int rc; 910 911 spdk_trace_record(TRACE_RDMA_QP_DESTROY, 0, 0, (uintptr_t)rqpair->cm_id, 0); 912 913 spdk_poller_unregister(&rqpair->destruct_poller); 914 915 if (rqpair->qd != 0) { 916 if (rqpair->srq == NULL) { 917 nvmf_rdma_dump_qpair_contents(rqpair); 918 } 919 SPDK_WARNLOG("Destroying qpair when queue depth is %d\n", rqpair->qd); 920 } 921 922 if (rqpair->poller) { 923 TAILQ_REMOVE(&rqpair->poller->qpairs, rqpair, link); 924 925 if (rqpair->srq != NULL && rqpair->resources != NULL) { 926 /* Drop all received but unprocessed commands for this queue and return them to SRQ */ 927 STAILQ_FOREACH_SAFE(rdma_recv, &rqpair->resources->incoming_queue, link, recv_tmp) { 928 if (rqpair == rdma_recv->qpair) { 929 STAILQ_REMOVE_HEAD(&rqpair->resources->incoming_queue, link); 930 rc = ibv_post_srq_recv(rqpair->srq, &rdma_recv->wr, &bad_recv_wr); 931 if (rc) { 932 SPDK_ERRLOG("Unable to re-post rx descriptor\n"); 933 } 934 } 935 } 936 } 937 } 938 939 if (rqpair->cm_id) { 940 if (rqpair->cm_id->qp != NULL) { 941 rdma_destroy_qp(rqpair->cm_id); 942 } 943 rdma_destroy_id(rqpair->cm_id); 944 945 if (rqpair->poller != NULL && rqpair->srq == NULL) { 946 rqpair->poller->required_num_wr -= MAX_WR_PER_QP(rqpair->max_queue_depth); 947 } 948 } 949 950 if (rqpair->srq == NULL && rqpair->resources != NULL) { 951 nvmf_rdma_resources_destroy(rqpair->resources); 952 } 953 954 free(rqpair); 955 } 956 957 static int 958 nvmf_rdma_resize_cq(struct spdk_nvmf_rdma_qpair *rqpair, struct spdk_nvmf_rdma_device *device) 959 { 960 struct spdk_nvmf_rdma_poller *rpoller; 961 int rc, num_cqe, required_num_wr; 962 963 /* Enlarge CQ size dynamically */ 964 rpoller = rqpair->poller; 965 required_num_wr = rpoller->required_num_wr + MAX_WR_PER_QP(rqpair->max_queue_depth); 966 num_cqe = rpoller->num_cqe; 967 if (num_cqe < required_num_wr) { 968 num_cqe = spdk_max(num_cqe * 2, required_num_wr); 969 num_cqe = spdk_min(num_cqe, device->attr.max_cqe); 970 } 971 972 if (rpoller->num_cqe != num_cqe) { 973 if (required_num_wr > device->attr.max_cqe) { 974 SPDK_ERRLOG("RDMA CQE requirement (%d) exceeds device max_cqe limitation (%d)\n", 975 required_num_wr, device->attr.max_cqe); 976 return -1; 977 } 978 979 SPDK_DEBUGLOG(SPDK_LOG_RDMA, "Resize RDMA CQ from %d to %d\n", rpoller->num_cqe, num_cqe); 980 rc = ibv_resize_cq(rpoller->cq, num_cqe); 981 if (rc) { 982 SPDK_ERRLOG("RDMA CQ resize failed: errno %d: %s\n", errno, spdk_strerror(errno)); 983 return -1; 984 } 985 986 rpoller->num_cqe = num_cqe; 987 } 988 989 rpoller->required_num_wr = required_num_wr; 990 return 0; 991 } 992 993 static int 994 spdk_nvmf_rdma_qpair_initialize(struct spdk_nvmf_qpair *qpair) 995 { 996 struct spdk_nvmf_rdma_qpair *rqpair; 997 int rc; 998 struct spdk_nvmf_rdma_transport *rtransport; 999 struct spdk_nvmf_transport *transport; 1000 struct spdk_nvmf_rdma_resource_opts opts; 1001 struct spdk_nvmf_rdma_device *device; 1002 struct ibv_qp_init_attr ibv_init_attr; 1003 1004 rqpair = SPDK_CONTAINEROF(qpair, struct spdk_nvmf_rdma_qpair, qpair); 1005 device = rqpair->port->device; 1006 1007 memset(&ibv_init_attr, 0, sizeof(struct ibv_qp_init_attr)); 1008 ibv_init_attr.qp_context = rqpair; 1009 ibv_init_attr.qp_type = IBV_QPT_RC; 1010 ibv_init_attr.send_cq = rqpair->poller->cq; 1011 ibv_init_attr.recv_cq = rqpair->poller->cq; 1012 1013 if (rqpair->srq) { 1014 ibv_init_attr.srq = rqpair->srq; 1015 } else { 1016 ibv_init_attr.cap.max_recv_wr = rqpair->max_queue_depth + 1017 1; /* RECV operations + dummy drain WR */ 1018 } 1019 1020 ibv_init_attr.cap.max_send_wr = rqpair->max_queue_depth * 1021 2 + 1; /* SEND, READ, and WRITE operations + dummy drain WR */ 1022 ibv_init_attr.cap.max_send_sge = spdk_min(device->attr.max_sge, NVMF_DEFAULT_TX_SGE); 1023 ibv_init_attr.cap.max_recv_sge = spdk_min(device->attr.max_sge, NVMF_DEFAULT_RX_SGE); 1024 1025 if (rqpair->srq == NULL && nvmf_rdma_resize_cq(rqpair, device) < 0) { 1026 SPDK_ERRLOG("Failed to resize the completion queue. Cannot initialize qpair.\n"); 1027 goto error; 1028 } 1029 1030 rc = rdma_create_qp(rqpair->cm_id, rqpair->port->device->pd, &ibv_init_attr); 1031 if (rc) { 1032 SPDK_ERRLOG("rdma_create_qp failed: errno %d: %s\n", errno, spdk_strerror(errno)); 1033 goto error; 1034 } 1035 1036 rqpair->max_send_depth = spdk_min((uint32_t)(rqpair->max_queue_depth * 2 + 1), 1037 ibv_init_attr.cap.max_send_wr); 1038 rqpair->max_send_sge = spdk_min(NVMF_DEFAULT_TX_SGE, ibv_init_attr.cap.max_send_sge); 1039 rqpair->max_recv_sge = spdk_min(NVMF_DEFAULT_RX_SGE, ibv_init_attr.cap.max_recv_sge); 1040 spdk_trace_record(TRACE_RDMA_QP_CREATE, 0, 0, (uintptr_t)rqpair->cm_id, 0); 1041 SPDK_DEBUGLOG(SPDK_LOG_RDMA, "New RDMA Connection: %p\n", qpair); 1042 1043 rqpair->sends_to_post.first = NULL; 1044 rqpair->sends_to_post.last = NULL; 1045 1046 if (rqpair->poller->srq == NULL) { 1047 rtransport = SPDK_CONTAINEROF(qpair->transport, struct spdk_nvmf_rdma_transport, transport); 1048 transport = &rtransport->transport; 1049 1050 opts.qp = rqpair->cm_id->qp; 1051 opts.pd = rqpair->cm_id->pd; 1052 opts.qpair = rqpair; 1053 opts.shared = false; 1054 opts.max_queue_depth = rqpair->max_queue_depth; 1055 opts.in_capsule_data_size = transport->opts.in_capsule_data_size; 1056 1057 rqpair->resources = nvmf_rdma_resources_create(&opts); 1058 1059 if (!rqpair->resources) { 1060 SPDK_ERRLOG("Unable to allocate resources for receive queue.\n"); 1061 rdma_destroy_qp(rqpair->cm_id); 1062 goto error; 1063 } 1064 } else { 1065 rqpair->resources = rqpair->poller->resources; 1066 } 1067 1068 rqpair->current_recv_depth = 0; 1069 STAILQ_INIT(&rqpair->pending_rdma_read_queue); 1070 STAILQ_INIT(&rqpair->pending_rdma_write_queue); 1071 1072 return 0; 1073 1074 error: 1075 rdma_destroy_id(rqpair->cm_id); 1076 rqpair->cm_id = NULL; 1077 return -1; 1078 } 1079 1080 /* Append the given recv wr structure to the resource structs outstanding recvs list. */ 1081 /* This function accepts either a single wr or the first wr in a linked list. */ 1082 static void 1083 nvmf_rdma_qpair_queue_recv_wrs(struct spdk_nvmf_rdma_qpair *rqpair, struct ibv_recv_wr *first) 1084 { 1085 struct ibv_recv_wr *last; 1086 1087 last = first; 1088 while (last->next != NULL) { 1089 last = last->next; 1090 } 1091 1092 if (rqpair->resources->recvs_to_post.first == NULL) { 1093 rqpair->resources->recvs_to_post.first = first; 1094 rqpair->resources->recvs_to_post.last = last; 1095 if (rqpair->srq == NULL) { 1096 STAILQ_INSERT_TAIL(&rqpair->poller->qpairs_pending_recv, rqpair, recv_link); 1097 } 1098 } else { 1099 rqpair->resources->recvs_to_post.last->next = first; 1100 rqpair->resources->recvs_to_post.last = last; 1101 } 1102 } 1103 1104 /* Append the given send wr structure to the qpair's outstanding sends list. */ 1105 /* This function accepts either a single wr or the first wr in a linked list. */ 1106 static void 1107 nvmf_rdma_qpair_queue_send_wrs(struct spdk_nvmf_rdma_qpair *rqpair, struct ibv_send_wr *first) 1108 { 1109 struct ibv_send_wr *last; 1110 1111 last = first; 1112 while (last->next != NULL) { 1113 last = last->next; 1114 } 1115 1116 if (rqpair->sends_to_post.first == NULL) { 1117 rqpair->sends_to_post.first = first; 1118 rqpair->sends_to_post.last = last; 1119 STAILQ_INSERT_TAIL(&rqpair->poller->qpairs_pending_send, rqpair, send_link); 1120 } else { 1121 rqpair->sends_to_post.last->next = first; 1122 rqpair->sends_to_post.last = last; 1123 } 1124 } 1125 1126 static int 1127 request_transfer_in(struct spdk_nvmf_request *req) 1128 { 1129 struct spdk_nvmf_rdma_request *rdma_req; 1130 struct spdk_nvmf_qpair *qpair; 1131 struct spdk_nvmf_rdma_qpair *rqpair; 1132 1133 qpair = req->qpair; 1134 rdma_req = SPDK_CONTAINEROF(req, struct spdk_nvmf_rdma_request, req); 1135 rqpair = SPDK_CONTAINEROF(qpair, struct spdk_nvmf_rdma_qpair, qpair); 1136 1137 assert(req->xfer == SPDK_NVME_DATA_HOST_TO_CONTROLLER); 1138 assert(rdma_req != NULL); 1139 1140 nvmf_rdma_qpair_queue_send_wrs(rqpair, &rdma_req->data.wr); 1141 rqpair->current_read_depth += rdma_req->num_outstanding_data_wr; 1142 rqpair->current_send_depth += rdma_req->num_outstanding_data_wr; 1143 return 0; 1144 } 1145 1146 static int 1147 request_transfer_out(struct spdk_nvmf_request *req, int *data_posted) 1148 { 1149 int num_outstanding_data_wr = 0; 1150 struct spdk_nvmf_rdma_request *rdma_req; 1151 struct spdk_nvmf_qpair *qpair; 1152 struct spdk_nvmf_rdma_qpair *rqpair; 1153 struct spdk_nvme_cpl *rsp; 1154 struct ibv_send_wr *first = NULL; 1155 1156 *data_posted = 0; 1157 qpair = req->qpair; 1158 rsp = &req->rsp->nvme_cpl; 1159 rdma_req = SPDK_CONTAINEROF(req, struct spdk_nvmf_rdma_request, req); 1160 rqpair = SPDK_CONTAINEROF(qpair, struct spdk_nvmf_rdma_qpair, qpair); 1161 1162 /* Advance our sq_head pointer */ 1163 if (qpair->sq_head == qpair->sq_head_max) { 1164 qpair->sq_head = 0; 1165 } else { 1166 qpair->sq_head++; 1167 } 1168 rsp->sqhd = qpair->sq_head; 1169 1170 /* queue the capsule for the recv buffer */ 1171 assert(rdma_req->recv != NULL); 1172 1173 nvmf_rdma_qpair_queue_recv_wrs(rqpair, &rdma_req->recv->wr); 1174 1175 rdma_req->recv = NULL; 1176 assert(rqpair->current_recv_depth > 0); 1177 rqpair->current_recv_depth--; 1178 1179 /* Build the response which consists of optional 1180 * RDMA WRITEs to transfer data, plus an RDMA SEND 1181 * containing the response. 1182 */ 1183 first = &rdma_req->rsp.wr; 1184 1185 if (rsp->status.sc == SPDK_NVME_SC_SUCCESS && 1186 req->xfer == SPDK_NVME_DATA_CONTROLLER_TO_HOST) { 1187 first = &rdma_req->data.wr; 1188 *data_posted = 1; 1189 num_outstanding_data_wr = rdma_req->num_outstanding_data_wr; 1190 } 1191 nvmf_rdma_qpair_queue_send_wrs(rqpair, first); 1192 /* +1 for the rsp wr */ 1193 rqpair->current_send_depth += num_outstanding_data_wr + 1; 1194 1195 return 0; 1196 } 1197 1198 static int 1199 spdk_nvmf_rdma_event_accept(struct rdma_cm_id *id, struct spdk_nvmf_rdma_qpair *rqpair) 1200 { 1201 struct spdk_nvmf_rdma_accept_private_data accept_data; 1202 struct rdma_conn_param ctrlr_event_data = {}; 1203 int rc; 1204 1205 accept_data.recfmt = 0; 1206 accept_data.crqsize = rqpair->max_queue_depth; 1207 1208 ctrlr_event_data.private_data = &accept_data; 1209 ctrlr_event_data.private_data_len = sizeof(accept_data); 1210 if (id->ps == RDMA_PS_TCP) { 1211 ctrlr_event_data.responder_resources = 0; /* We accept 0 reads from the host */ 1212 ctrlr_event_data.initiator_depth = rqpair->max_read_depth; 1213 } 1214 1215 /* Configure infinite retries for the initiator side qpair. 1216 * When using a shared receive queue on the target side, 1217 * we need to pass this value to the initiator to prevent the 1218 * initiator side NIC from completing SEND requests back to the 1219 * initiator with status rnr_retry_count_exceeded. */ 1220 if (rqpair->srq != NULL) { 1221 ctrlr_event_data.rnr_retry_count = 0x7; 1222 } 1223 1224 rc = rdma_accept(id, &ctrlr_event_data); 1225 if (rc) { 1226 SPDK_ERRLOG("Error %d on rdma_accept\n", errno); 1227 } else { 1228 SPDK_DEBUGLOG(SPDK_LOG_RDMA, "Sent back the accept\n"); 1229 } 1230 1231 return rc; 1232 } 1233 1234 static void 1235 spdk_nvmf_rdma_event_reject(struct rdma_cm_id *id, enum spdk_nvmf_rdma_transport_error error) 1236 { 1237 struct spdk_nvmf_rdma_reject_private_data rej_data; 1238 1239 rej_data.recfmt = 0; 1240 rej_data.sts = error; 1241 1242 rdma_reject(id, &rej_data, sizeof(rej_data)); 1243 } 1244 1245 static int 1246 nvmf_rdma_connect(struct spdk_nvmf_transport *transport, struct rdma_cm_event *event, 1247 new_qpair_fn cb_fn, void *cb_arg) 1248 { 1249 struct spdk_nvmf_rdma_transport *rtransport; 1250 struct spdk_nvmf_rdma_qpair *rqpair = NULL; 1251 struct spdk_nvmf_rdma_port *port; 1252 struct rdma_conn_param *rdma_param = NULL; 1253 const struct spdk_nvmf_rdma_request_private_data *private_data = NULL; 1254 uint16_t max_queue_depth; 1255 uint16_t max_read_depth; 1256 1257 rtransport = SPDK_CONTAINEROF(transport, struct spdk_nvmf_rdma_transport, transport); 1258 1259 assert(event->id != NULL); /* Impossible. Can't even reject the connection. */ 1260 assert(event->id->verbs != NULL); /* Impossible. No way to handle this. */ 1261 1262 rdma_param = &event->param.conn; 1263 if (rdma_param->private_data == NULL || 1264 rdma_param->private_data_len < sizeof(struct spdk_nvmf_rdma_request_private_data)) { 1265 SPDK_ERRLOG("connect request: no private data provided\n"); 1266 spdk_nvmf_rdma_event_reject(event->id, SPDK_NVMF_RDMA_ERROR_INVALID_PRIVATE_DATA_LENGTH); 1267 return -1; 1268 } 1269 1270 private_data = rdma_param->private_data; 1271 if (private_data->recfmt != 0) { 1272 SPDK_ERRLOG("Received RDMA private data with RECFMT != 0\n"); 1273 spdk_nvmf_rdma_event_reject(event->id, SPDK_NVMF_RDMA_ERROR_INVALID_RECFMT); 1274 return -1; 1275 } 1276 1277 SPDK_DEBUGLOG(SPDK_LOG_RDMA, "Connect Recv on fabric intf name %s, dev_name %s\n", 1278 event->id->verbs->device->name, event->id->verbs->device->dev_name); 1279 1280 port = event->listen_id->context; 1281 SPDK_DEBUGLOG(SPDK_LOG_RDMA, "Listen Id was %p with verbs %p. ListenAddr: %p\n", 1282 event->listen_id, event->listen_id->verbs, port); 1283 1284 /* Figure out the supported queue depth. This is a multi-step process 1285 * that takes into account hardware maximums, host provided values, 1286 * and our target's internal memory limits */ 1287 1288 SPDK_DEBUGLOG(SPDK_LOG_RDMA, "Calculating Queue Depth\n"); 1289 1290 /* Start with the maximum queue depth allowed by the target */ 1291 max_queue_depth = rtransport->transport.opts.max_queue_depth; 1292 max_read_depth = rtransport->transport.opts.max_queue_depth; 1293 SPDK_DEBUGLOG(SPDK_LOG_RDMA, "Target Max Queue Depth: %d\n", 1294 rtransport->transport.opts.max_queue_depth); 1295 1296 /* Next check the local NIC's hardware limitations */ 1297 SPDK_DEBUGLOG(SPDK_LOG_RDMA, 1298 "Local NIC Max Send/Recv Queue Depth: %d Max Read/Write Queue Depth: %d\n", 1299 port->device->attr.max_qp_wr, port->device->attr.max_qp_rd_atom); 1300 max_queue_depth = spdk_min(max_queue_depth, port->device->attr.max_qp_wr); 1301 max_read_depth = spdk_min(max_read_depth, port->device->attr.max_qp_init_rd_atom); 1302 1303 /* Next check the remote NIC's hardware limitations */ 1304 SPDK_DEBUGLOG(SPDK_LOG_RDMA, 1305 "Host (Initiator) NIC Max Incoming RDMA R/W operations: %d Max Outgoing RDMA R/W operations: %d\n", 1306 rdma_param->initiator_depth, rdma_param->responder_resources); 1307 if (rdma_param->initiator_depth > 0) { 1308 max_read_depth = spdk_min(max_read_depth, rdma_param->initiator_depth); 1309 } 1310 1311 /* Finally check for the host software requested values, which are 1312 * optional. */ 1313 if (rdma_param->private_data != NULL && 1314 rdma_param->private_data_len >= sizeof(struct spdk_nvmf_rdma_request_private_data)) { 1315 SPDK_DEBUGLOG(SPDK_LOG_RDMA, "Host Receive Queue Size: %d\n", private_data->hrqsize); 1316 SPDK_DEBUGLOG(SPDK_LOG_RDMA, "Host Send Queue Size: %d\n", private_data->hsqsize); 1317 max_queue_depth = spdk_min(max_queue_depth, private_data->hrqsize); 1318 max_queue_depth = spdk_min(max_queue_depth, private_data->hsqsize + 1); 1319 } 1320 1321 SPDK_DEBUGLOG(SPDK_LOG_RDMA, "Final Negotiated Queue Depth: %d R/W Depth: %d\n", 1322 max_queue_depth, max_read_depth); 1323 1324 rqpair = calloc(1, sizeof(struct spdk_nvmf_rdma_qpair)); 1325 if (rqpair == NULL) { 1326 SPDK_ERRLOG("Could not allocate new connection.\n"); 1327 spdk_nvmf_rdma_event_reject(event->id, SPDK_NVMF_RDMA_ERROR_NO_RESOURCES); 1328 return -1; 1329 } 1330 1331 rqpair->port = port; 1332 rqpair->max_queue_depth = max_queue_depth; 1333 rqpair->max_read_depth = max_read_depth; 1334 rqpair->cm_id = event->id; 1335 rqpair->listen_id = event->listen_id; 1336 rqpair->qpair.transport = transport; 1337 /* use qid from the private data to determine the qpair type 1338 qid will be set to the appropriate value when the controller is created */ 1339 rqpair->qpair.qid = private_data->qid; 1340 1341 event->id->context = &rqpair->qpair; 1342 1343 cb_fn(&rqpair->qpair, cb_arg); 1344 1345 return 0; 1346 } 1347 1348 static int 1349 spdk_nvmf_rdma_mem_notify(void *cb_ctx, struct spdk_mem_map *map, 1350 enum spdk_mem_map_notify_action action, 1351 void *vaddr, size_t size) 1352 { 1353 struct ibv_pd *pd = cb_ctx; 1354 struct ibv_mr *mr; 1355 int rc; 1356 1357 switch (action) { 1358 case SPDK_MEM_MAP_NOTIFY_REGISTER: 1359 if (!g_nvmf_hooks.get_rkey) { 1360 mr = ibv_reg_mr(pd, vaddr, size, 1361 IBV_ACCESS_LOCAL_WRITE | 1362 IBV_ACCESS_REMOTE_READ | 1363 IBV_ACCESS_REMOTE_WRITE); 1364 if (mr == NULL) { 1365 SPDK_ERRLOG("ibv_reg_mr() failed\n"); 1366 return -1; 1367 } else { 1368 rc = spdk_mem_map_set_translation(map, (uint64_t)vaddr, size, (uint64_t)mr); 1369 } 1370 } else { 1371 rc = spdk_mem_map_set_translation(map, (uint64_t)vaddr, size, 1372 g_nvmf_hooks.get_rkey(pd, vaddr, size)); 1373 } 1374 break; 1375 case SPDK_MEM_MAP_NOTIFY_UNREGISTER: 1376 if (!g_nvmf_hooks.get_rkey) { 1377 mr = (struct ibv_mr *)spdk_mem_map_translate(map, (uint64_t)vaddr, NULL); 1378 if (mr) { 1379 ibv_dereg_mr(mr); 1380 } 1381 } 1382 rc = spdk_mem_map_clear_translation(map, (uint64_t)vaddr, size); 1383 break; 1384 default: 1385 SPDK_UNREACHABLE(); 1386 } 1387 1388 return rc; 1389 } 1390 1391 static int 1392 spdk_nvmf_rdma_check_contiguous_entries(uint64_t addr_1, uint64_t addr_2) 1393 { 1394 /* Two contiguous mappings will point to the same address which is the start of the RDMA MR. */ 1395 return addr_1 == addr_2; 1396 } 1397 1398 typedef enum spdk_nvme_data_transfer spdk_nvme_data_transfer_t; 1399 1400 static spdk_nvme_data_transfer_t 1401 spdk_nvmf_rdma_request_get_xfer(struct spdk_nvmf_rdma_request *rdma_req) 1402 { 1403 enum spdk_nvme_data_transfer xfer; 1404 struct spdk_nvme_cmd *cmd = &rdma_req->req.cmd->nvme_cmd; 1405 struct spdk_nvme_sgl_descriptor *sgl = &cmd->dptr.sgl1; 1406 1407 #ifdef SPDK_CONFIG_RDMA_SEND_WITH_INVAL 1408 rdma_req->rsp.wr.opcode = IBV_WR_SEND; 1409 rdma_req->rsp.wr.imm_data = 0; 1410 #endif 1411 1412 /* Figure out data transfer direction */ 1413 if (cmd->opc == SPDK_NVME_OPC_FABRIC) { 1414 xfer = spdk_nvme_opc_get_data_transfer(rdma_req->req.cmd->nvmf_cmd.fctype); 1415 } else { 1416 xfer = spdk_nvme_opc_get_data_transfer(cmd->opc); 1417 1418 /* Some admin commands are special cases */ 1419 if ((rdma_req->req.qpair->qid == 0) && 1420 ((cmd->opc == SPDK_NVME_OPC_GET_FEATURES) || 1421 (cmd->opc == SPDK_NVME_OPC_SET_FEATURES))) { 1422 switch (cmd->cdw10 & 0xff) { 1423 case SPDK_NVME_FEAT_LBA_RANGE_TYPE: 1424 case SPDK_NVME_FEAT_AUTONOMOUS_POWER_STATE_TRANSITION: 1425 case SPDK_NVME_FEAT_HOST_IDENTIFIER: 1426 break; 1427 default: 1428 xfer = SPDK_NVME_DATA_NONE; 1429 } 1430 } 1431 } 1432 1433 if (xfer == SPDK_NVME_DATA_NONE) { 1434 return xfer; 1435 } 1436 1437 /* Even for commands that may transfer data, they could have specified 0 length. 1438 * We want those to show up with xfer SPDK_NVME_DATA_NONE. 1439 */ 1440 switch (sgl->generic.type) { 1441 case SPDK_NVME_SGL_TYPE_DATA_BLOCK: 1442 case SPDK_NVME_SGL_TYPE_BIT_BUCKET: 1443 case SPDK_NVME_SGL_TYPE_SEGMENT: 1444 case SPDK_NVME_SGL_TYPE_LAST_SEGMENT: 1445 case SPDK_NVME_SGL_TYPE_TRANSPORT_DATA_BLOCK: 1446 if (sgl->unkeyed.length == 0) { 1447 xfer = SPDK_NVME_DATA_NONE; 1448 } 1449 break; 1450 case SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK: 1451 if (sgl->keyed.length == 0) { 1452 xfer = SPDK_NVME_DATA_NONE; 1453 } 1454 break; 1455 } 1456 1457 return xfer; 1458 } 1459 1460 static inline void 1461 nvmf_rdma_setup_wr(struct ibv_send_wr *wr, struct ibv_send_wr *next, 1462 enum spdk_nvme_data_transfer xfer) 1463 { 1464 if (xfer == SPDK_NVME_DATA_CONTROLLER_TO_HOST) { 1465 wr->opcode = IBV_WR_RDMA_WRITE; 1466 wr->send_flags = 0; 1467 wr->next = next; 1468 } else if (xfer == SPDK_NVME_DATA_HOST_TO_CONTROLLER) { 1469 wr->opcode = IBV_WR_RDMA_READ; 1470 wr->send_flags = IBV_SEND_SIGNALED; 1471 wr->next = NULL; 1472 } else { 1473 assert(0); 1474 } 1475 } 1476 1477 static int 1478 nvmf_request_alloc_wrs(struct spdk_nvmf_rdma_transport *rtransport, 1479 struct spdk_nvmf_rdma_request *rdma_req, 1480 uint32_t num_sgl_descriptors) 1481 { 1482 struct spdk_nvmf_rdma_request_data *work_requests[SPDK_NVMF_MAX_SGL_ENTRIES]; 1483 struct spdk_nvmf_rdma_request_data *current_data_wr; 1484 uint32_t i; 1485 1486 if (num_sgl_descriptors > SPDK_NVMF_MAX_SGL_ENTRIES) { 1487 SPDK_ERRLOG("Requested too much entries (%u), the limit is %u\n", 1488 num_sgl_descriptors, SPDK_NVMF_MAX_SGL_ENTRIES); 1489 return -EINVAL; 1490 } 1491 1492 if (spdk_mempool_get_bulk(rtransport->data_wr_pool, (void **)work_requests, num_sgl_descriptors)) { 1493 return -ENOMEM; 1494 } 1495 1496 current_data_wr = &rdma_req->data; 1497 1498 for (i = 0; i < num_sgl_descriptors; i++) { 1499 nvmf_rdma_setup_wr(¤t_data_wr->wr, &work_requests[i]->wr, rdma_req->req.xfer); 1500 current_data_wr->wr.next = &work_requests[i]->wr; 1501 current_data_wr = work_requests[i]; 1502 current_data_wr->wr.sg_list = current_data_wr->sgl; 1503 current_data_wr->wr.wr_id = rdma_req->data.wr.wr_id; 1504 } 1505 1506 nvmf_rdma_setup_wr(¤t_data_wr->wr, &rdma_req->rsp.wr, rdma_req->req.xfer); 1507 1508 return 0; 1509 } 1510 1511 static inline void 1512 nvmf_rdma_setup_request(struct spdk_nvmf_rdma_request *rdma_req) 1513 { 1514 struct ibv_send_wr *wr = &rdma_req->data.wr; 1515 struct spdk_nvme_sgl_descriptor *sgl = &rdma_req->req.cmd->nvme_cmd.dptr.sgl1; 1516 1517 wr->wr.rdma.rkey = sgl->keyed.key; 1518 wr->wr.rdma.remote_addr = sgl->address; 1519 nvmf_rdma_setup_wr(wr, &rdma_req->rsp.wr, rdma_req->req.xfer); 1520 } 1521 1522 static inline void 1523 nvmf_rdma_update_remote_addr(struct spdk_nvmf_rdma_request *rdma_req, uint32_t num_wrs) 1524 { 1525 struct ibv_send_wr *wr = &rdma_req->data.wr; 1526 struct spdk_nvme_sgl_descriptor *sgl = &rdma_req->req.cmd->nvme_cmd.dptr.sgl1; 1527 uint32_t i; 1528 int j; 1529 uint64_t remote_addr_offset = 0; 1530 1531 for (i = 0; i < num_wrs; ++i) { 1532 wr->wr.rdma.rkey = sgl->keyed.key; 1533 wr->wr.rdma.remote_addr = sgl->address + remote_addr_offset; 1534 for (j = 0; j < wr->num_sge; ++j) { 1535 remote_addr_offset += wr->sg_list[j].length; 1536 } 1537 wr = wr->next; 1538 } 1539 } 1540 1541 /* This function is used in the rare case that we have a buffer split over multiple memory regions. */ 1542 static int 1543 nvmf_rdma_replace_buffer(struct spdk_nvmf_rdma_poll_group *rgroup, void **buf) 1544 { 1545 struct spdk_nvmf_transport_poll_group *group = &rgroup->group; 1546 struct spdk_nvmf_transport *transport = group->transport; 1547 struct spdk_nvmf_transport_pg_cache_buf *old_buf; 1548 void *new_buf; 1549 1550 if (!(STAILQ_EMPTY(&group->buf_cache))) { 1551 group->buf_cache_count--; 1552 new_buf = STAILQ_FIRST(&group->buf_cache); 1553 STAILQ_REMOVE_HEAD(&group->buf_cache, link); 1554 assert(*buf != NULL); 1555 } else { 1556 new_buf = spdk_mempool_get(transport->data_buf_pool); 1557 } 1558 1559 if (*buf == NULL) { 1560 return -ENOMEM; 1561 } 1562 1563 old_buf = *buf; 1564 STAILQ_INSERT_HEAD(&rgroup->retired_bufs, old_buf, link); 1565 *buf = new_buf; 1566 return 0; 1567 } 1568 1569 static bool 1570 nvmf_rdma_get_lkey(struct spdk_nvmf_rdma_device *device, struct iovec *iov, 1571 uint32_t *_lkey) 1572 { 1573 uint64_t translation_len; 1574 uint32_t lkey; 1575 1576 translation_len = iov->iov_len; 1577 1578 if (!g_nvmf_hooks.get_rkey) { 1579 lkey = ((struct ibv_mr *)spdk_mem_map_translate(device->map, 1580 (uint64_t)iov->iov_base, &translation_len))->lkey; 1581 } else { 1582 lkey = spdk_mem_map_translate(device->map, 1583 (uint64_t)iov->iov_base, &translation_len); 1584 } 1585 1586 if (spdk_unlikely(translation_len < iov->iov_len)) { 1587 return false; 1588 } 1589 1590 *_lkey = lkey; 1591 return true; 1592 } 1593 1594 static bool 1595 nvmf_rdma_fill_wr_sge(struct spdk_nvmf_rdma_device *device, 1596 struct iovec *iov, struct ibv_send_wr **_wr, 1597 uint32_t *_remaining_data_block, uint32_t *_offset, 1598 uint32_t *_num_extra_wrs, 1599 const struct spdk_dif_ctx *dif_ctx) 1600 { 1601 struct ibv_send_wr *wr = *_wr; 1602 struct ibv_sge *sg_ele = &wr->sg_list[wr->num_sge]; 1603 uint32_t lkey = 0; 1604 uint32_t remaining, data_block_size, md_size, sge_len; 1605 1606 if (spdk_unlikely(!nvmf_rdma_get_lkey(device, iov, &lkey))) { 1607 /* This is a very rare case that can occur when using DPDK version < 19.05 */ 1608 SPDK_ERRLOG("Data buffer split over multiple RDMA Memory Regions. Removing it from circulation.\n"); 1609 return false; 1610 } 1611 1612 if (spdk_likely(!dif_ctx)) { 1613 sg_ele->lkey = lkey; 1614 sg_ele->addr = (uintptr_t)(iov->iov_base); 1615 sg_ele->length = iov->iov_len; 1616 wr->num_sge++; 1617 } else { 1618 remaining = iov->iov_len - *_offset; 1619 data_block_size = dif_ctx->block_size - dif_ctx->md_size; 1620 md_size = dif_ctx->md_size; 1621 1622 while (remaining) { 1623 if (wr->num_sge >= SPDK_NVMF_MAX_SGL_ENTRIES) { 1624 if (*_num_extra_wrs > 0 && wr->next) { 1625 *_wr = wr->next; 1626 wr = *_wr; 1627 wr->num_sge = 0; 1628 sg_ele = &wr->sg_list[wr->num_sge]; 1629 (*_num_extra_wrs)--; 1630 } else { 1631 break; 1632 } 1633 } 1634 sg_ele->lkey = lkey; 1635 sg_ele->addr = (uintptr_t)((char *)iov->iov_base + *_offset); 1636 sge_len = spdk_min(remaining, *_remaining_data_block); 1637 sg_ele->length = sge_len; 1638 remaining -= sge_len; 1639 *_remaining_data_block -= sge_len; 1640 *_offset += sge_len; 1641 1642 sg_ele++; 1643 wr->num_sge++; 1644 1645 if (*_remaining_data_block == 0) { 1646 /* skip metadata */ 1647 *_offset += md_size; 1648 /* Metadata that do not fit this IO buffer will be included in the next IO buffer */ 1649 remaining -= spdk_min(remaining, md_size); 1650 *_remaining_data_block = data_block_size; 1651 } 1652 1653 if (remaining == 0) { 1654 /* By subtracting the size of the last IOV from the offset, we ensure that we skip 1655 the remaining metadata bits at the beginning of the next buffer */ 1656 *_offset -= iov->iov_len; 1657 } 1658 } 1659 } 1660 1661 return true; 1662 } 1663 1664 static int 1665 nvmf_rdma_fill_wr_sgl(struct spdk_nvmf_rdma_poll_group *rgroup, 1666 struct spdk_nvmf_rdma_device *device, 1667 struct spdk_nvmf_rdma_request *rdma_req, 1668 struct ibv_send_wr *wr, 1669 uint32_t length, 1670 uint32_t num_extra_wrs) 1671 { 1672 struct spdk_nvmf_request *req = &rdma_req->req; 1673 struct spdk_dif_ctx *dif_ctx = NULL; 1674 uint32_t remaining_data_block = 0; 1675 uint32_t offset = 0; 1676 1677 if (spdk_unlikely(rdma_req->req.dif.dif_insert_or_strip)) { 1678 dif_ctx = &rdma_req->req.dif.dif_ctx; 1679 remaining_data_block = dif_ctx->block_size - dif_ctx->md_size; 1680 } 1681 1682 wr->num_sge = 0; 1683 1684 while (length && (num_extra_wrs || wr->num_sge < SPDK_NVMF_MAX_SGL_ENTRIES)) { 1685 while (spdk_unlikely(!nvmf_rdma_fill_wr_sge(device, &req->iov[rdma_req->iovpos], &wr, 1686 &remaining_data_block, &offset, &num_extra_wrs, dif_ctx))) { 1687 if (nvmf_rdma_replace_buffer(rgroup, &req->buffers[rdma_req->iovpos]) == -ENOMEM) { 1688 return -ENOMEM; 1689 } 1690 req->iov[rdma_req->iovpos].iov_base = (void *)((uintptr_t)(req->buffers[rdma_req->iovpos] + 1691 NVMF_DATA_BUFFER_MASK) & 1692 ~NVMF_DATA_BUFFER_MASK); 1693 } 1694 1695 length -= req->iov[rdma_req->iovpos].iov_len; 1696 rdma_req->iovpos++; 1697 } 1698 1699 if (length) { 1700 SPDK_ERRLOG("Not enough SG entries to hold data buffer\n"); 1701 return -EINVAL; 1702 } 1703 1704 return 0; 1705 } 1706 1707 static inline uint32_t 1708 nvmf_rdma_calc_num_wrs(uint32_t length, uint32_t io_unit_size, uint32_t block_size) 1709 { 1710 /* estimate the number of SG entries and WRs needed to process the request */ 1711 uint32_t num_sge = 0; 1712 uint32_t i; 1713 uint32_t num_buffers = SPDK_CEIL_DIV(length, io_unit_size); 1714 1715 for (i = 0; i < num_buffers && length > 0; i++) { 1716 uint32_t buffer_len = spdk_min(length, io_unit_size); 1717 uint32_t num_sge_in_block = SPDK_CEIL_DIV(buffer_len, block_size); 1718 1719 if (num_sge_in_block * block_size > buffer_len) { 1720 ++num_sge_in_block; 1721 } 1722 num_sge += num_sge_in_block; 1723 length -= buffer_len; 1724 } 1725 return SPDK_CEIL_DIV(num_sge, SPDK_NVMF_MAX_SGL_ENTRIES); 1726 } 1727 1728 static int 1729 spdk_nvmf_rdma_request_fill_iovs(struct spdk_nvmf_rdma_transport *rtransport, 1730 struct spdk_nvmf_rdma_device *device, 1731 struct spdk_nvmf_rdma_request *rdma_req, 1732 uint32_t length) 1733 { 1734 struct spdk_nvmf_rdma_qpair *rqpair; 1735 struct spdk_nvmf_rdma_poll_group *rgroup; 1736 struct spdk_nvmf_request *req = &rdma_req->req; 1737 struct ibv_send_wr *wr = &rdma_req->data.wr; 1738 int rc; 1739 uint32_t num_wrs = 1; 1740 1741 rqpair = SPDK_CONTAINEROF(req->qpair, struct spdk_nvmf_rdma_qpair, qpair); 1742 rgroup = rqpair->poller->group; 1743 1744 /* rdma wr specifics */ 1745 nvmf_rdma_setup_request(rdma_req); 1746 1747 rc = spdk_nvmf_request_get_buffers(req, &rgroup->group, &rtransport->transport, 1748 length); 1749 if (rc != 0) { 1750 return rc; 1751 } 1752 1753 assert(req->iovcnt <= rqpair->max_send_sge); 1754 1755 rdma_req->iovpos = 0; 1756 1757 if (spdk_unlikely(req->dif.dif_insert_or_strip)) { 1758 num_wrs = nvmf_rdma_calc_num_wrs(length, rtransport->transport.opts.io_unit_size, 1759 req->dif.dif_ctx.block_size); 1760 if (num_wrs > 1) { 1761 rc = nvmf_request_alloc_wrs(rtransport, rdma_req, num_wrs - 1); 1762 if (rc != 0) { 1763 goto err_exit; 1764 } 1765 } 1766 } 1767 1768 rc = nvmf_rdma_fill_wr_sgl(rgroup, device, rdma_req, wr, length, num_wrs - 1); 1769 if (spdk_unlikely(rc != 0)) { 1770 goto err_exit; 1771 } 1772 1773 if (spdk_unlikely(num_wrs > 1)) { 1774 nvmf_rdma_update_remote_addr(rdma_req, num_wrs); 1775 } 1776 1777 /* set the number of outstanding data WRs for this request. */ 1778 rdma_req->num_outstanding_data_wr = num_wrs; 1779 1780 return rc; 1781 1782 err_exit: 1783 spdk_nvmf_request_free_buffers(req, &rgroup->group, &rtransport->transport); 1784 nvmf_rdma_request_free_data(rdma_req, rtransport); 1785 req->iovcnt = 0; 1786 return rc; 1787 } 1788 1789 static int 1790 nvmf_rdma_request_fill_iovs_multi_sgl(struct spdk_nvmf_rdma_transport *rtransport, 1791 struct spdk_nvmf_rdma_device *device, 1792 struct spdk_nvmf_rdma_request *rdma_req) 1793 { 1794 struct spdk_nvmf_rdma_qpair *rqpair; 1795 struct spdk_nvmf_rdma_poll_group *rgroup; 1796 struct ibv_send_wr *current_wr; 1797 struct spdk_nvmf_request *req = &rdma_req->req; 1798 struct spdk_nvme_sgl_descriptor *inline_segment, *desc; 1799 uint32_t num_sgl_descriptors; 1800 uint32_t lengths[SPDK_NVMF_MAX_SGL_ENTRIES]; 1801 uint32_t i; 1802 int rc; 1803 1804 rqpair = SPDK_CONTAINEROF(rdma_req->req.qpair, struct spdk_nvmf_rdma_qpair, qpair); 1805 rgroup = rqpair->poller->group; 1806 1807 inline_segment = &req->cmd->nvme_cmd.dptr.sgl1; 1808 assert(inline_segment->generic.type == SPDK_NVME_SGL_TYPE_LAST_SEGMENT); 1809 assert(inline_segment->unkeyed.subtype == SPDK_NVME_SGL_SUBTYPE_OFFSET); 1810 1811 num_sgl_descriptors = inline_segment->unkeyed.length / sizeof(struct spdk_nvme_sgl_descriptor); 1812 assert(num_sgl_descriptors <= SPDK_NVMF_MAX_SGL_ENTRIES); 1813 1814 if (nvmf_request_alloc_wrs(rtransport, rdma_req, num_sgl_descriptors - 1) != 0) { 1815 return -ENOMEM; 1816 } 1817 1818 desc = (struct spdk_nvme_sgl_descriptor *)rdma_req->recv->buf + inline_segment->address; 1819 for (i = 0; i < num_sgl_descriptors; i++) { 1820 if (spdk_likely(!req->dif.dif_insert_or_strip)) { 1821 lengths[i] = desc->keyed.length; 1822 } else { 1823 req->dif.orig_length += desc->keyed.length; 1824 lengths[i] = spdk_dif_get_length_with_md(desc->keyed.length, &req->dif.dif_ctx); 1825 req->dif.elba_length += lengths[i]; 1826 } 1827 desc++; 1828 } 1829 1830 rc = spdk_nvmf_request_get_buffers_multi(req, &rgroup->group, &rtransport->transport, 1831 lengths, num_sgl_descriptors); 1832 if (rc != 0) { 1833 nvmf_rdma_request_free_data(rdma_req, rtransport); 1834 return rc; 1835 } 1836 1837 /* The first WR must always be the embedded data WR. This is how we unwind them later. */ 1838 current_wr = &rdma_req->data.wr; 1839 assert(current_wr != NULL); 1840 1841 req->length = 0; 1842 rdma_req->iovpos = 0; 1843 desc = (struct spdk_nvme_sgl_descriptor *)rdma_req->recv->buf + inline_segment->address; 1844 for (i = 0; i < num_sgl_descriptors; i++) { 1845 /* The descriptors must be keyed data block descriptors with an address, not an offset. */ 1846 if (spdk_unlikely(desc->generic.type != SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK || 1847 desc->keyed.subtype != SPDK_NVME_SGL_SUBTYPE_ADDRESS)) { 1848 rc = -EINVAL; 1849 goto err_exit; 1850 } 1851 1852 current_wr->num_sge = 0; 1853 1854 rc = nvmf_rdma_fill_wr_sgl(rgroup, device, rdma_req, current_wr, lengths[i], 0); 1855 if (rc != 0) { 1856 rc = -ENOMEM; 1857 goto err_exit; 1858 } 1859 1860 req->length += desc->keyed.length; 1861 current_wr->wr.rdma.rkey = desc->keyed.key; 1862 current_wr->wr.rdma.remote_addr = desc->address; 1863 current_wr = current_wr->next; 1864 desc++; 1865 } 1866 1867 #ifdef SPDK_CONFIG_RDMA_SEND_WITH_INVAL 1868 /* Go back to the last descriptor in the list. */ 1869 desc--; 1870 if ((device->attr.device_cap_flags & IBV_DEVICE_MEM_MGT_EXTENSIONS) != 0) { 1871 if (desc->keyed.subtype == SPDK_NVME_SGL_SUBTYPE_INVALIDATE_KEY) { 1872 rdma_req->rsp.wr.opcode = IBV_WR_SEND_WITH_INV; 1873 rdma_req->rsp.wr.imm_data = desc->keyed.key; 1874 } 1875 } 1876 #endif 1877 1878 rdma_req->num_outstanding_data_wr = num_sgl_descriptors; 1879 1880 return 0; 1881 1882 err_exit: 1883 spdk_nvmf_request_free_buffers(req, &rgroup->group, &rtransport->transport); 1884 nvmf_rdma_request_free_data(rdma_req, rtransport); 1885 return rc; 1886 } 1887 1888 static int 1889 spdk_nvmf_rdma_request_parse_sgl(struct spdk_nvmf_rdma_transport *rtransport, 1890 struct spdk_nvmf_rdma_device *device, 1891 struct spdk_nvmf_rdma_request *rdma_req) 1892 { 1893 struct spdk_nvmf_request *req = &rdma_req->req; 1894 struct spdk_nvme_cpl *rsp; 1895 struct spdk_nvme_sgl_descriptor *sgl; 1896 int rc; 1897 uint32_t length; 1898 1899 rsp = &req->rsp->nvme_cpl; 1900 sgl = &req->cmd->nvme_cmd.dptr.sgl1; 1901 1902 if (sgl->generic.type == SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK && 1903 (sgl->keyed.subtype == SPDK_NVME_SGL_SUBTYPE_ADDRESS || 1904 sgl->keyed.subtype == SPDK_NVME_SGL_SUBTYPE_INVALIDATE_KEY)) { 1905 1906 length = sgl->keyed.length; 1907 if (length > rtransport->transport.opts.max_io_size) { 1908 SPDK_ERRLOG("SGL length 0x%x exceeds max io size 0x%x\n", 1909 length, rtransport->transport.opts.max_io_size); 1910 rsp->status.sc = SPDK_NVME_SC_DATA_SGL_LENGTH_INVALID; 1911 return -1; 1912 } 1913 #ifdef SPDK_CONFIG_RDMA_SEND_WITH_INVAL 1914 if ((device->attr.device_cap_flags & IBV_DEVICE_MEM_MGT_EXTENSIONS) != 0) { 1915 if (sgl->keyed.subtype == SPDK_NVME_SGL_SUBTYPE_INVALIDATE_KEY) { 1916 rdma_req->rsp.wr.opcode = IBV_WR_SEND_WITH_INV; 1917 rdma_req->rsp.wr.imm_data = sgl->keyed.key; 1918 } 1919 } 1920 #endif 1921 1922 /* fill request length and populate iovs */ 1923 req->length = length; 1924 1925 if (spdk_unlikely(req->dif.dif_insert_or_strip)) { 1926 req->dif.orig_length = length; 1927 length = spdk_dif_get_length_with_md(length, &req->dif.dif_ctx); 1928 req->dif.elba_length = length; 1929 } 1930 1931 rc = spdk_nvmf_rdma_request_fill_iovs(rtransport, device, rdma_req, length); 1932 if (spdk_unlikely(rc < 0)) { 1933 if (rc == -EINVAL) { 1934 SPDK_ERRLOG("SGL length exceeds the max I/O size\n"); 1935 return -1; 1936 } 1937 /* No available buffers. Queue this request up. */ 1938 SPDK_DEBUGLOG(SPDK_LOG_RDMA, "No available large data buffers. Queueing request %p\n", rdma_req); 1939 return 0; 1940 } 1941 1942 /* backward compatible */ 1943 req->data = req->iov[0].iov_base; 1944 1945 SPDK_DEBUGLOG(SPDK_LOG_RDMA, "Request %p took %d buffer/s from central pool\n", rdma_req, 1946 req->iovcnt); 1947 1948 return 0; 1949 } else if (sgl->generic.type == SPDK_NVME_SGL_TYPE_DATA_BLOCK && 1950 sgl->unkeyed.subtype == SPDK_NVME_SGL_SUBTYPE_OFFSET) { 1951 uint64_t offset = sgl->address; 1952 uint32_t max_len = rtransport->transport.opts.in_capsule_data_size; 1953 1954 SPDK_DEBUGLOG(SPDK_LOG_NVMF, "In-capsule data: offset 0x%" PRIx64 ", length 0x%x\n", 1955 offset, sgl->unkeyed.length); 1956 1957 if (offset > max_len) { 1958 SPDK_ERRLOG("In-capsule offset 0x%" PRIx64 " exceeds capsule length 0x%x\n", 1959 offset, max_len); 1960 rsp->status.sc = SPDK_NVME_SC_INVALID_SGL_OFFSET; 1961 return -1; 1962 } 1963 max_len -= (uint32_t)offset; 1964 1965 if (sgl->unkeyed.length > max_len) { 1966 SPDK_ERRLOG("In-capsule data length 0x%x exceeds capsule length 0x%x\n", 1967 sgl->unkeyed.length, max_len); 1968 rsp->status.sc = SPDK_NVME_SC_DATA_SGL_LENGTH_INVALID; 1969 return -1; 1970 } 1971 1972 rdma_req->num_outstanding_data_wr = 0; 1973 req->data = rdma_req->recv->buf + offset; 1974 req->data_from_pool = false; 1975 req->length = sgl->unkeyed.length; 1976 1977 req->iov[0].iov_base = req->data; 1978 req->iov[0].iov_len = req->length; 1979 req->iovcnt = 1; 1980 1981 return 0; 1982 } else if (sgl->generic.type == SPDK_NVME_SGL_TYPE_LAST_SEGMENT && 1983 sgl->unkeyed.subtype == SPDK_NVME_SGL_SUBTYPE_OFFSET) { 1984 1985 rc = nvmf_rdma_request_fill_iovs_multi_sgl(rtransport, device, rdma_req); 1986 if (rc == -ENOMEM) { 1987 SPDK_DEBUGLOG(SPDK_LOG_RDMA, "No available large data buffers. Queueing request %p\n", rdma_req); 1988 return 0; 1989 } else if (rc == -EINVAL) { 1990 SPDK_ERRLOG("Multi SGL element request length exceeds the max I/O size\n"); 1991 return -1; 1992 } 1993 1994 /* backward compatible */ 1995 req->data = req->iov[0].iov_base; 1996 1997 SPDK_DEBUGLOG(SPDK_LOG_RDMA, "Request %p took %d buffer/s from central pool\n", rdma_req, 1998 req->iovcnt); 1999 2000 return 0; 2001 } 2002 2003 SPDK_ERRLOG("Invalid NVMf I/O Command SGL: Type 0x%x, Subtype 0x%x\n", 2004 sgl->generic.type, sgl->generic.subtype); 2005 rsp->status.sc = SPDK_NVME_SC_SGL_DESCRIPTOR_TYPE_INVALID; 2006 return -1; 2007 } 2008 2009 static void 2010 nvmf_rdma_request_free(struct spdk_nvmf_rdma_request *rdma_req, 2011 struct spdk_nvmf_rdma_transport *rtransport) 2012 { 2013 struct spdk_nvmf_rdma_qpair *rqpair; 2014 struct spdk_nvmf_rdma_poll_group *rgroup; 2015 2016 rqpair = SPDK_CONTAINEROF(rdma_req->req.qpair, struct spdk_nvmf_rdma_qpair, qpair); 2017 if (rdma_req->req.data_from_pool) { 2018 rgroup = rqpair->poller->group; 2019 2020 spdk_nvmf_request_free_buffers(&rdma_req->req, &rgroup->group, &rtransport->transport); 2021 } 2022 nvmf_rdma_request_free_data(rdma_req, rtransport); 2023 rdma_req->req.length = 0; 2024 rdma_req->req.iovcnt = 0; 2025 rdma_req->req.data = NULL; 2026 rdma_req->rsp.wr.next = NULL; 2027 rdma_req->data.wr.next = NULL; 2028 memset(&rdma_req->req.dif, 0, sizeof(rdma_req->req.dif)); 2029 rqpair->qd--; 2030 2031 STAILQ_INSERT_HEAD(&rqpair->resources->free_queue, rdma_req, state_link); 2032 rdma_req->state = RDMA_REQUEST_STATE_FREE; 2033 } 2034 2035 static bool 2036 spdk_nvmf_rdma_request_process(struct spdk_nvmf_rdma_transport *rtransport, 2037 struct spdk_nvmf_rdma_request *rdma_req) 2038 { 2039 struct spdk_nvmf_rdma_qpair *rqpair; 2040 struct spdk_nvmf_rdma_device *device; 2041 struct spdk_nvmf_rdma_poll_group *rgroup; 2042 struct spdk_nvme_cpl *rsp = &rdma_req->req.rsp->nvme_cpl; 2043 int rc; 2044 struct spdk_nvmf_rdma_recv *rdma_recv; 2045 enum spdk_nvmf_rdma_request_state prev_state; 2046 bool progress = false; 2047 int data_posted; 2048 uint32_t num_blocks; 2049 2050 rqpair = SPDK_CONTAINEROF(rdma_req->req.qpair, struct spdk_nvmf_rdma_qpair, qpair); 2051 device = rqpair->port->device; 2052 rgroup = rqpair->poller->group; 2053 2054 assert(rdma_req->state != RDMA_REQUEST_STATE_FREE); 2055 2056 /* If the queue pair is in an error state, force the request to the completed state 2057 * to release resources. */ 2058 if (rqpair->ibv_state == IBV_QPS_ERR || rqpair->qpair.state != SPDK_NVMF_QPAIR_ACTIVE) { 2059 if (rdma_req->state == RDMA_REQUEST_STATE_NEED_BUFFER) { 2060 STAILQ_REMOVE(&rgroup->group.pending_buf_queue, &rdma_req->req, spdk_nvmf_request, buf_link); 2061 } else if (rdma_req->state == RDMA_REQUEST_STATE_DATA_TRANSFER_TO_CONTROLLER_PENDING) { 2062 STAILQ_REMOVE(&rqpair->pending_rdma_read_queue, rdma_req, spdk_nvmf_rdma_request, state_link); 2063 } else if (rdma_req->state == RDMA_REQUEST_STATE_DATA_TRANSFER_TO_HOST_PENDING) { 2064 STAILQ_REMOVE(&rqpair->pending_rdma_write_queue, rdma_req, spdk_nvmf_rdma_request, state_link); 2065 } 2066 rdma_req->state = RDMA_REQUEST_STATE_COMPLETED; 2067 } 2068 2069 /* The loop here is to allow for several back-to-back state changes. */ 2070 do { 2071 prev_state = rdma_req->state; 2072 2073 SPDK_DEBUGLOG(SPDK_LOG_RDMA, "Request %p entering state %d\n", rdma_req, prev_state); 2074 2075 switch (rdma_req->state) { 2076 case RDMA_REQUEST_STATE_FREE: 2077 /* Some external code must kick a request into RDMA_REQUEST_STATE_NEW 2078 * to escape this state. */ 2079 break; 2080 case RDMA_REQUEST_STATE_NEW: 2081 spdk_trace_record(TRACE_RDMA_REQUEST_STATE_NEW, 0, 0, 2082 (uintptr_t)rdma_req, (uintptr_t)rqpair->cm_id); 2083 rdma_recv = rdma_req->recv; 2084 2085 /* The first element of the SGL is the NVMe command */ 2086 rdma_req->req.cmd = (union nvmf_h2c_msg *)rdma_recv->sgl[0].addr; 2087 memset(rdma_req->req.rsp, 0, sizeof(*rdma_req->req.rsp)); 2088 2089 if (rqpair->ibv_state == IBV_QPS_ERR || rqpair->qpair.state != SPDK_NVMF_QPAIR_ACTIVE) { 2090 rdma_req->state = RDMA_REQUEST_STATE_COMPLETED; 2091 break; 2092 } 2093 2094 if (spdk_unlikely(spdk_nvmf_request_get_dif_ctx(&rdma_req->req, &rdma_req->req.dif.dif_ctx))) { 2095 rdma_req->req.dif.dif_insert_or_strip = true; 2096 } 2097 2098 /* The next state transition depends on the data transfer needs of this request. */ 2099 rdma_req->req.xfer = spdk_nvmf_rdma_request_get_xfer(rdma_req); 2100 2101 /* If no data to transfer, ready to execute. */ 2102 if (rdma_req->req.xfer == SPDK_NVME_DATA_NONE) { 2103 rdma_req->state = RDMA_REQUEST_STATE_READY_TO_EXECUTE; 2104 break; 2105 } 2106 2107 rdma_req->state = RDMA_REQUEST_STATE_NEED_BUFFER; 2108 STAILQ_INSERT_TAIL(&rgroup->group.pending_buf_queue, &rdma_req->req, buf_link); 2109 break; 2110 case RDMA_REQUEST_STATE_NEED_BUFFER: 2111 spdk_trace_record(TRACE_RDMA_REQUEST_STATE_NEED_BUFFER, 0, 0, 2112 (uintptr_t)rdma_req, (uintptr_t)rqpair->cm_id); 2113 2114 assert(rdma_req->req.xfer != SPDK_NVME_DATA_NONE); 2115 2116 if (&rdma_req->req != STAILQ_FIRST(&rgroup->group.pending_buf_queue)) { 2117 /* This request needs to wait in line to obtain a buffer */ 2118 break; 2119 } 2120 2121 /* Try to get a data buffer */ 2122 rc = spdk_nvmf_rdma_request_parse_sgl(rtransport, device, rdma_req); 2123 if (rc < 0) { 2124 STAILQ_REMOVE_HEAD(&rgroup->group.pending_buf_queue, buf_link); 2125 rsp->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; 2126 rdma_req->state = RDMA_REQUEST_STATE_READY_TO_COMPLETE; 2127 break; 2128 } 2129 2130 if (!rdma_req->req.data) { 2131 /* No buffers available. */ 2132 rgroup->stat.pending_data_buffer++; 2133 break; 2134 } 2135 2136 STAILQ_REMOVE_HEAD(&rgroup->group.pending_buf_queue, buf_link); 2137 2138 /* If data is transferring from host to controller and the data didn't 2139 * arrive using in capsule data, we need to do a transfer from the host. 2140 */ 2141 if (rdma_req->req.xfer == SPDK_NVME_DATA_HOST_TO_CONTROLLER && 2142 rdma_req->req.data_from_pool) { 2143 STAILQ_INSERT_TAIL(&rqpair->pending_rdma_read_queue, rdma_req, state_link); 2144 rdma_req->state = RDMA_REQUEST_STATE_DATA_TRANSFER_TO_CONTROLLER_PENDING; 2145 break; 2146 } 2147 2148 rdma_req->state = RDMA_REQUEST_STATE_READY_TO_EXECUTE; 2149 break; 2150 case RDMA_REQUEST_STATE_DATA_TRANSFER_TO_CONTROLLER_PENDING: 2151 spdk_trace_record(TRACE_RDMA_REQUEST_STATE_DATA_TRANSFER_TO_CONTROLLER_PENDING, 0, 0, 2152 (uintptr_t)rdma_req, (uintptr_t)rqpair->cm_id); 2153 2154 if (rdma_req != STAILQ_FIRST(&rqpair->pending_rdma_read_queue)) { 2155 /* This request needs to wait in line to perform RDMA */ 2156 break; 2157 } 2158 if (rqpair->current_send_depth + rdma_req->num_outstanding_data_wr > rqpair->max_send_depth 2159 || rqpair->current_read_depth + rdma_req->num_outstanding_data_wr > rqpair->max_read_depth) { 2160 /* We can only have so many WRs outstanding. we have to wait until some finish. */ 2161 rqpair->poller->stat.pending_rdma_read++; 2162 break; 2163 } 2164 2165 /* We have already verified that this request is the head of the queue. */ 2166 STAILQ_REMOVE_HEAD(&rqpair->pending_rdma_read_queue, state_link); 2167 2168 rc = request_transfer_in(&rdma_req->req); 2169 if (!rc) { 2170 rdma_req->state = RDMA_REQUEST_STATE_TRANSFERRING_HOST_TO_CONTROLLER; 2171 } else { 2172 rsp->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; 2173 rdma_req->state = RDMA_REQUEST_STATE_READY_TO_COMPLETE; 2174 } 2175 break; 2176 case RDMA_REQUEST_STATE_TRANSFERRING_HOST_TO_CONTROLLER: 2177 spdk_trace_record(TRACE_RDMA_REQUEST_STATE_TRANSFERRING_HOST_TO_CONTROLLER, 0, 0, 2178 (uintptr_t)rdma_req, (uintptr_t)rqpair->cm_id); 2179 /* Some external code must kick a request into RDMA_REQUEST_STATE_READY_TO_EXECUTE 2180 * to escape this state. */ 2181 break; 2182 case RDMA_REQUEST_STATE_READY_TO_EXECUTE: 2183 spdk_trace_record(TRACE_RDMA_REQUEST_STATE_READY_TO_EXECUTE, 0, 0, 2184 (uintptr_t)rdma_req, (uintptr_t)rqpair->cm_id); 2185 2186 if (spdk_unlikely(rdma_req->req.dif.dif_insert_or_strip)) { 2187 if (rdma_req->req.xfer == SPDK_NVME_DATA_HOST_TO_CONTROLLER) { 2188 /* generate DIF for write operation */ 2189 num_blocks = SPDK_CEIL_DIV(rdma_req->req.dif.elba_length, rdma_req->req.dif.dif_ctx.block_size); 2190 assert(num_blocks > 0); 2191 2192 rc = spdk_dif_generate(rdma_req->req.iov, rdma_req->req.iovcnt, 2193 num_blocks, &rdma_req->req.dif.dif_ctx); 2194 if (rc != 0) { 2195 SPDK_ERRLOG("DIF generation failed\n"); 2196 rdma_req->state = RDMA_REQUEST_STATE_COMPLETED; 2197 spdk_nvmf_rdma_start_disconnect(rqpair); 2198 break; 2199 } 2200 } 2201 2202 assert(rdma_req->req.dif.elba_length >= rdma_req->req.length); 2203 /* set extended length before IO operation */ 2204 rdma_req->req.length = rdma_req->req.dif.elba_length; 2205 } 2206 2207 rdma_req->state = RDMA_REQUEST_STATE_EXECUTING; 2208 spdk_nvmf_request_exec(&rdma_req->req); 2209 break; 2210 case RDMA_REQUEST_STATE_EXECUTING: 2211 spdk_trace_record(TRACE_RDMA_REQUEST_STATE_EXECUTING, 0, 0, 2212 (uintptr_t)rdma_req, (uintptr_t)rqpair->cm_id); 2213 /* Some external code must kick a request into RDMA_REQUEST_STATE_EXECUTED 2214 * to escape this state. */ 2215 break; 2216 case RDMA_REQUEST_STATE_EXECUTED: 2217 spdk_trace_record(TRACE_RDMA_REQUEST_STATE_EXECUTED, 0, 0, 2218 (uintptr_t)rdma_req, (uintptr_t)rqpair->cm_id); 2219 if (rdma_req->req.xfer == SPDK_NVME_DATA_CONTROLLER_TO_HOST) { 2220 STAILQ_INSERT_TAIL(&rqpair->pending_rdma_write_queue, rdma_req, state_link); 2221 rdma_req->state = RDMA_REQUEST_STATE_DATA_TRANSFER_TO_HOST_PENDING; 2222 } else { 2223 rdma_req->state = RDMA_REQUEST_STATE_READY_TO_COMPLETE; 2224 } 2225 if (spdk_unlikely(rdma_req->req.dif.dif_insert_or_strip)) { 2226 /* restore the original length */ 2227 rdma_req->req.length = rdma_req->req.dif.orig_length; 2228 2229 if (rdma_req->req.xfer == SPDK_NVME_DATA_CONTROLLER_TO_HOST) { 2230 struct spdk_dif_error error_blk; 2231 2232 num_blocks = SPDK_CEIL_DIV(rdma_req->req.dif.elba_length, rdma_req->req.dif.dif_ctx.block_size); 2233 2234 rc = spdk_dif_verify(rdma_req->req.iov, rdma_req->req.iovcnt, num_blocks, 2235 &rdma_req->req.dif.dif_ctx, &error_blk); 2236 if (rc) { 2237 struct spdk_nvme_cpl *rsp = &rdma_req->req.rsp->nvme_cpl; 2238 2239 SPDK_ERRLOG("DIF error detected. type=%d, offset=%" PRIu32 "\n", error_blk.err_type, 2240 error_blk.err_offset); 2241 rsp->status.sct = SPDK_NVME_SCT_MEDIA_ERROR; 2242 rsp->status.sc = spdk_nvmf_rdma_dif_error_to_compl_status(error_blk.err_type); 2243 rdma_req->state = RDMA_REQUEST_STATE_READY_TO_COMPLETE; 2244 STAILQ_REMOVE(&rqpair->pending_rdma_write_queue, rdma_req, spdk_nvmf_rdma_request, state_link); 2245 } 2246 } 2247 } 2248 break; 2249 case RDMA_REQUEST_STATE_DATA_TRANSFER_TO_HOST_PENDING: 2250 spdk_trace_record(TRACE_RDMA_REQUEST_STATE_DATA_TRANSFER_TO_HOST_PENDING, 0, 0, 2251 (uintptr_t)rdma_req, (uintptr_t)rqpair->cm_id); 2252 2253 if (rdma_req != STAILQ_FIRST(&rqpair->pending_rdma_write_queue)) { 2254 /* This request needs to wait in line to perform RDMA */ 2255 break; 2256 } 2257 if ((rqpair->current_send_depth + rdma_req->num_outstanding_data_wr + 1) > 2258 rqpair->max_send_depth) { 2259 /* We can only have so many WRs outstanding. we have to wait until some finish. 2260 * +1 since each request has an additional wr in the resp. */ 2261 rqpair->poller->stat.pending_rdma_write++; 2262 break; 2263 } 2264 2265 /* We have already verified that this request is the head of the queue. */ 2266 STAILQ_REMOVE_HEAD(&rqpair->pending_rdma_write_queue, state_link); 2267 2268 /* The data transfer will be kicked off from 2269 * RDMA_REQUEST_STATE_READY_TO_COMPLETE state. 2270 */ 2271 rdma_req->state = RDMA_REQUEST_STATE_READY_TO_COMPLETE; 2272 break; 2273 case RDMA_REQUEST_STATE_READY_TO_COMPLETE: 2274 spdk_trace_record(TRACE_RDMA_REQUEST_STATE_READY_TO_COMPLETE, 0, 0, 2275 (uintptr_t)rdma_req, (uintptr_t)rqpair->cm_id); 2276 rc = request_transfer_out(&rdma_req->req, &data_posted); 2277 assert(rc == 0); /* No good way to handle this currently */ 2278 if (rc) { 2279 rdma_req->state = RDMA_REQUEST_STATE_COMPLETED; 2280 } else { 2281 rdma_req->state = data_posted ? RDMA_REQUEST_STATE_TRANSFERRING_CONTROLLER_TO_HOST : 2282 RDMA_REQUEST_STATE_COMPLETING; 2283 } 2284 break; 2285 case RDMA_REQUEST_STATE_TRANSFERRING_CONTROLLER_TO_HOST: 2286 spdk_trace_record(TRACE_RDMA_REQUEST_STATE_TRANSFERRING_CONTROLLER_TO_HOST, 0, 0, 2287 (uintptr_t)rdma_req, (uintptr_t)rqpair->cm_id); 2288 /* Some external code must kick a request into RDMA_REQUEST_STATE_COMPLETED 2289 * to escape this state. */ 2290 break; 2291 case RDMA_REQUEST_STATE_COMPLETING: 2292 spdk_trace_record(TRACE_RDMA_REQUEST_STATE_COMPLETING, 0, 0, 2293 (uintptr_t)rdma_req, (uintptr_t)rqpair->cm_id); 2294 /* Some external code must kick a request into RDMA_REQUEST_STATE_COMPLETED 2295 * to escape this state. */ 2296 break; 2297 case RDMA_REQUEST_STATE_COMPLETED: 2298 spdk_trace_record(TRACE_RDMA_REQUEST_STATE_COMPLETED, 0, 0, 2299 (uintptr_t)rdma_req, (uintptr_t)rqpair->cm_id); 2300 2301 rqpair->poller->stat.request_latency += spdk_get_ticks() - rdma_req->receive_tsc; 2302 nvmf_rdma_request_free(rdma_req, rtransport); 2303 break; 2304 case RDMA_REQUEST_NUM_STATES: 2305 default: 2306 assert(0); 2307 break; 2308 } 2309 2310 if (rdma_req->state != prev_state) { 2311 progress = true; 2312 } 2313 } while (rdma_req->state != prev_state); 2314 2315 return progress; 2316 } 2317 2318 /* Public API callbacks begin here */ 2319 2320 #define SPDK_NVMF_RDMA_DEFAULT_MAX_QUEUE_DEPTH 128 2321 #define SPDK_NVMF_RDMA_DEFAULT_AQ_DEPTH 128 2322 #define SPDK_NVMF_RDMA_DEFAULT_SRQ_DEPTH 4096 2323 #define SPDK_NVMF_RDMA_DEFAULT_MAX_QPAIRS_PER_CTRLR 128 2324 #define SPDK_NVMF_RDMA_DEFAULT_IN_CAPSULE_DATA_SIZE 4096 2325 #define SPDK_NVMF_RDMA_DEFAULT_MAX_IO_SIZE 131072 2326 #define SPDK_NVMF_RDMA_MIN_IO_BUFFER_SIZE (SPDK_NVMF_RDMA_DEFAULT_MAX_IO_SIZE / SPDK_NVMF_MAX_SGL_ENTRIES) 2327 #define SPDK_NVMF_RDMA_DEFAULT_NUM_SHARED_BUFFERS 4095 2328 #define SPDK_NVMF_RDMA_DEFAULT_BUFFER_CACHE_SIZE 32 2329 #define SPDK_NVMF_RDMA_DEFAULT_NO_SRQ false 2330 #define SPDK_NVMF_RDMA_DIF_INSERT_OR_STRIP false 2331 2332 static void 2333 spdk_nvmf_rdma_opts_init(struct spdk_nvmf_transport_opts *opts) 2334 { 2335 opts->max_queue_depth = SPDK_NVMF_RDMA_DEFAULT_MAX_QUEUE_DEPTH; 2336 opts->max_qpairs_per_ctrlr = SPDK_NVMF_RDMA_DEFAULT_MAX_QPAIRS_PER_CTRLR; 2337 opts->in_capsule_data_size = SPDK_NVMF_RDMA_DEFAULT_IN_CAPSULE_DATA_SIZE; 2338 opts->max_io_size = SPDK_NVMF_RDMA_DEFAULT_MAX_IO_SIZE; 2339 opts->io_unit_size = SPDK_NVMF_RDMA_MIN_IO_BUFFER_SIZE; 2340 opts->max_aq_depth = SPDK_NVMF_RDMA_DEFAULT_AQ_DEPTH; 2341 opts->num_shared_buffers = SPDK_NVMF_RDMA_DEFAULT_NUM_SHARED_BUFFERS; 2342 opts->buf_cache_size = SPDK_NVMF_RDMA_DEFAULT_BUFFER_CACHE_SIZE; 2343 opts->max_srq_depth = SPDK_NVMF_RDMA_DEFAULT_SRQ_DEPTH; 2344 opts->no_srq = SPDK_NVMF_RDMA_DEFAULT_NO_SRQ; 2345 opts->dif_insert_or_strip = SPDK_NVMF_RDMA_DIF_INSERT_OR_STRIP; 2346 } 2347 2348 const struct spdk_mem_map_ops g_nvmf_rdma_map_ops = { 2349 .notify_cb = spdk_nvmf_rdma_mem_notify, 2350 .are_contiguous = spdk_nvmf_rdma_check_contiguous_entries 2351 }; 2352 2353 static int spdk_nvmf_rdma_destroy(struct spdk_nvmf_transport *transport); 2354 2355 static struct spdk_nvmf_transport * 2356 spdk_nvmf_rdma_create(struct spdk_nvmf_transport_opts *opts) 2357 { 2358 int rc; 2359 struct spdk_nvmf_rdma_transport *rtransport; 2360 struct spdk_nvmf_rdma_device *device, *tmp; 2361 struct ibv_context **contexts; 2362 uint32_t i; 2363 int flag; 2364 uint32_t sge_count; 2365 uint32_t min_shared_buffers; 2366 int max_device_sge = SPDK_NVMF_MAX_SGL_ENTRIES; 2367 pthread_mutexattr_t attr; 2368 2369 rtransport = calloc(1, sizeof(*rtransport)); 2370 if (!rtransport) { 2371 return NULL; 2372 } 2373 2374 if (pthread_mutexattr_init(&attr)) { 2375 SPDK_ERRLOG("pthread_mutexattr_init() failed\n"); 2376 free(rtransport); 2377 return NULL; 2378 } 2379 2380 if (pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE)) { 2381 SPDK_ERRLOG("pthread_mutexattr_settype() failed\n"); 2382 pthread_mutexattr_destroy(&attr); 2383 free(rtransport); 2384 return NULL; 2385 } 2386 2387 if (pthread_mutex_init(&rtransport->lock, &attr)) { 2388 SPDK_ERRLOG("pthread_mutex_init() failed\n"); 2389 pthread_mutexattr_destroy(&attr); 2390 free(rtransport); 2391 return NULL; 2392 } 2393 2394 pthread_mutexattr_destroy(&attr); 2395 2396 TAILQ_INIT(&rtransport->devices); 2397 TAILQ_INIT(&rtransport->ports); 2398 TAILQ_INIT(&rtransport->poll_groups); 2399 2400 rtransport->transport.ops = &spdk_nvmf_transport_rdma; 2401 2402 SPDK_INFOLOG(SPDK_LOG_RDMA, "*** RDMA Transport Init ***\n" 2403 " Transport opts: max_ioq_depth=%d, max_io_size=%d,\n" 2404 " max_qpairs_per_ctrlr=%d, io_unit_size=%d,\n" 2405 " in_capsule_data_size=%d, max_aq_depth=%d,\n" 2406 " num_shared_buffers=%d, max_srq_depth=%d, no_srq=%d\n", 2407 opts->max_queue_depth, 2408 opts->max_io_size, 2409 opts->max_qpairs_per_ctrlr, 2410 opts->io_unit_size, 2411 opts->in_capsule_data_size, 2412 opts->max_aq_depth, 2413 opts->num_shared_buffers, 2414 opts->max_srq_depth, 2415 opts->no_srq); 2416 2417 /* I/O unit size cannot be larger than max I/O size */ 2418 if (opts->io_unit_size > opts->max_io_size) { 2419 opts->io_unit_size = opts->max_io_size; 2420 } 2421 2422 if (opts->num_shared_buffers < (SPDK_NVMF_MAX_SGL_ENTRIES * 2)) { 2423 SPDK_ERRLOG("The number of shared data buffers (%d) is less than" 2424 "the minimum number required to guarantee that forward progress can be made (%d)\n", 2425 opts->num_shared_buffers, (SPDK_NVMF_MAX_SGL_ENTRIES * 2)); 2426 spdk_nvmf_rdma_destroy(&rtransport->transport); 2427 return NULL; 2428 } 2429 2430 min_shared_buffers = spdk_thread_get_count() * opts->buf_cache_size; 2431 if (min_shared_buffers > opts->num_shared_buffers) { 2432 SPDK_ERRLOG("There are not enough buffers to satisfy" 2433 "per-poll group caches for each thread. (%" PRIu32 ")" 2434 "supplied. (%" PRIu32 ") required\n", opts->num_shared_buffers, min_shared_buffers); 2435 SPDK_ERRLOG("Please specify a larger number of shared buffers\n"); 2436 spdk_nvmf_rdma_destroy(&rtransport->transport); 2437 return NULL; 2438 } 2439 2440 sge_count = opts->max_io_size / opts->io_unit_size; 2441 if (sge_count > NVMF_DEFAULT_TX_SGE) { 2442 SPDK_ERRLOG("Unsupported IO Unit size specified, %d bytes\n", opts->io_unit_size); 2443 spdk_nvmf_rdma_destroy(&rtransport->transport); 2444 return NULL; 2445 } 2446 2447 rtransport->event_channel = rdma_create_event_channel(); 2448 if (rtransport->event_channel == NULL) { 2449 SPDK_ERRLOG("rdma_create_event_channel() failed, %s\n", spdk_strerror(errno)); 2450 spdk_nvmf_rdma_destroy(&rtransport->transport); 2451 return NULL; 2452 } 2453 2454 flag = fcntl(rtransport->event_channel->fd, F_GETFL); 2455 if (fcntl(rtransport->event_channel->fd, F_SETFL, flag | O_NONBLOCK) < 0) { 2456 SPDK_ERRLOG("fcntl can't set nonblocking mode for socket, fd: %d (%s)\n", 2457 rtransport->event_channel->fd, spdk_strerror(errno)); 2458 spdk_nvmf_rdma_destroy(&rtransport->transport); 2459 return NULL; 2460 } 2461 2462 rtransport->data_wr_pool = spdk_mempool_create("spdk_nvmf_rdma_wr_data", 2463 opts->max_queue_depth * SPDK_NVMF_MAX_SGL_ENTRIES, 2464 sizeof(struct spdk_nvmf_rdma_request_data), 2465 SPDK_MEMPOOL_DEFAULT_CACHE_SIZE, 2466 SPDK_ENV_SOCKET_ID_ANY); 2467 if (!rtransport->data_wr_pool) { 2468 SPDK_ERRLOG("Unable to allocate work request pool for poll group\n"); 2469 spdk_nvmf_rdma_destroy(&rtransport->transport); 2470 return NULL; 2471 } 2472 2473 contexts = rdma_get_devices(NULL); 2474 if (contexts == NULL) { 2475 SPDK_ERRLOG("rdma_get_devices() failed: %s (%d)\n", spdk_strerror(errno), errno); 2476 spdk_nvmf_rdma_destroy(&rtransport->transport); 2477 return NULL; 2478 } 2479 2480 i = 0; 2481 rc = 0; 2482 while (contexts[i] != NULL) { 2483 device = calloc(1, sizeof(*device)); 2484 if (!device) { 2485 SPDK_ERRLOG("Unable to allocate memory for RDMA devices.\n"); 2486 rc = -ENOMEM; 2487 break; 2488 } 2489 device->context = contexts[i]; 2490 rc = ibv_query_device(device->context, &device->attr); 2491 if (rc < 0) { 2492 SPDK_ERRLOG("Failed to query RDMA device attributes.\n"); 2493 free(device); 2494 break; 2495 2496 } 2497 2498 max_device_sge = spdk_min(max_device_sge, device->attr.max_sge); 2499 2500 #ifdef SPDK_CONFIG_RDMA_SEND_WITH_INVAL 2501 if ((device->attr.device_cap_flags & IBV_DEVICE_MEM_MGT_EXTENSIONS) == 0) { 2502 SPDK_WARNLOG("The libibverbs on this system supports SEND_WITH_INVALIDATE,"); 2503 SPDK_WARNLOG("but the device with vendor ID %u does not.\n", device->attr.vendor_id); 2504 } 2505 2506 /** 2507 * The vendor ID is assigned by the IEEE and an ID of 0 implies Soft-RoCE. 2508 * The Soft-RoCE RXE driver does not currently support send with invalidate, 2509 * but incorrectly reports that it does. There are changes making their way 2510 * through the kernel now that will enable this feature. When they are merged, 2511 * we can conditionally enable this feature. 2512 * 2513 * TODO: enable this for versions of the kernel rxe driver that support it. 2514 */ 2515 if (device->attr.vendor_id == 0) { 2516 device->attr.device_cap_flags &= ~(IBV_DEVICE_MEM_MGT_EXTENSIONS); 2517 } 2518 #endif 2519 2520 /* set up device context async ev fd as NON_BLOCKING */ 2521 flag = fcntl(device->context->async_fd, F_GETFL); 2522 rc = fcntl(device->context->async_fd, F_SETFL, flag | O_NONBLOCK); 2523 if (rc < 0) { 2524 SPDK_ERRLOG("Failed to set context async fd to NONBLOCK.\n"); 2525 free(device); 2526 break; 2527 } 2528 2529 TAILQ_INSERT_TAIL(&rtransport->devices, device, link); 2530 i++; 2531 2532 if (g_nvmf_hooks.get_ibv_pd) { 2533 device->pd = g_nvmf_hooks.get_ibv_pd(NULL, device->context); 2534 } else { 2535 device->pd = ibv_alloc_pd(device->context); 2536 } 2537 2538 if (!device->pd) { 2539 SPDK_ERRLOG("Unable to allocate protection domain.\n"); 2540 rc = -ENOMEM; 2541 break; 2542 } 2543 2544 assert(device->map == NULL); 2545 2546 device->map = spdk_mem_map_alloc(0, &g_nvmf_rdma_map_ops, device->pd); 2547 if (!device->map) { 2548 SPDK_ERRLOG("Unable to allocate memory map for listen address\n"); 2549 rc = -ENOMEM; 2550 break; 2551 } 2552 2553 assert(device->map != NULL); 2554 assert(device->pd != NULL); 2555 } 2556 rdma_free_devices(contexts); 2557 2558 if (opts->io_unit_size * max_device_sge < opts->max_io_size) { 2559 /* divide and round up. */ 2560 opts->io_unit_size = (opts->max_io_size + max_device_sge - 1) / max_device_sge; 2561 2562 /* round up to the nearest 4k. */ 2563 opts->io_unit_size = (opts->io_unit_size + NVMF_DATA_BUFFER_ALIGNMENT - 1) & ~NVMF_DATA_BUFFER_MASK; 2564 2565 opts->io_unit_size = spdk_max(opts->io_unit_size, SPDK_NVMF_RDMA_MIN_IO_BUFFER_SIZE); 2566 SPDK_NOTICELOG("Adjusting the io unit size to fit the device's maximum I/O size. New I/O unit size %u\n", 2567 opts->io_unit_size); 2568 } 2569 2570 if (rc < 0) { 2571 spdk_nvmf_rdma_destroy(&rtransport->transport); 2572 return NULL; 2573 } 2574 2575 /* Set up poll descriptor array to monitor events from RDMA and IB 2576 * in a single poll syscall 2577 */ 2578 rtransport->npoll_fds = i + 1; 2579 i = 0; 2580 rtransport->poll_fds = calloc(rtransport->npoll_fds, sizeof(struct pollfd)); 2581 if (rtransport->poll_fds == NULL) { 2582 SPDK_ERRLOG("poll_fds allocation failed\n"); 2583 spdk_nvmf_rdma_destroy(&rtransport->transport); 2584 return NULL; 2585 } 2586 2587 rtransport->poll_fds[i].fd = rtransport->event_channel->fd; 2588 rtransport->poll_fds[i++].events = POLLIN; 2589 2590 TAILQ_FOREACH_SAFE(device, &rtransport->devices, link, tmp) { 2591 rtransport->poll_fds[i].fd = device->context->async_fd; 2592 rtransport->poll_fds[i++].events = POLLIN; 2593 } 2594 2595 return &rtransport->transport; 2596 } 2597 2598 static int 2599 spdk_nvmf_rdma_destroy(struct spdk_nvmf_transport *transport) 2600 { 2601 struct spdk_nvmf_rdma_transport *rtransport; 2602 struct spdk_nvmf_rdma_port *port, *port_tmp; 2603 struct spdk_nvmf_rdma_device *device, *device_tmp; 2604 2605 rtransport = SPDK_CONTAINEROF(transport, struct spdk_nvmf_rdma_transport, transport); 2606 2607 TAILQ_FOREACH_SAFE(port, &rtransport->ports, link, port_tmp) { 2608 TAILQ_REMOVE(&rtransport->ports, port, link); 2609 rdma_destroy_id(port->id); 2610 free(port); 2611 } 2612 2613 if (rtransport->poll_fds != NULL) { 2614 free(rtransport->poll_fds); 2615 } 2616 2617 if (rtransport->event_channel != NULL) { 2618 rdma_destroy_event_channel(rtransport->event_channel); 2619 } 2620 2621 TAILQ_FOREACH_SAFE(device, &rtransport->devices, link, device_tmp) { 2622 TAILQ_REMOVE(&rtransport->devices, device, link); 2623 if (device->map) { 2624 spdk_mem_map_free(&device->map); 2625 } 2626 if (device->pd) { 2627 if (!g_nvmf_hooks.get_ibv_pd) { 2628 ibv_dealloc_pd(device->pd); 2629 } 2630 } 2631 free(device); 2632 } 2633 2634 if (rtransport->data_wr_pool != NULL) { 2635 if (spdk_mempool_count(rtransport->data_wr_pool) != 2636 (transport->opts.max_queue_depth * SPDK_NVMF_MAX_SGL_ENTRIES)) { 2637 SPDK_ERRLOG("transport wr pool count is %zu but should be %u\n", 2638 spdk_mempool_count(rtransport->data_wr_pool), 2639 transport->opts.max_queue_depth * SPDK_NVMF_MAX_SGL_ENTRIES); 2640 } 2641 } 2642 2643 spdk_mempool_free(rtransport->data_wr_pool); 2644 2645 pthread_mutex_destroy(&rtransport->lock); 2646 free(rtransport); 2647 2648 return 0; 2649 } 2650 2651 static int 2652 spdk_nvmf_rdma_trid_from_cm_id(struct rdma_cm_id *id, 2653 struct spdk_nvme_transport_id *trid, 2654 bool peer); 2655 2656 static int 2657 spdk_nvmf_rdma_listen(struct spdk_nvmf_transport *transport, 2658 const struct spdk_nvme_transport_id *trid) 2659 { 2660 struct spdk_nvmf_rdma_transport *rtransport; 2661 struct spdk_nvmf_rdma_device *device; 2662 struct spdk_nvmf_rdma_port *port_tmp, *port; 2663 struct addrinfo *res; 2664 struct addrinfo hints; 2665 int family; 2666 int rc; 2667 2668 rtransport = SPDK_CONTAINEROF(transport, struct spdk_nvmf_rdma_transport, transport); 2669 2670 port = calloc(1, sizeof(*port)); 2671 if (!port) { 2672 return -ENOMEM; 2673 } 2674 2675 /* Selectively copy the trid. Things like NQN don't matter here - that 2676 * mapping is enforced elsewhere. 2677 */ 2678 port->trid.trtype = SPDK_NVME_TRANSPORT_RDMA; 2679 port->trid.adrfam = trid->adrfam; 2680 snprintf(port->trid.traddr, sizeof(port->trid.traddr), "%s", trid->traddr); 2681 snprintf(port->trid.trsvcid, sizeof(port->trid.trsvcid), "%s", trid->trsvcid); 2682 2683 switch (port->trid.adrfam) { 2684 case SPDK_NVMF_ADRFAM_IPV4: 2685 family = AF_INET; 2686 break; 2687 case SPDK_NVMF_ADRFAM_IPV6: 2688 family = AF_INET6; 2689 break; 2690 default: 2691 SPDK_ERRLOG("Unhandled ADRFAM %d\n", port->trid.adrfam); 2692 free(port); 2693 return -EINVAL; 2694 } 2695 2696 memset(&hints, 0, sizeof(hints)); 2697 hints.ai_family = family; 2698 hints.ai_flags = AI_NUMERICSERV; 2699 hints.ai_socktype = SOCK_STREAM; 2700 hints.ai_protocol = 0; 2701 2702 rc = getaddrinfo(port->trid.traddr, port->trid.trsvcid, &hints, &res); 2703 if (rc) { 2704 SPDK_ERRLOG("getaddrinfo failed: %s (%d)\n", gai_strerror(rc), rc); 2705 free(port); 2706 return -EINVAL; 2707 } 2708 2709 pthread_mutex_lock(&rtransport->lock); 2710 assert(rtransport->event_channel != NULL); 2711 TAILQ_FOREACH(port_tmp, &rtransport->ports, link) { 2712 if (spdk_nvme_transport_id_compare(&port_tmp->trid, &port->trid) == 0) { 2713 port_tmp->ref++; 2714 freeaddrinfo(res); 2715 free(port); 2716 /* Already listening at this address */ 2717 pthread_mutex_unlock(&rtransport->lock); 2718 return 0; 2719 } 2720 } 2721 2722 rc = rdma_create_id(rtransport->event_channel, &port->id, port, RDMA_PS_TCP); 2723 if (rc < 0) { 2724 SPDK_ERRLOG("rdma_create_id() failed\n"); 2725 freeaddrinfo(res); 2726 free(port); 2727 pthread_mutex_unlock(&rtransport->lock); 2728 return rc; 2729 } 2730 2731 rc = rdma_bind_addr(port->id, res->ai_addr); 2732 freeaddrinfo(res); 2733 2734 if (rc < 0) { 2735 SPDK_ERRLOG("rdma_bind_addr() failed\n"); 2736 rdma_destroy_id(port->id); 2737 free(port); 2738 pthread_mutex_unlock(&rtransport->lock); 2739 return rc; 2740 } 2741 2742 if (!port->id->verbs) { 2743 SPDK_ERRLOG("ibv_context is null\n"); 2744 rdma_destroy_id(port->id); 2745 free(port); 2746 pthread_mutex_unlock(&rtransport->lock); 2747 return -1; 2748 } 2749 2750 rc = rdma_listen(port->id, 10); /* 10 = backlog */ 2751 if (rc < 0) { 2752 SPDK_ERRLOG("rdma_listen() failed\n"); 2753 rdma_destroy_id(port->id); 2754 free(port); 2755 pthread_mutex_unlock(&rtransport->lock); 2756 return rc; 2757 } 2758 2759 TAILQ_FOREACH(device, &rtransport->devices, link) { 2760 if (device->context == port->id->verbs) { 2761 port->device = device; 2762 break; 2763 } 2764 } 2765 if (!port->device) { 2766 SPDK_ERRLOG("Accepted a connection with verbs %p, but unable to find a corresponding device.\n", 2767 port->id->verbs); 2768 rdma_destroy_id(port->id); 2769 free(port); 2770 pthread_mutex_unlock(&rtransport->lock); 2771 return -EINVAL; 2772 } 2773 2774 SPDK_INFOLOG(SPDK_LOG_RDMA, "*** NVMf Target Listening on %s port %d ***\n", 2775 port->trid.traddr, ntohs(rdma_get_src_port(port->id))); 2776 2777 port->ref = 1; 2778 2779 TAILQ_INSERT_TAIL(&rtransport->ports, port, link); 2780 pthread_mutex_unlock(&rtransport->lock); 2781 2782 return 0; 2783 } 2784 2785 static int 2786 spdk_nvmf_rdma_stop_listen(struct spdk_nvmf_transport *transport, 2787 const struct spdk_nvme_transport_id *_trid) 2788 { 2789 struct spdk_nvmf_rdma_transport *rtransport; 2790 struct spdk_nvmf_rdma_port *port, *tmp; 2791 struct spdk_nvme_transport_id trid = {}; 2792 2793 rtransport = SPDK_CONTAINEROF(transport, struct spdk_nvmf_rdma_transport, transport); 2794 2795 /* Selectively copy the trid. Things like NQN don't matter here - that 2796 * mapping is enforced elsewhere. 2797 */ 2798 trid.trtype = SPDK_NVME_TRANSPORT_RDMA; 2799 trid.adrfam = _trid->adrfam; 2800 snprintf(trid.traddr, sizeof(port->trid.traddr), "%s", _trid->traddr); 2801 snprintf(trid.trsvcid, sizeof(port->trid.trsvcid), "%s", _trid->trsvcid); 2802 2803 pthread_mutex_lock(&rtransport->lock); 2804 TAILQ_FOREACH_SAFE(port, &rtransport->ports, link, tmp) { 2805 if (spdk_nvme_transport_id_compare(&port->trid, &trid) == 0) { 2806 assert(port->ref > 0); 2807 port->ref--; 2808 if (port->ref == 0) { 2809 TAILQ_REMOVE(&rtransport->ports, port, link); 2810 rdma_destroy_id(port->id); 2811 free(port); 2812 } 2813 break; 2814 } 2815 } 2816 2817 pthread_mutex_unlock(&rtransport->lock); 2818 return 0; 2819 } 2820 2821 static void 2822 spdk_nvmf_rdma_qpair_process_pending(struct spdk_nvmf_rdma_transport *rtransport, 2823 struct spdk_nvmf_rdma_qpair *rqpair, bool drain) 2824 { 2825 struct spdk_nvmf_request *req, *tmp; 2826 struct spdk_nvmf_rdma_request *rdma_req, *req_tmp; 2827 struct spdk_nvmf_rdma_resources *resources; 2828 2829 /* We process I/O in the data transfer pending queue at the highest priority. RDMA reads first */ 2830 STAILQ_FOREACH_SAFE(rdma_req, &rqpair->pending_rdma_read_queue, state_link, req_tmp) { 2831 if (spdk_nvmf_rdma_request_process(rtransport, rdma_req) == false && drain == false) { 2832 break; 2833 } 2834 } 2835 2836 /* Then RDMA writes since reads have stronger restrictions than writes */ 2837 STAILQ_FOREACH_SAFE(rdma_req, &rqpair->pending_rdma_write_queue, state_link, req_tmp) { 2838 if (spdk_nvmf_rdma_request_process(rtransport, rdma_req) == false && drain == false) { 2839 break; 2840 } 2841 } 2842 2843 /* The second highest priority is I/O waiting on memory buffers. */ 2844 STAILQ_FOREACH_SAFE(req, &rqpair->poller->group->group.pending_buf_queue, buf_link, tmp) { 2845 rdma_req = SPDK_CONTAINEROF(req, struct spdk_nvmf_rdma_request, req); 2846 if (spdk_nvmf_rdma_request_process(rtransport, rdma_req) == false && drain == false) { 2847 break; 2848 } 2849 } 2850 2851 resources = rqpair->resources; 2852 while (!STAILQ_EMPTY(&resources->free_queue) && !STAILQ_EMPTY(&resources->incoming_queue)) { 2853 rdma_req = STAILQ_FIRST(&resources->free_queue); 2854 STAILQ_REMOVE_HEAD(&resources->free_queue, state_link); 2855 rdma_req->recv = STAILQ_FIRST(&resources->incoming_queue); 2856 STAILQ_REMOVE_HEAD(&resources->incoming_queue, link); 2857 2858 if (rqpair->srq != NULL) { 2859 rdma_req->req.qpair = &rdma_req->recv->qpair->qpair; 2860 rdma_req->recv->qpair->qd++; 2861 } else { 2862 rqpair->qd++; 2863 } 2864 2865 rdma_req->receive_tsc = rdma_req->recv->receive_tsc; 2866 rdma_req->state = RDMA_REQUEST_STATE_NEW; 2867 if (spdk_nvmf_rdma_request_process(rtransport, rdma_req) == false) { 2868 break; 2869 } 2870 } 2871 if (!STAILQ_EMPTY(&resources->incoming_queue) && STAILQ_EMPTY(&resources->free_queue)) { 2872 rqpair->poller->stat.pending_free_request++; 2873 } 2874 } 2875 2876 static void 2877 _nvmf_rdma_qpair_disconnect(void *ctx) 2878 { 2879 struct spdk_nvmf_qpair *qpair = ctx; 2880 2881 spdk_nvmf_qpair_disconnect(qpair, NULL, NULL); 2882 } 2883 2884 static void 2885 _nvmf_rdma_try_disconnect(void *ctx) 2886 { 2887 struct spdk_nvmf_qpair *qpair = ctx; 2888 struct spdk_nvmf_poll_group *group; 2889 2890 /* Read the group out of the qpair. This is normally set and accessed only from 2891 * the thread that created the group. Here, we're not on that thread necessarily. 2892 * The data member qpair->group begins it's life as NULL and then is assigned to 2893 * a pointer and never changes. So fortunately reading this and checking for 2894 * non-NULL is thread safe in the x86_64 memory model. */ 2895 group = qpair->group; 2896 2897 if (group == NULL) { 2898 /* The qpair hasn't been assigned to a group yet, so we can't 2899 * process a disconnect. Send a message to ourself and try again. */ 2900 spdk_thread_send_msg(spdk_get_thread(), _nvmf_rdma_try_disconnect, qpair); 2901 return; 2902 } 2903 2904 spdk_thread_send_msg(group->thread, _nvmf_rdma_qpair_disconnect, qpair); 2905 } 2906 2907 static inline void 2908 spdk_nvmf_rdma_start_disconnect(struct spdk_nvmf_rdma_qpair *rqpair) 2909 { 2910 if (!__atomic_test_and_set(&rqpair->disconnect_started, __ATOMIC_RELAXED)) { 2911 _nvmf_rdma_try_disconnect(&rqpair->qpair); 2912 } 2913 } 2914 2915 static void nvmf_rdma_destroy_drained_qpair(void *ctx) 2916 { 2917 struct spdk_nvmf_rdma_qpair *rqpair = ctx; 2918 struct spdk_nvmf_rdma_transport *rtransport = SPDK_CONTAINEROF(rqpair->qpair.transport, 2919 struct spdk_nvmf_rdma_transport, transport); 2920 2921 /* In non SRQ path, we will reach rqpair->max_queue_depth. In SRQ path, we will get the last_wqe event. */ 2922 if (rqpair->current_send_depth != 0) { 2923 return; 2924 } 2925 2926 if (rqpair->srq == NULL && rqpair->current_recv_depth != rqpair->max_queue_depth) { 2927 return; 2928 } 2929 2930 if (rqpair->srq != NULL && rqpair->last_wqe_reached == false) { 2931 return; 2932 } 2933 2934 spdk_nvmf_rdma_qpair_process_pending(rtransport, rqpair, true); 2935 2936 /* Qpair will be destroyed after nvmf layer closes this qpair */ 2937 if (rqpair->qpair.state != SPDK_NVMF_QPAIR_ERROR) { 2938 return; 2939 } 2940 2941 spdk_nvmf_rdma_qpair_destroy(rqpair); 2942 } 2943 2944 2945 static int 2946 nvmf_rdma_disconnect(struct rdma_cm_event *evt) 2947 { 2948 struct spdk_nvmf_qpair *qpair; 2949 struct spdk_nvmf_rdma_qpair *rqpair; 2950 2951 if (evt->id == NULL) { 2952 SPDK_ERRLOG("disconnect request: missing cm_id\n"); 2953 return -1; 2954 } 2955 2956 qpair = evt->id->context; 2957 if (qpair == NULL) { 2958 SPDK_ERRLOG("disconnect request: no active connection\n"); 2959 return -1; 2960 } 2961 2962 rqpair = SPDK_CONTAINEROF(qpair, struct spdk_nvmf_rdma_qpair, qpair); 2963 2964 spdk_trace_record(TRACE_RDMA_QP_DISCONNECT, 0, 0, (uintptr_t)rqpair->cm_id, 0); 2965 2966 spdk_nvmf_rdma_update_ibv_state(rqpair); 2967 2968 spdk_nvmf_rdma_start_disconnect(rqpair); 2969 2970 return 0; 2971 } 2972 2973 #ifdef DEBUG 2974 static const char *CM_EVENT_STR[] = { 2975 "RDMA_CM_EVENT_ADDR_RESOLVED", 2976 "RDMA_CM_EVENT_ADDR_ERROR", 2977 "RDMA_CM_EVENT_ROUTE_RESOLVED", 2978 "RDMA_CM_EVENT_ROUTE_ERROR", 2979 "RDMA_CM_EVENT_CONNECT_REQUEST", 2980 "RDMA_CM_EVENT_CONNECT_RESPONSE", 2981 "RDMA_CM_EVENT_CONNECT_ERROR", 2982 "RDMA_CM_EVENT_UNREACHABLE", 2983 "RDMA_CM_EVENT_REJECTED", 2984 "RDMA_CM_EVENT_ESTABLISHED", 2985 "RDMA_CM_EVENT_DISCONNECTED", 2986 "RDMA_CM_EVENT_DEVICE_REMOVAL", 2987 "RDMA_CM_EVENT_MULTICAST_JOIN", 2988 "RDMA_CM_EVENT_MULTICAST_ERROR", 2989 "RDMA_CM_EVENT_ADDR_CHANGE", 2990 "RDMA_CM_EVENT_TIMEWAIT_EXIT" 2991 }; 2992 #endif /* DEBUG */ 2993 2994 static void 2995 nvmf_rdma_handle_last_wqe_reached(void *ctx) 2996 { 2997 struct spdk_nvmf_rdma_qpair *rqpair = ctx; 2998 rqpair->last_wqe_reached = true; 2999 3000 nvmf_rdma_destroy_drained_qpair(rqpair); 3001 } 3002 3003 static void 3004 spdk_nvmf_process_cm_event(struct spdk_nvmf_transport *transport, new_qpair_fn cb_fn, void *cb_arg) 3005 { 3006 struct spdk_nvmf_rdma_transport *rtransport; 3007 struct rdma_cm_event *event; 3008 int rc; 3009 3010 rtransport = SPDK_CONTAINEROF(transport, struct spdk_nvmf_rdma_transport, transport); 3011 3012 if (rtransport->event_channel == NULL) { 3013 return; 3014 } 3015 3016 while (1) { 3017 rc = rdma_get_cm_event(rtransport->event_channel, &event); 3018 if (rc == 0) { 3019 SPDK_DEBUGLOG(SPDK_LOG_RDMA, "Acceptor Event: %s\n", CM_EVENT_STR[event->event]); 3020 3021 spdk_trace_record(TRACE_RDMA_CM_ASYNC_EVENT, 0, 0, 0, event->event); 3022 3023 switch (event->event) { 3024 case RDMA_CM_EVENT_ADDR_RESOLVED: 3025 case RDMA_CM_EVENT_ADDR_ERROR: 3026 case RDMA_CM_EVENT_ROUTE_RESOLVED: 3027 case RDMA_CM_EVENT_ROUTE_ERROR: 3028 /* No action required. The target never attempts to resolve routes. */ 3029 break; 3030 case RDMA_CM_EVENT_CONNECT_REQUEST: 3031 rc = nvmf_rdma_connect(transport, event, cb_fn, cb_arg); 3032 if (rc < 0) { 3033 SPDK_ERRLOG("Unable to process connect event. rc: %d\n", rc); 3034 break; 3035 } 3036 break; 3037 case RDMA_CM_EVENT_CONNECT_RESPONSE: 3038 /* The target never initiates a new connection. So this will not occur. */ 3039 break; 3040 case RDMA_CM_EVENT_CONNECT_ERROR: 3041 /* Can this happen? The docs say it can, but not sure what causes it. */ 3042 break; 3043 case RDMA_CM_EVENT_UNREACHABLE: 3044 case RDMA_CM_EVENT_REJECTED: 3045 /* These only occur on the client side. */ 3046 break; 3047 case RDMA_CM_EVENT_ESTABLISHED: 3048 /* TODO: Should we be waiting for this event anywhere? */ 3049 break; 3050 case RDMA_CM_EVENT_DISCONNECTED: 3051 case RDMA_CM_EVENT_DEVICE_REMOVAL: 3052 rc = nvmf_rdma_disconnect(event); 3053 if (rc < 0) { 3054 SPDK_ERRLOG("Unable to process disconnect event. rc: %d\n", rc); 3055 break; 3056 } 3057 break; 3058 case RDMA_CM_EVENT_MULTICAST_JOIN: 3059 case RDMA_CM_EVENT_MULTICAST_ERROR: 3060 /* Multicast is not used */ 3061 break; 3062 case RDMA_CM_EVENT_ADDR_CHANGE: 3063 /* Not utilizing this event */ 3064 break; 3065 case RDMA_CM_EVENT_TIMEWAIT_EXIT: 3066 /* For now, do nothing. The target never re-uses queue pairs. */ 3067 break; 3068 default: 3069 SPDK_ERRLOG("Unexpected Acceptor Event [%d]\n", event->event); 3070 break; 3071 } 3072 3073 rdma_ack_cm_event(event); 3074 } else { 3075 if (errno != EAGAIN && errno != EWOULDBLOCK) { 3076 SPDK_ERRLOG("Acceptor Event Error: %s\n", spdk_strerror(errno)); 3077 } 3078 break; 3079 } 3080 } 3081 } 3082 3083 static void 3084 spdk_nvmf_process_ib_event(struct spdk_nvmf_rdma_device *device) 3085 { 3086 int rc; 3087 struct spdk_nvmf_rdma_qpair *rqpair = NULL; 3088 struct ibv_async_event event; 3089 enum ibv_qp_state state; 3090 3091 rc = ibv_get_async_event(device->context, &event); 3092 3093 if (rc) { 3094 SPDK_ERRLOG("Failed to get async_event (%d): %s\n", 3095 errno, spdk_strerror(errno)); 3096 return; 3097 } 3098 3099 switch (event.event_type) { 3100 case IBV_EVENT_QP_FATAL: 3101 rqpair = event.element.qp->qp_context; 3102 SPDK_ERRLOG("Fatal event received for rqpair %p\n", rqpair); 3103 spdk_trace_record(TRACE_RDMA_IBV_ASYNC_EVENT, 0, 0, 3104 (uintptr_t)rqpair->cm_id, event.event_type); 3105 spdk_nvmf_rdma_update_ibv_state(rqpair); 3106 spdk_nvmf_rdma_start_disconnect(rqpair); 3107 break; 3108 case IBV_EVENT_QP_LAST_WQE_REACHED: 3109 /* This event only occurs for shared receive queues. */ 3110 rqpair = event.element.qp->qp_context; 3111 SPDK_DEBUGLOG(SPDK_LOG_RDMA, "Last WQE reached event received for rqpair %p\n", rqpair); 3112 /* This must be handled on the polling thread if it exists. Otherwise the timeout will catch it. */ 3113 if (rqpair->qpair.group) { 3114 spdk_thread_send_msg(rqpair->qpair.group->thread, nvmf_rdma_handle_last_wqe_reached, rqpair); 3115 } else { 3116 SPDK_ERRLOG("Unable to destroy the qpair %p since it does not have a poll group.\n", rqpair); 3117 rqpair->last_wqe_reached = true; 3118 } 3119 3120 break; 3121 case IBV_EVENT_SQ_DRAINED: 3122 /* This event occurs frequently in both error and non-error states. 3123 * Check if the qpair is in an error state before sending a message. 3124 * Note that we're not on the correct thread to access the qpair, but 3125 * the operations that the below calls make all happen to be thread 3126 * safe. */ 3127 rqpair = event.element.qp->qp_context; 3128 SPDK_DEBUGLOG(SPDK_LOG_RDMA, "Last sq drained event received for rqpair %p\n", rqpair); 3129 spdk_trace_record(TRACE_RDMA_IBV_ASYNC_EVENT, 0, 0, 3130 (uintptr_t)rqpair->cm_id, event.event_type); 3131 state = spdk_nvmf_rdma_update_ibv_state(rqpair); 3132 if (state == IBV_QPS_ERR) { 3133 spdk_nvmf_rdma_start_disconnect(rqpair); 3134 } 3135 break; 3136 case IBV_EVENT_QP_REQ_ERR: 3137 case IBV_EVENT_QP_ACCESS_ERR: 3138 case IBV_EVENT_COMM_EST: 3139 case IBV_EVENT_PATH_MIG: 3140 case IBV_EVENT_PATH_MIG_ERR: 3141 SPDK_NOTICELOG("Async event: %s\n", 3142 ibv_event_type_str(event.event_type)); 3143 rqpair = event.element.qp->qp_context; 3144 spdk_trace_record(TRACE_RDMA_IBV_ASYNC_EVENT, 0, 0, 3145 (uintptr_t)rqpair->cm_id, event.event_type); 3146 spdk_nvmf_rdma_update_ibv_state(rqpair); 3147 break; 3148 case IBV_EVENT_CQ_ERR: 3149 case IBV_EVENT_DEVICE_FATAL: 3150 case IBV_EVENT_PORT_ACTIVE: 3151 case IBV_EVENT_PORT_ERR: 3152 case IBV_EVENT_LID_CHANGE: 3153 case IBV_EVENT_PKEY_CHANGE: 3154 case IBV_EVENT_SM_CHANGE: 3155 case IBV_EVENT_SRQ_ERR: 3156 case IBV_EVENT_SRQ_LIMIT_REACHED: 3157 case IBV_EVENT_CLIENT_REREGISTER: 3158 case IBV_EVENT_GID_CHANGE: 3159 default: 3160 SPDK_NOTICELOG("Async event: %s\n", 3161 ibv_event_type_str(event.event_type)); 3162 spdk_trace_record(TRACE_RDMA_IBV_ASYNC_EVENT, 0, 0, 0, event.event_type); 3163 break; 3164 } 3165 ibv_ack_async_event(&event); 3166 } 3167 3168 static void 3169 spdk_nvmf_rdma_accept(struct spdk_nvmf_transport *transport, new_qpair_fn cb_fn, void *cb_arg) 3170 { 3171 int nfds, i = 0; 3172 struct spdk_nvmf_rdma_transport *rtransport; 3173 struct spdk_nvmf_rdma_device *device, *tmp; 3174 3175 rtransport = SPDK_CONTAINEROF(transport, struct spdk_nvmf_rdma_transport, transport); 3176 nfds = poll(rtransport->poll_fds, rtransport->npoll_fds, 0); 3177 3178 if (nfds <= 0) { 3179 return; 3180 } 3181 3182 /* The first poll descriptor is RDMA CM event */ 3183 if (rtransport->poll_fds[i++].revents & POLLIN) { 3184 spdk_nvmf_process_cm_event(transport, cb_fn, cb_arg); 3185 nfds--; 3186 } 3187 3188 if (nfds == 0) { 3189 return; 3190 } 3191 3192 /* Second and subsequent poll descriptors are IB async events */ 3193 TAILQ_FOREACH_SAFE(device, &rtransport->devices, link, tmp) { 3194 if (rtransport->poll_fds[i++].revents & POLLIN) { 3195 spdk_nvmf_process_ib_event(device); 3196 nfds--; 3197 } 3198 } 3199 /* check all flagged fd's have been served */ 3200 assert(nfds == 0); 3201 } 3202 3203 static void 3204 spdk_nvmf_rdma_discover(struct spdk_nvmf_transport *transport, 3205 struct spdk_nvme_transport_id *trid, 3206 struct spdk_nvmf_discovery_log_page_entry *entry) 3207 { 3208 entry->trtype = SPDK_NVMF_TRTYPE_RDMA; 3209 entry->adrfam = trid->adrfam; 3210 entry->treq.secure_channel = SPDK_NVMF_TREQ_SECURE_CHANNEL_NOT_REQUIRED; 3211 3212 spdk_strcpy_pad(entry->trsvcid, trid->trsvcid, sizeof(entry->trsvcid), ' '); 3213 spdk_strcpy_pad(entry->traddr, trid->traddr, sizeof(entry->traddr), ' '); 3214 3215 entry->tsas.rdma.rdma_qptype = SPDK_NVMF_RDMA_QPTYPE_RELIABLE_CONNECTED; 3216 entry->tsas.rdma.rdma_prtype = SPDK_NVMF_RDMA_PRTYPE_NONE; 3217 entry->tsas.rdma.rdma_cms = SPDK_NVMF_RDMA_CMS_RDMA_CM; 3218 } 3219 3220 static void 3221 spdk_nvmf_rdma_poll_group_destroy(struct spdk_nvmf_transport_poll_group *group); 3222 3223 static struct spdk_nvmf_transport_poll_group * 3224 spdk_nvmf_rdma_poll_group_create(struct spdk_nvmf_transport *transport) 3225 { 3226 struct spdk_nvmf_rdma_transport *rtransport; 3227 struct spdk_nvmf_rdma_poll_group *rgroup; 3228 struct spdk_nvmf_rdma_poller *poller; 3229 struct spdk_nvmf_rdma_device *device; 3230 struct ibv_srq_init_attr srq_init_attr; 3231 struct spdk_nvmf_rdma_resource_opts opts; 3232 int num_cqe; 3233 3234 rtransport = SPDK_CONTAINEROF(transport, struct spdk_nvmf_rdma_transport, transport); 3235 3236 rgroup = calloc(1, sizeof(*rgroup)); 3237 if (!rgroup) { 3238 return NULL; 3239 } 3240 3241 TAILQ_INIT(&rgroup->pollers); 3242 STAILQ_INIT(&rgroup->retired_bufs); 3243 3244 pthread_mutex_lock(&rtransport->lock); 3245 TAILQ_FOREACH(device, &rtransport->devices, link) { 3246 poller = calloc(1, sizeof(*poller)); 3247 if (!poller) { 3248 SPDK_ERRLOG("Unable to allocate memory for new RDMA poller\n"); 3249 spdk_nvmf_rdma_poll_group_destroy(&rgroup->group); 3250 pthread_mutex_unlock(&rtransport->lock); 3251 return NULL; 3252 } 3253 3254 poller->device = device; 3255 poller->group = rgroup; 3256 3257 TAILQ_INIT(&poller->qpairs); 3258 STAILQ_INIT(&poller->qpairs_pending_send); 3259 STAILQ_INIT(&poller->qpairs_pending_recv); 3260 3261 TAILQ_INSERT_TAIL(&rgroup->pollers, poller, link); 3262 if (transport->opts.no_srq == false && device->num_srq < device->attr.max_srq) { 3263 poller->max_srq_depth = transport->opts.max_srq_depth; 3264 3265 device->num_srq++; 3266 memset(&srq_init_attr, 0, sizeof(struct ibv_srq_init_attr)); 3267 srq_init_attr.attr.max_wr = poller->max_srq_depth; 3268 srq_init_attr.attr.max_sge = spdk_min(device->attr.max_sge, NVMF_DEFAULT_RX_SGE); 3269 poller->srq = ibv_create_srq(device->pd, &srq_init_attr); 3270 if (!poller->srq) { 3271 SPDK_ERRLOG("Unable to create shared receive queue, errno %d\n", errno); 3272 spdk_nvmf_rdma_poll_group_destroy(&rgroup->group); 3273 pthread_mutex_unlock(&rtransport->lock); 3274 return NULL; 3275 } 3276 3277 opts.qp = poller->srq; 3278 opts.pd = device->pd; 3279 opts.qpair = NULL; 3280 opts.shared = true; 3281 opts.max_queue_depth = poller->max_srq_depth; 3282 opts.in_capsule_data_size = transport->opts.in_capsule_data_size; 3283 3284 poller->resources = nvmf_rdma_resources_create(&opts); 3285 if (!poller->resources) { 3286 SPDK_ERRLOG("Unable to allocate resources for shared receive queue.\n"); 3287 spdk_nvmf_rdma_poll_group_destroy(&rgroup->group); 3288 pthread_mutex_unlock(&rtransport->lock); 3289 return NULL; 3290 } 3291 } 3292 3293 /* 3294 * When using an srq, we can limit the completion queue at startup. 3295 * The following formula represents the calculation: 3296 * num_cqe = num_recv + num_data_wr + num_send_wr. 3297 * where num_recv=num_data_wr=and num_send_wr=poller->max_srq_depth 3298 */ 3299 if (poller->srq) { 3300 num_cqe = poller->max_srq_depth * 3; 3301 } else { 3302 num_cqe = DEFAULT_NVMF_RDMA_CQ_SIZE; 3303 } 3304 3305 poller->cq = ibv_create_cq(device->context, num_cqe, poller, NULL, 0); 3306 if (!poller->cq) { 3307 SPDK_ERRLOG("Unable to create completion queue\n"); 3308 spdk_nvmf_rdma_poll_group_destroy(&rgroup->group); 3309 pthread_mutex_unlock(&rtransport->lock); 3310 return NULL; 3311 } 3312 poller->num_cqe = num_cqe; 3313 } 3314 3315 TAILQ_INSERT_TAIL(&rtransport->poll_groups, rgroup, link); 3316 if (rtransport->conn_sched.next_admin_pg == NULL) { 3317 rtransport->conn_sched.next_admin_pg = rgroup; 3318 rtransport->conn_sched.next_io_pg = rgroup; 3319 } 3320 3321 pthread_mutex_unlock(&rtransport->lock); 3322 return &rgroup->group; 3323 } 3324 3325 static struct spdk_nvmf_transport_poll_group * 3326 spdk_nvmf_rdma_get_optimal_poll_group(struct spdk_nvmf_qpair *qpair) 3327 { 3328 struct spdk_nvmf_rdma_transport *rtransport; 3329 struct spdk_nvmf_rdma_poll_group **pg; 3330 struct spdk_nvmf_transport_poll_group *result; 3331 3332 rtransport = SPDK_CONTAINEROF(qpair->transport, struct spdk_nvmf_rdma_transport, transport); 3333 3334 pthread_mutex_lock(&rtransport->lock); 3335 3336 if (TAILQ_EMPTY(&rtransport->poll_groups)) { 3337 pthread_mutex_unlock(&rtransport->lock); 3338 return NULL; 3339 } 3340 3341 if (qpair->qid == 0) { 3342 pg = &rtransport->conn_sched.next_admin_pg; 3343 } else { 3344 pg = &rtransport->conn_sched.next_io_pg; 3345 } 3346 3347 assert(*pg != NULL); 3348 3349 result = &(*pg)->group; 3350 3351 *pg = TAILQ_NEXT(*pg, link); 3352 if (*pg == NULL) { 3353 *pg = TAILQ_FIRST(&rtransport->poll_groups); 3354 } 3355 3356 pthread_mutex_unlock(&rtransport->lock); 3357 3358 return result; 3359 } 3360 3361 static void 3362 spdk_nvmf_rdma_poll_group_destroy(struct spdk_nvmf_transport_poll_group *group) 3363 { 3364 struct spdk_nvmf_rdma_poll_group *rgroup, *next_rgroup; 3365 struct spdk_nvmf_rdma_poller *poller, *tmp; 3366 struct spdk_nvmf_rdma_qpair *qpair, *tmp_qpair; 3367 struct spdk_nvmf_transport_pg_cache_buf *buf, *tmp_buf; 3368 struct spdk_nvmf_rdma_transport *rtransport; 3369 3370 rgroup = SPDK_CONTAINEROF(group, struct spdk_nvmf_rdma_poll_group, group); 3371 rtransport = SPDK_CONTAINEROF(rgroup->group.transport, struct spdk_nvmf_rdma_transport, transport); 3372 3373 if (!rgroup) { 3374 return; 3375 } 3376 3377 /* free all retired buffers back to the transport so we don't short the mempool. */ 3378 STAILQ_FOREACH_SAFE(buf, &rgroup->retired_bufs, link, tmp_buf) { 3379 STAILQ_REMOVE(&rgroup->retired_bufs, buf, spdk_nvmf_transport_pg_cache_buf, link); 3380 assert(group->transport != NULL); 3381 spdk_mempool_put(group->transport->data_buf_pool, buf); 3382 } 3383 3384 TAILQ_FOREACH_SAFE(poller, &rgroup->pollers, link, tmp) { 3385 TAILQ_REMOVE(&rgroup->pollers, poller, link); 3386 3387 TAILQ_FOREACH_SAFE(qpair, &poller->qpairs, link, tmp_qpair) { 3388 spdk_nvmf_rdma_qpair_destroy(qpair); 3389 } 3390 3391 if (poller->srq) { 3392 nvmf_rdma_resources_destroy(poller->resources); 3393 ibv_destroy_srq(poller->srq); 3394 SPDK_DEBUGLOG(SPDK_LOG_RDMA, "Destroyed RDMA shared queue %p\n", poller->srq); 3395 } 3396 3397 if (poller->cq) { 3398 ibv_destroy_cq(poller->cq); 3399 } 3400 3401 free(poller); 3402 } 3403 3404 pthread_mutex_lock(&rtransport->lock); 3405 next_rgroup = TAILQ_NEXT(rgroup, link); 3406 TAILQ_REMOVE(&rtransport->poll_groups, rgroup, link); 3407 if (next_rgroup == NULL) { 3408 next_rgroup = TAILQ_FIRST(&rtransport->poll_groups); 3409 } 3410 if (rtransport->conn_sched.next_admin_pg == rgroup) { 3411 rtransport->conn_sched.next_admin_pg = next_rgroup; 3412 } 3413 if (rtransport->conn_sched.next_io_pg == rgroup) { 3414 rtransport->conn_sched.next_io_pg = next_rgroup; 3415 } 3416 pthread_mutex_unlock(&rtransport->lock); 3417 3418 free(rgroup); 3419 } 3420 3421 static void 3422 spdk_nvmf_rdma_qpair_reject_connection(struct spdk_nvmf_rdma_qpair *rqpair) 3423 { 3424 if (rqpair->cm_id != NULL) { 3425 spdk_nvmf_rdma_event_reject(rqpair->cm_id, SPDK_NVMF_RDMA_ERROR_NO_RESOURCES); 3426 } 3427 spdk_nvmf_rdma_qpair_destroy(rqpair); 3428 } 3429 3430 static int 3431 spdk_nvmf_rdma_poll_group_add(struct spdk_nvmf_transport_poll_group *group, 3432 struct spdk_nvmf_qpair *qpair) 3433 { 3434 struct spdk_nvmf_rdma_poll_group *rgroup; 3435 struct spdk_nvmf_rdma_qpair *rqpair; 3436 struct spdk_nvmf_rdma_device *device; 3437 struct spdk_nvmf_rdma_poller *poller; 3438 int rc; 3439 3440 rgroup = SPDK_CONTAINEROF(group, struct spdk_nvmf_rdma_poll_group, group); 3441 rqpair = SPDK_CONTAINEROF(qpair, struct spdk_nvmf_rdma_qpair, qpair); 3442 3443 device = rqpair->port->device; 3444 3445 TAILQ_FOREACH(poller, &rgroup->pollers, link) { 3446 if (poller->device == device) { 3447 break; 3448 } 3449 } 3450 3451 if (!poller) { 3452 SPDK_ERRLOG("No poller found for device.\n"); 3453 return -1; 3454 } 3455 3456 TAILQ_INSERT_TAIL(&poller->qpairs, rqpair, link); 3457 rqpair->poller = poller; 3458 rqpair->srq = rqpair->poller->srq; 3459 3460 rc = spdk_nvmf_rdma_qpair_initialize(qpair); 3461 if (rc < 0) { 3462 SPDK_ERRLOG("Failed to initialize nvmf_rdma_qpair with qpair=%p\n", qpair); 3463 return -1; 3464 } 3465 3466 rc = spdk_nvmf_rdma_event_accept(rqpair->cm_id, rqpair); 3467 if (rc) { 3468 /* Try to reject, but we probably can't */ 3469 spdk_nvmf_rdma_qpair_reject_connection(rqpair); 3470 return -1; 3471 } 3472 3473 spdk_nvmf_rdma_update_ibv_state(rqpair); 3474 3475 return 0; 3476 } 3477 3478 static int 3479 spdk_nvmf_rdma_request_free(struct spdk_nvmf_request *req) 3480 { 3481 struct spdk_nvmf_rdma_request *rdma_req = SPDK_CONTAINEROF(req, struct spdk_nvmf_rdma_request, req); 3482 struct spdk_nvmf_rdma_transport *rtransport = SPDK_CONTAINEROF(req->qpair->transport, 3483 struct spdk_nvmf_rdma_transport, transport); 3484 3485 nvmf_rdma_request_free(rdma_req, rtransport); 3486 return 0; 3487 } 3488 3489 static int 3490 spdk_nvmf_rdma_request_complete(struct spdk_nvmf_request *req) 3491 { 3492 struct spdk_nvmf_rdma_transport *rtransport = SPDK_CONTAINEROF(req->qpair->transport, 3493 struct spdk_nvmf_rdma_transport, transport); 3494 struct spdk_nvmf_rdma_request *rdma_req = SPDK_CONTAINEROF(req, 3495 struct spdk_nvmf_rdma_request, req); 3496 struct spdk_nvmf_rdma_qpair *rqpair = SPDK_CONTAINEROF(rdma_req->req.qpair, 3497 struct spdk_nvmf_rdma_qpair, qpair); 3498 3499 if (rqpair->ibv_state != IBV_QPS_ERR) { 3500 /* The connection is alive, so process the request as normal */ 3501 rdma_req->state = RDMA_REQUEST_STATE_EXECUTED; 3502 } else { 3503 /* The connection is dead. Move the request directly to the completed state. */ 3504 rdma_req->state = RDMA_REQUEST_STATE_COMPLETED; 3505 } 3506 3507 spdk_nvmf_rdma_request_process(rtransport, rdma_req); 3508 3509 return 0; 3510 } 3511 3512 static int 3513 spdk_nvmf_rdma_destroy_defunct_qpair(void *ctx) 3514 { 3515 struct spdk_nvmf_rdma_qpair *rqpair = ctx; 3516 struct spdk_nvmf_rdma_transport *rtransport = SPDK_CONTAINEROF(rqpair->qpair.transport, 3517 struct spdk_nvmf_rdma_transport, transport); 3518 3519 SPDK_INFOLOG(SPDK_LOG_RDMA, "QP#%d hasn't been drained as expected, manually destroy it\n", 3520 rqpair->qpair.qid); 3521 3522 spdk_nvmf_rdma_qpair_process_pending(rtransport, rqpair, true); 3523 spdk_nvmf_rdma_qpair_destroy(rqpair); 3524 3525 return 0; 3526 } 3527 3528 static void 3529 spdk_nvmf_rdma_close_qpair(struct spdk_nvmf_qpair *qpair) 3530 { 3531 struct spdk_nvmf_rdma_qpair *rqpair = SPDK_CONTAINEROF(qpair, struct spdk_nvmf_rdma_qpair, qpair); 3532 3533 if (rqpair->disconnect_flags & RDMA_QP_DISCONNECTING) { 3534 return; 3535 } 3536 3537 rqpair->disconnect_flags |= RDMA_QP_DISCONNECTING; 3538 3539 /* This happens only when the qpair is disconnected before 3540 * it is added to the poll group. Since there is no poll group, 3541 * the RDMA qp has not been initialized yet and the RDMA CM 3542 * event has not yet been acknowledged, so we need to reject it. 3543 */ 3544 if (rqpair->qpair.state == SPDK_NVMF_QPAIR_UNINITIALIZED) { 3545 spdk_nvmf_rdma_qpair_reject_connection(rqpair); 3546 return; 3547 } 3548 3549 if (rqpair->ibv_state != IBV_QPS_ERR) { 3550 spdk_nvmf_rdma_set_ibv_state(rqpair, IBV_QPS_ERR); 3551 } 3552 3553 rqpair->destruct_poller = spdk_poller_register(spdk_nvmf_rdma_destroy_defunct_qpair, (void *)rqpair, 3554 NVMF_RDMA_QPAIR_DESTROY_TIMEOUT_US); 3555 } 3556 3557 static struct spdk_nvmf_rdma_qpair * 3558 get_rdma_qpair_from_wc(struct spdk_nvmf_rdma_poller *rpoller, struct ibv_wc *wc) 3559 { 3560 struct spdk_nvmf_rdma_qpair *rqpair; 3561 /* @todo: improve QP search */ 3562 TAILQ_FOREACH(rqpair, &rpoller->qpairs, link) { 3563 if (wc->qp_num == rqpair->cm_id->qp->qp_num) { 3564 return rqpair; 3565 } 3566 } 3567 SPDK_ERRLOG("Didn't find QP with qp_num %u\n", wc->qp_num); 3568 return NULL; 3569 } 3570 3571 #ifdef DEBUG 3572 static int 3573 spdk_nvmf_rdma_req_is_completing(struct spdk_nvmf_rdma_request *rdma_req) 3574 { 3575 return rdma_req->state == RDMA_REQUEST_STATE_TRANSFERRING_CONTROLLER_TO_HOST || 3576 rdma_req->state == RDMA_REQUEST_STATE_COMPLETING; 3577 } 3578 #endif 3579 3580 static void 3581 _poller_reset_failed_recvs(struct spdk_nvmf_rdma_poller *rpoller, struct ibv_recv_wr *bad_recv_wr, 3582 int rc) 3583 { 3584 struct spdk_nvmf_rdma_recv *rdma_recv; 3585 struct spdk_nvmf_rdma_wr *bad_rdma_wr; 3586 3587 SPDK_ERRLOG("Failed to post a recv for the poller %p with errno %d\n", rpoller, -rc); 3588 while (bad_recv_wr != NULL) { 3589 bad_rdma_wr = (struct spdk_nvmf_rdma_wr *)bad_recv_wr->wr_id; 3590 rdma_recv = SPDK_CONTAINEROF(bad_rdma_wr, struct spdk_nvmf_rdma_recv, rdma_wr); 3591 3592 rdma_recv->qpair->current_recv_depth++; 3593 bad_recv_wr = bad_recv_wr->next; 3594 SPDK_ERRLOG("Failed to post a recv for the qpair %p with errno %d\n", rdma_recv->qpair, -rc); 3595 spdk_nvmf_rdma_start_disconnect(rdma_recv->qpair); 3596 } 3597 } 3598 3599 static void 3600 _qp_reset_failed_recvs(struct spdk_nvmf_rdma_qpair *rqpair, struct ibv_recv_wr *bad_recv_wr, int rc) 3601 { 3602 SPDK_ERRLOG("Failed to post a recv for the qpair %p with errno %d\n", rqpair, -rc); 3603 while (bad_recv_wr != NULL) { 3604 bad_recv_wr = bad_recv_wr->next; 3605 rqpair->current_recv_depth++; 3606 } 3607 spdk_nvmf_rdma_start_disconnect(rqpair); 3608 } 3609 3610 static void 3611 _poller_submit_recvs(struct spdk_nvmf_rdma_transport *rtransport, 3612 struct spdk_nvmf_rdma_poller *rpoller) 3613 { 3614 struct spdk_nvmf_rdma_qpair *rqpair; 3615 struct ibv_recv_wr *bad_recv_wr; 3616 int rc; 3617 3618 if (rpoller->srq) { 3619 if (rpoller->resources->recvs_to_post.first != NULL) { 3620 rc = ibv_post_srq_recv(rpoller->srq, rpoller->resources->recvs_to_post.first, &bad_recv_wr); 3621 if (rc) { 3622 _poller_reset_failed_recvs(rpoller, bad_recv_wr, rc); 3623 } 3624 rpoller->resources->recvs_to_post.first = NULL; 3625 rpoller->resources->recvs_to_post.last = NULL; 3626 } 3627 } else { 3628 while (!STAILQ_EMPTY(&rpoller->qpairs_pending_recv)) { 3629 rqpair = STAILQ_FIRST(&rpoller->qpairs_pending_recv); 3630 assert(rqpair->resources->recvs_to_post.first != NULL); 3631 rc = ibv_post_recv(rqpair->cm_id->qp, rqpair->resources->recvs_to_post.first, &bad_recv_wr); 3632 if (rc) { 3633 _qp_reset_failed_recvs(rqpair, bad_recv_wr, rc); 3634 } 3635 rqpair->resources->recvs_to_post.first = NULL; 3636 rqpair->resources->recvs_to_post.last = NULL; 3637 STAILQ_REMOVE_HEAD(&rpoller->qpairs_pending_recv, recv_link); 3638 } 3639 } 3640 } 3641 3642 static void 3643 _qp_reset_failed_sends(struct spdk_nvmf_rdma_transport *rtransport, 3644 struct spdk_nvmf_rdma_qpair *rqpair, struct ibv_send_wr *bad_wr, int rc) 3645 { 3646 struct spdk_nvmf_rdma_wr *bad_rdma_wr; 3647 struct spdk_nvmf_rdma_request *prev_rdma_req = NULL, *cur_rdma_req = NULL; 3648 3649 SPDK_ERRLOG("Failed to post a send for the qpair %p with errno %d\n", rqpair, -rc); 3650 for (; bad_wr != NULL; bad_wr = bad_wr->next) { 3651 bad_rdma_wr = (struct spdk_nvmf_rdma_wr *)bad_wr->wr_id; 3652 assert(rqpair->current_send_depth > 0); 3653 rqpair->current_send_depth--; 3654 switch (bad_rdma_wr->type) { 3655 case RDMA_WR_TYPE_DATA: 3656 cur_rdma_req = SPDK_CONTAINEROF(bad_rdma_wr, struct spdk_nvmf_rdma_request, data.rdma_wr); 3657 if (bad_wr->opcode == IBV_WR_RDMA_READ) { 3658 assert(rqpair->current_read_depth > 0); 3659 rqpair->current_read_depth--; 3660 } 3661 break; 3662 case RDMA_WR_TYPE_SEND: 3663 cur_rdma_req = SPDK_CONTAINEROF(bad_rdma_wr, struct spdk_nvmf_rdma_request, rsp.rdma_wr); 3664 break; 3665 default: 3666 SPDK_ERRLOG("Found a RECV in the list of pending SEND requests for qpair %p\n", rqpair); 3667 prev_rdma_req = cur_rdma_req; 3668 continue; 3669 } 3670 3671 if (prev_rdma_req == cur_rdma_req) { 3672 /* this request was handled by an earlier wr. i.e. we were performing an nvme read. */ 3673 /* We only have to check against prev_wr since each requests wrs are contiguous in this list. */ 3674 continue; 3675 } 3676 3677 switch (cur_rdma_req->state) { 3678 case RDMA_REQUEST_STATE_TRANSFERRING_HOST_TO_CONTROLLER: 3679 cur_rdma_req->req.rsp->nvme_cpl.status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; 3680 cur_rdma_req->state = RDMA_REQUEST_STATE_READY_TO_COMPLETE; 3681 break; 3682 case RDMA_REQUEST_STATE_TRANSFERRING_CONTROLLER_TO_HOST: 3683 case RDMA_REQUEST_STATE_COMPLETING: 3684 cur_rdma_req->state = RDMA_REQUEST_STATE_COMPLETED; 3685 break; 3686 default: 3687 SPDK_ERRLOG("Found a request in a bad state %d when draining pending SEND requests for qpair %p\n", 3688 cur_rdma_req->state, rqpair); 3689 continue; 3690 } 3691 3692 spdk_nvmf_rdma_request_process(rtransport, cur_rdma_req); 3693 prev_rdma_req = cur_rdma_req; 3694 } 3695 3696 if (rqpair->qpair.state == SPDK_NVMF_QPAIR_ACTIVE) { 3697 /* Disconnect the connection. */ 3698 spdk_nvmf_rdma_start_disconnect(rqpair); 3699 } 3700 3701 } 3702 3703 static void 3704 _poller_submit_sends(struct spdk_nvmf_rdma_transport *rtransport, 3705 struct spdk_nvmf_rdma_poller *rpoller) 3706 { 3707 struct spdk_nvmf_rdma_qpair *rqpair; 3708 struct ibv_send_wr *bad_wr = NULL; 3709 int rc; 3710 3711 while (!STAILQ_EMPTY(&rpoller->qpairs_pending_send)) { 3712 rqpair = STAILQ_FIRST(&rpoller->qpairs_pending_send); 3713 assert(rqpair->sends_to_post.first != NULL); 3714 rc = ibv_post_send(rqpair->cm_id->qp, rqpair->sends_to_post.first, &bad_wr); 3715 3716 /* bad wr always points to the first wr that failed. */ 3717 if (rc) { 3718 _qp_reset_failed_sends(rtransport, rqpair, bad_wr, rc); 3719 } 3720 rqpair->sends_to_post.first = NULL; 3721 rqpair->sends_to_post.last = NULL; 3722 STAILQ_REMOVE_HEAD(&rpoller->qpairs_pending_send, send_link); 3723 } 3724 } 3725 3726 static int 3727 spdk_nvmf_rdma_poller_poll(struct spdk_nvmf_rdma_transport *rtransport, 3728 struct spdk_nvmf_rdma_poller *rpoller) 3729 { 3730 struct ibv_wc wc[32]; 3731 struct spdk_nvmf_rdma_wr *rdma_wr; 3732 struct spdk_nvmf_rdma_request *rdma_req; 3733 struct spdk_nvmf_rdma_recv *rdma_recv; 3734 struct spdk_nvmf_rdma_qpair *rqpair; 3735 int reaped, i; 3736 int count = 0; 3737 bool error = false; 3738 uint64_t poll_tsc = spdk_get_ticks(); 3739 3740 /* Poll for completing operations. */ 3741 reaped = ibv_poll_cq(rpoller->cq, 32, wc); 3742 if (reaped < 0) { 3743 SPDK_ERRLOG("Error polling CQ! (%d): %s\n", 3744 errno, spdk_strerror(errno)); 3745 return -1; 3746 } 3747 3748 rpoller->stat.polls++; 3749 rpoller->stat.completions += reaped; 3750 3751 for (i = 0; i < reaped; i++) { 3752 3753 rdma_wr = (struct spdk_nvmf_rdma_wr *)wc[i].wr_id; 3754 3755 switch (rdma_wr->type) { 3756 case RDMA_WR_TYPE_SEND: 3757 rdma_req = SPDK_CONTAINEROF(rdma_wr, struct spdk_nvmf_rdma_request, rsp.rdma_wr); 3758 rqpair = SPDK_CONTAINEROF(rdma_req->req.qpair, struct spdk_nvmf_rdma_qpair, qpair); 3759 3760 if (!wc[i].status) { 3761 count++; 3762 assert(wc[i].opcode == IBV_WC_SEND); 3763 assert(spdk_nvmf_rdma_req_is_completing(rdma_req)); 3764 } else { 3765 SPDK_ERRLOG("data=%p length=%u\n", rdma_req->req.data, rdma_req->req.length); 3766 } 3767 3768 rdma_req->state = RDMA_REQUEST_STATE_COMPLETED; 3769 /* +1 for the response wr */ 3770 rqpair->current_send_depth -= rdma_req->num_outstanding_data_wr + 1; 3771 rdma_req->num_outstanding_data_wr = 0; 3772 3773 spdk_nvmf_rdma_request_process(rtransport, rdma_req); 3774 break; 3775 case RDMA_WR_TYPE_RECV: 3776 /* rdma_recv->qpair will be invalid if using an SRQ. In that case we have to get the qpair from the wc. */ 3777 rdma_recv = SPDK_CONTAINEROF(rdma_wr, struct spdk_nvmf_rdma_recv, rdma_wr); 3778 if (rpoller->srq != NULL) { 3779 rdma_recv->qpair = get_rdma_qpair_from_wc(rpoller, &wc[i]); 3780 /* It is possible that there are still some completions for destroyed QP 3781 * associated with SRQ. We just ignore these late completions and re-post 3782 * receive WRs back to SRQ. 3783 */ 3784 if (spdk_unlikely(NULL == rdma_recv->qpair)) { 3785 struct ibv_recv_wr *bad_wr; 3786 int rc; 3787 3788 rdma_recv->wr.next = NULL; 3789 rc = ibv_post_srq_recv(rpoller->srq, 3790 &rdma_recv->wr, 3791 &bad_wr); 3792 if (rc) { 3793 SPDK_ERRLOG("Failed to re-post recv WR to SRQ, err %d\n", rc); 3794 } 3795 continue; 3796 } 3797 } 3798 rqpair = rdma_recv->qpair; 3799 3800 assert(rqpair != NULL); 3801 if (!wc[i].status) { 3802 assert(wc[i].opcode == IBV_WC_RECV); 3803 if (rqpair->current_recv_depth >= rqpair->max_queue_depth) { 3804 spdk_nvmf_rdma_start_disconnect(rqpair); 3805 break; 3806 } 3807 } 3808 3809 rdma_recv->wr.next = NULL; 3810 rqpair->current_recv_depth++; 3811 rdma_recv->receive_tsc = poll_tsc; 3812 rpoller->stat.requests++; 3813 STAILQ_INSERT_TAIL(&rqpair->resources->incoming_queue, rdma_recv, link); 3814 break; 3815 case RDMA_WR_TYPE_DATA: 3816 rdma_req = SPDK_CONTAINEROF(rdma_wr, struct spdk_nvmf_rdma_request, data.rdma_wr); 3817 rqpair = SPDK_CONTAINEROF(rdma_req->req.qpair, struct spdk_nvmf_rdma_qpair, qpair); 3818 3819 assert(rdma_req->num_outstanding_data_wr > 0); 3820 3821 rqpair->current_send_depth--; 3822 rdma_req->num_outstanding_data_wr--; 3823 if (!wc[i].status) { 3824 assert(wc[i].opcode == IBV_WC_RDMA_READ); 3825 rqpair->current_read_depth--; 3826 /* wait for all outstanding reads associated with the same rdma_req to complete before proceeding. */ 3827 if (rdma_req->num_outstanding_data_wr == 0) { 3828 rdma_req->state = RDMA_REQUEST_STATE_READY_TO_EXECUTE; 3829 spdk_nvmf_rdma_request_process(rtransport, rdma_req); 3830 } 3831 } else { 3832 /* If the data transfer fails still force the queue into the error state, 3833 * if we were performing an RDMA_READ, we need to force the request into a 3834 * completed state since it wasn't linked to a send. However, in the RDMA_WRITE 3835 * case, we should wait for the SEND to complete. */ 3836 SPDK_ERRLOG("data=%p length=%u\n", rdma_req->req.data, rdma_req->req.length); 3837 if (rdma_req->data.wr.opcode == IBV_WR_RDMA_READ) { 3838 rqpair->current_read_depth--; 3839 if (rdma_req->num_outstanding_data_wr == 0) { 3840 rdma_req->state = RDMA_REQUEST_STATE_COMPLETED; 3841 } 3842 } 3843 } 3844 break; 3845 default: 3846 SPDK_ERRLOG("Received an unknown opcode on the CQ: %d\n", wc[i].opcode); 3847 continue; 3848 } 3849 3850 /* Handle error conditions */ 3851 if (wc[i].status) { 3852 SPDK_DEBUGLOG(SPDK_LOG_RDMA, "CQ error on CQ %p, Request 0x%lu (%d): %s\n", 3853 rpoller->cq, wc[i].wr_id, wc[i].status, ibv_wc_status_str(wc[i].status)); 3854 3855 error = true; 3856 3857 if (rqpair->qpair.state == SPDK_NVMF_QPAIR_ACTIVE) { 3858 /* Disconnect the connection. */ 3859 spdk_nvmf_rdma_start_disconnect(rqpair); 3860 } else { 3861 nvmf_rdma_destroy_drained_qpair(rqpair); 3862 } 3863 continue; 3864 } 3865 3866 spdk_nvmf_rdma_qpair_process_pending(rtransport, rqpair, false); 3867 3868 if (rqpair->qpair.state != SPDK_NVMF_QPAIR_ACTIVE) { 3869 nvmf_rdma_destroy_drained_qpair(rqpair); 3870 } 3871 } 3872 3873 if (error == true) { 3874 return -1; 3875 } 3876 3877 /* submit outstanding work requests. */ 3878 _poller_submit_recvs(rtransport, rpoller); 3879 _poller_submit_sends(rtransport, rpoller); 3880 3881 return count; 3882 } 3883 3884 static int 3885 spdk_nvmf_rdma_poll_group_poll(struct spdk_nvmf_transport_poll_group *group) 3886 { 3887 struct spdk_nvmf_rdma_transport *rtransport; 3888 struct spdk_nvmf_rdma_poll_group *rgroup; 3889 struct spdk_nvmf_rdma_poller *rpoller; 3890 int count, rc; 3891 3892 rtransport = SPDK_CONTAINEROF(group->transport, struct spdk_nvmf_rdma_transport, transport); 3893 rgroup = SPDK_CONTAINEROF(group, struct spdk_nvmf_rdma_poll_group, group); 3894 3895 count = 0; 3896 TAILQ_FOREACH(rpoller, &rgroup->pollers, link) { 3897 rc = spdk_nvmf_rdma_poller_poll(rtransport, rpoller); 3898 if (rc < 0) { 3899 return rc; 3900 } 3901 count += rc; 3902 } 3903 3904 return count; 3905 } 3906 3907 static int 3908 spdk_nvmf_rdma_trid_from_cm_id(struct rdma_cm_id *id, 3909 struct spdk_nvme_transport_id *trid, 3910 bool peer) 3911 { 3912 struct sockaddr *saddr; 3913 uint16_t port; 3914 3915 trid->trtype = SPDK_NVME_TRANSPORT_RDMA; 3916 3917 if (peer) { 3918 saddr = rdma_get_peer_addr(id); 3919 } else { 3920 saddr = rdma_get_local_addr(id); 3921 } 3922 switch (saddr->sa_family) { 3923 case AF_INET: { 3924 struct sockaddr_in *saddr_in = (struct sockaddr_in *)saddr; 3925 3926 trid->adrfam = SPDK_NVMF_ADRFAM_IPV4; 3927 inet_ntop(AF_INET, &saddr_in->sin_addr, 3928 trid->traddr, sizeof(trid->traddr)); 3929 if (peer) { 3930 port = ntohs(rdma_get_dst_port(id)); 3931 } else { 3932 port = ntohs(rdma_get_src_port(id)); 3933 } 3934 snprintf(trid->trsvcid, sizeof(trid->trsvcid), "%u", port); 3935 break; 3936 } 3937 case AF_INET6: { 3938 struct sockaddr_in6 *saddr_in = (struct sockaddr_in6 *)saddr; 3939 trid->adrfam = SPDK_NVMF_ADRFAM_IPV6; 3940 inet_ntop(AF_INET6, &saddr_in->sin6_addr, 3941 trid->traddr, sizeof(trid->traddr)); 3942 if (peer) { 3943 port = ntohs(rdma_get_dst_port(id)); 3944 } else { 3945 port = ntohs(rdma_get_src_port(id)); 3946 } 3947 snprintf(trid->trsvcid, sizeof(trid->trsvcid), "%u", port); 3948 break; 3949 } 3950 default: 3951 return -1; 3952 3953 } 3954 3955 return 0; 3956 } 3957 3958 static int 3959 spdk_nvmf_rdma_qpair_get_peer_trid(struct spdk_nvmf_qpair *qpair, 3960 struct spdk_nvme_transport_id *trid) 3961 { 3962 struct spdk_nvmf_rdma_qpair *rqpair; 3963 3964 rqpair = SPDK_CONTAINEROF(qpair, struct spdk_nvmf_rdma_qpair, qpair); 3965 3966 return spdk_nvmf_rdma_trid_from_cm_id(rqpair->cm_id, trid, true); 3967 } 3968 3969 static int 3970 spdk_nvmf_rdma_qpair_get_local_trid(struct spdk_nvmf_qpair *qpair, 3971 struct spdk_nvme_transport_id *trid) 3972 { 3973 struct spdk_nvmf_rdma_qpair *rqpair; 3974 3975 rqpair = SPDK_CONTAINEROF(qpair, struct spdk_nvmf_rdma_qpair, qpair); 3976 3977 return spdk_nvmf_rdma_trid_from_cm_id(rqpair->cm_id, trid, false); 3978 } 3979 3980 static int 3981 spdk_nvmf_rdma_qpair_get_listen_trid(struct spdk_nvmf_qpair *qpair, 3982 struct spdk_nvme_transport_id *trid) 3983 { 3984 struct spdk_nvmf_rdma_qpair *rqpair; 3985 3986 rqpair = SPDK_CONTAINEROF(qpair, struct spdk_nvmf_rdma_qpair, qpair); 3987 3988 return spdk_nvmf_rdma_trid_from_cm_id(rqpair->listen_id, trid, false); 3989 } 3990 3991 void 3992 spdk_nvmf_rdma_init_hooks(struct spdk_nvme_rdma_hooks *hooks) 3993 { 3994 g_nvmf_hooks = *hooks; 3995 } 3996 3997 static int 3998 spdk_nvmf_rdma_poll_group_get_stat(struct spdk_nvmf_tgt *tgt, 3999 struct spdk_nvmf_transport_poll_group_stat **stat) 4000 { 4001 struct spdk_io_channel *ch; 4002 struct spdk_nvmf_poll_group *group; 4003 struct spdk_nvmf_transport_poll_group *tgroup; 4004 struct spdk_nvmf_rdma_poll_group *rgroup; 4005 struct spdk_nvmf_rdma_poller *rpoller; 4006 struct spdk_nvmf_rdma_device_stat *device_stat; 4007 uint64_t num_devices = 0; 4008 4009 if (tgt == NULL || stat == NULL) { 4010 return -EINVAL; 4011 } 4012 4013 ch = spdk_get_io_channel(tgt); 4014 group = spdk_io_channel_get_ctx(ch);; 4015 spdk_put_io_channel(ch); 4016 TAILQ_FOREACH(tgroup, &group->tgroups, link) { 4017 if (SPDK_NVME_TRANSPORT_RDMA == tgroup->transport->ops->type) { 4018 *stat = calloc(1, sizeof(struct spdk_nvmf_transport_poll_group_stat)); 4019 if (!*stat) { 4020 SPDK_ERRLOG("Failed to allocate memory for NVMf RDMA statistics\n"); 4021 return -ENOMEM; 4022 } 4023 (*stat)->trtype = SPDK_NVME_TRANSPORT_RDMA; 4024 4025 rgroup = SPDK_CONTAINEROF(tgroup, struct spdk_nvmf_rdma_poll_group, group); 4026 /* Count devices to allocate enough memory */ 4027 TAILQ_FOREACH(rpoller, &rgroup->pollers, link) { 4028 ++num_devices; 4029 } 4030 (*stat)->rdma.devices = calloc(num_devices, sizeof(struct spdk_nvmf_rdma_device_stat)); 4031 if (!(*stat)->rdma.devices) { 4032 SPDK_ERRLOG("Failed to allocate NVMf RDMA devices statistics\n"); 4033 free(*stat); 4034 return -ENOMEM; 4035 } 4036 4037 (*stat)->rdma.pending_data_buffer = rgroup->stat.pending_data_buffer; 4038 (*stat)->rdma.num_devices = num_devices; 4039 num_devices = 0; 4040 TAILQ_FOREACH(rpoller, &rgroup->pollers, link) { 4041 device_stat = &(*stat)->rdma.devices[num_devices++]; 4042 device_stat->name = ibv_get_device_name(rpoller->device->context->device); 4043 device_stat->polls = rpoller->stat.polls; 4044 device_stat->completions = rpoller->stat.completions; 4045 device_stat->requests = rpoller->stat.requests; 4046 device_stat->request_latency = rpoller->stat.request_latency; 4047 device_stat->pending_free_request = rpoller->stat.pending_free_request; 4048 device_stat->pending_rdma_read = rpoller->stat.pending_rdma_read; 4049 device_stat->pending_rdma_write = rpoller->stat.pending_rdma_write; 4050 } 4051 return 0; 4052 } 4053 } 4054 return -ENOENT; 4055 } 4056 4057 static void 4058 spdk_nvmf_rdma_poll_group_free_stat(struct spdk_nvmf_transport_poll_group_stat *stat) 4059 { 4060 if (stat) { 4061 free(stat->rdma.devices); 4062 } 4063 free(stat); 4064 } 4065 4066 const struct spdk_nvmf_transport_ops spdk_nvmf_transport_rdma = { 4067 .type = SPDK_NVME_TRANSPORT_RDMA, 4068 .opts_init = spdk_nvmf_rdma_opts_init, 4069 .create = spdk_nvmf_rdma_create, 4070 .destroy = spdk_nvmf_rdma_destroy, 4071 4072 .listen = spdk_nvmf_rdma_listen, 4073 .stop_listen = spdk_nvmf_rdma_stop_listen, 4074 .accept = spdk_nvmf_rdma_accept, 4075 4076 .listener_discover = spdk_nvmf_rdma_discover, 4077 4078 .poll_group_create = spdk_nvmf_rdma_poll_group_create, 4079 .get_optimal_poll_group = spdk_nvmf_rdma_get_optimal_poll_group, 4080 .poll_group_destroy = spdk_nvmf_rdma_poll_group_destroy, 4081 .poll_group_add = spdk_nvmf_rdma_poll_group_add, 4082 .poll_group_poll = spdk_nvmf_rdma_poll_group_poll, 4083 4084 .req_free = spdk_nvmf_rdma_request_free, 4085 .req_complete = spdk_nvmf_rdma_request_complete, 4086 4087 .qpair_fini = spdk_nvmf_rdma_close_qpair, 4088 .qpair_get_peer_trid = spdk_nvmf_rdma_qpair_get_peer_trid, 4089 .qpair_get_local_trid = spdk_nvmf_rdma_qpair_get_local_trid, 4090 .qpair_get_listen_trid = spdk_nvmf_rdma_qpair_get_listen_trid, 4091 4092 .poll_group_get_stat = spdk_nvmf_rdma_poll_group_get_stat, 4093 .poll_group_free_stat = spdk_nvmf_rdma_poll_group_free_stat, 4094 }; 4095 4096 SPDK_LOG_REGISTER_COMPONENT("rdma", SPDK_LOG_RDMA) 4097