1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. All rights reserved. 5 * Copyright (c) 2019, 2021 Mellanox Technologies LTD. All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "spdk/stdinc.h" 35 #include "spdk_cunit.h" 36 #include "common/lib/test_env.c" 37 #include "common/lib/test_rdma.c" 38 #include "nvmf/rdma.c" 39 #include "nvmf/transport.c" 40 41 #define RDMA_UT_UNITS_IN_MAX_IO 16 42 43 struct spdk_nvmf_transport_opts g_rdma_ut_transport_opts = { 44 .max_queue_depth = SPDK_NVMF_RDMA_DEFAULT_MAX_QUEUE_DEPTH, 45 .max_qpairs_per_ctrlr = SPDK_NVMF_RDMA_DEFAULT_MAX_QPAIRS_PER_CTRLR, 46 .in_capsule_data_size = SPDK_NVMF_RDMA_DEFAULT_IN_CAPSULE_DATA_SIZE, 47 .max_io_size = (SPDK_NVMF_RDMA_MIN_IO_BUFFER_SIZE * RDMA_UT_UNITS_IN_MAX_IO), 48 .io_unit_size = SPDK_NVMF_RDMA_MIN_IO_BUFFER_SIZE, 49 .max_aq_depth = SPDK_NVMF_RDMA_DEFAULT_AQ_DEPTH, 50 .num_shared_buffers = SPDK_NVMF_RDMA_DEFAULT_NUM_SHARED_BUFFERS, 51 }; 52 53 SPDK_LOG_REGISTER_COMPONENT(nvmf) 54 DEFINE_STUB(spdk_mem_map_set_translation, int, (struct spdk_mem_map *map, uint64_t vaddr, 55 uint64_t size, uint64_t translation), 0); 56 DEFINE_STUB(spdk_mem_map_clear_translation, int, (struct spdk_mem_map *map, uint64_t vaddr, 57 uint64_t size), 0); 58 DEFINE_STUB(spdk_mem_map_alloc, struct spdk_mem_map *, (uint64_t default_translation, 59 const struct spdk_mem_map_ops *ops, void *cb_ctx), NULL); 60 DEFINE_STUB(spdk_nvmf_qpair_disconnect, int, (struct spdk_nvmf_qpair *qpair, 61 nvmf_qpair_disconnect_cb cb_fn, void *ctx), 0); 62 DEFINE_STUB(spdk_nvmf_qpair_get_listen_trid, int, 63 (struct spdk_nvmf_qpair *qpair, struct spdk_nvme_transport_id *trid), 0); 64 DEFINE_STUB_V(spdk_mem_map_free, (struct spdk_mem_map **pmap)); 65 66 DEFINE_STUB_V(spdk_nvmf_ctrlr_data_init, (struct spdk_nvmf_transport_opts *opts, 67 struct spdk_nvmf_ctrlr_data *cdata)); 68 DEFINE_STUB_V(spdk_nvmf_request_exec, (struct spdk_nvmf_request *req)); 69 DEFINE_STUB(spdk_nvmf_request_complete, int, (struct spdk_nvmf_request *req), 0); 70 DEFINE_STUB(spdk_nvme_transport_id_compare, int, (const struct spdk_nvme_transport_id *trid1, 71 const struct spdk_nvme_transport_id *trid2), 0); 72 DEFINE_STUB_V(nvmf_ctrlr_abort_aer, (struct spdk_nvmf_ctrlr *ctrlr)); 73 DEFINE_STUB(spdk_nvmf_request_get_dif_ctx, bool, (struct spdk_nvmf_request *req, 74 struct spdk_dif_ctx *dif_ctx), false); 75 DEFINE_STUB_V(spdk_nvme_trid_populate_transport, (struct spdk_nvme_transport_id *trid, 76 enum spdk_nvme_transport_type trtype)); 77 DEFINE_STUB_V(spdk_nvmf_tgt_new_qpair, (struct spdk_nvmf_tgt *tgt, struct spdk_nvmf_qpair *qpair)); 78 DEFINE_STUB(nvmf_ctrlr_abort_request, int, (struct spdk_nvmf_request *req), 0); 79 DEFINE_STUB(spdk_nvme_transport_id_adrfam_str, const char *, (enum spdk_nvmf_adrfam adrfam), NULL); 80 DEFINE_STUB(ibv_dereg_mr, int, (struct ibv_mr *mr), 0); 81 82 /* ibv_reg_mr can be a macro, need to undefine it */ 83 #ifdef ibv_reg_mr 84 #undef ibv_reg_mr 85 #endif 86 87 DEFINE_RETURN_MOCK(ibv_reg_mr, struct ibv_mr *); 88 struct ibv_mr * 89 ibv_reg_mr(struct ibv_pd *pd, void *addr, size_t length, int access) 90 { 91 HANDLE_RETURN_MOCK(ibv_reg_mr); 92 if (length > 0) { 93 return &g_rdma_mr; 94 } else { 95 return NULL; 96 } 97 } 98 99 int ibv_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, 100 int attr_mask, struct ibv_qp_init_attr *init_attr) 101 { 102 if (qp == NULL) { 103 return -1; 104 } else { 105 attr->port_num = 80; 106 107 if (qp->state == IBV_QPS_ERR) { 108 attr->qp_state = 10; 109 } else { 110 attr->qp_state = IBV_QPS_INIT; 111 } 112 113 return 0; 114 } 115 } 116 117 const char * 118 spdk_nvme_transport_id_trtype_str(enum spdk_nvme_transport_type trtype) 119 { 120 switch (trtype) { 121 case SPDK_NVME_TRANSPORT_PCIE: 122 return "PCIe"; 123 case SPDK_NVME_TRANSPORT_RDMA: 124 return "RDMA"; 125 case SPDK_NVME_TRANSPORT_FC: 126 return "FC"; 127 default: 128 return NULL; 129 } 130 } 131 132 int 133 spdk_nvme_transport_id_populate_trstring(struct spdk_nvme_transport_id *trid, const char *trstring) 134 { 135 int len, i; 136 137 if (trstring == NULL) { 138 return -EINVAL; 139 } 140 141 len = strnlen(trstring, SPDK_NVMF_TRSTRING_MAX_LEN); 142 if (len == SPDK_NVMF_TRSTRING_MAX_LEN) { 143 return -EINVAL; 144 } 145 146 /* cast official trstring to uppercase version of input. */ 147 for (i = 0; i < len; i++) { 148 trid->trstring[i] = toupper(trstring[i]); 149 } 150 return 0; 151 } 152 153 static void reset_nvmf_rdma_request(struct spdk_nvmf_rdma_request *rdma_req) 154 { 155 int i; 156 157 rdma_req->req.length = 0; 158 rdma_req->req.data_from_pool = false; 159 rdma_req->req.data = NULL; 160 rdma_req->data.wr.num_sge = 0; 161 rdma_req->data.wr.wr.rdma.remote_addr = 0; 162 rdma_req->data.wr.wr.rdma.rkey = 0; 163 rdma_req->offset = 0; 164 memset(&rdma_req->req.dif, 0, sizeof(rdma_req->req.dif)); 165 166 for (i = 0; i < SPDK_NVMF_MAX_SGL_ENTRIES; i++) { 167 rdma_req->req.iov[i].iov_base = 0; 168 rdma_req->req.iov[i].iov_len = 0; 169 rdma_req->req.buffers[i] = 0; 170 rdma_req->data.wr.sg_list[i].addr = 0; 171 rdma_req->data.wr.sg_list[i].length = 0; 172 rdma_req->data.wr.sg_list[i].lkey = 0; 173 } 174 rdma_req->req.iovcnt = 0; 175 if (rdma_req->req.stripped_data) { 176 free(rdma_req->req.stripped_data); 177 rdma_req->req.stripped_data = NULL; 178 } 179 } 180 181 static void 182 test_spdk_nvmf_rdma_request_parse_sgl(void) 183 { 184 struct spdk_nvmf_rdma_transport rtransport; 185 struct spdk_nvmf_rdma_device device; 186 struct spdk_nvmf_rdma_request rdma_req = {}; 187 struct spdk_nvmf_rdma_recv recv; 188 struct spdk_nvmf_rdma_poll_group group; 189 struct spdk_nvmf_rdma_qpair rqpair; 190 struct spdk_nvmf_rdma_poller poller; 191 union nvmf_c2h_msg cpl; 192 union nvmf_h2c_msg cmd; 193 struct spdk_nvme_sgl_descriptor *sgl; 194 struct spdk_nvmf_transport_pg_cache_buf bufs[4]; 195 struct spdk_nvme_sgl_descriptor sgl_desc[SPDK_NVMF_MAX_SGL_ENTRIES] = {{0}}; 196 struct spdk_nvmf_rdma_request_data data; 197 int rc, i; 198 uint32_t sgl_length; 199 uintptr_t aligned_buffer_address; 200 201 data.wr.sg_list = data.sgl; 202 STAILQ_INIT(&group.group.buf_cache); 203 group.group.buf_cache_size = 0; 204 group.group.buf_cache_count = 0; 205 group.group.transport = &rtransport.transport; 206 poller.group = &group; 207 rqpair.poller = &poller; 208 rqpair.max_send_sge = SPDK_NVMF_MAX_SGL_ENTRIES; 209 210 sgl = &cmd.nvme_cmd.dptr.sgl1; 211 rdma_req.recv = &recv; 212 rdma_req.req.cmd = &cmd; 213 rdma_req.req.rsp = &cpl; 214 rdma_req.data.wr.sg_list = rdma_req.data.sgl; 215 rdma_req.req.qpair = &rqpair.qpair; 216 rdma_req.req.xfer = SPDK_NVME_DATA_CONTROLLER_TO_HOST; 217 218 rtransport.transport.opts = g_rdma_ut_transport_opts; 219 rtransport.data_wr_pool = NULL; 220 rtransport.transport.data_buf_pool = NULL; 221 222 device.attr.device_cap_flags = 0; 223 sgl->keyed.key = 0xEEEE; 224 sgl->address = 0xFFFF; 225 rdma_req.recv->buf = (void *)0xDDDD; 226 227 /* Test 1: sgl type: keyed data block subtype: address */ 228 sgl->generic.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; 229 sgl->keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; 230 231 /* Part 1: simple I/O, one SGL smaller than the transport io unit size */ 232 MOCK_SET(spdk_mempool_get, (void *)0x2000); 233 reset_nvmf_rdma_request(&rdma_req); 234 sgl->keyed.length = rtransport.transport.opts.io_unit_size / 2; 235 236 device.map = (void *)0x0; 237 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 238 CU_ASSERT(rc == 0); 239 CU_ASSERT(rdma_req.req.data_from_pool == true); 240 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size / 2); 241 CU_ASSERT((uint64_t)rdma_req.req.data == 0x2000); 242 CU_ASSERT(rdma_req.data.wr.num_sge == 1); 243 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 244 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 245 CU_ASSERT((uint64_t)rdma_req.req.buffers[0] == 0x2000); 246 CU_ASSERT(rdma_req.data.wr.sg_list[0].addr == 0x2000); 247 CU_ASSERT(rdma_req.data.wr.sg_list[0].length == rtransport.transport.opts.io_unit_size / 2); 248 CU_ASSERT(rdma_req.data.wr.sg_list[0].lkey == RDMA_UT_LKEY); 249 250 /* Part 2: simple I/O, one SGL larger than the transport io unit size (equal to the max io size) */ 251 reset_nvmf_rdma_request(&rdma_req); 252 sgl->keyed.length = rtransport.transport.opts.io_unit_size * RDMA_UT_UNITS_IN_MAX_IO; 253 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 254 255 CU_ASSERT(rc == 0); 256 CU_ASSERT(rdma_req.req.data_from_pool == true); 257 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size * RDMA_UT_UNITS_IN_MAX_IO); 258 CU_ASSERT(rdma_req.data.wr.num_sge == RDMA_UT_UNITS_IN_MAX_IO); 259 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 260 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 261 for (i = 0; i < RDMA_UT_UNITS_IN_MAX_IO; i++) { 262 CU_ASSERT((uint64_t)rdma_req.req.buffers[i] == 0x2000); 263 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == 0x2000); 264 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == rtransport.transport.opts.io_unit_size); 265 CU_ASSERT(rdma_req.data.wr.sg_list[i].lkey == RDMA_UT_LKEY); 266 } 267 268 /* Part 3: simple I/O one SGL larger than the transport max io size */ 269 reset_nvmf_rdma_request(&rdma_req); 270 sgl->keyed.length = rtransport.transport.opts.max_io_size * 2; 271 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 272 273 CU_ASSERT(rc == -1); 274 275 /* Part 4: Pretend there are no buffer pools */ 276 MOCK_SET(spdk_mempool_get, NULL); 277 reset_nvmf_rdma_request(&rdma_req); 278 sgl->keyed.length = rtransport.transport.opts.io_unit_size * RDMA_UT_UNITS_IN_MAX_IO; 279 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 280 281 CU_ASSERT(rc == 0); 282 CU_ASSERT(rdma_req.req.data_from_pool == false); 283 CU_ASSERT(rdma_req.req.data == NULL); 284 CU_ASSERT(rdma_req.data.wr.num_sge == 0); 285 CU_ASSERT(rdma_req.req.buffers[0] == NULL); 286 CU_ASSERT(rdma_req.data.wr.sg_list[0].addr == 0); 287 CU_ASSERT(rdma_req.data.wr.sg_list[0].length == 0); 288 CU_ASSERT(rdma_req.data.wr.sg_list[0].lkey == 0); 289 290 rdma_req.recv->buf = (void *)0xDDDD; 291 /* Test 2: sgl type: keyed data block subtype: offset (in capsule data) */ 292 sgl->generic.type = SPDK_NVME_SGL_TYPE_DATA_BLOCK; 293 sgl->unkeyed.subtype = SPDK_NVME_SGL_SUBTYPE_OFFSET; 294 295 /* Part 1: Normal I/O smaller than in capsule data size no offset */ 296 reset_nvmf_rdma_request(&rdma_req); 297 sgl->address = 0; 298 sgl->unkeyed.length = rtransport.transport.opts.in_capsule_data_size; 299 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 300 301 CU_ASSERT(rc == 0); 302 CU_ASSERT(rdma_req.req.data == (void *)0xDDDD); 303 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.in_capsule_data_size); 304 CU_ASSERT(rdma_req.req.data_from_pool == false); 305 306 /* Part 2: I/O offset + length too large */ 307 reset_nvmf_rdma_request(&rdma_req); 308 sgl->address = rtransport.transport.opts.in_capsule_data_size; 309 sgl->unkeyed.length = rtransport.transport.opts.in_capsule_data_size; 310 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 311 312 CU_ASSERT(rc == -1); 313 314 /* Part 3: I/O too large */ 315 reset_nvmf_rdma_request(&rdma_req); 316 sgl->address = 0; 317 sgl->unkeyed.length = rtransport.transport.opts.in_capsule_data_size * 2; 318 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 319 320 CU_ASSERT(rc == -1); 321 322 /* Test 3: Multi SGL */ 323 sgl->generic.type = SPDK_NVME_SGL_TYPE_LAST_SEGMENT; 324 sgl->unkeyed.subtype = SPDK_NVME_SGL_SUBTYPE_OFFSET; 325 sgl->address = 0; 326 rdma_req.recv->buf = (void *)&sgl_desc; 327 MOCK_SET(spdk_mempool_get, &data); 328 329 /* part 1: 2 segments each with 1 wr. */ 330 reset_nvmf_rdma_request(&rdma_req); 331 sgl->unkeyed.length = 2 * sizeof(struct spdk_nvme_sgl_descriptor); 332 for (i = 0; i < 2; i++) { 333 sgl_desc[i].keyed.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; 334 sgl_desc[i].keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; 335 sgl_desc[i].keyed.length = rtransport.transport.opts.io_unit_size; 336 sgl_desc[i].address = 0x4000 + i * rtransport.transport.opts.io_unit_size; 337 sgl_desc[i].keyed.key = 0x44; 338 } 339 340 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 341 342 CU_ASSERT(rc == 0); 343 CU_ASSERT(rdma_req.req.data_from_pool == true); 344 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size * 2); 345 CU_ASSERT(rdma_req.data.wr.num_sge == 1); 346 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0x44); 347 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0x4000); 348 CU_ASSERT(rdma_req.data.wr.next == &data.wr); 349 CU_ASSERT(data.wr.wr.rdma.rkey == 0x44); 350 CU_ASSERT(data.wr.wr.rdma.remote_addr == 0x4000 + rtransport.transport.opts.io_unit_size); 351 CU_ASSERT(data.wr.num_sge == 1); 352 CU_ASSERT(data.wr.next == &rdma_req.rsp.wr); 353 354 /* part 2: 2 segments, each with 1 wr containing 8 sge_elements */ 355 reset_nvmf_rdma_request(&rdma_req); 356 sgl->unkeyed.length = 2 * sizeof(struct spdk_nvme_sgl_descriptor); 357 for (i = 0; i < 2; i++) { 358 sgl_desc[i].keyed.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; 359 sgl_desc[i].keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; 360 sgl_desc[i].keyed.length = rtransport.transport.opts.io_unit_size * 8; 361 sgl_desc[i].address = 0x4000 + i * 8 * rtransport.transport.opts.io_unit_size; 362 sgl_desc[i].keyed.key = 0x44; 363 } 364 365 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 366 367 CU_ASSERT(rc == 0); 368 CU_ASSERT(rdma_req.req.data_from_pool == true); 369 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size * 16); 370 CU_ASSERT(rdma_req.req.iovcnt == 16); 371 CU_ASSERT(rdma_req.data.wr.num_sge == 8); 372 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0x44); 373 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0x4000); 374 CU_ASSERT(rdma_req.data.wr.next == &data.wr); 375 CU_ASSERT(data.wr.wr.rdma.rkey == 0x44); 376 CU_ASSERT(data.wr.wr.rdma.remote_addr == 0x4000 + rtransport.transport.opts.io_unit_size * 8); 377 CU_ASSERT(data.wr.num_sge == 8); 378 CU_ASSERT(data.wr.next == &rdma_req.rsp.wr); 379 380 /* part 3: 2 segments, one very large, one very small */ 381 reset_nvmf_rdma_request(&rdma_req); 382 for (i = 0; i < 2; i++) { 383 sgl_desc[i].keyed.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; 384 sgl_desc[i].keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; 385 sgl_desc[i].keyed.key = 0x44; 386 } 387 388 sgl_desc[0].keyed.length = rtransport.transport.opts.io_unit_size * 15 + 389 rtransport.transport.opts.io_unit_size / 2; 390 sgl_desc[0].address = 0x4000; 391 sgl_desc[1].keyed.length = rtransport.transport.opts.io_unit_size / 2; 392 sgl_desc[1].address = 0x4000 + rtransport.transport.opts.io_unit_size * 15 + 393 rtransport.transport.opts.io_unit_size / 2; 394 395 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 396 397 CU_ASSERT(rc == 0); 398 CU_ASSERT(rdma_req.req.data_from_pool == true); 399 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size * 16); 400 CU_ASSERT(rdma_req.req.iovcnt == 16); 401 CU_ASSERT(rdma_req.data.wr.num_sge == 16); 402 for (i = 0; i < 15; i++) { 403 CU_ASSERT(rdma_req.data.sgl[i].length == rtransport.transport.opts.io_unit_size); 404 } 405 CU_ASSERT(rdma_req.data.sgl[15].length == rtransport.transport.opts.io_unit_size / 2); 406 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0x44); 407 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0x4000); 408 CU_ASSERT(rdma_req.data.wr.next == &data.wr); 409 CU_ASSERT(data.wr.wr.rdma.rkey == 0x44); 410 CU_ASSERT(data.wr.wr.rdma.remote_addr == 0x4000 + rtransport.transport.opts.io_unit_size * 15 + 411 rtransport.transport.opts.io_unit_size / 2); 412 CU_ASSERT(data.sgl[0].length == rtransport.transport.opts.io_unit_size / 2); 413 CU_ASSERT(data.wr.num_sge == 1); 414 CU_ASSERT(data.wr.next == &rdma_req.rsp.wr); 415 416 /* part 4: 2 SGL descriptors, each length is transport buffer / 2 417 * 1 transport buffers should be allocated */ 418 reset_nvmf_rdma_request(&rdma_req); 419 aligned_buffer_address = ((uintptr_t)(&data) + NVMF_DATA_BUFFER_MASK) & ~NVMF_DATA_BUFFER_MASK; 420 sgl->unkeyed.length = 2 * sizeof(struct spdk_nvme_sgl_descriptor); 421 sgl_length = rtransport.transport.opts.io_unit_size / 2; 422 for (i = 0; i < 2; i++) { 423 sgl_desc[i].keyed.length = sgl_length; 424 sgl_desc[i].address = 0x4000 + i * sgl_length; 425 } 426 427 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 428 429 CU_ASSERT(rc == 0); 430 CU_ASSERT(rdma_req.req.data_from_pool == true); 431 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size); 432 CU_ASSERT(rdma_req.req.iovcnt == 1); 433 434 CU_ASSERT(rdma_req.data.sgl[0].length == sgl_length); 435 /* We mocked mempool_get to return address of data variable. Mempool is used 436 * to get both additional WRs and data buffers, so data points to &data */ 437 CU_ASSERT(rdma_req.data.sgl[0].addr == aligned_buffer_address); 438 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0x44); 439 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0x4000); 440 CU_ASSERT(rdma_req.data.wr.num_sge == 1); 441 CU_ASSERT(rdma_req.data.wr.next == &data.wr); 442 443 CU_ASSERT(data.wr.wr.rdma.rkey == 0x44); 444 CU_ASSERT(data.wr.wr.rdma.remote_addr == 0x4000 + sgl_length); 445 CU_ASSERT(data.sgl[0].length == sgl_length); 446 CU_ASSERT(data.sgl[0].addr == aligned_buffer_address + sgl_length); 447 CU_ASSERT(data.wr.num_sge == 1); 448 449 /* Test 4: use PG buffer cache */ 450 sgl->generic.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; 451 sgl->keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; 452 sgl->address = 0xFFFF; 453 rdma_req.recv->buf = (void *)0xDDDD; 454 sgl->keyed.key = 0xEEEE; 455 456 for (i = 0; i < 4; i++) { 457 STAILQ_INSERT_TAIL(&group.group.buf_cache, &bufs[i], link); 458 } 459 460 /* part 1: use the four buffers from the pg cache */ 461 group.group.buf_cache_size = 4; 462 group.group.buf_cache_count = 4; 463 MOCK_SET(spdk_mempool_get, (void *)0x2000); 464 reset_nvmf_rdma_request(&rdma_req); 465 sgl->keyed.length = rtransport.transport.opts.io_unit_size * 4; 466 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 467 468 SPDK_CU_ASSERT_FATAL(rc == 0); 469 CU_ASSERT(rdma_req.req.data_from_pool == true); 470 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size * 4); 471 CU_ASSERT((uint64_t)rdma_req.req.data == (((uint64_t)&bufs[0] + NVMF_DATA_BUFFER_MASK) & 472 ~NVMF_DATA_BUFFER_MASK)); 473 CU_ASSERT(rdma_req.data.wr.num_sge == 4); 474 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 475 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 476 CU_ASSERT(group.group.buf_cache_count == 0); 477 CU_ASSERT(STAILQ_EMPTY(&group.group.buf_cache)); 478 for (i = 0; i < 4; i++) { 479 CU_ASSERT((uint64_t)rdma_req.req.buffers[i] == (uint64_t)&bufs[i]); 480 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == (((uint64_t)&bufs[i] + NVMF_DATA_BUFFER_MASK) & 481 ~NVMF_DATA_BUFFER_MASK)); 482 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == rtransport.transport.opts.io_unit_size); 483 } 484 485 /* part 2: now that we have used the buffers from the cache, try again. We should get mempool buffers. */ 486 reset_nvmf_rdma_request(&rdma_req); 487 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 488 489 SPDK_CU_ASSERT_FATAL(rc == 0); 490 CU_ASSERT(rdma_req.req.data_from_pool == true); 491 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size * 4); 492 CU_ASSERT((uint64_t)rdma_req.req.data == 0x2000); 493 CU_ASSERT(rdma_req.data.wr.num_sge == 4); 494 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 495 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 496 CU_ASSERT(group.group.buf_cache_count == 0); 497 CU_ASSERT(STAILQ_EMPTY(&group.group.buf_cache)); 498 for (i = 0; i < 4; i++) { 499 CU_ASSERT((uint64_t)rdma_req.req.buffers[i] == 0x2000); 500 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == 0x2000); 501 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == rtransport.transport.opts.io_unit_size); 502 CU_ASSERT(group.group.buf_cache_count == 0); 503 } 504 505 /* part 3: half and half */ 506 group.group.buf_cache_count = 2; 507 508 for (i = 0; i < 2; i++) { 509 STAILQ_INSERT_TAIL(&group.group.buf_cache, &bufs[i], link); 510 } 511 reset_nvmf_rdma_request(&rdma_req); 512 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 513 514 SPDK_CU_ASSERT_FATAL(rc == 0); 515 CU_ASSERT(rdma_req.req.data_from_pool == true); 516 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size * 4); 517 CU_ASSERT((uint64_t)rdma_req.req.data == (((uint64_t)&bufs[0] + NVMF_DATA_BUFFER_MASK) & 518 ~NVMF_DATA_BUFFER_MASK)); 519 CU_ASSERT(rdma_req.data.wr.num_sge == 4); 520 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 521 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 522 CU_ASSERT(group.group.buf_cache_count == 0); 523 for (i = 0; i < 2; i++) { 524 CU_ASSERT((uint64_t)rdma_req.req.buffers[i] == (uint64_t)&bufs[i]); 525 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == (((uint64_t)&bufs[i] + NVMF_DATA_BUFFER_MASK) & 526 ~NVMF_DATA_BUFFER_MASK)); 527 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == rtransport.transport.opts.io_unit_size); 528 } 529 for (i = 2; i < 4; i++) { 530 CU_ASSERT((uint64_t)rdma_req.req.buffers[i] == 0x2000); 531 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == 0x2000); 532 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == rtransport.transport.opts.io_unit_size); 533 } 534 535 reset_nvmf_rdma_request(&rdma_req); 536 } 537 538 static struct spdk_nvmf_rdma_recv * 539 create_recv(struct spdk_nvmf_rdma_qpair *rqpair, enum spdk_nvme_nvm_opcode opc) 540 { 541 struct spdk_nvmf_rdma_recv *rdma_recv; 542 union nvmf_h2c_msg *cmd; 543 struct spdk_nvme_sgl_descriptor *sgl; 544 545 rdma_recv = calloc(1, sizeof(*rdma_recv)); 546 rdma_recv->qpair = rqpair; 547 cmd = calloc(1, sizeof(*cmd)); 548 rdma_recv->sgl[0].addr = (uintptr_t)cmd; 549 cmd->nvme_cmd.opc = opc; 550 sgl = &cmd->nvme_cmd.dptr.sgl1; 551 sgl->keyed.key = 0xEEEE; 552 sgl->address = 0xFFFF; 553 sgl->keyed.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; 554 sgl->keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; 555 sgl->keyed.length = 1; 556 557 return rdma_recv; 558 } 559 560 static void 561 free_recv(struct spdk_nvmf_rdma_recv *rdma_recv) 562 { 563 free((void *)rdma_recv->sgl[0].addr); 564 free(rdma_recv); 565 } 566 567 static struct spdk_nvmf_rdma_request * 568 create_req(struct spdk_nvmf_rdma_qpair *rqpair, 569 struct spdk_nvmf_rdma_recv *rdma_recv) 570 { 571 struct spdk_nvmf_rdma_request *rdma_req; 572 union nvmf_c2h_msg *cpl; 573 574 rdma_req = calloc(1, sizeof(*rdma_req)); 575 rdma_req->recv = rdma_recv; 576 rdma_req->req.qpair = &rqpair->qpair; 577 rdma_req->state = RDMA_REQUEST_STATE_NEW; 578 rdma_req->data.wr.wr_id = (uintptr_t)&rdma_req->data.rdma_wr; 579 rdma_req->data.wr.sg_list = rdma_req->data.sgl; 580 cpl = calloc(1, sizeof(*cpl)); 581 rdma_req->rsp.sgl[0].addr = (uintptr_t)cpl; 582 rdma_req->req.rsp = cpl; 583 584 return rdma_req; 585 } 586 587 static void 588 free_req(struct spdk_nvmf_rdma_request *rdma_req) 589 { 590 free((void *)rdma_req->rsp.sgl[0].addr); 591 free(rdma_req); 592 } 593 594 static void 595 qpair_reset(struct spdk_nvmf_rdma_qpair *rqpair, 596 struct spdk_nvmf_rdma_poller *poller, 597 struct spdk_nvmf_rdma_device *device, 598 struct spdk_nvmf_rdma_resources *resources, 599 struct spdk_nvmf_transport *transport) 600 { 601 memset(rqpair, 0, sizeof(*rqpair)); 602 STAILQ_INIT(&rqpair->pending_rdma_write_queue); 603 STAILQ_INIT(&rqpair->pending_rdma_read_queue); 604 rqpair->poller = poller; 605 rqpair->device = device; 606 rqpair->resources = resources; 607 rqpair->qpair.qid = 1; 608 rqpair->ibv_state = IBV_QPS_RTS; 609 rqpair->qpair.state = SPDK_NVMF_QPAIR_ACTIVE; 610 rqpair->max_send_sge = SPDK_NVMF_MAX_SGL_ENTRIES; 611 rqpair->max_send_depth = 16; 612 rqpair->max_read_depth = 16; 613 rqpair->qpair.transport = transport; 614 } 615 616 static void 617 poller_reset(struct spdk_nvmf_rdma_poller *poller, 618 struct spdk_nvmf_rdma_poll_group *group) 619 { 620 memset(poller, 0, sizeof(*poller)); 621 STAILQ_INIT(&poller->qpairs_pending_recv); 622 STAILQ_INIT(&poller->qpairs_pending_send); 623 poller->group = group; 624 } 625 626 static void 627 test_spdk_nvmf_rdma_request_process(void) 628 { 629 struct spdk_nvmf_rdma_transport rtransport = {}; 630 struct spdk_nvmf_rdma_poll_group group = {}; 631 struct spdk_nvmf_rdma_poller poller = {}; 632 struct spdk_nvmf_rdma_device device = {}; 633 struct spdk_nvmf_rdma_resources resources = {}; 634 struct spdk_nvmf_rdma_qpair rqpair = {}; 635 struct spdk_nvmf_rdma_recv *rdma_recv; 636 struct spdk_nvmf_rdma_request *rdma_req; 637 bool progress; 638 639 STAILQ_INIT(&group.group.buf_cache); 640 STAILQ_INIT(&group.group.pending_buf_queue); 641 group.group.buf_cache_size = 0; 642 group.group.buf_cache_count = 0; 643 poller_reset(&poller, &group); 644 qpair_reset(&rqpair, &poller, &device, &resources, &rtransport.transport); 645 646 rtransport.transport.opts = g_rdma_ut_transport_opts; 647 rtransport.transport.data_buf_pool = spdk_mempool_create("test_data_pool", 16, 128, 0, 0); 648 rtransport.data_wr_pool = spdk_mempool_create("test_wr_pool", 128, 649 sizeof(struct spdk_nvmf_rdma_request_data), 650 0, 0); 651 MOCK_CLEAR(spdk_mempool_get); 652 653 device.attr.device_cap_flags = 0; 654 device.map = (void *)0x0; 655 656 /* Test 1: single SGL READ request */ 657 rdma_recv = create_recv(&rqpair, SPDK_NVME_OPC_READ); 658 rdma_req = create_req(&rqpair, rdma_recv); 659 rqpair.current_recv_depth = 1; 660 /* NEW -> EXECUTING */ 661 progress = nvmf_rdma_request_process(&rtransport, rdma_req); 662 CU_ASSERT(progress == true); 663 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_EXECUTING); 664 CU_ASSERT(rdma_req->req.xfer == SPDK_NVME_DATA_CONTROLLER_TO_HOST); 665 /* EXECUTED -> TRANSFERRING_C2H */ 666 rdma_req->state = RDMA_REQUEST_STATE_EXECUTED; 667 progress = nvmf_rdma_request_process(&rtransport, rdma_req); 668 CU_ASSERT(progress == true); 669 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_TRANSFERRING_CONTROLLER_TO_HOST); 670 CU_ASSERT(rdma_req->recv == NULL); 671 /* COMPLETED -> FREE */ 672 rdma_req->state = RDMA_REQUEST_STATE_COMPLETED; 673 progress = nvmf_rdma_request_process(&rtransport, rdma_req); 674 CU_ASSERT(progress == true); 675 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_FREE); 676 677 free_recv(rdma_recv); 678 free_req(rdma_req); 679 poller_reset(&poller, &group); 680 qpair_reset(&rqpair, &poller, &device, &resources, &rtransport.transport); 681 682 /* Test 2: single SGL WRITE request */ 683 rdma_recv = create_recv(&rqpair, SPDK_NVME_OPC_WRITE); 684 rdma_req = create_req(&rqpair, rdma_recv); 685 rqpair.current_recv_depth = 1; 686 /* NEW -> TRANSFERRING_H2C */ 687 progress = nvmf_rdma_request_process(&rtransport, rdma_req); 688 CU_ASSERT(progress == true); 689 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_TRANSFERRING_HOST_TO_CONTROLLER); 690 CU_ASSERT(rdma_req->req.xfer == SPDK_NVME_DATA_HOST_TO_CONTROLLER); 691 STAILQ_INIT(&poller.qpairs_pending_send); 692 /* READY_TO_EXECUTE -> EXECUTING */ 693 rdma_req->state = RDMA_REQUEST_STATE_READY_TO_EXECUTE; 694 progress = nvmf_rdma_request_process(&rtransport, rdma_req); 695 CU_ASSERT(progress == true); 696 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_EXECUTING); 697 /* EXECUTED -> COMPLETING */ 698 rdma_req->state = RDMA_REQUEST_STATE_EXECUTED; 699 progress = nvmf_rdma_request_process(&rtransport, rdma_req); 700 CU_ASSERT(progress == true); 701 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_COMPLETING); 702 CU_ASSERT(rdma_req->recv == NULL); 703 /* COMPLETED -> FREE */ 704 rdma_req->state = RDMA_REQUEST_STATE_COMPLETED; 705 progress = nvmf_rdma_request_process(&rtransport, rdma_req); 706 CU_ASSERT(progress == true); 707 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_FREE); 708 709 free_recv(rdma_recv); 710 free_req(rdma_req); 711 poller_reset(&poller, &group); 712 qpair_reset(&rqpair, &poller, &device, &resources, &rtransport.transport); 713 714 /* Test 3: WRITE+WRITE ibv_send batching */ 715 { 716 struct spdk_nvmf_rdma_recv *recv1, *recv2; 717 struct spdk_nvmf_rdma_request *req1, *req2; 718 recv1 = create_recv(&rqpair, SPDK_NVME_OPC_WRITE); 719 req1 = create_req(&rqpair, recv1); 720 recv2 = create_recv(&rqpair, SPDK_NVME_OPC_WRITE); 721 req2 = create_req(&rqpair, recv2); 722 723 /* WRITE 1: NEW -> TRANSFERRING_H2C */ 724 rqpair.current_recv_depth = 1; 725 nvmf_rdma_request_process(&rtransport, req1); 726 CU_ASSERT(req1->state == RDMA_REQUEST_STATE_TRANSFERRING_HOST_TO_CONTROLLER); 727 728 /* WRITE 2: NEW -> TRANSFERRING_H2C */ 729 rqpair.current_recv_depth = 2; 730 nvmf_rdma_request_process(&rtransport, req2); 731 CU_ASSERT(req2->state == RDMA_REQUEST_STATE_TRANSFERRING_HOST_TO_CONTROLLER); 732 733 STAILQ_INIT(&poller.qpairs_pending_send); 734 735 /* WRITE 1 completes before WRITE 2 has finished RDMA reading */ 736 /* WRITE 1: READY_TO_EXECUTE -> EXECUTING */ 737 req1->state = RDMA_REQUEST_STATE_READY_TO_EXECUTE; 738 nvmf_rdma_request_process(&rtransport, req1); 739 CU_ASSERT(req1->state == RDMA_REQUEST_STATE_EXECUTING); 740 /* WRITE 1: EXECUTED -> COMPLETING */ 741 req1->state = RDMA_REQUEST_STATE_EXECUTED; 742 nvmf_rdma_request_process(&rtransport, req1); 743 CU_ASSERT(req1->state == RDMA_REQUEST_STATE_COMPLETING); 744 STAILQ_INIT(&poller.qpairs_pending_send); 745 /* WRITE 1: COMPLETED -> FREE */ 746 req1->state = RDMA_REQUEST_STATE_COMPLETED; 747 nvmf_rdma_request_process(&rtransport, req1); 748 CU_ASSERT(req1->state == RDMA_REQUEST_STATE_FREE); 749 750 /* Now WRITE 2 has finished reading and completes */ 751 /* WRITE 2: COMPLETED -> FREE */ 752 /* WRITE 2: READY_TO_EXECUTE -> EXECUTING */ 753 req2->state = RDMA_REQUEST_STATE_READY_TO_EXECUTE; 754 nvmf_rdma_request_process(&rtransport, req2); 755 CU_ASSERT(req2->state == RDMA_REQUEST_STATE_EXECUTING); 756 /* WRITE 1: EXECUTED -> COMPLETING */ 757 req2->state = RDMA_REQUEST_STATE_EXECUTED; 758 nvmf_rdma_request_process(&rtransport, req2); 759 CU_ASSERT(req2->state == RDMA_REQUEST_STATE_COMPLETING); 760 STAILQ_INIT(&poller.qpairs_pending_send); 761 /* WRITE 1: COMPLETED -> FREE */ 762 req2->state = RDMA_REQUEST_STATE_COMPLETED; 763 nvmf_rdma_request_process(&rtransport, req2); 764 CU_ASSERT(req2->state == RDMA_REQUEST_STATE_FREE); 765 766 free_recv(recv1); 767 free_req(req1); 768 free_recv(recv2); 769 free_req(req2); 770 poller_reset(&poller, &group); 771 qpair_reset(&rqpair, &poller, &device, &resources, &rtransport.transport); 772 } 773 774 /* Test 4, invalid command, check xfer type */ 775 { 776 struct spdk_nvmf_rdma_recv *rdma_recv_inv; 777 struct spdk_nvmf_rdma_request *rdma_req_inv; 778 /* construct an opcode that specifies BIDIRECTIONAL transfer */ 779 uint8_t opc = 0x10 | SPDK_NVME_DATA_BIDIRECTIONAL; 780 781 rdma_recv_inv = create_recv(&rqpair, opc); 782 rdma_req_inv = create_req(&rqpair, rdma_recv_inv); 783 784 /* NEW -> RDMA_REQUEST_STATE_COMPLETING */ 785 rqpair.current_recv_depth = 1; 786 progress = nvmf_rdma_request_process(&rtransport, rdma_req_inv); 787 CU_ASSERT(progress == true); 788 CU_ASSERT(rdma_req_inv->state == RDMA_REQUEST_STATE_COMPLETING); 789 CU_ASSERT(rdma_req_inv->req.rsp->nvme_cpl.status.sct == SPDK_NVME_SCT_GENERIC); 790 CU_ASSERT(rdma_req_inv->req.rsp->nvme_cpl.status.sc == SPDK_NVME_SC_INVALID_OPCODE); 791 792 /* RDMA_REQUEST_STATE_COMPLETED -> FREE */ 793 rdma_req_inv->state = RDMA_REQUEST_STATE_COMPLETED; 794 nvmf_rdma_request_process(&rtransport, rdma_req_inv); 795 CU_ASSERT(rdma_req_inv->state == RDMA_REQUEST_STATE_FREE); 796 797 free_recv(rdma_recv_inv); 798 free_req(rdma_req_inv); 799 poller_reset(&poller, &group); 800 qpair_reset(&rqpair, &poller, &device, &resources, &rtransport.transport); 801 } 802 803 spdk_mempool_free(rtransport.transport.data_buf_pool); 804 spdk_mempool_free(rtransport.data_wr_pool); 805 } 806 807 #define TEST_GROUPS_COUNT 5 808 static void 809 test_nvmf_rdma_get_optimal_poll_group(void) 810 { 811 struct spdk_nvmf_rdma_transport rtransport = {}; 812 struct spdk_nvmf_transport *transport = &rtransport.transport; 813 struct spdk_nvmf_rdma_qpair rqpair = {}; 814 struct spdk_nvmf_transport_poll_group *groups[TEST_GROUPS_COUNT]; 815 struct spdk_nvmf_rdma_poll_group *rgroups[TEST_GROUPS_COUNT]; 816 struct spdk_nvmf_transport_poll_group *result; 817 uint32_t i; 818 819 rqpair.qpair.transport = transport; 820 pthread_mutex_init(&rtransport.lock, NULL); 821 TAILQ_INIT(&rtransport.poll_groups); 822 823 for (i = 0; i < TEST_GROUPS_COUNT; i++) { 824 groups[i] = nvmf_rdma_poll_group_create(transport, NULL); 825 CU_ASSERT(groups[i] != NULL); 826 rgroups[i] = SPDK_CONTAINEROF(groups[i], struct spdk_nvmf_rdma_poll_group, group); 827 groups[i]->transport = transport; 828 } 829 CU_ASSERT(rtransport.conn_sched.next_admin_pg == rgroups[0]); 830 CU_ASSERT(rtransport.conn_sched.next_io_pg == rgroups[0]); 831 832 /* Emulate connection of %TEST_GROUPS_COUNT% initiators - each creates 1 admin and 1 io qp */ 833 for (i = 0; i < TEST_GROUPS_COUNT; i++) { 834 rqpair.qpair.qid = 0; 835 result = nvmf_rdma_get_optimal_poll_group(&rqpair.qpair); 836 CU_ASSERT(result == groups[i]); 837 CU_ASSERT(rtransport.conn_sched.next_admin_pg == rgroups[(i + 1) % TEST_GROUPS_COUNT]); 838 CU_ASSERT(rtransport.conn_sched.next_io_pg == rgroups[i]); 839 840 rqpair.qpair.qid = 1; 841 result = nvmf_rdma_get_optimal_poll_group(&rqpair.qpair); 842 CU_ASSERT(result == groups[i]); 843 CU_ASSERT(rtransport.conn_sched.next_admin_pg == rgroups[(i + 1) % TEST_GROUPS_COUNT]); 844 CU_ASSERT(rtransport.conn_sched.next_io_pg == rgroups[(i + 1) % TEST_GROUPS_COUNT]); 845 } 846 /* wrap around, admin/io pg point to the first pg 847 Destroy all poll groups except of the last one */ 848 for (i = 0; i < TEST_GROUPS_COUNT - 1; i++) { 849 nvmf_rdma_poll_group_destroy(groups[i]); 850 CU_ASSERT(rtransport.conn_sched.next_admin_pg == rgroups[i + 1]); 851 CU_ASSERT(rtransport.conn_sched.next_io_pg == rgroups[i + 1]); 852 } 853 854 CU_ASSERT(rtransport.conn_sched.next_admin_pg == rgroups[TEST_GROUPS_COUNT - 1]); 855 CU_ASSERT(rtransport.conn_sched.next_io_pg == rgroups[TEST_GROUPS_COUNT - 1]); 856 857 /* Check that pointers to the next admin/io poll groups are not changed */ 858 rqpair.qpair.qid = 0; 859 result = nvmf_rdma_get_optimal_poll_group(&rqpair.qpair); 860 CU_ASSERT(result == groups[TEST_GROUPS_COUNT - 1]); 861 CU_ASSERT(rtransport.conn_sched.next_admin_pg == rgroups[TEST_GROUPS_COUNT - 1]); 862 CU_ASSERT(rtransport.conn_sched.next_io_pg == rgroups[TEST_GROUPS_COUNT - 1]); 863 864 rqpair.qpair.qid = 1; 865 result = nvmf_rdma_get_optimal_poll_group(&rqpair.qpair); 866 CU_ASSERT(result == groups[TEST_GROUPS_COUNT - 1]); 867 CU_ASSERT(rtransport.conn_sched.next_admin_pg == rgroups[TEST_GROUPS_COUNT - 1]); 868 CU_ASSERT(rtransport.conn_sched.next_io_pg == rgroups[TEST_GROUPS_COUNT - 1]); 869 870 /* Remove the last poll group, check that pointers are NULL */ 871 nvmf_rdma_poll_group_destroy(groups[TEST_GROUPS_COUNT - 1]); 872 CU_ASSERT(rtransport.conn_sched.next_admin_pg == NULL); 873 CU_ASSERT(rtransport.conn_sched.next_io_pg == NULL); 874 875 /* Request optimal poll group, result must be NULL */ 876 rqpair.qpair.qid = 0; 877 result = nvmf_rdma_get_optimal_poll_group(&rqpair.qpair); 878 CU_ASSERT(result == NULL); 879 880 rqpair.qpair.qid = 1; 881 result = nvmf_rdma_get_optimal_poll_group(&rqpair.qpair); 882 CU_ASSERT(result == NULL); 883 884 pthread_mutex_destroy(&rtransport.lock); 885 } 886 #undef TEST_GROUPS_COUNT 887 888 static void 889 test_spdk_nvmf_rdma_request_parse_sgl_with_md(void) 890 { 891 struct spdk_nvmf_rdma_transport rtransport; 892 struct spdk_nvmf_rdma_device device; 893 struct spdk_nvmf_rdma_request rdma_req = {}; 894 struct spdk_nvmf_rdma_recv recv; 895 struct spdk_nvmf_rdma_poll_group group; 896 struct spdk_nvmf_rdma_qpair rqpair; 897 struct spdk_nvmf_rdma_poller poller; 898 union nvmf_c2h_msg cpl; 899 union nvmf_h2c_msg cmd; 900 struct spdk_nvme_sgl_descriptor *sgl; 901 struct spdk_nvme_sgl_descriptor sgl_desc[SPDK_NVMF_MAX_SGL_ENTRIES] = {{0}}; 902 char data_buffer[8192]; 903 struct spdk_nvmf_rdma_request_data *data = (struct spdk_nvmf_rdma_request_data *)data_buffer; 904 char data2_buffer[8192]; 905 struct spdk_nvmf_rdma_request_data *data2 = (struct spdk_nvmf_rdma_request_data *)data2_buffer; 906 const uint32_t data_bs = 512; 907 const uint32_t md_size = 8; 908 int rc, i; 909 void *aligned_buffer; 910 911 data->wr.sg_list = data->sgl; 912 STAILQ_INIT(&group.group.buf_cache); 913 group.group.buf_cache_size = 0; 914 group.group.buf_cache_count = 0; 915 group.group.transport = &rtransport.transport; 916 poller.group = &group; 917 rqpair.poller = &poller; 918 rqpair.max_send_sge = SPDK_NVMF_MAX_SGL_ENTRIES; 919 920 sgl = &cmd.nvme_cmd.dptr.sgl1; 921 rdma_req.recv = &recv; 922 rdma_req.req.cmd = &cmd; 923 rdma_req.req.rsp = &cpl; 924 rdma_req.data.wr.sg_list = rdma_req.data.sgl; 925 rdma_req.req.qpair = &rqpair.qpair; 926 rdma_req.req.xfer = SPDK_NVME_DATA_CONTROLLER_TO_HOST; 927 928 rtransport.transport.opts = g_rdma_ut_transport_opts; 929 rtransport.data_wr_pool = NULL; 930 rtransport.transport.data_buf_pool = NULL; 931 932 device.attr.device_cap_flags = 0; 933 device.map = NULL; 934 sgl->keyed.key = 0xEEEE; 935 sgl->address = 0xFFFF; 936 rdma_req.recv->buf = (void *)0xDDDD; 937 938 /* Test 1: sgl type: keyed data block subtype: address */ 939 sgl->generic.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; 940 sgl->keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; 941 942 /* Part 1: simple I/O, one SGL smaller than the transport io unit size, block size 512 */ 943 MOCK_SET(spdk_mempool_get, (void *)0x2000); 944 reset_nvmf_rdma_request(&rdma_req); 945 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 946 SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 947 0, 0, 0, 0, 0); 948 rdma_req.req.dif_enabled = true; 949 rtransport.transport.opts.io_unit_size = data_bs * 8; 950 rdma_req.req.qpair->transport = &rtransport.transport; 951 sgl->keyed.length = data_bs * 4; 952 953 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 954 955 CU_ASSERT(rc == 0); 956 CU_ASSERT(rdma_req.req.data_from_pool == true); 957 CU_ASSERT(rdma_req.req.length == data_bs * 4); 958 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 959 CU_ASSERT(rdma_req.req.dif.elba_length == (data_bs + md_size) * 4); 960 CU_ASSERT((uint64_t)rdma_req.req.data == 0x2000); 961 CU_ASSERT(rdma_req.data.wr.num_sge == 1); 962 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 963 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 964 CU_ASSERT((uint64_t)rdma_req.req.buffers[0] == 0x2000); 965 966 CU_ASSERT(rdma_req.data.wr.sg_list[0].addr == 0x2000); 967 CU_ASSERT(rdma_req.data.wr.sg_list[0].length == rdma_req.req.length); 968 CU_ASSERT(rdma_req.data.wr.sg_list[0].lkey == RDMA_UT_LKEY); 969 970 /* Part 2: simple I/O, one SGL equal to io unit size, io_unit_size is not aligned with md_size, 971 block size 512 */ 972 MOCK_SET(spdk_mempool_get, (void *)0x2000); 973 reset_nvmf_rdma_request(&rdma_req); 974 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 975 SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 976 0, 0, 0, 0, 0); 977 rdma_req.req.dif_enabled = true; 978 rtransport.transport.opts.io_unit_size = data_bs * 4; 979 sgl->keyed.length = data_bs * 4; 980 981 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 982 983 CU_ASSERT(rc == 0); 984 CU_ASSERT(rdma_req.req.data_from_pool == true); 985 CU_ASSERT(rdma_req.req.length == data_bs * 4); 986 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 987 CU_ASSERT(rdma_req.req.dif.elba_length == (data_bs + md_size) * 4); 988 CU_ASSERT((uint64_t)rdma_req.req.data == 0x2000); 989 CU_ASSERT(rdma_req.data.wr.num_sge == 5); 990 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 991 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 992 CU_ASSERT((uint64_t)rdma_req.req.buffers[0] == 0x2000); 993 994 for (i = 0; i < 3; ++i) { 995 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == 0x2000 + i * (data_bs + md_size)); 996 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == data_bs); 997 CU_ASSERT(rdma_req.data.wr.sg_list[i].lkey == RDMA_UT_LKEY); 998 } 999 CU_ASSERT(rdma_req.data.wr.sg_list[3].addr == 0x2000 + 3 * (data_bs + md_size)); 1000 CU_ASSERT(rdma_req.data.wr.sg_list[3].length == 488); 1001 CU_ASSERT(rdma_req.data.wr.sg_list[3].lkey == RDMA_UT_LKEY); 1002 1003 /* 2nd buffer consumed */ 1004 CU_ASSERT(rdma_req.data.wr.sg_list[4].addr == 0x2000); 1005 CU_ASSERT(rdma_req.data.wr.sg_list[4].length == 24); 1006 CU_ASSERT(rdma_req.data.wr.sg_list[4].lkey == RDMA_UT_LKEY); 1007 1008 /* Part 3: simple I/O, one SGL equal io unit size, io_unit_size is equal to block size 512 bytes */ 1009 MOCK_SET(spdk_mempool_get, (void *)0x2000); 1010 reset_nvmf_rdma_request(&rdma_req); 1011 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 1012 SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 1013 0, 0, 0, 0, 0); 1014 rdma_req.req.dif_enabled = true; 1015 rtransport.transport.opts.io_unit_size = data_bs; 1016 sgl->keyed.length = data_bs; 1017 1018 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 1019 1020 CU_ASSERT(rc == 0); 1021 CU_ASSERT(rdma_req.req.data_from_pool == true); 1022 CU_ASSERT(rdma_req.req.length == data_bs); 1023 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 1024 CU_ASSERT(rdma_req.req.dif.elba_length == data_bs + md_size); 1025 CU_ASSERT((uint64_t)rdma_req.req.data == 0x2000); 1026 CU_ASSERT(rdma_req.data.wr.num_sge == 1); 1027 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 1028 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 1029 CU_ASSERT((uint64_t)rdma_req.req.buffers[0] == 0x2000); 1030 1031 CU_ASSERT(rdma_req.data.wr.sg_list[0].addr == 0x2000); 1032 CU_ASSERT(rdma_req.data.wr.sg_list[0].length == data_bs); 1033 CU_ASSERT(rdma_req.data.wr.sg_list[0].lkey == RDMA_UT_LKEY); 1034 1035 CU_ASSERT(rdma_req.req.iovcnt == 2); 1036 CU_ASSERT(rdma_req.req.iov[0].iov_base == (void *)((unsigned long)0x2000)); 1037 CU_ASSERT(rdma_req.req.iov[0].iov_len == data_bs); 1038 /* 2nd buffer consumed for metadata */ 1039 CU_ASSERT(rdma_req.req.iov[1].iov_base == (void *)((unsigned long)0x2000)); 1040 CU_ASSERT(rdma_req.req.iov[1].iov_len == md_size); 1041 1042 /* Part 4: simple I/O, one SGL equal io unit size, io_unit_size is aligned with md_size, 1043 block size 512 */ 1044 MOCK_SET(spdk_mempool_get, (void *)0x2000); 1045 reset_nvmf_rdma_request(&rdma_req); 1046 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 1047 SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 1048 0, 0, 0, 0, 0); 1049 rdma_req.req.dif_enabled = true; 1050 rtransport.transport.opts.io_unit_size = (data_bs + md_size) * 4; 1051 sgl->keyed.length = data_bs * 4; 1052 1053 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 1054 1055 CU_ASSERT(rc == 0); 1056 CU_ASSERT(rdma_req.req.data_from_pool == true); 1057 CU_ASSERT(rdma_req.req.length == data_bs * 4); 1058 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 1059 CU_ASSERT(rdma_req.req.dif.elba_length == (data_bs + md_size) * 4); 1060 CU_ASSERT((uint64_t)rdma_req.req.data == 0x2000); 1061 CU_ASSERT(rdma_req.data.wr.num_sge == 1); 1062 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 1063 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 1064 CU_ASSERT((uint64_t)rdma_req.req.buffers[0] == 0x2000); 1065 1066 CU_ASSERT(rdma_req.data.wr.sg_list[0].addr == 0x2000); 1067 CU_ASSERT(rdma_req.data.wr.sg_list[0].length == rdma_req.req.length); 1068 CU_ASSERT(rdma_req.data.wr.sg_list[0].lkey == RDMA_UT_LKEY); 1069 1070 /* Part 5: simple I/O, one SGL equal to 2x io unit size, io_unit_size is aligned with md_size, 1071 block size 512 */ 1072 MOCK_SET(spdk_mempool_get, (void *)0x2000); 1073 reset_nvmf_rdma_request(&rdma_req); 1074 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 1075 SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 1076 0, 0, 0, 0, 0); 1077 rdma_req.req.dif_enabled = true; 1078 rtransport.transport.opts.io_unit_size = (data_bs + md_size) * 2; 1079 sgl->keyed.length = data_bs * 4; 1080 1081 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 1082 1083 CU_ASSERT(rc == 0); 1084 CU_ASSERT(rdma_req.req.data_from_pool == true); 1085 CU_ASSERT(rdma_req.req.length == data_bs * 4); 1086 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 1087 CU_ASSERT(rdma_req.req.dif.elba_length == (data_bs + md_size) * 4); 1088 CU_ASSERT((uint64_t)rdma_req.req.data == 0x2000); 1089 CU_ASSERT(rdma_req.data.wr.num_sge == 2); 1090 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 1091 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 1092 CU_ASSERT((uint64_t)rdma_req.req.buffers[0] == 0x2000); 1093 1094 for (i = 0; i < 2; ++i) { 1095 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == 0x2000); 1096 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == data_bs * 2); 1097 } 1098 1099 /* Part 6: simple I/O, one SGL larger than the transport io unit size, io_unit_size is not aligned to md_size, 1100 block size 512 */ 1101 MOCK_SET(spdk_mempool_get, (void *)0x2000); 1102 reset_nvmf_rdma_request(&rdma_req); 1103 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 1104 SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 1105 0, 0, 0, 0, 0); 1106 rdma_req.req.dif_enabled = true; 1107 rtransport.transport.opts.io_unit_size = data_bs * 4; 1108 sgl->keyed.length = data_bs * 6; 1109 1110 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 1111 1112 CU_ASSERT(rc == 0); 1113 CU_ASSERT(rdma_req.req.data_from_pool == true); 1114 CU_ASSERT(rdma_req.req.length == data_bs * 6); 1115 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 1116 CU_ASSERT(rdma_req.req.dif.elba_length == (data_bs + md_size) * 6); 1117 CU_ASSERT((uint64_t)rdma_req.req.data == 0x2000); 1118 CU_ASSERT(rdma_req.data.wr.num_sge == 7); 1119 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 1120 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 1121 CU_ASSERT((uint64_t)rdma_req.req.buffers[0] == 0x2000); 1122 1123 for (i = 0; i < 3; ++i) { 1124 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == 0x2000 + i * (data_bs + md_size)); 1125 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == data_bs); 1126 CU_ASSERT(rdma_req.data.wr.sg_list[i].lkey == RDMA_UT_LKEY); 1127 } 1128 CU_ASSERT(rdma_req.data.wr.sg_list[3].addr == 0x2000 + 3 * (data_bs + md_size)); 1129 CU_ASSERT(rdma_req.data.wr.sg_list[3].length == 488); 1130 CU_ASSERT(rdma_req.data.wr.sg_list[3].lkey == RDMA_UT_LKEY); 1131 1132 /* 2nd IO buffer consumed */ 1133 CU_ASSERT(rdma_req.data.wr.sg_list[4].addr == 0x2000); 1134 CU_ASSERT(rdma_req.data.wr.sg_list[4].length == 24); 1135 CU_ASSERT(rdma_req.data.wr.sg_list[4].lkey == RDMA_UT_LKEY); 1136 1137 CU_ASSERT(rdma_req.data.wr.sg_list[5].addr == 0x2000 + 24 + md_size); 1138 CU_ASSERT(rdma_req.data.wr.sg_list[5].length == 512); 1139 CU_ASSERT(rdma_req.data.wr.sg_list[5].lkey == RDMA_UT_LKEY); 1140 1141 CU_ASSERT(rdma_req.data.wr.sg_list[6].addr == 0x2000 + 24 + 512 + md_size * 2); 1142 CU_ASSERT(rdma_req.data.wr.sg_list[6].length == 512); 1143 CU_ASSERT(rdma_req.data.wr.sg_list[6].lkey == RDMA_UT_LKEY); 1144 1145 /* Part 7: simple I/O, number of SGL entries exceeds the number of entries 1146 one WR can hold. Additional WR is chained */ 1147 MOCK_SET(spdk_mempool_get, data2_buffer); 1148 aligned_buffer = (void *)((uintptr_t)(data2_buffer + NVMF_DATA_BUFFER_MASK) & 1149 ~NVMF_DATA_BUFFER_MASK); 1150 reset_nvmf_rdma_request(&rdma_req); 1151 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 1152 SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 1153 0, 0, 0, 0, 0); 1154 rdma_req.req.dif_enabled = true; 1155 rtransport.transport.opts.io_unit_size = data_bs * 16; 1156 sgl->keyed.length = data_bs * 16; 1157 1158 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 1159 1160 CU_ASSERT(rc == 0); 1161 CU_ASSERT(rdma_req.req.data_from_pool == true); 1162 CU_ASSERT(rdma_req.req.length == data_bs * 16); 1163 CU_ASSERT(rdma_req.req.iovcnt == 2); 1164 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 1165 CU_ASSERT(rdma_req.req.dif.elba_length == (data_bs + md_size) * 16); 1166 CU_ASSERT(rdma_req.req.data == aligned_buffer); 1167 CU_ASSERT(rdma_req.data.wr.num_sge == 16); 1168 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 1169 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 1170 1171 for (i = 0; i < 15; ++i) { 1172 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == (uintptr_t)aligned_buffer + i * (data_bs + md_size)); 1173 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == data_bs); 1174 CU_ASSERT(rdma_req.data.wr.sg_list[i].lkey == RDMA_UT_LKEY); 1175 } 1176 1177 /* 8192 - (512 + 8) * 15 = 392 */ 1178 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == (uintptr_t)aligned_buffer + i * (data_bs + md_size)); 1179 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == 392); 1180 CU_ASSERT(rdma_req.data.wr.sg_list[i].lkey == RDMA_UT_LKEY); 1181 1182 /* additional wr from pool */ 1183 CU_ASSERT(rdma_req.data.wr.next == (void *)&data2->wr); 1184 CU_ASSERT(rdma_req.data.wr.next->num_sge == 1); 1185 CU_ASSERT(rdma_req.data.wr.next->next == &rdma_req.rsp.wr); 1186 /* 2nd IO buffer */ 1187 CU_ASSERT(data2->wr.sg_list[0].addr == (uintptr_t)aligned_buffer); 1188 CU_ASSERT(data2->wr.sg_list[0].length == 120); 1189 CU_ASSERT(data2->wr.sg_list[0].lkey == RDMA_UT_LKEY); 1190 1191 /* Part 8: simple I/O, data with metadata do not fit to 1 io_buffer */ 1192 MOCK_SET(spdk_mempool_get, (void *)0x2000); 1193 reset_nvmf_rdma_request(&rdma_req); 1194 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 1195 SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 1196 0, 0, 0, 0, 0); 1197 rdma_req.req.dif_enabled = true; 1198 rtransport.transport.opts.io_unit_size = 516; 1199 sgl->keyed.length = data_bs * 2; 1200 1201 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 1202 1203 CU_ASSERT(rc == 0); 1204 CU_ASSERT(rdma_req.req.data_from_pool == true); 1205 CU_ASSERT(rdma_req.req.length == data_bs * 2); 1206 CU_ASSERT(rdma_req.req.iovcnt == 3); 1207 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 1208 CU_ASSERT(rdma_req.req.dif.elba_length == (data_bs + md_size) * 2); 1209 CU_ASSERT(rdma_req.req.data == (void *)0x2000); 1210 CU_ASSERT(rdma_req.data.wr.num_sge == 2); 1211 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 1212 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 1213 1214 CU_ASSERT(rdma_req.data.wr.sg_list[0].addr == 0x2000); 1215 CU_ASSERT(rdma_req.data.wr.sg_list[0].length == 512); 1216 CU_ASSERT(rdma_req.data.wr.sg_list[0].lkey == RDMA_UT_LKEY); 1217 1218 /* 2nd IO buffer consumed, offset 4 bytes due to part of the metadata 1219 is located at the beginning of that buffer */ 1220 CU_ASSERT(rdma_req.data.wr.sg_list[1].addr == 0x2000 + 4); 1221 CU_ASSERT(rdma_req.data.wr.sg_list[1].length == 512); 1222 CU_ASSERT(rdma_req.data.wr.sg_list[1].lkey == RDMA_UT_LKEY); 1223 1224 /* Test 2: Multi SGL */ 1225 sgl->generic.type = SPDK_NVME_SGL_TYPE_LAST_SEGMENT; 1226 sgl->unkeyed.subtype = SPDK_NVME_SGL_SUBTYPE_OFFSET; 1227 sgl->address = 0; 1228 rdma_req.recv->buf = (void *)&sgl_desc; 1229 MOCK_SET(spdk_mempool_get, data_buffer); 1230 aligned_buffer = (void *)((uintptr_t)(data_buffer + NVMF_DATA_BUFFER_MASK) & 1231 ~NVMF_DATA_BUFFER_MASK); 1232 1233 /* part 1: 2 segments each with 1 wr. io_unit_size is aligned with data_bs + md_size */ 1234 reset_nvmf_rdma_request(&rdma_req); 1235 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 1236 SPDK_DIF_TYPE1, 1237 SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 0, 0, 0, 0, 0); 1238 rdma_req.req.dif_enabled = true; 1239 rtransport.transport.opts.io_unit_size = (data_bs + md_size) * 4; 1240 sgl->unkeyed.length = 2 * sizeof(struct spdk_nvme_sgl_descriptor); 1241 1242 for (i = 0; i < 2; i++) { 1243 sgl_desc[i].keyed.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; 1244 sgl_desc[i].keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; 1245 sgl_desc[i].keyed.length = data_bs * 4; 1246 sgl_desc[i].address = 0x4000 + i * data_bs * 4; 1247 sgl_desc[i].keyed.key = 0x44; 1248 } 1249 1250 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 1251 1252 CU_ASSERT(rc == 0); 1253 CU_ASSERT(rdma_req.req.data_from_pool == true); 1254 CU_ASSERT(rdma_req.req.length == data_bs * 4 * 2); 1255 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 1256 CU_ASSERT(rdma_req.req.dif.elba_length == (data_bs + md_size) * 4 * 2); 1257 CU_ASSERT(rdma_req.data.wr.num_sge == 1); 1258 CU_ASSERT(rdma_req.data.wr.sg_list[0].addr == (uintptr_t)(aligned_buffer)); 1259 CU_ASSERT(rdma_req.data.wr.sg_list[0].length == data_bs * 4); 1260 1261 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0x44); 1262 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0x4000); 1263 CU_ASSERT(rdma_req.data.wr.next == &data->wr); 1264 CU_ASSERT(data->wr.wr.rdma.rkey == 0x44); 1265 CU_ASSERT(data->wr.wr.rdma.remote_addr == 0x4000 + data_bs * 4); 1266 CU_ASSERT(data->wr.num_sge == 1); 1267 CU_ASSERT(data->wr.sg_list[0].addr == (uintptr_t)(aligned_buffer)); 1268 CU_ASSERT(data->wr.sg_list[0].length == data_bs * 4); 1269 1270 CU_ASSERT(data->wr.next == &rdma_req.rsp.wr); 1271 reset_nvmf_rdma_request(&rdma_req); 1272 } 1273 1274 static void 1275 test_nvmf_rdma_opts_init(void) 1276 { 1277 struct spdk_nvmf_transport_opts opts = {}; 1278 1279 nvmf_rdma_opts_init(&opts); 1280 CU_ASSERT(opts.max_queue_depth == SPDK_NVMF_RDMA_DEFAULT_MAX_QUEUE_DEPTH); 1281 CU_ASSERT(opts.max_qpairs_per_ctrlr == SPDK_NVMF_RDMA_DEFAULT_MAX_QPAIRS_PER_CTRLR); 1282 CU_ASSERT(opts.in_capsule_data_size == SPDK_NVMF_RDMA_DEFAULT_IN_CAPSULE_DATA_SIZE); 1283 CU_ASSERT(opts.max_io_size == SPDK_NVMF_RDMA_DEFAULT_MAX_IO_SIZE); 1284 CU_ASSERT(opts.io_unit_size == SPDK_NVMF_RDMA_MIN_IO_BUFFER_SIZE); 1285 CU_ASSERT(opts.max_aq_depth == SPDK_NVMF_RDMA_DEFAULT_AQ_DEPTH); 1286 CU_ASSERT(opts.num_shared_buffers == SPDK_NVMF_RDMA_DEFAULT_NUM_SHARED_BUFFERS); 1287 CU_ASSERT(opts.buf_cache_size == SPDK_NVMF_RDMA_DEFAULT_BUFFER_CACHE_SIZE); 1288 CU_ASSERT(opts.dif_insert_or_strip == SPDK_NVMF_RDMA_DIF_INSERT_OR_STRIP); 1289 CU_ASSERT(opts.abort_timeout_sec == SPDK_NVMF_RDMA_DEFAULT_ABORT_TIMEOUT_SEC); 1290 CU_ASSERT(opts.transport_specific == NULL); 1291 } 1292 1293 static void 1294 test_nvmf_rdma_request_free_data(void) 1295 { 1296 struct spdk_nvmf_rdma_request rdma_req = {}; 1297 struct spdk_nvmf_rdma_transport rtransport = {}; 1298 struct spdk_nvmf_rdma_request_data *next_request_data = NULL; 1299 1300 MOCK_CLEAR(spdk_mempool_get); 1301 rtransport.data_wr_pool = spdk_mempool_create("spdk_nvmf_rdma_wr_data", 1302 SPDK_NVMF_MAX_SGL_ENTRIES, 1303 sizeof(struct spdk_nvmf_rdma_request_data), 1304 SPDK_MEMPOOL_DEFAULT_CACHE_SIZE, 1305 SPDK_ENV_SOCKET_ID_ANY); 1306 next_request_data = spdk_mempool_get(rtransport.data_wr_pool); 1307 SPDK_CU_ASSERT_FATAL(((struct test_mempool *)rtransport.data_wr_pool)->count == 1308 SPDK_NVMF_MAX_SGL_ENTRIES - 1); 1309 next_request_data->wr.wr_id = 1; 1310 next_request_data->wr.num_sge = 2; 1311 next_request_data->wr.next = NULL; 1312 rdma_req.data.wr.next = &next_request_data->wr; 1313 rdma_req.data.wr.wr_id = 1; 1314 rdma_req.data.wr.num_sge = 2; 1315 1316 nvmf_rdma_request_free_data(&rdma_req, &rtransport); 1317 /* Check if next_request_data put into memory pool */ 1318 CU_ASSERT(((struct test_mempool *)rtransport.data_wr_pool)->count == SPDK_NVMF_MAX_SGL_ENTRIES); 1319 CU_ASSERT(rdma_req.data.wr.num_sge == 0); 1320 1321 spdk_mempool_free(rtransport.data_wr_pool); 1322 } 1323 1324 static void 1325 test_nvmf_rdma_update_ibv_state(void) 1326 { 1327 struct spdk_nvmf_rdma_qpair rqpair = {}; 1328 struct spdk_rdma_qp rdma_qp = {}; 1329 struct ibv_qp qp = {}; 1330 int rc = 0; 1331 1332 rqpair.rdma_qp = &rdma_qp; 1333 1334 /* Case 1: Failed to get updated RDMA queue pair state */ 1335 rqpair.ibv_state = IBV_QPS_INIT; 1336 rqpair.rdma_qp->qp = NULL; 1337 1338 rc = nvmf_rdma_update_ibv_state(&rqpair); 1339 CU_ASSERT(rc == IBV_QPS_ERR + 1); 1340 1341 /* Case 2: Bad state updated */ 1342 rqpair.rdma_qp->qp = &qp; 1343 qp.state = IBV_QPS_ERR; 1344 rc = nvmf_rdma_update_ibv_state(&rqpair); 1345 CU_ASSERT(rqpair.ibv_state == 10); 1346 CU_ASSERT(rc == IBV_QPS_ERR + 1); 1347 1348 /* Case 3: Pass */ 1349 qp.state = IBV_QPS_INIT; 1350 rc = nvmf_rdma_update_ibv_state(&rqpair); 1351 CU_ASSERT(rqpair.ibv_state == IBV_QPS_INIT); 1352 CU_ASSERT(rc == IBV_QPS_INIT); 1353 } 1354 1355 static void 1356 test_nvmf_rdma_resources_create(void) 1357 { 1358 static struct spdk_nvmf_rdma_resources *rdma_resource; 1359 struct spdk_nvmf_rdma_resource_opts opts = {}; 1360 struct spdk_nvmf_rdma_qpair qpair = {}; 1361 struct spdk_nvmf_rdma_recv *recv = NULL; 1362 struct spdk_nvmf_rdma_request *req = NULL; 1363 const int DEPTH = 128; 1364 1365 opts.max_queue_depth = DEPTH; 1366 opts.in_capsule_data_size = 4096; 1367 opts.shared = true; 1368 opts.qpair = &qpair; 1369 1370 rdma_resource = nvmf_rdma_resources_create(&opts); 1371 CU_ASSERT(rdma_resource != NULL); 1372 /* Just check first and last entry */ 1373 recv = &rdma_resource->recvs[0]; 1374 req = &rdma_resource->reqs[0]; 1375 CU_ASSERT(recv->rdma_wr.type == RDMA_WR_TYPE_RECV); 1376 CU_ASSERT((uintptr_t)recv->buf == (uintptr_t)(rdma_resource->bufs)); 1377 CU_ASSERT(recv->sgl[0].addr == (uintptr_t)&rdma_resource->cmds[0]); 1378 CU_ASSERT(recv->sgl[0].length == sizeof(rdma_resource->cmds[0])); 1379 CU_ASSERT(recv->sgl[0].lkey == RDMA_UT_LKEY); 1380 CU_ASSERT(recv->wr.num_sge == 2); 1381 CU_ASSERT(recv->wr.wr_id == (uintptr_t)&rdma_resource->recvs[0].rdma_wr); 1382 CU_ASSERT(recv->wr.sg_list == rdma_resource->recvs[0].sgl); 1383 CU_ASSERT(req->req.rsp == &rdma_resource->cpls[0]); 1384 CU_ASSERT(req->rsp.sgl[0].addr == (uintptr_t)&rdma_resource->cpls[0]); 1385 CU_ASSERT(req->rsp.sgl[0].length == sizeof(rdma_resource->cpls[0])); 1386 CU_ASSERT(req->rsp.sgl[0].lkey == RDMA_UT_LKEY); 1387 CU_ASSERT(req->rsp.rdma_wr.type == RDMA_WR_TYPE_SEND); 1388 CU_ASSERT(req->rsp.wr.wr_id == (uintptr_t)&rdma_resource->reqs[0].rsp.rdma_wr); 1389 CU_ASSERT(req->rsp.wr.next == NULL); 1390 CU_ASSERT(req->rsp.wr.opcode == IBV_WR_SEND); 1391 CU_ASSERT(req->rsp.wr.send_flags == IBV_SEND_SIGNALED); 1392 CU_ASSERT(req->rsp.wr.sg_list == rdma_resource->reqs[0].rsp.sgl); 1393 CU_ASSERT(req->rsp.wr.num_sge == NVMF_DEFAULT_RSP_SGE); 1394 CU_ASSERT(req->data.rdma_wr.type == RDMA_WR_TYPE_DATA); 1395 CU_ASSERT(req->data.wr.wr_id == (uintptr_t)&rdma_resource->reqs[0].data.rdma_wr); 1396 CU_ASSERT(req->data.wr.next == NULL); 1397 CU_ASSERT(req->data.wr.send_flags == IBV_SEND_SIGNALED); 1398 CU_ASSERT(req->data.wr.sg_list == rdma_resource->reqs[0].data.sgl); 1399 CU_ASSERT(req->data.wr.num_sge == SPDK_NVMF_MAX_SGL_ENTRIES); 1400 CU_ASSERT(req->state == RDMA_REQUEST_STATE_FREE); 1401 1402 recv = &rdma_resource->recvs[DEPTH - 1]; 1403 req = &rdma_resource->reqs[DEPTH - 1]; 1404 CU_ASSERT(recv->rdma_wr.type == RDMA_WR_TYPE_RECV); 1405 CU_ASSERT((uintptr_t)recv->buf == (uintptr_t)(rdma_resource->bufs + 1406 (DEPTH - 1) * 4096)); 1407 CU_ASSERT(recv->sgl[0].addr == (uintptr_t)&rdma_resource->cmds[DEPTH - 1]); 1408 CU_ASSERT(recv->sgl[0].length == sizeof(rdma_resource->cmds[DEPTH - 1])); 1409 CU_ASSERT(recv->sgl[0].lkey == RDMA_UT_LKEY); 1410 CU_ASSERT(recv->wr.num_sge == 2); 1411 CU_ASSERT(recv->wr.wr_id == (uintptr_t)&rdma_resource->recvs[DEPTH - 1].rdma_wr); 1412 CU_ASSERT(recv->wr.sg_list == rdma_resource->recvs[DEPTH - 1].sgl); 1413 CU_ASSERT(req->req.rsp == &rdma_resource->cpls[DEPTH - 1]); 1414 CU_ASSERT(req->rsp.sgl[0].addr == (uintptr_t)&rdma_resource->cpls[DEPTH - 1]); 1415 CU_ASSERT(req->rsp.sgl[0].length == sizeof(rdma_resource->cpls[DEPTH - 1])); 1416 CU_ASSERT(req->rsp.sgl[0].lkey == RDMA_UT_LKEY); 1417 CU_ASSERT(req->rsp.rdma_wr.type == RDMA_WR_TYPE_SEND); 1418 CU_ASSERT(req->rsp.wr.wr_id == (uintptr_t) 1419 &req->rsp.rdma_wr); 1420 CU_ASSERT(req->rsp.wr.next == NULL); 1421 CU_ASSERT(req->rsp.wr.opcode == IBV_WR_SEND); 1422 CU_ASSERT(req->rsp.wr.send_flags == IBV_SEND_SIGNALED); 1423 CU_ASSERT(req->rsp.wr.sg_list == rdma_resource->reqs[DEPTH - 1].rsp.sgl); 1424 CU_ASSERT(req->rsp.wr.num_sge == NVMF_DEFAULT_RSP_SGE); 1425 CU_ASSERT(req->data.rdma_wr.type == RDMA_WR_TYPE_DATA); 1426 CU_ASSERT(req->data.wr.wr_id == (uintptr_t) 1427 &req->data.rdma_wr); 1428 CU_ASSERT(req->data.wr.next == NULL); 1429 CU_ASSERT(req->data.wr.send_flags == IBV_SEND_SIGNALED); 1430 CU_ASSERT(req->data.wr.sg_list == rdma_resource->reqs[DEPTH - 1].data.sgl); 1431 CU_ASSERT(req->data.wr.num_sge == SPDK_NVMF_MAX_SGL_ENTRIES); 1432 CU_ASSERT(req->state == RDMA_REQUEST_STATE_FREE); 1433 1434 nvmf_rdma_resources_destroy(rdma_resource); 1435 } 1436 1437 static void 1438 test_nvmf_rdma_qpair_compare(void) 1439 { 1440 struct spdk_nvmf_rdma_qpair rqpair1 = {}, rqpair2 = {}; 1441 1442 rqpair1.qp_num = 0; 1443 rqpair2.qp_num = UINT32_MAX; 1444 1445 CU_ASSERT(nvmf_rdma_qpair_compare(&rqpair1, &rqpair2) < 0); 1446 CU_ASSERT(nvmf_rdma_qpair_compare(&rqpair2, &rqpair1) > 0); 1447 } 1448 1449 int main(int argc, char **argv) 1450 { 1451 CU_pSuite suite = NULL; 1452 unsigned int num_failures; 1453 1454 CU_set_error_action(CUEA_ABORT); 1455 CU_initialize_registry(); 1456 1457 suite = CU_add_suite("nvmf", NULL, NULL); 1458 1459 CU_ADD_TEST(suite, test_spdk_nvmf_rdma_request_parse_sgl); 1460 CU_ADD_TEST(suite, test_spdk_nvmf_rdma_request_process); 1461 CU_ADD_TEST(suite, test_nvmf_rdma_get_optimal_poll_group); 1462 CU_ADD_TEST(suite, test_spdk_nvmf_rdma_request_parse_sgl_with_md); 1463 CU_ADD_TEST(suite, test_nvmf_rdma_opts_init); 1464 CU_ADD_TEST(suite, test_nvmf_rdma_request_free_data); 1465 CU_ADD_TEST(suite, test_nvmf_rdma_update_ibv_state); 1466 CU_ADD_TEST(suite, test_nvmf_rdma_resources_create); 1467 CU_ADD_TEST(suite, test_nvmf_rdma_qpair_compare); 1468 1469 CU_basic_set_mode(CU_BRM_VERBOSE); 1470 CU_basic_run_tests(); 1471 num_failures = CU_get_number_of_failures(); 1472 CU_cleanup_registry(); 1473 return num_failures; 1474 } 1475