1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. All rights reserved. 5 * Copyright (c) 2019, 2021 Mellanox Technologies LTD. All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "spdk/stdinc.h" 35 #include "spdk_cunit.h" 36 #include "common/lib/test_env.c" 37 #include "common/lib/test_rdma.c" 38 #include "nvmf/rdma.c" 39 #include "nvmf/transport.c" 40 41 #define RDMA_UT_UNITS_IN_MAX_IO 16 42 43 struct spdk_nvmf_transport_opts g_rdma_ut_transport_opts = { 44 .max_queue_depth = SPDK_NVMF_RDMA_DEFAULT_MAX_QUEUE_DEPTH, 45 .max_qpairs_per_ctrlr = SPDK_NVMF_RDMA_DEFAULT_MAX_QPAIRS_PER_CTRLR, 46 .in_capsule_data_size = SPDK_NVMF_RDMA_DEFAULT_IN_CAPSULE_DATA_SIZE, 47 .max_io_size = (SPDK_NVMF_RDMA_MIN_IO_BUFFER_SIZE * RDMA_UT_UNITS_IN_MAX_IO), 48 .io_unit_size = SPDK_NVMF_RDMA_MIN_IO_BUFFER_SIZE, 49 .max_aq_depth = SPDK_NVMF_RDMA_DEFAULT_AQ_DEPTH, 50 .num_shared_buffers = SPDK_NVMF_RDMA_DEFAULT_NUM_SHARED_BUFFERS, 51 }; 52 53 SPDK_LOG_REGISTER_COMPONENT(nvmf) 54 DEFINE_STUB(spdk_mem_map_set_translation, int, (struct spdk_mem_map *map, uint64_t vaddr, 55 uint64_t size, uint64_t translation), 0); 56 DEFINE_STUB(spdk_mem_map_clear_translation, int, (struct spdk_mem_map *map, uint64_t vaddr, 57 uint64_t size), 0); 58 DEFINE_STUB(spdk_mem_map_alloc, struct spdk_mem_map *, (uint64_t default_translation, 59 const struct spdk_mem_map_ops *ops, void *cb_ctx), NULL); 60 DEFINE_STUB(spdk_nvmf_qpair_disconnect, int, (struct spdk_nvmf_qpair *qpair, 61 nvmf_qpair_disconnect_cb cb_fn, void *ctx), 0); 62 DEFINE_STUB(spdk_nvmf_qpair_get_listen_trid, int, 63 (struct spdk_nvmf_qpair *qpair, struct spdk_nvme_transport_id *trid), 0); 64 DEFINE_STUB_V(spdk_mem_map_free, (struct spdk_mem_map **pmap)); 65 66 DEFINE_STUB_V(spdk_nvmf_ctrlr_data_init, (struct spdk_nvmf_transport_opts *opts, 67 struct spdk_nvmf_ctrlr_data *cdata)); 68 DEFINE_STUB_V(spdk_nvmf_request_exec, (struct spdk_nvmf_request *req)); 69 DEFINE_STUB(spdk_nvmf_request_complete, int, (struct spdk_nvmf_request *req), 0); 70 DEFINE_STUB(spdk_nvme_transport_id_compare, int, (const struct spdk_nvme_transport_id *trid1, 71 const struct spdk_nvme_transport_id *trid2), 0); 72 DEFINE_STUB_V(nvmf_ctrlr_abort_aer, (struct spdk_nvmf_ctrlr *ctrlr)); 73 DEFINE_STUB(spdk_nvmf_request_get_dif_ctx, bool, (struct spdk_nvmf_request *req, 74 struct spdk_dif_ctx *dif_ctx), false); 75 DEFINE_STUB_V(spdk_nvme_trid_populate_transport, (struct spdk_nvme_transport_id *trid, 76 enum spdk_nvme_transport_type trtype)); 77 DEFINE_STUB_V(spdk_nvmf_tgt_new_qpair, (struct spdk_nvmf_tgt *tgt, struct spdk_nvmf_qpair *qpair)); 78 DEFINE_STUB(nvmf_ctrlr_abort_request, int, (struct spdk_nvmf_request *req), 0); 79 DEFINE_STUB(spdk_nvme_transport_id_adrfam_str, const char *, (enum spdk_nvmf_adrfam adrfam), NULL); 80 DEFINE_STUB(ibv_dereg_mr, int, (struct ibv_mr *mr), 0); 81 82 /* ibv_reg_mr can be a macro, need to undefine it */ 83 #ifdef ibv_reg_mr 84 #undef ibv_reg_mr 85 #endif 86 87 DEFINE_RETURN_MOCK(ibv_reg_mr, struct ibv_mr *); 88 struct ibv_mr * 89 ibv_reg_mr(struct ibv_pd *pd, void *addr, size_t length, int access) 90 { 91 HANDLE_RETURN_MOCK(ibv_reg_mr); 92 if (length > 0) { 93 return &g_rdma_mr; 94 } else { 95 return NULL; 96 } 97 } 98 99 int ibv_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, 100 int attr_mask, struct ibv_qp_init_attr *init_attr) 101 { 102 if (qp == NULL) { 103 return -1; 104 } else { 105 attr->port_num = 80; 106 107 if (qp->state == IBV_QPS_ERR) { 108 attr->qp_state = 10; 109 } else { 110 attr->qp_state = IBV_QPS_INIT; 111 } 112 113 return 0; 114 } 115 } 116 117 const char * 118 spdk_nvme_transport_id_trtype_str(enum spdk_nvme_transport_type trtype) 119 { 120 switch (trtype) { 121 case SPDK_NVME_TRANSPORT_PCIE: 122 return "PCIe"; 123 case SPDK_NVME_TRANSPORT_RDMA: 124 return "RDMA"; 125 case SPDK_NVME_TRANSPORT_FC: 126 return "FC"; 127 default: 128 return NULL; 129 } 130 } 131 132 int 133 spdk_nvme_transport_id_populate_trstring(struct spdk_nvme_transport_id *trid, const char *trstring) 134 { 135 int len, i; 136 137 if (trstring == NULL) { 138 return -EINVAL; 139 } 140 141 len = strnlen(trstring, SPDK_NVMF_TRSTRING_MAX_LEN); 142 if (len == SPDK_NVMF_TRSTRING_MAX_LEN) { 143 return -EINVAL; 144 } 145 146 /* cast official trstring to uppercase version of input. */ 147 for (i = 0; i < len; i++) { 148 trid->trstring[i] = toupper(trstring[i]); 149 } 150 return 0; 151 } 152 153 static void reset_nvmf_rdma_request(struct spdk_nvmf_rdma_request *rdma_req) 154 { 155 int i; 156 157 rdma_req->req.length = 0; 158 rdma_req->req.data_from_pool = false; 159 rdma_req->req.data = NULL; 160 rdma_req->data.wr.num_sge = 0; 161 rdma_req->data.wr.wr.rdma.remote_addr = 0; 162 rdma_req->data.wr.wr.rdma.rkey = 0; 163 rdma_req->offset = 0; 164 memset(&rdma_req->req.dif, 0, sizeof(rdma_req->req.dif)); 165 166 for (i = 0; i < SPDK_NVMF_MAX_SGL_ENTRIES; i++) { 167 rdma_req->req.iov[i].iov_base = 0; 168 rdma_req->req.iov[i].iov_len = 0; 169 rdma_req->req.buffers[i] = 0; 170 rdma_req->data.wr.sg_list[i].addr = 0; 171 rdma_req->data.wr.sg_list[i].length = 0; 172 rdma_req->data.wr.sg_list[i].lkey = 0; 173 } 174 rdma_req->req.iovcnt = 0; 175 } 176 177 static void 178 test_spdk_nvmf_rdma_request_parse_sgl(void) 179 { 180 struct spdk_nvmf_rdma_transport rtransport; 181 struct spdk_nvmf_rdma_device device; 182 struct spdk_nvmf_rdma_request rdma_req = {}; 183 struct spdk_nvmf_rdma_recv recv; 184 struct spdk_nvmf_rdma_poll_group group; 185 struct spdk_nvmf_rdma_qpair rqpair; 186 struct spdk_nvmf_rdma_poller poller; 187 union nvmf_c2h_msg cpl; 188 union nvmf_h2c_msg cmd; 189 struct spdk_nvme_sgl_descriptor *sgl; 190 struct spdk_nvmf_transport_pg_cache_buf bufs[4]; 191 struct spdk_nvme_sgl_descriptor sgl_desc[SPDK_NVMF_MAX_SGL_ENTRIES] = {{0}}; 192 struct spdk_nvmf_rdma_request_data data; 193 int rc, i; 194 uint32_t sgl_length; 195 uintptr_t aligned_buffer_address; 196 197 data.wr.sg_list = data.sgl; 198 STAILQ_INIT(&group.group.buf_cache); 199 group.group.buf_cache_size = 0; 200 group.group.buf_cache_count = 0; 201 group.group.transport = &rtransport.transport; 202 poller.group = &group; 203 rqpair.poller = &poller; 204 rqpair.max_send_sge = SPDK_NVMF_MAX_SGL_ENTRIES; 205 206 sgl = &cmd.nvme_cmd.dptr.sgl1; 207 rdma_req.recv = &recv; 208 rdma_req.req.cmd = &cmd; 209 rdma_req.req.rsp = &cpl; 210 rdma_req.data.wr.sg_list = rdma_req.data.sgl; 211 rdma_req.req.qpair = &rqpair.qpair; 212 rdma_req.req.xfer = SPDK_NVME_DATA_CONTROLLER_TO_HOST; 213 214 rtransport.transport.opts = g_rdma_ut_transport_opts; 215 rtransport.data_wr_pool = NULL; 216 rtransport.transport.data_buf_pool = NULL; 217 218 device.attr.device_cap_flags = 0; 219 sgl->keyed.key = 0xEEEE; 220 sgl->address = 0xFFFF; 221 rdma_req.recv->buf = (void *)0xDDDD; 222 223 /* Test 1: sgl type: keyed data block subtype: address */ 224 sgl->generic.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; 225 sgl->keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; 226 227 /* Part 1: simple I/O, one SGL smaller than the transport io unit size */ 228 MOCK_SET(spdk_mempool_get, (void *)0x2000); 229 reset_nvmf_rdma_request(&rdma_req); 230 sgl->keyed.length = rtransport.transport.opts.io_unit_size / 2; 231 232 device.map = (void *)0x0; 233 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 234 CU_ASSERT(rc == 0); 235 CU_ASSERT(rdma_req.req.data_from_pool == true); 236 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size / 2); 237 CU_ASSERT((uint64_t)rdma_req.req.data == 0x2000); 238 CU_ASSERT(rdma_req.data.wr.num_sge == 1); 239 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 240 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 241 CU_ASSERT((uint64_t)rdma_req.req.buffers[0] == 0x2000); 242 CU_ASSERT(rdma_req.data.wr.sg_list[0].addr == 0x2000); 243 CU_ASSERT(rdma_req.data.wr.sg_list[0].length == rtransport.transport.opts.io_unit_size / 2); 244 CU_ASSERT(rdma_req.data.wr.sg_list[0].lkey == RDMA_UT_LKEY); 245 246 /* Part 2: simple I/O, one SGL larger than the transport io unit size (equal to the max io size) */ 247 reset_nvmf_rdma_request(&rdma_req); 248 sgl->keyed.length = rtransport.transport.opts.io_unit_size * RDMA_UT_UNITS_IN_MAX_IO; 249 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 250 251 CU_ASSERT(rc == 0); 252 CU_ASSERT(rdma_req.req.data_from_pool == true); 253 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size * RDMA_UT_UNITS_IN_MAX_IO); 254 CU_ASSERT(rdma_req.data.wr.num_sge == RDMA_UT_UNITS_IN_MAX_IO); 255 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 256 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 257 for (i = 0; i < RDMA_UT_UNITS_IN_MAX_IO; i++) { 258 CU_ASSERT((uint64_t)rdma_req.req.buffers[i] == 0x2000); 259 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == 0x2000); 260 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == rtransport.transport.opts.io_unit_size); 261 CU_ASSERT(rdma_req.data.wr.sg_list[i].lkey == RDMA_UT_LKEY); 262 } 263 264 /* Part 3: simple I/O one SGL larger than the transport max io size */ 265 reset_nvmf_rdma_request(&rdma_req); 266 sgl->keyed.length = rtransport.transport.opts.max_io_size * 2; 267 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 268 269 CU_ASSERT(rc == -1); 270 271 /* Part 4: Pretend there are no buffer pools */ 272 MOCK_SET(spdk_mempool_get, NULL); 273 reset_nvmf_rdma_request(&rdma_req); 274 sgl->keyed.length = rtransport.transport.opts.io_unit_size * RDMA_UT_UNITS_IN_MAX_IO; 275 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 276 277 CU_ASSERT(rc == 0); 278 CU_ASSERT(rdma_req.req.data_from_pool == false); 279 CU_ASSERT(rdma_req.req.data == NULL); 280 CU_ASSERT(rdma_req.data.wr.num_sge == 0); 281 CU_ASSERT(rdma_req.req.buffers[0] == NULL); 282 CU_ASSERT(rdma_req.data.wr.sg_list[0].addr == 0); 283 CU_ASSERT(rdma_req.data.wr.sg_list[0].length == 0); 284 CU_ASSERT(rdma_req.data.wr.sg_list[0].lkey == 0); 285 286 rdma_req.recv->buf = (void *)0xDDDD; 287 /* Test 2: sgl type: keyed data block subtype: offset (in capsule data) */ 288 sgl->generic.type = SPDK_NVME_SGL_TYPE_DATA_BLOCK; 289 sgl->unkeyed.subtype = SPDK_NVME_SGL_SUBTYPE_OFFSET; 290 291 /* Part 1: Normal I/O smaller than in capsule data size no offset */ 292 reset_nvmf_rdma_request(&rdma_req); 293 sgl->address = 0; 294 sgl->unkeyed.length = rtransport.transport.opts.in_capsule_data_size; 295 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 296 297 CU_ASSERT(rc == 0); 298 CU_ASSERT(rdma_req.req.data == (void *)0xDDDD); 299 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.in_capsule_data_size); 300 CU_ASSERT(rdma_req.req.data_from_pool == false); 301 302 /* Part 2: I/O offset + length too large */ 303 reset_nvmf_rdma_request(&rdma_req); 304 sgl->address = rtransport.transport.opts.in_capsule_data_size; 305 sgl->unkeyed.length = rtransport.transport.opts.in_capsule_data_size; 306 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 307 308 CU_ASSERT(rc == -1); 309 310 /* Part 3: I/O too large */ 311 reset_nvmf_rdma_request(&rdma_req); 312 sgl->address = 0; 313 sgl->unkeyed.length = rtransport.transport.opts.in_capsule_data_size * 2; 314 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 315 316 CU_ASSERT(rc == -1); 317 318 /* Test 3: Multi SGL */ 319 sgl->generic.type = SPDK_NVME_SGL_TYPE_LAST_SEGMENT; 320 sgl->unkeyed.subtype = SPDK_NVME_SGL_SUBTYPE_OFFSET; 321 sgl->address = 0; 322 rdma_req.recv->buf = (void *)&sgl_desc; 323 MOCK_SET(spdk_mempool_get, &data); 324 325 /* part 1: 2 segments each with 1 wr. */ 326 reset_nvmf_rdma_request(&rdma_req); 327 sgl->unkeyed.length = 2 * sizeof(struct spdk_nvme_sgl_descriptor); 328 for (i = 0; i < 2; i++) { 329 sgl_desc[i].keyed.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; 330 sgl_desc[i].keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; 331 sgl_desc[i].keyed.length = rtransport.transport.opts.io_unit_size; 332 sgl_desc[i].address = 0x4000 + i * rtransport.transport.opts.io_unit_size; 333 sgl_desc[i].keyed.key = 0x44; 334 } 335 336 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 337 338 CU_ASSERT(rc == 0); 339 CU_ASSERT(rdma_req.req.data_from_pool == true); 340 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size * 2); 341 CU_ASSERT(rdma_req.data.wr.num_sge == 1); 342 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0x44); 343 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0x4000); 344 CU_ASSERT(rdma_req.data.wr.next == &data.wr); 345 CU_ASSERT(data.wr.wr.rdma.rkey == 0x44); 346 CU_ASSERT(data.wr.wr.rdma.remote_addr == 0x4000 + rtransport.transport.opts.io_unit_size); 347 CU_ASSERT(data.wr.num_sge == 1); 348 CU_ASSERT(data.wr.next == &rdma_req.rsp.wr); 349 350 /* part 2: 2 segments, each with 1 wr containing 8 sge_elements */ 351 reset_nvmf_rdma_request(&rdma_req); 352 sgl->unkeyed.length = 2 * sizeof(struct spdk_nvme_sgl_descriptor); 353 for (i = 0; i < 2; i++) { 354 sgl_desc[i].keyed.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; 355 sgl_desc[i].keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; 356 sgl_desc[i].keyed.length = rtransport.transport.opts.io_unit_size * 8; 357 sgl_desc[i].address = 0x4000 + i * 8 * rtransport.transport.opts.io_unit_size; 358 sgl_desc[i].keyed.key = 0x44; 359 } 360 361 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 362 363 CU_ASSERT(rc == 0); 364 CU_ASSERT(rdma_req.req.data_from_pool == true); 365 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size * 16); 366 CU_ASSERT(rdma_req.req.iovcnt == 16); 367 CU_ASSERT(rdma_req.data.wr.num_sge == 8); 368 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0x44); 369 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0x4000); 370 CU_ASSERT(rdma_req.data.wr.next == &data.wr); 371 CU_ASSERT(data.wr.wr.rdma.rkey == 0x44); 372 CU_ASSERT(data.wr.wr.rdma.remote_addr == 0x4000 + rtransport.transport.opts.io_unit_size * 8); 373 CU_ASSERT(data.wr.num_sge == 8); 374 CU_ASSERT(data.wr.next == &rdma_req.rsp.wr); 375 376 /* part 3: 2 segments, one very large, one very small */ 377 reset_nvmf_rdma_request(&rdma_req); 378 for (i = 0; i < 2; i++) { 379 sgl_desc[i].keyed.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; 380 sgl_desc[i].keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; 381 sgl_desc[i].keyed.key = 0x44; 382 } 383 384 sgl_desc[0].keyed.length = rtransport.transport.opts.io_unit_size * 15 + 385 rtransport.transport.opts.io_unit_size / 2; 386 sgl_desc[0].address = 0x4000; 387 sgl_desc[1].keyed.length = rtransport.transport.opts.io_unit_size / 2; 388 sgl_desc[1].address = 0x4000 + rtransport.transport.opts.io_unit_size * 15 + 389 rtransport.transport.opts.io_unit_size / 2; 390 391 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 392 393 CU_ASSERT(rc == 0); 394 CU_ASSERT(rdma_req.req.data_from_pool == true); 395 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size * 16); 396 CU_ASSERT(rdma_req.req.iovcnt == 16); 397 CU_ASSERT(rdma_req.data.wr.num_sge == 16); 398 for (i = 0; i < 15; i++) { 399 CU_ASSERT(rdma_req.data.sgl[i].length == rtransport.transport.opts.io_unit_size); 400 } 401 CU_ASSERT(rdma_req.data.sgl[15].length == rtransport.transport.opts.io_unit_size / 2); 402 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0x44); 403 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0x4000); 404 CU_ASSERT(rdma_req.data.wr.next == &data.wr); 405 CU_ASSERT(data.wr.wr.rdma.rkey == 0x44); 406 CU_ASSERT(data.wr.wr.rdma.remote_addr == 0x4000 + rtransport.transport.opts.io_unit_size * 15 + 407 rtransport.transport.opts.io_unit_size / 2); 408 CU_ASSERT(data.sgl[0].length == rtransport.transport.opts.io_unit_size / 2); 409 CU_ASSERT(data.wr.num_sge == 1); 410 CU_ASSERT(data.wr.next == &rdma_req.rsp.wr); 411 412 /* part 4: 2 SGL descriptors, each length is transport buffer / 2 413 * 1 transport buffers should be allocated */ 414 reset_nvmf_rdma_request(&rdma_req); 415 aligned_buffer_address = ((uintptr_t)(&data) + NVMF_DATA_BUFFER_MASK) & ~NVMF_DATA_BUFFER_MASK; 416 sgl->unkeyed.length = 2 * sizeof(struct spdk_nvme_sgl_descriptor); 417 sgl_length = rtransport.transport.opts.io_unit_size / 2; 418 for (i = 0; i < 2; i++) { 419 sgl_desc[i].keyed.length = sgl_length; 420 sgl_desc[i].address = 0x4000 + i * sgl_length; 421 } 422 423 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 424 425 CU_ASSERT(rc == 0); 426 CU_ASSERT(rdma_req.req.data_from_pool == true); 427 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size); 428 CU_ASSERT(rdma_req.req.iovcnt == 1); 429 430 CU_ASSERT(rdma_req.data.sgl[0].length == sgl_length); 431 /* We mocked mempool_get to return address of data variable. Mempool is used 432 * to get both additional WRs and data buffers, so data points to &data */ 433 CU_ASSERT(rdma_req.data.sgl[0].addr == aligned_buffer_address); 434 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0x44); 435 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0x4000); 436 CU_ASSERT(rdma_req.data.wr.num_sge == 1); 437 CU_ASSERT(rdma_req.data.wr.next == &data.wr); 438 439 CU_ASSERT(data.wr.wr.rdma.rkey == 0x44); 440 CU_ASSERT(data.wr.wr.rdma.remote_addr == 0x4000 + sgl_length); 441 CU_ASSERT(data.sgl[0].length == sgl_length); 442 CU_ASSERT(data.sgl[0].addr == aligned_buffer_address + sgl_length); 443 CU_ASSERT(data.wr.num_sge == 1); 444 445 /* Test 4: use PG buffer cache */ 446 sgl->generic.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; 447 sgl->keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; 448 sgl->address = 0xFFFF; 449 rdma_req.recv->buf = (void *)0xDDDD; 450 sgl->keyed.key = 0xEEEE; 451 452 for (i = 0; i < 4; i++) { 453 STAILQ_INSERT_TAIL(&group.group.buf_cache, &bufs[i], link); 454 } 455 456 /* part 1: use the four buffers from the pg cache */ 457 group.group.buf_cache_size = 4; 458 group.group.buf_cache_count = 4; 459 MOCK_SET(spdk_mempool_get, (void *)0x2000); 460 reset_nvmf_rdma_request(&rdma_req); 461 sgl->keyed.length = rtransport.transport.opts.io_unit_size * 4; 462 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 463 464 SPDK_CU_ASSERT_FATAL(rc == 0); 465 CU_ASSERT(rdma_req.req.data_from_pool == true); 466 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size * 4); 467 CU_ASSERT((uint64_t)rdma_req.req.data == (((uint64_t)&bufs[0] + NVMF_DATA_BUFFER_MASK) & 468 ~NVMF_DATA_BUFFER_MASK)); 469 CU_ASSERT(rdma_req.data.wr.num_sge == 4); 470 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 471 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 472 CU_ASSERT(group.group.buf_cache_count == 0); 473 CU_ASSERT(STAILQ_EMPTY(&group.group.buf_cache)); 474 for (i = 0; i < 4; i++) { 475 CU_ASSERT((uint64_t)rdma_req.req.buffers[i] == (uint64_t)&bufs[i]); 476 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == (((uint64_t)&bufs[i] + NVMF_DATA_BUFFER_MASK) & 477 ~NVMF_DATA_BUFFER_MASK)); 478 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == rtransport.transport.opts.io_unit_size); 479 } 480 481 /* part 2: now that we have used the buffers from the cache, try again. We should get mempool buffers. */ 482 reset_nvmf_rdma_request(&rdma_req); 483 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 484 485 SPDK_CU_ASSERT_FATAL(rc == 0); 486 CU_ASSERT(rdma_req.req.data_from_pool == true); 487 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size * 4); 488 CU_ASSERT((uint64_t)rdma_req.req.data == 0x2000); 489 CU_ASSERT(rdma_req.data.wr.num_sge == 4); 490 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 491 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 492 CU_ASSERT(group.group.buf_cache_count == 0); 493 CU_ASSERT(STAILQ_EMPTY(&group.group.buf_cache)); 494 for (i = 0; i < 4; i++) { 495 CU_ASSERT((uint64_t)rdma_req.req.buffers[i] == 0x2000); 496 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == 0x2000); 497 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == rtransport.transport.opts.io_unit_size); 498 CU_ASSERT(group.group.buf_cache_count == 0); 499 } 500 501 /* part 3: half and half */ 502 group.group.buf_cache_count = 2; 503 504 for (i = 0; i < 2; i++) { 505 STAILQ_INSERT_TAIL(&group.group.buf_cache, &bufs[i], link); 506 } 507 reset_nvmf_rdma_request(&rdma_req); 508 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 509 510 SPDK_CU_ASSERT_FATAL(rc == 0); 511 CU_ASSERT(rdma_req.req.data_from_pool == true); 512 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size * 4); 513 CU_ASSERT((uint64_t)rdma_req.req.data == (((uint64_t)&bufs[0] + NVMF_DATA_BUFFER_MASK) & 514 ~NVMF_DATA_BUFFER_MASK)); 515 CU_ASSERT(rdma_req.data.wr.num_sge == 4); 516 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 517 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 518 CU_ASSERT(group.group.buf_cache_count == 0); 519 for (i = 0; i < 2; i++) { 520 CU_ASSERT((uint64_t)rdma_req.req.buffers[i] == (uint64_t)&bufs[i]); 521 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == (((uint64_t)&bufs[i] + NVMF_DATA_BUFFER_MASK) & 522 ~NVMF_DATA_BUFFER_MASK)); 523 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == rtransport.transport.opts.io_unit_size); 524 } 525 for (i = 2; i < 4; i++) { 526 CU_ASSERT((uint64_t)rdma_req.req.buffers[i] == 0x2000); 527 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == 0x2000); 528 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == rtransport.transport.opts.io_unit_size); 529 } 530 531 reset_nvmf_rdma_request(&rdma_req); 532 } 533 534 static struct spdk_nvmf_rdma_recv * 535 create_recv(struct spdk_nvmf_rdma_qpair *rqpair, enum spdk_nvme_nvm_opcode opc) 536 { 537 struct spdk_nvmf_rdma_recv *rdma_recv; 538 union nvmf_h2c_msg *cmd; 539 struct spdk_nvme_sgl_descriptor *sgl; 540 541 rdma_recv = calloc(1, sizeof(*rdma_recv)); 542 rdma_recv->qpair = rqpair; 543 cmd = calloc(1, sizeof(*cmd)); 544 rdma_recv->sgl[0].addr = (uintptr_t)cmd; 545 cmd->nvme_cmd.opc = opc; 546 sgl = &cmd->nvme_cmd.dptr.sgl1; 547 sgl->keyed.key = 0xEEEE; 548 sgl->address = 0xFFFF; 549 sgl->keyed.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; 550 sgl->keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; 551 sgl->keyed.length = 1; 552 553 return rdma_recv; 554 } 555 556 static void 557 free_recv(struct spdk_nvmf_rdma_recv *rdma_recv) 558 { 559 free((void *)rdma_recv->sgl[0].addr); 560 free(rdma_recv); 561 } 562 563 static struct spdk_nvmf_rdma_request * 564 create_req(struct spdk_nvmf_rdma_qpair *rqpair, 565 struct spdk_nvmf_rdma_recv *rdma_recv) 566 { 567 struct spdk_nvmf_rdma_request *rdma_req; 568 union nvmf_c2h_msg *cpl; 569 570 rdma_req = calloc(1, sizeof(*rdma_req)); 571 rdma_req->recv = rdma_recv; 572 rdma_req->req.qpair = &rqpair->qpair; 573 rdma_req->state = RDMA_REQUEST_STATE_NEW; 574 rdma_req->data.wr.wr_id = (uintptr_t)&rdma_req->data.rdma_wr; 575 rdma_req->data.wr.sg_list = rdma_req->data.sgl; 576 cpl = calloc(1, sizeof(*cpl)); 577 rdma_req->rsp.sgl[0].addr = (uintptr_t)cpl; 578 rdma_req->req.rsp = cpl; 579 580 return rdma_req; 581 } 582 583 static void 584 free_req(struct spdk_nvmf_rdma_request *rdma_req) 585 { 586 free((void *)rdma_req->rsp.sgl[0].addr); 587 free(rdma_req); 588 } 589 590 static void 591 qpair_reset(struct spdk_nvmf_rdma_qpair *rqpair, 592 struct spdk_nvmf_rdma_poller *poller, 593 struct spdk_nvmf_rdma_device *device, 594 struct spdk_nvmf_rdma_resources *resources, 595 struct spdk_nvmf_transport *transport) 596 { 597 memset(rqpair, 0, sizeof(*rqpair)); 598 STAILQ_INIT(&rqpair->pending_rdma_write_queue); 599 STAILQ_INIT(&rqpair->pending_rdma_read_queue); 600 rqpair->poller = poller; 601 rqpair->device = device; 602 rqpair->resources = resources; 603 rqpair->qpair.qid = 1; 604 rqpair->ibv_state = IBV_QPS_RTS; 605 rqpair->qpair.state = SPDK_NVMF_QPAIR_ACTIVE; 606 rqpair->max_send_sge = SPDK_NVMF_MAX_SGL_ENTRIES; 607 rqpair->max_send_depth = 16; 608 rqpair->max_read_depth = 16; 609 rqpair->qpair.transport = transport; 610 } 611 612 static void 613 poller_reset(struct spdk_nvmf_rdma_poller *poller, 614 struct spdk_nvmf_rdma_poll_group *group) 615 { 616 memset(poller, 0, sizeof(*poller)); 617 STAILQ_INIT(&poller->qpairs_pending_recv); 618 STAILQ_INIT(&poller->qpairs_pending_send); 619 poller->group = group; 620 } 621 622 static void 623 test_spdk_nvmf_rdma_request_process(void) 624 { 625 struct spdk_nvmf_rdma_transport rtransport = {}; 626 struct spdk_nvmf_rdma_poll_group group = {}; 627 struct spdk_nvmf_rdma_poller poller = {}; 628 struct spdk_nvmf_rdma_device device = {}; 629 struct spdk_nvmf_rdma_resources resources = {}; 630 struct spdk_nvmf_rdma_qpair rqpair = {}; 631 struct spdk_nvmf_rdma_recv *rdma_recv; 632 struct spdk_nvmf_rdma_request *rdma_req; 633 bool progress; 634 635 STAILQ_INIT(&group.group.buf_cache); 636 STAILQ_INIT(&group.group.pending_buf_queue); 637 group.group.buf_cache_size = 0; 638 group.group.buf_cache_count = 0; 639 poller_reset(&poller, &group); 640 qpair_reset(&rqpair, &poller, &device, &resources, &rtransport.transport); 641 642 rtransport.transport.opts = g_rdma_ut_transport_opts; 643 rtransport.transport.data_buf_pool = spdk_mempool_create("test_data_pool", 16, 128, 0, 0); 644 rtransport.data_wr_pool = spdk_mempool_create("test_wr_pool", 128, 645 sizeof(struct spdk_nvmf_rdma_request_data), 646 0, 0); 647 MOCK_CLEAR(spdk_mempool_get); 648 649 device.attr.device_cap_flags = 0; 650 device.map = (void *)0x0; 651 652 /* Test 1: single SGL READ request */ 653 rdma_recv = create_recv(&rqpair, SPDK_NVME_OPC_READ); 654 rdma_req = create_req(&rqpair, rdma_recv); 655 rqpair.current_recv_depth = 1; 656 /* NEW -> EXECUTING */ 657 progress = nvmf_rdma_request_process(&rtransport, rdma_req); 658 CU_ASSERT(progress == true); 659 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_EXECUTING); 660 CU_ASSERT(rdma_req->req.xfer == SPDK_NVME_DATA_CONTROLLER_TO_HOST); 661 /* EXECUTED -> TRANSFERRING_C2H */ 662 rdma_req->state = RDMA_REQUEST_STATE_EXECUTED; 663 progress = nvmf_rdma_request_process(&rtransport, rdma_req); 664 CU_ASSERT(progress == true); 665 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_TRANSFERRING_CONTROLLER_TO_HOST); 666 CU_ASSERT(rdma_req->recv == NULL); 667 /* COMPLETED -> FREE */ 668 rdma_req->state = RDMA_REQUEST_STATE_COMPLETED; 669 progress = nvmf_rdma_request_process(&rtransport, rdma_req); 670 CU_ASSERT(progress == true); 671 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_FREE); 672 673 free_recv(rdma_recv); 674 free_req(rdma_req); 675 poller_reset(&poller, &group); 676 qpair_reset(&rqpair, &poller, &device, &resources, &rtransport.transport); 677 678 /* Test 2: single SGL WRITE request */ 679 rdma_recv = create_recv(&rqpair, SPDK_NVME_OPC_WRITE); 680 rdma_req = create_req(&rqpair, rdma_recv); 681 rqpair.current_recv_depth = 1; 682 /* NEW -> TRANSFERRING_H2C */ 683 progress = nvmf_rdma_request_process(&rtransport, rdma_req); 684 CU_ASSERT(progress == true); 685 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_TRANSFERRING_HOST_TO_CONTROLLER); 686 CU_ASSERT(rdma_req->req.xfer == SPDK_NVME_DATA_HOST_TO_CONTROLLER); 687 STAILQ_INIT(&poller.qpairs_pending_send); 688 /* READY_TO_EXECUTE -> EXECUTING */ 689 rdma_req->state = RDMA_REQUEST_STATE_READY_TO_EXECUTE; 690 progress = nvmf_rdma_request_process(&rtransport, rdma_req); 691 CU_ASSERT(progress == true); 692 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_EXECUTING); 693 /* EXECUTED -> COMPLETING */ 694 rdma_req->state = RDMA_REQUEST_STATE_EXECUTED; 695 progress = nvmf_rdma_request_process(&rtransport, rdma_req); 696 CU_ASSERT(progress == true); 697 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_COMPLETING); 698 CU_ASSERT(rdma_req->recv == NULL); 699 /* COMPLETED -> FREE */ 700 rdma_req->state = RDMA_REQUEST_STATE_COMPLETED; 701 progress = nvmf_rdma_request_process(&rtransport, rdma_req); 702 CU_ASSERT(progress == true); 703 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_FREE); 704 705 free_recv(rdma_recv); 706 free_req(rdma_req); 707 poller_reset(&poller, &group); 708 qpair_reset(&rqpair, &poller, &device, &resources, &rtransport.transport); 709 710 /* Test 3: WRITE+WRITE ibv_send batching */ 711 { 712 struct spdk_nvmf_rdma_recv *recv1, *recv2; 713 struct spdk_nvmf_rdma_request *req1, *req2; 714 recv1 = create_recv(&rqpair, SPDK_NVME_OPC_WRITE); 715 req1 = create_req(&rqpair, recv1); 716 recv2 = create_recv(&rqpair, SPDK_NVME_OPC_WRITE); 717 req2 = create_req(&rqpair, recv2); 718 719 /* WRITE 1: NEW -> TRANSFERRING_H2C */ 720 rqpair.current_recv_depth = 1; 721 nvmf_rdma_request_process(&rtransport, req1); 722 CU_ASSERT(req1->state == RDMA_REQUEST_STATE_TRANSFERRING_HOST_TO_CONTROLLER); 723 724 /* WRITE 2: NEW -> TRANSFERRING_H2C */ 725 rqpair.current_recv_depth = 2; 726 nvmf_rdma_request_process(&rtransport, req2); 727 CU_ASSERT(req2->state == RDMA_REQUEST_STATE_TRANSFERRING_HOST_TO_CONTROLLER); 728 729 STAILQ_INIT(&poller.qpairs_pending_send); 730 731 /* WRITE 1 completes before WRITE 2 has finished RDMA reading */ 732 /* WRITE 1: READY_TO_EXECUTE -> EXECUTING */ 733 req1->state = RDMA_REQUEST_STATE_READY_TO_EXECUTE; 734 nvmf_rdma_request_process(&rtransport, req1); 735 CU_ASSERT(req1->state == RDMA_REQUEST_STATE_EXECUTING); 736 /* WRITE 1: EXECUTED -> COMPLETING */ 737 req1->state = RDMA_REQUEST_STATE_EXECUTED; 738 nvmf_rdma_request_process(&rtransport, req1); 739 CU_ASSERT(req1->state == RDMA_REQUEST_STATE_COMPLETING); 740 STAILQ_INIT(&poller.qpairs_pending_send); 741 /* WRITE 1: COMPLETED -> FREE */ 742 req1->state = RDMA_REQUEST_STATE_COMPLETED; 743 nvmf_rdma_request_process(&rtransport, req1); 744 CU_ASSERT(req1->state == RDMA_REQUEST_STATE_FREE); 745 746 /* Now WRITE 2 has finished reading and completes */ 747 /* WRITE 2: COMPLETED -> FREE */ 748 /* WRITE 2: READY_TO_EXECUTE -> EXECUTING */ 749 req2->state = RDMA_REQUEST_STATE_READY_TO_EXECUTE; 750 nvmf_rdma_request_process(&rtransport, req2); 751 CU_ASSERT(req2->state == RDMA_REQUEST_STATE_EXECUTING); 752 /* WRITE 1: EXECUTED -> COMPLETING */ 753 req2->state = RDMA_REQUEST_STATE_EXECUTED; 754 nvmf_rdma_request_process(&rtransport, req2); 755 CU_ASSERT(req2->state == RDMA_REQUEST_STATE_COMPLETING); 756 STAILQ_INIT(&poller.qpairs_pending_send); 757 /* WRITE 1: COMPLETED -> FREE */ 758 req2->state = RDMA_REQUEST_STATE_COMPLETED; 759 nvmf_rdma_request_process(&rtransport, req2); 760 CU_ASSERT(req2->state == RDMA_REQUEST_STATE_FREE); 761 762 free_recv(recv1); 763 free_req(req1); 764 free_recv(recv2); 765 free_req(req2); 766 poller_reset(&poller, &group); 767 qpair_reset(&rqpair, &poller, &device, &resources, &rtransport.transport); 768 } 769 770 /* Test 4, invalid command, check xfer type */ 771 { 772 struct spdk_nvmf_rdma_recv *rdma_recv_inv; 773 struct spdk_nvmf_rdma_request *rdma_req_inv; 774 /* construct an opcode that specifies BIDIRECTIONAL transfer */ 775 uint8_t opc = 0x10 | SPDK_NVME_DATA_BIDIRECTIONAL; 776 777 rdma_recv_inv = create_recv(&rqpair, opc); 778 rdma_req_inv = create_req(&rqpair, rdma_recv_inv); 779 780 /* NEW -> RDMA_REQUEST_STATE_COMPLETING */ 781 rqpair.current_recv_depth = 1; 782 progress = nvmf_rdma_request_process(&rtransport, rdma_req_inv); 783 CU_ASSERT(progress == true); 784 CU_ASSERT(rdma_req_inv->state == RDMA_REQUEST_STATE_COMPLETING); 785 CU_ASSERT(rdma_req_inv->req.rsp->nvme_cpl.status.sct == SPDK_NVME_SCT_GENERIC); 786 CU_ASSERT(rdma_req_inv->req.rsp->nvme_cpl.status.sc == SPDK_NVME_SC_INVALID_OPCODE); 787 788 /* RDMA_REQUEST_STATE_COMPLETED -> FREE */ 789 rdma_req_inv->state = RDMA_REQUEST_STATE_COMPLETED; 790 nvmf_rdma_request_process(&rtransport, rdma_req_inv); 791 CU_ASSERT(rdma_req_inv->state == RDMA_REQUEST_STATE_FREE); 792 793 free_recv(rdma_recv_inv); 794 free_req(rdma_req_inv); 795 poller_reset(&poller, &group); 796 qpair_reset(&rqpair, &poller, &device, &resources, &rtransport.transport); 797 } 798 799 spdk_mempool_free(rtransport.transport.data_buf_pool); 800 spdk_mempool_free(rtransport.data_wr_pool); 801 } 802 803 #define TEST_GROUPS_COUNT 5 804 static void 805 test_nvmf_rdma_get_optimal_poll_group(void) 806 { 807 struct spdk_nvmf_rdma_transport rtransport = {}; 808 struct spdk_nvmf_transport *transport = &rtransport.transport; 809 struct spdk_nvmf_rdma_qpair rqpair = {}; 810 struct spdk_nvmf_transport_poll_group *groups[TEST_GROUPS_COUNT]; 811 struct spdk_nvmf_rdma_poll_group *rgroups[TEST_GROUPS_COUNT]; 812 struct spdk_nvmf_transport_poll_group *result; 813 uint32_t i; 814 815 rqpair.qpair.transport = transport; 816 pthread_mutex_init(&rtransport.lock, NULL); 817 TAILQ_INIT(&rtransport.poll_groups); 818 819 for (i = 0; i < TEST_GROUPS_COUNT; i++) { 820 groups[i] = nvmf_rdma_poll_group_create(transport, NULL); 821 CU_ASSERT(groups[i] != NULL); 822 rgroups[i] = SPDK_CONTAINEROF(groups[i], struct spdk_nvmf_rdma_poll_group, group); 823 groups[i]->transport = transport; 824 } 825 CU_ASSERT(rtransport.conn_sched.next_admin_pg == rgroups[0]); 826 CU_ASSERT(rtransport.conn_sched.next_io_pg == rgroups[0]); 827 828 /* Emulate connection of %TEST_GROUPS_COUNT% initiators - each creates 1 admin and 1 io qp */ 829 for (i = 0; i < TEST_GROUPS_COUNT; i++) { 830 rqpair.qpair.qid = 0; 831 result = nvmf_rdma_get_optimal_poll_group(&rqpair.qpair); 832 CU_ASSERT(result == groups[i]); 833 CU_ASSERT(rtransport.conn_sched.next_admin_pg == rgroups[(i + 1) % TEST_GROUPS_COUNT]); 834 CU_ASSERT(rtransport.conn_sched.next_io_pg == rgroups[i]); 835 836 rqpair.qpair.qid = 1; 837 result = nvmf_rdma_get_optimal_poll_group(&rqpair.qpair); 838 CU_ASSERT(result == groups[i]); 839 CU_ASSERT(rtransport.conn_sched.next_admin_pg == rgroups[(i + 1) % TEST_GROUPS_COUNT]); 840 CU_ASSERT(rtransport.conn_sched.next_io_pg == rgroups[(i + 1) % TEST_GROUPS_COUNT]); 841 } 842 /* wrap around, admin/io pg point to the first pg 843 Destroy all poll groups except of the last one */ 844 for (i = 0; i < TEST_GROUPS_COUNT - 1; i++) { 845 nvmf_rdma_poll_group_destroy(groups[i]); 846 CU_ASSERT(rtransport.conn_sched.next_admin_pg == rgroups[i + 1]); 847 CU_ASSERT(rtransport.conn_sched.next_io_pg == rgroups[i + 1]); 848 } 849 850 CU_ASSERT(rtransport.conn_sched.next_admin_pg == rgroups[TEST_GROUPS_COUNT - 1]); 851 CU_ASSERT(rtransport.conn_sched.next_io_pg == rgroups[TEST_GROUPS_COUNT - 1]); 852 853 /* Check that pointers to the next admin/io poll groups are not changed */ 854 rqpair.qpair.qid = 0; 855 result = nvmf_rdma_get_optimal_poll_group(&rqpair.qpair); 856 CU_ASSERT(result == groups[TEST_GROUPS_COUNT - 1]); 857 CU_ASSERT(rtransport.conn_sched.next_admin_pg == rgroups[TEST_GROUPS_COUNT - 1]); 858 CU_ASSERT(rtransport.conn_sched.next_io_pg == rgroups[TEST_GROUPS_COUNT - 1]); 859 860 rqpair.qpair.qid = 1; 861 result = nvmf_rdma_get_optimal_poll_group(&rqpair.qpair); 862 CU_ASSERT(result == groups[TEST_GROUPS_COUNT - 1]); 863 CU_ASSERT(rtransport.conn_sched.next_admin_pg == rgroups[TEST_GROUPS_COUNT - 1]); 864 CU_ASSERT(rtransport.conn_sched.next_io_pg == rgroups[TEST_GROUPS_COUNT - 1]); 865 866 /* Remove the last poll group, check that pointers are NULL */ 867 nvmf_rdma_poll_group_destroy(groups[TEST_GROUPS_COUNT - 1]); 868 CU_ASSERT(rtransport.conn_sched.next_admin_pg == NULL); 869 CU_ASSERT(rtransport.conn_sched.next_io_pg == NULL); 870 871 /* Request optimal poll group, result must be NULL */ 872 rqpair.qpair.qid = 0; 873 result = nvmf_rdma_get_optimal_poll_group(&rqpair.qpair); 874 CU_ASSERT(result == NULL); 875 876 rqpair.qpair.qid = 1; 877 result = nvmf_rdma_get_optimal_poll_group(&rqpair.qpair); 878 CU_ASSERT(result == NULL); 879 880 pthread_mutex_destroy(&rtransport.lock); 881 } 882 #undef TEST_GROUPS_COUNT 883 884 static void 885 test_spdk_nvmf_rdma_request_parse_sgl_with_md(void) 886 { 887 struct spdk_nvmf_rdma_transport rtransport; 888 struct spdk_nvmf_rdma_device device; 889 struct spdk_nvmf_rdma_request rdma_req = {}; 890 struct spdk_nvmf_rdma_recv recv; 891 struct spdk_nvmf_rdma_poll_group group; 892 struct spdk_nvmf_rdma_qpair rqpair; 893 struct spdk_nvmf_rdma_poller poller; 894 union nvmf_c2h_msg cpl; 895 union nvmf_h2c_msg cmd; 896 struct spdk_nvme_sgl_descriptor *sgl; 897 struct spdk_nvme_sgl_descriptor sgl_desc[SPDK_NVMF_MAX_SGL_ENTRIES] = {{0}}; 898 char data_buffer[8192]; 899 struct spdk_nvmf_rdma_request_data *data = (struct spdk_nvmf_rdma_request_data *)data_buffer; 900 char data2_buffer[8192]; 901 struct spdk_nvmf_rdma_request_data *data2 = (struct spdk_nvmf_rdma_request_data *)data2_buffer; 902 const uint32_t data_bs = 512; 903 const uint32_t md_size = 8; 904 int rc, i; 905 void *aligned_buffer; 906 907 data->wr.sg_list = data->sgl; 908 STAILQ_INIT(&group.group.buf_cache); 909 group.group.buf_cache_size = 0; 910 group.group.buf_cache_count = 0; 911 group.group.transport = &rtransport.transport; 912 poller.group = &group; 913 rqpair.poller = &poller; 914 rqpair.max_send_sge = SPDK_NVMF_MAX_SGL_ENTRIES; 915 916 sgl = &cmd.nvme_cmd.dptr.sgl1; 917 rdma_req.recv = &recv; 918 rdma_req.req.cmd = &cmd; 919 rdma_req.req.rsp = &cpl; 920 rdma_req.data.wr.sg_list = rdma_req.data.sgl; 921 rdma_req.req.qpair = &rqpair.qpair; 922 rdma_req.req.xfer = SPDK_NVME_DATA_CONTROLLER_TO_HOST; 923 924 rtransport.transport.opts = g_rdma_ut_transport_opts; 925 rtransport.data_wr_pool = NULL; 926 rtransport.transport.data_buf_pool = NULL; 927 928 device.attr.device_cap_flags = 0; 929 device.map = NULL; 930 sgl->keyed.key = 0xEEEE; 931 sgl->address = 0xFFFF; 932 rdma_req.recv->buf = (void *)0xDDDD; 933 934 /* Test 1: sgl type: keyed data block subtype: address */ 935 sgl->generic.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; 936 sgl->keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; 937 938 /* Part 1: simple I/O, one SGL smaller than the transport io unit size, block size 512 */ 939 MOCK_SET(spdk_mempool_get, (void *)0x2000); 940 reset_nvmf_rdma_request(&rdma_req); 941 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 942 SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 943 0, 0, 0, 0, 0); 944 rdma_req.req.dif_enabled = true; 945 rtransport.transport.opts.io_unit_size = data_bs * 8; 946 sgl->keyed.length = data_bs * 4; 947 948 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 949 950 CU_ASSERT(rc == 0); 951 CU_ASSERT(rdma_req.req.data_from_pool == true); 952 CU_ASSERT(rdma_req.req.length == data_bs * 4); 953 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 954 CU_ASSERT(rdma_req.req.dif.elba_length == (data_bs + md_size) * 4); 955 CU_ASSERT((uint64_t)rdma_req.req.data == 0x2000); 956 CU_ASSERT(rdma_req.data.wr.num_sge == 4); 957 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 958 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 959 CU_ASSERT((uint64_t)rdma_req.req.buffers[0] == 0x2000); 960 961 for (i = 0; i < 4; ++i) { 962 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == 0x2000 + i * (data_bs + md_size)); 963 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == data_bs); 964 CU_ASSERT(rdma_req.data.wr.sg_list[i].lkey == RDMA_UT_LKEY); 965 } 966 967 /* Part 2: simple I/O, one SGL equal to io unit size, io_unit_size is not aligned with md_size, 968 block size 512 */ 969 MOCK_SET(spdk_mempool_get, (void *)0x2000); 970 reset_nvmf_rdma_request(&rdma_req); 971 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 972 SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 973 0, 0, 0, 0, 0); 974 rdma_req.req.dif_enabled = true; 975 rtransport.transport.opts.io_unit_size = data_bs * 4; 976 sgl->keyed.length = data_bs * 4; 977 978 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 979 980 CU_ASSERT(rc == 0); 981 CU_ASSERT(rdma_req.req.data_from_pool == true); 982 CU_ASSERT(rdma_req.req.length == data_bs * 4); 983 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 984 CU_ASSERT(rdma_req.req.dif.elba_length == (data_bs + md_size) * 4); 985 CU_ASSERT((uint64_t)rdma_req.req.data == 0x2000); 986 CU_ASSERT(rdma_req.data.wr.num_sge == 5); 987 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 988 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 989 CU_ASSERT((uint64_t)rdma_req.req.buffers[0] == 0x2000); 990 991 for (i = 0; i < 3; ++i) { 992 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == 0x2000 + i * (data_bs + md_size)); 993 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == data_bs); 994 CU_ASSERT(rdma_req.data.wr.sg_list[i].lkey == RDMA_UT_LKEY); 995 } 996 CU_ASSERT(rdma_req.data.wr.sg_list[3].addr == 0x2000 + 3 * (data_bs + md_size)); 997 CU_ASSERT(rdma_req.data.wr.sg_list[3].length == 488); 998 CU_ASSERT(rdma_req.data.wr.sg_list[3].lkey == RDMA_UT_LKEY); 999 1000 /* 2nd buffer consumed */ 1001 CU_ASSERT(rdma_req.data.wr.sg_list[4].addr == 0x2000); 1002 CU_ASSERT(rdma_req.data.wr.sg_list[4].length == 24); 1003 CU_ASSERT(rdma_req.data.wr.sg_list[4].lkey == RDMA_UT_LKEY); 1004 1005 /* Part 3: simple I/O, one SGL equal io unit size, io_unit_size is equal to block size 512 bytes */ 1006 MOCK_SET(spdk_mempool_get, (void *)0x2000); 1007 reset_nvmf_rdma_request(&rdma_req); 1008 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 1009 SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 1010 0, 0, 0, 0, 0); 1011 rdma_req.req.dif_enabled = true; 1012 rtransport.transport.opts.io_unit_size = data_bs; 1013 sgl->keyed.length = data_bs; 1014 1015 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 1016 1017 CU_ASSERT(rc == 0); 1018 CU_ASSERT(rdma_req.req.data_from_pool == true); 1019 CU_ASSERT(rdma_req.req.length == data_bs); 1020 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 1021 CU_ASSERT(rdma_req.req.dif.elba_length == data_bs + md_size); 1022 CU_ASSERT((uint64_t)rdma_req.req.data == 0x2000); 1023 CU_ASSERT(rdma_req.data.wr.num_sge == 1); 1024 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 1025 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 1026 CU_ASSERT((uint64_t)rdma_req.req.buffers[0] == 0x2000); 1027 1028 CU_ASSERT(rdma_req.data.wr.sg_list[0].addr == 0x2000); 1029 CU_ASSERT(rdma_req.data.wr.sg_list[0].length == data_bs); 1030 CU_ASSERT(rdma_req.data.wr.sg_list[0].lkey == RDMA_UT_LKEY); 1031 1032 CU_ASSERT(rdma_req.req.iovcnt == 2); 1033 CU_ASSERT(rdma_req.req.iov[0].iov_base == (void *)((unsigned long)0x2000)); 1034 CU_ASSERT(rdma_req.req.iov[0].iov_len == data_bs); 1035 /* 2nd buffer consumed for metadata */ 1036 CU_ASSERT(rdma_req.req.iov[1].iov_base == (void *)((unsigned long)0x2000)); 1037 CU_ASSERT(rdma_req.req.iov[1].iov_len == md_size); 1038 1039 /* Part 4: simple I/O, one SGL equal io unit size, io_unit_size is aligned with md_size, 1040 block size 512 */ 1041 MOCK_SET(spdk_mempool_get, (void *)0x2000); 1042 reset_nvmf_rdma_request(&rdma_req); 1043 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 1044 SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 1045 0, 0, 0, 0, 0); 1046 rdma_req.req.dif_enabled = true; 1047 rtransport.transport.opts.io_unit_size = (data_bs + md_size) * 4; 1048 sgl->keyed.length = data_bs * 4; 1049 1050 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 1051 1052 CU_ASSERT(rc == 0); 1053 CU_ASSERT(rdma_req.req.data_from_pool == true); 1054 CU_ASSERT(rdma_req.req.length == data_bs * 4); 1055 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 1056 CU_ASSERT(rdma_req.req.dif.elba_length == (data_bs + md_size) * 4); 1057 CU_ASSERT((uint64_t)rdma_req.req.data == 0x2000); 1058 CU_ASSERT(rdma_req.data.wr.num_sge == 4); 1059 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 1060 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 1061 CU_ASSERT((uint64_t)rdma_req.req.buffers[0] == 0x2000); 1062 1063 for (i = 0; i < 4; ++i) { 1064 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == 0x2000 + i * (data_bs + md_size)); 1065 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == data_bs); 1066 CU_ASSERT(rdma_req.data.wr.sg_list[i].lkey == RDMA_UT_LKEY); 1067 } 1068 1069 /* Part 5: simple I/O, one SGL equal to 2x io unit size, io_unit_size is aligned with md_size, 1070 block size 512 */ 1071 MOCK_SET(spdk_mempool_get, (void *)0x2000); 1072 reset_nvmf_rdma_request(&rdma_req); 1073 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 1074 SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 1075 0, 0, 0, 0, 0); 1076 rdma_req.req.dif_enabled = true; 1077 rtransport.transport.opts.io_unit_size = (data_bs + md_size) * 2; 1078 sgl->keyed.length = data_bs * 4; 1079 1080 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 1081 1082 CU_ASSERT(rc == 0); 1083 CU_ASSERT(rdma_req.req.data_from_pool == true); 1084 CU_ASSERT(rdma_req.req.length == data_bs * 4); 1085 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 1086 CU_ASSERT(rdma_req.req.dif.elba_length == (data_bs + md_size) * 4); 1087 CU_ASSERT((uint64_t)rdma_req.req.data == 0x2000); 1088 CU_ASSERT(rdma_req.data.wr.num_sge == 4); 1089 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 1090 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 1091 CU_ASSERT((uint64_t)rdma_req.req.buffers[0] == 0x2000); 1092 1093 for (i = 0; i < 2; ++i) { 1094 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == 0x2000 + i * (data_bs + md_size)); 1095 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == data_bs); 1096 } 1097 for (i = 0; i < 2; ++i) { 1098 CU_ASSERT(rdma_req.data.wr.sg_list[i + 2].addr == 0x2000 + i * (data_bs + md_size)); 1099 CU_ASSERT(rdma_req.data.wr.sg_list[i + 2].length == data_bs); 1100 } 1101 1102 /* Part 6: simple I/O, one SGL larger than the transport io unit size, io_unit_size is not aligned to md_size, 1103 block size 512 */ 1104 MOCK_SET(spdk_mempool_get, (void *)0x2000); 1105 reset_nvmf_rdma_request(&rdma_req); 1106 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 1107 SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 1108 0, 0, 0, 0, 0); 1109 rdma_req.req.dif_enabled = true; 1110 rtransport.transport.opts.io_unit_size = data_bs * 4; 1111 sgl->keyed.length = data_bs * 6; 1112 1113 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 1114 1115 CU_ASSERT(rc == 0); 1116 CU_ASSERT(rdma_req.req.data_from_pool == true); 1117 CU_ASSERT(rdma_req.req.length == data_bs * 6); 1118 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 1119 CU_ASSERT(rdma_req.req.dif.elba_length == (data_bs + md_size) * 6); 1120 CU_ASSERT((uint64_t)rdma_req.req.data == 0x2000); 1121 CU_ASSERT(rdma_req.data.wr.num_sge == 7); 1122 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 1123 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 1124 CU_ASSERT((uint64_t)rdma_req.req.buffers[0] == 0x2000); 1125 1126 for (i = 0; i < 3; ++i) { 1127 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == 0x2000 + i * (data_bs + md_size)); 1128 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == data_bs); 1129 CU_ASSERT(rdma_req.data.wr.sg_list[i].lkey == RDMA_UT_LKEY); 1130 } 1131 CU_ASSERT(rdma_req.data.wr.sg_list[3].addr == 0x2000 + 3 * (data_bs + md_size)); 1132 CU_ASSERT(rdma_req.data.wr.sg_list[3].length == 488); 1133 CU_ASSERT(rdma_req.data.wr.sg_list[3].lkey == RDMA_UT_LKEY); 1134 1135 /* 2nd IO buffer consumed */ 1136 CU_ASSERT(rdma_req.data.wr.sg_list[4].addr == 0x2000); 1137 CU_ASSERT(rdma_req.data.wr.sg_list[4].length == 24); 1138 CU_ASSERT(rdma_req.data.wr.sg_list[4].lkey == RDMA_UT_LKEY); 1139 1140 CU_ASSERT(rdma_req.data.wr.sg_list[5].addr == 0x2000 + 24 + md_size); 1141 CU_ASSERT(rdma_req.data.wr.sg_list[5].length == 512); 1142 CU_ASSERT(rdma_req.data.wr.sg_list[5].lkey == RDMA_UT_LKEY); 1143 1144 CU_ASSERT(rdma_req.data.wr.sg_list[6].addr == 0x2000 + 24 + 512 + md_size * 2); 1145 CU_ASSERT(rdma_req.data.wr.sg_list[6].length == 512); 1146 CU_ASSERT(rdma_req.data.wr.sg_list[6].lkey == RDMA_UT_LKEY); 1147 1148 /* Part 7: simple I/O, number of SGL entries exceeds the number of entries 1149 one WR can hold. Additional WR is chained */ 1150 MOCK_SET(spdk_mempool_get, data2_buffer); 1151 aligned_buffer = (void *)((uintptr_t)(data2_buffer + NVMF_DATA_BUFFER_MASK) & 1152 ~NVMF_DATA_BUFFER_MASK); 1153 reset_nvmf_rdma_request(&rdma_req); 1154 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 1155 SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 1156 0, 0, 0, 0, 0); 1157 rdma_req.req.dif_enabled = true; 1158 rtransport.transport.opts.io_unit_size = data_bs * 16; 1159 sgl->keyed.length = data_bs * 16; 1160 1161 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 1162 1163 CU_ASSERT(rc == 0); 1164 CU_ASSERT(rdma_req.req.data_from_pool == true); 1165 CU_ASSERT(rdma_req.req.length == data_bs * 16); 1166 CU_ASSERT(rdma_req.req.iovcnt == 2); 1167 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 1168 CU_ASSERT(rdma_req.req.dif.elba_length == (data_bs + md_size) * 16); 1169 CU_ASSERT(rdma_req.req.data == aligned_buffer); 1170 CU_ASSERT(rdma_req.data.wr.num_sge == 16); 1171 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 1172 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 1173 1174 for (i = 0; i < 15; ++i) { 1175 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == (uintptr_t)aligned_buffer + i * (data_bs + md_size)); 1176 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == data_bs); 1177 CU_ASSERT(rdma_req.data.wr.sg_list[i].lkey == RDMA_UT_LKEY); 1178 } 1179 1180 /* 8192 - (512 + 8) * 15 = 392 */ 1181 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == (uintptr_t)aligned_buffer + i * (data_bs + md_size)); 1182 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == 392); 1183 CU_ASSERT(rdma_req.data.wr.sg_list[i].lkey == RDMA_UT_LKEY); 1184 1185 /* additional wr from pool */ 1186 CU_ASSERT(rdma_req.data.wr.next == (void *)&data2->wr); 1187 CU_ASSERT(rdma_req.data.wr.next->num_sge == 1); 1188 CU_ASSERT(rdma_req.data.wr.next->next == &rdma_req.rsp.wr); 1189 /* 2nd IO buffer */ 1190 CU_ASSERT(data2->wr.sg_list[0].addr == (uintptr_t)aligned_buffer); 1191 CU_ASSERT(data2->wr.sg_list[0].length == 120); 1192 CU_ASSERT(data2->wr.sg_list[0].lkey == RDMA_UT_LKEY); 1193 1194 /* Part 8: simple I/O, data with metadata do not fit to 1 io_buffer */ 1195 MOCK_SET(spdk_mempool_get, (void *)0x2000); 1196 reset_nvmf_rdma_request(&rdma_req); 1197 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 1198 SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 1199 0, 0, 0, 0, 0); 1200 rdma_req.req.dif_enabled = true; 1201 rtransport.transport.opts.io_unit_size = 516; 1202 sgl->keyed.length = data_bs * 2; 1203 1204 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 1205 1206 CU_ASSERT(rc == 0); 1207 CU_ASSERT(rdma_req.req.data_from_pool == true); 1208 CU_ASSERT(rdma_req.req.length == data_bs * 2); 1209 CU_ASSERT(rdma_req.req.iovcnt == 3); 1210 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 1211 CU_ASSERT(rdma_req.req.dif.elba_length == (data_bs + md_size) * 2); 1212 CU_ASSERT(rdma_req.req.data == (void *)0x2000); 1213 CU_ASSERT(rdma_req.data.wr.num_sge == 2); 1214 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 1215 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 1216 1217 CU_ASSERT(rdma_req.data.wr.sg_list[0].addr == 0x2000); 1218 CU_ASSERT(rdma_req.data.wr.sg_list[0].length == 512); 1219 CU_ASSERT(rdma_req.data.wr.sg_list[0].lkey == RDMA_UT_LKEY); 1220 1221 /* 2nd IO buffer consumed, offset 4 bytes due to part of the metadata 1222 is located at the beginning of that buffer */ 1223 CU_ASSERT(rdma_req.data.wr.sg_list[1].addr == 0x2000 + 4); 1224 CU_ASSERT(rdma_req.data.wr.sg_list[1].length == 512); 1225 CU_ASSERT(rdma_req.data.wr.sg_list[1].lkey == RDMA_UT_LKEY); 1226 1227 /* Test 2: Multi SGL */ 1228 sgl->generic.type = SPDK_NVME_SGL_TYPE_LAST_SEGMENT; 1229 sgl->unkeyed.subtype = SPDK_NVME_SGL_SUBTYPE_OFFSET; 1230 sgl->address = 0; 1231 rdma_req.recv->buf = (void *)&sgl_desc; 1232 MOCK_SET(spdk_mempool_get, data_buffer); 1233 aligned_buffer = (void *)((uintptr_t)(data_buffer + NVMF_DATA_BUFFER_MASK) & 1234 ~NVMF_DATA_BUFFER_MASK); 1235 1236 /* part 1: 2 segments each with 1 wr. io_unit_size is aligned with data_bs + md_size */ 1237 reset_nvmf_rdma_request(&rdma_req); 1238 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 1239 SPDK_DIF_TYPE1, 1240 SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 0, 0, 0, 0, 0); 1241 rdma_req.req.dif_enabled = true; 1242 rtransport.transport.opts.io_unit_size = (data_bs + md_size) * 4; 1243 sgl->unkeyed.length = 2 * sizeof(struct spdk_nvme_sgl_descriptor); 1244 1245 for (i = 0; i < 2; i++) { 1246 sgl_desc[i].keyed.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; 1247 sgl_desc[i].keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; 1248 sgl_desc[i].keyed.length = data_bs * 4; 1249 sgl_desc[i].address = 0x4000 + i * data_bs * 4; 1250 sgl_desc[i].keyed.key = 0x44; 1251 } 1252 1253 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 1254 1255 CU_ASSERT(rc == 0); 1256 CU_ASSERT(rdma_req.req.data_from_pool == true); 1257 CU_ASSERT(rdma_req.req.length == data_bs * 4 * 2); 1258 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 1259 CU_ASSERT(rdma_req.req.dif.elba_length == (data_bs + md_size) * 4 * 2); 1260 CU_ASSERT(rdma_req.data.wr.num_sge == 4); 1261 for (i = 0; i < 4; ++i) { 1262 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == (uintptr_t)((unsigned char *)aligned_buffer) + i * 1263 (data_bs + md_size)); 1264 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == data_bs); 1265 } 1266 1267 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0x44); 1268 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0x4000); 1269 CU_ASSERT(rdma_req.data.wr.next == &data->wr); 1270 CU_ASSERT(data->wr.wr.rdma.rkey == 0x44); 1271 CU_ASSERT(data->wr.wr.rdma.remote_addr == 0x4000 + data_bs * 4); 1272 CU_ASSERT(data->wr.num_sge == 4); 1273 for (i = 0; i < 4; ++i) { 1274 CU_ASSERT(data->wr.sg_list[i].addr == (uintptr_t)((unsigned char *)aligned_buffer) + i * 1275 (data_bs + md_size)); 1276 CU_ASSERT(data->wr.sg_list[i].length == data_bs); 1277 } 1278 1279 CU_ASSERT(data->wr.next == &rdma_req.rsp.wr); 1280 } 1281 1282 static void 1283 test_nvmf_rdma_opts_init(void) 1284 { 1285 struct spdk_nvmf_transport_opts opts = {}; 1286 1287 nvmf_rdma_opts_init(&opts); 1288 CU_ASSERT(opts.max_queue_depth == SPDK_NVMF_RDMA_DEFAULT_MAX_QUEUE_DEPTH); 1289 CU_ASSERT(opts.max_qpairs_per_ctrlr == SPDK_NVMF_RDMA_DEFAULT_MAX_QPAIRS_PER_CTRLR); 1290 CU_ASSERT(opts.in_capsule_data_size == SPDK_NVMF_RDMA_DEFAULT_IN_CAPSULE_DATA_SIZE); 1291 CU_ASSERT(opts.max_io_size == SPDK_NVMF_RDMA_DEFAULT_MAX_IO_SIZE); 1292 CU_ASSERT(opts.io_unit_size == SPDK_NVMF_RDMA_MIN_IO_BUFFER_SIZE); 1293 CU_ASSERT(opts.max_aq_depth == SPDK_NVMF_RDMA_DEFAULT_AQ_DEPTH); 1294 CU_ASSERT(opts.num_shared_buffers == SPDK_NVMF_RDMA_DEFAULT_NUM_SHARED_BUFFERS); 1295 CU_ASSERT(opts.buf_cache_size == SPDK_NVMF_RDMA_DEFAULT_BUFFER_CACHE_SIZE); 1296 CU_ASSERT(opts.dif_insert_or_strip == SPDK_NVMF_RDMA_DIF_INSERT_OR_STRIP); 1297 CU_ASSERT(opts.abort_timeout_sec == SPDK_NVMF_RDMA_DEFAULT_ABORT_TIMEOUT_SEC); 1298 CU_ASSERT(opts.transport_specific == NULL); 1299 } 1300 1301 static void 1302 test_nvmf_rdma_request_free_data(void) 1303 { 1304 struct spdk_nvmf_rdma_request rdma_req = {}; 1305 struct spdk_nvmf_rdma_transport rtransport = {}; 1306 struct spdk_nvmf_rdma_request_data *next_request_data = NULL; 1307 1308 MOCK_CLEAR(spdk_mempool_get); 1309 rtransport.data_wr_pool = spdk_mempool_create("spdk_nvmf_rdma_wr_data", 1310 SPDK_NVMF_MAX_SGL_ENTRIES, 1311 sizeof(struct spdk_nvmf_rdma_request_data), 1312 SPDK_MEMPOOL_DEFAULT_CACHE_SIZE, 1313 SPDK_ENV_SOCKET_ID_ANY); 1314 next_request_data = spdk_mempool_get(rtransport.data_wr_pool); 1315 SPDK_CU_ASSERT_FATAL(((struct test_mempool *)rtransport.data_wr_pool)->count == 1316 SPDK_NVMF_MAX_SGL_ENTRIES - 1); 1317 next_request_data->wr.wr_id = 1; 1318 next_request_data->wr.num_sge = 2; 1319 next_request_data->wr.next = NULL; 1320 rdma_req.data.wr.next = &next_request_data->wr; 1321 rdma_req.data.wr.wr_id = 1; 1322 rdma_req.data.wr.num_sge = 2; 1323 1324 nvmf_rdma_request_free_data(&rdma_req, &rtransport); 1325 /* Check if next_request_data put into memory pool */ 1326 CU_ASSERT(((struct test_mempool *)rtransport.data_wr_pool)->count == SPDK_NVMF_MAX_SGL_ENTRIES); 1327 CU_ASSERT(rdma_req.data.wr.num_sge == 0); 1328 1329 spdk_mempool_free(rtransport.data_wr_pool); 1330 } 1331 1332 static void 1333 test_nvmf_rdma_update_ibv_state(void) 1334 { 1335 struct spdk_nvmf_rdma_qpair rqpair = {}; 1336 struct spdk_rdma_qp rdma_qp = {}; 1337 struct ibv_qp qp = {}; 1338 int rc = 0; 1339 1340 rqpair.rdma_qp = &rdma_qp; 1341 1342 /* Case 1: Failed to get updated RDMA queue pair state */ 1343 rqpair.ibv_state = IBV_QPS_INIT; 1344 rqpair.rdma_qp->qp = NULL; 1345 1346 rc = nvmf_rdma_update_ibv_state(&rqpair); 1347 CU_ASSERT(rc == IBV_QPS_ERR + 1); 1348 1349 /* Case 2: Bad state updated */ 1350 rqpair.rdma_qp->qp = &qp; 1351 qp.state = IBV_QPS_ERR; 1352 rc = nvmf_rdma_update_ibv_state(&rqpair); 1353 CU_ASSERT(rqpair.ibv_state == 10); 1354 CU_ASSERT(rc == IBV_QPS_ERR + 1); 1355 1356 /* Case 3: Pass */ 1357 qp.state = IBV_QPS_INIT; 1358 rc = nvmf_rdma_update_ibv_state(&rqpair); 1359 CU_ASSERT(rqpair.ibv_state == IBV_QPS_INIT); 1360 CU_ASSERT(rc == IBV_QPS_INIT); 1361 } 1362 1363 static void 1364 test_nvmf_rdma_resources_create(void) 1365 { 1366 static struct spdk_nvmf_rdma_resources *rdma_resource; 1367 struct spdk_nvmf_rdma_resource_opts opts = {}; 1368 struct spdk_nvmf_rdma_qpair qpair = {}; 1369 struct spdk_nvmf_rdma_recv *recv = NULL; 1370 struct spdk_nvmf_rdma_request *req = NULL; 1371 const int DEPTH = 128; 1372 1373 opts.max_queue_depth = DEPTH; 1374 opts.in_capsule_data_size = 4096; 1375 opts.shared = true; 1376 opts.qpair = &qpair; 1377 1378 rdma_resource = nvmf_rdma_resources_create(&opts); 1379 CU_ASSERT(rdma_resource != NULL); 1380 /* Just check first and last entry */ 1381 recv = &rdma_resource->recvs[0]; 1382 req = &rdma_resource->reqs[0]; 1383 CU_ASSERT(recv->rdma_wr.type == RDMA_WR_TYPE_RECV); 1384 CU_ASSERT((uintptr_t)recv->buf == (uintptr_t)(rdma_resource->bufs)); 1385 CU_ASSERT(recv->sgl[0].addr == (uintptr_t)&rdma_resource->cmds[0]); 1386 CU_ASSERT(recv->sgl[0].length == sizeof(rdma_resource->cmds[0])); 1387 CU_ASSERT(recv->sgl[0].lkey == RDMA_UT_LKEY); 1388 CU_ASSERT(recv->wr.num_sge == 2); 1389 CU_ASSERT(recv->wr.wr_id == (uintptr_t)&rdma_resource->recvs[0].rdma_wr); 1390 CU_ASSERT(recv->wr.sg_list == rdma_resource->recvs[0].sgl); 1391 CU_ASSERT(req->req.rsp == &rdma_resource->cpls[0]); 1392 CU_ASSERT(req->rsp.sgl[0].addr == (uintptr_t)&rdma_resource->cpls[0]); 1393 CU_ASSERT(req->rsp.sgl[0].length == sizeof(rdma_resource->cpls[0])); 1394 CU_ASSERT(req->rsp.sgl[0].lkey == RDMA_UT_LKEY); 1395 CU_ASSERT(req->rsp.rdma_wr.type == RDMA_WR_TYPE_SEND); 1396 CU_ASSERT(req->rsp.wr.wr_id == (uintptr_t)&rdma_resource->reqs[0].rsp.rdma_wr); 1397 CU_ASSERT(req->rsp.wr.next == NULL); 1398 CU_ASSERT(req->rsp.wr.opcode == IBV_WR_SEND); 1399 CU_ASSERT(req->rsp.wr.send_flags == IBV_SEND_SIGNALED); 1400 CU_ASSERT(req->rsp.wr.sg_list == rdma_resource->reqs[0].rsp.sgl); 1401 CU_ASSERT(req->rsp.wr.num_sge == NVMF_DEFAULT_RSP_SGE); 1402 CU_ASSERT(req->data.rdma_wr.type == RDMA_WR_TYPE_DATA); 1403 CU_ASSERT(req->data.wr.wr_id == (uintptr_t)&rdma_resource->reqs[0].data.rdma_wr); 1404 CU_ASSERT(req->data.wr.next == NULL); 1405 CU_ASSERT(req->data.wr.send_flags == IBV_SEND_SIGNALED); 1406 CU_ASSERT(req->data.wr.sg_list == rdma_resource->reqs[0].data.sgl); 1407 CU_ASSERT(req->data.wr.num_sge == SPDK_NVMF_MAX_SGL_ENTRIES); 1408 CU_ASSERT(req->state == RDMA_REQUEST_STATE_FREE); 1409 1410 recv = &rdma_resource->recvs[DEPTH - 1]; 1411 req = &rdma_resource->reqs[DEPTH - 1]; 1412 CU_ASSERT(recv->rdma_wr.type == RDMA_WR_TYPE_RECV); 1413 CU_ASSERT((uintptr_t)recv->buf == (uintptr_t)(rdma_resource->bufs + 1414 (DEPTH - 1) * 4096)); 1415 CU_ASSERT(recv->sgl[0].addr == (uintptr_t)&rdma_resource->cmds[DEPTH - 1]); 1416 CU_ASSERT(recv->sgl[0].length == sizeof(rdma_resource->cmds[DEPTH - 1])); 1417 CU_ASSERT(recv->sgl[0].lkey == RDMA_UT_LKEY); 1418 CU_ASSERT(recv->wr.num_sge == 2); 1419 CU_ASSERT(recv->wr.wr_id == (uintptr_t)&rdma_resource->recvs[DEPTH - 1].rdma_wr); 1420 CU_ASSERT(recv->wr.sg_list == rdma_resource->recvs[DEPTH - 1].sgl); 1421 CU_ASSERT(req->req.rsp == &rdma_resource->cpls[DEPTH - 1]); 1422 CU_ASSERT(req->rsp.sgl[0].addr == (uintptr_t)&rdma_resource->cpls[DEPTH - 1]); 1423 CU_ASSERT(req->rsp.sgl[0].length == sizeof(rdma_resource->cpls[DEPTH - 1])); 1424 CU_ASSERT(req->rsp.sgl[0].lkey == RDMA_UT_LKEY); 1425 CU_ASSERT(req->rsp.rdma_wr.type == RDMA_WR_TYPE_SEND); 1426 CU_ASSERT(req->rsp.wr.wr_id == (uintptr_t) 1427 &req->rsp.rdma_wr); 1428 CU_ASSERT(req->rsp.wr.next == NULL); 1429 CU_ASSERT(req->rsp.wr.opcode == IBV_WR_SEND); 1430 CU_ASSERT(req->rsp.wr.send_flags == IBV_SEND_SIGNALED); 1431 CU_ASSERT(req->rsp.wr.sg_list == rdma_resource->reqs[DEPTH - 1].rsp.sgl); 1432 CU_ASSERT(req->rsp.wr.num_sge == NVMF_DEFAULT_RSP_SGE); 1433 CU_ASSERT(req->data.rdma_wr.type == RDMA_WR_TYPE_DATA); 1434 CU_ASSERT(req->data.wr.wr_id == (uintptr_t) 1435 &req->data.rdma_wr); 1436 CU_ASSERT(req->data.wr.next == NULL); 1437 CU_ASSERT(req->data.wr.send_flags == IBV_SEND_SIGNALED); 1438 CU_ASSERT(req->data.wr.sg_list == rdma_resource->reqs[DEPTH - 1].data.sgl); 1439 CU_ASSERT(req->data.wr.num_sge == SPDK_NVMF_MAX_SGL_ENTRIES); 1440 CU_ASSERT(req->state == RDMA_REQUEST_STATE_FREE); 1441 1442 nvmf_rdma_resources_destroy(rdma_resource); 1443 } 1444 1445 int main(int argc, char **argv) 1446 { 1447 CU_pSuite suite = NULL; 1448 unsigned int num_failures; 1449 1450 CU_set_error_action(CUEA_ABORT); 1451 CU_initialize_registry(); 1452 1453 suite = CU_add_suite("nvmf", NULL, NULL); 1454 1455 CU_ADD_TEST(suite, test_spdk_nvmf_rdma_request_parse_sgl); 1456 CU_ADD_TEST(suite, test_spdk_nvmf_rdma_request_process); 1457 CU_ADD_TEST(suite, test_nvmf_rdma_get_optimal_poll_group); 1458 CU_ADD_TEST(suite, test_spdk_nvmf_rdma_request_parse_sgl_with_md); 1459 CU_ADD_TEST(suite, test_nvmf_rdma_opts_init); 1460 CU_ADD_TEST(suite, test_nvmf_rdma_request_free_data); 1461 CU_ADD_TEST(suite, test_nvmf_rdma_update_ibv_state); 1462 CU_ADD_TEST(suite, test_nvmf_rdma_resources_create); 1463 1464 CU_basic_set_mode(CU_BRM_VERBOSE); 1465 CU_basic_run_tests(); 1466 num_failures = CU_get_number_of_failures(); 1467 CU_cleanup_registry(); 1468 return num_failures; 1469 } 1470