1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. All rights reserved. 5 * Copyright (c) 2019, 2021 Mellanox Technologies LTD. All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "spdk/stdinc.h" 35 #include "spdk_cunit.h" 36 #include "common/lib/test_env.c" 37 #include "common/lib/test_rdma.c" 38 #include "nvmf/rdma.c" 39 #include "nvmf/transport.c" 40 41 #define RDMA_UT_UNITS_IN_MAX_IO 16 42 43 struct spdk_nvmf_transport_opts g_rdma_ut_transport_opts = { 44 .max_queue_depth = SPDK_NVMF_RDMA_DEFAULT_MAX_QUEUE_DEPTH, 45 .max_qpairs_per_ctrlr = SPDK_NVMF_RDMA_DEFAULT_MAX_QPAIRS_PER_CTRLR, 46 .in_capsule_data_size = SPDK_NVMF_RDMA_DEFAULT_IN_CAPSULE_DATA_SIZE, 47 .max_io_size = (SPDK_NVMF_RDMA_MIN_IO_BUFFER_SIZE * RDMA_UT_UNITS_IN_MAX_IO), 48 .io_unit_size = SPDK_NVMF_RDMA_MIN_IO_BUFFER_SIZE, 49 .max_aq_depth = SPDK_NVMF_RDMA_DEFAULT_AQ_DEPTH, 50 .num_shared_buffers = SPDK_NVMF_RDMA_DEFAULT_NUM_SHARED_BUFFERS, 51 }; 52 53 SPDK_LOG_REGISTER_COMPONENT(nvmf) 54 DEFINE_STUB(spdk_mem_map_set_translation, int, (struct spdk_mem_map *map, uint64_t vaddr, 55 uint64_t size, uint64_t translation), 0); 56 DEFINE_STUB(spdk_mem_map_clear_translation, int, (struct spdk_mem_map *map, uint64_t vaddr, 57 uint64_t size), 0); 58 DEFINE_STUB(spdk_mem_map_alloc, struct spdk_mem_map *, (uint64_t default_translation, 59 const struct spdk_mem_map_ops *ops, void *cb_ctx), NULL); 60 DEFINE_STUB(spdk_nvmf_qpair_disconnect, int, (struct spdk_nvmf_qpair *qpair, 61 nvmf_qpair_disconnect_cb cb_fn, void *ctx), 0); 62 DEFINE_STUB(spdk_nvmf_qpair_get_listen_trid, int, 63 (struct spdk_nvmf_qpair *qpair, struct spdk_nvme_transport_id *trid), 0); 64 DEFINE_STUB_V(spdk_mem_map_free, (struct spdk_mem_map **pmap)); 65 66 DEFINE_STUB_V(spdk_nvmf_ctrlr_data_init, (struct spdk_nvmf_transport_opts *opts, 67 struct spdk_nvmf_ctrlr_data *cdata)); 68 DEFINE_STUB_V(spdk_nvmf_request_exec, (struct spdk_nvmf_request *req)); 69 DEFINE_STUB(spdk_nvmf_request_complete, int, (struct spdk_nvmf_request *req), 0); 70 DEFINE_STUB(spdk_nvme_transport_id_compare, int, (const struct spdk_nvme_transport_id *trid1, 71 const struct spdk_nvme_transport_id *trid2), 0); 72 DEFINE_STUB_V(nvmf_ctrlr_abort_aer, (struct spdk_nvmf_ctrlr *ctrlr)); 73 DEFINE_STUB(spdk_nvmf_request_get_dif_ctx, bool, (struct spdk_nvmf_request *req, 74 struct spdk_dif_ctx *dif_ctx), false); 75 DEFINE_STUB_V(spdk_nvme_trid_populate_transport, (struct spdk_nvme_transport_id *trid, 76 enum spdk_nvme_transport_type trtype)); 77 DEFINE_STUB_V(spdk_nvmf_tgt_new_qpair, (struct spdk_nvmf_tgt *tgt, struct spdk_nvmf_qpair *qpair)); 78 DEFINE_STUB(nvmf_ctrlr_abort_request, int, (struct spdk_nvmf_request *req), 0); 79 80 const char * 81 spdk_nvme_transport_id_trtype_str(enum spdk_nvme_transport_type trtype) 82 { 83 switch (trtype) { 84 case SPDK_NVME_TRANSPORT_PCIE: 85 return "PCIe"; 86 case SPDK_NVME_TRANSPORT_RDMA: 87 return "RDMA"; 88 case SPDK_NVME_TRANSPORT_FC: 89 return "FC"; 90 default: 91 return NULL; 92 } 93 } 94 95 int 96 spdk_nvme_transport_id_populate_trstring(struct spdk_nvme_transport_id *trid, const char *trstring) 97 { 98 int len, i; 99 100 if (trstring == NULL) { 101 return -EINVAL; 102 } 103 104 len = strnlen(trstring, SPDK_NVMF_TRSTRING_MAX_LEN); 105 if (len == SPDK_NVMF_TRSTRING_MAX_LEN) { 106 return -EINVAL; 107 } 108 109 /* cast official trstring to uppercase version of input. */ 110 for (i = 0; i < len; i++) { 111 trid->trstring[i] = toupper(trstring[i]); 112 } 113 return 0; 114 } 115 116 static void reset_nvmf_rdma_request(struct spdk_nvmf_rdma_request *rdma_req) 117 { 118 int i; 119 120 rdma_req->req.length = 0; 121 rdma_req->req.data_from_pool = false; 122 rdma_req->req.data = NULL; 123 rdma_req->data.wr.num_sge = 0; 124 rdma_req->data.wr.wr.rdma.remote_addr = 0; 125 rdma_req->data.wr.wr.rdma.rkey = 0; 126 rdma_req->offset = 0; 127 memset(&rdma_req->req.dif, 0, sizeof(rdma_req->req.dif)); 128 129 for (i = 0; i < SPDK_NVMF_MAX_SGL_ENTRIES; i++) { 130 rdma_req->req.iov[i].iov_base = 0; 131 rdma_req->req.iov[i].iov_len = 0; 132 rdma_req->req.buffers[i] = 0; 133 rdma_req->data.wr.sg_list[i].addr = 0; 134 rdma_req->data.wr.sg_list[i].length = 0; 135 rdma_req->data.wr.sg_list[i].lkey = 0; 136 } 137 rdma_req->req.iovcnt = 0; 138 } 139 140 static void 141 test_spdk_nvmf_rdma_request_parse_sgl(void) 142 { 143 struct spdk_nvmf_rdma_transport rtransport; 144 struct spdk_nvmf_rdma_device device; 145 struct spdk_nvmf_rdma_request rdma_req = {}; 146 struct spdk_nvmf_rdma_recv recv; 147 struct spdk_nvmf_rdma_poll_group group; 148 struct spdk_nvmf_rdma_qpair rqpair; 149 struct spdk_nvmf_rdma_poller poller; 150 union nvmf_c2h_msg cpl; 151 union nvmf_h2c_msg cmd; 152 struct spdk_nvme_sgl_descriptor *sgl; 153 struct spdk_nvmf_transport_pg_cache_buf bufs[4]; 154 struct spdk_nvme_sgl_descriptor sgl_desc[SPDK_NVMF_MAX_SGL_ENTRIES] = {{0}}; 155 struct spdk_nvmf_rdma_request_data data; 156 int rc, i; 157 uint32_t sgl_length; 158 uintptr_t aligned_buffer_address; 159 160 data.wr.sg_list = data.sgl; 161 STAILQ_INIT(&group.group.buf_cache); 162 group.group.buf_cache_size = 0; 163 group.group.buf_cache_count = 0; 164 group.group.transport = &rtransport.transport; 165 poller.group = &group; 166 rqpair.poller = &poller; 167 rqpair.max_send_sge = SPDK_NVMF_MAX_SGL_ENTRIES; 168 169 sgl = &cmd.nvme_cmd.dptr.sgl1; 170 rdma_req.recv = &recv; 171 rdma_req.req.cmd = &cmd; 172 rdma_req.req.rsp = &cpl; 173 rdma_req.data.wr.sg_list = rdma_req.data.sgl; 174 rdma_req.req.qpair = &rqpair.qpair; 175 rdma_req.req.xfer = SPDK_NVME_DATA_CONTROLLER_TO_HOST; 176 177 rtransport.transport.opts = g_rdma_ut_transport_opts; 178 rtransport.data_wr_pool = NULL; 179 rtransport.transport.data_buf_pool = NULL; 180 181 device.attr.device_cap_flags = 0; 182 sgl->keyed.key = 0xEEEE; 183 sgl->address = 0xFFFF; 184 rdma_req.recv->buf = (void *)0xDDDD; 185 186 /* Test 1: sgl type: keyed data block subtype: address */ 187 sgl->generic.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; 188 sgl->keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; 189 190 /* Part 1: simple I/O, one SGL smaller than the transport io unit size */ 191 MOCK_SET(spdk_mempool_get, (void *)0x2000); 192 reset_nvmf_rdma_request(&rdma_req); 193 sgl->keyed.length = rtransport.transport.opts.io_unit_size / 2; 194 195 device.map = (void *)0x0; 196 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 197 CU_ASSERT(rc == 0); 198 CU_ASSERT(rdma_req.req.data_from_pool == true); 199 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size / 2); 200 CU_ASSERT((uint64_t)rdma_req.req.data == 0x2000); 201 CU_ASSERT(rdma_req.data.wr.num_sge == 1); 202 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 203 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 204 CU_ASSERT((uint64_t)rdma_req.req.buffers[0] == 0x2000); 205 CU_ASSERT(rdma_req.data.wr.sg_list[0].addr == 0x2000); 206 CU_ASSERT(rdma_req.data.wr.sg_list[0].length == rtransport.transport.opts.io_unit_size / 2); 207 CU_ASSERT(rdma_req.data.wr.sg_list[0].lkey == RDMA_UT_LKEY); 208 209 /* Part 2: simple I/O, one SGL larger than the transport io unit size (equal to the max io size) */ 210 reset_nvmf_rdma_request(&rdma_req); 211 sgl->keyed.length = rtransport.transport.opts.io_unit_size * RDMA_UT_UNITS_IN_MAX_IO; 212 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 213 214 CU_ASSERT(rc == 0); 215 CU_ASSERT(rdma_req.req.data_from_pool == true); 216 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size * RDMA_UT_UNITS_IN_MAX_IO); 217 CU_ASSERT(rdma_req.data.wr.num_sge == RDMA_UT_UNITS_IN_MAX_IO); 218 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 219 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 220 for (i = 0; i < RDMA_UT_UNITS_IN_MAX_IO; i++) { 221 CU_ASSERT((uint64_t)rdma_req.req.buffers[i] == 0x2000); 222 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == 0x2000); 223 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == rtransport.transport.opts.io_unit_size); 224 CU_ASSERT(rdma_req.data.wr.sg_list[i].lkey == RDMA_UT_LKEY); 225 } 226 227 /* Part 3: simple I/O one SGL larger than the transport max io size */ 228 reset_nvmf_rdma_request(&rdma_req); 229 sgl->keyed.length = rtransport.transport.opts.max_io_size * 2; 230 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 231 232 CU_ASSERT(rc == -1); 233 234 /* Part 4: Pretend there are no buffer pools */ 235 MOCK_SET(spdk_mempool_get, NULL); 236 reset_nvmf_rdma_request(&rdma_req); 237 sgl->keyed.length = rtransport.transport.opts.io_unit_size * RDMA_UT_UNITS_IN_MAX_IO; 238 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 239 240 CU_ASSERT(rc == 0); 241 CU_ASSERT(rdma_req.req.data_from_pool == false); 242 CU_ASSERT(rdma_req.req.data == NULL); 243 CU_ASSERT(rdma_req.data.wr.num_sge == 0); 244 CU_ASSERT(rdma_req.req.buffers[0] == NULL); 245 CU_ASSERT(rdma_req.data.wr.sg_list[0].addr == 0); 246 CU_ASSERT(rdma_req.data.wr.sg_list[0].length == 0); 247 CU_ASSERT(rdma_req.data.wr.sg_list[0].lkey == 0); 248 249 rdma_req.recv->buf = (void *)0xDDDD; 250 /* Test 2: sgl type: keyed data block subtype: offset (in capsule data) */ 251 sgl->generic.type = SPDK_NVME_SGL_TYPE_DATA_BLOCK; 252 sgl->unkeyed.subtype = SPDK_NVME_SGL_SUBTYPE_OFFSET; 253 254 /* Part 1: Normal I/O smaller than in capsule data size no offset */ 255 reset_nvmf_rdma_request(&rdma_req); 256 sgl->address = 0; 257 sgl->unkeyed.length = rtransport.transport.opts.in_capsule_data_size; 258 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 259 260 CU_ASSERT(rc == 0); 261 CU_ASSERT(rdma_req.req.data == (void *)0xDDDD); 262 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.in_capsule_data_size); 263 CU_ASSERT(rdma_req.req.data_from_pool == false); 264 265 /* Part 2: I/O offset + length too large */ 266 reset_nvmf_rdma_request(&rdma_req); 267 sgl->address = rtransport.transport.opts.in_capsule_data_size; 268 sgl->unkeyed.length = rtransport.transport.opts.in_capsule_data_size; 269 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 270 271 CU_ASSERT(rc == -1); 272 273 /* Part 3: I/O too large */ 274 reset_nvmf_rdma_request(&rdma_req); 275 sgl->address = 0; 276 sgl->unkeyed.length = rtransport.transport.opts.in_capsule_data_size * 2; 277 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 278 279 CU_ASSERT(rc == -1); 280 281 /* Test 3: Multi SGL */ 282 sgl->generic.type = SPDK_NVME_SGL_TYPE_LAST_SEGMENT; 283 sgl->unkeyed.subtype = SPDK_NVME_SGL_SUBTYPE_OFFSET; 284 sgl->address = 0; 285 rdma_req.recv->buf = (void *)&sgl_desc; 286 MOCK_SET(spdk_mempool_get, &data); 287 288 /* part 1: 2 segments each with 1 wr. */ 289 reset_nvmf_rdma_request(&rdma_req); 290 sgl->unkeyed.length = 2 * sizeof(struct spdk_nvme_sgl_descriptor); 291 for (i = 0; i < 2; i++) { 292 sgl_desc[i].keyed.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; 293 sgl_desc[i].keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; 294 sgl_desc[i].keyed.length = rtransport.transport.opts.io_unit_size; 295 sgl_desc[i].address = 0x4000 + i * rtransport.transport.opts.io_unit_size; 296 sgl_desc[i].keyed.key = 0x44; 297 } 298 299 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 300 301 CU_ASSERT(rc == 0); 302 CU_ASSERT(rdma_req.req.data_from_pool == true); 303 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size * 2); 304 CU_ASSERT(rdma_req.data.wr.num_sge == 1); 305 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0x44); 306 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0x4000); 307 CU_ASSERT(rdma_req.data.wr.next == &data.wr); 308 CU_ASSERT(data.wr.wr.rdma.rkey == 0x44); 309 CU_ASSERT(data.wr.wr.rdma.remote_addr == 0x4000 + rtransport.transport.opts.io_unit_size); 310 CU_ASSERT(data.wr.num_sge == 1); 311 CU_ASSERT(data.wr.next == &rdma_req.rsp.wr); 312 313 /* part 2: 2 segments, each with 1 wr containing 8 sge_elements */ 314 reset_nvmf_rdma_request(&rdma_req); 315 sgl->unkeyed.length = 2 * sizeof(struct spdk_nvme_sgl_descriptor); 316 for (i = 0; i < 2; i++) { 317 sgl_desc[i].keyed.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; 318 sgl_desc[i].keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; 319 sgl_desc[i].keyed.length = rtransport.transport.opts.io_unit_size * 8; 320 sgl_desc[i].address = 0x4000 + i * 8 * rtransport.transport.opts.io_unit_size; 321 sgl_desc[i].keyed.key = 0x44; 322 } 323 324 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 325 326 CU_ASSERT(rc == 0); 327 CU_ASSERT(rdma_req.req.data_from_pool == true); 328 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size * 16); 329 CU_ASSERT(rdma_req.req.iovcnt == 16); 330 CU_ASSERT(rdma_req.data.wr.num_sge == 8); 331 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0x44); 332 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0x4000); 333 CU_ASSERT(rdma_req.data.wr.next == &data.wr); 334 CU_ASSERT(data.wr.wr.rdma.rkey == 0x44); 335 CU_ASSERT(data.wr.wr.rdma.remote_addr == 0x4000 + rtransport.transport.opts.io_unit_size * 8); 336 CU_ASSERT(data.wr.num_sge == 8); 337 CU_ASSERT(data.wr.next == &rdma_req.rsp.wr); 338 339 /* part 3: 2 segments, one very large, one very small */ 340 reset_nvmf_rdma_request(&rdma_req); 341 for (i = 0; i < 2; i++) { 342 sgl_desc[i].keyed.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; 343 sgl_desc[i].keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; 344 sgl_desc[i].keyed.key = 0x44; 345 } 346 347 sgl_desc[0].keyed.length = rtransport.transport.opts.io_unit_size * 15 + 348 rtransport.transport.opts.io_unit_size / 2; 349 sgl_desc[0].address = 0x4000; 350 sgl_desc[1].keyed.length = rtransport.transport.opts.io_unit_size / 2; 351 sgl_desc[1].address = 0x4000 + rtransport.transport.opts.io_unit_size * 15 + 352 rtransport.transport.opts.io_unit_size / 2; 353 354 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 355 356 CU_ASSERT(rc == 0); 357 CU_ASSERT(rdma_req.req.data_from_pool == true); 358 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size * 16); 359 CU_ASSERT(rdma_req.req.iovcnt == 16); 360 CU_ASSERT(rdma_req.data.wr.num_sge == 16); 361 for (i = 0; i < 15; i++) { 362 CU_ASSERT(rdma_req.data.sgl[i].length == rtransport.transport.opts.io_unit_size); 363 } 364 CU_ASSERT(rdma_req.data.sgl[15].length == rtransport.transport.opts.io_unit_size / 2); 365 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0x44); 366 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0x4000); 367 CU_ASSERT(rdma_req.data.wr.next == &data.wr); 368 CU_ASSERT(data.wr.wr.rdma.rkey == 0x44); 369 CU_ASSERT(data.wr.wr.rdma.remote_addr == 0x4000 + rtransport.transport.opts.io_unit_size * 15 + 370 rtransport.transport.opts.io_unit_size / 2); 371 CU_ASSERT(data.sgl[0].length == rtransport.transport.opts.io_unit_size / 2); 372 CU_ASSERT(data.wr.num_sge == 1); 373 CU_ASSERT(data.wr.next == &rdma_req.rsp.wr); 374 375 /* part 4: 2 SGL descriptors, each length is transport buffer / 2 376 * 1 transport buffers should be allocated */ 377 reset_nvmf_rdma_request(&rdma_req); 378 aligned_buffer_address = ((uintptr_t)(&data) + NVMF_DATA_BUFFER_MASK) & ~NVMF_DATA_BUFFER_MASK; 379 sgl->unkeyed.length = 2 * sizeof(struct spdk_nvme_sgl_descriptor); 380 sgl_length = rtransport.transport.opts.io_unit_size / 2; 381 for (i = 0; i < 2; i++) { 382 sgl_desc[i].keyed.length = sgl_length; 383 sgl_desc[i].address = 0x4000 + i * sgl_length; 384 } 385 386 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 387 388 CU_ASSERT(rc == 0); 389 CU_ASSERT(rdma_req.req.data_from_pool == true); 390 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size); 391 CU_ASSERT(rdma_req.req.iovcnt == 1); 392 393 CU_ASSERT(rdma_req.data.sgl[0].length == sgl_length); 394 /* We mocked mempool_get to return address of data variable. Mempool is used 395 * to get both additional WRs and data buffers, so data points to &data */ 396 CU_ASSERT(rdma_req.data.sgl[0].addr == aligned_buffer_address); 397 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0x44); 398 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0x4000); 399 CU_ASSERT(rdma_req.data.wr.num_sge == 1); 400 CU_ASSERT(rdma_req.data.wr.next == &data.wr); 401 402 CU_ASSERT(data.wr.wr.rdma.rkey == 0x44); 403 CU_ASSERT(data.wr.wr.rdma.remote_addr == 0x4000 + sgl_length); 404 CU_ASSERT(data.sgl[0].length == sgl_length); 405 CU_ASSERT(data.sgl[0].addr == aligned_buffer_address + sgl_length); 406 CU_ASSERT(data.wr.num_sge == 1); 407 408 /* Test 4: use PG buffer cache */ 409 sgl->generic.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; 410 sgl->keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; 411 sgl->address = 0xFFFF; 412 rdma_req.recv->buf = (void *)0xDDDD; 413 sgl->keyed.key = 0xEEEE; 414 415 for (i = 0; i < 4; i++) { 416 STAILQ_INSERT_TAIL(&group.group.buf_cache, &bufs[i], link); 417 } 418 419 /* part 1: use the four buffers from the pg cache */ 420 group.group.buf_cache_size = 4; 421 group.group.buf_cache_count = 4; 422 MOCK_SET(spdk_mempool_get, (void *)0x2000); 423 reset_nvmf_rdma_request(&rdma_req); 424 sgl->keyed.length = rtransport.transport.opts.io_unit_size * 4; 425 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 426 427 SPDK_CU_ASSERT_FATAL(rc == 0); 428 CU_ASSERT(rdma_req.req.data_from_pool == true); 429 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size * 4); 430 CU_ASSERT((uint64_t)rdma_req.req.data == (((uint64_t)&bufs[0] + NVMF_DATA_BUFFER_MASK) & 431 ~NVMF_DATA_BUFFER_MASK)); 432 CU_ASSERT(rdma_req.data.wr.num_sge == 4); 433 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 434 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 435 CU_ASSERT(group.group.buf_cache_count == 0); 436 CU_ASSERT(STAILQ_EMPTY(&group.group.buf_cache)); 437 for (i = 0; i < 4; i++) { 438 CU_ASSERT((uint64_t)rdma_req.req.buffers[i] == (uint64_t)&bufs[i]); 439 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == (((uint64_t)&bufs[i] + NVMF_DATA_BUFFER_MASK) & 440 ~NVMF_DATA_BUFFER_MASK)); 441 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == rtransport.transport.opts.io_unit_size); 442 } 443 444 /* part 2: now that we have used the buffers from the cache, try again. We should get mempool buffers. */ 445 reset_nvmf_rdma_request(&rdma_req); 446 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 447 448 SPDK_CU_ASSERT_FATAL(rc == 0); 449 CU_ASSERT(rdma_req.req.data_from_pool == true); 450 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size * 4); 451 CU_ASSERT((uint64_t)rdma_req.req.data == 0x2000); 452 CU_ASSERT(rdma_req.data.wr.num_sge == 4); 453 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 454 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 455 CU_ASSERT(group.group.buf_cache_count == 0); 456 CU_ASSERT(STAILQ_EMPTY(&group.group.buf_cache)); 457 for (i = 0; i < 4; i++) { 458 CU_ASSERT((uint64_t)rdma_req.req.buffers[i] == 0x2000); 459 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == 0x2000); 460 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == rtransport.transport.opts.io_unit_size); 461 CU_ASSERT(group.group.buf_cache_count == 0); 462 } 463 464 /* part 3: half and half */ 465 group.group.buf_cache_count = 2; 466 467 for (i = 0; i < 2; i++) { 468 STAILQ_INSERT_TAIL(&group.group.buf_cache, &bufs[i], link); 469 } 470 reset_nvmf_rdma_request(&rdma_req); 471 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 472 473 SPDK_CU_ASSERT_FATAL(rc == 0); 474 CU_ASSERT(rdma_req.req.data_from_pool == true); 475 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size * 4); 476 CU_ASSERT((uint64_t)rdma_req.req.data == (((uint64_t)&bufs[0] + NVMF_DATA_BUFFER_MASK) & 477 ~NVMF_DATA_BUFFER_MASK)); 478 CU_ASSERT(rdma_req.data.wr.num_sge == 4); 479 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 480 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 481 CU_ASSERT(group.group.buf_cache_count == 0); 482 for (i = 0; i < 2; i++) { 483 CU_ASSERT((uint64_t)rdma_req.req.buffers[i] == (uint64_t)&bufs[i]); 484 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == (((uint64_t)&bufs[i] + NVMF_DATA_BUFFER_MASK) & 485 ~NVMF_DATA_BUFFER_MASK)); 486 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == rtransport.transport.opts.io_unit_size); 487 } 488 for (i = 2; i < 4; i++) { 489 CU_ASSERT((uint64_t)rdma_req.req.buffers[i] == 0x2000); 490 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == 0x2000); 491 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == rtransport.transport.opts.io_unit_size); 492 } 493 494 reset_nvmf_rdma_request(&rdma_req); 495 } 496 497 static struct spdk_nvmf_rdma_recv * 498 create_recv(struct spdk_nvmf_rdma_qpair *rqpair, enum spdk_nvme_nvm_opcode opc) 499 { 500 struct spdk_nvmf_rdma_recv *rdma_recv; 501 union nvmf_h2c_msg *cmd; 502 struct spdk_nvme_sgl_descriptor *sgl; 503 504 rdma_recv = calloc(1, sizeof(*rdma_recv)); 505 rdma_recv->qpair = rqpair; 506 cmd = calloc(1, sizeof(*cmd)); 507 rdma_recv->sgl[0].addr = (uintptr_t)cmd; 508 cmd->nvme_cmd.opc = opc; 509 sgl = &cmd->nvme_cmd.dptr.sgl1; 510 sgl->keyed.key = 0xEEEE; 511 sgl->address = 0xFFFF; 512 sgl->keyed.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; 513 sgl->keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; 514 sgl->keyed.length = 1; 515 516 return rdma_recv; 517 } 518 519 static void 520 free_recv(struct spdk_nvmf_rdma_recv *rdma_recv) 521 { 522 free((void *)rdma_recv->sgl[0].addr); 523 free(rdma_recv); 524 } 525 526 static struct spdk_nvmf_rdma_request * 527 create_req(struct spdk_nvmf_rdma_qpair *rqpair, 528 struct spdk_nvmf_rdma_recv *rdma_recv) 529 { 530 struct spdk_nvmf_rdma_request *rdma_req; 531 union nvmf_c2h_msg *cpl; 532 533 rdma_req = calloc(1, sizeof(*rdma_req)); 534 rdma_req->recv = rdma_recv; 535 rdma_req->req.qpair = &rqpair->qpair; 536 rdma_req->state = RDMA_REQUEST_STATE_NEW; 537 rdma_req->data.wr.wr_id = (uintptr_t)&rdma_req->data.rdma_wr; 538 rdma_req->data.wr.sg_list = rdma_req->data.sgl; 539 cpl = calloc(1, sizeof(*cpl)); 540 rdma_req->rsp.sgl[0].addr = (uintptr_t)cpl; 541 rdma_req->req.rsp = cpl; 542 543 return rdma_req; 544 } 545 546 static void 547 free_req(struct spdk_nvmf_rdma_request *rdma_req) 548 { 549 free((void *)rdma_req->rsp.sgl[0].addr); 550 free(rdma_req); 551 } 552 553 static void 554 qpair_reset(struct spdk_nvmf_rdma_qpair *rqpair, 555 struct spdk_nvmf_rdma_poller *poller, 556 struct spdk_nvmf_rdma_device *device, 557 struct spdk_nvmf_rdma_resources *resources, 558 struct spdk_nvmf_transport *transport) 559 { 560 memset(rqpair, 0, sizeof(*rqpair)); 561 STAILQ_INIT(&rqpair->pending_rdma_write_queue); 562 STAILQ_INIT(&rqpair->pending_rdma_read_queue); 563 rqpair->poller = poller; 564 rqpair->device = device; 565 rqpair->resources = resources; 566 rqpair->qpair.qid = 1; 567 rqpair->ibv_state = IBV_QPS_RTS; 568 rqpair->qpair.state = SPDK_NVMF_QPAIR_ACTIVE; 569 rqpair->max_send_sge = SPDK_NVMF_MAX_SGL_ENTRIES; 570 rqpair->max_send_depth = 16; 571 rqpair->max_read_depth = 16; 572 rqpair->qpair.transport = transport; 573 } 574 575 static void 576 poller_reset(struct spdk_nvmf_rdma_poller *poller, 577 struct spdk_nvmf_rdma_poll_group *group) 578 { 579 memset(poller, 0, sizeof(*poller)); 580 STAILQ_INIT(&poller->qpairs_pending_recv); 581 STAILQ_INIT(&poller->qpairs_pending_send); 582 poller->group = group; 583 } 584 585 static void 586 test_spdk_nvmf_rdma_request_process(void) 587 { 588 struct spdk_nvmf_rdma_transport rtransport = {}; 589 struct spdk_nvmf_rdma_poll_group group = {}; 590 struct spdk_nvmf_rdma_poller poller = {}; 591 struct spdk_nvmf_rdma_device device = {}; 592 struct spdk_nvmf_rdma_resources resources = {}; 593 struct spdk_nvmf_rdma_qpair rqpair = {}; 594 struct spdk_nvmf_rdma_recv *rdma_recv; 595 struct spdk_nvmf_rdma_request *rdma_req; 596 bool progress; 597 598 STAILQ_INIT(&group.group.buf_cache); 599 STAILQ_INIT(&group.group.pending_buf_queue); 600 group.group.buf_cache_size = 0; 601 group.group.buf_cache_count = 0; 602 poller_reset(&poller, &group); 603 qpair_reset(&rqpair, &poller, &device, &resources, &rtransport.transport); 604 605 rtransport.transport.opts = g_rdma_ut_transport_opts; 606 rtransport.transport.data_buf_pool = spdk_mempool_create("test_data_pool", 16, 128, 0, 0); 607 rtransport.data_wr_pool = spdk_mempool_create("test_wr_pool", 128, 608 sizeof(struct spdk_nvmf_rdma_request_data), 609 0, 0); 610 MOCK_CLEAR(spdk_mempool_get); 611 612 device.attr.device_cap_flags = 0; 613 device.map = (void *)0x0; 614 615 /* Test 1: single SGL READ request */ 616 rdma_recv = create_recv(&rqpair, SPDK_NVME_OPC_READ); 617 rdma_req = create_req(&rqpair, rdma_recv); 618 rqpair.current_recv_depth = 1; 619 /* NEW -> EXECUTING */ 620 progress = nvmf_rdma_request_process(&rtransport, rdma_req); 621 CU_ASSERT(progress == true); 622 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_EXECUTING); 623 CU_ASSERT(rdma_req->req.xfer == SPDK_NVME_DATA_CONTROLLER_TO_HOST); 624 /* EXECUTED -> TRANSFERRING_C2H */ 625 rdma_req->state = RDMA_REQUEST_STATE_EXECUTED; 626 progress = nvmf_rdma_request_process(&rtransport, rdma_req); 627 CU_ASSERT(progress == true); 628 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_TRANSFERRING_CONTROLLER_TO_HOST); 629 CU_ASSERT(rdma_req->recv == NULL); 630 /* COMPLETED -> FREE */ 631 rdma_req->state = RDMA_REQUEST_STATE_COMPLETED; 632 progress = nvmf_rdma_request_process(&rtransport, rdma_req); 633 CU_ASSERT(progress == true); 634 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_FREE); 635 636 free_recv(rdma_recv); 637 free_req(rdma_req); 638 poller_reset(&poller, &group); 639 qpair_reset(&rqpair, &poller, &device, &resources, &rtransport.transport); 640 641 /* Test 2: single SGL WRITE request */ 642 rdma_recv = create_recv(&rqpair, SPDK_NVME_OPC_WRITE); 643 rdma_req = create_req(&rqpair, rdma_recv); 644 rqpair.current_recv_depth = 1; 645 /* NEW -> TRANSFERRING_H2C */ 646 progress = nvmf_rdma_request_process(&rtransport, rdma_req); 647 CU_ASSERT(progress == true); 648 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_TRANSFERRING_HOST_TO_CONTROLLER); 649 CU_ASSERT(rdma_req->req.xfer == SPDK_NVME_DATA_HOST_TO_CONTROLLER); 650 STAILQ_INIT(&poller.qpairs_pending_send); 651 /* READY_TO_EXECUTE -> EXECUTING */ 652 rdma_req->state = RDMA_REQUEST_STATE_READY_TO_EXECUTE; 653 progress = nvmf_rdma_request_process(&rtransport, rdma_req); 654 CU_ASSERT(progress == true); 655 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_EXECUTING); 656 /* EXECUTED -> COMPLETING */ 657 rdma_req->state = RDMA_REQUEST_STATE_EXECUTED; 658 progress = nvmf_rdma_request_process(&rtransport, rdma_req); 659 CU_ASSERT(progress == true); 660 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_COMPLETING); 661 CU_ASSERT(rdma_req->recv == NULL); 662 /* COMPLETED -> FREE */ 663 rdma_req->state = RDMA_REQUEST_STATE_COMPLETED; 664 progress = nvmf_rdma_request_process(&rtransport, rdma_req); 665 CU_ASSERT(progress == true); 666 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_FREE); 667 668 free_recv(rdma_recv); 669 free_req(rdma_req); 670 poller_reset(&poller, &group); 671 qpair_reset(&rqpair, &poller, &device, &resources, &rtransport.transport); 672 673 /* Test 3: WRITE+WRITE ibv_send batching */ 674 { 675 struct spdk_nvmf_rdma_recv *recv1, *recv2; 676 struct spdk_nvmf_rdma_request *req1, *req2; 677 recv1 = create_recv(&rqpair, SPDK_NVME_OPC_WRITE); 678 req1 = create_req(&rqpair, recv1); 679 recv2 = create_recv(&rqpair, SPDK_NVME_OPC_WRITE); 680 req2 = create_req(&rqpair, recv2); 681 682 /* WRITE 1: NEW -> TRANSFERRING_H2C */ 683 rqpair.current_recv_depth = 1; 684 nvmf_rdma_request_process(&rtransport, req1); 685 CU_ASSERT(req1->state == RDMA_REQUEST_STATE_TRANSFERRING_HOST_TO_CONTROLLER); 686 687 /* WRITE 2: NEW -> TRANSFERRING_H2C */ 688 rqpair.current_recv_depth = 2; 689 nvmf_rdma_request_process(&rtransport, req2); 690 CU_ASSERT(req2->state == RDMA_REQUEST_STATE_TRANSFERRING_HOST_TO_CONTROLLER); 691 692 STAILQ_INIT(&poller.qpairs_pending_send); 693 694 /* WRITE 1 completes before WRITE 2 has finished RDMA reading */ 695 /* WRITE 1: READY_TO_EXECUTE -> EXECUTING */ 696 req1->state = RDMA_REQUEST_STATE_READY_TO_EXECUTE; 697 nvmf_rdma_request_process(&rtransport, req1); 698 CU_ASSERT(req1->state == RDMA_REQUEST_STATE_EXECUTING); 699 /* WRITE 1: EXECUTED -> COMPLETING */ 700 req1->state = RDMA_REQUEST_STATE_EXECUTED; 701 nvmf_rdma_request_process(&rtransport, req1); 702 CU_ASSERT(req1->state == RDMA_REQUEST_STATE_COMPLETING); 703 STAILQ_INIT(&poller.qpairs_pending_send); 704 /* WRITE 1: COMPLETED -> FREE */ 705 req1->state = RDMA_REQUEST_STATE_COMPLETED; 706 nvmf_rdma_request_process(&rtransport, req1); 707 CU_ASSERT(req1->state == RDMA_REQUEST_STATE_FREE); 708 709 /* Now WRITE 2 has finished reading and completes */ 710 /* WRITE 2: COMPLETED -> FREE */ 711 /* WRITE 2: READY_TO_EXECUTE -> EXECUTING */ 712 req2->state = RDMA_REQUEST_STATE_READY_TO_EXECUTE; 713 nvmf_rdma_request_process(&rtransport, req2); 714 CU_ASSERT(req2->state == RDMA_REQUEST_STATE_EXECUTING); 715 /* WRITE 1: EXECUTED -> COMPLETING */ 716 req2->state = RDMA_REQUEST_STATE_EXECUTED; 717 nvmf_rdma_request_process(&rtransport, req2); 718 CU_ASSERT(req2->state == RDMA_REQUEST_STATE_COMPLETING); 719 STAILQ_INIT(&poller.qpairs_pending_send); 720 /* WRITE 1: COMPLETED -> FREE */ 721 req2->state = RDMA_REQUEST_STATE_COMPLETED; 722 nvmf_rdma_request_process(&rtransport, req2); 723 CU_ASSERT(req2->state == RDMA_REQUEST_STATE_FREE); 724 725 free_recv(recv1); 726 free_req(req1); 727 free_recv(recv2); 728 free_req(req2); 729 poller_reset(&poller, &group); 730 qpair_reset(&rqpair, &poller, &device, &resources, &rtransport.transport); 731 } 732 733 /* Test 4, invalid command, check xfer type */ 734 { 735 struct spdk_nvmf_rdma_recv *rdma_recv_inv; 736 struct spdk_nvmf_rdma_request *rdma_req_inv; 737 /* construct an opcode that specifies BIDIRECTIONAL transfer */ 738 uint8_t opc = 0x10 | SPDK_NVME_DATA_BIDIRECTIONAL; 739 740 rdma_recv_inv = create_recv(&rqpair, opc); 741 rdma_req_inv = create_req(&rqpair, rdma_recv_inv); 742 743 /* NEW -> RDMA_REQUEST_STATE_COMPLETING */ 744 rqpair.current_recv_depth = 1; 745 progress = nvmf_rdma_request_process(&rtransport, rdma_req_inv); 746 CU_ASSERT(progress == true); 747 CU_ASSERT(rdma_req_inv->state == RDMA_REQUEST_STATE_COMPLETING); 748 CU_ASSERT(rdma_req_inv->req.rsp->nvme_cpl.status.sct == SPDK_NVME_SCT_GENERIC); 749 CU_ASSERT(rdma_req_inv->req.rsp->nvme_cpl.status.sc == SPDK_NVME_SC_INVALID_OPCODE); 750 751 /* RDMA_REQUEST_STATE_COMPLETED -> FREE */ 752 rdma_req_inv->state = RDMA_REQUEST_STATE_COMPLETED; 753 nvmf_rdma_request_process(&rtransport, rdma_req_inv); 754 CU_ASSERT(rdma_req_inv->state == RDMA_REQUEST_STATE_FREE); 755 756 free_recv(rdma_recv_inv); 757 free_req(rdma_req_inv); 758 poller_reset(&poller, &group); 759 qpair_reset(&rqpair, &poller, &device, &resources, &rtransport.transport); 760 } 761 762 spdk_mempool_free(rtransport.transport.data_buf_pool); 763 spdk_mempool_free(rtransport.data_wr_pool); 764 } 765 766 #define TEST_GROUPS_COUNT 5 767 static void 768 test_nvmf_rdma_get_optimal_poll_group(void) 769 { 770 struct spdk_nvmf_rdma_transport rtransport = {}; 771 struct spdk_nvmf_transport *transport = &rtransport.transport; 772 struct spdk_nvmf_rdma_qpair rqpair = {}; 773 struct spdk_nvmf_transport_poll_group *groups[TEST_GROUPS_COUNT]; 774 struct spdk_nvmf_rdma_poll_group *rgroups[TEST_GROUPS_COUNT]; 775 struct spdk_nvmf_transport_poll_group *result; 776 uint32_t i; 777 778 rqpair.qpair.transport = transport; 779 pthread_mutex_init(&rtransport.lock, NULL); 780 TAILQ_INIT(&rtransport.poll_groups); 781 782 for (i = 0; i < TEST_GROUPS_COUNT; i++) { 783 groups[i] = nvmf_rdma_poll_group_create(transport); 784 CU_ASSERT(groups[i] != NULL); 785 rgroups[i] = SPDK_CONTAINEROF(groups[i], struct spdk_nvmf_rdma_poll_group, group); 786 groups[i]->transport = transport; 787 } 788 CU_ASSERT(rtransport.conn_sched.next_admin_pg == rgroups[0]); 789 CU_ASSERT(rtransport.conn_sched.next_io_pg == rgroups[0]); 790 791 /* Emulate connection of %TEST_GROUPS_COUNT% initiators - each creates 1 admin and 1 io qp */ 792 for (i = 0; i < TEST_GROUPS_COUNT; i++) { 793 rqpair.qpair.qid = 0; 794 result = nvmf_rdma_get_optimal_poll_group(&rqpair.qpair); 795 CU_ASSERT(result == groups[i]); 796 CU_ASSERT(rtransport.conn_sched.next_admin_pg == rgroups[(i + 1) % TEST_GROUPS_COUNT]); 797 CU_ASSERT(rtransport.conn_sched.next_io_pg == rgroups[i]); 798 799 rqpair.qpair.qid = 1; 800 result = nvmf_rdma_get_optimal_poll_group(&rqpair.qpair); 801 CU_ASSERT(result == groups[i]); 802 CU_ASSERT(rtransport.conn_sched.next_admin_pg == rgroups[(i + 1) % TEST_GROUPS_COUNT]); 803 CU_ASSERT(rtransport.conn_sched.next_io_pg == rgroups[(i + 1) % TEST_GROUPS_COUNT]); 804 } 805 /* wrap around, admin/io pg point to the first pg 806 Destroy all poll groups except of the last one */ 807 for (i = 0; i < TEST_GROUPS_COUNT - 1; i++) { 808 nvmf_rdma_poll_group_destroy(groups[i]); 809 CU_ASSERT(rtransport.conn_sched.next_admin_pg == rgroups[i + 1]); 810 CU_ASSERT(rtransport.conn_sched.next_io_pg == rgroups[i + 1]); 811 } 812 813 CU_ASSERT(rtransport.conn_sched.next_admin_pg == rgroups[TEST_GROUPS_COUNT - 1]); 814 CU_ASSERT(rtransport.conn_sched.next_io_pg == rgroups[TEST_GROUPS_COUNT - 1]); 815 816 /* Check that pointers to the next admin/io poll groups are not changed */ 817 rqpair.qpair.qid = 0; 818 result = nvmf_rdma_get_optimal_poll_group(&rqpair.qpair); 819 CU_ASSERT(result == groups[TEST_GROUPS_COUNT - 1]); 820 CU_ASSERT(rtransport.conn_sched.next_admin_pg == rgroups[TEST_GROUPS_COUNT - 1]); 821 CU_ASSERT(rtransport.conn_sched.next_io_pg == rgroups[TEST_GROUPS_COUNT - 1]); 822 823 rqpair.qpair.qid = 1; 824 result = nvmf_rdma_get_optimal_poll_group(&rqpair.qpair); 825 CU_ASSERT(result == groups[TEST_GROUPS_COUNT - 1]); 826 CU_ASSERT(rtransport.conn_sched.next_admin_pg == rgroups[TEST_GROUPS_COUNT - 1]); 827 CU_ASSERT(rtransport.conn_sched.next_io_pg == rgroups[TEST_GROUPS_COUNT - 1]); 828 829 /* Remove the last poll group, check that pointers are NULL */ 830 nvmf_rdma_poll_group_destroy(groups[TEST_GROUPS_COUNT - 1]); 831 CU_ASSERT(rtransport.conn_sched.next_admin_pg == NULL); 832 CU_ASSERT(rtransport.conn_sched.next_io_pg == NULL); 833 834 /* Request optimal poll group, result must be NULL */ 835 rqpair.qpair.qid = 0; 836 result = nvmf_rdma_get_optimal_poll_group(&rqpair.qpair); 837 CU_ASSERT(result == NULL); 838 839 rqpair.qpair.qid = 1; 840 result = nvmf_rdma_get_optimal_poll_group(&rqpair.qpair); 841 CU_ASSERT(result == NULL); 842 843 pthread_mutex_destroy(&rtransport.lock); 844 } 845 #undef TEST_GROUPS_COUNT 846 847 static void 848 test_spdk_nvmf_rdma_request_parse_sgl_with_md(void) 849 { 850 struct spdk_nvmf_rdma_transport rtransport; 851 struct spdk_nvmf_rdma_device device; 852 struct spdk_nvmf_rdma_request rdma_req = {}; 853 struct spdk_nvmf_rdma_recv recv; 854 struct spdk_nvmf_rdma_poll_group group; 855 struct spdk_nvmf_rdma_qpair rqpair; 856 struct spdk_nvmf_rdma_poller poller; 857 union nvmf_c2h_msg cpl; 858 union nvmf_h2c_msg cmd; 859 struct spdk_nvme_sgl_descriptor *sgl; 860 struct spdk_nvme_sgl_descriptor sgl_desc[SPDK_NVMF_MAX_SGL_ENTRIES] = {{0}}; 861 char data_buffer[8192]; 862 struct spdk_nvmf_rdma_request_data *data = (struct spdk_nvmf_rdma_request_data *)data_buffer; 863 char data2_buffer[8192]; 864 struct spdk_nvmf_rdma_request_data *data2 = (struct spdk_nvmf_rdma_request_data *)data2_buffer; 865 const uint32_t data_bs = 512; 866 const uint32_t md_size = 8; 867 int rc, i; 868 void *aligned_buffer; 869 870 data->wr.sg_list = data->sgl; 871 STAILQ_INIT(&group.group.buf_cache); 872 group.group.buf_cache_size = 0; 873 group.group.buf_cache_count = 0; 874 group.group.transport = &rtransport.transport; 875 poller.group = &group; 876 rqpair.poller = &poller; 877 rqpair.max_send_sge = SPDK_NVMF_MAX_SGL_ENTRIES; 878 879 sgl = &cmd.nvme_cmd.dptr.sgl1; 880 rdma_req.recv = &recv; 881 rdma_req.req.cmd = &cmd; 882 rdma_req.req.rsp = &cpl; 883 rdma_req.data.wr.sg_list = rdma_req.data.sgl; 884 rdma_req.req.qpair = &rqpair.qpair; 885 rdma_req.req.xfer = SPDK_NVME_DATA_CONTROLLER_TO_HOST; 886 887 rtransport.transport.opts = g_rdma_ut_transport_opts; 888 rtransport.data_wr_pool = NULL; 889 rtransport.transport.data_buf_pool = NULL; 890 891 device.attr.device_cap_flags = 0; 892 device.map = NULL; 893 sgl->keyed.key = 0xEEEE; 894 sgl->address = 0xFFFF; 895 rdma_req.recv->buf = (void *)0xDDDD; 896 897 /* Test 1: sgl type: keyed data block subtype: address */ 898 sgl->generic.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; 899 sgl->keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; 900 901 /* Part 1: simple I/O, one SGL smaller than the transport io unit size, block size 512 */ 902 MOCK_SET(spdk_mempool_get, (void *)0x2000); 903 reset_nvmf_rdma_request(&rdma_req); 904 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 905 SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 906 0, 0, 0, 0, 0); 907 rdma_req.req.dif_enabled = true; 908 rtransport.transport.opts.io_unit_size = data_bs * 8; 909 sgl->keyed.length = data_bs * 4; 910 911 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 912 913 CU_ASSERT(rc == 0); 914 CU_ASSERT(rdma_req.req.data_from_pool == true); 915 CU_ASSERT(rdma_req.req.length == data_bs * 4); 916 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 917 CU_ASSERT(rdma_req.req.dif.elba_length == (data_bs + md_size) * 4); 918 CU_ASSERT((uint64_t)rdma_req.req.data == 0x2000); 919 CU_ASSERT(rdma_req.data.wr.num_sge == 4); 920 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 921 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 922 CU_ASSERT((uint64_t)rdma_req.req.buffers[0] == 0x2000); 923 924 for (i = 0; i < 4; ++i) { 925 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == 0x2000 + i * (data_bs + md_size)); 926 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == data_bs); 927 CU_ASSERT(rdma_req.data.wr.sg_list[i].lkey == RDMA_UT_LKEY); 928 } 929 930 /* Part 2: simple I/O, one SGL equal to io unit size, io_unit_size is not aligned with md_size, 931 block size 512 */ 932 MOCK_SET(spdk_mempool_get, (void *)0x2000); 933 reset_nvmf_rdma_request(&rdma_req); 934 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 935 SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 936 0, 0, 0, 0, 0); 937 rdma_req.req.dif_enabled = true; 938 rtransport.transport.opts.io_unit_size = data_bs * 4; 939 sgl->keyed.length = data_bs * 4; 940 941 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 942 943 CU_ASSERT(rc == 0); 944 CU_ASSERT(rdma_req.req.data_from_pool == true); 945 CU_ASSERT(rdma_req.req.length == data_bs * 4); 946 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 947 CU_ASSERT(rdma_req.req.dif.elba_length == (data_bs + md_size) * 4); 948 CU_ASSERT((uint64_t)rdma_req.req.data == 0x2000); 949 CU_ASSERT(rdma_req.data.wr.num_sge == 5); 950 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 951 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 952 CU_ASSERT((uint64_t)rdma_req.req.buffers[0] == 0x2000); 953 954 for (i = 0; i < 3; ++i) { 955 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == 0x2000 + i * (data_bs + md_size)); 956 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == data_bs); 957 CU_ASSERT(rdma_req.data.wr.sg_list[i].lkey == RDMA_UT_LKEY); 958 } 959 CU_ASSERT(rdma_req.data.wr.sg_list[3].addr == 0x2000 + 3 * (data_bs + md_size)); 960 CU_ASSERT(rdma_req.data.wr.sg_list[3].length == 488); 961 CU_ASSERT(rdma_req.data.wr.sg_list[3].lkey == RDMA_UT_LKEY); 962 963 /* 2nd buffer consumed */ 964 CU_ASSERT(rdma_req.data.wr.sg_list[4].addr == 0x2000); 965 CU_ASSERT(rdma_req.data.wr.sg_list[4].length == 24); 966 CU_ASSERT(rdma_req.data.wr.sg_list[4].lkey == RDMA_UT_LKEY); 967 968 /* Part 3: simple I/O, one SGL equal io unit size, io_unit_size is equal to block size 512 bytes */ 969 MOCK_SET(spdk_mempool_get, (void *)0x2000); 970 reset_nvmf_rdma_request(&rdma_req); 971 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 972 SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 973 0, 0, 0, 0, 0); 974 rdma_req.req.dif_enabled = true; 975 rtransport.transport.opts.io_unit_size = data_bs; 976 sgl->keyed.length = data_bs; 977 978 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 979 980 CU_ASSERT(rc == 0); 981 CU_ASSERT(rdma_req.req.data_from_pool == true); 982 CU_ASSERT(rdma_req.req.length == data_bs); 983 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 984 CU_ASSERT(rdma_req.req.dif.elba_length == data_bs + md_size); 985 CU_ASSERT((uint64_t)rdma_req.req.data == 0x2000); 986 CU_ASSERT(rdma_req.data.wr.num_sge == 1); 987 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 988 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 989 CU_ASSERT((uint64_t)rdma_req.req.buffers[0] == 0x2000); 990 991 CU_ASSERT(rdma_req.data.wr.sg_list[0].addr == 0x2000); 992 CU_ASSERT(rdma_req.data.wr.sg_list[0].length == data_bs); 993 CU_ASSERT(rdma_req.data.wr.sg_list[0].lkey == RDMA_UT_LKEY); 994 995 CU_ASSERT(rdma_req.req.iovcnt == 2); 996 CU_ASSERT(rdma_req.req.iov[0].iov_base == (void *)((unsigned long)0x2000)); 997 CU_ASSERT(rdma_req.req.iov[0].iov_len == data_bs); 998 /* 2nd buffer consumed for metadata */ 999 CU_ASSERT(rdma_req.req.iov[1].iov_base == (void *)((unsigned long)0x2000)); 1000 CU_ASSERT(rdma_req.req.iov[1].iov_len == md_size); 1001 1002 /* Part 4: simple I/O, one SGL equal io unit size, io_unit_size is aligned with md_size, 1003 block size 512 */ 1004 MOCK_SET(spdk_mempool_get, (void *)0x2000); 1005 reset_nvmf_rdma_request(&rdma_req); 1006 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 1007 SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 1008 0, 0, 0, 0, 0); 1009 rdma_req.req.dif_enabled = true; 1010 rtransport.transport.opts.io_unit_size = (data_bs + md_size) * 4; 1011 sgl->keyed.length = data_bs * 4; 1012 1013 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 1014 1015 CU_ASSERT(rc == 0); 1016 CU_ASSERT(rdma_req.req.data_from_pool == true); 1017 CU_ASSERT(rdma_req.req.length == data_bs * 4); 1018 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 1019 CU_ASSERT(rdma_req.req.dif.elba_length == (data_bs + md_size) * 4); 1020 CU_ASSERT((uint64_t)rdma_req.req.data == 0x2000); 1021 CU_ASSERT(rdma_req.data.wr.num_sge == 4); 1022 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 1023 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 1024 CU_ASSERT((uint64_t)rdma_req.req.buffers[0] == 0x2000); 1025 1026 for (i = 0; i < 4; ++i) { 1027 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == 0x2000 + i * (data_bs + md_size)); 1028 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == data_bs); 1029 CU_ASSERT(rdma_req.data.wr.sg_list[i].lkey == RDMA_UT_LKEY); 1030 } 1031 1032 /* Part 5: simple I/O, one SGL equal to 2x io unit size, io_unit_size is aligned with md_size, 1033 block size 512 */ 1034 MOCK_SET(spdk_mempool_get, (void *)0x2000); 1035 reset_nvmf_rdma_request(&rdma_req); 1036 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 1037 SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 1038 0, 0, 0, 0, 0); 1039 rdma_req.req.dif_enabled = true; 1040 rtransport.transport.opts.io_unit_size = (data_bs + md_size) * 2; 1041 sgl->keyed.length = data_bs * 4; 1042 1043 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 1044 1045 CU_ASSERT(rc == 0); 1046 CU_ASSERT(rdma_req.req.data_from_pool == true); 1047 CU_ASSERT(rdma_req.req.length == data_bs * 4); 1048 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 1049 CU_ASSERT(rdma_req.req.dif.elba_length == (data_bs + md_size) * 4); 1050 CU_ASSERT((uint64_t)rdma_req.req.data == 0x2000); 1051 CU_ASSERT(rdma_req.data.wr.num_sge == 4); 1052 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 1053 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 1054 CU_ASSERT((uint64_t)rdma_req.req.buffers[0] == 0x2000); 1055 1056 for (i = 0; i < 2; ++i) { 1057 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == 0x2000 + i * (data_bs + md_size)); 1058 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == data_bs); 1059 } 1060 for (i = 0; i < 2; ++i) { 1061 CU_ASSERT(rdma_req.data.wr.sg_list[i + 2].addr == 0x2000 + i * (data_bs + md_size)); 1062 CU_ASSERT(rdma_req.data.wr.sg_list[i + 2].length == data_bs); 1063 } 1064 1065 /* Part 6: simple I/O, one SGL larger than the transport io unit size, io_unit_size is not aligned to md_size, 1066 block size 512 */ 1067 MOCK_SET(spdk_mempool_get, (void *)0x2000); 1068 reset_nvmf_rdma_request(&rdma_req); 1069 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 1070 SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 1071 0, 0, 0, 0, 0); 1072 rdma_req.req.dif_enabled = true; 1073 rtransport.transport.opts.io_unit_size = data_bs * 4; 1074 sgl->keyed.length = data_bs * 6; 1075 1076 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 1077 1078 CU_ASSERT(rc == 0); 1079 CU_ASSERT(rdma_req.req.data_from_pool == true); 1080 CU_ASSERT(rdma_req.req.length == data_bs * 6); 1081 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 1082 CU_ASSERT(rdma_req.req.dif.elba_length == (data_bs + md_size) * 6); 1083 CU_ASSERT((uint64_t)rdma_req.req.data == 0x2000); 1084 CU_ASSERT(rdma_req.data.wr.num_sge == 7); 1085 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 1086 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 1087 CU_ASSERT((uint64_t)rdma_req.req.buffers[0] == 0x2000); 1088 1089 for (i = 0; i < 3; ++i) { 1090 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == 0x2000 + i * (data_bs + md_size)); 1091 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == data_bs); 1092 CU_ASSERT(rdma_req.data.wr.sg_list[i].lkey == RDMA_UT_LKEY); 1093 } 1094 CU_ASSERT(rdma_req.data.wr.sg_list[3].addr == 0x2000 + 3 * (data_bs + md_size)); 1095 CU_ASSERT(rdma_req.data.wr.sg_list[3].length == 488); 1096 CU_ASSERT(rdma_req.data.wr.sg_list[3].lkey == RDMA_UT_LKEY); 1097 1098 /* 2nd IO buffer consumed */ 1099 CU_ASSERT(rdma_req.data.wr.sg_list[4].addr == 0x2000); 1100 CU_ASSERT(rdma_req.data.wr.sg_list[4].length == 24); 1101 CU_ASSERT(rdma_req.data.wr.sg_list[4].lkey == RDMA_UT_LKEY); 1102 1103 CU_ASSERT(rdma_req.data.wr.sg_list[5].addr == 0x2000 + 24 + md_size); 1104 CU_ASSERT(rdma_req.data.wr.sg_list[5].length == 512); 1105 CU_ASSERT(rdma_req.data.wr.sg_list[5].lkey == RDMA_UT_LKEY); 1106 1107 CU_ASSERT(rdma_req.data.wr.sg_list[6].addr == 0x2000 + 24 + 512 + md_size * 2); 1108 CU_ASSERT(rdma_req.data.wr.sg_list[6].length == 512); 1109 CU_ASSERT(rdma_req.data.wr.sg_list[6].lkey == RDMA_UT_LKEY); 1110 1111 /* Part 7: simple I/O, number of SGL entries exceeds the number of entries 1112 one WR can hold. Additional WR is chained */ 1113 MOCK_SET(spdk_mempool_get, data2_buffer); 1114 aligned_buffer = (void *)((uintptr_t)(data2_buffer + NVMF_DATA_BUFFER_MASK) & 1115 ~NVMF_DATA_BUFFER_MASK); 1116 reset_nvmf_rdma_request(&rdma_req); 1117 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 1118 SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 1119 0, 0, 0, 0, 0); 1120 rdma_req.req.dif_enabled = true; 1121 rtransport.transport.opts.io_unit_size = data_bs * 16; 1122 sgl->keyed.length = data_bs * 16; 1123 1124 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 1125 1126 CU_ASSERT(rc == 0); 1127 CU_ASSERT(rdma_req.req.data_from_pool == true); 1128 CU_ASSERT(rdma_req.req.length == data_bs * 16); 1129 CU_ASSERT(rdma_req.req.iovcnt == 2); 1130 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 1131 CU_ASSERT(rdma_req.req.dif.elba_length == (data_bs + md_size) * 16); 1132 CU_ASSERT(rdma_req.req.data == aligned_buffer); 1133 CU_ASSERT(rdma_req.data.wr.num_sge == 16); 1134 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 1135 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 1136 1137 for (i = 0; i < 15; ++i) { 1138 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == (uintptr_t)aligned_buffer + i * (data_bs + md_size)); 1139 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == data_bs); 1140 CU_ASSERT(rdma_req.data.wr.sg_list[i].lkey == RDMA_UT_LKEY); 1141 } 1142 1143 /* 8192 - (512 + 8) * 15 = 392 */ 1144 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == (uintptr_t)aligned_buffer + i * (data_bs + md_size)); 1145 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == 392); 1146 CU_ASSERT(rdma_req.data.wr.sg_list[i].lkey == RDMA_UT_LKEY); 1147 1148 /* additional wr from pool */ 1149 CU_ASSERT(rdma_req.data.wr.next == (void *)&data2->wr); 1150 CU_ASSERT(rdma_req.data.wr.next->num_sge == 1); 1151 CU_ASSERT(rdma_req.data.wr.next->next == &rdma_req.rsp.wr); 1152 /* 2nd IO buffer */ 1153 CU_ASSERT(data2->wr.sg_list[0].addr == (uintptr_t)aligned_buffer); 1154 CU_ASSERT(data2->wr.sg_list[0].length == 120); 1155 CU_ASSERT(data2->wr.sg_list[0].lkey == RDMA_UT_LKEY); 1156 1157 /* Part 8: simple I/O, data with metadata do not fit to 1 io_buffer */ 1158 MOCK_SET(spdk_mempool_get, (void *)0x2000); 1159 reset_nvmf_rdma_request(&rdma_req); 1160 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 1161 SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 1162 0, 0, 0, 0, 0); 1163 rdma_req.req.dif_enabled = true; 1164 rtransport.transport.opts.io_unit_size = 516; 1165 sgl->keyed.length = data_bs * 2; 1166 1167 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 1168 1169 CU_ASSERT(rc == 0); 1170 CU_ASSERT(rdma_req.req.data_from_pool == true); 1171 CU_ASSERT(rdma_req.req.length == data_bs * 2); 1172 CU_ASSERT(rdma_req.req.iovcnt == 3); 1173 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 1174 CU_ASSERT(rdma_req.req.dif.elba_length == (data_bs + md_size) * 2); 1175 CU_ASSERT(rdma_req.req.data == (void *)0x2000); 1176 CU_ASSERT(rdma_req.data.wr.num_sge == 2); 1177 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 1178 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 1179 1180 CU_ASSERT(rdma_req.data.wr.sg_list[0].addr == 0x2000); 1181 CU_ASSERT(rdma_req.data.wr.sg_list[0].length == 512); 1182 CU_ASSERT(rdma_req.data.wr.sg_list[0].lkey == RDMA_UT_LKEY); 1183 1184 /* 2nd IO buffer consumed, offset 4 bytes due to part of the metadata 1185 is located at the beginning of that buffer */ 1186 CU_ASSERT(rdma_req.data.wr.sg_list[1].addr == 0x2000 + 4); 1187 CU_ASSERT(rdma_req.data.wr.sg_list[1].length == 512); 1188 CU_ASSERT(rdma_req.data.wr.sg_list[1].lkey == RDMA_UT_LKEY); 1189 1190 /* Test 2: Multi SGL */ 1191 sgl->generic.type = SPDK_NVME_SGL_TYPE_LAST_SEGMENT; 1192 sgl->unkeyed.subtype = SPDK_NVME_SGL_SUBTYPE_OFFSET; 1193 sgl->address = 0; 1194 rdma_req.recv->buf = (void *)&sgl_desc; 1195 MOCK_SET(spdk_mempool_get, data_buffer); 1196 aligned_buffer = (void *)((uintptr_t)(data_buffer + NVMF_DATA_BUFFER_MASK) & 1197 ~NVMF_DATA_BUFFER_MASK); 1198 1199 /* part 1: 2 segments each with 1 wr. io_unit_size is aligned with data_bs + md_size */ 1200 reset_nvmf_rdma_request(&rdma_req); 1201 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 1202 SPDK_DIF_TYPE1, 1203 SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 0, 0, 0, 0, 0); 1204 rdma_req.req.dif_enabled = true; 1205 rtransport.transport.opts.io_unit_size = (data_bs + md_size) * 4; 1206 sgl->unkeyed.length = 2 * sizeof(struct spdk_nvme_sgl_descriptor); 1207 1208 for (i = 0; i < 2; i++) { 1209 sgl_desc[i].keyed.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; 1210 sgl_desc[i].keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; 1211 sgl_desc[i].keyed.length = data_bs * 4; 1212 sgl_desc[i].address = 0x4000 + i * data_bs * 4; 1213 sgl_desc[i].keyed.key = 0x44; 1214 } 1215 1216 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 1217 1218 CU_ASSERT(rc == 0); 1219 CU_ASSERT(rdma_req.req.data_from_pool == true); 1220 CU_ASSERT(rdma_req.req.length == data_bs * 4 * 2); 1221 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 1222 CU_ASSERT(rdma_req.req.dif.elba_length == (data_bs + md_size) * 4 * 2); 1223 CU_ASSERT(rdma_req.data.wr.num_sge == 4); 1224 for (i = 0; i < 4; ++i) { 1225 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == (uintptr_t)((unsigned char *)aligned_buffer) + i * 1226 (data_bs + md_size)); 1227 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == data_bs); 1228 } 1229 1230 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0x44); 1231 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0x4000); 1232 CU_ASSERT(rdma_req.data.wr.next == &data->wr); 1233 CU_ASSERT(data->wr.wr.rdma.rkey == 0x44); 1234 CU_ASSERT(data->wr.wr.rdma.remote_addr == 0x4000 + data_bs * 4); 1235 CU_ASSERT(data->wr.num_sge == 4); 1236 for (i = 0; i < 4; ++i) { 1237 CU_ASSERT(data->wr.sg_list[i].addr == (uintptr_t)((unsigned char *)aligned_buffer) + i * 1238 (data_bs + md_size)); 1239 CU_ASSERT(data->wr.sg_list[i].length == data_bs); 1240 } 1241 1242 CU_ASSERT(data->wr.next == &rdma_req.rsp.wr); 1243 } 1244 1245 int main(int argc, char **argv) 1246 { 1247 CU_pSuite suite = NULL; 1248 unsigned int num_failures; 1249 1250 CU_set_error_action(CUEA_ABORT); 1251 CU_initialize_registry(); 1252 1253 suite = CU_add_suite("nvmf", NULL, NULL); 1254 1255 CU_ADD_TEST(suite, test_spdk_nvmf_rdma_request_parse_sgl); 1256 CU_ADD_TEST(suite, test_spdk_nvmf_rdma_request_process); 1257 CU_ADD_TEST(suite, test_nvmf_rdma_get_optimal_poll_group); 1258 CU_ADD_TEST(suite, test_spdk_nvmf_rdma_request_parse_sgl_with_md); 1259 1260 CU_basic_set_mode(CU_BRM_VERBOSE); 1261 CU_basic_run_tests(); 1262 num_failures = CU_get_number_of_failures(); 1263 CU_cleanup_registry(); 1264 return num_failures; 1265 } 1266