1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. All rights reserved. 5 * Copyright (c) 2019 Mellanox Technologies LTD. All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "spdk/stdinc.h" 35 #include "spdk_cunit.h" 36 #include "common/lib/test_env.c" 37 #include "common/lib/test_rdma.c" 38 #include "nvmf/rdma.c" 39 #include "nvmf/transport.c" 40 41 uint64_t g_mr_size; 42 uint64_t g_mr_next_size; 43 struct ibv_mr g_rdma_mr; 44 45 #define RDMA_UT_UNITS_IN_MAX_IO 16 46 47 struct spdk_nvmf_transport_opts g_rdma_ut_transport_opts = { 48 .max_queue_depth = SPDK_NVMF_RDMA_DEFAULT_MAX_QUEUE_DEPTH, 49 .max_qpairs_per_ctrlr = SPDK_NVMF_RDMA_DEFAULT_MAX_QPAIRS_PER_CTRLR, 50 .in_capsule_data_size = SPDK_NVMF_RDMA_DEFAULT_IN_CAPSULE_DATA_SIZE, 51 .max_io_size = (SPDK_NVMF_RDMA_MIN_IO_BUFFER_SIZE * RDMA_UT_UNITS_IN_MAX_IO), 52 .io_unit_size = SPDK_NVMF_RDMA_MIN_IO_BUFFER_SIZE, 53 .max_aq_depth = SPDK_NVMF_RDMA_DEFAULT_AQ_DEPTH, 54 .num_shared_buffers = SPDK_NVMF_RDMA_DEFAULT_NUM_SHARED_BUFFERS, 55 }; 56 57 SPDK_LOG_REGISTER_COMPONENT("nvmf", SPDK_LOG_NVMF) 58 DEFINE_STUB(spdk_mem_map_set_translation, int, (struct spdk_mem_map *map, uint64_t vaddr, 59 uint64_t size, uint64_t translation), 0); 60 DEFINE_STUB(spdk_mem_map_clear_translation, int, (struct spdk_mem_map *map, uint64_t vaddr, 61 uint64_t size), 0); 62 DEFINE_STUB(spdk_mem_map_alloc, struct spdk_mem_map *, (uint64_t default_translation, 63 const struct spdk_mem_map_ops *ops, void *cb_ctx), NULL); 64 DEFINE_STUB(spdk_nvmf_qpair_disconnect, int, (struct spdk_nvmf_qpair *qpair, 65 nvmf_qpair_disconnect_cb cb_fn, void *ctx), 0); 66 DEFINE_STUB_V(spdk_mem_map_free, (struct spdk_mem_map **pmap)); 67 68 struct spdk_trace_histories *g_trace_histories; 69 DEFINE_STUB_V(spdk_trace_add_register_fn, (struct spdk_trace_register_fn *reg_fn)); 70 DEFINE_STUB_V(spdk_trace_register_object, (uint8_t type, char id_prefix)); 71 DEFINE_STUB_V(spdk_trace_register_description, (const char *name, 72 uint16_t tpoint_id, uint8_t owner_type, uint8_t object_type, uint8_t new_object, 73 uint8_t arg1_type, const char *arg1_name)); 74 DEFINE_STUB_V(_spdk_trace_record, (uint64_t tsc, uint16_t tpoint_id, uint16_t poller_id, 75 uint32_t size, uint64_t object_id, uint64_t arg1)); 76 77 DEFINE_STUB_V(spdk_nvmf_ctrlr_data_init, (struct spdk_nvmf_transport_opts *opts, 78 struct spdk_nvmf_ctrlr_data *cdata)); 79 DEFINE_STUB_V(spdk_nvmf_request_exec, (struct spdk_nvmf_request *req)); 80 DEFINE_STUB(spdk_nvme_transport_id_compare, int, (const struct spdk_nvme_transport_id *trid1, 81 const struct spdk_nvme_transport_id *trid2), 0); 82 DEFINE_STUB_V(nvmf_ctrlr_abort_aer, (struct spdk_nvmf_ctrlr *ctrlr)); 83 DEFINE_STUB(spdk_nvmf_request_get_dif_ctx, bool, (struct spdk_nvmf_request *req, 84 struct spdk_dif_ctx *dif_ctx), false); 85 DEFINE_STUB_V(spdk_nvme_trid_populate_transport, (struct spdk_nvme_transport_id *trid, 86 enum spdk_nvme_transport_type trtype)); 87 88 const char * 89 spdk_nvme_transport_id_trtype_str(enum spdk_nvme_transport_type trtype) 90 { 91 switch (trtype) { 92 case SPDK_NVME_TRANSPORT_PCIE: 93 return "PCIe"; 94 case SPDK_NVME_TRANSPORT_RDMA: 95 return "RDMA"; 96 case SPDK_NVME_TRANSPORT_FC: 97 return "FC"; 98 default: 99 return NULL; 100 } 101 } 102 103 int 104 spdk_nvme_transport_id_populate_trstring(struct spdk_nvme_transport_id *trid, const char *trstring) 105 { 106 int len, i; 107 108 if (trstring == NULL) { 109 return -EINVAL; 110 } 111 112 len = strnlen(trstring, SPDK_NVMF_TRSTRING_MAX_LEN); 113 if (len == SPDK_NVMF_TRSTRING_MAX_LEN) { 114 return -EINVAL; 115 } 116 117 /* cast official trstring to uppercase version of input. */ 118 for (i = 0; i < len; i++) { 119 trid->trstring[i] = toupper(trstring[i]); 120 } 121 return 0; 122 } 123 124 uint64_t 125 spdk_mem_map_translate(const struct spdk_mem_map *map, uint64_t vaddr, uint64_t *size) 126 { 127 if (g_mr_size != 0) { 128 *(uint32_t *)size = g_mr_size; 129 if (g_mr_next_size != 0) { 130 g_mr_size = g_mr_next_size; 131 } 132 } 133 134 return (uint64_t)&g_rdma_mr; 135 } 136 137 static void reset_nvmf_rdma_request(struct spdk_nvmf_rdma_request *rdma_req) 138 { 139 int i; 140 141 rdma_req->req.length = 0; 142 rdma_req->req.data_from_pool = false; 143 rdma_req->req.data = NULL; 144 rdma_req->data.wr.num_sge = 0; 145 rdma_req->data.wr.wr.rdma.remote_addr = 0; 146 rdma_req->data.wr.wr.rdma.rkey = 0; 147 memset(&rdma_req->req.dif, 0, sizeof(rdma_req->req.dif)); 148 149 for (i = 0; i < SPDK_NVMF_MAX_SGL_ENTRIES; i++) { 150 rdma_req->req.iov[i].iov_base = 0; 151 rdma_req->req.iov[i].iov_len = 0; 152 rdma_req->req.buffers[i] = 0; 153 rdma_req->data.wr.sg_list[i].addr = 0; 154 rdma_req->data.wr.sg_list[i].length = 0; 155 rdma_req->data.wr.sg_list[i].lkey = 0; 156 } 157 rdma_req->req.iovcnt = 0; 158 } 159 160 static void 161 test_spdk_nvmf_rdma_request_parse_sgl(void) 162 { 163 struct spdk_nvmf_rdma_transport rtransport; 164 struct spdk_nvmf_rdma_device device; 165 struct spdk_nvmf_rdma_request rdma_req = {}; 166 struct spdk_nvmf_rdma_recv recv; 167 struct spdk_nvmf_rdma_poll_group group; 168 struct spdk_nvmf_rdma_qpair rqpair; 169 struct spdk_nvmf_rdma_poller poller; 170 union nvmf_c2h_msg cpl; 171 union nvmf_h2c_msg cmd; 172 struct spdk_nvme_sgl_descriptor *sgl; 173 struct spdk_nvmf_transport_pg_cache_buf bufs[4]; 174 struct spdk_nvme_sgl_descriptor sgl_desc[SPDK_NVMF_MAX_SGL_ENTRIES] = {{0}}; 175 struct spdk_nvmf_rdma_request_data data; 176 struct spdk_nvmf_transport_pg_cache_buf buffer; 177 struct spdk_nvmf_transport_pg_cache_buf *buffer_ptr; 178 int rc, i; 179 180 data.wr.sg_list = data.sgl; 181 STAILQ_INIT(&group.group.buf_cache); 182 group.group.buf_cache_size = 0; 183 group.group.buf_cache_count = 0; 184 group.group.transport = &rtransport.transport; 185 STAILQ_INIT(&group.retired_bufs); 186 poller.group = &group; 187 rqpair.poller = &poller; 188 rqpair.max_send_sge = SPDK_NVMF_MAX_SGL_ENTRIES; 189 190 sgl = &cmd.nvme_cmd.dptr.sgl1; 191 rdma_req.recv = &recv; 192 rdma_req.req.cmd = &cmd; 193 rdma_req.req.rsp = &cpl; 194 rdma_req.data.wr.sg_list = rdma_req.data.sgl; 195 rdma_req.req.qpair = &rqpair.qpair; 196 rdma_req.req.xfer = SPDK_NVME_DATA_CONTROLLER_TO_HOST; 197 198 rtransport.transport.opts = g_rdma_ut_transport_opts; 199 rtransport.data_wr_pool = NULL; 200 rtransport.transport.data_buf_pool = NULL; 201 202 device.attr.device_cap_flags = 0; 203 g_rdma_mr.lkey = 0xABCD; 204 sgl->keyed.key = 0xEEEE; 205 sgl->address = 0xFFFF; 206 rdma_req.recv->buf = (void *)0xDDDD; 207 208 /* Test 1: sgl type: keyed data block subtype: address */ 209 sgl->generic.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; 210 sgl->keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; 211 212 /* Part 1: simple I/O, one SGL smaller than the transport io unit size */ 213 MOCK_SET(spdk_mempool_get, (void *)0x2000); 214 reset_nvmf_rdma_request(&rdma_req); 215 sgl->keyed.length = rtransport.transport.opts.io_unit_size / 2; 216 217 device.map = (void *)0x0; 218 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 219 CU_ASSERT(rc == 0); 220 CU_ASSERT(rdma_req.req.data_from_pool == true); 221 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size / 2); 222 CU_ASSERT((uint64_t)rdma_req.req.data == 0x2000); 223 CU_ASSERT(rdma_req.data.wr.num_sge == 1); 224 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 225 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 226 CU_ASSERT((uint64_t)rdma_req.req.buffers[0] == 0x2000); 227 CU_ASSERT(rdma_req.data.wr.sg_list[0].addr == 0x2000); 228 CU_ASSERT(rdma_req.data.wr.sg_list[0].length == rtransport.transport.opts.io_unit_size / 2); 229 CU_ASSERT(rdma_req.data.wr.sg_list[0].lkey == g_rdma_mr.lkey); 230 231 /* Part 2: simple I/O, one SGL larger than the transport io unit size (equal to the max io size) */ 232 reset_nvmf_rdma_request(&rdma_req); 233 sgl->keyed.length = rtransport.transport.opts.io_unit_size * RDMA_UT_UNITS_IN_MAX_IO; 234 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 235 236 CU_ASSERT(rc == 0); 237 CU_ASSERT(rdma_req.req.data_from_pool == true); 238 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size * RDMA_UT_UNITS_IN_MAX_IO); 239 CU_ASSERT(rdma_req.data.wr.num_sge == RDMA_UT_UNITS_IN_MAX_IO); 240 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 241 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 242 for (i = 0; i < RDMA_UT_UNITS_IN_MAX_IO; i++) { 243 CU_ASSERT((uint64_t)rdma_req.req.buffers[i] == 0x2000); 244 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == 0x2000); 245 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == rtransport.transport.opts.io_unit_size); 246 CU_ASSERT(rdma_req.data.wr.sg_list[i].lkey == g_rdma_mr.lkey); 247 } 248 249 /* Part 3: simple I/O one SGL larger than the transport max io size */ 250 reset_nvmf_rdma_request(&rdma_req); 251 sgl->keyed.length = rtransport.transport.opts.max_io_size * 2; 252 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 253 254 CU_ASSERT(rc == -1); 255 256 /* Part 4: Pretend there are no buffer pools */ 257 MOCK_SET(spdk_mempool_get, NULL); 258 reset_nvmf_rdma_request(&rdma_req); 259 sgl->keyed.length = rtransport.transport.opts.io_unit_size * RDMA_UT_UNITS_IN_MAX_IO; 260 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 261 262 CU_ASSERT(rc == 0); 263 CU_ASSERT(rdma_req.req.data_from_pool == false); 264 CU_ASSERT(rdma_req.req.data == NULL); 265 CU_ASSERT(rdma_req.data.wr.num_sge == 0); 266 CU_ASSERT(rdma_req.req.buffers[0] == NULL); 267 CU_ASSERT(rdma_req.data.wr.sg_list[0].addr == 0); 268 CU_ASSERT(rdma_req.data.wr.sg_list[0].length == 0); 269 CU_ASSERT(rdma_req.data.wr.sg_list[0].lkey == 0); 270 271 rdma_req.recv->buf = (void *)0xDDDD; 272 /* Test 2: sgl type: keyed data block subtype: offset (in capsule data) */ 273 sgl->generic.type = SPDK_NVME_SGL_TYPE_DATA_BLOCK; 274 sgl->unkeyed.subtype = SPDK_NVME_SGL_SUBTYPE_OFFSET; 275 276 /* Part 1: Normal I/O smaller than in capsule data size no offset */ 277 reset_nvmf_rdma_request(&rdma_req); 278 sgl->address = 0; 279 sgl->unkeyed.length = rtransport.transport.opts.in_capsule_data_size; 280 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 281 282 CU_ASSERT(rc == 0); 283 CU_ASSERT(rdma_req.req.data == (void *)0xDDDD); 284 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.in_capsule_data_size); 285 CU_ASSERT(rdma_req.req.data_from_pool == false); 286 287 /* Part 2: I/O offset + length too large */ 288 reset_nvmf_rdma_request(&rdma_req); 289 sgl->address = rtransport.transport.opts.in_capsule_data_size; 290 sgl->unkeyed.length = rtransport.transport.opts.in_capsule_data_size; 291 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 292 293 CU_ASSERT(rc == -1); 294 295 /* Part 3: I/O too large */ 296 reset_nvmf_rdma_request(&rdma_req); 297 sgl->address = 0; 298 sgl->unkeyed.length = rtransport.transport.opts.in_capsule_data_size * 2; 299 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 300 301 CU_ASSERT(rc == -1); 302 303 /* Test 3: Multi SGL */ 304 sgl->generic.type = SPDK_NVME_SGL_TYPE_LAST_SEGMENT; 305 sgl->unkeyed.subtype = SPDK_NVME_SGL_SUBTYPE_OFFSET; 306 sgl->address = 0; 307 rdma_req.recv->buf = (void *)&sgl_desc; 308 MOCK_SET(spdk_mempool_get, &data); 309 310 /* part 1: 2 segments each with 1 wr. */ 311 reset_nvmf_rdma_request(&rdma_req); 312 sgl->unkeyed.length = 2 * sizeof(struct spdk_nvme_sgl_descriptor); 313 for (i = 0; i < 2; i++) { 314 sgl_desc[i].keyed.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; 315 sgl_desc[i].keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; 316 sgl_desc[i].keyed.length = rtransport.transport.opts.io_unit_size; 317 sgl_desc[i].address = 0x4000 + i * rtransport.transport.opts.io_unit_size; 318 sgl_desc[i].keyed.key = 0x44; 319 } 320 321 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 322 323 CU_ASSERT(rc == 0); 324 CU_ASSERT(rdma_req.req.data_from_pool == true); 325 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size * 2); 326 CU_ASSERT(rdma_req.data.wr.num_sge == 1); 327 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0x44); 328 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0x4000); 329 CU_ASSERT(rdma_req.data.wr.next == &data.wr); 330 CU_ASSERT(data.wr.wr.rdma.rkey == 0x44); 331 CU_ASSERT(data.wr.wr.rdma.remote_addr == 0x4000 + rtransport.transport.opts.io_unit_size); 332 CU_ASSERT(data.wr.num_sge == 1); 333 CU_ASSERT(data.wr.next == &rdma_req.rsp.wr); 334 335 /* part 2: 2 segments, each with 1 wr containing 8 sge_elements */ 336 reset_nvmf_rdma_request(&rdma_req); 337 sgl->unkeyed.length = 2 * sizeof(struct spdk_nvme_sgl_descriptor); 338 for (i = 0; i < 2; i++) { 339 sgl_desc[i].keyed.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; 340 sgl_desc[i].keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; 341 sgl_desc[i].keyed.length = rtransport.transport.opts.io_unit_size * 8; 342 sgl_desc[i].address = 0x4000 + i * 8 * rtransport.transport.opts.io_unit_size; 343 sgl_desc[i].keyed.key = 0x44; 344 } 345 346 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 347 348 CU_ASSERT(rc == 0); 349 CU_ASSERT(rdma_req.req.data_from_pool == true); 350 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size * 16); 351 CU_ASSERT(rdma_req.req.iovcnt == 16); 352 CU_ASSERT(rdma_req.data.wr.num_sge == 8); 353 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0x44); 354 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0x4000); 355 CU_ASSERT(rdma_req.data.wr.next == &data.wr); 356 CU_ASSERT(data.wr.wr.rdma.rkey == 0x44); 357 CU_ASSERT(data.wr.wr.rdma.remote_addr == 0x4000 + rtransport.transport.opts.io_unit_size * 8); 358 CU_ASSERT(data.wr.num_sge == 8); 359 CU_ASSERT(data.wr.next == &rdma_req.rsp.wr); 360 361 /* part 3: 2 segments, one very large, one very small */ 362 reset_nvmf_rdma_request(&rdma_req); 363 for (i = 0; i < 2; i++) { 364 sgl_desc[i].keyed.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; 365 sgl_desc[i].keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; 366 sgl_desc[i].keyed.key = 0x44; 367 } 368 369 sgl_desc[0].keyed.length = rtransport.transport.opts.io_unit_size * 15 + 370 rtransport.transport.opts.io_unit_size / 2; 371 sgl_desc[0].address = 0x4000; 372 sgl_desc[1].keyed.length = rtransport.transport.opts.io_unit_size / 2; 373 sgl_desc[1].address = 0x4000 + rtransport.transport.opts.io_unit_size * 15 + 374 rtransport.transport.opts.io_unit_size / 2; 375 376 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 377 378 CU_ASSERT(rc == 0); 379 CU_ASSERT(rdma_req.req.data_from_pool == true); 380 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size * 16); 381 CU_ASSERT(rdma_req.req.iovcnt == 17); 382 CU_ASSERT(rdma_req.data.wr.num_sge == 16); 383 for (i = 0; i < 15; i++) { 384 CU_ASSERT(rdma_req.data.sgl[i].length == rtransport.transport.opts.io_unit_size); 385 } 386 CU_ASSERT(rdma_req.data.sgl[15].length == rtransport.transport.opts.io_unit_size / 2); 387 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0x44); 388 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0x4000); 389 CU_ASSERT(rdma_req.data.wr.next == &data.wr); 390 CU_ASSERT(data.wr.wr.rdma.rkey == 0x44); 391 CU_ASSERT(data.wr.wr.rdma.remote_addr == 0x4000 + rtransport.transport.opts.io_unit_size * 15 + 392 rtransport.transport.opts.io_unit_size / 2); 393 CU_ASSERT(data.sgl[0].length == rtransport.transport.opts.io_unit_size / 2); 394 CU_ASSERT(data.wr.num_sge == 1); 395 CU_ASSERT(data.wr.next == &rdma_req.rsp.wr); 396 397 /* Test 4: use PG buffer cache */ 398 sgl->generic.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; 399 sgl->keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; 400 sgl->address = 0xFFFF; 401 rdma_req.recv->buf = (void *)0xDDDD; 402 g_rdma_mr.lkey = 0xABCD; 403 sgl->keyed.key = 0xEEEE; 404 405 for (i = 0; i < 4; i++) { 406 STAILQ_INSERT_TAIL(&group.group.buf_cache, &bufs[i], link); 407 } 408 409 /* part 1: use the four buffers from the pg cache */ 410 group.group.buf_cache_size = 4; 411 group.group.buf_cache_count = 4; 412 MOCK_SET(spdk_mempool_get, (void *)0x2000); 413 reset_nvmf_rdma_request(&rdma_req); 414 sgl->keyed.length = rtransport.transport.opts.io_unit_size * 4; 415 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 416 417 SPDK_CU_ASSERT_FATAL(rc == 0); 418 CU_ASSERT(rdma_req.req.data_from_pool == true); 419 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size * 4); 420 CU_ASSERT((uint64_t)rdma_req.req.data == (((uint64_t)&bufs[0] + NVMF_DATA_BUFFER_MASK) & 421 ~NVMF_DATA_BUFFER_MASK)); 422 CU_ASSERT(rdma_req.data.wr.num_sge == 4); 423 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 424 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 425 CU_ASSERT(group.group.buf_cache_count == 0); 426 CU_ASSERT(STAILQ_EMPTY(&group.group.buf_cache)); 427 for (i = 0; i < 4; i++) { 428 CU_ASSERT((uint64_t)rdma_req.req.buffers[i] == (uint64_t)&bufs[i]); 429 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == (((uint64_t)&bufs[i] + NVMF_DATA_BUFFER_MASK) & 430 ~NVMF_DATA_BUFFER_MASK)); 431 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == rtransport.transport.opts.io_unit_size); 432 } 433 434 /* part 2: now that we have used the buffers from the cache, try again. We should get mempool buffers. */ 435 reset_nvmf_rdma_request(&rdma_req); 436 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 437 438 SPDK_CU_ASSERT_FATAL(rc == 0); 439 CU_ASSERT(rdma_req.req.data_from_pool == true); 440 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size * 4); 441 CU_ASSERT((uint64_t)rdma_req.req.data == 0x2000); 442 CU_ASSERT(rdma_req.data.wr.num_sge == 4); 443 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 444 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 445 CU_ASSERT(group.group.buf_cache_count == 0); 446 CU_ASSERT(STAILQ_EMPTY(&group.group.buf_cache)); 447 for (i = 0; i < 4; i++) { 448 CU_ASSERT((uint64_t)rdma_req.req.buffers[i] == 0x2000); 449 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == 0x2000); 450 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == rtransport.transport.opts.io_unit_size); 451 CU_ASSERT(group.group.buf_cache_count == 0); 452 } 453 454 /* part 3: half and half */ 455 group.group.buf_cache_count = 2; 456 457 for (i = 0; i < 2; i++) { 458 STAILQ_INSERT_TAIL(&group.group.buf_cache, &bufs[i], link); 459 } 460 reset_nvmf_rdma_request(&rdma_req); 461 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 462 463 SPDK_CU_ASSERT_FATAL(rc == 0); 464 CU_ASSERT(rdma_req.req.data_from_pool == true); 465 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size * 4); 466 CU_ASSERT((uint64_t)rdma_req.req.data == (((uint64_t)&bufs[0] + NVMF_DATA_BUFFER_MASK) & 467 ~NVMF_DATA_BUFFER_MASK)); 468 CU_ASSERT(rdma_req.data.wr.num_sge == 4); 469 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 470 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 471 CU_ASSERT(group.group.buf_cache_count == 0); 472 for (i = 0; i < 2; i++) { 473 CU_ASSERT((uint64_t)rdma_req.req.buffers[i] == (uint64_t)&bufs[i]); 474 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == (((uint64_t)&bufs[i] + NVMF_DATA_BUFFER_MASK) & 475 ~NVMF_DATA_BUFFER_MASK)); 476 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == rtransport.transport.opts.io_unit_size); 477 } 478 for (i = 2; i < 4; i++) { 479 CU_ASSERT((uint64_t)rdma_req.req.buffers[i] == 0x2000); 480 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == 0x2000); 481 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == rtransport.transport.opts.io_unit_size); 482 } 483 484 reset_nvmf_rdma_request(&rdma_req); 485 /* Test 5 dealing with a buffer split over two Memory Regions */ 486 MOCK_SET(spdk_mempool_get, (void *)&buffer); 487 sgl->generic.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; 488 sgl->keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; 489 sgl->keyed.length = rtransport.transport.opts.io_unit_size / 2; 490 g_mr_size = rtransport.transport.opts.io_unit_size / 4; 491 g_mr_next_size = rtransport.transport.opts.io_unit_size / 2; 492 493 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 494 SPDK_CU_ASSERT_FATAL(rc == 0); 495 CU_ASSERT(rdma_req.req.data_from_pool == true); 496 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size / 2); 497 CU_ASSERT((uint64_t)rdma_req.req.data == (((uint64_t)&buffer + NVMF_DATA_BUFFER_MASK) & 498 ~NVMF_DATA_BUFFER_MASK)); 499 CU_ASSERT(rdma_req.data.wr.num_sge == 1); 500 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 501 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 502 CU_ASSERT(rdma_req.req.buffers[0] == &buffer); 503 CU_ASSERT(rdma_req.data.wr.sg_list[0].addr == (((uint64_t)&buffer + NVMF_DATA_BUFFER_MASK) & 504 ~NVMF_DATA_BUFFER_MASK)); 505 CU_ASSERT(rdma_req.data.wr.sg_list[0].length == rtransport.transport.opts.io_unit_size / 2); 506 CU_ASSERT(rdma_req.data.wr.sg_list[0].lkey == g_rdma_mr.lkey); 507 buffer_ptr = STAILQ_FIRST(&group.retired_bufs); 508 CU_ASSERT(buffer_ptr == &buffer); 509 STAILQ_REMOVE(&group.retired_bufs, buffer_ptr, spdk_nvmf_transport_pg_cache_buf, link); 510 CU_ASSERT(STAILQ_EMPTY(&group.retired_bufs)); 511 g_mr_size = 0; 512 g_mr_next_size = 0; 513 514 reset_nvmf_rdma_request(&rdma_req); 515 } 516 517 static struct spdk_nvmf_rdma_recv * 518 create_recv(struct spdk_nvmf_rdma_qpair *rqpair, enum spdk_nvme_nvm_opcode opc) 519 { 520 struct spdk_nvmf_rdma_recv *rdma_recv; 521 union nvmf_h2c_msg *cmd; 522 struct spdk_nvme_sgl_descriptor *sgl; 523 524 rdma_recv = calloc(1, sizeof(*rdma_recv)); 525 rdma_recv->qpair = rqpair; 526 cmd = calloc(1, sizeof(*cmd)); 527 rdma_recv->sgl[0].addr = (uintptr_t)cmd; 528 cmd->nvme_cmd.opc = opc; 529 sgl = &cmd->nvme_cmd.dptr.sgl1; 530 sgl->keyed.key = 0xEEEE; 531 sgl->address = 0xFFFF; 532 sgl->keyed.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; 533 sgl->keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; 534 sgl->keyed.length = 1; 535 536 return rdma_recv; 537 } 538 539 static void 540 free_recv(struct spdk_nvmf_rdma_recv *rdma_recv) 541 { 542 free((void *)rdma_recv->sgl[0].addr); 543 free(rdma_recv); 544 } 545 546 static struct spdk_nvmf_rdma_request * 547 create_req(struct spdk_nvmf_rdma_qpair *rqpair, 548 struct spdk_nvmf_rdma_recv *rdma_recv) 549 { 550 struct spdk_nvmf_rdma_request *rdma_req; 551 union nvmf_c2h_msg *cpl; 552 553 rdma_req = calloc(1, sizeof(*rdma_req)); 554 rdma_req->recv = rdma_recv; 555 rdma_req->req.qpair = &rqpair->qpair; 556 rdma_req->state = RDMA_REQUEST_STATE_NEW; 557 rdma_req->data.wr.wr_id = (uintptr_t)&rdma_req->data.rdma_wr; 558 rdma_req->data.wr.sg_list = rdma_req->data.sgl; 559 cpl = calloc(1, sizeof(*cpl)); 560 rdma_req->rsp.sgl[0].addr = (uintptr_t)cpl; 561 rdma_req->req.rsp = cpl; 562 563 return rdma_req; 564 } 565 566 static void 567 free_req(struct spdk_nvmf_rdma_request *rdma_req) 568 { 569 free((void *)rdma_req->rsp.sgl[0].addr); 570 free(rdma_req); 571 } 572 573 static void 574 qpair_reset(struct spdk_nvmf_rdma_qpair *rqpair, 575 struct spdk_nvmf_rdma_poller *poller, 576 struct spdk_nvmf_rdma_device *device, 577 struct spdk_nvmf_rdma_resources *resources) 578 { 579 memset(rqpair, 0, sizeof(*rqpair)); 580 STAILQ_INIT(&rqpair->pending_rdma_write_queue); 581 STAILQ_INIT(&rqpair->pending_rdma_read_queue); 582 rqpair->poller = poller; 583 rqpair->device = device; 584 rqpair->resources = resources; 585 rqpair->qpair.qid = 1; 586 rqpair->ibv_state = IBV_QPS_RTS; 587 rqpair->qpair.state = SPDK_NVMF_QPAIR_ACTIVE; 588 rqpair->max_send_sge = SPDK_NVMF_MAX_SGL_ENTRIES; 589 rqpair->max_send_depth = 16; 590 rqpair->max_read_depth = 16; 591 resources->recvs_to_post.first = resources->recvs_to_post.last = NULL; 592 } 593 594 static void 595 poller_reset(struct spdk_nvmf_rdma_poller *poller, 596 struct spdk_nvmf_rdma_poll_group *group) 597 { 598 memset(poller, 0, sizeof(*poller)); 599 STAILQ_INIT(&poller->qpairs_pending_recv); 600 STAILQ_INIT(&poller->qpairs_pending_send); 601 poller->group = group; 602 } 603 604 static void 605 test_spdk_nvmf_rdma_request_process(void) 606 { 607 struct spdk_nvmf_rdma_transport rtransport = {}; 608 struct spdk_nvmf_rdma_poll_group group = {}; 609 struct spdk_nvmf_rdma_poller poller = {}; 610 struct spdk_nvmf_rdma_device device = {}; 611 struct spdk_nvmf_rdma_resources resources = {}; 612 struct spdk_nvmf_rdma_qpair rqpair = {}; 613 struct spdk_nvmf_rdma_recv *rdma_recv; 614 struct spdk_nvmf_rdma_request *rdma_req; 615 bool progress; 616 617 STAILQ_INIT(&group.group.buf_cache); 618 STAILQ_INIT(&group.group.pending_buf_queue); 619 group.group.buf_cache_size = 0; 620 group.group.buf_cache_count = 0; 621 poller_reset(&poller, &group); 622 qpair_reset(&rqpair, &poller, &device, &resources); 623 624 rtransport.transport.opts = g_rdma_ut_transport_opts; 625 rtransport.transport.data_buf_pool = spdk_mempool_create("test_data_pool", 16, 128, 0, 0); 626 rtransport.data_wr_pool = spdk_mempool_create("test_wr_pool", 128, 627 sizeof(struct spdk_nvmf_rdma_request_data), 628 0, 0); 629 MOCK_CLEAR(spdk_mempool_get); 630 631 device.attr.device_cap_flags = 0; 632 device.map = (void *)0x0; 633 g_rdma_mr.lkey = 0xABCD; 634 635 /* Test 1: single SGL READ request */ 636 rdma_recv = create_recv(&rqpair, SPDK_NVME_OPC_READ); 637 rdma_req = create_req(&rqpair, rdma_recv); 638 rqpair.current_recv_depth = 1; 639 /* NEW -> EXECUTING */ 640 progress = nvmf_rdma_request_process(&rtransport, rdma_req); 641 CU_ASSERT(progress == true); 642 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_EXECUTING); 643 CU_ASSERT(rdma_req->req.xfer == SPDK_NVME_DATA_CONTROLLER_TO_HOST); 644 /* EXECUTED -> TRANSFERRING_C2H */ 645 rdma_req->state = RDMA_REQUEST_STATE_EXECUTED; 646 progress = nvmf_rdma_request_process(&rtransport, rdma_req); 647 CU_ASSERT(progress == true); 648 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_TRANSFERRING_CONTROLLER_TO_HOST); 649 CU_ASSERT(rdma_req->recv == NULL); 650 CU_ASSERT(resources.recvs_to_post.first == &rdma_recv->wr); 651 CU_ASSERT(resources.recvs_to_post.last == &rdma_recv->wr); 652 /* COMPLETED -> FREE */ 653 rdma_req->state = RDMA_REQUEST_STATE_COMPLETED; 654 progress = nvmf_rdma_request_process(&rtransport, rdma_req); 655 CU_ASSERT(progress == true); 656 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_FREE); 657 658 free_recv(rdma_recv); 659 free_req(rdma_req); 660 poller_reset(&poller, &group); 661 qpair_reset(&rqpair, &poller, &device, &resources); 662 663 /* Test 2: single SGL WRITE request */ 664 rdma_recv = create_recv(&rqpair, SPDK_NVME_OPC_WRITE); 665 rdma_req = create_req(&rqpair, rdma_recv); 666 rqpair.current_recv_depth = 1; 667 /* NEW -> TRANSFERRING_H2C */ 668 progress = nvmf_rdma_request_process(&rtransport, rdma_req); 669 CU_ASSERT(progress == true); 670 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_TRANSFERRING_HOST_TO_CONTROLLER); 671 CU_ASSERT(rdma_req->req.xfer == SPDK_NVME_DATA_HOST_TO_CONTROLLER); 672 STAILQ_INIT(&poller.qpairs_pending_send); 673 /* READY_TO_EXECUTE -> EXECUTING */ 674 rdma_req->state = RDMA_REQUEST_STATE_READY_TO_EXECUTE; 675 progress = nvmf_rdma_request_process(&rtransport, rdma_req); 676 CU_ASSERT(progress == true); 677 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_EXECUTING); 678 /* EXECUTED -> COMPLETING */ 679 rdma_req->state = RDMA_REQUEST_STATE_EXECUTED; 680 progress = nvmf_rdma_request_process(&rtransport, rdma_req); 681 CU_ASSERT(progress == true); 682 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_COMPLETING); 683 CU_ASSERT(rdma_req->recv == NULL); 684 CU_ASSERT(resources.recvs_to_post.first == &rdma_recv->wr); 685 CU_ASSERT(resources.recvs_to_post.last == &rdma_recv->wr); 686 /* COMPLETED -> FREE */ 687 rdma_req->state = RDMA_REQUEST_STATE_COMPLETED; 688 progress = nvmf_rdma_request_process(&rtransport, rdma_req); 689 CU_ASSERT(progress == true); 690 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_FREE); 691 692 free_recv(rdma_recv); 693 free_req(rdma_req); 694 poller_reset(&poller, &group); 695 qpair_reset(&rqpair, &poller, &device, &resources); 696 697 /* Test 3: WRITE+WRITE ibv_send batching */ 698 { 699 struct spdk_nvmf_rdma_recv *recv1, *recv2; 700 struct spdk_nvmf_rdma_request *req1, *req2; 701 recv1 = create_recv(&rqpair, SPDK_NVME_OPC_WRITE); 702 req1 = create_req(&rqpair, recv1); 703 recv2 = create_recv(&rqpair, SPDK_NVME_OPC_WRITE); 704 req2 = create_req(&rqpair, recv2); 705 706 /* WRITE 1: NEW -> TRANSFERRING_H2C */ 707 rqpair.current_recv_depth = 1; 708 nvmf_rdma_request_process(&rtransport, req1); 709 CU_ASSERT(req1->state == RDMA_REQUEST_STATE_TRANSFERRING_HOST_TO_CONTROLLER); 710 711 /* WRITE 2: NEW -> TRANSFERRING_H2C */ 712 rqpair.current_recv_depth = 2; 713 nvmf_rdma_request_process(&rtransport, req2); 714 CU_ASSERT(req2->state == RDMA_REQUEST_STATE_TRANSFERRING_HOST_TO_CONTROLLER); 715 716 STAILQ_INIT(&poller.qpairs_pending_send); 717 718 /* WRITE 1 completes before WRITE 2 has finished RDMA reading */ 719 /* WRITE 1: READY_TO_EXECUTE -> EXECUTING */ 720 req1->state = RDMA_REQUEST_STATE_READY_TO_EXECUTE; 721 nvmf_rdma_request_process(&rtransport, req1); 722 CU_ASSERT(req1->state == RDMA_REQUEST_STATE_EXECUTING); 723 /* WRITE 1: EXECUTED -> COMPLETING */ 724 req1->state = RDMA_REQUEST_STATE_EXECUTED; 725 nvmf_rdma_request_process(&rtransport, req1); 726 CU_ASSERT(req1->state == RDMA_REQUEST_STATE_COMPLETING); 727 STAILQ_INIT(&poller.qpairs_pending_send); 728 /* WRITE 1: COMPLETED -> FREE */ 729 req1->state = RDMA_REQUEST_STATE_COMPLETED; 730 nvmf_rdma_request_process(&rtransport, req1); 731 CU_ASSERT(req1->state == RDMA_REQUEST_STATE_FREE); 732 733 /* Now WRITE 2 has finished reading and completes */ 734 /* WRITE 2: COMPLETED -> FREE */ 735 /* WRITE 2: READY_TO_EXECUTE -> EXECUTING */ 736 req2->state = RDMA_REQUEST_STATE_READY_TO_EXECUTE; 737 nvmf_rdma_request_process(&rtransport, req2); 738 CU_ASSERT(req2->state == RDMA_REQUEST_STATE_EXECUTING); 739 /* WRITE 1: EXECUTED -> COMPLETING */ 740 req2->state = RDMA_REQUEST_STATE_EXECUTED; 741 nvmf_rdma_request_process(&rtransport, req2); 742 CU_ASSERT(req2->state == RDMA_REQUEST_STATE_COMPLETING); 743 STAILQ_INIT(&poller.qpairs_pending_send); 744 /* WRITE 1: COMPLETED -> FREE */ 745 req2->state = RDMA_REQUEST_STATE_COMPLETED; 746 nvmf_rdma_request_process(&rtransport, req2); 747 CU_ASSERT(req2->state == RDMA_REQUEST_STATE_FREE); 748 749 free_recv(recv1); 750 free_req(req1); 751 free_recv(recv2); 752 free_req(req2); 753 poller_reset(&poller, &group); 754 qpair_reset(&rqpair, &poller, &device, &resources); 755 } 756 757 spdk_mempool_free(rtransport.transport.data_buf_pool); 758 spdk_mempool_free(rtransport.data_wr_pool); 759 } 760 761 #define TEST_GROUPS_COUNT 5 762 static void 763 test_nvmf_rdma_get_optimal_poll_group(void) 764 { 765 struct spdk_nvmf_rdma_transport rtransport = {}; 766 struct spdk_nvmf_transport *transport = &rtransport.transport; 767 struct spdk_nvmf_rdma_qpair rqpair = {}; 768 struct spdk_nvmf_transport_poll_group *groups[TEST_GROUPS_COUNT]; 769 struct spdk_nvmf_rdma_poll_group *rgroups[TEST_GROUPS_COUNT]; 770 struct spdk_nvmf_transport_poll_group *result; 771 uint32_t i; 772 773 rqpair.qpair.transport = transport; 774 pthread_mutex_init(&rtransport.lock, NULL); 775 TAILQ_INIT(&rtransport.poll_groups); 776 777 for (i = 0; i < TEST_GROUPS_COUNT; i++) { 778 groups[i] = nvmf_rdma_poll_group_create(transport); 779 CU_ASSERT(groups[i] != NULL); 780 rgroups[i] = SPDK_CONTAINEROF(groups[i], struct spdk_nvmf_rdma_poll_group, group); 781 groups[i]->transport = transport; 782 } 783 CU_ASSERT(rtransport.conn_sched.next_admin_pg == rgroups[0]); 784 CU_ASSERT(rtransport.conn_sched.next_io_pg == rgroups[0]); 785 786 /* Emulate connection of %TEST_GROUPS_COUNT% initiators - each creates 1 admin and 1 io qp */ 787 for (i = 0; i < TEST_GROUPS_COUNT; i++) { 788 rqpair.qpair.qid = 0; 789 result = nvmf_rdma_get_optimal_poll_group(&rqpair.qpair); 790 CU_ASSERT(result == groups[i]); 791 CU_ASSERT(rtransport.conn_sched.next_admin_pg == rgroups[(i + 1) % TEST_GROUPS_COUNT]); 792 CU_ASSERT(rtransport.conn_sched.next_io_pg == rgroups[i]); 793 794 rqpair.qpair.qid = 1; 795 result = nvmf_rdma_get_optimal_poll_group(&rqpair.qpair); 796 CU_ASSERT(result == groups[i]); 797 CU_ASSERT(rtransport.conn_sched.next_admin_pg == rgroups[(i + 1) % TEST_GROUPS_COUNT]); 798 CU_ASSERT(rtransport.conn_sched.next_io_pg == rgroups[(i + 1) % TEST_GROUPS_COUNT]); 799 } 800 /* wrap around, admin/io pg point to the first pg 801 Destroy all poll groups except of the last one */ 802 for (i = 0; i < TEST_GROUPS_COUNT - 1; i++) { 803 nvmf_rdma_poll_group_destroy(groups[i]); 804 CU_ASSERT(rtransport.conn_sched.next_admin_pg == rgroups[i + 1]); 805 CU_ASSERT(rtransport.conn_sched.next_io_pg == rgroups[i + 1]); 806 } 807 808 CU_ASSERT(rtransport.conn_sched.next_admin_pg == rgroups[TEST_GROUPS_COUNT - 1]); 809 CU_ASSERT(rtransport.conn_sched.next_io_pg == rgroups[TEST_GROUPS_COUNT - 1]); 810 811 /* Check that pointers to the next admin/io poll groups are not changed */ 812 rqpair.qpair.qid = 0; 813 result = nvmf_rdma_get_optimal_poll_group(&rqpair.qpair); 814 CU_ASSERT(result == groups[TEST_GROUPS_COUNT - 1]); 815 CU_ASSERT(rtransport.conn_sched.next_admin_pg == rgroups[TEST_GROUPS_COUNT - 1]); 816 CU_ASSERT(rtransport.conn_sched.next_io_pg == rgroups[TEST_GROUPS_COUNT - 1]); 817 818 rqpair.qpair.qid = 1; 819 result = nvmf_rdma_get_optimal_poll_group(&rqpair.qpair); 820 CU_ASSERT(result == groups[TEST_GROUPS_COUNT - 1]); 821 CU_ASSERT(rtransport.conn_sched.next_admin_pg == rgroups[TEST_GROUPS_COUNT - 1]); 822 CU_ASSERT(rtransport.conn_sched.next_io_pg == rgroups[TEST_GROUPS_COUNT - 1]); 823 824 /* Remove the last poll group, check that pointers are NULL */ 825 nvmf_rdma_poll_group_destroy(groups[TEST_GROUPS_COUNT - 1]); 826 CU_ASSERT(rtransport.conn_sched.next_admin_pg == NULL); 827 CU_ASSERT(rtransport.conn_sched.next_io_pg == NULL); 828 829 /* Request optimal poll group, result must be NULL */ 830 rqpair.qpair.qid = 0; 831 result = nvmf_rdma_get_optimal_poll_group(&rqpair.qpair); 832 CU_ASSERT(result == NULL); 833 834 rqpair.qpair.qid = 1; 835 result = nvmf_rdma_get_optimal_poll_group(&rqpair.qpair); 836 CU_ASSERT(result == NULL); 837 838 pthread_mutex_destroy(&rtransport.lock); 839 } 840 #undef TEST_GROUPS_COUNT 841 842 static void 843 test_spdk_nvmf_rdma_request_parse_sgl_with_md(void) 844 { 845 struct spdk_nvmf_rdma_transport rtransport; 846 struct spdk_nvmf_rdma_device device; 847 struct spdk_nvmf_rdma_request rdma_req = {}; 848 struct spdk_nvmf_rdma_recv recv; 849 struct spdk_nvmf_rdma_poll_group group; 850 struct spdk_nvmf_rdma_qpair rqpair; 851 struct spdk_nvmf_rdma_poller poller; 852 union nvmf_c2h_msg cpl; 853 union nvmf_h2c_msg cmd; 854 struct spdk_nvme_sgl_descriptor *sgl; 855 struct spdk_nvme_sgl_descriptor sgl_desc[SPDK_NVMF_MAX_SGL_ENTRIES] = {{0}}; 856 struct spdk_nvmf_rdma_request_data data; 857 struct spdk_nvmf_transport_pg_cache_buf buffer; 858 struct spdk_nvmf_transport_pg_cache_buf *buffer_ptr; 859 const uint32_t data_bs = 512; 860 const uint32_t md_size = 8; 861 int rc, i; 862 void *aligned_buffer; 863 864 data.wr.sg_list = data.sgl; 865 STAILQ_INIT(&group.group.buf_cache); 866 group.group.buf_cache_size = 0; 867 group.group.buf_cache_count = 0; 868 group.group.transport = &rtransport.transport; 869 STAILQ_INIT(&group.retired_bufs); 870 poller.group = &group; 871 rqpair.poller = &poller; 872 rqpair.max_send_sge = SPDK_NVMF_MAX_SGL_ENTRIES; 873 874 sgl = &cmd.nvme_cmd.dptr.sgl1; 875 rdma_req.recv = &recv; 876 rdma_req.req.cmd = &cmd; 877 rdma_req.req.rsp = &cpl; 878 rdma_req.data.wr.sg_list = rdma_req.data.sgl; 879 rdma_req.req.qpair = &rqpair.qpair; 880 rdma_req.req.xfer = SPDK_NVME_DATA_CONTROLLER_TO_HOST; 881 882 rtransport.transport.opts = g_rdma_ut_transport_opts; 883 rtransport.data_wr_pool = NULL; 884 rtransport.transport.data_buf_pool = NULL; 885 886 device.attr.device_cap_flags = 0; 887 device.map = NULL; 888 g_rdma_mr.lkey = 0xABCD; 889 sgl->keyed.key = 0xEEEE; 890 sgl->address = 0xFFFF; 891 rdma_req.recv->buf = (void *)0xDDDD; 892 893 /* Test 1: sgl type: keyed data block subtype: address */ 894 sgl->generic.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; 895 sgl->keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; 896 897 /* Part 1: simple I/O, one SGL smaller than the transport io unit size, block size 512 */ 898 MOCK_SET(spdk_mempool_get, (void *)0x2000); 899 reset_nvmf_rdma_request(&rdma_req); 900 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 901 SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 902 0, 0, 0, 0, 0); 903 rdma_req.req.dif.dif_insert_or_strip = true; 904 rtransport.transport.opts.io_unit_size = data_bs * 8; 905 sgl->keyed.length = data_bs * 4; 906 907 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 908 909 CU_ASSERT(rc == 0); 910 CU_ASSERT(rdma_req.req.data_from_pool == true); 911 CU_ASSERT(rdma_req.req.length == data_bs * 4); 912 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 913 CU_ASSERT(rdma_req.req.dif.elba_length == (data_bs + md_size) * 4); 914 CU_ASSERT((uint64_t)rdma_req.req.data == 0x2000); 915 CU_ASSERT(rdma_req.data.wr.num_sge == 4); 916 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 917 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 918 CU_ASSERT((uint64_t)rdma_req.req.buffers[0] == 0x2000); 919 920 for (i = 0; i < 4; ++i) { 921 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == 0x2000 + i * (data_bs + md_size)); 922 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == data_bs); 923 CU_ASSERT(rdma_req.data.wr.sg_list[i].lkey == g_rdma_mr.lkey); 924 } 925 926 /* Part 2: simple I/O, one SGL equal to io unit size, io_unit_size is not aligned with md_size, 927 block size 512 */ 928 MOCK_SET(spdk_mempool_get, (void *)0x2000); 929 reset_nvmf_rdma_request(&rdma_req); 930 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 931 SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 932 0, 0, 0, 0, 0); 933 rdma_req.req.dif.dif_insert_or_strip = true; 934 rtransport.transport.opts.io_unit_size = data_bs * 4; 935 sgl->keyed.length = data_bs * 4; 936 937 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 938 939 CU_ASSERT(rc == 0); 940 CU_ASSERT(rdma_req.req.data_from_pool == true); 941 CU_ASSERT(rdma_req.req.length == data_bs * 4); 942 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 943 CU_ASSERT(rdma_req.req.dif.elba_length == (data_bs + md_size) * 4); 944 CU_ASSERT((uint64_t)rdma_req.req.data == 0x2000); 945 CU_ASSERT(rdma_req.data.wr.num_sge == 5); 946 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 947 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 948 CU_ASSERT((uint64_t)rdma_req.req.buffers[0] == 0x2000); 949 950 for (i = 0; i < 3; ++i) { 951 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == 0x2000 + i * (data_bs + md_size)); 952 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == data_bs); 953 CU_ASSERT(rdma_req.data.wr.sg_list[i].lkey == g_rdma_mr.lkey); 954 } 955 CU_ASSERT(rdma_req.data.wr.sg_list[3].addr == 0x2000 + 3 * (data_bs + md_size)); 956 CU_ASSERT(rdma_req.data.wr.sg_list[3].length == 488); 957 CU_ASSERT(rdma_req.data.wr.sg_list[3].lkey == g_rdma_mr.lkey); 958 959 /* 2nd buffer consumed */ 960 CU_ASSERT(rdma_req.data.wr.sg_list[4].addr == 0x2000); 961 CU_ASSERT(rdma_req.data.wr.sg_list[4].length == 24); 962 CU_ASSERT(rdma_req.data.wr.sg_list[4].lkey == g_rdma_mr.lkey); 963 964 /* Part 3: simple I/O, one SGL equal io unit size, io_unit_size is equal to block size 512 bytes */ 965 MOCK_SET(spdk_mempool_get, (void *)0x2000); 966 reset_nvmf_rdma_request(&rdma_req); 967 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 968 SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 969 0, 0, 0, 0, 0); 970 rdma_req.req.dif.dif_insert_or_strip = true; 971 rtransport.transport.opts.io_unit_size = data_bs; 972 sgl->keyed.length = data_bs; 973 974 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 975 976 CU_ASSERT(rc == 0); 977 CU_ASSERT(rdma_req.req.data_from_pool == true); 978 CU_ASSERT(rdma_req.req.length == data_bs); 979 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 980 CU_ASSERT(rdma_req.req.dif.elba_length == data_bs + md_size); 981 CU_ASSERT((uint64_t)rdma_req.req.data == 0x2000); 982 CU_ASSERT(rdma_req.data.wr.num_sge == 1); 983 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 984 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 985 CU_ASSERT((uint64_t)rdma_req.req.buffers[0] == 0x2000); 986 987 CU_ASSERT(rdma_req.data.wr.sg_list[0].addr == 0x2000); 988 CU_ASSERT(rdma_req.data.wr.sg_list[0].length == data_bs); 989 CU_ASSERT(rdma_req.data.wr.sg_list[0].lkey == g_rdma_mr.lkey); 990 991 CU_ASSERT(rdma_req.req.iovcnt == 2); 992 CU_ASSERT(rdma_req.req.iov[0].iov_base == (void *)((unsigned long)0x2000)); 993 CU_ASSERT(rdma_req.req.iov[0].iov_len == data_bs); 994 /* 2nd buffer consumed for metadata */ 995 CU_ASSERT(rdma_req.req.iov[1].iov_base == (void *)((unsigned long)0x2000)); 996 CU_ASSERT(rdma_req.req.iov[1].iov_len == md_size); 997 998 /* Part 4: simple I/O, one SGL equal io unit size, io_unit_size is aligned with md_size, 999 block size 512 */ 1000 MOCK_SET(spdk_mempool_get, (void *)0x2000); 1001 reset_nvmf_rdma_request(&rdma_req); 1002 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 1003 SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 1004 0, 0, 0, 0, 0); 1005 rdma_req.req.dif.dif_insert_or_strip = true; 1006 rtransport.transport.opts.io_unit_size = (data_bs + md_size) * 4; 1007 sgl->keyed.length = data_bs * 4; 1008 1009 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 1010 1011 CU_ASSERT(rc == 0); 1012 CU_ASSERT(rdma_req.req.data_from_pool == true); 1013 CU_ASSERT(rdma_req.req.length == data_bs * 4); 1014 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 1015 CU_ASSERT(rdma_req.req.dif.elba_length == (data_bs + md_size) * 4); 1016 CU_ASSERT((uint64_t)rdma_req.req.data == 0x2000); 1017 CU_ASSERT(rdma_req.data.wr.num_sge == 4); 1018 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 1019 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 1020 CU_ASSERT((uint64_t)rdma_req.req.buffers[0] == 0x2000); 1021 1022 for (i = 0; i < 4; ++i) { 1023 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == 0x2000 + i * (data_bs + md_size)); 1024 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == data_bs); 1025 CU_ASSERT(rdma_req.data.wr.sg_list[i].lkey == g_rdma_mr.lkey); 1026 } 1027 1028 /* Part 5: simple I/O, one SGL equal to 2x io unit size, io_unit_size is aligned with md_size, 1029 block size 512 */ 1030 MOCK_SET(spdk_mempool_get, (void *)0x2000); 1031 reset_nvmf_rdma_request(&rdma_req); 1032 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 1033 SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 1034 0, 0, 0, 0, 0); 1035 rdma_req.req.dif.dif_insert_or_strip = true; 1036 rtransport.transport.opts.io_unit_size = (data_bs + md_size) * 2; 1037 sgl->keyed.length = data_bs * 4; 1038 1039 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 1040 1041 CU_ASSERT(rc == 0); 1042 CU_ASSERT(rdma_req.req.data_from_pool == true); 1043 CU_ASSERT(rdma_req.req.length == data_bs * 4); 1044 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 1045 CU_ASSERT(rdma_req.req.dif.elba_length == (data_bs + md_size) * 4); 1046 CU_ASSERT((uint64_t)rdma_req.req.data == 0x2000); 1047 CU_ASSERT(rdma_req.data.wr.num_sge == 4); 1048 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 1049 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 1050 CU_ASSERT((uint64_t)rdma_req.req.buffers[0] == 0x2000); 1051 1052 for (i = 0; i < 2; ++i) { 1053 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == 0x2000 + i * (data_bs + md_size)); 1054 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == data_bs); 1055 } 1056 for (i = 0; i < 2; ++i) { 1057 CU_ASSERT(rdma_req.data.wr.sg_list[i + 2].addr == 0x2000 + i * (data_bs + md_size)); 1058 CU_ASSERT(rdma_req.data.wr.sg_list[i + 2].length == data_bs); 1059 } 1060 1061 /* Part 6: simple I/O, one SGL larger than the transport io unit size, io_unit_size is not aligned to md_size, 1062 block size 512 */ 1063 MOCK_SET(spdk_mempool_get, (void *)0x2000); 1064 reset_nvmf_rdma_request(&rdma_req); 1065 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 1066 SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 1067 0, 0, 0, 0, 0); 1068 rdma_req.req.dif.dif_insert_or_strip = true; 1069 rtransport.transport.opts.io_unit_size = data_bs * 4; 1070 sgl->keyed.length = data_bs * 6; 1071 1072 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 1073 1074 CU_ASSERT(rc == 0); 1075 CU_ASSERT(rdma_req.req.data_from_pool == true); 1076 CU_ASSERT(rdma_req.req.length == data_bs * 6); 1077 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 1078 CU_ASSERT(rdma_req.req.dif.elba_length == (data_bs + md_size) * 6); 1079 CU_ASSERT((uint64_t)rdma_req.req.data == 0x2000); 1080 CU_ASSERT(rdma_req.data.wr.num_sge == 7); 1081 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 1082 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 1083 CU_ASSERT((uint64_t)rdma_req.req.buffers[0] == 0x2000); 1084 1085 for (i = 0; i < 3; ++i) { 1086 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == 0x2000 + i * (data_bs + md_size)); 1087 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == data_bs); 1088 CU_ASSERT(rdma_req.data.wr.sg_list[i].lkey == g_rdma_mr.lkey); 1089 } 1090 CU_ASSERT(rdma_req.data.wr.sg_list[3].addr == 0x2000 + 3 * (data_bs + md_size)); 1091 CU_ASSERT(rdma_req.data.wr.sg_list[3].length == 488); 1092 CU_ASSERT(rdma_req.data.wr.sg_list[3].lkey == g_rdma_mr.lkey); 1093 1094 /* 2nd IO buffer consumed */ 1095 CU_ASSERT(rdma_req.data.wr.sg_list[4].addr == 0x2000); 1096 CU_ASSERT(rdma_req.data.wr.sg_list[4].length == 24); 1097 CU_ASSERT(rdma_req.data.wr.sg_list[4].lkey == g_rdma_mr.lkey); 1098 1099 CU_ASSERT(rdma_req.data.wr.sg_list[5].addr == 0x2000 + 24 + md_size); 1100 CU_ASSERT(rdma_req.data.wr.sg_list[5].length == 512); 1101 CU_ASSERT(rdma_req.data.wr.sg_list[5].lkey == g_rdma_mr.lkey); 1102 1103 CU_ASSERT(rdma_req.data.wr.sg_list[6].addr == 0x2000 + 24 + 512 + md_size * 2); 1104 CU_ASSERT(rdma_req.data.wr.sg_list[6].length == 512); 1105 CU_ASSERT(rdma_req.data.wr.sg_list[6].lkey == g_rdma_mr.lkey); 1106 1107 /* Part 7: simple I/O, number of SGL entries exceeds the number of entries 1108 one WR can hold. Additional WR is chained */ 1109 MOCK_SET(spdk_mempool_get, &data); 1110 aligned_buffer = (void *)((uintptr_t)((char *)&data + NVMF_DATA_BUFFER_MASK) & 1111 ~NVMF_DATA_BUFFER_MASK); 1112 reset_nvmf_rdma_request(&rdma_req); 1113 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 1114 SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 1115 0, 0, 0, 0, 0); 1116 rdma_req.req.dif.dif_insert_or_strip = true; 1117 rtransport.transport.opts.io_unit_size = data_bs * 16; 1118 sgl->keyed.length = data_bs * 16; 1119 1120 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 1121 1122 CU_ASSERT(rc == 0); 1123 CU_ASSERT(rdma_req.req.data_from_pool == true); 1124 CU_ASSERT(rdma_req.req.length == data_bs * 16); 1125 CU_ASSERT(rdma_req.req.iovcnt == 2); 1126 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 1127 CU_ASSERT(rdma_req.req.dif.elba_length == (data_bs + md_size) * 16); 1128 CU_ASSERT(rdma_req.req.data == aligned_buffer); 1129 CU_ASSERT(rdma_req.data.wr.num_sge == 16); 1130 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 1131 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 1132 /* additional wr from pool */ 1133 CU_ASSERT(rdma_req.data.wr.next == (void *)&data.wr); 1134 CU_ASSERT(rdma_req.data.wr.next->num_sge == 1); 1135 CU_ASSERT(rdma_req.data.wr.next->next == &rdma_req.rsp.wr); 1136 1137 /* Part 8: simple I/O, data with metadata do not fit to 1 io_buffer */ 1138 MOCK_SET(spdk_mempool_get, (void *)0x2000); 1139 reset_nvmf_rdma_request(&rdma_req); 1140 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 1141 SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 1142 0, 0, 0, 0, 0); 1143 rdma_req.req.dif.dif_insert_or_strip = true; 1144 rtransport.transport.opts.io_unit_size = 516; 1145 sgl->keyed.length = data_bs * 2; 1146 1147 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 1148 1149 CU_ASSERT(rc == 0); 1150 CU_ASSERT(rdma_req.req.data_from_pool == true); 1151 CU_ASSERT(rdma_req.req.length == data_bs * 2); 1152 CU_ASSERT(rdma_req.req.iovcnt == 3); 1153 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 1154 CU_ASSERT(rdma_req.req.dif.elba_length == (data_bs + md_size) * 2); 1155 CU_ASSERT(rdma_req.req.data == (void *)0x2000); 1156 CU_ASSERT(rdma_req.data.wr.num_sge == 2); 1157 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 1158 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 1159 1160 CU_ASSERT(rdma_req.data.wr.sg_list[0].addr == 0x2000); 1161 CU_ASSERT(rdma_req.data.wr.sg_list[0].length == 512); 1162 CU_ASSERT(rdma_req.data.wr.sg_list[0].lkey == g_rdma_mr.lkey); 1163 1164 /* 2nd IO buffer consumed, offset 4 bytes due to part of the metadata 1165 is located at the beginning of that buffer */ 1166 CU_ASSERT(rdma_req.data.wr.sg_list[1].addr == 0x2000 + 4); 1167 CU_ASSERT(rdma_req.data.wr.sg_list[1].length == 512); 1168 CU_ASSERT(rdma_req.data.wr.sg_list[1].lkey == g_rdma_mr.lkey); 1169 1170 /* Test 9 dealing with a buffer split over two Memory Regions */ 1171 MOCK_SET(spdk_mempool_get, (void *)&buffer); 1172 reset_nvmf_rdma_request(&rdma_req); 1173 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 1174 SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 1175 0, 0, 0, 0, 0); 1176 rdma_req.req.dif.dif_insert_or_strip = true; 1177 rtransport.transport.opts.io_unit_size = data_bs * 4; 1178 sgl->keyed.length = data_bs * 2; 1179 g_mr_size = data_bs; 1180 g_mr_next_size = rtransport.transport.opts.io_unit_size; 1181 1182 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 1183 SPDK_CU_ASSERT_FATAL(rc == 0); 1184 CU_ASSERT(rdma_req.req.data_from_pool == true); 1185 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size / 2); 1186 CU_ASSERT((uint64_t)rdma_req.req.data == (((uint64_t)&buffer + NVMF_DATA_BUFFER_MASK) & 1187 ~NVMF_DATA_BUFFER_MASK)); 1188 CU_ASSERT(rdma_req.data.wr.num_sge == 2); 1189 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 1190 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 1191 CU_ASSERT(rdma_req.req.buffers[0] == &buffer); 1192 for (i = 0; i < 2; i++) { 1193 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == (uint64_t)rdma_req.req.data + i * 1194 (data_bs + md_size)); 1195 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == data_bs); 1196 CU_ASSERT(rdma_req.data.wr.sg_list[i].lkey == g_rdma_mr.lkey); 1197 } 1198 buffer_ptr = STAILQ_FIRST(&group.retired_bufs); 1199 CU_ASSERT(buffer_ptr == &buffer); 1200 STAILQ_REMOVE(&group.retired_bufs, buffer_ptr, spdk_nvmf_transport_pg_cache_buf, link); 1201 CU_ASSERT(STAILQ_EMPTY(&group.retired_bufs)); 1202 g_mr_size = 0; 1203 g_mr_next_size = 0; 1204 1205 /* Test 2: Multi SGL */ 1206 sgl->generic.type = SPDK_NVME_SGL_TYPE_LAST_SEGMENT; 1207 sgl->unkeyed.subtype = SPDK_NVME_SGL_SUBTYPE_OFFSET; 1208 sgl->address = 0; 1209 rdma_req.recv->buf = (void *)&sgl_desc; 1210 MOCK_SET(spdk_mempool_get, &data); 1211 aligned_buffer = (void *)((uintptr_t)((char *)&data + NVMF_DATA_BUFFER_MASK) & 1212 ~NVMF_DATA_BUFFER_MASK); 1213 1214 /* part 1: 2 segments each with 1 wr. io_unit_size is aligned with data_bs + md_size */ 1215 reset_nvmf_rdma_request(&rdma_req); 1216 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 1217 SPDK_DIF_TYPE1, 1218 SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 0, 0, 0, 0, 0); 1219 rdma_req.req.dif.dif_insert_or_strip = true; 1220 rtransport.transport.opts.io_unit_size = (data_bs + md_size) * 4; 1221 sgl->unkeyed.length = 2 * sizeof(struct spdk_nvme_sgl_descriptor); 1222 1223 for (i = 0; i < 2; i++) { 1224 sgl_desc[i].keyed.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; 1225 sgl_desc[i].keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; 1226 sgl_desc[i].keyed.length = data_bs * 4; 1227 sgl_desc[i].address = 0x4000 + i * data_bs * 4; 1228 sgl_desc[i].keyed.key = 0x44; 1229 } 1230 1231 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 1232 1233 CU_ASSERT(rc == 0); 1234 CU_ASSERT(rdma_req.req.data_from_pool == true); 1235 CU_ASSERT(rdma_req.req.length == data_bs * 4 * 2); 1236 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 1237 CU_ASSERT(rdma_req.req.dif.elba_length == (data_bs + md_size) * 4 * 2); 1238 CU_ASSERT(rdma_req.data.wr.num_sge == 4); 1239 for (i = 0; i < 4; ++i) { 1240 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == (uintptr_t)((unsigned char *)aligned_buffer) + i * 1241 (data_bs + md_size)); 1242 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == data_bs); 1243 } 1244 1245 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0x44); 1246 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0x4000); 1247 CU_ASSERT(rdma_req.data.wr.next == &data.wr); 1248 CU_ASSERT(data.wr.wr.rdma.rkey == 0x44); 1249 CU_ASSERT(data.wr.wr.rdma.remote_addr == 0x4000 + data_bs * 4); 1250 CU_ASSERT(data.wr.num_sge == 4); 1251 for (i = 0; i < 4; ++i) { 1252 CU_ASSERT(data.wr.sg_list[i].addr == (uintptr_t)((unsigned char *)aligned_buffer) + i * 1253 (data_bs + md_size)); 1254 CU_ASSERT(data.wr.sg_list[i].length == data_bs); 1255 } 1256 1257 CU_ASSERT(data.wr.next == &rdma_req.rsp.wr); 1258 } 1259 1260 int main(int argc, char **argv) 1261 { 1262 CU_pSuite suite = NULL; 1263 unsigned int num_failures; 1264 1265 CU_set_error_action(CUEA_ABORT); 1266 CU_initialize_registry(); 1267 1268 suite = CU_add_suite("nvmf", NULL, NULL); 1269 1270 CU_ADD_TEST(suite, test_spdk_nvmf_rdma_request_parse_sgl); 1271 CU_ADD_TEST(suite, test_spdk_nvmf_rdma_request_process); 1272 CU_ADD_TEST(suite, test_nvmf_rdma_get_optimal_poll_group); 1273 CU_ADD_TEST(suite, test_spdk_nvmf_rdma_request_parse_sgl_with_md); 1274 1275 CU_basic_set_mode(CU_BRM_VERBOSE); 1276 CU_basic_run_tests(); 1277 num_failures = CU_get_number_of_failures(); 1278 CU_cleanup_registry(); 1279 return num_failures; 1280 } 1281