1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. All rights reserved. 5 * Copyright (c) 2019 Mellanox Technologies LTD. All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "spdk/stdinc.h" 35 #include "spdk_cunit.h" 36 #include "common/lib/test_env.c" 37 #include "nvmf/rdma.c" 38 39 uint64_t g_mr_size; 40 uint64_t g_mr_next_size; 41 struct ibv_mr g_rdma_mr; 42 43 #define RDMA_UT_UNITS_IN_MAX_IO 16 44 45 struct spdk_nvmf_transport_opts g_rdma_ut_transport_opts = { 46 .max_queue_depth = SPDK_NVMF_RDMA_DEFAULT_MAX_QUEUE_DEPTH, 47 .max_qpairs_per_ctrlr = SPDK_NVMF_RDMA_DEFAULT_MAX_QPAIRS_PER_CTRLR, 48 .in_capsule_data_size = SPDK_NVMF_RDMA_DEFAULT_IN_CAPSULE_DATA_SIZE, 49 .max_io_size = (SPDK_NVMF_RDMA_MIN_IO_BUFFER_SIZE * RDMA_UT_UNITS_IN_MAX_IO), 50 .io_unit_size = SPDK_NVMF_RDMA_MIN_IO_BUFFER_SIZE, 51 .max_aq_depth = SPDK_NVMF_RDMA_DEFAULT_AQ_DEPTH, 52 .num_shared_buffers = SPDK_NVMF_RDMA_DEFAULT_NUM_SHARED_BUFFERS, 53 }; 54 55 SPDK_LOG_REGISTER_COMPONENT("nvmf", SPDK_LOG_NVMF) 56 DEFINE_STUB(spdk_mem_map_set_translation, int, (struct spdk_mem_map *map, uint64_t vaddr, 57 uint64_t size, uint64_t translation), 0); 58 DEFINE_STUB(spdk_mem_map_clear_translation, int, (struct spdk_mem_map *map, uint64_t vaddr, 59 uint64_t size), 0); 60 DEFINE_STUB(spdk_mem_map_alloc, struct spdk_mem_map *, (uint64_t default_translation, 61 const struct spdk_mem_map_ops *ops, void *cb_ctx), NULL); 62 DEFINE_STUB(spdk_nvmf_qpair_disconnect, int, (struct spdk_nvmf_qpair *qpair, 63 nvmf_qpair_disconnect_cb cb_fn, void *ctx), 0); 64 DEFINE_STUB_V(spdk_mem_map_free, (struct spdk_mem_map **pmap)); 65 66 struct spdk_trace_histories *g_trace_histories; 67 DEFINE_STUB_V(spdk_trace_add_register_fn, (struct spdk_trace_register_fn *reg_fn)); 68 DEFINE_STUB_V(spdk_trace_register_object, (uint8_t type, char id_prefix)); 69 DEFINE_STUB_V(spdk_trace_register_description, (const char *name, 70 uint16_t tpoint_id, uint8_t owner_type, uint8_t object_type, uint8_t new_object, 71 uint8_t arg1_type, const char *arg1_name)); 72 DEFINE_STUB_V(_spdk_trace_record, (uint64_t tsc, uint16_t tpoint_id, uint16_t poller_id, 73 uint32_t size, uint64_t object_id, uint64_t arg1)); 74 75 DEFINE_STUB_V(spdk_nvmf_request_exec, (struct spdk_nvmf_request *req)); 76 DEFINE_STUB(spdk_nvme_transport_id_compare, int, (const struct spdk_nvme_transport_id *trid1, 77 const struct spdk_nvme_transport_id *trid2), 0); 78 DEFINE_STUB_V(spdk_nvmf_ctrlr_abort_aer, (struct spdk_nvmf_ctrlr *ctrlr)); 79 DEFINE_STUB(spdk_nvmf_request_get_dif_ctx, bool, (struct spdk_nvmf_request *req, 80 struct spdk_dif_ctx *dif_ctx), false); 81 82 void 83 spdk_nvmf_request_free_buffers(struct spdk_nvmf_request *req, 84 struct spdk_nvmf_transport_poll_group *group, 85 struct spdk_nvmf_transport *transport) 86 { 87 uint32_t i; 88 89 for (i = 0; i < req->num_buffers; i++) { 90 if (group->buf_cache_count < group->buf_cache_size) { 91 STAILQ_INSERT_HEAD(&group->buf_cache, 92 (struct spdk_nvmf_transport_pg_cache_buf *)req->buffers[i], 93 link); 94 group->buf_cache_count++; 95 } else { 96 spdk_mempool_put(transport->data_buf_pool, req->buffers[i]); 97 } 98 req->iov[i].iov_base = NULL; 99 req->buffers[i] = NULL; 100 req->iov[i].iov_len = 0; 101 } 102 req->num_buffers = 0; 103 req->data_from_pool = false; 104 } 105 106 int 107 spdk_nvmf_request_get_buffers(struct spdk_nvmf_request *req, 108 struct spdk_nvmf_transport_poll_group *group, 109 struct spdk_nvmf_transport *transport, 110 uint32_t length) 111 { 112 uint32_t num_buffers; 113 uint32_t i = 0; 114 115 /* If the number of buffers is too large, then we know the I/O is larger than allowed. 116 * Fail it. 117 */ 118 num_buffers = SPDK_CEIL_DIV(length, transport->opts.io_unit_size); 119 if (num_buffers + req->num_buffers > NVMF_REQ_MAX_BUFFERS) { 120 return -EINVAL; 121 } 122 123 while (i < num_buffers) { 124 if (!(STAILQ_EMPTY(&group->buf_cache))) { 125 group->buf_cache_count--; 126 req->buffers[req->num_buffers] = STAILQ_FIRST(&group->buf_cache); 127 STAILQ_REMOVE_HEAD(&group->buf_cache, link); 128 req->num_buffers++; 129 i++; 130 } else { 131 if (spdk_mempool_get_bulk(transport->data_buf_pool, 132 &req->buffers[req->num_buffers], 133 num_buffers - i)) { 134 goto err_exit; 135 } 136 req->num_buffers += num_buffers - i; 137 i += num_buffers - i; 138 } 139 } 140 141 return 0; 142 143 err_exit: 144 spdk_nvmf_request_free_buffers(req, group, transport); 145 return -ENOMEM; 146 } 147 148 uint64_t 149 spdk_mem_map_translate(const struct spdk_mem_map *map, uint64_t vaddr, uint64_t *size) 150 { 151 if (g_mr_size != 0) { 152 *(uint32_t *)size = g_mr_size; 153 if (g_mr_next_size != 0) { 154 g_mr_size = g_mr_next_size; 155 } 156 } 157 158 return (uint64_t)&g_rdma_mr; 159 } 160 161 static void reset_nvmf_rdma_request(struct spdk_nvmf_rdma_request *rdma_req) 162 { 163 int i; 164 165 rdma_req->req.length = 0; 166 rdma_req->req.data_from_pool = false; 167 rdma_req->req.data = NULL; 168 rdma_req->data.wr.num_sge = 0; 169 rdma_req->data.wr.wr.rdma.remote_addr = 0; 170 rdma_req->data.wr.wr.rdma.rkey = 0; 171 rdma_req->elba_length = 0; 172 rdma_req->orig_length = 0; 173 rdma_req->dif_insert_or_strip = false; 174 175 for (i = 0; i < SPDK_NVMF_MAX_SGL_ENTRIES; i++) { 176 rdma_req->req.iov[i].iov_base = 0; 177 rdma_req->req.iov[i].iov_len = 0; 178 rdma_req->req.buffers[i] = 0; 179 rdma_req->data.wr.sg_list[i].addr = 0; 180 rdma_req->data.wr.sg_list[i].length = 0; 181 rdma_req->data.wr.sg_list[i].lkey = 0; 182 } 183 rdma_req->req.iovcnt = 0; 184 rdma_req->req.num_buffers = 0; 185 } 186 187 static void 188 test_spdk_nvmf_rdma_request_parse_sgl(void) 189 { 190 struct spdk_nvmf_rdma_transport rtransport; 191 struct spdk_nvmf_rdma_device device; 192 struct spdk_nvmf_rdma_request rdma_req = {}; 193 struct spdk_nvmf_rdma_recv recv; 194 struct spdk_nvmf_rdma_poll_group group; 195 struct spdk_nvmf_rdma_qpair rqpair; 196 struct spdk_nvmf_rdma_poller poller; 197 union nvmf_c2h_msg cpl; 198 union nvmf_h2c_msg cmd; 199 struct spdk_nvme_sgl_descriptor *sgl; 200 struct spdk_nvmf_transport_pg_cache_buf bufs[4]; 201 struct spdk_nvme_sgl_descriptor sgl_desc[SPDK_NVMF_MAX_SGL_ENTRIES] = {{0}}; 202 struct spdk_nvmf_rdma_request_data data; 203 struct spdk_nvmf_transport_pg_cache_buf buffer; 204 struct spdk_nvmf_transport_pg_cache_buf *buffer_ptr; 205 int rc, i; 206 207 data.wr.sg_list = data.sgl; 208 STAILQ_INIT(&group.group.buf_cache); 209 group.group.buf_cache_size = 0; 210 group.group.buf_cache_count = 0; 211 group.group.transport = &rtransport.transport; 212 STAILQ_INIT(&group.retired_bufs); 213 poller.group = &group; 214 rqpair.poller = &poller; 215 rqpair.max_send_sge = SPDK_NVMF_MAX_SGL_ENTRIES; 216 217 sgl = &cmd.nvme_cmd.dptr.sgl1; 218 rdma_req.recv = &recv; 219 rdma_req.req.cmd = &cmd; 220 rdma_req.req.rsp = &cpl; 221 rdma_req.data.wr.sg_list = rdma_req.data.sgl; 222 rdma_req.req.qpair = &rqpair.qpair; 223 rdma_req.req.xfer = SPDK_NVME_DATA_CONTROLLER_TO_HOST; 224 225 rtransport.transport.opts = g_rdma_ut_transport_opts; 226 rtransport.data_wr_pool = NULL; 227 rtransport.transport.data_buf_pool = NULL; 228 229 device.attr.device_cap_flags = 0; 230 g_rdma_mr.lkey = 0xABCD; 231 sgl->keyed.key = 0xEEEE; 232 sgl->address = 0xFFFF; 233 rdma_req.recv->buf = (void *)0xDDDD; 234 235 /* Test 1: sgl type: keyed data block subtype: address */ 236 sgl->generic.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; 237 sgl->keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; 238 239 /* Part 1: simple I/O, one SGL smaller than the transport io unit size */ 240 MOCK_SET(spdk_mempool_get, (void *)0x2000); 241 reset_nvmf_rdma_request(&rdma_req); 242 sgl->keyed.length = rtransport.transport.opts.io_unit_size / 2; 243 244 device.map = (void *)0x0; 245 rc = spdk_nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 246 CU_ASSERT(rc == 0); 247 CU_ASSERT(rdma_req.req.data_from_pool == true); 248 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size / 2); 249 CU_ASSERT((uint64_t)rdma_req.req.data == 0x2000); 250 CU_ASSERT(rdma_req.data.wr.num_sge == 1); 251 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 252 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 253 CU_ASSERT((uint64_t)rdma_req.req.buffers[0] == 0x2000); 254 CU_ASSERT(rdma_req.data.wr.sg_list[0].addr == 0x2000); 255 CU_ASSERT(rdma_req.data.wr.sg_list[0].length == rtransport.transport.opts.io_unit_size / 2); 256 CU_ASSERT(rdma_req.data.wr.sg_list[0].lkey == g_rdma_mr.lkey); 257 258 /* Part 2: simple I/O, one SGL larger than the transport io unit size (equal to the max io size) */ 259 reset_nvmf_rdma_request(&rdma_req); 260 sgl->keyed.length = rtransport.transport.opts.io_unit_size * RDMA_UT_UNITS_IN_MAX_IO; 261 rc = spdk_nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 262 263 CU_ASSERT(rc == 0); 264 CU_ASSERT(rdma_req.req.data_from_pool == true); 265 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size * RDMA_UT_UNITS_IN_MAX_IO); 266 CU_ASSERT(rdma_req.data.wr.num_sge == RDMA_UT_UNITS_IN_MAX_IO); 267 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 268 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 269 for (i = 0; i < RDMA_UT_UNITS_IN_MAX_IO; i++) { 270 CU_ASSERT((uint64_t)rdma_req.req.buffers[i] == 0x2000); 271 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == 0x2000); 272 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == rtransport.transport.opts.io_unit_size); 273 CU_ASSERT(rdma_req.data.wr.sg_list[i].lkey == g_rdma_mr.lkey); 274 } 275 276 /* Part 3: simple I/O one SGL larger than the transport max io size */ 277 reset_nvmf_rdma_request(&rdma_req); 278 sgl->keyed.length = rtransport.transport.opts.max_io_size * 2; 279 rc = spdk_nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 280 281 CU_ASSERT(rc == -1); 282 283 /* Part 4: Pretend there are no buffer pools */ 284 MOCK_SET(spdk_mempool_get, NULL); 285 reset_nvmf_rdma_request(&rdma_req); 286 sgl->keyed.length = rtransport.transport.opts.io_unit_size * RDMA_UT_UNITS_IN_MAX_IO; 287 rc = spdk_nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 288 289 CU_ASSERT(rc == 0); 290 CU_ASSERT(rdma_req.req.data_from_pool == false); 291 CU_ASSERT(rdma_req.req.data == NULL); 292 CU_ASSERT(rdma_req.data.wr.num_sge == 0); 293 CU_ASSERT(rdma_req.req.buffers[0] == NULL); 294 CU_ASSERT(rdma_req.data.wr.sg_list[0].addr == 0); 295 CU_ASSERT(rdma_req.data.wr.sg_list[0].length == 0); 296 CU_ASSERT(rdma_req.data.wr.sg_list[0].lkey == 0); 297 298 rdma_req.recv->buf = (void *)0xDDDD; 299 /* Test 2: sgl type: keyed data block subtype: offset (in capsule data) */ 300 sgl->generic.type = SPDK_NVME_SGL_TYPE_DATA_BLOCK; 301 sgl->unkeyed.subtype = SPDK_NVME_SGL_SUBTYPE_OFFSET; 302 303 /* Part 1: Normal I/O smaller than in capsule data size no offset */ 304 reset_nvmf_rdma_request(&rdma_req); 305 sgl->address = 0; 306 sgl->unkeyed.length = rtransport.transport.opts.in_capsule_data_size; 307 rc = spdk_nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 308 309 CU_ASSERT(rc == 0); 310 CU_ASSERT(rdma_req.req.data == (void *)0xDDDD); 311 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.in_capsule_data_size); 312 CU_ASSERT(rdma_req.req.data_from_pool == false); 313 314 /* Part 2: I/O offset + length too large */ 315 reset_nvmf_rdma_request(&rdma_req); 316 sgl->address = rtransport.transport.opts.in_capsule_data_size; 317 sgl->unkeyed.length = rtransport.transport.opts.in_capsule_data_size; 318 rc = spdk_nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 319 320 CU_ASSERT(rc == -1); 321 322 /* Part 3: I/O too large */ 323 reset_nvmf_rdma_request(&rdma_req); 324 sgl->address = 0; 325 sgl->unkeyed.length = rtransport.transport.opts.in_capsule_data_size * 2; 326 rc = spdk_nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 327 328 CU_ASSERT(rc == -1); 329 330 /* Test 3: Multi SGL */ 331 sgl->generic.type = SPDK_NVME_SGL_TYPE_LAST_SEGMENT; 332 sgl->unkeyed.subtype = SPDK_NVME_SGL_SUBTYPE_OFFSET; 333 sgl->address = 0; 334 rdma_req.recv->buf = (void *)&sgl_desc; 335 MOCK_SET(spdk_mempool_get, &data); 336 337 /* part 1: 2 segments each with 1 wr. */ 338 reset_nvmf_rdma_request(&rdma_req); 339 sgl->unkeyed.length = 2 * sizeof(struct spdk_nvme_sgl_descriptor); 340 for (i = 0; i < 2; i++) { 341 sgl_desc[i].keyed.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; 342 sgl_desc[i].keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; 343 sgl_desc[i].keyed.length = rtransport.transport.opts.io_unit_size; 344 sgl_desc[i].address = 0x4000 + i * rtransport.transport.opts.io_unit_size; 345 sgl_desc[i].keyed.key = 0x44; 346 } 347 348 rc = spdk_nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 349 350 CU_ASSERT(rc == 0); 351 CU_ASSERT(rdma_req.req.data_from_pool == true); 352 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size * 2); 353 CU_ASSERT(rdma_req.data.wr.num_sge == 1); 354 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0x44); 355 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0x4000); 356 CU_ASSERT(rdma_req.data.wr.next == &data.wr); 357 CU_ASSERT(data.wr.wr.rdma.rkey == 0x44); 358 CU_ASSERT(data.wr.wr.rdma.remote_addr == 0x4000 + rtransport.transport.opts.io_unit_size); 359 CU_ASSERT(data.wr.num_sge == 1); 360 CU_ASSERT(data.wr.next == &rdma_req.rsp.wr); 361 362 /* part 2: 2 segments, each with 1 wr containing 8 sge_elements */ 363 reset_nvmf_rdma_request(&rdma_req); 364 sgl->unkeyed.length = 2 * sizeof(struct spdk_nvme_sgl_descriptor); 365 for (i = 0; i < 2; i++) { 366 sgl_desc[i].keyed.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; 367 sgl_desc[i].keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; 368 sgl_desc[i].keyed.length = rtransport.transport.opts.io_unit_size * 8; 369 sgl_desc[i].address = 0x4000 + i * 8 * rtransport.transport.opts.io_unit_size; 370 sgl_desc[i].keyed.key = 0x44; 371 } 372 373 rc = spdk_nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 374 375 CU_ASSERT(rc == 0); 376 CU_ASSERT(rdma_req.req.data_from_pool == true); 377 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size * 16); 378 CU_ASSERT(rdma_req.req.iovcnt == 16); 379 CU_ASSERT(rdma_req.data.wr.num_sge == 8); 380 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0x44); 381 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0x4000); 382 CU_ASSERT(rdma_req.data.wr.next == &data.wr); 383 CU_ASSERT(data.wr.wr.rdma.rkey == 0x44); 384 CU_ASSERT(data.wr.wr.rdma.remote_addr == 0x4000 + rtransport.transport.opts.io_unit_size * 8); 385 CU_ASSERT(data.wr.num_sge == 8); 386 CU_ASSERT(data.wr.next == &rdma_req.rsp.wr); 387 388 /* part 3: 2 segments, one very large, one very small */ 389 reset_nvmf_rdma_request(&rdma_req); 390 for (i = 0; i < 2; i++) { 391 sgl_desc[i].keyed.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; 392 sgl_desc[i].keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; 393 sgl_desc[i].keyed.key = 0x44; 394 } 395 396 sgl_desc[0].keyed.length = rtransport.transport.opts.io_unit_size * 15 + 397 rtransport.transport.opts.io_unit_size / 2; 398 sgl_desc[0].address = 0x4000; 399 sgl_desc[1].keyed.length = rtransport.transport.opts.io_unit_size / 2; 400 sgl_desc[1].address = 0x4000 + rtransport.transport.opts.io_unit_size * 15 + 401 rtransport.transport.opts.io_unit_size / 2; 402 403 rc = spdk_nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 404 405 CU_ASSERT(rc == 0); 406 CU_ASSERT(rdma_req.req.data_from_pool == true); 407 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size * 16); 408 CU_ASSERT(rdma_req.req.iovcnt == 17); 409 CU_ASSERT(rdma_req.data.wr.num_sge == 16); 410 for (i = 0; i < 15; i++) { 411 CU_ASSERT(rdma_req.data.sgl[i].length == rtransport.transport.opts.io_unit_size); 412 } 413 CU_ASSERT(rdma_req.data.sgl[15].length == rtransport.transport.opts.io_unit_size / 2); 414 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0x44); 415 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0x4000); 416 CU_ASSERT(rdma_req.data.wr.next == &data.wr); 417 CU_ASSERT(data.wr.wr.rdma.rkey == 0x44); 418 CU_ASSERT(data.wr.wr.rdma.remote_addr == 0x4000 + rtransport.transport.opts.io_unit_size * 15 + 419 rtransport.transport.opts.io_unit_size / 2); 420 CU_ASSERT(data.sgl[0].length == rtransport.transport.opts.io_unit_size / 2); 421 CU_ASSERT(data.wr.num_sge == 1); 422 CU_ASSERT(data.wr.next == &rdma_req.rsp.wr); 423 424 /* Test 4: use PG buffer cache */ 425 sgl->generic.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; 426 sgl->keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; 427 sgl->address = 0xFFFF; 428 rdma_req.recv->buf = (void *)0xDDDD; 429 g_rdma_mr.lkey = 0xABCD; 430 sgl->keyed.key = 0xEEEE; 431 432 for (i = 0; i < 4; i++) { 433 STAILQ_INSERT_TAIL(&group.group.buf_cache, &bufs[i], link); 434 } 435 436 /* part 1: use the four buffers from the pg cache */ 437 group.group.buf_cache_size = 4; 438 group.group.buf_cache_count = 4; 439 MOCK_SET(spdk_mempool_get, (void *)0x2000); 440 reset_nvmf_rdma_request(&rdma_req); 441 sgl->keyed.length = rtransport.transport.opts.io_unit_size * 4; 442 rc = spdk_nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 443 444 SPDK_CU_ASSERT_FATAL(rc == 0); 445 CU_ASSERT(rdma_req.req.data_from_pool == true); 446 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size * 4); 447 CU_ASSERT((uint64_t)rdma_req.req.data == (((uint64_t)&bufs[0] + NVMF_DATA_BUFFER_MASK) & 448 ~NVMF_DATA_BUFFER_MASK)); 449 CU_ASSERT(rdma_req.data.wr.num_sge == 4); 450 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 451 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 452 CU_ASSERT(group.group.buf_cache_count == 0); 453 CU_ASSERT(STAILQ_EMPTY(&group.group.buf_cache)); 454 for (i = 0; i < 4; i++) { 455 CU_ASSERT((uint64_t)rdma_req.req.buffers[i] == (uint64_t)&bufs[i]); 456 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == (((uint64_t)&bufs[i] + NVMF_DATA_BUFFER_MASK) & 457 ~NVMF_DATA_BUFFER_MASK)); 458 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == rtransport.transport.opts.io_unit_size); 459 } 460 461 /* part 2: now that we have used the buffers from the cache, try again. We should get mempool buffers. */ 462 reset_nvmf_rdma_request(&rdma_req); 463 rc = spdk_nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 464 465 SPDK_CU_ASSERT_FATAL(rc == 0); 466 CU_ASSERT(rdma_req.req.data_from_pool == true); 467 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size * 4); 468 CU_ASSERT((uint64_t)rdma_req.req.data == 0x2000); 469 CU_ASSERT(rdma_req.data.wr.num_sge == 4); 470 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 471 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 472 CU_ASSERT(group.group.buf_cache_count == 0); 473 CU_ASSERT(STAILQ_EMPTY(&group.group.buf_cache)); 474 for (i = 0; i < 4; i++) { 475 CU_ASSERT((uint64_t)rdma_req.req.buffers[i] == 0x2000); 476 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == 0x2000); 477 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == rtransport.transport.opts.io_unit_size); 478 CU_ASSERT(group.group.buf_cache_count == 0); 479 } 480 481 /* part 3: half and half */ 482 group.group.buf_cache_count = 2; 483 484 for (i = 0; i < 2; i++) { 485 STAILQ_INSERT_TAIL(&group.group.buf_cache, &bufs[i], link); 486 } 487 reset_nvmf_rdma_request(&rdma_req); 488 rc = spdk_nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 489 490 SPDK_CU_ASSERT_FATAL(rc == 0); 491 CU_ASSERT(rdma_req.req.data_from_pool == true); 492 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size * 4); 493 CU_ASSERT((uint64_t)rdma_req.req.data == (((uint64_t)&bufs[0] + NVMF_DATA_BUFFER_MASK) & 494 ~NVMF_DATA_BUFFER_MASK)); 495 CU_ASSERT(rdma_req.data.wr.num_sge == 4); 496 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 497 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 498 CU_ASSERT(group.group.buf_cache_count == 0); 499 for (i = 0; i < 2; i++) { 500 CU_ASSERT((uint64_t)rdma_req.req.buffers[i] == (uint64_t)&bufs[i]); 501 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == (((uint64_t)&bufs[i] + NVMF_DATA_BUFFER_MASK) & 502 ~NVMF_DATA_BUFFER_MASK)); 503 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == rtransport.transport.opts.io_unit_size); 504 } 505 for (i = 2; i < 4; i++) { 506 CU_ASSERT((uint64_t)rdma_req.req.buffers[i] == 0x2000); 507 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == 0x2000); 508 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == rtransport.transport.opts.io_unit_size); 509 } 510 511 reset_nvmf_rdma_request(&rdma_req); 512 /* Test 5 dealing with a buffer split over two Memory Regions */ 513 MOCK_SET(spdk_mempool_get, (void *)&buffer); 514 sgl->generic.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; 515 sgl->keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; 516 sgl->keyed.length = rtransport.transport.opts.io_unit_size / 2; 517 g_mr_size = rtransport.transport.opts.io_unit_size / 4; 518 g_mr_next_size = rtransport.transport.opts.io_unit_size / 2; 519 520 rc = spdk_nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 521 SPDK_CU_ASSERT_FATAL(rc == 0); 522 CU_ASSERT(rdma_req.req.data_from_pool == true); 523 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size / 2); 524 CU_ASSERT((uint64_t)rdma_req.req.data == (((uint64_t)&buffer + NVMF_DATA_BUFFER_MASK) & 525 ~NVMF_DATA_BUFFER_MASK)); 526 CU_ASSERT(rdma_req.data.wr.num_sge == 1); 527 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 528 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 529 CU_ASSERT(rdma_req.req.buffers[0] == &buffer); 530 CU_ASSERT(rdma_req.data.wr.sg_list[0].addr == (((uint64_t)&buffer + NVMF_DATA_BUFFER_MASK) & 531 ~NVMF_DATA_BUFFER_MASK)); 532 CU_ASSERT(rdma_req.data.wr.sg_list[0].length == rtransport.transport.opts.io_unit_size / 2); 533 CU_ASSERT(rdma_req.data.wr.sg_list[0].lkey == g_rdma_mr.lkey); 534 buffer_ptr = STAILQ_FIRST(&group.retired_bufs); 535 CU_ASSERT(buffer_ptr == &buffer); 536 STAILQ_REMOVE(&group.retired_bufs, buffer_ptr, spdk_nvmf_transport_pg_cache_buf, link); 537 CU_ASSERT(STAILQ_EMPTY(&group.retired_bufs)); 538 539 reset_nvmf_rdma_request(&rdma_req); 540 } 541 542 static struct spdk_nvmf_rdma_recv * 543 create_recv(struct spdk_nvmf_rdma_qpair *rqpair, enum spdk_nvme_nvm_opcode opc) 544 { 545 struct spdk_nvmf_rdma_recv *rdma_recv; 546 union nvmf_h2c_msg *cmd; 547 struct spdk_nvme_sgl_descriptor *sgl; 548 549 rdma_recv = calloc(1, sizeof(*rdma_recv)); 550 rdma_recv->qpair = rqpair; 551 cmd = calloc(1, sizeof(*cmd)); 552 rdma_recv->sgl[0].addr = (uintptr_t)cmd; 553 cmd->nvme_cmd.opc = opc; 554 sgl = &cmd->nvme_cmd.dptr.sgl1; 555 sgl->keyed.key = 0xEEEE; 556 sgl->address = 0xFFFF; 557 sgl->keyed.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; 558 sgl->keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; 559 sgl->keyed.length = 1; 560 561 return rdma_recv; 562 } 563 564 static void 565 free_recv(struct spdk_nvmf_rdma_recv *rdma_recv) 566 { 567 free((void *)rdma_recv->sgl[0].addr); 568 free(rdma_recv); 569 } 570 571 static struct spdk_nvmf_rdma_request * 572 create_req(struct spdk_nvmf_rdma_qpair *rqpair, 573 struct spdk_nvmf_rdma_recv *rdma_recv) 574 { 575 struct spdk_nvmf_rdma_request *rdma_req; 576 union nvmf_c2h_msg *cpl; 577 578 rdma_req = calloc(1, sizeof(*rdma_req)); 579 rdma_req->recv = rdma_recv; 580 rdma_req->req.qpair = &rqpair->qpair; 581 rdma_req->state = RDMA_REQUEST_STATE_NEW; 582 rdma_req->data.wr.wr_id = (uintptr_t)&rdma_req->data.rdma_wr; 583 rdma_req->data.wr.sg_list = rdma_req->data.sgl; 584 cpl = calloc(1, sizeof(*cpl)); 585 rdma_req->rsp.sgl[0].addr = (uintptr_t)cpl; 586 rdma_req->req.rsp = cpl; 587 588 return rdma_req; 589 } 590 591 static void 592 free_req(struct spdk_nvmf_rdma_request *rdma_req) 593 { 594 free((void *)rdma_req->rsp.sgl[0].addr); 595 free(rdma_req); 596 } 597 598 static void 599 qpair_reset(struct spdk_nvmf_rdma_qpair *rqpair, 600 struct spdk_nvmf_rdma_poller *poller, 601 struct spdk_nvmf_rdma_port *port, 602 struct spdk_nvmf_rdma_resources *resources) 603 { 604 memset(rqpair, 0, sizeof(*rqpair)); 605 STAILQ_INIT(&rqpair->pending_rdma_write_queue); 606 STAILQ_INIT(&rqpair->pending_rdma_read_queue); 607 rqpair->poller = poller; 608 rqpair->port = port; 609 rqpair->resources = resources; 610 rqpair->qpair.qid = 1; 611 rqpair->ibv_state = IBV_QPS_RTS; 612 rqpair->qpair.state = SPDK_NVMF_QPAIR_ACTIVE; 613 rqpair->max_send_sge = SPDK_NVMF_MAX_SGL_ENTRIES; 614 rqpair->max_send_depth = 16; 615 rqpair->max_read_depth = 16; 616 resources->recvs_to_post.first = resources->recvs_to_post.last = NULL; 617 } 618 619 static void 620 poller_reset(struct spdk_nvmf_rdma_poller *poller, 621 struct spdk_nvmf_rdma_poll_group *group) 622 { 623 memset(poller, 0, sizeof(*poller)); 624 STAILQ_INIT(&poller->qpairs_pending_recv); 625 STAILQ_INIT(&poller->qpairs_pending_send); 626 poller->group = group; 627 } 628 629 static void 630 test_spdk_nvmf_rdma_request_process(void) 631 { 632 struct spdk_nvmf_rdma_transport rtransport = {}; 633 struct spdk_nvmf_rdma_poll_group group = {}; 634 struct spdk_nvmf_rdma_poller poller = {}; 635 struct spdk_nvmf_rdma_port port = {}; 636 struct spdk_nvmf_rdma_device device = {}; 637 struct spdk_nvmf_rdma_resources resources = {}; 638 struct spdk_nvmf_rdma_qpair rqpair = {}; 639 struct spdk_nvmf_rdma_recv *rdma_recv; 640 struct spdk_nvmf_rdma_request *rdma_req; 641 bool progress; 642 643 STAILQ_INIT(&group.group.buf_cache); 644 STAILQ_INIT(&group.group.pending_buf_queue); 645 group.group.buf_cache_size = 0; 646 group.group.buf_cache_count = 0; 647 port.device = &device; 648 poller_reset(&poller, &group); 649 qpair_reset(&rqpair, &poller, &port, &resources); 650 651 rtransport.transport.opts = g_rdma_ut_transport_opts; 652 rtransport.transport.data_buf_pool = spdk_mempool_create("test_data_pool", 16, 128, 0, 0); 653 rtransport.data_wr_pool = spdk_mempool_create("test_wr_pool", 128, 654 sizeof(struct spdk_nvmf_rdma_request_data), 655 0, 0); 656 MOCK_CLEAR(spdk_mempool_get); 657 658 device.attr.device_cap_flags = 0; 659 device.map = (void *)0x0; 660 g_rdma_mr.lkey = 0xABCD; 661 662 /* Test 1: single SGL READ request */ 663 rdma_recv = create_recv(&rqpair, SPDK_NVME_OPC_READ); 664 rdma_req = create_req(&rqpair, rdma_recv); 665 rqpair.current_recv_depth = 1; 666 /* NEW -> EXECUTING */ 667 progress = spdk_nvmf_rdma_request_process(&rtransport, rdma_req); 668 CU_ASSERT(progress == true); 669 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_EXECUTING); 670 CU_ASSERT(rdma_req->req.xfer == SPDK_NVME_DATA_CONTROLLER_TO_HOST); 671 /* EXECUTED -> TRANSFERRING_C2H */ 672 rdma_req->state = RDMA_REQUEST_STATE_EXECUTED; 673 progress = spdk_nvmf_rdma_request_process(&rtransport, rdma_req); 674 CU_ASSERT(progress == true); 675 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_TRANSFERRING_CONTROLLER_TO_HOST); 676 CU_ASSERT(rdma_req->recv == NULL); 677 CU_ASSERT(rqpair.sends_to_post.first == &rdma_req->data.wr); 678 CU_ASSERT(rqpair.sends_to_post.last == &rdma_req->rsp.wr); 679 CU_ASSERT(resources.recvs_to_post.first == &rdma_recv->wr); 680 CU_ASSERT(resources.recvs_to_post.last == &rdma_recv->wr); 681 /* COMPLETED -> FREE */ 682 rdma_req->state = RDMA_REQUEST_STATE_COMPLETED; 683 progress = spdk_nvmf_rdma_request_process(&rtransport, rdma_req); 684 CU_ASSERT(progress == true); 685 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_FREE); 686 687 free_recv(rdma_recv); 688 free_req(rdma_req); 689 poller_reset(&poller, &group); 690 qpair_reset(&rqpair, &poller, &port, &resources); 691 692 /* Test 2: single SGL WRITE request */ 693 rdma_recv = create_recv(&rqpair, SPDK_NVME_OPC_WRITE); 694 rdma_req = create_req(&rqpair, rdma_recv); 695 rqpair.current_recv_depth = 1; 696 /* NEW -> TRANSFERRING_H2C */ 697 progress = spdk_nvmf_rdma_request_process(&rtransport, rdma_req); 698 CU_ASSERT(progress == true); 699 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_TRANSFERRING_HOST_TO_CONTROLLER); 700 CU_ASSERT(rdma_req->req.xfer == SPDK_NVME_DATA_HOST_TO_CONTROLLER); 701 CU_ASSERT(rqpair.sends_to_post.first == &rdma_req->data.wr); 702 CU_ASSERT(rqpair.sends_to_post.last == &rdma_req->data.wr); 703 rqpair.sends_to_post.first = rqpair.sends_to_post.last = NULL; 704 STAILQ_INIT(&poller.qpairs_pending_send); 705 /* READY_TO_EXECUTE -> EXECUTING */ 706 rdma_req->state = RDMA_REQUEST_STATE_READY_TO_EXECUTE; 707 progress = spdk_nvmf_rdma_request_process(&rtransport, rdma_req); 708 CU_ASSERT(progress == true); 709 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_EXECUTING); 710 /* EXECUTED -> COMPLETING */ 711 rdma_req->state = RDMA_REQUEST_STATE_EXECUTED; 712 progress = spdk_nvmf_rdma_request_process(&rtransport, rdma_req); 713 CU_ASSERT(progress == true); 714 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_COMPLETING); 715 CU_ASSERT(rdma_req->recv == NULL); 716 CU_ASSERT(rqpair.sends_to_post.first == &rdma_req->rsp.wr); 717 CU_ASSERT(rqpair.sends_to_post.last == &rdma_req->rsp.wr); 718 CU_ASSERT(resources.recvs_to_post.first == &rdma_recv->wr); 719 CU_ASSERT(resources.recvs_to_post.last == &rdma_recv->wr); 720 /* COMPLETED -> FREE */ 721 rdma_req->state = RDMA_REQUEST_STATE_COMPLETED; 722 progress = spdk_nvmf_rdma_request_process(&rtransport, rdma_req); 723 CU_ASSERT(progress == true); 724 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_FREE); 725 726 free_recv(rdma_recv); 727 free_req(rdma_req); 728 poller_reset(&poller, &group); 729 qpair_reset(&rqpair, &poller, &port, &resources); 730 731 /* Test 3: WRITE+WRITE ibv_send batching */ 732 { 733 struct spdk_nvmf_rdma_recv *recv1, *recv2; 734 struct spdk_nvmf_rdma_request *req1, *req2; 735 recv1 = create_recv(&rqpair, SPDK_NVME_OPC_WRITE); 736 req1 = create_req(&rqpair, recv1); 737 recv2 = create_recv(&rqpair, SPDK_NVME_OPC_WRITE); 738 req2 = create_req(&rqpair, recv2); 739 740 /* WRITE 1: NEW -> TRANSFERRING_H2C */ 741 rqpair.current_recv_depth = 1; 742 spdk_nvmf_rdma_request_process(&rtransport, req1); 743 CU_ASSERT(req1->state == RDMA_REQUEST_STATE_TRANSFERRING_HOST_TO_CONTROLLER); 744 /* WRITE 1 is the first in batching list */ 745 CU_ASSERT(rqpair.sends_to_post.first == &req1->data.wr); 746 CU_ASSERT(rqpair.sends_to_post.last == &req1->data.wr); 747 748 /* WRITE 2: NEW -> TRANSFERRING_H2C */ 749 rqpair.current_recv_depth = 2; 750 spdk_nvmf_rdma_request_process(&rtransport, req2); 751 CU_ASSERT(req2->state == RDMA_REQUEST_STATE_TRANSFERRING_HOST_TO_CONTROLLER); 752 /* WRITE 2 is now also in the batching list */ 753 CU_ASSERT(rqpair.sends_to_post.first->next == &req2->data.wr); 754 CU_ASSERT(rqpair.sends_to_post.last == &req2->data.wr); 755 756 /* Send everything */ 757 rqpair.sends_to_post.first = rqpair.sends_to_post.last = NULL; 758 STAILQ_INIT(&poller.qpairs_pending_send); 759 760 /* WRITE 1 completes before WRITE 2 has finished RDMA reading */ 761 /* WRITE 1: READY_TO_EXECUTE -> EXECUTING */ 762 req1->state = RDMA_REQUEST_STATE_READY_TO_EXECUTE; 763 spdk_nvmf_rdma_request_process(&rtransport, req1); 764 CU_ASSERT(req1->state == RDMA_REQUEST_STATE_EXECUTING); 765 /* WRITE 1: EXECUTED -> COMPLETING */ 766 req1->state = RDMA_REQUEST_STATE_EXECUTED; 767 spdk_nvmf_rdma_request_process(&rtransport, req1); 768 CU_ASSERT(req1->state == RDMA_REQUEST_STATE_COMPLETING); 769 CU_ASSERT(rqpair.sends_to_post.first == &req1->rsp.wr); 770 CU_ASSERT(rqpair.sends_to_post.last == &req1->rsp.wr); 771 rqpair.sends_to_post.first = rqpair.sends_to_post.last = NULL; 772 STAILQ_INIT(&poller.qpairs_pending_send); 773 /* WRITE 1: COMPLETED -> FREE */ 774 req1->state = RDMA_REQUEST_STATE_COMPLETED; 775 spdk_nvmf_rdma_request_process(&rtransport, req1); 776 CU_ASSERT(req1->state == RDMA_REQUEST_STATE_FREE); 777 778 /* Now WRITE 2 has finished reading and completes */ 779 /* WRITE 2: COMPLETED -> FREE */ 780 /* WRITE 2: READY_TO_EXECUTE -> EXECUTING */ 781 req2->state = RDMA_REQUEST_STATE_READY_TO_EXECUTE; 782 spdk_nvmf_rdma_request_process(&rtransport, req2); 783 CU_ASSERT(req2->state == RDMA_REQUEST_STATE_EXECUTING); 784 /* WRITE 1: EXECUTED -> COMPLETING */ 785 req2->state = RDMA_REQUEST_STATE_EXECUTED; 786 spdk_nvmf_rdma_request_process(&rtransport, req2); 787 CU_ASSERT(req2->state == RDMA_REQUEST_STATE_COMPLETING); 788 CU_ASSERT(rqpair.sends_to_post.first == &req2->rsp.wr); 789 CU_ASSERT(rqpair.sends_to_post.last == &req2->rsp.wr); 790 rqpair.sends_to_post.first = rqpair.sends_to_post.last = NULL; 791 STAILQ_INIT(&poller.qpairs_pending_send); 792 /* WRITE 1: COMPLETED -> FREE */ 793 req2->state = RDMA_REQUEST_STATE_COMPLETED; 794 spdk_nvmf_rdma_request_process(&rtransport, req2); 795 CU_ASSERT(req2->state == RDMA_REQUEST_STATE_FREE); 796 797 free_recv(recv1); 798 free_req(req1); 799 free_recv(recv2); 800 free_req(req2); 801 poller_reset(&poller, &group); 802 qpair_reset(&rqpair, &poller, &port, &resources); 803 } 804 805 spdk_mempool_free(rtransport.transport.data_buf_pool); 806 spdk_mempool_free(rtransport.data_wr_pool); 807 } 808 809 #define TEST_GROUPS_COUNT 5 810 static void 811 test_spdk_nvmf_rdma_get_optimal_poll_group(void) 812 { 813 struct spdk_nvmf_rdma_transport rtransport = {}; 814 struct spdk_nvmf_transport *transport = &rtransport.transport; 815 struct spdk_nvmf_rdma_qpair rqpair = {}; 816 struct spdk_nvmf_transport_poll_group *groups[TEST_GROUPS_COUNT]; 817 struct spdk_nvmf_rdma_poll_group *rgroups[TEST_GROUPS_COUNT]; 818 struct spdk_nvmf_transport_poll_group *result; 819 uint32_t i; 820 821 rqpair.qpair.transport = transport; 822 pthread_mutex_init(&rtransport.lock, NULL); 823 TAILQ_INIT(&rtransport.poll_groups); 824 825 for (i = 0; i < TEST_GROUPS_COUNT; i++) { 826 groups[i] = spdk_nvmf_rdma_poll_group_create(transport); 827 CU_ASSERT(groups[i] != NULL); 828 rgroups[i] = SPDK_CONTAINEROF(groups[i], struct spdk_nvmf_rdma_poll_group, group); 829 groups[i]->transport = transport; 830 } 831 CU_ASSERT(rtransport.conn_sched.next_admin_pg == rgroups[0]); 832 CU_ASSERT(rtransport.conn_sched.next_io_pg == rgroups[0]); 833 834 /* Emulate connection of %TEST_GROUPS_COUNT% initiators - each creates 1 admin and 1 io qp */ 835 for (i = 0; i < TEST_GROUPS_COUNT; i++) { 836 rqpair.qpair.qid = 0; 837 result = spdk_nvmf_rdma_get_optimal_poll_group(&rqpair.qpair); 838 CU_ASSERT(result == groups[i]); 839 CU_ASSERT(rtransport.conn_sched.next_admin_pg == rgroups[(i + 1) % TEST_GROUPS_COUNT]); 840 CU_ASSERT(rtransport.conn_sched.next_io_pg == rgroups[i]); 841 842 rqpair.qpair.qid = 1; 843 result = spdk_nvmf_rdma_get_optimal_poll_group(&rqpair.qpair); 844 CU_ASSERT(result == groups[i]); 845 CU_ASSERT(rtransport.conn_sched.next_admin_pg == rgroups[(i + 1) % TEST_GROUPS_COUNT]); 846 CU_ASSERT(rtransport.conn_sched.next_io_pg == rgroups[(i + 1) % TEST_GROUPS_COUNT]); 847 } 848 /* wrap around, admin/io pg point to the first pg 849 Destroy all poll groups except of the last one */ 850 for (i = 0; i < TEST_GROUPS_COUNT - 1; i++) { 851 spdk_nvmf_rdma_poll_group_destroy(groups[i]); 852 CU_ASSERT(rtransport.conn_sched.next_admin_pg == rgroups[i + 1]); 853 CU_ASSERT(rtransport.conn_sched.next_io_pg == rgroups[i + 1]); 854 } 855 856 CU_ASSERT(rtransport.conn_sched.next_admin_pg == rgroups[TEST_GROUPS_COUNT - 1]); 857 CU_ASSERT(rtransport.conn_sched.next_io_pg == rgroups[TEST_GROUPS_COUNT - 1]); 858 859 /* Check that pointers to the next admin/io poll groups are not changed */ 860 rqpair.qpair.qid = 0; 861 result = spdk_nvmf_rdma_get_optimal_poll_group(&rqpair.qpair); 862 CU_ASSERT(result == groups[TEST_GROUPS_COUNT - 1]); 863 CU_ASSERT(rtransport.conn_sched.next_admin_pg == rgroups[TEST_GROUPS_COUNT - 1]); 864 CU_ASSERT(rtransport.conn_sched.next_io_pg == rgroups[TEST_GROUPS_COUNT - 1]); 865 866 rqpair.qpair.qid = 1; 867 result = spdk_nvmf_rdma_get_optimal_poll_group(&rqpair.qpair); 868 CU_ASSERT(result == groups[TEST_GROUPS_COUNT - 1]); 869 CU_ASSERT(rtransport.conn_sched.next_admin_pg == rgroups[TEST_GROUPS_COUNT - 1]); 870 CU_ASSERT(rtransport.conn_sched.next_io_pg == rgroups[TEST_GROUPS_COUNT - 1]); 871 872 /* Remove the last poll group, check that pointers are NULL */ 873 spdk_nvmf_rdma_poll_group_destroy(groups[TEST_GROUPS_COUNT - 1]); 874 CU_ASSERT(rtransport.conn_sched.next_admin_pg == NULL); 875 CU_ASSERT(rtransport.conn_sched.next_io_pg == NULL); 876 877 /* Request optimal poll group, result must be NULL */ 878 rqpair.qpair.qid = 0; 879 result = spdk_nvmf_rdma_get_optimal_poll_group(&rqpair.qpair); 880 CU_ASSERT(result == NULL); 881 882 rqpair.qpair.qid = 1; 883 result = spdk_nvmf_rdma_get_optimal_poll_group(&rqpair.qpair); 884 CU_ASSERT(result == NULL); 885 886 pthread_mutex_destroy(&rtransport.lock); 887 } 888 #undef TEST_GROUPS_COUNT 889 890 int main(int argc, char **argv) 891 { 892 CU_pSuite suite = NULL; 893 unsigned int num_failures; 894 895 if (CU_initialize_registry() != CUE_SUCCESS) { 896 return CU_get_error(); 897 } 898 899 suite = CU_add_suite("nvmf", NULL, NULL); 900 if (suite == NULL) { 901 CU_cleanup_registry(); 902 return CU_get_error(); 903 } 904 905 if (!CU_add_test(suite, "test_parse_sgl", test_spdk_nvmf_rdma_request_parse_sgl) || 906 !CU_add_test(suite, "test_request_process", test_spdk_nvmf_rdma_request_process) || 907 !CU_add_test(suite, "test_optimal_pg", test_spdk_nvmf_rdma_get_optimal_poll_group)) { 908 CU_cleanup_registry(); 909 return CU_get_error(); 910 } 911 912 CU_basic_set_mode(CU_BRM_VERBOSE); 913 CU_basic_run_tests(); 914 num_failures = CU_get_number_of_failures(); 915 CU_cleanup_registry(); 916 return num_failures; 917 } 918