1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) Intel Corporation. All rights reserved. 3 * Copyright (c) 2019, 2021 Mellanox Technologies LTD. All rights reserved. 4 */ 5 6 #include "spdk/stdinc.h" 7 #include "spdk_cunit.h" 8 #include "common/lib/test_env.c" 9 #include "common/lib/test_rdma.c" 10 #include "nvmf/rdma.c" 11 #include "nvmf/transport.c" 12 13 #define RDMA_UT_UNITS_IN_MAX_IO 16 14 15 struct spdk_nvmf_transport_opts g_rdma_ut_transport_opts = { 16 .max_queue_depth = SPDK_NVMF_RDMA_DEFAULT_MAX_QUEUE_DEPTH, 17 .max_qpairs_per_ctrlr = SPDK_NVMF_RDMA_DEFAULT_MAX_QPAIRS_PER_CTRLR, 18 .in_capsule_data_size = SPDK_NVMF_RDMA_DEFAULT_IN_CAPSULE_DATA_SIZE, 19 .max_io_size = (SPDK_NVMF_RDMA_MIN_IO_BUFFER_SIZE * RDMA_UT_UNITS_IN_MAX_IO), 20 .io_unit_size = SPDK_NVMF_RDMA_MIN_IO_BUFFER_SIZE, 21 .max_aq_depth = SPDK_NVMF_RDMA_DEFAULT_AQ_DEPTH, 22 .num_shared_buffers = SPDK_NVMF_RDMA_DEFAULT_NUM_SHARED_BUFFERS, 23 }; 24 25 SPDK_LOG_REGISTER_COMPONENT(nvmf) 26 DEFINE_STUB(spdk_mem_map_set_translation, int, (struct spdk_mem_map *map, uint64_t vaddr, 27 uint64_t size, uint64_t translation), 0); 28 DEFINE_STUB(spdk_mem_map_clear_translation, int, (struct spdk_mem_map *map, uint64_t vaddr, 29 uint64_t size), 0); 30 DEFINE_STUB(spdk_mem_map_alloc, struct spdk_mem_map *, (uint64_t default_translation, 31 const struct spdk_mem_map_ops *ops, void *cb_ctx), NULL); 32 DEFINE_STUB(spdk_nvmf_qpair_disconnect, int, (struct spdk_nvmf_qpair *qpair, 33 nvmf_qpair_disconnect_cb cb_fn, void *ctx), 0); 34 DEFINE_STUB(spdk_nvmf_qpair_get_listen_trid, int, 35 (struct spdk_nvmf_qpair *qpair, struct spdk_nvme_transport_id *trid), 0); 36 DEFINE_STUB_V(spdk_mem_map_free, (struct spdk_mem_map **pmap)); 37 38 DEFINE_STUB_V(spdk_nvmf_ctrlr_data_init, (struct spdk_nvmf_transport_opts *opts, 39 struct spdk_nvmf_ctrlr_data *cdata)); 40 DEFINE_STUB_V(spdk_nvmf_request_exec, (struct spdk_nvmf_request *req)); 41 DEFINE_STUB(spdk_nvmf_request_complete, int, (struct spdk_nvmf_request *req), 0); 42 DEFINE_STUB(spdk_nvme_transport_id_compare, int, (const struct spdk_nvme_transport_id *trid1, 43 const struct spdk_nvme_transport_id *trid2), 0); 44 DEFINE_STUB_V(nvmf_ctrlr_abort_aer, (struct spdk_nvmf_ctrlr *ctrlr)); 45 DEFINE_STUB(spdk_nvmf_request_get_dif_ctx, bool, (struct spdk_nvmf_request *req, 46 struct spdk_dif_ctx *dif_ctx), false); 47 DEFINE_STUB_V(spdk_nvme_trid_populate_transport, (struct spdk_nvme_transport_id *trid, 48 enum spdk_nvme_transport_type trtype)); 49 DEFINE_STUB_V(spdk_nvmf_tgt_new_qpair, (struct spdk_nvmf_tgt *tgt, struct spdk_nvmf_qpair *qpair)); 50 DEFINE_STUB(nvmf_ctrlr_abort_request, int, (struct spdk_nvmf_request *req), 0); 51 DEFINE_STUB(spdk_nvme_transport_id_adrfam_str, const char *, (enum spdk_nvmf_adrfam adrfam), NULL); 52 DEFINE_STUB(ibv_dereg_mr, int, (struct ibv_mr *mr), 0); 53 DEFINE_STUB(ibv_resize_cq, int, (struct ibv_cq *cq, int cqe), 0); 54 55 /* ibv_reg_mr can be a macro, need to undefine it */ 56 #ifdef ibv_reg_mr 57 #undef ibv_reg_mr 58 #endif 59 60 DEFINE_RETURN_MOCK(ibv_reg_mr, struct ibv_mr *); 61 struct ibv_mr * 62 ibv_reg_mr(struct ibv_pd *pd, void *addr, size_t length, int access) 63 { 64 HANDLE_RETURN_MOCK(ibv_reg_mr); 65 if (length > 0) { 66 return &g_rdma_mr; 67 } else { 68 return NULL; 69 } 70 } 71 72 int 73 ibv_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, 74 int attr_mask, struct ibv_qp_init_attr *init_attr) 75 { 76 if (qp == NULL) { 77 return -1; 78 } else { 79 attr->port_num = 80; 80 81 if (qp->state == IBV_QPS_ERR) { 82 attr->qp_state = 10; 83 } else { 84 attr->qp_state = IBV_QPS_INIT; 85 } 86 87 return 0; 88 } 89 } 90 91 const char * 92 spdk_nvme_transport_id_trtype_str(enum spdk_nvme_transport_type trtype) 93 { 94 switch (trtype) { 95 case SPDK_NVME_TRANSPORT_PCIE: 96 return "PCIe"; 97 case SPDK_NVME_TRANSPORT_RDMA: 98 return "RDMA"; 99 case SPDK_NVME_TRANSPORT_FC: 100 return "FC"; 101 default: 102 return NULL; 103 } 104 } 105 106 int 107 spdk_nvme_transport_id_populate_trstring(struct spdk_nvme_transport_id *trid, const char *trstring) 108 { 109 int len, i; 110 111 if (trstring == NULL) { 112 return -EINVAL; 113 } 114 115 len = strnlen(trstring, SPDK_NVMF_TRSTRING_MAX_LEN); 116 if (len == SPDK_NVMF_TRSTRING_MAX_LEN) { 117 return -EINVAL; 118 } 119 120 /* cast official trstring to uppercase version of input. */ 121 for (i = 0; i < len; i++) { 122 trid->trstring[i] = toupper(trstring[i]); 123 } 124 return 0; 125 } 126 127 static void 128 reset_nvmf_rdma_request(struct spdk_nvmf_rdma_request *rdma_req) 129 { 130 int i; 131 132 rdma_req->req.length = 0; 133 rdma_req->req.data_from_pool = false; 134 rdma_req->req.data = NULL; 135 rdma_req->data.wr.num_sge = 0; 136 rdma_req->data.wr.wr.rdma.remote_addr = 0; 137 rdma_req->data.wr.wr.rdma.rkey = 0; 138 rdma_req->offset = 0; 139 memset(&rdma_req->req.dif, 0, sizeof(rdma_req->req.dif)); 140 141 for (i = 0; i < SPDK_NVMF_MAX_SGL_ENTRIES; i++) { 142 rdma_req->req.iov[i].iov_base = 0; 143 rdma_req->req.iov[i].iov_len = 0; 144 rdma_req->req.buffers[i] = 0; 145 rdma_req->data.wr.sg_list[i].addr = 0; 146 rdma_req->data.wr.sg_list[i].length = 0; 147 rdma_req->data.wr.sg_list[i].lkey = 0; 148 } 149 rdma_req->req.iovcnt = 0; 150 if (rdma_req->req.stripped_data) { 151 free(rdma_req->req.stripped_data); 152 rdma_req->req.stripped_data = NULL; 153 } 154 } 155 156 static void 157 test_spdk_nvmf_rdma_request_parse_sgl(void) 158 { 159 struct spdk_nvmf_rdma_transport rtransport; 160 struct spdk_nvmf_rdma_device device; 161 struct spdk_nvmf_rdma_request rdma_req = {}; 162 struct spdk_nvmf_rdma_recv recv; 163 struct spdk_nvmf_rdma_poll_group group; 164 struct spdk_nvmf_rdma_qpair rqpair; 165 struct spdk_nvmf_rdma_poller poller; 166 union nvmf_c2h_msg cpl; 167 union nvmf_h2c_msg cmd; 168 struct spdk_nvme_sgl_descriptor *sgl; 169 struct spdk_nvmf_transport_pg_cache_buf bufs[4]; 170 struct spdk_nvme_sgl_descriptor sgl_desc[SPDK_NVMF_MAX_SGL_ENTRIES] = {{0}}; 171 struct spdk_nvmf_rdma_request_data data; 172 int rc, i; 173 uint32_t sgl_length; 174 uintptr_t aligned_buffer_address; 175 176 data.wr.sg_list = data.sgl; 177 STAILQ_INIT(&group.group.buf_cache); 178 group.group.buf_cache_size = 0; 179 group.group.buf_cache_count = 0; 180 group.group.transport = &rtransport.transport; 181 poller.group = &group; 182 rqpair.poller = &poller; 183 rqpair.max_send_sge = SPDK_NVMF_MAX_SGL_ENTRIES; 184 185 sgl = &cmd.nvme_cmd.dptr.sgl1; 186 rdma_req.recv = &recv; 187 rdma_req.req.cmd = &cmd; 188 rdma_req.req.rsp = &cpl; 189 rdma_req.data.wr.sg_list = rdma_req.data.sgl; 190 rdma_req.req.qpair = &rqpair.qpair; 191 rdma_req.req.xfer = SPDK_NVME_DATA_CONTROLLER_TO_HOST; 192 193 rtransport.transport.opts = g_rdma_ut_transport_opts; 194 rtransport.data_wr_pool = NULL; 195 rtransport.transport.data_buf_pool = NULL; 196 197 device.attr.device_cap_flags = 0; 198 sgl->keyed.key = 0xEEEE; 199 sgl->address = 0xFFFF; 200 rdma_req.recv->buf = (void *)0xDDDD; 201 202 /* Test 1: sgl type: keyed data block subtype: address */ 203 sgl->generic.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; 204 sgl->keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; 205 206 /* Part 1: simple I/O, one SGL smaller than the transport io unit size */ 207 MOCK_SET(spdk_mempool_get, (void *)0x2000); 208 reset_nvmf_rdma_request(&rdma_req); 209 sgl->keyed.length = rtransport.transport.opts.io_unit_size / 2; 210 211 device.map = (void *)0x0; 212 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 213 CU_ASSERT(rc == 0); 214 CU_ASSERT(rdma_req.req.data_from_pool == true); 215 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size / 2); 216 CU_ASSERT((uint64_t)rdma_req.req.data == 0x2000); 217 CU_ASSERT(rdma_req.data.wr.num_sge == 1); 218 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 219 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 220 CU_ASSERT((uint64_t)rdma_req.req.buffers[0] == 0x2000); 221 CU_ASSERT(rdma_req.data.wr.sg_list[0].addr == 0x2000); 222 CU_ASSERT(rdma_req.data.wr.sg_list[0].length == rtransport.transport.opts.io_unit_size / 2); 223 CU_ASSERT(rdma_req.data.wr.sg_list[0].lkey == RDMA_UT_LKEY); 224 225 /* Part 2: simple I/O, one SGL larger than the transport io unit size (equal to the max io size) */ 226 reset_nvmf_rdma_request(&rdma_req); 227 sgl->keyed.length = rtransport.transport.opts.io_unit_size * RDMA_UT_UNITS_IN_MAX_IO; 228 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 229 230 CU_ASSERT(rc == 0); 231 CU_ASSERT(rdma_req.req.data_from_pool == true); 232 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size * RDMA_UT_UNITS_IN_MAX_IO); 233 CU_ASSERT(rdma_req.data.wr.num_sge == RDMA_UT_UNITS_IN_MAX_IO); 234 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 235 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 236 for (i = 0; i < RDMA_UT_UNITS_IN_MAX_IO; i++) { 237 CU_ASSERT((uint64_t)rdma_req.req.buffers[i] == 0x2000); 238 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == 0x2000); 239 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == rtransport.transport.opts.io_unit_size); 240 CU_ASSERT(rdma_req.data.wr.sg_list[i].lkey == RDMA_UT_LKEY); 241 } 242 243 /* Part 3: simple I/O one SGL larger than the transport max io size */ 244 reset_nvmf_rdma_request(&rdma_req); 245 sgl->keyed.length = rtransport.transport.opts.max_io_size * 2; 246 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 247 248 CU_ASSERT(rc == -1); 249 250 /* Part 4: Pretend there are no buffer pools */ 251 MOCK_SET(spdk_mempool_get, NULL); 252 reset_nvmf_rdma_request(&rdma_req); 253 sgl->keyed.length = rtransport.transport.opts.io_unit_size * RDMA_UT_UNITS_IN_MAX_IO; 254 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 255 256 CU_ASSERT(rc == 0); 257 CU_ASSERT(rdma_req.req.data_from_pool == false); 258 CU_ASSERT(rdma_req.req.data == NULL); 259 CU_ASSERT(rdma_req.data.wr.num_sge == 0); 260 CU_ASSERT(rdma_req.req.buffers[0] == NULL); 261 CU_ASSERT(rdma_req.data.wr.sg_list[0].addr == 0); 262 CU_ASSERT(rdma_req.data.wr.sg_list[0].length == 0); 263 CU_ASSERT(rdma_req.data.wr.sg_list[0].lkey == 0); 264 265 rdma_req.recv->buf = (void *)0xDDDD; 266 /* Test 2: sgl type: keyed data block subtype: offset (in capsule data) */ 267 sgl->generic.type = SPDK_NVME_SGL_TYPE_DATA_BLOCK; 268 sgl->unkeyed.subtype = SPDK_NVME_SGL_SUBTYPE_OFFSET; 269 270 /* Part 1: Normal I/O smaller than in capsule data size no offset */ 271 reset_nvmf_rdma_request(&rdma_req); 272 sgl->address = 0; 273 sgl->unkeyed.length = rtransport.transport.opts.in_capsule_data_size; 274 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 275 276 CU_ASSERT(rc == 0); 277 CU_ASSERT(rdma_req.req.data == (void *)0xDDDD); 278 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.in_capsule_data_size); 279 CU_ASSERT(rdma_req.req.data_from_pool == false); 280 281 /* Part 2: I/O offset + length too large */ 282 reset_nvmf_rdma_request(&rdma_req); 283 sgl->address = rtransport.transport.opts.in_capsule_data_size; 284 sgl->unkeyed.length = rtransport.transport.opts.in_capsule_data_size; 285 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 286 287 CU_ASSERT(rc == -1); 288 289 /* Part 3: I/O too large */ 290 reset_nvmf_rdma_request(&rdma_req); 291 sgl->address = 0; 292 sgl->unkeyed.length = rtransport.transport.opts.in_capsule_data_size * 2; 293 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 294 295 CU_ASSERT(rc == -1); 296 297 /* Test 3: Multi SGL */ 298 sgl->generic.type = SPDK_NVME_SGL_TYPE_LAST_SEGMENT; 299 sgl->unkeyed.subtype = SPDK_NVME_SGL_SUBTYPE_OFFSET; 300 sgl->address = 0; 301 rdma_req.recv->buf = (void *)&sgl_desc; 302 MOCK_SET(spdk_mempool_get, &data); 303 304 /* part 1: 2 segments each with 1 wr. */ 305 reset_nvmf_rdma_request(&rdma_req); 306 sgl->unkeyed.length = 2 * sizeof(struct spdk_nvme_sgl_descriptor); 307 for (i = 0; i < 2; i++) { 308 sgl_desc[i].keyed.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; 309 sgl_desc[i].keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; 310 sgl_desc[i].keyed.length = rtransport.transport.opts.io_unit_size; 311 sgl_desc[i].address = 0x4000 + i * rtransport.transport.opts.io_unit_size; 312 sgl_desc[i].keyed.key = 0x44; 313 } 314 315 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 316 317 CU_ASSERT(rc == 0); 318 CU_ASSERT(rdma_req.req.data_from_pool == true); 319 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size * 2); 320 CU_ASSERT(rdma_req.data.wr.num_sge == 1); 321 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0x44); 322 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0x4000); 323 CU_ASSERT(rdma_req.data.wr.next == &data.wr); 324 CU_ASSERT(data.wr.wr.rdma.rkey == 0x44); 325 CU_ASSERT(data.wr.wr.rdma.remote_addr == 0x4000 + rtransport.transport.opts.io_unit_size); 326 CU_ASSERT(data.wr.num_sge == 1); 327 CU_ASSERT(data.wr.next == &rdma_req.rsp.wr); 328 329 /* part 2: 2 segments, each with 1 wr containing 8 sge_elements */ 330 reset_nvmf_rdma_request(&rdma_req); 331 sgl->unkeyed.length = 2 * sizeof(struct spdk_nvme_sgl_descriptor); 332 for (i = 0; i < 2; i++) { 333 sgl_desc[i].keyed.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; 334 sgl_desc[i].keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; 335 sgl_desc[i].keyed.length = rtransport.transport.opts.io_unit_size * 8; 336 sgl_desc[i].address = 0x4000 + i * 8 * rtransport.transport.opts.io_unit_size; 337 sgl_desc[i].keyed.key = 0x44; 338 } 339 340 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 341 342 CU_ASSERT(rc == 0); 343 CU_ASSERT(rdma_req.req.data_from_pool == true); 344 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size * 16); 345 CU_ASSERT(rdma_req.req.iovcnt == 16); 346 CU_ASSERT(rdma_req.data.wr.num_sge == 8); 347 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0x44); 348 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0x4000); 349 CU_ASSERT(rdma_req.data.wr.next == &data.wr); 350 CU_ASSERT(data.wr.wr.rdma.rkey == 0x44); 351 CU_ASSERT(data.wr.wr.rdma.remote_addr == 0x4000 + rtransport.transport.opts.io_unit_size * 8); 352 CU_ASSERT(data.wr.num_sge == 8); 353 CU_ASSERT(data.wr.next == &rdma_req.rsp.wr); 354 355 /* part 3: 2 segments, one very large, one very small */ 356 reset_nvmf_rdma_request(&rdma_req); 357 for (i = 0; i < 2; i++) { 358 sgl_desc[i].keyed.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; 359 sgl_desc[i].keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; 360 sgl_desc[i].keyed.key = 0x44; 361 } 362 363 sgl_desc[0].keyed.length = rtransport.transport.opts.io_unit_size * 15 + 364 rtransport.transport.opts.io_unit_size / 2; 365 sgl_desc[0].address = 0x4000; 366 sgl_desc[1].keyed.length = rtransport.transport.opts.io_unit_size / 2; 367 sgl_desc[1].address = 0x4000 + rtransport.transport.opts.io_unit_size * 15 + 368 rtransport.transport.opts.io_unit_size / 2; 369 370 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 371 372 CU_ASSERT(rc == 0); 373 CU_ASSERT(rdma_req.req.data_from_pool == true); 374 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size * 16); 375 CU_ASSERT(rdma_req.req.iovcnt == 16); 376 CU_ASSERT(rdma_req.data.wr.num_sge == 16); 377 for (i = 0; i < 15; i++) { 378 CU_ASSERT(rdma_req.data.sgl[i].length == rtransport.transport.opts.io_unit_size); 379 } 380 CU_ASSERT(rdma_req.data.sgl[15].length == rtransport.transport.opts.io_unit_size / 2); 381 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0x44); 382 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0x4000); 383 CU_ASSERT(rdma_req.data.wr.next == &data.wr); 384 CU_ASSERT(data.wr.wr.rdma.rkey == 0x44); 385 CU_ASSERT(data.wr.wr.rdma.remote_addr == 0x4000 + rtransport.transport.opts.io_unit_size * 15 + 386 rtransport.transport.opts.io_unit_size / 2); 387 CU_ASSERT(data.sgl[0].length == rtransport.transport.opts.io_unit_size / 2); 388 CU_ASSERT(data.wr.num_sge == 1); 389 CU_ASSERT(data.wr.next == &rdma_req.rsp.wr); 390 391 /* part 4: 2 SGL descriptors, each length is transport buffer / 2 392 * 1 transport buffers should be allocated */ 393 reset_nvmf_rdma_request(&rdma_req); 394 aligned_buffer_address = ((uintptr_t)(&data) + NVMF_DATA_BUFFER_MASK) & ~NVMF_DATA_BUFFER_MASK; 395 sgl->unkeyed.length = 2 * sizeof(struct spdk_nvme_sgl_descriptor); 396 sgl_length = rtransport.transport.opts.io_unit_size / 2; 397 for (i = 0; i < 2; i++) { 398 sgl_desc[i].keyed.length = sgl_length; 399 sgl_desc[i].address = 0x4000 + i * sgl_length; 400 } 401 402 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 403 404 CU_ASSERT(rc == 0); 405 CU_ASSERT(rdma_req.req.data_from_pool == true); 406 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size); 407 CU_ASSERT(rdma_req.req.iovcnt == 1); 408 409 CU_ASSERT(rdma_req.data.sgl[0].length == sgl_length); 410 /* We mocked mempool_get to return address of data variable. Mempool is used 411 * to get both additional WRs and data buffers, so data points to &data */ 412 CU_ASSERT(rdma_req.data.sgl[0].addr == aligned_buffer_address); 413 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0x44); 414 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0x4000); 415 CU_ASSERT(rdma_req.data.wr.num_sge == 1); 416 CU_ASSERT(rdma_req.data.wr.next == &data.wr); 417 418 CU_ASSERT(data.wr.wr.rdma.rkey == 0x44); 419 CU_ASSERT(data.wr.wr.rdma.remote_addr == 0x4000 + sgl_length); 420 CU_ASSERT(data.sgl[0].length == sgl_length); 421 CU_ASSERT(data.sgl[0].addr == aligned_buffer_address + sgl_length); 422 CU_ASSERT(data.wr.num_sge == 1); 423 424 /* Test 4: use PG buffer cache */ 425 sgl->generic.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; 426 sgl->keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; 427 sgl->address = 0xFFFF; 428 rdma_req.recv->buf = (void *)0xDDDD; 429 sgl->keyed.key = 0xEEEE; 430 431 for (i = 0; i < 4; i++) { 432 STAILQ_INSERT_TAIL(&group.group.buf_cache, &bufs[i], link); 433 } 434 435 /* part 1: use the four buffers from the pg cache */ 436 group.group.buf_cache_size = 4; 437 group.group.buf_cache_count = 4; 438 MOCK_SET(spdk_mempool_get, (void *)0x2000); 439 reset_nvmf_rdma_request(&rdma_req); 440 sgl->keyed.length = rtransport.transport.opts.io_unit_size * 4; 441 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 442 443 SPDK_CU_ASSERT_FATAL(rc == 0); 444 CU_ASSERT(rdma_req.req.data_from_pool == true); 445 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size * 4); 446 CU_ASSERT((uint64_t)rdma_req.req.data == (((uint64_t)&bufs[0] + NVMF_DATA_BUFFER_MASK) & 447 ~NVMF_DATA_BUFFER_MASK)); 448 CU_ASSERT(rdma_req.data.wr.num_sge == 4); 449 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 450 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 451 CU_ASSERT(group.group.buf_cache_count == 0); 452 CU_ASSERT(STAILQ_EMPTY(&group.group.buf_cache)); 453 for (i = 0; i < 4; i++) { 454 CU_ASSERT((uint64_t)rdma_req.req.buffers[i] == (uint64_t)&bufs[i]); 455 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == (((uint64_t)&bufs[i] + NVMF_DATA_BUFFER_MASK) & 456 ~NVMF_DATA_BUFFER_MASK)); 457 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == rtransport.transport.opts.io_unit_size); 458 } 459 460 /* part 2: now that we have used the buffers from the cache, try again. We should get mempool buffers. */ 461 reset_nvmf_rdma_request(&rdma_req); 462 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 463 464 SPDK_CU_ASSERT_FATAL(rc == 0); 465 CU_ASSERT(rdma_req.req.data_from_pool == true); 466 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size * 4); 467 CU_ASSERT((uint64_t)rdma_req.req.data == 0x2000); 468 CU_ASSERT(rdma_req.data.wr.num_sge == 4); 469 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 470 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 471 CU_ASSERT(group.group.buf_cache_count == 0); 472 CU_ASSERT(STAILQ_EMPTY(&group.group.buf_cache)); 473 for (i = 0; i < 4; i++) { 474 CU_ASSERT((uint64_t)rdma_req.req.buffers[i] == 0x2000); 475 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == 0x2000); 476 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == rtransport.transport.opts.io_unit_size); 477 CU_ASSERT(group.group.buf_cache_count == 0); 478 } 479 480 /* part 3: half and half */ 481 group.group.buf_cache_count = 2; 482 483 for (i = 0; i < 2; i++) { 484 STAILQ_INSERT_TAIL(&group.group.buf_cache, &bufs[i], link); 485 } 486 reset_nvmf_rdma_request(&rdma_req); 487 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 488 489 SPDK_CU_ASSERT_FATAL(rc == 0); 490 CU_ASSERT(rdma_req.req.data_from_pool == true); 491 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size * 4); 492 CU_ASSERT((uint64_t)rdma_req.req.data == (((uint64_t)&bufs[0] + NVMF_DATA_BUFFER_MASK) & 493 ~NVMF_DATA_BUFFER_MASK)); 494 CU_ASSERT(rdma_req.data.wr.num_sge == 4); 495 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 496 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 497 CU_ASSERT(group.group.buf_cache_count == 0); 498 for (i = 0; i < 2; i++) { 499 CU_ASSERT((uint64_t)rdma_req.req.buffers[i] == (uint64_t)&bufs[i]); 500 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == (((uint64_t)&bufs[i] + NVMF_DATA_BUFFER_MASK) & 501 ~NVMF_DATA_BUFFER_MASK)); 502 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == rtransport.transport.opts.io_unit_size); 503 } 504 for (i = 2; i < 4; i++) { 505 CU_ASSERT((uint64_t)rdma_req.req.buffers[i] == 0x2000); 506 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == 0x2000); 507 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == rtransport.transport.opts.io_unit_size); 508 } 509 510 reset_nvmf_rdma_request(&rdma_req); 511 } 512 513 static struct spdk_nvmf_rdma_recv * 514 create_recv(struct spdk_nvmf_rdma_qpair *rqpair, enum spdk_nvme_nvm_opcode opc) 515 { 516 struct spdk_nvmf_rdma_recv *rdma_recv; 517 union nvmf_h2c_msg *cmd; 518 struct spdk_nvme_sgl_descriptor *sgl; 519 520 rdma_recv = calloc(1, sizeof(*rdma_recv)); 521 rdma_recv->qpair = rqpair; 522 cmd = calloc(1, sizeof(*cmd)); 523 rdma_recv->sgl[0].addr = (uintptr_t)cmd; 524 cmd->nvme_cmd.opc = opc; 525 sgl = &cmd->nvme_cmd.dptr.sgl1; 526 sgl->keyed.key = 0xEEEE; 527 sgl->address = 0xFFFF; 528 sgl->keyed.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; 529 sgl->keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; 530 sgl->keyed.length = 1; 531 532 return rdma_recv; 533 } 534 535 static void 536 free_recv(struct spdk_nvmf_rdma_recv *rdma_recv) 537 { 538 free((void *)rdma_recv->sgl[0].addr); 539 free(rdma_recv); 540 } 541 542 static struct spdk_nvmf_rdma_request * 543 create_req(struct spdk_nvmf_rdma_qpair *rqpair, 544 struct spdk_nvmf_rdma_recv *rdma_recv) 545 { 546 struct spdk_nvmf_rdma_request *rdma_req; 547 union nvmf_c2h_msg *cpl; 548 549 rdma_req = calloc(1, sizeof(*rdma_req)); 550 rdma_req->recv = rdma_recv; 551 rdma_req->req.qpair = &rqpair->qpair; 552 rdma_req->state = RDMA_REQUEST_STATE_NEW; 553 rdma_req->data.wr.wr_id = (uintptr_t)&rdma_req->data.rdma_wr; 554 rdma_req->data.wr.sg_list = rdma_req->data.sgl; 555 cpl = calloc(1, sizeof(*cpl)); 556 rdma_req->rsp.sgl[0].addr = (uintptr_t)cpl; 557 rdma_req->req.rsp = cpl; 558 559 return rdma_req; 560 } 561 562 static void 563 free_req(struct spdk_nvmf_rdma_request *rdma_req) 564 { 565 free((void *)rdma_req->rsp.sgl[0].addr); 566 free(rdma_req); 567 } 568 569 static void 570 qpair_reset(struct spdk_nvmf_rdma_qpair *rqpair, 571 struct spdk_nvmf_rdma_poller *poller, 572 struct spdk_nvmf_rdma_device *device, 573 struct spdk_nvmf_rdma_resources *resources, 574 struct spdk_nvmf_transport *transport) 575 { 576 memset(rqpair, 0, sizeof(*rqpair)); 577 STAILQ_INIT(&rqpair->pending_rdma_write_queue); 578 STAILQ_INIT(&rqpair->pending_rdma_read_queue); 579 rqpair->poller = poller; 580 rqpair->device = device; 581 rqpair->resources = resources; 582 rqpair->qpair.qid = 1; 583 rqpair->ibv_state = IBV_QPS_RTS; 584 rqpair->qpair.state = SPDK_NVMF_QPAIR_ACTIVE; 585 rqpair->max_send_sge = SPDK_NVMF_MAX_SGL_ENTRIES; 586 rqpair->max_send_depth = 16; 587 rqpair->max_read_depth = 16; 588 rqpair->qpair.transport = transport; 589 } 590 591 static void 592 poller_reset(struct spdk_nvmf_rdma_poller *poller, 593 struct spdk_nvmf_rdma_poll_group *group) 594 { 595 memset(poller, 0, sizeof(*poller)); 596 STAILQ_INIT(&poller->qpairs_pending_recv); 597 STAILQ_INIT(&poller->qpairs_pending_send); 598 poller->group = group; 599 } 600 601 static void 602 test_spdk_nvmf_rdma_request_process(void) 603 { 604 struct spdk_nvmf_rdma_transport rtransport = {}; 605 struct spdk_nvmf_rdma_poll_group group = {}; 606 struct spdk_nvmf_rdma_poller poller = {}; 607 struct spdk_nvmf_rdma_device device = {}; 608 struct spdk_nvmf_rdma_resources resources = {}; 609 struct spdk_nvmf_rdma_qpair rqpair = {}; 610 struct spdk_nvmf_rdma_recv *rdma_recv; 611 struct spdk_nvmf_rdma_request *rdma_req; 612 bool progress; 613 614 STAILQ_INIT(&group.group.buf_cache); 615 STAILQ_INIT(&group.group.pending_buf_queue); 616 group.group.buf_cache_size = 0; 617 group.group.buf_cache_count = 0; 618 poller_reset(&poller, &group); 619 qpair_reset(&rqpair, &poller, &device, &resources, &rtransport.transport); 620 621 rtransport.transport.opts = g_rdma_ut_transport_opts; 622 rtransport.transport.data_buf_pool = spdk_mempool_create("test_data_pool", 16, 128, 0, 0); 623 rtransport.data_wr_pool = spdk_mempool_create("test_wr_pool", 128, 624 sizeof(struct spdk_nvmf_rdma_request_data), 625 0, 0); 626 MOCK_CLEAR(spdk_mempool_get); 627 628 device.attr.device_cap_flags = 0; 629 device.map = (void *)0x0; 630 631 /* Test 1: single SGL READ request */ 632 rdma_recv = create_recv(&rqpair, SPDK_NVME_OPC_READ); 633 rdma_req = create_req(&rqpair, rdma_recv); 634 rqpair.current_recv_depth = 1; 635 /* NEW -> EXECUTING */ 636 progress = nvmf_rdma_request_process(&rtransport, rdma_req); 637 CU_ASSERT(progress == true); 638 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_EXECUTING); 639 CU_ASSERT(rdma_req->req.xfer == SPDK_NVME_DATA_CONTROLLER_TO_HOST); 640 /* EXECUTED -> TRANSFERRING_C2H */ 641 rdma_req->state = RDMA_REQUEST_STATE_EXECUTED; 642 progress = nvmf_rdma_request_process(&rtransport, rdma_req); 643 CU_ASSERT(progress == true); 644 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_TRANSFERRING_CONTROLLER_TO_HOST); 645 CU_ASSERT(rdma_req->recv == NULL); 646 /* COMPLETED -> FREE */ 647 rdma_req->state = RDMA_REQUEST_STATE_COMPLETED; 648 progress = nvmf_rdma_request_process(&rtransport, rdma_req); 649 CU_ASSERT(progress == true); 650 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_FREE); 651 652 free_recv(rdma_recv); 653 free_req(rdma_req); 654 poller_reset(&poller, &group); 655 qpair_reset(&rqpair, &poller, &device, &resources, &rtransport.transport); 656 657 /* Test 2: single SGL WRITE request */ 658 rdma_recv = create_recv(&rqpair, SPDK_NVME_OPC_WRITE); 659 rdma_req = create_req(&rqpair, rdma_recv); 660 rqpair.current_recv_depth = 1; 661 /* NEW -> TRANSFERRING_H2C */ 662 progress = nvmf_rdma_request_process(&rtransport, rdma_req); 663 CU_ASSERT(progress == true); 664 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_TRANSFERRING_HOST_TO_CONTROLLER); 665 CU_ASSERT(rdma_req->req.xfer == SPDK_NVME_DATA_HOST_TO_CONTROLLER); 666 STAILQ_INIT(&poller.qpairs_pending_send); 667 /* READY_TO_EXECUTE -> EXECUTING */ 668 rdma_req->state = RDMA_REQUEST_STATE_READY_TO_EXECUTE; 669 progress = nvmf_rdma_request_process(&rtransport, rdma_req); 670 CU_ASSERT(progress == true); 671 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_EXECUTING); 672 /* EXECUTED -> COMPLETING */ 673 rdma_req->state = RDMA_REQUEST_STATE_EXECUTED; 674 progress = nvmf_rdma_request_process(&rtransport, rdma_req); 675 CU_ASSERT(progress == true); 676 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_COMPLETING); 677 CU_ASSERT(rdma_req->recv == NULL); 678 /* COMPLETED -> FREE */ 679 rdma_req->state = RDMA_REQUEST_STATE_COMPLETED; 680 progress = nvmf_rdma_request_process(&rtransport, rdma_req); 681 CU_ASSERT(progress == true); 682 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_FREE); 683 684 free_recv(rdma_recv); 685 free_req(rdma_req); 686 poller_reset(&poller, &group); 687 qpair_reset(&rqpair, &poller, &device, &resources, &rtransport.transport); 688 689 /* Test 3: WRITE+WRITE ibv_send batching */ 690 { 691 struct spdk_nvmf_rdma_recv *recv1, *recv2; 692 struct spdk_nvmf_rdma_request *req1, *req2; 693 recv1 = create_recv(&rqpair, SPDK_NVME_OPC_WRITE); 694 req1 = create_req(&rqpair, recv1); 695 recv2 = create_recv(&rqpair, SPDK_NVME_OPC_WRITE); 696 req2 = create_req(&rqpair, recv2); 697 698 /* WRITE 1: NEW -> TRANSFERRING_H2C */ 699 rqpair.current_recv_depth = 1; 700 nvmf_rdma_request_process(&rtransport, req1); 701 CU_ASSERT(req1->state == RDMA_REQUEST_STATE_TRANSFERRING_HOST_TO_CONTROLLER); 702 703 /* WRITE 2: NEW -> TRANSFERRING_H2C */ 704 rqpair.current_recv_depth = 2; 705 nvmf_rdma_request_process(&rtransport, req2); 706 CU_ASSERT(req2->state == RDMA_REQUEST_STATE_TRANSFERRING_HOST_TO_CONTROLLER); 707 708 STAILQ_INIT(&poller.qpairs_pending_send); 709 710 /* WRITE 1 completes before WRITE 2 has finished RDMA reading */ 711 /* WRITE 1: READY_TO_EXECUTE -> EXECUTING */ 712 req1->state = RDMA_REQUEST_STATE_READY_TO_EXECUTE; 713 nvmf_rdma_request_process(&rtransport, req1); 714 CU_ASSERT(req1->state == RDMA_REQUEST_STATE_EXECUTING); 715 /* WRITE 1: EXECUTED -> COMPLETING */ 716 req1->state = RDMA_REQUEST_STATE_EXECUTED; 717 nvmf_rdma_request_process(&rtransport, req1); 718 CU_ASSERT(req1->state == RDMA_REQUEST_STATE_COMPLETING); 719 STAILQ_INIT(&poller.qpairs_pending_send); 720 /* WRITE 1: COMPLETED -> FREE */ 721 req1->state = RDMA_REQUEST_STATE_COMPLETED; 722 nvmf_rdma_request_process(&rtransport, req1); 723 CU_ASSERT(req1->state == RDMA_REQUEST_STATE_FREE); 724 725 /* Now WRITE 2 has finished reading and completes */ 726 /* WRITE 2: COMPLETED -> FREE */ 727 /* WRITE 2: READY_TO_EXECUTE -> EXECUTING */ 728 req2->state = RDMA_REQUEST_STATE_READY_TO_EXECUTE; 729 nvmf_rdma_request_process(&rtransport, req2); 730 CU_ASSERT(req2->state == RDMA_REQUEST_STATE_EXECUTING); 731 /* WRITE 1: EXECUTED -> COMPLETING */ 732 req2->state = RDMA_REQUEST_STATE_EXECUTED; 733 nvmf_rdma_request_process(&rtransport, req2); 734 CU_ASSERT(req2->state == RDMA_REQUEST_STATE_COMPLETING); 735 STAILQ_INIT(&poller.qpairs_pending_send); 736 /* WRITE 1: COMPLETED -> FREE */ 737 req2->state = RDMA_REQUEST_STATE_COMPLETED; 738 nvmf_rdma_request_process(&rtransport, req2); 739 CU_ASSERT(req2->state == RDMA_REQUEST_STATE_FREE); 740 741 free_recv(recv1); 742 free_req(req1); 743 free_recv(recv2); 744 free_req(req2); 745 poller_reset(&poller, &group); 746 qpair_reset(&rqpair, &poller, &device, &resources, &rtransport.transport); 747 } 748 749 /* Test 4, invalid command, check xfer type */ 750 { 751 struct spdk_nvmf_rdma_recv *rdma_recv_inv; 752 struct spdk_nvmf_rdma_request *rdma_req_inv; 753 /* construct an opcode that specifies BIDIRECTIONAL transfer */ 754 uint8_t opc = 0x10 | SPDK_NVME_DATA_BIDIRECTIONAL; 755 756 rdma_recv_inv = create_recv(&rqpair, opc); 757 rdma_req_inv = create_req(&rqpair, rdma_recv_inv); 758 759 /* NEW -> RDMA_REQUEST_STATE_COMPLETING */ 760 rqpair.current_recv_depth = 1; 761 progress = nvmf_rdma_request_process(&rtransport, rdma_req_inv); 762 CU_ASSERT(progress == true); 763 CU_ASSERT(rdma_req_inv->state == RDMA_REQUEST_STATE_COMPLETING); 764 CU_ASSERT(rdma_req_inv->req.rsp->nvme_cpl.status.sct == SPDK_NVME_SCT_GENERIC); 765 CU_ASSERT(rdma_req_inv->req.rsp->nvme_cpl.status.sc == SPDK_NVME_SC_INVALID_OPCODE); 766 767 /* RDMA_REQUEST_STATE_COMPLETED -> FREE */ 768 rdma_req_inv->state = RDMA_REQUEST_STATE_COMPLETED; 769 nvmf_rdma_request_process(&rtransport, rdma_req_inv); 770 CU_ASSERT(rdma_req_inv->state == RDMA_REQUEST_STATE_FREE); 771 772 free_recv(rdma_recv_inv); 773 free_req(rdma_req_inv); 774 poller_reset(&poller, &group); 775 qpair_reset(&rqpair, &poller, &device, &resources, &rtransport.transport); 776 } 777 778 spdk_mempool_free(rtransport.transport.data_buf_pool); 779 spdk_mempool_free(rtransport.data_wr_pool); 780 } 781 782 #define TEST_GROUPS_COUNT 5 783 static void 784 test_nvmf_rdma_get_optimal_poll_group(void) 785 { 786 struct spdk_nvmf_rdma_transport rtransport = {}; 787 struct spdk_nvmf_transport *transport = &rtransport.transport; 788 struct spdk_nvmf_rdma_qpair rqpair = {}; 789 struct spdk_nvmf_transport_poll_group *groups[TEST_GROUPS_COUNT]; 790 struct spdk_nvmf_rdma_poll_group *rgroups[TEST_GROUPS_COUNT]; 791 struct spdk_nvmf_transport_poll_group *result; 792 uint32_t i; 793 794 rqpair.qpair.transport = transport; 795 TAILQ_INIT(&rtransport.poll_groups); 796 797 for (i = 0; i < TEST_GROUPS_COUNT; i++) { 798 groups[i] = nvmf_rdma_poll_group_create(transport, NULL); 799 CU_ASSERT(groups[i] != NULL); 800 rgroups[i] = SPDK_CONTAINEROF(groups[i], struct spdk_nvmf_rdma_poll_group, group); 801 groups[i]->transport = transport; 802 } 803 CU_ASSERT(rtransport.conn_sched.next_admin_pg == rgroups[0]); 804 CU_ASSERT(rtransport.conn_sched.next_io_pg == rgroups[0]); 805 806 /* Emulate connection of %TEST_GROUPS_COUNT% initiators - each creates 1 admin and 1 io qp */ 807 for (i = 0; i < TEST_GROUPS_COUNT; i++) { 808 rqpair.qpair.qid = 0; 809 result = nvmf_rdma_get_optimal_poll_group(&rqpair.qpair); 810 CU_ASSERT(result == groups[i]); 811 CU_ASSERT(rtransport.conn_sched.next_admin_pg == rgroups[(i + 1) % TEST_GROUPS_COUNT]); 812 CU_ASSERT(rtransport.conn_sched.next_io_pg == rgroups[i]); 813 814 rqpair.qpair.qid = 1; 815 result = nvmf_rdma_get_optimal_poll_group(&rqpair.qpair); 816 CU_ASSERT(result == groups[i]); 817 CU_ASSERT(rtransport.conn_sched.next_admin_pg == rgroups[(i + 1) % TEST_GROUPS_COUNT]); 818 CU_ASSERT(rtransport.conn_sched.next_io_pg == rgroups[(i + 1) % TEST_GROUPS_COUNT]); 819 } 820 /* wrap around, admin/io pg point to the first pg 821 Destroy all poll groups except of the last one */ 822 for (i = 0; i < TEST_GROUPS_COUNT - 1; i++) { 823 nvmf_rdma_poll_group_destroy(groups[i]); 824 CU_ASSERT(rtransport.conn_sched.next_admin_pg == rgroups[i + 1]); 825 CU_ASSERT(rtransport.conn_sched.next_io_pg == rgroups[i + 1]); 826 } 827 828 CU_ASSERT(rtransport.conn_sched.next_admin_pg == rgroups[TEST_GROUPS_COUNT - 1]); 829 CU_ASSERT(rtransport.conn_sched.next_io_pg == rgroups[TEST_GROUPS_COUNT - 1]); 830 831 /* Check that pointers to the next admin/io poll groups are not changed */ 832 rqpair.qpair.qid = 0; 833 result = nvmf_rdma_get_optimal_poll_group(&rqpair.qpair); 834 CU_ASSERT(result == groups[TEST_GROUPS_COUNT - 1]); 835 CU_ASSERT(rtransport.conn_sched.next_admin_pg == rgroups[TEST_GROUPS_COUNT - 1]); 836 CU_ASSERT(rtransport.conn_sched.next_io_pg == rgroups[TEST_GROUPS_COUNT - 1]); 837 838 rqpair.qpair.qid = 1; 839 result = nvmf_rdma_get_optimal_poll_group(&rqpair.qpair); 840 CU_ASSERT(result == groups[TEST_GROUPS_COUNT - 1]); 841 CU_ASSERT(rtransport.conn_sched.next_admin_pg == rgroups[TEST_GROUPS_COUNT - 1]); 842 CU_ASSERT(rtransport.conn_sched.next_io_pg == rgroups[TEST_GROUPS_COUNT - 1]); 843 844 /* Remove the last poll group, check that pointers are NULL */ 845 nvmf_rdma_poll_group_destroy(groups[TEST_GROUPS_COUNT - 1]); 846 CU_ASSERT(rtransport.conn_sched.next_admin_pg == NULL); 847 CU_ASSERT(rtransport.conn_sched.next_io_pg == NULL); 848 849 /* Request optimal poll group, result must be NULL */ 850 rqpair.qpair.qid = 0; 851 result = nvmf_rdma_get_optimal_poll_group(&rqpair.qpair); 852 CU_ASSERT(result == NULL); 853 854 rqpair.qpair.qid = 1; 855 result = nvmf_rdma_get_optimal_poll_group(&rqpair.qpair); 856 CU_ASSERT(result == NULL); 857 } 858 #undef TEST_GROUPS_COUNT 859 860 static void 861 test_spdk_nvmf_rdma_request_parse_sgl_with_md(void) 862 { 863 struct spdk_nvmf_rdma_transport rtransport; 864 struct spdk_nvmf_rdma_device device; 865 struct spdk_nvmf_rdma_request rdma_req = {}; 866 struct spdk_nvmf_rdma_recv recv; 867 struct spdk_nvmf_rdma_poll_group group; 868 struct spdk_nvmf_rdma_qpair rqpair; 869 struct spdk_nvmf_rdma_poller poller; 870 union nvmf_c2h_msg cpl; 871 union nvmf_h2c_msg cmd; 872 struct spdk_nvme_sgl_descriptor *sgl; 873 struct spdk_nvme_sgl_descriptor sgl_desc[SPDK_NVMF_MAX_SGL_ENTRIES] = {{0}}; 874 char data_buffer[8192]; 875 struct spdk_nvmf_rdma_request_data *data = (struct spdk_nvmf_rdma_request_data *)data_buffer; 876 char data2_buffer[8192]; 877 struct spdk_nvmf_rdma_request_data *data2 = (struct spdk_nvmf_rdma_request_data *)data2_buffer; 878 const uint32_t data_bs = 512; 879 const uint32_t md_size = 8; 880 int rc, i; 881 void *aligned_buffer; 882 883 data->wr.sg_list = data->sgl; 884 STAILQ_INIT(&group.group.buf_cache); 885 group.group.buf_cache_size = 0; 886 group.group.buf_cache_count = 0; 887 group.group.transport = &rtransport.transport; 888 poller.group = &group; 889 rqpair.poller = &poller; 890 rqpair.max_send_sge = SPDK_NVMF_MAX_SGL_ENTRIES; 891 892 sgl = &cmd.nvme_cmd.dptr.sgl1; 893 rdma_req.recv = &recv; 894 rdma_req.req.cmd = &cmd; 895 rdma_req.req.rsp = &cpl; 896 rdma_req.data.wr.sg_list = rdma_req.data.sgl; 897 rdma_req.req.qpair = &rqpair.qpair; 898 rdma_req.req.xfer = SPDK_NVME_DATA_CONTROLLER_TO_HOST; 899 900 rtransport.transport.opts = g_rdma_ut_transport_opts; 901 rtransport.data_wr_pool = NULL; 902 rtransport.transport.data_buf_pool = NULL; 903 904 device.attr.device_cap_flags = 0; 905 device.map = NULL; 906 sgl->keyed.key = 0xEEEE; 907 sgl->address = 0xFFFF; 908 rdma_req.recv->buf = (void *)0xDDDD; 909 910 /* Test 1: sgl type: keyed data block subtype: address */ 911 sgl->generic.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; 912 sgl->keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; 913 914 /* Part 1: simple I/O, one SGL smaller than the transport io unit size, block size 512 */ 915 MOCK_SET(spdk_mempool_get, (void *)0x2000); 916 reset_nvmf_rdma_request(&rdma_req); 917 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 918 SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 919 0, 0, 0, 0, 0); 920 rdma_req.req.dif_enabled = true; 921 rtransport.transport.opts.io_unit_size = data_bs * 8; 922 rdma_req.req.qpair->transport = &rtransport.transport; 923 sgl->keyed.length = data_bs * 4; 924 925 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 926 927 CU_ASSERT(rc == 0); 928 CU_ASSERT(rdma_req.req.data_from_pool == true); 929 CU_ASSERT(rdma_req.req.length == data_bs * 4); 930 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 931 CU_ASSERT(rdma_req.req.dif.elba_length == (data_bs + md_size) * 4); 932 CU_ASSERT((uint64_t)rdma_req.req.data == 0x2000); 933 CU_ASSERT(rdma_req.data.wr.num_sge == 1); 934 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 935 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 936 CU_ASSERT((uint64_t)rdma_req.req.buffers[0] == 0x2000); 937 938 CU_ASSERT(rdma_req.data.wr.sg_list[0].addr == 0x2000); 939 CU_ASSERT(rdma_req.data.wr.sg_list[0].length == rdma_req.req.length); 940 CU_ASSERT(rdma_req.data.wr.sg_list[0].lkey == RDMA_UT_LKEY); 941 942 /* Part 2: simple I/O, one SGL equal to io unit size, io_unit_size is not aligned with md_size, 943 block size 512 */ 944 MOCK_SET(spdk_mempool_get, (void *)0x2000); 945 reset_nvmf_rdma_request(&rdma_req); 946 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 947 SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 948 0, 0, 0, 0, 0); 949 rdma_req.req.dif_enabled = true; 950 rtransport.transport.opts.io_unit_size = data_bs * 4; 951 sgl->keyed.length = data_bs * 4; 952 953 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 954 955 CU_ASSERT(rc == 0); 956 CU_ASSERT(rdma_req.req.data_from_pool == true); 957 CU_ASSERT(rdma_req.req.length == data_bs * 4); 958 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 959 CU_ASSERT(rdma_req.req.dif.elba_length == (data_bs + md_size) * 4); 960 CU_ASSERT((uint64_t)rdma_req.req.data == 0x2000); 961 CU_ASSERT(rdma_req.data.wr.num_sge == 5); 962 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 963 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 964 CU_ASSERT((uint64_t)rdma_req.req.buffers[0] == 0x2000); 965 966 for (i = 0; i < 3; ++i) { 967 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == 0x2000 + i * (data_bs + md_size)); 968 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == data_bs); 969 CU_ASSERT(rdma_req.data.wr.sg_list[i].lkey == RDMA_UT_LKEY); 970 } 971 CU_ASSERT(rdma_req.data.wr.sg_list[3].addr == 0x2000 + 3 * (data_bs + md_size)); 972 CU_ASSERT(rdma_req.data.wr.sg_list[3].length == 488); 973 CU_ASSERT(rdma_req.data.wr.sg_list[3].lkey == RDMA_UT_LKEY); 974 975 /* 2nd buffer consumed */ 976 CU_ASSERT(rdma_req.data.wr.sg_list[4].addr == 0x2000); 977 CU_ASSERT(rdma_req.data.wr.sg_list[4].length == 24); 978 CU_ASSERT(rdma_req.data.wr.sg_list[4].lkey == RDMA_UT_LKEY); 979 980 /* Part 3: simple I/O, one SGL equal io unit size, io_unit_size is equal to block size 512 bytes */ 981 MOCK_SET(spdk_mempool_get, (void *)0x2000); 982 reset_nvmf_rdma_request(&rdma_req); 983 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 984 SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 985 0, 0, 0, 0, 0); 986 rdma_req.req.dif_enabled = true; 987 rtransport.transport.opts.io_unit_size = data_bs; 988 sgl->keyed.length = data_bs; 989 990 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 991 992 CU_ASSERT(rc == 0); 993 CU_ASSERT(rdma_req.req.data_from_pool == true); 994 CU_ASSERT(rdma_req.req.length == data_bs); 995 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 996 CU_ASSERT(rdma_req.req.dif.elba_length == data_bs + md_size); 997 CU_ASSERT((uint64_t)rdma_req.req.data == 0x2000); 998 CU_ASSERT(rdma_req.data.wr.num_sge == 1); 999 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 1000 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 1001 CU_ASSERT((uint64_t)rdma_req.req.buffers[0] == 0x2000); 1002 1003 CU_ASSERT(rdma_req.data.wr.sg_list[0].addr == 0x2000); 1004 CU_ASSERT(rdma_req.data.wr.sg_list[0].length == data_bs); 1005 CU_ASSERT(rdma_req.data.wr.sg_list[0].lkey == RDMA_UT_LKEY); 1006 1007 CU_ASSERT(rdma_req.req.iovcnt == 2); 1008 CU_ASSERT(rdma_req.req.iov[0].iov_base == (void *)((unsigned long)0x2000)); 1009 CU_ASSERT(rdma_req.req.iov[0].iov_len == data_bs); 1010 /* 2nd buffer consumed for metadata */ 1011 CU_ASSERT(rdma_req.req.iov[1].iov_base == (void *)((unsigned long)0x2000)); 1012 CU_ASSERT(rdma_req.req.iov[1].iov_len == md_size); 1013 1014 /* Part 4: simple I/O, one SGL equal io unit size, io_unit_size is aligned with md_size, 1015 block size 512 */ 1016 MOCK_SET(spdk_mempool_get, (void *)0x2000); 1017 reset_nvmf_rdma_request(&rdma_req); 1018 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 1019 SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 1020 0, 0, 0, 0, 0); 1021 rdma_req.req.dif_enabled = true; 1022 rtransport.transport.opts.io_unit_size = (data_bs + md_size) * 4; 1023 sgl->keyed.length = data_bs * 4; 1024 1025 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 1026 1027 CU_ASSERT(rc == 0); 1028 CU_ASSERT(rdma_req.req.data_from_pool == true); 1029 CU_ASSERT(rdma_req.req.length == data_bs * 4); 1030 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 1031 CU_ASSERT(rdma_req.req.dif.elba_length == (data_bs + md_size) * 4); 1032 CU_ASSERT((uint64_t)rdma_req.req.data == 0x2000); 1033 CU_ASSERT(rdma_req.data.wr.num_sge == 1); 1034 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 1035 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 1036 CU_ASSERT((uint64_t)rdma_req.req.buffers[0] == 0x2000); 1037 1038 CU_ASSERT(rdma_req.data.wr.sg_list[0].addr == 0x2000); 1039 CU_ASSERT(rdma_req.data.wr.sg_list[0].length == rdma_req.req.length); 1040 CU_ASSERT(rdma_req.data.wr.sg_list[0].lkey == RDMA_UT_LKEY); 1041 1042 /* Part 5: simple I/O, one SGL equal to 2x io unit size, io_unit_size is aligned with md_size, 1043 block size 512 */ 1044 MOCK_SET(spdk_mempool_get, (void *)0x2000); 1045 reset_nvmf_rdma_request(&rdma_req); 1046 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 1047 SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 1048 0, 0, 0, 0, 0); 1049 rdma_req.req.dif_enabled = true; 1050 rtransport.transport.opts.io_unit_size = (data_bs + md_size) * 2; 1051 sgl->keyed.length = data_bs * 4; 1052 1053 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 1054 1055 CU_ASSERT(rc == 0); 1056 CU_ASSERT(rdma_req.req.data_from_pool == true); 1057 CU_ASSERT(rdma_req.req.length == data_bs * 4); 1058 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 1059 CU_ASSERT(rdma_req.req.dif.elba_length == (data_bs + md_size) * 4); 1060 CU_ASSERT((uint64_t)rdma_req.req.data == 0x2000); 1061 CU_ASSERT(rdma_req.data.wr.num_sge == 2); 1062 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 1063 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 1064 CU_ASSERT((uint64_t)rdma_req.req.buffers[0] == 0x2000); 1065 1066 for (i = 0; i < 2; ++i) { 1067 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == 0x2000); 1068 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == data_bs * 2); 1069 } 1070 1071 /* Part 6: simple I/O, one SGL larger than the transport io unit size, io_unit_size is not aligned to md_size, 1072 block size 512 */ 1073 MOCK_SET(spdk_mempool_get, (void *)0x2000); 1074 reset_nvmf_rdma_request(&rdma_req); 1075 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 1076 SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 1077 0, 0, 0, 0, 0); 1078 rdma_req.req.dif_enabled = true; 1079 rtransport.transport.opts.io_unit_size = data_bs * 4; 1080 sgl->keyed.length = data_bs * 6; 1081 1082 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 1083 1084 CU_ASSERT(rc == 0); 1085 CU_ASSERT(rdma_req.req.data_from_pool == true); 1086 CU_ASSERT(rdma_req.req.length == data_bs * 6); 1087 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 1088 CU_ASSERT(rdma_req.req.dif.elba_length == (data_bs + md_size) * 6); 1089 CU_ASSERT((uint64_t)rdma_req.req.data == 0x2000); 1090 CU_ASSERT(rdma_req.data.wr.num_sge == 7); 1091 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 1092 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 1093 CU_ASSERT((uint64_t)rdma_req.req.buffers[0] == 0x2000); 1094 1095 for (i = 0; i < 3; ++i) { 1096 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == 0x2000 + i * (data_bs + md_size)); 1097 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == data_bs); 1098 CU_ASSERT(rdma_req.data.wr.sg_list[i].lkey == RDMA_UT_LKEY); 1099 } 1100 CU_ASSERT(rdma_req.data.wr.sg_list[3].addr == 0x2000 + 3 * (data_bs + md_size)); 1101 CU_ASSERT(rdma_req.data.wr.sg_list[3].length == 488); 1102 CU_ASSERT(rdma_req.data.wr.sg_list[3].lkey == RDMA_UT_LKEY); 1103 1104 /* 2nd IO buffer consumed */ 1105 CU_ASSERT(rdma_req.data.wr.sg_list[4].addr == 0x2000); 1106 CU_ASSERT(rdma_req.data.wr.sg_list[4].length == 24); 1107 CU_ASSERT(rdma_req.data.wr.sg_list[4].lkey == RDMA_UT_LKEY); 1108 1109 CU_ASSERT(rdma_req.data.wr.sg_list[5].addr == 0x2000 + 24 + md_size); 1110 CU_ASSERT(rdma_req.data.wr.sg_list[5].length == 512); 1111 CU_ASSERT(rdma_req.data.wr.sg_list[5].lkey == RDMA_UT_LKEY); 1112 1113 CU_ASSERT(rdma_req.data.wr.sg_list[6].addr == 0x2000 + 24 + 512 + md_size * 2); 1114 CU_ASSERT(rdma_req.data.wr.sg_list[6].length == 512); 1115 CU_ASSERT(rdma_req.data.wr.sg_list[6].lkey == RDMA_UT_LKEY); 1116 1117 /* Part 7: simple I/O, number of SGL entries exceeds the number of entries 1118 one WR can hold. Additional WR is chained */ 1119 MOCK_SET(spdk_mempool_get, data2_buffer); 1120 aligned_buffer = (void *)((uintptr_t)(data2_buffer + NVMF_DATA_BUFFER_MASK) & 1121 ~NVMF_DATA_BUFFER_MASK); 1122 reset_nvmf_rdma_request(&rdma_req); 1123 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 1124 SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 1125 0, 0, 0, 0, 0); 1126 rdma_req.req.dif_enabled = true; 1127 rtransport.transport.opts.io_unit_size = data_bs * 16; 1128 sgl->keyed.length = data_bs * 16; 1129 1130 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 1131 1132 CU_ASSERT(rc == 0); 1133 CU_ASSERT(rdma_req.req.data_from_pool == true); 1134 CU_ASSERT(rdma_req.req.length == data_bs * 16); 1135 CU_ASSERT(rdma_req.req.iovcnt == 2); 1136 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 1137 CU_ASSERT(rdma_req.req.dif.elba_length == (data_bs + md_size) * 16); 1138 CU_ASSERT(rdma_req.req.data == aligned_buffer); 1139 CU_ASSERT(rdma_req.data.wr.num_sge == 16); 1140 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 1141 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 1142 1143 for (i = 0; i < 15; ++i) { 1144 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == (uintptr_t)aligned_buffer + i * (data_bs + md_size)); 1145 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == data_bs); 1146 CU_ASSERT(rdma_req.data.wr.sg_list[i].lkey == RDMA_UT_LKEY); 1147 } 1148 1149 /* 8192 - (512 + 8) * 15 = 392 */ 1150 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == (uintptr_t)aligned_buffer + i * (data_bs + md_size)); 1151 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == 392); 1152 CU_ASSERT(rdma_req.data.wr.sg_list[i].lkey == RDMA_UT_LKEY); 1153 1154 /* additional wr from pool */ 1155 CU_ASSERT(rdma_req.data.wr.next == (void *)&data2->wr); 1156 CU_ASSERT(rdma_req.data.wr.next->num_sge == 1); 1157 CU_ASSERT(rdma_req.data.wr.next->next == &rdma_req.rsp.wr); 1158 /* 2nd IO buffer */ 1159 CU_ASSERT(data2->wr.sg_list[0].addr == (uintptr_t)aligned_buffer); 1160 CU_ASSERT(data2->wr.sg_list[0].length == 120); 1161 CU_ASSERT(data2->wr.sg_list[0].lkey == RDMA_UT_LKEY); 1162 1163 /* Part 8: simple I/O, data with metadata do not fit to 1 io_buffer */ 1164 MOCK_SET(spdk_mempool_get, (void *)0x2000); 1165 reset_nvmf_rdma_request(&rdma_req); 1166 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 1167 SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 1168 0, 0, 0, 0, 0); 1169 rdma_req.req.dif_enabled = true; 1170 rtransport.transport.opts.io_unit_size = 516; 1171 sgl->keyed.length = data_bs * 2; 1172 1173 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 1174 1175 CU_ASSERT(rc == 0); 1176 CU_ASSERT(rdma_req.req.data_from_pool == true); 1177 CU_ASSERT(rdma_req.req.length == data_bs * 2); 1178 CU_ASSERT(rdma_req.req.iovcnt == 3); 1179 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 1180 CU_ASSERT(rdma_req.req.dif.elba_length == (data_bs + md_size) * 2); 1181 CU_ASSERT(rdma_req.req.data == (void *)0x2000); 1182 CU_ASSERT(rdma_req.data.wr.num_sge == 2); 1183 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 1184 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 1185 1186 CU_ASSERT(rdma_req.data.wr.sg_list[0].addr == 0x2000); 1187 CU_ASSERT(rdma_req.data.wr.sg_list[0].length == 512); 1188 CU_ASSERT(rdma_req.data.wr.sg_list[0].lkey == RDMA_UT_LKEY); 1189 1190 /* 2nd IO buffer consumed, offset 4 bytes due to part of the metadata 1191 is located at the beginning of that buffer */ 1192 CU_ASSERT(rdma_req.data.wr.sg_list[1].addr == 0x2000 + 4); 1193 CU_ASSERT(rdma_req.data.wr.sg_list[1].length == 512); 1194 CU_ASSERT(rdma_req.data.wr.sg_list[1].lkey == RDMA_UT_LKEY); 1195 1196 /* Test 2: Multi SGL */ 1197 sgl->generic.type = SPDK_NVME_SGL_TYPE_LAST_SEGMENT; 1198 sgl->unkeyed.subtype = SPDK_NVME_SGL_SUBTYPE_OFFSET; 1199 sgl->address = 0; 1200 rdma_req.recv->buf = (void *)&sgl_desc; 1201 MOCK_SET(spdk_mempool_get, data_buffer); 1202 aligned_buffer = (void *)((uintptr_t)(data_buffer + NVMF_DATA_BUFFER_MASK) & 1203 ~NVMF_DATA_BUFFER_MASK); 1204 1205 /* part 1: 2 segments each with 1 wr. io_unit_size is aligned with data_bs + md_size */ 1206 reset_nvmf_rdma_request(&rdma_req); 1207 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 1208 SPDK_DIF_TYPE1, 1209 SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 0, 0, 0, 0, 0); 1210 rdma_req.req.dif_enabled = true; 1211 rtransport.transport.opts.io_unit_size = (data_bs + md_size) * 4; 1212 sgl->unkeyed.length = 2 * sizeof(struct spdk_nvme_sgl_descriptor); 1213 1214 for (i = 0; i < 2; i++) { 1215 sgl_desc[i].keyed.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; 1216 sgl_desc[i].keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; 1217 sgl_desc[i].keyed.length = data_bs * 4; 1218 sgl_desc[i].address = 0x4000 + i * data_bs * 4; 1219 sgl_desc[i].keyed.key = 0x44; 1220 } 1221 1222 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 1223 1224 CU_ASSERT(rc == 0); 1225 CU_ASSERT(rdma_req.req.data_from_pool == true); 1226 CU_ASSERT(rdma_req.req.length == data_bs * 4 * 2); 1227 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 1228 CU_ASSERT(rdma_req.req.dif.elba_length == (data_bs + md_size) * 4 * 2); 1229 CU_ASSERT(rdma_req.data.wr.num_sge == 1); 1230 CU_ASSERT(rdma_req.data.wr.sg_list[0].addr == (uintptr_t)(aligned_buffer)); 1231 CU_ASSERT(rdma_req.data.wr.sg_list[0].length == data_bs * 4); 1232 1233 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0x44); 1234 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0x4000); 1235 CU_ASSERT(rdma_req.data.wr.next == &data->wr); 1236 CU_ASSERT(data->wr.wr.rdma.rkey == 0x44); 1237 CU_ASSERT(data->wr.wr.rdma.remote_addr == 0x4000 + data_bs * 4); 1238 CU_ASSERT(data->wr.num_sge == 1); 1239 CU_ASSERT(data->wr.sg_list[0].addr == (uintptr_t)(aligned_buffer)); 1240 CU_ASSERT(data->wr.sg_list[0].length == data_bs * 4); 1241 1242 CU_ASSERT(data->wr.next == &rdma_req.rsp.wr); 1243 reset_nvmf_rdma_request(&rdma_req); 1244 } 1245 1246 static void 1247 test_nvmf_rdma_opts_init(void) 1248 { 1249 struct spdk_nvmf_transport_opts opts = {}; 1250 1251 nvmf_rdma_opts_init(&opts); 1252 CU_ASSERT(opts.max_queue_depth == SPDK_NVMF_RDMA_DEFAULT_MAX_QUEUE_DEPTH); 1253 CU_ASSERT(opts.max_qpairs_per_ctrlr == SPDK_NVMF_RDMA_DEFAULT_MAX_QPAIRS_PER_CTRLR); 1254 CU_ASSERT(opts.in_capsule_data_size == SPDK_NVMF_RDMA_DEFAULT_IN_CAPSULE_DATA_SIZE); 1255 CU_ASSERT(opts.max_io_size == SPDK_NVMF_RDMA_DEFAULT_MAX_IO_SIZE); 1256 CU_ASSERT(opts.io_unit_size == SPDK_NVMF_RDMA_MIN_IO_BUFFER_SIZE); 1257 CU_ASSERT(opts.max_aq_depth == SPDK_NVMF_RDMA_DEFAULT_AQ_DEPTH); 1258 CU_ASSERT(opts.num_shared_buffers == SPDK_NVMF_RDMA_DEFAULT_NUM_SHARED_BUFFERS); 1259 CU_ASSERT(opts.buf_cache_size == SPDK_NVMF_RDMA_DEFAULT_BUFFER_CACHE_SIZE); 1260 CU_ASSERT(opts.dif_insert_or_strip == SPDK_NVMF_RDMA_DIF_INSERT_OR_STRIP); 1261 CU_ASSERT(opts.abort_timeout_sec == SPDK_NVMF_RDMA_DEFAULT_ABORT_TIMEOUT_SEC); 1262 CU_ASSERT(opts.transport_specific == NULL); 1263 } 1264 1265 static void 1266 test_nvmf_rdma_request_free_data(void) 1267 { 1268 struct spdk_nvmf_rdma_request rdma_req = {}; 1269 struct spdk_nvmf_rdma_transport rtransport = {}; 1270 struct spdk_nvmf_rdma_request_data *next_request_data = NULL; 1271 1272 MOCK_CLEAR(spdk_mempool_get); 1273 rtransport.data_wr_pool = spdk_mempool_create("spdk_nvmf_rdma_wr_data", 1274 SPDK_NVMF_MAX_SGL_ENTRIES, 1275 sizeof(struct spdk_nvmf_rdma_request_data), 1276 SPDK_MEMPOOL_DEFAULT_CACHE_SIZE, 1277 SPDK_ENV_SOCKET_ID_ANY); 1278 next_request_data = spdk_mempool_get(rtransport.data_wr_pool); 1279 SPDK_CU_ASSERT_FATAL(((struct test_mempool *)rtransport.data_wr_pool)->count == 1280 SPDK_NVMF_MAX_SGL_ENTRIES - 1); 1281 next_request_data->wr.wr_id = 1; 1282 next_request_data->wr.num_sge = 2; 1283 next_request_data->wr.next = NULL; 1284 rdma_req.data.wr.next = &next_request_data->wr; 1285 rdma_req.data.wr.wr_id = 1; 1286 rdma_req.data.wr.num_sge = 2; 1287 1288 nvmf_rdma_request_free_data(&rdma_req, &rtransport); 1289 /* Check if next_request_data put into memory pool */ 1290 CU_ASSERT(((struct test_mempool *)rtransport.data_wr_pool)->count == SPDK_NVMF_MAX_SGL_ENTRIES); 1291 CU_ASSERT(rdma_req.data.wr.num_sge == 0); 1292 1293 spdk_mempool_free(rtransport.data_wr_pool); 1294 } 1295 1296 static void 1297 test_nvmf_rdma_update_ibv_state(void) 1298 { 1299 struct spdk_nvmf_rdma_qpair rqpair = {}; 1300 struct spdk_rdma_qp rdma_qp = {}; 1301 struct ibv_qp qp = {}; 1302 int rc = 0; 1303 1304 rqpair.rdma_qp = &rdma_qp; 1305 1306 /* Case 1: Failed to get updated RDMA queue pair state */ 1307 rqpair.ibv_state = IBV_QPS_INIT; 1308 rqpair.rdma_qp->qp = NULL; 1309 1310 rc = nvmf_rdma_update_ibv_state(&rqpair); 1311 CU_ASSERT(rc == IBV_QPS_ERR + 1); 1312 1313 /* Case 2: Bad state updated */ 1314 rqpair.rdma_qp->qp = &qp; 1315 qp.state = IBV_QPS_ERR; 1316 rc = nvmf_rdma_update_ibv_state(&rqpair); 1317 CU_ASSERT(rqpair.ibv_state == 10); 1318 CU_ASSERT(rc == IBV_QPS_ERR + 1); 1319 1320 /* Case 3: Pass */ 1321 qp.state = IBV_QPS_INIT; 1322 rc = nvmf_rdma_update_ibv_state(&rqpair); 1323 CU_ASSERT(rqpair.ibv_state == IBV_QPS_INIT); 1324 CU_ASSERT(rc == IBV_QPS_INIT); 1325 } 1326 1327 static void 1328 test_nvmf_rdma_resources_create(void) 1329 { 1330 static struct spdk_nvmf_rdma_resources *rdma_resource; 1331 struct spdk_nvmf_rdma_resource_opts opts = {}; 1332 struct spdk_nvmf_rdma_qpair qpair = {}; 1333 struct spdk_nvmf_rdma_recv *recv = NULL; 1334 struct spdk_nvmf_rdma_request *req = NULL; 1335 const int DEPTH = 128; 1336 1337 opts.max_queue_depth = DEPTH; 1338 opts.in_capsule_data_size = 4096; 1339 opts.shared = true; 1340 opts.qpair = &qpair; 1341 1342 rdma_resource = nvmf_rdma_resources_create(&opts); 1343 CU_ASSERT(rdma_resource != NULL); 1344 /* Just check first and last entry */ 1345 recv = &rdma_resource->recvs[0]; 1346 req = &rdma_resource->reqs[0]; 1347 CU_ASSERT(recv->rdma_wr.type == RDMA_WR_TYPE_RECV); 1348 CU_ASSERT((uintptr_t)recv->buf == (uintptr_t)(rdma_resource->bufs)); 1349 CU_ASSERT(recv->sgl[0].addr == (uintptr_t)&rdma_resource->cmds[0]); 1350 CU_ASSERT(recv->sgl[0].length == sizeof(rdma_resource->cmds[0])); 1351 CU_ASSERT(recv->sgl[0].lkey == RDMA_UT_LKEY); 1352 CU_ASSERT(recv->wr.num_sge == 2); 1353 CU_ASSERT(recv->wr.wr_id == (uintptr_t)&rdma_resource->recvs[0].rdma_wr); 1354 CU_ASSERT(recv->wr.sg_list == rdma_resource->recvs[0].sgl); 1355 CU_ASSERT(req->req.rsp == &rdma_resource->cpls[0]); 1356 CU_ASSERT(req->rsp.sgl[0].addr == (uintptr_t)&rdma_resource->cpls[0]); 1357 CU_ASSERT(req->rsp.sgl[0].length == sizeof(rdma_resource->cpls[0])); 1358 CU_ASSERT(req->rsp.sgl[0].lkey == RDMA_UT_LKEY); 1359 CU_ASSERT(req->rsp.rdma_wr.type == RDMA_WR_TYPE_SEND); 1360 CU_ASSERT(req->rsp.wr.wr_id == (uintptr_t)&rdma_resource->reqs[0].rsp.rdma_wr); 1361 CU_ASSERT(req->rsp.wr.next == NULL); 1362 CU_ASSERT(req->rsp.wr.opcode == IBV_WR_SEND); 1363 CU_ASSERT(req->rsp.wr.send_flags == IBV_SEND_SIGNALED); 1364 CU_ASSERT(req->rsp.wr.sg_list == rdma_resource->reqs[0].rsp.sgl); 1365 CU_ASSERT(req->rsp.wr.num_sge == NVMF_DEFAULT_RSP_SGE); 1366 CU_ASSERT(req->data.rdma_wr.type == RDMA_WR_TYPE_DATA); 1367 CU_ASSERT(req->data.wr.wr_id == (uintptr_t)&rdma_resource->reqs[0].data.rdma_wr); 1368 CU_ASSERT(req->data.wr.next == NULL); 1369 CU_ASSERT(req->data.wr.send_flags == IBV_SEND_SIGNALED); 1370 CU_ASSERT(req->data.wr.sg_list == rdma_resource->reqs[0].data.sgl); 1371 CU_ASSERT(req->data.wr.num_sge == SPDK_NVMF_MAX_SGL_ENTRIES); 1372 CU_ASSERT(req->state == RDMA_REQUEST_STATE_FREE); 1373 1374 recv = &rdma_resource->recvs[DEPTH - 1]; 1375 req = &rdma_resource->reqs[DEPTH - 1]; 1376 CU_ASSERT(recv->rdma_wr.type == RDMA_WR_TYPE_RECV); 1377 CU_ASSERT((uintptr_t)recv->buf == (uintptr_t)(rdma_resource->bufs + 1378 (DEPTH - 1) * 4096)); 1379 CU_ASSERT(recv->sgl[0].addr == (uintptr_t)&rdma_resource->cmds[DEPTH - 1]); 1380 CU_ASSERT(recv->sgl[0].length == sizeof(rdma_resource->cmds[DEPTH - 1])); 1381 CU_ASSERT(recv->sgl[0].lkey == RDMA_UT_LKEY); 1382 CU_ASSERT(recv->wr.num_sge == 2); 1383 CU_ASSERT(recv->wr.wr_id == (uintptr_t)&rdma_resource->recvs[DEPTH - 1].rdma_wr); 1384 CU_ASSERT(recv->wr.sg_list == rdma_resource->recvs[DEPTH - 1].sgl); 1385 CU_ASSERT(req->req.rsp == &rdma_resource->cpls[DEPTH - 1]); 1386 CU_ASSERT(req->rsp.sgl[0].addr == (uintptr_t)&rdma_resource->cpls[DEPTH - 1]); 1387 CU_ASSERT(req->rsp.sgl[0].length == sizeof(rdma_resource->cpls[DEPTH - 1])); 1388 CU_ASSERT(req->rsp.sgl[0].lkey == RDMA_UT_LKEY); 1389 CU_ASSERT(req->rsp.rdma_wr.type == RDMA_WR_TYPE_SEND); 1390 CU_ASSERT(req->rsp.wr.wr_id == (uintptr_t) 1391 &req->rsp.rdma_wr); 1392 CU_ASSERT(req->rsp.wr.next == NULL); 1393 CU_ASSERT(req->rsp.wr.opcode == IBV_WR_SEND); 1394 CU_ASSERT(req->rsp.wr.send_flags == IBV_SEND_SIGNALED); 1395 CU_ASSERT(req->rsp.wr.sg_list == rdma_resource->reqs[DEPTH - 1].rsp.sgl); 1396 CU_ASSERT(req->rsp.wr.num_sge == NVMF_DEFAULT_RSP_SGE); 1397 CU_ASSERT(req->data.rdma_wr.type == RDMA_WR_TYPE_DATA); 1398 CU_ASSERT(req->data.wr.wr_id == (uintptr_t) 1399 &req->data.rdma_wr); 1400 CU_ASSERT(req->data.wr.next == NULL); 1401 CU_ASSERT(req->data.wr.send_flags == IBV_SEND_SIGNALED); 1402 CU_ASSERT(req->data.wr.sg_list == rdma_resource->reqs[DEPTH - 1].data.sgl); 1403 CU_ASSERT(req->data.wr.num_sge == SPDK_NVMF_MAX_SGL_ENTRIES); 1404 CU_ASSERT(req->state == RDMA_REQUEST_STATE_FREE); 1405 1406 nvmf_rdma_resources_destroy(rdma_resource); 1407 } 1408 1409 static void 1410 test_nvmf_rdma_qpair_compare(void) 1411 { 1412 struct spdk_nvmf_rdma_qpair rqpair1 = {}, rqpair2 = {}; 1413 1414 rqpair1.qp_num = 0; 1415 rqpair2.qp_num = UINT32_MAX; 1416 1417 CU_ASSERT(nvmf_rdma_qpair_compare(&rqpair1, &rqpair2) < 0); 1418 CU_ASSERT(nvmf_rdma_qpair_compare(&rqpair2, &rqpair1) > 0); 1419 } 1420 1421 static void 1422 test_nvmf_rdma_resize_cq(void) 1423 { 1424 int rc = -1; 1425 int tnum_wr = 0; 1426 int tnum_cqe = 0; 1427 struct spdk_nvmf_rdma_qpair rqpair = {}; 1428 struct spdk_nvmf_rdma_poller rpoller = {}; 1429 struct spdk_nvmf_rdma_device rdevice = {}; 1430 struct ibv_context ircontext = {}; 1431 struct ibv_device idevice = {}; 1432 1433 rdevice.context = &ircontext; 1434 rqpair.poller = &rpoller; 1435 ircontext.device = &idevice; 1436 1437 /* Test1: Current capacity support required size. */ 1438 rpoller.required_num_wr = 10; 1439 rpoller.num_cqe = 20; 1440 rqpair.max_queue_depth = 2; 1441 tnum_wr = rpoller.required_num_wr; 1442 tnum_cqe = rpoller.num_cqe; 1443 1444 rc = nvmf_rdma_resize_cq(&rqpair, &rdevice); 1445 CU_ASSERT(rc == 0); 1446 CU_ASSERT(rpoller.required_num_wr == 10 + MAX_WR_PER_QP(rqpair.max_queue_depth)); 1447 CU_ASSERT(rpoller.required_num_wr > tnum_wr); 1448 CU_ASSERT(rpoller.num_cqe == tnum_cqe); 1449 1450 /* Test2: iWARP doesn't support CQ resize. */ 1451 tnum_wr = rpoller.required_num_wr; 1452 tnum_cqe = rpoller.num_cqe; 1453 idevice.transport_type = IBV_TRANSPORT_IWARP; 1454 1455 rc = nvmf_rdma_resize_cq(&rqpair, &rdevice); 1456 CU_ASSERT(rc == -1); 1457 CU_ASSERT(rpoller.required_num_wr == tnum_wr); 1458 CU_ASSERT(rpoller.num_cqe == tnum_cqe); 1459 1460 1461 /* Test3: RDMA CQE requirement exceeds device max_cqe limitation. */ 1462 tnum_wr = rpoller.required_num_wr; 1463 tnum_cqe = rpoller.num_cqe; 1464 idevice.transport_type = IBV_TRANSPORT_UNKNOWN; 1465 rdevice.attr.max_cqe = 3; 1466 1467 rc = nvmf_rdma_resize_cq(&rqpair, &rdevice); 1468 CU_ASSERT(rc == -1); 1469 CU_ASSERT(rpoller.required_num_wr == tnum_wr); 1470 CU_ASSERT(rpoller.num_cqe == tnum_cqe); 1471 1472 /* Test4: RDMA CQ resize failed. */ 1473 tnum_wr = rpoller.required_num_wr; 1474 tnum_cqe = rpoller.num_cqe; 1475 idevice.transport_type = IBV_TRANSPORT_IB; 1476 rdevice.attr.max_cqe = 30; 1477 MOCK_SET(ibv_resize_cq, -1); 1478 1479 rc = nvmf_rdma_resize_cq(&rqpair, &rdevice); 1480 CU_ASSERT(rc == -1); 1481 CU_ASSERT(rpoller.required_num_wr == tnum_wr); 1482 CU_ASSERT(rpoller.num_cqe == tnum_cqe); 1483 1484 /* Test5: RDMA CQ resize success. rsize = MIN(MAX(num_cqe * 2, required_num_wr), device->attr.max_cqe). */ 1485 tnum_wr = rpoller.required_num_wr; 1486 tnum_cqe = rpoller.num_cqe; 1487 MOCK_SET(ibv_resize_cq, 0); 1488 1489 rc = nvmf_rdma_resize_cq(&rqpair, &rdevice); 1490 CU_ASSERT(rc == 0); 1491 CU_ASSERT(rpoller.num_cqe = 30); 1492 CU_ASSERT(rpoller.required_num_wr == 18 + MAX_WR_PER_QP(rqpair.max_queue_depth)); 1493 CU_ASSERT(rpoller.required_num_wr > tnum_wr); 1494 CU_ASSERT(rpoller.num_cqe > tnum_cqe); 1495 } 1496 1497 int 1498 main(int argc, char **argv) 1499 { 1500 CU_pSuite suite = NULL; 1501 unsigned int num_failures; 1502 1503 CU_set_error_action(CUEA_ABORT); 1504 CU_initialize_registry(); 1505 1506 suite = CU_add_suite("nvmf", NULL, NULL); 1507 1508 CU_ADD_TEST(suite, test_spdk_nvmf_rdma_request_parse_sgl); 1509 CU_ADD_TEST(suite, test_spdk_nvmf_rdma_request_process); 1510 CU_ADD_TEST(suite, test_nvmf_rdma_get_optimal_poll_group); 1511 CU_ADD_TEST(suite, test_spdk_nvmf_rdma_request_parse_sgl_with_md); 1512 CU_ADD_TEST(suite, test_nvmf_rdma_opts_init); 1513 CU_ADD_TEST(suite, test_nvmf_rdma_request_free_data); 1514 CU_ADD_TEST(suite, test_nvmf_rdma_update_ibv_state); 1515 CU_ADD_TEST(suite, test_nvmf_rdma_resources_create); 1516 CU_ADD_TEST(suite, test_nvmf_rdma_qpair_compare); 1517 CU_ADD_TEST(suite, test_nvmf_rdma_resize_cq); 1518 1519 CU_basic_set_mode(CU_BRM_VERBOSE); 1520 CU_basic_run_tests(); 1521 num_failures = CU_get_number_of_failures(); 1522 CU_cleanup_registry(); 1523 return num_failures; 1524 } 1525