1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2018 Intel Corporation. All rights reserved. 3 * Copyright (c) 2019, 2021 Mellanox Technologies LTD. All rights reserved. 4 */ 5 6 #include "spdk/stdinc.h" 7 #include "spdk_internal/cunit.h" 8 #include "common/lib/test_env.c" 9 #include "common/lib/test_iobuf.c" 10 #include "common/lib/test_rdma.c" 11 #include "nvmf/rdma.c" 12 #include "nvmf/transport.c" 13 14 #define RDMA_UT_UNITS_IN_MAX_IO 16 15 16 struct spdk_nvmf_transport_opts g_rdma_ut_transport_opts = { 17 .max_queue_depth = SPDK_NVMF_RDMA_DEFAULT_MAX_QUEUE_DEPTH, 18 .max_qpairs_per_ctrlr = SPDK_NVMF_RDMA_DEFAULT_MAX_QPAIRS_PER_CTRLR, 19 .in_capsule_data_size = SPDK_NVMF_RDMA_DEFAULT_IN_CAPSULE_DATA_SIZE, 20 .max_io_size = (SPDK_NVMF_RDMA_MIN_IO_BUFFER_SIZE * RDMA_UT_UNITS_IN_MAX_IO), 21 .io_unit_size = SPDK_NVMF_RDMA_MIN_IO_BUFFER_SIZE, 22 .max_aq_depth = SPDK_NVMF_RDMA_DEFAULT_AQ_DEPTH, 23 .num_shared_buffers = SPDK_NVMF_RDMA_DEFAULT_NUM_SHARED_BUFFERS, 24 }; 25 26 SPDK_LOG_REGISTER_COMPONENT(nvmf) 27 DEFINE_STUB(spdk_mem_map_set_translation, int, (struct spdk_mem_map *map, uint64_t vaddr, 28 uint64_t size, uint64_t translation), 0); 29 DEFINE_STUB(spdk_mem_map_clear_translation, int, (struct spdk_mem_map *map, uint64_t vaddr, 30 uint64_t size), 0); 31 DEFINE_STUB(spdk_mem_map_alloc, struct spdk_mem_map *, (uint64_t default_translation, 32 const struct spdk_mem_map_ops *ops, void *cb_ctx), NULL); 33 DEFINE_STUB(spdk_nvmf_qpair_disconnect, int, (struct spdk_nvmf_qpair *qpair, 34 nvmf_qpair_disconnect_cb cb_fn, void *ctx), 0); 35 DEFINE_STUB(spdk_nvmf_qpair_get_listen_trid, int, 36 (struct spdk_nvmf_qpair *qpair, struct spdk_nvme_transport_id *trid), 0); 37 DEFINE_STUB_V(spdk_mem_map_free, (struct spdk_mem_map **pmap)); 38 39 DEFINE_STUB_V(spdk_nvmf_ctrlr_data_init, (struct spdk_nvmf_transport_opts *opts, 40 struct spdk_nvmf_ctrlr_data *cdata)); 41 DEFINE_STUB_V(spdk_nvmf_request_exec, (struct spdk_nvmf_request *req)); 42 DEFINE_STUB(spdk_nvmf_request_complete, int, (struct spdk_nvmf_request *req), 0); 43 DEFINE_STUB(spdk_nvme_transport_id_compare, int, (const struct spdk_nvme_transport_id *trid1, 44 const struct spdk_nvme_transport_id *trid2), 0); 45 DEFINE_STUB_V(spdk_nvmf_ctrlr_abort_aer, (struct spdk_nvmf_ctrlr *ctrlr)); 46 DEFINE_STUB(spdk_nvmf_request_get_dif_ctx, bool, (struct spdk_nvmf_request *req, 47 struct spdk_dif_ctx *dif_ctx), false); 48 DEFINE_STUB_V(spdk_nvme_trid_populate_transport, (struct spdk_nvme_transport_id *trid, 49 enum spdk_nvme_transport_type trtype)); 50 DEFINE_STUB_V(spdk_nvmf_tgt_new_qpair, (struct spdk_nvmf_tgt *tgt, struct spdk_nvmf_qpair *qpair)); 51 DEFINE_STUB(nvmf_ctrlr_abort_request, int, (struct spdk_nvmf_request *req), 0); 52 DEFINE_STUB(spdk_nvme_transport_id_adrfam_str, const char *, (enum spdk_nvmf_adrfam adrfam), NULL); 53 DEFINE_STUB(ibv_dereg_mr, int, (struct ibv_mr *mr), 0); 54 DEFINE_STUB(ibv_resize_cq, int, (struct ibv_cq *cq, int cqe), 0); 55 DEFINE_STUB(spdk_mempool_lookup, struct spdk_mempool *, (const char *name), NULL); 56 57 /* ibv_reg_mr can be a macro, need to undefine it */ 58 #ifdef ibv_reg_mr 59 #undef ibv_reg_mr 60 #endif 61 62 DEFINE_RETURN_MOCK(ibv_reg_mr, struct ibv_mr *); 63 struct ibv_mr * 64 ibv_reg_mr(struct ibv_pd *pd, void *addr, size_t length, int access) 65 { 66 HANDLE_RETURN_MOCK(ibv_reg_mr); 67 if (length > 0) { 68 return &g_rdma_mr; 69 } else { 70 return NULL; 71 } 72 } 73 74 int 75 ibv_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, 76 int attr_mask, struct ibv_qp_init_attr *init_attr) 77 { 78 if (qp == NULL) { 79 return -1; 80 } else { 81 attr->port_num = 80; 82 83 if (qp->state == IBV_QPS_ERR) { 84 attr->qp_state = 10; 85 } else { 86 attr->qp_state = IBV_QPS_INIT; 87 } 88 89 return 0; 90 } 91 } 92 93 const char * 94 spdk_nvme_transport_id_trtype_str(enum spdk_nvme_transport_type trtype) 95 { 96 switch (trtype) { 97 case SPDK_NVME_TRANSPORT_PCIE: 98 return "PCIe"; 99 case SPDK_NVME_TRANSPORT_RDMA: 100 return "RDMA"; 101 case SPDK_NVME_TRANSPORT_FC: 102 return "FC"; 103 default: 104 return NULL; 105 } 106 } 107 108 int 109 spdk_nvme_transport_id_populate_trstring(struct spdk_nvme_transport_id *trid, const char *trstring) 110 { 111 int len, i; 112 113 if (trstring == NULL) { 114 return -EINVAL; 115 } 116 117 len = strnlen(trstring, SPDK_NVMF_TRSTRING_MAX_LEN); 118 if (len == SPDK_NVMF_TRSTRING_MAX_LEN) { 119 return -EINVAL; 120 } 121 122 /* cast official trstring to uppercase version of input. */ 123 for (i = 0; i < len; i++) { 124 trid->trstring[i] = toupper(trstring[i]); 125 } 126 return 0; 127 } 128 129 static void 130 reset_nvmf_rdma_request(struct spdk_nvmf_rdma_request *rdma_req) 131 { 132 int i; 133 134 rdma_req->req.length = 0; 135 rdma_req->req.data_from_pool = false; 136 rdma_req->data.wr.num_sge = 0; 137 rdma_req->data.wr.wr.rdma.remote_addr = 0; 138 rdma_req->data.wr.wr.rdma.rkey = 0; 139 rdma_req->offset = 0; 140 memset(&rdma_req->req.dif, 0, sizeof(rdma_req->req.dif)); 141 142 for (i = 0; i < SPDK_NVMF_MAX_SGL_ENTRIES; i++) { 143 rdma_req->req.iov[i].iov_base = 0; 144 rdma_req->req.iov[i].iov_len = 0; 145 rdma_req->data.wr.sg_list[i].addr = 0; 146 rdma_req->data.wr.sg_list[i].length = 0; 147 rdma_req->data.wr.sg_list[i].lkey = 0; 148 } 149 rdma_req->req.iovcnt = 0; 150 if (rdma_req->req.stripped_data) { 151 free(rdma_req->req.stripped_data); 152 rdma_req->req.stripped_data = NULL; 153 } 154 } 155 156 static void 157 test_spdk_nvmf_rdma_request_parse_sgl(void) 158 { 159 struct spdk_nvmf_rdma_transport rtransport; 160 struct spdk_nvmf_rdma_device device; 161 struct spdk_nvmf_rdma_request rdma_req = {}; 162 struct spdk_nvmf_rdma_recv recv; 163 struct spdk_nvmf_rdma_poll_group group; 164 struct spdk_nvmf_rdma_qpair rqpair; 165 struct spdk_nvmf_rdma_poller poller; 166 union nvmf_c2h_msg cpl; 167 union nvmf_h2c_msg cmd; 168 struct spdk_nvme_sgl_descriptor *sgl; 169 struct spdk_nvme_sgl_descriptor sgl_desc[SPDK_NVMF_MAX_SGL_ENTRIES] = {{0}}; 170 struct spdk_nvmf_rdma_request_data data; 171 int rc, i; 172 uint32_t sgl_length; 173 174 data.wr.sg_list = data.sgl; 175 group.group.transport = &rtransport.transport; 176 poller.group = &group; 177 rqpair.poller = &poller; 178 rqpair.max_send_sge = SPDK_NVMF_MAX_SGL_ENTRIES; 179 180 sgl = &cmd.nvme_cmd.dptr.sgl1; 181 rdma_req.recv = &recv; 182 rdma_req.req.cmd = &cmd; 183 rdma_req.req.rsp = &cpl; 184 rdma_req.data.wr.sg_list = rdma_req.data.sgl; 185 rdma_req.req.qpair = &rqpair.qpair; 186 rdma_req.req.xfer = SPDK_NVME_DATA_CONTROLLER_TO_HOST; 187 188 rtransport.transport.opts = g_rdma_ut_transport_opts; 189 rtransport.data_wr_pool = NULL; 190 191 device.attr.device_cap_flags = 0; 192 sgl->keyed.key = 0xEEEE; 193 sgl->address = 0xFFFF; 194 rdma_req.recv->buf = (void *)0xDDDD; 195 196 /* Test 1: sgl type: keyed data block subtype: address */ 197 sgl->generic.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; 198 sgl->keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; 199 200 /* Part 1: simple I/O, one SGL smaller than the transport io unit size */ 201 MOCK_SET(spdk_iobuf_get, (void *)0x2000); 202 reset_nvmf_rdma_request(&rdma_req); 203 sgl->keyed.length = rtransport.transport.opts.io_unit_size / 2; 204 205 device.map = (void *)0x0; 206 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 207 CU_ASSERT(rc == 0); 208 CU_ASSERT(rdma_req.req.data_from_pool == true); 209 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size / 2); 210 CU_ASSERT((uint64_t)rdma_req.req.iovcnt == 1); 211 CU_ASSERT((uint64_t)rdma_req.req.iov[0].iov_base == 0x2000); 212 CU_ASSERT(rdma_req.data.wr.num_sge == 1); 213 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 214 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 215 CU_ASSERT((uint64_t)rdma_req.req.iov[0].iov_base == 0x2000); 216 CU_ASSERT(rdma_req.data.wr.sg_list[0].addr == 0x2000); 217 CU_ASSERT(rdma_req.data.wr.sg_list[0].length == rtransport.transport.opts.io_unit_size / 2); 218 CU_ASSERT(rdma_req.data.wr.sg_list[0].lkey == RDMA_UT_LKEY); 219 220 /* Part 2: simple I/O, one SGL larger than the transport io unit size (equal to the max io size) */ 221 reset_nvmf_rdma_request(&rdma_req); 222 sgl->keyed.length = rtransport.transport.opts.io_unit_size * RDMA_UT_UNITS_IN_MAX_IO; 223 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 224 225 CU_ASSERT(rc == 0); 226 CU_ASSERT(rdma_req.req.data_from_pool == true); 227 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size * RDMA_UT_UNITS_IN_MAX_IO); 228 CU_ASSERT(rdma_req.data.wr.num_sge == RDMA_UT_UNITS_IN_MAX_IO); 229 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 230 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 231 for (i = 0; i < RDMA_UT_UNITS_IN_MAX_IO; i++) { 232 CU_ASSERT((uint64_t)rdma_req.req.iov[i].iov_base == 0x2000); 233 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == 0x2000); 234 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == rtransport.transport.opts.io_unit_size); 235 CU_ASSERT(rdma_req.data.wr.sg_list[i].lkey == RDMA_UT_LKEY); 236 } 237 238 /* Part 3: simple I/O one SGL larger than the transport max io size */ 239 reset_nvmf_rdma_request(&rdma_req); 240 sgl->keyed.length = rtransport.transport.opts.max_io_size * 2; 241 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 242 243 CU_ASSERT(rc == -1); 244 245 /* Part 4: Pretend there are no buffer pools */ 246 MOCK_SET(spdk_iobuf_get, NULL); 247 reset_nvmf_rdma_request(&rdma_req); 248 sgl->keyed.length = rtransport.transport.opts.io_unit_size * RDMA_UT_UNITS_IN_MAX_IO; 249 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 250 251 CU_ASSERT(rc == 0); 252 CU_ASSERT(rdma_req.req.data_from_pool == false); 253 CU_ASSERT(rdma_req.req.iovcnt == 0); 254 CU_ASSERT(rdma_req.data.wr.num_sge == 0); 255 CU_ASSERT(rdma_req.req.iov[0].iov_base == NULL); 256 CU_ASSERT(rdma_req.data.wr.sg_list[0].addr == 0); 257 CU_ASSERT(rdma_req.data.wr.sg_list[0].length == 0); 258 CU_ASSERT(rdma_req.data.wr.sg_list[0].lkey == 0); 259 260 rdma_req.recv->buf = (void *)0xDDDD; 261 /* Test 2: sgl type: keyed data block subtype: offset (in capsule data) */ 262 sgl->generic.type = SPDK_NVME_SGL_TYPE_DATA_BLOCK; 263 sgl->unkeyed.subtype = SPDK_NVME_SGL_SUBTYPE_OFFSET; 264 265 /* Part 1: Normal I/O smaller than in capsule data size no offset */ 266 reset_nvmf_rdma_request(&rdma_req); 267 sgl->address = 0; 268 sgl->unkeyed.length = rtransport.transport.opts.in_capsule_data_size; 269 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 270 271 CU_ASSERT(rc == 0); 272 CU_ASSERT(rdma_req.req.iovcnt == 1); 273 CU_ASSERT(rdma_req.req.iov[0].iov_base == (void *)0xDDDD); 274 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.in_capsule_data_size); 275 CU_ASSERT(rdma_req.req.data_from_pool == false); 276 277 /* Part 2: I/O offset + length too large */ 278 reset_nvmf_rdma_request(&rdma_req); 279 sgl->address = rtransport.transport.opts.in_capsule_data_size; 280 sgl->unkeyed.length = rtransport.transport.opts.in_capsule_data_size; 281 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 282 283 CU_ASSERT(rc == -1); 284 285 /* Part 3: I/O too large */ 286 reset_nvmf_rdma_request(&rdma_req); 287 sgl->address = 0; 288 sgl->unkeyed.length = rtransport.transport.opts.in_capsule_data_size * 2; 289 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 290 291 CU_ASSERT(rc == -1); 292 293 /* Test 3: Multi SGL */ 294 sgl->generic.type = SPDK_NVME_SGL_TYPE_LAST_SEGMENT; 295 sgl->unkeyed.subtype = SPDK_NVME_SGL_SUBTYPE_OFFSET; 296 sgl->address = 0; 297 rdma_req.recv->buf = (void *)&sgl_desc; 298 MOCK_SET(spdk_iobuf_get, &data); 299 MOCK_SET(spdk_mempool_get, &data); 300 301 /* part 1: 2 segments each with 1 wr. */ 302 reset_nvmf_rdma_request(&rdma_req); 303 sgl->unkeyed.length = 2 * sizeof(struct spdk_nvme_sgl_descriptor); 304 for (i = 0; i < 2; i++) { 305 sgl_desc[i].keyed.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; 306 sgl_desc[i].keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; 307 sgl_desc[i].keyed.length = rtransport.transport.opts.io_unit_size; 308 sgl_desc[i].address = 0x4000 + i * rtransport.transport.opts.io_unit_size; 309 sgl_desc[i].keyed.key = 0x44; 310 } 311 312 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 313 314 CU_ASSERT(rc == 0); 315 CU_ASSERT(rdma_req.req.data_from_pool == true); 316 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size * 2); 317 CU_ASSERT(rdma_req.data.wr.num_sge == 1); 318 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0x44); 319 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0x4000); 320 CU_ASSERT(rdma_req.data.wr.next == &data.wr); 321 CU_ASSERT(data.wr.wr.rdma.rkey == 0x44); 322 CU_ASSERT(data.wr.wr.rdma.remote_addr == 0x4000 + rtransport.transport.opts.io_unit_size); 323 CU_ASSERT(data.wr.num_sge == 1); 324 CU_ASSERT(data.wr.next == &rdma_req.rsp.wr); 325 326 /* part 2: 2 segments, each with 1 wr containing 8 sge_elements */ 327 reset_nvmf_rdma_request(&rdma_req); 328 sgl->unkeyed.length = 2 * sizeof(struct spdk_nvme_sgl_descriptor); 329 for (i = 0; i < 2; i++) { 330 sgl_desc[i].keyed.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; 331 sgl_desc[i].keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; 332 sgl_desc[i].keyed.length = rtransport.transport.opts.io_unit_size * 8; 333 sgl_desc[i].address = 0x4000 + i * 8 * rtransport.transport.opts.io_unit_size; 334 sgl_desc[i].keyed.key = 0x44; 335 } 336 337 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 338 339 CU_ASSERT(rc == 0); 340 CU_ASSERT(rdma_req.req.data_from_pool == true); 341 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size * 16); 342 CU_ASSERT(rdma_req.req.iovcnt == 16); 343 CU_ASSERT(rdma_req.data.wr.num_sge == 8); 344 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0x44); 345 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0x4000); 346 CU_ASSERT(rdma_req.data.wr.next == &data.wr); 347 CU_ASSERT(data.wr.wr.rdma.rkey == 0x44); 348 CU_ASSERT(data.wr.wr.rdma.remote_addr == 0x4000 + rtransport.transport.opts.io_unit_size * 8); 349 CU_ASSERT(data.wr.num_sge == 8); 350 CU_ASSERT(data.wr.next == &rdma_req.rsp.wr); 351 352 /* part 3: 2 segments, one very large, one very small */ 353 reset_nvmf_rdma_request(&rdma_req); 354 for (i = 0; i < 2; i++) { 355 sgl_desc[i].keyed.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; 356 sgl_desc[i].keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; 357 sgl_desc[i].keyed.key = 0x44; 358 } 359 360 sgl_desc[0].keyed.length = rtransport.transport.opts.io_unit_size * 15 + 361 rtransport.transport.opts.io_unit_size / 2; 362 sgl_desc[0].address = 0x4000; 363 sgl_desc[1].keyed.length = rtransport.transport.opts.io_unit_size / 2; 364 sgl_desc[1].address = 0x4000 + rtransport.transport.opts.io_unit_size * 15 + 365 rtransport.transport.opts.io_unit_size / 2; 366 367 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 368 369 CU_ASSERT(rc == 0); 370 CU_ASSERT(rdma_req.req.data_from_pool == true); 371 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size * 16); 372 CU_ASSERT(rdma_req.req.iovcnt == 16); 373 CU_ASSERT(rdma_req.data.wr.num_sge == 16); 374 for (i = 0; i < 15; i++) { 375 CU_ASSERT(rdma_req.data.sgl[i].length == rtransport.transport.opts.io_unit_size); 376 } 377 CU_ASSERT(rdma_req.data.sgl[15].length == rtransport.transport.opts.io_unit_size / 2); 378 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0x44); 379 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0x4000); 380 CU_ASSERT(rdma_req.data.wr.next == &data.wr); 381 CU_ASSERT(data.wr.wr.rdma.rkey == 0x44); 382 CU_ASSERT(data.wr.wr.rdma.remote_addr == 0x4000 + rtransport.transport.opts.io_unit_size * 15 + 383 rtransport.transport.opts.io_unit_size / 2); 384 CU_ASSERT(data.sgl[0].length == rtransport.transport.opts.io_unit_size / 2); 385 CU_ASSERT(data.wr.num_sge == 1); 386 CU_ASSERT(data.wr.next == &rdma_req.rsp.wr); 387 388 /* part 4: 2 SGL descriptors, each length is transport buffer / 2 389 * 1 transport buffers should be allocated */ 390 reset_nvmf_rdma_request(&rdma_req); 391 sgl->unkeyed.length = 2 * sizeof(struct spdk_nvme_sgl_descriptor); 392 sgl_length = rtransport.transport.opts.io_unit_size / 2; 393 for (i = 0; i < 2; i++) { 394 sgl_desc[i].keyed.length = sgl_length; 395 sgl_desc[i].address = 0x4000 + i * sgl_length; 396 } 397 398 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 399 400 CU_ASSERT(rc == 0); 401 CU_ASSERT(rdma_req.req.data_from_pool == true); 402 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size); 403 CU_ASSERT(rdma_req.req.iovcnt == 1); 404 405 CU_ASSERT(rdma_req.data.sgl[0].length == sgl_length); 406 /* We mocked mempool_get to return address of data variable. Mempool is used 407 * to get both additional WRs and data buffers, so data points to &data */ 408 CU_ASSERT(rdma_req.data.sgl[0].addr == (uint64_t)&data); 409 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0x44); 410 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0x4000); 411 CU_ASSERT(rdma_req.data.wr.num_sge == 1); 412 CU_ASSERT(rdma_req.data.wr.next == &data.wr); 413 414 CU_ASSERT(data.wr.wr.rdma.rkey == 0x44); 415 CU_ASSERT(data.wr.wr.rdma.remote_addr == 0x4000 + sgl_length); 416 CU_ASSERT(data.sgl[0].length == sgl_length); 417 CU_ASSERT(data.sgl[0].addr == (uint64_t)&data + sgl_length); 418 CU_ASSERT(data.wr.num_sge == 1); 419 420 MOCK_CLEAR(spdk_mempool_get); 421 MOCK_CLEAR(spdk_iobuf_get); 422 423 reset_nvmf_rdma_request(&rdma_req); 424 } 425 426 static struct spdk_nvmf_rdma_recv * 427 create_recv(struct spdk_nvmf_rdma_qpair *rqpair, enum spdk_nvme_nvm_opcode opc) 428 { 429 struct spdk_nvmf_rdma_recv *rdma_recv; 430 union nvmf_h2c_msg *cmd; 431 struct spdk_nvme_sgl_descriptor *sgl; 432 433 rdma_recv = calloc(1, sizeof(*rdma_recv)); 434 rdma_recv->qpair = rqpair; 435 cmd = calloc(1, sizeof(*cmd)); 436 rdma_recv->sgl[0].addr = (uintptr_t)cmd; 437 cmd->nvme_cmd.opc = opc; 438 sgl = &cmd->nvme_cmd.dptr.sgl1; 439 sgl->keyed.key = 0xEEEE; 440 sgl->address = 0xFFFF; 441 sgl->keyed.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; 442 sgl->keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; 443 sgl->keyed.length = 1; 444 445 return rdma_recv; 446 } 447 448 static void 449 free_recv(struct spdk_nvmf_rdma_recv *rdma_recv) 450 { 451 free((void *)rdma_recv->sgl[0].addr); 452 free(rdma_recv); 453 } 454 455 static struct spdk_nvmf_rdma_request * 456 create_req(struct spdk_nvmf_rdma_qpair *rqpair, 457 struct spdk_nvmf_rdma_recv *rdma_recv) 458 { 459 struct spdk_nvmf_rdma_request *rdma_req; 460 union nvmf_c2h_msg *cpl; 461 462 rdma_req = calloc(1, sizeof(*rdma_req)); 463 rdma_req->recv = rdma_recv; 464 rdma_req->req.qpair = &rqpair->qpair; 465 rdma_req->state = RDMA_REQUEST_STATE_NEW; 466 rdma_req->data.wr.wr_id = (uintptr_t)&rdma_req->data.rdma_wr; 467 rdma_req->data.wr.sg_list = rdma_req->data.sgl; 468 cpl = calloc(1, sizeof(*cpl)); 469 rdma_req->rsp.sgl[0].addr = (uintptr_t)cpl; 470 rdma_req->req.rsp = cpl; 471 472 return rdma_req; 473 } 474 475 static void 476 free_req(struct spdk_nvmf_rdma_request *rdma_req) 477 { 478 free((void *)rdma_req->rsp.sgl[0].addr); 479 free(rdma_req); 480 } 481 482 static void 483 qpair_reset(struct spdk_nvmf_rdma_qpair *rqpair, 484 struct spdk_nvmf_rdma_poller *poller, 485 struct spdk_nvmf_rdma_device *device, 486 struct spdk_nvmf_rdma_resources *resources, 487 struct spdk_nvmf_transport *transport) 488 { 489 memset(rqpair, 0, sizeof(*rqpair)); 490 STAILQ_INIT(&rqpair->pending_rdma_write_queue); 491 STAILQ_INIT(&rqpair->pending_rdma_read_queue); 492 rqpair->poller = poller; 493 rqpair->device = device; 494 rqpair->resources = resources; 495 rqpair->qpair.qid = 1; 496 rqpair->ibv_state = IBV_QPS_RTS; 497 rqpair->qpair.state = SPDK_NVMF_QPAIR_ACTIVE; 498 rqpair->max_send_sge = SPDK_NVMF_MAX_SGL_ENTRIES; 499 rqpair->max_send_depth = 16; 500 rqpair->max_read_depth = 16; 501 rqpair->qpair.transport = transport; 502 } 503 504 static void 505 poller_reset(struct spdk_nvmf_rdma_poller *poller, 506 struct spdk_nvmf_rdma_poll_group *group) 507 { 508 memset(poller, 0, sizeof(*poller)); 509 STAILQ_INIT(&poller->qpairs_pending_recv); 510 STAILQ_INIT(&poller->qpairs_pending_send); 511 poller->group = group; 512 } 513 514 static void 515 test_spdk_nvmf_rdma_request_process(void) 516 { 517 struct spdk_nvmf_rdma_transport rtransport = {}; 518 struct spdk_nvmf_rdma_poll_group group = {}; 519 struct spdk_nvmf_rdma_poller poller = {}; 520 struct spdk_nvmf_rdma_device device = {}; 521 struct spdk_nvmf_rdma_resources resources = {}; 522 struct spdk_nvmf_rdma_qpair rqpair = {}; 523 struct spdk_nvmf_rdma_recv *rdma_recv; 524 struct spdk_nvmf_rdma_request *rdma_req; 525 bool progress; 526 527 STAILQ_INIT(&group.group.pending_buf_queue); 528 poller_reset(&poller, &group); 529 qpair_reset(&rqpair, &poller, &device, &resources, &rtransport.transport); 530 531 rtransport.transport.opts = g_rdma_ut_transport_opts; 532 rtransport.data_wr_pool = spdk_mempool_create("test_wr_pool", 128, 533 sizeof(struct spdk_nvmf_rdma_request_data), 534 0, 0); 535 MOCK_CLEAR(spdk_iobuf_get); 536 537 device.attr.device_cap_flags = 0; 538 device.map = (void *)0x0; 539 540 /* Test 1: single SGL READ request */ 541 rdma_recv = create_recv(&rqpair, SPDK_NVME_OPC_READ); 542 rdma_req = create_req(&rqpair, rdma_recv); 543 rqpair.current_recv_depth = 1; 544 /* NEW -> EXECUTING */ 545 progress = nvmf_rdma_request_process(&rtransport, rdma_req); 546 CU_ASSERT(progress == true); 547 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_EXECUTING); 548 CU_ASSERT(rdma_req->req.xfer == SPDK_NVME_DATA_CONTROLLER_TO_HOST); 549 /* EXECUTED -> TRANSFERRING_C2H */ 550 rdma_req->state = RDMA_REQUEST_STATE_EXECUTED; 551 progress = nvmf_rdma_request_process(&rtransport, rdma_req); 552 CU_ASSERT(progress == true); 553 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_TRANSFERRING_CONTROLLER_TO_HOST); 554 CU_ASSERT(rdma_req->recv == NULL); 555 /* COMPLETED -> FREE */ 556 rdma_req->state = RDMA_REQUEST_STATE_COMPLETED; 557 progress = nvmf_rdma_request_process(&rtransport, rdma_req); 558 CU_ASSERT(progress == true); 559 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_FREE); 560 561 free_recv(rdma_recv); 562 free_req(rdma_req); 563 poller_reset(&poller, &group); 564 qpair_reset(&rqpair, &poller, &device, &resources, &rtransport.transport); 565 566 /* Test 2: single SGL WRITE request */ 567 rdma_recv = create_recv(&rqpair, SPDK_NVME_OPC_WRITE); 568 rdma_req = create_req(&rqpair, rdma_recv); 569 rqpair.current_recv_depth = 1; 570 /* NEW -> TRANSFERRING_H2C */ 571 progress = nvmf_rdma_request_process(&rtransport, rdma_req); 572 CU_ASSERT(progress == true); 573 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_TRANSFERRING_HOST_TO_CONTROLLER); 574 CU_ASSERT(rdma_req->req.xfer == SPDK_NVME_DATA_HOST_TO_CONTROLLER); 575 STAILQ_INIT(&poller.qpairs_pending_send); 576 /* READY_TO_EXECUTE -> EXECUTING */ 577 rdma_req->state = RDMA_REQUEST_STATE_READY_TO_EXECUTE; 578 progress = nvmf_rdma_request_process(&rtransport, rdma_req); 579 CU_ASSERT(progress == true); 580 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_EXECUTING); 581 /* EXECUTED -> COMPLETING */ 582 rdma_req->state = RDMA_REQUEST_STATE_EXECUTED; 583 progress = nvmf_rdma_request_process(&rtransport, rdma_req); 584 CU_ASSERT(progress == true); 585 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_COMPLETING); 586 CU_ASSERT(rdma_req->recv == NULL); 587 /* COMPLETED -> FREE */ 588 rdma_req->state = RDMA_REQUEST_STATE_COMPLETED; 589 progress = nvmf_rdma_request_process(&rtransport, rdma_req); 590 CU_ASSERT(progress == true); 591 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_FREE); 592 593 free_recv(rdma_recv); 594 free_req(rdma_req); 595 poller_reset(&poller, &group); 596 qpair_reset(&rqpair, &poller, &device, &resources, &rtransport.transport); 597 598 /* Test 3: WRITE+WRITE ibv_send batching */ 599 { 600 struct spdk_nvmf_rdma_recv *recv1, *recv2; 601 struct spdk_nvmf_rdma_request *req1, *req2; 602 recv1 = create_recv(&rqpair, SPDK_NVME_OPC_WRITE); 603 req1 = create_req(&rqpair, recv1); 604 recv2 = create_recv(&rqpair, SPDK_NVME_OPC_WRITE); 605 req2 = create_req(&rqpair, recv2); 606 607 /* WRITE 1: NEW -> TRANSFERRING_H2C */ 608 rqpair.current_recv_depth = 1; 609 nvmf_rdma_request_process(&rtransport, req1); 610 CU_ASSERT(req1->state == RDMA_REQUEST_STATE_TRANSFERRING_HOST_TO_CONTROLLER); 611 612 /* WRITE 2: NEW -> TRANSFERRING_H2C */ 613 rqpair.current_recv_depth = 2; 614 nvmf_rdma_request_process(&rtransport, req2); 615 CU_ASSERT(req2->state == RDMA_REQUEST_STATE_TRANSFERRING_HOST_TO_CONTROLLER); 616 617 STAILQ_INIT(&poller.qpairs_pending_send); 618 619 /* WRITE 1 completes before WRITE 2 has finished RDMA reading */ 620 /* WRITE 1: READY_TO_EXECUTE -> EXECUTING */ 621 req1->state = RDMA_REQUEST_STATE_READY_TO_EXECUTE; 622 nvmf_rdma_request_process(&rtransport, req1); 623 CU_ASSERT(req1->state == RDMA_REQUEST_STATE_EXECUTING); 624 /* WRITE 1: EXECUTED -> COMPLETING */ 625 req1->state = RDMA_REQUEST_STATE_EXECUTED; 626 nvmf_rdma_request_process(&rtransport, req1); 627 CU_ASSERT(req1->state == RDMA_REQUEST_STATE_COMPLETING); 628 STAILQ_INIT(&poller.qpairs_pending_send); 629 /* WRITE 1: COMPLETED -> FREE */ 630 req1->state = RDMA_REQUEST_STATE_COMPLETED; 631 nvmf_rdma_request_process(&rtransport, req1); 632 CU_ASSERT(req1->state == RDMA_REQUEST_STATE_FREE); 633 634 /* Now WRITE 2 has finished reading and completes */ 635 /* WRITE 2: COMPLETED -> FREE */ 636 /* WRITE 2: READY_TO_EXECUTE -> EXECUTING */ 637 req2->state = RDMA_REQUEST_STATE_READY_TO_EXECUTE; 638 nvmf_rdma_request_process(&rtransport, req2); 639 CU_ASSERT(req2->state == RDMA_REQUEST_STATE_EXECUTING); 640 /* WRITE 1: EXECUTED -> COMPLETING */ 641 req2->state = RDMA_REQUEST_STATE_EXECUTED; 642 nvmf_rdma_request_process(&rtransport, req2); 643 CU_ASSERT(req2->state == RDMA_REQUEST_STATE_COMPLETING); 644 STAILQ_INIT(&poller.qpairs_pending_send); 645 /* WRITE 1: COMPLETED -> FREE */ 646 req2->state = RDMA_REQUEST_STATE_COMPLETED; 647 nvmf_rdma_request_process(&rtransport, req2); 648 CU_ASSERT(req2->state == RDMA_REQUEST_STATE_FREE); 649 650 free_recv(recv1); 651 free_req(req1); 652 free_recv(recv2); 653 free_req(req2); 654 poller_reset(&poller, &group); 655 qpair_reset(&rqpair, &poller, &device, &resources, &rtransport.transport); 656 } 657 658 /* Test 4, invalid command, check xfer type */ 659 { 660 struct spdk_nvmf_rdma_recv *rdma_recv_inv; 661 struct spdk_nvmf_rdma_request *rdma_req_inv; 662 /* construct an opcode that specifies BIDIRECTIONAL transfer */ 663 uint8_t opc = 0x10 | SPDK_NVME_DATA_BIDIRECTIONAL; 664 665 rdma_recv_inv = create_recv(&rqpair, opc); 666 rdma_req_inv = create_req(&rqpair, rdma_recv_inv); 667 668 /* NEW -> RDMA_REQUEST_STATE_COMPLETING */ 669 rqpair.current_recv_depth = 1; 670 progress = nvmf_rdma_request_process(&rtransport, rdma_req_inv); 671 CU_ASSERT(progress == true); 672 CU_ASSERT(rdma_req_inv->state == RDMA_REQUEST_STATE_COMPLETING); 673 CU_ASSERT(rdma_req_inv->req.rsp->nvme_cpl.status.sct == SPDK_NVME_SCT_GENERIC); 674 CU_ASSERT(rdma_req_inv->req.rsp->nvme_cpl.status.sc == SPDK_NVME_SC_INVALID_OPCODE); 675 676 /* RDMA_REQUEST_STATE_COMPLETED -> FREE */ 677 rdma_req_inv->state = RDMA_REQUEST_STATE_COMPLETED; 678 nvmf_rdma_request_process(&rtransport, rdma_req_inv); 679 CU_ASSERT(rdma_req_inv->state == RDMA_REQUEST_STATE_FREE); 680 681 free_recv(rdma_recv_inv); 682 free_req(rdma_req_inv); 683 poller_reset(&poller, &group); 684 qpair_reset(&rqpair, &poller, &device, &resources, &rtransport.transport); 685 } 686 687 spdk_mempool_free(rtransport.data_wr_pool); 688 } 689 690 #define TEST_GROUPS_COUNT 5 691 static void 692 test_nvmf_rdma_get_optimal_poll_group(void) 693 { 694 struct spdk_nvmf_rdma_transport rtransport = {}; 695 struct spdk_nvmf_transport *transport = &rtransport.transport; 696 struct spdk_nvmf_rdma_qpair rqpair = {}; 697 struct spdk_nvmf_transport_poll_group *groups[TEST_GROUPS_COUNT]; 698 struct spdk_nvmf_rdma_poll_group *rgroups[TEST_GROUPS_COUNT]; 699 struct spdk_nvmf_transport_poll_group *result; 700 struct spdk_nvmf_poll_group group = {}; 701 uint32_t i; 702 703 rqpair.qpair.transport = transport; 704 TAILQ_INIT(&rtransport.poll_groups); 705 706 for (i = 0; i < TEST_GROUPS_COUNT; i++) { 707 groups[i] = nvmf_rdma_poll_group_create(transport, NULL); 708 CU_ASSERT(groups[i] != NULL); 709 groups[i]->group = &group; 710 rgroups[i] = SPDK_CONTAINEROF(groups[i], struct spdk_nvmf_rdma_poll_group, group); 711 groups[i]->transport = transport; 712 } 713 CU_ASSERT(rtransport.conn_sched.next_admin_pg == rgroups[0]); 714 CU_ASSERT(rtransport.conn_sched.next_io_pg == rgroups[0]); 715 716 /* Emulate connection of %TEST_GROUPS_COUNT% initiators - each creates 1 admin and 1 io qp */ 717 for (i = 0; i < TEST_GROUPS_COUNT; i++) { 718 rqpair.qpair.qid = 0; 719 result = nvmf_rdma_get_optimal_poll_group(&rqpair.qpair); 720 CU_ASSERT(result == groups[i]); 721 CU_ASSERT(rtransport.conn_sched.next_admin_pg == rgroups[(i + 1) % TEST_GROUPS_COUNT]); 722 CU_ASSERT(rtransport.conn_sched.next_io_pg == rgroups[i]); 723 724 rqpair.qpair.qid = 1; 725 result = nvmf_rdma_get_optimal_poll_group(&rqpair.qpair); 726 CU_ASSERT(result == groups[i]); 727 CU_ASSERT(rtransport.conn_sched.next_admin_pg == rgroups[(i + 1) % TEST_GROUPS_COUNT]); 728 CU_ASSERT(rtransport.conn_sched.next_io_pg == rgroups[(i + 1) % TEST_GROUPS_COUNT]); 729 } 730 /* wrap around, admin/io pg point to the first pg 731 Destroy all poll groups except of the last one */ 732 for (i = 0; i < TEST_GROUPS_COUNT - 1; i++) { 733 nvmf_rdma_poll_group_destroy(groups[i]); 734 CU_ASSERT(rtransport.conn_sched.next_admin_pg == rgroups[i + 1]); 735 CU_ASSERT(rtransport.conn_sched.next_io_pg == rgroups[i + 1]); 736 } 737 738 CU_ASSERT(rtransport.conn_sched.next_admin_pg == rgroups[TEST_GROUPS_COUNT - 1]); 739 CU_ASSERT(rtransport.conn_sched.next_io_pg == rgroups[TEST_GROUPS_COUNT - 1]); 740 741 /* Check that pointers to the next admin/io poll groups are not changed */ 742 rqpair.qpair.qid = 0; 743 result = nvmf_rdma_get_optimal_poll_group(&rqpair.qpair); 744 CU_ASSERT(result == groups[TEST_GROUPS_COUNT - 1]); 745 CU_ASSERT(rtransport.conn_sched.next_admin_pg == rgroups[TEST_GROUPS_COUNT - 1]); 746 CU_ASSERT(rtransport.conn_sched.next_io_pg == rgroups[TEST_GROUPS_COUNT - 1]); 747 748 rqpair.qpair.qid = 1; 749 result = nvmf_rdma_get_optimal_poll_group(&rqpair.qpair); 750 CU_ASSERT(result == groups[TEST_GROUPS_COUNT - 1]); 751 CU_ASSERT(rtransport.conn_sched.next_admin_pg == rgroups[TEST_GROUPS_COUNT - 1]); 752 CU_ASSERT(rtransport.conn_sched.next_io_pg == rgroups[TEST_GROUPS_COUNT - 1]); 753 754 /* Remove the last poll group, check that pointers are NULL */ 755 nvmf_rdma_poll_group_destroy(groups[TEST_GROUPS_COUNT - 1]); 756 CU_ASSERT(rtransport.conn_sched.next_admin_pg == NULL); 757 CU_ASSERT(rtransport.conn_sched.next_io_pg == NULL); 758 759 /* Request optimal poll group, result must be NULL */ 760 rqpair.qpair.qid = 0; 761 result = nvmf_rdma_get_optimal_poll_group(&rqpair.qpair); 762 CU_ASSERT(result == NULL); 763 764 rqpair.qpair.qid = 1; 765 result = nvmf_rdma_get_optimal_poll_group(&rqpair.qpair); 766 CU_ASSERT(result == NULL); 767 } 768 #undef TEST_GROUPS_COUNT 769 770 static void 771 test_spdk_nvmf_rdma_request_parse_sgl_with_md(void) 772 { 773 struct spdk_nvmf_rdma_transport rtransport; 774 struct spdk_nvmf_rdma_device device; 775 struct spdk_nvmf_rdma_request rdma_req = {}; 776 struct spdk_nvmf_rdma_recv recv; 777 struct spdk_nvmf_rdma_poll_group group; 778 struct spdk_nvmf_rdma_qpair rqpair; 779 struct spdk_nvmf_rdma_poller poller; 780 union nvmf_c2h_msg cpl; 781 union nvmf_h2c_msg cmd; 782 struct spdk_nvme_sgl_descriptor *sgl; 783 struct spdk_nvme_sgl_descriptor sgl_desc[SPDK_NVMF_MAX_SGL_ENTRIES] = {{0}}; 784 char data_buffer[8192]; 785 struct spdk_nvmf_rdma_request_data *data = (struct spdk_nvmf_rdma_request_data *)data_buffer; 786 char data2_buffer[8192]; 787 struct spdk_nvmf_rdma_request_data *data2 = (struct spdk_nvmf_rdma_request_data *)data2_buffer; 788 const uint32_t data_bs = 512; 789 const uint32_t md_size = 8; 790 int rc, i; 791 struct spdk_dif_ctx_init_ext_opts dif_opts; 792 793 MOCK_CLEAR(spdk_mempool_get); 794 MOCK_CLEAR(spdk_iobuf_get); 795 796 data->wr.sg_list = data->sgl; 797 group.group.transport = &rtransport.transport; 798 poller.group = &group; 799 rqpair.poller = &poller; 800 rqpair.max_send_sge = SPDK_NVMF_MAX_SGL_ENTRIES; 801 802 sgl = &cmd.nvme_cmd.dptr.sgl1; 803 rdma_req.recv = &recv; 804 rdma_req.req.cmd = &cmd; 805 rdma_req.req.rsp = &cpl; 806 rdma_req.data.wr.sg_list = rdma_req.data.sgl; 807 rdma_req.req.qpair = &rqpair.qpair; 808 rdma_req.req.xfer = SPDK_NVME_DATA_CONTROLLER_TO_HOST; 809 810 rtransport.transport.opts = g_rdma_ut_transport_opts; 811 rtransport.data_wr_pool = NULL; 812 813 device.attr.device_cap_flags = 0; 814 device.map = NULL; 815 sgl->keyed.key = 0xEEEE; 816 sgl->address = 0xFFFF; 817 rdma_req.recv->buf = (void *)0xDDDD; 818 819 /* Test 1: sgl type: keyed data block subtype: address */ 820 sgl->generic.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; 821 sgl->keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; 822 823 /* Part 1: simple I/O, one SGL smaller than the transport io unit size, block size 512 */ 824 MOCK_SET(spdk_iobuf_get, (void *)0x2000); 825 reset_nvmf_rdma_request(&rdma_req); 826 dif_opts.size = SPDK_SIZEOF(&dif_opts, dif_pi_format); 827 dif_opts.dif_pi_format = SPDK_DIF_PI_FORMAT_16; 828 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 829 SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 830 0, 0, 0, 0, 0, &dif_opts); 831 rdma_req.req.dif_enabled = true; 832 rtransport.transport.opts.io_unit_size = data_bs * 8; 833 rdma_req.req.qpair->transport = &rtransport.transport; 834 sgl->keyed.length = data_bs * 4; 835 836 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 837 838 CU_ASSERT(rc == 0); 839 CU_ASSERT(rdma_req.req.data_from_pool == true); 840 CU_ASSERT(rdma_req.req.length == data_bs * 4); 841 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 842 CU_ASSERT(rdma_req.req.dif.elba_length == (data_bs + md_size) * 4); 843 CU_ASSERT(rdma_req.req.iovcnt == 1); 844 CU_ASSERT((uint64_t)rdma_req.req.iov[0].iov_base == 0x2000); 845 CU_ASSERT(rdma_req.data.wr.num_sge == 1); 846 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 847 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 848 CU_ASSERT((uint64_t)rdma_req.req.iov[0].iov_base == 0x2000); 849 850 CU_ASSERT(rdma_req.data.wr.sg_list[0].addr == 0x2000); 851 CU_ASSERT(rdma_req.data.wr.sg_list[0].length == rdma_req.req.length); 852 CU_ASSERT(rdma_req.data.wr.sg_list[0].lkey == RDMA_UT_LKEY); 853 854 /* Part 2: simple I/O, one SGL equal to io unit size, io_unit_size is not aligned with md_size, 855 block size 512 */ 856 MOCK_SET(spdk_iobuf_get, (void *)0x2000); 857 reset_nvmf_rdma_request(&rdma_req); 858 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 859 SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 860 0, 0, 0, 0, 0, &dif_opts); 861 rdma_req.req.dif_enabled = true; 862 rtransport.transport.opts.io_unit_size = data_bs * 4; 863 sgl->keyed.length = data_bs * 4; 864 865 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 866 867 CU_ASSERT(rc == 0); 868 CU_ASSERT(rdma_req.req.data_from_pool == true); 869 CU_ASSERT(rdma_req.req.length == data_bs * 4); 870 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 871 CU_ASSERT(rdma_req.req.dif.elba_length == (data_bs + md_size) * 4); 872 CU_ASSERT(rdma_req.req.iovcnt == 2); 873 CU_ASSERT((uint64_t)rdma_req.req.iov[0].iov_base == 0x2000); 874 CU_ASSERT(rdma_req.data.wr.num_sge == 5); 875 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 876 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 877 CU_ASSERT((uint64_t)rdma_req.req.iov[0].iov_base == 0x2000); 878 879 for (i = 0; i < 3; ++i) { 880 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == 0x2000 + i * (data_bs + md_size)); 881 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == data_bs); 882 CU_ASSERT(rdma_req.data.wr.sg_list[i].lkey == RDMA_UT_LKEY); 883 } 884 CU_ASSERT(rdma_req.data.wr.sg_list[3].addr == 0x2000 + 3 * (data_bs + md_size)); 885 CU_ASSERT(rdma_req.data.wr.sg_list[3].length == 488); 886 CU_ASSERT(rdma_req.data.wr.sg_list[3].lkey == RDMA_UT_LKEY); 887 888 /* 2nd buffer consumed */ 889 CU_ASSERT(rdma_req.data.wr.sg_list[4].addr == 0x2000); 890 CU_ASSERT(rdma_req.data.wr.sg_list[4].length == 24); 891 CU_ASSERT(rdma_req.data.wr.sg_list[4].lkey == RDMA_UT_LKEY); 892 893 /* Part 3: simple I/O, one SGL equal io unit size, io_unit_size is equal to block size 512 bytes */ 894 MOCK_SET(spdk_iobuf_get, (void *)0x2000); 895 reset_nvmf_rdma_request(&rdma_req); 896 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 897 SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 898 0, 0, 0, 0, 0, &dif_opts); 899 rdma_req.req.dif_enabled = true; 900 rtransport.transport.opts.io_unit_size = data_bs; 901 sgl->keyed.length = data_bs; 902 903 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 904 905 CU_ASSERT(rc == 0); 906 CU_ASSERT(rdma_req.req.data_from_pool == true); 907 CU_ASSERT(rdma_req.req.length == data_bs); 908 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 909 CU_ASSERT(rdma_req.req.dif.elba_length == data_bs + md_size); 910 CU_ASSERT(rdma_req.req.iovcnt == 2); 911 CU_ASSERT((uint64_t)rdma_req.req.iov[0].iov_base == 0x2000); 912 CU_ASSERT(rdma_req.data.wr.num_sge == 1); 913 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 914 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 915 CU_ASSERT((uint64_t)rdma_req.req.iov[0].iov_base == 0x2000); 916 917 CU_ASSERT(rdma_req.data.wr.sg_list[0].addr == 0x2000); 918 CU_ASSERT(rdma_req.data.wr.sg_list[0].length == data_bs); 919 CU_ASSERT(rdma_req.data.wr.sg_list[0].lkey == RDMA_UT_LKEY); 920 921 CU_ASSERT(rdma_req.req.iovcnt == 2); 922 CU_ASSERT(rdma_req.req.iov[0].iov_base == (void *)((unsigned long)0x2000)); 923 CU_ASSERT(rdma_req.req.iov[0].iov_len == data_bs); 924 /* 2nd buffer consumed for metadata */ 925 CU_ASSERT(rdma_req.req.iov[1].iov_base == (void *)((unsigned long)0x2000)); 926 CU_ASSERT(rdma_req.req.iov[1].iov_len == md_size); 927 928 /* Part 4: simple I/O, one SGL equal io unit size, io_unit_size is aligned with md_size, 929 block size 512 */ 930 MOCK_SET(spdk_iobuf_get, (void *)0x2000); 931 reset_nvmf_rdma_request(&rdma_req); 932 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 933 SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 934 0, 0, 0, 0, 0, &dif_opts); 935 rdma_req.req.dif_enabled = true; 936 rtransport.transport.opts.io_unit_size = (data_bs + md_size) * 4; 937 sgl->keyed.length = data_bs * 4; 938 939 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 940 941 CU_ASSERT(rc == 0); 942 CU_ASSERT(rdma_req.req.data_from_pool == true); 943 CU_ASSERT(rdma_req.req.length == data_bs * 4); 944 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 945 CU_ASSERT(rdma_req.req.dif.elba_length == (data_bs + md_size) * 4); 946 CU_ASSERT(rdma_req.req.iovcnt == 1); 947 CU_ASSERT((uint64_t)rdma_req.req.iov[0].iov_base == 0x2000); 948 CU_ASSERT(rdma_req.data.wr.num_sge == 1); 949 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 950 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 951 CU_ASSERT((uint64_t)rdma_req.req.iov[0].iov_base == 0x2000); 952 953 CU_ASSERT(rdma_req.data.wr.sg_list[0].addr == 0x2000); 954 CU_ASSERT(rdma_req.data.wr.sg_list[0].length == rdma_req.req.length); 955 CU_ASSERT(rdma_req.data.wr.sg_list[0].lkey == RDMA_UT_LKEY); 956 957 /* Part 5: simple I/O, one SGL equal to 2x io unit size, io_unit_size is aligned with md_size, 958 block size 512 */ 959 MOCK_SET(spdk_iobuf_get, (void *)0x2000); 960 reset_nvmf_rdma_request(&rdma_req); 961 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 962 SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 963 0, 0, 0, 0, 0, &dif_opts); 964 rdma_req.req.dif_enabled = true; 965 rtransport.transport.opts.io_unit_size = (data_bs + md_size) * 2; 966 sgl->keyed.length = data_bs * 4; 967 968 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 969 970 CU_ASSERT(rc == 0); 971 CU_ASSERT(rdma_req.req.data_from_pool == true); 972 CU_ASSERT(rdma_req.req.length == data_bs * 4); 973 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 974 CU_ASSERT(rdma_req.req.dif.elba_length == (data_bs + md_size) * 4); 975 CU_ASSERT(rdma_req.req.iovcnt == 2); 976 CU_ASSERT((uint64_t)rdma_req.req.iov[0].iov_base == 0x2000); 977 CU_ASSERT(rdma_req.data.wr.num_sge == 2); 978 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 979 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 980 CU_ASSERT((uint64_t)rdma_req.req.iov[0].iov_base == 0x2000); 981 982 for (i = 0; i < 2; ++i) { 983 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == 0x2000); 984 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == data_bs * 2); 985 } 986 987 /* Part 6: simple I/O, one SGL larger than the transport io unit size, io_unit_size is not aligned to md_size, 988 block size 512 */ 989 MOCK_SET(spdk_iobuf_get, (void *)0x2000); 990 reset_nvmf_rdma_request(&rdma_req); 991 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 992 SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 993 0, 0, 0, 0, 0, &dif_opts); 994 rdma_req.req.dif_enabled = true; 995 rtransport.transport.opts.io_unit_size = data_bs * 4; 996 sgl->keyed.length = data_bs * 6; 997 998 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 999 1000 CU_ASSERT(rc == 0); 1001 CU_ASSERT(rdma_req.req.data_from_pool == true); 1002 CU_ASSERT(rdma_req.req.length == data_bs * 6); 1003 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 1004 CU_ASSERT(rdma_req.req.dif.elba_length == (data_bs + md_size) * 6); 1005 CU_ASSERT(rdma_req.req.iovcnt == 2); 1006 CU_ASSERT((uint64_t)rdma_req.req.iov[0].iov_base == 0x2000); 1007 CU_ASSERT(rdma_req.data.wr.num_sge == 7); 1008 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 1009 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 1010 CU_ASSERT((uint64_t)rdma_req.req.iov[0].iov_base == 0x2000); 1011 1012 for (i = 0; i < 3; ++i) { 1013 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == 0x2000 + i * (data_bs + md_size)); 1014 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == data_bs); 1015 CU_ASSERT(rdma_req.data.wr.sg_list[i].lkey == RDMA_UT_LKEY); 1016 } 1017 CU_ASSERT(rdma_req.data.wr.sg_list[3].addr == 0x2000 + 3 * (data_bs + md_size)); 1018 CU_ASSERT(rdma_req.data.wr.sg_list[3].length == 488); 1019 CU_ASSERT(rdma_req.data.wr.sg_list[3].lkey == RDMA_UT_LKEY); 1020 1021 /* 2nd IO buffer consumed */ 1022 CU_ASSERT(rdma_req.data.wr.sg_list[4].addr == 0x2000); 1023 CU_ASSERT(rdma_req.data.wr.sg_list[4].length == 24); 1024 CU_ASSERT(rdma_req.data.wr.sg_list[4].lkey == RDMA_UT_LKEY); 1025 1026 CU_ASSERT(rdma_req.data.wr.sg_list[5].addr == 0x2000 + 24 + md_size); 1027 CU_ASSERT(rdma_req.data.wr.sg_list[5].length == 512); 1028 CU_ASSERT(rdma_req.data.wr.sg_list[5].lkey == RDMA_UT_LKEY); 1029 1030 CU_ASSERT(rdma_req.data.wr.sg_list[6].addr == 0x2000 + 24 + 512 + md_size * 2); 1031 CU_ASSERT(rdma_req.data.wr.sg_list[6].length == 512); 1032 CU_ASSERT(rdma_req.data.wr.sg_list[6].lkey == RDMA_UT_LKEY); 1033 1034 /* Part 7: simple I/O, number of SGL entries exceeds the number of entries 1035 one WR can hold. Additional WR is chained */ 1036 MOCK_SET(spdk_iobuf_get, data2_buffer); 1037 MOCK_SET(spdk_mempool_get, data2_buffer); 1038 reset_nvmf_rdma_request(&rdma_req); 1039 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 1040 SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 1041 0, 0, 0, 0, 0, &dif_opts); 1042 rdma_req.req.dif_enabled = true; 1043 rtransport.transport.opts.io_unit_size = data_bs * 16; 1044 sgl->keyed.length = data_bs * 16; 1045 1046 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 1047 1048 CU_ASSERT(rc == 0); 1049 CU_ASSERT(rdma_req.req.data_from_pool == true); 1050 CU_ASSERT(rdma_req.req.length == data_bs * 16); 1051 CU_ASSERT(rdma_req.req.iovcnt == 2); 1052 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 1053 CU_ASSERT(rdma_req.req.dif.elba_length == (data_bs + md_size) * 16); 1054 CU_ASSERT(rdma_req.req.iov[0].iov_base == data2_buffer); 1055 CU_ASSERT(rdma_req.data.wr.num_sge == 16); 1056 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 1057 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 1058 1059 for (i = 0; i < 15; ++i) { 1060 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == (uintptr_t)data2_buffer + i * (data_bs + md_size)); 1061 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == data_bs); 1062 CU_ASSERT(rdma_req.data.wr.sg_list[i].lkey == RDMA_UT_LKEY); 1063 } 1064 1065 /* 8192 - (512 + 8) * 15 = 392 */ 1066 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == (uintptr_t)data2_buffer + i * (data_bs + md_size)); 1067 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == 392); 1068 CU_ASSERT(rdma_req.data.wr.sg_list[i].lkey == RDMA_UT_LKEY); 1069 1070 /* additional wr from pool */ 1071 CU_ASSERT(rdma_req.data.wr.next == (void *)&data2->wr); 1072 CU_ASSERT(rdma_req.data.wr.next->num_sge == 1); 1073 CU_ASSERT(rdma_req.data.wr.next->next == &rdma_req.rsp.wr); 1074 /* 2nd IO buffer */ 1075 CU_ASSERT(data2->wr.sg_list[0].addr == (uintptr_t)data2_buffer); 1076 CU_ASSERT(data2->wr.sg_list[0].length == 120); 1077 CU_ASSERT(data2->wr.sg_list[0].lkey == RDMA_UT_LKEY); 1078 1079 /* Part 8: simple I/O, data with metadata do not fit to 1 io_buffer */ 1080 MOCK_SET(spdk_iobuf_get, (void *)0x2000); 1081 reset_nvmf_rdma_request(&rdma_req); 1082 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 1083 SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 1084 0, 0, 0, 0, 0, &dif_opts); 1085 rdma_req.req.dif_enabled = true; 1086 rtransport.transport.opts.io_unit_size = 516; 1087 sgl->keyed.length = data_bs * 2; 1088 1089 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 1090 1091 CU_ASSERT(rc == 0); 1092 CU_ASSERT(rdma_req.req.data_from_pool == true); 1093 CU_ASSERT(rdma_req.req.length == data_bs * 2); 1094 CU_ASSERT(rdma_req.req.iovcnt == 3); 1095 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 1096 CU_ASSERT(rdma_req.req.dif.elba_length == (data_bs + md_size) * 2); 1097 CU_ASSERT(rdma_req.req.iov[0].iov_base == (void *)0x2000); 1098 CU_ASSERT(rdma_req.data.wr.num_sge == 2); 1099 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 1100 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 1101 1102 CU_ASSERT(rdma_req.data.wr.sg_list[0].addr == 0x2000); 1103 CU_ASSERT(rdma_req.data.wr.sg_list[0].length == 512); 1104 CU_ASSERT(rdma_req.data.wr.sg_list[0].lkey == RDMA_UT_LKEY); 1105 1106 /* 2nd IO buffer consumed, offset 4 bytes due to part of the metadata 1107 is located at the beginning of that buffer */ 1108 CU_ASSERT(rdma_req.data.wr.sg_list[1].addr == 0x2000 + 4); 1109 CU_ASSERT(rdma_req.data.wr.sg_list[1].length == 512); 1110 CU_ASSERT(rdma_req.data.wr.sg_list[1].lkey == RDMA_UT_LKEY); 1111 1112 /* Test 2: Multi SGL */ 1113 sgl->generic.type = SPDK_NVME_SGL_TYPE_LAST_SEGMENT; 1114 sgl->unkeyed.subtype = SPDK_NVME_SGL_SUBTYPE_OFFSET; 1115 sgl->address = 0; 1116 rdma_req.recv->buf = (void *)&sgl_desc; 1117 MOCK_SET(spdk_mempool_get, data_buffer); 1118 MOCK_SET(spdk_iobuf_get, data_buffer); 1119 1120 /* part 1: 2 segments each with 1 wr. io_unit_size is aligned with data_bs + md_size */ 1121 reset_nvmf_rdma_request(&rdma_req); 1122 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 1123 SPDK_DIF_TYPE1, 1124 SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 1125 0, 0, 0, 0, 0, &dif_opts); 1126 rdma_req.req.dif_enabled = true; 1127 rtransport.transport.opts.io_unit_size = (data_bs + md_size) * 4; 1128 sgl->unkeyed.length = 2 * sizeof(struct spdk_nvme_sgl_descriptor); 1129 1130 for (i = 0; i < 2; i++) { 1131 sgl_desc[i].keyed.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; 1132 sgl_desc[i].keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; 1133 sgl_desc[i].keyed.length = data_bs * 4; 1134 sgl_desc[i].address = 0x4000 + i * data_bs * 4; 1135 sgl_desc[i].keyed.key = 0x44; 1136 } 1137 1138 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 1139 1140 CU_ASSERT(rc == 0); 1141 CU_ASSERT(rdma_req.req.data_from_pool == true); 1142 CU_ASSERT(rdma_req.req.length == data_bs * 4 * 2); 1143 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 1144 CU_ASSERT(rdma_req.req.dif.elba_length == (data_bs + md_size) * 4 * 2); 1145 CU_ASSERT(rdma_req.data.wr.num_sge == 1); 1146 CU_ASSERT(rdma_req.data.wr.sg_list[0].addr == (uintptr_t)(data_buffer)); 1147 CU_ASSERT(rdma_req.data.wr.sg_list[0].length == data_bs * 4); 1148 1149 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0x44); 1150 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0x4000); 1151 CU_ASSERT(rdma_req.data.wr.next == &data->wr); 1152 CU_ASSERT(data->wr.wr.rdma.rkey == 0x44); 1153 CU_ASSERT(data->wr.wr.rdma.remote_addr == 0x4000 + data_bs * 4); 1154 CU_ASSERT(data->wr.num_sge == 1); 1155 CU_ASSERT(data->wr.sg_list[0].addr == (uintptr_t)(data_buffer)); 1156 CU_ASSERT(data->wr.sg_list[0].length == data_bs * 4); 1157 1158 CU_ASSERT(data->wr.next == &rdma_req.rsp.wr); 1159 reset_nvmf_rdma_request(&rdma_req); 1160 } 1161 1162 static void 1163 test_nvmf_rdma_opts_init(void) 1164 { 1165 struct spdk_nvmf_transport_opts opts = {}; 1166 1167 nvmf_rdma_opts_init(&opts); 1168 CU_ASSERT(opts.max_queue_depth == SPDK_NVMF_RDMA_DEFAULT_MAX_QUEUE_DEPTH); 1169 CU_ASSERT(opts.max_qpairs_per_ctrlr == SPDK_NVMF_RDMA_DEFAULT_MAX_QPAIRS_PER_CTRLR); 1170 CU_ASSERT(opts.in_capsule_data_size == SPDK_NVMF_RDMA_DEFAULT_IN_CAPSULE_DATA_SIZE); 1171 CU_ASSERT(opts.max_io_size == SPDK_NVMF_RDMA_DEFAULT_MAX_IO_SIZE); 1172 CU_ASSERT(opts.io_unit_size == SPDK_NVMF_RDMA_MIN_IO_BUFFER_SIZE); 1173 CU_ASSERT(opts.max_aq_depth == SPDK_NVMF_RDMA_DEFAULT_AQ_DEPTH); 1174 CU_ASSERT(opts.num_shared_buffers == SPDK_NVMF_RDMA_DEFAULT_NUM_SHARED_BUFFERS); 1175 CU_ASSERT(opts.buf_cache_size == SPDK_NVMF_RDMA_DEFAULT_BUFFER_CACHE_SIZE); 1176 CU_ASSERT(opts.dif_insert_or_strip == SPDK_NVMF_RDMA_DIF_INSERT_OR_STRIP); 1177 CU_ASSERT(opts.abort_timeout_sec == SPDK_NVMF_RDMA_DEFAULT_ABORT_TIMEOUT_SEC); 1178 CU_ASSERT(opts.transport_specific == NULL); 1179 } 1180 1181 static void 1182 test_nvmf_rdma_request_free_data(void) 1183 { 1184 struct spdk_nvmf_rdma_request rdma_req = {}; 1185 struct spdk_nvmf_rdma_transport rtransport = {}; 1186 struct spdk_nvmf_rdma_request_data *next_request_data = NULL; 1187 1188 MOCK_CLEAR(spdk_mempool_get); 1189 rtransport.data_wr_pool = spdk_mempool_create("spdk_nvmf_rdma_wr_data", 1190 SPDK_NVMF_MAX_SGL_ENTRIES, 1191 sizeof(struct spdk_nvmf_rdma_request_data), 1192 SPDK_MEMPOOL_DEFAULT_CACHE_SIZE, 1193 SPDK_ENV_SOCKET_ID_ANY); 1194 next_request_data = spdk_mempool_get(rtransport.data_wr_pool); 1195 SPDK_CU_ASSERT_FATAL(((struct test_mempool *)rtransport.data_wr_pool)->count == 1196 SPDK_NVMF_MAX_SGL_ENTRIES - 1); 1197 next_request_data->wr.wr_id = 1; 1198 next_request_data->wr.num_sge = 2; 1199 next_request_data->wr.next = NULL; 1200 rdma_req.data.wr.next = &next_request_data->wr; 1201 rdma_req.data.wr.wr_id = 1; 1202 rdma_req.data.wr.num_sge = 2; 1203 1204 nvmf_rdma_request_free_data(&rdma_req, &rtransport); 1205 /* Check if next_request_data put into memory pool */ 1206 CU_ASSERT(((struct test_mempool *)rtransport.data_wr_pool)->count == SPDK_NVMF_MAX_SGL_ENTRIES); 1207 CU_ASSERT(rdma_req.data.wr.num_sge == 0); 1208 1209 spdk_mempool_free(rtransport.data_wr_pool); 1210 } 1211 1212 static void 1213 test_nvmf_rdma_update_ibv_state(void) 1214 { 1215 struct spdk_nvmf_rdma_qpair rqpair = {}; 1216 struct spdk_rdma_qp rdma_qp = {}; 1217 struct ibv_qp qp = {}; 1218 int rc = 0; 1219 1220 rqpair.rdma_qp = &rdma_qp; 1221 1222 /* Case 1: Failed to get updated RDMA queue pair state */ 1223 rqpair.ibv_state = IBV_QPS_INIT; 1224 rqpair.rdma_qp->qp = NULL; 1225 1226 rc = nvmf_rdma_update_ibv_state(&rqpair); 1227 CU_ASSERT(rc == IBV_QPS_ERR + 1); 1228 1229 /* Case 2: Bad state updated */ 1230 rqpair.rdma_qp->qp = &qp; 1231 qp.state = IBV_QPS_ERR; 1232 rc = nvmf_rdma_update_ibv_state(&rqpair); 1233 CU_ASSERT(rqpair.ibv_state == 10); 1234 CU_ASSERT(rc == IBV_QPS_ERR + 1); 1235 1236 /* Case 3: Pass */ 1237 qp.state = IBV_QPS_INIT; 1238 rc = nvmf_rdma_update_ibv_state(&rqpair); 1239 CU_ASSERT(rqpair.ibv_state == IBV_QPS_INIT); 1240 CU_ASSERT(rc == IBV_QPS_INIT); 1241 } 1242 1243 static void 1244 test_nvmf_rdma_resources_create(void) 1245 { 1246 static struct spdk_nvmf_rdma_resources *rdma_resource; 1247 struct spdk_nvmf_rdma_resource_opts opts = {}; 1248 struct spdk_nvmf_rdma_qpair qpair = {}; 1249 struct spdk_nvmf_rdma_recv *recv = NULL; 1250 struct spdk_nvmf_rdma_request *req = NULL; 1251 const int DEPTH = 128; 1252 1253 opts.max_queue_depth = DEPTH; 1254 opts.in_capsule_data_size = 4096; 1255 opts.shared = true; 1256 opts.qpair = &qpair; 1257 1258 rdma_resource = nvmf_rdma_resources_create(&opts); 1259 CU_ASSERT(rdma_resource != NULL); 1260 /* Just check first and last entry */ 1261 recv = &rdma_resource->recvs[0]; 1262 req = &rdma_resource->reqs[0]; 1263 CU_ASSERT(recv->rdma_wr.type == RDMA_WR_TYPE_RECV); 1264 CU_ASSERT((uintptr_t)recv->buf == (uintptr_t)(rdma_resource->bufs)); 1265 CU_ASSERT(recv->sgl[0].addr == (uintptr_t)&rdma_resource->cmds[0]); 1266 CU_ASSERT(recv->sgl[0].length == sizeof(rdma_resource->cmds[0])); 1267 CU_ASSERT(recv->sgl[0].lkey == RDMA_UT_LKEY); 1268 CU_ASSERT(recv->wr.num_sge == 2); 1269 CU_ASSERT(recv->wr.wr_id == (uintptr_t)&rdma_resource->recvs[0].rdma_wr); 1270 CU_ASSERT(recv->wr.sg_list == rdma_resource->recvs[0].sgl); 1271 CU_ASSERT(req->req.rsp == &rdma_resource->cpls[0]); 1272 CU_ASSERT(req->rsp.sgl[0].addr == (uintptr_t)&rdma_resource->cpls[0]); 1273 CU_ASSERT(req->rsp.sgl[0].length == sizeof(rdma_resource->cpls[0])); 1274 CU_ASSERT(req->rsp.sgl[0].lkey == RDMA_UT_LKEY); 1275 CU_ASSERT(req->rsp.rdma_wr.type == RDMA_WR_TYPE_SEND); 1276 CU_ASSERT(req->rsp.wr.wr_id == (uintptr_t)&rdma_resource->reqs[0].rsp.rdma_wr); 1277 CU_ASSERT(req->rsp.wr.next == NULL); 1278 CU_ASSERT(req->rsp.wr.opcode == IBV_WR_SEND); 1279 CU_ASSERT(req->rsp.wr.send_flags == IBV_SEND_SIGNALED); 1280 CU_ASSERT(req->rsp.wr.sg_list == rdma_resource->reqs[0].rsp.sgl); 1281 CU_ASSERT(req->rsp.wr.num_sge == NVMF_DEFAULT_RSP_SGE); 1282 CU_ASSERT(req->data.rdma_wr.type == RDMA_WR_TYPE_DATA); 1283 CU_ASSERT(req->data.wr.wr_id == (uintptr_t)&rdma_resource->reqs[0].data.rdma_wr); 1284 CU_ASSERT(req->data.wr.next == NULL); 1285 CU_ASSERT(req->data.wr.send_flags == IBV_SEND_SIGNALED); 1286 CU_ASSERT(req->data.wr.sg_list == rdma_resource->reqs[0].data.sgl); 1287 CU_ASSERT(req->data.wr.num_sge == SPDK_NVMF_MAX_SGL_ENTRIES); 1288 CU_ASSERT(req->state == RDMA_REQUEST_STATE_FREE); 1289 1290 recv = &rdma_resource->recvs[DEPTH - 1]; 1291 req = &rdma_resource->reqs[DEPTH - 1]; 1292 CU_ASSERT(recv->rdma_wr.type == RDMA_WR_TYPE_RECV); 1293 CU_ASSERT((uintptr_t)recv->buf == (uintptr_t)(rdma_resource->bufs + 1294 (DEPTH - 1) * 4096)); 1295 CU_ASSERT(recv->sgl[0].addr == (uintptr_t)&rdma_resource->cmds[DEPTH - 1]); 1296 CU_ASSERT(recv->sgl[0].length == sizeof(rdma_resource->cmds[DEPTH - 1])); 1297 CU_ASSERT(recv->sgl[0].lkey == RDMA_UT_LKEY); 1298 CU_ASSERT(recv->wr.num_sge == 2); 1299 CU_ASSERT(recv->wr.wr_id == (uintptr_t)&rdma_resource->recvs[DEPTH - 1].rdma_wr); 1300 CU_ASSERT(recv->wr.sg_list == rdma_resource->recvs[DEPTH - 1].sgl); 1301 CU_ASSERT(req->req.rsp == &rdma_resource->cpls[DEPTH - 1]); 1302 CU_ASSERT(req->rsp.sgl[0].addr == (uintptr_t)&rdma_resource->cpls[DEPTH - 1]); 1303 CU_ASSERT(req->rsp.sgl[0].length == sizeof(rdma_resource->cpls[DEPTH - 1])); 1304 CU_ASSERT(req->rsp.sgl[0].lkey == RDMA_UT_LKEY); 1305 CU_ASSERT(req->rsp.rdma_wr.type == RDMA_WR_TYPE_SEND); 1306 CU_ASSERT(req->rsp.wr.wr_id == (uintptr_t) 1307 &req->rsp.rdma_wr); 1308 CU_ASSERT(req->rsp.wr.next == NULL); 1309 CU_ASSERT(req->rsp.wr.opcode == IBV_WR_SEND); 1310 CU_ASSERT(req->rsp.wr.send_flags == IBV_SEND_SIGNALED); 1311 CU_ASSERT(req->rsp.wr.sg_list == rdma_resource->reqs[DEPTH - 1].rsp.sgl); 1312 CU_ASSERT(req->rsp.wr.num_sge == NVMF_DEFAULT_RSP_SGE); 1313 CU_ASSERT(req->data.rdma_wr.type == RDMA_WR_TYPE_DATA); 1314 CU_ASSERT(req->data.wr.wr_id == (uintptr_t) 1315 &req->data.rdma_wr); 1316 CU_ASSERT(req->data.wr.next == NULL); 1317 CU_ASSERT(req->data.wr.send_flags == IBV_SEND_SIGNALED); 1318 CU_ASSERT(req->data.wr.sg_list == rdma_resource->reqs[DEPTH - 1].data.sgl); 1319 CU_ASSERT(req->data.wr.num_sge == SPDK_NVMF_MAX_SGL_ENTRIES); 1320 CU_ASSERT(req->state == RDMA_REQUEST_STATE_FREE); 1321 1322 nvmf_rdma_resources_destroy(rdma_resource); 1323 } 1324 1325 static void 1326 test_nvmf_rdma_qpair_compare(void) 1327 { 1328 struct spdk_nvmf_rdma_qpair rqpair1 = {}, rqpair2 = {}; 1329 1330 rqpair1.qp_num = 0; 1331 rqpair2.qp_num = UINT32_MAX; 1332 1333 CU_ASSERT(nvmf_rdma_qpair_compare(&rqpair1, &rqpair2) < 0); 1334 CU_ASSERT(nvmf_rdma_qpair_compare(&rqpair2, &rqpair1) > 0); 1335 } 1336 1337 static void 1338 test_nvmf_rdma_resize_cq(void) 1339 { 1340 int rc = -1; 1341 int tnum_wr = 0; 1342 int tnum_cqe = 0; 1343 struct spdk_nvmf_rdma_qpair rqpair = {}; 1344 struct spdk_nvmf_rdma_poller rpoller = {}; 1345 struct spdk_nvmf_rdma_device rdevice = {}; 1346 struct ibv_context ircontext = {}; 1347 struct ibv_device idevice = {}; 1348 1349 rdevice.context = &ircontext; 1350 rqpair.poller = &rpoller; 1351 ircontext.device = &idevice; 1352 1353 /* Test1: Current capacity support required size. */ 1354 rpoller.required_num_wr = 10; 1355 rpoller.num_cqe = 20; 1356 rqpair.max_queue_depth = 2; 1357 tnum_wr = rpoller.required_num_wr; 1358 tnum_cqe = rpoller.num_cqe; 1359 1360 rc = nvmf_rdma_resize_cq(&rqpair, &rdevice); 1361 CU_ASSERT(rc == 0); 1362 CU_ASSERT(rpoller.required_num_wr == 10 + MAX_WR_PER_QP(rqpair.max_queue_depth)); 1363 CU_ASSERT(rpoller.required_num_wr > tnum_wr); 1364 CU_ASSERT(rpoller.num_cqe == tnum_cqe); 1365 1366 /* Test2: iWARP doesn't support CQ resize. */ 1367 tnum_wr = rpoller.required_num_wr; 1368 tnum_cqe = rpoller.num_cqe; 1369 idevice.transport_type = IBV_TRANSPORT_IWARP; 1370 1371 rc = nvmf_rdma_resize_cq(&rqpair, &rdevice); 1372 CU_ASSERT(rc == -1); 1373 CU_ASSERT(rpoller.required_num_wr == tnum_wr); 1374 CU_ASSERT(rpoller.num_cqe == tnum_cqe); 1375 1376 1377 /* Test3: RDMA CQE requirement exceeds device max_cqe limitation. */ 1378 tnum_wr = rpoller.required_num_wr; 1379 tnum_cqe = rpoller.num_cqe; 1380 idevice.transport_type = IBV_TRANSPORT_UNKNOWN; 1381 rdevice.attr.max_cqe = 3; 1382 1383 rc = nvmf_rdma_resize_cq(&rqpair, &rdevice); 1384 CU_ASSERT(rc == -1); 1385 CU_ASSERT(rpoller.required_num_wr == tnum_wr); 1386 CU_ASSERT(rpoller.num_cqe == tnum_cqe); 1387 1388 /* Test4: RDMA CQ resize failed. */ 1389 tnum_wr = rpoller.required_num_wr; 1390 tnum_cqe = rpoller.num_cqe; 1391 idevice.transport_type = IBV_TRANSPORT_IB; 1392 rdevice.attr.max_cqe = 30; 1393 MOCK_SET(ibv_resize_cq, -1); 1394 1395 rc = nvmf_rdma_resize_cq(&rqpair, &rdevice); 1396 CU_ASSERT(rc == -1); 1397 CU_ASSERT(rpoller.required_num_wr == tnum_wr); 1398 CU_ASSERT(rpoller.num_cqe == tnum_cqe); 1399 1400 /* Test5: RDMA CQ resize success. rsize = MIN(MAX(num_cqe * 2, required_num_wr), device->attr.max_cqe). */ 1401 tnum_wr = rpoller.required_num_wr; 1402 tnum_cqe = rpoller.num_cqe; 1403 MOCK_SET(ibv_resize_cq, 0); 1404 1405 rc = nvmf_rdma_resize_cq(&rqpair, &rdevice); 1406 CU_ASSERT(rc == 0); 1407 CU_ASSERT(rpoller.num_cqe = 30); 1408 CU_ASSERT(rpoller.required_num_wr == 18 + MAX_WR_PER_QP(rqpair.max_queue_depth)); 1409 CU_ASSERT(rpoller.required_num_wr > tnum_wr); 1410 CU_ASSERT(rpoller.num_cqe > tnum_cqe); 1411 } 1412 1413 int 1414 main(int argc, char **argv) 1415 { 1416 CU_pSuite suite = NULL; 1417 unsigned int num_failures; 1418 1419 CU_initialize_registry(); 1420 1421 suite = CU_add_suite("nvmf", NULL, NULL); 1422 1423 CU_ADD_TEST(suite, test_spdk_nvmf_rdma_request_parse_sgl); 1424 CU_ADD_TEST(suite, test_spdk_nvmf_rdma_request_process); 1425 CU_ADD_TEST(suite, test_nvmf_rdma_get_optimal_poll_group); 1426 CU_ADD_TEST(suite, test_spdk_nvmf_rdma_request_parse_sgl_with_md); 1427 CU_ADD_TEST(suite, test_nvmf_rdma_opts_init); 1428 CU_ADD_TEST(suite, test_nvmf_rdma_request_free_data); 1429 CU_ADD_TEST(suite, test_nvmf_rdma_update_ibv_state); 1430 CU_ADD_TEST(suite, test_nvmf_rdma_resources_create); 1431 CU_ADD_TEST(suite, test_nvmf_rdma_qpair_compare); 1432 CU_ADD_TEST(suite, test_nvmf_rdma_resize_cq); 1433 1434 num_failures = spdk_ut_run_tests(argc, argv, NULL); 1435 CU_cleanup_registry(); 1436 return num_failures; 1437 } 1438