1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2018 Intel Corporation. All rights reserved. 3 * Copyright (c) 2019, 2021 Mellanox Technologies LTD. All rights reserved. 4 * Copyright (c) 2023, 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 5 */ 6 7 #include "spdk/stdinc.h" 8 #include "spdk_internal/cunit.h" 9 #include "common/lib/test_env.c" 10 #include "common/lib/test_iobuf.c" 11 #include "common/lib/test_rdma.c" 12 #include "nvmf/rdma.c" 13 #include "nvmf/transport.c" 14 15 #define RDMA_UT_UNITS_IN_MAX_IO 16 16 17 struct spdk_nvmf_transport_opts g_rdma_ut_transport_opts = { 18 .max_queue_depth = SPDK_NVMF_RDMA_DEFAULT_MAX_QUEUE_DEPTH, 19 .max_qpairs_per_ctrlr = SPDK_NVMF_RDMA_DEFAULT_MAX_QPAIRS_PER_CTRLR, 20 .in_capsule_data_size = SPDK_NVMF_RDMA_DEFAULT_IN_CAPSULE_DATA_SIZE, 21 .max_io_size = (SPDK_NVMF_RDMA_MIN_IO_BUFFER_SIZE * RDMA_UT_UNITS_IN_MAX_IO), 22 .io_unit_size = SPDK_NVMF_RDMA_MIN_IO_BUFFER_SIZE, 23 .max_aq_depth = SPDK_NVMF_RDMA_DEFAULT_AQ_DEPTH, 24 .num_shared_buffers = SPDK_NVMF_RDMA_DEFAULT_NUM_SHARED_BUFFERS, 25 }; 26 27 SPDK_LOG_REGISTER_COMPONENT(nvmf) 28 DEFINE_STUB(spdk_mem_map_set_translation, int, (struct spdk_mem_map *map, uint64_t vaddr, 29 uint64_t size, uint64_t translation), 0); 30 DEFINE_STUB(spdk_mem_map_clear_translation, int, (struct spdk_mem_map *map, uint64_t vaddr, 31 uint64_t size), 0); 32 DEFINE_STUB(spdk_mem_map_alloc, struct spdk_mem_map *, (uint64_t default_translation, 33 const struct spdk_mem_map_ops *ops, void *cb_ctx), NULL); 34 DEFINE_STUB(spdk_nvmf_qpair_disconnect, int, (struct spdk_nvmf_qpair *qpair), 0); 35 DEFINE_STUB(spdk_nvmf_qpair_get_listen_trid, int, 36 (struct spdk_nvmf_qpair *qpair, struct spdk_nvme_transport_id *trid), 0); 37 DEFINE_STUB_V(spdk_mem_map_free, (struct spdk_mem_map **pmap)); 38 39 DEFINE_STUB_V(spdk_nvmf_request_exec, (struct spdk_nvmf_request *req)); 40 DEFINE_STUB(spdk_nvmf_request_complete, int, (struct spdk_nvmf_request *req), 0); 41 DEFINE_STUB(spdk_nvme_transport_id_compare, int, (const struct spdk_nvme_transport_id *trid1, 42 const struct spdk_nvme_transport_id *trid2), 0); 43 DEFINE_STUB_V(spdk_nvmf_ctrlr_abort_aer, (struct spdk_nvmf_ctrlr *ctrlr)); 44 DEFINE_STUB(spdk_nvmf_request_get_dif_ctx, bool, (struct spdk_nvmf_request *req, 45 struct spdk_dif_ctx *dif_ctx), false); 46 DEFINE_STUB_V(spdk_nvme_trid_populate_transport, (struct spdk_nvme_transport_id *trid, 47 enum spdk_nvme_transport_type trtype)); 48 DEFINE_STUB_V(spdk_nvmf_tgt_new_qpair, (struct spdk_nvmf_tgt *tgt, struct spdk_nvmf_qpair *qpair)); 49 DEFINE_STUB(nvmf_ctrlr_abort_request, int, (struct spdk_nvmf_request *req), 0); 50 DEFINE_STUB(spdk_nvme_transport_id_adrfam_str, const char *, (enum spdk_nvmf_adrfam adrfam), NULL); 51 DEFINE_STUB(ibv_dereg_mr, int, (struct ibv_mr *mr), 0); 52 DEFINE_STUB(ibv_resize_cq, int, (struct ibv_cq *cq, int cqe), 0); 53 DEFINE_STUB(spdk_mempool_lookup, struct spdk_mempool *, (const char *name), NULL); 54 55 /* ibv_reg_mr can be a macro, need to undefine it */ 56 #ifdef ibv_reg_mr 57 #undef ibv_reg_mr 58 #endif 59 60 DEFINE_RETURN_MOCK(ibv_reg_mr, struct ibv_mr *); 61 struct ibv_mr * 62 ibv_reg_mr(struct ibv_pd *pd, void *addr, size_t length, int access) 63 { 64 HANDLE_RETURN_MOCK(ibv_reg_mr); 65 if (length > 0) { 66 return &g_rdma_mr; 67 } else { 68 return NULL; 69 } 70 } 71 72 int 73 ibv_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, 74 int attr_mask, struct ibv_qp_init_attr *init_attr) 75 { 76 if (qp == NULL) { 77 return -1; 78 } else { 79 attr->port_num = 80; 80 81 if (qp->state == IBV_QPS_ERR) { 82 attr->qp_state = 10; 83 } else { 84 attr->qp_state = IBV_QPS_INIT; 85 } 86 87 return 0; 88 } 89 } 90 91 const char * 92 spdk_nvme_transport_id_trtype_str(enum spdk_nvme_transport_type trtype) 93 { 94 switch (trtype) { 95 case SPDK_NVME_TRANSPORT_PCIE: 96 return "PCIe"; 97 case SPDK_NVME_TRANSPORT_RDMA: 98 return "RDMA"; 99 case SPDK_NVME_TRANSPORT_FC: 100 return "FC"; 101 default: 102 return NULL; 103 } 104 } 105 106 int 107 spdk_nvme_transport_id_populate_trstring(struct spdk_nvme_transport_id *trid, const char *trstring) 108 { 109 int len, i; 110 111 if (trstring == NULL) { 112 return -EINVAL; 113 } 114 115 len = strnlen(trstring, SPDK_NVMF_TRSTRING_MAX_LEN); 116 if (len == SPDK_NVMF_TRSTRING_MAX_LEN) { 117 return -EINVAL; 118 } 119 120 /* cast official trstring to uppercase version of input. */ 121 for (i = 0; i < len; i++) { 122 trid->trstring[i] = toupper(trstring[i]); 123 } 124 return 0; 125 } 126 127 static void 128 reset_nvmf_rdma_request(struct spdk_nvmf_rdma_request *rdma_req) 129 { 130 int i; 131 132 rdma_req->req.length = 0; 133 rdma_req->req.data_from_pool = false; 134 rdma_req->data.wr.num_sge = 0; 135 rdma_req->data.wr.wr.rdma.remote_addr = 0; 136 rdma_req->data.wr.wr.rdma.rkey = 0; 137 rdma_req->offset = 0; 138 memset(&rdma_req->req.dif, 0, sizeof(rdma_req->req.dif)); 139 140 for (i = 0; i < SPDK_NVMF_MAX_SGL_ENTRIES; i++) { 141 rdma_req->req.iov[i].iov_base = 0; 142 rdma_req->req.iov[i].iov_len = 0; 143 rdma_req->data.wr.sg_list[i].addr = 0; 144 rdma_req->data.wr.sg_list[i].length = 0; 145 rdma_req->data.wr.sg_list[i].lkey = 0; 146 } 147 rdma_req->req.iovcnt = 0; 148 if (rdma_req->req.stripped_data) { 149 free(rdma_req->req.stripped_data); 150 rdma_req->req.stripped_data = NULL; 151 } 152 } 153 154 static void 155 test_spdk_nvmf_rdma_request_parse_sgl(void) 156 { 157 struct spdk_nvmf_rdma_transport rtransport; 158 struct spdk_nvmf_rdma_device device; 159 struct spdk_nvmf_rdma_request rdma_req = {}; 160 struct spdk_nvmf_rdma_recv recv; 161 struct spdk_nvmf_rdma_poll_group group; 162 struct spdk_nvmf_rdma_qpair rqpair; 163 struct spdk_nvmf_rdma_poller poller; 164 union nvmf_c2h_msg cpl; 165 union nvmf_h2c_msg cmd; 166 struct spdk_nvme_sgl_descriptor *sgl; 167 struct spdk_nvme_sgl_descriptor sgl_desc[SPDK_NVMF_MAX_SGL_ENTRIES] = {{0}}; 168 struct spdk_nvmf_rdma_request_data data; 169 int rc, i; 170 uint32_t sgl_length; 171 172 data.wr.sg_list = data.sgl; 173 group.group.transport = &rtransport.transport; 174 poller.group = &group; 175 rqpair.poller = &poller; 176 rqpair.max_send_sge = SPDK_NVMF_MAX_SGL_ENTRIES; 177 178 sgl = &cmd.nvme_cmd.dptr.sgl1; 179 rdma_req.recv = &recv; 180 rdma_req.req.cmd = &cmd; 181 rdma_req.req.rsp = &cpl; 182 rdma_req.data.wr.sg_list = rdma_req.data.sgl; 183 rdma_req.req.qpair = &rqpair.qpair; 184 rdma_req.req.xfer = SPDK_NVME_DATA_CONTROLLER_TO_HOST; 185 186 rtransport.transport.opts = g_rdma_ut_transport_opts; 187 rtransport.data_wr_pool = NULL; 188 189 device.attr.device_cap_flags = 0; 190 sgl->keyed.key = 0xEEEE; 191 sgl->address = 0xFFFF; 192 rdma_req.recv->buf = (void *)0xDDDD; 193 194 /* Test 1: sgl type: keyed data block subtype: address */ 195 sgl->generic.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; 196 sgl->keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; 197 198 /* Part 1: simple I/O, one SGL smaller than the transport io unit size */ 199 MOCK_SET(spdk_iobuf_get, (void *)0x2000); 200 reset_nvmf_rdma_request(&rdma_req); 201 sgl->keyed.length = rtransport.transport.opts.io_unit_size / 2; 202 203 device.map = (void *)0x0; 204 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 205 CU_ASSERT(rc == 0); 206 CU_ASSERT(rdma_req.req.data_from_pool == true); 207 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size / 2); 208 CU_ASSERT((uint64_t)rdma_req.req.iovcnt == 1); 209 CU_ASSERT((uint64_t)rdma_req.req.iov[0].iov_base == 0x2000); 210 CU_ASSERT(rdma_req.data.wr.num_sge == 1); 211 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 212 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 213 CU_ASSERT((uint64_t)rdma_req.req.iov[0].iov_base == 0x2000); 214 CU_ASSERT(rdma_req.data.wr.sg_list[0].addr == 0x2000); 215 CU_ASSERT(rdma_req.data.wr.sg_list[0].length == rtransport.transport.opts.io_unit_size / 2); 216 CU_ASSERT(rdma_req.data.wr.sg_list[0].lkey == RDMA_UT_LKEY); 217 218 /* Part 2: simple I/O, one SGL larger than the transport io unit size (equal to the max io size) */ 219 reset_nvmf_rdma_request(&rdma_req); 220 sgl->keyed.length = rtransport.transport.opts.io_unit_size * RDMA_UT_UNITS_IN_MAX_IO; 221 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 222 223 CU_ASSERT(rc == 0); 224 CU_ASSERT(rdma_req.req.data_from_pool == true); 225 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size * RDMA_UT_UNITS_IN_MAX_IO); 226 CU_ASSERT(rdma_req.data.wr.num_sge == RDMA_UT_UNITS_IN_MAX_IO); 227 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 228 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 229 for (i = 0; i < RDMA_UT_UNITS_IN_MAX_IO; i++) { 230 CU_ASSERT((uint64_t)rdma_req.req.iov[i].iov_base == 0x2000); 231 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == 0x2000); 232 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == rtransport.transport.opts.io_unit_size); 233 CU_ASSERT(rdma_req.data.wr.sg_list[i].lkey == RDMA_UT_LKEY); 234 } 235 236 /* Part 3: simple I/O one SGL larger than the transport max io size */ 237 reset_nvmf_rdma_request(&rdma_req); 238 sgl->keyed.length = rtransport.transport.opts.max_io_size * 2; 239 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 240 241 CU_ASSERT(rc == -1); 242 243 /* Part 4: Pretend there are no buffer pools */ 244 MOCK_SET(spdk_iobuf_get, NULL); 245 reset_nvmf_rdma_request(&rdma_req); 246 sgl->keyed.length = rtransport.transport.opts.io_unit_size * RDMA_UT_UNITS_IN_MAX_IO; 247 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 248 249 CU_ASSERT(rc == 0); 250 CU_ASSERT(rdma_req.req.data_from_pool == false); 251 CU_ASSERT(rdma_req.req.iovcnt == 0); 252 CU_ASSERT(rdma_req.data.wr.num_sge == 0); 253 CU_ASSERT(rdma_req.req.iov[0].iov_base == NULL); 254 CU_ASSERT(rdma_req.data.wr.sg_list[0].addr == 0); 255 CU_ASSERT(rdma_req.data.wr.sg_list[0].length == 0); 256 CU_ASSERT(rdma_req.data.wr.sg_list[0].lkey == 0); 257 258 rdma_req.recv->buf = (void *)0xDDDD; 259 /* Test 2: sgl type: keyed data block subtype: offset (in capsule data) */ 260 sgl->generic.type = SPDK_NVME_SGL_TYPE_DATA_BLOCK; 261 sgl->unkeyed.subtype = SPDK_NVME_SGL_SUBTYPE_OFFSET; 262 263 /* Part 1: Normal I/O smaller than in capsule data size no offset */ 264 reset_nvmf_rdma_request(&rdma_req); 265 sgl->address = 0; 266 sgl->unkeyed.length = rtransport.transport.opts.in_capsule_data_size; 267 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 268 269 CU_ASSERT(rc == 0); 270 CU_ASSERT(rdma_req.req.iovcnt == 1); 271 CU_ASSERT(rdma_req.req.iov[0].iov_base == (void *)0xDDDD); 272 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.in_capsule_data_size); 273 CU_ASSERT(rdma_req.req.data_from_pool == false); 274 275 /* Part 2: I/O offset + length too large */ 276 reset_nvmf_rdma_request(&rdma_req); 277 sgl->address = rtransport.transport.opts.in_capsule_data_size; 278 sgl->unkeyed.length = rtransport.transport.opts.in_capsule_data_size; 279 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 280 281 CU_ASSERT(rc == -1); 282 283 /* Part 3: I/O too large */ 284 reset_nvmf_rdma_request(&rdma_req); 285 sgl->address = 0; 286 sgl->unkeyed.length = rtransport.transport.opts.in_capsule_data_size * 2; 287 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 288 289 CU_ASSERT(rc == -1); 290 291 /* Test 3: Multi SGL */ 292 sgl->generic.type = SPDK_NVME_SGL_TYPE_LAST_SEGMENT; 293 sgl->unkeyed.subtype = SPDK_NVME_SGL_SUBTYPE_OFFSET; 294 sgl->address = 0; 295 rdma_req.recv->buf = (void *)&sgl_desc; 296 MOCK_SET(spdk_iobuf_get, &data); 297 MOCK_SET(spdk_mempool_get, &data); 298 299 /* part 1: 2 segments each with 1 wr. */ 300 reset_nvmf_rdma_request(&rdma_req); 301 sgl->unkeyed.length = 2 * sizeof(struct spdk_nvme_sgl_descriptor); 302 for (i = 0; i < 2; i++) { 303 sgl_desc[i].keyed.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; 304 sgl_desc[i].keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; 305 sgl_desc[i].keyed.length = rtransport.transport.opts.io_unit_size; 306 sgl_desc[i].address = 0x4000 + i * rtransport.transport.opts.io_unit_size; 307 sgl_desc[i].keyed.key = 0x44; 308 } 309 310 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 311 312 CU_ASSERT(rc == 0); 313 CU_ASSERT(rdma_req.req.data_from_pool == true); 314 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size * 2); 315 CU_ASSERT(rdma_req.data.wr.num_sge == 1); 316 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0x44); 317 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0x4000); 318 CU_ASSERT(rdma_req.data.wr.next == &data.wr); 319 CU_ASSERT(data.wr.wr.rdma.rkey == 0x44); 320 CU_ASSERT(data.wr.wr.rdma.remote_addr == 0x4000 + rtransport.transport.opts.io_unit_size); 321 CU_ASSERT(data.wr.num_sge == 1); 322 CU_ASSERT(data.wr.next == &rdma_req.rsp.wr); 323 324 /* part 2: 2 segments, each with 1 wr containing 8 sge_elements */ 325 reset_nvmf_rdma_request(&rdma_req); 326 sgl->unkeyed.length = 2 * sizeof(struct spdk_nvme_sgl_descriptor); 327 for (i = 0; i < 2; i++) { 328 sgl_desc[i].keyed.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; 329 sgl_desc[i].keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; 330 sgl_desc[i].keyed.length = rtransport.transport.opts.io_unit_size * 8; 331 sgl_desc[i].address = 0x4000 + i * 8 * rtransport.transport.opts.io_unit_size; 332 sgl_desc[i].keyed.key = 0x44; 333 } 334 335 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 336 337 CU_ASSERT(rc == 0); 338 CU_ASSERT(rdma_req.req.data_from_pool == true); 339 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size * 16); 340 CU_ASSERT(rdma_req.req.iovcnt == 16); 341 CU_ASSERT(rdma_req.data.wr.num_sge == 8); 342 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0x44); 343 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0x4000); 344 CU_ASSERT(rdma_req.data.wr.next == &data.wr); 345 CU_ASSERT(data.wr.wr.rdma.rkey == 0x44); 346 CU_ASSERT(data.wr.wr.rdma.remote_addr == 0x4000 + rtransport.transport.opts.io_unit_size * 8); 347 CU_ASSERT(data.wr.num_sge == 8); 348 CU_ASSERT(data.wr.next == &rdma_req.rsp.wr); 349 350 /* part 3: 2 segments, one very large, one very small */ 351 reset_nvmf_rdma_request(&rdma_req); 352 for (i = 0; i < 2; i++) { 353 sgl_desc[i].keyed.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; 354 sgl_desc[i].keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; 355 sgl_desc[i].keyed.key = 0x44; 356 } 357 358 sgl_desc[0].keyed.length = rtransport.transport.opts.io_unit_size * 15 + 359 rtransport.transport.opts.io_unit_size / 2; 360 sgl_desc[0].address = 0x4000; 361 sgl_desc[1].keyed.length = rtransport.transport.opts.io_unit_size / 2; 362 sgl_desc[1].address = 0x4000 + rtransport.transport.opts.io_unit_size * 15 + 363 rtransport.transport.opts.io_unit_size / 2; 364 365 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 366 367 CU_ASSERT(rc == 0); 368 CU_ASSERT(rdma_req.req.data_from_pool == true); 369 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size * 16); 370 CU_ASSERT(rdma_req.req.iovcnt == 16); 371 CU_ASSERT(rdma_req.data.wr.num_sge == 16); 372 for (i = 0; i < 15; i++) { 373 CU_ASSERT(rdma_req.data.sgl[i].length == rtransport.transport.opts.io_unit_size); 374 } 375 CU_ASSERT(rdma_req.data.sgl[15].length == rtransport.transport.opts.io_unit_size / 2); 376 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0x44); 377 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0x4000); 378 CU_ASSERT(rdma_req.data.wr.next == &data.wr); 379 CU_ASSERT(data.wr.wr.rdma.rkey == 0x44); 380 CU_ASSERT(data.wr.wr.rdma.remote_addr == 0x4000 + rtransport.transport.opts.io_unit_size * 15 + 381 rtransport.transport.opts.io_unit_size / 2); 382 CU_ASSERT(data.sgl[0].length == rtransport.transport.opts.io_unit_size / 2); 383 CU_ASSERT(data.wr.num_sge == 1); 384 CU_ASSERT(data.wr.next == &rdma_req.rsp.wr); 385 386 /* part 4: 2 SGL descriptors, each length is transport buffer / 2 387 * 1 transport buffers should be allocated */ 388 reset_nvmf_rdma_request(&rdma_req); 389 sgl->unkeyed.length = 2 * sizeof(struct spdk_nvme_sgl_descriptor); 390 sgl_length = rtransport.transport.opts.io_unit_size / 2; 391 for (i = 0; i < 2; i++) { 392 sgl_desc[i].keyed.length = sgl_length; 393 sgl_desc[i].address = 0x4000 + i * sgl_length; 394 } 395 396 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 397 398 CU_ASSERT(rc == 0); 399 CU_ASSERT(rdma_req.req.data_from_pool == true); 400 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size); 401 CU_ASSERT(rdma_req.req.iovcnt == 1); 402 403 CU_ASSERT(rdma_req.data.sgl[0].length == sgl_length); 404 /* We mocked mempool_get to return address of data variable. Mempool is used 405 * to get both additional WRs and data buffers, so data points to &data */ 406 CU_ASSERT(rdma_req.data.sgl[0].addr == (uint64_t)&data); 407 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0x44); 408 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0x4000); 409 CU_ASSERT(rdma_req.data.wr.num_sge == 1); 410 CU_ASSERT(rdma_req.data.wr.next == &data.wr); 411 412 CU_ASSERT(data.wr.wr.rdma.rkey == 0x44); 413 CU_ASSERT(data.wr.wr.rdma.remote_addr == 0x4000 + sgl_length); 414 CU_ASSERT(data.sgl[0].length == sgl_length); 415 CU_ASSERT(data.sgl[0].addr == (uint64_t)&data + sgl_length); 416 CU_ASSERT(data.wr.num_sge == 1); 417 418 MOCK_CLEAR(spdk_mempool_get); 419 MOCK_CLEAR(spdk_iobuf_get); 420 421 reset_nvmf_rdma_request(&rdma_req); 422 } 423 424 static struct spdk_nvmf_rdma_recv * 425 create_recv(struct spdk_nvmf_rdma_qpair *rqpair, enum spdk_nvme_nvm_opcode opc) 426 { 427 struct spdk_nvmf_rdma_recv *rdma_recv; 428 union nvmf_h2c_msg *cmd; 429 struct spdk_nvme_sgl_descriptor *sgl; 430 431 rdma_recv = calloc(1, sizeof(*rdma_recv)); 432 rdma_recv->qpair = rqpair; 433 cmd = calloc(1, sizeof(*cmd)); 434 rdma_recv->sgl[0].addr = (uintptr_t)cmd; 435 cmd->nvme_cmd.opc = opc; 436 sgl = &cmd->nvme_cmd.dptr.sgl1; 437 sgl->keyed.key = 0xEEEE; 438 sgl->address = 0xFFFF; 439 sgl->keyed.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; 440 sgl->keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; 441 sgl->keyed.length = 1; 442 443 return rdma_recv; 444 } 445 446 static void 447 free_recv(struct spdk_nvmf_rdma_recv *rdma_recv) 448 { 449 free((void *)rdma_recv->sgl[0].addr); 450 free(rdma_recv); 451 } 452 453 static struct spdk_nvmf_rdma_request * 454 create_req(struct spdk_nvmf_rdma_qpair *rqpair, 455 struct spdk_nvmf_rdma_recv *rdma_recv) 456 { 457 struct spdk_nvmf_rdma_request *rdma_req; 458 union nvmf_c2h_msg *cpl; 459 460 rdma_req = calloc(1, sizeof(*rdma_req)); 461 rdma_req->recv = rdma_recv; 462 rdma_req->req.qpair = &rqpair->qpair; 463 rdma_req->state = RDMA_REQUEST_STATE_NEW; 464 rdma_req->data.wr.wr_id = (uintptr_t)&rdma_req->data_wr; 465 rdma_req->data.wr.sg_list = rdma_req->data.sgl; 466 cpl = calloc(1, sizeof(*cpl)); 467 rdma_req->rsp.sgl[0].addr = (uintptr_t)cpl; 468 rdma_req->req.rsp = cpl; 469 470 return rdma_req; 471 } 472 473 static void 474 free_req(struct spdk_nvmf_rdma_request *rdma_req) 475 { 476 free((void *)rdma_req->rsp.sgl[0].addr); 477 free(rdma_req); 478 } 479 480 static void 481 qpair_reset(struct spdk_nvmf_rdma_qpair *rqpair, 482 struct spdk_nvmf_rdma_poller *poller, 483 struct spdk_nvmf_rdma_device *device, 484 struct spdk_nvmf_rdma_resources *resources, 485 struct spdk_nvmf_transport *transport) 486 { 487 memset(rqpair, 0, sizeof(*rqpair)); 488 STAILQ_INIT(&rqpair->pending_rdma_write_queue); 489 STAILQ_INIT(&rqpair->pending_rdma_read_queue); 490 STAILQ_INIT(&rqpair->pending_rdma_send_queue); 491 rqpair->poller = poller; 492 rqpair->device = device; 493 rqpair->resources = resources; 494 rqpair->qpair.qid = 1; 495 rqpair->qpair.state = SPDK_NVMF_QPAIR_ENABLED; 496 rqpair->max_send_sge = SPDK_NVMF_MAX_SGL_ENTRIES; 497 rqpair->max_send_depth = 16; 498 rqpair->max_read_depth = 16; 499 rqpair->qpair.transport = transport; 500 } 501 502 static void 503 poller_reset(struct spdk_nvmf_rdma_poller *poller, 504 struct spdk_nvmf_rdma_poll_group *group) 505 { 506 memset(poller, 0, sizeof(*poller)); 507 STAILQ_INIT(&poller->qpairs_pending_recv); 508 STAILQ_INIT(&poller->qpairs_pending_send); 509 poller->group = group; 510 } 511 512 static void 513 test_spdk_nvmf_rdma_request_process(void) 514 { 515 struct spdk_nvmf_rdma_transport rtransport = {}; 516 struct spdk_nvmf_rdma_poll_group group = {}; 517 struct spdk_nvmf_rdma_poller poller = {}; 518 struct spdk_nvmf_rdma_device device = {}; 519 struct spdk_nvmf_rdma_resources resources = {}; 520 struct spdk_nvmf_rdma_qpair rqpair = {}; 521 struct spdk_nvmf_rdma_recv *rdma_recv; 522 struct spdk_nvmf_rdma_request *rdma_req; 523 struct spdk_iobuf_channel ch = {}; 524 bool progress; 525 526 group.group.buf_cache = &ch; 527 528 STAILQ_INIT(&group.group.pending_buf_queue); 529 poller_reset(&poller, &group); 530 qpair_reset(&rqpair, &poller, &device, &resources, &rtransport.transport); 531 532 rtransport.transport.opts = g_rdma_ut_transport_opts; 533 rtransport.data_wr_pool = spdk_mempool_create("test_wr_pool", 128, 534 sizeof(struct spdk_nvmf_rdma_request_data), 535 0, 0); 536 MOCK_CLEAR(spdk_iobuf_get); 537 538 device.attr.device_cap_flags = 0; 539 device.map = (void *)0x0; 540 541 /* Test 1: single SGL READ request */ 542 rdma_recv = create_recv(&rqpair, SPDK_NVME_OPC_READ); 543 rdma_req = create_req(&rqpair, rdma_recv); 544 rqpair.current_recv_depth = 1; 545 /* NEW -> EXECUTING */ 546 progress = nvmf_rdma_request_process(&rtransport, rdma_req); 547 CU_ASSERT(progress == true); 548 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_EXECUTING); 549 CU_ASSERT(rdma_req->req.xfer == SPDK_NVME_DATA_CONTROLLER_TO_HOST); 550 /* EXECUTED -> TRANSFERRING_C2H */ 551 rdma_req->state = RDMA_REQUEST_STATE_EXECUTED; 552 progress = nvmf_rdma_request_process(&rtransport, rdma_req); 553 CU_ASSERT(progress == true); 554 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_TRANSFERRING_CONTROLLER_TO_HOST); 555 CU_ASSERT(rdma_req->recv == NULL); 556 /* COMPLETED -> FREE */ 557 rdma_req->state = RDMA_REQUEST_STATE_COMPLETED; 558 progress = nvmf_rdma_request_process(&rtransport, rdma_req); 559 CU_ASSERT(progress == true); 560 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_FREE); 561 562 free_recv(rdma_recv); 563 free_req(rdma_req); 564 poller_reset(&poller, &group); 565 qpair_reset(&rqpair, &poller, &device, &resources, &rtransport.transport); 566 567 /* Test 2: single SGL WRITE request */ 568 rdma_recv = create_recv(&rqpair, SPDK_NVME_OPC_WRITE); 569 rdma_req = create_req(&rqpair, rdma_recv); 570 rqpair.current_recv_depth = 1; 571 /* NEW -> TRANSFERRING_H2C */ 572 progress = nvmf_rdma_request_process(&rtransport, rdma_req); 573 CU_ASSERT(progress == true); 574 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_TRANSFERRING_HOST_TO_CONTROLLER); 575 CU_ASSERT(rdma_req->req.xfer == SPDK_NVME_DATA_HOST_TO_CONTROLLER); 576 STAILQ_INIT(&poller.qpairs_pending_send); 577 /* READY_TO_EXECUTE -> EXECUTING */ 578 rdma_req->state = RDMA_REQUEST_STATE_READY_TO_EXECUTE; 579 progress = nvmf_rdma_request_process(&rtransport, rdma_req); 580 CU_ASSERT(progress == true); 581 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_EXECUTING); 582 /* EXECUTED -> COMPLETING */ 583 rdma_req->state = RDMA_REQUEST_STATE_EXECUTED; 584 progress = nvmf_rdma_request_process(&rtransport, rdma_req); 585 CU_ASSERT(progress == true); 586 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_COMPLETING); 587 CU_ASSERT(rdma_req->recv == NULL); 588 /* COMPLETED -> FREE */ 589 rdma_req->state = RDMA_REQUEST_STATE_COMPLETED; 590 progress = nvmf_rdma_request_process(&rtransport, rdma_req); 591 CU_ASSERT(progress == true); 592 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_FREE); 593 594 free_recv(rdma_recv); 595 free_req(rdma_req); 596 poller_reset(&poller, &group); 597 qpair_reset(&rqpair, &poller, &device, &resources, &rtransport.transport); 598 599 /* Test 3: WRITE+WRITE ibv_send batching */ 600 { 601 struct spdk_nvmf_rdma_recv *recv1, *recv2; 602 struct spdk_nvmf_rdma_request *req1, *req2; 603 recv1 = create_recv(&rqpair, SPDK_NVME_OPC_WRITE); 604 req1 = create_req(&rqpair, recv1); 605 recv2 = create_recv(&rqpair, SPDK_NVME_OPC_WRITE); 606 req2 = create_req(&rqpair, recv2); 607 608 /* WRITE 1: NEW -> TRANSFERRING_H2C */ 609 rqpair.current_recv_depth = 1; 610 nvmf_rdma_request_process(&rtransport, req1); 611 CU_ASSERT(req1->state == RDMA_REQUEST_STATE_TRANSFERRING_HOST_TO_CONTROLLER); 612 613 /* WRITE 2: NEW -> TRANSFERRING_H2C */ 614 rqpair.current_recv_depth = 2; 615 nvmf_rdma_request_process(&rtransport, req2); 616 CU_ASSERT(req2->state == RDMA_REQUEST_STATE_TRANSFERRING_HOST_TO_CONTROLLER); 617 618 STAILQ_INIT(&poller.qpairs_pending_send); 619 620 /* WRITE 1 completes before WRITE 2 has finished RDMA reading */ 621 /* WRITE 1: READY_TO_EXECUTE -> EXECUTING */ 622 req1->state = RDMA_REQUEST_STATE_READY_TO_EXECUTE; 623 nvmf_rdma_request_process(&rtransport, req1); 624 CU_ASSERT(req1->state == RDMA_REQUEST_STATE_EXECUTING); 625 /* WRITE 1: EXECUTED -> COMPLETING */ 626 req1->state = RDMA_REQUEST_STATE_EXECUTED; 627 nvmf_rdma_request_process(&rtransport, req1); 628 CU_ASSERT(req1->state == RDMA_REQUEST_STATE_COMPLETING); 629 STAILQ_INIT(&poller.qpairs_pending_send); 630 /* WRITE 1: COMPLETED -> FREE */ 631 req1->state = RDMA_REQUEST_STATE_COMPLETED; 632 nvmf_rdma_request_process(&rtransport, req1); 633 CU_ASSERT(req1->state == RDMA_REQUEST_STATE_FREE); 634 635 /* Now WRITE 2 has finished reading and completes */ 636 /* WRITE 2: COMPLETED -> FREE */ 637 /* WRITE 2: READY_TO_EXECUTE -> EXECUTING */ 638 req2->state = RDMA_REQUEST_STATE_READY_TO_EXECUTE; 639 nvmf_rdma_request_process(&rtransport, req2); 640 CU_ASSERT(req2->state == RDMA_REQUEST_STATE_EXECUTING); 641 /* WRITE 1: EXECUTED -> COMPLETING */ 642 req2->state = RDMA_REQUEST_STATE_EXECUTED; 643 nvmf_rdma_request_process(&rtransport, req2); 644 CU_ASSERT(req2->state == RDMA_REQUEST_STATE_COMPLETING); 645 STAILQ_INIT(&poller.qpairs_pending_send); 646 /* WRITE 1: COMPLETED -> FREE */ 647 req2->state = RDMA_REQUEST_STATE_COMPLETED; 648 nvmf_rdma_request_process(&rtransport, req2); 649 CU_ASSERT(req2->state == RDMA_REQUEST_STATE_FREE); 650 651 free_recv(recv1); 652 free_req(req1); 653 free_recv(recv2); 654 free_req(req2); 655 poller_reset(&poller, &group); 656 qpair_reset(&rqpair, &poller, &device, &resources, &rtransport.transport); 657 } 658 659 /* Test 4, invalid command, check xfer type */ 660 { 661 struct spdk_nvmf_rdma_recv *rdma_recv_inv; 662 struct spdk_nvmf_rdma_request *rdma_req_inv; 663 /* construct an opcode that specifies BIDIRECTIONAL transfer */ 664 uint8_t opc = 0x10 | SPDK_NVME_DATA_BIDIRECTIONAL; 665 666 rdma_recv_inv = create_recv(&rqpair, opc); 667 rdma_req_inv = create_req(&rqpair, rdma_recv_inv); 668 669 /* NEW -> RDMA_REQUEST_STATE_COMPLETING */ 670 rqpair.current_recv_depth = 1; 671 progress = nvmf_rdma_request_process(&rtransport, rdma_req_inv); 672 CU_ASSERT(progress == true); 673 CU_ASSERT(rdma_req_inv->state == RDMA_REQUEST_STATE_COMPLETING); 674 CU_ASSERT(rdma_req_inv->req.rsp->nvme_cpl.status.sct == SPDK_NVME_SCT_GENERIC); 675 CU_ASSERT(rdma_req_inv->req.rsp->nvme_cpl.status.sc == SPDK_NVME_SC_INVALID_OPCODE); 676 677 /* RDMA_REQUEST_STATE_COMPLETED -> FREE */ 678 rdma_req_inv->state = RDMA_REQUEST_STATE_COMPLETED; 679 nvmf_rdma_request_process(&rtransport, rdma_req_inv); 680 CU_ASSERT(rdma_req_inv->state == RDMA_REQUEST_STATE_FREE); 681 682 free_recv(rdma_recv_inv); 683 free_req(rdma_req_inv); 684 poller_reset(&poller, &group); 685 qpair_reset(&rqpair, &poller, &device, &resources, &rtransport.transport); 686 } 687 688 /* Test 5: Write response waits in queue */ 689 { 690 rdma_recv = create_recv(&rqpair, SPDK_NVME_OPC_WRITE); 691 rdma_req = create_req(&rqpair, rdma_recv); 692 rqpair.current_recv_depth = 1; 693 /* NEW -> TRANSFERRING_H2C */ 694 progress = nvmf_rdma_request_process(&rtransport, rdma_req); 695 CU_ASSERT(progress == true); 696 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_TRANSFERRING_HOST_TO_CONTROLLER); 697 CU_ASSERT(rdma_req->req.xfer == SPDK_NVME_DATA_HOST_TO_CONTROLLER); 698 STAILQ_INIT(&poller.qpairs_pending_send); 699 /* READY_TO_EXECUTE -> EXECUTING */ 700 rdma_req->state = RDMA_REQUEST_STATE_READY_TO_EXECUTE; 701 progress = nvmf_rdma_request_process(&rtransport, rdma_req); 702 CU_ASSERT(progress == true); 703 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_EXECUTING); 704 /* EXECUTED -> COMPLETING */ 705 rdma_req->state = RDMA_REQUEST_STATE_EXECUTED; 706 /* Send queue is full */ 707 rqpair.current_send_depth = rqpair.max_send_depth; 708 progress = nvmf_rdma_request_process(&rtransport, rdma_req); 709 CU_ASSERT(progress == true); 710 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_READY_TO_COMPLETE_PENDING); 711 CU_ASSERT(rdma_req == STAILQ_FIRST(&rqpair.pending_rdma_send_queue)); 712 713 /* Send queue is still full */ 714 progress = nvmf_rdma_request_process(&rtransport, rdma_req); 715 CU_ASSERT(progress == false); 716 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_READY_TO_COMPLETE_PENDING); 717 CU_ASSERT(rdma_req == STAILQ_FIRST(&rqpair.pending_rdma_send_queue)); 718 719 /* Slot is available */ 720 rqpair.current_send_depth = rqpair.max_send_depth - 1; 721 progress = nvmf_rdma_request_process(&rtransport, rdma_req); 722 CU_ASSERT(progress == true); 723 CU_ASSERT(STAILQ_EMPTY(&rqpair.pending_rdma_send_queue)); 724 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_COMPLETING); 725 CU_ASSERT(rdma_req->recv == NULL); 726 /* COMPLETED -> FREE */ 727 rdma_req->state = RDMA_REQUEST_STATE_COMPLETED; 728 progress = nvmf_rdma_request_process(&rtransport, rdma_req); 729 CU_ASSERT(progress == true); 730 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_FREE); 731 732 free_recv(rdma_recv); 733 free_req(rdma_req); 734 poller_reset(&poller, &group); 735 qpair_reset(&rqpair, &poller, &device, &resources, &rtransport.transport); 736 737 } 738 739 spdk_mempool_free(rtransport.data_wr_pool); 740 } 741 742 #define TEST_GROUPS_COUNT 5 743 static void 744 test_nvmf_rdma_get_optimal_poll_group(void) 745 { 746 struct spdk_nvmf_rdma_transport rtransport = {}; 747 struct spdk_nvmf_transport *transport = &rtransport.transport; 748 struct spdk_nvmf_rdma_qpair rqpair = {}; 749 struct spdk_nvmf_transport_poll_group *groups[TEST_GROUPS_COUNT]; 750 struct spdk_nvmf_rdma_poll_group *rgroups[TEST_GROUPS_COUNT]; 751 struct spdk_nvmf_transport_poll_group *result; 752 struct spdk_nvmf_poll_group group = {}; 753 uint32_t i; 754 755 rqpair.qpair.transport = transport; 756 TAILQ_INIT(&rtransport.poll_groups); 757 758 for (i = 0; i < TEST_GROUPS_COUNT; i++) { 759 groups[i] = nvmf_rdma_poll_group_create(transport, NULL); 760 CU_ASSERT(groups[i] != NULL); 761 groups[i]->group = &group; 762 rgroups[i] = SPDK_CONTAINEROF(groups[i], struct spdk_nvmf_rdma_poll_group, group); 763 groups[i]->transport = transport; 764 } 765 CU_ASSERT(rtransport.conn_sched.next_admin_pg == rgroups[0]); 766 CU_ASSERT(rtransport.conn_sched.next_io_pg == rgroups[0]); 767 768 /* Emulate connection of %TEST_GROUPS_COUNT% initiators - each creates 1 admin and 1 io qp */ 769 for (i = 0; i < TEST_GROUPS_COUNT; i++) { 770 rqpair.qpair.qid = 0; 771 result = nvmf_rdma_get_optimal_poll_group(&rqpair.qpair); 772 CU_ASSERT(result == groups[i]); 773 CU_ASSERT(rtransport.conn_sched.next_admin_pg == rgroups[(i + 1) % TEST_GROUPS_COUNT]); 774 CU_ASSERT(rtransport.conn_sched.next_io_pg == rgroups[i]); 775 776 rqpair.qpair.qid = 1; 777 result = nvmf_rdma_get_optimal_poll_group(&rqpair.qpair); 778 CU_ASSERT(result == groups[i]); 779 CU_ASSERT(rtransport.conn_sched.next_admin_pg == rgroups[(i + 1) % TEST_GROUPS_COUNT]); 780 CU_ASSERT(rtransport.conn_sched.next_io_pg == rgroups[(i + 1) % TEST_GROUPS_COUNT]); 781 } 782 /* wrap around, admin/io pg point to the first pg 783 Destroy all poll groups except of the last one */ 784 for (i = 0; i < TEST_GROUPS_COUNT - 1; i++) { 785 nvmf_rdma_poll_group_destroy(groups[i]); 786 CU_ASSERT(rtransport.conn_sched.next_admin_pg == rgroups[i + 1]); 787 CU_ASSERT(rtransport.conn_sched.next_io_pg == rgroups[i + 1]); 788 } 789 790 CU_ASSERT(rtransport.conn_sched.next_admin_pg == rgroups[TEST_GROUPS_COUNT - 1]); 791 CU_ASSERT(rtransport.conn_sched.next_io_pg == rgroups[TEST_GROUPS_COUNT - 1]); 792 793 /* Check that pointers to the next admin/io poll groups are not changed */ 794 rqpair.qpair.qid = 0; 795 result = nvmf_rdma_get_optimal_poll_group(&rqpair.qpair); 796 CU_ASSERT(result == groups[TEST_GROUPS_COUNT - 1]); 797 CU_ASSERT(rtransport.conn_sched.next_admin_pg == rgroups[TEST_GROUPS_COUNT - 1]); 798 CU_ASSERT(rtransport.conn_sched.next_io_pg == rgroups[TEST_GROUPS_COUNT - 1]); 799 800 rqpair.qpair.qid = 1; 801 result = nvmf_rdma_get_optimal_poll_group(&rqpair.qpair); 802 CU_ASSERT(result == groups[TEST_GROUPS_COUNT - 1]); 803 CU_ASSERT(rtransport.conn_sched.next_admin_pg == rgroups[TEST_GROUPS_COUNT - 1]); 804 CU_ASSERT(rtransport.conn_sched.next_io_pg == rgroups[TEST_GROUPS_COUNT - 1]); 805 806 /* Remove the last poll group, check that pointers are NULL */ 807 nvmf_rdma_poll_group_destroy(groups[TEST_GROUPS_COUNT - 1]); 808 CU_ASSERT(rtransport.conn_sched.next_admin_pg == NULL); 809 CU_ASSERT(rtransport.conn_sched.next_io_pg == NULL); 810 811 /* Request optimal poll group, result must be NULL */ 812 rqpair.qpair.qid = 0; 813 result = nvmf_rdma_get_optimal_poll_group(&rqpair.qpair); 814 CU_ASSERT(result == NULL); 815 816 rqpair.qpair.qid = 1; 817 result = nvmf_rdma_get_optimal_poll_group(&rqpair.qpair); 818 CU_ASSERT(result == NULL); 819 } 820 #undef TEST_GROUPS_COUNT 821 822 static void 823 test_spdk_nvmf_rdma_request_parse_sgl_with_md(void) 824 { 825 struct spdk_nvmf_rdma_transport rtransport; 826 struct spdk_nvmf_rdma_device device; 827 struct spdk_nvmf_rdma_request rdma_req = {}; 828 struct spdk_nvmf_rdma_recv recv; 829 struct spdk_nvmf_rdma_poll_group group; 830 struct spdk_nvmf_rdma_qpair rqpair; 831 struct spdk_nvmf_rdma_poller poller; 832 union nvmf_c2h_msg cpl; 833 union nvmf_h2c_msg cmd; 834 struct spdk_nvme_sgl_descriptor *sgl; 835 struct spdk_nvme_sgl_descriptor sgl_desc[SPDK_NVMF_MAX_SGL_ENTRIES] = {{0}}; 836 char data_buffer[8192]; 837 struct spdk_nvmf_rdma_request_data *data = (struct spdk_nvmf_rdma_request_data *)data_buffer; 838 char data2_buffer[8192]; 839 struct spdk_nvmf_rdma_request_data *data2 = (struct spdk_nvmf_rdma_request_data *)data2_buffer; 840 const uint32_t data_bs = 512; 841 const uint32_t md_size = 8; 842 int rc, i; 843 struct spdk_dif_ctx_init_ext_opts dif_opts; 844 845 MOCK_CLEAR(spdk_mempool_get); 846 MOCK_CLEAR(spdk_iobuf_get); 847 848 data->wr.sg_list = data->sgl; 849 group.group.transport = &rtransport.transport; 850 poller.group = &group; 851 rqpair.poller = &poller; 852 rqpair.max_send_sge = SPDK_NVMF_MAX_SGL_ENTRIES; 853 854 sgl = &cmd.nvme_cmd.dptr.sgl1; 855 rdma_req.recv = &recv; 856 rdma_req.req.cmd = &cmd; 857 rdma_req.req.rsp = &cpl; 858 rdma_req.data.wr.sg_list = rdma_req.data.sgl; 859 rdma_req.req.qpair = &rqpair.qpair; 860 rdma_req.req.xfer = SPDK_NVME_DATA_CONTROLLER_TO_HOST; 861 862 rtransport.transport.opts = g_rdma_ut_transport_opts; 863 rtransport.data_wr_pool = NULL; 864 865 device.attr.device_cap_flags = 0; 866 device.map = NULL; 867 sgl->keyed.key = 0xEEEE; 868 sgl->address = 0xFFFF; 869 rdma_req.recv->buf = (void *)0xDDDD; 870 871 /* Test 1: sgl type: keyed data block subtype: address */ 872 sgl->generic.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; 873 sgl->keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; 874 875 /* Part 1: simple I/O, one SGL smaller than the transport io unit size, block size 512 */ 876 MOCK_SET(spdk_iobuf_get, (void *)0x2000); 877 reset_nvmf_rdma_request(&rdma_req); 878 dif_opts.size = SPDK_SIZEOF(&dif_opts, dif_pi_format); 879 dif_opts.dif_pi_format = SPDK_DIF_PI_FORMAT_16; 880 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 881 SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 882 0, 0, 0, 0, 0, &dif_opts); 883 rdma_req.req.dif_enabled = true; 884 rtransport.transport.opts.io_unit_size = data_bs * 8; 885 rdma_req.req.qpair->transport = &rtransport.transport; 886 sgl->keyed.length = data_bs * 4; 887 888 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 889 890 CU_ASSERT(rc == 0); 891 CU_ASSERT(rdma_req.req.data_from_pool == true); 892 CU_ASSERT(rdma_req.req.length == data_bs * 4); 893 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 894 CU_ASSERT(rdma_req.req.dif.elba_length == (data_bs + md_size) * 4); 895 CU_ASSERT(rdma_req.req.iovcnt == 1); 896 CU_ASSERT((uint64_t)rdma_req.req.iov[0].iov_base == 0x2000); 897 CU_ASSERT(rdma_req.data.wr.num_sge == 1); 898 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 899 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 900 CU_ASSERT((uint64_t)rdma_req.req.iov[0].iov_base == 0x2000); 901 902 CU_ASSERT(rdma_req.data.wr.sg_list[0].addr == 0x2000); 903 CU_ASSERT(rdma_req.data.wr.sg_list[0].length == rdma_req.req.length); 904 CU_ASSERT(rdma_req.data.wr.sg_list[0].lkey == RDMA_UT_LKEY); 905 906 /* Part 2: simple I/O, one SGL equal to io unit size, io_unit_size is not aligned with md_size, 907 block size 512 */ 908 MOCK_SET(spdk_iobuf_get, (void *)0x2000); 909 reset_nvmf_rdma_request(&rdma_req); 910 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 911 SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 912 0, 0, 0, 0, 0, &dif_opts); 913 rdma_req.req.dif_enabled = true; 914 rtransport.transport.opts.io_unit_size = data_bs * 4; 915 sgl->keyed.length = data_bs * 4; 916 917 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 918 919 CU_ASSERT(rc == 0); 920 CU_ASSERT(rdma_req.req.data_from_pool == true); 921 CU_ASSERT(rdma_req.req.length == data_bs * 4); 922 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 923 CU_ASSERT(rdma_req.req.dif.elba_length == (data_bs + md_size) * 4); 924 CU_ASSERT(rdma_req.req.iovcnt == 2); 925 CU_ASSERT((uint64_t)rdma_req.req.iov[0].iov_base == 0x2000); 926 CU_ASSERT(rdma_req.data.wr.num_sge == 5); 927 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 928 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 929 CU_ASSERT((uint64_t)rdma_req.req.iov[0].iov_base == 0x2000); 930 931 for (i = 0; i < 3; ++i) { 932 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == 0x2000 + i * (data_bs + md_size)); 933 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == data_bs); 934 CU_ASSERT(rdma_req.data.wr.sg_list[i].lkey == RDMA_UT_LKEY); 935 } 936 CU_ASSERT(rdma_req.data.wr.sg_list[3].addr == 0x2000 + 3 * (data_bs + md_size)); 937 CU_ASSERT(rdma_req.data.wr.sg_list[3].length == 488); 938 CU_ASSERT(rdma_req.data.wr.sg_list[3].lkey == RDMA_UT_LKEY); 939 940 /* 2nd buffer consumed */ 941 CU_ASSERT(rdma_req.data.wr.sg_list[4].addr == 0x2000); 942 CU_ASSERT(rdma_req.data.wr.sg_list[4].length == 24); 943 CU_ASSERT(rdma_req.data.wr.sg_list[4].lkey == RDMA_UT_LKEY); 944 945 /* Part 3: simple I/O, one SGL equal io unit size, io_unit_size is equal to block size 512 bytes */ 946 MOCK_SET(spdk_iobuf_get, (void *)0x2000); 947 reset_nvmf_rdma_request(&rdma_req); 948 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 949 SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 950 0, 0, 0, 0, 0, &dif_opts); 951 rdma_req.req.dif_enabled = true; 952 rtransport.transport.opts.io_unit_size = data_bs; 953 sgl->keyed.length = data_bs; 954 955 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 956 957 CU_ASSERT(rc == 0); 958 CU_ASSERT(rdma_req.req.data_from_pool == true); 959 CU_ASSERT(rdma_req.req.length == data_bs); 960 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 961 CU_ASSERT(rdma_req.req.dif.elba_length == data_bs + md_size); 962 CU_ASSERT(rdma_req.req.iovcnt == 2); 963 CU_ASSERT((uint64_t)rdma_req.req.iov[0].iov_base == 0x2000); 964 CU_ASSERT(rdma_req.data.wr.num_sge == 1); 965 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 966 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 967 CU_ASSERT((uint64_t)rdma_req.req.iov[0].iov_base == 0x2000); 968 969 CU_ASSERT(rdma_req.data.wr.sg_list[0].addr == 0x2000); 970 CU_ASSERT(rdma_req.data.wr.sg_list[0].length == data_bs); 971 CU_ASSERT(rdma_req.data.wr.sg_list[0].lkey == RDMA_UT_LKEY); 972 973 CU_ASSERT(rdma_req.req.iovcnt == 2); 974 CU_ASSERT(rdma_req.req.iov[0].iov_base == (void *)((unsigned long)0x2000)); 975 CU_ASSERT(rdma_req.req.iov[0].iov_len == data_bs); 976 /* 2nd buffer consumed for metadata */ 977 CU_ASSERT(rdma_req.req.iov[1].iov_base == (void *)((unsigned long)0x2000)); 978 CU_ASSERT(rdma_req.req.iov[1].iov_len == md_size); 979 980 /* Part 4: simple I/O, one SGL equal io unit size, io_unit_size is aligned with md_size, 981 block size 512 */ 982 MOCK_SET(spdk_iobuf_get, (void *)0x2000); 983 reset_nvmf_rdma_request(&rdma_req); 984 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 985 SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 986 0, 0, 0, 0, 0, &dif_opts); 987 rdma_req.req.dif_enabled = true; 988 rtransport.transport.opts.io_unit_size = (data_bs + md_size) * 4; 989 sgl->keyed.length = data_bs * 4; 990 991 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 992 993 CU_ASSERT(rc == 0); 994 CU_ASSERT(rdma_req.req.data_from_pool == true); 995 CU_ASSERT(rdma_req.req.length == data_bs * 4); 996 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 997 CU_ASSERT(rdma_req.req.dif.elba_length == (data_bs + md_size) * 4); 998 CU_ASSERT(rdma_req.req.iovcnt == 1); 999 CU_ASSERT((uint64_t)rdma_req.req.iov[0].iov_base == 0x2000); 1000 CU_ASSERT(rdma_req.data.wr.num_sge == 1); 1001 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 1002 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 1003 CU_ASSERT((uint64_t)rdma_req.req.iov[0].iov_base == 0x2000); 1004 1005 CU_ASSERT(rdma_req.data.wr.sg_list[0].addr == 0x2000); 1006 CU_ASSERT(rdma_req.data.wr.sg_list[0].length == rdma_req.req.length); 1007 CU_ASSERT(rdma_req.data.wr.sg_list[0].lkey == RDMA_UT_LKEY); 1008 1009 /* Part 5: simple I/O, one SGL equal to 2x io unit size, io_unit_size is aligned with md_size, 1010 block size 512 */ 1011 MOCK_SET(spdk_iobuf_get, (void *)0x2000); 1012 reset_nvmf_rdma_request(&rdma_req); 1013 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 1014 SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 1015 0, 0, 0, 0, 0, &dif_opts); 1016 rdma_req.req.dif_enabled = true; 1017 rtransport.transport.opts.io_unit_size = (data_bs + md_size) * 2; 1018 sgl->keyed.length = data_bs * 4; 1019 1020 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 1021 1022 CU_ASSERT(rc == 0); 1023 CU_ASSERT(rdma_req.req.data_from_pool == true); 1024 CU_ASSERT(rdma_req.req.length == data_bs * 4); 1025 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 1026 CU_ASSERT(rdma_req.req.dif.elba_length == (data_bs + md_size) * 4); 1027 CU_ASSERT(rdma_req.req.iovcnt == 2); 1028 CU_ASSERT((uint64_t)rdma_req.req.iov[0].iov_base == 0x2000); 1029 CU_ASSERT(rdma_req.data.wr.num_sge == 2); 1030 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 1031 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 1032 CU_ASSERT((uint64_t)rdma_req.req.iov[0].iov_base == 0x2000); 1033 1034 for (i = 0; i < 2; ++i) { 1035 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == 0x2000); 1036 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == data_bs * 2); 1037 } 1038 1039 /* Part 6: simple I/O, one SGL larger than the transport io unit size, io_unit_size is not aligned to md_size, 1040 block size 512 */ 1041 MOCK_SET(spdk_iobuf_get, (void *)0x2000); 1042 reset_nvmf_rdma_request(&rdma_req); 1043 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 1044 SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 1045 0, 0, 0, 0, 0, &dif_opts); 1046 rdma_req.req.dif_enabled = true; 1047 rtransport.transport.opts.io_unit_size = data_bs * 4; 1048 sgl->keyed.length = data_bs * 6; 1049 1050 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 1051 1052 CU_ASSERT(rc == 0); 1053 CU_ASSERT(rdma_req.req.data_from_pool == true); 1054 CU_ASSERT(rdma_req.req.length == data_bs * 6); 1055 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 1056 CU_ASSERT(rdma_req.req.dif.elba_length == (data_bs + md_size) * 6); 1057 CU_ASSERT(rdma_req.req.iovcnt == 2); 1058 CU_ASSERT((uint64_t)rdma_req.req.iov[0].iov_base == 0x2000); 1059 CU_ASSERT(rdma_req.data.wr.num_sge == 7); 1060 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 1061 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 1062 CU_ASSERT((uint64_t)rdma_req.req.iov[0].iov_base == 0x2000); 1063 1064 for (i = 0; i < 3; ++i) { 1065 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == 0x2000 + i * (data_bs + md_size)); 1066 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == data_bs); 1067 CU_ASSERT(rdma_req.data.wr.sg_list[i].lkey == RDMA_UT_LKEY); 1068 } 1069 CU_ASSERT(rdma_req.data.wr.sg_list[3].addr == 0x2000 + 3 * (data_bs + md_size)); 1070 CU_ASSERT(rdma_req.data.wr.sg_list[3].length == 488); 1071 CU_ASSERT(rdma_req.data.wr.sg_list[3].lkey == RDMA_UT_LKEY); 1072 1073 /* 2nd IO buffer consumed */ 1074 CU_ASSERT(rdma_req.data.wr.sg_list[4].addr == 0x2000); 1075 CU_ASSERT(rdma_req.data.wr.sg_list[4].length == 24); 1076 CU_ASSERT(rdma_req.data.wr.sg_list[4].lkey == RDMA_UT_LKEY); 1077 1078 CU_ASSERT(rdma_req.data.wr.sg_list[5].addr == 0x2000 + 24 + md_size); 1079 CU_ASSERT(rdma_req.data.wr.sg_list[5].length == 512); 1080 CU_ASSERT(rdma_req.data.wr.sg_list[5].lkey == RDMA_UT_LKEY); 1081 1082 CU_ASSERT(rdma_req.data.wr.sg_list[6].addr == 0x2000 + 24 + 512 + md_size * 2); 1083 CU_ASSERT(rdma_req.data.wr.sg_list[6].length == 512); 1084 CU_ASSERT(rdma_req.data.wr.sg_list[6].lkey == RDMA_UT_LKEY); 1085 1086 /* Part 7: simple I/O, number of SGL entries exceeds the number of entries 1087 one WR can hold. Additional WR is chained */ 1088 MOCK_SET(spdk_iobuf_get, data2_buffer); 1089 MOCK_SET(spdk_mempool_get, data2_buffer); 1090 reset_nvmf_rdma_request(&rdma_req); 1091 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 1092 SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 1093 0, 0, 0, 0, 0, &dif_opts); 1094 rdma_req.req.dif_enabled = true; 1095 rtransport.transport.opts.io_unit_size = data_bs * 16; 1096 sgl->keyed.length = data_bs * 16; 1097 1098 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 1099 1100 CU_ASSERT(rc == 0); 1101 CU_ASSERT(rdma_req.req.data_from_pool == true); 1102 CU_ASSERT(rdma_req.req.length == data_bs * 16); 1103 CU_ASSERT(rdma_req.req.iovcnt == 2); 1104 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 1105 CU_ASSERT(rdma_req.req.dif.elba_length == (data_bs + md_size) * 16); 1106 CU_ASSERT(rdma_req.req.iov[0].iov_base == data2_buffer); 1107 CU_ASSERT(rdma_req.data.wr.num_sge == 16); 1108 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 1109 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 1110 1111 for (i = 0; i < 15; ++i) { 1112 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == (uintptr_t)data2_buffer + i * (data_bs + md_size)); 1113 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == data_bs); 1114 CU_ASSERT(rdma_req.data.wr.sg_list[i].lkey == RDMA_UT_LKEY); 1115 } 1116 1117 /* 8192 - (512 + 8) * 15 = 392 */ 1118 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == (uintptr_t)data2_buffer + i * (data_bs + md_size)); 1119 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == 392); 1120 CU_ASSERT(rdma_req.data.wr.sg_list[i].lkey == RDMA_UT_LKEY); 1121 1122 /* additional wr from pool */ 1123 CU_ASSERT(rdma_req.data.wr.next == (void *)&data2->wr); 1124 CU_ASSERT(rdma_req.data.wr.next->num_sge == 1); 1125 CU_ASSERT(rdma_req.data.wr.next->next == &rdma_req.rsp.wr); 1126 /* 2nd IO buffer */ 1127 CU_ASSERT(data2->wr.sg_list[0].addr == (uintptr_t)data2_buffer); 1128 CU_ASSERT(data2->wr.sg_list[0].length == 120); 1129 CU_ASSERT(data2->wr.sg_list[0].lkey == RDMA_UT_LKEY); 1130 1131 /* Part 8: simple I/O, data with metadata do not fit to 1 io_buffer */ 1132 MOCK_SET(spdk_iobuf_get, (void *)0x2000); 1133 reset_nvmf_rdma_request(&rdma_req); 1134 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 1135 SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 1136 0, 0, 0, 0, 0, &dif_opts); 1137 rdma_req.req.dif_enabled = true; 1138 rtransport.transport.opts.io_unit_size = 516; 1139 sgl->keyed.length = data_bs * 2; 1140 1141 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 1142 1143 CU_ASSERT(rc == 0); 1144 CU_ASSERT(rdma_req.req.data_from_pool == true); 1145 CU_ASSERT(rdma_req.req.length == data_bs * 2); 1146 CU_ASSERT(rdma_req.req.iovcnt == 3); 1147 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 1148 CU_ASSERT(rdma_req.req.dif.elba_length == (data_bs + md_size) * 2); 1149 CU_ASSERT(rdma_req.req.iov[0].iov_base == (void *)0x2000); 1150 CU_ASSERT(rdma_req.data.wr.num_sge == 2); 1151 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 1152 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 1153 1154 CU_ASSERT(rdma_req.data.wr.sg_list[0].addr == 0x2000); 1155 CU_ASSERT(rdma_req.data.wr.sg_list[0].length == 512); 1156 CU_ASSERT(rdma_req.data.wr.sg_list[0].lkey == RDMA_UT_LKEY); 1157 1158 /* 2nd IO buffer consumed, offset 4 bytes due to part of the metadata 1159 is located at the beginning of that buffer */ 1160 CU_ASSERT(rdma_req.data.wr.sg_list[1].addr == 0x2000 + 4); 1161 CU_ASSERT(rdma_req.data.wr.sg_list[1].length == 512); 1162 CU_ASSERT(rdma_req.data.wr.sg_list[1].lkey == RDMA_UT_LKEY); 1163 1164 /* Test 2: Multi SGL */ 1165 sgl->generic.type = SPDK_NVME_SGL_TYPE_LAST_SEGMENT; 1166 sgl->unkeyed.subtype = SPDK_NVME_SGL_SUBTYPE_OFFSET; 1167 sgl->address = 0; 1168 rdma_req.recv->buf = (void *)&sgl_desc; 1169 MOCK_SET(spdk_mempool_get, data_buffer); 1170 MOCK_SET(spdk_iobuf_get, data_buffer); 1171 1172 /* part 1: 2 segments each with 1 wr. io_unit_size is aligned with data_bs + md_size */ 1173 reset_nvmf_rdma_request(&rdma_req); 1174 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 1175 SPDK_DIF_TYPE1, 1176 SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 1177 0, 0, 0, 0, 0, &dif_opts); 1178 rdma_req.req.dif_enabled = true; 1179 rtransport.transport.opts.io_unit_size = (data_bs + md_size) * 4; 1180 sgl->unkeyed.length = 2 * sizeof(struct spdk_nvme_sgl_descriptor); 1181 1182 for (i = 0; i < 2; i++) { 1183 sgl_desc[i].keyed.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; 1184 sgl_desc[i].keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; 1185 sgl_desc[i].keyed.length = data_bs * 4; 1186 sgl_desc[i].address = 0x4000 + i * data_bs * 4; 1187 sgl_desc[i].keyed.key = 0x44; 1188 } 1189 1190 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 1191 1192 CU_ASSERT(rc == 0); 1193 CU_ASSERT(rdma_req.req.data_from_pool == true); 1194 CU_ASSERT(rdma_req.req.length == data_bs * 4 * 2); 1195 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 1196 CU_ASSERT(rdma_req.req.dif.elba_length == (data_bs + md_size) * 4 * 2); 1197 CU_ASSERT(rdma_req.data.wr.num_sge == 1); 1198 CU_ASSERT(rdma_req.data.wr.sg_list[0].addr == (uintptr_t)(data_buffer)); 1199 CU_ASSERT(rdma_req.data.wr.sg_list[0].length == data_bs * 4); 1200 1201 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0x44); 1202 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0x4000); 1203 CU_ASSERT(rdma_req.data.wr.next == &data->wr); 1204 CU_ASSERT(data->wr.wr.rdma.rkey == 0x44); 1205 CU_ASSERT(data->wr.wr.rdma.remote_addr == 0x4000 + data_bs * 4); 1206 CU_ASSERT(data->wr.num_sge == 1); 1207 CU_ASSERT(data->wr.sg_list[0].addr == (uintptr_t)(data_buffer)); 1208 CU_ASSERT(data->wr.sg_list[0].length == data_bs * 4); 1209 1210 CU_ASSERT(data->wr.next == &rdma_req.rsp.wr); 1211 reset_nvmf_rdma_request(&rdma_req); 1212 } 1213 1214 static void 1215 test_nvmf_rdma_opts_init(void) 1216 { 1217 struct spdk_nvmf_transport_opts opts = {}; 1218 1219 nvmf_rdma_opts_init(&opts); 1220 CU_ASSERT(opts.max_queue_depth == SPDK_NVMF_RDMA_DEFAULT_MAX_QUEUE_DEPTH); 1221 CU_ASSERT(opts.max_qpairs_per_ctrlr == SPDK_NVMF_RDMA_DEFAULT_MAX_QPAIRS_PER_CTRLR); 1222 CU_ASSERT(opts.in_capsule_data_size == SPDK_NVMF_RDMA_DEFAULT_IN_CAPSULE_DATA_SIZE); 1223 CU_ASSERT(opts.max_io_size == SPDK_NVMF_RDMA_DEFAULT_MAX_IO_SIZE); 1224 CU_ASSERT(opts.io_unit_size == SPDK_NVMF_RDMA_MIN_IO_BUFFER_SIZE); 1225 CU_ASSERT(opts.max_aq_depth == SPDK_NVMF_RDMA_DEFAULT_AQ_DEPTH); 1226 CU_ASSERT(opts.num_shared_buffers == SPDK_NVMF_RDMA_DEFAULT_NUM_SHARED_BUFFERS); 1227 CU_ASSERT(opts.buf_cache_size == SPDK_NVMF_RDMA_DEFAULT_BUFFER_CACHE_SIZE); 1228 CU_ASSERT(opts.dif_insert_or_strip == SPDK_NVMF_RDMA_DIF_INSERT_OR_STRIP); 1229 CU_ASSERT(opts.abort_timeout_sec == SPDK_NVMF_RDMA_DEFAULT_ABORT_TIMEOUT_SEC); 1230 CU_ASSERT(opts.transport_specific == NULL); 1231 } 1232 1233 static void 1234 test_nvmf_rdma_request_free_data(void) 1235 { 1236 struct spdk_nvmf_rdma_request rdma_req = {}; 1237 struct spdk_nvmf_rdma_transport rtransport = {}; 1238 struct spdk_nvmf_rdma_request_data *next_request_data = NULL; 1239 1240 MOCK_CLEAR(spdk_mempool_get); 1241 rtransport.data_wr_pool = spdk_mempool_create("spdk_nvmf_rdma_wr_data", 1242 SPDK_NVMF_MAX_SGL_ENTRIES, 1243 sizeof(struct spdk_nvmf_rdma_request_data), 1244 SPDK_MEMPOOL_DEFAULT_CACHE_SIZE, 1245 SPDK_ENV_SOCKET_ID_ANY); 1246 next_request_data = spdk_mempool_get(rtransport.data_wr_pool); 1247 SPDK_CU_ASSERT_FATAL(((struct test_mempool *)rtransport.data_wr_pool)->count == 1248 SPDK_NVMF_MAX_SGL_ENTRIES - 1); 1249 next_request_data->wr.wr_id = (uint64_t)&rdma_req.data_wr; 1250 next_request_data->wr.num_sge = 2; 1251 next_request_data->wr.next = NULL; 1252 rdma_req.data.wr.next = &next_request_data->wr; 1253 rdma_req.data.wr.wr_id = (uint64_t)&rdma_req.data_wr; 1254 rdma_req.data.wr.num_sge = 2; 1255 rdma_req.transfer_wr = &rdma_req.data.wr; 1256 1257 nvmf_rdma_request_free_data(&rdma_req, &rtransport); 1258 /* Check if next_request_data put into memory pool */ 1259 CU_ASSERT(((struct test_mempool *)rtransport.data_wr_pool)->count == SPDK_NVMF_MAX_SGL_ENTRIES); 1260 CU_ASSERT(rdma_req.data.wr.num_sge == 0); 1261 1262 spdk_mempool_free(rtransport.data_wr_pool); 1263 } 1264 1265 static void 1266 test_nvmf_rdma_resources_create(void) 1267 { 1268 static struct spdk_nvmf_rdma_resources *rdma_resource; 1269 struct spdk_nvmf_rdma_resource_opts opts = {}; 1270 struct spdk_nvmf_rdma_qpair qpair = {}; 1271 struct spdk_nvmf_rdma_recv *recv = NULL; 1272 struct spdk_nvmf_rdma_request *req = NULL; 1273 const int DEPTH = 128; 1274 1275 opts.max_queue_depth = DEPTH; 1276 opts.in_capsule_data_size = 4096; 1277 opts.shared = true; 1278 opts.qpair = &qpair; 1279 1280 rdma_resource = nvmf_rdma_resources_create(&opts); 1281 CU_ASSERT(rdma_resource != NULL); 1282 /* Just check first and last entry */ 1283 recv = &rdma_resource->recvs[0]; 1284 req = &rdma_resource->reqs[0]; 1285 CU_ASSERT(recv->rdma_wr.type == RDMA_WR_TYPE_RECV); 1286 CU_ASSERT((uintptr_t)recv->buf == (uintptr_t)(rdma_resource->bufs)); 1287 CU_ASSERT(recv->sgl[0].addr == (uintptr_t)&rdma_resource->cmds[0]); 1288 CU_ASSERT(recv->sgl[0].length == sizeof(rdma_resource->cmds[0])); 1289 CU_ASSERT(recv->sgl[0].lkey == RDMA_UT_LKEY); 1290 CU_ASSERT(recv->wr.num_sge == 2); 1291 CU_ASSERT(recv->wr.wr_id == (uintptr_t)&rdma_resource->recvs[0].rdma_wr); 1292 CU_ASSERT(recv->wr.sg_list == rdma_resource->recvs[0].sgl); 1293 CU_ASSERT(req->req.rsp == &rdma_resource->cpls[0]); 1294 CU_ASSERT(req->rsp.sgl[0].addr == (uintptr_t)&rdma_resource->cpls[0]); 1295 CU_ASSERT(req->rsp.sgl[0].length == sizeof(rdma_resource->cpls[0])); 1296 CU_ASSERT(req->rsp.sgl[0].lkey == RDMA_UT_LKEY); 1297 CU_ASSERT(req->rsp_wr.type == RDMA_WR_TYPE_SEND); 1298 CU_ASSERT(req->rsp.wr.wr_id == (uintptr_t)&rdma_resource->reqs[0].rsp_wr); 1299 CU_ASSERT(req->rsp.wr.next == NULL); 1300 CU_ASSERT(req->rsp.wr.opcode == IBV_WR_SEND); 1301 CU_ASSERT(req->rsp.wr.send_flags == IBV_SEND_SIGNALED); 1302 CU_ASSERT(req->rsp.wr.sg_list == rdma_resource->reqs[0].rsp.sgl); 1303 CU_ASSERT(req->rsp.wr.num_sge == NVMF_DEFAULT_RSP_SGE); 1304 CU_ASSERT(req->data_wr.type == RDMA_WR_TYPE_DATA); 1305 CU_ASSERT(req->data.wr.wr_id == (uintptr_t)&rdma_resource->reqs[0].data_wr); 1306 CU_ASSERT(req->data.wr.next == NULL); 1307 CU_ASSERT(req->data.wr.send_flags == IBV_SEND_SIGNALED); 1308 CU_ASSERT(req->data.wr.sg_list == rdma_resource->reqs[0].data.sgl); 1309 CU_ASSERT(req->data.wr.num_sge == SPDK_NVMF_MAX_SGL_ENTRIES); 1310 CU_ASSERT(req->state == RDMA_REQUEST_STATE_FREE); 1311 1312 recv = &rdma_resource->recvs[DEPTH - 1]; 1313 req = &rdma_resource->reqs[DEPTH - 1]; 1314 CU_ASSERT(recv->rdma_wr.type == RDMA_WR_TYPE_RECV); 1315 CU_ASSERT((uintptr_t)recv->buf == (uintptr_t)(rdma_resource->bufs + 1316 (DEPTH - 1) * 4096)); 1317 CU_ASSERT(recv->sgl[0].addr == (uintptr_t)&rdma_resource->cmds[DEPTH - 1]); 1318 CU_ASSERT(recv->sgl[0].length == sizeof(rdma_resource->cmds[DEPTH - 1])); 1319 CU_ASSERT(recv->sgl[0].lkey == RDMA_UT_LKEY); 1320 CU_ASSERT(recv->wr.num_sge == 2); 1321 CU_ASSERT(recv->wr.wr_id == (uintptr_t)&rdma_resource->recvs[DEPTH - 1].rdma_wr); 1322 CU_ASSERT(recv->wr.sg_list == rdma_resource->recvs[DEPTH - 1].sgl); 1323 CU_ASSERT(req->req.rsp == &rdma_resource->cpls[DEPTH - 1]); 1324 CU_ASSERT(req->rsp.sgl[0].addr == (uintptr_t)&rdma_resource->cpls[DEPTH - 1]); 1325 CU_ASSERT(req->rsp.sgl[0].length == sizeof(rdma_resource->cpls[DEPTH - 1])); 1326 CU_ASSERT(req->rsp.sgl[0].lkey == RDMA_UT_LKEY); 1327 CU_ASSERT(req->rsp_wr.type == RDMA_WR_TYPE_SEND); 1328 CU_ASSERT(req->rsp.wr.wr_id == (uintptr_t)&req->rsp_wr); 1329 CU_ASSERT(req->rsp.wr.next == NULL); 1330 CU_ASSERT(req->rsp.wr.opcode == IBV_WR_SEND); 1331 CU_ASSERT(req->rsp.wr.send_flags == IBV_SEND_SIGNALED); 1332 CU_ASSERT(req->rsp.wr.sg_list == rdma_resource->reqs[DEPTH - 1].rsp.sgl); 1333 CU_ASSERT(req->rsp.wr.num_sge == NVMF_DEFAULT_RSP_SGE); 1334 CU_ASSERT(req->data_wr.type == RDMA_WR_TYPE_DATA); 1335 CU_ASSERT(req->data.wr.wr_id == (uintptr_t)&req->data_wr); 1336 CU_ASSERT(req->data.wr.next == NULL); 1337 CU_ASSERT(req->data.wr.send_flags == IBV_SEND_SIGNALED); 1338 CU_ASSERT(req->data.wr.sg_list == rdma_resource->reqs[DEPTH - 1].data.sgl); 1339 CU_ASSERT(req->data.wr.num_sge == SPDK_NVMF_MAX_SGL_ENTRIES); 1340 CU_ASSERT(req->state == RDMA_REQUEST_STATE_FREE); 1341 1342 nvmf_rdma_resources_destroy(rdma_resource); 1343 } 1344 1345 static void 1346 test_nvmf_rdma_qpair_compare(void) 1347 { 1348 struct spdk_nvmf_rdma_qpair rqpair1 = {}, rqpair2 = {}; 1349 1350 rqpair1.qp_num = 0; 1351 rqpair2.qp_num = UINT32_MAX; 1352 1353 CU_ASSERT(nvmf_rdma_qpair_compare(&rqpair1, &rqpair2) < 0); 1354 CU_ASSERT(nvmf_rdma_qpair_compare(&rqpair2, &rqpair1) > 0); 1355 } 1356 1357 static void 1358 test_nvmf_rdma_resize_cq(void) 1359 { 1360 int rc = -1; 1361 int tnum_wr = 0; 1362 int tnum_cqe = 0; 1363 struct spdk_nvmf_rdma_qpair rqpair = {}; 1364 struct spdk_nvmf_rdma_poller rpoller = {}; 1365 struct spdk_nvmf_rdma_device rdevice = {}; 1366 struct ibv_context ircontext = {}; 1367 struct ibv_device idevice = {}; 1368 1369 rdevice.context = &ircontext; 1370 rqpair.poller = &rpoller; 1371 ircontext.device = &idevice; 1372 1373 /* Test1: Current capacity support required size. */ 1374 rpoller.required_num_wr = 10; 1375 rpoller.num_cqe = 20; 1376 rqpair.max_queue_depth = 2; 1377 tnum_wr = rpoller.required_num_wr; 1378 tnum_cqe = rpoller.num_cqe; 1379 1380 rc = nvmf_rdma_resize_cq(&rqpair, &rdevice); 1381 CU_ASSERT(rc == 0); 1382 CU_ASSERT(rpoller.required_num_wr == 10 + MAX_WR_PER_QP(rqpair.max_queue_depth)); 1383 CU_ASSERT(rpoller.required_num_wr > tnum_wr); 1384 CU_ASSERT(rpoller.num_cqe == tnum_cqe); 1385 1386 /* Test2: iWARP doesn't support CQ resize. */ 1387 tnum_wr = rpoller.required_num_wr; 1388 tnum_cqe = rpoller.num_cqe; 1389 idevice.transport_type = IBV_TRANSPORT_IWARP; 1390 1391 rc = nvmf_rdma_resize_cq(&rqpair, &rdevice); 1392 CU_ASSERT(rc == -1); 1393 CU_ASSERT(rpoller.required_num_wr == tnum_wr); 1394 CU_ASSERT(rpoller.num_cqe == tnum_cqe); 1395 1396 1397 /* Test3: RDMA CQE requirement exceeds device max_cqe limitation. */ 1398 tnum_wr = rpoller.required_num_wr; 1399 tnum_cqe = rpoller.num_cqe; 1400 idevice.transport_type = IBV_TRANSPORT_UNKNOWN; 1401 rdevice.attr.max_cqe = 3; 1402 1403 rc = nvmf_rdma_resize_cq(&rqpair, &rdevice); 1404 CU_ASSERT(rc == -1); 1405 CU_ASSERT(rpoller.required_num_wr == tnum_wr); 1406 CU_ASSERT(rpoller.num_cqe == tnum_cqe); 1407 1408 /* Test4: RDMA CQ resize failed. */ 1409 tnum_wr = rpoller.required_num_wr; 1410 tnum_cqe = rpoller.num_cqe; 1411 idevice.transport_type = IBV_TRANSPORT_IB; 1412 rdevice.attr.max_cqe = 30; 1413 MOCK_SET(ibv_resize_cq, -1); 1414 1415 rc = nvmf_rdma_resize_cq(&rqpair, &rdevice); 1416 CU_ASSERT(rc == -1); 1417 CU_ASSERT(rpoller.required_num_wr == tnum_wr); 1418 CU_ASSERT(rpoller.num_cqe == tnum_cqe); 1419 1420 /* Test5: RDMA CQ resize success. rsize = MIN(MAX(num_cqe * 2, required_num_wr), device->attr.max_cqe). */ 1421 tnum_wr = rpoller.required_num_wr; 1422 tnum_cqe = rpoller.num_cqe; 1423 MOCK_SET(ibv_resize_cq, 0); 1424 1425 rc = nvmf_rdma_resize_cq(&rqpair, &rdevice); 1426 CU_ASSERT(rc == 0); 1427 CU_ASSERT(rpoller.num_cqe = 30); 1428 CU_ASSERT(rpoller.required_num_wr == 18 + MAX_WR_PER_QP(rqpair.max_queue_depth)); 1429 CU_ASSERT(rpoller.required_num_wr > tnum_wr); 1430 CU_ASSERT(rpoller.num_cqe > tnum_cqe); 1431 } 1432 1433 int 1434 main(int argc, char **argv) 1435 { 1436 CU_pSuite suite = NULL; 1437 unsigned int num_failures; 1438 1439 CU_initialize_registry(); 1440 1441 suite = CU_add_suite("nvmf", NULL, NULL); 1442 1443 CU_ADD_TEST(suite, test_spdk_nvmf_rdma_request_parse_sgl); 1444 CU_ADD_TEST(suite, test_spdk_nvmf_rdma_request_process); 1445 CU_ADD_TEST(suite, test_nvmf_rdma_get_optimal_poll_group); 1446 CU_ADD_TEST(suite, test_spdk_nvmf_rdma_request_parse_sgl_with_md); 1447 CU_ADD_TEST(suite, test_nvmf_rdma_opts_init); 1448 CU_ADD_TEST(suite, test_nvmf_rdma_request_free_data); 1449 CU_ADD_TEST(suite, test_nvmf_rdma_resources_create); 1450 CU_ADD_TEST(suite, test_nvmf_rdma_qpair_compare); 1451 CU_ADD_TEST(suite, test_nvmf_rdma_resize_cq); 1452 1453 num_failures = spdk_ut_run_tests(argc, argv, NULL); 1454 CU_cleanup_registry(); 1455 return num_failures; 1456 } 1457