1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2018 Intel Corporation. All rights reserved. 3 * Copyright (c) 2019, 2021 Mellanox Technologies LTD. All rights reserved. 4 * Copyright (c) 2023, 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 5 */ 6 7 #include "spdk/stdinc.h" 8 #include "spdk_internal/cunit.h" 9 #include "common/lib/test_env.c" 10 #include "common/lib/test_iobuf.c" 11 #include "common/lib/test_rdma.c" 12 #include "nvmf/rdma.c" 13 #include "nvmf/transport.c" 14 15 #define RDMA_UT_UNITS_IN_MAX_IO 16 16 17 struct spdk_nvmf_transport_opts g_rdma_ut_transport_opts = { 18 .max_queue_depth = SPDK_NVMF_RDMA_DEFAULT_MAX_QUEUE_DEPTH, 19 .max_qpairs_per_ctrlr = SPDK_NVMF_RDMA_DEFAULT_MAX_QPAIRS_PER_CTRLR, 20 .in_capsule_data_size = SPDK_NVMF_RDMA_DEFAULT_IN_CAPSULE_DATA_SIZE, 21 .max_io_size = (SPDK_NVMF_RDMA_MIN_IO_BUFFER_SIZE * RDMA_UT_UNITS_IN_MAX_IO), 22 .io_unit_size = SPDK_NVMF_RDMA_MIN_IO_BUFFER_SIZE, 23 .max_aq_depth = SPDK_NVMF_RDMA_DEFAULT_AQ_DEPTH, 24 .num_shared_buffers = SPDK_NVMF_RDMA_DEFAULT_NUM_SHARED_BUFFERS, 25 }; 26 27 SPDK_LOG_REGISTER_COMPONENT(nvmf) 28 DEFINE_STUB(spdk_mem_map_set_translation, int, (struct spdk_mem_map *map, uint64_t vaddr, 29 uint64_t size, uint64_t translation), 0); 30 DEFINE_STUB(spdk_mem_map_clear_translation, int, (struct spdk_mem_map *map, uint64_t vaddr, 31 uint64_t size), 0); 32 DEFINE_STUB(spdk_mem_map_alloc, struct spdk_mem_map *, (uint64_t default_translation, 33 const struct spdk_mem_map_ops *ops, void *cb_ctx), NULL); 34 DEFINE_STUB(spdk_nvmf_qpair_disconnect, int, (struct spdk_nvmf_qpair *qpair), 0); 35 DEFINE_STUB(spdk_nvmf_qpair_get_listen_trid, int, 36 (struct spdk_nvmf_qpair *qpair, struct spdk_nvme_transport_id *trid), 0); 37 DEFINE_STUB_V(spdk_mem_map_free, (struct spdk_mem_map **pmap)); 38 39 DEFINE_STUB_V(spdk_nvmf_request_exec, (struct spdk_nvmf_request *req)); 40 DEFINE_STUB(spdk_nvmf_request_complete, int, (struct spdk_nvmf_request *req), 0); 41 DEFINE_STUB(spdk_nvme_transport_id_compare, int, (const struct spdk_nvme_transport_id *trid1, 42 const struct spdk_nvme_transport_id *trid2), 0); 43 DEFINE_STUB_V(spdk_nvmf_ctrlr_abort_aer, (struct spdk_nvmf_ctrlr *ctrlr)); 44 DEFINE_STUB(spdk_nvmf_request_get_dif_ctx, bool, (struct spdk_nvmf_request *req, 45 struct spdk_dif_ctx *dif_ctx), false); 46 DEFINE_STUB_V(spdk_nvme_trid_populate_transport, (struct spdk_nvme_transport_id *trid, 47 enum spdk_nvme_transport_type trtype)); 48 DEFINE_STUB_V(spdk_nvmf_tgt_new_qpair, (struct spdk_nvmf_tgt *tgt, struct spdk_nvmf_qpair *qpair)); 49 DEFINE_STUB(nvmf_ctrlr_abort_request, int, (struct spdk_nvmf_request *req), 0); 50 DEFINE_STUB(spdk_nvme_transport_id_adrfam_str, const char *, (enum spdk_nvmf_adrfam adrfam), NULL); 51 DEFINE_STUB(ibv_dereg_mr, int, (struct ibv_mr *mr), 0); 52 DEFINE_STUB(ibv_resize_cq, int, (struct ibv_cq *cq, int cqe), 0); 53 DEFINE_STUB(spdk_mempool_lookup, struct spdk_mempool *, (const char *name), NULL); 54 DEFINE_STUB(spdk_rdma_cm_id_get_numa_id, int32_t, (struct rdma_cm_id *cm_id), 0); 55 56 /* ibv_reg_mr can be a macro, need to undefine it */ 57 #ifdef ibv_reg_mr 58 #undef ibv_reg_mr 59 #endif 60 61 DEFINE_RETURN_MOCK(ibv_reg_mr, struct ibv_mr *); 62 struct ibv_mr * 63 ibv_reg_mr(struct ibv_pd *pd, void *addr, size_t length, int access) 64 { 65 HANDLE_RETURN_MOCK(ibv_reg_mr); 66 if (length > 0) { 67 return &g_rdma_mr; 68 } else { 69 return NULL; 70 } 71 } 72 73 int 74 ibv_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, 75 int attr_mask, struct ibv_qp_init_attr *init_attr) 76 { 77 if (qp == NULL) { 78 return -1; 79 } else { 80 attr->port_num = 80; 81 82 if (qp->state == IBV_QPS_ERR) { 83 attr->qp_state = 10; 84 } else { 85 attr->qp_state = IBV_QPS_INIT; 86 } 87 88 return 0; 89 } 90 } 91 92 const char * 93 spdk_nvme_transport_id_trtype_str(enum spdk_nvme_transport_type trtype) 94 { 95 switch (trtype) { 96 case SPDK_NVME_TRANSPORT_PCIE: 97 return "PCIe"; 98 case SPDK_NVME_TRANSPORT_RDMA: 99 return "RDMA"; 100 case SPDK_NVME_TRANSPORT_FC: 101 return "FC"; 102 default: 103 return NULL; 104 } 105 } 106 107 int 108 spdk_nvme_transport_id_populate_trstring(struct spdk_nvme_transport_id *trid, const char *trstring) 109 { 110 int len, i; 111 112 if (trstring == NULL) { 113 return -EINVAL; 114 } 115 116 len = strnlen(trstring, SPDK_NVMF_TRSTRING_MAX_LEN); 117 if (len == SPDK_NVMF_TRSTRING_MAX_LEN) { 118 return -EINVAL; 119 } 120 121 /* cast official trstring to uppercase version of input. */ 122 for (i = 0; i < len; i++) { 123 trid->trstring[i] = toupper(trstring[i]); 124 } 125 return 0; 126 } 127 128 static void 129 reset_nvmf_rdma_request(struct spdk_nvmf_rdma_request *rdma_req) 130 { 131 int i; 132 133 rdma_req->req.length = 0; 134 rdma_req->req.data_from_pool = false; 135 rdma_req->data.wr.num_sge = 0; 136 rdma_req->data.wr.wr.rdma.remote_addr = 0; 137 rdma_req->data.wr.wr.rdma.rkey = 0; 138 rdma_req->offset = 0; 139 memset(&rdma_req->req.dif, 0, sizeof(rdma_req->req.dif)); 140 141 for (i = 0; i < SPDK_NVMF_MAX_SGL_ENTRIES; i++) { 142 rdma_req->req.iov[i].iov_base = 0; 143 rdma_req->req.iov[i].iov_len = 0; 144 rdma_req->data.wr.sg_list[i].addr = 0; 145 rdma_req->data.wr.sg_list[i].length = 0; 146 rdma_req->data.wr.sg_list[i].lkey = 0; 147 } 148 rdma_req->req.iovcnt = 0; 149 if (rdma_req->req.stripped_data) { 150 free(rdma_req->req.stripped_data); 151 rdma_req->req.stripped_data = NULL; 152 } 153 } 154 155 static void 156 test_spdk_nvmf_rdma_request_parse_sgl(void) 157 { 158 struct spdk_nvmf_rdma_transport rtransport = {}; 159 struct spdk_nvmf_transport_ops ops = {}; 160 struct spdk_nvmf_rdma_device device; 161 struct spdk_nvmf_rdma_request rdma_req = {}; 162 struct spdk_nvmf_rdma_recv recv; 163 struct spdk_nvmf_rdma_poll_group group; 164 struct spdk_nvmf_rdma_qpair rqpair; 165 struct spdk_nvmf_rdma_poller poller; 166 union nvmf_c2h_msg cpl; 167 union nvmf_h2c_msg cmd; 168 struct spdk_nvme_sgl_descriptor *sgl; 169 struct spdk_nvme_sgl_descriptor sgl_desc[SPDK_NVMF_MAX_SGL_ENTRIES] = {{0}}; 170 struct spdk_nvmf_rdma_request_data data; 171 int rc, i; 172 uint32_t sgl_length; 173 174 data.wr.sg_list = data.sgl; 175 group.group.transport = &rtransport.transport; 176 poller.group = &group; 177 rqpair.poller = &poller; 178 rqpair.max_send_sge = SPDK_NVMF_MAX_SGL_ENTRIES; 179 180 sgl = &cmd.nvme_cmd.dptr.sgl1; 181 rdma_req.recv = &recv; 182 rdma_req.req.cmd = &cmd; 183 rdma_req.req.rsp = &cpl; 184 rdma_req.data.wr.sg_list = rdma_req.data.sgl; 185 rdma_req.req.qpair = &rqpair.qpair; 186 rdma_req.req.xfer = SPDK_NVME_DATA_CONTROLLER_TO_HOST; 187 188 rtransport.transport.opts = g_rdma_ut_transport_opts; 189 rtransport.data_wr_pool = NULL; 190 rtransport.transport.ops = &ops; 191 192 device.attr.device_cap_flags = 0; 193 sgl->keyed.key = 0xEEEE; 194 sgl->address = 0xFFFF; 195 rdma_req.recv->buf = (void *)0xDDDD; 196 197 /* Test 1: sgl type: keyed data block subtype: address */ 198 sgl->generic.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; 199 sgl->keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; 200 201 /* Part 1: simple I/O, one SGL smaller than the transport io unit size */ 202 MOCK_SET(spdk_iobuf_get, (void *)0x2000); 203 reset_nvmf_rdma_request(&rdma_req); 204 sgl->keyed.length = rtransport.transport.opts.io_unit_size / 2; 205 206 device.map = (void *)0x0; 207 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 208 CU_ASSERT(rc == 0); 209 CU_ASSERT(rdma_req.req.data_from_pool == true); 210 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size / 2); 211 CU_ASSERT((uint64_t)rdma_req.req.iovcnt == 1); 212 CU_ASSERT((uint64_t)rdma_req.req.iov[0].iov_base == 0x2000); 213 CU_ASSERT(rdma_req.data.wr.num_sge == 1); 214 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 215 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 216 CU_ASSERT((uint64_t)rdma_req.req.iov[0].iov_base == 0x2000); 217 CU_ASSERT(rdma_req.data.wr.sg_list[0].addr == 0x2000); 218 CU_ASSERT(rdma_req.data.wr.sg_list[0].length == rtransport.transport.opts.io_unit_size / 2); 219 CU_ASSERT(rdma_req.data.wr.sg_list[0].lkey == RDMA_UT_LKEY); 220 221 /* Part 2: simple I/O, one SGL larger than the transport io unit size (equal to the max io size) */ 222 reset_nvmf_rdma_request(&rdma_req); 223 sgl->keyed.length = rtransport.transport.opts.io_unit_size * RDMA_UT_UNITS_IN_MAX_IO; 224 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 225 226 CU_ASSERT(rc == 0); 227 CU_ASSERT(rdma_req.req.data_from_pool == true); 228 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size * RDMA_UT_UNITS_IN_MAX_IO); 229 CU_ASSERT(rdma_req.data.wr.num_sge == RDMA_UT_UNITS_IN_MAX_IO); 230 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 231 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 232 for (i = 0; i < RDMA_UT_UNITS_IN_MAX_IO; i++) { 233 CU_ASSERT((uint64_t)rdma_req.req.iov[i].iov_base == 0x2000); 234 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == 0x2000); 235 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == rtransport.transport.opts.io_unit_size); 236 CU_ASSERT(rdma_req.data.wr.sg_list[i].lkey == RDMA_UT_LKEY); 237 } 238 239 /* Part 3: simple I/O one SGL larger than the transport max io size */ 240 reset_nvmf_rdma_request(&rdma_req); 241 sgl->keyed.length = rtransport.transport.opts.max_io_size * 2; 242 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 243 244 CU_ASSERT(rc == -1); 245 246 /* Part 4: Pretend there are no buffer pools */ 247 MOCK_SET(spdk_iobuf_get, NULL); 248 reset_nvmf_rdma_request(&rdma_req); 249 sgl->keyed.length = rtransport.transport.opts.io_unit_size * RDMA_UT_UNITS_IN_MAX_IO; 250 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 251 252 CU_ASSERT(rc == 0); 253 CU_ASSERT(rdma_req.req.data_from_pool == false); 254 CU_ASSERT(rdma_req.req.iovcnt == 0); 255 CU_ASSERT(rdma_req.data.wr.num_sge == 0); 256 CU_ASSERT(rdma_req.req.iov[0].iov_base == NULL); 257 CU_ASSERT(rdma_req.data.wr.sg_list[0].addr == 0); 258 CU_ASSERT(rdma_req.data.wr.sg_list[0].length == 0); 259 CU_ASSERT(rdma_req.data.wr.sg_list[0].lkey == 0); 260 261 rdma_req.recv->buf = (void *)0xDDDD; 262 /* Test 2: sgl type: keyed data block subtype: offset (in capsule data) */ 263 sgl->generic.type = SPDK_NVME_SGL_TYPE_DATA_BLOCK; 264 sgl->unkeyed.subtype = SPDK_NVME_SGL_SUBTYPE_OFFSET; 265 266 /* Part 1: Normal I/O smaller than in capsule data size no offset */ 267 reset_nvmf_rdma_request(&rdma_req); 268 sgl->address = 0; 269 sgl->unkeyed.length = rtransport.transport.opts.in_capsule_data_size; 270 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 271 272 CU_ASSERT(rc == 0); 273 CU_ASSERT(rdma_req.req.iovcnt == 1); 274 CU_ASSERT(rdma_req.req.iov[0].iov_base == (void *)0xDDDD); 275 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.in_capsule_data_size); 276 CU_ASSERT(rdma_req.req.data_from_pool == false); 277 278 /* Part 2: I/O offset + length too large */ 279 reset_nvmf_rdma_request(&rdma_req); 280 sgl->address = rtransport.transport.opts.in_capsule_data_size; 281 sgl->unkeyed.length = rtransport.transport.opts.in_capsule_data_size; 282 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 283 284 CU_ASSERT(rc == -1); 285 286 /* Part 3: I/O too large */ 287 reset_nvmf_rdma_request(&rdma_req); 288 sgl->address = 0; 289 sgl->unkeyed.length = rtransport.transport.opts.in_capsule_data_size * 2; 290 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 291 292 CU_ASSERT(rc == -1); 293 294 /* Test 3: Multi SGL */ 295 sgl->generic.type = SPDK_NVME_SGL_TYPE_LAST_SEGMENT; 296 sgl->unkeyed.subtype = SPDK_NVME_SGL_SUBTYPE_OFFSET; 297 sgl->address = 0; 298 rdma_req.recv->buf = (void *)&sgl_desc; 299 MOCK_SET(spdk_iobuf_get, &data); 300 MOCK_SET(spdk_mempool_get, &data); 301 302 /* part 1: 2 segments each with 1 wr. */ 303 reset_nvmf_rdma_request(&rdma_req); 304 sgl->unkeyed.length = 2 * sizeof(struct spdk_nvme_sgl_descriptor); 305 for (i = 0; i < 2; i++) { 306 sgl_desc[i].keyed.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; 307 sgl_desc[i].keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; 308 sgl_desc[i].keyed.length = rtransport.transport.opts.io_unit_size; 309 sgl_desc[i].address = 0x4000 + i * rtransport.transport.opts.io_unit_size; 310 sgl_desc[i].keyed.key = 0x44; 311 } 312 313 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 314 315 CU_ASSERT(rc == 0); 316 CU_ASSERT(rdma_req.req.data_from_pool == true); 317 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size * 2); 318 CU_ASSERT(rdma_req.data.wr.num_sge == 1); 319 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0x44); 320 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0x4000); 321 CU_ASSERT(rdma_req.data.wr.next == &data.wr); 322 CU_ASSERT(data.wr.wr.rdma.rkey == 0x44); 323 CU_ASSERT(data.wr.wr.rdma.remote_addr == 0x4000 + rtransport.transport.opts.io_unit_size); 324 CU_ASSERT(data.wr.num_sge == 1); 325 CU_ASSERT(data.wr.next == &rdma_req.rsp.wr); 326 327 /* part 2: 2 segments, each with 1 wr containing 8 sge_elements */ 328 reset_nvmf_rdma_request(&rdma_req); 329 sgl->unkeyed.length = 2 * sizeof(struct spdk_nvme_sgl_descriptor); 330 for (i = 0; i < 2; i++) { 331 sgl_desc[i].keyed.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; 332 sgl_desc[i].keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; 333 sgl_desc[i].keyed.length = rtransport.transport.opts.io_unit_size * 8; 334 sgl_desc[i].address = 0x4000 + i * 8 * rtransport.transport.opts.io_unit_size; 335 sgl_desc[i].keyed.key = 0x44; 336 } 337 338 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 339 340 CU_ASSERT(rc == 0); 341 CU_ASSERT(rdma_req.req.data_from_pool == true); 342 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size * 16); 343 CU_ASSERT(rdma_req.req.iovcnt == 16); 344 CU_ASSERT(rdma_req.data.wr.num_sge == 8); 345 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0x44); 346 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0x4000); 347 CU_ASSERT(rdma_req.data.wr.next == &data.wr); 348 CU_ASSERT(data.wr.wr.rdma.rkey == 0x44); 349 CU_ASSERT(data.wr.wr.rdma.remote_addr == 0x4000 + rtransport.transport.opts.io_unit_size * 8); 350 CU_ASSERT(data.wr.num_sge == 8); 351 CU_ASSERT(data.wr.next == &rdma_req.rsp.wr); 352 353 /* part 3: 2 segments, one very large, one very small */ 354 reset_nvmf_rdma_request(&rdma_req); 355 for (i = 0; i < 2; i++) { 356 sgl_desc[i].keyed.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; 357 sgl_desc[i].keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; 358 sgl_desc[i].keyed.key = 0x44; 359 } 360 361 sgl_desc[0].keyed.length = rtransport.transport.opts.io_unit_size * 15 + 362 rtransport.transport.opts.io_unit_size / 2; 363 sgl_desc[0].address = 0x4000; 364 sgl_desc[1].keyed.length = rtransport.transport.opts.io_unit_size / 2; 365 sgl_desc[1].address = 0x4000 + rtransport.transport.opts.io_unit_size * 15 + 366 rtransport.transport.opts.io_unit_size / 2; 367 368 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 369 370 CU_ASSERT(rc == 0); 371 CU_ASSERT(rdma_req.req.data_from_pool == true); 372 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size * 16); 373 CU_ASSERT(rdma_req.req.iovcnt == 16); 374 CU_ASSERT(rdma_req.data.wr.num_sge == 16); 375 for (i = 0; i < 15; i++) { 376 CU_ASSERT(rdma_req.data.sgl[i].length == rtransport.transport.opts.io_unit_size); 377 } 378 CU_ASSERT(rdma_req.data.sgl[15].length == rtransport.transport.opts.io_unit_size / 2); 379 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0x44); 380 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0x4000); 381 CU_ASSERT(rdma_req.data.wr.next == &data.wr); 382 CU_ASSERT(data.wr.wr.rdma.rkey == 0x44); 383 CU_ASSERT(data.wr.wr.rdma.remote_addr == 0x4000 + rtransport.transport.opts.io_unit_size * 15 + 384 rtransport.transport.opts.io_unit_size / 2); 385 CU_ASSERT(data.sgl[0].length == rtransport.transport.opts.io_unit_size / 2); 386 CU_ASSERT(data.wr.num_sge == 1); 387 CU_ASSERT(data.wr.next == &rdma_req.rsp.wr); 388 389 /* part 4: 2 SGL descriptors, each length is transport buffer / 2 390 * 1 transport buffers should be allocated */ 391 reset_nvmf_rdma_request(&rdma_req); 392 sgl->unkeyed.length = 2 * sizeof(struct spdk_nvme_sgl_descriptor); 393 sgl_length = rtransport.transport.opts.io_unit_size / 2; 394 for (i = 0; i < 2; i++) { 395 sgl_desc[i].keyed.length = sgl_length; 396 sgl_desc[i].address = 0x4000 + i * sgl_length; 397 } 398 399 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 400 401 CU_ASSERT(rc == 0); 402 CU_ASSERT(rdma_req.req.data_from_pool == true); 403 CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size); 404 CU_ASSERT(rdma_req.req.iovcnt == 1); 405 406 CU_ASSERT(rdma_req.data.sgl[0].length == sgl_length); 407 /* We mocked mempool_get to return address of data variable. Mempool is used 408 * to get both additional WRs and data buffers, so data points to &data */ 409 CU_ASSERT(rdma_req.data.sgl[0].addr == (uint64_t)&data); 410 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0x44); 411 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0x4000); 412 CU_ASSERT(rdma_req.data.wr.num_sge == 1); 413 CU_ASSERT(rdma_req.data.wr.next == &data.wr); 414 415 CU_ASSERT(data.wr.wr.rdma.rkey == 0x44); 416 CU_ASSERT(data.wr.wr.rdma.remote_addr == 0x4000 + sgl_length); 417 CU_ASSERT(data.sgl[0].length == sgl_length); 418 CU_ASSERT(data.sgl[0].addr == (uint64_t)&data + sgl_length); 419 CU_ASSERT(data.wr.num_sge == 1); 420 421 MOCK_CLEAR(spdk_mempool_get); 422 MOCK_CLEAR(spdk_iobuf_get); 423 424 reset_nvmf_rdma_request(&rdma_req); 425 } 426 427 static struct spdk_nvmf_rdma_recv * 428 create_recv(struct spdk_nvmf_rdma_qpair *rqpair, enum spdk_nvme_nvm_opcode opc) 429 { 430 struct spdk_nvmf_rdma_recv *rdma_recv; 431 union nvmf_h2c_msg *cmd; 432 struct spdk_nvme_sgl_descriptor *sgl; 433 434 rdma_recv = calloc(1, sizeof(*rdma_recv)); 435 rdma_recv->qpair = rqpair; 436 cmd = calloc(1, sizeof(*cmd)); 437 rdma_recv->sgl[0].addr = (uintptr_t)cmd; 438 cmd->nvme_cmd.opc = opc; 439 sgl = &cmd->nvme_cmd.dptr.sgl1; 440 sgl->keyed.key = 0xEEEE; 441 sgl->address = 0xFFFF; 442 sgl->keyed.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; 443 sgl->keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; 444 sgl->keyed.length = 1; 445 446 return rdma_recv; 447 } 448 449 static void 450 free_recv(struct spdk_nvmf_rdma_recv *rdma_recv) 451 { 452 free((void *)rdma_recv->sgl[0].addr); 453 free(rdma_recv); 454 } 455 456 static struct spdk_nvmf_rdma_request * 457 create_req(struct spdk_nvmf_rdma_qpair *rqpair, 458 struct spdk_nvmf_rdma_recv *rdma_recv) 459 { 460 struct spdk_nvmf_rdma_request *rdma_req; 461 union nvmf_c2h_msg *cpl; 462 463 rdma_req = calloc(1, sizeof(*rdma_req)); 464 rdma_req->recv = rdma_recv; 465 rdma_req->req.qpair = &rqpair->qpair; 466 rdma_req->state = RDMA_REQUEST_STATE_NEW; 467 rdma_req->data.wr.wr_id = (uintptr_t)&rdma_req->data_wr; 468 rdma_req->data.wr.sg_list = rdma_req->data.sgl; 469 cpl = calloc(1, sizeof(*cpl)); 470 rdma_req->rsp.sgl[0].addr = (uintptr_t)cpl; 471 rdma_req->req.rsp = cpl; 472 473 return rdma_req; 474 } 475 476 static void 477 free_req(struct spdk_nvmf_rdma_request *rdma_req) 478 { 479 free((void *)rdma_req->rsp.sgl[0].addr); 480 free(rdma_req); 481 } 482 483 static void 484 qpair_reset(struct spdk_nvmf_rdma_qpair *rqpair, 485 struct spdk_nvmf_rdma_poller *poller, 486 struct spdk_nvmf_rdma_device *device, 487 struct spdk_nvmf_rdma_resources *resources, 488 struct spdk_nvmf_transport *transport) 489 { 490 memset(rqpair, 0, sizeof(*rqpair)); 491 STAILQ_INIT(&rqpair->pending_rdma_write_queue); 492 STAILQ_INIT(&rqpair->pending_rdma_read_queue); 493 STAILQ_INIT(&rqpair->pending_rdma_send_queue); 494 rqpair->poller = poller; 495 rqpair->device = device; 496 rqpair->resources = resources; 497 rqpair->qpair.qid = 1; 498 rqpair->qpair.state = SPDK_NVMF_QPAIR_ENABLED; 499 rqpair->max_send_sge = SPDK_NVMF_MAX_SGL_ENTRIES; 500 rqpair->max_send_depth = 16; 501 rqpair->max_read_depth = 16; 502 rqpair->qpair.transport = transport; 503 } 504 505 static void 506 poller_reset(struct spdk_nvmf_rdma_poller *poller, 507 struct spdk_nvmf_rdma_poll_group *group) 508 { 509 memset(poller, 0, sizeof(*poller)); 510 STAILQ_INIT(&poller->qpairs_pending_recv); 511 STAILQ_INIT(&poller->qpairs_pending_send); 512 poller->group = group; 513 } 514 515 static void 516 test_spdk_nvmf_rdma_request_process(void) 517 { 518 struct spdk_nvmf_rdma_transport rtransport = {}; 519 struct spdk_nvmf_transport_ops ops = {}; 520 struct spdk_nvmf_rdma_poll_group group = {}; 521 struct spdk_nvmf_rdma_poller poller = {}; 522 struct spdk_nvmf_rdma_device device = {}; 523 struct spdk_nvmf_rdma_resources resources = {}; 524 struct spdk_nvmf_rdma_qpair rqpair = {}; 525 struct spdk_nvmf_rdma_recv *rdma_recv; 526 struct spdk_nvmf_rdma_request *rdma_req; 527 struct spdk_iobuf_channel ch = {}; 528 bool progress; 529 530 group.group.buf_cache = &ch; 531 532 STAILQ_INIT(&group.group.pending_buf_queue); 533 poller_reset(&poller, &group); 534 qpair_reset(&rqpair, &poller, &device, &resources, &rtransport.transport); 535 536 rtransport.transport.opts = g_rdma_ut_transport_opts; 537 rtransport.data_wr_pool = spdk_mempool_create("test_wr_pool", 128, 538 sizeof(struct spdk_nvmf_rdma_request_data), 539 0, 0); 540 rtransport.transport.ops = &ops; 541 MOCK_CLEAR(spdk_iobuf_get); 542 543 device.attr.device_cap_flags = 0; 544 device.map = (void *)0x0; 545 546 /* Test 1: single SGL READ request */ 547 rdma_recv = create_recv(&rqpair, SPDK_NVME_OPC_READ); 548 rdma_req = create_req(&rqpair, rdma_recv); 549 rqpair.current_recv_depth = 1; 550 /* NEW -> EXECUTING */ 551 progress = nvmf_rdma_request_process(&rtransport, rdma_req); 552 CU_ASSERT(progress == true); 553 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_EXECUTING); 554 CU_ASSERT(rdma_req->req.xfer == SPDK_NVME_DATA_CONTROLLER_TO_HOST); 555 /* EXECUTED -> TRANSFERRING_C2H */ 556 rdma_req->state = RDMA_REQUEST_STATE_EXECUTED; 557 progress = nvmf_rdma_request_process(&rtransport, rdma_req); 558 CU_ASSERT(progress == true); 559 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_TRANSFERRING_CONTROLLER_TO_HOST); 560 CU_ASSERT(rdma_req->recv == NULL); 561 /* COMPLETED -> FREE */ 562 rdma_req->state = RDMA_REQUEST_STATE_COMPLETED; 563 progress = nvmf_rdma_request_process(&rtransport, rdma_req); 564 CU_ASSERT(progress == true); 565 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_FREE); 566 567 free_recv(rdma_recv); 568 free_req(rdma_req); 569 poller_reset(&poller, &group); 570 qpair_reset(&rqpair, &poller, &device, &resources, &rtransport.transport); 571 572 /* Test 2: single SGL WRITE request */ 573 rdma_recv = create_recv(&rqpair, SPDK_NVME_OPC_WRITE); 574 rdma_req = create_req(&rqpair, rdma_recv); 575 rqpair.current_recv_depth = 1; 576 /* NEW -> TRANSFERRING_H2C */ 577 progress = nvmf_rdma_request_process(&rtransport, rdma_req); 578 CU_ASSERT(progress == true); 579 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_TRANSFERRING_HOST_TO_CONTROLLER); 580 CU_ASSERT(rdma_req->req.xfer == SPDK_NVME_DATA_HOST_TO_CONTROLLER); 581 STAILQ_INIT(&poller.qpairs_pending_send); 582 /* READY_TO_EXECUTE -> EXECUTING */ 583 rdma_req->state = RDMA_REQUEST_STATE_READY_TO_EXECUTE; 584 progress = nvmf_rdma_request_process(&rtransport, rdma_req); 585 CU_ASSERT(progress == true); 586 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_EXECUTING); 587 /* EXECUTED -> COMPLETING */ 588 rdma_req->state = RDMA_REQUEST_STATE_EXECUTED; 589 progress = nvmf_rdma_request_process(&rtransport, rdma_req); 590 CU_ASSERT(progress == true); 591 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_COMPLETING); 592 CU_ASSERT(rdma_req->recv == NULL); 593 /* COMPLETED -> FREE */ 594 rdma_req->state = RDMA_REQUEST_STATE_COMPLETED; 595 progress = nvmf_rdma_request_process(&rtransport, rdma_req); 596 CU_ASSERT(progress == true); 597 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_FREE); 598 599 free_recv(rdma_recv); 600 free_req(rdma_req); 601 poller_reset(&poller, &group); 602 qpair_reset(&rqpair, &poller, &device, &resources, &rtransport.transport); 603 604 /* Test 3: WRITE+WRITE ibv_send batching */ 605 { 606 struct spdk_nvmf_rdma_recv *recv1, *recv2; 607 struct spdk_nvmf_rdma_request *req1, *req2; 608 recv1 = create_recv(&rqpair, SPDK_NVME_OPC_WRITE); 609 req1 = create_req(&rqpair, recv1); 610 recv2 = create_recv(&rqpair, SPDK_NVME_OPC_WRITE); 611 req2 = create_req(&rqpair, recv2); 612 613 /* WRITE 1: NEW -> TRANSFERRING_H2C */ 614 rqpair.current_recv_depth = 1; 615 nvmf_rdma_request_process(&rtransport, req1); 616 CU_ASSERT(req1->state == RDMA_REQUEST_STATE_TRANSFERRING_HOST_TO_CONTROLLER); 617 618 /* WRITE 2: NEW -> TRANSFERRING_H2C */ 619 rqpair.current_recv_depth = 2; 620 nvmf_rdma_request_process(&rtransport, req2); 621 CU_ASSERT(req2->state == RDMA_REQUEST_STATE_TRANSFERRING_HOST_TO_CONTROLLER); 622 623 STAILQ_INIT(&poller.qpairs_pending_send); 624 625 /* WRITE 1 completes before WRITE 2 has finished RDMA reading */ 626 /* WRITE 1: READY_TO_EXECUTE -> EXECUTING */ 627 req1->state = RDMA_REQUEST_STATE_READY_TO_EXECUTE; 628 nvmf_rdma_request_process(&rtransport, req1); 629 CU_ASSERT(req1->state == RDMA_REQUEST_STATE_EXECUTING); 630 /* WRITE 1: EXECUTED -> COMPLETING */ 631 req1->state = RDMA_REQUEST_STATE_EXECUTED; 632 nvmf_rdma_request_process(&rtransport, req1); 633 CU_ASSERT(req1->state == RDMA_REQUEST_STATE_COMPLETING); 634 STAILQ_INIT(&poller.qpairs_pending_send); 635 /* WRITE 1: COMPLETED -> FREE */ 636 req1->state = RDMA_REQUEST_STATE_COMPLETED; 637 nvmf_rdma_request_process(&rtransport, req1); 638 CU_ASSERT(req1->state == RDMA_REQUEST_STATE_FREE); 639 640 /* Now WRITE 2 has finished reading and completes */ 641 /* WRITE 2: COMPLETED -> FREE */ 642 /* WRITE 2: READY_TO_EXECUTE -> EXECUTING */ 643 req2->state = RDMA_REQUEST_STATE_READY_TO_EXECUTE; 644 nvmf_rdma_request_process(&rtransport, req2); 645 CU_ASSERT(req2->state == RDMA_REQUEST_STATE_EXECUTING); 646 /* WRITE 1: EXECUTED -> COMPLETING */ 647 req2->state = RDMA_REQUEST_STATE_EXECUTED; 648 nvmf_rdma_request_process(&rtransport, req2); 649 CU_ASSERT(req2->state == RDMA_REQUEST_STATE_COMPLETING); 650 STAILQ_INIT(&poller.qpairs_pending_send); 651 /* WRITE 1: COMPLETED -> FREE */ 652 req2->state = RDMA_REQUEST_STATE_COMPLETED; 653 nvmf_rdma_request_process(&rtransport, req2); 654 CU_ASSERT(req2->state == RDMA_REQUEST_STATE_FREE); 655 656 free_recv(recv1); 657 free_req(req1); 658 free_recv(recv2); 659 free_req(req2); 660 poller_reset(&poller, &group); 661 qpair_reset(&rqpair, &poller, &device, &resources, &rtransport.transport); 662 } 663 664 /* Test 4, invalid command, check xfer type */ 665 { 666 struct spdk_nvmf_rdma_recv *rdma_recv_inv; 667 struct spdk_nvmf_rdma_request *rdma_req_inv; 668 /* construct an opcode that specifies BIDIRECTIONAL transfer */ 669 uint8_t opc = 0x10 | SPDK_NVME_DATA_BIDIRECTIONAL; 670 671 rdma_recv_inv = create_recv(&rqpair, opc); 672 rdma_req_inv = create_req(&rqpair, rdma_recv_inv); 673 674 /* NEW -> RDMA_REQUEST_STATE_COMPLETING */ 675 rqpair.current_recv_depth = 1; 676 progress = nvmf_rdma_request_process(&rtransport, rdma_req_inv); 677 CU_ASSERT(progress == true); 678 CU_ASSERT(rdma_req_inv->state == RDMA_REQUEST_STATE_COMPLETING); 679 CU_ASSERT(rdma_req_inv->req.rsp->nvme_cpl.status.sct == SPDK_NVME_SCT_GENERIC); 680 CU_ASSERT(rdma_req_inv->req.rsp->nvme_cpl.status.sc == SPDK_NVME_SC_INVALID_OPCODE); 681 682 /* RDMA_REQUEST_STATE_COMPLETED -> FREE */ 683 rdma_req_inv->state = RDMA_REQUEST_STATE_COMPLETED; 684 nvmf_rdma_request_process(&rtransport, rdma_req_inv); 685 CU_ASSERT(rdma_req_inv->state == RDMA_REQUEST_STATE_FREE); 686 687 free_recv(rdma_recv_inv); 688 free_req(rdma_req_inv); 689 poller_reset(&poller, &group); 690 qpair_reset(&rqpair, &poller, &device, &resources, &rtransport.transport); 691 } 692 693 /* Test 5: Write response waits in queue */ 694 { 695 rdma_recv = create_recv(&rqpair, SPDK_NVME_OPC_WRITE); 696 rdma_req = create_req(&rqpair, rdma_recv); 697 rqpair.current_recv_depth = 1; 698 /* NEW -> TRANSFERRING_H2C */ 699 progress = nvmf_rdma_request_process(&rtransport, rdma_req); 700 CU_ASSERT(progress == true); 701 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_TRANSFERRING_HOST_TO_CONTROLLER); 702 CU_ASSERT(rdma_req->req.xfer == SPDK_NVME_DATA_HOST_TO_CONTROLLER); 703 STAILQ_INIT(&poller.qpairs_pending_send); 704 /* READY_TO_EXECUTE -> EXECUTING */ 705 rdma_req->state = RDMA_REQUEST_STATE_READY_TO_EXECUTE; 706 progress = nvmf_rdma_request_process(&rtransport, rdma_req); 707 CU_ASSERT(progress == true); 708 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_EXECUTING); 709 /* EXECUTED -> COMPLETING */ 710 rdma_req->state = RDMA_REQUEST_STATE_EXECUTED; 711 /* Send queue is full */ 712 rqpair.current_send_depth = rqpair.max_send_depth; 713 progress = nvmf_rdma_request_process(&rtransport, rdma_req); 714 CU_ASSERT(progress == true); 715 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_READY_TO_COMPLETE_PENDING); 716 CU_ASSERT(rdma_req == STAILQ_FIRST(&rqpair.pending_rdma_send_queue)); 717 718 /* Send queue is still full */ 719 progress = nvmf_rdma_request_process(&rtransport, rdma_req); 720 CU_ASSERT(progress == false); 721 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_READY_TO_COMPLETE_PENDING); 722 CU_ASSERT(rdma_req == STAILQ_FIRST(&rqpair.pending_rdma_send_queue)); 723 724 /* Slot is available */ 725 rqpair.current_send_depth = rqpair.max_send_depth - 1; 726 progress = nvmf_rdma_request_process(&rtransport, rdma_req); 727 CU_ASSERT(progress == true); 728 CU_ASSERT(STAILQ_EMPTY(&rqpair.pending_rdma_send_queue)); 729 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_COMPLETING); 730 CU_ASSERT(rdma_req->recv == NULL); 731 /* COMPLETED -> FREE */ 732 rdma_req->state = RDMA_REQUEST_STATE_COMPLETED; 733 progress = nvmf_rdma_request_process(&rtransport, rdma_req); 734 CU_ASSERT(progress == true); 735 CU_ASSERT(rdma_req->state == RDMA_REQUEST_STATE_FREE); 736 737 free_recv(rdma_recv); 738 free_req(rdma_req); 739 poller_reset(&poller, &group); 740 qpair_reset(&rqpair, &poller, &device, &resources, &rtransport.transport); 741 742 } 743 744 spdk_mempool_free(rtransport.data_wr_pool); 745 } 746 747 #define TEST_GROUPS_COUNT 5 748 static void 749 test_nvmf_rdma_get_optimal_poll_group(void) 750 { 751 struct spdk_nvmf_rdma_transport rtransport = {}; 752 struct spdk_nvmf_transport *transport = &rtransport.transport; 753 struct spdk_nvmf_rdma_qpair rqpair = {}; 754 struct spdk_nvmf_transport_poll_group *groups[TEST_GROUPS_COUNT]; 755 struct spdk_nvmf_rdma_poll_group *rgroups[TEST_GROUPS_COUNT]; 756 struct spdk_nvmf_transport_poll_group *result; 757 struct spdk_nvmf_poll_group group = {}; 758 uint32_t i; 759 760 rqpair.qpair.transport = transport; 761 TAILQ_INIT(&rtransport.poll_groups); 762 763 for (i = 0; i < TEST_GROUPS_COUNT; i++) { 764 groups[i] = nvmf_rdma_poll_group_create(transport, NULL); 765 CU_ASSERT(groups[i] != NULL); 766 groups[i]->group = &group; 767 rgroups[i] = SPDK_CONTAINEROF(groups[i], struct spdk_nvmf_rdma_poll_group, group); 768 groups[i]->transport = transport; 769 } 770 CU_ASSERT(rtransport.conn_sched.next_admin_pg == rgroups[0]); 771 CU_ASSERT(rtransport.conn_sched.next_io_pg == rgroups[0]); 772 773 /* Emulate connection of %TEST_GROUPS_COUNT% initiators - each creates 1 admin and 1 io qp */ 774 for (i = 0; i < TEST_GROUPS_COUNT; i++) { 775 rqpair.qpair.qid = 0; 776 result = nvmf_rdma_get_optimal_poll_group(&rqpair.qpair); 777 CU_ASSERT(result == groups[i]); 778 CU_ASSERT(rtransport.conn_sched.next_admin_pg == rgroups[(i + 1) % TEST_GROUPS_COUNT]); 779 CU_ASSERT(rtransport.conn_sched.next_io_pg == rgroups[i]); 780 781 rqpair.qpair.qid = 1; 782 result = nvmf_rdma_get_optimal_poll_group(&rqpair.qpair); 783 CU_ASSERT(result == groups[i]); 784 CU_ASSERT(rtransport.conn_sched.next_admin_pg == rgroups[(i + 1) % TEST_GROUPS_COUNT]); 785 CU_ASSERT(rtransport.conn_sched.next_io_pg == rgroups[(i + 1) % TEST_GROUPS_COUNT]); 786 } 787 /* wrap around, admin/io pg point to the first pg 788 Destroy all poll groups except of the last one */ 789 for (i = 0; i < TEST_GROUPS_COUNT - 1; i++) { 790 nvmf_rdma_poll_group_destroy(groups[i]); 791 CU_ASSERT(rtransport.conn_sched.next_admin_pg == rgroups[i + 1]); 792 CU_ASSERT(rtransport.conn_sched.next_io_pg == rgroups[i + 1]); 793 } 794 795 CU_ASSERT(rtransport.conn_sched.next_admin_pg == rgroups[TEST_GROUPS_COUNT - 1]); 796 CU_ASSERT(rtransport.conn_sched.next_io_pg == rgroups[TEST_GROUPS_COUNT - 1]); 797 798 /* Check that pointers to the next admin/io poll groups are not changed */ 799 rqpair.qpair.qid = 0; 800 result = nvmf_rdma_get_optimal_poll_group(&rqpair.qpair); 801 CU_ASSERT(result == groups[TEST_GROUPS_COUNT - 1]); 802 CU_ASSERT(rtransport.conn_sched.next_admin_pg == rgroups[TEST_GROUPS_COUNT - 1]); 803 CU_ASSERT(rtransport.conn_sched.next_io_pg == rgroups[TEST_GROUPS_COUNT - 1]); 804 805 rqpair.qpair.qid = 1; 806 result = nvmf_rdma_get_optimal_poll_group(&rqpair.qpair); 807 CU_ASSERT(result == groups[TEST_GROUPS_COUNT - 1]); 808 CU_ASSERT(rtransport.conn_sched.next_admin_pg == rgroups[TEST_GROUPS_COUNT - 1]); 809 CU_ASSERT(rtransport.conn_sched.next_io_pg == rgroups[TEST_GROUPS_COUNT - 1]); 810 811 /* Remove the last poll group, check that pointers are NULL */ 812 nvmf_rdma_poll_group_destroy(groups[TEST_GROUPS_COUNT - 1]); 813 CU_ASSERT(rtransport.conn_sched.next_admin_pg == NULL); 814 CU_ASSERT(rtransport.conn_sched.next_io_pg == NULL); 815 816 /* Request optimal poll group, result must be NULL */ 817 rqpair.qpair.qid = 0; 818 result = nvmf_rdma_get_optimal_poll_group(&rqpair.qpair); 819 CU_ASSERT(result == NULL); 820 821 rqpair.qpair.qid = 1; 822 result = nvmf_rdma_get_optimal_poll_group(&rqpair.qpair); 823 CU_ASSERT(result == NULL); 824 } 825 #undef TEST_GROUPS_COUNT 826 827 static void 828 test_spdk_nvmf_rdma_request_parse_sgl_with_md(void) 829 { 830 struct spdk_nvmf_rdma_transport rtransport = {}; 831 struct spdk_nvmf_transport_ops ops = {}; 832 struct spdk_nvmf_rdma_device device; 833 struct spdk_nvmf_rdma_request rdma_req = {}; 834 struct spdk_nvmf_rdma_recv recv; 835 struct spdk_nvmf_rdma_poll_group group; 836 struct spdk_nvmf_rdma_qpair rqpair; 837 struct spdk_nvmf_rdma_poller poller; 838 union nvmf_c2h_msg cpl; 839 union nvmf_h2c_msg cmd; 840 struct spdk_nvme_sgl_descriptor *sgl; 841 struct spdk_nvme_sgl_descriptor sgl_desc[SPDK_NVMF_MAX_SGL_ENTRIES] = {{0}}; 842 char data_buffer[8192]; 843 struct spdk_nvmf_rdma_request_data *data = (struct spdk_nvmf_rdma_request_data *)data_buffer; 844 char data2_buffer[8192]; 845 struct spdk_nvmf_rdma_request_data *data2 = (struct spdk_nvmf_rdma_request_data *)data2_buffer; 846 const uint32_t data_bs = 512; 847 const uint32_t md_size = 8; 848 int rc, i; 849 struct spdk_dif_ctx_init_ext_opts dif_opts; 850 851 MOCK_CLEAR(spdk_mempool_get); 852 MOCK_CLEAR(spdk_iobuf_get); 853 854 data->wr.sg_list = data->sgl; 855 group.group.transport = &rtransport.transport; 856 poller.group = &group; 857 rqpair.poller = &poller; 858 rqpair.max_send_sge = SPDK_NVMF_MAX_SGL_ENTRIES; 859 860 sgl = &cmd.nvme_cmd.dptr.sgl1; 861 rdma_req.recv = &recv; 862 rdma_req.req.cmd = &cmd; 863 rdma_req.req.rsp = &cpl; 864 rdma_req.data.wr.sg_list = rdma_req.data.sgl; 865 rdma_req.req.qpair = &rqpair.qpair; 866 rdma_req.req.xfer = SPDK_NVME_DATA_CONTROLLER_TO_HOST; 867 868 rtransport.transport.opts = g_rdma_ut_transport_opts; 869 rtransport.data_wr_pool = NULL; 870 rtransport.transport.ops = &ops; 871 872 device.attr.device_cap_flags = 0; 873 device.map = NULL; 874 sgl->keyed.key = 0xEEEE; 875 sgl->address = 0xFFFF; 876 rdma_req.recv->buf = (void *)0xDDDD; 877 878 /* Test 1: sgl type: keyed data block subtype: address */ 879 sgl->generic.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; 880 sgl->keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; 881 882 /* Part 1: simple I/O, one SGL smaller than the transport io unit size, block size 512 */ 883 MOCK_SET(spdk_iobuf_get, (void *)0x2000); 884 reset_nvmf_rdma_request(&rdma_req); 885 dif_opts.size = SPDK_SIZEOF(&dif_opts, dif_pi_format); 886 dif_opts.dif_pi_format = SPDK_DIF_PI_FORMAT_16; 887 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 888 SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 889 0, 0, 0, 0, 0, &dif_opts); 890 rdma_req.req.dif_enabled = true; 891 rtransport.transport.opts.io_unit_size = data_bs * 8; 892 rdma_req.req.qpair->transport = &rtransport.transport; 893 sgl->keyed.length = data_bs * 4; 894 895 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 896 897 CU_ASSERT(rc == 0); 898 CU_ASSERT(rdma_req.req.data_from_pool == true); 899 CU_ASSERT(rdma_req.req.length == data_bs * 4); 900 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 901 CU_ASSERT(rdma_req.req.dif.elba_length == (data_bs + md_size) * 4); 902 CU_ASSERT(rdma_req.req.iovcnt == 1); 903 CU_ASSERT((uint64_t)rdma_req.req.iov[0].iov_base == 0x2000); 904 CU_ASSERT(rdma_req.data.wr.num_sge == 1); 905 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 906 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 907 CU_ASSERT((uint64_t)rdma_req.req.iov[0].iov_base == 0x2000); 908 909 CU_ASSERT(rdma_req.data.wr.sg_list[0].addr == 0x2000); 910 CU_ASSERT(rdma_req.data.wr.sg_list[0].length == rdma_req.req.length); 911 CU_ASSERT(rdma_req.data.wr.sg_list[0].lkey == RDMA_UT_LKEY); 912 913 /* Part 2: simple I/O, one SGL equal to io unit size, io_unit_size is not aligned with md_size, 914 block size 512 */ 915 MOCK_SET(spdk_iobuf_get, (void *)0x2000); 916 reset_nvmf_rdma_request(&rdma_req); 917 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 918 SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 919 0, 0, 0, 0, 0, &dif_opts); 920 rdma_req.req.dif_enabled = true; 921 rtransport.transport.opts.io_unit_size = data_bs * 4; 922 sgl->keyed.length = data_bs * 4; 923 924 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 925 926 CU_ASSERT(rc == 0); 927 CU_ASSERT(rdma_req.req.data_from_pool == true); 928 CU_ASSERT(rdma_req.req.length == data_bs * 4); 929 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 930 CU_ASSERT(rdma_req.req.dif.elba_length == (data_bs + md_size) * 4); 931 CU_ASSERT(rdma_req.req.iovcnt == 2); 932 CU_ASSERT((uint64_t)rdma_req.req.iov[0].iov_base == 0x2000); 933 CU_ASSERT(rdma_req.data.wr.num_sge == 5); 934 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 935 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 936 CU_ASSERT((uint64_t)rdma_req.req.iov[0].iov_base == 0x2000); 937 938 for (i = 0; i < 3; ++i) { 939 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == 0x2000 + i * (data_bs + md_size)); 940 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == data_bs); 941 CU_ASSERT(rdma_req.data.wr.sg_list[i].lkey == RDMA_UT_LKEY); 942 } 943 CU_ASSERT(rdma_req.data.wr.sg_list[3].addr == 0x2000 + 3 * (data_bs + md_size)); 944 CU_ASSERT(rdma_req.data.wr.sg_list[3].length == 488); 945 CU_ASSERT(rdma_req.data.wr.sg_list[3].lkey == RDMA_UT_LKEY); 946 947 /* 2nd buffer consumed */ 948 CU_ASSERT(rdma_req.data.wr.sg_list[4].addr == 0x2000); 949 CU_ASSERT(rdma_req.data.wr.sg_list[4].length == 24); 950 CU_ASSERT(rdma_req.data.wr.sg_list[4].lkey == RDMA_UT_LKEY); 951 952 /* Part 3: simple I/O, one SGL equal io unit size, io_unit_size is equal to block size 512 bytes */ 953 MOCK_SET(spdk_iobuf_get, (void *)0x2000); 954 reset_nvmf_rdma_request(&rdma_req); 955 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 956 SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 957 0, 0, 0, 0, 0, &dif_opts); 958 rdma_req.req.dif_enabled = true; 959 rtransport.transport.opts.io_unit_size = data_bs; 960 sgl->keyed.length = data_bs; 961 962 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 963 964 CU_ASSERT(rc == 0); 965 CU_ASSERT(rdma_req.req.data_from_pool == true); 966 CU_ASSERT(rdma_req.req.length == data_bs); 967 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 968 CU_ASSERT(rdma_req.req.dif.elba_length == data_bs + md_size); 969 CU_ASSERT(rdma_req.req.iovcnt == 2); 970 CU_ASSERT((uint64_t)rdma_req.req.iov[0].iov_base == 0x2000); 971 CU_ASSERT(rdma_req.data.wr.num_sge == 1); 972 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 973 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 974 CU_ASSERT((uint64_t)rdma_req.req.iov[0].iov_base == 0x2000); 975 976 CU_ASSERT(rdma_req.data.wr.sg_list[0].addr == 0x2000); 977 CU_ASSERT(rdma_req.data.wr.sg_list[0].length == data_bs); 978 CU_ASSERT(rdma_req.data.wr.sg_list[0].lkey == RDMA_UT_LKEY); 979 980 CU_ASSERT(rdma_req.req.iovcnt == 2); 981 CU_ASSERT(rdma_req.req.iov[0].iov_base == (void *)((unsigned long)0x2000)); 982 CU_ASSERT(rdma_req.req.iov[0].iov_len == data_bs); 983 /* 2nd buffer consumed for metadata */ 984 CU_ASSERT(rdma_req.req.iov[1].iov_base == (void *)((unsigned long)0x2000)); 985 CU_ASSERT(rdma_req.req.iov[1].iov_len == md_size); 986 987 /* Part 4: simple I/O, one SGL equal io unit size, io_unit_size is aligned with md_size, 988 block size 512 */ 989 MOCK_SET(spdk_iobuf_get, (void *)0x2000); 990 reset_nvmf_rdma_request(&rdma_req); 991 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 992 SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 993 0, 0, 0, 0, 0, &dif_opts); 994 rdma_req.req.dif_enabled = true; 995 rtransport.transport.opts.io_unit_size = (data_bs + md_size) * 4; 996 sgl->keyed.length = data_bs * 4; 997 998 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 999 1000 CU_ASSERT(rc == 0); 1001 CU_ASSERT(rdma_req.req.data_from_pool == true); 1002 CU_ASSERT(rdma_req.req.length == data_bs * 4); 1003 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 1004 CU_ASSERT(rdma_req.req.dif.elba_length == (data_bs + md_size) * 4); 1005 CU_ASSERT(rdma_req.req.iovcnt == 1); 1006 CU_ASSERT((uint64_t)rdma_req.req.iov[0].iov_base == 0x2000); 1007 CU_ASSERT(rdma_req.data.wr.num_sge == 1); 1008 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 1009 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 1010 CU_ASSERT((uint64_t)rdma_req.req.iov[0].iov_base == 0x2000); 1011 1012 CU_ASSERT(rdma_req.data.wr.sg_list[0].addr == 0x2000); 1013 CU_ASSERT(rdma_req.data.wr.sg_list[0].length == rdma_req.req.length); 1014 CU_ASSERT(rdma_req.data.wr.sg_list[0].lkey == RDMA_UT_LKEY); 1015 1016 /* Part 5: simple I/O, one SGL equal to 2x io unit size, io_unit_size is aligned with md_size, 1017 block size 512 */ 1018 MOCK_SET(spdk_iobuf_get, (void *)0x2000); 1019 reset_nvmf_rdma_request(&rdma_req); 1020 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 1021 SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 1022 0, 0, 0, 0, 0, &dif_opts); 1023 rdma_req.req.dif_enabled = true; 1024 rtransport.transport.opts.io_unit_size = (data_bs + md_size) * 2; 1025 sgl->keyed.length = data_bs * 4; 1026 1027 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 1028 1029 CU_ASSERT(rc == 0); 1030 CU_ASSERT(rdma_req.req.data_from_pool == true); 1031 CU_ASSERT(rdma_req.req.length == data_bs * 4); 1032 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 1033 CU_ASSERT(rdma_req.req.dif.elba_length == (data_bs + md_size) * 4); 1034 CU_ASSERT(rdma_req.req.iovcnt == 2); 1035 CU_ASSERT((uint64_t)rdma_req.req.iov[0].iov_base == 0x2000); 1036 CU_ASSERT(rdma_req.data.wr.num_sge == 2); 1037 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 1038 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 1039 CU_ASSERT((uint64_t)rdma_req.req.iov[0].iov_base == 0x2000); 1040 1041 for (i = 0; i < 2; ++i) { 1042 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == 0x2000); 1043 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == data_bs * 2); 1044 } 1045 1046 /* Part 6: simple I/O, one SGL larger than the transport io unit size, io_unit_size is not aligned to md_size, 1047 block size 512 */ 1048 MOCK_SET(spdk_iobuf_get, (void *)0x2000); 1049 reset_nvmf_rdma_request(&rdma_req); 1050 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 1051 SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 1052 0, 0, 0, 0, 0, &dif_opts); 1053 rdma_req.req.dif_enabled = true; 1054 rtransport.transport.opts.io_unit_size = data_bs * 4; 1055 sgl->keyed.length = data_bs * 6; 1056 1057 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 1058 1059 CU_ASSERT(rc == 0); 1060 CU_ASSERT(rdma_req.req.data_from_pool == true); 1061 CU_ASSERT(rdma_req.req.length == data_bs * 6); 1062 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 1063 CU_ASSERT(rdma_req.req.dif.elba_length == (data_bs + md_size) * 6); 1064 CU_ASSERT(rdma_req.req.iovcnt == 2); 1065 CU_ASSERT((uint64_t)rdma_req.req.iov[0].iov_base == 0x2000); 1066 CU_ASSERT(rdma_req.data.wr.num_sge == 7); 1067 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 1068 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 1069 CU_ASSERT((uint64_t)rdma_req.req.iov[0].iov_base == 0x2000); 1070 1071 for (i = 0; i < 3; ++i) { 1072 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == 0x2000 + i * (data_bs + md_size)); 1073 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == data_bs); 1074 CU_ASSERT(rdma_req.data.wr.sg_list[i].lkey == RDMA_UT_LKEY); 1075 } 1076 CU_ASSERT(rdma_req.data.wr.sg_list[3].addr == 0x2000 + 3 * (data_bs + md_size)); 1077 CU_ASSERT(rdma_req.data.wr.sg_list[3].length == 488); 1078 CU_ASSERT(rdma_req.data.wr.sg_list[3].lkey == RDMA_UT_LKEY); 1079 1080 /* 2nd IO buffer consumed */ 1081 CU_ASSERT(rdma_req.data.wr.sg_list[4].addr == 0x2000); 1082 CU_ASSERT(rdma_req.data.wr.sg_list[4].length == 24); 1083 CU_ASSERT(rdma_req.data.wr.sg_list[4].lkey == RDMA_UT_LKEY); 1084 1085 CU_ASSERT(rdma_req.data.wr.sg_list[5].addr == 0x2000 + 24 + md_size); 1086 CU_ASSERT(rdma_req.data.wr.sg_list[5].length == 512); 1087 CU_ASSERT(rdma_req.data.wr.sg_list[5].lkey == RDMA_UT_LKEY); 1088 1089 CU_ASSERT(rdma_req.data.wr.sg_list[6].addr == 0x2000 + 24 + 512 + md_size * 2); 1090 CU_ASSERT(rdma_req.data.wr.sg_list[6].length == 512); 1091 CU_ASSERT(rdma_req.data.wr.sg_list[6].lkey == RDMA_UT_LKEY); 1092 1093 /* Part 7: simple I/O, number of SGL entries exceeds the number of entries 1094 one WR can hold. Additional WR is chained */ 1095 MOCK_SET(spdk_iobuf_get, data2_buffer); 1096 MOCK_SET(spdk_mempool_get, data2_buffer); 1097 reset_nvmf_rdma_request(&rdma_req); 1098 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 1099 SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 1100 0, 0, 0, 0, 0, &dif_opts); 1101 rdma_req.req.dif_enabled = true; 1102 rtransport.transport.opts.io_unit_size = data_bs * 16; 1103 sgl->keyed.length = data_bs * 16; 1104 1105 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 1106 1107 CU_ASSERT(rc == 0); 1108 CU_ASSERT(rdma_req.req.data_from_pool == true); 1109 CU_ASSERT(rdma_req.req.length == data_bs * 16); 1110 CU_ASSERT(rdma_req.req.iovcnt == 2); 1111 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 1112 CU_ASSERT(rdma_req.req.dif.elba_length == (data_bs + md_size) * 16); 1113 CU_ASSERT(rdma_req.req.iov[0].iov_base == data2_buffer); 1114 CU_ASSERT(rdma_req.data.wr.num_sge == 16); 1115 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 1116 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 1117 1118 for (i = 0; i < 15; ++i) { 1119 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == (uintptr_t)data2_buffer + i * (data_bs + md_size)); 1120 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == data_bs); 1121 CU_ASSERT(rdma_req.data.wr.sg_list[i].lkey == RDMA_UT_LKEY); 1122 } 1123 1124 /* 8192 - (512 + 8) * 15 = 392 */ 1125 CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == (uintptr_t)data2_buffer + i * (data_bs + md_size)); 1126 CU_ASSERT(rdma_req.data.wr.sg_list[i].length == 392); 1127 CU_ASSERT(rdma_req.data.wr.sg_list[i].lkey == RDMA_UT_LKEY); 1128 1129 /* additional wr from pool */ 1130 CU_ASSERT(rdma_req.data.wr.next == (void *)&data2->wr); 1131 CU_ASSERT(rdma_req.data.wr.next->num_sge == 1); 1132 CU_ASSERT(rdma_req.data.wr.next->next == &rdma_req.rsp.wr); 1133 /* 2nd IO buffer */ 1134 CU_ASSERT(data2->wr.sg_list[0].addr == (uintptr_t)data2_buffer); 1135 CU_ASSERT(data2->wr.sg_list[0].length == 120); 1136 CU_ASSERT(data2->wr.sg_list[0].lkey == RDMA_UT_LKEY); 1137 1138 /* Part 8: simple I/O, data with metadata do not fit to 1 io_buffer */ 1139 MOCK_SET(spdk_iobuf_get, (void *)0x2000); 1140 reset_nvmf_rdma_request(&rdma_req); 1141 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 1142 SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 1143 0, 0, 0, 0, 0, &dif_opts); 1144 rdma_req.req.dif_enabled = true; 1145 rtransport.transport.opts.io_unit_size = 516; 1146 sgl->keyed.length = data_bs * 2; 1147 1148 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 1149 1150 CU_ASSERT(rc == 0); 1151 CU_ASSERT(rdma_req.req.data_from_pool == true); 1152 CU_ASSERT(rdma_req.req.length == data_bs * 2); 1153 CU_ASSERT(rdma_req.req.iovcnt == 3); 1154 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 1155 CU_ASSERT(rdma_req.req.dif.elba_length == (data_bs + md_size) * 2); 1156 CU_ASSERT(rdma_req.req.iov[0].iov_base == (void *)0x2000); 1157 CU_ASSERT(rdma_req.data.wr.num_sge == 2); 1158 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); 1159 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); 1160 1161 CU_ASSERT(rdma_req.data.wr.sg_list[0].addr == 0x2000); 1162 CU_ASSERT(rdma_req.data.wr.sg_list[0].length == 512); 1163 CU_ASSERT(rdma_req.data.wr.sg_list[0].lkey == RDMA_UT_LKEY); 1164 1165 /* 2nd IO buffer consumed, offset 4 bytes due to part of the metadata 1166 is located at the beginning of that buffer */ 1167 CU_ASSERT(rdma_req.data.wr.sg_list[1].addr == 0x2000 + 4); 1168 CU_ASSERT(rdma_req.data.wr.sg_list[1].length == 512); 1169 CU_ASSERT(rdma_req.data.wr.sg_list[1].lkey == RDMA_UT_LKEY); 1170 1171 /* Test 2: Multi SGL */ 1172 sgl->generic.type = SPDK_NVME_SGL_TYPE_LAST_SEGMENT; 1173 sgl->unkeyed.subtype = SPDK_NVME_SGL_SUBTYPE_OFFSET; 1174 sgl->address = 0; 1175 rdma_req.recv->buf = (void *)&sgl_desc; 1176 MOCK_SET(spdk_mempool_get, data_buffer); 1177 MOCK_SET(spdk_iobuf_get, data_buffer); 1178 1179 /* part 1: 2 segments each with 1 wr. io_unit_size is aligned with data_bs + md_size */ 1180 reset_nvmf_rdma_request(&rdma_req); 1181 spdk_dif_ctx_init(&rdma_req.req.dif.dif_ctx, data_bs + md_size, md_size, true, false, 1182 SPDK_DIF_TYPE1, 1183 SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK, 1184 0, 0, 0, 0, 0, &dif_opts); 1185 rdma_req.req.dif_enabled = true; 1186 rtransport.transport.opts.io_unit_size = (data_bs + md_size) * 4; 1187 sgl->unkeyed.length = 2 * sizeof(struct spdk_nvme_sgl_descriptor); 1188 1189 for (i = 0; i < 2; i++) { 1190 sgl_desc[i].keyed.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; 1191 sgl_desc[i].keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; 1192 sgl_desc[i].keyed.length = data_bs * 4; 1193 sgl_desc[i].address = 0x4000 + i * data_bs * 4; 1194 sgl_desc[i].keyed.key = 0x44; 1195 } 1196 1197 rc = nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); 1198 1199 CU_ASSERT(rc == 0); 1200 CU_ASSERT(rdma_req.req.data_from_pool == true); 1201 CU_ASSERT(rdma_req.req.length == data_bs * 4 * 2); 1202 CU_ASSERT(rdma_req.req.dif.orig_length == rdma_req.req.length); 1203 CU_ASSERT(rdma_req.req.dif.elba_length == (data_bs + md_size) * 4 * 2); 1204 CU_ASSERT(rdma_req.data.wr.num_sge == 1); 1205 CU_ASSERT(rdma_req.data.wr.sg_list[0].addr == (uintptr_t)(data_buffer)); 1206 CU_ASSERT(rdma_req.data.wr.sg_list[0].length == data_bs * 4); 1207 1208 CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0x44); 1209 CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0x4000); 1210 CU_ASSERT(rdma_req.data.wr.next == &data->wr); 1211 CU_ASSERT(data->wr.wr.rdma.rkey == 0x44); 1212 CU_ASSERT(data->wr.wr.rdma.remote_addr == 0x4000 + data_bs * 4); 1213 CU_ASSERT(data->wr.num_sge == 1); 1214 CU_ASSERT(data->wr.sg_list[0].addr == (uintptr_t)(data_buffer)); 1215 CU_ASSERT(data->wr.sg_list[0].length == data_bs * 4); 1216 1217 CU_ASSERT(data->wr.next == &rdma_req.rsp.wr); 1218 reset_nvmf_rdma_request(&rdma_req); 1219 } 1220 1221 static void 1222 test_nvmf_rdma_opts_init(void) 1223 { 1224 struct spdk_nvmf_transport_opts opts = {}; 1225 1226 nvmf_rdma_opts_init(&opts); 1227 CU_ASSERT(opts.max_queue_depth == SPDK_NVMF_RDMA_DEFAULT_MAX_QUEUE_DEPTH); 1228 CU_ASSERT(opts.max_qpairs_per_ctrlr == SPDK_NVMF_RDMA_DEFAULT_MAX_QPAIRS_PER_CTRLR); 1229 CU_ASSERT(opts.in_capsule_data_size == SPDK_NVMF_RDMA_DEFAULT_IN_CAPSULE_DATA_SIZE); 1230 CU_ASSERT(opts.max_io_size == SPDK_NVMF_RDMA_DEFAULT_MAX_IO_SIZE); 1231 CU_ASSERT(opts.io_unit_size == SPDK_NVMF_RDMA_MIN_IO_BUFFER_SIZE); 1232 CU_ASSERT(opts.max_aq_depth == SPDK_NVMF_RDMA_DEFAULT_AQ_DEPTH); 1233 CU_ASSERT(opts.num_shared_buffers == SPDK_NVMF_RDMA_DEFAULT_NUM_SHARED_BUFFERS); 1234 CU_ASSERT(opts.buf_cache_size == SPDK_NVMF_RDMA_DEFAULT_BUFFER_CACHE_SIZE); 1235 CU_ASSERT(opts.dif_insert_or_strip == SPDK_NVMF_RDMA_DIF_INSERT_OR_STRIP); 1236 CU_ASSERT(opts.abort_timeout_sec == SPDK_NVMF_RDMA_DEFAULT_ABORT_TIMEOUT_SEC); 1237 CU_ASSERT(opts.transport_specific == NULL); 1238 } 1239 1240 static void 1241 test_nvmf_rdma_request_free_data(void) 1242 { 1243 struct spdk_nvmf_rdma_request rdma_req = {}; 1244 struct spdk_nvmf_rdma_transport rtransport = {}; 1245 struct spdk_nvmf_rdma_request_data *next_request_data = NULL; 1246 1247 MOCK_CLEAR(spdk_mempool_get); 1248 rtransport.data_wr_pool = spdk_mempool_create("spdk_nvmf_rdma_wr_data", 1249 SPDK_NVMF_MAX_SGL_ENTRIES, 1250 sizeof(struct spdk_nvmf_rdma_request_data), 1251 SPDK_MEMPOOL_DEFAULT_CACHE_SIZE, 1252 SPDK_ENV_NUMA_ID_ANY); 1253 next_request_data = spdk_mempool_get(rtransport.data_wr_pool); 1254 SPDK_CU_ASSERT_FATAL(((struct test_mempool *)rtransport.data_wr_pool)->count == 1255 SPDK_NVMF_MAX_SGL_ENTRIES - 1); 1256 next_request_data->wr.wr_id = (uint64_t)&rdma_req.data_wr; 1257 next_request_data->wr.num_sge = 2; 1258 next_request_data->wr.next = NULL; 1259 rdma_req.data.wr.next = &next_request_data->wr; 1260 rdma_req.data.wr.wr_id = (uint64_t)&rdma_req.data_wr; 1261 rdma_req.data.wr.num_sge = 2; 1262 rdma_req.transfer_wr = &rdma_req.data.wr; 1263 1264 nvmf_rdma_request_free_data(&rdma_req, &rtransport); 1265 /* Check if next_request_data put into memory pool */ 1266 CU_ASSERT(((struct test_mempool *)rtransport.data_wr_pool)->count == SPDK_NVMF_MAX_SGL_ENTRIES); 1267 CU_ASSERT(rdma_req.data.wr.num_sge == 0); 1268 1269 spdk_mempool_free(rtransport.data_wr_pool); 1270 } 1271 1272 static void 1273 test_nvmf_rdma_resources_create(void) 1274 { 1275 static struct spdk_nvmf_rdma_resources *rdma_resource; 1276 struct spdk_nvmf_rdma_resource_opts opts = {}; 1277 struct spdk_nvmf_rdma_qpair qpair = {}; 1278 struct spdk_nvmf_rdma_recv *recv = NULL; 1279 struct spdk_nvmf_rdma_request *req = NULL; 1280 const int DEPTH = 128; 1281 1282 opts.max_queue_depth = DEPTH; 1283 opts.in_capsule_data_size = 4096; 1284 opts.shared = true; 1285 opts.qpair = &qpair; 1286 1287 rdma_resource = nvmf_rdma_resources_create(&opts); 1288 CU_ASSERT(rdma_resource != NULL); 1289 /* Just check first and last entry */ 1290 recv = &rdma_resource->recvs[0]; 1291 req = &rdma_resource->reqs[0]; 1292 CU_ASSERT(recv->rdma_wr.type == RDMA_WR_TYPE_RECV); 1293 CU_ASSERT((uintptr_t)recv->buf == (uintptr_t)(rdma_resource->bufs)); 1294 CU_ASSERT(recv->sgl[0].addr == (uintptr_t)&rdma_resource->cmds[0]); 1295 CU_ASSERT(recv->sgl[0].length == sizeof(rdma_resource->cmds[0])); 1296 CU_ASSERT(recv->sgl[0].lkey == RDMA_UT_LKEY); 1297 CU_ASSERT(recv->wr.num_sge == 2); 1298 CU_ASSERT(recv->wr.wr_id == (uintptr_t)&rdma_resource->recvs[0].rdma_wr); 1299 CU_ASSERT(recv->wr.sg_list == rdma_resource->recvs[0].sgl); 1300 CU_ASSERT(req->req.rsp == &rdma_resource->cpls[0]); 1301 CU_ASSERT(req->rsp.sgl[0].addr == (uintptr_t)&rdma_resource->cpls[0]); 1302 CU_ASSERT(req->rsp.sgl[0].length == sizeof(rdma_resource->cpls[0])); 1303 CU_ASSERT(req->rsp.sgl[0].lkey == RDMA_UT_LKEY); 1304 CU_ASSERT(req->rsp_wr.type == RDMA_WR_TYPE_SEND); 1305 CU_ASSERT(req->rsp.wr.wr_id == (uintptr_t)&rdma_resource->reqs[0].rsp_wr); 1306 CU_ASSERT(req->rsp.wr.next == NULL); 1307 CU_ASSERT(req->rsp.wr.opcode == IBV_WR_SEND); 1308 CU_ASSERT(req->rsp.wr.send_flags == IBV_SEND_SIGNALED); 1309 CU_ASSERT(req->rsp.wr.sg_list == rdma_resource->reqs[0].rsp.sgl); 1310 CU_ASSERT(req->rsp.wr.num_sge == NVMF_DEFAULT_RSP_SGE); 1311 CU_ASSERT(req->data_wr.type == RDMA_WR_TYPE_DATA); 1312 CU_ASSERT(req->data.wr.wr_id == (uintptr_t)&rdma_resource->reqs[0].data_wr); 1313 CU_ASSERT(req->data.wr.next == NULL); 1314 CU_ASSERT(req->data.wr.send_flags == IBV_SEND_SIGNALED); 1315 CU_ASSERT(req->data.wr.sg_list == rdma_resource->reqs[0].data.sgl); 1316 CU_ASSERT(req->data.wr.num_sge == SPDK_NVMF_MAX_SGL_ENTRIES); 1317 CU_ASSERT(req->state == RDMA_REQUEST_STATE_FREE); 1318 1319 recv = &rdma_resource->recvs[DEPTH - 1]; 1320 req = &rdma_resource->reqs[DEPTH - 1]; 1321 CU_ASSERT(recv->rdma_wr.type == RDMA_WR_TYPE_RECV); 1322 CU_ASSERT((uintptr_t)recv->buf == (uintptr_t)(rdma_resource->bufs + 1323 (DEPTH - 1) * 4096)); 1324 CU_ASSERT(recv->sgl[0].addr == (uintptr_t)&rdma_resource->cmds[DEPTH - 1]); 1325 CU_ASSERT(recv->sgl[0].length == sizeof(rdma_resource->cmds[DEPTH - 1])); 1326 CU_ASSERT(recv->sgl[0].lkey == RDMA_UT_LKEY); 1327 CU_ASSERT(recv->wr.num_sge == 2); 1328 CU_ASSERT(recv->wr.wr_id == (uintptr_t)&rdma_resource->recvs[DEPTH - 1].rdma_wr); 1329 CU_ASSERT(recv->wr.sg_list == rdma_resource->recvs[DEPTH - 1].sgl); 1330 CU_ASSERT(req->req.rsp == &rdma_resource->cpls[DEPTH - 1]); 1331 CU_ASSERT(req->rsp.sgl[0].addr == (uintptr_t)&rdma_resource->cpls[DEPTH - 1]); 1332 CU_ASSERT(req->rsp.sgl[0].length == sizeof(rdma_resource->cpls[DEPTH - 1])); 1333 CU_ASSERT(req->rsp.sgl[0].lkey == RDMA_UT_LKEY); 1334 CU_ASSERT(req->rsp_wr.type == RDMA_WR_TYPE_SEND); 1335 CU_ASSERT(req->rsp.wr.wr_id == (uintptr_t)&req->rsp_wr); 1336 CU_ASSERT(req->rsp.wr.next == NULL); 1337 CU_ASSERT(req->rsp.wr.opcode == IBV_WR_SEND); 1338 CU_ASSERT(req->rsp.wr.send_flags == IBV_SEND_SIGNALED); 1339 CU_ASSERT(req->rsp.wr.sg_list == rdma_resource->reqs[DEPTH - 1].rsp.sgl); 1340 CU_ASSERT(req->rsp.wr.num_sge == NVMF_DEFAULT_RSP_SGE); 1341 CU_ASSERT(req->data_wr.type == RDMA_WR_TYPE_DATA); 1342 CU_ASSERT(req->data.wr.wr_id == (uintptr_t)&req->data_wr); 1343 CU_ASSERT(req->data.wr.next == NULL); 1344 CU_ASSERT(req->data.wr.send_flags == IBV_SEND_SIGNALED); 1345 CU_ASSERT(req->data.wr.sg_list == rdma_resource->reqs[DEPTH - 1].data.sgl); 1346 CU_ASSERT(req->data.wr.num_sge == SPDK_NVMF_MAX_SGL_ENTRIES); 1347 CU_ASSERT(req->state == RDMA_REQUEST_STATE_FREE); 1348 1349 nvmf_rdma_resources_destroy(rdma_resource); 1350 } 1351 1352 static void 1353 test_nvmf_rdma_qpair_compare(void) 1354 { 1355 struct spdk_nvmf_rdma_qpair rqpair1 = {}, rqpair2 = {}; 1356 1357 rqpair1.qp_num = 0; 1358 rqpair2.qp_num = UINT32_MAX; 1359 1360 CU_ASSERT(nvmf_rdma_qpair_compare(&rqpair1, &rqpair2) < 0); 1361 CU_ASSERT(nvmf_rdma_qpair_compare(&rqpair2, &rqpair1) > 0); 1362 } 1363 1364 static void 1365 test_nvmf_rdma_resize_cq(void) 1366 { 1367 int rc = -1; 1368 int tnum_wr = 0; 1369 int tnum_cqe = 0; 1370 struct spdk_nvmf_rdma_qpair rqpair = {}; 1371 struct spdk_nvmf_rdma_poller rpoller = {}; 1372 struct spdk_nvmf_rdma_device rdevice = {}; 1373 struct ibv_context ircontext = {}; 1374 struct ibv_device idevice = {}; 1375 1376 rdevice.context = &ircontext; 1377 rqpair.poller = &rpoller; 1378 ircontext.device = &idevice; 1379 1380 /* Test1: Current capacity support required size. */ 1381 rpoller.required_num_wr = 10; 1382 rpoller.num_cqe = 20; 1383 rqpair.max_queue_depth = 2; 1384 tnum_wr = rpoller.required_num_wr; 1385 tnum_cqe = rpoller.num_cqe; 1386 1387 rc = nvmf_rdma_resize_cq(&rqpair, &rdevice); 1388 CU_ASSERT(rc == 0); 1389 CU_ASSERT(rpoller.required_num_wr == 10 + MAX_WR_PER_QP(rqpair.max_queue_depth)); 1390 CU_ASSERT(rpoller.required_num_wr > tnum_wr); 1391 CU_ASSERT(rpoller.num_cqe == tnum_cqe); 1392 1393 /* Test2: iWARP doesn't support CQ resize. */ 1394 tnum_wr = rpoller.required_num_wr; 1395 tnum_cqe = rpoller.num_cqe; 1396 idevice.transport_type = IBV_TRANSPORT_IWARP; 1397 1398 rc = nvmf_rdma_resize_cq(&rqpair, &rdevice); 1399 CU_ASSERT(rc == -1); 1400 CU_ASSERT(rpoller.required_num_wr == tnum_wr); 1401 CU_ASSERT(rpoller.num_cqe == tnum_cqe); 1402 1403 1404 /* Test3: RDMA CQE requirement exceeds device max_cqe limitation. */ 1405 tnum_wr = rpoller.required_num_wr; 1406 tnum_cqe = rpoller.num_cqe; 1407 idevice.transport_type = IBV_TRANSPORT_UNKNOWN; 1408 rdevice.attr.max_cqe = 3; 1409 1410 rc = nvmf_rdma_resize_cq(&rqpair, &rdevice); 1411 CU_ASSERT(rc == -1); 1412 CU_ASSERT(rpoller.required_num_wr == tnum_wr); 1413 CU_ASSERT(rpoller.num_cqe == tnum_cqe); 1414 1415 /* Test4: RDMA CQ resize failed. */ 1416 tnum_wr = rpoller.required_num_wr; 1417 tnum_cqe = rpoller.num_cqe; 1418 idevice.transport_type = IBV_TRANSPORT_IB; 1419 rdevice.attr.max_cqe = 30; 1420 MOCK_SET(ibv_resize_cq, -1); 1421 1422 rc = nvmf_rdma_resize_cq(&rqpair, &rdevice); 1423 CU_ASSERT(rc == -1); 1424 CU_ASSERT(rpoller.required_num_wr == tnum_wr); 1425 CU_ASSERT(rpoller.num_cqe == tnum_cqe); 1426 1427 /* Test5: RDMA CQ resize success. rsize = MIN(MAX(num_cqe * 2, required_num_wr), device->attr.max_cqe). */ 1428 tnum_wr = rpoller.required_num_wr; 1429 tnum_cqe = rpoller.num_cqe; 1430 MOCK_SET(ibv_resize_cq, 0); 1431 1432 rc = nvmf_rdma_resize_cq(&rqpair, &rdevice); 1433 CU_ASSERT(rc == 0); 1434 CU_ASSERT(rpoller.num_cqe == 30); 1435 CU_ASSERT(rpoller.required_num_wr == 18 + MAX_WR_PER_QP(rqpair.max_queue_depth)); 1436 CU_ASSERT(rpoller.required_num_wr > tnum_wr); 1437 CU_ASSERT(rpoller.num_cqe > tnum_cqe); 1438 } 1439 1440 int 1441 main(int argc, char **argv) 1442 { 1443 CU_pSuite suite = NULL; 1444 unsigned int num_failures; 1445 1446 CU_initialize_registry(); 1447 1448 suite = CU_add_suite("nvmf", NULL, NULL); 1449 1450 CU_ADD_TEST(suite, test_spdk_nvmf_rdma_request_parse_sgl); 1451 CU_ADD_TEST(suite, test_spdk_nvmf_rdma_request_process); 1452 CU_ADD_TEST(suite, test_nvmf_rdma_get_optimal_poll_group); 1453 CU_ADD_TEST(suite, test_spdk_nvmf_rdma_request_parse_sgl_with_md); 1454 CU_ADD_TEST(suite, test_nvmf_rdma_opts_init); 1455 CU_ADD_TEST(suite, test_nvmf_rdma_request_free_data); 1456 CU_ADD_TEST(suite, test_nvmf_rdma_resources_create); 1457 CU_ADD_TEST(suite, test_nvmf_rdma_qpair_compare); 1458 CU_ADD_TEST(suite, test_nvmf_rdma_resize_cq); 1459 1460 num_failures = spdk_ut_run_tests(argc, argv, NULL); 1461 CU_cleanup_registry(); 1462 return num_failures; 1463 } 1464