1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2022 Intel Corporation. 3 * All rights reserved. 4 */ 5 6 #include "spdk/stdinc.h" 7 #include "spdk_internal/cunit.h" 8 #include "spdk/env.h" 9 10 #include "common/lib/ut_multithread.c" 11 12 #include "bdev/raid/raid1.c" 13 #include "../common.c" 14 15 DEFINE_STUB_V(raid_bdev_module_list_add, (struct raid_bdev_module *raid_module)); 16 DEFINE_STUB_V(raid_bdev_module_stop_done, (struct raid_bdev *raid_bdev)); 17 DEFINE_STUB_V(spdk_bdev_free_io, (struct spdk_bdev_io *bdev_io)); 18 DEFINE_STUB_V(raid_bdev_queue_io_wait, (struct raid_bdev_io *raid_io, struct spdk_bdev *bdev, 19 struct spdk_io_channel *ch, spdk_bdev_io_wait_cb cb_fn)); 20 DEFINE_STUB(spdk_bdev_readv_blocks_with_md, int, (struct spdk_bdev_desc *desc, 21 struct spdk_io_channel *ch, 22 struct iovec *iov, int iovcnt, void *md, 23 uint64_t offset_blocks, uint64_t num_blocks, 24 spdk_bdev_io_completion_cb cb, void *cb_arg), 0); 25 DEFINE_STUB(spdk_bdev_writev_blocks_with_md, int, (struct spdk_bdev_desc *desc, 26 struct spdk_io_channel *ch, 27 struct iovec *iov, int iovcnt, void *md, 28 uint64_t offset_blocks, uint64_t num_blocks, 29 spdk_bdev_io_completion_cb cb, void *cb_arg), 0); 30 DEFINE_STUB(spdk_bdev_readv_blocks_ext, int, (struct spdk_bdev_desc *desc, 31 struct spdk_io_channel *ch, 32 struct iovec *iov, int iovcnt, uint64_t offset_blocks, uint64_t num_blocks, 33 spdk_bdev_io_completion_cb cb, void *cb_arg, struct spdk_bdev_ext_io_opts *opts), 0); 34 DEFINE_STUB(spdk_bdev_writev_blocks_ext, int, (struct spdk_bdev_desc *desc, 35 struct spdk_io_channel *ch, 36 struct iovec *iov, int iovcnt, uint64_t offset_blocks, uint64_t num_blocks, 37 spdk_bdev_io_completion_cb cb, void *cb_arg, struct spdk_bdev_ext_io_opts *opts), 0); 38 DEFINE_STUB_V(raid_bdev_process_request_complete, (struct raid_bdev_process_request *process_req, 39 int status)); 40 DEFINE_STUB_V(raid_bdev_io_init, (struct raid_bdev_io *raid_io, 41 struct raid_bdev_io_channel *raid_ch, 42 enum spdk_bdev_io_type type, uint64_t offset_blocks, 43 uint64_t num_blocks, struct iovec *iovs, int iovcnt, void *md_buf, 44 struct spdk_memory_domain *memory_domain, void *memory_domain_ctx)); 45 46 static int 47 test_setup(void) 48 { 49 uint8_t num_base_bdevs_values[] = { 2, 3 }; 50 uint64_t base_bdev_blockcnt_values[] = { 1, 1024, 1024 * 1024 }; 51 uint32_t base_bdev_blocklen_values[] = { 512, 4096 }; 52 uint8_t *num_base_bdevs; 53 uint64_t *base_bdev_blockcnt; 54 uint32_t *base_bdev_blocklen; 55 struct raid_params params; 56 uint64_t params_count; 57 int rc; 58 59 params_count = SPDK_COUNTOF(num_base_bdevs_values) * 60 SPDK_COUNTOF(base_bdev_blockcnt_values) * 61 SPDK_COUNTOF(base_bdev_blocklen_values); 62 rc = raid_test_params_alloc(params_count); 63 if (rc) { 64 return rc; 65 } 66 67 ARRAY_FOR_EACH(num_base_bdevs_values, num_base_bdevs) { 68 ARRAY_FOR_EACH(base_bdev_blockcnt_values, base_bdev_blockcnt) { 69 ARRAY_FOR_EACH(base_bdev_blocklen_values, base_bdev_blocklen) { 70 params.num_base_bdevs = *num_base_bdevs; 71 params.base_bdev_blockcnt = *base_bdev_blockcnt; 72 params.base_bdev_blocklen = *base_bdev_blocklen; 73 params.strip_size = 0; 74 params.md_len = 0; 75 raid_test_params_add(¶ms); 76 } 77 } 78 } 79 80 return 0; 81 } 82 83 static int 84 test_cleanup(void) 85 { 86 raid_test_params_free(); 87 return 0; 88 } 89 90 static struct raid1_info * 91 create_raid1(struct raid_params *params) 92 { 93 struct raid_bdev *raid_bdev = raid_test_create_raid_bdev(params, &g_raid1_module); 94 95 SPDK_CU_ASSERT_FATAL(raid1_start(raid_bdev) == 0); 96 97 return raid_bdev->module_private; 98 } 99 100 static void 101 delete_raid1(struct raid1_info *r1_info) 102 { 103 struct raid_bdev *raid_bdev = r1_info->raid_bdev; 104 105 raid1_stop(raid_bdev); 106 107 raid_test_delete_raid_bdev(raid_bdev); 108 } 109 110 static void 111 test_raid1_start(void) 112 { 113 struct raid_params *params; 114 115 RAID_PARAMS_FOR_EACH(params) { 116 struct raid1_info *r1_info; 117 118 r1_info = create_raid1(params); 119 120 SPDK_CU_ASSERT_FATAL(r1_info != NULL); 121 122 CU_ASSERT_EQUAL(r1_info->raid_bdev->level, RAID1); 123 CU_ASSERT_EQUAL(r1_info->raid_bdev->bdev.blockcnt, params->base_bdev_blockcnt); 124 CU_ASSERT_PTR_EQUAL(r1_info->raid_bdev->module, &g_raid1_module); 125 126 delete_raid1(r1_info); 127 } 128 } 129 130 static struct raid_bdev_io * 131 get_raid_io(struct raid1_info *r1_info, struct raid_bdev_io_channel *raid_ch, 132 enum spdk_bdev_io_type io_type, uint64_t num_blocks) 133 { 134 struct raid_bdev_io *raid_io; 135 136 raid_io = calloc(1, sizeof(*raid_io)); 137 SPDK_CU_ASSERT_FATAL(raid_io != NULL); 138 139 raid_test_bdev_io_init(raid_io, r1_info->raid_bdev, raid_ch, io_type, 0, num_blocks, NULL, 0, NULL); 140 141 return raid_io; 142 } 143 144 static void 145 put_raid_io(struct raid_bdev_io *raid_io) 146 { 147 free(raid_io); 148 } 149 150 static void 151 raid_test_bdev_io_complete(struct raid_bdev_io *raid_io, enum spdk_bdev_io_status status) 152 { 153 CU_ASSERT(status == SPDK_BDEV_IO_STATUS_SUCCESS); 154 155 put_raid_io(raid_io); 156 } 157 158 static void 159 run_for_each_raid1_config(void (*test_fn)(struct raid_bdev *raid_bdev, 160 struct raid_bdev_io_channel *raid_ch)) 161 { 162 struct raid_params *params; 163 164 RAID_PARAMS_FOR_EACH(params) { 165 struct raid1_info *r1_info; 166 struct raid_bdev_io_channel *raid_ch; 167 168 r1_info = create_raid1(params); 169 raid_ch = raid_test_create_io_channel(r1_info->raid_bdev); 170 171 test_fn(r1_info->raid_bdev, raid_ch); 172 173 raid_test_destroy_io_channel(raid_ch); 174 delete_raid1(r1_info); 175 } 176 } 177 178 static void 179 _test_raid1_read_balancing(struct raid_bdev *raid_bdev, struct raid_bdev_io_channel *raid_ch) 180 { 181 struct raid1_info *r1_info = raid_bdev->module_private; 182 struct raid1_io_channel *raid1_ch = raid_bdev_channel_get_module_ctx(raid_ch); 183 uint8_t big_io_base_bdev_idx; 184 const uint64_t big_io_blocks = 256; 185 const uint64_t small_io_blocks = 4; 186 uint64_t blocks_remaining; 187 struct raid_bdev_io *raid_io; 188 uint8_t i; 189 int n; 190 191 /* same sized IOs should be be spread evenly across all base bdevs */ 192 for (n = 0; n < 3; n++) { 193 for (i = 0; i < raid_bdev->num_base_bdevs; i++) { 194 raid_io = get_raid_io(r1_info, raid_ch, SPDK_BDEV_IO_TYPE_READ, small_io_blocks); 195 raid1_submit_read_request(raid_io); 196 CU_ASSERT(raid_io->base_bdev_io_submitted == i); 197 put_raid_io(raid_io); 198 } 199 } 200 201 for (i = 0; i < raid_bdev->num_base_bdevs; i++) { 202 CU_ASSERT(raid1_ch->read_blocks_outstanding[i] == n * small_io_blocks); 203 raid1_ch->read_blocks_outstanding[i] = 0; 204 } 205 206 /* 207 * Submit one big and many small IOs. The small IOs should not land on the same base bdev 208 * as the big until the submitted block count is matched. 209 */ 210 raid_io = get_raid_io(r1_info, raid_ch, SPDK_BDEV_IO_TYPE_READ, big_io_blocks); 211 raid1_submit_read_request(raid_io); 212 big_io_base_bdev_idx = raid_io->base_bdev_io_submitted; 213 put_raid_io(raid_io); 214 215 blocks_remaining = big_io_blocks * (raid_bdev->num_base_bdevs - 1); 216 while (blocks_remaining > 0) { 217 raid_io = get_raid_io(r1_info, raid_ch, SPDK_BDEV_IO_TYPE_READ, small_io_blocks); 218 raid1_submit_read_request(raid_io); 219 CU_ASSERT(raid_io->base_bdev_io_submitted != big_io_base_bdev_idx); 220 put_raid_io(raid_io); 221 blocks_remaining -= small_io_blocks; 222 } 223 224 for (i = 0; i < raid_bdev->num_base_bdevs; i++) { 225 CU_ASSERT(raid1_ch->read_blocks_outstanding[i] == big_io_blocks); 226 } 227 228 raid_io = get_raid_io(r1_info, raid_ch, SPDK_BDEV_IO_TYPE_READ, small_io_blocks); 229 raid1_submit_read_request(raid_io); 230 CU_ASSERT(raid_io->base_bdev_io_submitted == big_io_base_bdev_idx); 231 put_raid_io(raid_io); 232 } 233 234 static void 235 test_raid1_read_balancing(void) 236 { 237 run_for_each_raid1_config(_test_raid1_read_balancing); 238 } 239 240 int 241 main(int argc, char **argv) 242 { 243 CU_pSuite suite = NULL; 244 unsigned int num_failures; 245 246 CU_initialize_registry(); 247 248 suite = CU_add_suite("raid1", test_setup, test_cleanup); 249 CU_ADD_TEST(suite, test_raid1_start); 250 CU_ADD_TEST(suite, test_raid1_read_balancing); 251 252 allocate_threads(1); 253 set_thread(0); 254 255 num_failures = spdk_ut_run_tests(argc, argv, NULL); 256 CU_cleanup_registry(); 257 258 free_threads(); 259 260 return num_failures; 261 } 262