1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2022 Intel Corporation. 3 * All rights reserved. 4 */ 5 6 #include "spdk/stdinc.h" 7 #include "spdk_internal/cunit.h" 8 #include "spdk/env.h" 9 10 #include "common/lib/ut_multithread.c" 11 12 #include "bdev/raid/raid1.c" 13 #include "../common.c" 14 15 DEFINE_STUB_V(raid_bdev_module_list_add, (struct raid_bdev_module *raid_module)); 16 DEFINE_STUB_V(raid_bdev_module_stop_done, (struct raid_bdev *raid_bdev)); 17 DEFINE_STUB_V(spdk_bdev_free_io, (struct spdk_bdev_io *bdev_io)); 18 DEFINE_STUB_V(raid_bdev_queue_io_wait, (struct raid_bdev_io *raid_io, struct spdk_bdev *bdev, 19 struct spdk_io_channel *ch, spdk_bdev_io_wait_cb cb_fn)); 20 DEFINE_STUB(spdk_bdev_readv_blocks_with_md, int, (struct spdk_bdev_desc *desc, 21 struct spdk_io_channel *ch, 22 struct iovec *iov, int iovcnt, void *md, 23 uint64_t offset_blocks, uint64_t num_blocks, 24 spdk_bdev_io_completion_cb cb, void *cb_arg), 0); 25 DEFINE_STUB(spdk_bdev_writev_blocks_with_md, int, (struct spdk_bdev_desc *desc, 26 struct spdk_io_channel *ch, 27 struct iovec *iov, int iovcnt, void *md, 28 uint64_t offset_blocks, uint64_t num_blocks, 29 spdk_bdev_io_completion_cb cb, void *cb_arg), 0); 30 DEFINE_STUB(spdk_bdev_readv_blocks_ext, int, (struct spdk_bdev_desc *desc, 31 struct spdk_io_channel *ch, 32 struct iovec *iov, int iovcnt, uint64_t offset_blocks, uint64_t num_blocks, 33 spdk_bdev_io_completion_cb cb, void *cb_arg, struct spdk_bdev_ext_io_opts *opts), 0); 34 DEFINE_STUB(spdk_bdev_writev_blocks_ext, int, (struct spdk_bdev_desc *desc, 35 struct spdk_io_channel *ch, 36 struct iovec *iov, int iovcnt, uint64_t offset_blocks, uint64_t num_blocks, 37 spdk_bdev_io_completion_cb cb, void *cb_arg, struct spdk_bdev_ext_io_opts *opts), 0); 38 DEFINE_STUB_V(raid_bdev_process_request_complete, (struct raid_bdev_process_request *process_req, 39 int status)); 40 DEFINE_STUB_V(raid_bdev_io_init, (struct raid_bdev_io *raid_io, 41 struct raid_bdev_io_channel *raid_ch, 42 enum spdk_bdev_io_type type, uint64_t offset_blocks, 43 uint64_t num_blocks, struct iovec *iovs, int iovcnt, void *md_buf, 44 struct spdk_memory_domain *memory_domain, void *memory_domain_ctx)); 45 46 static int 47 test_setup(void) 48 { 49 uint8_t num_base_bdevs_values[] = { 2, 3 }; 50 uint64_t base_bdev_blockcnt_values[] = { 1, 1024, 1024 * 1024 }; 51 uint32_t base_bdev_blocklen_values[] = { 512, 4096 }; 52 uint8_t *num_base_bdevs; 53 uint64_t *base_bdev_blockcnt; 54 uint32_t *base_bdev_blocklen; 55 uint64_t params_count; 56 int rc; 57 58 params_count = SPDK_COUNTOF(num_base_bdevs_values) * 59 SPDK_COUNTOF(base_bdev_blockcnt_values) * 60 SPDK_COUNTOF(base_bdev_blocklen_values); 61 rc = raid_test_params_alloc(params_count); 62 if (rc) { 63 return rc; 64 } 65 66 ARRAY_FOR_EACH(num_base_bdevs_values, num_base_bdevs) { 67 ARRAY_FOR_EACH(base_bdev_blockcnt_values, base_bdev_blockcnt) { 68 ARRAY_FOR_EACH(base_bdev_blocklen_values, base_bdev_blocklen) { 69 struct raid_params params = { 70 .num_base_bdevs = *num_base_bdevs, 71 .base_bdev_blockcnt = *base_bdev_blockcnt, 72 .base_bdev_blocklen = *base_bdev_blocklen, 73 }; 74 raid_test_params_add(¶ms); 75 } 76 } 77 } 78 79 return 0; 80 } 81 82 static int 83 test_cleanup(void) 84 { 85 raid_test_params_free(); 86 return 0; 87 } 88 89 static struct raid1_info * 90 create_raid1(struct raid_params *params) 91 { 92 struct raid_bdev *raid_bdev = raid_test_create_raid_bdev(params, &g_raid1_module); 93 94 SPDK_CU_ASSERT_FATAL(raid1_start(raid_bdev) == 0); 95 96 return raid_bdev->module_private; 97 } 98 99 static void 100 delete_raid1(struct raid1_info *r1_info) 101 { 102 struct raid_bdev *raid_bdev = r1_info->raid_bdev; 103 104 raid1_stop(raid_bdev); 105 106 raid_test_delete_raid_bdev(raid_bdev); 107 } 108 109 static void 110 test_raid1_start(void) 111 { 112 struct raid_params *params; 113 114 RAID_PARAMS_FOR_EACH(params) { 115 struct raid1_info *r1_info; 116 117 r1_info = create_raid1(params); 118 119 SPDK_CU_ASSERT_FATAL(r1_info != NULL); 120 121 CU_ASSERT_EQUAL(r1_info->raid_bdev->level, RAID1); 122 CU_ASSERT_EQUAL(r1_info->raid_bdev->bdev.blockcnt, params->base_bdev_blockcnt); 123 CU_ASSERT_PTR_EQUAL(r1_info->raid_bdev->module, &g_raid1_module); 124 125 delete_raid1(r1_info); 126 } 127 } 128 129 static struct raid_bdev_io * 130 get_raid_io(struct raid1_info *r1_info, struct raid_bdev_io_channel *raid_ch, 131 enum spdk_bdev_io_type io_type, uint64_t num_blocks) 132 { 133 struct raid_bdev_io *raid_io; 134 135 raid_io = calloc(1, sizeof(*raid_io)); 136 SPDK_CU_ASSERT_FATAL(raid_io != NULL); 137 138 raid_test_bdev_io_init(raid_io, r1_info->raid_bdev, raid_ch, io_type, 0, num_blocks, NULL, 0, NULL); 139 140 return raid_io; 141 } 142 143 static void 144 put_raid_io(struct raid_bdev_io *raid_io) 145 { 146 free(raid_io); 147 } 148 149 static void 150 raid_test_bdev_io_complete(struct raid_bdev_io *raid_io, enum spdk_bdev_io_status status) 151 { 152 CU_ASSERT(status == SPDK_BDEV_IO_STATUS_SUCCESS); 153 154 put_raid_io(raid_io); 155 } 156 157 static void 158 run_for_each_raid1_config(void (*test_fn)(struct raid_bdev *raid_bdev, 159 struct raid_bdev_io_channel *raid_ch)) 160 { 161 struct raid_params *params; 162 163 RAID_PARAMS_FOR_EACH(params) { 164 struct raid1_info *r1_info; 165 struct raid_bdev_io_channel *raid_ch; 166 167 r1_info = create_raid1(params); 168 raid_ch = raid_test_create_io_channel(r1_info->raid_bdev); 169 170 test_fn(r1_info->raid_bdev, raid_ch); 171 172 raid_test_destroy_io_channel(raid_ch); 173 delete_raid1(r1_info); 174 } 175 } 176 177 static void 178 _test_raid1_read_balancing(struct raid_bdev *raid_bdev, struct raid_bdev_io_channel *raid_ch) 179 { 180 struct raid1_info *r1_info = raid_bdev->module_private; 181 struct raid1_io_channel *raid1_ch = raid_bdev_channel_get_module_ctx(raid_ch); 182 uint8_t big_io_base_bdev_idx; 183 const uint64_t big_io_blocks = 256; 184 const uint64_t small_io_blocks = 4; 185 uint64_t blocks_remaining; 186 struct raid_bdev_io *raid_io; 187 uint8_t i; 188 int n; 189 190 /* same sized IOs should be be spread evenly across all base bdevs */ 191 for (n = 0; n < 3; n++) { 192 for (i = 0; i < raid_bdev->num_base_bdevs; i++) { 193 raid_io = get_raid_io(r1_info, raid_ch, SPDK_BDEV_IO_TYPE_READ, small_io_blocks); 194 raid1_submit_read_request(raid_io); 195 CU_ASSERT(raid_io->base_bdev_io_submitted == i); 196 put_raid_io(raid_io); 197 } 198 } 199 200 for (i = 0; i < raid_bdev->num_base_bdevs; i++) { 201 CU_ASSERT(raid1_ch->read_blocks_outstanding[i] == n * small_io_blocks); 202 raid1_ch->read_blocks_outstanding[i] = 0; 203 } 204 205 /* 206 * Submit one big and many small IOs. The small IOs should not land on the same base bdev 207 * as the big until the submitted block count is matched. 208 */ 209 raid_io = get_raid_io(r1_info, raid_ch, SPDK_BDEV_IO_TYPE_READ, big_io_blocks); 210 raid1_submit_read_request(raid_io); 211 big_io_base_bdev_idx = raid_io->base_bdev_io_submitted; 212 put_raid_io(raid_io); 213 214 blocks_remaining = big_io_blocks * (raid_bdev->num_base_bdevs - 1); 215 while (blocks_remaining > 0) { 216 raid_io = get_raid_io(r1_info, raid_ch, SPDK_BDEV_IO_TYPE_READ, small_io_blocks); 217 raid1_submit_read_request(raid_io); 218 CU_ASSERT(raid_io->base_bdev_io_submitted != big_io_base_bdev_idx); 219 put_raid_io(raid_io); 220 blocks_remaining -= small_io_blocks; 221 } 222 223 for (i = 0; i < raid_bdev->num_base_bdevs; i++) { 224 CU_ASSERT(raid1_ch->read_blocks_outstanding[i] == big_io_blocks); 225 } 226 227 raid_io = get_raid_io(r1_info, raid_ch, SPDK_BDEV_IO_TYPE_READ, small_io_blocks); 228 raid1_submit_read_request(raid_io); 229 CU_ASSERT(raid_io->base_bdev_io_submitted == big_io_base_bdev_idx); 230 put_raid_io(raid_io); 231 } 232 233 static void 234 test_raid1_read_balancing(void) 235 { 236 run_for_each_raid1_config(_test_raid1_read_balancing); 237 } 238 239 int 240 main(int argc, char **argv) 241 { 242 CU_pSuite suite = NULL; 243 unsigned int num_failures; 244 245 CU_initialize_registry(); 246 247 suite = CU_add_suite("raid1", test_setup, test_cleanup); 248 CU_ADD_TEST(suite, test_raid1_start); 249 CU_ADD_TEST(suite, test_raid1_read_balancing); 250 251 allocate_threads(1); 252 set_thread(0); 253 254 num_failures = spdk_ut_run_tests(argc, argv, NULL); 255 CU_cleanup_registry(); 256 257 free_threads(); 258 259 return num_failures; 260 } 261