xref: /spdk/test/unit/lib/bdev/raid/raid1.c/raid1_ut.c (revision 8afdeef3becfe9409cc9e7372bd0bc10e8b7d46d)
1 /*   SPDX-License-Identifier: BSD-3-Clause
2  *   Copyright (C) 2022 Intel Corporation.
3  *   All rights reserved.
4  */
5 
6 #include "spdk/stdinc.h"
7 #include "spdk_internal/cunit.h"
8 #include "spdk/env.h"
9 
10 #include "common/lib/ut_multithread.c"
11 
12 #include "bdev/raid/raid1.c"
13 #include "../common.c"
14 
15 static enum spdk_bdev_io_status g_io_status;
16 static struct spdk_bdev_desc *g_last_io_desc;
17 static spdk_bdev_io_completion_cb g_last_io_cb;
18 
19 DEFINE_STUB_V(raid_bdev_module_list_add, (struct raid_bdev_module *raid_module));
20 DEFINE_STUB_V(raid_bdev_module_stop_done, (struct raid_bdev *raid_bdev));
21 DEFINE_STUB_V(spdk_bdev_free_io, (struct spdk_bdev_io *bdev_io));
22 DEFINE_STUB_V(raid_bdev_queue_io_wait, (struct raid_bdev_io *raid_io, struct spdk_bdev *bdev,
23 					struct spdk_io_channel *ch, spdk_bdev_io_wait_cb cb_fn));
24 DEFINE_STUB_V(raid_bdev_process_request_complete, (struct raid_bdev_process_request *process_req,
25 		int status));
26 DEFINE_STUB_V(raid_bdev_io_init, (struct raid_bdev_io *raid_io,
27 				  struct raid_bdev_io_channel *raid_ch,
28 				  enum spdk_bdev_io_type type, uint64_t offset_blocks,
29 				  uint64_t num_blocks, struct iovec *iovs, int iovcnt, void *md_buf,
30 				  struct spdk_memory_domain *memory_domain, void *memory_domain_ctx));
31 DEFINE_STUB(raid_bdev_remap_dix_reftag, int, (void *md_buf, uint64_t num_blocks,
32 		struct spdk_bdev *bdev, uint32_t remapped_offset), -1);
33 DEFINE_STUB(spdk_bdev_notify_blockcnt_change, int, (struct spdk_bdev *bdev, uint64_t size), 0);
34 
35 int
36 spdk_bdev_readv_blocks_ext(struct spdk_bdev_desc *desc,
37 			   struct spdk_io_channel *ch,
38 			   struct iovec *iov, int iovcnt, uint64_t offset_blocks, uint64_t num_blocks,
39 			   spdk_bdev_io_completion_cb cb, void *cb_arg, struct spdk_bdev_ext_io_opts *opts)
40 {
41 	g_last_io_desc = desc;
42 	g_last_io_cb = cb;
43 
44 	return 0;
45 }
46 
47 int
48 spdk_bdev_writev_blocks_ext(struct spdk_bdev_desc *desc,
49 			    struct spdk_io_channel *ch,
50 			    struct iovec *iov, int iovcnt, uint64_t offset_blocks, uint64_t num_blocks,
51 			    spdk_bdev_io_completion_cb cb, void *cb_arg, struct spdk_bdev_ext_io_opts *opts)
52 {
53 	g_last_io_desc = desc;
54 	g_last_io_cb = cb;
55 
56 	return 0;
57 }
58 
59 void
60 raid_bdev_fail_base_bdev(struct raid_base_bdev_info *base_info)
61 {
62 	base_info->is_failed = true;
63 }
64 
65 static int
66 test_setup(void)
67 {
68 	uint8_t num_base_bdevs_values[] = { 2, 3 };
69 	uint64_t base_bdev_blockcnt_values[] = { 1, 1024, 1024 * 1024 };
70 	uint32_t base_bdev_blocklen_values[] = { 512, 4096 };
71 	uint8_t *num_base_bdevs;
72 	uint64_t *base_bdev_blockcnt;
73 	uint32_t *base_bdev_blocklen;
74 	uint64_t params_count;
75 	int rc;
76 
77 	params_count = SPDK_COUNTOF(num_base_bdevs_values) *
78 		       SPDK_COUNTOF(base_bdev_blockcnt_values) *
79 		       SPDK_COUNTOF(base_bdev_blocklen_values);
80 	rc = raid_test_params_alloc(params_count);
81 	if (rc) {
82 		return rc;
83 	}
84 
85 	ARRAY_FOR_EACH(num_base_bdevs_values, num_base_bdevs) {
86 		ARRAY_FOR_EACH(base_bdev_blockcnt_values, base_bdev_blockcnt) {
87 			ARRAY_FOR_EACH(base_bdev_blocklen_values, base_bdev_blocklen) {
88 				struct raid_params params = {
89 					.num_base_bdevs = *num_base_bdevs,
90 					.base_bdev_blockcnt = *base_bdev_blockcnt,
91 					.base_bdev_blocklen = *base_bdev_blocklen,
92 				};
93 				raid_test_params_add(&params);
94 			}
95 		}
96 	}
97 
98 	return 0;
99 }
100 
101 static int
102 test_cleanup(void)
103 {
104 	raid_test_params_free();
105 	return 0;
106 }
107 
108 static struct raid1_info *
109 create_raid1(struct raid_params *params)
110 {
111 	struct raid_bdev *raid_bdev = raid_test_create_raid_bdev(params, &g_raid1_module);
112 
113 	SPDK_CU_ASSERT_FATAL(raid1_start(raid_bdev) == 0);
114 
115 	return raid_bdev->module_private;
116 }
117 
118 static void
119 delete_raid1(struct raid1_info *r1_info)
120 {
121 	struct raid_bdev *raid_bdev = r1_info->raid_bdev;
122 
123 	raid1_stop(raid_bdev);
124 
125 	raid_test_delete_raid_bdev(raid_bdev);
126 }
127 
128 static void
129 test_raid1_start(void)
130 {
131 	struct raid_params *params;
132 
133 	RAID_PARAMS_FOR_EACH(params) {
134 		struct raid1_info *r1_info;
135 
136 		r1_info = create_raid1(params);
137 
138 		SPDK_CU_ASSERT_FATAL(r1_info != NULL);
139 
140 		CU_ASSERT_EQUAL(r1_info->raid_bdev->level, RAID1);
141 		CU_ASSERT_EQUAL(r1_info->raid_bdev->bdev.blockcnt, params->base_bdev_blockcnt);
142 		CU_ASSERT_PTR_EQUAL(r1_info->raid_bdev->module, &g_raid1_module);
143 
144 		delete_raid1(r1_info);
145 	}
146 }
147 
148 static struct raid_bdev_io *
149 get_raid_io(struct raid1_info *r1_info, struct raid_bdev_io_channel *raid_ch,
150 	    enum spdk_bdev_io_type io_type, uint64_t num_blocks)
151 {
152 	struct raid_bdev_io *raid_io;
153 
154 	raid_io = calloc(1, sizeof(*raid_io));
155 	SPDK_CU_ASSERT_FATAL(raid_io != NULL);
156 
157 	raid_test_bdev_io_init(raid_io, r1_info->raid_bdev, raid_ch, io_type, 0, num_blocks, NULL, 0, NULL);
158 
159 	return raid_io;
160 }
161 
162 static void
163 put_raid_io(struct raid_bdev_io *raid_io)
164 {
165 	free(raid_io);
166 }
167 
168 void
169 raid_test_bdev_io_complete(struct raid_bdev_io *raid_io, enum spdk_bdev_io_status status)
170 {
171 	g_io_status = status;
172 
173 	put_raid_io(raid_io);
174 }
175 
176 static void
177 run_for_each_raid1_config(void (*test_fn)(struct raid_bdev *raid_bdev,
178 			  struct raid_bdev_io_channel *raid_ch))
179 {
180 	struct raid_params *params;
181 
182 	RAID_PARAMS_FOR_EACH(params) {
183 		struct raid1_info *r1_info;
184 		struct raid_bdev_io_channel *raid_ch;
185 
186 		r1_info = create_raid1(params);
187 		raid_ch = raid_test_create_io_channel(r1_info->raid_bdev);
188 
189 		test_fn(r1_info->raid_bdev, raid_ch);
190 
191 		raid_test_destroy_io_channel(raid_ch);
192 		delete_raid1(r1_info);
193 	}
194 }
195 
196 static void
197 _test_raid1_read_balancing(struct raid_bdev *raid_bdev, struct raid_bdev_io_channel *raid_ch)
198 {
199 	struct raid1_info *r1_info = raid_bdev->module_private;
200 	struct raid1_io_channel *raid1_ch = raid_bdev_channel_get_module_ctx(raid_ch);
201 	uint8_t big_io_base_bdev_idx;
202 	const uint64_t big_io_blocks = 256;
203 	const uint64_t small_io_blocks = 4;
204 	uint64_t blocks_remaining;
205 	struct raid_bdev_io *raid_io;
206 	uint8_t i;
207 	int n;
208 
209 	/* same sized IOs should be be spread evenly across all base bdevs */
210 	for (n = 0; n < 3; n++) {
211 		for (i = 0; i < raid_bdev->num_base_bdevs; i++) {
212 			raid_io = get_raid_io(r1_info, raid_ch, SPDK_BDEV_IO_TYPE_READ, small_io_blocks);
213 			raid1_submit_read_request(raid_io);
214 			CU_ASSERT(raid_io->base_bdev_io_submitted == i);
215 			put_raid_io(raid_io);
216 		}
217 	}
218 
219 	for (i = 0; i < raid_bdev->num_base_bdevs; i++) {
220 		CU_ASSERT(raid1_ch->read_blocks_outstanding[i] == n * small_io_blocks);
221 		raid1_ch->read_blocks_outstanding[i] = 0;
222 	}
223 
224 	/*
225 	 * Submit one big and many small IOs. The small IOs should not land on the same base bdev
226 	 * as the big until the submitted block count is matched.
227 	 */
228 	raid_io = get_raid_io(r1_info, raid_ch, SPDK_BDEV_IO_TYPE_READ, big_io_blocks);
229 	raid1_submit_read_request(raid_io);
230 	big_io_base_bdev_idx = raid_io->base_bdev_io_submitted;
231 	put_raid_io(raid_io);
232 
233 	blocks_remaining = big_io_blocks * (raid_bdev->num_base_bdevs - 1);
234 	while (blocks_remaining > 0) {
235 		raid_io = get_raid_io(r1_info, raid_ch, SPDK_BDEV_IO_TYPE_READ, small_io_blocks);
236 		raid1_submit_read_request(raid_io);
237 		CU_ASSERT(raid_io->base_bdev_io_submitted != big_io_base_bdev_idx);
238 		put_raid_io(raid_io);
239 		blocks_remaining -= small_io_blocks;
240 	}
241 
242 	for (i = 0; i < raid_bdev->num_base_bdevs; i++) {
243 		CU_ASSERT(raid1_ch->read_blocks_outstanding[i] == big_io_blocks);
244 	}
245 
246 	raid_io = get_raid_io(r1_info, raid_ch, SPDK_BDEV_IO_TYPE_READ, small_io_blocks);
247 	raid1_submit_read_request(raid_io);
248 	CU_ASSERT(raid_io->base_bdev_io_submitted == big_io_base_bdev_idx);
249 	put_raid_io(raid_io);
250 }
251 
252 static void
253 test_raid1_read_balancing(void)
254 {
255 	run_for_each_raid1_config(_test_raid1_read_balancing);
256 }
257 
258 static void
259 _test_raid1_write_error(struct raid_bdev *raid_bdev, struct raid_bdev_io_channel *raid_ch)
260 {
261 	struct raid1_info *r1_info = raid_bdev->module_private;
262 	struct raid_bdev_io *raid_io;
263 	struct raid_base_bdev_info *base_info;
264 	struct spdk_bdev_io bdev_io = {};
265 	bool bdev_io_success;
266 
267 	/* first completion failed */
268 	g_io_status = SPDK_BDEV_IO_STATUS_PENDING;
269 	raid_io = get_raid_io(r1_info, raid_ch, SPDK_BDEV_IO_TYPE_WRITE, 64);
270 	raid1_submit_write_request(raid_io);
271 
272 	RAID_FOR_EACH_BASE_BDEV(raid_bdev, base_info) {
273 		base_info->is_failed = false;
274 		if (raid_bdev_base_bdev_slot(base_info) == 0) {
275 			bdev_io_success = false;
276 		} else {
277 			bdev_io_success = true;
278 		}
279 		bdev_io.bdev = base_info->desc->bdev;
280 		raid1_write_bdev_io_completion(&bdev_io, bdev_io_success, raid_io);
281 		CU_ASSERT(base_info->is_failed == !bdev_io_success);
282 	}
283 	CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_SUCCESS);
284 
285 	/* all except first completion failed */
286 	g_io_status = SPDK_BDEV_IO_STATUS_PENDING;
287 	raid_io = get_raid_io(r1_info, raid_ch, SPDK_BDEV_IO_TYPE_WRITE, 64);
288 	raid1_submit_write_request(raid_io);
289 
290 	RAID_FOR_EACH_BASE_BDEV(raid_bdev, base_info) {
291 		base_info->is_failed = false;
292 		if (raid_bdev_base_bdev_slot(base_info) != 0) {
293 			bdev_io_success = false;
294 		} else {
295 			bdev_io_success = true;
296 		}
297 		bdev_io.bdev = base_info->desc->bdev;
298 		raid1_write_bdev_io_completion(&bdev_io, bdev_io_success, raid_io);
299 		CU_ASSERT(base_info->is_failed == !bdev_io_success);
300 	}
301 	CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_SUCCESS);
302 
303 	/* all completions failed */
304 	g_io_status = SPDK_BDEV_IO_STATUS_PENDING;
305 	raid_io = get_raid_io(r1_info, raid_ch, SPDK_BDEV_IO_TYPE_WRITE, 64);
306 	raid1_submit_write_request(raid_io);
307 
308 	bdev_io_success = false;
309 	RAID_FOR_EACH_BASE_BDEV(raid_bdev, base_info) {
310 		base_info->is_failed = false;
311 		bdev_io.bdev = base_info->desc->bdev;
312 		raid1_write_bdev_io_completion(&bdev_io, bdev_io_success, raid_io);
313 		CU_ASSERT(base_info->is_failed == !bdev_io_success);
314 	}
315 	CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_FAILED);
316 }
317 
318 static void
319 test_raid1_write_error(void)
320 {
321 	run_for_each_raid1_config(_test_raid1_write_error);
322 }
323 
324 static void
325 _test_raid1_read_error(struct raid_bdev *raid_bdev, struct raid_bdev_io_channel *raid_ch)
326 {
327 	struct raid1_info *r1_info = raid_bdev->module_private;
328 	struct raid_base_bdev_info *base_info = &raid_bdev->base_bdev_info[0];
329 	struct raid1_io_channel *raid1_ch = raid_bdev_channel_get_module_ctx(raid_ch);
330 	struct raid_bdev_io *raid_io;
331 	struct spdk_bdev_io bdev_io = {};
332 
333 	/* first read fails, the second succeeds */
334 	base_info->is_failed = false;
335 	g_io_status = SPDK_BDEV_IO_STATUS_PENDING;
336 	raid_io = get_raid_io(r1_info, raid_ch, SPDK_BDEV_IO_TYPE_READ, 64);
337 	raid1_submit_read_request(raid_io);
338 	CU_ASSERT(raid_io->base_bdev_io_submitted == 0);
339 	CU_ASSERT(raid_io->base_bdev_io_remaining == 0);
340 
341 	CU_ASSERT(g_last_io_desc == base_info->desc);
342 	CU_ASSERT(g_last_io_cb == raid1_read_bdev_io_completion);
343 	raid1_read_bdev_io_completion(&bdev_io, false, raid_io);
344 	CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_PENDING);
345 	CU_ASSERT((uint8_t)raid_io->base_bdev_io_remaining == (raid_bdev->num_base_bdevs - 1));
346 
347 	CU_ASSERT(g_last_io_desc == raid_bdev->base_bdev_info[1].desc);
348 	CU_ASSERT(g_last_io_cb == raid1_read_other_completion);
349 	raid1_read_other_completion(&bdev_io, true, raid_io);
350 	CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_PENDING);
351 
352 	CU_ASSERT(g_last_io_desc == base_info->desc);
353 	CU_ASSERT(g_last_io_cb == raid1_correct_read_error_completion);
354 	raid1_correct_read_error_completion(&bdev_io, true, raid_io);
355 	CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_SUCCESS);
356 	CU_ASSERT(base_info->is_failed == false);
357 
358 	/* rewrite fails */
359 	base_info->is_failed = false;
360 	g_io_status = SPDK_BDEV_IO_STATUS_PENDING;
361 	raid_io = get_raid_io(r1_info, raid_ch, SPDK_BDEV_IO_TYPE_READ, 64);
362 	raid1_submit_read_request(raid_io);
363 	CU_ASSERT(raid_io->base_bdev_io_submitted == 0);
364 	CU_ASSERT(raid_io->base_bdev_io_remaining == 0);
365 
366 	CU_ASSERT(g_last_io_desc == base_info->desc);
367 	CU_ASSERT(g_last_io_cb == raid1_read_bdev_io_completion);
368 	raid1_read_bdev_io_completion(&bdev_io, false, raid_io);
369 	CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_PENDING);
370 	CU_ASSERT((uint8_t)raid_io->base_bdev_io_remaining == (raid_bdev->num_base_bdevs - 1));
371 
372 	CU_ASSERT(g_last_io_desc == raid_bdev->base_bdev_info[1].desc);
373 	CU_ASSERT(g_last_io_cb == raid1_read_other_completion);
374 	raid1_read_other_completion(&bdev_io, true, raid_io);
375 	CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_PENDING);
376 
377 	CU_ASSERT(g_last_io_desc == base_info->desc);
378 	CU_ASSERT(g_last_io_cb == raid1_correct_read_error_completion);
379 	raid1_correct_read_error_completion(&bdev_io, false, raid_io);
380 	CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_SUCCESS);
381 	CU_ASSERT(base_info->is_failed == true);
382 
383 	/* only the last read succeeds */
384 	base_info->is_failed = false;
385 	g_io_status = SPDK_BDEV_IO_STATUS_PENDING;
386 	raid_io = get_raid_io(r1_info, raid_ch, SPDK_BDEV_IO_TYPE_READ, 64);
387 	raid1_submit_read_request(raid_io);
388 	CU_ASSERT(raid_io->base_bdev_io_submitted == 0);
389 	CU_ASSERT(raid_io->base_bdev_io_remaining == 0);
390 
391 	CU_ASSERT(g_last_io_desc == base_info->desc);
392 	CU_ASSERT(g_last_io_cb == raid1_read_bdev_io_completion);
393 	raid1_read_bdev_io_completion(&bdev_io, false, raid_io);
394 	CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_PENDING);
395 	CU_ASSERT((uint8_t)raid_io->base_bdev_io_remaining == (raid_bdev->num_base_bdevs - 1));
396 
397 	while (raid_io->base_bdev_io_remaining > 1) {
398 		CU_ASSERT(g_last_io_cb == raid1_read_other_completion);
399 		raid1_read_other_completion(&bdev_io, false, raid_io);
400 		CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_PENDING);
401 	}
402 
403 	CU_ASSERT(g_last_io_desc == raid_bdev->base_bdev_info[raid_bdev->num_base_bdevs - 1].desc);
404 	CU_ASSERT(g_last_io_cb == raid1_read_other_completion);
405 	raid1_read_other_completion(&bdev_io, true, raid_io);
406 	CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_PENDING);
407 
408 	CU_ASSERT(g_last_io_desc == base_info->desc);
409 	CU_ASSERT(g_last_io_cb == raid1_correct_read_error_completion);
410 	raid1_correct_read_error_completion(&bdev_io, true, raid_io);
411 	CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_SUCCESS);
412 	CU_ASSERT(base_info->is_failed == false);
413 
414 	/* all reads fail */
415 	base_info->is_failed = false;
416 	g_io_status = SPDK_BDEV_IO_STATUS_PENDING;
417 	raid_io = get_raid_io(r1_info, raid_ch, SPDK_BDEV_IO_TYPE_READ, 64);
418 	raid1_submit_read_request(raid_io);
419 	CU_ASSERT(raid_io->base_bdev_io_submitted == 0);
420 	CU_ASSERT(raid_io->base_bdev_io_remaining == 0);
421 
422 	CU_ASSERT(g_last_io_desc == base_info->desc);
423 	CU_ASSERT(g_last_io_cb == raid1_read_bdev_io_completion);
424 	raid1_read_bdev_io_completion(&bdev_io, false, raid_io);
425 	CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_PENDING);
426 	CU_ASSERT((uint8_t)raid_io->base_bdev_io_remaining == (raid_bdev->num_base_bdevs - 1));
427 
428 	while (raid_io->base_bdev_io_remaining > 1) {
429 		CU_ASSERT(g_last_io_cb == raid1_read_other_completion);
430 		raid1_read_other_completion(&bdev_io, false, raid_io);
431 		CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_PENDING);
432 	}
433 
434 	CU_ASSERT(g_last_io_desc == raid_bdev->base_bdev_info[raid_bdev->num_base_bdevs - 1].desc);
435 	CU_ASSERT(g_last_io_cb == raid1_read_other_completion);
436 	raid1_read_other_completion(&bdev_io, false, raid_io);
437 	CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_FAILED);
438 	CU_ASSERT(base_info->is_failed == true);
439 
440 	/* read from base bdev #1 fails, read from #0 succeeds */
441 	base_info->is_failed = false;
442 	base_info = &raid_bdev->base_bdev_info[1];
443 	raid1_ch->read_blocks_outstanding[0] = 123;
444 	g_io_status = SPDK_BDEV_IO_STATUS_PENDING;
445 	raid_io = get_raid_io(r1_info, raid_ch, SPDK_BDEV_IO_TYPE_READ, 64);
446 	raid1_submit_read_request(raid_io);
447 	CU_ASSERT(raid_io->base_bdev_io_submitted == 1);
448 	CU_ASSERT(raid_io->base_bdev_io_remaining == 0);
449 
450 	CU_ASSERT(g_last_io_desc == base_info->desc);
451 	CU_ASSERT(g_last_io_cb == raid1_read_bdev_io_completion);
452 	raid1_read_bdev_io_completion(&bdev_io, false, raid_io);
453 	CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_PENDING);
454 	CU_ASSERT((uint8_t)raid_io->base_bdev_io_remaining == raid_bdev->num_base_bdevs);
455 
456 	CU_ASSERT(g_last_io_desc == raid_bdev->base_bdev_info[0].desc);
457 	CU_ASSERT(g_last_io_cb == raid1_read_other_completion);
458 	raid1_read_other_completion(&bdev_io, true, raid_io);
459 	CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_PENDING);
460 
461 	CU_ASSERT(g_last_io_desc == base_info->desc);
462 	CU_ASSERT(g_last_io_cb == raid1_correct_read_error_completion);
463 	raid1_correct_read_error_completion(&bdev_io, true, raid_io);
464 	CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_SUCCESS);
465 	CU_ASSERT(base_info->is_failed == false);
466 
467 	/* base bdev #0 is failed, read from #1 fails, read from next succeeds if N > 2 */
468 	base_info->is_failed = false;
469 	raid_ch->_base_channels[0] = NULL;
470 	g_io_status = SPDK_BDEV_IO_STATUS_PENDING;
471 	raid_io = get_raid_io(r1_info, raid_ch, SPDK_BDEV_IO_TYPE_READ, 64);
472 	raid1_submit_read_request(raid_io);
473 	CU_ASSERT(raid_io->base_bdev_io_submitted == 1);
474 	CU_ASSERT(raid_io->base_bdev_io_remaining == 0);
475 
476 	CU_ASSERT(g_last_io_desc == base_info->desc);
477 	CU_ASSERT(g_last_io_cb == raid1_read_bdev_io_completion);
478 	raid1_read_bdev_io_completion(&bdev_io, false, raid_io);
479 	if (raid_bdev->num_base_bdevs > 2) {
480 		CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_PENDING);
481 		CU_ASSERT((uint8_t)raid_io->base_bdev_io_remaining == (raid_bdev->num_base_bdevs - 2));
482 
483 		CU_ASSERT(g_last_io_desc == raid_bdev->base_bdev_info[2].desc);
484 		CU_ASSERT(g_last_io_cb == raid1_read_other_completion);
485 		raid1_read_other_completion(&bdev_io, true, raid_io);
486 		CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_PENDING);
487 
488 		CU_ASSERT(g_last_io_desc == base_info->desc);
489 		CU_ASSERT(g_last_io_cb == raid1_correct_read_error_completion);
490 		raid1_correct_read_error_completion(&bdev_io, true, raid_io);
491 		CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_SUCCESS);
492 		CU_ASSERT(base_info->is_failed == false);
493 	} else {
494 		CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_FAILED);
495 		CU_ASSERT(base_info->is_failed == true);
496 	}
497 }
498 
499 static void
500 test_raid1_read_error(void)
501 {
502 	run_for_each_raid1_config(_test_raid1_read_error);
503 }
504 
505 int
506 main(int argc, char **argv)
507 {
508 	CU_pSuite suite = NULL;
509 	unsigned int num_failures;
510 
511 	CU_initialize_registry();
512 
513 	suite = CU_add_suite("raid1", test_setup, test_cleanup);
514 	CU_ADD_TEST(suite, test_raid1_start);
515 	CU_ADD_TEST(suite, test_raid1_read_balancing);
516 	CU_ADD_TEST(suite, test_raid1_write_error);
517 	CU_ADD_TEST(suite, test_raid1_read_error);
518 
519 	allocate_threads(1);
520 	set_thread(0);
521 
522 	num_failures = spdk_ut_run_tests(argc, argv, NULL);
523 	CU_cleanup_registry();
524 
525 	free_threads();
526 
527 	return num_failures;
528 }
529