xref: /spdk/test/unit/lib/bdev/raid/raid1.c/raid1_ut.c (revision 8a01b4d6366393ba157b7c42f076389b9cebaafa)
1 /*   SPDX-License-Identifier: BSD-3-Clause
2  *   Copyright (C) 2022 Intel Corporation.
3  *   All rights reserved.
4  */
5 
6 #include "spdk/stdinc.h"
7 #include "spdk_internal/cunit.h"
8 #include "spdk/env.h"
9 
10 #include "common/lib/ut_multithread.c"
11 
12 #include "bdev/raid/raid1.c"
13 #include "../common.c"
14 
15 static enum spdk_bdev_io_status g_io_status;
16 static struct spdk_bdev_desc *g_last_io_desc;
17 static spdk_bdev_io_completion_cb g_last_io_cb;
18 
19 DEFINE_STUB_V(raid_bdev_module_list_add, (struct raid_bdev_module *raid_module));
20 DEFINE_STUB_V(raid_bdev_module_stop_done, (struct raid_bdev *raid_bdev));
21 DEFINE_STUB_V(spdk_bdev_free_io, (struct spdk_bdev_io *bdev_io));
22 DEFINE_STUB_V(raid_bdev_queue_io_wait, (struct raid_bdev_io *raid_io, struct spdk_bdev *bdev,
23 					struct spdk_io_channel *ch, spdk_bdev_io_wait_cb cb_fn));
24 DEFINE_STUB_V(raid_bdev_process_request_complete, (struct raid_bdev_process_request *process_req,
25 		int status));
26 DEFINE_STUB_V(raid_bdev_io_init, (struct raid_bdev_io *raid_io,
27 				  struct raid_bdev_io_channel *raid_ch,
28 				  enum spdk_bdev_io_type type, uint64_t offset_blocks,
29 				  uint64_t num_blocks, struct iovec *iovs, int iovcnt, void *md_buf,
30 				  struct spdk_memory_domain *memory_domain, void *memory_domain_ctx));
31 DEFINE_STUB(raid_bdev_remap_dix_reftag, int, (void *md_buf, uint64_t num_blocks,
32 		struct spdk_bdev *bdev, uint32_t remapped_offset), -1);
33 
34 int
35 spdk_bdev_readv_blocks_ext(struct spdk_bdev_desc *desc,
36 			   struct spdk_io_channel *ch,
37 			   struct iovec *iov, int iovcnt, uint64_t offset_blocks, uint64_t num_blocks,
38 			   spdk_bdev_io_completion_cb cb, void *cb_arg, struct spdk_bdev_ext_io_opts *opts)
39 {
40 	g_last_io_desc = desc;
41 	g_last_io_cb = cb;
42 
43 	return 0;
44 }
45 
46 int
47 spdk_bdev_writev_blocks_ext(struct spdk_bdev_desc *desc,
48 			    struct spdk_io_channel *ch,
49 			    struct iovec *iov, int iovcnt, uint64_t offset_blocks, uint64_t num_blocks,
50 			    spdk_bdev_io_completion_cb cb, void *cb_arg, struct spdk_bdev_ext_io_opts *opts)
51 {
52 	g_last_io_desc = desc;
53 	g_last_io_cb = cb;
54 
55 	return 0;
56 }
57 
58 void
59 raid_bdev_fail_base_bdev(struct raid_base_bdev_info *base_info)
60 {
61 	base_info->is_failed = true;
62 }
63 
64 static int
65 test_setup(void)
66 {
67 	uint8_t num_base_bdevs_values[] = { 2, 3 };
68 	uint64_t base_bdev_blockcnt_values[] = { 1, 1024, 1024 * 1024 };
69 	uint32_t base_bdev_blocklen_values[] = { 512, 4096 };
70 	uint8_t *num_base_bdevs;
71 	uint64_t *base_bdev_blockcnt;
72 	uint32_t *base_bdev_blocklen;
73 	uint64_t params_count;
74 	int rc;
75 
76 	params_count = SPDK_COUNTOF(num_base_bdevs_values) *
77 		       SPDK_COUNTOF(base_bdev_blockcnt_values) *
78 		       SPDK_COUNTOF(base_bdev_blocklen_values);
79 	rc = raid_test_params_alloc(params_count);
80 	if (rc) {
81 		return rc;
82 	}
83 
84 	ARRAY_FOR_EACH(num_base_bdevs_values, num_base_bdevs) {
85 		ARRAY_FOR_EACH(base_bdev_blockcnt_values, base_bdev_blockcnt) {
86 			ARRAY_FOR_EACH(base_bdev_blocklen_values, base_bdev_blocklen) {
87 				struct raid_params params = {
88 					.num_base_bdevs = *num_base_bdevs,
89 					.base_bdev_blockcnt = *base_bdev_blockcnt,
90 					.base_bdev_blocklen = *base_bdev_blocklen,
91 				};
92 				raid_test_params_add(&params);
93 			}
94 		}
95 	}
96 
97 	return 0;
98 }
99 
100 static int
101 test_cleanup(void)
102 {
103 	raid_test_params_free();
104 	return 0;
105 }
106 
107 static struct raid1_info *
108 create_raid1(struct raid_params *params)
109 {
110 	struct raid_bdev *raid_bdev = raid_test_create_raid_bdev(params, &g_raid1_module);
111 
112 	SPDK_CU_ASSERT_FATAL(raid1_start(raid_bdev) == 0);
113 
114 	return raid_bdev->module_private;
115 }
116 
117 static void
118 delete_raid1(struct raid1_info *r1_info)
119 {
120 	struct raid_bdev *raid_bdev = r1_info->raid_bdev;
121 
122 	raid1_stop(raid_bdev);
123 
124 	raid_test_delete_raid_bdev(raid_bdev);
125 }
126 
127 static void
128 test_raid1_start(void)
129 {
130 	struct raid_params *params;
131 
132 	RAID_PARAMS_FOR_EACH(params) {
133 		struct raid1_info *r1_info;
134 
135 		r1_info = create_raid1(params);
136 
137 		SPDK_CU_ASSERT_FATAL(r1_info != NULL);
138 
139 		CU_ASSERT_EQUAL(r1_info->raid_bdev->level, RAID1);
140 		CU_ASSERT_EQUAL(r1_info->raid_bdev->bdev.blockcnt, params->base_bdev_blockcnt);
141 		CU_ASSERT_PTR_EQUAL(r1_info->raid_bdev->module, &g_raid1_module);
142 
143 		delete_raid1(r1_info);
144 	}
145 }
146 
147 static struct raid_bdev_io *
148 get_raid_io(struct raid1_info *r1_info, struct raid_bdev_io_channel *raid_ch,
149 	    enum spdk_bdev_io_type io_type, uint64_t num_blocks)
150 {
151 	struct raid_bdev_io *raid_io;
152 
153 	raid_io = calloc(1, sizeof(*raid_io));
154 	SPDK_CU_ASSERT_FATAL(raid_io != NULL);
155 
156 	raid_test_bdev_io_init(raid_io, r1_info->raid_bdev, raid_ch, io_type, 0, num_blocks, NULL, 0, NULL);
157 
158 	return raid_io;
159 }
160 
161 static void
162 put_raid_io(struct raid_bdev_io *raid_io)
163 {
164 	free(raid_io);
165 }
166 
167 void
168 raid_test_bdev_io_complete(struct raid_bdev_io *raid_io, enum spdk_bdev_io_status status)
169 {
170 	g_io_status = status;
171 
172 	put_raid_io(raid_io);
173 }
174 
175 static void
176 run_for_each_raid1_config(void (*test_fn)(struct raid_bdev *raid_bdev,
177 			  struct raid_bdev_io_channel *raid_ch))
178 {
179 	struct raid_params *params;
180 
181 	RAID_PARAMS_FOR_EACH(params) {
182 		struct raid1_info *r1_info;
183 		struct raid_bdev_io_channel *raid_ch;
184 
185 		r1_info = create_raid1(params);
186 		raid_ch = raid_test_create_io_channel(r1_info->raid_bdev);
187 
188 		test_fn(r1_info->raid_bdev, raid_ch);
189 
190 		raid_test_destroy_io_channel(raid_ch);
191 		delete_raid1(r1_info);
192 	}
193 }
194 
195 static void
196 _test_raid1_read_balancing(struct raid_bdev *raid_bdev, struct raid_bdev_io_channel *raid_ch)
197 {
198 	struct raid1_info *r1_info = raid_bdev->module_private;
199 	struct raid1_io_channel *raid1_ch = raid_bdev_channel_get_module_ctx(raid_ch);
200 	uint8_t big_io_base_bdev_idx;
201 	const uint64_t big_io_blocks = 256;
202 	const uint64_t small_io_blocks = 4;
203 	uint64_t blocks_remaining;
204 	struct raid_bdev_io *raid_io;
205 	uint8_t i;
206 	int n;
207 
208 	/* same sized IOs should be be spread evenly across all base bdevs */
209 	for (n = 0; n < 3; n++) {
210 		for (i = 0; i < raid_bdev->num_base_bdevs; i++) {
211 			raid_io = get_raid_io(r1_info, raid_ch, SPDK_BDEV_IO_TYPE_READ, small_io_blocks);
212 			raid1_submit_read_request(raid_io);
213 			CU_ASSERT(raid_io->base_bdev_io_submitted == i);
214 			put_raid_io(raid_io);
215 		}
216 	}
217 
218 	for (i = 0; i < raid_bdev->num_base_bdevs; i++) {
219 		CU_ASSERT(raid1_ch->read_blocks_outstanding[i] == n * small_io_blocks);
220 		raid1_ch->read_blocks_outstanding[i] = 0;
221 	}
222 
223 	/*
224 	 * Submit one big and many small IOs. The small IOs should not land on the same base bdev
225 	 * as the big until the submitted block count is matched.
226 	 */
227 	raid_io = get_raid_io(r1_info, raid_ch, SPDK_BDEV_IO_TYPE_READ, big_io_blocks);
228 	raid1_submit_read_request(raid_io);
229 	big_io_base_bdev_idx = raid_io->base_bdev_io_submitted;
230 	put_raid_io(raid_io);
231 
232 	blocks_remaining = big_io_blocks * (raid_bdev->num_base_bdevs - 1);
233 	while (blocks_remaining > 0) {
234 		raid_io = get_raid_io(r1_info, raid_ch, SPDK_BDEV_IO_TYPE_READ, small_io_blocks);
235 		raid1_submit_read_request(raid_io);
236 		CU_ASSERT(raid_io->base_bdev_io_submitted != big_io_base_bdev_idx);
237 		put_raid_io(raid_io);
238 		blocks_remaining -= small_io_blocks;
239 	}
240 
241 	for (i = 0; i < raid_bdev->num_base_bdevs; i++) {
242 		CU_ASSERT(raid1_ch->read_blocks_outstanding[i] == big_io_blocks);
243 	}
244 
245 	raid_io = get_raid_io(r1_info, raid_ch, SPDK_BDEV_IO_TYPE_READ, small_io_blocks);
246 	raid1_submit_read_request(raid_io);
247 	CU_ASSERT(raid_io->base_bdev_io_submitted == big_io_base_bdev_idx);
248 	put_raid_io(raid_io);
249 }
250 
251 static void
252 test_raid1_read_balancing(void)
253 {
254 	run_for_each_raid1_config(_test_raid1_read_balancing);
255 }
256 
257 static void
258 _test_raid1_write_error(struct raid_bdev *raid_bdev, struct raid_bdev_io_channel *raid_ch)
259 {
260 	struct raid1_info *r1_info = raid_bdev->module_private;
261 	struct raid_bdev_io *raid_io;
262 	struct raid_base_bdev_info *base_info;
263 	struct spdk_bdev_io bdev_io = {};
264 	bool bdev_io_success;
265 
266 	/* first completion failed */
267 	g_io_status = SPDK_BDEV_IO_STATUS_PENDING;
268 	raid_io = get_raid_io(r1_info, raid_ch, SPDK_BDEV_IO_TYPE_WRITE, 64);
269 	raid1_submit_write_request(raid_io);
270 
271 	RAID_FOR_EACH_BASE_BDEV(raid_bdev, base_info) {
272 		base_info->is_failed = false;
273 		if (raid_bdev_base_bdev_slot(base_info) == 0) {
274 			bdev_io_success = false;
275 		} else {
276 			bdev_io_success = true;
277 		}
278 		bdev_io.bdev = base_info->desc->bdev;
279 		raid1_write_bdev_io_completion(&bdev_io, bdev_io_success, raid_io);
280 		CU_ASSERT(base_info->is_failed == !bdev_io_success);
281 	}
282 	CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_SUCCESS);
283 
284 	/* all except first completion failed */
285 	g_io_status = SPDK_BDEV_IO_STATUS_PENDING;
286 	raid_io = get_raid_io(r1_info, raid_ch, SPDK_BDEV_IO_TYPE_WRITE, 64);
287 	raid1_submit_write_request(raid_io);
288 
289 	RAID_FOR_EACH_BASE_BDEV(raid_bdev, base_info) {
290 		base_info->is_failed = false;
291 		if (raid_bdev_base_bdev_slot(base_info) != 0) {
292 			bdev_io_success = false;
293 		} else {
294 			bdev_io_success = true;
295 		}
296 		bdev_io.bdev = base_info->desc->bdev;
297 		raid1_write_bdev_io_completion(&bdev_io, bdev_io_success, raid_io);
298 		CU_ASSERT(base_info->is_failed == !bdev_io_success);
299 	}
300 	CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_SUCCESS);
301 
302 	/* all completions failed */
303 	g_io_status = SPDK_BDEV_IO_STATUS_PENDING;
304 	raid_io = get_raid_io(r1_info, raid_ch, SPDK_BDEV_IO_TYPE_WRITE, 64);
305 	raid1_submit_write_request(raid_io);
306 
307 	bdev_io_success = false;
308 	RAID_FOR_EACH_BASE_BDEV(raid_bdev, base_info) {
309 		base_info->is_failed = false;
310 		bdev_io.bdev = base_info->desc->bdev;
311 		raid1_write_bdev_io_completion(&bdev_io, bdev_io_success, raid_io);
312 		CU_ASSERT(base_info->is_failed == !bdev_io_success);
313 	}
314 	CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_FAILED);
315 }
316 
317 static void
318 test_raid1_write_error(void)
319 {
320 	run_for_each_raid1_config(_test_raid1_write_error);
321 }
322 
323 static void
324 _test_raid1_read_error(struct raid_bdev *raid_bdev, struct raid_bdev_io_channel *raid_ch)
325 {
326 	struct raid1_info *r1_info = raid_bdev->module_private;
327 	struct raid_base_bdev_info *base_info = &raid_bdev->base_bdev_info[0];
328 	struct raid1_io_channel *raid1_ch = raid_bdev_channel_get_module_ctx(raid_ch);
329 	struct raid_bdev_io *raid_io;
330 	struct spdk_bdev_io bdev_io = {};
331 
332 	/* first read fails, the second succeeds */
333 	base_info->is_failed = false;
334 	g_io_status = SPDK_BDEV_IO_STATUS_PENDING;
335 	raid_io = get_raid_io(r1_info, raid_ch, SPDK_BDEV_IO_TYPE_READ, 64);
336 	raid1_submit_read_request(raid_io);
337 	CU_ASSERT(raid_io->base_bdev_io_submitted == 0);
338 	CU_ASSERT(raid_io->base_bdev_io_remaining == 0);
339 
340 	CU_ASSERT(g_last_io_desc == base_info->desc);
341 	CU_ASSERT(g_last_io_cb == raid1_read_bdev_io_completion);
342 	raid1_read_bdev_io_completion(&bdev_io, false, raid_io);
343 	CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_PENDING);
344 	CU_ASSERT((uint8_t)raid_io->base_bdev_io_remaining == (raid_bdev->num_base_bdevs - 1));
345 
346 	CU_ASSERT(g_last_io_desc == raid_bdev->base_bdev_info[1].desc);
347 	CU_ASSERT(g_last_io_cb == raid1_read_other_completion);
348 	raid1_read_other_completion(&bdev_io, true, raid_io);
349 	CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_PENDING);
350 
351 	CU_ASSERT(g_last_io_desc == base_info->desc);
352 	CU_ASSERT(g_last_io_cb == raid1_correct_read_error_completion);
353 	raid1_correct_read_error_completion(&bdev_io, true, raid_io);
354 	CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_SUCCESS);
355 	CU_ASSERT(base_info->is_failed == false);
356 
357 	/* rewrite fails */
358 	base_info->is_failed = false;
359 	g_io_status = SPDK_BDEV_IO_STATUS_PENDING;
360 	raid_io = get_raid_io(r1_info, raid_ch, SPDK_BDEV_IO_TYPE_READ, 64);
361 	raid1_submit_read_request(raid_io);
362 	CU_ASSERT(raid_io->base_bdev_io_submitted == 0);
363 	CU_ASSERT(raid_io->base_bdev_io_remaining == 0);
364 
365 	CU_ASSERT(g_last_io_desc == base_info->desc);
366 	CU_ASSERT(g_last_io_cb == raid1_read_bdev_io_completion);
367 	raid1_read_bdev_io_completion(&bdev_io, false, raid_io);
368 	CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_PENDING);
369 	CU_ASSERT((uint8_t)raid_io->base_bdev_io_remaining == (raid_bdev->num_base_bdevs - 1));
370 
371 	CU_ASSERT(g_last_io_desc == raid_bdev->base_bdev_info[1].desc);
372 	CU_ASSERT(g_last_io_cb == raid1_read_other_completion);
373 	raid1_read_other_completion(&bdev_io, true, raid_io);
374 	CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_PENDING);
375 
376 	CU_ASSERT(g_last_io_desc == base_info->desc);
377 	CU_ASSERT(g_last_io_cb == raid1_correct_read_error_completion);
378 	raid1_correct_read_error_completion(&bdev_io, false, raid_io);
379 	CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_SUCCESS);
380 	CU_ASSERT(base_info->is_failed == true);
381 
382 	/* only the last read succeeds */
383 	base_info->is_failed = false;
384 	g_io_status = SPDK_BDEV_IO_STATUS_PENDING;
385 	raid_io = get_raid_io(r1_info, raid_ch, SPDK_BDEV_IO_TYPE_READ, 64);
386 	raid1_submit_read_request(raid_io);
387 	CU_ASSERT(raid_io->base_bdev_io_submitted == 0);
388 	CU_ASSERT(raid_io->base_bdev_io_remaining == 0);
389 
390 	CU_ASSERT(g_last_io_desc == base_info->desc);
391 	CU_ASSERT(g_last_io_cb == raid1_read_bdev_io_completion);
392 	raid1_read_bdev_io_completion(&bdev_io, false, raid_io);
393 	CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_PENDING);
394 	CU_ASSERT((uint8_t)raid_io->base_bdev_io_remaining == (raid_bdev->num_base_bdevs - 1));
395 
396 	while (raid_io->base_bdev_io_remaining > 1) {
397 		CU_ASSERT(g_last_io_cb == raid1_read_other_completion);
398 		raid1_read_other_completion(&bdev_io, false, raid_io);
399 		CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_PENDING);
400 	}
401 
402 	CU_ASSERT(g_last_io_desc == raid_bdev->base_bdev_info[raid_bdev->num_base_bdevs - 1].desc);
403 	CU_ASSERT(g_last_io_cb == raid1_read_other_completion);
404 	raid1_read_other_completion(&bdev_io, true, raid_io);
405 	CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_PENDING);
406 
407 	CU_ASSERT(g_last_io_desc == base_info->desc);
408 	CU_ASSERT(g_last_io_cb == raid1_correct_read_error_completion);
409 	raid1_correct_read_error_completion(&bdev_io, true, raid_io);
410 	CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_SUCCESS);
411 	CU_ASSERT(base_info->is_failed == false);
412 
413 	/* all reads fail */
414 	base_info->is_failed = false;
415 	g_io_status = SPDK_BDEV_IO_STATUS_PENDING;
416 	raid_io = get_raid_io(r1_info, raid_ch, SPDK_BDEV_IO_TYPE_READ, 64);
417 	raid1_submit_read_request(raid_io);
418 	CU_ASSERT(raid_io->base_bdev_io_submitted == 0);
419 	CU_ASSERT(raid_io->base_bdev_io_remaining == 0);
420 
421 	CU_ASSERT(g_last_io_desc == base_info->desc);
422 	CU_ASSERT(g_last_io_cb == raid1_read_bdev_io_completion);
423 	raid1_read_bdev_io_completion(&bdev_io, false, raid_io);
424 	CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_PENDING);
425 	CU_ASSERT((uint8_t)raid_io->base_bdev_io_remaining == (raid_bdev->num_base_bdevs - 1));
426 
427 	while (raid_io->base_bdev_io_remaining > 1) {
428 		CU_ASSERT(g_last_io_cb == raid1_read_other_completion);
429 		raid1_read_other_completion(&bdev_io, false, raid_io);
430 		CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_PENDING);
431 	}
432 
433 	CU_ASSERT(g_last_io_desc == raid_bdev->base_bdev_info[raid_bdev->num_base_bdevs - 1].desc);
434 	CU_ASSERT(g_last_io_cb == raid1_read_other_completion);
435 	raid1_read_other_completion(&bdev_io, false, raid_io);
436 	CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_FAILED);
437 	CU_ASSERT(base_info->is_failed == true);
438 
439 	/* read from base bdev #1 fails, read from #0 succeeds */
440 	base_info->is_failed = false;
441 	base_info = &raid_bdev->base_bdev_info[1];
442 	raid1_ch->read_blocks_outstanding[0] = 123;
443 	g_io_status = SPDK_BDEV_IO_STATUS_PENDING;
444 	raid_io = get_raid_io(r1_info, raid_ch, SPDK_BDEV_IO_TYPE_READ, 64);
445 	raid1_submit_read_request(raid_io);
446 	CU_ASSERT(raid_io->base_bdev_io_submitted == 1);
447 	CU_ASSERT(raid_io->base_bdev_io_remaining == 0);
448 
449 	CU_ASSERT(g_last_io_desc == base_info->desc);
450 	CU_ASSERT(g_last_io_cb == raid1_read_bdev_io_completion);
451 	raid1_read_bdev_io_completion(&bdev_io, false, raid_io);
452 	CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_PENDING);
453 	CU_ASSERT((uint8_t)raid_io->base_bdev_io_remaining == raid_bdev->num_base_bdevs);
454 
455 	CU_ASSERT(g_last_io_desc == raid_bdev->base_bdev_info[0].desc);
456 	CU_ASSERT(g_last_io_cb == raid1_read_other_completion);
457 	raid1_read_other_completion(&bdev_io, true, raid_io);
458 	CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_PENDING);
459 
460 	CU_ASSERT(g_last_io_desc == base_info->desc);
461 	CU_ASSERT(g_last_io_cb == raid1_correct_read_error_completion);
462 	raid1_correct_read_error_completion(&bdev_io, true, raid_io);
463 	CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_SUCCESS);
464 	CU_ASSERT(base_info->is_failed == false);
465 
466 	/* base bdev #0 is failed, read from #1 fails, read from next succeeds if N > 2 */
467 	base_info->is_failed = false;
468 	raid_ch->_base_channels[0] = NULL;
469 	g_io_status = SPDK_BDEV_IO_STATUS_PENDING;
470 	raid_io = get_raid_io(r1_info, raid_ch, SPDK_BDEV_IO_TYPE_READ, 64);
471 	raid1_submit_read_request(raid_io);
472 	CU_ASSERT(raid_io->base_bdev_io_submitted == 1);
473 	CU_ASSERT(raid_io->base_bdev_io_remaining == 0);
474 
475 	CU_ASSERT(g_last_io_desc == base_info->desc);
476 	CU_ASSERT(g_last_io_cb == raid1_read_bdev_io_completion);
477 	raid1_read_bdev_io_completion(&bdev_io, false, raid_io);
478 	if (raid_bdev->num_base_bdevs > 2) {
479 		CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_PENDING);
480 		CU_ASSERT((uint8_t)raid_io->base_bdev_io_remaining == (raid_bdev->num_base_bdevs - 2));
481 
482 		CU_ASSERT(g_last_io_desc == raid_bdev->base_bdev_info[2].desc);
483 		CU_ASSERT(g_last_io_cb == raid1_read_other_completion);
484 		raid1_read_other_completion(&bdev_io, true, raid_io);
485 		CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_PENDING);
486 
487 		CU_ASSERT(g_last_io_desc == base_info->desc);
488 		CU_ASSERT(g_last_io_cb == raid1_correct_read_error_completion);
489 		raid1_correct_read_error_completion(&bdev_io, true, raid_io);
490 		CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_SUCCESS);
491 		CU_ASSERT(base_info->is_failed == false);
492 	} else {
493 		CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_FAILED);
494 		CU_ASSERT(base_info->is_failed == true);
495 	}
496 }
497 
498 static void
499 test_raid1_read_error(void)
500 {
501 	run_for_each_raid1_config(_test_raid1_read_error);
502 }
503 
504 int
505 main(int argc, char **argv)
506 {
507 	CU_pSuite suite = NULL;
508 	unsigned int num_failures;
509 
510 	CU_initialize_registry();
511 
512 	suite = CU_add_suite("raid1", test_setup, test_cleanup);
513 	CU_ADD_TEST(suite, test_raid1_start);
514 	CU_ADD_TEST(suite, test_raid1_read_balancing);
515 	CU_ADD_TEST(suite, test_raid1_write_error);
516 	CU_ADD_TEST(suite, test_raid1_read_error);
517 
518 	allocate_threads(1);
519 	set_thread(0);
520 
521 	num_failures = spdk_ut_run_tests(argc, argv, NULL);
522 	CU_cleanup_registry();
523 
524 	free_threads();
525 
526 	return num_failures;
527 }
528