xref: /spdk/test/unit/lib/blobfs/blobfs_sync_ut/blobfs_sync_ut.c (revision 488570ebd418ba07c9e69e65106dcc964f3bb41b)
1 /*   SPDX-License-Identifier: BSD-3-Clause
2  *   Copyright (c) Intel Corporation.
3  *   All rights reserved.
4  */
5 
6 #include "spdk/stdinc.h"
7 
8 #include "spdk/blobfs.h"
9 #include "spdk/env.h"
10 #include "spdk/log.h"
11 #include "spdk/barrier.h"
12 #include "thread/thread_internal.h"
13 
14 #include "spdk_cunit.h"
15 #include "unit/lib/blob/bs_dev_common.c"
16 #include "common/lib/test_env.c"
17 #include "blobfs/blobfs.c"
18 #include "blobfs/tree.c"
19 
20 struct spdk_filesystem *g_fs;
21 struct spdk_file *g_file;
22 int g_fserrno;
23 struct spdk_thread *g_dispatch_thread = NULL;
24 
25 struct ut_request {
26 	fs_request_fn fn;
27 	void *arg;
28 	volatile int done;
29 };
30 
31 DEFINE_STUB(spdk_memory_domain_memzero, int, (struct spdk_memory_domain *src_domain,
32 		void *src_domain_ctx, struct iovec *iov, uint32_t iovcnt, void (*cpl_cb)(void *, int),
33 		void *cpl_cb_arg), 0);
34 
35 static void
36 send_request(fs_request_fn fn, void *arg)
37 {
38 	spdk_thread_send_msg(g_dispatch_thread, (spdk_msg_fn)fn, arg);
39 }
40 
41 static void
42 ut_call_fn(void *arg)
43 {
44 	struct ut_request *req = arg;
45 
46 	req->fn(req->arg);
47 	req->done = 1;
48 }
49 
50 static void
51 ut_send_request(fs_request_fn fn, void *arg)
52 {
53 	struct ut_request req;
54 
55 	req.fn = fn;
56 	req.arg = arg;
57 	req.done = 0;
58 
59 	spdk_thread_send_msg(g_dispatch_thread, ut_call_fn, &req);
60 
61 	/* Wait for this to finish */
62 	while (req.done == 0) {	}
63 }
64 
65 static void
66 fs_op_complete(void *ctx, int fserrno)
67 {
68 	g_fserrno = fserrno;
69 }
70 
71 static void
72 fs_op_with_handle_complete(void *ctx, struct spdk_filesystem *fs, int fserrno)
73 {
74 	g_fs = fs;
75 	g_fserrno = fserrno;
76 }
77 
78 static void
79 fs_thread_poll(void)
80 {
81 	struct spdk_thread *thread;
82 
83 	thread = spdk_get_thread();
84 	while (spdk_thread_poll(thread, 0, 0) > 0) {}
85 	while (spdk_thread_poll(g_cache_pool_thread, 0, 0) > 0) {}
86 }
87 
88 static void
89 _fs_init(void *arg)
90 {
91 	struct spdk_bs_dev *dev;
92 
93 	g_fs = NULL;
94 	g_fserrno = -1;
95 	dev = init_dev();
96 	spdk_fs_init(dev, NULL, send_request, fs_op_with_handle_complete, NULL);
97 
98 	fs_thread_poll();
99 
100 	SPDK_CU_ASSERT_FATAL(g_fs != NULL);
101 	SPDK_CU_ASSERT_FATAL(g_fs->bdev == dev);
102 	CU_ASSERT(g_fserrno == 0);
103 }
104 
105 static void
106 _fs_load(void *arg)
107 {
108 	struct spdk_bs_dev *dev;
109 
110 	g_fs = NULL;
111 	g_fserrno = -1;
112 	dev = init_dev();
113 	spdk_fs_load(dev, send_request, fs_op_with_handle_complete, NULL);
114 
115 	fs_thread_poll();
116 
117 	SPDK_CU_ASSERT_FATAL(g_fs != NULL);
118 	SPDK_CU_ASSERT_FATAL(g_fs->bdev == dev);
119 	CU_ASSERT(g_fserrno == 0);
120 }
121 
122 static void
123 _fs_unload(void *arg)
124 {
125 	g_fserrno = -1;
126 	spdk_fs_unload(g_fs, fs_op_complete, NULL);
127 
128 	fs_thread_poll();
129 
130 	CU_ASSERT(g_fserrno == 0);
131 	g_fs = NULL;
132 }
133 
134 static void
135 _nop(void *arg)
136 {
137 }
138 
139 static void
140 cache_read_after_write(void)
141 {
142 	uint64_t length;
143 	int rc;
144 	char w_buf[100], r_buf[100];
145 	struct spdk_fs_thread_ctx *channel;
146 	struct spdk_file_stat stat = {0};
147 
148 	ut_send_request(_fs_init, NULL);
149 
150 	channel = spdk_fs_alloc_thread_ctx(g_fs);
151 
152 	rc = spdk_fs_open_file(g_fs, channel, "testfile", SPDK_BLOBFS_OPEN_CREATE, &g_file);
153 	CU_ASSERT(rc == 0);
154 	SPDK_CU_ASSERT_FATAL(g_file != NULL);
155 
156 	length = (4 * 1024 * 1024);
157 	rc = spdk_file_truncate(g_file, channel, length);
158 	CU_ASSERT(rc == 0);
159 
160 	memset(w_buf, 0x5a, sizeof(w_buf));
161 	spdk_file_write(g_file, channel, w_buf, 0, sizeof(w_buf));
162 
163 	CU_ASSERT(spdk_file_get_length(g_file) == length);
164 
165 	rc = spdk_file_truncate(g_file, channel, sizeof(w_buf));
166 	CU_ASSERT(rc == 0);
167 
168 	spdk_file_close(g_file, channel);
169 
170 	fs_thread_poll();
171 
172 	rc = spdk_fs_file_stat(g_fs, channel, "testfile", &stat);
173 	CU_ASSERT(rc == 0);
174 	CU_ASSERT(sizeof(w_buf) == stat.size);
175 
176 	rc = spdk_fs_open_file(g_fs, channel, "testfile", 0, &g_file);
177 	CU_ASSERT(rc == 0);
178 	SPDK_CU_ASSERT_FATAL(g_file != NULL);
179 
180 	memset(r_buf, 0, sizeof(r_buf));
181 	spdk_file_read(g_file, channel, r_buf, 0, sizeof(r_buf));
182 	CU_ASSERT(memcmp(w_buf, r_buf, sizeof(r_buf)) == 0);
183 
184 	spdk_file_close(g_file, channel);
185 
186 	fs_thread_poll();
187 
188 	rc = spdk_fs_delete_file(g_fs, channel, "testfile");
189 	CU_ASSERT(rc == 0);
190 
191 	rc = spdk_fs_delete_file(g_fs, channel, "testfile");
192 	CU_ASSERT(rc == -ENOENT);
193 
194 	spdk_fs_free_thread_ctx(channel);
195 
196 	ut_send_request(_fs_unload, NULL);
197 }
198 
199 static void
200 file_length(void)
201 {
202 	int rc;
203 	char *buf;
204 	uint64_t buf_length;
205 	volatile uint64_t *length_flushed;
206 	struct spdk_fs_thread_ctx *channel;
207 	struct spdk_file_stat stat = {0};
208 
209 	ut_send_request(_fs_init, NULL);
210 
211 	channel = spdk_fs_alloc_thread_ctx(g_fs);
212 
213 	g_file = NULL;
214 	rc = spdk_fs_open_file(g_fs, channel, "testfile", SPDK_BLOBFS_OPEN_CREATE, &g_file);
215 	CU_ASSERT(rc == 0);
216 	SPDK_CU_ASSERT_FATAL(g_file != NULL);
217 
218 	/* Write one CACHE_BUFFER.  Filling at least one cache buffer triggers
219 	 * a flush to disk.
220 	 */
221 	buf_length = CACHE_BUFFER_SIZE;
222 	buf = calloc(1, buf_length);
223 	spdk_file_write(g_file, channel, buf, 0, buf_length);
224 	free(buf);
225 
226 	/* Spin until all of the data has been flushed to the SSD.  There's been no
227 	 * sync operation yet, so the xattr on the file is still 0.
228 	 *
229 	 * length_flushed: This variable is modified by a different thread in this unit
230 	 * test. So we need to dereference it as a volatile to ensure the value is always
231 	 * re-read.
232 	 */
233 	length_flushed = &g_file->length_flushed;
234 	while (*length_flushed != buf_length) {}
235 
236 	/* Close the file.  This causes an implicit sync which should write the
237 	 * length_flushed value as the "length" xattr on the file.
238 	 */
239 	spdk_file_close(g_file, channel);
240 
241 	fs_thread_poll();
242 
243 	rc = spdk_fs_file_stat(g_fs, channel, "testfile", &stat);
244 	CU_ASSERT(rc == 0);
245 	CU_ASSERT(buf_length == stat.size);
246 
247 	spdk_fs_free_thread_ctx(channel);
248 
249 	/* Unload and reload the filesystem.  The file length will be
250 	 * read during load from the length xattr.  We want to make sure
251 	 * it matches what was written when the file was originally
252 	 * written and closed.
253 	 */
254 	ut_send_request(_fs_unload, NULL);
255 
256 	ut_send_request(_fs_load, NULL);
257 
258 	channel = spdk_fs_alloc_thread_ctx(g_fs);
259 
260 	rc = spdk_fs_file_stat(g_fs, channel, "testfile", &stat);
261 	CU_ASSERT(rc == 0);
262 	CU_ASSERT(buf_length == stat.size);
263 
264 	g_file = NULL;
265 	rc = spdk_fs_open_file(g_fs, channel, "testfile", 0, &g_file);
266 	CU_ASSERT(rc == 0);
267 	SPDK_CU_ASSERT_FATAL(g_file != NULL);
268 
269 	spdk_file_close(g_file, channel);
270 
271 	fs_thread_poll();
272 
273 	rc = spdk_fs_delete_file(g_fs, channel, "testfile");
274 	CU_ASSERT(rc == 0);
275 
276 	spdk_fs_free_thread_ctx(channel);
277 
278 	ut_send_request(_fs_unload, NULL);
279 }
280 
281 static void
282 append_write_to_extend_blob(void)
283 {
284 	uint64_t blob_size, buf_length;
285 	char *buf, append_buf[64];
286 	int rc;
287 	struct spdk_fs_thread_ctx *channel;
288 
289 	ut_send_request(_fs_init, NULL);
290 
291 	channel = spdk_fs_alloc_thread_ctx(g_fs);
292 
293 	/* create a file and write the file with blob_size - 1 data length */
294 	rc = spdk_fs_open_file(g_fs, channel, "testfile", SPDK_BLOBFS_OPEN_CREATE, &g_file);
295 	CU_ASSERT(rc == 0);
296 	SPDK_CU_ASSERT_FATAL(g_file != NULL);
297 
298 	blob_size = __file_get_blob_size(g_file);
299 
300 	buf_length = blob_size - 1;
301 	buf = calloc(1, buf_length);
302 	rc = spdk_file_write(g_file, channel, buf, 0, buf_length);
303 	CU_ASSERT(rc == 0);
304 	free(buf);
305 
306 	spdk_file_close(g_file, channel);
307 	fs_thread_poll();
308 	spdk_fs_free_thread_ctx(channel);
309 	ut_send_request(_fs_unload, NULL);
310 
311 	/* load existing file and write extra 2 bytes to cross blob boundary */
312 	ut_send_request(_fs_load, NULL);
313 
314 	channel = spdk_fs_alloc_thread_ctx(g_fs);
315 	g_file = NULL;
316 	rc = spdk_fs_open_file(g_fs, channel, "testfile", 0, &g_file);
317 	CU_ASSERT(rc == 0);
318 	SPDK_CU_ASSERT_FATAL(g_file != NULL);
319 
320 	CU_ASSERT(g_file->length == buf_length);
321 	CU_ASSERT(g_file->last == NULL);
322 	CU_ASSERT(g_file->append_pos == buf_length);
323 
324 	rc = spdk_file_write(g_file, channel, append_buf, buf_length, 2);
325 	CU_ASSERT(rc == 0);
326 	CU_ASSERT(2 * blob_size == __file_get_blob_size(g_file));
327 	spdk_file_close(g_file, channel);
328 	fs_thread_poll();
329 	CU_ASSERT(g_file->length == buf_length + 2);
330 
331 	spdk_fs_free_thread_ctx(channel);
332 	ut_send_request(_fs_unload, NULL);
333 }
334 
335 static void
336 partial_buffer(void)
337 {
338 	int rc;
339 	char *buf;
340 	uint64_t buf_length;
341 	struct spdk_fs_thread_ctx *channel;
342 	struct spdk_file_stat stat = {0};
343 
344 	ut_send_request(_fs_init, NULL);
345 
346 	channel = spdk_fs_alloc_thread_ctx(g_fs);
347 
348 	g_file = NULL;
349 	rc = spdk_fs_open_file(g_fs, channel, "testfile", SPDK_BLOBFS_OPEN_CREATE, &g_file);
350 	CU_ASSERT(rc == 0);
351 	SPDK_CU_ASSERT_FATAL(g_file != NULL);
352 
353 	/* Write one CACHE_BUFFER plus one byte.  Filling at least one cache buffer triggers
354 	 * a flush to disk.  We want to make sure the extra byte is not implicitly flushed.
355 	 * It should only get flushed once we sync or close the file.
356 	 */
357 	buf_length = CACHE_BUFFER_SIZE + 1;
358 	buf = calloc(1, buf_length);
359 	spdk_file_write(g_file, channel, buf, 0, buf_length);
360 	free(buf);
361 
362 	/* Send some nop messages to the dispatch thread.  This will ensure any of the
363 	 * pending write operations are completed.  A well-functioning blobfs should only
364 	 * issue one write for the filled CACHE_BUFFER - a buggy one might try to write
365 	 * the extra byte.  So do a bunch of _nops to make sure all of them (even the buggy
366 	 * ones) get a chance to run.  Note that we can't just send a message to the
367 	 * dispatch thread to call spdk_thread_poll() because the messages are themselves
368 	 * run in the context of spdk_thread_poll().
369 	 */
370 	ut_send_request(_nop, NULL);
371 	ut_send_request(_nop, NULL);
372 	ut_send_request(_nop, NULL);
373 	ut_send_request(_nop, NULL);
374 	ut_send_request(_nop, NULL);
375 	ut_send_request(_nop, NULL);
376 
377 	CU_ASSERT(g_file->length_flushed == CACHE_BUFFER_SIZE);
378 
379 	/* Close the file.  This causes an implicit sync which should write the
380 	 * length_flushed value as the "length" xattr on the file.
381 	 */
382 	spdk_file_close(g_file, channel);
383 
384 	fs_thread_poll();
385 
386 	rc = spdk_fs_file_stat(g_fs, channel, "testfile", &stat);
387 	CU_ASSERT(rc == 0);
388 	CU_ASSERT(buf_length == stat.size);
389 
390 	rc = spdk_fs_delete_file(g_fs, channel, "testfile");
391 	CU_ASSERT(rc == 0);
392 
393 	spdk_fs_free_thread_ctx(channel);
394 
395 	ut_send_request(_fs_unload, NULL);
396 }
397 
398 static void
399 cache_write_null_buffer(void)
400 {
401 	uint64_t length;
402 	int rc;
403 	struct spdk_fs_thread_ctx *channel;
404 	struct spdk_thread *thread;
405 
406 	ut_send_request(_fs_init, NULL);
407 
408 	channel = spdk_fs_alloc_thread_ctx(g_fs);
409 
410 	rc = spdk_fs_open_file(g_fs, channel, "testfile", SPDK_BLOBFS_OPEN_CREATE, &g_file);
411 	CU_ASSERT(rc == 0);
412 	SPDK_CU_ASSERT_FATAL(g_file != NULL);
413 
414 	length = 0;
415 	rc = spdk_file_truncate(g_file, channel, length);
416 	CU_ASSERT(rc == 0);
417 
418 	rc = spdk_file_write(g_file, channel, NULL, 0, 0);
419 	CU_ASSERT(rc == 0);
420 
421 	spdk_file_close(g_file, channel);
422 
423 	fs_thread_poll();
424 
425 	rc = spdk_fs_delete_file(g_fs, channel, "testfile");
426 	CU_ASSERT(rc == 0);
427 
428 	spdk_fs_free_thread_ctx(channel);
429 
430 	thread = spdk_get_thread();
431 	while (spdk_thread_poll(thread, 0, 0) > 0) {}
432 
433 	ut_send_request(_fs_unload, NULL);
434 }
435 
436 static void
437 fs_create_sync(void)
438 {
439 	int rc;
440 	struct spdk_fs_thread_ctx *channel;
441 
442 	ut_send_request(_fs_init, NULL);
443 
444 	channel = spdk_fs_alloc_thread_ctx(g_fs);
445 	CU_ASSERT(channel != NULL);
446 
447 	rc = spdk_fs_create_file(g_fs, channel, "testfile");
448 	CU_ASSERT(rc == 0);
449 
450 	/* Create should fail, because the file already exists. */
451 	rc = spdk_fs_create_file(g_fs, channel, "testfile");
452 	CU_ASSERT(rc != 0);
453 
454 	rc = spdk_fs_delete_file(g_fs, channel, "testfile");
455 	CU_ASSERT(rc == 0);
456 
457 	spdk_fs_free_thread_ctx(channel);
458 
459 	fs_thread_poll();
460 
461 	ut_send_request(_fs_unload, NULL);
462 }
463 
464 static void
465 fs_rename_sync(void)
466 {
467 	int rc;
468 	struct spdk_fs_thread_ctx *channel;
469 
470 	ut_send_request(_fs_init, NULL);
471 
472 	channel = spdk_fs_alloc_thread_ctx(g_fs);
473 	CU_ASSERT(channel != NULL);
474 
475 	rc = spdk_fs_open_file(g_fs, channel, "testfile", SPDK_BLOBFS_OPEN_CREATE, &g_file);
476 	CU_ASSERT(rc == 0);
477 	SPDK_CU_ASSERT_FATAL(g_file != NULL);
478 
479 	CU_ASSERT(strcmp(spdk_file_get_name(g_file), "testfile") == 0);
480 
481 	rc = spdk_fs_rename_file(g_fs, channel, "testfile", "newtestfile");
482 	CU_ASSERT(rc == 0);
483 	CU_ASSERT(strcmp(spdk_file_get_name(g_file), "newtestfile") == 0);
484 
485 	spdk_file_close(g_file, channel);
486 
487 	fs_thread_poll();
488 
489 	spdk_fs_free_thread_ctx(channel);
490 
491 	ut_send_request(_fs_unload, NULL);
492 }
493 
494 static void
495 cache_append_no_cache(void)
496 {
497 	int rc;
498 	char buf[100];
499 	struct spdk_fs_thread_ctx *channel;
500 
501 	ut_send_request(_fs_init, NULL);
502 
503 	channel = spdk_fs_alloc_thread_ctx(g_fs);
504 
505 	rc = spdk_fs_open_file(g_fs, channel, "testfile", SPDK_BLOBFS_OPEN_CREATE, &g_file);
506 	CU_ASSERT(rc == 0);
507 	SPDK_CU_ASSERT_FATAL(g_file != NULL);
508 
509 	spdk_file_write(g_file, channel, buf, 0 * sizeof(buf), sizeof(buf));
510 	CU_ASSERT(spdk_file_get_length(g_file) == 1 * sizeof(buf));
511 	spdk_file_write(g_file, channel, buf, 1 * sizeof(buf), sizeof(buf));
512 	CU_ASSERT(spdk_file_get_length(g_file) == 2 * sizeof(buf));
513 	spdk_file_sync(g_file, channel);
514 
515 	fs_thread_poll();
516 
517 	spdk_file_write(g_file, channel, buf, 2 * sizeof(buf), sizeof(buf));
518 	CU_ASSERT(spdk_file_get_length(g_file) == 3 * sizeof(buf));
519 	spdk_file_write(g_file, channel, buf, 3 * sizeof(buf), sizeof(buf));
520 	CU_ASSERT(spdk_file_get_length(g_file) == 4 * sizeof(buf));
521 	spdk_file_write(g_file, channel, buf, 4 * sizeof(buf), sizeof(buf));
522 	CU_ASSERT(spdk_file_get_length(g_file) == 5 * sizeof(buf));
523 
524 	spdk_file_close(g_file, channel);
525 
526 	fs_thread_poll();
527 
528 	rc = spdk_fs_delete_file(g_fs, channel, "testfile");
529 	CU_ASSERT(rc == 0);
530 
531 	spdk_fs_free_thread_ctx(channel);
532 
533 	ut_send_request(_fs_unload, NULL);
534 }
535 
536 static void
537 fs_delete_file_without_close(void)
538 {
539 	int rc;
540 	struct spdk_fs_thread_ctx *channel;
541 	struct spdk_file *file;
542 
543 	ut_send_request(_fs_init, NULL);
544 	channel = spdk_fs_alloc_thread_ctx(g_fs);
545 	CU_ASSERT(channel != NULL);
546 
547 	rc = spdk_fs_open_file(g_fs, channel, "testfile", SPDK_BLOBFS_OPEN_CREATE, &g_file);
548 	CU_ASSERT(rc == 0);
549 	SPDK_CU_ASSERT_FATAL(g_file != NULL);
550 
551 	rc = spdk_fs_delete_file(g_fs, channel, "testfile");
552 	CU_ASSERT(rc == 0);
553 	CU_ASSERT(g_file->ref_count != 0);
554 	CU_ASSERT(g_file->is_deleted == true);
555 
556 	rc = spdk_fs_open_file(g_fs, channel, "testfile", 0, &file);
557 	CU_ASSERT(rc != 0);
558 
559 	spdk_file_close(g_file, channel);
560 
561 	fs_thread_poll();
562 
563 	rc = spdk_fs_open_file(g_fs, channel, "testfile", 0, &file);
564 	CU_ASSERT(rc != 0);
565 
566 	spdk_fs_free_thread_ctx(channel);
567 
568 	ut_send_request(_fs_unload, NULL);
569 
570 }
571 
572 static bool g_thread_exit = false;
573 
574 static void
575 terminate_spdk_thread(void *arg)
576 {
577 	g_thread_exit = true;
578 }
579 
580 static void *
581 spdk_thread(void *arg)
582 {
583 	struct spdk_thread *thread = arg;
584 
585 	spdk_set_thread(thread);
586 
587 	while (!g_thread_exit) {
588 		spdk_thread_poll(thread, 0, 0);
589 	}
590 
591 	return NULL;
592 }
593 
594 int main(int argc, char **argv)
595 {
596 	struct spdk_thread *thread;
597 	CU_pSuite	suite = NULL;
598 	pthread_t	spdk_tid;
599 	unsigned int	num_failures;
600 
601 	CU_set_error_action(CUEA_ABORT);
602 	CU_initialize_registry();
603 
604 	suite = CU_add_suite("blobfs_sync_ut", NULL, NULL);
605 
606 	CU_ADD_TEST(suite, cache_read_after_write);
607 	CU_ADD_TEST(suite, file_length);
608 	CU_ADD_TEST(suite, append_write_to_extend_blob);
609 	CU_ADD_TEST(suite, partial_buffer);
610 	CU_ADD_TEST(suite, cache_write_null_buffer);
611 	CU_ADD_TEST(suite, fs_create_sync);
612 	CU_ADD_TEST(suite, fs_rename_sync);
613 	CU_ADD_TEST(suite, cache_append_no_cache);
614 	CU_ADD_TEST(suite, fs_delete_file_without_close);
615 
616 	spdk_thread_lib_init(NULL, 0);
617 
618 	thread = spdk_thread_create("test_thread", NULL);
619 	spdk_set_thread(thread);
620 
621 	g_dispatch_thread = spdk_thread_create("dispatch_thread", NULL);
622 	pthread_create(&spdk_tid, NULL, spdk_thread, g_dispatch_thread);
623 
624 	g_dev_buffer = calloc(1, DEV_BUFFER_SIZE);
625 
626 	CU_basic_set_mode(CU_BRM_VERBOSE);
627 	CU_basic_run_tests();
628 	num_failures = CU_get_number_of_failures();
629 	CU_cleanup_registry();
630 
631 	free(g_dev_buffer);
632 
633 	ut_send_request(terminate_spdk_thread, NULL);
634 	pthread_join(spdk_tid, NULL);
635 
636 	while (spdk_thread_poll(g_dispatch_thread, 0, 0) > 0) {}
637 	while (spdk_thread_poll(thread, 0, 0) > 0) {}
638 
639 	spdk_set_thread(thread);
640 	spdk_thread_exit(thread);
641 	while (!spdk_thread_is_exited(thread)) {
642 		spdk_thread_poll(thread, 0, 0);
643 	}
644 	spdk_thread_destroy(thread);
645 
646 	spdk_set_thread(g_dispatch_thread);
647 	spdk_thread_exit(g_dispatch_thread);
648 	while (!spdk_thread_is_exited(g_dispatch_thread)) {
649 		spdk_thread_poll(g_dispatch_thread, 0, 0);
650 	}
651 	spdk_thread_destroy(g_dispatch_thread);
652 
653 	spdk_thread_lib_fini();
654 
655 	return num_failures;
656 }
657