xref: /spdk/test/unit/lib/blobfs/blobfs_sync_ut/blobfs_sync_ut.c (revision ea941caeaf896fdf2aef7685f86f37023060faed)
1 /*   SPDX-License-Identifier: BSD-3-Clause
2  *   Copyright (C) 2017 Intel Corporation.
3  *   All rights reserved.
4  */
5 
6 #include "spdk/stdinc.h"
7 
8 #include "spdk/blobfs.h"
9 #include "spdk/env.h"
10 #include "spdk/log.h"
11 #include "spdk/barrier.h"
12 #include "thread/thread_internal.h"
13 
14 #include "spdk_internal/cunit.h"
15 #include "unit/lib/blob/bs_dev_common.c"
16 #include "common/lib/test_env.c"
17 #include "blobfs/blobfs.c"
18 #include "blobfs/tree.c"
19 
20 struct spdk_filesystem *g_fs;
21 struct spdk_file *g_file;
22 int g_fserrno;
23 struct spdk_thread *g_dispatch_thread = NULL;
24 
25 struct ut_request {
26 	fs_request_fn fn;
27 	void *arg;
28 	volatile int done;
29 };
30 
31 DEFINE_STUB(spdk_memory_domain_memzero, int, (struct spdk_memory_domain *src_domain,
32 		void *src_domain_ctx, struct iovec *iov, uint32_t iovcnt, void (*cpl_cb)(void *, int),
33 		void *cpl_cb_arg), 0);
34 DEFINE_STUB(spdk_mempool_lookup, struct spdk_mempool *, (const char *name), NULL);
35 
36 static void
send_request(fs_request_fn fn,void * arg)37 send_request(fs_request_fn fn, void *arg)
38 {
39 	spdk_thread_send_msg(g_dispatch_thread, (spdk_msg_fn)fn, arg);
40 }
41 
42 static void
ut_call_fn(void * arg)43 ut_call_fn(void *arg)
44 {
45 	struct ut_request *req = arg;
46 
47 	req->fn(req->arg);
48 	req->done = 1;
49 }
50 
51 static void
ut_send_request(fs_request_fn fn,void * arg)52 ut_send_request(fs_request_fn fn, void *arg)
53 {
54 	struct ut_request req;
55 
56 	req.fn = fn;
57 	req.arg = arg;
58 	req.done = 0;
59 
60 	spdk_thread_send_msg(g_dispatch_thread, ut_call_fn, &req);
61 
62 	/* Wait for this to finish */
63 	while (req.done == 0) {	}
64 }
65 
66 static void
fs_op_complete(void * ctx,int fserrno)67 fs_op_complete(void *ctx, int fserrno)
68 {
69 	g_fserrno = fserrno;
70 }
71 
72 static void
fs_op_with_handle_complete(void * ctx,struct spdk_filesystem * fs,int fserrno)73 fs_op_with_handle_complete(void *ctx, struct spdk_filesystem *fs, int fserrno)
74 {
75 	g_fs = fs;
76 	g_fserrno = fserrno;
77 }
78 
79 static void
fs_thread_poll(void)80 fs_thread_poll(void)
81 {
82 	struct spdk_thread *thread;
83 
84 	thread = spdk_get_thread();
85 	while (spdk_thread_poll(thread, 0, 0) > 0) {}
86 	while (spdk_thread_poll(g_cache_pool_thread, 0, 0) > 0) {}
87 }
88 
89 static void
_fs_init(void * arg)90 _fs_init(void *arg)
91 {
92 	struct spdk_bs_dev *dev;
93 
94 	g_fs = NULL;
95 	g_fserrno = -1;
96 	dev = init_dev();
97 	spdk_fs_init(dev, NULL, send_request, fs_op_with_handle_complete, NULL);
98 
99 	fs_thread_poll();
100 
101 	SPDK_CU_ASSERT_FATAL(g_fs != NULL);
102 	SPDK_CU_ASSERT_FATAL(g_fs->bdev == dev);
103 	CU_ASSERT(g_fserrno == 0);
104 }
105 
106 static void
_fs_load(void * arg)107 _fs_load(void *arg)
108 {
109 	struct spdk_bs_dev *dev;
110 
111 	g_fs = NULL;
112 	g_fserrno = -1;
113 	dev = init_dev();
114 	spdk_fs_load(dev, send_request, fs_op_with_handle_complete, NULL);
115 
116 	fs_thread_poll();
117 
118 	SPDK_CU_ASSERT_FATAL(g_fs != NULL);
119 	SPDK_CU_ASSERT_FATAL(g_fs->bdev == dev);
120 	CU_ASSERT(g_fserrno == 0);
121 }
122 
123 static void
_fs_unload(void * arg)124 _fs_unload(void *arg)
125 {
126 	g_fserrno = -1;
127 	spdk_fs_unload(g_fs, fs_op_complete, NULL);
128 
129 	fs_thread_poll();
130 
131 	CU_ASSERT(g_fserrno == 0);
132 	g_fs = NULL;
133 }
134 
135 static void
_nop(void * arg)136 _nop(void *arg)
137 {
138 }
139 
140 static void
cache_read_after_write(void)141 cache_read_after_write(void)
142 {
143 	uint64_t length;
144 	int rc;
145 	char w_buf[100], r_buf[100];
146 	struct spdk_fs_thread_ctx *channel;
147 	struct spdk_file_stat stat = {0};
148 
149 	ut_send_request(_fs_init, NULL);
150 
151 	channel = spdk_fs_alloc_thread_ctx(g_fs);
152 
153 	rc = spdk_fs_open_file(g_fs, channel, "testfile", SPDK_BLOBFS_OPEN_CREATE, &g_file);
154 	CU_ASSERT(rc == 0);
155 	SPDK_CU_ASSERT_FATAL(g_file != NULL);
156 
157 	length = (4 * 1024 * 1024);
158 	rc = spdk_file_truncate(g_file, channel, length);
159 	CU_ASSERT(rc == 0);
160 
161 	memset(w_buf, 0x5a, sizeof(w_buf));
162 	spdk_file_write(g_file, channel, w_buf, 0, sizeof(w_buf));
163 
164 	CU_ASSERT(spdk_file_get_length(g_file) == length);
165 
166 	rc = spdk_file_truncate(g_file, channel, sizeof(w_buf));
167 	CU_ASSERT(rc == 0);
168 
169 	spdk_file_close(g_file, channel);
170 
171 	fs_thread_poll();
172 
173 	rc = spdk_fs_file_stat(g_fs, channel, "testfile", &stat);
174 	CU_ASSERT(rc == 0);
175 	CU_ASSERT(sizeof(w_buf) == stat.size);
176 
177 	rc = spdk_fs_open_file(g_fs, channel, "testfile", 0, &g_file);
178 	CU_ASSERT(rc == 0);
179 	SPDK_CU_ASSERT_FATAL(g_file != NULL);
180 
181 	memset(r_buf, 0, sizeof(r_buf));
182 	spdk_file_read(g_file, channel, r_buf, 0, sizeof(r_buf));
183 	CU_ASSERT(memcmp(w_buf, r_buf, sizeof(r_buf)) == 0);
184 
185 	spdk_file_close(g_file, channel);
186 
187 	fs_thread_poll();
188 
189 	rc = spdk_fs_delete_file(g_fs, channel, "testfile");
190 	CU_ASSERT(rc == 0);
191 
192 	rc = spdk_fs_delete_file(g_fs, channel, "testfile");
193 	CU_ASSERT(rc == -ENOENT);
194 
195 	spdk_fs_free_thread_ctx(channel);
196 
197 	ut_send_request(_fs_unload, NULL);
198 }
199 
200 static void
file_length(void)201 file_length(void)
202 {
203 	int rc;
204 	char *buf;
205 	uint64_t buf_length;
206 	volatile uint64_t *length_flushed;
207 	struct spdk_fs_thread_ctx *channel;
208 	struct spdk_file_stat stat = {0};
209 
210 	ut_send_request(_fs_init, NULL);
211 
212 	channel = spdk_fs_alloc_thread_ctx(g_fs);
213 
214 	g_file = NULL;
215 	rc = spdk_fs_open_file(g_fs, channel, "testfile", SPDK_BLOBFS_OPEN_CREATE, &g_file);
216 	CU_ASSERT(rc == 0);
217 	SPDK_CU_ASSERT_FATAL(g_file != NULL);
218 
219 	/* Write one CACHE_BUFFER.  Filling at least one cache buffer triggers
220 	 * a flush to disk.
221 	 */
222 	buf_length = CACHE_BUFFER_SIZE;
223 	buf = calloc(1, buf_length);
224 	spdk_file_write(g_file, channel, buf, 0, buf_length);
225 	free(buf);
226 
227 	/* Spin until all of the data has been flushed to the SSD.  There's been no
228 	 * sync operation yet, so the xattr on the file is still 0.
229 	 *
230 	 * length_flushed: This variable is modified by a different thread in this unit
231 	 * test. So we need to dereference it as a volatile to ensure the value is always
232 	 * re-read.
233 	 */
234 	length_flushed = &g_file->length_flushed;
235 	while (*length_flushed != buf_length) {}
236 
237 	/* Close the file.  This causes an implicit sync which should write the
238 	 * length_flushed value as the "length" xattr on the file.
239 	 */
240 	spdk_file_close(g_file, channel);
241 
242 	fs_thread_poll();
243 
244 	rc = spdk_fs_file_stat(g_fs, channel, "testfile", &stat);
245 	CU_ASSERT(rc == 0);
246 	CU_ASSERT(buf_length == stat.size);
247 
248 	spdk_fs_free_thread_ctx(channel);
249 
250 	/* Unload and reload the filesystem.  The file length will be
251 	 * read during load from the length xattr.  We want to make sure
252 	 * it matches what was written when the file was originally
253 	 * written and closed.
254 	 */
255 	ut_send_request(_fs_unload, NULL);
256 
257 	ut_send_request(_fs_load, NULL);
258 
259 	channel = spdk_fs_alloc_thread_ctx(g_fs);
260 
261 	rc = spdk_fs_file_stat(g_fs, channel, "testfile", &stat);
262 	CU_ASSERT(rc == 0);
263 	CU_ASSERT(buf_length == stat.size);
264 
265 	g_file = NULL;
266 	rc = spdk_fs_open_file(g_fs, channel, "testfile", 0, &g_file);
267 	CU_ASSERT(rc == 0);
268 	SPDK_CU_ASSERT_FATAL(g_file != NULL);
269 
270 	spdk_file_close(g_file, channel);
271 
272 	fs_thread_poll();
273 
274 	rc = spdk_fs_delete_file(g_fs, channel, "testfile");
275 	CU_ASSERT(rc == 0);
276 
277 	spdk_fs_free_thread_ctx(channel);
278 
279 	ut_send_request(_fs_unload, NULL);
280 }
281 
282 static void
append_write_to_extend_blob(void)283 append_write_to_extend_blob(void)
284 {
285 	uint64_t blob_size, buf_length;
286 	char *buf, append_buf[64];
287 	int rc;
288 	struct spdk_fs_thread_ctx *channel;
289 
290 	ut_send_request(_fs_init, NULL);
291 
292 	channel = spdk_fs_alloc_thread_ctx(g_fs);
293 
294 	/* create a file and write the file with blob_size - 1 data length */
295 	rc = spdk_fs_open_file(g_fs, channel, "testfile", SPDK_BLOBFS_OPEN_CREATE, &g_file);
296 	CU_ASSERT(rc == 0);
297 	SPDK_CU_ASSERT_FATAL(g_file != NULL);
298 
299 	blob_size = __file_get_blob_size(g_file);
300 
301 	buf_length = blob_size - 1;
302 	buf = calloc(1, buf_length);
303 	rc = spdk_file_write(g_file, channel, buf, 0, buf_length);
304 	CU_ASSERT(rc == 0);
305 	free(buf);
306 
307 	spdk_file_close(g_file, channel);
308 	fs_thread_poll();
309 	spdk_fs_free_thread_ctx(channel);
310 	ut_send_request(_fs_unload, NULL);
311 
312 	/* load existing file and write extra 2 bytes to cross blob boundary */
313 	ut_send_request(_fs_load, NULL);
314 
315 	channel = spdk_fs_alloc_thread_ctx(g_fs);
316 	g_file = NULL;
317 	rc = spdk_fs_open_file(g_fs, channel, "testfile", 0, &g_file);
318 	CU_ASSERT(rc == 0);
319 	SPDK_CU_ASSERT_FATAL(g_file != NULL);
320 
321 	CU_ASSERT(g_file->length == buf_length);
322 	CU_ASSERT(g_file->last == NULL);
323 	CU_ASSERT(g_file->append_pos == buf_length);
324 
325 	rc = spdk_file_write(g_file, channel, append_buf, buf_length, 2);
326 	CU_ASSERT(rc == 0);
327 	CU_ASSERT(2 * blob_size == __file_get_blob_size(g_file));
328 	spdk_file_close(g_file, channel);
329 	fs_thread_poll();
330 	CU_ASSERT(g_file->length == buf_length + 2);
331 
332 	spdk_fs_free_thread_ctx(channel);
333 	ut_send_request(_fs_unload, NULL);
334 }
335 
336 static void
partial_buffer(void)337 partial_buffer(void)
338 {
339 	int rc;
340 	char *buf;
341 	uint64_t buf_length;
342 	struct spdk_fs_thread_ctx *channel;
343 	struct spdk_file_stat stat = {0};
344 
345 	ut_send_request(_fs_init, NULL);
346 
347 	channel = spdk_fs_alloc_thread_ctx(g_fs);
348 
349 	g_file = NULL;
350 	rc = spdk_fs_open_file(g_fs, channel, "testfile", SPDK_BLOBFS_OPEN_CREATE, &g_file);
351 	CU_ASSERT(rc == 0);
352 	SPDK_CU_ASSERT_FATAL(g_file != NULL);
353 
354 	/* Write one CACHE_BUFFER plus one byte.  Filling at least one cache buffer triggers
355 	 * a flush to disk.  We want to make sure the extra byte is not implicitly flushed.
356 	 * It should only get flushed once we sync or close the file.
357 	 */
358 	buf_length = CACHE_BUFFER_SIZE + 1;
359 	buf = calloc(1, buf_length);
360 	spdk_file_write(g_file, channel, buf, 0, buf_length);
361 	free(buf);
362 
363 	/* Send some nop messages to the dispatch thread.  This will ensure any of the
364 	 * pending write operations are completed.  A well-functioning blobfs should only
365 	 * issue one write for the filled CACHE_BUFFER - a buggy one might try to write
366 	 * the extra byte.  So do a bunch of _nops to make sure all of them (even the buggy
367 	 * ones) get a chance to run.  Note that we can't just send a message to the
368 	 * dispatch thread to call spdk_thread_poll() because the messages are themselves
369 	 * run in the context of spdk_thread_poll().
370 	 */
371 	ut_send_request(_nop, NULL);
372 	ut_send_request(_nop, NULL);
373 	ut_send_request(_nop, NULL);
374 	ut_send_request(_nop, NULL);
375 	ut_send_request(_nop, NULL);
376 	ut_send_request(_nop, NULL);
377 
378 	CU_ASSERT(g_file->length_flushed == CACHE_BUFFER_SIZE);
379 
380 	/* Close the file.  This causes an implicit sync which should write the
381 	 * length_flushed value as the "length" xattr on the file.
382 	 */
383 	spdk_file_close(g_file, channel);
384 
385 	fs_thread_poll();
386 
387 	rc = spdk_fs_file_stat(g_fs, channel, "testfile", &stat);
388 	CU_ASSERT(rc == 0);
389 	CU_ASSERT(buf_length == stat.size);
390 
391 	rc = spdk_fs_delete_file(g_fs, channel, "testfile");
392 	CU_ASSERT(rc == 0);
393 
394 	spdk_fs_free_thread_ctx(channel);
395 
396 	ut_send_request(_fs_unload, NULL);
397 }
398 
399 static void
cache_write_null_buffer(void)400 cache_write_null_buffer(void)
401 {
402 	uint64_t length;
403 	int rc;
404 	struct spdk_fs_thread_ctx *channel;
405 	struct spdk_thread *thread;
406 
407 	ut_send_request(_fs_init, NULL);
408 
409 	channel = spdk_fs_alloc_thread_ctx(g_fs);
410 
411 	rc = spdk_fs_open_file(g_fs, channel, "testfile", SPDK_BLOBFS_OPEN_CREATE, &g_file);
412 	CU_ASSERT(rc == 0);
413 	SPDK_CU_ASSERT_FATAL(g_file != NULL);
414 
415 	length = 0;
416 	rc = spdk_file_truncate(g_file, channel, length);
417 	CU_ASSERT(rc == 0);
418 
419 	rc = spdk_file_write(g_file, channel, NULL, 0, 0);
420 	CU_ASSERT(rc == 0);
421 
422 	spdk_file_close(g_file, channel);
423 
424 	fs_thread_poll();
425 
426 	rc = spdk_fs_delete_file(g_fs, channel, "testfile");
427 	CU_ASSERT(rc == 0);
428 
429 	spdk_fs_free_thread_ctx(channel);
430 
431 	thread = spdk_get_thread();
432 	while (spdk_thread_poll(thread, 0, 0) > 0) {}
433 
434 	ut_send_request(_fs_unload, NULL);
435 }
436 
437 static void
fs_create_sync(void)438 fs_create_sync(void)
439 {
440 	int rc;
441 	struct spdk_fs_thread_ctx *channel;
442 
443 	ut_send_request(_fs_init, NULL);
444 
445 	channel = spdk_fs_alloc_thread_ctx(g_fs);
446 	CU_ASSERT(channel != NULL);
447 
448 	rc = spdk_fs_create_file(g_fs, channel, "testfile");
449 	CU_ASSERT(rc == 0);
450 
451 	/* Create should fail, because the file already exists. */
452 	rc = spdk_fs_create_file(g_fs, channel, "testfile");
453 	CU_ASSERT(rc != 0);
454 
455 	rc = spdk_fs_delete_file(g_fs, channel, "testfile");
456 	CU_ASSERT(rc == 0);
457 
458 	spdk_fs_free_thread_ctx(channel);
459 
460 	fs_thread_poll();
461 
462 	ut_send_request(_fs_unload, NULL);
463 }
464 
465 static void
fs_rename_sync(void)466 fs_rename_sync(void)
467 {
468 	int rc;
469 	struct spdk_fs_thread_ctx *channel;
470 
471 	ut_send_request(_fs_init, NULL);
472 
473 	channel = spdk_fs_alloc_thread_ctx(g_fs);
474 	CU_ASSERT(channel != NULL);
475 
476 	rc = spdk_fs_open_file(g_fs, channel, "testfile", SPDK_BLOBFS_OPEN_CREATE, &g_file);
477 	CU_ASSERT(rc == 0);
478 	SPDK_CU_ASSERT_FATAL(g_file != NULL);
479 
480 	CU_ASSERT(strcmp(spdk_file_get_name(g_file), "testfile") == 0);
481 
482 	rc = spdk_fs_rename_file(g_fs, channel, "testfile", "newtestfile");
483 	CU_ASSERT(rc == 0);
484 	CU_ASSERT(strcmp(spdk_file_get_name(g_file), "newtestfile") == 0);
485 
486 	spdk_file_close(g_file, channel);
487 
488 	fs_thread_poll();
489 
490 	spdk_fs_free_thread_ctx(channel);
491 
492 	ut_send_request(_fs_unload, NULL);
493 }
494 
495 static void
cache_append_no_cache(void)496 cache_append_no_cache(void)
497 {
498 	int rc;
499 	char buf[100];
500 	struct spdk_fs_thread_ctx *channel;
501 
502 	ut_send_request(_fs_init, NULL);
503 
504 	channel = spdk_fs_alloc_thread_ctx(g_fs);
505 
506 	rc = spdk_fs_open_file(g_fs, channel, "testfile", SPDK_BLOBFS_OPEN_CREATE, &g_file);
507 	CU_ASSERT(rc == 0);
508 	SPDK_CU_ASSERT_FATAL(g_file != NULL);
509 
510 	spdk_file_write(g_file, channel, buf, 0 * sizeof(buf), sizeof(buf));
511 	CU_ASSERT(spdk_file_get_length(g_file) == 1 * sizeof(buf));
512 	spdk_file_write(g_file, channel, buf, 1 * sizeof(buf), sizeof(buf));
513 	CU_ASSERT(spdk_file_get_length(g_file) == 2 * sizeof(buf));
514 	spdk_file_sync(g_file, channel);
515 
516 	fs_thread_poll();
517 
518 	spdk_file_write(g_file, channel, buf, 2 * sizeof(buf), sizeof(buf));
519 	CU_ASSERT(spdk_file_get_length(g_file) == 3 * sizeof(buf));
520 	spdk_file_write(g_file, channel, buf, 3 * sizeof(buf), sizeof(buf));
521 	CU_ASSERT(spdk_file_get_length(g_file) == 4 * sizeof(buf));
522 	spdk_file_write(g_file, channel, buf, 4 * sizeof(buf), sizeof(buf));
523 	CU_ASSERT(spdk_file_get_length(g_file) == 5 * sizeof(buf));
524 
525 	spdk_file_close(g_file, channel);
526 
527 	fs_thread_poll();
528 
529 	rc = spdk_fs_delete_file(g_fs, channel, "testfile");
530 	CU_ASSERT(rc == 0);
531 
532 	spdk_fs_free_thread_ctx(channel);
533 
534 	ut_send_request(_fs_unload, NULL);
535 }
536 
537 static void
fs_delete_file_without_close(void)538 fs_delete_file_without_close(void)
539 {
540 	int rc;
541 	struct spdk_fs_thread_ctx *channel;
542 	struct spdk_file *file;
543 
544 	ut_send_request(_fs_init, NULL);
545 	channel = spdk_fs_alloc_thread_ctx(g_fs);
546 	CU_ASSERT(channel != NULL);
547 
548 	rc = spdk_fs_open_file(g_fs, channel, "testfile", SPDK_BLOBFS_OPEN_CREATE, &g_file);
549 	CU_ASSERT(rc == 0);
550 	SPDK_CU_ASSERT_FATAL(g_file != NULL);
551 
552 	rc = spdk_fs_delete_file(g_fs, channel, "testfile");
553 	CU_ASSERT(rc == 0);
554 	CU_ASSERT(g_file->ref_count != 0);
555 	CU_ASSERT(g_file->is_deleted == true);
556 
557 	rc = spdk_fs_open_file(g_fs, channel, "testfile", 0, &file);
558 	CU_ASSERT(rc != 0);
559 
560 	spdk_file_close(g_file, channel);
561 
562 	fs_thread_poll();
563 
564 	rc = spdk_fs_open_file(g_fs, channel, "testfile", 0, &file);
565 	CU_ASSERT(rc != 0);
566 
567 	spdk_fs_free_thread_ctx(channel);
568 
569 	ut_send_request(_fs_unload, NULL);
570 
571 }
572 
573 static bool g_thread_exit = false;
574 
575 static void
terminate_spdk_thread(void * arg)576 terminate_spdk_thread(void *arg)
577 {
578 	g_thread_exit = true;
579 }
580 
581 static void *
spdk_thread(void * arg)582 spdk_thread(void *arg)
583 {
584 	struct spdk_thread *thread = arg;
585 
586 	spdk_set_thread(thread);
587 
588 	while (!g_thread_exit) {
589 		spdk_thread_poll(thread, 0, 0);
590 	}
591 
592 	return NULL;
593 }
594 
595 int
main(int argc,char ** argv)596 main(int argc, char **argv)
597 {
598 	struct spdk_thread *thread;
599 	CU_pSuite	suite = NULL;
600 	pthread_t	spdk_tid;
601 	unsigned int	num_failures;
602 
603 	CU_initialize_registry();
604 
605 	suite = CU_add_suite("blobfs_sync_ut", NULL, NULL);
606 
607 	CU_ADD_TEST(suite, cache_read_after_write);
608 	CU_ADD_TEST(suite, file_length);
609 	CU_ADD_TEST(suite, append_write_to_extend_blob);
610 	CU_ADD_TEST(suite, partial_buffer);
611 	CU_ADD_TEST(suite, cache_write_null_buffer);
612 	CU_ADD_TEST(suite, fs_create_sync);
613 	CU_ADD_TEST(suite, fs_rename_sync);
614 	CU_ADD_TEST(suite, cache_append_no_cache);
615 	CU_ADD_TEST(suite, fs_delete_file_without_close);
616 
617 	spdk_thread_lib_init(NULL, 0);
618 
619 	thread = spdk_thread_create("test_thread", NULL);
620 	spdk_set_thread(thread);
621 
622 	g_dispatch_thread = spdk_thread_create("dispatch_thread", NULL);
623 	pthread_create(&spdk_tid, NULL, spdk_thread, g_dispatch_thread);
624 
625 	g_dev_buffer = calloc(1, DEV_BUFFER_SIZE);
626 
627 	num_failures = spdk_ut_run_tests(argc, argv, NULL);
628 	CU_cleanup_registry();
629 
630 	free(g_dev_buffer);
631 
632 	ut_send_request(terminate_spdk_thread, NULL);
633 	pthread_join(spdk_tid, NULL);
634 
635 	while (spdk_thread_poll(g_dispatch_thread, 0, 0) > 0) {}
636 	while (spdk_thread_poll(thread, 0, 0) > 0) {}
637 
638 	spdk_set_thread(thread);
639 	spdk_thread_exit(thread);
640 	while (!spdk_thread_is_exited(thread)) {
641 		spdk_thread_poll(thread, 0, 0);
642 	}
643 	spdk_thread_destroy(thread);
644 
645 	spdk_set_thread(g_dispatch_thread);
646 	spdk_thread_exit(g_dispatch_thread);
647 	while (!spdk_thread_is_exited(g_dispatch_thread)) {
648 		spdk_thread_poll(g_dispatch_thread, 0, 0);
649 	}
650 	spdk_thread_destroy(g_dispatch_thread);
651 
652 	spdk_thread_lib_fini();
653 
654 	return num_failures;
655 }
656