xref: /spdk/test/unit/lib/blobfs/blobfs_sync_ut/blobfs_sync_ut.c (revision 4e9e220e7c74f8e69c428f13485d198ea5428220)
1 /*-
2  *   BSD LICENSE
3  *
4  *   Copyright (c) Intel Corporation.
5  *   All rights reserved.
6  *
7  *   Redistribution and use in source and binary forms, with or without
8  *   modification, are permitted provided that the following conditions
9  *   are met:
10  *
11  *     * Redistributions of source code must retain the above copyright
12  *       notice, this list of conditions and the following disclaimer.
13  *     * Redistributions in binary form must reproduce the above copyright
14  *       notice, this list of conditions and the following disclaimer in
15  *       the documentation and/or other materials provided with the
16  *       distribution.
17  *     * Neither the name of Intel Corporation nor the names of its
18  *       contributors may be used to endorse or promote products derived
19  *       from this software without specific prior written permission.
20  *
21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32  */
33 
34 #include "spdk/stdinc.h"
35 
36 #include "spdk/blobfs.h"
37 #include "spdk/env.h"
38 #include "spdk/log.h"
39 #include "spdk/thread.h"
40 #include "spdk/barrier.h"
41 #include "spdk_internal/thread.h"
42 
43 #include "spdk_cunit.h"
44 #include "unit/lib/blob/bs_dev_common.c"
45 #include "common/lib/test_env.c"
46 #include "blobfs/blobfs.c"
47 #include "blobfs/tree.c"
48 
49 struct spdk_filesystem *g_fs;
50 struct spdk_file *g_file;
51 int g_fserrno;
52 struct spdk_thread *g_dispatch_thread = NULL;
53 struct spdk_trace_histories *g_trace_histories;
54 DEFINE_STUB_V(spdk_trace_add_register_fn, (struct spdk_trace_register_fn *reg_fn));
55 DEFINE_STUB_V(spdk_trace_register_description, (const char *name,
56 		uint16_t tpoint_id, uint8_t owner_type,
57 		uint8_t object_type, uint8_t new_object,
58 		uint8_t arg1_is_ptr, const char *arg1_name));
59 DEFINE_STUB_V(_spdk_trace_record, (uint64_t tsc, uint16_t tpoint_id, uint16_t poller_id,
60 				   uint32_t size, uint64_t object_id, uint64_t arg1));
61 
62 /* Return NULL to test hardcoded defaults. */
63 struct spdk_conf_section *
64 spdk_conf_find_section(struct spdk_conf *cp, const char *name)
65 {
66 	return NULL;
67 }
68 
69 /* Return -1 to test hardcoded defaults. */
70 int
71 spdk_conf_section_get_intval(struct spdk_conf_section *sp, const char *key)
72 {
73 	return -1;
74 }
75 
76 struct ut_request {
77 	fs_request_fn fn;
78 	void *arg;
79 	volatile int done;
80 };
81 
82 static void
83 send_request(fs_request_fn fn, void *arg)
84 {
85 	spdk_thread_send_msg(g_dispatch_thread, (spdk_msg_fn)fn, arg);
86 }
87 
88 static void
89 ut_call_fn(void *arg)
90 {
91 	struct ut_request *req = arg;
92 
93 	req->fn(req->arg);
94 	req->done = 1;
95 }
96 
97 static void
98 ut_send_request(fs_request_fn fn, void *arg)
99 {
100 	struct ut_request req;
101 
102 	req.fn = fn;
103 	req.arg = arg;
104 	req.done = 0;
105 
106 	spdk_thread_send_msg(g_dispatch_thread, ut_call_fn, &req);
107 
108 	/* Wait for this to finish */
109 	while (req.done == 0) {	}
110 }
111 
112 static void
113 fs_op_complete(void *ctx, int fserrno)
114 {
115 	g_fserrno = fserrno;
116 }
117 
118 static void
119 fs_op_with_handle_complete(void *ctx, struct spdk_filesystem *fs, int fserrno)
120 {
121 	g_fs = fs;
122 	g_fserrno = fserrno;
123 }
124 
125 static void
126 _fs_init(void *arg)
127 {
128 	struct spdk_thread *thread;
129 	struct spdk_bs_dev *dev;
130 
131 	g_fs = NULL;
132 	g_fserrno = -1;
133 	dev = init_dev();
134 	spdk_fs_init(dev, NULL, send_request, fs_op_with_handle_complete, NULL);
135 	thread = spdk_get_thread();
136 	while (spdk_thread_poll(thread, 0, 0) > 0) {}
137 
138 	SPDK_CU_ASSERT_FATAL(g_fs != NULL);
139 	SPDK_CU_ASSERT_FATAL(g_fs->bdev == dev);
140 	CU_ASSERT(g_fserrno == 0);
141 }
142 
143 static void
144 _fs_load(void *arg)
145 {
146 	struct spdk_thread *thread;
147 	struct spdk_bs_dev *dev;
148 
149 	g_fs = NULL;
150 	g_fserrno = -1;
151 	dev = init_dev();
152 	spdk_fs_load(dev, send_request, fs_op_with_handle_complete, NULL);
153 	thread = spdk_get_thread();
154 	while (spdk_thread_poll(thread, 0, 0) > 0) {}
155 
156 	SPDK_CU_ASSERT_FATAL(g_fs != NULL);
157 	SPDK_CU_ASSERT_FATAL(g_fs->bdev == dev);
158 	CU_ASSERT(g_fserrno == 0);
159 }
160 
161 static void
162 _fs_unload(void *arg)
163 {
164 	struct spdk_thread *thread;
165 
166 	g_fserrno = -1;
167 	spdk_fs_unload(g_fs, fs_op_complete, NULL);
168 	thread = spdk_get_thread();
169 	while (spdk_thread_poll(thread, 0, 0) > 0) {}
170 	CU_ASSERT(g_fserrno == 0);
171 	g_fs = NULL;
172 }
173 
174 static void
175 _nop(void *arg)
176 {
177 }
178 
179 static void
180 cache_read_after_write(void)
181 {
182 	uint64_t length;
183 	int rc;
184 	char w_buf[100], r_buf[100];
185 	struct spdk_fs_thread_ctx *channel;
186 	struct spdk_file_stat stat = {0};
187 
188 	ut_send_request(_fs_init, NULL);
189 
190 	channel = spdk_fs_alloc_thread_ctx(g_fs);
191 
192 	rc = spdk_fs_open_file(g_fs, channel, "testfile", SPDK_BLOBFS_OPEN_CREATE, &g_file);
193 	CU_ASSERT(rc == 0);
194 	SPDK_CU_ASSERT_FATAL(g_file != NULL);
195 
196 	length = (4 * 1024 * 1024);
197 	rc = spdk_file_truncate(g_file, channel, length);
198 	CU_ASSERT(rc == 0);
199 
200 	memset(w_buf, 0x5a, sizeof(w_buf));
201 	spdk_file_write(g_file, channel, w_buf, 0, sizeof(w_buf));
202 
203 	CU_ASSERT(spdk_file_get_length(g_file) == length);
204 
205 	rc = spdk_file_truncate(g_file, channel, sizeof(w_buf));
206 	CU_ASSERT(rc == 0);
207 
208 	spdk_file_close(g_file, channel);
209 
210 	rc = spdk_fs_file_stat(g_fs, channel, "testfile", &stat);
211 	CU_ASSERT(rc == 0);
212 	CU_ASSERT(sizeof(w_buf) == stat.size);
213 
214 	rc = spdk_fs_open_file(g_fs, channel, "testfile", 0, &g_file);
215 	CU_ASSERT(rc == 0);
216 	SPDK_CU_ASSERT_FATAL(g_file != NULL);
217 
218 	memset(r_buf, 0, sizeof(r_buf));
219 	spdk_file_read(g_file, channel, r_buf, 0, sizeof(r_buf));
220 	CU_ASSERT(memcmp(w_buf, r_buf, sizeof(r_buf)) == 0);
221 
222 	spdk_file_close(g_file, channel);
223 	rc = spdk_fs_delete_file(g_fs, channel, "testfile");
224 	CU_ASSERT(rc == 0);
225 
226 	rc = spdk_fs_delete_file(g_fs, channel, "testfile");
227 	CU_ASSERT(rc == -ENOENT);
228 
229 	spdk_fs_free_thread_ctx(channel);
230 
231 	ut_send_request(_fs_unload, NULL);
232 }
233 
234 static void
235 file_length(void)
236 {
237 	int rc;
238 	char *buf;
239 	uint64_t buf_length;
240 	struct spdk_fs_thread_ctx *channel;
241 	struct spdk_file_stat stat = {0};
242 
243 	ut_send_request(_fs_init, NULL);
244 
245 	channel = spdk_fs_alloc_thread_ctx(g_fs);
246 
247 	g_file = NULL;
248 	rc = spdk_fs_open_file(g_fs, channel, "testfile", SPDK_BLOBFS_OPEN_CREATE, &g_file);
249 	CU_ASSERT(rc == 0);
250 	SPDK_CU_ASSERT_FATAL(g_file != NULL);
251 
252 	/* Write one CACHE_BUFFER.  Filling at least one cache buffer triggers
253 	 * a flush to disk.
254 	 */
255 	buf_length = CACHE_BUFFER_SIZE;
256 	buf = calloc(1, buf_length);
257 	spdk_file_write(g_file, channel, buf, 0, buf_length);
258 	free(buf);
259 
260 	/* Spin until all of the data has been flushed to the SSD.  There's been no
261 	 * sync operation yet, so the xattr on the file is still 0.
262 	 */
263 	while (g_file->length_flushed != buf_length) {}
264 
265 	/* Close the file.  This causes an implicit sync which should write the
266 	 * length_flushed value as the "length" xattr on the file.
267 	 */
268 	spdk_file_close(g_file, channel);
269 
270 	rc = spdk_fs_file_stat(g_fs, channel, "testfile", &stat);
271 	CU_ASSERT(rc == 0);
272 	CU_ASSERT(buf_length == stat.size);
273 
274 	spdk_fs_free_thread_ctx(channel);
275 
276 	/* Unload and reload the filesystem.  The file length will be
277 	 * read during load from the length xattr.  We want to make sure
278 	 * it matches what was written when the file was originally
279 	 * written and closed.
280 	 */
281 	ut_send_request(_fs_unload, NULL);
282 
283 	ut_send_request(_fs_load, NULL);
284 
285 	channel = spdk_fs_alloc_thread_ctx(g_fs);
286 
287 	rc = spdk_fs_file_stat(g_fs, channel, "testfile", &stat);
288 	CU_ASSERT(rc == 0);
289 	CU_ASSERT(buf_length == stat.size);
290 
291 	g_file = NULL;
292 	rc = spdk_fs_open_file(g_fs, channel, "testfile", 0, &g_file);
293 	CU_ASSERT(rc == 0);
294 	SPDK_CU_ASSERT_FATAL(g_file != NULL);
295 
296 	spdk_file_close(g_file, channel);
297 
298 	rc = spdk_fs_delete_file(g_fs, channel, "testfile");
299 	CU_ASSERT(rc == 0);
300 
301 	spdk_fs_free_thread_ctx(channel);
302 
303 	ut_send_request(_fs_unload, NULL);
304 }
305 
306 static void
307 partial_buffer(void)
308 {
309 	int rc;
310 	char *buf;
311 	uint64_t buf_length;
312 	struct spdk_fs_thread_ctx *channel;
313 	struct spdk_file_stat stat = {0};
314 
315 	ut_send_request(_fs_init, NULL);
316 
317 	channel = spdk_fs_alloc_thread_ctx(g_fs);
318 
319 	g_file = NULL;
320 	rc = spdk_fs_open_file(g_fs, channel, "testfile", SPDK_BLOBFS_OPEN_CREATE, &g_file);
321 	CU_ASSERT(rc == 0);
322 	SPDK_CU_ASSERT_FATAL(g_file != NULL);
323 
324 	/* Write one CACHE_BUFFER plus one byte.  Filling at least one cache buffer triggers
325 	 * a flush to disk.  We want to make sure the extra byte is not implicitly flushed.
326 	 * It should only get flushed once we sync or close the file.
327 	 */
328 	buf_length = CACHE_BUFFER_SIZE + 1;
329 	buf = calloc(1, buf_length);
330 	spdk_file_write(g_file, channel, buf, 0, buf_length);
331 	free(buf);
332 
333 	/* Send some nop messages to the dispatch thread.  This will ensure any of the
334 	 * pending write operations are completed.  A well-functioning blobfs should only
335 	 * issue one write for the filled CACHE_BUFFER - a buggy one might try to write
336 	 * the extra byte.  So do a bunch of _nops to make sure all of them (even the buggy
337 	 * ones) get a chance to run.  Note that we can't just send a message to the
338 	 * dispatch thread to call spdk_thread_poll() because the messages are themselves
339 	 * run in the context of spdk_thread_poll().
340 	 */
341 	ut_send_request(_nop, NULL);
342 	ut_send_request(_nop, NULL);
343 	ut_send_request(_nop, NULL);
344 	ut_send_request(_nop, NULL);
345 	ut_send_request(_nop, NULL);
346 	ut_send_request(_nop, NULL);
347 
348 	CU_ASSERT(g_file->length_flushed == CACHE_BUFFER_SIZE);
349 
350 	/* Close the file.  This causes an implicit sync which should write the
351 	 * length_flushed value as the "length" xattr on the file.
352 	 */
353 	spdk_file_close(g_file, channel);
354 
355 	rc = spdk_fs_file_stat(g_fs, channel, "testfile", &stat);
356 	CU_ASSERT(rc == 0);
357 	CU_ASSERT(buf_length == stat.size);
358 
359 	rc = spdk_fs_delete_file(g_fs, channel, "testfile");
360 	CU_ASSERT(rc == 0);
361 
362 	spdk_fs_free_thread_ctx(channel);
363 
364 	ut_send_request(_fs_unload, NULL);
365 }
366 
367 static void
368 cache_write_null_buffer(void)
369 {
370 	uint64_t length;
371 	int rc;
372 	struct spdk_fs_thread_ctx *channel;
373 	struct spdk_thread *thread;
374 
375 	ut_send_request(_fs_init, NULL);
376 
377 	channel = spdk_fs_alloc_thread_ctx(g_fs);
378 
379 	rc = spdk_fs_open_file(g_fs, channel, "testfile", SPDK_BLOBFS_OPEN_CREATE, &g_file);
380 	CU_ASSERT(rc == 0);
381 	SPDK_CU_ASSERT_FATAL(g_file != NULL);
382 
383 	length = 0;
384 	rc = spdk_file_truncate(g_file, channel, length);
385 	CU_ASSERT(rc == 0);
386 
387 	rc = spdk_file_write(g_file, channel, NULL, 0, 0);
388 	CU_ASSERT(rc == 0);
389 
390 	spdk_file_close(g_file, channel);
391 	rc = spdk_fs_delete_file(g_fs, channel, "testfile");
392 	CU_ASSERT(rc == 0);
393 
394 	spdk_fs_free_thread_ctx(channel);
395 
396 	thread = spdk_get_thread();
397 	while (spdk_thread_poll(thread, 0, 0) > 0) {}
398 
399 	ut_send_request(_fs_unload, NULL);
400 }
401 
402 static void
403 fs_create_sync(void)
404 {
405 	int rc;
406 	struct spdk_fs_thread_ctx *channel;
407 	struct spdk_thread *thread;
408 
409 	ut_send_request(_fs_init, NULL);
410 
411 	channel = spdk_fs_alloc_thread_ctx(g_fs);
412 	CU_ASSERT(channel != NULL);
413 
414 	rc = spdk_fs_create_file(g_fs, channel, "testfile");
415 	CU_ASSERT(rc == 0);
416 
417 	/* Create should fail, because the file already exists. */
418 	rc = spdk_fs_create_file(g_fs, channel, "testfile");
419 	CU_ASSERT(rc != 0);
420 
421 	rc = spdk_fs_delete_file(g_fs, channel, "testfile");
422 	CU_ASSERT(rc == 0);
423 
424 	spdk_fs_free_thread_ctx(channel);
425 
426 	thread = spdk_get_thread();
427 	while (spdk_thread_poll(thread, 0, 0) > 0) {}
428 
429 	ut_send_request(_fs_unload, NULL);
430 }
431 
432 static void
433 fs_rename_sync(void)
434 {
435 	int rc;
436 	struct spdk_fs_thread_ctx *channel;
437 	struct spdk_thread *thread;
438 
439 	ut_send_request(_fs_init, NULL);
440 
441 	channel = spdk_fs_alloc_thread_ctx(g_fs);
442 	CU_ASSERT(channel != NULL);
443 
444 	rc = spdk_fs_open_file(g_fs, channel, "testfile", SPDK_BLOBFS_OPEN_CREATE, &g_file);
445 	CU_ASSERT(rc == 0);
446 	SPDK_CU_ASSERT_FATAL(g_file != NULL);
447 
448 	CU_ASSERT(strcmp(spdk_file_get_name(g_file), "testfile") == 0);
449 
450 	rc = spdk_fs_rename_file(g_fs, channel, "testfile", "newtestfile");
451 	CU_ASSERT(rc == 0);
452 	CU_ASSERT(strcmp(spdk_file_get_name(g_file), "newtestfile") == 0);
453 
454 	spdk_file_close(g_file, channel);
455 	spdk_fs_free_thread_ctx(channel);
456 
457 	thread = spdk_get_thread();
458 	while (spdk_thread_poll(thread, 0, 0) > 0) {}
459 
460 	ut_send_request(_fs_unload, NULL);
461 }
462 
463 static void
464 cache_append_no_cache(void)
465 {
466 	int rc;
467 	char buf[100];
468 	struct spdk_fs_thread_ctx *channel;
469 	struct spdk_thread *thread;
470 
471 	ut_send_request(_fs_init, NULL);
472 
473 	channel = spdk_fs_alloc_thread_ctx(g_fs);
474 
475 	rc = spdk_fs_open_file(g_fs, channel, "testfile", SPDK_BLOBFS_OPEN_CREATE, &g_file);
476 	CU_ASSERT(rc == 0);
477 	SPDK_CU_ASSERT_FATAL(g_file != NULL);
478 
479 	spdk_file_write(g_file, channel, buf, 0 * sizeof(buf), sizeof(buf));
480 	CU_ASSERT(spdk_file_get_length(g_file) == 1 * sizeof(buf));
481 	spdk_file_write(g_file, channel, buf, 1 * sizeof(buf), sizeof(buf));
482 	CU_ASSERT(spdk_file_get_length(g_file) == 2 * sizeof(buf));
483 	spdk_file_sync(g_file, channel);
484 	cache_free_buffers(g_file);
485 	spdk_file_write(g_file, channel, buf, 2 * sizeof(buf), sizeof(buf));
486 	CU_ASSERT(spdk_file_get_length(g_file) == 3 * sizeof(buf));
487 	spdk_file_write(g_file, channel, buf, 3 * sizeof(buf), sizeof(buf));
488 	CU_ASSERT(spdk_file_get_length(g_file) == 4 * sizeof(buf));
489 	spdk_file_write(g_file, channel, buf, 4 * sizeof(buf), sizeof(buf));
490 	CU_ASSERT(spdk_file_get_length(g_file) == 5 * sizeof(buf));
491 
492 	spdk_file_close(g_file, channel);
493 	rc = spdk_fs_delete_file(g_fs, channel, "testfile");
494 	CU_ASSERT(rc == 0);
495 
496 	spdk_fs_free_thread_ctx(channel);
497 
498 	thread = spdk_get_thread();
499 	while (spdk_thread_poll(thread, 0, 0) > 0) {}
500 
501 	ut_send_request(_fs_unload, NULL);
502 }
503 
504 static void
505 fs_delete_file_without_close(void)
506 {
507 	int rc;
508 	struct spdk_fs_thread_ctx *channel;
509 	struct spdk_file *file;
510 	struct spdk_thread *thread;
511 
512 	ut_send_request(_fs_init, NULL);
513 	channel = spdk_fs_alloc_thread_ctx(g_fs);
514 	CU_ASSERT(channel != NULL);
515 
516 	rc = spdk_fs_open_file(g_fs, channel, "testfile", SPDK_BLOBFS_OPEN_CREATE, &g_file);
517 	CU_ASSERT(rc == 0);
518 	SPDK_CU_ASSERT_FATAL(g_file != NULL);
519 
520 	rc = spdk_fs_delete_file(g_fs, channel, "testfile");
521 	CU_ASSERT(rc == 0);
522 	CU_ASSERT(g_file->ref_count != 0);
523 	CU_ASSERT(g_file->is_deleted == true);
524 
525 	rc = spdk_fs_open_file(g_fs, channel, "testfile", 0, &file);
526 	CU_ASSERT(rc != 0);
527 
528 	spdk_file_close(g_file, channel);
529 
530 	rc = spdk_fs_open_file(g_fs, channel, "testfile", 0, &file);
531 	CU_ASSERT(rc != 0);
532 
533 	spdk_fs_free_thread_ctx(channel);
534 
535 	thread = spdk_get_thread();
536 	while (spdk_thread_poll(thread, 0, 0) > 0) {}
537 
538 	ut_send_request(_fs_unload, NULL);
539 
540 }
541 
542 static bool g_thread_exit = false;
543 
544 static void
545 terminate_spdk_thread(void *arg)
546 {
547 	g_thread_exit = true;
548 }
549 
550 static void *
551 spdk_thread(void *arg)
552 {
553 	struct spdk_thread *thread = arg;
554 
555 	spdk_set_thread(thread);
556 
557 	while (!g_thread_exit) {
558 		spdk_thread_poll(thread, 0, 0);
559 	}
560 
561 	return NULL;
562 }
563 
564 int main(int argc, char **argv)
565 {
566 	struct spdk_thread *thread;
567 	CU_pSuite	suite = NULL;
568 	pthread_t	spdk_tid;
569 	unsigned int	num_failures;
570 
571 	if (CU_initialize_registry() != CUE_SUCCESS) {
572 		return CU_get_error();
573 	}
574 
575 	suite = CU_add_suite("blobfs_sync_ut", NULL, NULL);
576 	if (suite == NULL) {
577 		CU_cleanup_registry();
578 		return CU_get_error();
579 	}
580 
581 	if (
582 		CU_add_test(suite, "cache read after write", cache_read_after_write) == NULL ||
583 		CU_add_test(suite, "file length", file_length) == NULL ||
584 		CU_add_test(suite, "partial buffer", partial_buffer) == NULL ||
585 		CU_add_test(suite, "write_null_buffer", cache_write_null_buffer) == NULL ||
586 		CU_add_test(suite, "create_sync", fs_create_sync) == NULL ||
587 		CU_add_test(suite, "rename_sync", fs_rename_sync) == NULL ||
588 		CU_add_test(suite, "append_no_cache", cache_append_no_cache) == NULL ||
589 		CU_add_test(suite, "delete_file_without_close", fs_delete_file_without_close) == NULL
590 	) {
591 		CU_cleanup_registry();
592 		return CU_get_error();
593 	}
594 
595 	spdk_thread_lib_init(NULL, 0);
596 
597 	thread = spdk_thread_create("test_thread", NULL);
598 	spdk_set_thread(thread);
599 
600 	g_dispatch_thread = spdk_thread_create("dispatch_thread", NULL);
601 	pthread_create(&spdk_tid, NULL, spdk_thread, g_dispatch_thread);
602 
603 	g_dev_buffer = calloc(1, DEV_BUFFER_SIZE);
604 
605 	CU_basic_set_mode(CU_BRM_VERBOSE);
606 	CU_basic_run_tests();
607 	num_failures = CU_get_number_of_failures();
608 	CU_cleanup_registry();
609 
610 	free(g_dev_buffer);
611 
612 	ut_send_request(terminate_spdk_thread, NULL);
613 	pthread_join(spdk_tid, NULL);
614 
615 	while (spdk_thread_poll(g_dispatch_thread, 0, 0) > 0) {}
616 	while (spdk_thread_poll(thread, 0, 0) > 0) {}
617 
618 	spdk_set_thread(thread);
619 	spdk_thread_exit(thread);
620 	spdk_thread_destroy(thread);
621 
622 	spdk_set_thread(g_dispatch_thread);
623 	spdk_thread_exit(g_dispatch_thread);
624 	spdk_thread_destroy(g_dispatch_thread);
625 
626 	spdk_thread_lib_fini();
627 
628 	return num_failures;
629 }
630