xref: /spdk/test/unit/lib/event/reactor.c/reactor_ut.c (revision c680e3a05b1a903c18bf3f75b732765607126f45)
1 /*   SPDX-License-Identifier: BSD-3-Clause
2  *   Copyright (C) 2020 Intel Corporation.
3  *   All rights reserved.
4  */
5 
6 #include "spdk/stdinc.h"
7 
8 #include "spdk_cunit.h"
9 #include "common/lib/test_env.c"
10 #include "event/reactor.c"
11 #include "spdk/thread.h"
12 #include "spdk_internal/thread.h"
13 #include "event/scheduler_static.c"
14 #include "../module/scheduler/dynamic/scheduler_dynamic.c"
15 
16 struct spdk_thread *
17 _spdk_get_app_thread(void)
18 {
19 	struct spdk_lw_thread *lw_thread;
20 	struct spdk_thread *thread;
21 
22 	/* Assume there has to be at least one thread on main
23 	 * reactor, that has at least one thread. */
24 	lw_thread = TAILQ_FIRST(&g_scheduling_reactor->threads);
25 	SPDK_CU_ASSERT_FATAL(lw_thread != NULL);
26 	thread = spdk_thread_get_from_ctx(lw_thread);
27 	SPDK_CU_ASSERT_FATAL(thread != NULL);
28 
29 	return thread;
30 }
31 
32 static void
33 test_create_reactor(void)
34 {
35 	struct spdk_reactor reactor = {};
36 
37 	g_reactors = &reactor;
38 	g_reactor_count = 1;
39 
40 	reactor_construct(&reactor, 0);
41 
42 	CU_ASSERT(spdk_reactor_get(0) == &reactor);
43 
44 	spdk_ring_free(reactor.events);
45 	reactor_interrupt_fini(&reactor);
46 	g_reactors = NULL;
47 }
48 
49 static void
50 test_init_reactors(void)
51 {
52 	uint32_t core;
53 
54 	MOCK_SET(spdk_env_get_current_core, 0);
55 
56 	allocate_cores(3);
57 
58 	CU_ASSERT(spdk_reactors_init(SPDK_DEFAULT_MSG_MEMPOOL_SIZE) == 0);
59 
60 	CU_ASSERT(g_reactor_state == SPDK_REACTOR_STATE_INITIALIZED);
61 	for (core = 0; core < 3; core++) {
62 		CU_ASSERT(spdk_reactor_get(core) != NULL);
63 	}
64 
65 	spdk_reactors_fini();
66 
67 	free_cores();
68 
69 	MOCK_CLEAR(spdk_env_get_current_core);
70 }
71 
72 static void
73 ut_event_fn(void *arg1, void *arg2)
74 {
75 	uint8_t *test1 = arg1;
76 	uint8_t *test2 = arg2;
77 
78 	*test1 = 1;
79 	*test2 = 0xFF;
80 }
81 
82 static void
83 test_event_call(void)
84 {
85 	uint8_t test1 = 0, test2 = 0;
86 	struct spdk_event *evt;
87 	struct spdk_reactor *reactor;
88 
89 	MOCK_SET(spdk_env_get_current_core, 0);
90 
91 	allocate_cores(1);
92 
93 	CU_ASSERT(spdk_reactors_init(SPDK_DEFAULT_MSG_MEMPOOL_SIZE) == 0);
94 
95 	evt = spdk_event_allocate(0, ut_event_fn, &test1, &test2);
96 	CU_ASSERT(evt != NULL);
97 
98 	MOCK_SET(spdk_env_get_current_core, 0);
99 
100 	spdk_event_call(evt);
101 
102 	reactor = spdk_reactor_get(0);
103 	CU_ASSERT(reactor != NULL);
104 
105 	CU_ASSERT(event_queue_run_batch(reactor) == 1);
106 	CU_ASSERT(test1 == 1);
107 	CU_ASSERT(test2 == 0xFF);
108 
109 	MOCK_CLEAR(spdk_env_get_current_core);
110 
111 	spdk_reactors_fini();
112 
113 	free_cores();
114 
115 	MOCK_CLEAR(spdk_env_get_current_core);
116 }
117 
118 static void
119 test_schedule_thread(void)
120 {
121 	struct spdk_cpuset cpuset = {};
122 	struct spdk_thread *thread;
123 	struct spdk_reactor *reactor;
124 	struct spdk_lw_thread *lw_thread;
125 
126 	MOCK_SET(spdk_env_get_current_core, 0);
127 
128 	allocate_cores(5);
129 
130 	CU_ASSERT(spdk_reactors_init(SPDK_DEFAULT_MSG_MEMPOOL_SIZE) == 0);
131 
132 	spdk_cpuset_set_cpu(&cpuset, 3, true);
133 	g_next_core = 4;
134 
135 	MOCK_SET(spdk_env_get_current_core, 3);
136 
137 	/* _reactor_schedule_thread() will be called in spdk_thread_create()
138 	 * at its end because it is passed to SPDK thread library by
139 	 * spdk_thread_lib_init().
140 	 */
141 	thread = spdk_thread_create(NULL, &cpuset);
142 	CU_ASSERT(thread != NULL);
143 
144 	reactor = spdk_reactor_get(3);
145 	CU_ASSERT(reactor != NULL);
146 
147 	CU_ASSERT(event_queue_run_batch(reactor) == 1);
148 
149 	MOCK_CLEAR(spdk_env_get_current_core);
150 
151 	lw_thread = TAILQ_FIRST(&reactor->threads);
152 	CU_ASSERT(lw_thread != NULL);
153 	CU_ASSERT(spdk_thread_get_from_ctx(lw_thread) == thread);
154 
155 	TAILQ_REMOVE(&reactor->threads, lw_thread, link);
156 	reactor->thread_count--;
157 	spdk_set_thread(thread);
158 	spdk_thread_exit(thread);
159 	while (!spdk_thread_is_exited(thread)) {
160 		spdk_thread_poll(thread, 0, 0);
161 	}
162 	spdk_thread_destroy(thread);
163 	spdk_set_thread(NULL);
164 
165 	spdk_reactors_fini();
166 
167 	free_cores();
168 }
169 
170 static void
171 test_reschedule_thread(void)
172 {
173 	struct spdk_cpuset cpuset = {};
174 	struct spdk_thread *thread;
175 	struct spdk_reactor *reactor;
176 	struct spdk_lw_thread *lw_thread;
177 
178 	MOCK_SET(spdk_env_get_current_core, 0);
179 
180 	allocate_cores(3);
181 
182 	CU_ASSERT(spdk_reactors_init(SPDK_DEFAULT_MSG_MEMPOOL_SIZE) == 0);
183 
184 	spdk_cpuset_set_cpu(&g_reactor_core_mask, 0, true);
185 	spdk_cpuset_set_cpu(&g_reactor_core_mask, 1, true);
186 	spdk_cpuset_set_cpu(&g_reactor_core_mask, 2, true);
187 	g_next_core = 0;
188 
189 	MOCK_SET(spdk_env_get_current_core, 1);
190 	/* Create and schedule the thread to core 1. */
191 	spdk_cpuset_set_cpu(&cpuset, 1, true);
192 
193 	thread = spdk_thread_create(NULL, &cpuset);
194 	CU_ASSERT(thread != NULL);
195 	lw_thread = spdk_thread_get_ctx(thread);
196 
197 	reactor = spdk_reactor_get(1);
198 	CU_ASSERT(reactor != NULL);
199 
200 	CU_ASSERT(event_queue_run_batch(reactor) == 1);
201 	CU_ASSERT(TAILQ_FIRST(&reactor->threads) == lw_thread);
202 
203 	spdk_set_thread(thread);
204 
205 	/* Call spdk_thread_set_cpumask() twice with different cpumask values.
206 	 * The cpumask of the 2nd call will be used in reschedule operation.
207 	 */
208 
209 	spdk_cpuset_zero(&cpuset);
210 	spdk_cpuset_set_cpu(&cpuset, 0, true);
211 	CU_ASSERT(spdk_thread_set_cpumask(&cpuset) == 0);
212 
213 	spdk_cpuset_zero(&cpuset);
214 	spdk_cpuset_set_cpu(&cpuset, 2, true);
215 	CU_ASSERT(spdk_thread_set_cpumask(&cpuset) == 0);
216 
217 	CU_ASSERT(lw_thread->resched == true);
218 
219 	reactor_run(reactor);
220 
221 	CU_ASSERT(lw_thread->resched == false);
222 	CU_ASSERT(TAILQ_EMPTY(&reactor->threads));
223 
224 	reactor = spdk_reactor_get(0);
225 	CU_ASSERT(reactor != NULL);
226 	MOCK_SET(spdk_env_get_current_core, 0);
227 
228 	CU_ASSERT(event_queue_run_batch(reactor) == 0);
229 
230 	reactor = spdk_reactor_get(2);
231 	CU_ASSERT(reactor != NULL);
232 	MOCK_SET(spdk_env_get_current_core, 2);
233 
234 	CU_ASSERT(event_queue_run_batch(reactor) == 1);
235 
236 	CU_ASSERT(TAILQ_FIRST(&reactor->threads) == lw_thread);
237 
238 	MOCK_CLEAR(spdk_env_get_current_core);
239 
240 	TAILQ_REMOVE(&reactor->threads, lw_thread, link);
241 	reactor->thread_count--;
242 	spdk_set_thread(thread);
243 	spdk_thread_exit(thread);
244 	while (!spdk_thread_is_exited(thread)) {
245 		spdk_thread_poll(thread, 0, 0);
246 	}
247 	spdk_thread_destroy(thread);
248 	spdk_set_thread(NULL);
249 
250 	spdk_reactors_fini();
251 
252 	free_cores();
253 }
254 
255 static void
256 for_each_reactor_done(void *arg1, void *arg2)
257 {
258 	uint32_t *count = arg1;
259 	bool *done = arg2;
260 
261 	(*count)++;
262 	*done = true;
263 }
264 
265 static void
266 for_each_reactor_cb(void *arg1, void *arg2)
267 {
268 	uint32_t *count = arg1;
269 
270 	(*count)++;
271 }
272 
273 static void
274 test_for_each_reactor(void)
275 {
276 	uint32_t count = 0, i;
277 	bool done = false;
278 	struct spdk_reactor *reactor;
279 
280 	MOCK_SET(spdk_env_get_current_core, 0);
281 
282 	allocate_cores(5);
283 
284 	CU_ASSERT(spdk_reactors_init(SPDK_DEFAULT_MSG_MEMPOOL_SIZE) == 0);
285 
286 	spdk_for_each_reactor(for_each_reactor_cb, &count, &done, for_each_reactor_done);
287 
288 	MOCK_CLEAR(spdk_env_get_current_core);
289 
290 	/* We have not processed any event yet, so count and done should be 0 and false,
291 	 *  respectively.
292 	 */
293 	CU_ASSERT(count == 0);
294 
295 	/* Poll each reactor to verify the event is passed to each */
296 	for (i = 0; i < 5; i++) {
297 		reactor = spdk_reactor_get(i);
298 		CU_ASSERT(reactor != NULL);
299 		MOCK_SET(spdk_env_get_current_core, i);
300 
301 		event_queue_run_batch(reactor);
302 		CU_ASSERT(count == (i + 1));
303 		CU_ASSERT(done == false);
304 		MOCK_CLEAR(spdk_env_get_current_core);
305 	}
306 
307 	MOCK_SET(spdk_env_get_current_core, 0);
308 	/* After each reactor is called, the completion calls it one more time. */
309 	reactor = spdk_reactor_get(0);
310 	CU_ASSERT(reactor != NULL);
311 
312 	event_queue_run_batch(reactor);
313 	CU_ASSERT(count == 6);
314 	CU_ASSERT(done == true);
315 	MOCK_CLEAR(spdk_env_get_current_core);
316 
317 	spdk_reactors_fini();
318 
319 	free_cores();
320 }
321 
322 static int
323 poller_run_idle(void *ctx)
324 {
325 	uint64_t delay_us = (uint64_t)ctx;
326 
327 	spdk_delay_us(delay_us);
328 
329 	return 0;
330 }
331 
332 static int
333 poller_run_busy(void *ctx)
334 {
335 	uint64_t delay_us = (uint64_t)ctx;
336 
337 	spdk_delay_us(delay_us);
338 
339 	return 1;
340 }
341 
342 static void
343 test_reactor_stats(void)
344 {
345 	struct spdk_cpuset cpuset = {};
346 	struct spdk_thread *thread1, *thread2;
347 	struct spdk_reactor *reactor;
348 	struct spdk_poller *busy1, *idle1, *busy2, *idle2;
349 	struct spdk_thread_stats stats;
350 	int rc __attribute__((unused));
351 
352 	/* Test case is the following:
353 	 * Create a reactor on CPU core0.
354 	 * Create thread1 and thread2 simultaneously on reactor0 at TSC = 100.
355 	 * Reactor runs
356 	 * - thread1 for 100 with busy
357 	 * - thread2 for 200 with idle
358 	 * - thread1 for 300 with idle
359 	 * - thread2 for 400 with busy.
360 	 * Then,
361 	 * - both elapsed TSC of thread1 and thread2 should be 1100 (= 100 + 1000).
362 	 * - busy TSC of reactor should be 500 (= 100 + 400).
363 	 * - idle TSC of reactor should be 500 (= 200 + 300).
364 	 *
365 	 * After that reactor0 runs with no threads for 900 TSC.
366 	 * Create thread1 on reactor0 at TSC = 2000.
367 	 * Reactor runs
368 	 * - thread1 for 100 with busy
369 	 * Then,
370 	 * - elapsed TSC of thread1 should be 2100 (= 2000+ 100).
371 	 * - busy TSC of reactor should be 600 (= 500 + 100).
372 	 * - idle TSC of reactor should be 500 (= 500 + 900).
373 	 */
374 
375 	MOCK_SET(spdk_env_get_current_core, 0);
376 
377 	allocate_cores(1);
378 
379 	CU_ASSERT(spdk_reactors_init(SPDK_DEFAULT_MSG_MEMPOOL_SIZE) == 0);
380 
381 	spdk_cpuset_set_cpu(&cpuset, 0, true);
382 
383 	reactor = spdk_reactor_get(0);
384 	SPDK_CU_ASSERT_FATAL(reactor != NULL);
385 
386 	/* First reactor_run() sets the tsc_last. */
387 	MOCK_SET(spdk_get_ticks, 100);
388 	reactor->tsc_last = spdk_get_ticks();
389 
390 	thread1 = spdk_thread_create(NULL, &cpuset);
391 	SPDK_CU_ASSERT_FATAL(thread1 != NULL);
392 
393 	thread2 = spdk_thread_create(NULL, &cpuset);
394 	SPDK_CU_ASSERT_FATAL(thread2 != NULL);
395 
396 	spdk_set_thread(thread1);
397 	busy1 = spdk_poller_register(poller_run_busy, (void *)100, 0);
398 	CU_ASSERT(busy1 != NULL);
399 
400 	spdk_set_thread(thread2);
401 	idle2 = spdk_poller_register(poller_run_idle, (void *)300, 0);
402 	CU_ASSERT(idle2 != NULL);
403 
404 	_reactor_run(reactor);
405 
406 	spdk_set_thread(thread1);
407 	CU_ASSERT(spdk_thread_get_last_tsc(thread1) == 200);
408 	CU_ASSERT(spdk_thread_get_stats(&stats) == 0);
409 	CU_ASSERT(stats.busy_tsc == 100);
410 	CU_ASSERT(stats.idle_tsc == 0);
411 	spdk_set_thread(thread2);
412 	CU_ASSERT(spdk_thread_get_last_tsc(thread2) == 500);
413 	CU_ASSERT(spdk_thread_get_stats(&stats) == 0);
414 	CU_ASSERT(stats.busy_tsc == 0);
415 	CU_ASSERT(stats.idle_tsc == 300);
416 
417 	CU_ASSERT(reactor->busy_tsc == 100);
418 	CU_ASSERT(reactor->idle_tsc == 300);
419 
420 	/* 100 + 100 + 300 = 500 ticks elapsed */
421 	CU_ASSERT(reactor->tsc_last == 500);
422 
423 	spdk_set_thread(thread1);
424 	spdk_poller_unregister(&busy1);
425 	idle1 = spdk_poller_register(poller_run_idle, (void *)200, 0);
426 	CU_ASSERT(idle1 != NULL);
427 
428 	spdk_set_thread(thread2);
429 	spdk_poller_unregister(&idle2);
430 	busy2 = spdk_poller_register(poller_run_busy, (void *)400, 0);
431 	CU_ASSERT(busy2 != NULL);
432 
433 	_reactor_run(reactor);
434 
435 	spdk_set_thread(thread1);
436 	CU_ASSERT(spdk_thread_get_last_tsc(thread1) == 700);
437 	CU_ASSERT(spdk_thread_get_stats(&stats) == 0);
438 	CU_ASSERT(stats.busy_tsc == 100);
439 	CU_ASSERT(stats.idle_tsc == 200);
440 	spdk_set_thread(thread2);
441 	CU_ASSERT(spdk_thread_get_last_tsc(thread2) == 1100);
442 	CU_ASSERT(spdk_thread_get_stats(&stats) == 0);
443 	CU_ASSERT(stats.busy_tsc == 400);
444 	CU_ASSERT(stats.idle_tsc == 300);
445 
446 	CU_ASSERT(reactor->busy_tsc == 500);
447 	CU_ASSERT(reactor->idle_tsc == 500);
448 
449 	/* 500 + 200 + 400 = 1100 ticks elapsed */
450 	CU_ASSERT(reactor->tsc_last == 1100);
451 
452 	spdk_set_thread(thread1);
453 	spdk_poller_unregister(&idle1);
454 	spdk_thread_exit(thread1);
455 
456 	spdk_set_thread(thread2);
457 	spdk_poller_unregister(&busy2);
458 	spdk_thread_exit(thread2);
459 
460 	_reactor_run(reactor);
461 
462 	/* After 900 ticks new thread is created. */
463 	/* 1100 + 900 = 2000 ticks elapsed */
464 	MOCK_SET(spdk_get_ticks, 2000);
465 	_reactor_run(reactor);
466 	CU_ASSERT(reactor->tsc_last == 2000);
467 
468 	thread1 = spdk_thread_create(NULL, &cpuset);
469 	SPDK_CU_ASSERT_FATAL(thread1 != NULL);
470 
471 	spdk_set_thread(thread1);
472 	busy1 = spdk_poller_register(poller_run_busy, (void *)100, 0);
473 	CU_ASSERT(busy1 != NULL);
474 
475 	_reactor_run(reactor);
476 
477 	spdk_set_thread(thread1);
478 	CU_ASSERT(spdk_thread_get_last_tsc(thread1) == 2100);
479 	CU_ASSERT(spdk_thread_get_stats(&stats) == 0);
480 	CU_ASSERT(stats.busy_tsc == 100);
481 	CU_ASSERT(stats.idle_tsc == 0);
482 
483 	CU_ASSERT(reactor->busy_tsc == 600);
484 	CU_ASSERT(reactor->idle_tsc == 1400);
485 
486 	/* 2000 + 100 = 2100 ticks elapsed */
487 	CU_ASSERT(reactor->tsc_last == 2100);
488 
489 	spdk_set_thread(thread1);
490 	spdk_poller_unregister(&busy1);
491 	spdk_thread_exit(thread1);
492 
493 	_reactor_run(reactor);
494 
495 	CU_ASSERT(TAILQ_EMPTY(&reactor->threads));
496 
497 	/* No further than 2100 ticks elapsed */
498 	CU_ASSERT(reactor->tsc_last == 2100);
499 
500 	spdk_reactors_fini();
501 
502 	free_cores();
503 
504 	MOCK_CLEAR(spdk_env_get_current_core);
505 }
506 
507 static uint32_t
508 _run_events_till_completion(uint32_t reactor_count)
509 {
510 	struct spdk_reactor *reactor;
511 	struct spdk_thread *app_thread = _spdk_get_app_thread();
512 	uint32_t i, events;
513 	uint32_t total_events = 0;
514 
515 	do {
516 		events = 0;
517 		for (i = 0; i < reactor_count; i++) {
518 			reactor = spdk_reactor_get(i);
519 			CU_ASSERT(reactor != NULL);
520 			MOCK_SET(spdk_env_get_current_core, i);
521 			events += event_queue_run_batch(reactor);
522 
523 			/* Some events still require app_thread to run */
524 			MOCK_SET(spdk_env_get_current_core, g_scheduling_reactor->lcore);
525 			spdk_thread_poll(app_thread, 0, 0);
526 
527 			MOCK_CLEAR(spdk_env_get_current_core);
528 		}
529 		total_events += events;
530 	} while (events > 0);
531 
532 	return total_events;
533 }
534 
535 static void
536 test_scheduler(void)
537 {
538 	struct spdk_cpuset cpuset = {};
539 	struct spdk_thread *thread[3];
540 	struct spdk_reactor *reactor;
541 	struct spdk_poller *busy, *idle;
542 	uint64_t reactor_busy_tsc[3], reactor_idle_tsc[3];
543 	uint64_t thread_busy_tsc[3], thread_idle_tsc[3];
544 	uint64_t current_time, end_time, busy_time, idle_time;
545 	struct spdk_thread_stats stats;
546 	int i;
547 
548 	MOCK_SET(spdk_env_get_current_core, 0);
549 
550 	allocate_cores(3);
551 
552 	CU_ASSERT(spdk_reactors_init(SPDK_DEFAULT_MSG_MEMPOOL_SIZE) == 0);
553 
554 	spdk_scheduler_set("dynamic");
555 
556 	for (i = 0; i < 3; i++) {
557 		spdk_cpuset_set_cpu(&g_reactor_core_mask, i, true);
558 	}
559 	g_next_core = 0;
560 
561 	/* Create threads. */
562 	for (i = 0; i < 3; i++) {
563 		spdk_cpuset_zero(&cpuset);
564 		spdk_cpuset_set_cpu(&cpuset, i, true);
565 		thread[i] = spdk_thread_create(NULL, &cpuset);
566 		CU_ASSERT(thread[i] != NULL);
567 		thread_busy_tsc[i] = 0;
568 		thread_idle_tsc[i] = 0;
569 	}
570 
571 	for (i = 0; i < 3; i++) {
572 		reactor = spdk_reactor_get(i);
573 		CU_ASSERT(reactor != NULL);
574 		MOCK_SET(spdk_env_get_current_core, i);
575 		event_queue_run_batch(reactor);
576 		CU_ASSERT(!TAILQ_EMPTY(&reactor->threads));
577 		reactor_busy_tsc[i] = 0;
578 		reactor_idle_tsc[i] = 0;
579 	}
580 
581 	g_reactor_state = SPDK_REACTOR_STATE_RUNNING;
582 
583 	MOCK_SET(spdk_env_get_current_core, 0);
584 
585 	/* Init threads stats (low load) */
586 	/* Each reactor starts at 100 tsc,
587 	 * ends at 100 + 100 = 200 tsc. */
588 	current_time = 100;
589 	idle_time = 100;
590 	busy_time = 0;
591 	end_time = current_time + idle_time + busy_time;
592 	for (i = 0; i < 3; i++) {
593 		spdk_set_thread(thread[i]);
594 		idle = spdk_poller_register(poller_run_idle, (void *)idle_time, 0);
595 		reactor = spdk_reactor_get(i);
596 		CU_ASSERT(reactor != NULL);
597 		MOCK_SET(spdk_get_ticks, current_time);
598 		reactor->tsc_last = spdk_get_ticks();
599 		_reactor_run(reactor);
600 		CU_ASSERT(reactor->tsc_last == end_time);
601 		spdk_poller_unregister(&idle);
602 
603 		CU_ASSERT(spdk_thread_get_last_tsc(thread[i]) == end_time);
604 		CU_ASSERT(spdk_thread_get_stats(&stats) == 0);
605 		CU_ASSERT(stats.busy_tsc == busy_time);
606 		thread_busy_tsc[i] = stats.busy_tsc;
607 		CU_ASSERT(stats.idle_tsc == idle_time);
608 		thread_idle_tsc[i] = stats.idle_tsc;
609 		CU_ASSERT(reactor->busy_tsc == busy_time);
610 		reactor_busy_tsc[i] = reactor->busy_tsc;
611 		CU_ASSERT(reactor->idle_tsc == idle_time);
612 		reactor_idle_tsc[i] = reactor->idle_tsc;
613 	}
614 	CU_ASSERT(spdk_get_ticks() == end_time);
615 	current_time = 200;
616 
617 	MOCK_SET(spdk_env_get_current_core, 0);
618 	_reactors_scheduler_gather_metrics(NULL, NULL);
619 
620 	_run_events_till_completion(3);
621 	MOCK_SET(spdk_env_get_current_core, 0);
622 
623 	/* Threads were idle, so all of them should be placed on core 0.
624 	 * All reactors start and end at 200 tsc, since for this iteration
625 	 * the threads have no pollers (so they consume no idle or busy tsc).
626 	 */
627 	for (i = 0; i < 3; i++) {
628 		reactor = spdk_reactor_get(i);
629 		CU_ASSERT(reactor != NULL);
630 		MOCK_SET(spdk_get_ticks, current_time);
631 		_reactor_run(reactor);
632 		CU_ASSERT(reactor->tsc_last == current_time);
633 		CU_ASSERT(reactor->busy_tsc == reactor_busy_tsc[i]);
634 		CU_ASSERT(reactor->idle_tsc == reactor_idle_tsc[i]);
635 		spdk_set_thread(thread[i]);
636 		CU_ASSERT(spdk_thread_get_last_tsc(thread[i]) == current_time);
637 		CU_ASSERT(spdk_thread_get_stats(&stats) == 0);
638 		CU_ASSERT(stats.busy_tsc == thread_busy_tsc[i]);
639 		CU_ASSERT(stats.idle_tsc == thread_idle_tsc[i]);
640 	}
641 	CU_ASSERT(spdk_get_ticks() == current_time);
642 
643 	/* 2 threads should be scheduled to core 0 */
644 	reactor = spdk_reactor_get(0);
645 	CU_ASSERT(reactor != NULL);
646 	MOCK_SET(spdk_env_get_current_core, 0);
647 	event_queue_run_batch(reactor);
648 
649 	reactor = spdk_reactor_get(0);
650 	CU_ASSERT(reactor != NULL);
651 	CU_ASSERT(!TAILQ_EMPTY(&reactor->threads));
652 	reactor = spdk_reactor_get(1);
653 	CU_ASSERT(reactor != NULL);
654 	CU_ASSERT(TAILQ_EMPTY(&reactor->threads));
655 	reactor = spdk_reactor_get(2);
656 	CU_ASSERT(reactor != NULL);
657 	CU_ASSERT(TAILQ_EMPTY(&reactor->threads));
658 
659 	/* Make threads busy */
660 	reactor = spdk_reactor_get(0);
661 	CU_ASSERT(reactor != NULL);
662 
663 	/* All threads run on single reactor,
664 	 * reactor 0 starts at 200 tsc,
665 	 * ending at 200 + (100 * 3) = 500 tsc. */
666 	MOCK_SET(spdk_get_ticks, current_time);
667 	busy_time = 100;
668 	idle_time = 0;
669 	for (i = 0; i < 3; i++) {
670 		spdk_set_thread(thread[i]);
671 		busy = spdk_poller_register(poller_run_busy, (void *)busy_time, 0);
672 		_reactor_run(reactor);
673 		spdk_poller_unregister(&busy);
674 		current_time += busy_time;
675 
676 		CU_ASSERT(reactor->tsc_last == current_time);
677 		CU_ASSERT(spdk_thread_get_last_tsc(thread[i]) == current_time);
678 		CU_ASSERT(spdk_thread_get_stats(&stats) == 0);
679 		CU_ASSERT(stats.busy_tsc == thread_busy_tsc[i] + busy_time);
680 		CU_ASSERT(stats.idle_tsc == thread_idle_tsc[i] + idle_time);;
681 	}
682 	CU_ASSERT(reactor->busy_tsc == reactor_busy_tsc[0] + 3 * busy_time);
683 	CU_ASSERT(reactor->idle_tsc == reactor_idle_tsc[0] + 3 * idle_time);
684 	CU_ASSERT(spdk_get_ticks() == current_time);
685 
686 	/* Run scheduler again, this time all threads are busy */
687 	MOCK_SET(spdk_env_get_current_core, 0);
688 	_reactors_scheduler_gather_metrics(NULL, NULL);
689 
690 	_run_events_till_completion(3);
691 	MOCK_SET(spdk_env_get_current_core, 0);
692 
693 	/* Threads were busy, 2 will stay on core 0, 1 will move to core 1 */
694 	for (i = 0; i < 3; i++) {
695 		MOCK_SET(spdk_env_get_current_core, i);
696 		reactor = spdk_reactor_get(i);
697 		CU_ASSERT(reactor != NULL);
698 		_reactor_run(reactor);
699 	}
700 
701 	for (i = 0; i < 3; i++) {
702 		reactor = spdk_reactor_get(i);
703 		CU_ASSERT(reactor != NULL);
704 		CU_ASSERT(!TAILQ_EMPTY(&reactor->threads));
705 	}
706 
707 	g_reactor_state = SPDK_REACTOR_STATE_INITIALIZED;
708 
709 	/* Destroy threads */
710 	for (i = 0; i < 3; i++) {
711 		spdk_set_thread(thread[i]);
712 		spdk_thread_exit(thread[i]);
713 	}
714 	for (i = 0; i < 3; i++) {
715 		reactor = spdk_reactor_get(i);
716 		CU_ASSERT(reactor != NULL);
717 		reactor_run(reactor);
718 	}
719 
720 	spdk_set_thread(NULL);
721 
722 	MOCK_CLEAR(spdk_env_get_current_core);
723 
724 	spdk_reactors_fini();
725 
726 	free_cores();
727 }
728 
729 uint8_t g_curr_freq;
730 
731 static int
732 core_freq_up(uint32_t lcore)
733 {
734 	if (g_curr_freq != UINT8_MAX) {
735 		g_curr_freq++;
736 	}
737 
738 	return 0;
739 }
740 
741 static int
742 core_freq_down(uint32_t lcore)
743 {
744 	if (g_curr_freq != 0) {
745 		g_curr_freq--;
746 	}
747 
748 	return 0;
749 }
750 
751 static int
752 core_freq_max(uint32_t lcore)
753 {
754 	g_curr_freq = UINT8_MAX;
755 
756 	return 0;
757 }
758 
759 DEFINE_STUB(core_freq_min, int, (uint32_t lcore_id), 0);
760 DEFINE_STUB(core_caps, int,
761 	    (uint32_t lcore_id, struct spdk_governor_capabilities *capabilities), 0);
762 DEFINE_STUB(governor_init, int, (void), 0);
763 DEFINE_STUB_V(governor_deinit, (void));
764 
765 static struct spdk_governor governor = {
766 	.name = "dpdk_governor",
767 	.get_core_curr_freq = NULL,
768 	.core_freq_up = core_freq_up,
769 	.core_freq_down = core_freq_down,
770 	.set_core_freq_max = core_freq_max,
771 	.set_core_freq_min = core_freq_min,
772 	.get_core_capabilities = core_caps,
773 	.init = governor_init,
774 	.deinit = governor_deinit,
775 };
776 
777 static void
778 test_governor(void)
779 {
780 	struct spdk_cpuset cpuset = {};
781 	struct spdk_thread *thread[2];
782 	struct spdk_lw_thread *lw_thread;
783 	struct spdk_reactor *reactor;
784 	struct spdk_poller *busy, *idle;
785 	uint8_t last_freq = 100;
786 	int i;
787 
788 	MOCK_SET(spdk_env_get_current_core, 0);
789 
790 	g_curr_freq = last_freq;
791 	spdk_governor_register(&governor);
792 
793 	allocate_cores(2);
794 
795 	CU_ASSERT(spdk_reactors_init(SPDK_DEFAULT_MSG_MEMPOOL_SIZE) == 0);
796 
797 	spdk_scheduler_set("dynamic");
798 	spdk_governor_set("dpdk_governor");
799 
800 	for (i = 0; i < 2; i++) {
801 		spdk_cpuset_set_cpu(&g_reactor_core_mask, i, true);
802 	}
803 
804 	/* Create threads. */
805 	for (i = 0; i < 2; i++) {
806 		spdk_cpuset_zero(&cpuset);
807 		spdk_cpuset_set_cpu(&cpuset, i, true);
808 		thread[i] = spdk_thread_create(NULL, &cpuset);
809 		CU_ASSERT(thread[i] != NULL);
810 	}
811 
812 	for (i = 0; i < 2; i++) {
813 		reactor = spdk_reactor_get(i);
814 		CU_ASSERT(reactor != NULL);
815 		MOCK_SET(spdk_env_get_current_core, i);
816 		CU_ASSERT(event_queue_run_batch(reactor) == 1);
817 		CU_ASSERT(!TAILQ_EMPTY(&reactor->threads));
818 	}
819 
820 	reactor = spdk_reactor_get(0);
821 	CU_ASSERT(reactor != NULL);
822 	MOCK_SET(spdk_env_get_current_core, 0);
823 
824 	g_reactor_state = SPDK_REACTOR_STATE_RUNNING;
825 
826 	/* TEST 1 */
827 	/* Init thread stats (low load) */
828 	MOCK_SET(spdk_get_ticks, 100);
829 	reactor->tsc_last = 100;
830 
831 	for (i = 0; i < 2; i++) {
832 		spdk_set_thread(thread[i]);
833 		idle = spdk_poller_register(poller_run_idle, (void *)200, 0);
834 		reactor = spdk_reactor_get(i);
835 		CU_ASSERT(reactor != NULL);
836 		MOCK_SET(spdk_env_get_current_core, i);
837 		_reactor_run(reactor);
838 		spdk_poller_unregister(&idle);
839 
840 		/* Update last stats so that we don't have to call scheduler twice */
841 		lw_thread = spdk_thread_get_ctx(thread[i]);
842 		lw_thread->current_stats.idle_tsc = 1;
843 	}
844 
845 	MOCK_SET(spdk_env_get_current_core, 0);
846 	_reactors_scheduler_gather_metrics(NULL, NULL);
847 
848 	CU_ASSERT(_run_events_till_completion(2) == 2);
849 	MOCK_SET(spdk_env_get_current_core, 0);
850 
851 	/* Threads were idle, so all of them should be placed on core 0 */
852 	for (i = 0; i < 2; i++) {
853 		reactor = spdk_reactor_get(i);
854 		CU_ASSERT(reactor != NULL);
855 		_reactor_run(reactor);
856 	}
857 
858 	/* 1 thread should be scheduled to core 0 */
859 	reactor = spdk_reactor_get(0);
860 	CU_ASSERT(reactor != NULL);
861 	MOCK_SET(spdk_env_get_current_core, 0);
862 	CU_ASSERT(event_queue_run_batch(reactor) == 1);
863 
864 	/* Main core should be busy less than 50% time now - frequency should be lowered */
865 	CU_ASSERT(g_curr_freq == last_freq - 1);
866 
867 	last_freq = g_curr_freq;
868 
869 	/* TEST 2 */
870 	/* Make first threads busy - both threads will be still on core 0, but frequency will have to be raised */
871 	spdk_set_thread(thread[0]);
872 	busy = spdk_poller_register(poller_run_busy, (void *)1000, 0);
873 	_reactor_run(reactor);
874 	spdk_poller_unregister(&busy);
875 
876 	spdk_set_thread(thread[1]);
877 	idle = spdk_poller_register(poller_run_idle, (void *)100, 0);
878 	_reactor_run(reactor);
879 	spdk_poller_unregister(&idle);
880 
881 	/* Run scheduler again */
882 	MOCK_SET(spdk_env_get_current_core, 0);
883 	_reactors_scheduler_gather_metrics(NULL, NULL);
884 
885 	i = _run_events_till_completion(2);
886 	/* Six runs when interrupt mode is supported, two if not. */
887 	CU_ASSERT(i == 6 || i == 2);
888 	MOCK_SET(spdk_env_get_current_core, 0);
889 
890 	/* Main core should be busy more than 50% time now - frequency should be raised */
891 	CU_ASSERT(g_curr_freq == last_freq + 1);
892 
893 	/* TEST 3 */
894 	/* Make second thread very busy so that it will be moved to second core */
895 	spdk_set_thread(thread[1]);
896 	busy = spdk_poller_register(poller_run_busy, (void *)2000, 0);
897 	_reactor_run(reactor);
898 	spdk_poller_unregister(&busy);
899 
900 	/* Update first thread stats */
901 	spdk_set_thread(thread[0]);
902 	idle = spdk_poller_register(poller_run_idle, (void *)100, 0);
903 	_reactor_run(reactor);
904 	spdk_poller_unregister(&idle);
905 
906 	/* Run scheduler again */
907 	MOCK_SET(spdk_env_get_current_core, 0);
908 	_reactors_scheduler_gather_metrics(NULL, NULL);
909 
910 	i = _run_events_till_completion(2);
911 	/* Six runs when interrupt mode is supported, two if not. */
912 	CU_ASSERT(i == 6 || i == 2);
913 	MOCK_SET(spdk_env_get_current_core, 0);
914 
915 	for (i = 0; i < 2; i++) {
916 		reactor = spdk_reactor_get(i);
917 		CU_ASSERT(reactor != NULL);
918 		_reactor_run(reactor);
919 	}
920 
921 	/* Main core frequency should be set to max when we have busy threads on other cores */
922 	CU_ASSERT(g_curr_freq == UINT8_MAX);
923 
924 	g_reactor_state = SPDK_REACTOR_STATE_INITIALIZED;
925 
926 	/* Destroy threads */
927 	for (i = 0; i < 2; i++) {
928 		spdk_set_thread(thread[i]);
929 		spdk_thread_exit(thread[i]);
930 	}
931 	for (i = 0; i < 2; i++) {
932 		reactor = spdk_reactor_get(i);
933 		CU_ASSERT(reactor != NULL);
934 		reactor_run(reactor);
935 	}
936 
937 	spdk_set_thread(NULL);
938 
939 	MOCK_CLEAR(spdk_env_get_current_core);
940 
941 	spdk_reactors_fini();
942 
943 	free_cores();
944 }
945 
946 int
947 main(int argc, char **argv)
948 {
949 	CU_pSuite suite = NULL;
950 	unsigned int num_failures;
951 
952 	CU_set_error_action(CUEA_ABORT);
953 	CU_initialize_registry();
954 
955 	suite = CU_add_suite("app_suite", NULL, NULL);
956 
957 	CU_ADD_TEST(suite, test_create_reactor);
958 	CU_ADD_TEST(suite, test_init_reactors);
959 	CU_ADD_TEST(suite, test_event_call);
960 	CU_ADD_TEST(suite, test_schedule_thread);
961 	CU_ADD_TEST(suite, test_reschedule_thread);
962 	CU_ADD_TEST(suite, test_for_each_reactor);
963 	CU_ADD_TEST(suite, test_reactor_stats);
964 	CU_ADD_TEST(suite, test_scheduler);
965 	CU_ADD_TEST(suite, test_governor);
966 
967 	CU_basic_set_mode(CU_BRM_VERBOSE);
968 	CU_basic_run_tests();
969 	num_failures = CU_get_number_of_failures();
970 	CU_cleanup_registry();
971 
972 	return num_failures;
973 }
974