xref: /spdk/app/spdk_top/spdk_top.c (revision 86e4665b4e7382b1ad6ab9fe98c3454253406fc8)
1 /*   SPDX-License-Identifier: BSD-3-Clause
2  *   Copyright (C) 2020 Intel Corporation.
3  *   All rights reserved.
4  */
5 
6 #include "spdk/stdinc.h"
7 #include "spdk/jsonrpc.h"
8 #include "spdk/rpc.h"
9 #include "spdk/event.h"
10 #include "spdk/util.h"
11 #include "spdk/env.h"
12 
13 #if defined __has_include
14 #if __has_include(<ncurses/panel.h>)
15 #include <ncurses/ncurses.h>
16 #include <ncurses/panel.h>
17 #include <ncurses/menu.h>
18 #else
19 #include <ncurses.h>
20 #include <panel.h>
21 #include <menu.h>
22 #endif
23 #else
24 #include <ncurses.h>
25 #include <panel.h>
26 #include <menu.h>
27 #endif
28 
29 #define RPC_MAX_THREADS 1024
30 #define RPC_MAX_POLLERS 1024
31 #define RPC_MAX_CORES 255
32 #define MAX_THREAD_NAME 128
33 #define MAX_POLLER_NAME 128
34 #define MAX_THREADS 4096
35 #define RR_MAX_VALUE 255
36 
37 #define MAX_STRING_LEN 12289 /* 3x 4k monitors + 1 */
38 #define TAB_WIN_HEIGHT 3
39 #define TAB_WIN_LOCATION_ROW 1
40 #define TABS_SPACING 2
41 #define TABS_LOCATION_ROW 4
42 #define TABS_LOCATION_COL 0
43 #define TABS_DATA_START_ROW 3
44 #define TABS_DATA_START_COL 2
45 #define TABS_COL_COUNT 13
46 #define MENU_WIN_HEIGHT 3
47 #define MENU_WIN_SPACING 4
48 #define MENU_WIN_LOCATION_COL 0
49 #define RR_WIN_WIDTH 32
50 #define RR_WIN_HEIGHT 5
51 #define MAX_THREAD_NAME_LEN 26
52 #define MAX_THREAD_COUNT_STR_LEN 14
53 #define MAX_POLLER_NAME_LEN 36
54 #define MAX_POLLER_COUNT_STR_LEN 16
55 #define MAX_POLLER_TYPE_STR_LEN 8
56 #define MAX_STATUS_IND_STR_LEN 8
57 #define MAX_POLLER_IND_STR_LEN 28
58 #define MAX_CORE_MASK_STR_LEN 16
59 #define MAX_CORE_STR_LEN 6
60 #define MAX_CORE_FREQ_STR_LEN 18
61 #define MAX_TIME_STR_LEN 12
62 #define MAX_FLOAT_STR_LEN 8
63 #define MAX_POLLER_RUN_COUNT 20
64 #define MAX_PERIOD_STR_LEN 12
65 #define MAX_INTR_LEN 6
66 #define WINDOW_HEADER 12
67 #define FROM_HEX 16
68 #define THREAD_WIN_WIDTH 69
69 #define THREAD_WIN_HEIGHT 9
70 #define THREAD_WIN_FIRST_COL 2
71 #define CORE_WIN_FIRST_COL 16
72 #define CORE_WIN_WIDTH 48
73 #define CORE_WIN_HEIGHT 11
74 #define POLLER_WIN_HEIGHT 8
75 #define POLLER_WIN_WIDTH 64
76 #define POLLER_WIN_FIRST_COL 14
77 #define FIRST_DATA_ROW 7
78 #define HELP_WIN_WIDTH 88
79 #define HELP_WIN_HEIGHT 24
80 #define SCHEDULER_WIN_HEIGHT 7
81 #define SCHEDULER_WIN_FIRST_COL 2
82 #define MAX_SCHEDULER_PERIOD_STR_LEN 10
83 
84 enum tabs {
85 	THREADS_TAB,
86 	POLLERS_TAB,
87 	CORES_TAB,
88 	NUMBER_OF_TABS,
89 };
90 
91 enum column_threads_type {
92 	COL_THREADS_NAME,
93 	COL_THREADS_CORE,
94 	COL_THREADS_ACTIVE_POLLERS,
95 	COL_THREADS_TIMED_POLLERS,
96 	COL_THREADS_PAUSED_POLLERS,
97 	COL_THREADS_IDLE_TIME,
98 	COL_THREADS_BUSY_TIME,
99 	COL_THREADS_CPU_USAGE,
100 	COL_THREADS_STATUS,
101 	COL_THREADS_NONE = 255,
102 };
103 
104 enum column_pollers_type {
105 	COL_POLLERS_NAME,
106 	COL_POLLERS_TYPE,
107 	COL_POLLERS_THREAD_NAME,
108 	COL_POLLERS_RUN_COUNTER,
109 	COL_POLLERS_PERIOD,
110 	COL_POLLERS_BUSY_COUNT,
111 	COL_POLLERS_NONE = 255,
112 };
113 
114 enum column_cores_type {
115 	COL_CORES_CORE,
116 	COL_CORES_THREADS,
117 	COL_CORES_POLLERS,
118 	COL_CORES_IDLE_TIME,
119 	COL_CORES_BUSY_TIME,
120 	COL_CORES_BUSY_PCT,
121 	COL_CORES_STATUS,
122 	COL_CORES_INTR,
123 	COL_CORES_CORE_FREQ,
124 	COL_CORES_NONE = 255,
125 };
126 
127 enum spdk_poller_type {
128 	SPDK_ACTIVE_POLLER,
129 	SPDK_TIMED_POLLER,
130 	SPDK_PAUSED_POLLER,
131 	SPDK_POLLER_TYPES_COUNT,
132 };
133 
134 struct col_desc {
135 	const char *name;
136 	uint8_t name_len;
137 	uint8_t max_data_string;
138 	bool disabled;
139 };
140 
141 struct run_counter_history {
142 	uint64_t poller_id;
143 	uint64_t thread_id;
144 	uint64_t last_run_counter;
145 	uint64_t last_busy_counter;
146 	TAILQ_ENTRY(run_counter_history) link;
147 };
148 
149 uint8_t g_sleep_time = 1;
150 uint16_t g_selected_row;
151 uint16_t g_max_selected_row;
152 uint64_t g_tick_rate;
153 const char *poller_type_str[SPDK_POLLER_TYPES_COUNT] = {"Active", "Timed", "Paused"};
154 const char *g_tab_title[NUMBER_OF_TABS] = {"[1] THREADS", "[2] POLLERS", "[3] CORES"};
155 struct spdk_jsonrpc_client *g_rpc_client;
156 static TAILQ_HEAD(, run_counter_history) g_run_counter_history = TAILQ_HEAD_INITIALIZER(
157 			g_run_counter_history);
158 WINDOW *g_menu_win, *g_tab_win[NUMBER_OF_TABS], *g_tabs[NUMBER_OF_TABS];
159 PANEL *g_panels[NUMBER_OF_TABS];
160 uint16_t g_max_row, g_max_col;
161 uint16_t g_data_win_size, g_max_data_rows;
162 uint32_t g_last_threads_count, g_last_pollers_count, g_last_cores_count;
163 uint8_t g_current_sort_col[NUMBER_OF_TABS] = {COL_THREADS_NAME, COL_POLLERS_NAME, COL_CORES_CORE};
164 uint8_t g_current_sort_col2[NUMBER_OF_TABS] = {COL_THREADS_NONE, COL_POLLERS_NONE, COL_CORES_NONE};
165 bool g_interval_data = true;
166 bool g_quit_app = false;
167 pthread_mutex_t g_thread_lock;
168 static struct col_desc g_col_desc[NUMBER_OF_TABS][TABS_COL_COUNT] = {
169 	{	{.name = "Thread name", .max_data_string = MAX_THREAD_NAME_LEN},
170 		{.name = "Core", .max_data_string = MAX_CORE_STR_LEN},
171 		{.name = "Active pollers", .max_data_string = MAX_POLLER_COUNT_STR_LEN},
172 		{.name = "Timed pollers", .max_data_string = MAX_POLLER_COUNT_STR_LEN},
173 		{.name = "Paused pollers", .max_data_string = MAX_POLLER_COUNT_STR_LEN},
174 		{.name = "Idle [us]", .max_data_string = MAX_TIME_STR_LEN},
175 		{.name = "Busy [us]", .max_data_string = MAX_TIME_STR_LEN},
176 		{.name = "CPU %", .max_data_string = MAX_FLOAT_STR_LEN},
177 		{.name = "Status", .max_data_string = MAX_STATUS_IND_STR_LEN},
178 		{.name = (char *)NULL}
179 	},
180 	{	{.name = "Poller name", .max_data_string = MAX_POLLER_NAME_LEN},
181 		{.name = "Type", .max_data_string = MAX_POLLER_TYPE_STR_LEN},
182 		{.name = "On thread", .max_data_string = MAX_THREAD_NAME_LEN},
183 		{.name = "Run count", .max_data_string = MAX_POLLER_RUN_COUNT},
184 		{.name = "Period [us]", .max_data_string = MAX_PERIOD_STR_LEN},
185 		{.name = "Status (busy count)", .max_data_string = MAX_POLLER_IND_STR_LEN},
186 		{.name = (char *)NULL}
187 	},
188 	{	{.name = "Core", .max_data_string = MAX_CORE_STR_LEN},
189 		{.name = "Threads", .max_data_string = MAX_THREAD_COUNT_STR_LEN},
190 		{.name = "Pollers", .max_data_string = MAX_POLLER_COUNT_STR_LEN},
191 		{.name = "Idle [us]", .max_data_string = MAX_TIME_STR_LEN},
192 		{.name = "Busy [us]", .max_data_string = MAX_TIME_STR_LEN},
193 		{.name = "Busy %", .max_data_string = MAX_FLOAT_STR_LEN},
194 		{.name = "Status", .max_data_string = MAX_STATUS_IND_STR_LEN},
195 		{.name = "Intr", .max_data_string = MAX_INTR_LEN},
196 		{.name = "Frequency [MHz]", .max_data_string = MAX_CORE_FREQ_STR_LEN},
197 		{.name = (char *)NULL}
198 	}
199 };
200 
201 struct rpc_thread_info {
202 	char *name;
203 	uint64_t id;
204 	int core_num;
205 	int core_idx;
206 	char *cpumask;
207 	uint64_t busy;
208 	uint64_t last_busy;
209 	uint64_t idle;
210 	uint64_t last_idle;
211 	uint64_t active_pollers_count;
212 	uint64_t timed_pollers_count;
213 	uint64_t paused_pollers_count;
214 };
215 
216 struct rpc_poller_info {
217 	char *name;
218 	char *state;
219 	uint64_t id;
220 	uint64_t run_count;
221 	uint64_t busy_count;
222 	uint64_t period_ticks;
223 	enum spdk_poller_type type;
224 	char thread_name[MAX_THREAD_NAME];
225 	uint64_t thread_id;
226 };
227 
228 struct rpc_core_thread_info {
229 	char *name;
230 	uint64_t id;
231 	char *cpumask;
232 	uint64_t elapsed;
233 };
234 
235 struct rpc_core_threads {
236 	uint64_t threads_count;
237 	struct rpc_core_thread_info *thread;
238 };
239 
240 struct rpc_core_info {
241 	uint32_t lcore;
242 	uint64_t pollers_count;
243 	uint64_t busy;
244 	uint64_t idle;
245 	uint32_t core_freq;
246 	uint64_t last_idle;
247 	uint64_t last_busy;
248 	bool in_interrupt;
249 	struct rpc_core_threads threads;
250 };
251 
252 struct rpc_scheduler {
253 	char *scheduler_name;
254 	char *governor_name;
255 	uint64_t scheduler_period;
256 };
257 
258 struct rpc_thread_info g_threads_info[RPC_MAX_THREADS];
259 struct rpc_poller_info g_pollers_info[RPC_MAX_POLLERS];
260 struct rpc_core_info g_cores_info[RPC_MAX_CORES];
261 struct rpc_scheduler g_scheduler_info;
262 
263 static void
264 init_str_len(void)
265 {
266 	int i, j;
267 
268 	for (i = 0; i < NUMBER_OF_TABS; i++) {
269 		for (j = 0; g_col_desc[i][j].name != NULL; j++) {
270 			g_col_desc[i][j].name_len = strlen(g_col_desc[i][j].name);
271 		}
272 	}
273 }
274 
275 static void
276 free_rpc_threads_stats(struct rpc_thread_info *req)
277 {
278 	free(req->name);
279 	req->name = NULL;
280 	free(req->cpumask);
281 	req->cpumask = NULL;
282 }
283 
284 static const struct spdk_json_object_decoder rpc_thread_info_decoders[] = {
285 	{"name", offsetof(struct rpc_thread_info, name), spdk_json_decode_string},
286 	{"id", offsetof(struct rpc_thread_info, id), spdk_json_decode_uint64},
287 	{"cpumask", offsetof(struct rpc_thread_info, cpumask), spdk_json_decode_string},
288 	{"busy", offsetof(struct rpc_thread_info, busy), spdk_json_decode_uint64},
289 	{"idle", offsetof(struct rpc_thread_info, idle), spdk_json_decode_uint64},
290 	{"active_pollers_count", offsetof(struct rpc_thread_info, active_pollers_count), spdk_json_decode_uint64},
291 	{"timed_pollers_count", offsetof(struct rpc_thread_info, timed_pollers_count), spdk_json_decode_uint64},
292 	{"paused_pollers_count", offsetof(struct rpc_thread_info, paused_pollers_count), spdk_json_decode_uint64},
293 };
294 
295 static int
296 rpc_decode_threads_array(struct spdk_json_val *val, struct rpc_thread_info *out,
297 			 uint64_t *current_threads_count)
298 {
299 	struct spdk_json_val *thread = val;
300 	uint64_t i = 0;
301 	int rc;
302 
303 	/* Fetch the beginning of threads array */
304 	rc = spdk_json_find_array(thread, "threads", NULL, &thread);
305 	if (rc) {
306 		printf("Could not fetch threads array from JSON.\n");
307 		goto end;
308 	}
309 
310 	for (thread = spdk_json_array_first(thread); thread != NULL; thread = spdk_json_next(thread)) {
311 		rc = spdk_json_decode_object(thread, rpc_thread_info_decoders,
312 					     SPDK_COUNTOF(rpc_thread_info_decoders), &out[i]);
313 		if (rc) {
314 			printf("Could not decode thread object from JSON.\n");
315 			break;
316 		}
317 
318 		i++;
319 	}
320 
321 end:
322 
323 	*current_threads_count = i;
324 	return rc;
325 }
326 
327 static void
328 free_rpc_poller(struct rpc_poller_info *poller)
329 {
330 	free(poller->name);
331 	poller->name = NULL;
332 	free(poller->state);
333 	poller->state = NULL;
334 }
335 
336 static void
337 free_rpc_core_info(struct rpc_core_info *core_info, size_t size)
338 {
339 	struct rpc_core_threads *threads;
340 	struct rpc_core_thread_info *thread;
341 	uint64_t i, core_number;
342 
343 	for (core_number = 0; core_number < size; core_number++) {
344 		threads = &core_info[core_number].threads;
345 		for (i = 0; i < threads->threads_count; i++) {
346 			thread = &threads->thread[i];
347 			free(thread->name);
348 			free(thread->cpumask);
349 		}
350 		free(threads->thread);
351 	}
352 }
353 
354 static const struct spdk_json_object_decoder rpc_pollers_decoders[] = {
355 	{"name", offsetof(struct rpc_poller_info, name), spdk_json_decode_string},
356 	{"state", offsetof(struct rpc_poller_info, state), spdk_json_decode_string},
357 	{"id", offsetof(struct rpc_poller_info, id), spdk_json_decode_uint64},
358 	{"run_count", offsetof(struct rpc_poller_info, run_count), spdk_json_decode_uint64},
359 	{"busy_count", offsetof(struct rpc_poller_info, busy_count), spdk_json_decode_uint64},
360 	{"period_ticks", offsetof(struct rpc_poller_info, period_ticks), spdk_json_decode_uint64, true},
361 };
362 
363 static int
364 rpc_decode_pollers_array(struct spdk_json_val *poller, struct rpc_poller_info *out,
365 			 uint64_t *poller_count,
366 			 const char *thread_name, uint64_t thread_name_length, uint64_t thread_id,
367 			 enum spdk_poller_type poller_type)
368 {
369 	int rc;
370 
371 	for (poller = spdk_json_array_first(poller); poller != NULL; poller = spdk_json_next(poller)) {
372 		out[*poller_count].thread_id = thread_id;
373 		memcpy(out[*poller_count].thread_name, thread_name, sizeof(char) * thread_name_length);
374 		out[*poller_count].type = poller_type;
375 
376 		rc = spdk_json_decode_object(poller, rpc_pollers_decoders,
377 					     SPDK_COUNTOF(rpc_pollers_decoders), &out[*poller_count]);
378 		if (rc) {
379 			printf("Could not decode poller object from JSON.\n");
380 			return rc;
381 		}
382 
383 		(*poller_count)++;
384 		if (*poller_count == RPC_MAX_POLLERS) {
385 			return -1;
386 		}
387 	}
388 
389 	return 0;
390 }
391 
392 static const struct spdk_json_object_decoder rpc_thread_pollers_decoders[] = {
393 	{"name", offsetof(struct rpc_thread_info, name), spdk_json_decode_string},
394 	{"id", offsetof(struct rpc_thread_info, id), spdk_json_decode_uint64},
395 };
396 
397 static int
398 rpc_decode_pollers_threads_array(struct spdk_json_val *val, struct rpc_poller_info *out,
399 				 uint32_t *num_pollers)
400 {
401 	struct spdk_json_val *thread = val, *poller;
402 	/* This is a temporary poller structure to hold thread name and id.
403 	 * It is filled with data only once per thread change and then
404 	 * that memory is copied to each poller running on that thread. */
405 	struct rpc_thread_info thread_info = {};
406 	uint64_t poller_count = 0, i, thread_name_length;
407 	int rc;
408 	const char *poller_typenames[] = { "active_pollers", "timed_pollers", "paused_pollers" };
409 	enum spdk_poller_type poller_types[] = { SPDK_ACTIVE_POLLER, SPDK_TIMED_POLLER, SPDK_PAUSED_POLLER };
410 
411 	/* Fetch the beginning of threads array */
412 	rc = spdk_json_find_array(thread, "threads", NULL, &thread);
413 	if (rc) {
414 		printf("Could not fetch threads array from JSON.\n");
415 		goto end;
416 	}
417 
418 	for (thread = spdk_json_array_first(thread); thread != NULL; thread = spdk_json_next(thread)) {
419 		rc = spdk_json_decode_object_relaxed(thread, rpc_thread_pollers_decoders,
420 						     SPDK_COUNTOF(rpc_thread_pollers_decoders), &thread_info);
421 		if (rc) {
422 			printf("Could not decode thread info from JSON.\n");
423 			goto end;
424 		}
425 
426 		thread_name_length = strlen(thread_info.name);
427 
428 		for (i = 0; i < SPDK_COUNTOF(poller_types); i++) {
429 			/* Find poller array */
430 			rc = spdk_json_find(thread, poller_typenames[i], NULL, &poller,
431 					    SPDK_JSON_VAL_ARRAY_BEGIN);
432 			if (rc) {
433 				printf("Could not fetch pollers array from JSON.\n");
434 				goto end;
435 			}
436 
437 			rc = rpc_decode_pollers_array(poller, out, &poller_count, thread_info.name,
438 						      thread_name_length,
439 						      thread_info.id, poller_types[i]);
440 			if (rc) {
441 				printf("Could not decode the first object in pollers array.\n");
442 				goto end;
443 			}
444 		}
445 	}
446 
447 	*num_pollers = poller_count;
448 
449 end:
450 	/* Since we rely in spdk_json_object_decode() to free this value
451 	 * each time we rewrite it, we need to free the last allocation
452 	 * manually. */
453 	free(thread_info.name);
454 
455 	if (rc) {
456 		*num_pollers = 0;
457 		for (i = 0; i < poller_count; i++) {
458 			free_rpc_poller(&out[i]);
459 		}
460 	}
461 
462 	return rc;
463 }
464 
465 static const struct spdk_json_object_decoder rpc_core_thread_info_decoders[] = {
466 	{"name", offsetof(struct rpc_core_thread_info, name), spdk_json_decode_string},
467 	{"id", offsetof(struct rpc_core_thread_info, id), spdk_json_decode_uint64},
468 	{"cpumask", offsetof(struct rpc_core_thread_info, cpumask), spdk_json_decode_string},
469 	{"elapsed", offsetof(struct rpc_core_thread_info, elapsed), spdk_json_decode_uint64},
470 };
471 
472 static int
473 rpc_decode_core_threads_object(const struct spdk_json_val *val, void *out)
474 {
475 	struct rpc_core_thread_info *info = out;
476 
477 	return spdk_json_decode_object(val, rpc_core_thread_info_decoders,
478 				       SPDK_COUNTOF(rpc_core_thread_info_decoders), info);
479 }
480 
481 #define RPC_THREAD_ENTRY_SIZE (SPDK_COUNTOF(rpc_core_thread_info_decoders) * 2)
482 
483 static int
484 rpc_decode_cores_lw_threads(const struct spdk_json_val *val, void *out)
485 {
486 	struct rpc_core_threads *threads = out;
487 	/* The number of thread entries received from RPC can be calculated using
488 	 * above define value (each JSON line = key + value, hence '* 2' ) and JSON
489 	 * 'val' value (-2 is to subtract VAL_OBJECT_BEGIN/END). */
490 	uint16_t threads_count = (spdk_json_val_len(val) - 2) / RPC_THREAD_ENTRY_SIZE;
491 
492 	assert(threads != NULL);
493 	threads->thread = calloc(threads_count, sizeof(struct rpc_core_thread_info));
494 
495 	return spdk_json_decode_array(val, rpc_decode_core_threads_object, threads->thread, threads_count,
496 				      &threads->threads_count, sizeof(struct rpc_core_thread_info));
497 }
498 
499 static const struct spdk_json_object_decoder rpc_core_info_decoders[] = {
500 	{"lcore", offsetof(struct rpc_core_info, lcore), spdk_json_decode_uint32},
501 	{"busy", offsetof(struct rpc_core_info, busy), spdk_json_decode_uint64},
502 	{"idle", offsetof(struct rpc_core_info, idle), spdk_json_decode_uint64},
503 	{"core_freq", offsetof(struct rpc_core_info, core_freq), spdk_json_decode_uint32, true},
504 	{"in_interrupt", offsetof(struct rpc_core_info, in_interrupt), spdk_json_decode_bool},
505 	{"lw_threads", offsetof(struct rpc_core_info, threads), rpc_decode_cores_lw_threads},
506 };
507 
508 static int
509 rpc_decode_core_object(const struct spdk_json_val *val, void *out)
510 {
511 	struct rpc_core_info *info = out;
512 
513 	return spdk_json_decode_object(val, rpc_core_info_decoders,
514 				       SPDK_COUNTOF(rpc_core_info_decoders), info);
515 }
516 
517 static int
518 rpc_decode_cores_array(struct spdk_json_val *val, struct rpc_core_info *out,
519 		       uint32_t *current_cores_count)
520 {
521 	struct spdk_json_val *core = val;
522 	size_t cores_count;
523 	int rc;
524 
525 	/* Fetch the beginning of reactors array. */
526 	rc = spdk_json_find_array(core, "reactors", NULL, &core);
527 	if (rc) {
528 		printf("Could not fetch cores array from JSON.");
529 		goto end;
530 	}
531 
532 	rc = spdk_json_decode_array(core, rpc_decode_core_object, out, RPC_MAX_CORES, &cores_count,
533 				    sizeof(struct rpc_core_info));
534 
535 	*current_cores_count = (uint32_t)cores_count;
536 
537 end:
538 	return rc;
539 }
540 
541 static void
542 free_rpc_scheduler(struct rpc_scheduler *req)
543 {
544 	free(req->scheduler_name);
545 	free(req->governor_name);
546 }
547 
548 static const struct spdk_json_object_decoder rpc_scheduler_decoders[] = {
549 	{"scheduler_name", offsetof(struct rpc_scheduler, scheduler_name), spdk_json_decode_string, true},
550 	{"governor_name", offsetof(struct rpc_scheduler, governor_name), spdk_json_decode_string, true},
551 	{"scheduler_period", offsetof(struct rpc_scheduler, scheduler_period), spdk_json_decode_uint64},
552 };
553 
554 static int
555 rpc_send_req(char *rpc_name, struct spdk_jsonrpc_client_response **resp)
556 {
557 	struct spdk_jsonrpc_client_response *json_resp = NULL;
558 	struct spdk_json_write_ctx *w;
559 	struct spdk_jsonrpc_client_request *request;
560 	int rc;
561 
562 	request = spdk_jsonrpc_client_create_request();
563 	if (request == NULL) {
564 		return -ENOMEM;
565 	}
566 
567 	w = spdk_jsonrpc_begin_request(request, 1, rpc_name);
568 	spdk_jsonrpc_end_request(request, w);
569 	spdk_jsonrpc_client_send_request(g_rpc_client, request);
570 
571 	do {
572 		rc = spdk_jsonrpc_client_poll(g_rpc_client, 1);
573 	} while (rc == 0 || rc == -ENOTCONN);
574 
575 	if (rc <= 0) {
576 		return -1;
577 	}
578 
579 	json_resp = spdk_jsonrpc_client_get_response(g_rpc_client);
580 	if (json_resp == NULL) {
581 		return -1;
582 	}
583 
584 	/* Check for error response */
585 	if (json_resp->error != NULL) {
586 		spdk_jsonrpc_client_free_response(json_resp);
587 		return -1;
588 	}
589 
590 	assert(json_resp->result);
591 
592 	*resp = json_resp;
593 
594 	return 0;
595 }
596 
597 static uint64_t
598 get_cpu_usage(uint64_t busy_ticks, uint64_t idle_ticks)
599 {
600 	if (busy_ticks + idle_ticks > 0) {
601 		/* Increase numerator to convert fraction into decimal with
602 		 * additional precision */
603 		return busy_ticks * 10000 / (busy_ticks + idle_ticks);
604 	}
605 
606 	return 0;
607 }
608 
609 static int
610 subsort_threads(enum column_threads_type sort_column, const void *p1, const void *p2)
611 {
612 	const struct rpc_thread_info thread_info1 = *(struct rpc_thread_info *)p1;
613 	const struct rpc_thread_info thread_info2 = *(struct rpc_thread_info *)p2;
614 	uint64_t count1, count2;
615 
616 	switch (sort_column) {
617 	case COL_THREADS_NAME:
618 		return strcmp(thread_info1.name, thread_info2.name);
619 	case COL_THREADS_CORE:
620 		count2 = thread_info1.core_num;
621 		count1 = thread_info2.core_num;
622 		break;
623 	case COL_THREADS_ACTIVE_POLLERS:
624 		count1 = thread_info1.active_pollers_count;
625 		count2 = thread_info2.active_pollers_count;
626 		break;
627 	case COL_THREADS_TIMED_POLLERS:
628 		count1 = thread_info1.timed_pollers_count;
629 		count2 = thread_info2.timed_pollers_count;
630 		break;
631 	case COL_THREADS_PAUSED_POLLERS:
632 		count1 = thread_info1.paused_pollers_count;
633 		count2 = thread_info2.paused_pollers_count;
634 		break;
635 	case COL_THREADS_IDLE_TIME:
636 		if (g_interval_data) {
637 			count1 = thread_info1.idle - thread_info1.last_idle;
638 			count2 = thread_info2.idle - thread_info2.last_idle;
639 		} else {
640 			count1 = thread_info1.idle;
641 			count2 = thread_info2.idle;
642 		}
643 		break;
644 	case COL_THREADS_BUSY_TIME:
645 		if (g_interval_data) {
646 			count1 = thread_info1.busy - thread_info1.last_busy;
647 			count2 = thread_info2.busy - thread_info2.last_busy;
648 		} else {
649 			count1 = thread_info1.busy;
650 			count2 = thread_info2.busy;
651 		}
652 		break;
653 	case COL_THREADS_CPU_USAGE:
654 		count1 = get_cpu_usage(thread_info1.busy - thread_info1.last_busy,
655 				       g_cores_info[thread_info1.core_idx].busy + g_cores_info[thread_info1.core_idx].idle);
656 		count2 = get_cpu_usage(thread_info2.busy - thread_info2.last_busy,
657 				       g_cores_info[thread_info2.core_idx].busy + g_cores_info[thread_info2.core_idx].idle);
658 		break;
659 	case COL_THREADS_NONE:
660 	default:
661 		return 0;
662 	}
663 
664 	if (count2 > count1) {
665 		return 1;
666 	} else if (count2 < count1) {
667 		return -1;
668 	} else {
669 		return 0;
670 	}
671 }
672 
673 static int
674 sort_threads(const void *p1, const void *p2)
675 {
676 	int res;
677 
678 	res = subsort_threads(g_current_sort_col[THREADS_TAB], p1, p2);
679 	if (res == 0) {
680 		res = subsort_threads(g_current_sort_col2[THREADS_TAB], p1, p2);
681 	}
682 	return res;
683 }
684 
685 static void
686 store_last_counters(uint64_t poller_id, uint64_t thread_id, uint64_t last_run_counter,
687 		    uint64_t last_busy_counter)
688 {
689 	struct run_counter_history *history;
690 
691 	TAILQ_FOREACH(history, &g_run_counter_history, link) {
692 		if ((history->poller_id == poller_id) && (history->thread_id == thread_id)) {
693 			history->last_run_counter = last_run_counter;
694 			history->last_busy_counter = last_busy_counter;
695 			return;
696 		}
697 	}
698 
699 	history = calloc(1, sizeof(*history));
700 	if (history == NULL) {
701 		fprintf(stderr, "Unable to allocate a history object in store_last_counters.\n");
702 		return;
703 	}
704 	history->poller_id = poller_id;
705 	history->thread_id = thread_id;
706 	history->last_run_counter = last_run_counter;
707 	history->last_busy_counter = last_busy_counter;
708 
709 	TAILQ_INSERT_TAIL(&g_run_counter_history, history, link);
710 }
711 
712 static int
713 get_thread_data(void)
714 {
715 	struct spdk_jsonrpc_client_response *json_resp = NULL;
716 	struct rpc_thread_info thread_info[RPC_MAX_THREADS], *thread;
717 	struct rpc_core_info *core_info;
718 	uint64_t i, j, k, current_threads_count = 0;
719 	int rc = 0;
720 
721 	rc = rpc_send_req("thread_get_stats", &json_resp);
722 	if (rc) {
723 		return rc;
724 	}
725 
726 	/* Decode json */
727 	memset(thread_info, 0, sizeof(struct rpc_thread_info) * RPC_MAX_THREADS);
728 	if (rpc_decode_threads_array(json_resp->result, thread_info, &current_threads_count)) {
729 		rc = -EINVAL;
730 		for (i = 0; i < current_threads_count; i++) {
731 			free_rpc_threads_stats(&thread_info[i]);
732 		}
733 		goto end;
734 	}
735 
736 	pthread_mutex_lock(&g_thread_lock);
737 
738 	/* This is to free allocated char arrays with old thread names */
739 	for (i = 0; i < g_last_threads_count; i++) {
740 		free_rpc_threads_stats(&g_threads_info[i]);
741 	}
742 
743 	for (i = 0; i < current_threads_count; i++) {
744 		for (j = 0; j < g_last_threads_count; j++) {
745 			if (thread_info[i].id == g_threads_info[j].id) {
746 				thread_info[i].last_busy = g_threads_info[j].busy;
747 				thread_info[i].last_idle = g_threads_info[j].idle;
748 			}
749 		}
750 	}
751 	g_last_threads_count = current_threads_count;
752 
753 	memcpy(g_threads_info, thread_info, sizeof(struct rpc_thread_info) * RPC_MAX_THREADS);
754 
755 	for (i = 0; i < g_last_threads_count; i++) {
756 		g_threads_info[i].core_num = -1;
757 	}
758 
759 	for (i = 0; i < g_last_cores_count; i++) {
760 		core_info = &g_cores_info[i];
761 
762 		for (j = 0; j < core_info->threads.threads_count; j++) {
763 			for (k = 0; k < g_last_threads_count; k++) {
764 				/* For each thread on current core: check if it's ID also exists
765 				 * in g_thread_info data structure. If it does then assign current
766 				 * core's number to that thread, otherwise application state is inconsistent
767 				 * (e.g. scheduler is moving threads between cores). */
768 				thread = &g_threads_info[k];
769 				if (thread->id == core_info->threads.thread[j].id) {
770 					thread->core_num = core_info->lcore;
771 					thread->core_idx = i;
772 					break;
773 				}
774 			}
775 		}
776 	}
777 
778 	qsort(g_threads_info, g_last_threads_count, sizeof(struct rpc_thread_info), sort_threads);
779 
780 	pthread_mutex_unlock(&g_thread_lock);
781 
782 end:
783 	spdk_jsonrpc_client_free_response(json_resp);
784 	return rc;
785 }
786 
787 static uint64_t
788 get_last_run_counter(uint64_t poller_id, uint64_t thread_id)
789 {
790 	struct run_counter_history *history;
791 
792 	TAILQ_FOREACH(history, &g_run_counter_history, link) {
793 		if ((history->poller_id == poller_id) && (history->thread_id == thread_id)) {
794 			return history->last_run_counter;
795 		}
796 	}
797 
798 	return 0;
799 }
800 
801 static uint64_t
802 get_last_busy_counter(uint64_t poller_id, uint64_t thread_id)
803 {
804 	struct run_counter_history *history;
805 
806 	TAILQ_FOREACH(history, &g_run_counter_history, link) {
807 		if ((history->poller_id == poller_id) && (history->thread_id == thread_id)) {
808 			return history->last_busy_counter;
809 		}
810 	}
811 
812 	return 0;
813 }
814 
815 static int
816 subsort_pollers(enum column_pollers_type sort_column, const void *p1, const void *p2)
817 {
818 	const struct rpc_poller_info *poller1 = (struct rpc_poller_info *)p1;
819 	const struct rpc_poller_info *poller2 = (struct rpc_poller_info *)p2;
820 	uint64_t count1, count2;
821 	uint64_t last_busy_counter1, last_busy_counter2;
822 
823 	switch (sort_column) {
824 	case COL_POLLERS_NAME:
825 		return strcmp(poller1->name, poller2->name);
826 	case COL_POLLERS_TYPE:
827 		return poller1->type - poller2->type;
828 	case COL_POLLERS_THREAD_NAME:
829 		return strcmp(poller1->thread_name, poller2->thread_name);
830 	case COL_POLLERS_RUN_COUNTER:
831 		if (g_interval_data) {
832 			count1 = poller1->run_count - get_last_run_counter(poller1->id, poller1->thread_id);
833 			count2 = poller2->run_count - get_last_run_counter(poller2->id, poller2->thread_id);
834 		} else {
835 			count1 = poller1->run_count;
836 			count2 = poller2->run_count;
837 		}
838 		break;
839 	case COL_POLLERS_PERIOD:
840 		count1 = poller1->period_ticks;
841 		count2 = poller2->period_ticks;
842 		break;
843 	case COL_POLLERS_BUSY_COUNT:
844 		count1 = poller1->busy_count;
845 		count2 = poller2->busy_count;
846 		if (g_interval_data) {
847 			last_busy_counter1 = get_last_busy_counter(poller1->id, poller1->thread_id);
848 			last_busy_counter2 = get_last_busy_counter(poller2->id, poller2->thread_id);
849 			if (count1 > last_busy_counter1) {
850 				count1 -= last_busy_counter1;
851 			}
852 			if (count2 > last_busy_counter2) {
853 				count2 -= last_busy_counter2;
854 			}
855 		}
856 		break;
857 	case COL_POLLERS_NONE:
858 	default:
859 		return 0;
860 	}
861 
862 	if (count2 > count1) {
863 		return 1;
864 	} else if (count2 < count1) {
865 		return -1;
866 	} else {
867 		return 0;
868 	}
869 }
870 
871 static int
872 sort_pollers(const void *p1, const void *p2)
873 {
874 	int rc;
875 
876 	rc = subsort_pollers(g_current_sort_col[POLLERS_TAB], p1, p2);
877 	if (rc == 0) {
878 		rc = subsort_pollers(g_current_sort_col2[POLLERS_TAB], p1, p2);
879 	}
880 	return rc;
881 }
882 
883 static int
884 get_pollers_data(void)
885 {
886 	struct spdk_jsonrpc_client_response *json_resp = NULL;
887 	int rc = 0;
888 	uint64_t i = 0;
889 	uint32_t current_pollers_count;
890 	struct rpc_poller_info pollers_info[RPC_MAX_POLLERS];
891 
892 	rc = rpc_send_req("thread_get_pollers", &json_resp);
893 	if (rc) {
894 		return rc;
895 	}
896 
897 	/* Decode json */
898 	memset(&pollers_info, 0, sizeof(pollers_info));
899 	if (rpc_decode_pollers_threads_array(json_resp->result, pollers_info, &current_pollers_count)) {
900 		rc = -EINVAL;
901 		for (i = 0; i < current_pollers_count; i++) {
902 			free_rpc_poller(&pollers_info[i]);
903 		}
904 		goto end;
905 	}
906 
907 	pthread_mutex_lock(&g_thread_lock);
908 
909 	/* Save last run counter of each poller before updating g_pollers_stats. */
910 	for (i = 0; i < g_last_pollers_count; i++) {
911 		store_last_counters(g_pollers_info[i].id, g_pollers_info[i].thread_id,
912 				    g_pollers_info[i].run_count, g_pollers_info[i].busy_count);
913 	}
914 
915 	/* Free old pollers values before allocating memory for new ones */
916 	for (i = 0; i < g_last_pollers_count; i++) {
917 		free_rpc_poller(&g_pollers_info[i]);
918 	}
919 
920 	g_last_pollers_count = current_pollers_count;
921 
922 	qsort(&pollers_info, g_last_pollers_count, sizeof(struct rpc_poller_info), sort_pollers);
923 
924 	memcpy(&g_pollers_info, &pollers_info, sizeof(struct rpc_poller_info) * g_last_pollers_count);
925 
926 	pthread_mutex_unlock(&g_thread_lock);
927 
928 end:
929 	spdk_jsonrpc_client_free_response(json_resp);
930 	return rc;
931 }
932 
933 static int
934 subsort_cores(enum column_cores_type sort_column, const void *p1, const void *p2)
935 {
936 	const struct rpc_core_info core_info1 = *(struct rpc_core_info *)p1;
937 	const struct rpc_core_info core_info2 = *(struct rpc_core_info *)p2;
938 	uint64_t count1, count2;
939 
940 	switch (sort_column) {
941 	case COL_CORES_CORE:
942 		count1 = core_info2.lcore;
943 		count2 = core_info1.lcore;
944 		break;
945 	case COL_CORES_THREADS:
946 		count1 = core_info1.threads.threads_count;
947 		count2 = core_info2.threads.threads_count;
948 		break;
949 	case COL_CORES_POLLERS:
950 		count1 = core_info1.pollers_count;
951 		count2 = core_info2.pollers_count;
952 		break;
953 	case COL_CORES_IDLE_TIME:
954 		if (g_interval_data) {
955 			count1 = core_info1.last_idle - core_info1.idle;
956 			count2 = core_info2.last_idle - core_info2.idle;
957 		} else {
958 			count1 = core_info1.idle;
959 			count2 = core_info2.idle;
960 		}
961 		break;
962 	case COL_CORES_BUSY_TIME:
963 		if (g_interval_data) {
964 			count1 = core_info1.last_busy - core_info1.busy;
965 			count2 = core_info2.last_busy - core_info2.busy;
966 		} else {
967 			count1 = core_info1.busy;
968 			count2 = core_info2.busy;
969 		}
970 		break;
971 	case COL_CORES_CORE_FREQ:
972 		count1 = core_info1.core_freq;
973 		count2 = core_info2.core_freq;
974 		break;
975 	case COL_CORES_INTR:
976 		count1 = core_info1.in_interrupt;
977 		count2 = core_info2.in_interrupt;
978 		break;
979 	case COL_CORES_BUSY_PCT:
980 		count1 = get_cpu_usage(core_info1.last_busy - core_info1.busy,
981 				       core_info1.last_idle - core_info1.idle);
982 		count2 = get_cpu_usage(core_info2.last_busy - core_info2.busy,
983 				       core_info2.last_idle - core_info2.idle);
984 		break;
985 	case COL_CORES_NONE:
986 	default:
987 		return 0;
988 	}
989 
990 	if (count2 > count1) {
991 		return 1;
992 	} else if (count2 < count1) {
993 		return -1;
994 	} else {
995 		return 0;
996 	}
997 }
998 
999 static int
1000 sort_cores(const void *p1, const void *p2)
1001 {
1002 	int rc;
1003 
1004 	rc = subsort_cores(g_current_sort_col[CORES_TAB], p1, p2);
1005 	if (rc == 0) {
1006 		return subsort_cores(g_current_sort_col[CORES_TAB], p1, p2);
1007 	}
1008 	return rc;
1009 }
1010 
1011 static int
1012 get_cores_data(void)
1013 {
1014 	struct spdk_jsonrpc_client_response *json_resp = NULL;
1015 	struct rpc_core_info *core_info;
1016 	uint64_t i, j, k;
1017 	uint32_t current_cores_count;
1018 	struct rpc_core_info cores_info[RPC_MAX_CORES];
1019 	int rc = 0;
1020 
1021 	rc = rpc_send_req("framework_get_reactors", &json_resp);
1022 	if (rc) {
1023 		return rc;
1024 	}
1025 
1026 	/* Decode json */
1027 	memset(cores_info, 0, sizeof(struct rpc_core_info) * RPC_MAX_CORES);
1028 	if (rpc_decode_cores_array(json_resp->result, cores_info, &current_cores_count)) {
1029 		rc = -EINVAL;
1030 		goto end;
1031 	}
1032 
1033 	pthread_mutex_lock(&g_thread_lock);
1034 	for (i = 0; i < current_cores_count; i++) {
1035 		for (j = 0; j < g_last_cores_count; j++) {
1036 			if (cores_info[i].lcore == g_cores_info[j].lcore) {
1037 				cores_info[i].last_busy = g_cores_info[j].busy;
1038 				cores_info[i].last_idle = g_cores_info[j].idle;
1039 			}
1040 		}
1041 	}
1042 
1043 	/* Free old cores values before allocating memory for new ones */
1044 	free_rpc_core_info(g_cores_info, current_cores_count);
1045 	memcpy(g_cores_info, cores_info, sizeof(struct rpc_core_info) * current_cores_count);
1046 
1047 	for (i = 0; i < g_last_cores_count; i++) {
1048 		core_info = &g_cores_info[i];
1049 
1050 		core_info->threads.thread = cores_info[i].threads.thread;
1051 
1052 		for (j = 0; j < core_info->threads.threads_count; j++) {
1053 			memcpy(&core_info->threads.thread[j], &cores_info[i].threads.thread[j],
1054 			       sizeof(struct rpc_core_thread_info));
1055 			for (k = 0; k < g_last_threads_count; k++) {
1056 				if (core_info->threads.thread[j].id == g_threads_info[k].id) {
1057 					g_threads_info[k].core_num = core_info->lcore;
1058 					/* Do not consider threads which changed cores when issuing
1059 					 * RPCs to get_core_data and get_thread_data and threads
1060 					 * not currently assigned to this core. */
1061 					core_info->pollers_count += g_threads_info[k].active_pollers_count +
1062 								    g_threads_info[k].timed_pollers_count +
1063 								    g_threads_info[k].paused_pollers_count;
1064 				}
1065 			}
1066 		}
1067 	}
1068 
1069 	g_last_cores_count = current_cores_count;
1070 
1071 	qsort(&g_cores_info, g_last_cores_count, sizeof(struct rpc_core_info), sort_cores);
1072 
1073 end:
1074 	pthread_mutex_unlock(&g_thread_lock);
1075 	spdk_jsonrpc_client_free_response(json_resp);
1076 	return rc;
1077 }
1078 
1079 static int
1080 get_scheduler_data(void)
1081 {
1082 	struct spdk_jsonrpc_client_response *json_resp = NULL;
1083 	struct rpc_scheduler scheduler_info;
1084 	int rc = 0;
1085 
1086 	rc = rpc_send_req("framework_get_scheduler", &json_resp);
1087 	if (rc) {
1088 		return rc;
1089 	}
1090 
1091 	memset(&scheduler_info, 0, sizeof(scheduler_info));
1092 	if (spdk_json_decode_object_relaxed(json_resp->result, rpc_scheduler_decoders,
1093 					    SPDK_COUNTOF(rpc_scheduler_decoders), &scheduler_info)) {
1094 		rc = -EINVAL;
1095 	} else {
1096 		pthread_mutex_lock(&g_thread_lock);
1097 
1098 		free_rpc_scheduler(&g_scheduler_info);
1099 
1100 		memcpy(&g_scheduler_info, &scheduler_info, sizeof(struct rpc_scheduler));
1101 		pthread_mutex_unlock(&g_thread_lock);
1102 	}
1103 
1104 	spdk_jsonrpc_client_free_response(json_resp);
1105 	return rc;
1106 }
1107 
1108 enum str_alignment {
1109 	ALIGN_LEFT,
1110 	ALIGN_RIGHT,
1111 };
1112 
1113 static void
1114 print_max_len(WINDOW *win, int row, uint16_t col, uint16_t max_len, enum str_alignment alignment,
1115 	      const char *string)
1116 {
1117 	const char dots[] = "...";
1118 	int DOTS_STR_LEN = sizeof(dots) / sizeof(dots[0]);
1119 	char tmp_str[MAX_STRING_LEN];
1120 	int len, max_col, max_str, cmp_len;
1121 	int max_row;
1122 
1123 	len = strlen(string);
1124 	getmaxyx(win, max_row, max_col);
1125 
1126 	if (row > max_row) {
1127 		/* We are in a process of resizing and this may happen */
1128 		return;
1129 	}
1130 
1131 	if (max_len != 0 && col + max_len < max_col) {
1132 		max_col = col + max_len;
1133 	}
1134 
1135 	max_str = max_col - col;
1136 
1137 	if (max_str <= DOTS_STR_LEN + 1) {
1138 		/* No space to print anything, but we have to let a user know about it */
1139 		mvwprintw(win, row, max_col - DOTS_STR_LEN - 1, "...");
1140 		wnoutrefresh(win);
1141 		return;
1142 	}
1143 
1144 	if (max_len) {
1145 		if (alignment == ALIGN_LEFT) {
1146 			snprintf(tmp_str, max_str, "%s%*c", string, max_len - len - 1, ' ');
1147 		} else {
1148 			snprintf(tmp_str, max_str, "%*c%s", max_len - len - 1, ' ', string);
1149 		}
1150 		cmp_len = max_len - 1;
1151 	} else {
1152 		snprintf(tmp_str, max_str, "%s", string);
1153 		cmp_len = len;
1154 	}
1155 
1156 	if (col + cmp_len > max_col - 1) {
1157 		snprintf(&tmp_str[max_str - DOTS_STR_LEN - 2], DOTS_STR_LEN, "%s", dots);
1158 	}
1159 
1160 	mvwprintw(win, row, col, "%s", tmp_str);
1161 
1162 	wnoutrefresh(win);
1163 }
1164 
1165 static void
1166 draw_menu_win(void)
1167 {
1168 	wbkgd(g_menu_win, COLOR_PAIR(2));
1169 	box(g_menu_win, 0, 0);
1170 	print_max_len(g_menu_win, 1, 1, 0, ALIGN_LEFT,
1171 		      "  [q] Quit  |  [1-3][Tab] Switch tab  |  [PgUp] Previous page  |  [PgDown] Next page  |  [Enter] Item details  |  [h] Help");
1172 }
1173 
1174 static void
1175 draw_tab_win(enum tabs tab)
1176 {
1177 	uint16_t col;
1178 	uint8_t white_spaces = TABS_SPACING * NUMBER_OF_TABS;
1179 
1180 	wbkgd(g_tab_win[tab], COLOR_PAIR(10));
1181 	box(g_tab_win[tab], 0, 0);
1182 
1183 	col = ((g_max_col - white_spaces) / NUMBER_OF_TABS / 2) - (strlen(g_tab_title[tab]) / 2) -
1184 	      TABS_SPACING;
1185 	print_max_len(g_tab_win[tab], 1, col, 0, ALIGN_LEFT, g_tab_title[tab]);
1186 }
1187 
1188 static void
1189 draw_tabs(enum tabs tab_index, uint8_t sort_col, uint8_t sort_col2)
1190 {
1191 	struct col_desc *col_desc = g_col_desc[tab_index];
1192 	WINDOW *tab = g_tabs[tab_index];
1193 	int i, j;
1194 	uint16_t offset, draw_offset;
1195 	uint16_t tab_height = g_max_row - MENU_WIN_HEIGHT - TAB_WIN_HEIGHT - 3;
1196 
1197 	for (i = 0; col_desc[i].name != NULL; i++) {
1198 		if (col_desc[i].disabled) {
1199 			continue;
1200 		}
1201 
1202 		offset = 1;
1203 		for (j = i; j != 0; j--) {
1204 			if (!col_desc[j - 1].disabled) {
1205 				offset += col_desc[j - 1].max_data_string;
1206 				offset += col_desc[j - 1].name_len % 2 + 1;
1207 			}
1208 		}
1209 
1210 		draw_offset = offset + (col_desc[i].max_data_string / 2) - (col_desc[i].name_len / 2);
1211 
1212 		if (i == sort_col) {
1213 			wattron(tab, COLOR_PAIR(4));
1214 			print_max_len(tab, 1, draw_offset, 0, ALIGN_LEFT, col_desc[i].name);
1215 			wattroff(tab, COLOR_PAIR(4));
1216 		} else if (i == sort_col2) {
1217 			wattron(tab, COLOR_PAIR(3));
1218 			print_max_len(tab, 1, draw_offset, 0, ALIGN_LEFT, col_desc[i].name);
1219 			wattroff(tab, COLOR_PAIR(3));
1220 		} else {
1221 			print_max_len(tab, 1, draw_offset, 0, ALIGN_LEFT, col_desc[i].name);
1222 		}
1223 
1224 		if (offset != 1) {
1225 			print_max_len(tab, 1, offset - 1, 0, ALIGN_LEFT, "|");
1226 		}
1227 	}
1228 
1229 	print_max_len(tab, 2, 1, 0, ALIGN_LEFT, ""); /* Move to next line */
1230 	whline(tab, ACS_HLINE, g_max_col - 2);
1231 
1232 	/* Border lines */
1233 	mvwhline(tab, 0, 1, ACS_HLINE, g_max_col - 2);
1234 	mvwhline(tab, tab_height, 1, ACS_HLINE, g_max_col - 2);
1235 
1236 	wrefresh(tab);
1237 }
1238 
1239 static void
1240 resize_interface(enum tabs tab)
1241 {
1242 	int i;
1243 
1244 	clear();
1245 	wclear(g_menu_win);
1246 	mvwin(g_menu_win, g_max_row - MENU_WIN_SPACING, MENU_WIN_LOCATION_COL);
1247 	wresize(g_menu_win, MENU_WIN_HEIGHT, g_max_col);
1248 	draw_menu_win();
1249 
1250 	for (i = 0; i < NUMBER_OF_TABS; i++) {
1251 		wclear(g_tabs[i]);
1252 		wresize(g_tabs[i], g_max_row - MENU_WIN_HEIGHT - TAB_WIN_HEIGHT - 2, g_max_col);
1253 		mvwin(g_tabs[i], TABS_LOCATION_ROW, TABS_LOCATION_COL);
1254 	}
1255 
1256 	draw_tabs(tab, g_current_sort_col[tab], g_current_sort_col2[tab]);
1257 
1258 	for (i = 0; i < NUMBER_OF_TABS; i++) {
1259 		wclear(g_tab_win[i]);
1260 		wresize(g_tab_win[i], TAB_WIN_HEIGHT,
1261 			(g_max_col - (TABS_SPACING * NUMBER_OF_TABS)) / NUMBER_OF_TABS);
1262 		mvwin(g_tab_win[i], TAB_WIN_LOCATION_ROW, 1 + (g_max_col / NUMBER_OF_TABS) * i);
1263 		draw_tab_win(i);
1264 	}
1265 
1266 	update_panels();
1267 	doupdate();
1268 }
1269 
1270 static void
1271 switch_tab(enum tabs tab)
1272 {
1273 	wclear(g_tabs[tab]);
1274 	draw_tabs(tab, g_current_sort_col[tab], g_current_sort_col2[tab]);
1275 	top_panel(g_panels[tab]);
1276 	update_panels();
1277 	doupdate();
1278 }
1279 
1280 static void
1281 get_time_str(uint64_t ticks, char *time_str)
1282 {
1283 	uint64_t time;
1284 
1285 	time = ticks * SPDK_SEC_TO_USEC / g_tick_rate;
1286 	snprintf(time_str, MAX_TIME_STR_LEN, "%" PRIu64, time);
1287 }
1288 
1289 static void
1290 draw_row_background(uint8_t item_index, uint8_t tab)
1291 {
1292 	int k;
1293 
1294 	if (item_index == g_selected_row) {
1295 		wattron(g_tabs[tab], COLOR_PAIR(2));
1296 	}
1297 	for (k = 1; k < g_max_col - 1; k++) {
1298 		mvwprintw(g_tabs[tab], TABS_DATA_START_ROW + item_index, k, " ");
1299 	}
1300 }
1301 
1302 static void
1303 get_cpu_usage_str(uint64_t busy_ticks, uint64_t total_ticks, char *cpu_str)
1304 {
1305 	if (total_ticks > 0) {
1306 		snprintf(cpu_str, MAX_FLOAT_STR_LEN, "%.2f",
1307 			 (double)(busy_ticks) * 100 / (double)(total_ticks));
1308 	} else {
1309 		cpu_str[0] = '\0';
1310 	}
1311 }
1312 
1313 static void
1314 draw_thread_tab_row(uint64_t current_row, uint8_t item_index)
1315 {
1316 	struct col_desc *col_desc = g_col_desc[THREADS_TAB];
1317 	uint16_t col = TABS_DATA_START_COL;
1318 	int core_idx, color_attr = COLOR_PAIR(6);
1319 	char pollers_number[MAX_POLLER_COUNT_STR_LEN], idle_time[MAX_TIME_STR_LEN],
1320 	     busy_time[MAX_TIME_STR_LEN], core_str[MAX_CORE_MASK_STR_LEN],
1321 	     cpu_usage[MAX_FLOAT_STR_LEN], *status_str;
1322 
1323 	if (!col_desc[COL_THREADS_NAME].disabled) {
1324 		print_max_len(g_tabs[THREADS_TAB], TABS_DATA_START_ROW + item_index, col,
1325 			      col_desc[COL_THREADS_NAME].max_data_string, ALIGN_LEFT, g_threads_info[current_row].name);
1326 		col += col_desc[COL_THREADS_NAME].max_data_string;
1327 	}
1328 
1329 	if (!col_desc[COL_THREADS_CORE].disabled) {
1330 		snprintf(core_str, MAX_CORE_STR_LEN, "%d", g_threads_info[current_row].core_num);
1331 		print_max_len(g_tabs[THREADS_TAB], TABS_DATA_START_ROW + item_index,
1332 			      col, col_desc[COL_THREADS_CORE].max_data_string, ALIGN_RIGHT, core_str);
1333 		col += col_desc[COL_THREADS_CORE].max_data_string + 2;
1334 	}
1335 
1336 	if (!col_desc[COL_THREADS_ACTIVE_POLLERS].disabled) {
1337 		snprintf(pollers_number, MAX_POLLER_COUNT_STR_LEN, "%ld",
1338 			 g_threads_info[current_row].active_pollers_count);
1339 		print_max_len(g_tabs[THREADS_TAB], TABS_DATA_START_ROW + item_index,
1340 			      col + (col_desc[COL_THREADS_ACTIVE_POLLERS].name_len / 2),
1341 			      col_desc[COL_THREADS_ACTIVE_POLLERS].max_data_string, ALIGN_LEFT, pollers_number);
1342 		col += col_desc[COL_THREADS_ACTIVE_POLLERS].max_data_string + 2;
1343 	}
1344 
1345 	if (!col_desc[COL_THREADS_TIMED_POLLERS].disabled) {
1346 		snprintf(pollers_number, MAX_POLLER_COUNT_STR_LEN, "%ld",
1347 			 g_threads_info[current_row].timed_pollers_count);
1348 		print_max_len(g_tabs[THREADS_TAB], TABS_DATA_START_ROW + item_index,
1349 			      col + (col_desc[COL_THREADS_TIMED_POLLERS].name_len / 2),
1350 			      col_desc[COL_THREADS_TIMED_POLLERS].max_data_string, ALIGN_LEFT, pollers_number);
1351 		col += col_desc[COL_THREADS_TIMED_POLLERS].max_data_string + 1;
1352 	}
1353 
1354 	if (!col_desc[COL_THREADS_PAUSED_POLLERS].disabled) {
1355 		snprintf(pollers_number, MAX_POLLER_COUNT_STR_LEN, "%ld",
1356 			 g_threads_info[current_row].paused_pollers_count);
1357 		print_max_len(g_tabs[THREADS_TAB], TABS_DATA_START_ROW + item_index,
1358 			      col + (col_desc[COL_THREADS_PAUSED_POLLERS].name_len / 2),
1359 			      col_desc[COL_THREADS_PAUSED_POLLERS].max_data_string, ALIGN_LEFT, pollers_number);
1360 		col += col_desc[COL_THREADS_PAUSED_POLLERS].max_data_string + 2;
1361 	}
1362 
1363 	uint64_t idle_period = g_threads_info[current_row].idle - g_threads_info[current_row].last_idle;
1364 	uint64_t busy_period = g_threads_info[current_row].busy - g_threads_info[current_row].last_busy;
1365 	if (!col_desc[COL_THREADS_IDLE_TIME].disabled) {
1366 		if (g_interval_data == true) {
1367 			get_time_str(idle_period, idle_time);
1368 		} else {
1369 			get_time_str(g_threads_info[current_row].idle, idle_time);
1370 		}
1371 		print_max_len(g_tabs[THREADS_TAB], TABS_DATA_START_ROW + item_index, col,
1372 			      col_desc[COL_THREADS_IDLE_TIME].max_data_string, ALIGN_RIGHT, idle_time);
1373 		col += col_desc[COL_THREADS_IDLE_TIME].max_data_string;
1374 	}
1375 
1376 	if (!col_desc[COL_THREADS_BUSY_TIME].disabled) {
1377 		if (g_interval_data == true) {
1378 			get_time_str(busy_period, busy_time);
1379 		} else {
1380 			get_time_str(g_threads_info[current_row].busy, busy_time);
1381 		}
1382 		print_max_len(g_tabs[THREADS_TAB], TABS_DATA_START_ROW + item_index, col,
1383 			      col_desc[COL_THREADS_BUSY_TIME].max_data_string, ALIGN_RIGHT, busy_time);
1384 		col += col_desc[COL_THREADS_BUSY_TIME].max_data_string + 3;
1385 	}
1386 
1387 	if (!col_desc[COL_THREADS_CPU_USAGE].disabled) {
1388 		core_idx = g_threads_info[current_row].core_idx;
1389 		if (core_idx >= 0 && core_idx < RPC_MAX_CORES) {
1390 			uint64_t core_busy_period = g_cores_info[core_idx].busy - g_cores_info[core_idx].last_busy;
1391 			uint64_t core_idle_period = g_cores_info[core_idx].idle - g_cores_info[core_idx].last_idle;
1392 			get_cpu_usage_str(busy_period, core_busy_period + core_idle_period, cpu_usage);
1393 		} else {
1394 			snprintf(cpu_usage, sizeof(cpu_usage), "n/a");
1395 		}
1396 
1397 		print_max_len(g_tabs[THREADS_TAB], TABS_DATA_START_ROW + item_index, col,
1398 			      col_desc[COL_THREADS_CPU_USAGE].max_data_string, ALIGN_RIGHT, cpu_usage);
1399 		col += col_desc[COL_THREADS_CPU_USAGE].max_data_string + 2;
1400 	}
1401 
1402 	if (!col_desc[COL_THREADS_STATUS].disabled) {
1403 		if (busy_period > idle_period) {
1404 			if (item_index != g_selected_row) {
1405 				color_attr = COLOR_PAIR(6);
1406 			} else {
1407 				color_attr = COLOR_PAIR(8);
1408 			}
1409 			status_str = "Busy";
1410 		} else {
1411 			if (item_index != g_selected_row) {
1412 				color_attr = COLOR_PAIR(7);
1413 			} else {
1414 				color_attr = COLOR_PAIR(9);
1415 			}
1416 			status_str = "Idle";
1417 		}
1418 		wattron(g_tabs[THREADS_TAB], color_attr);
1419 		print_max_len(g_tabs[THREADS_TAB], TABS_DATA_START_ROW + item_index, col,
1420 			      col_desc[COL_THREADS_STATUS].max_data_string, ALIGN_RIGHT, status_str);
1421 		wattroff(g_tabs[THREADS_TAB], color_attr);
1422 	}
1423 }
1424 
1425 static uint8_t
1426 refresh_threads_tab(uint8_t current_page)
1427 {
1428 	uint64_t i, j, threads_count;
1429 	uint16_t empty_col = 0;
1430 	uint8_t max_pages, item_index;
1431 
1432 	threads_count = g_last_threads_count;
1433 
1434 	max_pages = (threads_count + g_max_data_rows - 1) / g_max_data_rows;
1435 
1436 	for (i = current_page * g_max_data_rows;
1437 	     i < (uint64_t)((current_page + 1) * g_max_data_rows);
1438 	     i++) {
1439 		item_index = i - (current_page * g_max_data_rows);
1440 
1441 		/* When number of threads decreases, this will print spaces in places
1442 		 * where non existent threads were previously displayed. */
1443 		if (i >= threads_count) {
1444 			for (j = 1; j < (uint64_t)g_max_col - 1; j++) {
1445 				mvwprintw(g_tabs[THREADS_TAB], item_index + TABS_DATA_START_ROW, j, " ");
1446 			}
1447 
1448 			empty_col++;
1449 			continue;
1450 		}
1451 
1452 		draw_row_background(item_index, THREADS_TAB);
1453 		draw_thread_tab_row(i, item_index);
1454 
1455 		if (item_index == g_selected_row) {
1456 			wattroff(g_tabs[THREADS_TAB], COLOR_PAIR(2));
1457 		}
1458 	}
1459 
1460 	g_max_selected_row = i - current_page * g_max_data_rows - 1 - empty_col;
1461 
1462 	return max_pages;
1463 }
1464 
1465 static void
1466 draw_poller_tab_row(uint64_t current_row, uint8_t item_index)
1467 {
1468 	struct col_desc *col_desc = g_col_desc[POLLERS_TAB];
1469 	uint64_t last_run_counter, last_busy_counter;
1470 	uint16_t col = TABS_DATA_START_COL;
1471 	char run_count[MAX_POLLER_RUN_COUNT], period_ticks[MAX_PERIOD_STR_LEN],
1472 	     status[MAX_POLLER_IND_STR_LEN];
1473 
1474 	last_busy_counter = get_last_busy_counter(g_pollers_info[current_row].id,
1475 			    g_pollers_info[current_row].thread_id);
1476 
1477 	if (!col_desc[COL_POLLERS_NAME].disabled) {
1478 		print_max_len(g_tabs[POLLERS_TAB], TABS_DATA_START_ROW + item_index, col + 1,
1479 			      col_desc[COL_POLLERS_NAME].max_data_string, ALIGN_LEFT, g_pollers_info[current_row].name);
1480 		col += col_desc[COL_POLLERS_NAME].max_data_string + 2;
1481 	}
1482 
1483 	if (!col_desc[COL_POLLERS_TYPE].disabled) {
1484 		print_max_len(g_tabs[POLLERS_TAB], TABS_DATA_START_ROW + item_index, col,
1485 			      col_desc[COL_POLLERS_TYPE].max_data_string, ALIGN_LEFT,
1486 			      poller_type_str[g_pollers_info[current_row].type]);
1487 		col += col_desc[COL_POLLERS_TYPE].max_data_string + 2;
1488 	}
1489 
1490 	if (!col_desc[COL_POLLERS_THREAD_NAME].disabled) {
1491 		print_max_len(g_tabs[POLLERS_TAB], TABS_DATA_START_ROW + item_index, col,
1492 			      col_desc[COL_POLLERS_THREAD_NAME].max_data_string, ALIGN_LEFT,
1493 			      g_pollers_info[current_row].thread_name);
1494 		col += col_desc[COL_POLLERS_THREAD_NAME].max_data_string + 1;
1495 	}
1496 
1497 	if (!col_desc[COL_POLLERS_RUN_COUNTER].disabled) {
1498 		last_run_counter = get_last_run_counter(g_pollers_info[current_row].id,
1499 							g_pollers_info[current_row].thread_id);
1500 		if (g_interval_data == true) {
1501 			snprintf(run_count, MAX_POLLER_RUN_COUNT, "%" PRIu64,
1502 				 g_pollers_info[current_row].run_count - last_run_counter);
1503 		} else {
1504 			snprintf(run_count, MAX_POLLER_RUN_COUNT, "%" PRIu64, g_pollers_info[current_row].run_count);
1505 		}
1506 		print_max_len(g_tabs[POLLERS_TAB], TABS_DATA_START_ROW + item_index, col,
1507 			      col_desc[COL_POLLERS_RUN_COUNTER].max_data_string, ALIGN_RIGHT, run_count);
1508 		col += col_desc[COL_POLLERS_RUN_COUNTER].max_data_string;
1509 	}
1510 
1511 	if (!col_desc[COL_POLLERS_PERIOD].disabled) {
1512 		if (g_pollers_info[current_row].period_ticks != 0) {
1513 			get_time_str(g_pollers_info[current_row].period_ticks, period_ticks);
1514 			print_max_len(g_tabs[POLLERS_TAB], TABS_DATA_START_ROW + item_index, col,
1515 				      col_desc[COL_POLLERS_PERIOD].max_data_string, ALIGN_RIGHT, period_ticks);
1516 		}
1517 		col += col_desc[COL_POLLERS_PERIOD].max_data_string + 7;
1518 	}
1519 
1520 	if (!col_desc[COL_POLLERS_BUSY_COUNT].disabled) {
1521 		if (g_pollers_info[current_row].busy_count > last_busy_counter) {
1522 			if (g_interval_data == true) {
1523 				snprintf(status, MAX_POLLER_IND_STR_LEN, "Busy (%" PRIu64 ")",
1524 					 g_pollers_info[current_row].busy_count - last_busy_counter);
1525 			} else {
1526 				snprintf(status, MAX_POLLER_IND_STR_LEN, "Busy (%" PRIu64 ")",
1527 					 g_pollers_info[current_row].busy_count);
1528 			}
1529 
1530 			if (item_index != g_selected_row) {
1531 				wattron(g_tabs[POLLERS_TAB], COLOR_PAIR(6));
1532 				print_max_len(g_tabs[POLLERS_TAB], TABS_DATA_START_ROW + item_index, col,
1533 					      col_desc[COL_POLLERS_BUSY_COUNT].max_data_string, ALIGN_LEFT, status);
1534 				wattroff(g_tabs[POLLERS_TAB], COLOR_PAIR(6));
1535 			} else {
1536 				wattron(g_tabs[POLLERS_TAB], COLOR_PAIR(8));
1537 				print_max_len(g_tabs[POLLERS_TAB], TABS_DATA_START_ROW + item_index, col,
1538 					      col_desc[COL_POLLERS_BUSY_COUNT].max_data_string, ALIGN_LEFT, status);
1539 				wattroff(g_tabs[POLLERS_TAB], COLOR_PAIR(8));
1540 			}
1541 		} else {
1542 			if (g_interval_data == true) {
1543 				snprintf(status, MAX_POLLER_IND_STR_LEN, "%s", "Idle");
1544 			} else {
1545 				snprintf(status, MAX_POLLER_IND_STR_LEN, "Idle (%" PRIu64 ")",
1546 					 g_pollers_info[current_row].busy_count);
1547 			}
1548 
1549 			if (item_index != g_selected_row) {
1550 				wattron(g_tabs[POLLERS_TAB], COLOR_PAIR(7));
1551 				print_max_len(g_tabs[POLLERS_TAB], TABS_DATA_START_ROW + item_index, col,
1552 					      col_desc[COL_POLLERS_BUSY_COUNT].max_data_string, ALIGN_LEFT, status);
1553 				wattroff(g_tabs[POLLERS_TAB], COLOR_PAIR(7));
1554 			} else {
1555 				wattron(g_tabs[POLLERS_TAB], COLOR_PAIR(9));
1556 				print_max_len(g_tabs[POLLERS_TAB], TABS_DATA_START_ROW + item_index, col,
1557 					      col_desc[COL_POLLERS_BUSY_COUNT].max_data_string, ALIGN_LEFT, status);
1558 				wattroff(g_tabs[POLLERS_TAB], COLOR_PAIR(9));
1559 			}
1560 		}
1561 	}
1562 }
1563 
1564 static uint8_t
1565 refresh_pollers_tab(uint8_t current_page)
1566 {
1567 	uint64_t i, j;
1568 	uint16_t empty_col = 0;
1569 	uint8_t max_pages, item_index;
1570 
1571 	max_pages = (g_last_pollers_count + g_max_data_rows - 1) / g_max_data_rows;
1572 
1573 	/* Display info */
1574 	for (i = current_page * g_max_data_rows;
1575 	     i < (uint64_t)((current_page + 1) * g_max_data_rows);
1576 	     i++) {
1577 		item_index = i - (current_page * g_max_data_rows);
1578 
1579 		/* When number of pollers decreases, this will print spaces in places
1580 		 * where non existent pollers were previously displayed. */
1581 		if (i >= g_last_pollers_count) {
1582 			for (j = 1; j < (uint64_t)g_max_col - 1; j++) {
1583 				mvwprintw(g_tabs[POLLERS_TAB], item_index + TABS_DATA_START_ROW, j, " ");
1584 			}
1585 
1586 			empty_col++;
1587 			continue;
1588 		}
1589 
1590 		draw_row_background(item_index, POLLERS_TAB);
1591 		draw_poller_tab_row(i, item_index);
1592 
1593 		if (item_index == g_selected_row) {
1594 			wattroff(g_tabs[POLLERS_TAB], COLOR_PAIR(2));
1595 		}
1596 	}
1597 
1598 	g_max_selected_row = i - current_page * g_max_data_rows - 1 - empty_col;
1599 
1600 	return max_pages;
1601 }
1602 
1603 static void
1604 draw_core_tab_row(uint64_t current_row, uint8_t item_index)
1605 {
1606 	struct col_desc *col_desc = g_col_desc[CORES_TAB];
1607 	uint16_t col = 1;
1608 	int color_attr = COLOR_PAIR(6);
1609 	char core[MAX_CORE_STR_LEN], threads_number[MAX_THREAD_COUNT_STR_LEN], cpu_usage[MAX_FLOAT_STR_LEN],
1610 	     pollers_number[MAX_POLLER_COUNT_STR_LEN], idle_time[MAX_TIME_STR_LEN],
1611 	     busy_time[MAX_TIME_STR_LEN], core_freq[MAX_CORE_FREQ_STR_LEN],
1612 	     in_interrupt[MAX_INTR_LEN], *status_str;
1613 
1614 	snprintf(threads_number, MAX_THREAD_COUNT_STR_LEN, "%ld",
1615 		 g_cores_info[current_row].threads.threads_count);
1616 	snprintf(pollers_number, MAX_POLLER_COUNT_STR_LEN, "%ld", g_cores_info[current_row].pollers_count);
1617 
1618 	if (!col_desc[COL_CORES_CORE].disabled) {
1619 		snprintf(core, MAX_CORE_STR_LEN, "%d", g_cores_info[current_row].lcore);
1620 		print_max_len(g_tabs[CORES_TAB], TABS_DATA_START_ROW + item_index, col,
1621 			      col_desc[COL_CORES_CORE].max_data_string, ALIGN_RIGHT, core);
1622 		col += col_desc[COL_CORES_CORE].max_data_string + 2;
1623 	}
1624 
1625 	if (!col_desc[COL_CORES_THREADS].disabled) {
1626 		print_max_len(g_tabs[CORES_TAB], TABS_DATA_START_ROW + item_index,
1627 			      col + (col_desc[COL_CORES_THREADS].name_len / 2), col_desc[COL_CORES_THREADS].max_data_string,
1628 			      ALIGN_LEFT, threads_number);
1629 		col += col_desc[COL_CORES_THREADS].max_data_string + 2;
1630 	}
1631 
1632 	if (!col_desc[COL_CORES_POLLERS].disabled) {
1633 		print_max_len(g_tabs[CORES_TAB], TABS_DATA_START_ROW + item_index,
1634 			      col + (col_desc[COL_CORES_POLLERS].name_len / 2), col_desc[COL_CORES_POLLERS].max_data_string,
1635 			      ALIGN_LEFT, pollers_number);
1636 		col += col_desc[COL_CORES_POLLERS].max_data_string;
1637 	}
1638 
1639 	uint64_t idle_period = g_cores_info[current_row].idle - g_cores_info[current_row].last_idle;
1640 	uint64_t busy_period = g_cores_info[current_row].busy - g_cores_info[current_row].last_busy;
1641 	if (!col_desc[COL_CORES_IDLE_TIME].disabled) {
1642 		if (g_interval_data == true) {
1643 			get_time_str(idle_period, idle_time);
1644 		} else {
1645 			get_time_str(g_cores_info[current_row].idle, idle_time);
1646 		}
1647 		print_max_len(g_tabs[CORES_TAB], TABS_DATA_START_ROW + item_index, col,
1648 			      col_desc[COL_CORES_IDLE_TIME].max_data_string, ALIGN_RIGHT, idle_time);
1649 		col += col_desc[COL_CORES_IDLE_TIME].max_data_string + 2;
1650 	}
1651 
1652 	if (!col_desc[COL_CORES_BUSY_TIME].disabled) {
1653 		if (g_interval_data == true) {
1654 			get_time_str(busy_period, busy_time);
1655 		} else {
1656 			get_time_str(g_cores_info[current_row].busy, busy_time);
1657 		}
1658 		print_max_len(g_tabs[CORES_TAB], TABS_DATA_START_ROW + item_index, col,
1659 			      col_desc[COL_CORES_BUSY_TIME].max_data_string, ALIGN_RIGHT, busy_time);
1660 		col += col_desc[COL_CORES_BUSY_TIME].max_data_string + 2;
1661 	}
1662 
1663 	if (!col_desc[COL_CORES_BUSY_PCT].disabled) {
1664 		get_cpu_usage_str(busy_period, busy_period + idle_period, cpu_usage);
1665 		print_max_len(g_tabs[CORES_TAB], TABS_DATA_START_ROW + item_index, col,
1666 			      col_desc[COL_CORES_BUSY_PCT].max_data_string, ALIGN_RIGHT, cpu_usage);
1667 		col += col_desc[COL_CORES_BUSY_PCT].max_data_string + 1;
1668 	}
1669 
1670 	if (!col_desc[COL_CORES_STATUS].disabled) {
1671 		if (busy_period > idle_period) {
1672 			if (item_index != g_selected_row) {
1673 				color_attr = COLOR_PAIR(6);
1674 			} else {
1675 				color_attr = COLOR_PAIR(8);
1676 			}
1677 			status_str = "Busy";
1678 		} else {
1679 			if (item_index != g_selected_row) {
1680 				color_attr = COLOR_PAIR(7);
1681 			} else {
1682 				color_attr = COLOR_PAIR(9);
1683 			}
1684 			status_str = "Idle";
1685 		}
1686 		wattron(g_tabs[CORES_TAB], color_attr);
1687 		print_max_len(g_tabs[CORES_TAB], TABS_DATA_START_ROW + item_index, col,
1688 			      col_desc[COL_CORES_STATUS].max_data_string, ALIGN_RIGHT, status_str);
1689 		wattroff(g_tabs[CORES_TAB], color_attr);
1690 		if (item_index == g_selected_row) {
1691 			wattron(g_tabs[CORES_TAB], COLOR_PAIR(2));
1692 		} else {
1693 			wattron(g_tabs[CORES_TAB], COLOR_PAIR(0));
1694 		}
1695 		col += col_desc[COL_CORES_STATUS].max_data_string + 1;
1696 	}
1697 
1698 	if (!col_desc[COL_CORES_INTR].disabled) {
1699 		snprintf(in_interrupt, MAX_INTR_LEN, "%s",
1700 			 g_cores_info[current_row].in_interrupt ? "Yes" : "No");
1701 		print_max_len(g_tabs[CORES_TAB], TABS_DATA_START_ROW + item_index,
1702 			      col + (col_desc[COL_CORES_INTR].name_len / 2),
1703 			      col_desc[COL_CORES_INTR].max_data_string,
1704 			      ALIGN_LEFT, in_interrupt);
1705 		col += col_desc[COL_CORES_INTR].max_data_string + 1;
1706 	}
1707 
1708 	if (!col_desc[COL_CORES_CORE_FREQ].disabled) {
1709 		if (!g_cores_info[current_row].core_freq) {
1710 			snprintf(core_freq, MAX_CORE_FREQ_STR_LEN, "%s", "N/A");
1711 		} else {
1712 			snprintf(core_freq, MAX_CORE_FREQ_STR_LEN, "%" PRIu32,
1713 				 g_cores_info[current_row].core_freq);
1714 		}
1715 		print_max_len(g_tabs[CORES_TAB], TABS_DATA_START_ROW + item_index, col,
1716 			      col_desc[COL_CORES_CORE_FREQ].max_data_string, ALIGN_RIGHT, core_freq);
1717 	}
1718 }
1719 
1720 static uint8_t
1721 refresh_cores_tab(uint8_t current_page)
1722 {
1723 	uint64_t i;
1724 	uint16_t count = 0;
1725 	uint8_t max_pages, item_index;
1726 
1727 	count = g_last_cores_count;
1728 
1729 	max_pages = (count + g_max_row - WINDOW_HEADER - 1) / (g_max_row - WINDOW_HEADER);
1730 
1731 	for (i = current_page * g_max_data_rows;
1732 	     i < spdk_min(count, (uint64_t)((current_page + 1) * g_max_data_rows));
1733 	     i++) {
1734 		item_index = i - (current_page * g_max_data_rows);
1735 
1736 		draw_row_background(item_index, CORES_TAB);
1737 		draw_core_tab_row(i, item_index);
1738 
1739 		if (item_index == g_selected_row) {
1740 			wattroff(g_tabs[CORES_TAB], COLOR_PAIR(2));
1741 		}
1742 	}
1743 
1744 	g_max_selected_row = i - current_page * g_max_data_rows - 1;
1745 
1746 	return max_pages;
1747 }
1748 
1749 static uint8_t
1750 refresh_tab(enum tabs tab, uint8_t current_page)
1751 {
1752 	uint8_t (*refresh_function[NUMBER_OF_TABS])(uint8_t current_page) = {refresh_threads_tab, refresh_pollers_tab, refresh_cores_tab};
1753 	int color_pair[NUMBER_OF_TABS] = {COLOR_PAIR(2), COLOR_PAIR(2), COLOR_PAIR(2)};
1754 	int i;
1755 	uint8_t max_pages = 0;
1756 
1757 	color_pair[tab] = COLOR_PAIR(1);
1758 
1759 	for (i = 0; i < NUMBER_OF_TABS; i++) {
1760 		wbkgd(g_tab_win[i], color_pair[i]);
1761 	}
1762 
1763 	max_pages = (*refresh_function[tab])(current_page);
1764 
1765 	for (i = 0; i < NUMBER_OF_TABS; i++) {
1766 		wnoutrefresh(g_tab_win[i]);
1767 	}
1768 	draw_menu_win();
1769 
1770 	return max_pages;
1771 }
1772 
1773 static void
1774 print_in_middle(WINDOW *win, int starty, int startx, int width, char *string, chtype color)
1775 {
1776 	int length, temp;
1777 
1778 	length = strlen(string);
1779 	temp = (width - length) / 2;
1780 	wattron(win, color);
1781 	mvwprintw(win, starty, startx + temp, "%s", string);
1782 	wattroff(win, color);
1783 }
1784 
1785 static void
1786 print_left(WINDOW *win, int starty, int startx, int width, const char *string, chtype color)
1787 {
1788 	wattron(win, color);
1789 	mvwprintw(win, starty, startx, "%s", string);
1790 	wattroff(win, color);
1791 }
1792 
1793 static void
1794 apply_filters(enum tabs tab)
1795 {
1796 	wclear(g_tabs[tab]);
1797 	draw_tabs(tab, g_current_sort_col[tab], g_current_sort_col2[tab]);
1798 }
1799 
1800 static ITEM **
1801 draw_filtering_menu(uint8_t position, WINDOW *filter_win, uint8_t tab, MENU **my_menu)
1802 {
1803 	const int ADDITIONAL_ELEMENTS = 3;
1804 	const int ROW_PADDING = 6;
1805 	const int WINDOW_START_X = 1;
1806 	const int WINDOW_START_Y = 3;
1807 	const int WINDOW_COLUMNS = 2;
1808 	struct col_desc *col_desc = g_col_desc[tab];
1809 	ITEM **my_items;
1810 	MENU *menu;
1811 	int i, elements;
1812 	uint8_t len = 0;
1813 
1814 	for (i = 0; col_desc[i].name != NULL; ++i) {
1815 		len = spdk_max(col_desc[i].name_len, len);
1816 	}
1817 
1818 	elements = i;
1819 
1820 	my_items = (ITEM **)calloc(elements * WINDOW_COLUMNS + ADDITIONAL_ELEMENTS, sizeof(ITEM *));
1821 	if (my_items == NULL) {
1822 		fprintf(stderr, "Unable to allocate an item list in draw_filtering_menu.\n");
1823 		return NULL;
1824 	}
1825 
1826 	for (i = 0; i < elements * 2; i++) {
1827 		my_items[i] = new_item(col_desc[i / WINDOW_COLUMNS].name, NULL);
1828 		i++;
1829 		my_items[i] = new_item(col_desc[i / WINDOW_COLUMNS].disabled ? "[ ]" : "[*]", NULL);
1830 	}
1831 
1832 	my_items[i] = new_item("     CLOSE", NULL);
1833 	set_item_userptr(my_items[i], apply_filters);
1834 
1835 	menu = new_menu((ITEM **)my_items);
1836 
1837 	menu_opts_off(menu, O_SHOWDESC);
1838 	set_menu_format(menu, elements + 1, WINDOW_COLUMNS);
1839 
1840 	set_menu_win(menu, filter_win);
1841 	set_menu_sub(menu, derwin(filter_win, elements + 1, len + ROW_PADDING, WINDOW_START_Y,
1842 				  WINDOW_START_X));
1843 
1844 	*my_menu = menu;
1845 
1846 	post_menu(menu);
1847 	refresh();
1848 	wrefresh(filter_win);
1849 
1850 	for (i = 0; i < position / WINDOW_COLUMNS; i++) {
1851 		menu_driver(menu, REQ_DOWN_ITEM);
1852 	}
1853 
1854 	return my_items;
1855 }
1856 
1857 static void
1858 delete_filtering_menu(MENU *my_menu, ITEM **my_items, uint8_t elements)
1859 {
1860 	int i;
1861 
1862 	unpost_menu(my_menu);
1863 	free_menu(my_menu);
1864 	for (i = 0; i < elements * 2 + 2; ++i) {
1865 		free_item(my_items[i]);
1866 	}
1867 	free(my_items);
1868 }
1869 
1870 static ITEM **
1871 refresh_filtering_menu(MENU **my_menu, WINDOW *filter_win, uint8_t tab, ITEM **my_items,
1872 		       uint8_t elements, uint8_t position)
1873 {
1874 	delete_filtering_menu(*my_menu, my_items, elements);
1875 	return draw_filtering_menu(position, filter_win, tab, my_menu);
1876 }
1877 
1878 static void
1879 filter_columns(uint8_t tab)
1880 {
1881 	const int WINDOW_HEADER_LEN = 5;
1882 	const int WINDOW_BORDER_LEN = 8;
1883 	const int WINDOW_HEADER_END_LINE = 2;
1884 	const int WINDOW_COLUMNS = 2;
1885 	struct col_desc *col_desc = g_col_desc[tab];
1886 	PANEL *filter_panel;
1887 	WINDOW *filter_win;
1888 	ITEM **my_items;
1889 	MENU *my_menu = NULL;
1890 	int i, c, elements;
1891 	bool stop_loop = false;
1892 	ITEM *cur;
1893 	void (*p)(enum tabs tab);
1894 	uint8_t current_index, len = 0;
1895 	bool disabled[TABS_COL_COUNT];
1896 
1897 	for (i = 0; col_desc[i].name != NULL; ++i) {
1898 		len = spdk_max(col_desc[i].name_len, len);
1899 	}
1900 
1901 	elements = i;
1902 
1903 	filter_win = newwin(elements + WINDOW_HEADER_LEN, len + WINDOW_BORDER_LEN,
1904 			    (g_max_row - elements - 1) / 2, (g_max_col - len) / 2);
1905 	assert(filter_win != NULL);
1906 	keypad(filter_win, TRUE);
1907 	filter_panel = new_panel(filter_win);
1908 	assert(filter_panel != NULL);
1909 
1910 	top_panel(filter_panel);
1911 	update_panels();
1912 	doupdate();
1913 
1914 	box(filter_win, 0, 0);
1915 
1916 	print_in_middle(filter_win, 1, 0, len + WINDOW_BORDER_LEN, "Filtering", COLOR_PAIR(3));
1917 	mvwaddch(filter_win, WINDOW_HEADER_END_LINE, 0, ACS_LTEE);
1918 	mvwhline(filter_win, WINDOW_HEADER_END_LINE, 1, ACS_HLINE, len + WINDOW_BORDER_LEN - 2);
1919 	mvwaddch(filter_win, WINDOW_HEADER_END_LINE, len + WINDOW_BORDER_LEN - 1, ACS_RTEE);
1920 
1921 	my_items = draw_filtering_menu(0, filter_win, tab, &my_menu);
1922 	if (my_items == NULL || my_menu == NULL) {
1923 		goto fail;
1924 	}
1925 
1926 	for (int i = 0; i < TABS_COL_COUNT; i++) {
1927 		disabled[i] = col_desc[i].disabled;
1928 	}
1929 
1930 	while (!stop_loop) {
1931 		c = wgetch(filter_win);
1932 
1933 		switch (c) {
1934 		case KEY_DOWN:
1935 			menu_driver(my_menu, REQ_DOWN_ITEM);
1936 			break;
1937 		case KEY_UP:
1938 			menu_driver(my_menu, REQ_UP_ITEM);
1939 			break;
1940 		case 27: /* ESC */
1941 		case 'q':
1942 			for (int i = 0; i < TABS_COL_COUNT; i++) {
1943 				cur = current_item(my_menu);
1944 				col_desc[i].disabled = disabled[i];
1945 
1946 				my_items = refresh_filtering_menu(&my_menu, filter_win, tab, my_items, elements,
1947 								  item_index(cur) + 1);
1948 				if (my_items == NULL || my_menu == NULL) {
1949 					goto fail;
1950 				}
1951 			}
1952 
1953 			stop_loop = true;
1954 			break;
1955 		case ' ': /* Space */
1956 			cur = current_item(my_menu);
1957 			current_index = item_index(cur) / WINDOW_COLUMNS;
1958 			col_desc[current_index].disabled = !col_desc[current_index].disabled;
1959 			my_items = refresh_filtering_menu(&my_menu, filter_win, tab, my_items, elements,
1960 							  item_index(cur) + 1);
1961 			if (my_items == NULL || my_menu == NULL) {
1962 				goto fail;
1963 			}
1964 			break;
1965 		case 10: /* Enter */
1966 			cur = current_item(my_menu);
1967 			current_index = item_index(cur) / WINDOW_COLUMNS;
1968 			if (current_index == elements) {
1969 				stop_loop = true;
1970 				p = item_userptr(cur);
1971 				p(tab);
1972 			} else {
1973 				col_desc[current_index].disabled = !col_desc[current_index].disabled;
1974 				my_items = refresh_filtering_menu(&my_menu, filter_win, tab, my_items, elements,
1975 								  item_index(cur) + 1);
1976 				if (my_items == NULL || my_menu == NULL) {
1977 					goto fail;
1978 				}
1979 			}
1980 			break;
1981 		}
1982 		wrefresh(filter_win);
1983 	}
1984 
1985 	delete_filtering_menu(my_menu, my_items, elements);
1986 
1987 	del_panel(filter_panel);
1988 	delwin(filter_win);
1989 
1990 	wclear(g_menu_win);
1991 	draw_menu_win();
1992 	return;
1993 
1994 fail:
1995 	fprintf(stderr, "Unable to filter the columns due to allocation failure.\n");
1996 	assert(false);
1997 }
1998 
1999 static void
2000 sort_type(enum tabs tab, int item_index)
2001 {
2002 	g_current_sort_col[tab] = item_index;
2003 }
2004 
2005 static void
2006 sort_type2(enum tabs tab, int item_index)
2007 {
2008 	g_current_sort_col2[tab] = item_index;
2009 }
2010 
2011 static void
2012 change_sorting(uint8_t tab, int winnum, bool *pstop_loop)
2013 {
2014 	const int WINDOW_HEADER_LEN = 4;
2015 	const int WINDOW_BORDER_LEN = 3;
2016 	const int WINDOW_START_X = 1;
2017 	const int WINDOW_START_Y = 4;
2018 	const int WINDOW_HEADER_END_LINE = 3;
2019 	const int WINDOW_MIN_WIDTH = 31;
2020 	PANEL *sort_panel;
2021 	WINDOW *sort_win;
2022 	ITEM **my_items;
2023 	MENU *my_menu;
2024 	int i, c, elements;
2025 	bool stop_loop = false;
2026 	ITEM *cur;
2027 	char *name;
2028 	char *help;
2029 	void (*p)(enum tabs tab, int item_index);
2030 	uint8_t len = WINDOW_MIN_WIDTH;
2031 
2032 	for (i = 0; g_col_desc[tab][i].name != NULL; ++i) {
2033 		len = spdk_max(len, g_col_desc[tab][i].name_len);
2034 	}
2035 
2036 	elements = i;
2037 
2038 	my_items = (ITEM **)calloc(elements + 1, sizeof(ITEM *));
2039 	if (my_items == NULL) {
2040 		fprintf(stderr, "Unable to allocate an item list in change_sorting.\n");
2041 		return;
2042 	}
2043 
2044 	for (i = 0; i < elements; ++i) {
2045 		my_items[i] = new_item(g_col_desc[tab][i].name, NULL);
2046 		set_item_userptr(my_items[i], (winnum == 0) ? sort_type : sort_type2);
2047 	}
2048 
2049 	my_menu = new_menu((ITEM **)my_items);
2050 
2051 	menu_opts_off(my_menu, O_SHOWDESC);
2052 
2053 	sort_win = newwin(elements + WINDOW_HEADER_LEN + 1, len + WINDOW_BORDER_LEN,
2054 			  (g_max_row - elements) / 2,
2055 			  (g_max_col - len) / 2 - len + len * winnum);
2056 	assert(sort_win != NULL);
2057 	keypad(sort_win, TRUE);
2058 	sort_panel = new_panel(sort_win);
2059 	assert(sort_panel != NULL);
2060 
2061 	top_panel(sort_panel);
2062 	update_panels();
2063 	doupdate();
2064 
2065 	set_menu_win(my_menu, sort_win);
2066 	set_menu_sub(my_menu, derwin(sort_win, elements, len + 1, WINDOW_START_Y, WINDOW_START_X));
2067 	box(sort_win, 0, 0);
2068 
2069 	if (winnum == 0) {
2070 		name = "Sorting #1";
2071 		help = "Right key for second sorting";
2072 	} else {
2073 		name = "Sorting #2";
2074 		help = "Left key for first sorting";
2075 	}
2076 
2077 	print_in_middle(sort_win, 1, 0, len + WINDOW_BORDER_LEN, name, COLOR_PAIR(3));
2078 	print_in_middle(sort_win, 2, 0, len + WINDOW_BORDER_LEN, help, COLOR_PAIR(3));
2079 	mvwaddch(sort_win, WINDOW_HEADER_END_LINE, 0, ACS_LTEE);
2080 	mvwhline(sort_win, WINDOW_HEADER_END_LINE, 1, ACS_HLINE, len + 1);
2081 	mvwaddch(sort_win, WINDOW_HEADER_END_LINE, len + WINDOW_BORDER_LEN - 1, ACS_RTEE);
2082 
2083 	post_menu(my_menu);
2084 	refresh();
2085 	wrefresh(sort_win);
2086 
2087 	while (!stop_loop) {
2088 		c = wgetch(sort_win);
2089 		/*
2090 		 * First sorting window:
2091 		 * Up/Down - select first sorting column;
2092 		 * Enter - apply current column;
2093 		 * Right - open second sorting window.
2094 		 * Second sorting window:
2095 		 * Up/Down - select second sorting column;
2096 		 * Enter - apply current column of both sorting windows;
2097 		 * Left - exit second window and reset second sorting key;
2098 		 * Right - do nothing.
2099 		 */
2100 		switch (c) {
2101 		case KEY_DOWN:
2102 			menu_driver(my_menu, REQ_DOWN_ITEM);
2103 			break;
2104 		case KEY_UP:
2105 			menu_driver(my_menu, REQ_UP_ITEM);
2106 			break;
2107 		case KEY_RIGHT:
2108 			if (winnum > 0) {
2109 				break;
2110 			}
2111 			change_sorting(tab, winnum + 1, &stop_loop);
2112 			/* Restore input. */
2113 			keypad(sort_win, TRUE);
2114 			post_menu(my_menu);
2115 			refresh();
2116 			wrefresh(sort_win);
2117 			redrawwin(sort_win);
2118 			if (winnum == 0) {
2119 				cur = current_item(my_menu);
2120 				p = item_userptr(cur);
2121 				p(tab, item_index(cur));
2122 			}
2123 			break;
2124 		case 27: /* ESC */
2125 			stop_loop = true;
2126 			break;
2127 		case KEY_LEFT:
2128 			if (winnum > 0) {
2129 				sort_type2(tab, COL_THREADS_NONE);
2130 			}
2131 		/* FALLTHROUGH */
2132 		case 10: /* Enter */
2133 			stop_loop = true;
2134 			if (winnum > 0 && c == 10) {
2135 				*pstop_loop = true;
2136 			}
2137 			if (c == 10) {
2138 				cur = current_item(my_menu);
2139 				p = item_userptr(cur);
2140 				p(tab, item_index(cur));
2141 			}
2142 			break;
2143 		}
2144 		wrefresh(sort_win);
2145 	}
2146 
2147 	if (winnum == 0) {
2148 		wclear(g_tabs[tab]);
2149 		draw_tabs(tab, g_current_sort_col[tab], g_current_sort_col2[tab]);
2150 	}
2151 
2152 	unpost_menu(my_menu);
2153 	free_menu(my_menu);
2154 
2155 	for (i = 0; i < elements; ++i) {
2156 		free_item(my_items[i]);
2157 	}
2158 
2159 	free(my_items);
2160 
2161 	del_panel(sort_panel);
2162 	delwin(sort_win);
2163 
2164 	if (winnum == 0) {
2165 		wclear(g_menu_win);
2166 		draw_menu_win();
2167 	}
2168 }
2169 
2170 static int
2171 check_resize_interface(uint8_t active_tab, uint8_t *current_page)
2172 {
2173 	int max_row, max_col;
2174 	uint16_t required_size = WINDOW_HEADER + 1;
2175 
2176 	/* Check if interface has to be resized (terminal size changed) */
2177 	getmaxyx(stdscr, max_row, max_col);
2178 
2179 	if (max_row != g_max_row || max_col != g_max_col) {
2180 		if (max_row != g_max_row) {
2181 			*current_page = 0;
2182 		}
2183 		g_max_row = spdk_max(max_row, required_size);
2184 		g_max_col = max_col;
2185 		g_data_win_size = g_max_row - required_size + 1;
2186 		g_max_data_rows = g_max_row - WINDOW_HEADER;
2187 		resize_interface(active_tab);
2188 
2189 		return 1;
2190 	}
2191 
2192 	return 0;
2193 }
2194 
2195 static void
2196 change_refresh_rate(void)
2197 {
2198 	const int WINDOW_HEADER_END_LINE = 2;
2199 	PANEL *refresh_panel;
2200 	WINDOW *refresh_win;
2201 	int c;
2202 	bool stop_loop = false;
2203 	uint32_t rr_tmp, refresh_rate = 0;
2204 	char refresh_rate_str[MAX_STRING_LEN];
2205 
2206 	refresh_win = newwin(RR_WIN_HEIGHT, RR_WIN_WIDTH, (g_max_row - RR_WIN_HEIGHT - 1) / 2,
2207 			     (g_max_col - RR_WIN_WIDTH) / 2);
2208 	assert(refresh_win != NULL);
2209 	keypad(refresh_win, TRUE);
2210 	refresh_panel = new_panel(refresh_win);
2211 	assert(refresh_panel != NULL);
2212 
2213 	top_panel(refresh_panel);
2214 	update_panels();
2215 	doupdate();
2216 
2217 	box(refresh_win, 0, 0);
2218 
2219 	print_in_middle(refresh_win, 1, 0, RR_WIN_WIDTH + 1, "Enter refresh rate value [s]", COLOR_PAIR(3));
2220 	mvwaddch(refresh_win, WINDOW_HEADER_END_LINE, 0, ACS_LTEE);
2221 	mvwhline(refresh_win, WINDOW_HEADER_END_LINE, 1, ACS_HLINE, RR_WIN_WIDTH - 2);
2222 	mvwaddch(refresh_win, WINDOW_HEADER_END_LINE, RR_WIN_WIDTH, ACS_RTEE);
2223 	mvwprintw(refresh_win, WINDOW_HEADER_END_LINE + 1, (RR_WIN_WIDTH - 1) / 2, "%d", refresh_rate);
2224 
2225 	refresh();
2226 	wrefresh(refresh_win);
2227 
2228 	while (!stop_loop) {
2229 		c = wgetch(refresh_win);
2230 
2231 		switch (c) {
2232 		case '0':
2233 		case '1':
2234 		case '2':
2235 		case '3':
2236 		case '4':
2237 		case '5':
2238 		case '6':
2239 		case '7':
2240 		case '8':
2241 		case '9':
2242 			rr_tmp = refresh_rate * 10 + c - '0';
2243 
2244 			if (rr_tmp <= RR_MAX_VALUE) {
2245 				refresh_rate = rr_tmp;
2246 				snprintf(refresh_rate_str, MAX_STRING_LEN - 1, "%d", refresh_rate);
2247 				mvwprintw(refresh_win, WINDOW_HEADER_END_LINE + 1,
2248 					  (RR_WIN_WIDTH - 1 - strlen(refresh_rate_str)) / 2, "%d", refresh_rate);
2249 				refresh();
2250 				wrefresh(refresh_win);
2251 			}
2252 			break;
2253 		case KEY_BACKSPACE:
2254 		case 127:
2255 		case '\b':
2256 			refresh_rate = refresh_rate / 10;
2257 			snprintf(refresh_rate_str, MAX_STRING_LEN - 1, "%d", refresh_rate);
2258 			mvwprintw(refresh_win, WINDOW_HEADER_END_LINE + 1,
2259 				  (RR_WIN_WIDTH - 1 - strlen(refresh_rate_str) - 2) / 2, "       ");
2260 			mvwprintw(refresh_win, WINDOW_HEADER_END_LINE + 1,
2261 				  (RR_WIN_WIDTH - 1 - strlen(refresh_rate_str)) / 2, "%d", refresh_rate);
2262 			refresh();
2263 			wrefresh(refresh_win);
2264 			break;
2265 		case 27: /* ESC */
2266 		case 'q':
2267 			stop_loop = true;
2268 			break;
2269 		case 10: /* Enter */
2270 			pthread_mutex_lock(&g_thread_lock);
2271 			g_sleep_time = refresh_rate;
2272 			pthread_mutex_unlock(&g_thread_lock);
2273 			stop_loop = true;
2274 			break;
2275 		}
2276 		wrefresh(refresh_win);
2277 	}
2278 
2279 	del_panel(refresh_panel);
2280 	delwin(refresh_win);
2281 }
2282 
2283 static void
2284 free_poller_history(void)
2285 {
2286 	struct run_counter_history *history, *tmp;
2287 
2288 	TAILQ_FOREACH_SAFE(history, &g_run_counter_history, link, tmp) {
2289 		TAILQ_REMOVE(&g_run_counter_history, history, link);
2290 		free(history);
2291 	}
2292 }
2293 
2294 static uint64_t
2295 get_position_for_window(uint64_t window_size, uint64_t max_size)
2296 {
2297 	/* This function calculates position for pop-up detail window.
2298 	 * Since horizontal and vertical positions are calculated the same way
2299 	 * there is no need for separate functions. */
2300 	window_size = spdk_min(window_size, max_size);
2301 
2302 	return (max_size - window_size) / 2;
2303 }
2304 
2305 static void
2306 print_bottom_message(char *msg)
2307 {
2308 	uint64_t i;
2309 
2310 	for (i = 1; i < (uint64_t)g_max_col - 1; i++) {
2311 		mvprintw(g_max_row - 1, i, " ");
2312 	}
2313 	mvprintw(g_max_row - 1, g_max_col - strlen(msg) - 2, "%s", msg);
2314 }
2315 
2316 static void
2317 draw_thread_win_content(WINDOW *thread_win, struct rpc_thread_info *thread_info)
2318 {
2319 	uint64_t current_row, i, time;
2320 	char idle_time[MAX_TIME_STR_LEN], busy_time[MAX_TIME_STR_LEN];
2321 
2322 	box(thread_win, 0, 0);
2323 
2324 	print_in_middle(thread_win, 1, 0, THREAD_WIN_WIDTH, thread_info->name,
2325 			COLOR_PAIR(3));
2326 	mvwhline(thread_win, 2, 1, ACS_HLINE, THREAD_WIN_WIDTH - 2);
2327 	mvwaddch(thread_win, 2, THREAD_WIN_WIDTH, ACS_RTEE);
2328 
2329 	print_left(thread_win, 3, THREAD_WIN_FIRST_COL, THREAD_WIN_WIDTH,
2330 		   "Core:                Idle [us]:            Busy [us]:", COLOR_PAIR(5));
2331 	mvwprintw(thread_win, 3, THREAD_WIN_FIRST_COL + 6, "%d",
2332 		  thread_info->core_num);
2333 
2334 	if (g_interval_data) {
2335 		get_time_str(thread_info->idle - thread_info->last_idle, idle_time);
2336 		mvwprintw(thread_win, 3, THREAD_WIN_FIRST_COL + 32, "%s", idle_time);
2337 		get_time_str(thread_info->busy - thread_info->last_busy, busy_time);
2338 		mvwprintw(thread_win, 3, THREAD_WIN_FIRST_COL + 54, "%s", busy_time);
2339 	} else {
2340 		get_time_str(thread_info->idle, idle_time);
2341 		mvwprintw(thread_win, 3, THREAD_WIN_FIRST_COL + 32, "%s", idle_time);
2342 		get_time_str(thread_info->busy, busy_time);
2343 		mvwprintw(thread_win, 3, THREAD_WIN_FIRST_COL + 54, "%s", busy_time);
2344 	}
2345 
2346 	print_left(thread_win, 4, THREAD_WIN_FIRST_COL, THREAD_WIN_WIDTH,
2347 		   "Active pollers:      Timed pollers:        Paused pollers:", COLOR_PAIR(5));
2348 	mvwprintw(thread_win, 4, THREAD_WIN_FIRST_COL + 17, "%" PRIu64,
2349 		  thread_info->active_pollers_count);
2350 	mvwprintw(thread_win, 4, THREAD_WIN_FIRST_COL + 36, "%" PRIu64,
2351 		  thread_info->timed_pollers_count);
2352 	mvwprintw(thread_win, 4, THREAD_WIN_FIRST_COL + 59, "%" PRIu64,
2353 		  thread_info->paused_pollers_count);
2354 
2355 	mvwhline(thread_win, 5, 1, ACS_HLINE, THREAD_WIN_WIDTH - 2);
2356 
2357 	print_in_middle(thread_win, 6, 0, THREAD_WIN_WIDTH,
2358 			"Pollers                          Type    Total run count   Period", COLOR_PAIR(5));
2359 
2360 	mvwhline(thread_win, 7, 1, ACS_HLINE, THREAD_WIN_WIDTH - 2);
2361 
2362 	current_row = 8;
2363 
2364 	for (i = 0; i < g_last_pollers_count; i++) {
2365 		if (g_pollers_info[i].thread_id == thread_info->id) {
2366 			mvwprintw(thread_win, current_row, THREAD_WIN_FIRST_COL, "%s", g_pollers_info[i].name);
2367 			if (g_pollers_info[i].type == SPDK_ACTIVE_POLLER) {
2368 				mvwprintw(thread_win, current_row, THREAD_WIN_FIRST_COL + 33, "Active");
2369 			} else if (g_pollers_info[i].type == SPDK_TIMED_POLLER) {
2370 				mvwprintw(thread_win, current_row, THREAD_WIN_FIRST_COL + 33, "Timed");
2371 			} else {
2372 				mvwprintw(thread_win, current_row, THREAD_WIN_FIRST_COL + 33, "Paused");
2373 			}
2374 			mvwprintw(thread_win, current_row, THREAD_WIN_FIRST_COL + 41, "%" PRIu64,
2375 				  g_pollers_info[i].run_count);
2376 			if (g_pollers_info[i].period_ticks) {
2377 				time = g_pollers_info[i].period_ticks * SPDK_SEC_TO_USEC / g_tick_rate;
2378 				mvwprintw(thread_win, current_row, THREAD_WIN_FIRST_COL + 59, "%" PRIu64, time);
2379 			}
2380 			current_row++;
2381 		}
2382 	}
2383 
2384 	wnoutrefresh(thread_win);
2385 }
2386 
2387 static int
2388 get_single_thread_info(uint64_t thread_id, struct rpc_thread_info *thread_info)
2389 {
2390 	uint64_t i;
2391 
2392 	for (i = 0; i < g_last_threads_count; i++) {
2393 		if (g_threads_info[i].id == thread_id) {
2394 			memcpy(thread_info, &g_threads_info[i], sizeof(struct rpc_thread_info));
2395 			thread_info->name = strdup(g_threads_info[i].name);
2396 			thread_info->cpumask = strdup(g_threads_info[i].cpumask);
2397 
2398 			if (thread_info->name == NULL || thread_info->cpumask == NULL) {
2399 				print_bottom_message("Unable to allocate memory for thread name and cpumask. Exiting pop-up.");
2400 				return -1;
2401 			}
2402 
2403 			return 0;
2404 		}
2405 	}
2406 
2407 	print_bottom_message("Selected thread no longer exists. Exiting pop-up.");
2408 	return -1;
2409 }
2410 
2411 static void
2412 draw_core_win_content(WINDOW *core_win, struct rpc_core_info *core_info)
2413 {
2414 	uint64_t i;
2415 	char core_win_title[25];
2416 	char idle_time[MAX_TIME_STR_LEN], busy_time[MAX_TIME_STR_LEN];
2417 
2418 	box(core_win, 0, 0);
2419 	snprintf(core_win_title, sizeof(core_win_title), "Core %" PRIu32 " details",
2420 		 core_info->lcore);
2421 	print_in_middle(core_win, 1, 0, CORE_WIN_WIDTH, core_win_title, COLOR_PAIR(3));
2422 
2423 	mvwhline(core_win, 2, 1, ACS_HLINE, CORE_WIN_WIDTH - 2);
2424 	print_left(core_win, 3, 1, CORE_WIN_WIDTH - (CORE_WIN_WIDTH / 3),
2425 		   "Frequency:             Intr:", COLOR_PAIR(5));
2426 	if (core_info->core_freq) {
2427 		mvwprintw(core_win, 3, CORE_WIN_FIRST_COL - 3, "%" PRIu32,
2428 			  core_info->core_freq);
2429 	} else {
2430 		mvwprintw(core_win, 3, CORE_WIN_FIRST_COL - 3, "%s", "N/A");
2431 	}
2432 
2433 	mvwprintw(core_win, 3, CORE_WIN_FIRST_COL + 15, "%s",
2434 		  core_info->in_interrupt ? "Yes" : "No");
2435 
2436 	mvwhline(core_win, 4, 1, ACS_HLINE, CORE_WIN_WIDTH - 2);
2437 	print_left(core_win, 5, 1, CORE_WIN_WIDTH, "Thread count:          Idle time:", COLOR_PAIR(5));
2438 
2439 	mvwprintw(core_win, 5, CORE_WIN_FIRST_COL, "%" PRIu64,
2440 		  core_info->threads.threads_count);
2441 
2442 	if (g_interval_data == true) {
2443 		get_time_str(core_info->idle - core_info->last_idle, idle_time);
2444 		get_time_str(core_info->busy - core_info->last_busy, busy_time);
2445 	} else {
2446 		get_time_str(core_info->idle, idle_time);
2447 		get_time_str(core_info->busy, busy_time);
2448 	}
2449 	mvwprintw(core_win, 5, CORE_WIN_FIRST_COL + 20, "%s", idle_time);
2450 	mvwhline(core_win, 6, 1, ACS_HLINE, CORE_WIN_WIDTH - 2);
2451 
2452 	print_left(core_win, 7, 1, CORE_WIN_WIDTH, "Poller count:          Busy time:", COLOR_PAIR(5));
2453 	mvwprintw(core_win, 7, CORE_WIN_FIRST_COL, "%" PRIu64,
2454 		  core_info->pollers_count);
2455 
2456 	mvwprintw(core_win, 7, CORE_WIN_FIRST_COL + 20, "%s", busy_time);
2457 
2458 	mvwhline(core_win, 8, 1, ACS_HLINE, CORE_WIN_WIDTH - 2);
2459 	print_left(core_win, 9, 1, CORE_WIN_WIDTH, "Threads on this core", COLOR_PAIR(5));
2460 
2461 	for (i = 0; i < core_info->threads.threads_count; i++) {
2462 		mvwprintw(core_win, i + CORE_WIN_HEIGHT - 1, 1, "%s", core_info->threads.thread[i].name);
2463 	}
2464 	pthread_mutex_unlock(&g_thread_lock);
2465 
2466 	wnoutrefresh(core_win);
2467 }
2468 
2469 static void
2470 display_thread(uint64_t thread_id, uint8_t current_page, uint8_t active_tab,
2471 	       WINDOW *core_popup, struct rpc_core_info *core_info)
2472 {
2473 	PANEL *thread_panel = NULL;
2474 	WINDOW *thread_win = NULL;
2475 	struct rpc_thread_info thread_info;
2476 	uint64_t pollers_count, threads_count, last_pollers_count = 0;
2477 	int c;
2478 	bool stop_loop = false;
2479 	long int time_last, time_dif;
2480 	struct timespec time_now;
2481 
2482 	clock_gettime(CLOCK_MONOTONIC, &time_now);
2483 	time_last = time_now.tv_sec;
2484 
2485 	memset(&thread_info, 0, sizeof(thread_info));
2486 
2487 	while (!stop_loop) {
2488 		pthread_mutex_lock(&g_thread_lock);
2489 		if (get_single_thread_info(thread_id, &thread_info)) {
2490 			pthread_mutex_unlock(&g_thread_lock);
2491 			free(thread_info.name);
2492 			free(thread_info.cpumask);
2493 			thread_info.name = NULL;
2494 			thread_info.cpumask = NULL;
2495 			return;
2496 		}
2497 		pollers_count = thread_info.active_pollers_count +
2498 				thread_info.timed_pollers_count +
2499 				thread_info.paused_pollers_count;
2500 		if (pollers_count != last_pollers_count) {
2501 			if (thread_win != NULL) {
2502 				assert(thread_panel != NULL);
2503 				del_panel(thread_panel);
2504 				delwin(thread_win);
2505 			}
2506 
2507 			thread_win = newwin(pollers_count + THREAD_WIN_HEIGHT, THREAD_WIN_WIDTH,
2508 					    get_position_for_window(THREAD_WIN_HEIGHT + pollers_count, g_max_row),
2509 					    get_position_for_window(THREAD_WIN_WIDTH, g_max_col));
2510 			keypad(thread_win, TRUE);
2511 			thread_panel = new_panel(thread_win);
2512 
2513 			top_panel(thread_panel);
2514 			update_panels();
2515 			doupdate();
2516 			draw_thread_win_content(thread_win, &thread_info);
2517 			refresh();
2518 		}
2519 		pthread_mutex_unlock(&g_thread_lock);
2520 
2521 		if (check_resize_interface(active_tab, &current_page)) {
2522 			/* This clear is to avoid remaining artifacts after window has been moved */
2523 			wclear(thread_win);
2524 			wclear(core_popup);
2525 			resize_interface(active_tab);
2526 			draw_tabs(active_tab, g_current_sort_col[active_tab], g_current_sort_col2[active_tab]);
2527 			if (core_popup != NULL) {
2528 				pthread_mutex_lock(&g_thread_lock);
2529 				threads_count = g_cores_info[core_info->lcore].threads.threads_count;
2530 				pthread_mutex_unlock(&g_thread_lock);
2531 				mvwin(core_popup, get_position_for_window(CORE_WIN_HEIGHT + threads_count, g_max_row),
2532 				      get_position_for_window(CORE_WIN_WIDTH, g_max_col));
2533 			}
2534 			mvwin(thread_win, get_position_for_window(THREAD_WIN_HEIGHT + pollers_count, g_max_row),
2535 			      get_position_for_window(THREAD_WIN_WIDTH, g_max_col));
2536 		}
2537 
2538 		c = getch();
2539 
2540 		switch (c) {
2541 		case 27: /* ESC */
2542 			stop_loop = true;
2543 			break;
2544 		default:
2545 			break;
2546 		}
2547 
2548 		clock_gettime(CLOCK_MONOTONIC, &time_now);
2549 		time_dif = time_now.tv_sec - time_last;
2550 
2551 		if (time_dif >= g_sleep_time) {
2552 			time_last = time_now.tv_sec;
2553 			pthread_mutex_lock(&g_thread_lock);
2554 			refresh_tab(active_tab, current_page);
2555 			if (core_popup != NULL) {
2556 				draw_core_win_content(core_popup, core_info);
2557 			}
2558 			draw_thread_win_content(thread_win, &thread_info);
2559 			refresh();
2560 			pthread_mutex_unlock(&g_thread_lock);
2561 		}
2562 
2563 		last_pollers_count = pollers_count;
2564 		free(thread_info.name);
2565 		free(thread_info.cpumask);
2566 		thread_info.name = NULL;
2567 		thread_info.cpumask = NULL;
2568 	}
2569 
2570 	del_panel(thread_panel);
2571 	delwin(thread_win);
2572 }
2573 
2574 static void
2575 show_thread(uint8_t current_page, uint8_t active_tab)
2576 {
2577 	uint64_t thread_number = current_page * g_max_data_rows + g_selected_row;
2578 	uint64_t thread_id;
2579 
2580 	pthread_mutex_lock(&g_thread_lock);
2581 	assert(thread_number < g_last_threads_count);
2582 	thread_id = g_threads_info[thread_number].id;
2583 	pthread_mutex_unlock(&g_thread_lock);
2584 
2585 	display_thread(thread_id, current_page, active_tab, NULL, NULL);
2586 }
2587 
2588 static void
2589 show_single_thread(uint64_t thread_id, uint8_t current_page, uint8_t active_tab, WINDOW *core_popup,
2590 		   struct rpc_core_info *core_info)
2591 {
2592 	uint64_t i;
2593 
2594 	pthread_mutex_lock(&g_thread_lock);
2595 	for (i = 0; i < g_last_threads_count; i++) {
2596 		if (g_threads_info[i].id == thread_id) {
2597 			pthread_mutex_unlock(&g_thread_lock);
2598 			display_thread(thread_id, current_page, active_tab, core_popup, core_info);
2599 			return;
2600 		}
2601 	}
2602 	pthread_mutex_unlock(&g_thread_lock);
2603 }
2604 
2605 static void
2606 show_core(uint8_t current_page, uint8_t active_tab)
2607 {
2608 	PANEL *core_panel;
2609 	WINDOW *core_win;
2610 	uint64_t core_number = current_page * g_max_data_rows + g_selected_row;
2611 	struct rpc_core_info *core_info = &g_cores_info[core_number];
2612 	uint64_t threads_count, i;
2613 	uint64_t thread_id = 0;
2614 	uint16_t current_threads_row;
2615 	int c;
2616 	long int time_last, time_dif;
2617 	struct timespec time_now;
2618 
2619 	clock_gettime(CLOCK_MONOTONIC, &time_now);
2620 	time_last = time_now.tv_sec;
2621 
2622 	bool stop_loop = false;
2623 
2624 	pthread_mutex_lock(&g_thread_lock);
2625 	assert(core_number < g_last_cores_count);
2626 
2627 	threads_count = g_cores_info[core_number].threads.threads_count;
2628 
2629 	core_win = newwin(threads_count + CORE_WIN_HEIGHT, CORE_WIN_WIDTH,
2630 			  get_position_for_window(CORE_WIN_HEIGHT + threads_count, g_max_row),
2631 			  get_position_for_window(CORE_WIN_WIDTH, g_max_col));
2632 
2633 	keypad(core_win, TRUE);
2634 	core_panel = new_panel(core_win);
2635 
2636 	top_panel(core_panel);
2637 	update_panels();
2638 	doupdate();
2639 	draw_core_win_content(core_win, core_info);
2640 	refresh();
2641 
2642 	current_threads_row = 0;
2643 
2644 	while (!stop_loop) {
2645 		pthread_mutex_lock(&g_thread_lock);
2646 		for (i = 0; i < core_info->threads.threads_count; i++) {
2647 			if (i != current_threads_row) {
2648 				mvwprintw(core_win, i + CORE_WIN_HEIGHT - 1, 1,
2649 					  "%s", core_info->threads.thread[i].name);
2650 			} else {
2651 				print_left(core_win, i + CORE_WIN_HEIGHT - 1, 1, CORE_WIN_WIDTH - 2,
2652 					   core_info->threads.thread[i].name, COLOR_PAIR(2));
2653 			}
2654 		}
2655 		pthread_mutex_unlock(&g_thread_lock);
2656 
2657 		wrefresh(core_win);
2658 		if (check_resize_interface(active_tab, &current_page)) {
2659 			wclear(core_win);
2660 			resize_interface(active_tab);
2661 			draw_tab_win(active_tab);
2662 			mvwin(core_win, get_position_for_window(CORE_WIN_HEIGHT + threads_count, g_max_row),
2663 			      get_position_for_window(CORE_WIN_WIDTH, g_max_col));
2664 		}
2665 
2666 		c = getch();
2667 		switch (c) {
2668 		case 10: /* ENTER */
2669 			pthread_mutex_lock(&g_thread_lock);
2670 			if (core_info->threads.threads_count > 0) {
2671 				thread_id = core_info->threads.thread[current_threads_row].id;
2672 			}
2673 			pthread_mutex_unlock(&g_thread_lock);
2674 
2675 			if (thread_id != 0) {
2676 				show_single_thread(thread_id, current_page, active_tab, core_win, core_info);
2677 			}
2678 
2679 			/* This refreshes tab and core_pop-up after exiting threads pop-up. */
2680 			pthread_mutex_lock(&g_thread_lock);
2681 			refresh_tab(active_tab, current_page);
2682 			wnoutrefresh(core_win);
2683 			refresh();
2684 			pthread_mutex_unlock(&g_thread_lock);
2685 			break;
2686 		case 27: /* ESC */
2687 			stop_loop = true;
2688 			break;
2689 		case KEY_UP:
2690 			if (current_threads_row != 0) {
2691 				current_threads_row--;
2692 			}
2693 			break;
2694 		case KEY_DOWN:
2695 			pthread_mutex_lock(&g_thread_lock);
2696 			if (current_threads_row != core_info->threads.threads_count - 1) {
2697 				current_threads_row++;
2698 			}
2699 			pthread_mutex_unlock(&g_thread_lock);
2700 			break;
2701 		default:
2702 			break;
2703 		}
2704 
2705 		clock_gettime(CLOCK_MONOTONIC, &time_now);
2706 		time_dif = time_now.tv_sec - time_last;
2707 
2708 		if (time_dif >= g_sleep_time) {
2709 			time_last = time_now.tv_sec;
2710 			pthread_mutex_lock(&g_thread_lock);
2711 			refresh_tab(active_tab, current_page);
2712 			draw_core_win_content(core_win, core_info);
2713 			refresh();
2714 			pthread_mutex_unlock(&g_thread_lock);
2715 		}
2716 	}
2717 
2718 	del_panel(core_panel);
2719 	delwin(core_win);
2720 }
2721 
2722 static void
2723 draw_poller_win_content(WINDOW *poller_win, struct rpc_poller_info *poller_info)
2724 {
2725 	uint64_t last_run_counter, last_busy_counter, busy_count;
2726 	char poller_period[MAX_TIME_STR_LEN];
2727 
2728 	box(poller_win, 0, 0);
2729 
2730 	print_in_middle(poller_win, 1, 0, POLLER_WIN_WIDTH, poller_info->name, COLOR_PAIR(3));
2731 	mvwhline(poller_win, 2, 1, ACS_HLINE, POLLER_WIN_WIDTH - 2);
2732 	mvwaddch(poller_win, 2, POLLER_WIN_WIDTH, ACS_RTEE);
2733 
2734 	print_left(poller_win, 3, 2, POLLER_WIN_WIDTH, "Type:                  On thread:", COLOR_PAIR(5));
2735 	mvwprintw(poller_win, 3, POLLER_WIN_FIRST_COL, "%s",
2736 		  poller_type_str[poller_info->type]);
2737 	mvwprintw(poller_win, 3, POLLER_WIN_FIRST_COL + 23, "%s", poller_info->thread_name);
2738 
2739 	print_left(poller_win, 4, 2, POLLER_WIN_WIDTH, "Run count:", COLOR_PAIR(5));
2740 
2741 	last_run_counter = get_last_run_counter(poller_info->id, poller_info->thread_id);
2742 	last_busy_counter = get_last_busy_counter(poller_info->id, poller_info->thread_id);
2743 	if (g_interval_data) {
2744 		mvwprintw(poller_win, 4, POLLER_WIN_FIRST_COL, "%" PRIu64,
2745 			  poller_info->run_count - last_run_counter);
2746 	} else {
2747 		mvwprintw(poller_win, 4, POLLER_WIN_FIRST_COL, "%" PRIu64, poller_info->run_count);
2748 	}
2749 
2750 	if (poller_info->period_ticks != 0) {
2751 		print_left(poller_win, 4, 28, POLLER_WIN_WIDTH, "Period:", COLOR_PAIR(5));
2752 		get_time_str(poller_info->period_ticks, poller_period);
2753 		mvwprintw(poller_win, 4, POLLER_WIN_FIRST_COL + 23, "%s", poller_period);
2754 	}
2755 	mvwhline(poller_win, 5, 1, ACS_HLINE, POLLER_WIN_WIDTH - 2);
2756 
2757 	busy_count = g_interval_data ? poller_info->busy_count - last_busy_counter :
2758 		     poller_info->busy_count;
2759 	if (busy_count != 0) {
2760 		print_left(poller_win, 6, 2, POLLER_WIN_WIDTH,  "Status:               Busy count:", COLOR_PAIR(5));
2761 
2762 		if (g_interval_data == false && poller_info->busy_count == last_busy_counter) {
2763 			print_left(poller_win, 6, POLLER_WIN_FIRST_COL, POLLER_WIN_WIDTH, "Idle", COLOR_PAIR(7));
2764 		} else {
2765 			print_left(poller_win, 6, POLLER_WIN_FIRST_COL, POLLER_WIN_WIDTH, "Busy", COLOR_PAIR(6));
2766 		}
2767 
2768 		mvwprintw(poller_win, 6, POLLER_WIN_FIRST_COL + 23, "%" PRIu64, busy_count);
2769 	} else {
2770 		print_in_middle(poller_win, 6, 1, POLLER_WIN_WIDTH - 7, "Status:", COLOR_PAIR(5));
2771 		print_in_middle(poller_win, 6, 1, POLLER_WIN_WIDTH + 6, "Idle", COLOR_PAIR(7));
2772 	}
2773 
2774 	wnoutrefresh(poller_win);
2775 }
2776 
2777 static void
2778 show_poller(uint8_t current_page, uint8_t active_tab)
2779 {
2780 	PANEL *poller_panel;
2781 	WINDOW *poller_win;
2782 	uint64_t poller_number = current_page * g_max_data_rows + g_selected_row;
2783 	struct rpc_poller_info *poller;
2784 	bool stop_loop = false;
2785 	int c;
2786 	long int time_last, time_dif;
2787 	struct timespec time_now;
2788 
2789 	clock_gettime(CLOCK_MONOTONIC, &time_now);
2790 	time_last = time_now.tv_sec;
2791 
2792 
2793 	pthread_mutex_lock(&g_thread_lock);
2794 
2795 	assert(poller_number < g_last_pollers_count);
2796 	poller = &g_pollers_info[poller_number];
2797 
2798 	poller_win = newwin(POLLER_WIN_HEIGHT, POLLER_WIN_WIDTH,
2799 			    get_position_for_window(POLLER_WIN_HEIGHT, g_max_row),
2800 			    get_position_for_window(POLLER_WIN_WIDTH, g_max_col));
2801 
2802 	keypad(poller_win, TRUE);
2803 	poller_panel = new_panel(poller_win);
2804 
2805 	top_panel(poller_panel);
2806 	update_panels();
2807 	doupdate();
2808 	draw_poller_win_content(poller_win, poller);
2809 	refresh();
2810 
2811 	pthread_mutex_unlock(&g_thread_lock);
2812 	while (!stop_loop) {
2813 		if (check_resize_interface(active_tab, &current_page)) {
2814 			/* This clear is to avoid remaining artifacts after window has been moved */
2815 			wclear(poller_win);
2816 			resize_interface(active_tab);
2817 			draw_tabs(active_tab, g_current_sort_col[active_tab], g_current_sort_col2[active_tab]);
2818 			mvwin(poller_win, get_position_for_window(POLLER_WIN_HEIGHT, g_max_row),
2819 			      get_position_for_window(POLLER_WIN_WIDTH, g_max_col));
2820 		}
2821 		c = getch();
2822 		switch (c) {
2823 		case 27: /* ESC */
2824 			stop_loop = true;
2825 			break;
2826 		default:
2827 			break;
2828 		}
2829 
2830 		clock_gettime(CLOCK_MONOTONIC, &time_now);
2831 		time_dif = time_now.tv_sec - time_last;
2832 
2833 		if (time_dif >= g_sleep_time) {
2834 			time_last = time_now.tv_sec;
2835 			pthread_mutex_lock(&g_thread_lock);
2836 			refresh_tab(active_tab, current_page);
2837 			draw_poller_win_content(poller_win, poller);
2838 			refresh();
2839 			pthread_mutex_unlock(&g_thread_lock);
2840 		}
2841 	}
2842 
2843 	del_panel(poller_panel);
2844 	delwin(poller_win);
2845 }
2846 
2847 static uint64_t
2848 get_max_scheduler_win_width(uint8_t sched_name_label_len, uint8_t sched_period_label_len,
2849 			    uint8_t gov_name_label_len)
2850 {
2851 	uint8_t window_borders = 4;
2852 	uint64_t s_name, s_period, g_name, max_name_len;
2853 	char scheduler_period[MAX_SCHEDULER_PERIOD_STR_LEN];
2854 
2855 	snprintf(scheduler_period, MAX_SCHEDULER_PERIOD_STR_LEN, "%" PRIu64,
2856 		 g_scheduler_info.scheduler_period);
2857 
2858 	s_name = strlen(g_scheduler_info.scheduler_name) + sched_name_label_len;
2859 	if (g_scheduler_info.governor_name != NULL) {
2860 		g_name = strlen(g_scheduler_info.governor_name) + gov_name_label_len;
2861 	} else {
2862 		g_name = strlen("none") + gov_name_label_len;
2863 	}
2864 	s_period = strlen(scheduler_period) + sched_period_label_len;
2865 
2866 	max_name_len = spdk_max(s_name, g_name);
2867 	/* This function relies on the fact that scheduler/governor
2868 	 * names will not change during runtime. Otherwise the scheduler
2869 	 * pop-up would need dynamic resizing. */
2870 
2871 	return spdk_max(max_name_len, s_period) + window_borders;
2872 }
2873 
2874 static void
2875 draw_scheduler_popup(WINDOW *scheduler_win, uint64_t scheduler_win_width, uint8_t active_tab,
2876 		     uint8_t current_page, const char *scheduler_name_label,
2877 		     const char *scheduler_period_label, const char *governor_name_label)
2878 {
2879 	char scheduler_period[MAX_SCHEDULER_PERIOD_STR_LEN];
2880 
2881 	box(scheduler_win, 0, 0);
2882 
2883 	print_left(scheduler_win, 1, SCHEDULER_WIN_FIRST_COL, scheduler_win_width, scheduler_name_label,
2884 		   COLOR_PAIR(5));
2885 	print_left(scheduler_win, 1, SCHEDULER_WIN_FIRST_COL + strlen(scheduler_name_label),
2886 		   scheduler_win_width,
2887 		   g_scheduler_info.scheduler_name, COLOR_PAIR(3));
2888 
2889 	mvwhline(scheduler_win, 2, 1, ACS_HLINE, scheduler_win_width - 2);
2890 	mvwaddch(scheduler_win, 2, scheduler_win_width, ACS_RTEE);
2891 
2892 	print_left(scheduler_win, 3, SCHEDULER_WIN_FIRST_COL, scheduler_win_width, scheduler_period_label,
2893 		   COLOR_PAIR(5));
2894 	snprintf(scheduler_period, MAX_SCHEDULER_PERIOD_STR_LEN, "%" PRIu64,
2895 		 g_scheduler_info.scheduler_period);
2896 	mvwprintw(scheduler_win, 3, SCHEDULER_WIN_FIRST_COL + strlen(scheduler_period_label), "%s",
2897 		  scheduler_period);
2898 
2899 	mvwhline(scheduler_win, 4, 1, ACS_HLINE, scheduler_win_width - 2);
2900 	mvwaddch(scheduler_win, 4, scheduler_win_width, ACS_RTEE);
2901 
2902 	print_left(scheduler_win, 5, SCHEDULER_WIN_FIRST_COL, scheduler_win_width, governor_name_label,
2903 		   COLOR_PAIR(5));
2904 
2905 	if (g_scheduler_info.governor_name != NULL) {
2906 		mvwprintw(scheduler_win, 5, SCHEDULER_WIN_FIRST_COL + strlen(governor_name_label), "%s",
2907 			  g_scheduler_info.governor_name);
2908 	} else {
2909 		mvwprintw(scheduler_win, 5, SCHEDULER_WIN_FIRST_COL + strlen(governor_name_label), "%s", "none");
2910 	}
2911 
2912 	refresh_tab(active_tab, current_page);
2913 	wnoutrefresh(scheduler_win);
2914 	refresh();
2915 }
2916 
2917 static void
2918 show_scheduler(uint8_t active_tab, uint8_t current_page)
2919 {
2920 	PANEL *scheduler_panel;
2921 	WINDOW *scheduler_win;
2922 	uint64_t scheduler_win_width;
2923 	bool stop_loop = false;
2924 	int c;
2925 	const char *scheduler_name_label = "Scheduler:  ";
2926 	const char *scheduler_period_label = "Period [us]:  ";
2927 	const char *governor_name_label = "Governor:  ";
2928 
2929 	pthread_mutex_lock(&g_thread_lock);
2930 	scheduler_win_width = get_max_scheduler_win_width(strlen(scheduler_name_label),
2931 			      strlen(scheduler_period_label),
2932 			      strlen(governor_name_label));
2933 
2934 	scheduler_win = newwin(SCHEDULER_WIN_HEIGHT, scheduler_win_width,
2935 			       get_position_for_window(SCHEDULER_WIN_HEIGHT, g_max_row),
2936 			       get_position_for_window(scheduler_win_width, g_max_col));
2937 
2938 	keypad(scheduler_win, TRUE);
2939 	scheduler_panel = new_panel(scheduler_win);
2940 
2941 	top_panel(scheduler_panel);
2942 	update_panels();
2943 	doupdate();
2944 
2945 	draw_scheduler_popup(scheduler_win, scheduler_win_width, active_tab, current_page,
2946 			     scheduler_name_label, scheduler_period_label, governor_name_label);
2947 	pthread_mutex_unlock(&g_thread_lock);
2948 
2949 	while (!stop_loop) {
2950 		c = wgetch(scheduler_win);
2951 
2952 		switch (c) {
2953 		case 27: /* ESC */
2954 			stop_loop = true;
2955 			break;
2956 		default:
2957 			break;
2958 		}
2959 	}
2960 
2961 	del_panel(scheduler_panel);
2962 	delwin(scheduler_win);
2963 }
2964 
2965 static void *
2966 data_thread_routine(void *arg)
2967 {
2968 	int rc;
2969 	uint64_t refresh_rate;
2970 
2971 	while (1) {
2972 		pthread_mutex_lock(&g_thread_lock);
2973 		if (g_quit_app) {
2974 			pthread_mutex_unlock(&g_thread_lock);
2975 			break;
2976 		}
2977 
2978 		if (g_sleep_time == 0) {
2979 			/* Give display thread time to redraw all windows */
2980 			refresh_rate = SPDK_SEC_TO_USEC / 100;
2981 		} else {
2982 			refresh_rate = g_sleep_time * SPDK_SEC_TO_USEC;
2983 		}
2984 		pthread_mutex_unlock(&g_thread_lock);
2985 
2986 		/* Get data from RPC for each object type.
2987 		 * Start with cores since their number should not change. */
2988 		rc = get_cores_data();
2989 		if (rc) {
2990 			print_bottom_message("ERROR occurred while getting cores data");
2991 		}
2992 		rc = get_thread_data();
2993 		if (rc) {
2994 			print_bottom_message("ERROR occurred while getting threads data");
2995 		}
2996 
2997 		rc = get_pollers_data();
2998 		if (rc) {
2999 			print_bottom_message("ERROR occurred while getting pollers data");
3000 		}
3001 		rc = get_scheduler_data();
3002 		if (rc) {
3003 			print_bottom_message("ERROR occurred while getting scheduler data");
3004 		}
3005 
3006 		usleep(refresh_rate);
3007 	}
3008 
3009 	return NULL;
3010 }
3011 
3012 static void
3013 help_window_display(void)
3014 {
3015 	PANEL *help_panel;
3016 	WINDOW *help_win;
3017 	bool stop_loop = false;
3018 	int c;
3019 	uint64_t row = 1, col = 2, desc_second_row_col = 26, header_footer_col = 0;
3020 
3021 	help_win = newwin(HELP_WIN_HEIGHT, HELP_WIN_WIDTH,
3022 			  get_position_for_window(HELP_WIN_HEIGHT, g_max_row),
3023 			  get_position_for_window(HELP_WIN_WIDTH, g_max_col));
3024 	help_panel = new_panel(help_win);
3025 	top_panel(help_panel);
3026 	update_panels();
3027 	doupdate();
3028 
3029 	box(help_win, 0, 0);
3030 
3031 	/* Header */
3032 	print_in_middle(help_win, row, header_footer_col, HELP_WIN_WIDTH, "HELP", COLOR_PAIR(3));
3033 	mvwhline(help_win, 2, 1, ACS_HLINE, HELP_WIN_WIDTH - 2);
3034 	mvwaddch(help_win, 2, HELP_WIN_WIDTH, ACS_RTEE);
3035 	row = 3;
3036 
3037 	/* Content */
3038 	print_left(help_win, row, col, HELP_WIN_WIDTH, "MENU options", COLOR_PAIR(5));
3039 	print_left(help_win, ++row, ++col, HELP_WIN_WIDTH, "[q] Quit		- quit this application",
3040 		   COLOR_PAIR(10));
3041 	print_left(help_win, ++row, col,  HELP_WIN_WIDTH,
3042 		   "[Tab] Next tab	- switch to next tab", COLOR_PAIR(10));
3043 	print_left(help_win, ++row, col,  HELP_WIN_WIDTH,
3044 		   "[1-3] Select tab	- switch to THREADS, POLLERS or CORES tab", COLOR_PAIR(10));
3045 	print_left(help_win, ++row, col,  HELP_WIN_WIDTH,
3046 		   "[PgUp] Previous page	- scroll up to previous page", COLOR_PAIR(10));
3047 	print_left(help_win, ++row, col,  HELP_WIN_WIDTH,
3048 		   "[PgDown] Next page	- scroll down to next page", COLOR_PAIR(10));
3049 	print_left(help_win, ++row, col,  HELP_WIN_WIDTH,
3050 		   "[Up] Arrow key	- go to previous data row", COLOR_PAIR(10));
3051 	print_left(help_win, ++row, col,  HELP_WIN_WIDTH,
3052 		   "[Down] Arrow key	- go to next data row", COLOR_PAIR(10));
3053 	print_left(help_win, ++row, col,  HELP_WIN_WIDTH,
3054 		   "[Right] Arrow key	- go to second sorting window", COLOR_PAIR(10));
3055 	print_left(help_win, ++row, col,  HELP_WIN_WIDTH,
3056 		   "[Left] Arrow key	- close second sorting window", COLOR_PAIR(10));
3057 	print_left(help_win, ++row, col,  HELP_WIN_WIDTH,
3058 		   "[c] Columns		- choose data columns to display", COLOR_PAIR(10));
3059 	print_left(help_win, ++row, col,  HELP_WIN_WIDTH,
3060 		   "[s] Sorting		- change sorting by column", COLOR_PAIR(10));
3061 	print_left(help_win, ++row, col,  HELP_WIN_WIDTH,
3062 		   "[r] Refresh rate	- set refresh rate <0, 255> in seconds", COLOR_PAIR(10));
3063 	print_left(help_win, ++row, desc_second_row_col,  HELP_WIN_WIDTH, "that value in seconds",
3064 		   COLOR_PAIR(10));
3065 	print_left(help_win, ++row, col,  HELP_WIN_WIDTH,
3066 		   "[Enter] Item details	- show current data row details (Enter to open, Esc to close)",
3067 		   COLOR_PAIR(10));
3068 	print_left(help_win, ++row, col,  HELP_WIN_WIDTH,
3069 		   "[t] Total/Interval	- switch to display data measured from the start of SPDK", COLOR_PAIR(10));
3070 	print_left(help_win, ++row, desc_second_row_col,  HELP_WIN_WIDTH,
3071 		   "application or last refresh", COLOR_PAIR(10));
3072 	print_left(help_win, ++row, col,  HELP_WIN_WIDTH,
3073 		   "[g] Scheduler pop-up - display current scheduler information", COLOR_PAIR(10));
3074 	print_left(help_win, ++row, col,  HELP_WIN_WIDTH, "[h] Help		- show this help window",
3075 		   COLOR_PAIR(10));
3076 
3077 	/* Footer */
3078 	mvwhline(help_win, HELP_WIN_HEIGHT - 3, 1, ACS_HLINE, HELP_WIN_WIDTH - 2);
3079 	mvwaddch(help_win, HELP_WIN_HEIGHT - 3, HELP_WIN_WIDTH, ACS_RTEE);
3080 
3081 	print_in_middle(help_win, HELP_WIN_HEIGHT - 2, header_footer_col, HELP_WIN_WIDTH,
3082 			"[Esc] Close this window", COLOR_PAIR(10));
3083 
3084 	refresh();
3085 	wrefresh(help_win);
3086 
3087 	while (!stop_loop) {
3088 		c = wgetch(help_win);
3089 
3090 		switch (c) {
3091 		case 27: /* ESC */
3092 			stop_loop = true;
3093 			break;
3094 		default:
3095 			break;
3096 		}
3097 	}
3098 
3099 	del_panel(help_panel);
3100 	delwin(help_win);
3101 
3102 }
3103 
3104 static void
3105 refresh_after_popup(uint8_t active_tab, uint8_t *max_pages, uint8_t current_page)
3106 {
3107 	int i;
3108 
3109 	/* After closing pop-up there would be unrefreshed parts
3110 	 * of the tab, so this is to refresh them */
3111 	draw_tabs(active_tab, g_current_sort_col[active_tab], g_current_sort_col2[active_tab]);
3112 	pthread_mutex_lock(&g_thread_lock);
3113 	*max_pages = refresh_tab(active_tab, current_page);
3114 	pthread_mutex_unlock(&g_thread_lock);
3115 	top_panel(g_panels[active_tab]);
3116 
3117 	for (i = 0; i < NUMBER_OF_TABS; i++) {
3118 		wclear(g_tab_win[i]);
3119 		wresize(g_tab_win[i], TAB_WIN_HEIGHT,
3120 			(g_max_col - (TABS_SPACING * NUMBER_OF_TABS)) / NUMBER_OF_TABS);
3121 		mvwin(g_tab_win[i], TAB_WIN_LOCATION_ROW, 1 + (g_max_col / NUMBER_OF_TABS) * i);
3122 		draw_tab_win(i);
3123 	}
3124 
3125 	update_panels();
3126 	refresh();
3127 }
3128 
3129 static void
3130 show_stats(pthread_t *data_thread)
3131 {
3132 	const int CURRENT_PAGE_STR_LEN = 50;
3133 	long int time_last, time_dif;
3134 	struct timespec time_now;
3135 	int c;
3136 	uint8_t active_tab = THREADS_TAB;
3137 	uint8_t current_page = 0;
3138 	uint8_t max_pages = 1;
3139 	uint64_t i;
3140 	char current_page_str[CURRENT_PAGE_STR_LEN];
3141 	bool force_refresh = true;
3142 
3143 	clock_gettime(CLOCK_MONOTONIC, &time_now);
3144 	time_last = time_now.tv_sec;
3145 
3146 	switch_tab(THREADS_TAB);
3147 
3148 	while (1) {
3149 		check_resize_interface(active_tab, &current_page);
3150 
3151 		clock_gettime(CLOCK_MONOTONIC, &time_now);
3152 		time_dif = time_now.tv_sec - time_last;
3153 		if (time_dif < 0) {
3154 			time_dif = g_sleep_time;
3155 		}
3156 
3157 		if (time_dif >= g_sleep_time || force_refresh) {
3158 			time_last = time_now.tv_sec;
3159 			pthread_mutex_lock(&g_thread_lock);
3160 			max_pages = refresh_tab(active_tab, current_page);
3161 			pthread_mutex_unlock(&g_thread_lock);
3162 
3163 			snprintf(current_page_str, CURRENT_PAGE_STR_LEN - 1, "Page: %d/%d", current_page + 1, max_pages);
3164 			mvprintw(g_max_row - 1, 1, "%s", current_page_str);
3165 
3166 			refresh();
3167 		}
3168 
3169 		c = getch();
3170 		if (c == 'q') {
3171 			pthread_mutex_lock(&g_thread_lock);
3172 			g_quit_app = true;
3173 			pthread_mutex_unlock(&g_thread_lock);
3174 			break;
3175 		}
3176 
3177 		force_refresh = true;
3178 
3179 		switch (c) {
3180 		case '1':
3181 		case '2':
3182 		case '3':
3183 			active_tab = c - '1';
3184 			current_page = 0;
3185 			g_selected_row = 0;
3186 			switch_tab(active_tab);
3187 			break;
3188 		case '\t':
3189 			if (active_tab < NUMBER_OF_TABS - 1) {
3190 				active_tab++;
3191 			} else {
3192 				active_tab = THREADS_TAB;
3193 			}
3194 			g_selected_row = 0;
3195 			current_page = 0;
3196 			switch_tab(active_tab);
3197 			break;
3198 		case 's':
3199 			sort_type2(active_tab, COL_THREADS_NONE);
3200 			change_sorting(active_tab, 0, NULL);
3201 			break;
3202 		case 'c':
3203 			filter_columns(active_tab);
3204 			break;
3205 		case 'r':
3206 			change_refresh_rate();
3207 			break;
3208 		case 't':
3209 			g_interval_data = !g_interval_data;
3210 			break;
3211 		case 'g':
3212 			show_scheduler(active_tab, current_page);
3213 			break;
3214 		case KEY_NPAGE: /* PgDown */
3215 			if (current_page + 1 < max_pages) {
3216 				current_page++;
3217 			}
3218 			wclear(g_tabs[active_tab]);
3219 			g_selected_row = 0;
3220 			draw_tabs(active_tab, g_current_sort_col[active_tab], g_current_sort_col2[active_tab]);
3221 			break;
3222 		case KEY_PPAGE: /* PgUp */
3223 			if (current_page > 0) {
3224 				current_page--;
3225 			}
3226 			wclear(g_tabs[active_tab]);
3227 			g_selected_row = 0;
3228 			draw_tabs(active_tab, g_current_sort_col[active_tab], g_current_sort_col2[active_tab]);
3229 			break;
3230 		case KEY_UP: /* Arrow up */
3231 			if (g_selected_row > 0) {
3232 				g_selected_row--;
3233 			} else if (g_selected_row == 0) {
3234 				if (current_page > 0) {
3235 					current_page--;
3236 					g_selected_row = g_max_data_rows - 1;
3237 					wclear(g_tabs[active_tab]);
3238 					draw_tabs(active_tab, g_current_sort_col[active_tab], g_current_sort_col2[active_tab]);
3239 				}
3240 			}
3241 			break;
3242 		case KEY_DOWN: /* Arrow down */
3243 			if (g_selected_row < g_max_selected_row) {
3244 				g_selected_row++;
3245 			} else if (g_selected_row == g_max_selected_row) {
3246 				if (current_page + 1 < max_pages) {
3247 					current_page++;
3248 					g_selected_row = 0;
3249 					wclear(g_tabs[active_tab]);
3250 					draw_tabs(active_tab, g_current_sort_col[active_tab], g_current_sort_col2[active_tab]);
3251 				}
3252 			}
3253 			break;
3254 		case 10: /* Enter */
3255 			if (active_tab == THREADS_TAB) {
3256 				show_thread(current_page, active_tab);
3257 			} else if (active_tab == CORES_TAB) {
3258 				show_core(current_page, active_tab);
3259 			} else if (active_tab == POLLERS_TAB) {
3260 				show_poller(current_page, active_tab);
3261 			}
3262 			snprintf(current_page_str, CURRENT_PAGE_STR_LEN - 1, "Page: %d/%d", current_page + 1, max_pages);
3263 			mvprintw(g_max_row - 1, 1, "%s", current_page_str);
3264 			refresh_after_popup(active_tab, &max_pages, current_page);
3265 			break;
3266 		case 'h':
3267 			help_window_display();
3268 			refresh_after_popup(active_tab, &max_pages, current_page);
3269 			break;
3270 		default:
3271 			force_refresh = false;
3272 			break;
3273 		}
3274 	}
3275 
3276 	pthread_join(*data_thread, NULL);
3277 
3278 	free_poller_history();
3279 
3280 	/* Free memory holding current data states before quitting application */
3281 	for (i = 0; i < g_last_pollers_count; i++) {
3282 		free_rpc_poller(&g_pollers_info[i]);
3283 	}
3284 	for (i = 0; i < g_last_threads_count; i++) {
3285 		free_rpc_threads_stats(&g_threads_info[i]);
3286 	}
3287 	free_rpc_core_info(g_cores_info, g_last_cores_count);
3288 	free_rpc_scheduler(&g_scheduler_info);
3289 }
3290 
3291 static void
3292 draw_interface(void)
3293 {
3294 	int i;
3295 	uint16_t required_size =  WINDOW_HEADER + 1;
3296 
3297 	getmaxyx(stdscr, g_max_row, g_max_col);
3298 	g_max_row = spdk_max(g_max_row, required_size);
3299 	g_data_win_size = g_max_row - required_size;
3300 	g_max_data_rows = g_max_row - WINDOW_HEADER;
3301 
3302 	g_menu_win = newwin(MENU_WIN_HEIGHT, g_max_col, g_max_row - MENU_WIN_HEIGHT - 1,
3303 			    MENU_WIN_LOCATION_COL);
3304 	assert(g_menu_win != NULL);
3305 	draw_menu_win();
3306 
3307 	for (i = 0; i < NUMBER_OF_TABS; i++) {
3308 		g_tab_win[i] = newwin(TAB_WIN_HEIGHT, g_max_col / NUMBER_OF_TABS - TABS_SPACING,
3309 				      TAB_WIN_LOCATION_ROW, g_max_col / NUMBER_OF_TABS * i + 1);
3310 		assert(g_tab_win[i] != NULL);
3311 		draw_tab_win(i);
3312 
3313 		g_tabs[i] = newwin(g_max_row - MENU_WIN_HEIGHT - TAB_WIN_HEIGHT - 2, g_max_col, TABS_LOCATION_ROW,
3314 				   TABS_LOCATION_COL);
3315 		draw_tabs(i, g_current_sort_col[i], g_current_sort_col2[i]);
3316 		g_panels[i] = new_panel(g_tabs[i]);
3317 		assert(g_panels[i] != NULL);
3318 	}
3319 
3320 	update_panels();
3321 	doupdate();
3322 }
3323 
3324 static void
3325 finish(int sig)
3326 {
3327 	/* End ncurses mode */
3328 	endwin();
3329 	spdk_jsonrpc_client_close(g_rpc_client);
3330 	exit(0);
3331 }
3332 
3333 static void
3334 setup_ncurses(void)
3335 {
3336 	clear();
3337 	noecho();
3338 	timeout(1);
3339 	curs_set(0);
3340 	keypad(stdscr, TRUE);
3341 	start_color();
3342 	init_pair(1, COLOR_BLACK, COLOR_GREEN);
3343 	init_pair(2, COLOR_BLACK, COLOR_WHITE);
3344 	init_pair(3, COLOR_YELLOW, COLOR_BLACK);
3345 	init_pair(4, COLOR_BLACK, COLOR_YELLOW);
3346 	init_pair(5, COLOR_GREEN, COLOR_BLACK);
3347 	init_pair(6, COLOR_RED, COLOR_BLACK);
3348 	init_pair(7, COLOR_BLUE, COLOR_BLACK);
3349 	init_pair(8, COLOR_RED, COLOR_WHITE);
3350 	init_pair(9, COLOR_BLUE, COLOR_WHITE);
3351 	init_pair(10, COLOR_WHITE, COLOR_BLACK);
3352 
3353 	if (has_colors() == FALSE) {
3354 		endwin();
3355 		printf("Your terminal does not support color\n");
3356 		exit(1);
3357 	}
3358 
3359 	/* Handle signals to exit gracefully cleaning up ncurses */
3360 	(void) signal(SIGINT, finish);
3361 	(void) signal(SIGPIPE, finish);
3362 	(void) signal(SIGABRT, finish);
3363 }
3364 
3365 static void
3366 usage(const char *program_name)
3367 {
3368 	printf("%s [options]", program_name);
3369 	printf("\n");
3370 	printf("options:\n");
3371 	printf(" -r <path>  RPC connect address (default: /var/tmp/spdk.sock)\n");
3372 	printf(" -h         show this usage\n");
3373 }
3374 
3375 static int
3376 rpc_decode_tick_rate(struct spdk_json_val *val, uint64_t *tick_rate)
3377 {
3378 	struct t_rate {
3379 		uint64_t tr;
3380 	};
3381 
3382 	const struct spdk_json_object_decoder rpc_tick_rate_decoder[] = {
3383 		{"tick_rate", offsetof(struct t_rate, tr), spdk_json_decode_uint64}
3384 	};
3385 
3386 	int rc;
3387 	struct t_rate tmp;
3388 
3389 	rc = spdk_json_decode_object_relaxed(val, rpc_tick_rate_decoder,
3390 					     SPDK_COUNTOF(rpc_tick_rate_decoder), &tmp);
3391 
3392 	*tick_rate = tmp.tr;
3393 
3394 	return rc;
3395 }
3396 
3397 static int
3398 wait_init(pthread_t *data_thread)
3399 {
3400 	struct spdk_jsonrpc_client_response *json_resp = NULL;
3401 	char *uninit_log = "Waiting for SPDK target application to initialize...",
3402 	      *uninit_error = "Unable to read SPDK application state!";
3403 	int c, max_col, rc = 0;
3404 	uint64_t tick_rate;
3405 
3406 	max_col = getmaxx(stdscr);
3407 	print_in_middle(stdscr, FIRST_DATA_ROW, 1, max_col, uninit_log, COLOR_PAIR(5));
3408 	rc = rpc_send_req("framework_wait_init", &json_resp);
3409 	if (rc) {
3410 		while (1) {
3411 			print_in_middle(stdscr, FIRST_DATA_ROW, 1, max_col, uninit_error, COLOR_PAIR(8));
3412 			c = getch();
3413 			if (c == 'q') {
3414 				return -1;
3415 			}
3416 		}
3417 	}
3418 
3419 	spdk_jsonrpc_client_free_response(json_resp);
3420 
3421 	rc = pthread_mutex_init(&g_thread_lock, NULL);
3422 	if (rc) {
3423 		fprintf(stderr, "mutex lock failed to initialize: %d\n", errno);
3424 		return -1;
3425 	}
3426 
3427 	memset(&g_threads_info, 0, sizeof(struct rpc_thread_info) * RPC_MAX_THREADS);
3428 	memset(&g_cores_info, 0, sizeof(struct rpc_core_info) * RPC_MAX_CORES);
3429 
3430 	/* Decode tick rate */
3431 	rc = rpc_send_req("framework_get_reactors", &json_resp);
3432 	if (rc) {
3433 		return rc;
3434 	}
3435 
3436 	if (rpc_decode_tick_rate(json_resp->result, &tick_rate)) {
3437 		spdk_jsonrpc_client_free_response(json_resp);
3438 		return -EINVAL;
3439 	}
3440 
3441 	spdk_jsonrpc_client_free_response(json_resp);
3442 
3443 	g_tick_rate = tick_rate;
3444 
3445 	/* This is to get first batch of data for display functions.
3446 	 * Since data thread makes RPC calls that take more time than
3447 	 * startup of display functions on main thread, without these
3448 	 * calls both threads would be subject to a race condition. */
3449 	rc = get_thread_data();
3450 	if (rc) {
3451 		return -1;
3452 	}
3453 
3454 	rc = get_pollers_data();
3455 	if (rc) {
3456 		return -1;
3457 	}
3458 
3459 	rc = get_cores_data();
3460 	if (rc) {
3461 		return -1;
3462 	}
3463 
3464 	rc = pthread_create(data_thread, NULL, &data_thread_routine, NULL);
3465 	if (rc) {
3466 		fprintf(stderr, "data thread creation failed: %d\n", errno);
3467 		return -1;
3468 	}
3469 	return 0;
3470 }
3471 
3472 int
3473 main(int argc, char **argv)
3474 {
3475 	int op, rc;
3476 	char *socket = SPDK_DEFAULT_RPC_ADDR;
3477 	pthread_t data_thread;
3478 
3479 	while ((op = getopt(argc, argv, "r:h")) != -1) {
3480 		switch (op) {
3481 		case 'r':
3482 			socket = optarg;
3483 			break;
3484 		default:
3485 			usage(argv[0]);
3486 			return op == 'h' ? 0 : 1;
3487 		}
3488 	}
3489 
3490 	g_rpc_client = spdk_jsonrpc_client_connect(socket, socket[0] == '/' ? AF_UNIX : AF_INET);
3491 	if (!g_rpc_client) {
3492 		fprintf(stderr, "spdk_jsonrpc_client_connect() failed: %d\n", errno);
3493 		return 1;
3494 	}
3495 
3496 	initscr();
3497 	init_str_len();
3498 	setup_ncurses();
3499 	draw_interface();
3500 
3501 	rc = wait_init(&data_thread);
3502 	if (!rc) {
3503 		show_stats(&data_thread);
3504 	}
3505 
3506 	finish(0);
3507 
3508 	return (0);
3509 }
3510