xref: /spdk/app/spdk_top/spdk_top.c (revision cc6920a4763d4b9a43aa40583c8397d8f14fa100)
1 /*-
2  *   BSD LICENSE
3  *
4  *   Copyright (c) Intel Corporation.
5  *   All rights reserved.
6  *
7  *   Redistribution and use in source and binary forms, with or without
8  *   modification, are permitted provided that the following conditions
9  *   are met:
10  *
11  *     * Redistributions of source code must retain the above copyright
12  *       notice, this list of conditions and the following disclaimer.
13  *     * Redistributions in binary form must reproduce the above copyright
14  *       notice, this list of conditions and the following disclaimer in
15  *       the documentation and/or other materials provided with the
16  *       distribution.
17  *     * Neither the name of Intel Corporation nor the names of its
18  *       contributors may be used to endorse or promote products derived
19  *       from this software without specific prior written permission.
20  *
21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32  */
33 
34 #include "spdk/stdinc.h"
35 #include "spdk/jsonrpc.h"
36 #include "spdk/rpc.h"
37 #include "spdk/event.h"
38 #include "spdk/util.h"
39 #include "spdk/env.h"
40 
41 #if defined __has_include
42 #if __has_include(<ncurses/panel.h>)
43 #include <ncurses/ncurses.h>
44 #include <ncurses/panel.h>
45 #include <ncurses/menu.h>
46 #else
47 #include <ncurses.h>
48 #include <panel.h>
49 #include <menu.h>
50 #endif
51 #else
52 #include <ncurses.h>
53 #include <panel.h>
54 #include <menu.h>
55 #endif
56 
57 #define RPC_MAX_THREADS 1024
58 #define RPC_MAX_POLLERS 1024
59 #define RPC_MAX_CORES 255
60 #define MAX_THREAD_NAME 128
61 #define MAX_POLLER_NAME 128
62 #define MAX_THREADS 4096
63 #define RR_MAX_VALUE 255
64 
65 #define MAX_STRING_LEN 12289 /* 3x 4k monitors + 1 */
66 #define TAB_WIN_HEIGHT 3
67 #define TAB_WIN_LOCATION_ROW 1
68 #define TABS_SPACING 2
69 #define TABS_LOCATION_ROW 4
70 #define TABS_LOCATION_COL 0
71 #define TABS_DATA_START_ROW 3
72 #define TABS_DATA_START_COL 2
73 #define TABS_COL_COUNT 10
74 #define MENU_WIN_HEIGHT 3
75 #define MENU_WIN_SPACING 4
76 #define MENU_WIN_LOCATION_COL 0
77 #define RR_WIN_WIDTH 32
78 #define RR_WIN_HEIGHT 5
79 #define MAX_THREAD_NAME_LEN 26
80 #define MAX_THREAD_COUNT_STR_LEN 14
81 #define MAX_POLLER_NAME_LEN 36
82 #define MAX_POLLER_COUNT_STR_LEN 16
83 #define MAX_POLLER_TYPE_STR_LEN 8
84 #define MAX_POLLER_IND_STR_LEN 28
85 #define MAX_CORE_MASK_STR_LEN 16
86 #define MAX_CORE_STR_LEN 6
87 #define MAX_CORE_FREQ_STR_LEN 18
88 #define MAX_TIME_STR_LEN 12
89 #define MAX_CPU_STR_LEN 8
90 #define MAX_POLLER_RUN_COUNT 20
91 #define MAX_PERIOD_STR_LEN 12
92 #define MAX_INTR_LEN 6
93 #define WINDOW_HEADER 12
94 #define FROM_HEX 16
95 #define THREAD_WIN_WIDTH 69
96 #define THREAD_WIN_HEIGHT 9
97 #define THREAD_WIN_FIRST_COL 2
98 #define CORE_WIN_FIRST_COL 16
99 #define CORE_WIN_WIDTH 48
100 #define CORE_WIN_HEIGHT 11
101 #define POLLER_WIN_HEIGHT 8
102 #define POLLER_WIN_WIDTH 64
103 #define POLLER_WIN_FIRST_COL 14
104 #define FIRST_DATA_ROW 7
105 #define HELP_WIN_WIDTH 88
106 #define HELP_WIN_HEIGHT 22
107 
108 enum tabs {
109 	THREADS_TAB,
110 	POLLERS_TAB,
111 	CORES_TAB,
112 	NUMBER_OF_TABS,
113 };
114 
115 enum column_threads_type {
116 	COL_THREADS_NAME,
117 	COL_THREADS_CORE,
118 	COL_THREADS_ACTIVE_POLLERS,
119 	COL_THREADS_TIMED_POLLERS,
120 	COL_THREADS_PAUSED_POLLERS,
121 	COL_THREADS_IDLE_TIME,
122 	COL_THREADS_BUSY_TIME,
123 	COL_THREADS_CPU_USAGE,
124 	COL_THREADS_NONE = 255,
125 };
126 
127 enum column_pollers_type {
128 	COL_POLLERS_NAME,
129 	COL_POLLERS_TYPE,
130 	COL_POLLERS_THREAD_NAME,
131 	COL_POLLERS_RUN_COUNTER,
132 	COL_POLLERS_PERIOD,
133 	COL_POLLERS_BUSY_COUNT,
134 	COL_POLLERS_NONE = 255,
135 };
136 
137 enum column_cores_type {
138 	COL_CORES_CORE,
139 	COL_CORES_THREADS,
140 	COL_CORES_POLLERS,
141 	COL_CORES_IDLE_TIME,
142 	COL_CORES_BUSY_TIME,
143 	COL_CORES_CORE_FREQ,
144 	COL_CORES_INTR,
145 	COL_CORES_CPU_USAGE,
146 	COL_CORES_NONE = 255,
147 };
148 
149 enum spdk_poller_type {
150 	SPDK_ACTIVE_POLLER,
151 	SPDK_TIMED_POLLER,
152 	SPDK_PAUSED_POLLER,
153 	SPDK_POLLER_TYPES_COUNT,
154 };
155 
156 struct col_desc {
157 	const char *name;
158 	uint8_t name_len;
159 	uint8_t max_data_string;
160 	bool disabled;
161 };
162 
163 struct run_counter_history {
164 	uint64_t poller_id;
165 	uint64_t thread_id;
166 	uint64_t last_run_counter;
167 	uint64_t last_busy_counter;
168 	TAILQ_ENTRY(run_counter_history) link;
169 };
170 
171 uint8_t g_sleep_time = 1;
172 uint16_t g_selected_row;
173 uint16_t g_max_selected_row;
174 uint64_t g_tick_rate;
175 const char *poller_type_str[SPDK_POLLER_TYPES_COUNT] = {"Active", "Timed", "Paused"};
176 const char *g_tab_title[NUMBER_OF_TABS] = {"[1] THREADS", "[2] POLLERS", "[3] CORES"};
177 struct spdk_jsonrpc_client *g_rpc_client;
178 static TAILQ_HEAD(, run_counter_history) g_run_counter_history = TAILQ_HEAD_INITIALIZER(
179 			g_run_counter_history);
180 WINDOW *g_menu_win, *g_tab_win[NUMBER_OF_TABS], *g_tabs[NUMBER_OF_TABS];
181 PANEL *g_panels[NUMBER_OF_TABS];
182 uint16_t g_max_row, g_max_col;
183 uint16_t g_data_win_size, g_max_data_rows;
184 uint32_t g_last_threads_count, g_last_pollers_count, g_last_cores_count;
185 uint8_t g_current_sort_col[NUMBER_OF_TABS] = {COL_THREADS_NAME, COL_POLLERS_NAME, COL_CORES_CORE};
186 uint8_t g_current_sort_col2[NUMBER_OF_TABS] = {COL_THREADS_NONE, COL_POLLERS_NONE, COL_CORES_NONE};
187 bool g_interval_data = true;
188 bool g_quit_app = false;
189 pthread_mutex_t g_thread_lock;
190 static struct col_desc g_col_desc[NUMBER_OF_TABS][TABS_COL_COUNT] = {
191 	{	{.name = "Thread name", .max_data_string = MAX_THREAD_NAME_LEN},
192 		{.name = "Core", .max_data_string = MAX_CORE_STR_LEN},
193 		{.name = "Active pollers", .max_data_string = MAX_POLLER_COUNT_STR_LEN},
194 		{.name = "Timed pollers", .max_data_string = MAX_POLLER_COUNT_STR_LEN},
195 		{.name = "Paused pollers", .max_data_string = MAX_POLLER_COUNT_STR_LEN},
196 		{.name = "Idle [us]", .max_data_string = MAX_TIME_STR_LEN},
197 		{.name = "Busy [us]", .max_data_string = MAX_TIME_STR_LEN},
198 		{.name = "CPU %", .max_data_string = MAX_CPU_STR_LEN},
199 		{.name = (char *)NULL}
200 	},
201 	{	{.name = "Poller name", .max_data_string = MAX_POLLER_NAME_LEN},
202 		{.name = "Type", .max_data_string = MAX_POLLER_TYPE_STR_LEN},
203 		{.name = "On thread", .max_data_string = MAX_THREAD_NAME_LEN},
204 		{.name = "Run count", .max_data_string = MAX_POLLER_RUN_COUNT},
205 		{.name = "Period [us]", .max_data_string = MAX_PERIOD_STR_LEN},
206 		{.name = "Status (busy count)", .max_data_string = MAX_POLLER_IND_STR_LEN},
207 		{.name = (char *)NULL}
208 	},
209 	{	{.name = "Core", .max_data_string = MAX_CORE_STR_LEN},
210 		{.name = "Thread count", .max_data_string = MAX_THREAD_COUNT_STR_LEN},
211 		{.name = "Poller count", .max_data_string = MAX_POLLER_COUNT_STR_LEN},
212 		{.name = "Idle [us]", .max_data_string = MAX_TIME_STR_LEN},
213 		{.name = "Busy [us]", .max_data_string = MAX_TIME_STR_LEN},
214 		{.name = "Frequency [MHz]", .max_data_string = MAX_CORE_FREQ_STR_LEN},
215 		{.name = "Intr", .max_data_string = MAX_INTR_LEN},
216 		{.name = "CPU %", .max_data_string = MAX_CPU_STR_LEN},
217 		{.name = (char *)NULL}
218 	}
219 };
220 
221 struct rpc_thread_info {
222 	char *name;
223 	uint64_t id;
224 	int core_num;
225 	char *cpumask;
226 	uint64_t busy;
227 	uint64_t last_busy;
228 	uint64_t idle;
229 	uint64_t last_idle;
230 	uint64_t active_pollers_count;
231 	uint64_t timed_pollers_count;
232 	uint64_t paused_pollers_count;
233 };
234 
235 struct rpc_poller_info {
236 	char *name;
237 	char *state;
238 	uint64_t id;
239 	uint64_t run_count;
240 	uint64_t busy_count;
241 	uint64_t period_ticks;
242 	enum spdk_poller_type type;
243 	char thread_name[MAX_THREAD_NAME];
244 	uint64_t thread_id;
245 };
246 
247 struct rpc_core_thread_info {
248 	char *name;
249 	uint64_t id;
250 	char *cpumask;
251 	uint64_t elapsed;
252 };
253 
254 struct rpc_core_threads {
255 	uint64_t threads_count;
256 	struct rpc_core_thread_info *thread;
257 };
258 
259 struct rpc_core_info {
260 	uint32_t lcore;
261 	uint64_t pollers_count;
262 	uint64_t busy;
263 	uint64_t idle;
264 	uint32_t core_freq;
265 	uint64_t last_idle;
266 	uint64_t last_busy;
267 	bool in_interrupt;
268 	struct rpc_core_threads threads;
269 };
270 
271 struct rpc_thread_info g_threads_info[RPC_MAX_THREADS];
272 struct rpc_poller_info g_pollers_info[RPC_MAX_POLLERS];
273 struct rpc_core_info g_cores_info[RPC_MAX_CORES];
274 
275 static void
276 init_str_len(void)
277 {
278 	int i, j;
279 
280 	for (i = 0; i < NUMBER_OF_TABS; i++) {
281 		for (j = 0; g_col_desc[i][j].name != NULL; j++) {
282 			g_col_desc[i][j].name_len = strlen(g_col_desc[i][j].name);
283 		}
284 	}
285 }
286 
287 static void
288 free_rpc_threads_stats(struct rpc_thread_info *req)
289 {
290 	free(req->name);
291 	req->name = NULL;
292 	free(req->cpumask);
293 	req->cpumask = NULL;
294 }
295 
296 static const struct spdk_json_object_decoder rpc_thread_info_decoders[] = {
297 	{"name", offsetof(struct rpc_thread_info, name), spdk_json_decode_string},
298 	{"id", offsetof(struct rpc_thread_info, id), spdk_json_decode_uint64},
299 	{"cpumask", offsetof(struct rpc_thread_info, cpumask), spdk_json_decode_string},
300 	{"busy", offsetof(struct rpc_thread_info, busy), spdk_json_decode_uint64},
301 	{"idle", offsetof(struct rpc_thread_info, idle), spdk_json_decode_uint64},
302 	{"active_pollers_count", offsetof(struct rpc_thread_info, active_pollers_count), spdk_json_decode_uint64},
303 	{"timed_pollers_count", offsetof(struct rpc_thread_info, timed_pollers_count), spdk_json_decode_uint64},
304 	{"paused_pollers_count", offsetof(struct rpc_thread_info, paused_pollers_count), spdk_json_decode_uint64},
305 };
306 
307 static int
308 rpc_decode_threads_array(struct spdk_json_val *val, struct rpc_thread_info *out,
309 			 uint64_t *current_threads_count)
310 {
311 	struct spdk_json_val *thread = val;
312 	uint64_t i = 0;
313 	int rc;
314 
315 	/* Fetch the beginning of threads array */
316 	rc = spdk_json_find_array(thread, "threads", NULL, &thread);
317 	if (rc) {
318 		printf("Could not fetch threads array from JSON.\n");
319 		goto end;
320 	}
321 
322 	for (thread = spdk_json_array_first(thread); thread != NULL; thread = spdk_json_next(thread)) {
323 		rc = spdk_json_decode_object(thread, rpc_thread_info_decoders,
324 					     SPDK_COUNTOF(rpc_thread_info_decoders), &out[i]);
325 		if (rc) {
326 			printf("Could not decode thread object from JSON.\n");
327 			break;
328 		}
329 
330 		i++;
331 	}
332 
333 end:
334 
335 	*current_threads_count = i;
336 	return rc;
337 }
338 
339 static void
340 free_rpc_poller(struct rpc_poller_info *poller)
341 {
342 	free(poller->name);
343 	poller->name = NULL;
344 	free(poller->state);
345 	poller->state = NULL;
346 }
347 
348 static void
349 free_rpc_core_info(struct rpc_core_info *core_info, size_t size)
350 {
351 	struct rpc_core_threads *threads;
352 	struct rpc_core_thread_info *thread;
353 	uint64_t i, core_number;
354 
355 	for (core_number = 0; core_number < size; core_number++) {
356 		threads = &core_info[core_number].threads;
357 		for (i = 0; i < threads->threads_count; i++) {
358 			thread = &threads->thread[i];
359 			free(thread->name);
360 			free(thread->cpumask);
361 		}
362 		free(threads->thread);
363 	}
364 }
365 
366 static const struct spdk_json_object_decoder rpc_pollers_decoders[] = {
367 	{"name", offsetof(struct rpc_poller_info, name), spdk_json_decode_string},
368 	{"state", offsetof(struct rpc_poller_info, state), spdk_json_decode_string},
369 	{"id", offsetof(struct rpc_poller_info, id), spdk_json_decode_uint64},
370 	{"run_count", offsetof(struct rpc_poller_info, run_count), spdk_json_decode_uint64},
371 	{"busy_count", offsetof(struct rpc_poller_info, busy_count), spdk_json_decode_uint64},
372 	{"period_ticks", offsetof(struct rpc_poller_info, period_ticks), spdk_json_decode_uint64, true},
373 };
374 
375 static int
376 rpc_decode_pollers_array(struct spdk_json_val *poller, struct rpc_poller_info *out,
377 			 uint64_t *poller_count,
378 			 const char *thread_name, uint64_t thread_name_length, uint64_t thread_id,
379 			 enum spdk_poller_type poller_type)
380 {
381 	int rc;
382 
383 	for (poller = spdk_json_array_first(poller); poller != NULL; poller = spdk_json_next(poller)) {
384 		out[*poller_count].thread_id = thread_id;
385 		memcpy(out[*poller_count].thread_name, thread_name, sizeof(char) * thread_name_length);
386 		out[*poller_count].type = poller_type;
387 
388 		rc = spdk_json_decode_object(poller, rpc_pollers_decoders,
389 					     SPDK_COUNTOF(rpc_pollers_decoders), &out[*poller_count]);
390 		if (rc) {
391 			printf("Could not decode poller object from JSON.\n");
392 			return rc;
393 		}
394 
395 		(*poller_count)++;
396 		if (*poller_count == RPC_MAX_POLLERS) {
397 			return -1;
398 		}
399 	}
400 
401 	return 0;
402 }
403 
404 static const struct spdk_json_object_decoder rpc_thread_pollers_decoders[] = {
405 	{"name", offsetof(struct rpc_thread_info, name), spdk_json_decode_string},
406 	{"id", offsetof(struct rpc_thread_info, id), spdk_json_decode_uint64},
407 };
408 
409 static int
410 rpc_decode_pollers_threads_array(struct spdk_json_val *val, struct rpc_poller_info *out,
411 				 uint32_t *num_pollers)
412 {
413 	struct spdk_json_val *thread = val, *poller;
414 	/* This is a temporary poller structure to hold thread name and id.
415 	 * It is filled with data only once per thread change and then
416 	 * that memory is copied to each poller running on that thread. */
417 	struct rpc_thread_info thread_info = {};
418 	uint64_t poller_count = 0, i, thread_name_length;
419 	int rc;
420 	const char *poller_typenames[] = { "active_pollers", "timed_pollers", "paused_pollers" };
421 	enum spdk_poller_type poller_types[] = { SPDK_ACTIVE_POLLER, SPDK_TIMED_POLLER, SPDK_PAUSED_POLLER };
422 
423 	/* Fetch the beginning of threads array */
424 	rc = spdk_json_find_array(thread, "threads", NULL, &thread);
425 	if (rc) {
426 		printf("Could not fetch threads array from JSON.\n");
427 		goto end;
428 	}
429 
430 	for (thread = spdk_json_array_first(thread); thread != NULL; thread = spdk_json_next(thread)) {
431 		rc = spdk_json_decode_object_relaxed(thread, rpc_thread_pollers_decoders,
432 						     SPDK_COUNTOF(rpc_thread_pollers_decoders), &thread_info);
433 		if (rc) {
434 			printf("Could not decode thread info from JSON.\n");
435 			goto end;
436 		}
437 
438 		thread_name_length = strlen(thread_info.name);
439 
440 		for (i = 0; i < SPDK_COUNTOF(poller_types); i++) {
441 			/* Find poller array */
442 			rc = spdk_json_find(thread, poller_typenames[i], NULL, &poller,
443 					    SPDK_JSON_VAL_ARRAY_BEGIN);
444 			if (rc) {
445 				printf("Could not fetch pollers array from JSON.\n");
446 				goto end;
447 			}
448 
449 			rc = rpc_decode_pollers_array(poller, out, &poller_count, thread_info.name,
450 						      thread_name_length,
451 						      thread_info.id, poller_types[i]);
452 			if (rc) {
453 				printf("Could not decode the first object in pollers array.\n");
454 				goto end;
455 			}
456 		}
457 	}
458 
459 	*num_pollers = poller_count;
460 
461 end:
462 	/* Since we rely in spdk_json_object_decode() to free this value
463 	 * each time we rewrite it, we need to free the last allocation
464 	 * manually. */
465 	free(thread_info.name);
466 
467 	if (rc) {
468 		*num_pollers = 0;
469 		for (i = 0; i < poller_count; i++) {
470 			free_rpc_poller(&out[i]);
471 		}
472 	}
473 
474 	return rc;
475 }
476 
477 static const struct spdk_json_object_decoder rpc_core_thread_info_decoders[] = {
478 	{"name", offsetof(struct rpc_core_thread_info, name), spdk_json_decode_string},
479 	{"id", offsetof(struct rpc_core_thread_info, id), spdk_json_decode_uint64},
480 	{"cpumask", offsetof(struct rpc_core_thread_info, cpumask), spdk_json_decode_string},
481 	{"elapsed", offsetof(struct rpc_core_thread_info, elapsed), spdk_json_decode_uint64},
482 };
483 
484 static int
485 rpc_decode_core_threads_object(const struct spdk_json_val *val, void *out)
486 {
487 	struct rpc_core_thread_info *info = out;
488 
489 	return spdk_json_decode_object(val, rpc_core_thread_info_decoders,
490 				       SPDK_COUNTOF(rpc_core_thread_info_decoders), info);
491 }
492 
493 #define RPC_THREAD_ENTRY_SIZE (SPDK_COUNTOF(rpc_core_thread_info_decoders) * 2)
494 
495 static int
496 rpc_decode_cores_lw_threads(const struct spdk_json_val *val, void *out)
497 {
498 	struct rpc_core_threads *threads = out;
499 	/* The number of thread entries received from RPC can be calculated using
500 	 * above define value (each JSON line = key + value, hence '* 2' ) and JSON
501 	 * 'val' value (-2 is to subtract VAL_OBJECT_BEGIN/END). */
502 	size_t threads_count = (spdk_json_val_len(val) - 2) / RPC_THREAD_ENTRY_SIZE;
503 
504 	threads->thread = calloc(threads_count, sizeof(struct rpc_core_thread_info));
505 	if (!out) {
506 		fprintf(stderr, "Unable to allocate memory for a thread array.\n");
507 		return -1;
508 	}
509 
510 	return spdk_json_decode_array(val, rpc_decode_core_threads_object, threads->thread, threads_count,
511 				      &threads->threads_count, sizeof(struct rpc_core_thread_info));
512 }
513 
514 static const struct spdk_json_object_decoder rpc_core_info_decoders[] = {
515 	{"lcore", offsetof(struct rpc_core_info, lcore), spdk_json_decode_uint32},
516 	{"busy", offsetof(struct rpc_core_info, busy), spdk_json_decode_uint64},
517 	{"idle", offsetof(struct rpc_core_info, idle), spdk_json_decode_uint64},
518 	{"core_freq", offsetof(struct rpc_core_info, core_freq), spdk_json_decode_uint32, true},
519 	{"in_interrupt", offsetof(struct rpc_core_info, in_interrupt), spdk_json_decode_bool},
520 	{"lw_threads", offsetof(struct rpc_core_info, threads), rpc_decode_cores_lw_threads},
521 };
522 
523 static int
524 rpc_decode_core_object(const struct spdk_json_val *val, void *out)
525 {
526 	struct rpc_core_info *info = out;
527 
528 	return spdk_json_decode_object(val, rpc_core_info_decoders,
529 				       SPDK_COUNTOF(rpc_core_info_decoders), info);
530 }
531 
532 static int
533 rpc_decode_cores_array(struct spdk_json_val *val, struct rpc_core_info *out,
534 		       uint32_t *current_cores_count)
535 {
536 	struct spdk_json_val *core = val;
537 	size_t cores_count;
538 	int rc;
539 
540 	/* Fetch the beginning of reactors array. */
541 	rc = spdk_json_find_array(core, "reactors", NULL, &core);
542 	if (rc) {
543 		printf("Could not fetch cores array from JSON.");
544 		goto end;
545 	}
546 
547 	rc = spdk_json_decode_array(core, rpc_decode_core_object, out, RPC_MAX_CORES, &cores_count,
548 				    sizeof(struct rpc_core_info));
549 
550 	*current_cores_count = (uint32_t)cores_count;
551 
552 end:
553 	return rc;
554 }
555 
556 static int
557 rpc_send_req(char *rpc_name, struct spdk_jsonrpc_client_response **resp)
558 {
559 	struct spdk_jsonrpc_client_response *json_resp = NULL;
560 	struct spdk_json_write_ctx *w;
561 	struct spdk_jsonrpc_client_request *request;
562 	int rc;
563 
564 	request = spdk_jsonrpc_client_create_request();
565 	if (request == NULL) {
566 		return -ENOMEM;
567 	}
568 
569 	w = spdk_jsonrpc_begin_request(request, 1, rpc_name);
570 	spdk_jsonrpc_end_request(request, w);
571 	spdk_jsonrpc_client_send_request(g_rpc_client, request);
572 
573 	do {
574 		rc = spdk_jsonrpc_client_poll(g_rpc_client, 1);
575 	} while (rc == 0 || rc == -ENOTCONN);
576 
577 	if (rc <= 0) {
578 		return -1;
579 	}
580 
581 	json_resp = spdk_jsonrpc_client_get_response(g_rpc_client);
582 	if (json_resp == NULL) {
583 		return -1;
584 	}
585 
586 	/* Check for error response */
587 	if (json_resp->error != NULL) {
588 		spdk_jsonrpc_client_free_response(json_resp);
589 		return -1;
590 	}
591 
592 	assert(json_resp->result);
593 
594 	*resp = json_resp;
595 
596 	return 0;
597 }
598 
599 static uint64_t
600 get_cpu_usage(uint64_t busy_ticks, uint64_t idle_ticks)
601 {
602 	if (busy_ticks + idle_ticks > 0) {
603 		/* Increase numerator to convert fraction into decimal with
604 		 * additional precision */
605 		return busy_ticks * 10000 / (busy_ticks + idle_ticks);
606 	}
607 
608 	return 0;
609 }
610 
611 static int
612 subsort_threads(enum column_threads_type sort_column, const void *p1, const void *p2)
613 {
614 	const struct rpc_thread_info thread_info1 = *(struct rpc_thread_info *)p1;
615 	const struct rpc_thread_info thread_info2 = *(struct rpc_thread_info *)p2;
616 	uint64_t count1, count2;
617 
618 	switch (sort_column) {
619 	case COL_THREADS_NAME:
620 		return strcmp(thread_info1.name, thread_info2.name);
621 	case COL_THREADS_CORE:
622 		count2 = thread_info1.core_num;
623 		count1 = thread_info2.core_num;
624 		break;
625 	case COL_THREADS_ACTIVE_POLLERS:
626 		count1 = thread_info1.active_pollers_count;
627 		count2 = thread_info2.active_pollers_count;
628 		break;
629 	case COL_THREADS_TIMED_POLLERS:
630 		count1 = thread_info1.timed_pollers_count;
631 		count2 = thread_info2.timed_pollers_count;
632 		break;
633 	case COL_THREADS_PAUSED_POLLERS:
634 		count1 = thread_info1.paused_pollers_count;
635 		count2 = thread_info2.paused_pollers_count;
636 		break;
637 	case COL_THREADS_IDLE_TIME:
638 		if (g_interval_data) {
639 			count1 = thread_info1.idle - thread_info1.last_idle;
640 			count2 = thread_info2.idle - thread_info2.last_idle;
641 		} else {
642 			count1 = thread_info1.idle;
643 			count2 = thread_info2.idle;
644 		}
645 		break;
646 	case COL_THREADS_BUSY_TIME:
647 		if (g_interval_data) {
648 			count1 = thread_info1.busy - thread_info1.last_busy;
649 			count2 = thread_info2.busy - thread_info2.last_busy;
650 		} else {
651 			count1 = thread_info1.busy;
652 			count2 = thread_info2.busy;
653 		}
654 		break;
655 	case COL_THREADS_CPU_USAGE:
656 		count1 = get_cpu_usage(thread_info1.busy - thread_info1.last_busy,
657 				       g_cores_info[thread_info1.core_num].busy + g_cores_info[thread_info1.core_num].idle);
658 		count2 = get_cpu_usage(thread_info2.busy - thread_info2.last_busy,
659 				       g_cores_info[thread_info2.core_num].busy + g_cores_info[thread_info2.core_num].idle);
660 		break;
661 	case COL_THREADS_NONE:
662 	default:
663 		return 0;
664 	}
665 
666 	if (count2 > count1) {
667 		return 1;
668 	} else if (count2 < count1) {
669 		return -1;
670 	} else {
671 		return 0;
672 	}
673 }
674 
675 static int
676 sort_threads(const void *p1, const void *p2)
677 {
678 	int res;
679 
680 	res = subsort_threads(g_current_sort_col[THREADS_TAB], p1, p2);
681 	if (res == 0) {
682 		res = subsort_threads(g_current_sort_col2[THREADS_TAB], p1, p2);
683 	}
684 	return res;
685 }
686 
687 static void
688 store_last_counters(uint64_t poller_id, uint64_t thread_id, uint64_t last_run_counter,
689 		    uint64_t last_busy_counter)
690 {
691 	struct run_counter_history *history;
692 
693 	TAILQ_FOREACH(history, &g_run_counter_history, link) {
694 		if ((history->poller_id == poller_id) && (history->thread_id == thread_id)) {
695 			history->last_run_counter = last_run_counter;
696 			history->last_busy_counter = last_busy_counter;
697 			return;
698 		}
699 	}
700 
701 	history = calloc(1, sizeof(*history));
702 	if (history == NULL) {
703 		fprintf(stderr, "Unable to allocate a history object in store_last_counters.\n");
704 		return;
705 	}
706 	history->poller_id = poller_id;
707 	history->thread_id = thread_id;
708 	history->last_run_counter = last_run_counter;
709 	history->last_busy_counter = last_busy_counter;
710 
711 	TAILQ_INSERT_TAIL(&g_run_counter_history, history, link);
712 }
713 
714 static int
715 get_thread_data(void)
716 {
717 	struct spdk_jsonrpc_client_response *json_resp = NULL;
718 	struct rpc_thread_info thread_info[RPC_MAX_THREADS], *thread;
719 	struct rpc_core_info *core_info;
720 	uint64_t i, j, k, current_threads_count = 0;
721 	int rc = 0;
722 
723 	rc = rpc_send_req("thread_get_stats", &json_resp);
724 	if (rc) {
725 		return rc;
726 	}
727 
728 	/* Decode json */
729 	memset(thread_info, 0, sizeof(struct rpc_thread_info) * RPC_MAX_THREADS);
730 	if (rpc_decode_threads_array(json_resp->result, thread_info, &current_threads_count)) {
731 		rc = -EINVAL;
732 		for (i = 0; i < current_threads_count; i++) {
733 			free_rpc_threads_stats(&thread_info[i]);
734 		}
735 		goto end;
736 	}
737 
738 	pthread_mutex_lock(&g_thread_lock);
739 
740 	/* This is to free allocated char arrays with old thread names */
741 	for (i = 0; i < g_last_threads_count; i++) {
742 		free_rpc_threads_stats(&g_threads_info[i]);
743 	}
744 
745 	for (i = 0; i < current_threads_count; i++) {
746 		for (j = 0; j < g_last_threads_count; j++) {
747 			if (thread_info[i].id == g_threads_info[j].id) {
748 				thread_info[i].last_busy = g_threads_info[j].busy;
749 				thread_info[i].last_idle = g_threads_info[j].idle;
750 			}
751 		}
752 	}
753 	g_last_threads_count = current_threads_count;
754 
755 	memcpy(g_threads_info, thread_info, sizeof(struct rpc_thread_info) * RPC_MAX_THREADS);
756 
757 	for (i = 0; i < g_last_threads_count; i++) {
758 		g_threads_info[i].core_num = -1;
759 	}
760 
761 	for (i = 0; i < g_last_cores_count; i++) {
762 		core_info = &g_cores_info[i];
763 
764 		for (j = 0; j < core_info->threads.threads_count; j++) {
765 			for (k = 0; k < g_last_threads_count; k++) {
766 				/* For each thread on current core: check if it's ID also exists
767 				 * in g_thread_info data structure. If it does then assign current
768 				 * core's number to that thread, otherwise application state is inconsistent
769 				 * (e.g. scheduler is moving threads between cores). */
770 				thread = &g_threads_info[k];
771 				if (thread->id == core_info->threads.thread[j].id) {
772 					thread->core_num = core_info->lcore;
773 					break;
774 				}
775 			}
776 		}
777 	}
778 
779 	qsort(g_threads_info, g_last_threads_count, sizeof(struct rpc_thread_info), sort_threads);
780 
781 	pthread_mutex_unlock(&g_thread_lock);
782 
783 end:
784 	spdk_jsonrpc_client_free_response(json_resp);
785 	return rc;
786 }
787 
788 static uint64_t
789 get_last_run_counter(uint64_t poller_id, uint64_t thread_id)
790 {
791 	struct run_counter_history *history;
792 
793 	TAILQ_FOREACH(history, &g_run_counter_history, link) {
794 		if ((history->poller_id == poller_id) && (history->thread_id == thread_id)) {
795 			return history->last_run_counter;
796 		}
797 	}
798 
799 	return 0;
800 }
801 
802 static uint64_t
803 get_last_busy_counter(uint64_t poller_id, uint64_t thread_id)
804 {
805 	struct run_counter_history *history;
806 
807 	TAILQ_FOREACH(history, &g_run_counter_history, link) {
808 		if ((history->poller_id == poller_id) && (history->thread_id == thread_id)) {
809 			return history->last_busy_counter;
810 		}
811 	}
812 
813 	return 0;
814 }
815 
816 static int
817 subsort_pollers(enum column_pollers_type sort_column, const void *p1, const void *p2)
818 {
819 	const struct rpc_poller_info *poller1 = (struct rpc_poller_info *)p1;
820 	const struct rpc_poller_info *poller2 = (struct rpc_poller_info *)p2;
821 	uint64_t count1, count2;
822 	uint64_t last_busy_counter1, last_busy_counter2;
823 
824 	switch (sort_column) {
825 	case COL_POLLERS_NAME:
826 		return strcmp(poller1->name, poller2->name);
827 	case COL_POLLERS_TYPE:
828 		return poller1->type - poller2->type;
829 	case COL_POLLERS_THREAD_NAME:
830 		return strcmp(poller1->thread_name, poller2->thread_name);
831 	case COL_POLLERS_RUN_COUNTER:
832 		if (g_interval_data) {
833 			count1 = poller1->run_count - get_last_run_counter(poller1->id, poller1->thread_id);
834 			count2 = poller2->run_count - get_last_run_counter(poller2->id, poller2->thread_id);
835 		} else {
836 			count1 = poller1->run_count;
837 			count2 = poller2->run_count;
838 		}
839 		break;
840 	case COL_POLLERS_PERIOD:
841 		count1 = poller1->period_ticks;
842 		count2 = poller2->period_ticks;
843 		break;
844 	case COL_POLLERS_BUSY_COUNT:
845 		count1 = poller1->busy_count;
846 		count2 = poller2->busy_count;
847 		if (g_interval_data) {
848 			last_busy_counter1 = get_last_busy_counter(poller1->id, poller1->thread_id);
849 			last_busy_counter2 = get_last_busy_counter(poller2->id, poller2->thread_id);
850 			if (count1 > last_busy_counter1) {
851 				count1 -= last_busy_counter1;
852 			}
853 			if (count2 > last_busy_counter2) {
854 				count2 -= last_busy_counter2;
855 			}
856 		}
857 		break;
858 	case COL_POLLERS_NONE:
859 	default:
860 		return 0;
861 	}
862 
863 	if (count2 > count1) {
864 		return 1;
865 	} else if (count2 < count1) {
866 		return -1;
867 	} else {
868 		return 0;
869 	}
870 }
871 
872 static int
873 sort_pollers(const void *p1, const void *p2)
874 {
875 	int rc;
876 
877 	rc = subsort_pollers(g_current_sort_col[POLLERS_TAB], p1, p2);
878 	if (rc == 0) {
879 		rc = subsort_pollers(g_current_sort_col2[POLLERS_TAB], p1, p2);
880 	}
881 	return rc;
882 }
883 
884 static int
885 get_pollers_data(void)
886 {
887 	struct spdk_jsonrpc_client_response *json_resp = NULL;
888 	int rc = 0;
889 	uint64_t i = 0;
890 	uint32_t current_pollers_count;
891 	struct rpc_poller_info pollers_info[RPC_MAX_POLLERS];
892 
893 	rc = rpc_send_req("thread_get_pollers", &json_resp);
894 	if (rc) {
895 		return rc;
896 	}
897 
898 	/* Decode json */
899 	memset(&pollers_info, 0, sizeof(pollers_info));
900 	if (rpc_decode_pollers_threads_array(json_resp->result, pollers_info, &current_pollers_count)) {
901 		rc = -EINVAL;
902 		for (i = 0; i < current_pollers_count; i++) {
903 			free_rpc_poller(&pollers_info[i]);
904 		}
905 		goto end;
906 	}
907 
908 	pthread_mutex_lock(&g_thread_lock);
909 
910 	/* Save last run counter of each poller before updating g_pollers_stats. */
911 	for (i = 0; i < g_last_pollers_count; i++) {
912 		store_last_counters(g_pollers_info[i].id, g_pollers_info[i].thread_id,
913 				    g_pollers_info[i].run_count, g_pollers_info[i].busy_count);
914 	}
915 
916 	/* Free old pollers values before allocating memory for new ones */
917 	for (i = 0; i < g_last_pollers_count; i++) {
918 		free_rpc_poller(&g_pollers_info[i]);
919 	}
920 
921 	g_last_pollers_count = current_pollers_count;
922 
923 	qsort(&pollers_info, g_last_pollers_count, sizeof(struct rpc_poller_info), sort_pollers);
924 
925 	memcpy(&g_pollers_info, &pollers_info, sizeof(struct rpc_poller_info) * g_last_pollers_count);
926 
927 	pthread_mutex_unlock(&g_thread_lock);
928 
929 end:
930 	spdk_jsonrpc_client_free_response(json_resp);
931 	return rc;
932 }
933 
934 static int
935 subsort_cores(enum column_cores_type sort_column, const void *p1, const void *p2)
936 {
937 	const struct rpc_core_info core_info1 = *(struct rpc_core_info *)p1;
938 	const struct rpc_core_info core_info2 = *(struct rpc_core_info *)p2;
939 	uint64_t count1, count2;
940 
941 	switch (sort_column) {
942 	case COL_CORES_CORE:
943 		count1 = core_info2.lcore;
944 		count2 = core_info1.lcore;
945 		break;
946 	case COL_CORES_THREADS:
947 		count1 = core_info1.threads.threads_count;
948 		count2 = core_info2.threads.threads_count;
949 		break;
950 	case COL_CORES_POLLERS:
951 		count1 = core_info1.pollers_count;
952 		count2 = core_info2.pollers_count;
953 		break;
954 	case COL_CORES_IDLE_TIME:
955 		if (g_interval_data) {
956 			count1 = core_info1.last_idle - core_info1.idle;
957 			count2 = core_info2.last_idle - core_info2.idle;
958 		} else {
959 			count1 = core_info1.idle;
960 			count2 = core_info2.idle;
961 		}
962 		break;
963 	case COL_CORES_BUSY_TIME:
964 		if (g_interval_data) {
965 			count1 = core_info1.last_busy - core_info1.busy;
966 			count2 = core_info2.last_busy - core_info2.busy;
967 		} else {
968 			count1 = core_info1.busy;
969 			count2 = core_info2.busy;
970 		}
971 		break;
972 	case COL_CORES_CORE_FREQ:
973 		count1 = core_info1.core_freq;
974 		count2 = core_info2.core_freq;
975 		break;
976 	case COL_CORES_INTR:
977 		count1 = core_info1.in_interrupt;
978 		count2 = core_info2.in_interrupt;
979 		break;
980 	case COL_CORES_CPU_USAGE:
981 		count1 = get_cpu_usage(core_info1.last_busy - core_info1.busy,
982 				       core_info1.last_idle - core_info1.idle);
983 		count2 = get_cpu_usage(core_info2.last_busy - core_info2.busy,
984 				       core_info2.last_idle - core_info2.idle);
985 		break;
986 	case COL_CORES_NONE:
987 	default:
988 		return 0;
989 	}
990 
991 	if (count2 > count1) {
992 		return 1;
993 	} else if (count2 < count1) {
994 		return -1;
995 	} else {
996 		return 0;
997 	}
998 }
999 
1000 static int
1001 sort_cores(const void *p1, const void *p2)
1002 {
1003 	int rc;
1004 
1005 	rc = subsort_cores(g_current_sort_col[CORES_TAB], p1, p2);
1006 	if (rc == 0) {
1007 		return subsort_cores(g_current_sort_col[CORES_TAB], p1, p2);
1008 	}
1009 	return rc;
1010 }
1011 
1012 static int
1013 get_cores_data(void)
1014 {
1015 	struct spdk_jsonrpc_client_response *json_resp = NULL;
1016 	struct rpc_core_info *core_info;
1017 	uint64_t i, j, k;
1018 	uint32_t current_cores_count;
1019 	struct rpc_core_info cores_info[RPC_MAX_CORES];
1020 	int rc = 0;
1021 
1022 	rc = rpc_send_req("framework_get_reactors", &json_resp);
1023 	if (rc) {
1024 		return rc;
1025 	}
1026 
1027 	/* Decode json */
1028 	memset(cores_info, 0, sizeof(struct rpc_core_info) * RPC_MAX_CORES);
1029 	if (rpc_decode_cores_array(json_resp->result, cores_info, &current_cores_count)) {
1030 		rc = -EINVAL;
1031 		goto end;
1032 	}
1033 
1034 	pthread_mutex_lock(&g_thread_lock);
1035 	for (i = 0; i < current_cores_count; i++) {
1036 		for (j = 0; j < g_last_cores_count; j++) {
1037 			if (cores_info[i].lcore == g_cores_info[j].lcore) {
1038 				cores_info[i].last_busy = g_cores_info[j].busy;
1039 				cores_info[i].last_idle = g_cores_info[j].idle;
1040 			}
1041 			/* Do not consider threads which changed cores when issuing
1042 			 * RPCs to get_core_data and get_thread_data and threads
1043 			 * not currently assigned to this core. */
1044 			if ((int)cores_info[i].lcore == g_threads_info[j].core_num) {
1045 				cores_info[i].pollers_count += g_threads_info[j].active_pollers_count +
1046 							       g_threads_info[j].timed_pollers_count +
1047 							       g_threads_info[j].paused_pollers_count;
1048 			}
1049 		}
1050 	}
1051 
1052 	/* Free old cores values before allocating memory for new ones */
1053 	free_rpc_core_info(g_cores_info, current_cores_count);
1054 	memcpy(g_cores_info, cores_info, sizeof(struct rpc_core_info) * current_cores_count);
1055 
1056 	for (i = 0; i < g_last_cores_count; i++) {
1057 		core_info = &g_cores_info[i];
1058 
1059 		core_info->threads.thread = cores_info[i].threads.thread;
1060 
1061 		for (j = 0; j < core_info->threads.threads_count; j++) {
1062 			memcpy(&core_info->threads.thread[j], &cores_info[i].threads.thread[j],
1063 			       sizeof(struct rpc_core_thread_info));
1064 			for (k = 0; k < g_last_threads_count; k++) {
1065 				if (core_info->threads.thread[j].id == g_threads_info[k].id) {
1066 					g_threads_info[k].core_num = core_info->lcore;
1067 				}
1068 			}
1069 		}
1070 	}
1071 
1072 	g_last_cores_count = current_cores_count;
1073 
1074 	qsort(&g_cores_info, g_last_cores_count, sizeof(struct rpc_core_info), sort_cores);
1075 
1076 end:
1077 	pthread_mutex_unlock(&g_thread_lock);
1078 	spdk_jsonrpc_client_free_response(json_resp);
1079 	return rc;
1080 }
1081 
1082 enum str_alignment {
1083 	ALIGN_LEFT,
1084 	ALIGN_RIGHT,
1085 };
1086 
1087 static void
1088 print_max_len(WINDOW *win, int row, uint16_t col, uint16_t max_len, enum str_alignment alignment,
1089 	      const char *string)
1090 {
1091 	const char dots[] = "...";
1092 	int DOTS_STR_LEN = sizeof(dots) / sizeof(dots[0]);
1093 	char tmp_str[MAX_STRING_LEN];
1094 	int len, max_col, max_str, cmp_len;
1095 	int max_row;
1096 
1097 	len = strlen(string);
1098 	getmaxyx(win, max_row, max_col);
1099 
1100 	if (row > max_row) {
1101 		/* We are in a process of resizing and this may happen */
1102 		return;
1103 	}
1104 
1105 	if (max_len != 0 && col + max_len < max_col) {
1106 		max_col = col + max_len;
1107 	}
1108 
1109 	max_str = max_col - col;
1110 
1111 	if (max_str <= DOTS_STR_LEN + 1) {
1112 		/* No space to print anything, but we have to let a user know about it */
1113 		mvwprintw(win, row, max_col - DOTS_STR_LEN - 1, "...");
1114 		refresh();
1115 		wrefresh(win);
1116 		return;
1117 	}
1118 
1119 	if (max_len) {
1120 		if (alignment == ALIGN_LEFT) {
1121 			snprintf(tmp_str, max_str, "%s%*c", string, max_len - len - 1, ' ');
1122 		} else {
1123 			snprintf(tmp_str, max_str, "%*c%s", max_len - len - 1, ' ', string);
1124 		}
1125 		cmp_len = max_len - 1;
1126 	} else {
1127 		snprintf(tmp_str, max_str, "%s", string);
1128 		cmp_len = len;
1129 	}
1130 
1131 	if (col + cmp_len > max_col - 1) {
1132 		snprintf(&tmp_str[max_str - DOTS_STR_LEN - 2], DOTS_STR_LEN, "%s", dots);
1133 	}
1134 
1135 	mvwprintw(win, row, col, "%s", tmp_str);
1136 
1137 	refresh();
1138 	wrefresh(win);
1139 }
1140 
1141 static void
1142 draw_menu_win(void)
1143 {
1144 	wbkgd(g_menu_win, COLOR_PAIR(2));
1145 	box(g_menu_win, 0, 0);
1146 	print_max_len(g_menu_win, 1, 1, 0, ALIGN_LEFT,
1147 		      "  [q] Quit  |  [1-3][Tab] Switch tab  |  [PgUp] Previous page  |  [PgDown] Next page  |  [Enter] Item details  |  [h] Help");
1148 }
1149 
1150 static void
1151 draw_tab_win(enum tabs tab)
1152 {
1153 	uint16_t col;
1154 	uint8_t white_spaces = TABS_SPACING * NUMBER_OF_TABS;
1155 
1156 	wbkgd(g_tab_win[tab], COLOR_PAIR(2));
1157 	box(g_tab_win[tab], 0, 0);
1158 
1159 	col = ((g_max_col - white_spaces) / NUMBER_OF_TABS / 2) - (strlen(g_tab_title[tab]) / 2) -
1160 	      TABS_SPACING;
1161 	print_max_len(g_tab_win[tab], 1, col, 0, ALIGN_LEFT, g_tab_title[tab]);
1162 }
1163 
1164 static void
1165 draw_tabs(enum tabs tab_index, uint8_t sort_col, uint8_t sort_col2)
1166 {
1167 	struct col_desc *col_desc = g_col_desc[tab_index];
1168 	WINDOW *tab = g_tabs[tab_index];
1169 	int i, j;
1170 	uint16_t offset, draw_offset;
1171 	uint16_t tab_height = g_max_row - MENU_WIN_HEIGHT - TAB_WIN_HEIGHT - 3;
1172 
1173 	for (i = 0; col_desc[i].name != NULL; i++) {
1174 		if (col_desc[i].disabled) {
1175 			continue;
1176 		}
1177 
1178 		offset = 1;
1179 		for (j = i; j != 0; j--) {
1180 			if (!col_desc[j - 1].disabled) {
1181 				offset += col_desc[j - 1].max_data_string;
1182 				offset += col_desc[j - 1].name_len % 2 + 1;
1183 			}
1184 		}
1185 
1186 		draw_offset = offset + (col_desc[i].max_data_string / 2) - (col_desc[i].name_len / 2);
1187 
1188 		if (i == sort_col) {
1189 			wattron(tab, COLOR_PAIR(4));
1190 			print_max_len(tab, 1, draw_offset, 0, ALIGN_LEFT, col_desc[i].name);
1191 			wattroff(tab, COLOR_PAIR(4));
1192 		} else if (i == sort_col2) {
1193 			wattron(tab, COLOR_PAIR(3));
1194 			print_max_len(tab, 1, draw_offset, 0, ALIGN_LEFT, col_desc[i].name);
1195 			wattroff(tab, COLOR_PAIR(3));
1196 		} else {
1197 			print_max_len(tab, 1, draw_offset, 0, ALIGN_LEFT, col_desc[i].name);
1198 		}
1199 
1200 		if (offset != 1) {
1201 			print_max_len(tab, 1, offset - 1, 0, ALIGN_LEFT, "|");
1202 		}
1203 	}
1204 
1205 	print_max_len(tab, 2, 1, 0, ALIGN_LEFT, ""); /* Move to next line */
1206 	whline(tab, ACS_HLINE, g_max_col - 2);
1207 
1208 	/* Border lines */
1209 	mvwhline(tab, 0, 1, ACS_HLINE, g_max_col - 2);
1210 	mvwhline(tab, tab_height, 1, ACS_HLINE, g_max_col - 2);
1211 
1212 	wrefresh(tab);
1213 }
1214 
1215 static void
1216 resize_interface(enum tabs tab)
1217 {
1218 	int i;
1219 
1220 	clear();
1221 	wclear(g_menu_win);
1222 	mvwin(g_menu_win, g_max_row - MENU_WIN_SPACING, MENU_WIN_LOCATION_COL);
1223 	wresize(g_menu_win, MENU_WIN_HEIGHT, g_max_col);
1224 	draw_menu_win();
1225 
1226 	for (i = 0; i < NUMBER_OF_TABS; i++) {
1227 		wclear(g_tabs[i]);
1228 		wresize(g_tabs[i], g_max_row - MENU_WIN_HEIGHT - TAB_WIN_HEIGHT - 2, g_max_col);
1229 		mvwin(g_tabs[i], TABS_LOCATION_ROW, TABS_LOCATION_COL);
1230 		draw_tabs(i, g_current_sort_col[i], g_current_sort_col2[i]);
1231 	}
1232 
1233 	draw_tabs(tab, g_current_sort_col[tab], g_current_sort_col2[tab]);
1234 
1235 	for (i = 0; i < NUMBER_OF_TABS; i++) {
1236 		wclear(g_tab_win[i]);
1237 		wresize(g_tab_win[i], TAB_WIN_HEIGHT,
1238 			(g_max_col - (TABS_SPACING * NUMBER_OF_TABS)) / NUMBER_OF_TABS);
1239 		mvwin(g_tab_win[i], TAB_WIN_LOCATION_ROW, 1 + (g_max_col / NUMBER_OF_TABS) * i);
1240 		draw_tab_win(i);
1241 	}
1242 
1243 	update_panels();
1244 	doupdate();
1245 }
1246 
1247 static void
1248 switch_tab(enum tabs tab)
1249 {
1250 	wclear(g_tabs[tab]);
1251 	draw_tabs(tab, g_current_sort_col[tab], g_current_sort_col2[tab]);
1252 	top_panel(g_panels[tab]);
1253 	update_panels();
1254 	doupdate();
1255 }
1256 
1257 static void
1258 get_time_str(uint64_t ticks, char *time_str)
1259 {
1260 	uint64_t time;
1261 
1262 	time = ticks * SPDK_SEC_TO_USEC / g_tick_rate;
1263 	snprintf(time_str, MAX_TIME_STR_LEN, "%" PRIu64, time);
1264 }
1265 
1266 static void
1267 draw_row_background(uint8_t item_index, uint8_t tab)
1268 {
1269 	int k;
1270 
1271 	if (item_index == g_selected_row) {
1272 		wattron(g_tabs[tab], COLOR_PAIR(2));
1273 	}
1274 	for (k = 1; k < g_max_col - 1; k++) {
1275 		mvwprintw(g_tabs[tab], TABS_DATA_START_ROW + item_index, k, " ");
1276 	}
1277 }
1278 
1279 static void
1280 get_cpu_usage_str(uint64_t busy_ticks, uint64_t total_ticks, char *cpu_str)
1281 {
1282 	if (total_ticks > 0) {
1283 		snprintf(cpu_str, MAX_CPU_STR_LEN, "%.2f",
1284 			 (double)(busy_ticks) * 100 / (double)(total_ticks));
1285 	} else {
1286 		cpu_str[0] = '\0';
1287 	}
1288 }
1289 
1290 static void
1291 draw_thread_tab_row(uint64_t current_row, uint8_t item_index)
1292 {
1293 	struct col_desc *col_desc = g_col_desc[THREADS_TAB];
1294 	uint16_t col = TABS_DATA_START_COL;
1295 	int core_num;
1296 	char pollers_number[MAX_POLLER_COUNT_STR_LEN], idle_time[MAX_TIME_STR_LEN],
1297 	     busy_time[MAX_TIME_STR_LEN], core_str[MAX_CORE_MASK_STR_LEN],
1298 	     cpu_usage[MAX_CPU_STR_LEN];
1299 
1300 	if (!col_desc[COL_THREADS_NAME].disabled) {
1301 		print_max_len(g_tabs[THREADS_TAB], TABS_DATA_START_ROW + item_index, col,
1302 			      col_desc[COL_THREADS_NAME].max_data_string, ALIGN_LEFT, g_threads_info[current_row].name);
1303 		col += col_desc[COL_THREADS_NAME].max_data_string;
1304 	}
1305 
1306 	if (!col_desc[COL_THREADS_CORE].disabled) {
1307 		snprintf(core_str, MAX_CORE_STR_LEN, "%d", g_threads_info[current_row].core_num);
1308 		print_max_len(g_tabs[THREADS_TAB], TABS_DATA_START_ROW + item_index,
1309 			      col, col_desc[COL_THREADS_CORE].max_data_string, ALIGN_RIGHT, core_str);
1310 		col += col_desc[COL_THREADS_CORE].max_data_string + 2;
1311 	}
1312 
1313 	if (!col_desc[COL_THREADS_ACTIVE_POLLERS].disabled) {
1314 		snprintf(pollers_number, MAX_POLLER_COUNT_STR_LEN, "%ld",
1315 			 g_threads_info[current_row].active_pollers_count);
1316 		print_max_len(g_tabs[THREADS_TAB], TABS_DATA_START_ROW + item_index,
1317 			      col + (col_desc[COL_THREADS_ACTIVE_POLLERS].name_len / 2),
1318 			      col_desc[COL_THREADS_ACTIVE_POLLERS].max_data_string, ALIGN_LEFT, pollers_number);
1319 		col += col_desc[COL_THREADS_ACTIVE_POLLERS].max_data_string + 2;
1320 	}
1321 
1322 	if (!col_desc[COL_THREADS_TIMED_POLLERS].disabled) {
1323 		snprintf(pollers_number, MAX_POLLER_COUNT_STR_LEN, "%ld",
1324 			 g_threads_info[current_row].timed_pollers_count);
1325 		print_max_len(g_tabs[THREADS_TAB], TABS_DATA_START_ROW + item_index,
1326 			      col + (col_desc[COL_THREADS_TIMED_POLLERS].name_len / 2),
1327 			      col_desc[COL_THREADS_TIMED_POLLERS].max_data_string, ALIGN_LEFT, pollers_number);
1328 		col += col_desc[COL_THREADS_TIMED_POLLERS].max_data_string + 1;
1329 	}
1330 
1331 	if (!col_desc[COL_THREADS_PAUSED_POLLERS].disabled) {
1332 		snprintf(pollers_number, MAX_POLLER_COUNT_STR_LEN, "%ld",
1333 			 g_threads_info[current_row].paused_pollers_count);
1334 		print_max_len(g_tabs[THREADS_TAB], TABS_DATA_START_ROW + item_index,
1335 			      col + (col_desc[COL_THREADS_PAUSED_POLLERS].name_len / 2),
1336 			      col_desc[COL_THREADS_PAUSED_POLLERS].max_data_string, ALIGN_LEFT, pollers_number);
1337 		col += col_desc[COL_THREADS_PAUSED_POLLERS].max_data_string + 2;
1338 	}
1339 
1340 	if (!col_desc[COL_THREADS_IDLE_TIME].disabled) {
1341 		if (g_interval_data == true) {
1342 			get_time_str(g_threads_info[current_row].idle - g_threads_info[current_row].last_idle, idle_time);
1343 		} else {
1344 			get_time_str(g_threads_info[current_row].idle, idle_time);
1345 		}
1346 		print_max_len(g_tabs[THREADS_TAB], TABS_DATA_START_ROW + item_index, col,
1347 			      col_desc[COL_THREADS_IDLE_TIME].max_data_string, ALIGN_RIGHT, idle_time);
1348 		col += col_desc[COL_THREADS_IDLE_TIME].max_data_string;
1349 	}
1350 
1351 	if (!col_desc[COL_THREADS_BUSY_TIME].disabled) {
1352 		if (g_interval_data == true) {
1353 			get_time_str(g_threads_info[current_row].busy - g_threads_info[current_row].last_busy, busy_time);
1354 		} else {
1355 			get_time_str(g_threads_info[current_row].busy, busy_time);
1356 		}
1357 		print_max_len(g_tabs[THREADS_TAB], TABS_DATA_START_ROW + item_index, col,
1358 			      col_desc[COL_THREADS_BUSY_TIME].max_data_string, ALIGN_RIGHT, busy_time);
1359 		col += col_desc[COL_THREADS_BUSY_TIME].max_data_string + 3;
1360 	}
1361 
1362 	if (!col_desc[COL_THREADS_CPU_USAGE].disabled) {
1363 		core_num = g_threads_info[current_row].core_num;
1364 		if (core_num >= 0 && core_num < RPC_MAX_CORES) {
1365 			get_cpu_usage_str(g_threads_info[current_row].busy - g_threads_info[current_row].last_busy,
1366 					  g_cores_info[core_num].busy + g_cores_info[core_num].idle,
1367 					  cpu_usage);
1368 		} else {
1369 			snprintf(cpu_usage, sizeof(cpu_usage), "n/a");
1370 		}
1371 
1372 		print_max_len(g_tabs[THREADS_TAB], TABS_DATA_START_ROW + item_index, col,
1373 			      col_desc[COL_THREADS_CPU_USAGE].max_data_string, ALIGN_RIGHT, cpu_usage);
1374 	}
1375 }
1376 
1377 static uint8_t
1378 refresh_threads_tab(uint8_t current_page)
1379 {
1380 	uint64_t i, j, threads_count;
1381 	uint16_t empty_col = 0;
1382 	uint8_t max_pages, item_index;
1383 
1384 	threads_count = g_last_threads_count;
1385 
1386 	max_pages = (threads_count + g_max_data_rows - 1) / g_max_data_rows;
1387 
1388 	for (i = current_page * g_max_data_rows;
1389 	     i < (uint64_t)((current_page + 1) * g_max_data_rows);
1390 	     i++) {
1391 		item_index = i - (current_page * g_max_data_rows);
1392 
1393 		/* When number of threads decreases, this will print spaces in places
1394 		 * where non existent threads were previously displayed. */
1395 		if (i >= threads_count) {
1396 			for (j = 1; j < (uint64_t)g_max_col - 1; j++) {
1397 				mvwprintw(g_tabs[THREADS_TAB], item_index + TABS_DATA_START_ROW, j, " ");
1398 			}
1399 
1400 			empty_col++;
1401 			continue;
1402 		}
1403 
1404 		draw_row_background(item_index, THREADS_TAB);
1405 		draw_thread_tab_row(i, item_index);
1406 
1407 		if (item_index == g_selected_row) {
1408 			wattroff(g_tabs[THREADS_TAB], COLOR_PAIR(2));
1409 		}
1410 	}
1411 
1412 	g_max_selected_row = i - current_page * g_max_data_rows - 1 - empty_col;
1413 
1414 	return max_pages;
1415 }
1416 
1417 static void
1418 draw_poller_tab_row(uint64_t current_row, uint8_t item_index)
1419 {
1420 	struct col_desc *col_desc = g_col_desc[POLLERS_TAB];
1421 	uint64_t last_run_counter, last_busy_counter;
1422 	uint16_t col = TABS_DATA_START_COL;
1423 	char run_count[MAX_POLLER_RUN_COUNT], period_ticks[MAX_PERIOD_STR_LEN],
1424 	     status[MAX_POLLER_IND_STR_LEN];
1425 
1426 	last_busy_counter = get_last_busy_counter(g_pollers_info[current_row].id,
1427 			    g_pollers_info[current_row].thread_id);
1428 
1429 	if (!col_desc[COL_POLLERS_NAME].disabled) {
1430 		print_max_len(g_tabs[POLLERS_TAB], TABS_DATA_START_ROW + item_index, col + 1,
1431 			      col_desc[COL_POLLERS_NAME].max_data_string, ALIGN_LEFT, g_pollers_info[current_row].name);
1432 		col += col_desc[COL_POLLERS_NAME].max_data_string + 2;
1433 	}
1434 
1435 	if (!col_desc[COL_POLLERS_TYPE].disabled) {
1436 		print_max_len(g_tabs[POLLERS_TAB], TABS_DATA_START_ROW + item_index, col,
1437 			      col_desc[COL_POLLERS_TYPE].max_data_string, ALIGN_LEFT,
1438 			      poller_type_str[g_pollers_info[current_row].type]);
1439 		col += col_desc[COL_POLLERS_TYPE].max_data_string + 2;
1440 	}
1441 
1442 	if (!col_desc[COL_POLLERS_THREAD_NAME].disabled) {
1443 		print_max_len(g_tabs[POLLERS_TAB], TABS_DATA_START_ROW + item_index, col,
1444 			      col_desc[COL_POLLERS_THREAD_NAME].max_data_string, ALIGN_LEFT,
1445 			      g_pollers_info[current_row].thread_name);
1446 		col += col_desc[COL_POLLERS_THREAD_NAME].max_data_string + 1;
1447 	}
1448 
1449 	if (!col_desc[COL_POLLERS_RUN_COUNTER].disabled) {
1450 		last_run_counter = get_last_run_counter(g_pollers_info[current_row].id,
1451 							g_pollers_info[current_row].thread_id);
1452 		if (g_interval_data == true) {
1453 			snprintf(run_count, MAX_POLLER_RUN_COUNT, "%" PRIu64,
1454 				 g_pollers_info[current_row].run_count - last_run_counter);
1455 		} else {
1456 			snprintf(run_count, MAX_POLLER_RUN_COUNT, "%" PRIu64, g_pollers_info[current_row].run_count);
1457 		}
1458 		print_max_len(g_tabs[POLLERS_TAB], TABS_DATA_START_ROW + item_index, col,
1459 			      col_desc[COL_POLLERS_RUN_COUNTER].max_data_string, ALIGN_RIGHT, run_count);
1460 		col += col_desc[COL_POLLERS_RUN_COUNTER].max_data_string;
1461 	}
1462 
1463 	if (!col_desc[COL_POLLERS_PERIOD].disabled) {
1464 		if (g_pollers_info[current_row].period_ticks != 0) {
1465 			get_time_str(g_pollers_info[current_row].period_ticks, period_ticks);
1466 			print_max_len(g_tabs[POLLERS_TAB], TABS_DATA_START_ROW + item_index, col,
1467 				      col_desc[COL_POLLERS_PERIOD].max_data_string, ALIGN_RIGHT, period_ticks);
1468 		}
1469 		col += col_desc[COL_POLLERS_PERIOD].max_data_string + 7;
1470 	}
1471 
1472 	if (!col_desc[COL_POLLERS_BUSY_COUNT].disabled) {
1473 		if (g_pollers_info[current_row].busy_count > last_busy_counter) {
1474 			if (g_interval_data == true) {
1475 				snprintf(status, MAX_POLLER_IND_STR_LEN, "Busy (%" PRIu64 ")",
1476 					 g_pollers_info[current_row].busy_count - last_busy_counter);
1477 			} else {
1478 				snprintf(status, MAX_POLLER_IND_STR_LEN, "Busy (%" PRIu64 ")",
1479 					 g_pollers_info[current_row].busy_count);
1480 			}
1481 
1482 			if (item_index != g_selected_row) {
1483 				wattron(g_tabs[POLLERS_TAB], COLOR_PAIR(6));
1484 				print_max_len(g_tabs[POLLERS_TAB], TABS_DATA_START_ROW + item_index, col,
1485 					      col_desc[COL_POLLERS_BUSY_COUNT].max_data_string, ALIGN_LEFT, status);
1486 				wattroff(g_tabs[POLLERS_TAB], COLOR_PAIR(6));
1487 			} else {
1488 				wattron(g_tabs[POLLERS_TAB], COLOR_PAIR(8));
1489 				print_max_len(g_tabs[POLLERS_TAB], TABS_DATA_START_ROW + item_index, col,
1490 					      col_desc[COL_POLLERS_BUSY_COUNT].max_data_string, ALIGN_LEFT, status);
1491 				wattroff(g_tabs[POLLERS_TAB], COLOR_PAIR(8));
1492 			}
1493 		} else {
1494 			if (g_interval_data == true) {
1495 				snprintf(status, MAX_POLLER_IND_STR_LEN, "%s", "Idle");
1496 			} else {
1497 				snprintf(status, MAX_POLLER_IND_STR_LEN, "Idle (%" PRIu64 ")",
1498 					 g_pollers_info[current_row].busy_count);
1499 			}
1500 
1501 			if (item_index != g_selected_row) {
1502 				wattron(g_tabs[POLLERS_TAB], COLOR_PAIR(7));
1503 				print_max_len(g_tabs[POLLERS_TAB], TABS_DATA_START_ROW + item_index, col,
1504 					      col_desc[COL_POLLERS_BUSY_COUNT].max_data_string, ALIGN_LEFT, status);
1505 				wattroff(g_tabs[POLLERS_TAB], COLOR_PAIR(7));
1506 			} else {
1507 				wattron(g_tabs[POLLERS_TAB], COLOR_PAIR(9));
1508 				print_max_len(g_tabs[POLLERS_TAB], TABS_DATA_START_ROW + item_index, col,
1509 					      col_desc[COL_POLLERS_BUSY_COUNT].max_data_string, ALIGN_LEFT, status);
1510 				wattroff(g_tabs[POLLERS_TAB], COLOR_PAIR(9));
1511 			}
1512 		}
1513 	}
1514 }
1515 
1516 static uint8_t
1517 refresh_pollers_tab(uint8_t current_page)
1518 {
1519 	uint64_t i, j;
1520 	uint16_t empty_col = 0;
1521 	uint8_t max_pages, item_index;
1522 
1523 	max_pages = (g_last_pollers_count + g_max_data_rows - 1) / g_max_data_rows;
1524 
1525 	/* Display info */
1526 	for (i = current_page * g_max_data_rows;
1527 	     i < (uint64_t)((current_page + 1) * g_max_data_rows);
1528 	     i++) {
1529 		item_index = i - (current_page * g_max_data_rows);
1530 
1531 		/* When number of pollers decreases, this will print spaces in places
1532 		 * where non existent pollers were previously displayed. */
1533 		if (i >= g_last_pollers_count) {
1534 			for (j = 1; j < (uint64_t)g_max_col - 1; j++) {
1535 				mvwprintw(g_tabs[POLLERS_TAB], item_index + TABS_DATA_START_ROW, j, " ");
1536 			}
1537 
1538 			empty_col++;
1539 			continue;
1540 		}
1541 
1542 		draw_row_background(item_index, POLLERS_TAB);
1543 		draw_poller_tab_row(i, item_index);
1544 
1545 		if (item_index == g_selected_row) {
1546 			wattroff(g_tabs[POLLERS_TAB], COLOR_PAIR(2));
1547 		}
1548 	}
1549 
1550 	g_max_selected_row = i - current_page * g_max_data_rows - 1 - empty_col;
1551 
1552 	return max_pages;
1553 }
1554 
1555 static void
1556 draw_core_tab_row(uint64_t current_row, uint8_t item_index)
1557 {
1558 	struct col_desc *col_desc = g_col_desc[CORES_TAB];
1559 	uint16_t col = 1;
1560 	char core[MAX_CORE_STR_LEN], threads_number[MAX_THREAD_COUNT_STR_LEN],  cpu_usage[MAX_CPU_STR_LEN],
1561 	     pollers_number[MAX_POLLER_COUNT_STR_LEN], idle_time[MAX_TIME_STR_LEN],
1562 	     busy_time[MAX_TIME_STR_LEN], core_freq[MAX_CORE_FREQ_STR_LEN],
1563 	     in_interrupt[MAX_INTR_LEN];
1564 
1565 	snprintf(threads_number, MAX_THREAD_COUNT_STR_LEN, "%ld",
1566 		 g_cores_info[current_row].threads.threads_count);
1567 	snprintf(pollers_number, MAX_POLLER_COUNT_STR_LEN, "%ld", g_cores_info[current_row].pollers_count);
1568 
1569 	if (!col_desc[COL_CORES_CORE].disabled) {
1570 		snprintf(core, MAX_CORE_STR_LEN, "%d", g_cores_info[current_row].lcore);
1571 		print_max_len(g_tabs[CORES_TAB], TABS_DATA_START_ROW + item_index, col,
1572 			      col_desc[COL_CORES_CORE].max_data_string, ALIGN_RIGHT, core);
1573 		col += col_desc[COL_CORES_CORE].max_data_string + 2;
1574 	}
1575 
1576 	if (!col_desc[COL_CORES_THREADS].disabled) {
1577 		print_max_len(g_tabs[CORES_TAB], TABS_DATA_START_ROW + item_index,
1578 			      col + (col_desc[COL_CORES_THREADS].name_len / 2), col_desc[COL_CORES_THREADS].max_data_string,
1579 			      ALIGN_LEFT, threads_number);
1580 		col += col_desc[COL_CORES_THREADS].max_data_string + 2;
1581 	}
1582 
1583 	if (!col_desc[COL_CORES_POLLERS].disabled) {
1584 		print_max_len(g_tabs[CORES_TAB], TABS_DATA_START_ROW + item_index,
1585 			      col + (col_desc[COL_CORES_POLLERS].name_len / 2), col_desc[COL_CORES_POLLERS].max_data_string,
1586 			      ALIGN_LEFT, pollers_number);
1587 		col += col_desc[COL_CORES_POLLERS].max_data_string;
1588 	}
1589 
1590 	if (!col_desc[COL_CORES_IDLE_TIME].disabled) {
1591 		if (g_interval_data == true) {
1592 			get_time_str(g_cores_info[current_row].idle - g_cores_info[current_row].last_idle, idle_time);
1593 		} else {
1594 			get_time_str(g_cores_info[current_row].idle, idle_time);
1595 		}
1596 		print_max_len(g_tabs[CORES_TAB], TABS_DATA_START_ROW + item_index, col,
1597 			      col_desc[COL_CORES_IDLE_TIME].max_data_string, ALIGN_RIGHT, idle_time);
1598 		col += col_desc[COL_CORES_IDLE_TIME].max_data_string + 2;
1599 	}
1600 
1601 	if (!col_desc[COL_CORES_BUSY_TIME].disabled) {
1602 		if (g_interval_data == true) {
1603 			get_time_str(g_cores_info[current_row].busy - g_cores_info[current_row].last_busy, busy_time);
1604 		} else {
1605 			get_time_str(g_cores_info[current_row].busy, busy_time);
1606 		}
1607 		print_max_len(g_tabs[CORES_TAB], TABS_DATA_START_ROW + item_index, col,
1608 			      col_desc[COL_CORES_BUSY_TIME].max_data_string, ALIGN_RIGHT, busy_time);
1609 		col += col_desc[COL_CORES_BUSY_TIME].max_data_string + 2;
1610 	}
1611 
1612 	if (!col_desc[COL_CORES_CORE_FREQ].disabled) {
1613 		if (!g_cores_info[current_row].core_freq) {
1614 			snprintf(core_freq, MAX_CORE_FREQ_STR_LEN, "%s", "N/A");
1615 		} else {
1616 			snprintf(core_freq, MAX_CORE_FREQ_STR_LEN, "%" PRIu32,
1617 				 g_cores_info[current_row].core_freq);
1618 		}
1619 		print_max_len(g_tabs[CORES_TAB], TABS_DATA_START_ROW + item_index, col,
1620 			      col_desc[COL_CORES_CORE_FREQ].max_data_string, ALIGN_RIGHT, core_freq);
1621 		col += col_desc[COL_CORES_CORE_FREQ].max_data_string + 2;
1622 	}
1623 
1624 	if (!col_desc[COL_CORES_INTR].disabled) {
1625 		snprintf(in_interrupt, MAX_INTR_LEN, "%s", g_cores_info[current_row].in_interrupt ? "Yes" : "No");
1626 		print_max_len(g_tabs[CORES_TAB], TABS_DATA_START_ROW + item_index,
1627 			      col + (col_desc[COL_CORES_INTR].name_len / 2), col_desc[COL_CORES_INTR].max_data_string,
1628 			      ALIGN_LEFT, in_interrupt);
1629 		col += col_desc[COL_CORES_INTR].max_data_string + 1;
1630 	}
1631 
1632 	if (!col_desc[COL_CORES_CPU_USAGE].disabled) {
1633 		get_cpu_usage_str(g_cores_info[current_row].busy - g_cores_info[current_row].last_busy,
1634 				  g_cores_info[current_row].busy + g_cores_info[current_row].idle,
1635 				  cpu_usage);
1636 		print_max_len(g_tabs[CORES_TAB], TABS_DATA_START_ROW + item_index, col,
1637 			      col_desc[COL_CORES_CPU_USAGE].max_data_string, ALIGN_RIGHT, cpu_usage);
1638 	}
1639 }
1640 
1641 static uint8_t
1642 refresh_cores_tab(uint8_t current_page)
1643 {
1644 	uint64_t i;
1645 	uint16_t count = 0;
1646 	uint8_t max_pages, item_index;
1647 
1648 	count = g_last_cores_count;
1649 
1650 	max_pages = (count + g_max_row - WINDOW_HEADER - 1) / (g_max_row - WINDOW_HEADER);
1651 
1652 	for (i = current_page * g_max_data_rows;
1653 	     i < spdk_min(count, (uint64_t)((current_page + 1) * g_max_data_rows));
1654 	     i++) {
1655 		item_index = i - (current_page * g_max_data_rows);
1656 
1657 		draw_row_background(item_index, CORES_TAB);
1658 		draw_core_tab_row(i, item_index);
1659 
1660 		if (item_index == g_selected_row) {
1661 			wattroff(g_tabs[CORES_TAB], COLOR_PAIR(2));
1662 		}
1663 	}
1664 
1665 	g_max_selected_row = i - current_page * g_max_data_rows - 1;
1666 
1667 	return max_pages;
1668 }
1669 
1670 static uint8_t
1671 refresh_tab(enum tabs tab, uint8_t current_page)
1672 {
1673 	uint8_t (*refresh_function[NUMBER_OF_TABS])(uint8_t current_page) = {refresh_threads_tab, refresh_pollers_tab, refresh_cores_tab};
1674 	int color_pair[NUMBER_OF_TABS] = {COLOR_PAIR(2), COLOR_PAIR(2), COLOR_PAIR(2)};
1675 	int i;
1676 	uint8_t max_pages = 0;
1677 
1678 	color_pair[tab] = COLOR_PAIR(1);
1679 
1680 	for (i = 0; i < NUMBER_OF_TABS; i++) {
1681 		wbkgd(g_tab_win[i], color_pair[i]);
1682 	}
1683 
1684 	max_pages = (*refresh_function[tab])(current_page);
1685 	refresh();
1686 
1687 	for (i = 0; i < NUMBER_OF_TABS; i++) {
1688 		wrefresh(g_tab_win[i]);
1689 	}
1690 	draw_menu_win();
1691 
1692 	return max_pages;
1693 }
1694 
1695 static void
1696 print_in_middle(WINDOW *win, int starty, int startx, int width, char *string, chtype color)
1697 {
1698 	int length, temp;
1699 
1700 	length = strlen(string);
1701 	temp = (width - length) / 2;
1702 	wattron(win, color);
1703 	mvwprintw(win, starty, startx + temp, "%s", string);
1704 	wattroff(win, color);
1705 	refresh();
1706 }
1707 
1708 static void
1709 print_left(WINDOW *win, int starty, int startx, int width, char *string, chtype color)
1710 {
1711 	wattron(win, color);
1712 	mvwprintw(win, starty, startx, "%s", string);
1713 	wattroff(win, color);
1714 	refresh();
1715 }
1716 
1717 static void
1718 apply_filters(enum tabs tab)
1719 {
1720 	wclear(g_tabs[tab]);
1721 	draw_tabs(tab, g_current_sort_col[tab], g_current_sort_col2[tab]);
1722 }
1723 
1724 static ITEM **
1725 draw_filtering_menu(uint8_t position, WINDOW *filter_win, uint8_t tab, MENU **my_menu)
1726 {
1727 	const int ADDITIONAL_ELEMENTS = 3;
1728 	const int ROW_PADDING = 6;
1729 	const int WINDOW_START_X = 1;
1730 	const int WINDOW_START_Y = 3;
1731 	const int WINDOW_COLUMNS = 2;
1732 	struct col_desc *col_desc = g_col_desc[tab];
1733 	ITEM **my_items;
1734 	MENU *menu;
1735 	int i, elements;
1736 	uint8_t len = 0;
1737 
1738 	for (i = 0; col_desc[i].name != NULL; ++i) {
1739 		len = spdk_max(col_desc[i].name_len, len);
1740 	}
1741 
1742 	elements = i;
1743 
1744 	my_items = (ITEM **)calloc(elements * WINDOW_COLUMNS + ADDITIONAL_ELEMENTS, sizeof(ITEM *));
1745 	if (my_items == NULL) {
1746 		fprintf(stderr, "Unable to allocate an item list in draw_filtering_menu.\n");
1747 		return NULL;
1748 	}
1749 
1750 	for (i = 0; i < elements * 2; i++) {
1751 		my_items[i] = new_item(col_desc[i / WINDOW_COLUMNS].name, NULL);
1752 		i++;
1753 		my_items[i] = new_item(col_desc[i / WINDOW_COLUMNS].disabled ? "[ ]" : "[*]", NULL);
1754 	}
1755 
1756 	my_items[i] = new_item("     CLOSE", NULL);
1757 	set_item_userptr(my_items[i], apply_filters);
1758 
1759 	menu = new_menu((ITEM **)my_items);
1760 
1761 	menu_opts_off(menu, O_SHOWDESC);
1762 	set_menu_format(menu, elements + 1, WINDOW_COLUMNS);
1763 
1764 	set_menu_win(menu, filter_win);
1765 	set_menu_sub(menu, derwin(filter_win, elements + 1, len + ROW_PADDING, WINDOW_START_Y,
1766 				  WINDOW_START_X));
1767 
1768 	*my_menu = menu;
1769 
1770 	post_menu(menu);
1771 	refresh();
1772 	wrefresh(filter_win);
1773 
1774 	for (i = 0; i < position / WINDOW_COLUMNS; i++) {
1775 		menu_driver(menu, REQ_DOWN_ITEM);
1776 	}
1777 
1778 	return my_items;
1779 }
1780 
1781 static void
1782 delete_filtering_menu(MENU *my_menu, ITEM **my_items, uint8_t elements)
1783 {
1784 	int i;
1785 
1786 	unpost_menu(my_menu);
1787 	free_menu(my_menu);
1788 	for (i = 0; i < elements * 2 + 2; ++i) {
1789 		free_item(my_items[i]);
1790 	}
1791 	free(my_items);
1792 }
1793 
1794 static ITEM **
1795 refresh_filtering_menu(MENU **my_menu, WINDOW *filter_win, uint8_t tab, ITEM **my_items,
1796 		       uint8_t elements, uint8_t position)
1797 {
1798 	delete_filtering_menu(*my_menu, my_items, elements);
1799 	return draw_filtering_menu(position, filter_win, tab, my_menu);
1800 }
1801 
1802 static void
1803 filter_columns(uint8_t tab)
1804 {
1805 	const int WINDOW_HEADER_LEN = 5;
1806 	const int WINDOW_BORDER_LEN = 8;
1807 	const int WINDOW_HEADER_END_LINE = 2;
1808 	const int WINDOW_COLUMNS = 2;
1809 	struct col_desc *col_desc = g_col_desc[tab];
1810 	PANEL *filter_panel;
1811 	WINDOW *filter_win;
1812 	ITEM **my_items;
1813 	MENU *my_menu = NULL;
1814 	int i, c, elements;
1815 	bool stop_loop = false;
1816 	ITEM *cur;
1817 	void (*p)(enum tabs tab);
1818 	uint8_t current_index, len = 0;
1819 	bool disabled[TABS_COL_COUNT];
1820 
1821 	for (i = 0; col_desc[i].name != NULL; ++i) {
1822 		len = spdk_max(col_desc[i].name_len, len);
1823 	}
1824 
1825 	elements = i;
1826 
1827 	filter_win = newwin(elements + WINDOW_HEADER_LEN, len + WINDOW_BORDER_LEN,
1828 			    (g_max_row - elements - 1) / 2, (g_max_col - len) / 2);
1829 	assert(filter_win != NULL);
1830 	keypad(filter_win, TRUE);
1831 	filter_panel = new_panel(filter_win);
1832 	assert(filter_panel != NULL);
1833 
1834 	top_panel(filter_panel);
1835 	update_panels();
1836 	doupdate();
1837 
1838 	box(filter_win, 0, 0);
1839 
1840 	print_in_middle(filter_win, 1, 0, len + WINDOW_BORDER_LEN, "Filtering", COLOR_PAIR(3));
1841 	mvwaddch(filter_win, WINDOW_HEADER_END_LINE, 0, ACS_LTEE);
1842 	mvwhline(filter_win, WINDOW_HEADER_END_LINE, 1, ACS_HLINE, len + WINDOW_BORDER_LEN - 2);
1843 	mvwaddch(filter_win, WINDOW_HEADER_END_LINE, len + WINDOW_BORDER_LEN - 1, ACS_RTEE);
1844 
1845 	my_items = draw_filtering_menu(0, filter_win, tab, &my_menu);
1846 	if (my_items == NULL || my_menu == NULL) {
1847 		goto fail;
1848 	}
1849 
1850 	for (int i = 0; i < TABS_COL_COUNT; i++) {
1851 		disabled[i] = col_desc[i].disabled;
1852 	}
1853 
1854 	while (!stop_loop) {
1855 		c = wgetch(filter_win);
1856 
1857 		switch (c) {
1858 		case KEY_DOWN:
1859 			menu_driver(my_menu, REQ_DOWN_ITEM);
1860 			break;
1861 		case KEY_UP:
1862 			menu_driver(my_menu, REQ_UP_ITEM);
1863 			break;
1864 		case 27: /* ESC */
1865 		case 'q':
1866 			for (int i = 0; i < TABS_COL_COUNT; i++) {
1867 				cur = current_item(my_menu);
1868 				col_desc[i].disabled = disabled[i];
1869 
1870 				my_items = refresh_filtering_menu(&my_menu, filter_win, tab, my_items, elements,
1871 								  item_index(cur) + 1);
1872 				if (my_items == NULL || my_menu == NULL) {
1873 					goto fail;
1874 				}
1875 			}
1876 
1877 			stop_loop = true;
1878 			break;
1879 		case ' ': /* Space */
1880 			cur = current_item(my_menu);
1881 			current_index = item_index(cur) / WINDOW_COLUMNS;
1882 			col_desc[current_index].disabled = !col_desc[current_index].disabled;
1883 			my_items = refresh_filtering_menu(&my_menu, filter_win, tab, my_items, elements,
1884 							  item_index(cur) + 1);
1885 			if (my_items == NULL || my_menu == NULL) {
1886 				goto fail;
1887 			}
1888 			break;
1889 		case 10: /* Enter */
1890 			cur = current_item(my_menu);
1891 			current_index = item_index(cur) / WINDOW_COLUMNS;
1892 			if (current_index == elements) {
1893 				stop_loop = true;
1894 				p = item_userptr(cur);
1895 				p(tab);
1896 			} else {
1897 				col_desc[current_index].disabled = !col_desc[current_index].disabled;
1898 				my_items = refresh_filtering_menu(&my_menu, filter_win, tab, my_items, elements,
1899 								  item_index(cur) + 1);
1900 				if (my_items == NULL || my_menu == NULL) {
1901 					goto fail;
1902 				}
1903 			}
1904 			break;
1905 		}
1906 		wrefresh(filter_win);
1907 	}
1908 
1909 	delete_filtering_menu(my_menu, my_items, elements);
1910 
1911 	del_panel(filter_panel);
1912 	delwin(filter_win);
1913 
1914 	wclear(g_menu_win);
1915 	draw_menu_win();
1916 	return;
1917 
1918 fail:
1919 	fprintf(stderr, "Unable to filter the columns due to allocation failure.\n");
1920 	assert(false);
1921 }
1922 
1923 static void
1924 sort_type(enum tabs tab, int item_index)
1925 {
1926 	g_current_sort_col[tab] = item_index;
1927 }
1928 
1929 static void
1930 sort_type2(enum tabs tab, int item_index)
1931 {
1932 	g_current_sort_col2[tab] = item_index;
1933 }
1934 
1935 static void
1936 change_sorting(uint8_t tab, int winnum, bool *pstop_loop)
1937 {
1938 	const int WINDOW_HEADER_LEN = 4;
1939 	const int WINDOW_BORDER_LEN = 3;
1940 	const int WINDOW_START_X = 1;
1941 	const int WINDOW_START_Y = 4;
1942 	const int WINDOW_HEADER_END_LINE = 3;
1943 	const int WINDOW_MIN_WIDTH = 31;
1944 	PANEL *sort_panel;
1945 	WINDOW *sort_win;
1946 	ITEM **my_items;
1947 	MENU *my_menu;
1948 	int i, c, elements;
1949 	bool stop_loop = false;
1950 	ITEM *cur;
1951 	char *name;
1952 	char *help;
1953 	void (*p)(enum tabs tab, int item_index);
1954 	uint8_t len = WINDOW_MIN_WIDTH;
1955 
1956 	for (i = 0; g_col_desc[tab][i].name != NULL; ++i) {
1957 		len = spdk_max(len, g_col_desc[tab][i].name_len);
1958 	}
1959 
1960 	elements = i;
1961 
1962 	my_items = (ITEM **)calloc(elements + 1, sizeof(ITEM *));
1963 	if (my_items == NULL) {
1964 		fprintf(stderr, "Unable to allocate an item list in change_sorting.\n");
1965 		return;
1966 	}
1967 
1968 	for (i = 0; i < elements; ++i) {
1969 		my_items[i] = new_item(g_col_desc[tab][i].name, NULL);
1970 		set_item_userptr(my_items[i], (winnum == 0) ? sort_type : sort_type2);
1971 	}
1972 
1973 	my_menu = new_menu((ITEM **)my_items);
1974 
1975 	menu_opts_off(my_menu, O_SHOWDESC);
1976 
1977 	sort_win = newwin(elements + WINDOW_HEADER_LEN + 1, len + WINDOW_BORDER_LEN,
1978 			  (g_max_row - elements) / 2,
1979 			  (g_max_col - len) / 2 - len + len * winnum);
1980 	assert(sort_win != NULL);
1981 	keypad(sort_win, TRUE);
1982 	sort_panel = new_panel(sort_win);
1983 	assert(sort_panel != NULL);
1984 
1985 	top_panel(sort_panel);
1986 	update_panels();
1987 	doupdate();
1988 
1989 	set_menu_win(my_menu, sort_win);
1990 	set_menu_sub(my_menu, derwin(sort_win, elements, len + 1, WINDOW_START_Y, WINDOW_START_X));
1991 	box(sort_win, 0, 0);
1992 
1993 	if (winnum == 0) {
1994 		name = "Sorting #1";
1995 		help = "Right key for second sorting";
1996 	} else {
1997 		name = "Sorting #2";
1998 		help = "Left key for first sorting";
1999 	}
2000 
2001 	print_in_middle(sort_win, 1, 0, len + WINDOW_BORDER_LEN, name, COLOR_PAIR(3));
2002 	print_in_middle(sort_win, 2, 0, len + WINDOW_BORDER_LEN, help, COLOR_PAIR(3));
2003 	mvwaddch(sort_win, WINDOW_HEADER_END_LINE, 0, ACS_LTEE);
2004 	mvwhline(sort_win, WINDOW_HEADER_END_LINE, 1, ACS_HLINE, len + 1);
2005 	mvwaddch(sort_win, WINDOW_HEADER_END_LINE, len + WINDOW_BORDER_LEN - 1, ACS_RTEE);
2006 
2007 	post_menu(my_menu);
2008 	refresh();
2009 	wrefresh(sort_win);
2010 
2011 	while (!stop_loop) {
2012 		c = wgetch(sort_win);
2013 		/*
2014 		 * First sorting window:
2015 		 * Up/Down - select first sorting column;
2016 		 * Enter - apply current column;
2017 		 * Right - open second sorting window.
2018 		 * Second sorting window:
2019 		 * Up/Down - select second sorting column;
2020 		 * Enter - apply current column of both sorting windows;
2021 		 * Left - exit second window and reset second sorting key;
2022 		 * Right - do nothing.
2023 		 */
2024 		switch (c) {
2025 		case KEY_DOWN:
2026 			menu_driver(my_menu, REQ_DOWN_ITEM);
2027 			break;
2028 		case KEY_UP:
2029 			menu_driver(my_menu, REQ_UP_ITEM);
2030 			break;
2031 		case KEY_RIGHT:
2032 			if (winnum > 0) {
2033 				break;
2034 			}
2035 			change_sorting(tab, winnum + 1, &stop_loop);
2036 			/* Restore input. */
2037 			keypad(sort_win, TRUE);
2038 			post_menu(my_menu);
2039 			refresh();
2040 			wrefresh(sort_win);
2041 			redrawwin(sort_win);
2042 			if (winnum == 0) {
2043 				cur = current_item(my_menu);
2044 				p = item_userptr(cur);
2045 				p(tab, item_index(cur));
2046 			}
2047 			break;
2048 		case 27: /* ESC */
2049 			stop_loop = true;
2050 			break;
2051 		case KEY_LEFT:
2052 			if (winnum > 0) {
2053 				sort_type2(tab, COL_THREADS_NONE);
2054 			}
2055 		/* FALLTHROUGH */
2056 		case 10: /* Enter */
2057 			stop_loop = true;
2058 			if (winnum > 0 && c == 10) {
2059 				*pstop_loop = true;
2060 			}
2061 			if (c == 10) {
2062 				cur = current_item(my_menu);
2063 				p = item_userptr(cur);
2064 				p(tab, item_index(cur));
2065 			}
2066 			break;
2067 		}
2068 		wrefresh(sort_win);
2069 	}
2070 
2071 	if (winnum == 0) {
2072 		wclear(g_tabs[tab]);
2073 		draw_tabs(tab, g_current_sort_col[tab], g_current_sort_col2[tab]);
2074 	}
2075 
2076 	unpost_menu(my_menu);
2077 	free_menu(my_menu);
2078 
2079 	for (i = 0; i < elements; ++i) {
2080 		free_item(my_items[i]);
2081 	}
2082 
2083 	free(my_items);
2084 
2085 	del_panel(sort_panel);
2086 	delwin(sort_win);
2087 
2088 	if (winnum == 0) {
2089 		wclear(g_menu_win);
2090 		draw_menu_win();
2091 	}
2092 }
2093 
2094 static void
2095 change_refresh_rate(void)
2096 {
2097 	const int WINDOW_HEADER_END_LINE = 2;
2098 	PANEL *refresh_panel;
2099 	WINDOW *refresh_win;
2100 	int c;
2101 	bool stop_loop = false;
2102 	uint32_t rr_tmp, refresh_rate = 0;
2103 	char refresh_rate_str[MAX_STRING_LEN];
2104 
2105 	refresh_win = newwin(RR_WIN_HEIGHT, RR_WIN_WIDTH, (g_max_row - RR_WIN_HEIGHT - 1) / 2,
2106 			     (g_max_col - RR_WIN_WIDTH) / 2);
2107 	assert(refresh_win != NULL);
2108 	keypad(refresh_win, TRUE);
2109 	refresh_panel = new_panel(refresh_win);
2110 	assert(refresh_panel != NULL);
2111 
2112 	top_panel(refresh_panel);
2113 	update_panels();
2114 	doupdate();
2115 
2116 	box(refresh_win, 0, 0);
2117 
2118 	print_in_middle(refresh_win, 1, 0, RR_WIN_WIDTH + 1, "Enter refresh rate value [s]", COLOR_PAIR(3));
2119 	mvwaddch(refresh_win, WINDOW_HEADER_END_LINE, 0, ACS_LTEE);
2120 	mvwhline(refresh_win, WINDOW_HEADER_END_LINE, 1, ACS_HLINE, RR_WIN_WIDTH - 2);
2121 	mvwaddch(refresh_win, WINDOW_HEADER_END_LINE, RR_WIN_WIDTH, ACS_RTEE);
2122 	mvwprintw(refresh_win, WINDOW_HEADER_END_LINE + 1, (RR_WIN_WIDTH - 1) / 2, "%d", refresh_rate);
2123 
2124 	refresh();
2125 	wrefresh(refresh_win);
2126 
2127 	while (!stop_loop) {
2128 		c = wgetch(refresh_win);
2129 
2130 		switch (c) {
2131 		case '0':
2132 		case '1':
2133 		case '2':
2134 		case '3':
2135 		case '4':
2136 		case '5':
2137 		case '6':
2138 		case '7':
2139 		case '8':
2140 		case '9':
2141 			rr_tmp = refresh_rate * 10 + c - '0';
2142 
2143 			if (rr_tmp <= RR_MAX_VALUE) {
2144 				refresh_rate = rr_tmp;
2145 				snprintf(refresh_rate_str, MAX_STRING_LEN - 1, "%d", refresh_rate);
2146 				mvwprintw(refresh_win, WINDOW_HEADER_END_LINE + 1,
2147 					  (RR_WIN_WIDTH - 1 - strlen(refresh_rate_str)) / 2, "%d", refresh_rate);
2148 				refresh();
2149 				wrefresh(refresh_win);
2150 			}
2151 			break;
2152 		case KEY_BACKSPACE:
2153 		case 127:
2154 		case '\b':
2155 			refresh_rate = refresh_rate / 10;
2156 			snprintf(refresh_rate_str, MAX_STRING_LEN - 1, "%d", refresh_rate);
2157 			mvwprintw(refresh_win, WINDOW_HEADER_END_LINE + 1,
2158 				  (RR_WIN_WIDTH - 1 - strlen(refresh_rate_str) - 2) / 2, "       ");
2159 			mvwprintw(refresh_win, WINDOW_HEADER_END_LINE + 1,
2160 				  (RR_WIN_WIDTH - 1 - strlen(refresh_rate_str)) / 2, "%d", refresh_rate);
2161 			refresh();
2162 			wrefresh(refresh_win);
2163 			break;
2164 		case 27: /* ESC */
2165 		case 'q':
2166 			stop_loop = true;
2167 			break;
2168 		case 10: /* Enter */
2169 			g_sleep_time = refresh_rate;
2170 			stop_loop = true;
2171 			break;
2172 		}
2173 		wrefresh(refresh_win);
2174 	}
2175 
2176 	del_panel(refresh_panel);
2177 	delwin(refresh_win);
2178 }
2179 
2180 static void
2181 free_poller_history(void)
2182 {
2183 	struct run_counter_history *history, *tmp;
2184 
2185 	TAILQ_FOREACH_SAFE(history, &g_run_counter_history, link, tmp) {
2186 		TAILQ_REMOVE(&g_run_counter_history, history, link);
2187 		free(history);
2188 	}
2189 }
2190 
2191 static uint64_t
2192 get_position_for_window(uint64_t window_size, uint64_t max_size)
2193 {
2194 	/* This function calculates position for pop-up detail window.
2195 	 * Since horizontal and vertical positions are calculated the same way
2196 	 * there is no need for separate functions. */
2197 	window_size = spdk_min(window_size, max_size);
2198 
2199 	return (max_size - window_size) / 2;
2200 }
2201 
2202 static void
2203 print_bottom_error_message(char *msg)
2204 {
2205 	mvprintw(g_max_row - 1, g_max_col - strlen(msg) - 2, "%s", msg);
2206 }
2207 
2208 static void
2209 display_thread(struct rpc_thread_info *thread_info)
2210 {
2211 	PANEL *thread_panel;
2212 	WINDOW *thread_win;
2213 	uint64_t pollers_count, current_row, i, time;
2214 	int c;
2215 	bool stop_loop = false;
2216 	char idle_time[MAX_TIME_STR_LEN], busy_time[MAX_TIME_STR_LEN];
2217 
2218 	pthread_mutex_lock(&g_thread_lock);
2219 	pollers_count = thread_info->active_pollers_count +
2220 			thread_info->timed_pollers_count +
2221 			thread_info->paused_pollers_count;
2222 
2223 	thread_win = newwin(pollers_count + THREAD_WIN_HEIGHT, THREAD_WIN_WIDTH,
2224 			    get_position_for_window(THREAD_WIN_HEIGHT + pollers_count, g_max_row),
2225 			    get_position_for_window(THREAD_WIN_WIDTH, g_max_col));
2226 	keypad(thread_win, TRUE);
2227 	thread_panel = new_panel(thread_win);
2228 
2229 	top_panel(thread_panel);
2230 	update_panels();
2231 	doupdate();
2232 
2233 	box(thread_win, 0, 0);
2234 
2235 	print_in_middle(thread_win, 1, 0, THREAD_WIN_WIDTH, thread_info->name,
2236 			COLOR_PAIR(3));
2237 	mvwhline(thread_win, 2, 1, ACS_HLINE, THREAD_WIN_WIDTH - 2);
2238 	mvwaddch(thread_win, 2, THREAD_WIN_WIDTH, ACS_RTEE);
2239 
2240 	print_left(thread_win, 3, THREAD_WIN_FIRST_COL, THREAD_WIN_WIDTH,
2241 		   "Core:                Idle [us]:            Busy [us]:", COLOR_PAIR(5));
2242 	mvwprintw(thread_win, 3, THREAD_WIN_FIRST_COL + 6, "%d",
2243 		  thread_info->core_num);
2244 
2245 	if (g_interval_data) {
2246 		get_time_str(thread_info->idle - thread_info->last_idle, idle_time);
2247 		mvwprintw(thread_win, 3, THREAD_WIN_FIRST_COL + 32, "%s", idle_time);
2248 		get_time_str(thread_info->busy - thread_info->last_busy, busy_time);
2249 		mvwprintw(thread_win, 3, THREAD_WIN_FIRST_COL + 54, "%s", busy_time);
2250 	} else {
2251 		get_time_str(thread_info->idle, idle_time);
2252 		mvwprintw(thread_win, 3, THREAD_WIN_FIRST_COL + 32, "%s", idle_time);
2253 		get_time_str(thread_info->busy, busy_time);
2254 		mvwprintw(thread_win, 3, THREAD_WIN_FIRST_COL + 54, "%s", busy_time);
2255 	}
2256 
2257 	print_left(thread_win, 4, THREAD_WIN_FIRST_COL, THREAD_WIN_WIDTH,
2258 		   "Active pollers:      Timed pollers:        Paused pollers:", COLOR_PAIR(5));
2259 	mvwprintw(thread_win, 4, THREAD_WIN_FIRST_COL + 17, "%" PRIu64,
2260 		  thread_info->active_pollers_count);
2261 	mvwprintw(thread_win, 4, THREAD_WIN_FIRST_COL + 36, "%" PRIu64,
2262 		  thread_info->timed_pollers_count);
2263 	mvwprintw(thread_win, 4, THREAD_WIN_FIRST_COL + 59, "%" PRIu64,
2264 		  thread_info->paused_pollers_count);
2265 
2266 	mvwhline(thread_win, 5, 1, ACS_HLINE, THREAD_WIN_WIDTH - 2);
2267 
2268 	print_in_middle(thread_win, 6, 0, THREAD_WIN_WIDTH,
2269 			"Pollers                          Type    Total run count   Period", COLOR_PAIR(5));
2270 
2271 	mvwhline(thread_win, 7, 1, ACS_HLINE, THREAD_WIN_WIDTH - 2);
2272 
2273 	current_row = 8;
2274 
2275 	for (i = 0; i < g_last_pollers_count; i++) {
2276 		if (g_pollers_info[i].thread_id == thread_info->id) {
2277 			mvwprintw(thread_win, current_row, THREAD_WIN_FIRST_COL, "%s", g_pollers_info[i].name);
2278 			if (g_pollers_info[i].type == SPDK_ACTIVE_POLLER) {
2279 				mvwprintw(thread_win, current_row, THREAD_WIN_FIRST_COL + 33, "Active");
2280 			} else if (g_pollers_info[i].type == SPDK_TIMED_POLLER) {
2281 				mvwprintw(thread_win, current_row, THREAD_WIN_FIRST_COL + 33, "Timed");
2282 			} else {
2283 				mvwprintw(thread_win, current_row, THREAD_WIN_FIRST_COL + 33, "Paused");
2284 			}
2285 			mvwprintw(thread_win, current_row, THREAD_WIN_FIRST_COL + 41, "%" PRIu64,
2286 				  g_pollers_info[i].run_count);
2287 			if (g_pollers_info[i].period_ticks) {
2288 				time = g_pollers_info[i].period_ticks * SPDK_SEC_TO_USEC / g_tick_rate;
2289 				mvwprintw(thread_win, current_row, THREAD_WIN_FIRST_COL + 59, "%" PRIu64, time);
2290 			}
2291 			current_row++;
2292 		}
2293 	}
2294 
2295 	refresh();
2296 	wrefresh(thread_win);
2297 
2298 	pthread_mutex_unlock(&g_thread_lock);
2299 	while (!stop_loop) {
2300 		c = wgetch(thread_win);
2301 
2302 		switch (c) {
2303 		case 27: /* ESC */
2304 			stop_loop = true;
2305 			break;
2306 		default:
2307 			break;
2308 		}
2309 	}
2310 
2311 	del_panel(thread_panel);
2312 	delwin(thread_win);
2313 }
2314 
2315 static void
2316 show_thread(uint8_t current_page)
2317 {
2318 	struct rpc_thread_info thread_info;
2319 	uint64_t thread_number = current_page * g_max_data_rows + g_selected_row;
2320 
2321 	pthread_mutex_lock(&g_thread_lock);
2322 	assert(thread_number < g_last_threads_count);
2323 	thread_info = g_threads_info[thread_number];
2324 	pthread_mutex_unlock(&g_thread_lock);
2325 
2326 	display_thread(&thread_info);
2327 }
2328 
2329 static void
2330 show_single_thread(uint64_t thread_id)
2331 {
2332 	uint64_t i;
2333 	struct rpc_thread_info thread_info;
2334 
2335 	pthread_mutex_lock(&g_thread_lock);
2336 	for (i = 0; i < g_last_threads_count; i++) {
2337 		if (g_threads_info[i].id == thread_id) {
2338 			thread_info = g_threads_info[i];
2339 			pthread_mutex_unlock(&g_thread_lock);
2340 			display_thread(&thread_info);
2341 			return;
2342 		}
2343 	}
2344 	pthread_mutex_unlock(&g_thread_lock);
2345 }
2346 
2347 static void
2348 show_core(uint8_t current_page)
2349 {
2350 	PANEL *core_panel;
2351 	WINDOW *core_win;
2352 	uint64_t core_number = current_page * g_max_data_rows + g_selected_row;
2353 	struct rpc_core_info *core_info = &g_cores_info[core_number];
2354 	uint64_t threads_count, i;
2355 	uint64_t thread_id = 0;
2356 	uint16_t current_threads_row;
2357 	int c;
2358 	char core_win_title[25];
2359 	bool stop_loop = false;
2360 	char idle_time[MAX_TIME_STR_LEN], busy_time[MAX_TIME_STR_LEN];
2361 
2362 	pthread_mutex_lock(&g_thread_lock);
2363 	assert(core_number < g_last_cores_count);
2364 
2365 	threads_count = g_cores_info[core_number].threads.threads_count;
2366 
2367 	core_win = newwin(threads_count + CORE_WIN_HEIGHT, CORE_WIN_WIDTH,
2368 			  get_position_for_window(CORE_WIN_HEIGHT + threads_count, g_max_row),
2369 			  get_position_for_window(CORE_WIN_WIDTH, g_max_col));
2370 
2371 	keypad(core_win, TRUE);
2372 	core_panel = new_panel(core_win);
2373 
2374 	top_panel(core_panel);
2375 	update_panels();
2376 	doupdate();
2377 
2378 	box(core_win, 0, 0);
2379 	snprintf(core_win_title, sizeof(core_win_title), "Core %" PRIu32 " details",
2380 		 core_info->lcore);
2381 	print_in_middle(core_win, 1, 0, CORE_WIN_WIDTH, core_win_title, COLOR_PAIR(3));
2382 
2383 	mvwaddch(core_win, -1, 0, ACS_LTEE);
2384 	mvwhline(core_win, 2, 1, ACS_HLINE, CORE_WIN_WIDTH - 2);
2385 	mvwaddch(core_win, 2, CORE_WIN_WIDTH, ACS_RTEE);
2386 	print_left(core_win, 3, 1, CORE_WIN_WIDTH - (CORE_WIN_WIDTH / 3),
2387 		   "Frequency:             Intr:", COLOR_PAIR(5));
2388 	if (core_info->core_freq) {
2389 		mvwprintw(core_win, 3, CORE_WIN_FIRST_COL - 3, "%" PRIu32,
2390 			  core_info->core_freq);
2391 	} else {
2392 		mvwprintw(core_win, 3, CORE_WIN_FIRST_COL - 3, "%s", "N/A");
2393 	}
2394 
2395 	mvwprintw(core_win, 3, CORE_WIN_FIRST_COL + 15, "%s",
2396 		  core_info->in_interrupt ? "Yes" : "No");
2397 
2398 	mvwaddch(core_win, -1, 0, ACS_LTEE);
2399 	mvwhline(core_win, 4, 1, ACS_HLINE, CORE_WIN_WIDTH - 2);
2400 	mvwaddch(core_win, 4, CORE_WIN_WIDTH, ACS_RTEE);
2401 	print_left(core_win, 5, 1, CORE_WIN_WIDTH, "Thread count:          Idle time:", COLOR_PAIR(5));
2402 
2403 	mvwprintw(core_win, 5, CORE_WIN_FIRST_COL, "%" PRIu64,
2404 		  core_info->threads.threads_count);
2405 
2406 	if (g_interval_data == true) {
2407 		get_time_str(core_info->idle - core_info->last_idle, idle_time);
2408 		get_time_str(core_info->busy - core_info->last_busy, busy_time);
2409 	} else {
2410 		get_time_str(core_info->idle, idle_time);
2411 		get_time_str(core_info->busy, busy_time);
2412 	}
2413 	mvwprintw(core_win, 5, CORE_WIN_FIRST_COL + 20, "%s", idle_time);
2414 	mvwhline(core_win, 6, 1, ACS_HLINE, CORE_WIN_WIDTH - 2);
2415 
2416 	print_left(core_win, 7, 1, CORE_WIN_WIDTH, "Poller count:          Busy time:", COLOR_PAIR(5));
2417 	mvwprintw(core_win, 7, CORE_WIN_FIRST_COL, "%" PRIu64,
2418 		  core_info->pollers_count);
2419 
2420 	mvwprintw(core_win, 7, CORE_WIN_FIRST_COL + 20, "%s", busy_time);
2421 
2422 	mvwhline(core_win, 8, 1, ACS_HLINE, CORE_WIN_WIDTH - 2);
2423 	print_left(core_win, 9, 1, CORE_WIN_WIDTH, "Threads on this core", COLOR_PAIR(5));
2424 
2425 	for (i = 0; i < core_info->threads.threads_count; i++) {
2426 		mvwprintw(core_win, i + 10, 1, "%s", core_info->threads.thread[i].name);
2427 	}
2428 	pthread_mutex_unlock(&g_thread_lock);
2429 
2430 	refresh();
2431 	wrefresh(core_win);
2432 
2433 	current_threads_row = 0;
2434 
2435 	while (!stop_loop) {
2436 		pthread_mutex_lock(&g_thread_lock);
2437 		for (i = 0; i < core_info->threads.threads_count; i++) {
2438 			if (i != current_threads_row) {
2439 				mvwprintw(core_win, i + 10, 1, "%s", core_info->threads.thread[i].name);
2440 			} else {
2441 				print_left(core_win, i + 10, 1, CORE_WIN_WIDTH - 2,
2442 					   core_info->threads.thread[i].name, COLOR_PAIR(2));
2443 			}
2444 		}
2445 		pthread_mutex_unlock(&g_thread_lock);
2446 
2447 		wrefresh(core_win);
2448 
2449 		c = wgetch(core_win);
2450 		switch (c) {
2451 		case 10: /* ENTER */
2452 			pthread_mutex_lock(&g_thread_lock);
2453 			if (core_info->threads.threads_count > 0) {
2454 				thread_id = core_info->threads.thread[current_threads_row].id;
2455 			}
2456 			pthread_mutex_unlock(&g_thread_lock);
2457 			if (thread_id != 0) {
2458 				show_single_thread(thread_id);
2459 			}
2460 			break;
2461 		case 27: /* ESC */
2462 			stop_loop = true;
2463 			break;
2464 		case KEY_UP:
2465 			if (current_threads_row != 0) {
2466 				current_threads_row--;
2467 			}
2468 			break;
2469 		case KEY_DOWN:
2470 			pthread_mutex_lock(&g_thread_lock);
2471 			if (current_threads_row != core_info->threads.threads_count - 1) {
2472 				current_threads_row++;
2473 			}
2474 			pthread_mutex_unlock(&g_thread_lock);
2475 			break;
2476 		default:
2477 			break;
2478 		}
2479 	}
2480 
2481 	del_panel(core_panel);
2482 	delwin(core_win);
2483 }
2484 
2485 static void
2486 show_poller(uint8_t current_page)
2487 {
2488 	PANEL *poller_panel;
2489 	WINDOW *poller_win;
2490 	uint64_t last_run_counter, last_busy_counter, busy_count;
2491 	uint64_t poller_number = current_page * g_max_data_rows + g_selected_row;
2492 	struct rpc_poller_info *poller;
2493 	bool stop_loop = false;
2494 	char poller_period[MAX_TIME_STR_LEN];
2495 	int c;
2496 
2497 	pthread_mutex_lock(&g_thread_lock);
2498 
2499 	assert(poller_number < g_last_pollers_count);
2500 	poller = &g_pollers_info[poller_number];
2501 
2502 	poller_win = newwin(POLLER_WIN_HEIGHT, POLLER_WIN_WIDTH,
2503 			    get_position_for_window(POLLER_WIN_HEIGHT, g_max_row),
2504 			    get_position_for_window(POLLER_WIN_WIDTH, g_max_col));
2505 
2506 	keypad(poller_win, TRUE);
2507 	poller_panel = new_panel(poller_win);
2508 
2509 	top_panel(poller_panel);
2510 	update_panels();
2511 	doupdate();
2512 
2513 	box(poller_win, 0, 0);
2514 
2515 	print_in_middle(poller_win, 1, 0, POLLER_WIN_WIDTH, poller->name, COLOR_PAIR(3));
2516 	mvwhline(poller_win, 2, 1, ACS_HLINE, POLLER_WIN_WIDTH - 2);
2517 	mvwaddch(poller_win, 2, POLLER_WIN_WIDTH, ACS_RTEE);
2518 
2519 	print_left(poller_win, 3, 2, POLLER_WIN_WIDTH, "Type:                  On thread:", COLOR_PAIR(5));
2520 	mvwprintw(poller_win, 3, POLLER_WIN_FIRST_COL, "%s",
2521 		  poller_type_str[poller->type]);
2522 	mvwprintw(poller_win, 3, POLLER_WIN_FIRST_COL + 23, "%s", poller->thread_name);
2523 
2524 	print_left(poller_win, 4, 2, POLLER_WIN_WIDTH, "Run count:", COLOR_PAIR(5));
2525 
2526 	last_run_counter = get_last_run_counter(poller->id, poller->thread_id);
2527 	last_busy_counter = get_last_busy_counter(poller->id, poller->thread_id);
2528 	if (g_interval_data) {
2529 		mvwprintw(poller_win, 4, POLLER_WIN_FIRST_COL, "%" PRIu64, poller->run_count - last_run_counter);
2530 	} else {
2531 		mvwprintw(poller_win, 4, POLLER_WIN_FIRST_COL, "%" PRIu64, poller->run_count);
2532 	}
2533 
2534 	if (poller->period_ticks != 0) {
2535 		print_left(poller_win, 4, 28, POLLER_WIN_WIDTH, "Period:", COLOR_PAIR(5));
2536 		get_time_str(poller->period_ticks, poller_period);
2537 		mvwprintw(poller_win, 4, POLLER_WIN_FIRST_COL + 23, "%s", poller_period);
2538 	}
2539 	mvwhline(poller_win, 5, 1, ACS_HLINE, POLLER_WIN_WIDTH - 2);
2540 
2541 	busy_count = g_interval_data ? poller->busy_count - last_busy_counter : poller->busy_count;
2542 	if (busy_count != 0) {
2543 		print_left(poller_win, 6, 2, POLLER_WIN_WIDTH,  "Status:               Busy count:", COLOR_PAIR(5));
2544 
2545 		if (g_interval_data == false && poller->busy_count == last_busy_counter) {
2546 			print_left(poller_win, 6, POLLER_WIN_FIRST_COL, POLLER_WIN_WIDTH, "Idle", COLOR_PAIR(7));
2547 		} else {
2548 			print_left(poller_win, 6, POLLER_WIN_FIRST_COL, POLLER_WIN_WIDTH, "Busy", COLOR_PAIR(6));
2549 		}
2550 
2551 		mvwprintw(poller_win, 6, POLLER_WIN_FIRST_COL + 23, "%" PRIu64, busy_count);
2552 	} else {
2553 		print_in_middle(poller_win, 6, 1, POLLER_WIN_WIDTH - 7, "Status:", COLOR_PAIR(5));
2554 		print_in_middle(poller_win, 6, 1, POLLER_WIN_WIDTH + 6, "Idle", COLOR_PAIR(7));
2555 	}
2556 
2557 	refresh();
2558 	wrefresh(poller_win);
2559 
2560 	pthread_mutex_unlock(&g_thread_lock);
2561 	while (!stop_loop) {
2562 		c = wgetch(poller_win);
2563 		switch (c) {
2564 		case 27: /* ESC */
2565 			stop_loop = true;
2566 			break;
2567 		default:
2568 			break;
2569 		}
2570 	}
2571 
2572 	del_panel(poller_panel);
2573 	delwin(poller_win);
2574 }
2575 
2576 static void *
2577 data_thread_routine(void *arg)
2578 {
2579 	int rc;
2580 
2581 	while (1) {
2582 		pthread_mutex_lock(&g_thread_lock);
2583 		if (g_quit_app) {
2584 			pthread_mutex_unlock(&g_thread_lock);
2585 			break;
2586 		}
2587 		pthread_mutex_unlock(&g_thread_lock);
2588 
2589 		/* Get data from RPC for each object type.
2590 		 * Start with cores since their number should not change. */
2591 		rc = get_cores_data();
2592 		if (rc) {
2593 			print_bottom_error_message("ERROR occurred while getting cores data");
2594 		}
2595 		rc = get_thread_data();
2596 		if (rc) {
2597 			print_bottom_error_message("ERROR occurred while getting threads data");
2598 		}
2599 
2600 		rc = get_pollers_data();
2601 		if (rc) {
2602 			print_bottom_error_message("ERROR occurred while getting pollers data");
2603 		}
2604 
2605 		usleep(g_sleep_time * SPDK_SEC_TO_USEC);
2606 	}
2607 
2608 	return NULL;
2609 }
2610 
2611 static void
2612 help_window_display(void)
2613 {
2614 	PANEL *help_panel;
2615 	WINDOW *help_win;
2616 	bool stop_loop = false;
2617 	int c;
2618 	uint64_t row = 1, col = 2, desc_second_row_col = 26, header_footer_col = 0;
2619 
2620 	help_win = newwin(HELP_WIN_HEIGHT, HELP_WIN_WIDTH,
2621 			  get_position_for_window(HELP_WIN_HEIGHT, g_max_row),
2622 			  get_position_for_window(HELP_WIN_WIDTH, g_max_col));
2623 	help_panel = new_panel(help_win);
2624 	top_panel(help_panel);
2625 	update_panels();
2626 	doupdate();
2627 
2628 	box(help_win, 0, 0);
2629 
2630 	/* Header */
2631 	print_in_middle(help_win, row, header_footer_col, HELP_WIN_WIDTH, "HELP", COLOR_PAIR(3));
2632 	mvwhline(help_win, 2, 1, ACS_HLINE, HELP_WIN_WIDTH - 2);
2633 	mvwaddch(help_win, 2, HELP_WIN_WIDTH, ACS_RTEE);
2634 	row = 3;
2635 
2636 	/* Content */
2637 	print_left(help_win, row, col, HELP_WIN_WIDTH, "MENU options", COLOR_PAIR(5));
2638 	print_left(help_win, ++row, ++col, HELP_WIN_WIDTH, "[q] Quit		- quit this application",
2639 		   COLOR_PAIR(10));
2640 	print_left(help_win, ++row, col,  HELP_WIN_WIDTH,
2641 		   "[Tab] Next tab	- switch to next tab", COLOR_PAIR(10));
2642 	print_left(help_win, ++row, col,  HELP_WIN_WIDTH,
2643 		   "[1-3] Select tab	- switch to THREADS, POLLERS or CORES tab", COLOR_PAIR(10));
2644 	print_left(help_win, ++row, col,  HELP_WIN_WIDTH,
2645 		   "[PgUp] Previous page	- scroll up to previous page", COLOR_PAIR(10));
2646 	print_left(help_win, ++row, col,  HELP_WIN_WIDTH,
2647 		   "[PgDown] Next page	- scroll down to next page", COLOR_PAIR(10));
2648 	print_left(help_win, ++row, col,  HELP_WIN_WIDTH,
2649 		   "[Up] Arrow key	- go to previous data row", COLOR_PAIR(10));
2650 	print_left(help_win, ++row, col,  HELP_WIN_WIDTH,
2651 		   "[Down] Arrow key	- go to next data row", COLOR_PAIR(10));
2652 	print_left(help_win, ++row, col,  HELP_WIN_WIDTH,
2653 		   "[Right] Arrow key	- go to second sorting window", COLOR_PAIR(10));
2654 	print_left(help_win, ++row, col,  HELP_WIN_WIDTH,
2655 		   "[Left] Arrow key	- close second sorting window", COLOR_PAIR(10));
2656 	print_left(help_win, ++row, col,  HELP_WIN_WIDTH,
2657 		   "[c] Columns		- choose data columns to display", COLOR_PAIR(10));
2658 	print_left(help_win, ++row, col,  HELP_WIN_WIDTH,
2659 		   "[s] Sorting		- change sorting by column", COLOR_PAIR(10));
2660 	print_left(help_win, ++row, col,  HELP_WIN_WIDTH,
2661 		   "[r] Refresh rate	- set refresh rate <0, 255> in seconds", COLOR_PAIR(10));
2662 	print_left(help_win, ++row, desc_second_row_col,  HELP_WIN_WIDTH, "that value in seconds",
2663 		   COLOR_PAIR(10));
2664 	print_left(help_win, ++row, col,  HELP_WIN_WIDTH,
2665 		   "[Enter] Item details	- show current data row details (Enter to open, Esc to close)",
2666 		   COLOR_PAIR(10));
2667 	print_left(help_win, ++row, col,  HELP_WIN_WIDTH,
2668 		   "[t] Total/Interval	- switch to display data measured from the start of SPDK", COLOR_PAIR(10));
2669 	print_left(help_win, ++row, desc_second_row_col,  HELP_WIN_WIDTH,
2670 		   "application or last refresh", COLOR_PAIR(10));
2671 	print_left(help_win, ++row, col,  HELP_WIN_WIDTH, "[h] Help		- show this help window",
2672 		   COLOR_PAIR(10));
2673 
2674 	/* Footer */
2675 	mvwhline(help_win, HELP_WIN_HEIGHT - 3, 1, ACS_HLINE, HELP_WIN_WIDTH - 2);
2676 	mvwaddch(help_win, HELP_WIN_HEIGHT - 3, HELP_WIN_WIDTH, ACS_RTEE);
2677 
2678 	print_in_middle(help_win, HELP_WIN_HEIGHT - 2, header_footer_col, HELP_WIN_WIDTH,
2679 			"[Esc] Close this window", COLOR_PAIR(10));
2680 
2681 	refresh();
2682 	wrefresh(help_win);
2683 
2684 	while (!stop_loop) {
2685 		c = wgetch(help_win);
2686 
2687 		switch (c) {
2688 		case 27: /* ESC */
2689 			stop_loop = true;
2690 			break;
2691 		default:
2692 			break;
2693 		}
2694 	}
2695 
2696 	del_panel(help_panel);
2697 	delwin(help_win);
2698 
2699 }
2700 
2701 static void
2702 show_stats(pthread_t *data_thread)
2703 {
2704 	const int CURRENT_PAGE_STR_LEN = 50;
2705 	long int time_last, time_dif;
2706 	struct timespec time_now;
2707 	int c;
2708 	int max_row, max_col;
2709 	uint8_t active_tab = THREADS_TAB;
2710 	uint8_t current_page = 0;
2711 	uint8_t max_pages = 1;
2712 	uint16_t required_size = WINDOW_HEADER + 1;
2713 	uint64_t i;
2714 	char current_page_str[CURRENT_PAGE_STR_LEN];
2715 	bool force_refresh = true;
2716 
2717 	clock_gettime(CLOCK_REALTIME, &time_now);
2718 	time_last = time_now.tv_sec;
2719 
2720 	switch_tab(THREADS_TAB);
2721 
2722 	while (1) {
2723 		/* Check if interface has to be resized (terminal size changed) */
2724 		getmaxyx(stdscr, max_row, max_col);
2725 
2726 		if (max_row != g_max_row || max_col != g_max_col) {
2727 			if (max_row != g_max_row) {
2728 				current_page = 0;
2729 			}
2730 			g_max_row = spdk_max(max_row, required_size);
2731 			g_max_col = max_col;
2732 			g_data_win_size = g_max_row - required_size + 1;
2733 			g_max_data_rows = g_max_row - WINDOW_HEADER;
2734 			resize_interface(active_tab);
2735 		}
2736 
2737 		clock_gettime(CLOCK_REALTIME, &time_now);
2738 		time_dif = time_now.tv_sec - time_last;
2739 		if (time_dif < 0) {
2740 			time_dif = g_sleep_time;
2741 		}
2742 
2743 		if (time_dif >= g_sleep_time || force_refresh) {
2744 			time_last = time_now.tv_sec;
2745 			pthread_mutex_lock(&g_thread_lock);
2746 			max_pages = refresh_tab(active_tab, current_page);
2747 			pthread_mutex_unlock(&g_thread_lock);
2748 
2749 			snprintf(current_page_str, CURRENT_PAGE_STR_LEN - 1, "Page: %d/%d", current_page + 1, max_pages);
2750 			mvprintw(g_max_row - 1, 1, "%s", current_page_str);
2751 
2752 			refresh();
2753 		}
2754 
2755 		c = getch();
2756 		if (c == 'q') {
2757 			pthread_mutex_lock(&g_thread_lock);
2758 			g_quit_app = true;
2759 			pthread_mutex_unlock(&g_thread_lock);
2760 			break;
2761 		}
2762 
2763 		force_refresh = true;
2764 
2765 		switch (c) {
2766 		case '1':
2767 		case '2':
2768 		case '3':
2769 			active_tab = c - '1';
2770 			current_page = 0;
2771 			g_selected_row = 0;
2772 			switch_tab(active_tab);
2773 			break;
2774 		case '\t':
2775 			if (active_tab < NUMBER_OF_TABS - 1) {
2776 				active_tab++;
2777 			} else {
2778 				active_tab = THREADS_TAB;
2779 			}
2780 			g_selected_row = 0;
2781 			current_page = 0;
2782 			switch_tab(active_tab);
2783 			break;
2784 		case 's':
2785 			sort_type2(active_tab, COL_THREADS_NONE);
2786 			change_sorting(active_tab, 0, NULL);
2787 			break;
2788 		case 'c':
2789 			filter_columns(active_tab);
2790 			break;
2791 		case 'r':
2792 			change_refresh_rate();
2793 			break;
2794 		case 't':
2795 			g_interval_data = !g_interval_data;
2796 			break;
2797 		case KEY_NPAGE: /* PgDown */
2798 			if (current_page + 1 < max_pages) {
2799 				current_page++;
2800 			}
2801 			wclear(g_tabs[active_tab]);
2802 			g_selected_row = 0;
2803 			draw_tabs(active_tab, g_current_sort_col[active_tab], g_current_sort_col2[active_tab]);
2804 			break;
2805 		case KEY_PPAGE: /* PgUp */
2806 			if (current_page > 0) {
2807 				current_page--;
2808 			}
2809 			wclear(g_tabs[active_tab]);
2810 			g_selected_row = 0;
2811 			draw_tabs(active_tab, g_current_sort_col[active_tab], g_current_sort_col2[active_tab]);
2812 			break;
2813 		case KEY_UP: /* Arrow up */
2814 			if (g_selected_row > 0) {
2815 				g_selected_row--;
2816 			}
2817 			break;
2818 		case KEY_DOWN: /* Arrow down */
2819 			if (g_selected_row < g_max_selected_row) {
2820 				g_selected_row++;
2821 			}
2822 			break;
2823 		case 10: /* Enter */
2824 			if (active_tab == THREADS_TAB) {
2825 				show_thread(current_page);
2826 			} else if (active_tab == CORES_TAB) {
2827 				show_core(current_page);
2828 			} else if (active_tab == POLLERS_TAB) {
2829 				show_poller(current_page);
2830 			}
2831 			break;
2832 		case 'h':
2833 			help_window_display();
2834 			break;
2835 		default:
2836 			force_refresh = false;
2837 			break;
2838 		}
2839 	}
2840 
2841 	pthread_join(*data_thread, NULL);
2842 
2843 	free_poller_history();
2844 
2845 	/* Free memory holding current data states before quitting application */
2846 	for (i = 0; i < g_last_pollers_count; i++) {
2847 		free_rpc_poller(&g_pollers_info[i]);
2848 	}
2849 	for (i = 0; i < g_last_threads_count; i++) {
2850 		free_rpc_threads_stats(&g_threads_info[i]);
2851 	}
2852 	free_rpc_core_info(g_cores_info, g_last_cores_count);
2853 }
2854 
2855 static void
2856 draw_interface(void)
2857 {
2858 	int i;
2859 	uint16_t required_size =  WINDOW_HEADER + 1;
2860 
2861 	getmaxyx(stdscr, g_max_row, g_max_col);
2862 	g_max_row = spdk_max(g_max_row, required_size);
2863 	g_data_win_size = g_max_row - required_size;
2864 	g_max_data_rows = g_max_row - WINDOW_HEADER;
2865 
2866 	g_menu_win = newwin(MENU_WIN_HEIGHT, g_max_col, g_max_row - MENU_WIN_HEIGHT - 1,
2867 			    MENU_WIN_LOCATION_COL);
2868 	assert(g_menu_win != NULL);
2869 	draw_menu_win();
2870 
2871 	for (i = 0; i < NUMBER_OF_TABS; i++) {
2872 		g_tab_win[i] = newwin(TAB_WIN_HEIGHT, g_max_col / NUMBER_OF_TABS - TABS_SPACING,
2873 				      TAB_WIN_LOCATION_ROW, g_max_col / NUMBER_OF_TABS * i + 1);
2874 		assert(g_tab_win[i] != NULL);
2875 		draw_tab_win(i);
2876 
2877 		g_tabs[i] = newwin(g_max_row - MENU_WIN_HEIGHT - TAB_WIN_HEIGHT - 2, g_max_col, TABS_LOCATION_ROW,
2878 				   TABS_LOCATION_COL);
2879 		draw_tabs(i, g_current_sort_col[i], g_current_sort_col2[i]);
2880 		g_panels[i] = new_panel(g_tabs[i]);
2881 		assert(g_panels[i] != NULL);
2882 	}
2883 
2884 	update_panels();
2885 	doupdate();
2886 }
2887 
2888 static void
2889 finish(int sig)
2890 {
2891 	/* End ncurses mode */
2892 	endwin();
2893 	spdk_jsonrpc_client_close(g_rpc_client);
2894 	exit(0);
2895 }
2896 
2897 static void
2898 setup_ncurses(void)
2899 {
2900 	clear();
2901 	noecho();
2902 	timeout(1);
2903 	curs_set(0);
2904 	keypad(stdscr, TRUE);
2905 	start_color();
2906 	init_pair(1, COLOR_BLACK, COLOR_GREEN);
2907 	init_pair(2, COLOR_BLACK, COLOR_WHITE);
2908 	init_pair(3, COLOR_YELLOW, COLOR_BLACK);
2909 	init_pair(4, COLOR_BLACK, COLOR_YELLOW);
2910 	init_pair(5, COLOR_GREEN, COLOR_BLACK);
2911 	init_pair(6, COLOR_RED, COLOR_BLACK);
2912 	init_pair(7, COLOR_BLUE, COLOR_BLACK);
2913 	init_pair(8, COLOR_RED, COLOR_WHITE);
2914 	init_pair(9, COLOR_BLUE, COLOR_WHITE);
2915 	init_pair(10, COLOR_WHITE, COLOR_BLACK);
2916 
2917 	if (has_colors() == FALSE) {
2918 		endwin();
2919 		printf("Your terminal does not support color\n");
2920 		exit(1);
2921 	}
2922 
2923 	/* Handle signals to exit gracefully cleaning up ncurses */
2924 	(void) signal(SIGINT, finish);
2925 	(void) signal(SIGPIPE, finish);
2926 	(void) signal(SIGABRT, finish);
2927 }
2928 
2929 static void
2930 usage(const char *program_name)
2931 {
2932 	printf("%s [options]", program_name);
2933 	printf("\n");
2934 	printf("options:\n");
2935 	printf(" -r <path>  RPC connect address (default: /var/tmp/spdk.sock)\n");
2936 	printf(" -h         show this usage\n");
2937 }
2938 
2939 static int
2940 rpc_decode_tick_rate(struct spdk_json_val *val, uint64_t *tick_rate)
2941 {
2942 	struct t_rate {
2943 		uint64_t tr;
2944 	};
2945 
2946 	const struct spdk_json_object_decoder rpc_tick_rate_decoder[] = {
2947 		{"tick_rate", offsetof(struct t_rate, tr), spdk_json_decode_uint64}
2948 	};
2949 
2950 	int rc;
2951 	struct t_rate tmp;
2952 
2953 	rc = spdk_json_decode_object_relaxed(val, rpc_tick_rate_decoder,
2954 					     SPDK_COUNTOF(rpc_tick_rate_decoder), &tmp);
2955 
2956 	*tick_rate = tmp.tr;
2957 
2958 	return rc;
2959 }
2960 
2961 static int
2962 wait_init(pthread_t *data_thread)
2963 {
2964 	struct spdk_jsonrpc_client_response *json_resp = NULL;
2965 	char *uninit_log = "Waiting for SPDK target application to initialize...",
2966 	      *uninit_error = "Unable to read SPDK application state!";
2967 	int c, max_col, rc = 0;
2968 	uint64_t tick_rate;
2969 
2970 	max_col = getmaxx(stdscr);
2971 	print_in_middle(stdscr, FIRST_DATA_ROW, 1, max_col, uninit_log, COLOR_PAIR(5));
2972 	rc = rpc_send_req("framework_wait_init", &json_resp);
2973 	if (rc) {
2974 		while (1) {
2975 			print_in_middle(stdscr, FIRST_DATA_ROW, 1, max_col, uninit_error, COLOR_PAIR(8));
2976 			c = getch();
2977 			if (c == 'q') {
2978 				return -1;
2979 			}
2980 		}
2981 	}
2982 
2983 	spdk_jsonrpc_client_free_response(json_resp);
2984 
2985 	rc = pthread_mutex_init(&g_thread_lock, NULL);
2986 	if (rc) {
2987 		fprintf(stderr, "mutex lock failed to initialize: %d\n", errno);
2988 		return -1;
2989 	}
2990 
2991 	memset(&g_threads_info, 0, sizeof(struct rpc_thread_info) * RPC_MAX_THREADS);
2992 	memset(&g_cores_info, 0, sizeof(struct rpc_core_info) * RPC_MAX_CORES);
2993 
2994 	/* Decode tick rate */
2995 	rc = rpc_send_req("framework_get_reactors", &json_resp);
2996 	if (rc) {
2997 		return rc;
2998 	}
2999 
3000 	if (rpc_decode_tick_rate(json_resp->result, &tick_rate)) {
3001 		spdk_jsonrpc_client_free_response(json_resp);
3002 		return -EINVAL;
3003 	}
3004 
3005 	spdk_jsonrpc_client_free_response(json_resp);
3006 
3007 	g_tick_rate = tick_rate;
3008 
3009 	/* This is to get first batch of data for display functions.
3010 	 * Since data thread makes RPC calls that take more time than
3011 	 * startup of display functions on main thread, without these
3012 	 * calls both threads would be subject to a race condition. */
3013 	rc = get_thread_data();
3014 	if (rc) {
3015 		return -1;
3016 	}
3017 
3018 	rc = get_pollers_data();
3019 	if (rc) {
3020 		return -1;
3021 	}
3022 
3023 	rc = get_cores_data();
3024 	if (rc) {
3025 		return -1;
3026 	}
3027 
3028 	rc = pthread_create(data_thread, NULL, &data_thread_routine, NULL);
3029 	if (rc) {
3030 		fprintf(stderr, "data thread creation failed: %d\n", errno);
3031 		return -1;
3032 	}
3033 	return 0;
3034 }
3035 
3036 int main(int argc, char **argv)
3037 {
3038 	int op, rc;
3039 	char *socket = SPDK_DEFAULT_RPC_ADDR;
3040 	pthread_t data_thread;
3041 
3042 	while ((op = getopt(argc, argv, "r:h")) != -1) {
3043 		switch (op) {
3044 		case 'r':
3045 			socket = optarg;
3046 			break;
3047 		default:
3048 			usage(argv[0]);
3049 			return op == 'h' ? 0 : 1;
3050 		}
3051 	}
3052 
3053 	g_rpc_client = spdk_jsonrpc_client_connect(socket, AF_UNIX);
3054 	if (!g_rpc_client) {
3055 		fprintf(stderr, "spdk_jsonrpc_client_connect() failed: %d\n", errno);
3056 		return 1;
3057 	}
3058 
3059 	initscr();
3060 	init_str_len();
3061 	setup_ncurses();
3062 	draw_interface();
3063 
3064 	rc = wait_init(&data_thread);
3065 	if (!rc) {
3066 		show_stats(&data_thread);
3067 	}
3068 
3069 	finish(0);
3070 
3071 	return (0);
3072 }
3073