xref: /spdk/app/spdk_top/spdk_top.c (revision 32999ab917f67af61872f868585fd3d78ad6fb8a)
1 /*-
2  *   BSD LICENSE
3  *
4  *   Copyright (c) Intel Corporation.
5  *   All rights reserved.
6  *
7  *   Redistribution and use in source and binary forms, with or without
8  *   modification, are permitted provided that the following conditions
9  *   are met:
10  *
11  *     * Redistributions of source code must retain the above copyright
12  *       notice, this list of conditions and the following disclaimer.
13  *     * Redistributions in binary form must reproduce the above copyright
14  *       notice, this list of conditions and the following disclaimer in
15  *       the documentation and/or other materials provided with the
16  *       distribution.
17  *     * Neither the name of Intel Corporation nor the names of its
18  *       contributors may be used to endorse or promote products derived
19  *       from this software without specific prior written permission.
20  *
21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32  */
33 
34 #include "spdk/stdinc.h"
35 #include "spdk/jsonrpc.h"
36 #include "spdk/rpc.h"
37 #include "spdk/event.h"
38 #include "spdk/util.h"
39 #include "spdk/env.h"
40 
41 #if defined __has_include
42 #if __has_include(<ncurses/panel.h>)
43 #include <ncurses/ncurses.h>
44 #include <ncurses/panel.h>
45 #include <ncurses/menu.h>
46 #else
47 #include <ncurses.h>
48 #include <panel.h>
49 #include <menu.h>
50 #endif
51 #else
52 #include <ncurses.h>
53 #include <panel.h>
54 #include <menu.h>
55 #endif
56 
57 #define RPC_MAX_THREADS 1024
58 #define RPC_MAX_POLLERS 1024
59 #define RPC_MAX_CORES 255
60 #define MAX_THREAD_NAME 128
61 #define MAX_POLLER_NAME 128
62 #define MAX_THREADS 4096
63 #define RR_MAX_VALUE 255
64 
65 #define MAX_STRING_LEN 12289 /* 3x 4k monitors + 1 */
66 #define TAB_WIN_HEIGHT 3
67 #define TAB_WIN_LOCATION_ROW 1
68 #define TABS_SPACING 2
69 #define TABS_LOCATION_ROW 4
70 #define TABS_LOCATION_COL 0
71 #define TABS_DATA_START_ROW 3
72 #define TABS_DATA_START_COL 2
73 #define TABS_COL_COUNT 10
74 #define MENU_WIN_HEIGHT 3
75 #define MENU_WIN_SPACING 4
76 #define MENU_WIN_LOCATION_COL 0
77 #define RR_WIN_WIDTH 32
78 #define RR_WIN_HEIGHT 5
79 #define MAX_THREAD_NAME_LEN 26
80 #define MAX_THREAD_COUNT_STR_LEN 14
81 #define MAX_POLLER_NAME_LEN 36
82 #define MAX_POLLER_COUNT_STR_LEN 16
83 #define MAX_POLLER_TYPE_STR_LEN 8
84 #define MAX_POLLER_IND_STR_LEN 8
85 #define MAX_CORE_MASK_STR_LEN 16
86 #define MAX_CORE_STR_LEN 6
87 #define MAX_CORE_FREQ_STR_LEN 18
88 #define MAX_TIME_STR_LEN 12
89 #define MAX_POLLER_RUN_COUNT 20
90 #define MAX_PERIOD_STR_LEN 12
91 #define WINDOW_HEADER 12
92 #define FROM_HEX 16
93 #define THREAD_WIN_WIDTH 69
94 #define THREAD_WIN_HEIGHT 9
95 #define THREAD_WIN_FIRST_COL 2
96 #define CORE_WIN_FIRST_COL 16
97 #define CORE_WIN_WIDTH 48
98 #define CORE_WIN_HEIGHT 11
99 #define POLLER_WIN_HEIGHT 8
100 #define POLLER_WIN_WIDTH 64
101 #define POLLER_WIN_FIRST_COL 14
102 #define FIRST_DATA_ROW 7
103 
104 enum tabs {
105 	THREADS_TAB,
106 	POLLERS_TAB,
107 	CORES_TAB,
108 	NUMBER_OF_TABS,
109 };
110 
111 enum spdk_poller_type {
112 	SPDK_ACTIVE_POLLER,
113 	SPDK_TIMED_POLLER,
114 	SPDK_PAUSED_POLLER,
115 	SPDK_POLLER_TYPES_COUNT,
116 };
117 
118 struct col_desc {
119 	const char *name;
120 	uint8_t name_len;
121 	uint8_t max_data_string;
122 	bool disabled;
123 };
124 
125 struct run_counter_history {
126 	char *poller_name;
127 	uint64_t thread_id;
128 	uint64_t last_run_counter;
129 	TAILQ_ENTRY(run_counter_history) link;
130 };
131 
132 struct core_info {
133 	uint32_t core;
134 	uint64_t threads_count;
135 	uint64_t pollers_count;
136 	uint64_t idle;
137 	uint64_t last_idle;
138 	uint64_t busy;
139 	uint64_t last_busy;
140 };
141 
142 uint8_t g_sleep_time = 1;
143 uint16_t g_selected_row;
144 uint16_t g_max_selected_row;
145 struct rpc_thread_info *g_thread_info[MAX_THREADS];
146 const char *poller_type_str[SPDK_POLLER_TYPES_COUNT] = {"Active", "Timed", "Paused"};
147 const char *g_tab_title[NUMBER_OF_TABS] = {"[1] THREADS", "[2] POLLERS", "[3] CORES"};
148 struct spdk_jsonrpc_client *g_rpc_client;
149 static TAILQ_HEAD(, run_counter_history) g_run_counter_history = TAILQ_HEAD_INITIALIZER(
150 			g_run_counter_history);
151 struct core_info g_cores_history[RPC_MAX_CORES];
152 WINDOW *g_menu_win, *g_tab_win[NUMBER_OF_TABS], *g_tabs[NUMBER_OF_TABS];
153 PANEL *g_panels[NUMBER_OF_TABS];
154 uint16_t g_max_row, g_max_col;
155 uint16_t g_data_win_size, g_max_data_rows;
156 uint32_t g_last_threads_count, g_last_pollers_count, g_last_cores_count;
157 uint8_t g_current_sort_col[NUMBER_OF_TABS] = {0, 0, 0};
158 bool g_interval_data = true;
159 static struct col_desc g_col_desc[NUMBER_OF_TABS][TABS_COL_COUNT] = {
160 	{	{.name = "Thread name", .max_data_string = MAX_THREAD_NAME_LEN},
161 		{.name = "Core", .max_data_string = MAX_CORE_STR_LEN},
162 		{.name = "Active pollers", .max_data_string = MAX_POLLER_COUNT_STR_LEN},
163 		{.name = "Timed pollers", .max_data_string = MAX_POLLER_COUNT_STR_LEN},
164 		{.name = "Paused pollers", .max_data_string = MAX_POLLER_COUNT_STR_LEN},
165 		{.name = "Idle [us]", .max_data_string = MAX_TIME_STR_LEN},
166 		{.name = "Busy [us]", .max_data_string = MAX_TIME_STR_LEN},
167 		{.name = (char *)NULL}
168 	},
169 	{	{.name = "Poller name", .max_data_string = MAX_POLLER_NAME_LEN},
170 		{.name = "Type", .max_data_string = MAX_POLLER_TYPE_STR_LEN},
171 		{.name = "On thread", .max_data_string = MAX_THREAD_NAME_LEN},
172 		{.name = "Run count", .max_data_string = MAX_POLLER_RUN_COUNT},
173 		{.name = "Period [us]", .max_data_string = MAX_PERIOD_STR_LEN},
174 		{.name = "Status", .max_data_string = MAX_POLLER_IND_STR_LEN},
175 		{.name = (char *)NULL}
176 	},
177 	{	{.name = "Core", .max_data_string = MAX_CORE_STR_LEN},
178 		{.name = "Thread count", .max_data_string = MAX_THREAD_COUNT_STR_LEN},
179 		{.name = "Poller count", .max_data_string = MAX_POLLER_COUNT_STR_LEN},
180 		{.name = "Idle [us]", .max_data_string = MAX_TIME_STR_LEN},
181 		{.name = "Busy [us]", .max_data_string = MAX_TIME_STR_LEN},
182 		{.name = "Frequency [MHz]", .max_data_string = MAX_CORE_FREQ_STR_LEN},
183 		{.name = (char *)NULL}
184 	}
185 };
186 
187 struct rpc_thread_info {
188 	char *name;
189 	uint64_t id;
190 	uint32_t core_num;
191 	char *cpumask;
192 	uint64_t busy;
193 	uint64_t last_busy;
194 	uint64_t idle;
195 	uint64_t last_idle;
196 	uint64_t active_pollers_count;
197 	uint64_t timed_pollers_count;
198 	uint64_t paused_pollers_count;
199 };
200 
201 struct rpc_threads {
202 	uint64_t threads_count;
203 	struct rpc_thread_info thread_info[RPC_MAX_THREADS];
204 };
205 
206 struct rpc_threads_stats {
207 	uint64_t tick_rate;
208 	struct rpc_threads threads;
209 };
210 
211 struct rpc_poller_info {
212 	char *name;
213 	char *state;
214 	uint64_t run_count;
215 	uint64_t busy_count;
216 	uint64_t period_ticks;
217 	enum spdk_poller_type type;
218 	char thread_name[MAX_THREAD_NAME];
219 	uint64_t thread_id;
220 };
221 
222 struct rpc_pollers {
223 	uint64_t pollers_count;
224 	struct rpc_poller_info pollers[RPC_MAX_POLLERS];
225 };
226 
227 struct rpc_poller_thread_info {
228 	char *name;
229 	uint64_t id;
230 	struct rpc_pollers active_pollers;
231 	struct rpc_pollers timed_pollers;
232 	struct rpc_pollers paused_pollers;
233 };
234 
235 struct rpc_pollers_threads {
236 	uint64_t threads_count;
237 	struct rpc_poller_thread_info threads[RPC_MAX_THREADS];
238 };
239 
240 struct rpc_pollers_stats {
241 	uint64_t tick_rate;
242 	struct rpc_pollers_threads pollers_threads;
243 };
244 
245 struct rpc_core_thread_info {
246 	char *name;
247 	uint64_t id;
248 	char *cpumask;
249 	uint64_t elapsed;
250 };
251 
252 struct rpc_core_threads {
253 	uint64_t threads_count;
254 	struct rpc_core_thread_info thread[RPC_MAX_THREADS];
255 };
256 
257 struct rpc_core_info {
258 	uint32_t lcore;
259 	uint64_t busy;
260 	uint64_t idle;
261 	uint32_t core_freq;
262 	struct rpc_core_threads threads;
263 };
264 
265 struct rpc_cores {
266 	uint64_t cores_count;
267 	struct rpc_core_info core[RPC_MAX_CORES];
268 };
269 
270 struct rpc_cores_stats {
271 	uint64_t tick_rate;
272 	struct rpc_cores cores;
273 };
274 
275 struct rpc_threads_stats g_threads_stats;
276 struct rpc_pollers_stats g_pollers_stats;
277 struct rpc_cores_stats g_cores_stats;
278 struct rpc_poller_info g_pollers_history[RPC_MAX_POLLERS];
279 struct rpc_thread_info g_thread_history[RPC_MAX_THREADS];
280 
281 static void
282 init_str_len(void)
283 {
284 	int i, j;
285 
286 	for (i = 0; i < NUMBER_OF_TABS; i++) {
287 		for (j = 0; g_col_desc[i][j].name != NULL; j++) {
288 			g_col_desc[i][j].name_len = strlen(g_col_desc[i][j].name);
289 		}
290 	}
291 }
292 
293 static void
294 free_rpc_threads_stats(struct rpc_threads_stats *req)
295 {
296 	uint64_t i;
297 
298 	for (i = 0; i < req->threads.threads_count; i++) {
299 		free(req->threads.thread_info[i].name);
300 		req->threads.thread_info[i].name = NULL;
301 		free(req->threads.thread_info[i].cpumask);
302 		req->threads.thread_info[i].cpumask = NULL;
303 	}
304 }
305 
306 static const struct spdk_json_object_decoder rpc_thread_info_decoders[] = {
307 	{"name", offsetof(struct rpc_thread_info, name), spdk_json_decode_string},
308 	{"id", offsetof(struct rpc_thread_info, id), spdk_json_decode_uint64},
309 	{"cpumask", offsetof(struct rpc_thread_info, cpumask), spdk_json_decode_string},
310 	{"busy", offsetof(struct rpc_thread_info, busy), spdk_json_decode_uint64},
311 	{"idle", offsetof(struct rpc_thread_info, idle), spdk_json_decode_uint64},
312 	{"active_pollers_count", offsetof(struct rpc_thread_info, active_pollers_count), spdk_json_decode_uint64},
313 	{"timed_pollers_count", offsetof(struct rpc_thread_info, timed_pollers_count), spdk_json_decode_uint64},
314 	{"paused_pollers_count", offsetof(struct rpc_thread_info, paused_pollers_count), spdk_json_decode_uint64},
315 };
316 
317 static int
318 rpc_decode_threads_object(const struct spdk_json_val *val, void *out)
319 {
320 	struct rpc_thread_info *info = out;
321 
322 	return spdk_json_decode_object(val, rpc_thread_info_decoders,
323 				       SPDK_COUNTOF(rpc_thread_info_decoders), info);
324 }
325 
326 static int
327 rpc_decode_threads_array(const struct spdk_json_val *val, void *out)
328 {
329 	struct rpc_threads *threads = out;
330 
331 	return spdk_json_decode_array(val, rpc_decode_threads_object, threads->thread_info, RPC_MAX_THREADS,
332 				      &threads->threads_count, sizeof(struct rpc_thread_info));
333 }
334 
335 static const struct spdk_json_object_decoder rpc_threads_stats_decoders[] = {
336 	{"tick_rate", offsetof(struct rpc_threads_stats, tick_rate), spdk_json_decode_uint64},
337 	{"threads", offsetof(struct rpc_threads_stats, threads), rpc_decode_threads_array},
338 };
339 
340 static void
341 free_rpc_poller(struct rpc_poller_info *poller)
342 {
343 	free(poller->name);
344 	poller->name = NULL;
345 	free(poller->state);
346 	poller->state = NULL;
347 }
348 
349 static void
350 free_rpc_pollers_stats(struct rpc_pollers_stats *req)
351 {
352 	struct rpc_poller_thread_info *thread;
353 	uint64_t i, j;
354 
355 	for (i = 0; i < req->pollers_threads.threads_count; i++) {
356 		thread = &req->pollers_threads.threads[i];
357 
358 		for (j = 0; j < thread->active_pollers.pollers_count; j++) {
359 			free_rpc_poller(&thread->active_pollers.pollers[j]);
360 		}
361 
362 		for (j = 0; j < thread->timed_pollers.pollers_count; j++) {
363 			free_rpc_poller(&thread->timed_pollers.pollers[j]);
364 		}
365 
366 		for (j = 0; j < thread->paused_pollers.pollers_count; j++) {
367 			free_rpc_poller(&thread->paused_pollers.pollers[j]);
368 		}
369 
370 		free(thread->name);
371 		thread->name = NULL;
372 	}
373 }
374 
375 static void
376 free_rpc_cores_stats(struct rpc_cores_stats *req)
377 {
378 	struct rpc_core_info *core;
379 	struct rpc_core_thread_info *thread;
380 	uint64_t i, j;
381 
382 	for (i = 0; i < req->cores.cores_count; i++) {
383 		core = &req->cores.core[i];
384 
385 		for (j = 0; j < core->threads.threads_count; j++) {
386 			thread = &core->threads.thread[j];
387 
388 			free(thread->name);
389 			free(thread->cpumask);
390 		}
391 	}
392 }
393 
394 static const struct spdk_json_object_decoder rpc_pollers_decoders[] = {
395 	{"name", offsetof(struct rpc_poller_info, name), spdk_json_decode_string},
396 	{"state", offsetof(struct rpc_poller_info, state), spdk_json_decode_string},
397 	{"run_count", offsetof(struct rpc_poller_info, run_count), spdk_json_decode_uint64},
398 	{"busy_count", offsetof(struct rpc_poller_info, busy_count), spdk_json_decode_uint64},
399 	{"period_ticks", offsetof(struct rpc_poller_info, period_ticks), spdk_json_decode_uint64, true},
400 };
401 
402 static int
403 rpc_decode_pollers_object(const struct spdk_json_val *val, void *out)
404 {
405 	struct rpc_poller_info *info = out;
406 
407 	return spdk_json_decode_object(val, rpc_pollers_decoders, SPDK_COUNTOF(rpc_pollers_decoders), info);
408 }
409 
410 static int
411 rpc_decode_pollers_array(const struct spdk_json_val *val, void *out)
412 {
413 	struct rpc_pollers *pollers = out;
414 
415 	return spdk_json_decode_array(val, rpc_decode_pollers_object, pollers->pollers, RPC_MAX_THREADS,
416 				      &pollers->pollers_count, sizeof(struct rpc_poller_info));
417 }
418 
419 static const struct spdk_json_object_decoder rpc_pollers_threads_decoders[] = {
420 	{"name", offsetof(struct rpc_poller_thread_info, name), spdk_json_decode_string},
421 	{"id", offsetof(struct rpc_poller_thread_info, id), spdk_json_decode_uint64},
422 	{"active_pollers", offsetof(struct rpc_poller_thread_info, active_pollers), rpc_decode_pollers_array},
423 	{"timed_pollers", offsetof(struct rpc_poller_thread_info, timed_pollers), rpc_decode_pollers_array},
424 	{"paused_pollers", offsetof(struct rpc_poller_thread_info, paused_pollers), rpc_decode_pollers_array},
425 };
426 
427 static int
428 rpc_decode_pollers_threads_object(const struct spdk_json_val *val, void *out)
429 {
430 	struct rpc_poller_thread_info *info = out;
431 
432 	return spdk_json_decode_object(val, rpc_pollers_threads_decoders,
433 				       SPDK_COUNTOF(rpc_pollers_threads_decoders), info);
434 }
435 
436 static int
437 rpc_decode_pollers_threads_array(const struct spdk_json_val *val, void *out)
438 {
439 	struct rpc_pollers_threads *pollers_threads = out;
440 
441 	return spdk_json_decode_array(val, rpc_decode_pollers_threads_object, pollers_threads->threads,
442 				      RPC_MAX_THREADS, &pollers_threads->threads_count, sizeof(struct rpc_poller_thread_info));
443 }
444 
445 static const struct spdk_json_object_decoder rpc_pollers_stats_decoders[] = {
446 	{"tick_rate", offsetof(struct rpc_pollers_stats, tick_rate), spdk_json_decode_uint64},
447 	{"threads", offsetof(struct rpc_pollers_stats, pollers_threads), rpc_decode_pollers_threads_array},
448 };
449 
450 static const struct spdk_json_object_decoder rpc_core_thread_info_decoders[] = {
451 	{"name", offsetof(struct rpc_core_thread_info, name), spdk_json_decode_string},
452 	{"id", offsetof(struct rpc_core_thread_info, id), spdk_json_decode_uint64},
453 	{"cpumask", offsetof(struct rpc_core_thread_info, cpumask), spdk_json_decode_string},
454 	{"elapsed", offsetof(struct rpc_core_thread_info, elapsed), spdk_json_decode_uint64},
455 };
456 
457 static int
458 rpc_decode_core_threads_object(const struct spdk_json_val *val, void *out)
459 {
460 	struct rpc_core_thread_info *info = out;
461 
462 	return spdk_json_decode_object(val, rpc_core_thread_info_decoders,
463 				       SPDK_COUNTOF(rpc_core_thread_info_decoders), info);
464 }
465 
466 static int
467 rpc_decode_cores_lw_threads(const struct spdk_json_val *val, void *out)
468 {
469 	struct rpc_core_threads *threads = out;
470 
471 	return spdk_json_decode_array(val, rpc_decode_core_threads_object, threads->thread, RPC_MAX_THREADS,
472 				      &threads->threads_count, sizeof(struct rpc_core_thread_info));
473 }
474 
475 static const struct spdk_json_object_decoder rpc_core_info_decoders[] = {
476 	{"lcore", offsetof(struct rpc_core_info, lcore), spdk_json_decode_uint32},
477 	{"busy", offsetof(struct rpc_core_info, busy), spdk_json_decode_uint64},
478 	{"idle", offsetof(struct rpc_core_info, idle), spdk_json_decode_uint64},
479 	{"core_freq", offsetof(struct rpc_core_info, core_freq), spdk_json_decode_uint32, true},
480 	{"lw_threads", offsetof(struct rpc_core_info, threads), rpc_decode_cores_lw_threads},
481 };
482 
483 static int
484 rpc_decode_core_object(const struct spdk_json_val *val, void *out)
485 {
486 	struct rpc_core_info *info = out;
487 
488 	return spdk_json_decode_object(val, rpc_core_info_decoders,
489 				       SPDK_COUNTOF(rpc_core_info_decoders), info);
490 }
491 
492 static int
493 rpc_decode_cores_array(const struct spdk_json_val *val, void *out)
494 {
495 	struct rpc_cores *cores = out;
496 
497 	return spdk_json_decode_array(val, rpc_decode_core_object, cores->core,
498 				      RPC_MAX_THREADS, &cores->cores_count, sizeof(struct rpc_core_info));
499 }
500 
501 static const struct spdk_json_object_decoder rpc_cores_stats_decoders[] = {
502 	{"tick_rate", offsetof(struct rpc_cores_stats, tick_rate), spdk_json_decode_uint64},
503 	{"reactors", offsetof(struct rpc_cores_stats, cores), rpc_decode_cores_array},
504 };
505 
506 
507 static int
508 rpc_send_req(char *rpc_name, struct spdk_jsonrpc_client_response **resp)
509 {
510 	struct spdk_jsonrpc_client_response *json_resp = NULL;
511 	struct spdk_json_write_ctx *w;
512 	struct spdk_jsonrpc_client_request *request;
513 	int rc;
514 
515 	request = spdk_jsonrpc_client_create_request();
516 	if (request == NULL) {
517 		return -ENOMEM;
518 	}
519 
520 	w = spdk_jsonrpc_begin_request(request, 1, rpc_name);
521 	spdk_jsonrpc_end_request(request, w);
522 	spdk_jsonrpc_client_send_request(g_rpc_client, request);
523 
524 	do {
525 		rc = spdk_jsonrpc_client_poll(g_rpc_client, 1);
526 	} while (rc == 0 || rc == -ENOTCONN);
527 
528 	if (rc <= 0) {
529 		return -1;
530 	}
531 
532 	json_resp = spdk_jsonrpc_client_get_response(g_rpc_client);
533 	if (json_resp == NULL) {
534 		return -1;
535 	}
536 
537 	/* Check for error response */
538 	if (json_resp->error != NULL) {
539 		return -1;
540 	}
541 
542 	assert(json_resp->result);
543 
544 	*resp = json_resp;
545 
546 	return 0;
547 }
548 
549 static int
550 get_data(void)
551 {
552 	struct spdk_jsonrpc_client_response *json_resp = NULL;
553 	struct rpc_thread_info *thread_info;
554 	struct rpc_core_info *core_info;
555 	uint64_t i, j;
556 	int rc = 0;
557 
558 	rc = rpc_send_req("thread_get_stats", &json_resp);
559 	if (rc) {
560 		goto end;
561 	}
562 
563 	/* Decode json */
564 	memset(&g_threads_stats, 0, sizeof(g_threads_stats));
565 	if (spdk_json_decode_object(json_resp->result, rpc_threads_stats_decoders,
566 				    SPDK_COUNTOF(rpc_threads_stats_decoders), &g_threads_stats)) {
567 		rc = -EINVAL;
568 		goto end;
569 	}
570 
571 	spdk_jsonrpc_client_free_response(json_resp);
572 
573 	for (i = 0; i < g_threads_stats.threads.threads_count; i++) {
574 		thread_info = &g_threads_stats.threads.thread_info[i];
575 		g_thread_info[thread_info->id] = thread_info;
576 	}
577 
578 	rc = rpc_send_req("thread_get_pollers", &json_resp);
579 	if (rc) {
580 		goto end;
581 	}
582 
583 	/* Decode json */
584 	memset(&g_pollers_stats, 0, sizeof(g_pollers_stats));
585 	if (spdk_json_decode_object(json_resp->result, rpc_pollers_stats_decoders,
586 				    SPDK_COUNTOF(rpc_pollers_stats_decoders), &g_pollers_stats)) {
587 		rc = -EINVAL;
588 		goto end;
589 	}
590 
591 	spdk_jsonrpc_client_free_response(json_resp);
592 
593 	rc = rpc_send_req("framework_get_reactors", &json_resp);
594 	if (rc) {
595 		goto end;
596 	}
597 
598 	/* Decode json */
599 	memset(&g_cores_stats, 0, sizeof(g_cores_stats));
600 	if (spdk_json_decode_object(json_resp->result, rpc_cores_stats_decoders,
601 				    SPDK_COUNTOF(rpc_cores_stats_decoders), &g_cores_stats)) {
602 		rc = -EINVAL;
603 		goto end;
604 	}
605 
606 	for (i = 0; i < g_cores_stats.cores.cores_count; i++) {
607 		core_info = &g_cores_stats.cores.core[i];
608 
609 		for (j = 0; j < core_info->threads.threads_count; j++) {
610 			g_thread_info[core_info->threads.thread[j].id]->core_num = core_info->lcore;
611 		}
612 	}
613 
614 end:
615 	spdk_jsonrpc_client_free_response(json_resp);
616 	return rc;
617 }
618 
619 static void
620 free_data(void)
621 {
622 	free_rpc_threads_stats(&g_threads_stats);
623 	free_rpc_pollers_stats(&g_pollers_stats);
624 	free_rpc_cores_stats(&g_cores_stats);
625 }
626 
627 enum str_alignment {
628 	ALIGN_LEFT,
629 	ALIGN_RIGHT,
630 };
631 
632 static void
633 print_max_len(WINDOW *win, int row, uint16_t col, uint16_t max_len, enum str_alignment alignment,
634 	      const char *string)
635 {
636 	const char dots[] = "...";
637 	int DOTS_STR_LEN = sizeof(dots) / sizeof(dots[0]);
638 	char tmp_str[MAX_STRING_LEN];
639 	int len, max_col, max_str, cmp_len;
640 	int max_row;
641 
642 	len = strlen(string);
643 	getmaxyx(win, max_row, max_col);
644 
645 	if (row > max_row) {
646 		/* We are in a process of resizing and this may happen */
647 		return;
648 	}
649 
650 	if (max_len != 0 && col + max_len < max_col) {
651 		max_col = col + max_len;
652 	}
653 
654 	max_str = max_col - col;
655 
656 	if (max_str <= DOTS_STR_LEN + 1) {
657 		/* No space to print anything, but we have to let a user know about it */
658 		mvwprintw(win, row, max_col - DOTS_STR_LEN - 1, "...");
659 		refresh();
660 		wrefresh(win);
661 		return;
662 	}
663 
664 	if (max_len) {
665 		if (alignment == ALIGN_LEFT) {
666 			snprintf(tmp_str, max_str, "%s%*c", string, max_len - len - 1, ' ');
667 		} else {
668 			snprintf(tmp_str, max_str, "%*c%s", max_len - len - 1, ' ', string);
669 		}
670 		cmp_len = max_len - 1;
671 	} else {
672 		snprintf(tmp_str, max_str, "%s", string);
673 		cmp_len = len;
674 	}
675 
676 	if (col + cmp_len > max_col - 1) {
677 		snprintf(&tmp_str[max_str - DOTS_STR_LEN - 2], DOTS_STR_LEN, "%s", dots);
678 	}
679 
680 	mvwprintw(win, row, col, tmp_str);
681 
682 	refresh();
683 	wrefresh(win);
684 }
685 
686 static void
687 draw_menu_win(void)
688 {
689 	wbkgd(g_menu_win, COLOR_PAIR(2));
690 	box(g_menu_win, 0, 0);
691 	print_max_len(g_menu_win, 1, 1, 0, ALIGN_LEFT,
692 		      "  [q] Quit  |  [1-3] TAB selection  |  [PgUp] Previous page  |  [PgDown] Next page  |  [c] Columns  |  [s] Sorting  |  [r] Refresh rate  |  [Enter] Item details  |  [t] Total/Interval");
693 }
694 
695 static void
696 draw_tab_win(enum tabs tab)
697 {
698 	uint16_t col;
699 	uint8_t white_spaces = TABS_SPACING * NUMBER_OF_TABS;
700 
701 	wbkgd(g_tab_win[tab], COLOR_PAIR(2));
702 	box(g_tab_win[tab], 0, 0);
703 
704 	col = ((g_max_col - white_spaces) / NUMBER_OF_TABS / 2) - (strlen(g_tab_title[tab]) / 2) -
705 	      TABS_SPACING;
706 	print_max_len(g_tab_win[tab], 1, col, 0, ALIGN_LEFT, g_tab_title[tab]);
707 }
708 
709 static void
710 draw_tabs(enum tabs tab_index, uint8_t sort_col)
711 {
712 	struct col_desc *col_desc = g_col_desc[tab_index];
713 	WINDOW *tab = g_tabs[tab_index];
714 	int i, j;
715 	uint16_t offset, draw_offset;
716 
717 	for (i = 0; col_desc[i].name != NULL; i++) {
718 		if (col_desc[i].disabled) {
719 			continue;
720 		}
721 
722 		offset = 1;
723 		for (j = i; j != 0; j--) {
724 			if (!col_desc[j - 1].disabled) {
725 				offset += col_desc[j - 1].max_data_string;
726 				offset += col_desc[j - 1].name_len % 2 + 1;
727 			}
728 		}
729 
730 		draw_offset = offset + (col_desc[i].max_data_string / 2) - (col_desc[i].name_len / 2);
731 
732 		if (i == sort_col) {
733 			wattron(tab, COLOR_PAIR(3));
734 			print_max_len(tab, 1, draw_offset, 0, ALIGN_LEFT, col_desc[i].name);
735 			wattroff(tab, COLOR_PAIR(3));
736 		} else {
737 			print_max_len(tab, 1, draw_offset, 0, ALIGN_LEFT, col_desc[i].name);
738 		}
739 
740 		if (offset != 1) {
741 			print_max_len(tab, 1, offset - 1, 0, ALIGN_LEFT, "|");
742 		}
743 	}
744 
745 	print_max_len(tab, 2, 1, 0, ALIGN_LEFT, ""); /* Move to next line */
746 	whline(tab, ACS_HLINE, MAX_STRING_LEN);
747 	box(tab, 0, 0);
748 	wrefresh(tab);
749 }
750 
751 static void
752 resize_interface(enum tabs tab)
753 {
754 	int i;
755 
756 	clear();
757 	wclear(g_menu_win);
758 	mvwin(g_menu_win, g_max_row - MENU_WIN_SPACING, MENU_WIN_LOCATION_COL);
759 	wresize(g_menu_win, MENU_WIN_HEIGHT, g_max_col);
760 	draw_menu_win();
761 
762 	for (i = 0; i < NUMBER_OF_TABS; i++) {
763 		wclear(g_tabs[i]);
764 		wresize(g_tabs[i], g_max_row - MENU_WIN_HEIGHT - TAB_WIN_HEIGHT - 2, g_max_col);
765 		mvwin(g_tabs[i], TABS_LOCATION_ROW, TABS_LOCATION_COL);
766 		draw_tabs(i, g_current_sort_col[i]);
767 	}
768 
769 	draw_tabs(tab, g_current_sort_col[tab]);
770 
771 	for (i = 0; i < NUMBER_OF_TABS; i++) {
772 		wclear(g_tab_win[i]);
773 		wresize(g_tab_win[i], TAB_WIN_HEIGHT,
774 			(g_max_col - (TABS_SPACING * NUMBER_OF_TABS)) / NUMBER_OF_TABS);
775 		mvwin(g_tab_win[i], TAB_WIN_LOCATION_ROW, 1 + (g_max_col / NUMBER_OF_TABS) * i);
776 		draw_tab_win(i);
777 	}
778 
779 	update_panels();
780 	doupdate();
781 }
782 
783 static void
784 switch_tab(enum tabs tab)
785 {
786 	top_panel(g_panels[tab]);
787 	update_panels();
788 	doupdate();
789 }
790 
791 static void
792 get_time_str(uint64_t ticks, char *time_str)
793 {
794 	uint64_t time;
795 
796 	time = ticks * SPDK_SEC_TO_USEC / g_cores_stats.tick_rate;
797 	snprintf(time_str, MAX_TIME_STR_LEN, "%" PRIu64, time);
798 }
799 
800 static int
801 sort_threads(const void *p1, const void *p2)
802 {
803 	const struct rpc_thread_info *thread_info1 = *(struct rpc_thread_info **)p1;
804 	const struct rpc_thread_info *thread_info2 = *(struct rpc_thread_info **)p2;
805 	uint64_t count1, count2;
806 
807 	/* thread IDs may not be allocated contiguously, so we need
808 	 * to account for NULL thread_info pointers */
809 	if (thread_info1 == NULL && thread_info2 == NULL) {
810 		return 0;
811 	} else if (thread_info1 == NULL) {
812 		return 1;
813 	} else if (thread_info2 == NULL) {
814 		return -1;
815 	}
816 
817 	switch (g_current_sort_col[THREADS_TAB]) {
818 	case 0: /* Sort by name */
819 		return strcmp(thread_info1->name, thread_info2->name);
820 	case 1: /* Sort by core */
821 		count2 = thread_info1->core_num;
822 		count1 = thread_info2->core_num;
823 		break;
824 	case 2: /* Sort by active pollers number */
825 		count1 = thread_info1->active_pollers_count;
826 		count2 = thread_info2->active_pollers_count;
827 		break;
828 	case 3: /* Sort by timed pollers number */
829 		count1 = thread_info1->timed_pollers_count;
830 		count2 = thread_info2->timed_pollers_count;
831 		break;
832 	case 4: /* Sort by paused pollers number */
833 		count1 = thread_info1->paused_pollers_count;
834 		count2 = thread_info2->paused_pollers_count;
835 		break;
836 	case 5: /* Sort by idle time */
837 		count1 = thread_info1->idle - thread_info1->last_idle;
838 		count2 = thread_info2->idle - thread_info2->last_idle;
839 		break;
840 	case 6: /* Sort by busy time */
841 		count1 = thread_info1->busy - thread_info1->last_busy;
842 		count2 = thread_info2->busy - thread_info2->last_busy;
843 		break;
844 	default:
845 		return 0;
846 	}
847 
848 	if (count2 > count1) {
849 		return 1;
850 	} else if (count2 < count1) {
851 		return -1;
852 	} else {
853 		return 0;
854 	}
855 }
856 
857 static void
858 draw_row_background(uint8_t item_index, uint8_t tab)
859 {
860 	int k;
861 
862 	if (item_index == g_selected_row) {
863 		wattron(g_tabs[tab], COLOR_PAIR(2));
864 	}
865 	for (k = 1; k < g_max_col - 1; k++) {
866 		mvwprintw(g_tabs[tab], TABS_DATA_START_ROW + item_index, k, " ");
867 	}
868 }
869 
870 static uint8_t
871 refresh_threads_tab(uint8_t current_page)
872 {
873 	struct col_desc *col_desc = g_col_desc[THREADS_TAB];
874 	uint64_t i, threads_count;
875 	uint16_t j, k;
876 	uint16_t col;
877 	uint8_t max_pages, item_index;
878 	static uint8_t last_page = 0;
879 	char pollers_number[MAX_POLLER_COUNT_STR_LEN], idle_time[MAX_TIME_STR_LEN],
880 	     busy_time[MAX_TIME_STR_LEN], core_str[MAX_CORE_MASK_STR_LEN];
881 	struct rpc_thread_info *thread_info[g_threads_stats.threads.threads_count];
882 
883 	threads_count = g_threads_stats.threads.threads_count;
884 
885 	/* Clear screen if number of threads changed */
886 	if (g_last_threads_count != threads_count) {
887 		for (i = TABS_DATA_START_ROW; i < g_data_win_size; i++) {
888 			for (j = 1; j < (uint64_t)g_max_col - 1; j++) {
889 				mvwprintw(g_tabs[THREADS_TAB], i, j, " ");
890 			}
891 		}
892 
893 		g_last_threads_count = threads_count;
894 	}
895 
896 	/* From g_thread_info copy to thread_info without null elements.
897 	 * The index of g_thread_info equals to Thread IDs, so it starts from '1'. */
898 	for (i = 0, j = 1; i <  g_threads_stats.threads.threads_count; i++) {
899 		while (g_thread_info[j] == NULL) {
900 			j++;
901 		}
902 		memcpy(&thread_info[i], &g_thread_info[j], sizeof(struct rpc_thread_info *));
903 		j++;
904 	}
905 
906 	if (last_page != current_page) {
907 		for (i = 0; i < threads_count; i++) {
908 			/* Thread IDs start from 1, so we have to do i + 1 */
909 			g_threads_stats.threads.thread_info[i].last_idle = g_thread_info[i + 1]->idle;
910 			g_threads_stats.threads.thread_info[i].last_busy = g_thread_info[i + 1]->busy;
911 		}
912 
913 		last_page = current_page;
914 	}
915 
916 	max_pages = (threads_count + g_max_data_rows - 1) / g_max_data_rows;
917 
918 	qsort(thread_info, threads_count, sizeof(thread_info[0]), sort_threads);
919 
920 	for (k = 0; k < threads_count; k++) {
921 		g_thread_history[thread_info[k]->id].busy = thread_info[k]->busy - thread_info[k]->last_busy;
922 		g_thread_history[thread_info[k]->id].idle = thread_info[k]->idle - thread_info[k]->last_idle;
923 	}
924 
925 	for (i = current_page * g_max_data_rows;
926 	     i < spdk_min(threads_count, (uint64_t)((current_page + 1) * g_max_data_rows));
927 	     i++) {
928 		item_index = i - (current_page * g_max_data_rows);
929 
930 		col = TABS_DATA_START_COL;
931 
932 		draw_row_background(item_index, THREADS_TAB);
933 
934 		if (!col_desc[0].disabled) {
935 			print_max_len(g_tabs[THREADS_TAB], TABS_DATA_START_ROW + item_index, col,
936 				      col_desc[0].max_data_string, ALIGN_LEFT, thread_info[i]->name);
937 			col += col_desc[0].max_data_string;
938 		}
939 
940 		if (!col_desc[1].disabled) {
941 			snprintf(core_str, MAX_CORE_STR_LEN, "%d", thread_info[i]->core_num);
942 			print_max_len(g_tabs[THREADS_TAB], TABS_DATA_START_ROW + item_index,
943 				      col, col_desc[1].max_data_string, ALIGN_RIGHT, core_str);
944 			col += col_desc[1].max_data_string + 2;
945 		}
946 
947 		if (!col_desc[2].disabled) {
948 			snprintf(pollers_number, MAX_POLLER_COUNT_STR_LEN, "%ld", thread_info[i]->active_pollers_count);
949 			print_max_len(g_tabs[THREADS_TAB], TABS_DATA_START_ROW + item_index,
950 				      col + (col_desc[2].name_len / 2), col_desc[2].max_data_string, ALIGN_LEFT, pollers_number);
951 			col += col_desc[2].max_data_string + 2;
952 		}
953 
954 		if (!col_desc[3].disabled) {
955 			snprintf(pollers_number, MAX_POLLER_COUNT_STR_LEN, "%ld", thread_info[i]->timed_pollers_count);
956 			print_max_len(g_tabs[THREADS_TAB], TABS_DATA_START_ROW + item_index,
957 				      col + (col_desc[3].name_len / 2), col_desc[3].max_data_string, ALIGN_LEFT, pollers_number);
958 			col += col_desc[3].max_data_string + 1;
959 		}
960 
961 		if (!col_desc[4].disabled) {
962 			snprintf(pollers_number, MAX_POLLER_COUNT_STR_LEN, "%ld", thread_info[i]->paused_pollers_count);
963 			print_max_len(g_tabs[THREADS_TAB], TABS_DATA_START_ROW + item_index,
964 				      col + (col_desc[4].name_len / 2), col_desc[4].max_data_string, ALIGN_LEFT, pollers_number);
965 			col += col_desc[4].max_data_string + 2;
966 		}
967 
968 		g_thread_history[thread_info[i]->id].idle = thread_info[i]->idle - thread_info[i]->last_idle;
969 		if (!col_desc[5].disabled) {
970 			if (g_interval_data == true) {
971 				get_time_str(thread_info[i]->idle - thread_info[i]->last_idle, idle_time);
972 			} else {
973 				get_time_str(thread_info[i]->idle, idle_time);
974 			}
975 			print_max_len(g_tabs[THREADS_TAB], TABS_DATA_START_ROW + item_index, col,
976 				      col_desc[5].max_data_string, ALIGN_RIGHT, idle_time);
977 			col += col_desc[5].max_data_string;
978 		}
979 
980 		g_thread_history[thread_info[i]->id].busy = thread_info[i]->busy - thread_info[i]->last_busy;
981 		if (!col_desc[6].disabled) {
982 			if (g_interval_data == true) {
983 				get_time_str(thread_info[i]->busy - thread_info[i]->last_busy, busy_time);
984 			} else {
985 				get_time_str(thread_info[i]->busy, busy_time);
986 			}
987 			print_max_len(g_tabs[THREADS_TAB], TABS_DATA_START_ROW + item_index, col,
988 				      col_desc[6].max_data_string, ALIGN_RIGHT, busy_time);
989 		}
990 
991 		if (item_index == g_selected_row) {
992 			wattroff(g_tabs[THREADS_TAB], COLOR_PAIR(2));
993 		}
994 	}
995 
996 	for (k = 0; k < threads_count; k++) {
997 		thread_info[k]->last_idle = thread_info[k]->idle;
998 		thread_info[k]->last_busy = thread_info[k]->busy;
999 	}
1000 
1001 	g_max_selected_row = i - current_page * g_max_data_rows - 1;
1002 
1003 	return max_pages;
1004 }
1005 
1006 static uint64_t *
1007 get_last_run_counter(const char *poller_name, uint64_t thread_id)
1008 {
1009 	struct run_counter_history *history;
1010 
1011 	TAILQ_FOREACH(history, &g_run_counter_history, link) {
1012 		if (!strcmp(history->poller_name, poller_name) && history->thread_id == thread_id) {
1013 			return &history->last_run_counter;
1014 		}
1015 	}
1016 
1017 	return NULL;
1018 }
1019 
1020 static void
1021 store_last_run_counter(const char *poller_name, uint64_t thread_id, uint64_t last_run_counter)
1022 {
1023 	struct run_counter_history *history;
1024 
1025 	TAILQ_FOREACH(history, &g_run_counter_history, link) {
1026 		if (!strcmp(history->poller_name, poller_name) && history->thread_id == thread_id) {
1027 			history->last_run_counter = last_run_counter;
1028 			return;
1029 		}
1030 	}
1031 
1032 	history = calloc(1, sizeof(*history));
1033 	if (history == NULL) {
1034 		fprintf(stderr, "Unable to allocate a history object in store_last_run_counter.\n");
1035 		return;
1036 	}
1037 	history->poller_name = strdup(poller_name);
1038 	history->thread_id = thread_id;
1039 	history->last_run_counter = last_run_counter;
1040 
1041 	TAILQ_INSERT_TAIL(&g_run_counter_history, history, link);
1042 }
1043 
1044 enum sort_type {
1045 	BY_NAME,
1046 	USE_GLOBAL,
1047 };
1048 
1049 static int
1050 #ifdef __FreeBSD__
1051 sort_pollers(void *arg, const void *p1, const void *p2)
1052 #else
1053 sort_pollers(const void *p1, const void *p2, void *arg)
1054 #endif
1055 {
1056 	const struct rpc_poller_info *poller1 = *(struct rpc_poller_info **)p1;
1057 	const struct rpc_poller_info *poller2 = *(struct rpc_poller_info **)p2;
1058 	enum sort_type sorting = *(enum sort_type *)arg;
1059 	uint64_t count1, count2;
1060 	uint64_t *last_run_counter;
1061 
1062 	if (sorting == BY_NAME) {
1063 		/* Sorting by name requested explicitly */
1064 		return strcmp(poller1->name, poller2->name);
1065 	} else {
1066 		/* Use globaly set sorting */
1067 		switch (g_current_sort_col[POLLERS_TAB]) {
1068 		case 0: /* Sort by name */
1069 			return strcmp(poller1->name, poller2->name);
1070 		case 1: /* Sort by type */
1071 			return poller1->type - poller2->type;
1072 		case 2: /* Sort by thread */
1073 			return strcmp(poller1->thread_name, poller2->thread_name);
1074 		case 3: /* Sort by run counter */
1075 			last_run_counter = get_last_run_counter(poller1->name, poller1->thread_id);
1076 			assert(last_run_counter != NULL);
1077 			count1 = poller1->run_count - *last_run_counter;
1078 			last_run_counter = get_last_run_counter(poller2->name, poller2->thread_id);
1079 			assert(last_run_counter != NULL);
1080 			count2 = poller2->run_count - *last_run_counter;
1081 			break;
1082 		case 4: /* Sort by period */
1083 			count1 = poller1->period_ticks;
1084 			count2 = poller2->period_ticks;
1085 			break;
1086 		default:
1087 			return 0;
1088 		}
1089 	}
1090 
1091 	if (count2 > count1) {
1092 		return 1;
1093 	} else if (count2 < count1) {
1094 		return -1;
1095 	} else {
1096 		return 0;
1097 	}
1098 }
1099 
1100 static void
1101 copy_pollers(struct rpc_pollers *pollers, uint64_t pollers_count, enum spdk_poller_type type,
1102 	     struct rpc_poller_thread_info *thread, uint64_t *current_count, bool reset_last_counter,
1103 	     struct rpc_poller_info **pollers_info)
1104 {
1105 	uint64_t *last_run_counter;
1106 	uint64_t i;
1107 
1108 	for (i = 0; i < pollers_count; i++) {
1109 		if (reset_last_counter) {
1110 			last_run_counter = get_last_run_counter(pollers->pollers[i].name, thread->id);
1111 			if (last_run_counter == NULL) {
1112 				store_last_run_counter(pollers->pollers[i].name, thread->id, pollers->pollers[i].run_count);
1113 				last_run_counter = get_last_run_counter(pollers->pollers[i].name, thread->id);
1114 			}
1115 
1116 			assert(last_run_counter != NULL);
1117 			*last_run_counter = pollers->pollers[i].run_count;
1118 		}
1119 		pollers_info[*current_count] = &pollers->pollers[i];
1120 		snprintf(pollers_info[*current_count]->thread_name, MAX_POLLER_NAME - 1, "%s", thread->name);
1121 		pollers_info[*current_count]->thread_id = thread->id;
1122 		pollers_info[(*current_count)++]->type = type;
1123 	}
1124 }
1125 
1126 static void
1127 store_pollers_last_stats(uint64_t poller, uint64_t run_counter, uint64_t period_ticks_counter)
1128 {
1129 	g_pollers_history[poller].run_count = run_counter;
1130 	g_pollers_history[poller].period_ticks = period_ticks_counter;
1131 }
1132 
1133 static uint8_t
1134 prepare_poller_data(uint8_t current_page, struct rpc_poller_info **pollers,
1135 		    uint64_t *count, uint8_t last_page)
1136 {
1137 	struct rpc_poller_thread_info *thread;
1138 	uint64_t i;
1139 	bool reset_last_counter = false;
1140 	enum sort_type sorting;
1141 
1142 	for (i = 0; i < g_pollers_stats.pollers_threads.threads_count; i++) {
1143 		thread = &g_pollers_stats.pollers_threads.threads[i];
1144 		if (last_page != current_page) {
1145 			reset_last_counter = true;
1146 		}
1147 
1148 		copy_pollers(&thread->active_pollers, thread->active_pollers.pollers_count, SPDK_ACTIVE_POLLER,
1149 			     thread, count, reset_last_counter, pollers);
1150 		copy_pollers(&thread->timed_pollers, thread->timed_pollers.pollers_count, SPDK_TIMED_POLLER, thread,
1151 			     count, reset_last_counter, pollers);
1152 		copy_pollers(&thread->paused_pollers, thread->paused_pollers.pollers_count, SPDK_PAUSED_POLLER,
1153 			     thread, count, reset_last_counter, pollers);
1154 	}
1155 
1156 	if (last_page != current_page) {
1157 		last_page = current_page;
1158 	}
1159 
1160 	/* Timed pollers can switch their possition on a list because of how they work.
1161 	 * Let's sort them by name first so that they won't switch on data refresh */
1162 	sorting = BY_NAME;
1163 	qsort_r(pollers, *count, sizeof(pollers[0]), sort_pollers, (void *)&sorting);
1164 	sorting = USE_GLOBAL;
1165 	qsort_r(pollers, *count, sizeof(pollers[0]), sort_pollers, (void *)&sorting);
1166 
1167 	return last_page;
1168 }
1169 
1170 static uint8_t
1171 refresh_pollers_tab(uint8_t current_page)
1172 {
1173 	struct col_desc *col_desc = g_col_desc[POLLERS_TAB];
1174 	uint64_t *last_run_counter;
1175 	uint64_t i, count = 0;
1176 	uint16_t col, j;
1177 	uint8_t max_pages, item_index;
1178 	static uint8_t g_last_page = 0xF;
1179 	/* Init g_last_page with value != 0 to force store_last_run_counter() call in copy_pollers()
1180 	 * so that initial values for run_counter are stored in g_run_counter_history */
1181 	char run_count[MAX_TIME_STR_LEN], period_ticks[MAX_PERIOD_STR_LEN];
1182 	struct rpc_poller_info *pollers[RPC_MAX_POLLERS];
1183 
1184 	g_last_page = prepare_poller_data(current_page, pollers, &count, g_last_page);
1185 
1186 	max_pages = (count + g_max_data_rows - 1) / g_max_data_rows;
1187 
1188 	/* Clear screen if number of pollers changed */
1189 	if (g_last_pollers_count != count) {
1190 		for (i = TABS_DATA_START_ROW; i < g_data_win_size; i++) {
1191 			for (j = 1; j < (uint64_t)g_max_col - 1; j++) {
1192 				mvwprintw(g_tabs[POLLERS_TAB], i, j, " ");
1193 			}
1194 		}
1195 
1196 		g_last_pollers_count = count;
1197 
1198 		/* We need to run store_last_run_counter() again, so the easiest way is to call this function
1199 		 * again with changed g_last_page value */
1200 		g_last_page = 0xF;
1201 		refresh_pollers_tab(current_page);
1202 		return max_pages;
1203 	}
1204 
1205 	/* Display info */
1206 	for (i = current_page * g_max_data_rows;
1207 	     i < spdk_min(count, (uint64_t)((current_page + 1) * g_max_data_rows));
1208 	     i++) {
1209 		item_index = i - (current_page * g_max_data_rows);
1210 
1211 		col = TABS_DATA_START_COL;
1212 
1213 		draw_row_background(item_index, POLLERS_TAB);
1214 
1215 		last_run_counter = get_last_run_counter(pollers[i]->name, pollers[i]->thread_id);
1216 		assert(last_run_counter != NULL);
1217 
1218 		if (!col_desc[0].disabled) {
1219 			print_max_len(g_tabs[POLLERS_TAB], TABS_DATA_START_ROW + item_index, col + 1,
1220 				      col_desc[0].max_data_string, ALIGN_LEFT, pollers[i]->name);
1221 			col += col_desc[0].max_data_string + 2;
1222 		}
1223 
1224 		if (!col_desc[1].disabled) {
1225 			print_max_len(g_tabs[POLLERS_TAB], TABS_DATA_START_ROW + item_index, col,
1226 				      col_desc[1].max_data_string, ALIGN_LEFT, poller_type_str[pollers[i]->type]);
1227 			col += col_desc[1].max_data_string + 2;
1228 		}
1229 
1230 		if (!col_desc[2].disabled) {
1231 			print_max_len(g_tabs[POLLERS_TAB], TABS_DATA_START_ROW + item_index, col,
1232 				      col_desc[2].max_data_string, ALIGN_LEFT, pollers[i]->thread_name);
1233 			col += col_desc[2].max_data_string + 1;
1234 		}
1235 
1236 		store_pollers_last_stats(i, pollers[i]->run_count - *last_run_counter, pollers[i]->period_ticks);
1237 		if (!col_desc[3].disabled) {
1238 			if (g_interval_data == true) {
1239 				snprintf(run_count, MAX_TIME_STR_LEN, "%" PRIu64, pollers[i]->run_count - *last_run_counter);
1240 			} else {
1241 				snprintf(run_count, MAX_TIME_STR_LEN, "%" PRIu64, pollers[i]->run_count);
1242 			}
1243 			print_max_len(g_tabs[POLLERS_TAB], TABS_DATA_START_ROW + item_index, col,
1244 				      col_desc[3].max_data_string, ALIGN_RIGHT, run_count);
1245 			col += col_desc[3].max_data_string;
1246 		}
1247 
1248 		if (!col_desc[4].disabled) {
1249 			if (pollers[i]->period_ticks != 0) {
1250 				get_time_str(pollers[i]->period_ticks, period_ticks);
1251 				print_max_len(g_tabs[POLLERS_TAB], TABS_DATA_START_ROW + item_index, col,
1252 					      col_desc[4].max_data_string, ALIGN_RIGHT, period_ticks);
1253 			}
1254 			col += col_desc[3].max_data_string + 4;
1255 		}
1256 
1257 		store_last_run_counter(pollers[i]->name, pollers[i]->thread_id, pollers[i]->run_count);
1258 
1259 		if (!col_desc[5].disabled) {
1260 			if (pollers[i]->busy_count > 0) {
1261 				if (item_index != g_selected_row) {
1262 					wattron(g_tabs[POLLERS_TAB], COLOR_PAIR(6));
1263 					print_max_len(g_tabs[POLLERS_TAB], TABS_DATA_START_ROW + item_index, col,
1264 						      col_desc[5].max_data_string, ALIGN_RIGHT, "Busy");
1265 					wattroff(g_tabs[POLLERS_TAB], COLOR_PAIR(6));
1266 				} else {
1267 					wattron(g_tabs[POLLERS_TAB], COLOR_PAIR(8));
1268 					print_max_len(g_tabs[POLLERS_TAB], TABS_DATA_START_ROW + item_index, col,
1269 						      col_desc[5].max_data_string, ALIGN_RIGHT, "Busy");
1270 					wattroff(g_tabs[POLLERS_TAB], COLOR_PAIR(8));
1271 				}
1272 			} else {
1273 				if (item_index != g_selected_row) {
1274 					wattron(g_tabs[POLLERS_TAB], COLOR_PAIR(7));
1275 					print_max_len(g_tabs[POLLERS_TAB], TABS_DATA_START_ROW + item_index, col,
1276 						      col_desc[5].max_data_string, ALIGN_RIGHT, "Idle");
1277 					wattroff(g_tabs[POLLERS_TAB], COLOR_PAIR(7));
1278 				} else {
1279 					wattron(g_tabs[POLLERS_TAB], COLOR_PAIR(9));
1280 					print_max_len(g_tabs[POLLERS_TAB], TABS_DATA_START_ROW + item_index, col,
1281 						      col_desc[5].max_data_string, ALIGN_RIGHT, "Idle");
1282 					wattroff(g_tabs[POLLERS_TAB], COLOR_PAIR(9));
1283 				}
1284 			}
1285 		}
1286 
1287 		if (item_index == g_selected_row) {
1288 			wattroff(g_tabs[POLLERS_TAB], COLOR_PAIR(2));
1289 		}
1290 	}
1291 
1292 	g_max_selected_row = i - current_page * g_max_data_rows - 1;
1293 
1294 	return max_pages;
1295 }
1296 
1297 static int
1298 sort_cores(const void *p1, const void *p2)
1299 {
1300 	const struct core_info core_info1 = *(struct core_info *)p1;
1301 	const struct core_info core_info2 = *(struct core_info *)p2;
1302 	uint64_t count1, count2;
1303 
1304 	switch (g_current_sort_col[CORES_TAB]) {
1305 	case 0: /* Sort by core */
1306 		count1 = core_info2.core;
1307 		count2 = core_info1.core;
1308 		break;
1309 	case 1: /* Sort by threads number */
1310 		count1 = core_info1.threads_count;
1311 		count2 = core_info2.threads_count;
1312 		break;
1313 	case 2: /* Sort by pollers number */
1314 		count1 = core_info1.pollers_count;
1315 		count2 = core_info2.pollers_count;
1316 		break;
1317 	case 3: /* Sort by idle time */
1318 		count2 = g_cores_history[core_info1.core].last_idle - core_info1.idle;
1319 		count1 = g_cores_history[core_info2.core].last_idle - core_info2.idle;
1320 		break;
1321 	case 4: /* Sort by busy time */
1322 		count2 = g_cores_history[core_info1.core].last_busy - core_info1.busy;
1323 		count1 = g_cores_history[core_info2.core].last_busy - core_info2.busy;
1324 		break;
1325 	default:
1326 		return 0;
1327 	}
1328 
1329 	if (count2 > count1) {
1330 		return 1;
1331 	} else if (count2 < count1) {
1332 		return -1;
1333 	} else {
1334 		return 0;
1335 	}
1336 }
1337 
1338 static void
1339 store_core_last_stats(uint32_t core, uint64_t idle, uint64_t busy)
1340 {
1341 	g_cores_history[core].last_idle = idle;
1342 	g_cores_history[core].last_busy = busy;
1343 }
1344 
1345 static void
1346 get_core_last_stats(uint32_t core, uint64_t *idle, uint64_t *busy)
1347 {
1348 	*idle = g_cores_history[core].last_idle;
1349 	*busy = g_cores_history[core].last_busy;
1350 }
1351 
1352 static void
1353 store_core_stats(uint32_t core, uint64_t threads, uint64_t pollers, uint64_t idle, uint64_t busy)
1354 {
1355 	g_cores_history[core].threads_count = threads;
1356 	g_cores_history[core].pollers_count = pollers;
1357 	g_cores_history[core].idle = idle;
1358 	g_cores_history[core].busy = busy;
1359 }
1360 
1361 static uint8_t
1362 refresh_cores_tab(uint8_t current_page)
1363 {
1364 	struct col_desc *col_desc = g_col_desc[CORES_TAB];
1365 	uint64_t i;
1366 	uint32_t core_num;
1367 	uint16_t offset, count = 0;
1368 	uint8_t max_pages, item_index;
1369 	static uint8_t last_page = 0;
1370 	char core[MAX_CORE_STR_LEN], threads_number[MAX_THREAD_COUNT_STR_LEN],
1371 	     pollers_number[MAX_POLLER_COUNT_STR_LEN], idle_time[MAX_TIME_STR_LEN],
1372 	     busy_time[MAX_TIME_STR_LEN], core_freq[MAX_CORE_FREQ_STR_LEN];
1373 	struct core_info cores[RPC_MAX_CORES];
1374 
1375 	memset(&cores, 0, sizeof(cores));
1376 
1377 	for (i = 0; i < g_threads_stats.threads.threads_count; i++) {
1378 		core_num = g_threads_stats.threads.thread_info[i].core_num;
1379 		cores[core_num].threads_count++;
1380 		cores[core_num].pollers_count += g_threads_stats.threads.thread_info[i].active_pollers_count +
1381 						 g_threads_stats.threads.thread_info[i].timed_pollers_count +
1382 						 g_threads_stats.threads.thread_info[i].paused_pollers_count;
1383 	}
1384 
1385 	count = g_cores_stats.cores.cores_count;
1386 
1387 	for (i = 0; i < count; i++) {
1388 		core_num = g_cores_stats.cores.core[i].lcore;
1389 		cores[core_num].core = core_num;
1390 		cores[core_num].busy = g_cores_stats.cores.core[i].busy;
1391 		cores[core_num].idle = g_cores_stats.cores.core[i].idle;
1392 		if (last_page != current_page) {
1393 			store_core_last_stats(cores[core_num].core, cores[core_num].idle, cores[core_num].busy);
1394 		}
1395 	}
1396 
1397 	if (last_page != current_page) {
1398 		last_page = current_page;
1399 	}
1400 
1401 	max_pages = (count + g_max_row - WINDOW_HEADER - 1) / (g_max_row - WINDOW_HEADER);
1402 
1403 	qsort(&cores, count, sizeof(cores[0]), sort_cores);
1404 
1405 	for (i = current_page * g_max_data_rows;
1406 	     i < spdk_min(count, (uint64_t)((current_page + 1) * g_max_data_rows));
1407 	     i++) {
1408 		item_index = i - (current_page * g_max_data_rows);
1409 
1410 		core_num = g_cores_stats.cores.core[i].lcore;
1411 
1412 		snprintf(threads_number, MAX_THREAD_COUNT_STR_LEN, "%ld", cores[core_num].threads_count);
1413 		snprintf(pollers_number, MAX_POLLER_COUNT_STR_LEN, "%ld", cores[core_num].pollers_count);
1414 		get_core_last_stats(cores[core_num].core, &cores[core_num].last_idle, &cores[core_num].last_busy);
1415 
1416 		offset = 1;
1417 
1418 		draw_row_background(item_index, CORES_TAB);
1419 
1420 		if (!col_desc[0].disabled) {
1421 			snprintf(core, MAX_CORE_STR_LEN, "%d", cores[core_num].core);
1422 			print_max_len(g_tabs[CORES_TAB], TABS_DATA_START_ROW + item_index, offset,
1423 				      col_desc[0].max_data_string, ALIGN_RIGHT, core);
1424 			offset += col_desc[0].max_data_string + 2;
1425 		}
1426 
1427 		if (!col_desc[1].disabled) {
1428 			print_max_len(g_tabs[CORES_TAB], TABS_DATA_START_ROW + item_index,
1429 				      offset + (col_desc[1].name_len / 2), col_desc[1].max_data_string, ALIGN_LEFT, threads_number);
1430 			offset += col_desc[1].max_data_string + 2;
1431 		}
1432 
1433 		if (!col_desc[2].disabled) {
1434 			print_max_len(g_tabs[CORES_TAB], TABS_DATA_START_ROW + item_index,
1435 				      offset + (col_desc[2].name_len / 2), col_desc[2].max_data_string, ALIGN_LEFT, pollers_number);
1436 			offset += col_desc[2].max_data_string;
1437 		}
1438 
1439 		if (!col_desc[3].disabled) {
1440 			if (g_interval_data == true) {
1441 				get_time_str(cores[core_num].idle - cores[core_num].last_idle, idle_time);
1442 			} else {
1443 				get_time_str(cores[core_num].idle, idle_time);
1444 			}
1445 			print_max_len(g_tabs[CORES_TAB], TABS_DATA_START_ROW + item_index, offset,
1446 				      col_desc[3].max_data_string, ALIGN_RIGHT, idle_time);
1447 			offset += col_desc[3].max_data_string + 2;
1448 		}
1449 
1450 		if (!col_desc[4].disabled) {
1451 			if (g_interval_data == true) {
1452 				get_time_str(cores[core_num].busy - cores[core_num].last_busy, busy_time);
1453 			} else {
1454 				get_time_str(cores[core_num].busy, busy_time);
1455 			}
1456 			print_max_len(g_tabs[CORES_TAB], TABS_DATA_START_ROW + item_index, offset,
1457 				      col_desc[4].max_data_string, ALIGN_RIGHT, busy_time);
1458 			offset += col_desc[4].max_data_string + 2;
1459 		}
1460 
1461 		if (!col_desc[5].disabled) {
1462 			if (!g_cores_stats.cores.core[core_num].core_freq) {
1463 				snprintf(core_freq,  MAX_CORE_FREQ_STR_LEN, "%s", "N/A");
1464 			} else {
1465 				snprintf(core_freq, MAX_CORE_FREQ_STR_LEN, "%" PRIu32,
1466 					 g_cores_stats.cores.core[core_num].core_freq);
1467 			}
1468 			print_max_len(g_tabs[CORES_TAB], TABS_DATA_START_ROW + item_index, offset,
1469 				      col_desc[5].max_data_string, ALIGN_RIGHT, core_freq);
1470 		}
1471 
1472 		store_core_last_stats(cores[core_num].core, cores[core_num].idle, cores[core_num].busy);
1473 		store_core_stats(cores[core_num].core, cores[core_num].threads_count, cores[core_num].pollers_count,
1474 				 cores[core_num].idle - cores[core_num].last_idle, cores[core_num].busy - cores[core_num].last_busy);
1475 
1476 		if (item_index == g_selected_row) {
1477 			wattroff(g_tabs[CORES_TAB], COLOR_PAIR(2));
1478 		}
1479 	}
1480 
1481 	g_max_selected_row = i - current_page * g_max_data_rows - 1;
1482 
1483 	return max_pages;
1484 }
1485 
1486 static uint8_t
1487 refresh_tab(enum tabs tab, uint8_t current_page)
1488 {
1489 	uint8_t (*refresh_function[NUMBER_OF_TABS])(uint8_t current_page) = {refresh_threads_tab, refresh_pollers_tab, refresh_cores_tab};
1490 	int color_pair[NUMBER_OF_TABS] = {COLOR_PAIR(2), COLOR_PAIR(2), COLOR_PAIR(2)};
1491 	int i;
1492 	uint8_t max_pages = 0;
1493 
1494 	color_pair[tab] = COLOR_PAIR(1);
1495 
1496 	for (i = 0; i < NUMBER_OF_TABS; i++) {
1497 		wbkgd(g_tab_win[i], color_pair[i]);
1498 	}
1499 
1500 	max_pages = (*refresh_function[tab])(current_page);
1501 	refresh();
1502 
1503 	for (i = 0; i < NUMBER_OF_TABS; i++) {
1504 		wrefresh(g_tab_win[i]);
1505 	}
1506 	draw_menu_win();
1507 
1508 	return max_pages;
1509 }
1510 
1511 static void
1512 print_in_middle(WINDOW *win, int starty, int startx, int width, char *string, chtype color)
1513 {
1514 	int length, temp;
1515 
1516 	length = strlen(string);
1517 	temp = (width - length) / 2;
1518 	wattron(win, color);
1519 	mvwprintw(win, starty, startx + temp, "%s", string);
1520 	wattroff(win, color);
1521 	refresh();
1522 }
1523 
1524 static void
1525 print_left(WINDOW *win, int starty, int startx, int width, char *string, chtype color)
1526 {
1527 	wattron(win, color);
1528 	mvwprintw(win, starty, startx, "%s", string);
1529 	wattroff(win, color);
1530 	refresh();
1531 }
1532 
1533 static void
1534 apply_filters(enum tabs tab)
1535 {
1536 	wclear(g_tabs[tab]);
1537 	draw_tabs(tab, g_current_sort_col[tab]);
1538 }
1539 
1540 static ITEM **
1541 draw_filtering_menu(uint8_t position, WINDOW *filter_win, uint8_t tab, MENU **my_menu)
1542 {
1543 	const int ADDITIONAL_ELEMENTS = 3;
1544 	const int ROW_PADDING = 6;
1545 	const int WINDOW_START_X = 1;
1546 	const int WINDOW_START_Y = 3;
1547 	const int WINDOW_COLUMNS = 2;
1548 	struct col_desc *col_desc = g_col_desc[tab];
1549 	ITEM **my_items;
1550 	MENU *menu;
1551 	int i, elements;
1552 	uint8_t len = 0;
1553 
1554 	for (i = 0; col_desc[i].name != NULL; ++i) {
1555 		len = spdk_max(col_desc[i].name_len, len);
1556 	}
1557 
1558 	elements = i;
1559 
1560 	my_items = (ITEM **)calloc(elements * WINDOW_COLUMNS + ADDITIONAL_ELEMENTS, sizeof(ITEM *));
1561 	if (my_items == NULL) {
1562 		fprintf(stderr, "Unable to allocate an item list in draw_filtering_menu.\n");
1563 		return NULL;
1564 	}
1565 
1566 	for (i = 0; i < elements * 2; i++) {
1567 		my_items[i] = new_item(col_desc[i / WINDOW_COLUMNS].name, NULL);
1568 		i++;
1569 		my_items[i] = new_item(col_desc[i / WINDOW_COLUMNS].disabled ? "[ ]" : "[*]", NULL);
1570 	}
1571 
1572 	my_items[i] = new_item("     CLOSE", NULL);
1573 	set_item_userptr(my_items[i], apply_filters);
1574 
1575 	menu = new_menu((ITEM **)my_items);
1576 
1577 	menu_opts_off(menu, O_SHOWDESC);
1578 	set_menu_format(menu, elements + 1, WINDOW_COLUMNS);
1579 
1580 	set_menu_win(menu, filter_win);
1581 	set_menu_sub(menu, derwin(filter_win, elements + 1, len + ROW_PADDING, WINDOW_START_Y,
1582 				  WINDOW_START_X));
1583 
1584 	*my_menu = menu;
1585 
1586 	post_menu(menu);
1587 	refresh();
1588 	wrefresh(filter_win);
1589 
1590 	for (i = 0; i < position / WINDOW_COLUMNS; i++) {
1591 		menu_driver(menu, REQ_DOWN_ITEM);
1592 	}
1593 
1594 	return my_items;
1595 }
1596 
1597 static void
1598 delete_filtering_menu(MENU *my_menu, ITEM **my_items, uint8_t elements)
1599 {
1600 	int i;
1601 
1602 	unpost_menu(my_menu);
1603 	free_menu(my_menu);
1604 	for (i = 0; i < elements * 2 + 2; ++i) {
1605 		free_item(my_items[i]);
1606 	}
1607 	free(my_items);
1608 }
1609 
1610 static ITEM **
1611 refresh_filtering_menu(MENU **my_menu, WINDOW *filter_win, uint8_t tab, ITEM **my_items,
1612 		       uint8_t elements, uint8_t position)
1613 {
1614 	delete_filtering_menu(*my_menu, my_items, elements);
1615 	return draw_filtering_menu(position, filter_win, tab, my_menu);
1616 }
1617 
1618 static void
1619 filter_columns(uint8_t tab)
1620 {
1621 	const int WINDOW_HEADER_LEN = 5;
1622 	const int WINDOW_BORDER_LEN = 8;
1623 	const int WINDOW_HEADER_END_LINE = 2;
1624 	const int WINDOW_COLUMNS = 2;
1625 	struct col_desc *col_desc = g_col_desc[tab];
1626 	PANEL *filter_panel;
1627 	WINDOW *filter_win;
1628 	ITEM **my_items;
1629 	MENU *my_menu = NULL;
1630 	int i, c, elements;
1631 	bool stop_loop = false;
1632 	ITEM *cur;
1633 	void (*p)(enum tabs tab);
1634 	uint8_t current_index, len = 0;
1635 	bool disabled[TABS_COL_COUNT];
1636 
1637 	for (i = 0; col_desc[i].name != NULL; ++i) {
1638 		len = spdk_max(col_desc[i].name_len, len);
1639 	}
1640 
1641 	elements = i;
1642 
1643 	filter_win = newwin(elements + WINDOW_HEADER_LEN, len + WINDOW_BORDER_LEN,
1644 			    (g_max_row - elements - 1) / 2, (g_max_col - len) / 2);
1645 	assert(filter_win != NULL);
1646 	keypad(filter_win, TRUE);
1647 	filter_panel = new_panel(filter_win);
1648 	assert(filter_panel != NULL);
1649 
1650 	top_panel(filter_panel);
1651 	update_panels();
1652 	doupdate();
1653 
1654 	box(filter_win, 0, 0);
1655 
1656 	print_in_middle(filter_win, 1, 0, len + WINDOW_BORDER_LEN, "Filtering", COLOR_PAIR(3));
1657 	mvwaddch(filter_win, WINDOW_HEADER_END_LINE, 0, ACS_LTEE);
1658 	mvwhline(filter_win, WINDOW_HEADER_END_LINE, 1, ACS_HLINE, len + WINDOW_BORDER_LEN - 2);
1659 	mvwaddch(filter_win, WINDOW_HEADER_END_LINE, len + WINDOW_BORDER_LEN - 1, ACS_RTEE);
1660 
1661 	my_items = draw_filtering_menu(0, filter_win, tab, &my_menu);
1662 	if (my_items == NULL || my_menu == NULL) {
1663 		goto fail;
1664 	}
1665 
1666 	for (int i = 0; i < TABS_COL_COUNT; i++) {
1667 		disabled[i] = col_desc[i].disabled;
1668 	}
1669 
1670 	while (!stop_loop) {
1671 		c = wgetch(filter_win);
1672 
1673 		switch (c) {
1674 		case KEY_DOWN:
1675 			menu_driver(my_menu, REQ_DOWN_ITEM);
1676 			break;
1677 		case KEY_UP:
1678 			menu_driver(my_menu, REQ_UP_ITEM);
1679 			break;
1680 		case 27: /* ESC */
1681 		case 'q':
1682 			for (int i = 0; i < TABS_COL_COUNT; i++) {
1683 				cur = current_item(my_menu);
1684 				col_desc[i].disabled = disabled[i];
1685 
1686 				my_items = refresh_filtering_menu(&my_menu, filter_win, tab, my_items, elements,
1687 								  item_index(cur) + 1);
1688 				if (my_items == NULL || my_menu == NULL) {
1689 					goto fail;
1690 				}
1691 			}
1692 
1693 			stop_loop = true;
1694 			break;
1695 		case ' ': /* Space */
1696 			cur = current_item(my_menu);
1697 			current_index = item_index(cur) / WINDOW_COLUMNS;
1698 			col_desc[current_index].disabled = !col_desc[current_index].disabled;
1699 			my_items = refresh_filtering_menu(&my_menu, filter_win, tab, my_items, elements,
1700 							  item_index(cur) + 1);
1701 			if (my_items == NULL || my_menu == NULL) {
1702 				goto fail;
1703 			}
1704 			break;
1705 		case 10: /* Enter */
1706 			cur = current_item(my_menu);
1707 			current_index = item_index(cur) / WINDOW_COLUMNS;
1708 			if (current_index == elements) {
1709 				stop_loop = true;
1710 				p = item_userptr(cur);
1711 				p(tab);
1712 			} else {
1713 				col_desc[current_index].disabled = !col_desc[current_index].disabled;
1714 				my_items = refresh_filtering_menu(&my_menu, filter_win, tab, my_items, elements,
1715 								  item_index(cur) + 1);
1716 				if (my_items == NULL || my_menu == NULL) {
1717 					goto fail;
1718 				}
1719 			}
1720 			break;
1721 		}
1722 		wrefresh(filter_win);
1723 	}
1724 
1725 	delete_filtering_menu(my_menu, my_items, elements);
1726 
1727 	del_panel(filter_panel);
1728 	delwin(filter_win);
1729 
1730 	wclear(g_menu_win);
1731 	draw_menu_win();
1732 	return;
1733 
1734 fail:
1735 	fprintf(stderr, "Unable to filter the columns due to allocation failure.\n");
1736 	assert(false);
1737 }
1738 
1739 static void
1740 sort_type(enum tabs tab, int item_index)
1741 {
1742 	g_current_sort_col[tab] = item_index;
1743 	wclear(g_tabs[tab]);
1744 	draw_tabs(tab, g_current_sort_col[tab]);
1745 }
1746 
1747 static void
1748 change_sorting(uint8_t tab)
1749 {
1750 	const int WINDOW_HEADER_LEN = 4;
1751 	const int WINDOW_BORDER_LEN = 3;
1752 	const int WINDOW_START_X = 1;
1753 	const int WINDOW_START_Y = 3;
1754 	const int WINDOW_HEADER_END_LINE = 2;
1755 	PANEL *sort_panel;
1756 	WINDOW *sort_win;
1757 	ITEM **my_items;
1758 	MENU *my_menu;
1759 	int i, c, elements;
1760 	bool stop_loop = false;
1761 	ITEM *cur;
1762 	void (*p)(enum tabs tab, int item_index);
1763 	uint8_t len = 0;
1764 
1765 	for (i = 0; g_col_desc[tab][i].name != NULL; ++i) {
1766 		len = spdk_max(len, g_col_desc[tab][i].name_len);
1767 	}
1768 
1769 	elements = i;
1770 
1771 	my_items = (ITEM **)calloc(elements + 1, sizeof(ITEM *));
1772 	if (my_items == NULL) {
1773 		fprintf(stderr, "Unable to allocate an item list in change_sorting.\n");
1774 		return;
1775 	}
1776 
1777 	for (i = 0; i < elements; ++i) {
1778 		my_items[i] = new_item(g_col_desc[tab][i].name, NULL);
1779 		set_item_userptr(my_items[i], sort_type);
1780 	}
1781 
1782 	my_menu = new_menu((ITEM **)my_items);
1783 
1784 	menu_opts_off(my_menu, O_SHOWDESC);
1785 
1786 	sort_win = newwin(elements + WINDOW_HEADER_LEN, len + WINDOW_BORDER_LEN, (g_max_row - elements) / 2,
1787 			  (g_max_col - len) / 2);
1788 	assert(sort_win != NULL);
1789 	keypad(sort_win, TRUE);
1790 	sort_panel = new_panel(sort_win);
1791 	assert(sort_panel != NULL);
1792 
1793 	top_panel(sort_panel);
1794 	update_panels();
1795 	doupdate();
1796 
1797 	set_menu_win(my_menu, sort_win);
1798 	set_menu_sub(my_menu, derwin(sort_win, elements, len + 1, WINDOW_START_Y, WINDOW_START_X));
1799 	box(sort_win, 0, 0);
1800 
1801 	print_in_middle(sort_win, 1, 0, len + WINDOW_BORDER_LEN, "Sorting", COLOR_PAIR(3));
1802 	mvwaddch(sort_win, WINDOW_HEADER_END_LINE, 0, ACS_LTEE);
1803 	mvwhline(sort_win, WINDOW_HEADER_END_LINE, 1, ACS_HLINE, len + 1);
1804 	mvwaddch(sort_win, WINDOW_HEADER_END_LINE, len + WINDOW_BORDER_LEN - 1, ACS_RTEE);
1805 
1806 	post_menu(my_menu);
1807 	refresh();
1808 	wrefresh(sort_win);
1809 
1810 	while (!stop_loop) {
1811 		c = wgetch(sort_win);
1812 
1813 		switch (c) {
1814 		case KEY_DOWN:
1815 			menu_driver(my_menu, REQ_DOWN_ITEM);
1816 			break;
1817 		case KEY_UP:
1818 			menu_driver(my_menu, REQ_UP_ITEM);
1819 			break;
1820 		case 27: /* ESC */
1821 			stop_loop = true;
1822 			break;
1823 		case 10: /* Enter */
1824 			stop_loop = true;
1825 			cur = current_item(my_menu);
1826 			p = item_userptr(cur);
1827 			p(tab, item_index(cur));
1828 			break;
1829 		}
1830 		wrefresh(sort_win);
1831 	}
1832 
1833 	unpost_menu(my_menu);
1834 	free_menu(my_menu);
1835 
1836 	for (i = 0; i < elements; ++i) {
1837 		free_item(my_items[i]);
1838 	}
1839 
1840 	free(my_items);
1841 
1842 	del_panel(sort_panel);
1843 	delwin(sort_win);
1844 
1845 	wclear(g_menu_win);
1846 	draw_menu_win();
1847 }
1848 
1849 static void
1850 change_refresh_rate(void)
1851 {
1852 	const int WINDOW_HEADER_END_LINE = 2;
1853 	PANEL *refresh_panel;
1854 	WINDOW *refresh_win;
1855 	int c;
1856 	bool stop_loop = false;
1857 	uint32_t rr_tmp, refresh_rate = 0;
1858 	char refresh_rate_str[MAX_STRING_LEN];
1859 
1860 	refresh_win = newwin(RR_WIN_HEIGHT, RR_WIN_WIDTH, (g_max_row - RR_WIN_HEIGHT - 1) / 2,
1861 			     (g_max_col - RR_WIN_WIDTH) / 2);
1862 	assert(refresh_win != NULL);
1863 	keypad(refresh_win, TRUE);
1864 	refresh_panel = new_panel(refresh_win);
1865 	assert(refresh_panel != NULL);
1866 
1867 	top_panel(refresh_panel);
1868 	update_panels();
1869 	doupdate();
1870 
1871 	box(refresh_win, 0, 0);
1872 
1873 	print_in_middle(refresh_win, 1, 0, RR_WIN_WIDTH + 1, "Enter refresh rate value [s]", COLOR_PAIR(3));
1874 	mvwaddch(refresh_win, WINDOW_HEADER_END_LINE, 0, ACS_LTEE);
1875 	mvwhline(refresh_win, WINDOW_HEADER_END_LINE, 1, ACS_HLINE, RR_WIN_WIDTH - 2);
1876 	mvwaddch(refresh_win, WINDOW_HEADER_END_LINE, RR_WIN_WIDTH, ACS_RTEE);
1877 	mvwprintw(refresh_win, WINDOW_HEADER_END_LINE + 1, (RR_WIN_WIDTH - 1) / 2, "%d", refresh_rate);
1878 
1879 	refresh();
1880 	wrefresh(refresh_win);
1881 
1882 	while (!stop_loop) {
1883 		c = wgetch(refresh_win);
1884 
1885 		switch (c) {
1886 		case '0':
1887 		case '1':
1888 		case '2':
1889 		case '3':
1890 		case '4':
1891 		case '5':
1892 		case '6':
1893 		case '7':
1894 		case '8':
1895 		case '9':
1896 			rr_tmp = refresh_rate * 10 + c - '0';
1897 
1898 			if (rr_tmp <= RR_MAX_VALUE) {
1899 				refresh_rate = rr_tmp;
1900 				snprintf(refresh_rate_str, MAX_STRING_LEN - 1, "%d", refresh_rate);
1901 				mvwprintw(refresh_win, WINDOW_HEADER_END_LINE + 1,
1902 					  (RR_WIN_WIDTH - 1 - strlen(refresh_rate_str)) / 2, "%d", refresh_rate);
1903 				refresh();
1904 				wrefresh(refresh_win);
1905 			}
1906 			break;
1907 		case KEY_BACKSPACE:
1908 		case 127:
1909 		case '\b':
1910 			refresh_rate = refresh_rate / 10;
1911 			snprintf(refresh_rate_str, MAX_STRING_LEN - 1, "%d", refresh_rate);
1912 			mvwprintw(refresh_win, WINDOW_HEADER_END_LINE + 1,
1913 				  (RR_WIN_WIDTH - 1 - strlen(refresh_rate_str) - 2) / 2, "       ");
1914 			mvwprintw(refresh_win, WINDOW_HEADER_END_LINE + 1,
1915 				  (RR_WIN_WIDTH - 1 - strlen(refresh_rate_str)) / 2, "%d", refresh_rate);
1916 			refresh();
1917 			wrefresh(refresh_win);
1918 			break;
1919 		case 27: /* ESC */
1920 		case 'q':
1921 			stop_loop = true;
1922 			break;
1923 		case 10: /* Enter */
1924 			g_sleep_time = refresh_rate;
1925 			stop_loop = true;
1926 			break;
1927 		}
1928 		wrefresh(refresh_win);
1929 	}
1930 
1931 	del_panel(refresh_panel);
1932 	delwin(refresh_win);
1933 }
1934 
1935 static void
1936 free_resources(void)
1937 {
1938 	struct run_counter_history *history, *tmp;
1939 
1940 	TAILQ_FOREACH_SAFE(history, &g_run_counter_history, link, tmp) {
1941 		TAILQ_REMOVE(&g_run_counter_history, history, link);
1942 		free(history->poller_name);
1943 		free(history);
1944 	}
1945 }
1946 
1947 static uint64_t
1948 get_position_for_window(uint64_t window_size, uint64_t max_size)
1949 {
1950 	/* This function calculates position for pop-up detail window.
1951 	 * Since horizontal and vertical positions are calculated the same way
1952 	 * there is no need for separate functions. */
1953 	window_size = spdk_min(window_size, max_size);
1954 
1955 	return (max_size - window_size) / 2;
1956 }
1957 
1958 static void
1959 display_thread(struct rpc_thread_info *thread_info)
1960 {
1961 	PANEL *thread_panel;
1962 	WINDOW *thread_win;
1963 	struct rpc_poller_thread_info *thread;
1964 	struct rpc_pollers *pollers;
1965 	struct rpc_poller_info *poller;
1966 	uint64_t pollers_count, current_row, i, j, time;
1967 	int c;
1968 	bool stop_loop = false;
1969 	char idle_time[MAX_TIME_STR_LEN], busy_time[MAX_TIME_STR_LEN], run_count[MAX_POLLER_COUNT_STR_LEN];
1970 
1971 	pollers_count = thread_info->active_pollers_count +
1972 			thread_info->timed_pollers_count +
1973 			thread_info->paused_pollers_count;
1974 
1975 	thread_win = newwin(pollers_count + THREAD_WIN_HEIGHT, THREAD_WIN_WIDTH,
1976 			    get_position_for_window(THREAD_WIN_HEIGHT + pollers_count, g_max_row),
1977 			    get_position_for_window(THREAD_WIN_WIDTH, g_max_col));
1978 	keypad(thread_win, TRUE);
1979 	thread_panel = new_panel(thread_win);
1980 
1981 	top_panel(thread_panel);
1982 	update_panels();
1983 	doupdate();
1984 
1985 	box(thread_win, 0, 0);
1986 
1987 	print_in_middle(thread_win, 1, 0, THREAD_WIN_WIDTH, thread_info->name,
1988 			COLOR_PAIR(3));
1989 	mvwhline(thread_win, 2, 1, ACS_HLINE, THREAD_WIN_WIDTH - 2);
1990 	mvwaddch(thread_win, 2, THREAD_WIN_WIDTH, ACS_RTEE);
1991 
1992 	print_left(thread_win, 3, THREAD_WIN_FIRST_COL, THREAD_WIN_WIDTH,
1993 		   "Core:                Idle [us]:            Busy [us]:", COLOR_PAIR(5));
1994 	mvwprintw(thread_win, 3, THREAD_WIN_FIRST_COL + 6, "%" PRIu64,
1995 		  thread_info->core_num);
1996 
1997 	if (g_interval_data) {
1998 		get_time_str(g_thread_history[thread_info->id].idle, idle_time);
1999 		mvwprintw(thread_win, 3, THREAD_WIN_FIRST_COL + 32, idle_time);
2000 		get_time_str(g_thread_history[thread_info->id].busy, busy_time);
2001 		mvwprintw(thread_win, 3, THREAD_WIN_FIRST_COL + 54, busy_time);
2002 	} else {
2003 		get_time_str(thread_info->idle, idle_time);
2004 		mvwprintw(thread_win, 3, THREAD_WIN_FIRST_COL + 32, idle_time);
2005 		get_time_str(thread_info->busy, busy_time);
2006 		mvwprintw(thread_win, 3, THREAD_WIN_FIRST_COL + 54, busy_time);
2007 	}
2008 
2009 	print_left(thread_win, 4, THREAD_WIN_FIRST_COL, THREAD_WIN_WIDTH,
2010 		   "Active pollers:      Timed pollers:        Paused pollers:", COLOR_PAIR(5));
2011 	mvwprintw(thread_win, 4, THREAD_WIN_FIRST_COL + 17, "%" PRIu64,
2012 		  thread_info->active_pollers_count);
2013 	mvwprintw(thread_win, 4, THREAD_WIN_FIRST_COL + 36, "%" PRIu64,
2014 		  thread_info->timed_pollers_count);
2015 	mvwprintw(thread_win, 4, THREAD_WIN_FIRST_COL + 59, "%" PRIu64,
2016 		  thread_info->paused_pollers_count);
2017 
2018 	mvwhline(thread_win, 5, 1, ACS_HLINE, THREAD_WIN_WIDTH - 2);
2019 
2020 	print_in_middle(thread_win, 6, 0, THREAD_WIN_WIDTH,
2021 			"Pollers                          Type    Total run count   Period", COLOR_PAIR(5));
2022 
2023 	mvwhline(thread_win, 7, 1, ACS_HLINE, THREAD_WIN_WIDTH - 2);
2024 
2025 	current_row = 8;
2026 
2027 	for (i = 0; i < g_pollers_stats.pollers_threads.threads_count; i++) {
2028 		thread = &g_pollers_stats.pollers_threads.threads[i];
2029 		if (thread->id == thread_info->id) {
2030 			pollers = &thread->active_pollers;
2031 			for (j = 0; j < pollers->pollers_count; j++) {
2032 				poller = &pollers->pollers[j];
2033 				mvwprintw(thread_win, current_row, THREAD_WIN_FIRST_COL, "%s", poller->name);
2034 				mvwprintw(thread_win, current_row, THREAD_WIN_FIRST_COL + 33, "Active");
2035 				snprintf(run_count, MAX_POLLER_COUNT_STR_LEN, "%" PRIu64, poller->run_count);
2036 				mvwprintw(thread_win, current_row, THREAD_WIN_FIRST_COL + 41, run_count);
2037 				current_row++;
2038 			}
2039 			pollers = &thread->timed_pollers;
2040 			for (j = 0; j < pollers->pollers_count; j++) {
2041 				poller = &pollers->pollers[j];
2042 				mvwprintw(thread_win, current_row, THREAD_WIN_FIRST_COL, "%s", poller->name);
2043 				mvwprintw(thread_win, current_row, THREAD_WIN_FIRST_COL + 33, "Timed");
2044 				mvwprintw(thread_win, current_row, THREAD_WIN_FIRST_COL + 41, "%" PRIu64, poller->run_count);
2045 				time = poller->period_ticks * SPDK_SEC_TO_USEC / g_cores_stats.tick_rate;
2046 				mvwprintw(thread_win, current_row, THREAD_WIN_FIRST_COL + 59, "%" PRIu64, time);
2047 				current_row++;
2048 			}
2049 			pollers = &thread->paused_pollers;
2050 			for (j = 0; j < pollers->pollers_count; j++) {
2051 				poller = &pollers->pollers[j];
2052 				mvwprintw(thread_win, current_row, THREAD_WIN_FIRST_COL, "%s", poller->name);
2053 				mvwprintw(thread_win, current_row, THREAD_WIN_FIRST_COL + 33, "Paused");
2054 				mvwprintw(thread_win, current_row, THREAD_WIN_FIRST_COL + 41, "%" PRIu64, poller->run_count);
2055 				current_row++;
2056 			}
2057 		}
2058 	}
2059 
2060 	refresh();
2061 	wrefresh(thread_win);
2062 
2063 	while (!stop_loop) {
2064 		c = wgetch(thread_win);
2065 
2066 		switch (c) {
2067 		case 27: /* ESC */
2068 			stop_loop = true;
2069 			break;
2070 		default:
2071 			break;
2072 		}
2073 	}
2074 
2075 	del_panel(thread_panel);
2076 	delwin(thread_win);
2077 }
2078 
2079 static void
2080 show_thread(uint8_t current_page)
2081 {
2082 	struct rpc_thread_info *thread_info[g_threads_stats.threads.threads_count];
2083 	uint64_t thread_number = current_page * g_max_data_rows + g_selected_row;
2084 	uint64_t i;
2085 
2086 	get_data();
2087 
2088 	assert(thread_number < g_threads_stats.threads.threads_count);
2089 	for (i = 0; i < g_threads_stats.threads.threads_count; i++) {
2090 		thread_info[i] = &g_threads_stats.threads.thread_info[i];
2091 	}
2092 
2093 	qsort(thread_info, g_threads_stats.threads.threads_count, sizeof(thread_info[0]), sort_threads);
2094 
2095 	display_thread(thread_info[thread_number]);
2096 
2097 	free_data();
2098 }
2099 
2100 static void
2101 show_single_thread(uint64_t thread_id)
2102 {
2103 	uint64_t i;
2104 
2105 	for (i = 0; i < g_threads_stats.threads.threads_count; i++) {
2106 		if (g_threads_stats.threads.thread_info[i].id == thread_id) {
2107 			display_thread(&g_threads_stats.threads.thread_info[i]);
2108 			break;
2109 		}
2110 	}
2111 }
2112 
2113 static void
2114 show_core(uint8_t current_page)
2115 {
2116 	PANEL *core_panel;
2117 	WINDOW *core_win;
2118 	uint64_t core_number = current_page * g_max_data_rows + g_selected_row;
2119 	struct rpc_core_info *core_info[g_cores_stats.cores.cores_count];
2120 	uint64_t threads_count, i, j;
2121 	uint16_t current_threads_row;
2122 	int c;
2123 	char core_win_title[25];
2124 	bool stop_loop = false;
2125 	char idle_time[MAX_TIME_STR_LEN], busy_time[MAX_TIME_STR_LEN];
2126 
2127 	get_data();
2128 
2129 	assert(core_number < g_cores_stats.cores.cores_count);
2130 	for (i = 0; i < g_cores_stats.cores.cores_count; i++) {
2131 		core_info[i] = &g_cores_stats.cores.core[i];
2132 	}
2133 
2134 	threads_count = g_cores_stats.cores.core->threads.threads_count;
2135 	core_win = newwin(threads_count + CORE_WIN_HEIGHT, CORE_WIN_WIDTH,
2136 			  get_position_for_window(CORE_WIN_HEIGHT + threads_count, g_max_row),
2137 			  get_position_for_window(CORE_WIN_WIDTH, g_max_col));
2138 
2139 	keypad(core_win, TRUE);
2140 	core_panel = new_panel(core_win);
2141 
2142 	top_panel(core_panel);
2143 	update_panels();
2144 	doupdate();
2145 
2146 	box(core_win, 0, 0);
2147 	snprintf(core_win_title, sizeof(core_win_title), "Core %" PRIu64 " details", core_number);
2148 	print_in_middle(core_win, 1, 0, CORE_WIN_WIDTH, core_win_title, COLOR_PAIR(3));
2149 
2150 	mvwaddch(core_win, -1, 0, ACS_LTEE);
2151 	mvwhline(core_win, 2, 1, ACS_HLINE, CORE_WIN_WIDTH - 2);
2152 	mvwaddch(core_win, 2, CORE_WIN_WIDTH, ACS_RTEE);
2153 	print_in_middle(core_win, 3, 0, CORE_WIN_WIDTH - (CORE_WIN_WIDTH / 3), "Frequency:", COLOR_PAIR(5));
2154 	if (core_info[core_number]->core_freq) {
2155 		mvwprintw(core_win, 3, CORE_WIN_FIRST_COL + 15, "%" PRIu32,
2156 			  core_info[core_number]->core_freq);
2157 	} else {
2158 		mvwprintw(core_win, 3, CORE_WIN_FIRST_COL + 15, "%s", "N/A");
2159 	}
2160 
2161 	mvwaddch(core_win, -1, 0, ACS_LTEE);
2162 	mvwhline(core_win, 4, 1, ACS_HLINE, CORE_WIN_WIDTH - 2);
2163 	mvwaddch(core_win, 4, CORE_WIN_WIDTH, ACS_RTEE);
2164 	print_left(core_win, 5, 1, CORE_WIN_WIDTH, "Thread count:          Idle time:", COLOR_PAIR(5));
2165 
2166 	mvwprintw(core_win, 5, CORE_WIN_FIRST_COL, "%" PRIu64,
2167 		  g_cores_history[core_number].threads_count);
2168 
2169 	if (g_interval_data == true) {
2170 		get_time_str(g_cores_history[core_number].idle, idle_time);
2171 		get_time_str(g_cores_history[core_number].busy, busy_time);
2172 	} else {
2173 		get_time_str(core_info[core_number]->idle, idle_time);
2174 		get_time_str(core_info[core_number]->busy, busy_time);
2175 	}
2176 	mvwprintw(core_win, 5, CORE_WIN_FIRST_COL + 20, idle_time);
2177 
2178 	print_left(core_win, 7, 1, CORE_WIN_WIDTH, "Poller count:          Busy time:", COLOR_PAIR(5));
2179 	mvwprintw(core_win, 7, CORE_WIN_FIRST_COL, "%" PRIu64,
2180 		  g_cores_history[core_number].pollers_count);
2181 
2182 	mvwprintw(core_win, 7, CORE_WIN_FIRST_COL + 20, busy_time);
2183 
2184 	mvwhline(core_win, 6, 1, ACS_HLINE, CORE_WIN_WIDTH - 2);
2185 	mvwhline(core_win, 8, 1, ACS_HLINE, CORE_WIN_WIDTH - 2);
2186 	print_left(core_win, 9, 1, CORE_WIN_WIDTH, "Threads on this core", COLOR_PAIR(5));
2187 
2188 	for (j = 0; j < core_info[core_number]->threads.threads_count; j++) {
2189 		mvwprintw(core_win, j + 10, 1, core_info[core_number]->threads.thread[j].name);
2190 	}
2191 
2192 	refresh();
2193 	wrefresh(core_win);
2194 
2195 	current_threads_row = 0;
2196 
2197 	while (!stop_loop) {
2198 		for (j = 0; j < core_info[core_number]->threads.threads_count; j++) {
2199 			if (j != current_threads_row) {
2200 				mvwprintw(core_win, j + 10, 1, core_info[core_number]->threads.thread[j].name);
2201 			} else {
2202 				print_left(core_win, j + 10, 1, CORE_WIN_WIDTH - 2,
2203 					   core_info[core_number]->threads.thread[j].name, COLOR_PAIR(2));
2204 			}
2205 		}
2206 
2207 		wrefresh(core_win);
2208 
2209 		c = wgetch(core_win);
2210 		switch (c) {
2211 		case 10: /* ENTER */
2212 			show_single_thread(core_info[core_number]->threads.thread[current_threads_row].id);
2213 			break;
2214 		case 27: /* ESC */
2215 			stop_loop = true;
2216 			break;
2217 		case KEY_UP:
2218 			if (current_threads_row != 0) {
2219 				current_threads_row--;
2220 			}
2221 			break;
2222 		case KEY_DOWN:
2223 			if (current_threads_row != core_info[core_number]->threads.threads_count - 1) {
2224 				current_threads_row++;
2225 			}
2226 			break;
2227 		default:
2228 			break;
2229 		}
2230 	}
2231 
2232 	del_panel(core_panel);
2233 	delwin(core_win);
2234 
2235 	free_data();
2236 }
2237 
2238 static void
2239 show_poller(uint8_t current_page)
2240 {
2241 	PANEL *poller_panel;
2242 	WINDOW *poller_win;
2243 	uint64_t count = 0;
2244 	uint64_t poller_number = current_page * g_max_data_rows + g_selected_row;
2245 	struct rpc_poller_info *pollers[RPC_MAX_POLLERS];
2246 	bool stop_loop = false;
2247 	char poller_period[MAX_TIME_STR_LEN];
2248 	int c;
2249 
2250 	get_data();
2251 
2252 	prepare_poller_data(current_page, pollers, &count, current_page);
2253 	assert(poller_number < count);
2254 
2255 	poller_win = newwin(POLLER_WIN_HEIGHT, POLLER_WIN_WIDTH,
2256 			    get_position_for_window(POLLER_WIN_HEIGHT, g_max_row),
2257 			    get_position_for_window(POLLER_WIN_WIDTH, g_max_col));
2258 
2259 	keypad(poller_win, TRUE);
2260 	poller_panel = new_panel(poller_win);
2261 
2262 	top_panel(poller_panel);
2263 	update_panels();
2264 	doupdate();
2265 
2266 	box(poller_win, 0, 0);
2267 
2268 	print_in_middle(poller_win, 1, 0, POLLER_WIN_WIDTH, pollers[poller_number]->name, COLOR_PAIR(3));
2269 	mvwhline(poller_win, 2, 1, ACS_HLINE, POLLER_WIN_WIDTH - 2);
2270 	mvwaddch(poller_win, 2, POLLER_WIN_WIDTH, ACS_RTEE);
2271 
2272 	print_left(poller_win, 3, 2, POLLER_WIN_WIDTH, "Type:                  On thread:", COLOR_PAIR(5));
2273 	mvwprintw(poller_win, 3, POLLER_WIN_FIRST_COL,
2274 		  poller_type_str[pollers[poller_number]->type]);
2275 	mvwprintw(poller_win, 3, POLLER_WIN_FIRST_COL + 23, pollers[poller_number]->thread_name);
2276 
2277 	print_left(poller_win, 4, 2, POLLER_WIN_WIDTH, "Run count:", COLOR_PAIR(5));
2278 
2279 	if (g_interval_data) {
2280 		mvwprintw(poller_win, 4, POLLER_WIN_FIRST_COL, "%" PRIu64,
2281 			  g_pollers_history[poller_number].run_count);
2282 	} else {
2283 		mvwprintw(poller_win, 4, POLLER_WIN_FIRST_COL, "%" PRIu64,
2284 			  pollers[poller_number]->run_count);
2285 	}
2286 
2287 	if (pollers[poller_number]->period_ticks != 0) {
2288 		print_left(poller_win, 4, 28, POLLER_WIN_WIDTH, "Period:", COLOR_PAIR(5));
2289 		get_time_str(g_pollers_history[poller_number].period_ticks, poller_period);
2290 		mvwprintw(poller_win, 4, POLLER_WIN_FIRST_COL + 23, poller_period);
2291 	}
2292 	mvwhline(poller_win, 5, 1, ACS_HLINE, POLLER_WIN_WIDTH - 2);
2293 	print_in_middle(poller_win, 6, 1, POLLER_WIN_WIDTH - 7, "Status:", COLOR_PAIR(5));
2294 
2295 	if (pollers[poller_number]->busy_count > 0) {
2296 		print_in_middle(poller_win, 6, 1, POLLER_WIN_WIDTH + 6, "Busy", COLOR_PAIR(6));
2297 	} else {
2298 		print_in_middle(poller_win, 6, 1, POLLER_WIN_WIDTH + 6, "Idle", COLOR_PAIR(7));
2299 	}
2300 
2301 	refresh();
2302 	wrefresh(poller_win);
2303 
2304 	while (!stop_loop) {
2305 		c = wgetch(poller_win);
2306 		switch (c) {
2307 		case 27: /* ESC */
2308 			stop_loop = true;
2309 			break;
2310 		default:
2311 			break;
2312 		}
2313 	}
2314 
2315 	del_panel(poller_panel);
2316 	delwin(poller_win);
2317 
2318 	free_data();
2319 }
2320 
2321 static void
2322 show_stats(void)
2323 {
2324 	const int CURRENT_PAGE_STR_LEN = 50;
2325 	const char *refresh_error = "ERROR occurred while getting data";
2326 	long int time_last, time_dif;
2327 	struct timespec time_now;
2328 	int c, rc;
2329 	int max_row, max_col;
2330 	uint8_t active_tab = THREADS_TAB;
2331 	uint8_t current_page = 0;
2332 	uint8_t max_pages = 1;
2333 	uint16_t required_size = WINDOW_HEADER + 1;
2334 	char current_page_str[CURRENT_PAGE_STR_LEN];
2335 	bool force_refresh = true;
2336 
2337 	clock_gettime(CLOCK_REALTIME, &time_now);
2338 	time_last = time_now.tv_sec;
2339 
2340 	switch_tab(THREADS_TAB);
2341 
2342 	while (1) {
2343 		/* Check if interface has to be resized (terminal size changed) */
2344 		getmaxyx(stdscr, max_row, max_col);
2345 
2346 		if (max_row != g_max_row || max_col != g_max_col) {
2347 			g_max_row = spdk_max(max_row, required_size);
2348 			g_max_col = max_col;
2349 			g_data_win_size = g_max_row - required_size + 1;
2350 			g_max_data_rows = g_max_row - WINDOW_HEADER;
2351 			resize_interface(active_tab);
2352 		}
2353 
2354 		c = getch();
2355 		if (c == 'q') {
2356 			free_resources();
2357 			break;
2358 		}
2359 
2360 		force_refresh = true;
2361 
2362 		switch (c) {
2363 		case '1':
2364 		case '2':
2365 		case '3':
2366 			active_tab = c - '1';
2367 			current_page = 0;
2368 			g_selected_row = 0;
2369 			switch_tab(active_tab);
2370 			break;
2371 		case '\t':
2372 			if (active_tab < NUMBER_OF_TABS - 1) {
2373 				active_tab++;
2374 			} else {
2375 				active_tab = THREADS_TAB;
2376 			}
2377 			g_selected_row = 0;
2378 			current_page = 0;
2379 			switch_tab(active_tab);
2380 			break;
2381 		case 's':
2382 			change_sorting(active_tab);
2383 			break;
2384 		case 'c':
2385 			filter_columns(active_tab);
2386 			break;
2387 		case 'r':
2388 			change_refresh_rate();
2389 			break;
2390 		case 't':
2391 			g_interval_data = !g_interval_data;
2392 			break;
2393 		case KEY_NPAGE: /* PgDown */
2394 			if (current_page + 1 < max_pages) {
2395 				current_page++;
2396 			}
2397 			wclear(g_tabs[active_tab]);
2398 			g_selected_row = 0;
2399 			draw_tabs(active_tab, g_current_sort_col[active_tab]);
2400 			break;
2401 		case KEY_PPAGE: /* PgUp */
2402 			if (current_page > 0) {
2403 				current_page--;
2404 			}
2405 			wclear(g_tabs[active_tab]);
2406 			g_selected_row = 0;
2407 			draw_tabs(active_tab, g_current_sort_col[active_tab]);
2408 			break;
2409 		case KEY_UP: /* Arrow up */
2410 			if (g_selected_row > 0) {
2411 				g_selected_row--;
2412 			}
2413 			break;
2414 		case KEY_DOWN: /* Arrow down */
2415 			if (g_selected_row < g_max_selected_row) {
2416 				g_selected_row++;
2417 			}
2418 			break;
2419 		case 10: /* Enter */
2420 			if (active_tab == THREADS_TAB) {
2421 				show_thread(current_page);
2422 			} else if (active_tab == CORES_TAB) {
2423 				show_core(current_page);
2424 			} else if (active_tab == POLLERS_TAB) {
2425 				show_poller(current_page);
2426 			}
2427 			break;
2428 		default:
2429 			force_refresh = false;
2430 			break;
2431 		}
2432 
2433 		clock_gettime(CLOCK_REALTIME, &time_now);
2434 		time_dif = time_now.tv_sec - time_last;
2435 		if (time_dif < 0) {
2436 			time_dif = g_sleep_time;
2437 		}
2438 
2439 		if (time_dif >= g_sleep_time || force_refresh) {
2440 			time_last = time_now.tv_sec;
2441 			rc = get_data();
2442 			if (rc) {
2443 				mvprintw(g_max_row - 1, g_max_col - strlen(refresh_error) - 2, refresh_error);
2444 			}
2445 
2446 			max_pages = refresh_tab(active_tab, current_page);
2447 
2448 			snprintf(current_page_str, CURRENT_PAGE_STR_LEN - 1, "Page: %d/%d", current_page + 1, max_pages);
2449 			mvprintw(g_max_row - 1, 1, current_page_str);
2450 
2451 			free_data();
2452 
2453 			refresh();
2454 		}
2455 	}
2456 }
2457 
2458 static void
2459 draw_interface(void)
2460 {
2461 	int i;
2462 	uint16_t required_size =  WINDOW_HEADER + 1;
2463 
2464 	getmaxyx(stdscr, g_max_row, g_max_col);
2465 	g_max_row = spdk_max(g_max_row, required_size);
2466 	g_data_win_size = g_max_row - required_size;
2467 	g_max_data_rows = g_max_row - WINDOW_HEADER;
2468 
2469 	g_menu_win = newwin(MENU_WIN_HEIGHT, g_max_col, g_max_row - MENU_WIN_HEIGHT - 1,
2470 			    MENU_WIN_LOCATION_COL);
2471 	assert(g_menu_win != NULL);
2472 	draw_menu_win();
2473 
2474 	for (i = 0; i < NUMBER_OF_TABS; i++) {
2475 		g_tab_win[i] = newwin(TAB_WIN_HEIGHT, g_max_col / NUMBER_OF_TABS - TABS_SPACING,
2476 				      TAB_WIN_LOCATION_ROW, g_max_col / NUMBER_OF_TABS * i + 1);
2477 		assert(g_tab_win[i] != NULL);
2478 		draw_tab_win(i);
2479 
2480 		g_tabs[i] = newwin(g_max_row - MENU_WIN_HEIGHT - TAB_WIN_HEIGHT - 2, g_max_col, TABS_LOCATION_ROW,
2481 				   TABS_LOCATION_COL);
2482 		draw_tabs(i, g_current_sort_col[i]);
2483 		g_panels[i] = new_panel(g_tabs[i]);
2484 		assert(g_panels[i] != NULL);
2485 	}
2486 
2487 	update_panels();
2488 	doupdate();
2489 }
2490 
2491 static void finish(int sig)
2492 {
2493 	/* End ncurses mode */
2494 	endwin();
2495 	spdk_jsonrpc_client_close(g_rpc_client);
2496 	exit(0);
2497 }
2498 
2499 static void
2500 setup_ncurses(void)
2501 {
2502 	clear();
2503 	noecho();
2504 	timeout(1);
2505 	curs_set(0);
2506 	keypad(stdscr, TRUE);
2507 	start_color();
2508 	init_pair(1, COLOR_BLACK, COLOR_GREEN);
2509 	init_pair(2, COLOR_BLACK, COLOR_WHITE);
2510 	init_pair(3, COLOR_YELLOW, COLOR_BLACK);
2511 	init_pair(4, COLOR_BLACK, COLOR_YELLOW);
2512 	init_pair(5, COLOR_GREEN, COLOR_BLACK);
2513 	init_pair(6, COLOR_RED, COLOR_BLACK);
2514 	init_pair(7, COLOR_BLUE, COLOR_BLACK);
2515 	init_pair(8, COLOR_RED, COLOR_WHITE);
2516 	init_pair(9, COLOR_BLUE, COLOR_WHITE);
2517 
2518 	if (has_colors() == FALSE) {
2519 		endwin();
2520 		printf("Your terminal does not support color\n");
2521 		exit(1);
2522 	}
2523 
2524 	/* Handle signals to exit gracfully cleaning up ncurses */
2525 	(void) signal(SIGINT, finish);
2526 	(void) signal(SIGPIPE, finish);
2527 	(void) signal(SIGABRT, finish);
2528 }
2529 
2530 static void
2531 usage(const char *program_name)
2532 {
2533 	printf("%s [options]", program_name);
2534 	printf("\n");
2535 	printf("options:\n");
2536 	printf(" -r <path>  RPC connect address (default: /var/tmp/spdk.sock)\n");
2537 	printf(" -h         show this usage\n");
2538 }
2539 
2540 static int
2541 wait_init(void)
2542 {
2543 	struct spdk_jsonrpc_client_response *json_resp = NULL;
2544 	char *uninit_log = "Waiting for SPDK target application to initialize...",
2545 	      *uninit_error = "Unable to read SPDK application state!";
2546 	int c, max_col, rc = 0;
2547 
2548 	max_col = getmaxx(stdscr);
2549 	print_in_middle(stdscr, FIRST_DATA_ROW, 1, max_col, uninit_log, COLOR_PAIR(5));
2550 	rc = rpc_send_req("framework_wait_init", &json_resp);
2551 	if (rc) {
2552 		spdk_jsonrpc_client_free_response(json_resp);
2553 
2554 		while (1) {
2555 			print_in_middle(stdscr, FIRST_DATA_ROW, 1, max_col, uninit_error, COLOR_PAIR(8));
2556 			c = getch();
2557 			if (c == 'q') {
2558 				return -1;
2559 			}
2560 		}
2561 	}
2562 
2563 	spdk_jsonrpc_client_free_response(json_resp);
2564 	return 0;
2565 }
2566 
2567 int main(int argc, char **argv)
2568 {
2569 	int op, rc;
2570 	char *socket = SPDK_DEFAULT_RPC_ADDR;
2571 
2572 	while ((op = getopt(argc, argv, "r:h")) != -1) {
2573 		switch (op) {
2574 		case 'r':
2575 			socket = optarg;
2576 			break;
2577 		default:
2578 			usage(argv[0]);
2579 			return op == 'h' ? 0 : 1;
2580 		}
2581 	}
2582 
2583 	g_rpc_client = spdk_jsonrpc_client_connect(socket, AF_UNIX);
2584 	if (!g_rpc_client) {
2585 		fprintf(stderr, "spdk_jsonrpc_client_connect() failed: %d\n", errno);
2586 		return 1;
2587 	}
2588 
2589 	initscr();
2590 	init_str_len();
2591 	setup_ncurses();
2592 	draw_interface();
2593 
2594 	rc = wait_init();
2595 	if (!rc) {
2596 		show_stats();
2597 	}
2598 
2599 	finish(0);
2600 
2601 	return (0);
2602 }
2603