xref: /spdk/app/spdk_top/spdk_top.c (revision efa33b8590b42a802ade9d0d06f23f21082135ed)
1 /*   SPDX-License-Identifier: BSD-3-Clause
2  *   Copyright (c) Intel Corporation.
3  *   All rights reserved.
4  */
5 
6 #include "spdk/stdinc.h"
7 #include "spdk/jsonrpc.h"
8 #include "spdk/rpc.h"
9 #include "spdk/event.h"
10 #include "spdk/util.h"
11 #include "spdk/env.h"
12 
13 #if defined __has_include
14 #if __has_include(<ncurses/panel.h>)
15 #include <ncurses/ncurses.h>
16 #include <ncurses/panel.h>
17 #include <ncurses/menu.h>
18 #else
19 #include <ncurses.h>
20 #include <panel.h>
21 #include <menu.h>
22 #endif
23 #else
24 #include <ncurses.h>
25 #include <panel.h>
26 #include <menu.h>
27 #endif
28 
29 #define RPC_MAX_THREADS 1024
30 #define RPC_MAX_POLLERS 1024
31 #define RPC_MAX_CORES 255
32 #define MAX_THREAD_NAME 128
33 #define MAX_POLLER_NAME 128
34 #define MAX_THREADS 4096
35 #define RR_MAX_VALUE 255
36 
37 #define MAX_STRING_LEN 12289 /* 3x 4k monitors + 1 */
38 #define TAB_WIN_HEIGHT 3
39 #define TAB_WIN_LOCATION_ROW 1
40 #define TABS_SPACING 2
41 #define TABS_LOCATION_ROW 4
42 #define TABS_LOCATION_COL 0
43 #define TABS_DATA_START_ROW 3
44 #define TABS_DATA_START_COL 2
45 #define TABS_COL_COUNT 10
46 #define MENU_WIN_HEIGHT 3
47 #define MENU_WIN_SPACING 4
48 #define MENU_WIN_LOCATION_COL 0
49 #define RR_WIN_WIDTH 32
50 #define RR_WIN_HEIGHT 5
51 #define MAX_THREAD_NAME_LEN 26
52 #define MAX_THREAD_COUNT_STR_LEN 14
53 #define MAX_POLLER_NAME_LEN 36
54 #define MAX_POLLER_COUNT_STR_LEN 16
55 #define MAX_POLLER_TYPE_STR_LEN 8
56 #define MAX_POLLER_IND_STR_LEN 28
57 #define MAX_CORE_MASK_STR_LEN 16
58 #define MAX_CORE_STR_LEN 6
59 #define MAX_CORE_FREQ_STR_LEN 18
60 #define MAX_TIME_STR_LEN 12
61 #define MAX_CPU_STR_LEN 8
62 #define MAX_POLLER_RUN_COUNT 20
63 #define MAX_PERIOD_STR_LEN 12
64 #define MAX_INTR_LEN 6
65 #define WINDOW_HEADER 12
66 #define FROM_HEX 16
67 #define THREAD_WIN_WIDTH 69
68 #define THREAD_WIN_HEIGHT 9
69 #define THREAD_WIN_FIRST_COL 2
70 #define CORE_WIN_FIRST_COL 16
71 #define CORE_WIN_WIDTH 48
72 #define CORE_WIN_HEIGHT 11
73 #define POLLER_WIN_HEIGHT 8
74 #define POLLER_WIN_WIDTH 64
75 #define POLLER_WIN_FIRST_COL 14
76 #define FIRST_DATA_ROW 7
77 #define HELP_WIN_WIDTH 88
78 #define HELP_WIN_HEIGHT 22
79 
80 enum tabs {
81 	THREADS_TAB,
82 	POLLERS_TAB,
83 	CORES_TAB,
84 	NUMBER_OF_TABS,
85 };
86 
87 enum column_threads_type {
88 	COL_THREADS_NAME,
89 	COL_THREADS_CORE,
90 	COL_THREADS_ACTIVE_POLLERS,
91 	COL_THREADS_TIMED_POLLERS,
92 	COL_THREADS_PAUSED_POLLERS,
93 	COL_THREADS_IDLE_TIME,
94 	COL_THREADS_BUSY_TIME,
95 	COL_THREADS_CPU_USAGE,
96 	COL_THREADS_NONE = 255,
97 };
98 
99 enum column_pollers_type {
100 	COL_POLLERS_NAME,
101 	COL_POLLERS_TYPE,
102 	COL_POLLERS_THREAD_NAME,
103 	COL_POLLERS_RUN_COUNTER,
104 	COL_POLLERS_PERIOD,
105 	COL_POLLERS_BUSY_COUNT,
106 	COL_POLLERS_NONE = 255,
107 };
108 
109 enum column_cores_type {
110 	COL_CORES_CORE,
111 	COL_CORES_THREADS,
112 	COL_CORES_POLLERS,
113 	COL_CORES_IDLE_TIME,
114 	COL_CORES_BUSY_TIME,
115 	COL_CORES_CORE_FREQ,
116 	COL_CORES_INTR,
117 	COL_CORES_CPU_USAGE,
118 	COL_CORES_NONE = 255,
119 };
120 
121 enum spdk_poller_type {
122 	SPDK_ACTIVE_POLLER,
123 	SPDK_TIMED_POLLER,
124 	SPDK_PAUSED_POLLER,
125 	SPDK_POLLER_TYPES_COUNT,
126 };
127 
128 struct col_desc {
129 	const char *name;
130 	uint8_t name_len;
131 	uint8_t max_data_string;
132 	bool disabled;
133 };
134 
135 struct run_counter_history {
136 	uint64_t poller_id;
137 	uint64_t thread_id;
138 	uint64_t last_run_counter;
139 	uint64_t last_busy_counter;
140 	TAILQ_ENTRY(run_counter_history) link;
141 };
142 
143 uint8_t g_sleep_time = 1;
144 uint16_t g_selected_row;
145 uint16_t g_max_selected_row;
146 uint64_t g_tick_rate;
147 const char *poller_type_str[SPDK_POLLER_TYPES_COUNT] = {"Active", "Timed", "Paused"};
148 const char *g_tab_title[NUMBER_OF_TABS] = {"[1] THREADS", "[2] POLLERS", "[3] CORES"};
149 struct spdk_jsonrpc_client *g_rpc_client;
150 static TAILQ_HEAD(, run_counter_history) g_run_counter_history = TAILQ_HEAD_INITIALIZER(
151 			g_run_counter_history);
152 WINDOW *g_menu_win, *g_tab_win[NUMBER_OF_TABS], *g_tabs[NUMBER_OF_TABS];
153 PANEL *g_panels[NUMBER_OF_TABS];
154 uint16_t g_max_row, g_max_col;
155 uint16_t g_data_win_size, g_max_data_rows;
156 uint32_t g_last_threads_count, g_last_pollers_count, g_last_cores_count;
157 uint8_t g_current_sort_col[NUMBER_OF_TABS] = {COL_THREADS_NAME, COL_POLLERS_NAME, COL_CORES_CORE};
158 uint8_t g_current_sort_col2[NUMBER_OF_TABS] = {COL_THREADS_NONE, COL_POLLERS_NONE, COL_CORES_NONE};
159 bool g_interval_data = true;
160 bool g_quit_app = false;
161 pthread_mutex_t g_thread_lock;
162 static struct col_desc g_col_desc[NUMBER_OF_TABS][TABS_COL_COUNT] = {
163 	{	{.name = "Thread name", .max_data_string = MAX_THREAD_NAME_LEN},
164 		{.name = "Core", .max_data_string = MAX_CORE_STR_LEN},
165 		{.name = "Active pollers", .max_data_string = MAX_POLLER_COUNT_STR_LEN},
166 		{.name = "Timed pollers", .max_data_string = MAX_POLLER_COUNT_STR_LEN},
167 		{.name = "Paused pollers", .max_data_string = MAX_POLLER_COUNT_STR_LEN},
168 		{.name = "Idle [us]", .max_data_string = MAX_TIME_STR_LEN},
169 		{.name = "Busy [us]", .max_data_string = MAX_TIME_STR_LEN},
170 		{.name = "CPU %", .max_data_string = MAX_CPU_STR_LEN},
171 		{.name = (char *)NULL}
172 	},
173 	{	{.name = "Poller name", .max_data_string = MAX_POLLER_NAME_LEN},
174 		{.name = "Type", .max_data_string = MAX_POLLER_TYPE_STR_LEN},
175 		{.name = "On thread", .max_data_string = MAX_THREAD_NAME_LEN},
176 		{.name = "Run count", .max_data_string = MAX_POLLER_RUN_COUNT},
177 		{.name = "Period [us]", .max_data_string = MAX_PERIOD_STR_LEN},
178 		{.name = "Status (busy count)", .max_data_string = MAX_POLLER_IND_STR_LEN},
179 		{.name = (char *)NULL}
180 	},
181 	{	{.name = "Core", .max_data_string = MAX_CORE_STR_LEN},
182 		{.name = "Thread count", .max_data_string = MAX_THREAD_COUNT_STR_LEN},
183 		{.name = "Poller count", .max_data_string = MAX_POLLER_COUNT_STR_LEN},
184 		{.name = "Idle [us]", .max_data_string = MAX_TIME_STR_LEN},
185 		{.name = "Busy [us]", .max_data_string = MAX_TIME_STR_LEN},
186 		{.name = "Frequency [MHz]", .max_data_string = MAX_CORE_FREQ_STR_LEN},
187 		{.name = "Intr", .max_data_string = MAX_INTR_LEN},
188 		{.name = "CPU %", .max_data_string = MAX_CPU_STR_LEN},
189 		{.name = (char *)NULL}
190 	}
191 };
192 
193 struct rpc_thread_info {
194 	char *name;
195 	uint64_t id;
196 	int core_num;
197 	char *cpumask;
198 	uint64_t busy;
199 	uint64_t last_busy;
200 	uint64_t idle;
201 	uint64_t last_idle;
202 	uint64_t active_pollers_count;
203 	uint64_t timed_pollers_count;
204 	uint64_t paused_pollers_count;
205 };
206 
207 struct rpc_poller_info {
208 	char *name;
209 	char *state;
210 	uint64_t id;
211 	uint64_t run_count;
212 	uint64_t busy_count;
213 	uint64_t period_ticks;
214 	enum spdk_poller_type type;
215 	char thread_name[MAX_THREAD_NAME];
216 	uint64_t thread_id;
217 };
218 
219 struct rpc_core_thread_info {
220 	char *name;
221 	uint64_t id;
222 	char *cpumask;
223 	uint64_t elapsed;
224 };
225 
226 struct rpc_core_threads {
227 	uint64_t threads_count;
228 	struct rpc_core_thread_info *thread;
229 };
230 
231 struct rpc_core_info {
232 	uint32_t lcore;
233 	uint64_t pollers_count;
234 	uint64_t busy;
235 	uint64_t idle;
236 	uint32_t core_freq;
237 	uint64_t last_idle;
238 	uint64_t last_busy;
239 	bool in_interrupt;
240 	struct rpc_core_threads threads;
241 };
242 
243 struct rpc_scheduler {
244 	char *scheduler_name;
245 	uint64_t scheduler_period;
246 };
247 
248 struct rpc_thread_info g_threads_info[RPC_MAX_THREADS];
249 struct rpc_poller_info g_pollers_info[RPC_MAX_POLLERS];
250 struct rpc_core_info g_cores_info[RPC_MAX_CORES];
251 struct rpc_scheduler g_scheduler_info;
252 
253 static void
254 init_str_len(void)
255 {
256 	int i, j;
257 
258 	for (i = 0; i < NUMBER_OF_TABS; i++) {
259 		for (j = 0; g_col_desc[i][j].name != NULL; j++) {
260 			g_col_desc[i][j].name_len = strlen(g_col_desc[i][j].name);
261 		}
262 	}
263 }
264 
265 static void
266 free_rpc_threads_stats(struct rpc_thread_info *req)
267 {
268 	free(req->name);
269 	req->name = NULL;
270 	free(req->cpumask);
271 	req->cpumask = NULL;
272 }
273 
274 static const struct spdk_json_object_decoder rpc_thread_info_decoders[] = {
275 	{"name", offsetof(struct rpc_thread_info, name), spdk_json_decode_string},
276 	{"id", offsetof(struct rpc_thread_info, id), spdk_json_decode_uint64},
277 	{"cpumask", offsetof(struct rpc_thread_info, cpumask), spdk_json_decode_string},
278 	{"busy", offsetof(struct rpc_thread_info, busy), spdk_json_decode_uint64},
279 	{"idle", offsetof(struct rpc_thread_info, idle), spdk_json_decode_uint64},
280 	{"active_pollers_count", offsetof(struct rpc_thread_info, active_pollers_count), spdk_json_decode_uint64},
281 	{"timed_pollers_count", offsetof(struct rpc_thread_info, timed_pollers_count), spdk_json_decode_uint64},
282 	{"paused_pollers_count", offsetof(struct rpc_thread_info, paused_pollers_count), spdk_json_decode_uint64},
283 };
284 
285 static int
286 rpc_decode_threads_array(struct spdk_json_val *val, struct rpc_thread_info *out,
287 			 uint64_t *current_threads_count)
288 {
289 	struct spdk_json_val *thread = val;
290 	uint64_t i = 0;
291 	int rc;
292 
293 	/* Fetch the beginning of threads array */
294 	rc = spdk_json_find_array(thread, "threads", NULL, &thread);
295 	if (rc) {
296 		printf("Could not fetch threads array from JSON.\n");
297 		goto end;
298 	}
299 
300 	for (thread = spdk_json_array_first(thread); thread != NULL; thread = spdk_json_next(thread)) {
301 		rc = spdk_json_decode_object(thread, rpc_thread_info_decoders,
302 					     SPDK_COUNTOF(rpc_thread_info_decoders), &out[i]);
303 		if (rc) {
304 			printf("Could not decode thread object from JSON.\n");
305 			break;
306 		}
307 
308 		i++;
309 	}
310 
311 end:
312 
313 	*current_threads_count = i;
314 	return rc;
315 }
316 
317 static void
318 free_rpc_poller(struct rpc_poller_info *poller)
319 {
320 	free(poller->name);
321 	poller->name = NULL;
322 	free(poller->state);
323 	poller->state = NULL;
324 }
325 
326 static void
327 free_rpc_core_info(struct rpc_core_info *core_info, size_t size)
328 {
329 	struct rpc_core_threads *threads;
330 	struct rpc_core_thread_info *thread;
331 	uint64_t i, core_number;
332 
333 	for (core_number = 0; core_number < size; core_number++) {
334 		threads = &core_info[core_number].threads;
335 		for (i = 0; i < threads->threads_count; i++) {
336 			thread = &threads->thread[i];
337 			free(thread->name);
338 			free(thread->cpumask);
339 		}
340 		free(threads->thread);
341 	}
342 }
343 
344 static const struct spdk_json_object_decoder rpc_pollers_decoders[] = {
345 	{"name", offsetof(struct rpc_poller_info, name), spdk_json_decode_string},
346 	{"state", offsetof(struct rpc_poller_info, state), spdk_json_decode_string},
347 	{"id", offsetof(struct rpc_poller_info, id), spdk_json_decode_uint64},
348 	{"run_count", offsetof(struct rpc_poller_info, run_count), spdk_json_decode_uint64},
349 	{"busy_count", offsetof(struct rpc_poller_info, busy_count), spdk_json_decode_uint64},
350 	{"period_ticks", offsetof(struct rpc_poller_info, period_ticks), spdk_json_decode_uint64, true},
351 };
352 
353 static int
354 rpc_decode_pollers_array(struct spdk_json_val *poller, struct rpc_poller_info *out,
355 			 uint64_t *poller_count,
356 			 const char *thread_name, uint64_t thread_name_length, uint64_t thread_id,
357 			 enum spdk_poller_type poller_type)
358 {
359 	int rc;
360 
361 	for (poller = spdk_json_array_first(poller); poller != NULL; poller = spdk_json_next(poller)) {
362 		out[*poller_count].thread_id = thread_id;
363 		memcpy(out[*poller_count].thread_name, thread_name, sizeof(char) * thread_name_length);
364 		out[*poller_count].type = poller_type;
365 
366 		rc = spdk_json_decode_object(poller, rpc_pollers_decoders,
367 					     SPDK_COUNTOF(rpc_pollers_decoders), &out[*poller_count]);
368 		if (rc) {
369 			printf("Could not decode poller object from JSON.\n");
370 			return rc;
371 		}
372 
373 		(*poller_count)++;
374 		if (*poller_count == RPC_MAX_POLLERS) {
375 			return -1;
376 		}
377 	}
378 
379 	return 0;
380 }
381 
382 static const struct spdk_json_object_decoder rpc_thread_pollers_decoders[] = {
383 	{"name", offsetof(struct rpc_thread_info, name), spdk_json_decode_string},
384 	{"id", offsetof(struct rpc_thread_info, id), spdk_json_decode_uint64},
385 };
386 
387 static int
388 rpc_decode_pollers_threads_array(struct spdk_json_val *val, struct rpc_poller_info *out,
389 				 uint32_t *num_pollers)
390 {
391 	struct spdk_json_val *thread = val, *poller;
392 	/* This is a temporary poller structure to hold thread name and id.
393 	 * It is filled with data only once per thread change and then
394 	 * that memory is copied to each poller running on that thread. */
395 	struct rpc_thread_info thread_info = {};
396 	uint64_t poller_count = 0, i, thread_name_length;
397 	int rc;
398 	const char *poller_typenames[] = { "active_pollers", "timed_pollers", "paused_pollers" };
399 	enum spdk_poller_type poller_types[] = { SPDK_ACTIVE_POLLER, SPDK_TIMED_POLLER, SPDK_PAUSED_POLLER };
400 
401 	/* Fetch the beginning of threads array */
402 	rc = spdk_json_find_array(thread, "threads", NULL, &thread);
403 	if (rc) {
404 		printf("Could not fetch threads array from JSON.\n");
405 		goto end;
406 	}
407 
408 	for (thread = spdk_json_array_first(thread); thread != NULL; thread = spdk_json_next(thread)) {
409 		rc = spdk_json_decode_object_relaxed(thread, rpc_thread_pollers_decoders,
410 						     SPDK_COUNTOF(rpc_thread_pollers_decoders), &thread_info);
411 		if (rc) {
412 			printf("Could not decode thread info from JSON.\n");
413 			goto end;
414 		}
415 
416 		thread_name_length = strlen(thread_info.name);
417 
418 		for (i = 0; i < SPDK_COUNTOF(poller_types); i++) {
419 			/* Find poller array */
420 			rc = spdk_json_find(thread, poller_typenames[i], NULL, &poller,
421 					    SPDK_JSON_VAL_ARRAY_BEGIN);
422 			if (rc) {
423 				printf("Could not fetch pollers array from JSON.\n");
424 				goto end;
425 			}
426 
427 			rc = rpc_decode_pollers_array(poller, out, &poller_count, thread_info.name,
428 						      thread_name_length,
429 						      thread_info.id, poller_types[i]);
430 			if (rc) {
431 				printf("Could not decode the first object in pollers array.\n");
432 				goto end;
433 			}
434 		}
435 	}
436 
437 	*num_pollers = poller_count;
438 
439 end:
440 	/* Since we rely in spdk_json_object_decode() to free this value
441 	 * each time we rewrite it, we need to free the last allocation
442 	 * manually. */
443 	free(thread_info.name);
444 
445 	if (rc) {
446 		*num_pollers = 0;
447 		for (i = 0; i < poller_count; i++) {
448 			free_rpc_poller(&out[i]);
449 		}
450 	}
451 
452 	return rc;
453 }
454 
455 static const struct spdk_json_object_decoder rpc_core_thread_info_decoders[] = {
456 	{"name", offsetof(struct rpc_core_thread_info, name), spdk_json_decode_string},
457 	{"id", offsetof(struct rpc_core_thread_info, id), spdk_json_decode_uint64},
458 	{"cpumask", offsetof(struct rpc_core_thread_info, cpumask), spdk_json_decode_string},
459 	{"elapsed", offsetof(struct rpc_core_thread_info, elapsed), spdk_json_decode_uint64},
460 };
461 
462 static int
463 rpc_decode_core_threads_object(const struct spdk_json_val *val, void *out)
464 {
465 	struct rpc_core_thread_info *info = out;
466 
467 	return spdk_json_decode_object(val, rpc_core_thread_info_decoders,
468 				       SPDK_COUNTOF(rpc_core_thread_info_decoders), info);
469 }
470 
471 #define RPC_THREAD_ENTRY_SIZE (SPDK_COUNTOF(rpc_core_thread_info_decoders) * 2)
472 
473 static int
474 rpc_decode_cores_lw_threads(const struct spdk_json_val *val, void *out)
475 {
476 	struct rpc_core_threads *threads = out;
477 	/* The number of thread entries received from RPC can be calculated using
478 	 * above define value (each JSON line = key + value, hence '* 2' ) and JSON
479 	 * 'val' value (-2 is to subtract VAL_OBJECT_BEGIN/END). */
480 	uint16_t threads_count = (spdk_json_val_len(val) - 2) / RPC_THREAD_ENTRY_SIZE;
481 
482 	threads->thread = calloc(threads_count, sizeof(struct rpc_core_thread_info));
483 	if (!out) {
484 		fprintf(stderr, "Unable to allocate memory for a thread array.\n");
485 		return -1;
486 	}
487 
488 	return spdk_json_decode_array(val, rpc_decode_core_threads_object, threads->thread, threads_count,
489 				      &threads->threads_count, sizeof(struct rpc_core_thread_info));
490 }
491 
492 static const struct spdk_json_object_decoder rpc_core_info_decoders[] = {
493 	{"lcore", offsetof(struct rpc_core_info, lcore), spdk_json_decode_uint32},
494 	{"busy", offsetof(struct rpc_core_info, busy), spdk_json_decode_uint64},
495 	{"idle", offsetof(struct rpc_core_info, idle), spdk_json_decode_uint64},
496 	{"core_freq", offsetof(struct rpc_core_info, core_freq), spdk_json_decode_uint32, true},
497 	{"in_interrupt", offsetof(struct rpc_core_info, in_interrupt), spdk_json_decode_bool},
498 	{"lw_threads", offsetof(struct rpc_core_info, threads), rpc_decode_cores_lw_threads},
499 };
500 
501 static int
502 rpc_decode_core_object(const struct spdk_json_val *val, void *out)
503 {
504 	struct rpc_core_info *info = out;
505 
506 	return spdk_json_decode_object(val, rpc_core_info_decoders,
507 				       SPDK_COUNTOF(rpc_core_info_decoders), info);
508 }
509 
510 static int
511 rpc_decode_cores_array(struct spdk_json_val *val, struct rpc_core_info *out,
512 		       uint32_t *current_cores_count)
513 {
514 	struct spdk_json_val *core = val;
515 	size_t cores_count;
516 	int rc;
517 
518 	/* Fetch the beginning of reactors array. */
519 	rc = spdk_json_find_array(core, "reactors", NULL, &core);
520 	if (rc) {
521 		printf("Could not fetch cores array from JSON.");
522 		goto end;
523 	}
524 
525 	rc = spdk_json_decode_array(core, rpc_decode_core_object, out, RPC_MAX_CORES, &cores_count,
526 				    sizeof(struct rpc_core_info));
527 
528 	*current_cores_count = (uint32_t)cores_count;
529 
530 end:
531 	return rc;
532 }
533 
534 static void
535 free_rpc_scheduler(struct rpc_scheduler *req)
536 {
537 	free(req->scheduler_name);
538 }
539 
540 static const struct spdk_json_object_decoder rpc_scheduler_decoders[] = {
541 	{"scheduler_name", offsetof(struct rpc_scheduler, scheduler_name), spdk_json_decode_string, true},
542 	{"scheduler_period", offsetof(struct rpc_scheduler, scheduler_period), spdk_json_decode_uint64},
543 };
544 
545 static int
546 rpc_send_req(char *rpc_name, struct spdk_jsonrpc_client_response **resp)
547 {
548 	struct spdk_jsonrpc_client_response *json_resp = NULL;
549 	struct spdk_json_write_ctx *w;
550 	struct spdk_jsonrpc_client_request *request;
551 	int rc;
552 
553 	request = spdk_jsonrpc_client_create_request();
554 	if (request == NULL) {
555 		return -ENOMEM;
556 	}
557 
558 	w = spdk_jsonrpc_begin_request(request, 1, rpc_name);
559 	spdk_jsonrpc_end_request(request, w);
560 	spdk_jsonrpc_client_send_request(g_rpc_client, request);
561 
562 	do {
563 		rc = spdk_jsonrpc_client_poll(g_rpc_client, 1);
564 	} while (rc == 0 || rc == -ENOTCONN);
565 
566 	if (rc <= 0) {
567 		return -1;
568 	}
569 
570 	json_resp = spdk_jsonrpc_client_get_response(g_rpc_client);
571 	if (json_resp == NULL) {
572 		return -1;
573 	}
574 
575 	/* Check for error response */
576 	if (json_resp->error != NULL) {
577 		spdk_jsonrpc_client_free_response(json_resp);
578 		return -1;
579 	}
580 
581 	assert(json_resp->result);
582 
583 	*resp = json_resp;
584 
585 	return 0;
586 }
587 
588 static uint64_t
589 get_cpu_usage(uint64_t busy_ticks, uint64_t idle_ticks)
590 {
591 	if (busy_ticks + idle_ticks > 0) {
592 		/* Increase numerator to convert fraction into decimal with
593 		 * additional precision */
594 		return busy_ticks * 10000 / (busy_ticks + idle_ticks);
595 	}
596 
597 	return 0;
598 }
599 
600 static int
601 subsort_threads(enum column_threads_type sort_column, const void *p1, const void *p2)
602 {
603 	const struct rpc_thread_info thread_info1 = *(struct rpc_thread_info *)p1;
604 	const struct rpc_thread_info thread_info2 = *(struct rpc_thread_info *)p2;
605 	uint64_t count1, count2;
606 
607 	switch (sort_column) {
608 	case COL_THREADS_NAME:
609 		return strcmp(thread_info1.name, thread_info2.name);
610 	case COL_THREADS_CORE:
611 		count2 = thread_info1.core_num;
612 		count1 = thread_info2.core_num;
613 		break;
614 	case COL_THREADS_ACTIVE_POLLERS:
615 		count1 = thread_info1.active_pollers_count;
616 		count2 = thread_info2.active_pollers_count;
617 		break;
618 	case COL_THREADS_TIMED_POLLERS:
619 		count1 = thread_info1.timed_pollers_count;
620 		count2 = thread_info2.timed_pollers_count;
621 		break;
622 	case COL_THREADS_PAUSED_POLLERS:
623 		count1 = thread_info1.paused_pollers_count;
624 		count2 = thread_info2.paused_pollers_count;
625 		break;
626 	case COL_THREADS_IDLE_TIME:
627 		if (g_interval_data) {
628 			count1 = thread_info1.idle - thread_info1.last_idle;
629 			count2 = thread_info2.idle - thread_info2.last_idle;
630 		} else {
631 			count1 = thread_info1.idle;
632 			count2 = thread_info2.idle;
633 		}
634 		break;
635 	case COL_THREADS_BUSY_TIME:
636 		if (g_interval_data) {
637 			count1 = thread_info1.busy - thread_info1.last_busy;
638 			count2 = thread_info2.busy - thread_info2.last_busy;
639 		} else {
640 			count1 = thread_info1.busy;
641 			count2 = thread_info2.busy;
642 		}
643 		break;
644 	case COL_THREADS_CPU_USAGE:
645 		count1 = get_cpu_usage(thread_info1.busy - thread_info1.last_busy,
646 				       g_cores_info[thread_info1.core_num].busy + g_cores_info[thread_info1.core_num].idle);
647 		count2 = get_cpu_usage(thread_info2.busy - thread_info2.last_busy,
648 				       g_cores_info[thread_info2.core_num].busy + g_cores_info[thread_info2.core_num].idle);
649 		break;
650 	case COL_THREADS_NONE:
651 	default:
652 		return 0;
653 	}
654 
655 	if (count2 > count1) {
656 		return 1;
657 	} else if (count2 < count1) {
658 		return -1;
659 	} else {
660 		return 0;
661 	}
662 }
663 
664 static int
665 sort_threads(const void *p1, const void *p2)
666 {
667 	int res;
668 
669 	res = subsort_threads(g_current_sort_col[THREADS_TAB], p1, p2);
670 	if (res == 0) {
671 		res = subsort_threads(g_current_sort_col2[THREADS_TAB], p1, p2);
672 	}
673 	return res;
674 }
675 
676 static void
677 store_last_counters(uint64_t poller_id, uint64_t thread_id, uint64_t last_run_counter,
678 		    uint64_t last_busy_counter)
679 {
680 	struct run_counter_history *history;
681 
682 	TAILQ_FOREACH(history, &g_run_counter_history, link) {
683 		if ((history->poller_id == poller_id) && (history->thread_id == thread_id)) {
684 			history->last_run_counter = last_run_counter;
685 			history->last_busy_counter = last_busy_counter;
686 			return;
687 		}
688 	}
689 
690 	history = calloc(1, sizeof(*history));
691 	if (history == NULL) {
692 		fprintf(stderr, "Unable to allocate a history object in store_last_counters.\n");
693 		return;
694 	}
695 	history->poller_id = poller_id;
696 	history->thread_id = thread_id;
697 	history->last_run_counter = last_run_counter;
698 	history->last_busy_counter = last_busy_counter;
699 
700 	TAILQ_INSERT_TAIL(&g_run_counter_history, history, link);
701 }
702 
703 static int
704 get_thread_data(void)
705 {
706 	struct spdk_jsonrpc_client_response *json_resp = NULL;
707 	struct rpc_thread_info thread_info[RPC_MAX_THREADS], *thread;
708 	struct rpc_core_info *core_info;
709 	uint64_t i, j, k, current_threads_count = 0;
710 	int rc = 0;
711 
712 	rc = rpc_send_req("thread_get_stats", &json_resp);
713 	if (rc) {
714 		return rc;
715 	}
716 
717 	/* Decode json */
718 	memset(thread_info, 0, sizeof(struct rpc_thread_info) * RPC_MAX_THREADS);
719 	if (rpc_decode_threads_array(json_resp->result, thread_info, &current_threads_count)) {
720 		rc = -EINVAL;
721 		for (i = 0; i < current_threads_count; i++) {
722 			free_rpc_threads_stats(&thread_info[i]);
723 		}
724 		goto end;
725 	}
726 
727 	pthread_mutex_lock(&g_thread_lock);
728 
729 	/* This is to free allocated char arrays with old thread names */
730 	for (i = 0; i < g_last_threads_count; i++) {
731 		free_rpc_threads_stats(&g_threads_info[i]);
732 	}
733 
734 	for (i = 0; i < current_threads_count; i++) {
735 		for (j = 0; j < g_last_threads_count; j++) {
736 			if (thread_info[i].id == g_threads_info[j].id) {
737 				thread_info[i].last_busy = g_threads_info[j].busy;
738 				thread_info[i].last_idle = g_threads_info[j].idle;
739 			}
740 		}
741 	}
742 	g_last_threads_count = current_threads_count;
743 
744 	memcpy(g_threads_info, thread_info, sizeof(struct rpc_thread_info) * RPC_MAX_THREADS);
745 
746 	for (i = 0; i < g_last_threads_count; i++) {
747 		g_threads_info[i].core_num = -1;
748 	}
749 
750 	for (i = 0; i < g_last_cores_count; i++) {
751 		core_info = &g_cores_info[i];
752 
753 		for (j = 0; j < core_info->threads.threads_count; j++) {
754 			for (k = 0; k < g_last_threads_count; k++) {
755 				/* For each thread on current core: check if it's ID also exists
756 				 * in g_thread_info data structure. If it does then assign current
757 				 * core's number to that thread, otherwise application state is inconsistent
758 				 * (e.g. scheduler is moving threads between cores). */
759 				thread = &g_threads_info[k];
760 				if (thread->id == core_info->threads.thread[j].id) {
761 					thread->core_num = core_info->lcore;
762 					break;
763 				}
764 			}
765 		}
766 	}
767 
768 	qsort(g_threads_info, g_last_threads_count, sizeof(struct rpc_thread_info), sort_threads);
769 
770 	pthread_mutex_unlock(&g_thread_lock);
771 
772 end:
773 	spdk_jsonrpc_client_free_response(json_resp);
774 	return rc;
775 }
776 
777 static uint64_t
778 get_last_run_counter(uint64_t poller_id, uint64_t thread_id)
779 {
780 	struct run_counter_history *history;
781 
782 	TAILQ_FOREACH(history, &g_run_counter_history, link) {
783 		if ((history->poller_id == poller_id) && (history->thread_id == thread_id)) {
784 			return history->last_run_counter;
785 		}
786 	}
787 
788 	return 0;
789 }
790 
791 static uint64_t
792 get_last_busy_counter(uint64_t poller_id, uint64_t thread_id)
793 {
794 	struct run_counter_history *history;
795 
796 	TAILQ_FOREACH(history, &g_run_counter_history, link) {
797 		if ((history->poller_id == poller_id) && (history->thread_id == thread_id)) {
798 			return history->last_busy_counter;
799 		}
800 	}
801 
802 	return 0;
803 }
804 
805 static int
806 subsort_pollers(enum column_pollers_type sort_column, const void *p1, const void *p2)
807 {
808 	const struct rpc_poller_info *poller1 = (struct rpc_poller_info *)p1;
809 	const struct rpc_poller_info *poller2 = (struct rpc_poller_info *)p2;
810 	uint64_t count1, count2;
811 	uint64_t last_busy_counter1, last_busy_counter2;
812 
813 	switch (sort_column) {
814 	case COL_POLLERS_NAME:
815 		return strcmp(poller1->name, poller2->name);
816 	case COL_POLLERS_TYPE:
817 		return poller1->type - poller2->type;
818 	case COL_POLLERS_THREAD_NAME:
819 		return strcmp(poller1->thread_name, poller2->thread_name);
820 	case COL_POLLERS_RUN_COUNTER:
821 		if (g_interval_data) {
822 			count1 = poller1->run_count - get_last_run_counter(poller1->id, poller1->thread_id);
823 			count2 = poller2->run_count - get_last_run_counter(poller2->id, poller2->thread_id);
824 		} else {
825 			count1 = poller1->run_count;
826 			count2 = poller2->run_count;
827 		}
828 		break;
829 	case COL_POLLERS_PERIOD:
830 		count1 = poller1->period_ticks;
831 		count2 = poller2->period_ticks;
832 		break;
833 	case COL_POLLERS_BUSY_COUNT:
834 		count1 = poller1->busy_count;
835 		count2 = poller2->busy_count;
836 		if (g_interval_data) {
837 			last_busy_counter1 = get_last_busy_counter(poller1->id, poller1->thread_id);
838 			last_busy_counter2 = get_last_busy_counter(poller2->id, poller2->thread_id);
839 			if (count1 > last_busy_counter1) {
840 				count1 -= last_busy_counter1;
841 			}
842 			if (count2 > last_busy_counter2) {
843 				count2 -= last_busy_counter2;
844 			}
845 		}
846 		break;
847 	case COL_POLLERS_NONE:
848 	default:
849 		return 0;
850 	}
851 
852 	if (count2 > count1) {
853 		return 1;
854 	} else if (count2 < count1) {
855 		return -1;
856 	} else {
857 		return 0;
858 	}
859 }
860 
861 static int
862 sort_pollers(const void *p1, const void *p2)
863 {
864 	int rc;
865 
866 	rc = subsort_pollers(g_current_sort_col[POLLERS_TAB], p1, p2);
867 	if (rc == 0) {
868 		rc = subsort_pollers(g_current_sort_col2[POLLERS_TAB], p1, p2);
869 	}
870 	return rc;
871 }
872 
873 static int
874 get_pollers_data(void)
875 {
876 	struct spdk_jsonrpc_client_response *json_resp = NULL;
877 	int rc = 0;
878 	uint64_t i = 0;
879 	uint32_t current_pollers_count;
880 	struct rpc_poller_info pollers_info[RPC_MAX_POLLERS];
881 
882 	rc = rpc_send_req("thread_get_pollers", &json_resp);
883 	if (rc) {
884 		return rc;
885 	}
886 
887 	/* Decode json */
888 	memset(&pollers_info, 0, sizeof(pollers_info));
889 	if (rpc_decode_pollers_threads_array(json_resp->result, pollers_info, &current_pollers_count)) {
890 		rc = -EINVAL;
891 		for (i = 0; i < current_pollers_count; i++) {
892 			free_rpc_poller(&pollers_info[i]);
893 		}
894 		goto end;
895 	}
896 
897 	pthread_mutex_lock(&g_thread_lock);
898 
899 	/* Save last run counter of each poller before updating g_pollers_stats. */
900 	for (i = 0; i < g_last_pollers_count; i++) {
901 		store_last_counters(g_pollers_info[i].id, g_pollers_info[i].thread_id,
902 				    g_pollers_info[i].run_count, g_pollers_info[i].busy_count);
903 	}
904 
905 	/* Free old pollers values before allocating memory for new ones */
906 	for (i = 0; i < g_last_pollers_count; i++) {
907 		free_rpc_poller(&g_pollers_info[i]);
908 	}
909 
910 	g_last_pollers_count = current_pollers_count;
911 
912 	qsort(&pollers_info, g_last_pollers_count, sizeof(struct rpc_poller_info), sort_pollers);
913 
914 	memcpy(&g_pollers_info, &pollers_info, sizeof(struct rpc_poller_info) * g_last_pollers_count);
915 
916 	pthread_mutex_unlock(&g_thread_lock);
917 
918 end:
919 	spdk_jsonrpc_client_free_response(json_resp);
920 	return rc;
921 }
922 
923 static int
924 subsort_cores(enum column_cores_type sort_column, const void *p1, const void *p2)
925 {
926 	const struct rpc_core_info core_info1 = *(struct rpc_core_info *)p1;
927 	const struct rpc_core_info core_info2 = *(struct rpc_core_info *)p2;
928 	uint64_t count1, count2;
929 
930 	switch (sort_column) {
931 	case COL_CORES_CORE:
932 		count1 = core_info2.lcore;
933 		count2 = core_info1.lcore;
934 		break;
935 	case COL_CORES_THREADS:
936 		count1 = core_info1.threads.threads_count;
937 		count2 = core_info2.threads.threads_count;
938 		break;
939 	case COL_CORES_POLLERS:
940 		count1 = core_info1.pollers_count;
941 		count2 = core_info2.pollers_count;
942 		break;
943 	case COL_CORES_IDLE_TIME:
944 		if (g_interval_data) {
945 			count1 = core_info1.last_idle - core_info1.idle;
946 			count2 = core_info2.last_idle - core_info2.idle;
947 		} else {
948 			count1 = core_info1.idle;
949 			count2 = core_info2.idle;
950 		}
951 		break;
952 	case COL_CORES_BUSY_TIME:
953 		if (g_interval_data) {
954 			count1 = core_info1.last_busy - core_info1.busy;
955 			count2 = core_info2.last_busy - core_info2.busy;
956 		} else {
957 			count1 = core_info1.busy;
958 			count2 = core_info2.busy;
959 		}
960 		break;
961 	case COL_CORES_CORE_FREQ:
962 		count1 = core_info1.core_freq;
963 		count2 = core_info2.core_freq;
964 		break;
965 	case COL_CORES_INTR:
966 		count1 = core_info1.in_interrupt;
967 		count2 = core_info2.in_interrupt;
968 		break;
969 	case COL_CORES_CPU_USAGE:
970 		count1 = get_cpu_usage(core_info1.last_busy - core_info1.busy,
971 				       core_info1.last_idle - core_info1.idle);
972 		count2 = get_cpu_usage(core_info2.last_busy - core_info2.busy,
973 				       core_info2.last_idle - core_info2.idle);
974 		break;
975 	case COL_CORES_NONE:
976 	default:
977 		return 0;
978 	}
979 
980 	if (count2 > count1) {
981 		return 1;
982 	} else if (count2 < count1) {
983 		return -1;
984 	} else {
985 		return 0;
986 	}
987 }
988 
989 static int
990 sort_cores(const void *p1, const void *p2)
991 {
992 	int rc;
993 
994 	rc = subsort_cores(g_current_sort_col[CORES_TAB], p1, p2);
995 	if (rc == 0) {
996 		return subsort_cores(g_current_sort_col[CORES_TAB], p1, p2);
997 	}
998 	return rc;
999 }
1000 
1001 static int
1002 get_cores_data(void)
1003 {
1004 	struct spdk_jsonrpc_client_response *json_resp = NULL;
1005 	struct rpc_core_info *core_info;
1006 	uint64_t i, j, k;
1007 	uint32_t current_cores_count;
1008 	struct rpc_core_info cores_info[RPC_MAX_CORES];
1009 	int rc = 0;
1010 
1011 	rc = rpc_send_req("framework_get_reactors", &json_resp);
1012 	if (rc) {
1013 		return rc;
1014 	}
1015 
1016 	/* Decode json */
1017 	memset(cores_info, 0, sizeof(struct rpc_core_info) * RPC_MAX_CORES);
1018 	if (rpc_decode_cores_array(json_resp->result, cores_info, &current_cores_count)) {
1019 		rc = -EINVAL;
1020 		goto end;
1021 	}
1022 
1023 	pthread_mutex_lock(&g_thread_lock);
1024 	for (i = 0; i < current_cores_count; i++) {
1025 		for (j = 0; j < g_last_cores_count; j++) {
1026 			if (cores_info[i].lcore == g_cores_info[j].lcore) {
1027 				cores_info[i].last_busy = g_cores_info[j].busy;
1028 				cores_info[i].last_idle = g_cores_info[j].idle;
1029 			}
1030 		}
1031 	}
1032 
1033 	/* Free old cores values before allocating memory for new ones */
1034 	free_rpc_core_info(g_cores_info, current_cores_count);
1035 	memcpy(g_cores_info, cores_info, sizeof(struct rpc_core_info) * current_cores_count);
1036 
1037 	for (i = 0; i < g_last_cores_count; i++) {
1038 		core_info = &g_cores_info[i];
1039 
1040 		core_info->threads.thread = cores_info[i].threads.thread;
1041 
1042 		for (j = 0; j < core_info->threads.threads_count; j++) {
1043 			memcpy(&core_info->threads.thread[j], &cores_info[i].threads.thread[j],
1044 			       sizeof(struct rpc_core_thread_info));
1045 			for (k = 0; k < g_last_threads_count; k++) {
1046 				if (core_info->threads.thread[j].id == g_threads_info[k].id) {
1047 					g_threads_info[k].core_num = core_info->lcore;
1048 					/* Do not consider threads which changed cores when issuing
1049 					 * RPCs to get_core_data and get_thread_data and threads
1050 					 * not currently assigned to this core. */
1051 					core_info->pollers_count += g_threads_info[k].active_pollers_count +
1052 								    g_threads_info[k].timed_pollers_count +
1053 								    g_threads_info[k].paused_pollers_count;
1054 				}
1055 			}
1056 		}
1057 	}
1058 
1059 	g_last_cores_count = current_cores_count;
1060 
1061 	qsort(&g_cores_info, g_last_cores_count, sizeof(struct rpc_core_info), sort_cores);
1062 
1063 end:
1064 	pthread_mutex_unlock(&g_thread_lock);
1065 	spdk_jsonrpc_client_free_response(json_resp);
1066 	return rc;
1067 }
1068 
1069 static int
1070 get_scheduler_data(void)
1071 {
1072 	struct spdk_jsonrpc_client_response *json_resp = NULL;
1073 	struct rpc_scheduler scheduler_info;
1074 	int rc = 0;
1075 
1076 	rc = rpc_send_req("framework_get_scheduler", &json_resp);
1077 	if (rc) {
1078 		return rc;
1079 	}
1080 
1081 	memset(&scheduler_info, 0, sizeof(scheduler_info));
1082 	if (spdk_json_decode_object_relaxed(json_resp->result, rpc_scheduler_decoders,
1083 					    SPDK_COUNTOF(rpc_scheduler_decoders), &scheduler_info)) {
1084 		rc = -EINVAL;
1085 	} else {
1086 		pthread_mutex_lock(&g_thread_lock);
1087 
1088 		free_rpc_scheduler(&g_scheduler_info);
1089 
1090 		memcpy(&g_scheduler_info, &scheduler_info, sizeof(struct rpc_scheduler));
1091 		pthread_mutex_unlock(&g_thread_lock);
1092 	}
1093 
1094 	spdk_jsonrpc_client_free_response(json_resp);
1095 	return rc;
1096 }
1097 
1098 enum str_alignment {
1099 	ALIGN_LEFT,
1100 	ALIGN_RIGHT,
1101 };
1102 
1103 static void
1104 print_max_len(WINDOW *win, int row, uint16_t col, uint16_t max_len, enum str_alignment alignment,
1105 	      const char *string)
1106 {
1107 	const char dots[] = "...";
1108 	int DOTS_STR_LEN = sizeof(dots) / sizeof(dots[0]);
1109 	char tmp_str[MAX_STRING_LEN];
1110 	int len, max_col, max_str, cmp_len;
1111 	int max_row;
1112 
1113 	len = strlen(string);
1114 	getmaxyx(win, max_row, max_col);
1115 
1116 	if (row > max_row) {
1117 		/* We are in a process of resizing and this may happen */
1118 		return;
1119 	}
1120 
1121 	if (max_len != 0 && col + max_len < max_col) {
1122 		max_col = col + max_len;
1123 	}
1124 
1125 	max_str = max_col - col;
1126 
1127 	if (max_str <= DOTS_STR_LEN + 1) {
1128 		/* No space to print anything, but we have to let a user know about it */
1129 		mvwprintw(win, row, max_col - DOTS_STR_LEN - 1, "...");
1130 		wnoutrefresh(win);
1131 		return;
1132 	}
1133 
1134 	if (max_len) {
1135 		if (alignment == ALIGN_LEFT) {
1136 			snprintf(tmp_str, max_str, "%s%*c", string, max_len - len - 1, ' ');
1137 		} else {
1138 			snprintf(tmp_str, max_str, "%*c%s", max_len - len - 1, ' ', string);
1139 		}
1140 		cmp_len = max_len - 1;
1141 	} else {
1142 		snprintf(tmp_str, max_str, "%s", string);
1143 		cmp_len = len;
1144 	}
1145 
1146 	if (col + cmp_len > max_col - 1) {
1147 		snprintf(&tmp_str[max_str - DOTS_STR_LEN - 2], DOTS_STR_LEN, "%s", dots);
1148 	}
1149 
1150 	mvwprintw(win, row, col, "%s", tmp_str);
1151 
1152 	wnoutrefresh(win);
1153 }
1154 
1155 static void
1156 draw_menu_win(void)
1157 {
1158 	wbkgd(g_menu_win, COLOR_PAIR(2));
1159 	box(g_menu_win, 0, 0);
1160 	print_max_len(g_menu_win, 1, 1, 0, ALIGN_LEFT,
1161 		      "  [q] Quit  |  [1-3][Tab] Switch tab  |  [PgUp] Previous page  |  [PgDown] Next page  |  [Enter] Item details  |  [h] Help");
1162 }
1163 
1164 static void
1165 draw_tab_win(enum tabs tab)
1166 {
1167 	uint16_t col;
1168 	uint8_t white_spaces = TABS_SPACING * NUMBER_OF_TABS;
1169 
1170 	wbkgd(g_tab_win[tab], COLOR_PAIR(2));
1171 	box(g_tab_win[tab], 0, 0);
1172 
1173 	col = ((g_max_col - white_spaces) / NUMBER_OF_TABS / 2) - (strlen(g_tab_title[tab]) / 2) -
1174 	      TABS_SPACING;
1175 	print_max_len(g_tab_win[tab], 1, col, 0, ALIGN_LEFT, g_tab_title[tab]);
1176 }
1177 
1178 static void
1179 draw_tabs(enum tabs tab_index, uint8_t sort_col, uint8_t sort_col2)
1180 {
1181 	struct col_desc *col_desc = g_col_desc[tab_index];
1182 	WINDOW *tab = g_tabs[tab_index];
1183 	int i, j;
1184 	uint16_t offset, draw_offset;
1185 	uint16_t tab_height = g_max_row - MENU_WIN_HEIGHT - TAB_WIN_HEIGHT - 3;
1186 
1187 	for (i = 0; col_desc[i].name != NULL; i++) {
1188 		if (col_desc[i].disabled) {
1189 			continue;
1190 		}
1191 
1192 		offset = 1;
1193 		for (j = i; j != 0; j--) {
1194 			if (!col_desc[j - 1].disabled) {
1195 				offset += col_desc[j - 1].max_data_string;
1196 				offset += col_desc[j - 1].name_len % 2 + 1;
1197 			}
1198 		}
1199 
1200 		draw_offset = offset + (col_desc[i].max_data_string / 2) - (col_desc[i].name_len / 2);
1201 
1202 		if (i == sort_col) {
1203 			wattron(tab, COLOR_PAIR(4));
1204 			print_max_len(tab, 1, draw_offset, 0, ALIGN_LEFT, col_desc[i].name);
1205 			wattroff(tab, COLOR_PAIR(4));
1206 		} else if (i == sort_col2) {
1207 			wattron(tab, COLOR_PAIR(3));
1208 			print_max_len(tab, 1, draw_offset, 0, ALIGN_LEFT, col_desc[i].name);
1209 			wattroff(tab, COLOR_PAIR(3));
1210 		} else {
1211 			print_max_len(tab, 1, draw_offset, 0, ALIGN_LEFT, col_desc[i].name);
1212 		}
1213 
1214 		if (offset != 1) {
1215 			print_max_len(tab, 1, offset - 1, 0, ALIGN_LEFT, "|");
1216 		}
1217 	}
1218 
1219 	print_max_len(tab, 2, 1, 0, ALIGN_LEFT, ""); /* Move to next line */
1220 	whline(tab, ACS_HLINE, g_max_col - 2);
1221 
1222 	/* Border lines */
1223 	mvwhline(tab, 0, 1, ACS_HLINE, g_max_col - 2);
1224 	mvwhline(tab, tab_height, 1, ACS_HLINE, g_max_col - 2);
1225 
1226 	wrefresh(tab);
1227 }
1228 
1229 static void
1230 resize_interface(enum tabs tab)
1231 {
1232 	int i;
1233 
1234 	clear();
1235 	wclear(g_menu_win);
1236 	mvwin(g_menu_win, g_max_row - MENU_WIN_SPACING, MENU_WIN_LOCATION_COL);
1237 	wresize(g_menu_win, MENU_WIN_HEIGHT, g_max_col);
1238 	draw_menu_win();
1239 
1240 	for (i = 0; i < NUMBER_OF_TABS; i++) {
1241 		wclear(g_tabs[i]);
1242 		wresize(g_tabs[i], g_max_row - MENU_WIN_HEIGHT - TAB_WIN_HEIGHT - 2, g_max_col);
1243 		mvwin(g_tabs[i], TABS_LOCATION_ROW, TABS_LOCATION_COL);
1244 	}
1245 
1246 	draw_tabs(tab, g_current_sort_col[tab], g_current_sort_col2[tab]);
1247 
1248 	for (i = 0; i < NUMBER_OF_TABS; i++) {
1249 		wclear(g_tab_win[i]);
1250 		wresize(g_tab_win[i], TAB_WIN_HEIGHT,
1251 			(g_max_col - (TABS_SPACING * NUMBER_OF_TABS)) / NUMBER_OF_TABS);
1252 		mvwin(g_tab_win[i], TAB_WIN_LOCATION_ROW, 1 + (g_max_col / NUMBER_OF_TABS) * i);
1253 		draw_tab_win(i);
1254 	}
1255 
1256 	update_panels();
1257 	doupdate();
1258 }
1259 
1260 static void
1261 switch_tab(enum tabs tab)
1262 {
1263 	wclear(g_tabs[tab]);
1264 	draw_tabs(tab, g_current_sort_col[tab], g_current_sort_col2[tab]);
1265 	top_panel(g_panels[tab]);
1266 	update_panels();
1267 	doupdate();
1268 }
1269 
1270 static void
1271 get_time_str(uint64_t ticks, char *time_str)
1272 {
1273 	uint64_t time;
1274 
1275 	time = ticks * SPDK_SEC_TO_USEC / g_tick_rate;
1276 	snprintf(time_str, MAX_TIME_STR_LEN, "%" PRIu64, time);
1277 }
1278 
1279 static void
1280 draw_row_background(uint8_t item_index, uint8_t tab)
1281 {
1282 	int k;
1283 
1284 	if (item_index == g_selected_row) {
1285 		wattron(g_tabs[tab], COLOR_PAIR(2));
1286 	}
1287 	for (k = 1; k < g_max_col - 1; k++) {
1288 		mvwprintw(g_tabs[tab], TABS_DATA_START_ROW + item_index, k, " ");
1289 	}
1290 }
1291 
1292 static void
1293 get_cpu_usage_str(uint64_t busy_ticks, uint64_t total_ticks, char *cpu_str)
1294 {
1295 	if (total_ticks > 0) {
1296 		snprintf(cpu_str, MAX_CPU_STR_LEN, "%.2f",
1297 			 (double)(busy_ticks) * 100 / (double)(total_ticks));
1298 	} else {
1299 		cpu_str[0] = '\0';
1300 	}
1301 }
1302 
1303 static void
1304 draw_thread_tab_row(uint64_t current_row, uint8_t item_index)
1305 {
1306 	struct col_desc *col_desc = g_col_desc[THREADS_TAB];
1307 	uint16_t col = TABS_DATA_START_COL;
1308 	int core_num;
1309 	char pollers_number[MAX_POLLER_COUNT_STR_LEN], idle_time[MAX_TIME_STR_LEN],
1310 	     busy_time[MAX_TIME_STR_LEN], core_str[MAX_CORE_MASK_STR_LEN],
1311 	     cpu_usage[MAX_CPU_STR_LEN];
1312 
1313 	if (!col_desc[COL_THREADS_NAME].disabled) {
1314 		print_max_len(g_tabs[THREADS_TAB], TABS_DATA_START_ROW + item_index, col,
1315 			      col_desc[COL_THREADS_NAME].max_data_string, ALIGN_LEFT, g_threads_info[current_row].name);
1316 		col += col_desc[COL_THREADS_NAME].max_data_string;
1317 	}
1318 
1319 	if (!col_desc[COL_THREADS_CORE].disabled) {
1320 		snprintf(core_str, MAX_CORE_STR_LEN, "%d", g_threads_info[current_row].core_num);
1321 		print_max_len(g_tabs[THREADS_TAB], TABS_DATA_START_ROW + item_index,
1322 			      col, col_desc[COL_THREADS_CORE].max_data_string, ALIGN_RIGHT, core_str);
1323 		col += col_desc[COL_THREADS_CORE].max_data_string + 2;
1324 	}
1325 
1326 	if (!col_desc[COL_THREADS_ACTIVE_POLLERS].disabled) {
1327 		snprintf(pollers_number, MAX_POLLER_COUNT_STR_LEN, "%ld",
1328 			 g_threads_info[current_row].active_pollers_count);
1329 		print_max_len(g_tabs[THREADS_TAB], TABS_DATA_START_ROW + item_index,
1330 			      col + (col_desc[COL_THREADS_ACTIVE_POLLERS].name_len / 2),
1331 			      col_desc[COL_THREADS_ACTIVE_POLLERS].max_data_string, ALIGN_LEFT, pollers_number);
1332 		col += col_desc[COL_THREADS_ACTIVE_POLLERS].max_data_string + 2;
1333 	}
1334 
1335 	if (!col_desc[COL_THREADS_TIMED_POLLERS].disabled) {
1336 		snprintf(pollers_number, MAX_POLLER_COUNT_STR_LEN, "%ld",
1337 			 g_threads_info[current_row].timed_pollers_count);
1338 		print_max_len(g_tabs[THREADS_TAB], TABS_DATA_START_ROW + item_index,
1339 			      col + (col_desc[COL_THREADS_TIMED_POLLERS].name_len / 2),
1340 			      col_desc[COL_THREADS_TIMED_POLLERS].max_data_string, ALIGN_LEFT, pollers_number);
1341 		col += col_desc[COL_THREADS_TIMED_POLLERS].max_data_string + 1;
1342 	}
1343 
1344 	if (!col_desc[COL_THREADS_PAUSED_POLLERS].disabled) {
1345 		snprintf(pollers_number, MAX_POLLER_COUNT_STR_LEN, "%ld",
1346 			 g_threads_info[current_row].paused_pollers_count);
1347 		print_max_len(g_tabs[THREADS_TAB], TABS_DATA_START_ROW + item_index,
1348 			      col + (col_desc[COL_THREADS_PAUSED_POLLERS].name_len / 2),
1349 			      col_desc[COL_THREADS_PAUSED_POLLERS].max_data_string, ALIGN_LEFT, pollers_number);
1350 		col += col_desc[COL_THREADS_PAUSED_POLLERS].max_data_string + 2;
1351 	}
1352 
1353 	if (!col_desc[COL_THREADS_IDLE_TIME].disabled) {
1354 		if (g_interval_data == true) {
1355 			get_time_str(g_threads_info[current_row].idle - g_threads_info[current_row].last_idle, idle_time);
1356 		} else {
1357 			get_time_str(g_threads_info[current_row].idle, idle_time);
1358 		}
1359 		print_max_len(g_tabs[THREADS_TAB], TABS_DATA_START_ROW + item_index, col,
1360 			      col_desc[COL_THREADS_IDLE_TIME].max_data_string, ALIGN_RIGHT, idle_time);
1361 		col += col_desc[COL_THREADS_IDLE_TIME].max_data_string;
1362 	}
1363 
1364 	if (!col_desc[COL_THREADS_BUSY_TIME].disabled) {
1365 		if (g_interval_data == true) {
1366 			get_time_str(g_threads_info[current_row].busy - g_threads_info[current_row].last_busy, busy_time);
1367 		} else {
1368 			get_time_str(g_threads_info[current_row].busy, busy_time);
1369 		}
1370 		print_max_len(g_tabs[THREADS_TAB], TABS_DATA_START_ROW + item_index, col,
1371 			      col_desc[COL_THREADS_BUSY_TIME].max_data_string, ALIGN_RIGHT, busy_time);
1372 		col += col_desc[COL_THREADS_BUSY_TIME].max_data_string + 3;
1373 	}
1374 
1375 	if (!col_desc[COL_THREADS_CPU_USAGE].disabled) {
1376 		core_num = g_threads_info[current_row].core_num;
1377 		if (core_num >= 0 && core_num < RPC_MAX_CORES) {
1378 			get_cpu_usage_str(g_threads_info[current_row].busy - g_threads_info[current_row].last_busy,
1379 					  (g_cores_info[core_num].busy - g_cores_info[core_num].last_busy) +
1380 					  (g_cores_info[core_num].idle - g_cores_info[core_num].last_idle),
1381 					  cpu_usage);
1382 		} else {
1383 			snprintf(cpu_usage, sizeof(cpu_usage), "n/a");
1384 		}
1385 
1386 		print_max_len(g_tabs[THREADS_TAB], TABS_DATA_START_ROW + item_index, col,
1387 			      col_desc[COL_THREADS_CPU_USAGE].max_data_string, ALIGN_RIGHT, cpu_usage);
1388 	}
1389 }
1390 
1391 static uint8_t
1392 refresh_threads_tab(uint8_t current_page)
1393 {
1394 	uint64_t i, j, threads_count;
1395 	uint16_t empty_col = 0;
1396 	uint8_t max_pages, item_index;
1397 
1398 	threads_count = g_last_threads_count;
1399 
1400 	max_pages = (threads_count + g_max_data_rows - 1) / g_max_data_rows;
1401 
1402 	for (i = current_page * g_max_data_rows;
1403 	     i < (uint64_t)((current_page + 1) * g_max_data_rows);
1404 	     i++) {
1405 		item_index = i - (current_page * g_max_data_rows);
1406 
1407 		/* When number of threads decreases, this will print spaces in places
1408 		 * where non existent threads were previously displayed. */
1409 		if (i >= threads_count) {
1410 			for (j = 1; j < (uint64_t)g_max_col - 1; j++) {
1411 				mvwprintw(g_tabs[THREADS_TAB], item_index + TABS_DATA_START_ROW, j, " ");
1412 			}
1413 
1414 			empty_col++;
1415 			continue;
1416 		}
1417 
1418 		draw_row_background(item_index, THREADS_TAB);
1419 		draw_thread_tab_row(i, item_index);
1420 
1421 		if (item_index == g_selected_row) {
1422 			wattroff(g_tabs[THREADS_TAB], COLOR_PAIR(2));
1423 		}
1424 	}
1425 
1426 	g_max_selected_row = i - current_page * g_max_data_rows - 1 - empty_col;
1427 
1428 	return max_pages;
1429 }
1430 
1431 static void
1432 draw_poller_tab_row(uint64_t current_row, uint8_t item_index)
1433 {
1434 	struct col_desc *col_desc = g_col_desc[POLLERS_TAB];
1435 	uint64_t last_run_counter, last_busy_counter;
1436 	uint16_t col = TABS_DATA_START_COL;
1437 	char run_count[MAX_POLLER_RUN_COUNT], period_ticks[MAX_PERIOD_STR_LEN],
1438 	     status[MAX_POLLER_IND_STR_LEN];
1439 
1440 	last_busy_counter = get_last_busy_counter(g_pollers_info[current_row].id,
1441 			    g_pollers_info[current_row].thread_id);
1442 
1443 	if (!col_desc[COL_POLLERS_NAME].disabled) {
1444 		print_max_len(g_tabs[POLLERS_TAB], TABS_DATA_START_ROW + item_index, col + 1,
1445 			      col_desc[COL_POLLERS_NAME].max_data_string, ALIGN_LEFT, g_pollers_info[current_row].name);
1446 		col += col_desc[COL_POLLERS_NAME].max_data_string + 2;
1447 	}
1448 
1449 	if (!col_desc[COL_POLLERS_TYPE].disabled) {
1450 		print_max_len(g_tabs[POLLERS_TAB], TABS_DATA_START_ROW + item_index, col,
1451 			      col_desc[COL_POLLERS_TYPE].max_data_string, ALIGN_LEFT,
1452 			      poller_type_str[g_pollers_info[current_row].type]);
1453 		col += col_desc[COL_POLLERS_TYPE].max_data_string + 2;
1454 	}
1455 
1456 	if (!col_desc[COL_POLLERS_THREAD_NAME].disabled) {
1457 		print_max_len(g_tabs[POLLERS_TAB], TABS_DATA_START_ROW + item_index, col,
1458 			      col_desc[COL_POLLERS_THREAD_NAME].max_data_string, ALIGN_LEFT,
1459 			      g_pollers_info[current_row].thread_name);
1460 		col += col_desc[COL_POLLERS_THREAD_NAME].max_data_string + 1;
1461 	}
1462 
1463 	if (!col_desc[COL_POLLERS_RUN_COUNTER].disabled) {
1464 		last_run_counter = get_last_run_counter(g_pollers_info[current_row].id,
1465 							g_pollers_info[current_row].thread_id);
1466 		if (g_interval_data == true) {
1467 			snprintf(run_count, MAX_POLLER_RUN_COUNT, "%" PRIu64,
1468 				 g_pollers_info[current_row].run_count - last_run_counter);
1469 		} else {
1470 			snprintf(run_count, MAX_POLLER_RUN_COUNT, "%" PRIu64, g_pollers_info[current_row].run_count);
1471 		}
1472 		print_max_len(g_tabs[POLLERS_TAB], TABS_DATA_START_ROW + item_index, col,
1473 			      col_desc[COL_POLLERS_RUN_COUNTER].max_data_string, ALIGN_RIGHT, run_count);
1474 		col += col_desc[COL_POLLERS_RUN_COUNTER].max_data_string;
1475 	}
1476 
1477 	if (!col_desc[COL_POLLERS_PERIOD].disabled) {
1478 		if (g_pollers_info[current_row].period_ticks != 0) {
1479 			get_time_str(g_pollers_info[current_row].period_ticks, period_ticks);
1480 			print_max_len(g_tabs[POLLERS_TAB], TABS_DATA_START_ROW + item_index, col,
1481 				      col_desc[COL_POLLERS_PERIOD].max_data_string, ALIGN_RIGHT, period_ticks);
1482 		}
1483 		col += col_desc[COL_POLLERS_PERIOD].max_data_string + 7;
1484 	}
1485 
1486 	if (!col_desc[COL_POLLERS_BUSY_COUNT].disabled) {
1487 		if (g_pollers_info[current_row].busy_count > last_busy_counter) {
1488 			if (g_interval_data == true) {
1489 				snprintf(status, MAX_POLLER_IND_STR_LEN, "Busy (%" PRIu64 ")",
1490 					 g_pollers_info[current_row].busy_count - last_busy_counter);
1491 			} else {
1492 				snprintf(status, MAX_POLLER_IND_STR_LEN, "Busy (%" PRIu64 ")",
1493 					 g_pollers_info[current_row].busy_count);
1494 			}
1495 
1496 			if (item_index != g_selected_row) {
1497 				wattron(g_tabs[POLLERS_TAB], COLOR_PAIR(6));
1498 				print_max_len(g_tabs[POLLERS_TAB], TABS_DATA_START_ROW + item_index, col,
1499 					      col_desc[COL_POLLERS_BUSY_COUNT].max_data_string, ALIGN_LEFT, status);
1500 				wattroff(g_tabs[POLLERS_TAB], COLOR_PAIR(6));
1501 			} else {
1502 				wattron(g_tabs[POLLERS_TAB], COLOR_PAIR(8));
1503 				print_max_len(g_tabs[POLLERS_TAB], TABS_DATA_START_ROW + item_index, col,
1504 					      col_desc[COL_POLLERS_BUSY_COUNT].max_data_string, ALIGN_LEFT, status);
1505 				wattroff(g_tabs[POLLERS_TAB], COLOR_PAIR(8));
1506 			}
1507 		} else {
1508 			if (g_interval_data == true) {
1509 				snprintf(status, MAX_POLLER_IND_STR_LEN, "%s", "Idle");
1510 			} else {
1511 				snprintf(status, MAX_POLLER_IND_STR_LEN, "Idle (%" PRIu64 ")",
1512 					 g_pollers_info[current_row].busy_count);
1513 			}
1514 
1515 			if (item_index != g_selected_row) {
1516 				wattron(g_tabs[POLLERS_TAB], COLOR_PAIR(7));
1517 				print_max_len(g_tabs[POLLERS_TAB], TABS_DATA_START_ROW + item_index, col,
1518 					      col_desc[COL_POLLERS_BUSY_COUNT].max_data_string, ALIGN_LEFT, status);
1519 				wattroff(g_tabs[POLLERS_TAB], COLOR_PAIR(7));
1520 			} else {
1521 				wattron(g_tabs[POLLERS_TAB], COLOR_PAIR(9));
1522 				print_max_len(g_tabs[POLLERS_TAB], TABS_DATA_START_ROW + item_index, col,
1523 					      col_desc[COL_POLLERS_BUSY_COUNT].max_data_string, ALIGN_LEFT, status);
1524 				wattroff(g_tabs[POLLERS_TAB], COLOR_PAIR(9));
1525 			}
1526 		}
1527 	}
1528 }
1529 
1530 static uint8_t
1531 refresh_pollers_tab(uint8_t current_page)
1532 {
1533 	uint64_t i, j;
1534 	uint16_t empty_col = 0;
1535 	uint8_t max_pages, item_index;
1536 
1537 	max_pages = (g_last_pollers_count + g_max_data_rows - 1) / g_max_data_rows;
1538 
1539 	/* Display info */
1540 	for (i = current_page * g_max_data_rows;
1541 	     i < (uint64_t)((current_page + 1) * g_max_data_rows);
1542 	     i++) {
1543 		item_index = i - (current_page * g_max_data_rows);
1544 
1545 		/* When number of pollers decreases, this will print spaces in places
1546 		 * where non existent pollers were previously displayed. */
1547 		if (i >= g_last_pollers_count) {
1548 			for (j = 1; j < (uint64_t)g_max_col - 1; j++) {
1549 				mvwprintw(g_tabs[POLLERS_TAB], item_index + TABS_DATA_START_ROW, j, " ");
1550 			}
1551 
1552 			empty_col++;
1553 			continue;
1554 		}
1555 
1556 		draw_row_background(item_index, POLLERS_TAB);
1557 		draw_poller_tab_row(i, item_index);
1558 
1559 		if (item_index == g_selected_row) {
1560 			wattroff(g_tabs[POLLERS_TAB], COLOR_PAIR(2));
1561 		}
1562 	}
1563 
1564 	g_max_selected_row = i - current_page * g_max_data_rows - 1 - empty_col;
1565 
1566 	return max_pages;
1567 }
1568 
1569 static void
1570 draw_core_tab_row(uint64_t current_row, uint8_t item_index)
1571 {
1572 	struct col_desc *col_desc = g_col_desc[CORES_TAB];
1573 	uint16_t col = 1;
1574 	char core[MAX_CORE_STR_LEN], threads_number[MAX_THREAD_COUNT_STR_LEN],  cpu_usage[MAX_CPU_STR_LEN],
1575 	     pollers_number[MAX_POLLER_COUNT_STR_LEN], idle_time[MAX_TIME_STR_LEN],
1576 	     busy_time[MAX_TIME_STR_LEN], core_freq[MAX_CORE_FREQ_STR_LEN],
1577 	     in_interrupt[MAX_INTR_LEN];
1578 
1579 	snprintf(threads_number, MAX_THREAD_COUNT_STR_LEN, "%ld",
1580 		 g_cores_info[current_row].threads.threads_count);
1581 	snprintf(pollers_number, MAX_POLLER_COUNT_STR_LEN, "%ld", g_cores_info[current_row].pollers_count);
1582 
1583 	if (!col_desc[COL_CORES_CORE].disabled) {
1584 		snprintf(core, MAX_CORE_STR_LEN, "%d", g_cores_info[current_row].lcore);
1585 		print_max_len(g_tabs[CORES_TAB], TABS_DATA_START_ROW + item_index, col,
1586 			      col_desc[COL_CORES_CORE].max_data_string, ALIGN_RIGHT, core);
1587 		col += col_desc[COL_CORES_CORE].max_data_string + 2;
1588 	}
1589 
1590 	if (!col_desc[COL_CORES_THREADS].disabled) {
1591 		print_max_len(g_tabs[CORES_TAB], TABS_DATA_START_ROW + item_index,
1592 			      col + (col_desc[COL_CORES_THREADS].name_len / 2), col_desc[COL_CORES_THREADS].max_data_string,
1593 			      ALIGN_LEFT, threads_number);
1594 		col += col_desc[COL_CORES_THREADS].max_data_string + 2;
1595 	}
1596 
1597 	if (!col_desc[COL_CORES_POLLERS].disabled) {
1598 		print_max_len(g_tabs[CORES_TAB], TABS_DATA_START_ROW + item_index,
1599 			      col + (col_desc[COL_CORES_POLLERS].name_len / 2), col_desc[COL_CORES_POLLERS].max_data_string,
1600 			      ALIGN_LEFT, pollers_number);
1601 		col += col_desc[COL_CORES_POLLERS].max_data_string;
1602 	}
1603 
1604 	if (!col_desc[COL_CORES_IDLE_TIME].disabled) {
1605 		if (g_interval_data == true) {
1606 			get_time_str(g_cores_info[current_row].idle - g_cores_info[current_row].last_idle, idle_time);
1607 		} else {
1608 			get_time_str(g_cores_info[current_row].idle, idle_time);
1609 		}
1610 		print_max_len(g_tabs[CORES_TAB], TABS_DATA_START_ROW + item_index, col,
1611 			      col_desc[COL_CORES_IDLE_TIME].max_data_string, ALIGN_RIGHT, idle_time);
1612 		col += col_desc[COL_CORES_IDLE_TIME].max_data_string + 2;
1613 	}
1614 
1615 	if (!col_desc[COL_CORES_BUSY_TIME].disabled) {
1616 		if (g_interval_data == true) {
1617 			get_time_str(g_cores_info[current_row].busy - g_cores_info[current_row].last_busy, busy_time);
1618 		} else {
1619 			get_time_str(g_cores_info[current_row].busy, busy_time);
1620 		}
1621 		print_max_len(g_tabs[CORES_TAB], TABS_DATA_START_ROW + item_index, col,
1622 			      col_desc[COL_CORES_BUSY_TIME].max_data_string, ALIGN_RIGHT, busy_time);
1623 		col += col_desc[COL_CORES_BUSY_TIME].max_data_string + 2;
1624 	}
1625 
1626 	if (!col_desc[COL_CORES_CORE_FREQ].disabled) {
1627 		if (!g_cores_info[current_row].core_freq) {
1628 			snprintf(core_freq, MAX_CORE_FREQ_STR_LEN, "%s", "N/A");
1629 		} else {
1630 			snprintf(core_freq, MAX_CORE_FREQ_STR_LEN, "%" PRIu32,
1631 				 g_cores_info[current_row].core_freq);
1632 		}
1633 		print_max_len(g_tabs[CORES_TAB], TABS_DATA_START_ROW + item_index, col,
1634 			      col_desc[COL_CORES_CORE_FREQ].max_data_string, ALIGN_RIGHT, core_freq);
1635 		col += col_desc[COL_CORES_CORE_FREQ].max_data_string + 2;
1636 	}
1637 
1638 	if (!col_desc[COL_CORES_INTR].disabled) {
1639 		snprintf(in_interrupt, MAX_INTR_LEN, "%s", g_cores_info[current_row].in_interrupt ? "Yes" : "No");
1640 		print_max_len(g_tabs[CORES_TAB], TABS_DATA_START_ROW + item_index,
1641 			      col + (col_desc[COL_CORES_INTR].name_len / 2), col_desc[COL_CORES_INTR].max_data_string,
1642 			      ALIGN_LEFT, in_interrupt);
1643 		col += col_desc[COL_CORES_INTR].max_data_string + 1;
1644 	}
1645 
1646 	if (!col_desc[COL_CORES_CPU_USAGE].disabled) {
1647 		get_cpu_usage_str(g_cores_info[current_row].busy - g_cores_info[current_row].last_busy,
1648 				  (g_cores_info[current_row].busy - g_cores_info[current_row].last_busy) +
1649 				  (g_cores_info[current_row].idle - g_cores_info[current_row].last_idle),
1650 				  cpu_usage);
1651 		print_max_len(g_tabs[CORES_TAB], TABS_DATA_START_ROW + item_index, col,
1652 			      col_desc[COL_CORES_CPU_USAGE].max_data_string, ALIGN_RIGHT, cpu_usage);
1653 	}
1654 }
1655 
1656 static uint8_t
1657 refresh_cores_tab(uint8_t current_page)
1658 {
1659 	uint64_t i;
1660 	uint16_t count = 0;
1661 	uint8_t max_pages, item_index;
1662 
1663 	count = g_last_cores_count;
1664 
1665 	max_pages = (count + g_max_row - WINDOW_HEADER - 1) / (g_max_row - WINDOW_HEADER);
1666 
1667 	for (i = current_page * g_max_data_rows;
1668 	     i < spdk_min(count, (uint64_t)((current_page + 1) * g_max_data_rows));
1669 	     i++) {
1670 		item_index = i - (current_page * g_max_data_rows);
1671 
1672 		draw_row_background(item_index, CORES_TAB);
1673 		draw_core_tab_row(i, item_index);
1674 
1675 		if (item_index == g_selected_row) {
1676 			wattroff(g_tabs[CORES_TAB], COLOR_PAIR(2));
1677 		}
1678 	}
1679 
1680 	g_max_selected_row = i - current_page * g_max_data_rows - 1;
1681 
1682 	return max_pages;
1683 }
1684 
1685 static uint8_t
1686 refresh_tab(enum tabs tab, uint8_t current_page)
1687 {
1688 	uint8_t (*refresh_function[NUMBER_OF_TABS])(uint8_t current_page) = {refresh_threads_tab, refresh_pollers_tab, refresh_cores_tab};
1689 	int color_pair[NUMBER_OF_TABS] = {COLOR_PAIR(2), COLOR_PAIR(2), COLOR_PAIR(2)};
1690 	int i;
1691 	uint8_t max_pages = 0;
1692 
1693 	color_pair[tab] = COLOR_PAIR(1);
1694 
1695 	for (i = 0; i < NUMBER_OF_TABS; i++) {
1696 		wbkgd(g_tab_win[i], color_pair[i]);
1697 	}
1698 
1699 	max_pages = (*refresh_function[tab])(current_page);
1700 
1701 	for (i = 0; i < NUMBER_OF_TABS; i++) {
1702 		wnoutrefresh(g_tab_win[i]);
1703 	}
1704 	draw_menu_win();
1705 
1706 	return max_pages;
1707 }
1708 
1709 static void
1710 print_in_middle(WINDOW *win, int starty, int startx, int width, char *string, chtype color)
1711 {
1712 	int length, temp;
1713 
1714 	length = strlen(string);
1715 	temp = (width - length) / 2;
1716 	wattron(win, color);
1717 	mvwprintw(win, starty, startx + temp, "%s", string);
1718 	wattroff(win, color);
1719 }
1720 
1721 static void
1722 print_left(WINDOW *win, int starty, int startx, int width, char *string, chtype color)
1723 {
1724 	wattron(win, color);
1725 	mvwprintw(win, starty, startx, "%s", string);
1726 	wattroff(win, color);
1727 }
1728 
1729 static void
1730 apply_filters(enum tabs tab)
1731 {
1732 	wclear(g_tabs[tab]);
1733 	draw_tabs(tab, g_current_sort_col[tab], g_current_sort_col2[tab]);
1734 }
1735 
1736 static ITEM **
1737 draw_filtering_menu(uint8_t position, WINDOW *filter_win, uint8_t tab, MENU **my_menu)
1738 {
1739 	const int ADDITIONAL_ELEMENTS = 3;
1740 	const int ROW_PADDING = 6;
1741 	const int WINDOW_START_X = 1;
1742 	const int WINDOW_START_Y = 3;
1743 	const int WINDOW_COLUMNS = 2;
1744 	struct col_desc *col_desc = g_col_desc[tab];
1745 	ITEM **my_items;
1746 	MENU *menu;
1747 	int i, elements;
1748 	uint8_t len = 0;
1749 
1750 	for (i = 0; col_desc[i].name != NULL; ++i) {
1751 		len = spdk_max(col_desc[i].name_len, len);
1752 	}
1753 
1754 	elements = i;
1755 
1756 	my_items = (ITEM **)calloc(elements * WINDOW_COLUMNS + ADDITIONAL_ELEMENTS, sizeof(ITEM *));
1757 	if (my_items == NULL) {
1758 		fprintf(stderr, "Unable to allocate an item list in draw_filtering_menu.\n");
1759 		return NULL;
1760 	}
1761 
1762 	for (i = 0; i < elements * 2; i++) {
1763 		my_items[i] = new_item(col_desc[i / WINDOW_COLUMNS].name, NULL);
1764 		i++;
1765 		my_items[i] = new_item(col_desc[i / WINDOW_COLUMNS].disabled ? "[ ]" : "[*]", NULL);
1766 	}
1767 
1768 	my_items[i] = new_item("     CLOSE", NULL);
1769 	set_item_userptr(my_items[i], apply_filters);
1770 
1771 	menu = new_menu((ITEM **)my_items);
1772 
1773 	menu_opts_off(menu, O_SHOWDESC);
1774 	set_menu_format(menu, elements + 1, WINDOW_COLUMNS);
1775 
1776 	set_menu_win(menu, filter_win);
1777 	set_menu_sub(menu, derwin(filter_win, elements + 1, len + ROW_PADDING, WINDOW_START_Y,
1778 				  WINDOW_START_X));
1779 
1780 	*my_menu = menu;
1781 
1782 	post_menu(menu);
1783 	refresh();
1784 	wrefresh(filter_win);
1785 
1786 	for (i = 0; i < position / WINDOW_COLUMNS; i++) {
1787 		menu_driver(menu, REQ_DOWN_ITEM);
1788 	}
1789 
1790 	return my_items;
1791 }
1792 
1793 static void
1794 delete_filtering_menu(MENU *my_menu, ITEM **my_items, uint8_t elements)
1795 {
1796 	int i;
1797 
1798 	unpost_menu(my_menu);
1799 	free_menu(my_menu);
1800 	for (i = 0; i < elements * 2 + 2; ++i) {
1801 		free_item(my_items[i]);
1802 	}
1803 	free(my_items);
1804 }
1805 
1806 static ITEM **
1807 refresh_filtering_menu(MENU **my_menu, WINDOW *filter_win, uint8_t tab, ITEM **my_items,
1808 		       uint8_t elements, uint8_t position)
1809 {
1810 	delete_filtering_menu(*my_menu, my_items, elements);
1811 	return draw_filtering_menu(position, filter_win, tab, my_menu);
1812 }
1813 
1814 static void
1815 filter_columns(uint8_t tab)
1816 {
1817 	const int WINDOW_HEADER_LEN = 5;
1818 	const int WINDOW_BORDER_LEN = 8;
1819 	const int WINDOW_HEADER_END_LINE = 2;
1820 	const int WINDOW_COLUMNS = 2;
1821 	struct col_desc *col_desc = g_col_desc[tab];
1822 	PANEL *filter_panel;
1823 	WINDOW *filter_win;
1824 	ITEM **my_items;
1825 	MENU *my_menu = NULL;
1826 	int i, c, elements;
1827 	bool stop_loop = false;
1828 	ITEM *cur;
1829 	void (*p)(enum tabs tab);
1830 	uint8_t current_index, len = 0;
1831 	bool disabled[TABS_COL_COUNT];
1832 
1833 	for (i = 0; col_desc[i].name != NULL; ++i) {
1834 		len = spdk_max(col_desc[i].name_len, len);
1835 	}
1836 
1837 	elements = i;
1838 
1839 	filter_win = newwin(elements + WINDOW_HEADER_LEN, len + WINDOW_BORDER_LEN,
1840 			    (g_max_row - elements - 1) / 2, (g_max_col - len) / 2);
1841 	assert(filter_win != NULL);
1842 	keypad(filter_win, TRUE);
1843 	filter_panel = new_panel(filter_win);
1844 	assert(filter_panel != NULL);
1845 
1846 	top_panel(filter_panel);
1847 	update_panels();
1848 	doupdate();
1849 
1850 	box(filter_win, 0, 0);
1851 
1852 	print_in_middle(filter_win, 1, 0, len + WINDOW_BORDER_LEN, "Filtering", COLOR_PAIR(3));
1853 	mvwaddch(filter_win, WINDOW_HEADER_END_LINE, 0, ACS_LTEE);
1854 	mvwhline(filter_win, WINDOW_HEADER_END_LINE, 1, ACS_HLINE, len + WINDOW_BORDER_LEN - 2);
1855 	mvwaddch(filter_win, WINDOW_HEADER_END_LINE, len + WINDOW_BORDER_LEN - 1, ACS_RTEE);
1856 
1857 	my_items = draw_filtering_menu(0, filter_win, tab, &my_menu);
1858 	if (my_items == NULL || my_menu == NULL) {
1859 		goto fail;
1860 	}
1861 
1862 	for (int i = 0; i < TABS_COL_COUNT; i++) {
1863 		disabled[i] = col_desc[i].disabled;
1864 	}
1865 
1866 	while (!stop_loop) {
1867 		c = wgetch(filter_win);
1868 
1869 		switch (c) {
1870 		case KEY_DOWN:
1871 			menu_driver(my_menu, REQ_DOWN_ITEM);
1872 			break;
1873 		case KEY_UP:
1874 			menu_driver(my_menu, REQ_UP_ITEM);
1875 			break;
1876 		case 27: /* ESC */
1877 		case 'q':
1878 			for (int i = 0; i < TABS_COL_COUNT; i++) {
1879 				cur = current_item(my_menu);
1880 				col_desc[i].disabled = disabled[i];
1881 
1882 				my_items = refresh_filtering_menu(&my_menu, filter_win, tab, my_items, elements,
1883 								  item_index(cur) + 1);
1884 				if (my_items == NULL || my_menu == NULL) {
1885 					goto fail;
1886 				}
1887 			}
1888 
1889 			stop_loop = true;
1890 			break;
1891 		case ' ': /* Space */
1892 			cur = current_item(my_menu);
1893 			current_index = item_index(cur) / WINDOW_COLUMNS;
1894 			col_desc[current_index].disabled = !col_desc[current_index].disabled;
1895 			my_items = refresh_filtering_menu(&my_menu, filter_win, tab, my_items, elements,
1896 							  item_index(cur) + 1);
1897 			if (my_items == NULL || my_menu == NULL) {
1898 				goto fail;
1899 			}
1900 			break;
1901 		case 10: /* Enter */
1902 			cur = current_item(my_menu);
1903 			current_index = item_index(cur) / WINDOW_COLUMNS;
1904 			if (current_index == elements) {
1905 				stop_loop = true;
1906 				p = item_userptr(cur);
1907 				p(tab);
1908 			} else {
1909 				col_desc[current_index].disabled = !col_desc[current_index].disabled;
1910 				my_items = refresh_filtering_menu(&my_menu, filter_win, tab, my_items, elements,
1911 								  item_index(cur) + 1);
1912 				if (my_items == NULL || my_menu == NULL) {
1913 					goto fail;
1914 				}
1915 			}
1916 			break;
1917 		}
1918 		wrefresh(filter_win);
1919 	}
1920 
1921 	delete_filtering_menu(my_menu, my_items, elements);
1922 
1923 	del_panel(filter_panel);
1924 	delwin(filter_win);
1925 
1926 	wclear(g_menu_win);
1927 	draw_menu_win();
1928 	return;
1929 
1930 fail:
1931 	fprintf(stderr, "Unable to filter the columns due to allocation failure.\n");
1932 	assert(false);
1933 }
1934 
1935 static void
1936 sort_type(enum tabs tab, int item_index)
1937 {
1938 	g_current_sort_col[tab] = item_index;
1939 }
1940 
1941 static void
1942 sort_type2(enum tabs tab, int item_index)
1943 {
1944 	g_current_sort_col2[tab] = item_index;
1945 }
1946 
1947 static void
1948 change_sorting(uint8_t tab, int winnum, bool *pstop_loop)
1949 {
1950 	const int WINDOW_HEADER_LEN = 4;
1951 	const int WINDOW_BORDER_LEN = 3;
1952 	const int WINDOW_START_X = 1;
1953 	const int WINDOW_START_Y = 4;
1954 	const int WINDOW_HEADER_END_LINE = 3;
1955 	const int WINDOW_MIN_WIDTH = 31;
1956 	PANEL *sort_panel;
1957 	WINDOW *sort_win;
1958 	ITEM **my_items;
1959 	MENU *my_menu;
1960 	int i, c, elements;
1961 	bool stop_loop = false;
1962 	ITEM *cur;
1963 	char *name;
1964 	char *help;
1965 	void (*p)(enum tabs tab, int item_index);
1966 	uint8_t len = WINDOW_MIN_WIDTH;
1967 
1968 	for (i = 0; g_col_desc[tab][i].name != NULL; ++i) {
1969 		len = spdk_max(len, g_col_desc[tab][i].name_len);
1970 	}
1971 
1972 	elements = i;
1973 
1974 	my_items = (ITEM **)calloc(elements + 1, sizeof(ITEM *));
1975 	if (my_items == NULL) {
1976 		fprintf(stderr, "Unable to allocate an item list in change_sorting.\n");
1977 		return;
1978 	}
1979 
1980 	for (i = 0; i < elements; ++i) {
1981 		my_items[i] = new_item(g_col_desc[tab][i].name, NULL);
1982 		set_item_userptr(my_items[i], (winnum == 0) ? sort_type : sort_type2);
1983 	}
1984 
1985 	my_menu = new_menu((ITEM **)my_items);
1986 
1987 	menu_opts_off(my_menu, O_SHOWDESC);
1988 
1989 	sort_win = newwin(elements + WINDOW_HEADER_LEN + 1, len + WINDOW_BORDER_LEN,
1990 			  (g_max_row - elements) / 2,
1991 			  (g_max_col - len) / 2 - len + len * winnum);
1992 	assert(sort_win != NULL);
1993 	keypad(sort_win, TRUE);
1994 	sort_panel = new_panel(sort_win);
1995 	assert(sort_panel != NULL);
1996 
1997 	top_panel(sort_panel);
1998 	update_panels();
1999 	doupdate();
2000 
2001 	set_menu_win(my_menu, sort_win);
2002 	set_menu_sub(my_menu, derwin(sort_win, elements, len + 1, WINDOW_START_Y, WINDOW_START_X));
2003 	box(sort_win, 0, 0);
2004 
2005 	if (winnum == 0) {
2006 		name = "Sorting #1";
2007 		help = "Right key for second sorting";
2008 	} else {
2009 		name = "Sorting #2";
2010 		help = "Left key for first sorting";
2011 	}
2012 
2013 	print_in_middle(sort_win, 1, 0, len + WINDOW_BORDER_LEN, name, COLOR_PAIR(3));
2014 	print_in_middle(sort_win, 2, 0, len + WINDOW_BORDER_LEN, help, COLOR_PAIR(3));
2015 	mvwaddch(sort_win, WINDOW_HEADER_END_LINE, 0, ACS_LTEE);
2016 	mvwhline(sort_win, WINDOW_HEADER_END_LINE, 1, ACS_HLINE, len + 1);
2017 	mvwaddch(sort_win, WINDOW_HEADER_END_LINE, len + WINDOW_BORDER_LEN - 1, ACS_RTEE);
2018 
2019 	post_menu(my_menu);
2020 	refresh();
2021 	wrefresh(sort_win);
2022 
2023 	while (!stop_loop) {
2024 		c = wgetch(sort_win);
2025 		/*
2026 		 * First sorting window:
2027 		 * Up/Down - select first sorting column;
2028 		 * Enter - apply current column;
2029 		 * Right - open second sorting window.
2030 		 * Second sorting window:
2031 		 * Up/Down - select second sorting column;
2032 		 * Enter - apply current column of both sorting windows;
2033 		 * Left - exit second window and reset second sorting key;
2034 		 * Right - do nothing.
2035 		 */
2036 		switch (c) {
2037 		case KEY_DOWN:
2038 			menu_driver(my_menu, REQ_DOWN_ITEM);
2039 			break;
2040 		case KEY_UP:
2041 			menu_driver(my_menu, REQ_UP_ITEM);
2042 			break;
2043 		case KEY_RIGHT:
2044 			if (winnum > 0) {
2045 				break;
2046 			}
2047 			change_sorting(tab, winnum + 1, &stop_loop);
2048 			/* Restore input. */
2049 			keypad(sort_win, TRUE);
2050 			post_menu(my_menu);
2051 			refresh();
2052 			wrefresh(sort_win);
2053 			redrawwin(sort_win);
2054 			if (winnum == 0) {
2055 				cur = current_item(my_menu);
2056 				p = item_userptr(cur);
2057 				p(tab, item_index(cur));
2058 			}
2059 			break;
2060 		case 27: /* ESC */
2061 			stop_loop = true;
2062 			break;
2063 		case KEY_LEFT:
2064 			if (winnum > 0) {
2065 				sort_type2(tab, COL_THREADS_NONE);
2066 			}
2067 		/* FALLTHROUGH */
2068 		case 10: /* Enter */
2069 			stop_loop = true;
2070 			if (winnum > 0 && c == 10) {
2071 				*pstop_loop = true;
2072 			}
2073 			if (c == 10) {
2074 				cur = current_item(my_menu);
2075 				p = item_userptr(cur);
2076 				p(tab, item_index(cur));
2077 			}
2078 			break;
2079 		}
2080 		wrefresh(sort_win);
2081 	}
2082 
2083 	if (winnum == 0) {
2084 		wclear(g_tabs[tab]);
2085 		draw_tabs(tab, g_current_sort_col[tab], g_current_sort_col2[tab]);
2086 	}
2087 
2088 	unpost_menu(my_menu);
2089 	free_menu(my_menu);
2090 
2091 	for (i = 0; i < elements; ++i) {
2092 		free_item(my_items[i]);
2093 	}
2094 
2095 	free(my_items);
2096 
2097 	del_panel(sort_panel);
2098 	delwin(sort_win);
2099 
2100 	if (winnum == 0) {
2101 		wclear(g_menu_win);
2102 		draw_menu_win();
2103 	}
2104 }
2105 
2106 static int
2107 check_resize_interface(uint8_t active_tab, uint8_t *current_page)
2108 {
2109 	int max_row, max_col;
2110 	uint16_t required_size = WINDOW_HEADER + 1;
2111 
2112 	/* Check if interface has to be resized (terminal size changed) */
2113 	getmaxyx(stdscr, max_row, max_col);
2114 
2115 	if (max_row != g_max_row || max_col != g_max_col) {
2116 		if (max_row != g_max_row) {
2117 			*current_page = 0;
2118 		}
2119 		g_max_row = spdk_max(max_row, required_size);
2120 		g_max_col = max_col;
2121 		g_data_win_size = g_max_row - required_size + 1;
2122 		g_max_data_rows = g_max_row - WINDOW_HEADER;
2123 		resize_interface(active_tab);
2124 
2125 		return 1;
2126 	}
2127 
2128 	return 0;
2129 }
2130 
2131 static void
2132 change_refresh_rate(void)
2133 {
2134 	const int WINDOW_HEADER_END_LINE = 2;
2135 	PANEL *refresh_panel;
2136 	WINDOW *refresh_win;
2137 	int c;
2138 	bool stop_loop = false;
2139 	uint32_t rr_tmp, refresh_rate = 0;
2140 	char refresh_rate_str[MAX_STRING_LEN];
2141 
2142 	refresh_win = newwin(RR_WIN_HEIGHT, RR_WIN_WIDTH, (g_max_row - RR_WIN_HEIGHT - 1) / 2,
2143 			     (g_max_col - RR_WIN_WIDTH) / 2);
2144 	assert(refresh_win != NULL);
2145 	keypad(refresh_win, TRUE);
2146 	refresh_panel = new_panel(refresh_win);
2147 	assert(refresh_panel != NULL);
2148 
2149 	top_panel(refresh_panel);
2150 	update_panels();
2151 	doupdate();
2152 
2153 	box(refresh_win, 0, 0);
2154 
2155 	print_in_middle(refresh_win, 1, 0, RR_WIN_WIDTH + 1, "Enter refresh rate value [s]", COLOR_PAIR(3));
2156 	mvwaddch(refresh_win, WINDOW_HEADER_END_LINE, 0, ACS_LTEE);
2157 	mvwhline(refresh_win, WINDOW_HEADER_END_LINE, 1, ACS_HLINE, RR_WIN_WIDTH - 2);
2158 	mvwaddch(refresh_win, WINDOW_HEADER_END_LINE, RR_WIN_WIDTH, ACS_RTEE);
2159 	mvwprintw(refresh_win, WINDOW_HEADER_END_LINE + 1, (RR_WIN_WIDTH - 1) / 2, "%d", refresh_rate);
2160 
2161 	refresh();
2162 	wrefresh(refresh_win);
2163 
2164 	while (!stop_loop) {
2165 		c = wgetch(refresh_win);
2166 
2167 		switch (c) {
2168 		case '0':
2169 		case '1':
2170 		case '2':
2171 		case '3':
2172 		case '4':
2173 		case '5':
2174 		case '6':
2175 		case '7':
2176 		case '8':
2177 		case '9':
2178 			rr_tmp = refresh_rate * 10 + c - '0';
2179 
2180 			if (rr_tmp <= RR_MAX_VALUE) {
2181 				refresh_rate = rr_tmp;
2182 				snprintf(refresh_rate_str, MAX_STRING_LEN - 1, "%d", refresh_rate);
2183 				mvwprintw(refresh_win, WINDOW_HEADER_END_LINE + 1,
2184 					  (RR_WIN_WIDTH - 1 - strlen(refresh_rate_str)) / 2, "%d", refresh_rate);
2185 				refresh();
2186 				wrefresh(refresh_win);
2187 			}
2188 			break;
2189 		case KEY_BACKSPACE:
2190 		case 127:
2191 		case '\b':
2192 			refresh_rate = refresh_rate / 10;
2193 			snprintf(refresh_rate_str, MAX_STRING_LEN - 1, "%d", refresh_rate);
2194 			mvwprintw(refresh_win, WINDOW_HEADER_END_LINE + 1,
2195 				  (RR_WIN_WIDTH - 1 - strlen(refresh_rate_str) - 2) / 2, "       ");
2196 			mvwprintw(refresh_win, WINDOW_HEADER_END_LINE + 1,
2197 				  (RR_WIN_WIDTH - 1 - strlen(refresh_rate_str)) / 2, "%d", refresh_rate);
2198 			refresh();
2199 			wrefresh(refresh_win);
2200 			break;
2201 		case 27: /* ESC */
2202 		case 'q':
2203 			stop_loop = true;
2204 			break;
2205 		case 10: /* Enter */
2206 			pthread_mutex_lock(&g_thread_lock);
2207 			g_sleep_time = refresh_rate;
2208 			pthread_mutex_unlock(&g_thread_lock);
2209 			stop_loop = true;
2210 			break;
2211 		}
2212 		wrefresh(refresh_win);
2213 	}
2214 
2215 	del_panel(refresh_panel);
2216 	delwin(refresh_win);
2217 }
2218 
2219 static void
2220 free_poller_history(void)
2221 {
2222 	struct run_counter_history *history, *tmp;
2223 
2224 	TAILQ_FOREACH_SAFE(history, &g_run_counter_history, link, tmp) {
2225 		TAILQ_REMOVE(&g_run_counter_history, history, link);
2226 		free(history);
2227 	}
2228 }
2229 
2230 static uint64_t
2231 get_position_for_window(uint64_t window_size, uint64_t max_size)
2232 {
2233 	/* This function calculates position for pop-up detail window.
2234 	 * Since horizontal and vertical positions are calculated the same way
2235 	 * there is no need for separate functions. */
2236 	window_size = spdk_min(window_size, max_size);
2237 
2238 	return (max_size - window_size) / 2;
2239 }
2240 
2241 static void
2242 print_bottom_message(char *msg)
2243 {
2244 	uint64_t i;
2245 
2246 	for (i = 1; i < (uint64_t)g_max_col - 1; i++) {
2247 		mvprintw(g_max_row - 1, i, " ");
2248 	}
2249 	mvprintw(g_max_row - 1, g_max_col - strlen(msg) - 2, "%s", msg);
2250 }
2251 
2252 static void
2253 draw_thread_win_content(WINDOW *thread_win, struct rpc_thread_info *thread_info)
2254 {
2255 	uint64_t current_row, i, time;
2256 	char idle_time[MAX_TIME_STR_LEN], busy_time[MAX_TIME_STR_LEN];
2257 
2258 	box(thread_win, 0, 0);
2259 
2260 	print_in_middle(thread_win, 1, 0, THREAD_WIN_WIDTH, thread_info->name,
2261 			COLOR_PAIR(3));
2262 	mvwhline(thread_win, 2, 1, ACS_HLINE, THREAD_WIN_WIDTH - 2);
2263 	mvwaddch(thread_win, 2, THREAD_WIN_WIDTH, ACS_RTEE);
2264 
2265 	print_left(thread_win, 3, THREAD_WIN_FIRST_COL, THREAD_WIN_WIDTH,
2266 		   "Core:                Idle [us]:            Busy [us]:", COLOR_PAIR(5));
2267 	mvwprintw(thread_win, 3, THREAD_WIN_FIRST_COL + 6, "%d",
2268 		  thread_info->core_num);
2269 
2270 	if (g_interval_data) {
2271 		get_time_str(thread_info->idle - thread_info->last_idle, idle_time);
2272 		mvwprintw(thread_win, 3, THREAD_WIN_FIRST_COL + 32, "%s", idle_time);
2273 		get_time_str(thread_info->busy - thread_info->last_busy, busy_time);
2274 		mvwprintw(thread_win, 3, THREAD_WIN_FIRST_COL + 54, "%s", busy_time);
2275 	} else {
2276 		get_time_str(thread_info->idle, idle_time);
2277 		mvwprintw(thread_win, 3, THREAD_WIN_FIRST_COL + 32, "%s", idle_time);
2278 		get_time_str(thread_info->busy, busy_time);
2279 		mvwprintw(thread_win, 3, THREAD_WIN_FIRST_COL + 54, "%s", busy_time);
2280 	}
2281 
2282 	print_left(thread_win, 4, THREAD_WIN_FIRST_COL, THREAD_WIN_WIDTH,
2283 		   "Active pollers:      Timed pollers:        Paused pollers:", COLOR_PAIR(5));
2284 	mvwprintw(thread_win, 4, THREAD_WIN_FIRST_COL + 17, "%" PRIu64,
2285 		  thread_info->active_pollers_count);
2286 	mvwprintw(thread_win, 4, THREAD_WIN_FIRST_COL + 36, "%" PRIu64,
2287 		  thread_info->timed_pollers_count);
2288 	mvwprintw(thread_win, 4, THREAD_WIN_FIRST_COL + 59, "%" PRIu64,
2289 		  thread_info->paused_pollers_count);
2290 
2291 	mvwhline(thread_win, 5, 1, ACS_HLINE, THREAD_WIN_WIDTH - 2);
2292 
2293 	print_in_middle(thread_win, 6, 0, THREAD_WIN_WIDTH,
2294 			"Pollers                          Type    Total run count   Period", COLOR_PAIR(5));
2295 
2296 	mvwhline(thread_win, 7, 1, ACS_HLINE, THREAD_WIN_WIDTH - 2);
2297 
2298 	current_row = 8;
2299 
2300 	for (i = 0; i < g_last_pollers_count; i++) {
2301 		if (g_pollers_info[i].thread_id == thread_info->id) {
2302 			mvwprintw(thread_win, current_row, THREAD_WIN_FIRST_COL, "%s", g_pollers_info[i].name);
2303 			if (g_pollers_info[i].type == SPDK_ACTIVE_POLLER) {
2304 				mvwprintw(thread_win, current_row, THREAD_WIN_FIRST_COL + 33, "Active");
2305 			} else if (g_pollers_info[i].type == SPDK_TIMED_POLLER) {
2306 				mvwprintw(thread_win, current_row, THREAD_WIN_FIRST_COL + 33, "Timed");
2307 			} else {
2308 				mvwprintw(thread_win, current_row, THREAD_WIN_FIRST_COL + 33, "Paused");
2309 			}
2310 			mvwprintw(thread_win, current_row, THREAD_WIN_FIRST_COL + 41, "%" PRIu64,
2311 				  g_pollers_info[i].run_count);
2312 			if (g_pollers_info[i].period_ticks) {
2313 				time = g_pollers_info[i].period_ticks * SPDK_SEC_TO_USEC / g_tick_rate;
2314 				mvwprintw(thread_win, current_row, THREAD_WIN_FIRST_COL + 59, "%" PRIu64, time);
2315 			}
2316 			current_row++;
2317 		}
2318 	}
2319 
2320 	wnoutrefresh(thread_win);
2321 }
2322 
2323 static int
2324 get_single_thread_info(uint64_t thread_id, struct rpc_thread_info *thread_info)
2325 {
2326 	uint64_t i;
2327 
2328 	for (i = 0; i < g_last_threads_count; i++) {
2329 		if (g_threads_info[i].id == thread_id) {
2330 			memcpy(thread_info, &g_threads_info[i], sizeof(struct rpc_thread_info));
2331 			return 0;
2332 		}
2333 	}
2334 
2335 	print_bottom_message("Selected thread no longer exists. Exiting pop-up.");
2336 	return -1;
2337 }
2338 
2339 static void
2340 draw_core_win_content(WINDOW *core_win, struct rpc_core_info *core_info)
2341 {
2342 	uint64_t i;
2343 	char core_win_title[25];
2344 	char idle_time[MAX_TIME_STR_LEN], busy_time[MAX_TIME_STR_LEN];
2345 
2346 	box(core_win, 0, 0);
2347 	snprintf(core_win_title, sizeof(core_win_title), "Core %" PRIu32 " details",
2348 		 core_info->lcore);
2349 	print_in_middle(core_win, 1, 0, CORE_WIN_WIDTH, core_win_title, COLOR_PAIR(3));
2350 
2351 	mvwaddch(core_win, -1, 0, ACS_LTEE);
2352 	mvwhline(core_win, 2, 1, ACS_HLINE, CORE_WIN_WIDTH - 2);
2353 	mvwaddch(core_win, 2, CORE_WIN_WIDTH, ACS_RTEE);
2354 	print_left(core_win, 3, 1, CORE_WIN_WIDTH - (CORE_WIN_WIDTH / 3),
2355 		   "Frequency:             Intr:", COLOR_PAIR(5));
2356 	if (core_info->core_freq) {
2357 		mvwprintw(core_win, 3, CORE_WIN_FIRST_COL - 3, "%" PRIu32,
2358 			  core_info->core_freq);
2359 	} else {
2360 		mvwprintw(core_win, 3, CORE_WIN_FIRST_COL - 3, "%s", "N/A");
2361 	}
2362 
2363 	mvwprintw(core_win, 3, CORE_WIN_FIRST_COL + 15, "%s",
2364 		  core_info->in_interrupt ? "Yes" : "No");
2365 
2366 	mvwaddch(core_win, -1, 0, ACS_LTEE);
2367 	mvwhline(core_win, 4, 1, ACS_HLINE, CORE_WIN_WIDTH - 2);
2368 	mvwaddch(core_win, 4, CORE_WIN_WIDTH, ACS_RTEE);
2369 	print_left(core_win, 5, 1, CORE_WIN_WIDTH, "Thread count:          Idle time:", COLOR_PAIR(5));
2370 
2371 	mvwprintw(core_win, 5, CORE_WIN_FIRST_COL, "%" PRIu64,
2372 		  core_info->threads.threads_count);
2373 
2374 	if (g_interval_data == true) {
2375 		get_time_str(core_info->idle - core_info->last_idle, idle_time);
2376 		get_time_str(core_info->busy - core_info->last_busy, busy_time);
2377 	} else {
2378 		get_time_str(core_info->idle, idle_time);
2379 		get_time_str(core_info->busy, busy_time);
2380 	}
2381 	mvwprintw(core_win, 5, CORE_WIN_FIRST_COL + 20, "%s", idle_time);
2382 	mvwhline(core_win, 6, 1, ACS_HLINE, CORE_WIN_WIDTH - 2);
2383 
2384 	print_left(core_win, 7, 1, CORE_WIN_WIDTH, "Poller count:          Busy time:", COLOR_PAIR(5));
2385 	mvwprintw(core_win, 7, CORE_WIN_FIRST_COL, "%" PRIu64,
2386 		  core_info->pollers_count);
2387 
2388 	mvwprintw(core_win, 7, CORE_WIN_FIRST_COL + 20, "%s", busy_time);
2389 
2390 	mvwhline(core_win, 8, 1, ACS_HLINE, CORE_WIN_WIDTH - 2);
2391 	print_left(core_win, 9, 1, CORE_WIN_WIDTH, "Threads on this core", COLOR_PAIR(5));
2392 
2393 	for (i = 0; i < core_info->threads.threads_count; i++) {
2394 		mvwprintw(core_win, i + 10, 1, "%s", core_info->threads.thread[i].name);
2395 	}
2396 	pthread_mutex_unlock(&g_thread_lock);
2397 
2398 	wnoutrefresh(core_win);
2399 }
2400 
2401 static void
2402 display_thread(uint64_t thread_id, uint8_t current_page, uint8_t active_tab,
2403 	       WINDOW *core_popup, struct rpc_core_info *core_info)
2404 {
2405 	PANEL *thread_panel = NULL;
2406 	WINDOW *thread_win = NULL;
2407 	struct rpc_thread_info thread_info;
2408 	uint64_t pollers_count, threads_count, last_pollers_count = 0;
2409 	int c;
2410 	bool stop_loop = false;
2411 	long int time_last, time_dif;
2412 	struct timespec time_now;
2413 
2414 	clock_gettime(CLOCK_MONOTONIC, &time_now);
2415 	time_last = time_now.tv_sec;
2416 
2417 	memset(&thread_info, 0, sizeof(thread_info));
2418 
2419 	while (!stop_loop) {
2420 		pthread_mutex_lock(&g_thread_lock);
2421 		if (get_single_thread_info(thread_id, &thread_info)) {
2422 			pthread_mutex_unlock(&g_thread_lock);
2423 			return;
2424 		}
2425 		pollers_count = thread_info.active_pollers_count +
2426 				thread_info.timed_pollers_count +
2427 				thread_info.paused_pollers_count;
2428 		if (pollers_count != last_pollers_count) {
2429 			if (thread_win != NULL) {
2430 				assert(thread_panel != NULL);
2431 				del_panel(thread_panel);
2432 				delwin(thread_win);
2433 			}
2434 
2435 			thread_win = newwin(pollers_count + THREAD_WIN_HEIGHT, THREAD_WIN_WIDTH,
2436 					    get_position_for_window(THREAD_WIN_HEIGHT + pollers_count, g_max_row),
2437 					    get_position_for_window(THREAD_WIN_WIDTH, g_max_col));
2438 			keypad(thread_win, TRUE);
2439 			thread_panel = new_panel(thread_win);
2440 
2441 			top_panel(thread_panel);
2442 			update_panels();
2443 			doupdate();
2444 			draw_thread_win_content(thread_win, &thread_info);
2445 			refresh();
2446 		}
2447 		pthread_mutex_unlock(&g_thread_lock);
2448 
2449 		if (check_resize_interface(active_tab, &current_page)) {
2450 			/* This clear is to avoid remaining artifacts after window has been moved */
2451 			wclear(thread_win);
2452 			wclear(core_popup);
2453 			resize_interface(active_tab);
2454 			draw_tabs(active_tab, g_current_sort_col[active_tab], g_current_sort_col2[active_tab]);
2455 			if (core_popup != NULL) {
2456 				pthread_mutex_lock(&g_thread_lock);
2457 				threads_count = g_cores_info[core_info->lcore].threads.threads_count;
2458 				pthread_mutex_unlock(&g_thread_lock);
2459 				mvwin(core_popup, get_position_for_window(CORE_WIN_HEIGHT + threads_count, g_max_row),
2460 				      get_position_for_window(CORE_WIN_WIDTH, g_max_col));
2461 			}
2462 			mvwin(thread_win, get_position_for_window(THREAD_WIN_HEIGHT + pollers_count, g_max_row),
2463 			      get_position_for_window(THREAD_WIN_WIDTH, g_max_col));
2464 		}
2465 
2466 		c = getch();
2467 
2468 		switch (c) {
2469 		case 27: /* ESC */
2470 			stop_loop = true;
2471 			break;
2472 		default:
2473 			break;
2474 		}
2475 
2476 		clock_gettime(CLOCK_MONOTONIC, &time_now);
2477 		time_dif = time_now.tv_sec - time_last;
2478 
2479 		if (time_dif >= g_sleep_time) {
2480 			time_last = time_now.tv_sec;
2481 			pthread_mutex_lock(&g_thread_lock);
2482 			refresh_tab(active_tab, current_page);
2483 			if (core_popup != NULL) {
2484 				draw_core_win_content(core_popup, core_info);
2485 			}
2486 			draw_thread_win_content(thread_win, &thread_info);
2487 			refresh();
2488 			pthread_mutex_unlock(&g_thread_lock);
2489 		}
2490 
2491 		last_pollers_count = pollers_count;
2492 	}
2493 
2494 	del_panel(thread_panel);
2495 	delwin(thread_win);
2496 }
2497 
2498 static void
2499 show_thread(uint8_t current_page, uint8_t active_tab)
2500 {
2501 	uint64_t thread_number = current_page * g_max_data_rows + g_selected_row;
2502 	uint64_t thread_id;
2503 
2504 	pthread_mutex_lock(&g_thread_lock);
2505 	assert(thread_number < g_last_threads_count);
2506 	thread_id = g_threads_info[thread_number].id;
2507 	pthread_mutex_unlock(&g_thread_lock);
2508 
2509 	display_thread(thread_id, current_page, active_tab, NULL, NULL);
2510 }
2511 
2512 static void
2513 show_single_thread(uint64_t thread_id, uint8_t current_page, uint8_t active_tab, WINDOW *core_popup,
2514 		   struct rpc_core_info *core_info)
2515 {
2516 	uint64_t i;
2517 
2518 	pthread_mutex_lock(&g_thread_lock);
2519 	for (i = 0; i < g_last_threads_count; i++) {
2520 		if (g_threads_info[i].id == thread_id) {
2521 			pthread_mutex_unlock(&g_thread_lock);
2522 			display_thread(thread_id, current_page, active_tab, core_popup, core_info);
2523 			return;
2524 		}
2525 	}
2526 	pthread_mutex_unlock(&g_thread_lock);
2527 }
2528 
2529 static void
2530 show_core(uint8_t current_page, uint8_t active_tab)
2531 {
2532 	PANEL *core_panel;
2533 	WINDOW *core_win;
2534 	uint64_t core_number = current_page * g_max_data_rows + g_selected_row;
2535 	struct rpc_core_info *core_info = &g_cores_info[core_number];
2536 	uint64_t threads_count, i;
2537 	uint64_t thread_id = 0;
2538 	uint16_t current_threads_row;
2539 	int c;
2540 	long int time_last, time_dif;
2541 	struct timespec time_now;
2542 
2543 	clock_gettime(CLOCK_MONOTONIC, &time_now);
2544 	time_last = time_now.tv_sec;
2545 
2546 	bool stop_loop = false;
2547 
2548 	pthread_mutex_lock(&g_thread_lock);
2549 	assert(core_number < g_last_cores_count);
2550 
2551 	threads_count = g_cores_info[core_number].threads.threads_count;
2552 
2553 	core_win = newwin(threads_count + CORE_WIN_HEIGHT, CORE_WIN_WIDTH,
2554 			  get_position_for_window(CORE_WIN_HEIGHT + threads_count, g_max_row),
2555 			  get_position_for_window(CORE_WIN_WIDTH, g_max_col));
2556 
2557 	keypad(core_win, TRUE);
2558 	core_panel = new_panel(core_win);
2559 
2560 	top_panel(core_panel);
2561 	update_panels();
2562 	doupdate();
2563 	draw_core_win_content(core_win, core_info);
2564 	refresh();
2565 
2566 	current_threads_row = 0;
2567 
2568 	while (!stop_loop) {
2569 		pthread_mutex_lock(&g_thread_lock);
2570 		for (i = 0; i < core_info->threads.threads_count; i++) {
2571 			if (i != current_threads_row) {
2572 				mvwprintw(core_win, i + 10, 1, "%s", core_info->threads.thread[i].name);
2573 			} else {
2574 				print_left(core_win, i + 10, 1, CORE_WIN_WIDTH - 2,
2575 					   core_info->threads.thread[i].name, COLOR_PAIR(2));
2576 			}
2577 		}
2578 		pthread_mutex_unlock(&g_thread_lock);
2579 
2580 		wrefresh(core_win);
2581 		if (check_resize_interface(active_tab, &current_page)) {
2582 			wclear(core_win);
2583 			resize_interface(active_tab);
2584 			draw_tab_win(active_tab);
2585 			mvwin(core_win, get_position_for_window(CORE_WIN_HEIGHT + threads_count, g_max_row),
2586 			      get_position_for_window(CORE_WIN_WIDTH, g_max_col));
2587 		}
2588 
2589 		c = getch();
2590 		switch (c) {
2591 		case 10: /* ENTER */
2592 			pthread_mutex_lock(&g_thread_lock);
2593 			if (core_info->threads.threads_count > 0) {
2594 				thread_id = core_info->threads.thread[current_threads_row].id;
2595 			}
2596 			pthread_mutex_unlock(&g_thread_lock);
2597 
2598 			if (thread_id != 0) {
2599 				show_single_thread(thread_id, current_page, active_tab, core_win, core_info);
2600 			}
2601 
2602 			/* This refreshes tab and core_pop-up after exiting threads pop-up. */
2603 			pthread_mutex_lock(&g_thread_lock);
2604 			refresh_tab(active_tab, current_page);
2605 			wnoutrefresh(core_win);
2606 			refresh();
2607 			pthread_mutex_unlock(&g_thread_lock);
2608 			break;
2609 		case 27: /* ESC */
2610 			stop_loop = true;
2611 			break;
2612 		case KEY_UP:
2613 			if (current_threads_row != 0) {
2614 				current_threads_row--;
2615 			}
2616 			break;
2617 		case KEY_DOWN:
2618 			pthread_mutex_lock(&g_thread_lock);
2619 			if (current_threads_row != core_info->threads.threads_count - 1) {
2620 				current_threads_row++;
2621 			}
2622 			pthread_mutex_unlock(&g_thread_lock);
2623 			break;
2624 		default:
2625 			break;
2626 		}
2627 
2628 		clock_gettime(CLOCK_MONOTONIC, &time_now);
2629 		time_dif = time_now.tv_sec - time_last;
2630 
2631 		if (time_dif >= g_sleep_time) {
2632 			time_last = time_now.tv_sec;
2633 			pthread_mutex_lock(&g_thread_lock);
2634 			refresh_tab(active_tab, current_page);
2635 			draw_core_win_content(core_win, core_info);
2636 			refresh();
2637 			pthread_mutex_unlock(&g_thread_lock);
2638 		}
2639 	}
2640 
2641 	del_panel(core_panel);
2642 	delwin(core_win);
2643 }
2644 
2645 static void
2646 draw_poller_win_content(WINDOW *poller_win, struct rpc_poller_info *poller_info)
2647 {
2648 	uint64_t last_run_counter, last_busy_counter, busy_count;
2649 	char poller_period[MAX_TIME_STR_LEN];
2650 
2651 	box(poller_win, 0, 0);
2652 
2653 	print_in_middle(poller_win, 1, 0, POLLER_WIN_WIDTH, poller_info->name, COLOR_PAIR(3));
2654 	mvwhline(poller_win, 2, 1, ACS_HLINE, POLLER_WIN_WIDTH - 2);
2655 	mvwaddch(poller_win, 2, POLLER_WIN_WIDTH, ACS_RTEE);
2656 
2657 	print_left(poller_win, 3, 2, POLLER_WIN_WIDTH, "Type:                  On thread:", COLOR_PAIR(5));
2658 	mvwprintw(poller_win, 3, POLLER_WIN_FIRST_COL, "%s",
2659 		  poller_type_str[poller_info->type]);
2660 	mvwprintw(poller_win, 3, POLLER_WIN_FIRST_COL + 23, "%s", poller_info->thread_name);
2661 
2662 	print_left(poller_win, 4, 2, POLLER_WIN_WIDTH, "Run count:", COLOR_PAIR(5));
2663 
2664 	last_run_counter = get_last_run_counter(poller_info->id, poller_info->thread_id);
2665 	last_busy_counter = get_last_busy_counter(poller_info->id, poller_info->thread_id);
2666 	if (g_interval_data) {
2667 		mvwprintw(poller_win, 4, POLLER_WIN_FIRST_COL, "%" PRIu64,
2668 			  poller_info->run_count - last_run_counter);
2669 	} else {
2670 		mvwprintw(poller_win, 4, POLLER_WIN_FIRST_COL, "%" PRIu64, poller_info->run_count);
2671 	}
2672 
2673 	if (poller_info->period_ticks != 0) {
2674 		print_left(poller_win, 4, 28, POLLER_WIN_WIDTH, "Period:", COLOR_PAIR(5));
2675 		get_time_str(poller_info->period_ticks, poller_period);
2676 		mvwprintw(poller_win, 4, POLLER_WIN_FIRST_COL + 23, "%s", poller_period);
2677 	}
2678 	mvwhline(poller_win, 5, 1, ACS_HLINE, POLLER_WIN_WIDTH - 2);
2679 
2680 	busy_count = g_interval_data ? poller_info->busy_count - last_busy_counter :
2681 		     poller_info->busy_count;
2682 	if (busy_count != 0) {
2683 		print_left(poller_win, 6, 2, POLLER_WIN_WIDTH,  "Status:               Busy count:", COLOR_PAIR(5));
2684 
2685 		if (g_interval_data == false && poller_info->busy_count == last_busy_counter) {
2686 			print_left(poller_win, 6, POLLER_WIN_FIRST_COL, POLLER_WIN_WIDTH, "Idle", COLOR_PAIR(7));
2687 		} else {
2688 			print_left(poller_win, 6, POLLER_WIN_FIRST_COL, POLLER_WIN_WIDTH, "Busy", COLOR_PAIR(6));
2689 		}
2690 
2691 		mvwprintw(poller_win, 6, POLLER_WIN_FIRST_COL + 23, "%" PRIu64, busy_count);
2692 	} else {
2693 		print_in_middle(poller_win, 6, 1, POLLER_WIN_WIDTH - 7, "Status:", COLOR_PAIR(5));
2694 		print_in_middle(poller_win, 6, 1, POLLER_WIN_WIDTH + 6, "Idle", COLOR_PAIR(7));
2695 	}
2696 
2697 	wnoutrefresh(poller_win);
2698 }
2699 
2700 static void
2701 show_poller(uint8_t current_page, uint8_t active_tab)
2702 {
2703 	PANEL *poller_panel;
2704 	WINDOW *poller_win;
2705 	uint64_t poller_number = current_page * g_max_data_rows + g_selected_row;
2706 	struct rpc_poller_info *poller;
2707 	bool stop_loop = false;
2708 	int c;
2709 	long int time_last, time_dif;
2710 	struct timespec time_now;
2711 
2712 	clock_gettime(CLOCK_MONOTONIC, &time_now);
2713 	time_last = time_now.tv_sec;
2714 
2715 
2716 	pthread_mutex_lock(&g_thread_lock);
2717 
2718 	assert(poller_number < g_last_pollers_count);
2719 	poller = &g_pollers_info[poller_number];
2720 
2721 	poller_win = newwin(POLLER_WIN_HEIGHT, POLLER_WIN_WIDTH,
2722 			    get_position_for_window(POLLER_WIN_HEIGHT, g_max_row),
2723 			    get_position_for_window(POLLER_WIN_WIDTH, g_max_col));
2724 
2725 	keypad(poller_win, TRUE);
2726 	poller_panel = new_panel(poller_win);
2727 
2728 	top_panel(poller_panel);
2729 	update_panels();
2730 	doupdate();
2731 	draw_poller_win_content(poller_win, poller);
2732 	refresh();
2733 
2734 	pthread_mutex_unlock(&g_thread_lock);
2735 	while (!stop_loop) {
2736 		if (check_resize_interface(active_tab, &current_page)) {
2737 			/* This clear is to avoid remaining artifacts after window has been moved */
2738 			wclear(poller_win);
2739 			resize_interface(active_tab);
2740 			draw_tabs(active_tab, g_current_sort_col[active_tab], g_current_sort_col2[active_tab]);
2741 			mvwin(poller_win, get_position_for_window(POLLER_WIN_HEIGHT, g_max_row),
2742 			      get_position_for_window(POLLER_WIN_WIDTH, g_max_col));
2743 		}
2744 		c = getch();
2745 		switch (c) {
2746 		case 27: /* ESC */
2747 			stop_loop = true;
2748 			break;
2749 		default:
2750 			break;
2751 		}
2752 
2753 		clock_gettime(CLOCK_MONOTONIC, &time_now);
2754 		time_dif = time_now.tv_sec - time_last;
2755 
2756 		if (time_dif >= g_sleep_time) {
2757 			time_last = time_now.tv_sec;
2758 			pthread_mutex_lock(&g_thread_lock);
2759 			refresh_tab(active_tab, current_page);
2760 			draw_poller_win_content(poller_win, poller);
2761 			refresh();
2762 			pthread_mutex_unlock(&g_thread_lock);
2763 		}
2764 	}
2765 
2766 	del_panel(poller_panel);
2767 	delwin(poller_win);
2768 }
2769 
2770 static void *
2771 data_thread_routine(void *arg)
2772 {
2773 	int rc;
2774 	uint64_t refresh_rate;
2775 
2776 	while (1) {
2777 		pthread_mutex_lock(&g_thread_lock);
2778 		if (g_quit_app) {
2779 			pthread_mutex_unlock(&g_thread_lock);
2780 			break;
2781 		}
2782 
2783 		if (g_sleep_time == 0) {
2784 			/* Give display thread time to redraw all windows */
2785 			refresh_rate = SPDK_SEC_TO_USEC / 100;
2786 		} else {
2787 			refresh_rate = g_sleep_time * SPDK_SEC_TO_USEC;
2788 		}
2789 		pthread_mutex_unlock(&g_thread_lock);
2790 
2791 		/* Get data from RPC for each object type.
2792 		 * Start with cores since their number should not change. */
2793 		rc = get_cores_data();
2794 		if (rc) {
2795 			print_bottom_message("ERROR occurred while getting cores data");
2796 		}
2797 		rc = get_thread_data();
2798 		if (rc) {
2799 			print_bottom_message("ERROR occurred while getting threads data");
2800 		}
2801 
2802 		rc = get_pollers_data();
2803 		if (rc) {
2804 			print_bottom_message("ERROR occurred while getting pollers data");
2805 		}
2806 		rc = get_scheduler_data();
2807 		if (rc) {
2808 			print_bottom_message("ERROR occurred while getting scheduler data");
2809 		}
2810 
2811 		usleep(refresh_rate);
2812 	}
2813 
2814 	return NULL;
2815 }
2816 
2817 static void
2818 help_window_display(void)
2819 {
2820 	PANEL *help_panel;
2821 	WINDOW *help_win;
2822 	bool stop_loop = false;
2823 	int c;
2824 	uint64_t row = 1, col = 2, desc_second_row_col = 26, header_footer_col = 0;
2825 
2826 	help_win = newwin(HELP_WIN_HEIGHT, HELP_WIN_WIDTH,
2827 			  get_position_for_window(HELP_WIN_HEIGHT, g_max_row),
2828 			  get_position_for_window(HELP_WIN_WIDTH, g_max_col));
2829 	help_panel = new_panel(help_win);
2830 	top_panel(help_panel);
2831 	update_panels();
2832 	doupdate();
2833 
2834 	box(help_win, 0, 0);
2835 
2836 	/* Header */
2837 	print_in_middle(help_win, row, header_footer_col, HELP_WIN_WIDTH, "HELP", COLOR_PAIR(3));
2838 	mvwhline(help_win, 2, 1, ACS_HLINE, HELP_WIN_WIDTH - 2);
2839 	mvwaddch(help_win, 2, HELP_WIN_WIDTH, ACS_RTEE);
2840 	row = 3;
2841 
2842 	/* Content */
2843 	print_left(help_win, row, col, HELP_WIN_WIDTH, "MENU options", COLOR_PAIR(5));
2844 	print_left(help_win, ++row, ++col, HELP_WIN_WIDTH, "[q] Quit		- quit this application",
2845 		   COLOR_PAIR(10));
2846 	print_left(help_win, ++row, col,  HELP_WIN_WIDTH,
2847 		   "[Tab] Next tab	- switch to next tab", COLOR_PAIR(10));
2848 	print_left(help_win, ++row, col,  HELP_WIN_WIDTH,
2849 		   "[1-3] Select tab	- switch to THREADS, POLLERS or CORES tab", COLOR_PAIR(10));
2850 	print_left(help_win, ++row, col,  HELP_WIN_WIDTH,
2851 		   "[PgUp] Previous page	- scroll up to previous page", COLOR_PAIR(10));
2852 	print_left(help_win, ++row, col,  HELP_WIN_WIDTH,
2853 		   "[PgDown] Next page	- scroll down to next page", COLOR_PAIR(10));
2854 	print_left(help_win, ++row, col,  HELP_WIN_WIDTH,
2855 		   "[Up] Arrow key	- go to previous data row", COLOR_PAIR(10));
2856 	print_left(help_win, ++row, col,  HELP_WIN_WIDTH,
2857 		   "[Down] Arrow key	- go to next data row", COLOR_PAIR(10));
2858 	print_left(help_win, ++row, col,  HELP_WIN_WIDTH,
2859 		   "[Right] Arrow key	- go to second sorting window", COLOR_PAIR(10));
2860 	print_left(help_win, ++row, col,  HELP_WIN_WIDTH,
2861 		   "[Left] Arrow key	- close second sorting window", COLOR_PAIR(10));
2862 	print_left(help_win, ++row, col,  HELP_WIN_WIDTH,
2863 		   "[c] Columns		- choose data columns to display", COLOR_PAIR(10));
2864 	print_left(help_win, ++row, col,  HELP_WIN_WIDTH,
2865 		   "[s] Sorting		- change sorting by column", COLOR_PAIR(10));
2866 	print_left(help_win, ++row, col,  HELP_WIN_WIDTH,
2867 		   "[r] Refresh rate	- set refresh rate <0, 255> in seconds", COLOR_PAIR(10));
2868 	print_left(help_win, ++row, desc_second_row_col,  HELP_WIN_WIDTH, "that value in seconds",
2869 		   COLOR_PAIR(10));
2870 	print_left(help_win, ++row, col,  HELP_WIN_WIDTH,
2871 		   "[Enter] Item details	- show current data row details (Enter to open, Esc to close)",
2872 		   COLOR_PAIR(10));
2873 	print_left(help_win, ++row, col,  HELP_WIN_WIDTH,
2874 		   "[t] Total/Interval	- switch to display data measured from the start of SPDK", COLOR_PAIR(10));
2875 	print_left(help_win, ++row, desc_second_row_col,  HELP_WIN_WIDTH,
2876 		   "application or last refresh", COLOR_PAIR(10));
2877 	print_left(help_win, ++row, col,  HELP_WIN_WIDTH, "[h] Help		- show this help window",
2878 		   COLOR_PAIR(10));
2879 
2880 	/* Footer */
2881 	mvwhline(help_win, HELP_WIN_HEIGHT - 3, 1, ACS_HLINE, HELP_WIN_WIDTH - 2);
2882 	mvwaddch(help_win, HELP_WIN_HEIGHT - 3, HELP_WIN_WIDTH, ACS_RTEE);
2883 
2884 	print_in_middle(help_win, HELP_WIN_HEIGHT - 2, header_footer_col, HELP_WIN_WIDTH,
2885 			"[Esc] Close this window", COLOR_PAIR(10));
2886 
2887 	refresh();
2888 	wrefresh(help_win);
2889 
2890 	while (!stop_loop) {
2891 		c = wgetch(help_win);
2892 
2893 		switch (c) {
2894 		case 27: /* ESC */
2895 			stop_loop = true;
2896 			break;
2897 		default:
2898 			break;
2899 		}
2900 	}
2901 
2902 	del_panel(help_panel);
2903 	delwin(help_win);
2904 
2905 }
2906 
2907 static void
2908 show_stats(pthread_t *data_thread)
2909 {
2910 	const int CURRENT_PAGE_STR_LEN = 50;
2911 	long int time_last, time_dif;
2912 	struct timespec time_now;
2913 	int c;
2914 	uint8_t active_tab = THREADS_TAB;
2915 	uint8_t current_page = 0;
2916 	uint8_t max_pages = 1;
2917 	uint64_t i;
2918 	char current_page_str[CURRENT_PAGE_STR_LEN];
2919 	bool force_refresh = true;
2920 
2921 	clock_gettime(CLOCK_MONOTONIC, &time_now);
2922 	time_last = time_now.tv_sec;
2923 
2924 	switch_tab(THREADS_TAB);
2925 
2926 	while (1) {
2927 		check_resize_interface(active_tab, &current_page);
2928 
2929 		clock_gettime(CLOCK_MONOTONIC, &time_now);
2930 		time_dif = time_now.tv_sec - time_last;
2931 		if (time_dif < 0) {
2932 			time_dif = g_sleep_time;
2933 		}
2934 
2935 		if (time_dif >= g_sleep_time || force_refresh) {
2936 			time_last = time_now.tv_sec;
2937 			pthread_mutex_lock(&g_thread_lock);
2938 			max_pages = refresh_tab(active_tab, current_page);
2939 			pthread_mutex_unlock(&g_thread_lock);
2940 
2941 			snprintf(current_page_str, CURRENT_PAGE_STR_LEN - 1, "Page: %d/%d", current_page + 1, max_pages);
2942 			mvprintw(g_max_row - 1, 1, "%s", current_page_str);
2943 
2944 			refresh();
2945 		}
2946 
2947 		c = getch();
2948 		if (c == 'q') {
2949 			pthread_mutex_lock(&g_thread_lock);
2950 			g_quit_app = true;
2951 			pthread_mutex_unlock(&g_thread_lock);
2952 			break;
2953 		}
2954 
2955 		force_refresh = true;
2956 
2957 		switch (c) {
2958 		case '1':
2959 		case '2':
2960 		case '3':
2961 			active_tab = c - '1';
2962 			current_page = 0;
2963 			g_selected_row = 0;
2964 			switch_tab(active_tab);
2965 			break;
2966 		case '\t':
2967 			if (active_tab < NUMBER_OF_TABS - 1) {
2968 				active_tab++;
2969 			} else {
2970 				active_tab = THREADS_TAB;
2971 			}
2972 			g_selected_row = 0;
2973 			current_page = 0;
2974 			switch_tab(active_tab);
2975 			break;
2976 		case 's':
2977 			sort_type2(active_tab, COL_THREADS_NONE);
2978 			change_sorting(active_tab, 0, NULL);
2979 			break;
2980 		case 'c':
2981 			filter_columns(active_tab);
2982 			break;
2983 		case 'r':
2984 			change_refresh_rate();
2985 			break;
2986 		case 't':
2987 			g_interval_data = !g_interval_data;
2988 			break;
2989 		case KEY_NPAGE: /* PgDown */
2990 			if (current_page + 1 < max_pages) {
2991 				current_page++;
2992 			}
2993 			wclear(g_tabs[active_tab]);
2994 			g_selected_row = 0;
2995 			draw_tabs(active_tab, g_current_sort_col[active_tab], g_current_sort_col2[active_tab]);
2996 			break;
2997 		case KEY_PPAGE: /* PgUp */
2998 			if (current_page > 0) {
2999 				current_page--;
3000 			}
3001 			wclear(g_tabs[active_tab]);
3002 			g_selected_row = 0;
3003 			draw_tabs(active_tab, g_current_sort_col[active_tab], g_current_sort_col2[active_tab]);
3004 			break;
3005 		case KEY_UP: /* Arrow up */
3006 			if (g_selected_row > 0) {
3007 				g_selected_row--;
3008 			}
3009 			break;
3010 		case KEY_DOWN: /* Arrow down */
3011 			if (g_selected_row < g_max_selected_row) {
3012 				g_selected_row++;
3013 			}
3014 			break;
3015 		case 10: /* Enter */
3016 			if (active_tab == THREADS_TAB) {
3017 				show_thread(current_page, active_tab);
3018 			} else if (active_tab == CORES_TAB) {
3019 				show_core(current_page, active_tab);
3020 			} else if (active_tab == POLLERS_TAB) {
3021 				show_poller(current_page, active_tab);
3022 			}
3023 			/* After closing pop-up there would be unrefreshed parts
3024 			 * of the tab, so this is to refresh them */
3025 			draw_tabs(active_tab, g_current_sort_col[active_tab], g_current_sort_col2[active_tab]);
3026 			pthread_mutex_lock(&g_thread_lock);
3027 			max_pages = refresh_tab(active_tab, current_page);
3028 			pthread_mutex_unlock(&g_thread_lock);
3029 			snprintf(current_page_str, CURRENT_PAGE_STR_LEN - 1, "Page: %d/%d", current_page + 1, max_pages);
3030 			mvprintw(g_max_row - 1, 1, "%s", current_page_str);
3031 			top_panel(g_panels[active_tab]);
3032 			update_panels();
3033 			refresh();
3034 			break;
3035 		case 'h':
3036 			help_window_display();
3037 			break;
3038 		default:
3039 			force_refresh = false;
3040 			break;
3041 		}
3042 	}
3043 
3044 	pthread_join(*data_thread, NULL);
3045 
3046 	free_poller_history();
3047 
3048 	/* Free memory holding current data states before quitting application */
3049 	for (i = 0; i < g_last_pollers_count; i++) {
3050 		free_rpc_poller(&g_pollers_info[i]);
3051 	}
3052 	for (i = 0; i < g_last_threads_count; i++) {
3053 		free_rpc_threads_stats(&g_threads_info[i]);
3054 	}
3055 	free_rpc_core_info(g_cores_info, g_last_cores_count);
3056 	free_rpc_scheduler(&g_scheduler_info);
3057 }
3058 
3059 static void
3060 draw_interface(void)
3061 {
3062 	int i;
3063 	uint16_t required_size =  WINDOW_HEADER + 1;
3064 
3065 	getmaxyx(stdscr, g_max_row, g_max_col);
3066 	g_max_row = spdk_max(g_max_row, required_size);
3067 	g_data_win_size = g_max_row - required_size;
3068 	g_max_data_rows = g_max_row - WINDOW_HEADER;
3069 
3070 	g_menu_win = newwin(MENU_WIN_HEIGHT, g_max_col, g_max_row - MENU_WIN_HEIGHT - 1,
3071 			    MENU_WIN_LOCATION_COL);
3072 	assert(g_menu_win != NULL);
3073 	draw_menu_win();
3074 
3075 	for (i = 0; i < NUMBER_OF_TABS; i++) {
3076 		g_tab_win[i] = newwin(TAB_WIN_HEIGHT, g_max_col / NUMBER_OF_TABS - TABS_SPACING,
3077 				      TAB_WIN_LOCATION_ROW, g_max_col / NUMBER_OF_TABS * i + 1);
3078 		assert(g_tab_win[i] != NULL);
3079 		draw_tab_win(i);
3080 
3081 		g_tabs[i] = newwin(g_max_row - MENU_WIN_HEIGHT - TAB_WIN_HEIGHT - 2, g_max_col, TABS_LOCATION_ROW,
3082 				   TABS_LOCATION_COL);
3083 		draw_tabs(i, g_current_sort_col[i], g_current_sort_col2[i]);
3084 		g_panels[i] = new_panel(g_tabs[i]);
3085 		assert(g_panels[i] != NULL);
3086 	}
3087 
3088 	update_panels();
3089 	doupdate();
3090 }
3091 
3092 static void
3093 finish(int sig)
3094 {
3095 	/* End ncurses mode */
3096 	endwin();
3097 	spdk_jsonrpc_client_close(g_rpc_client);
3098 	exit(0);
3099 }
3100 
3101 static void
3102 setup_ncurses(void)
3103 {
3104 	clear();
3105 	noecho();
3106 	timeout(1);
3107 	curs_set(0);
3108 	keypad(stdscr, TRUE);
3109 	start_color();
3110 	init_pair(1, COLOR_BLACK, COLOR_GREEN);
3111 	init_pair(2, COLOR_BLACK, COLOR_WHITE);
3112 	init_pair(3, COLOR_YELLOW, COLOR_BLACK);
3113 	init_pair(4, COLOR_BLACK, COLOR_YELLOW);
3114 	init_pair(5, COLOR_GREEN, COLOR_BLACK);
3115 	init_pair(6, COLOR_RED, COLOR_BLACK);
3116 	init_pair(7, COLOR_BLUE, COLOR_BLACK);
3117 	init_pair(8, COLOR_RED, COLOR_WHITE);
3118 	init_pair(9, COLOR_BLUE, COLOR_WHITE);
3119 	init_pair(10, COLOR_WHITE, COLOR_BLACK);
3120 
3121 	if (has_colors() == FALSE) {
3122 		endwin();
3123 		printf("Your terminal does not support color\n");
3124 		exit(1);
3125 	}
3126 
3127 	/* Handle signals to exit gracefully cleaning up ncurses */
3128 	(void) signal(SIGINT, finish);
3129 	(void) signal(SIGPIPE, finish);
3130 	(void) signal(SIGABRT, finish);
3131 }
3132 
3133 static void
3134 usage(const char *program_name)
3135 {
3136 	printf("%s [options]", program_name);
3137 	printf("\n");
3138 	printf("options:\n");
3139 	printf(" -r <path>  RPC connect address (default: /var/tmp/spdk.sock)\n");
3140 	printf(" -h         show this usage\n");
3141 }
3142 
3143 static int
3144 rpc_decode_tick_rate(struct spdk_json_val *val, uint64_t *tick_rate)
3145 {
3146 	struct t_rate {
3147 		uint64_t tr;
3148 	};
3149 
3150 	const struct spdk_json_object_decoder rpc_tick_rate_decoder[] = {
3151 		{"tick_rate", offsetof(struct t_rate, tr), spdk_json_decode_uint64}
3152 	};
3153 
3154 	int rc;
3155 	struct t_rate tmp;
3156 
3157 	rc = spdk_json_decode_object_relaxed(val, rpc_tick_rate_decoder,
3158 					     SPDK_COUNTOF(rpc_tick_rate_decoder), &tmp);
3159 
3160 	*tick_rate = tmp.tr;
3161 
3162 	return rc;
3163 }
3164 
3165 static int
3166 wait_init(pthread_t *data_thread)
3167 {
3168 	struct spdk_jsonrpc_client_response *json_resp = NULL;
3169 	char *uninit_log = "Waiting for SPDK target application to initialize...",
3170 	      *uninit_error = "Unable to read SPDK application state!";
3171 	int c, max_col, rc = 0;
3172 	uint64_t tick_rate;
3173 
3174 	max_col = getmaxx(stdscr);
3175 	print_in_middle(stdscr, FIRST_DATA_ROW, 1, max_col, uninit_log, COLOR_PAIR(5));
3176 	rc = rpc_send_req("framework_wait_init", &json_resp);
3177 	if (rc) {
3178 		while (1) {
3179 			print_in_middle(stdscr, FIRST_DATA_ROW, 1, max_col, uninit_error, COLOR_PAIR(8));
3180 			c = getch();
3181 			if (c == 'q') {
3182 				return -1;
3183 			}
3184 		}
3185 	}
3186 
3187 	spdk_jsonrpc_client_free_response(json_resp);
3188 
3189 	rc = pthread_mutex_init(&g_thread_lock, NULL);
3190 	if (rc) {
3191 		fprintf(stderr, "mutex lock failed to initialize: %d\n", errno);
3192 		return -1;
3193 	}
3194 
3195 	memset(&g_threads_info, 0, sizeof(struct rpc_thread_info) * RPC_MAX_THREADS);
3196 	memset(&g_cores_info, 0, sizeof(struct rpc_core_info) * RPC_MAX_CORES);
3197 
3198 	/* Decode tick rate */
3199 	rc = rpc_send_req("framework_get_reactors", &json_resp);
3200 	if (rc) {
3201 		return rc;
3202 	}
3203 
3204 	if (rpc_decode_tick_rate(json_resp->result, &tick_rate)) {
3205 		spdk_jsonrpc_client_free_response(json_resp);
3206 		return -EINVAL;
3207 	}
3208 
3209 	spdk_jsonrpc_client_free_response(json_resp);
3210 
3211 	g_tick_rate = tick_rate;
3212 
3213 	/* This is to get first batch of data for display functions.
3214 	 * Since data thread makes RPC calls that take more time than
3215 	 * startup of display functions on main thread, without these
3216 	 * calls both threads would be subject to a race condition. */
3217 	rc = get_thread_data();
3218 	if (rc) {
3219 		return -1;
3220 	}
3221 
3222 	rc = get_pollers_data();
3223 	if (rc) {
3224 		return -1;
3225 	}
3226 
3227 	rc = get_cores_data();
3228 	if (rc) {
3229 		return -1;
3230 	}
3231 
3232 	rc = pthread_create(data_thread, NULL, &data_thread_routine, NULL);
3233 	if (rc) {
3234 		fprintf(stderr, "data thread creation failed: %d\n", errno);
3235 		return -1;
3236 	}
3237 	return 0;
3238 }
3239 
3240 int
3241 main(int argc, char **argv)
3242 {
3243 	int op, rc;
3244 	char *socket = SPDK_DEFAULT_RPC_ADDR;
3245 	pthread_t data_thread;
3246 
3247 	while ((op = getopt(argc, argv, "r:h")) != -1) {
3248 		switch (op) {
3249 		case 'r':
3250 			socket = optarg;
3251 			break;
3252 		default:
3253 			usage(argv[0]);
3254 			return op == 'h' ? 0 : 1;
3255 		}
3256 	}
3257 
3258 	g_rpc_client = spdk_jsonrpc_client_connect(socket, socket[0] == '/' ? AF_UNIX : AF_INET);
3259 	if (!g_rpc_client) {
3260 		fprintf(stderr, "spdk_jsonrpc_client_connect() failed: %d\n", errno);
3261 		return 1;
3262 	}
3263 
3264 	initscr();
3265 	init_str_len();
3266 	setup_ncurses();
3267 	draw_interface();
3268 
3269 	rc = wait_init(&data_thread);
3270 	if (!rc) {
3271 		show_stats(&data_thread);
3272 	}
3273 
3274 	finish(0);
3275 
3276 	return (0);
3277 }
3278