1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(C) 2020 Marvell International Ltd. 3 */ 4 5 #include <fnmatch.h> 6 #include <stdbool.h> 7 #include <stdlib.h> 8 9 #include <rte_common.h> 10 #include <rte_errno.h> 11 #include <rte_malloc.h> 12 13 #include "graph_private.h" 14 15 /* Capture all graphs of cluster */ 16 struct cluster { 17 rte_graph_t nb_graphs; 18 rte_graph_t size; 19 20 struct graph **graphs; 21 }; 22 23 /* Capture same node ID across cluster */ 24 struct cluster_node { 25 struct rte_graph_cluster_node_stats stat; 26 rte_node_t nb_nodes; 27 28 struct rte_node *nodes[]; 29 }; 30 31 struct __rte_cache_aligned rte_graph_cluster_stats { 32 /* Header */ 33 rte_graph_cluster_stats_cb_t fn; 34 uint32_t cluster_node_size; /* Size of struct cluster_node */ 35 rte_node_t max_nodes; 36 int socket_id; 37 void *cookie; 38 size_t sz; 39 40 struct cluster_node clusters[]; 41 }; 42 43 #define boarder_model_dispatch() \ 44 fprintf(f, "+-------------------------------+---------------+--------" \ 45 "-------+---------------+---------------+---------------+" \ 46 "---------------+---------------+-" \ 47 "----------+\n") 48 49 #define boarder() \ 50 fprintf(f, "+-------------------------------+---------------+--------" \ 51 "-------+---------------+---------------+---------------+-" \ 52 "----------+\n") 53 54 static inline void 55 print_banner_default(FILE *f) 56 { 57 boarder(); 58 fprintf(f, "%-32s%-16s%-16s%-16s%-16s%-16s%-16s\n", "|Node", "|calls", 59 "|objs", "|realloc_count", "|objs/call", "|objs/sec(10E6)", 60 "|cycles/call|"); 61 boarder(); 62 } 63 64 static inline void 65 print_banner_dispatch(FILE *f) 66 { 67 boarder_model_dispatch(); 68 fprintf(f, "%-32s%-16s%-16s%-16s%-16s%-16s%-16s%-16s%-16s\n", 69 "|Node", "|calls", 70 "|objs", "|sched objs", "|sched fail", 71 "|realloc_count", "|objs/call", "|objs/sec(10E6)", 72 "|cycles/call|"); 73 boarder_model_dispatch(); 74 } 75 76 static inline void 77 print_banner(FILE *f) 78 { 79 if (rte_graph_worker_model_get(STAILQ_FIRST(graph_list_head_get())->graph) == 80 RTE_GRAPH_MODEL_MCORE_DISPATCH) 81 print_banner_dispatch(f); 82 else 83 print_banner_default(f); 84 } 85 86 static inline void 87 print_node(FILE *f, const struct rte_graph_cluster_node_stats *stat) 88 { 89 double objs_per_call, objs_per_sec, cycles_per_call, ts_per_hz; 90 const uint64_t prev_calls = stat->prev_calls; 91 const uint64_t prev_objs = stat->prev_objs; 92 const uint64_t cycles = stat->cycles; 93 const uint64_t calls = stat->calls; 94 const uint64_t objs = stat->objs; 95 uint64_t call_delta; 96 97 call_delta = calls - prev_calls; 98 objs_per_call = 99 call_delta ? (double)((objs - prev_objs) / call_delta) : 0; 100 cycles_per_call = 101 call_delta ? (double)((cycles - stat->prev_cycles) / call_delta) 102 : 0; 103 ts_per_hz = (double)((stat->ts - stat->prev_ts) / stat->hz); 104 objs_per_sec = ts_per_hz ? (objs - prev_objs) / ts_per_hz : 0; 105 objs_per_sec /= 1000000; 106 107 if (rte_graph_worker_model_get(STAILQ_FIRST(graph_list_head_get())->graph) == 108 RTE_GRAPH_MODEL_MCORE_DISPATCH) { 109 fprintf(f, 110 "|%-31s|%-15" PRIu64 "|%-15" PRIu64 "|%-15" PRIu64 111 "|%-15" PRIu64 "|%-15" PRIu64 112 "|%-15.3f|%-15.6f|%-11.4f|\n", 113 stat->name, calls, objs, stat->dispatch.sched_objs, 114 stat->dispatch.sched_fail, stat->realloc_count, objs_per_call, 115 objs_per_sec, cycles_per_call); 116 } else { 117 fprintf(f, 118 "|%-31s|%-15" PRIu64 "|%-15" PRIu64 "|%-15" PRIu64 119 "|%-15.3f|%-15.6f|%-11.4f|\n", 120 stat->name, calls, objs, stat->realloc_count, objs_per_call, 121 objs_per_sec, cycles_per_call); 122 } 123 } 124 125 static int 126 graph_cluster_stats_cb(bool is_first, bool is_last, void *cookie, 127 const struct rte_graph_cluster_node_stats *stat) 128 { 129 FILE *f = cookie; 130 int model; 131 132 model = rte_graph_worker_model_get(STAILQ_FIRST(graph_list_head_get())->graph); 133 134 if (unlikely(is_first)) 135 print_banner(f); 136 if (stat->objs) 137 print_node(f, stat); 138 if (unlikely(is_last)) { 139 if (model == RTE_GRAPH_MODEL_MCORE_DISPATCH) 140 boarder_model_dispatch(); 141 else 142 boarder(); 143 } 144 145 return 0; 146 }; 147 148 static struct rte_graph_cluster_stats * 149 stats_mem_init(struct cluster *cluster, 150 const struct rte_graph_cluster_stats_param *prm) 151 { 152 size_t sz = sizeof(struct rte_graph_cluster_stats); 153 struct rte_graph_cluster_stats *stats; 154 rte_graph_cluster_stats_cb_t fn; 155 int socket_id = prm->socket_id; 156 uint32_t cluster_node_size; 157 158 /* Fix up callback */ 159 fn = prm->fn; 160 if (fn == NULL) 161 fn = graph_cluster_stats_cb; 162 163 cluster_node_size = sizeof(struct cluster_node); 164 /* For a given cluster, max nodes will be the max number of graphs */ 165 cluster_node_size += cluster->nb_graphs * sizeof(struct rte_node *); 166 cluster_node_size = RTE_ALIGN(cluster_node_size, RTE_CACHE_LINE_SIZE); 167 168 stats = realloc(NULL, sz); 169 if (stats) { 170 memset(stats, 0, sz); 171 stats->fn = fn; 172 stats->cluster_node_size = cluster_node_size; 173 stats->max_nodes = 0; 174 stats->socket_id = socket_id; 175 stats->cookie = prm->cookie; 176 stats->sz = sz; 177 } 178 179 return stats; 180 } 181 182 static int 183 stats_mem_populate(struct rte_graph_cluster_stats **stats_in, 184 struct rte_graph *graph, struct graph_node *graph_node) 185 { 186 struct rte_graph_cluster_stats *stats = *stats_in; 187 rte_node_t id = graph_node->node->id; 188 struct cluster_node *cluster; 189 struct rte_node *node; 190 rte_node_t count; 191 192 cluster = stats->clusters; 193 194 /* Iterate over cluster node array to find node ID match */ 195 for (count = 0; count < stats->max_nodes; count++) { 196 /* Found an existing node in the reel */ 197 if (cluster->stat.id == id) { 198 node = graph_node_id_to_ptr(graph, id); 199 if (node == NULL) 200 SET_ERR_JMP( 201 ENOENT, err, 202 "Failed to find node %s in graph %s", 203 graph_node->node->name, graph->name); 204 205 cluster->nodes[cluster->nb_nodes++] = node; 206 return 0; 207 } 208 cluster = RTE_PTR_ADD(cluster, stats->cluster_node_size); 209 } 210 211 /* Hey, it is a new node, allocate space for it in the reel */ 212 stats = realloc(stats, stats->sz + stats->cluster_node_size); 213 if (stats == NULL) 214 SET_ERR_JMP(ENOMEM, err, "Realloc failed"); 215 *stats_in = NULL; 216 217 /* Clear the new struct cluster_node area */ 218 cluster = RTE_PTR_ADD(stats, stats->sz), 219 memset(cluster, 0, stats->cluster_node_size); 220 memcpy(cluster->stat.name, graph_node->node->name, RTE_NODE_NAMESIZE); 221 cluster->stat.id = graph_node->node->id; 222 cluster->stat.hz = rte_get_timer_hz(); 223 node = graph_node_id_to_ptr(graph, id); 224 if (node == NULL) 225 SET_ERR_JMP(ENOENT, free, "Failed to find node %s in graph %s", 226 graph_node->node->name, graph->name); 227 cluster->nodes[cluster->nb_nodes++] = node; 228 229 stats->sz += stats->cluster_node_size; 230 stats->max_nodes++; 231 *stats_in = stats; 232 233 return 0; 234 free: 235 free(stats); 236 err: 237 return -rte_errno; 238 } 239 240 static void 241 stats_mem_fini(struct rte_graph_cluster_stats *stats) 242 { 243 free(stats); 244 } 245 246 static void 247 cluster_init(struct cluster *cluster) 248 { 249 memset(cluster, 0, sizeof(*cluster)); 250 } 251 252 static int 253 cluster_add(struct cluster *cluster, struct graph *graph) 254 { 255 rte_graph_t count; 256 size_t sz; 257 258 /* Skip the if graph is already added to cluster */ 259 for (count = 0; count < cluster->nb_graphs; count++) 260 if (cluster->graphs[count] == graph) 261 return 0; 262 263 /* Expand the cluster if required to store graph objects */ 264 if (cluster->nb_graphs + 1 > cluster->size) { 265 cluster->size = RTE_MAX(1, cluster->size * 2); 266 sz = sizeof(struct graph *) * cluster->size; 267 cluster->graphs = realloc(cluster->graphs, sz); 268 if (cluster->graphs == NULL) 269 SET_ERR_JMP(ENOMEM, free, "Failed to realloc"); 270 } 271 272 /* Add graph to cluster */ 273 cluster->graphs[cluster->nb_graphs++] = graph; 274 return 0; 275 276 free: 277 return -rte_errno; 278 } 279 280 static void 281 cluster_fini(struct cluster *cluster) 282 { 283 free(cluster->graphs); 284 } 285 286 static int 287 expand_pattern_to_cluster(struct cluster *cluster, const char *pattern) 288 { 289 struct graph_head *graph_head = graph_list_head_get(); 290 struct graph *graph; 291 bool found = false; 292 293 /* Check for pattern match */ 294 STAILQ_FOREACH(graph, graph_head, next) { 295 if (fnmatch(pattern, graph->name, 0) == 0) { 296 if (cluster_add(cluster, graph)) 297 goto fail; 298 found = true; 299 } 300 } 301 if (found == false) 302 SET_ERR_JMP(EFAULT, fail, "Pattern %s graph not found", 303 pattern); 304 305 return 0; 306 fail: 307 return -rte_errno; 308 } 309 310 struct rte_graph_cluster_stats * 311 rte_graph_cluster_stats_create(const struct rte_graph_cluster_stats_param *prm) 312 { 313 struct rte_graph_cluster_stats *stats, *rc = NULL; 314 struct graph_node *graph_node; 315 struct cluster cluster; 316 struct graph *graph; 317 const char *pattern; 318 rte_graph_t i; 319 320 /* Sanity checks */ 321 if (!rte_graph_has_stats_feature()) 322 SET_ERR_JMP(EINVAL, fail, "Stats feature is not enabled"); 323 324 if (prm == NULL) 325 SET_ERR_JMP(EINVAL, fail, "Invalid param"); 326 327 if (prm->graph_patterns == NULL || prm->nb_graph_patterns == 0) 328 SET_ERR_JMP(EINVAL, fail, "Invalid graph param"); 329 330 cluster_init(&cluster); 331 332 graph_spinlock_lock(); 333 /* Expand graph pattern and add the graph to the cluster */ 334 for (i = 0; i < prm->nb_graph_patterns; i++) { 335 pattern = prm->graph_patterns[i]; 336 if (expand_pattern_to_cluster(&cluster, pattern)) 337 goto bad_pattern; 338 } 339 340 /* Alloc the stats memory */ 341 stats = stats_mem_init(&cluster, prm); 342 if (stats == NULL) 343 SET_ERR_JMP(ENOMEM, bad_pattern, "Failed alloc stats memory"); 344 345 /* Iterate over M(Graph) x N (Nodes in graph) */ 346 for (i = 0; i < cluster.nb_graphs; i++) { 347 graph = cluster.graphs[i]; 348 STAILQ_FOREACH(graph_node, &graph->node_list, next) { 349 struct rte_graph *graph_fp = graph->graph; 350 if (stats_mem_populate(&stats, graph_fp, graph_node)) 351 goto realloc_fail; 352 } 353 } 354 355 /* Finally copy to hugepage memory to avoid pressure on rte_realloc */ 356 rc = rte_malloc_socket(NULL, stats->sz, 0, stats->socket_id); 357 if (rc) 358 rte_memcpy(rc, stats, stats->sz); 359 else 360 SET_ERR_JMP(ENOMEM, realloc_fail, "rte_malloc failed"); 361 362 realloc_fail: 363 stats_mem_fini(stats); 364 bad_pattern: 365 graph_spinlock_unlock(); 366 cluster_fini(&cluster); 367 fail: 368 return rc; 369 } 370 371 void 372 rte_graph_cluster_stats_destroy(struct rte_graph_cluster_stats *stat) 373 { 374 return rte_free(stat); 375 } 376 377 static inline void 378 cluster_node_arregate_stats(struct cluster_node *cluster) 379 { 380 uint64_t calls = 0, cycles = 0, objs = 0, realloc_count = 0; 381 struct rte_graph_cluster_node_stats *stat = &cluster->stat; 382 uint64_t sched_objs = 0, sched_fail = 0; 383 struct rte_node *node; 384 rte_node_t count; 385 int model; 386 387 model = rte_graph_worker_model_get(STAILQ_FIRST(graph_list_head_get())->graph); 388 for (count = 0; count < cluster->nb_nodes; count++) { 389 node = cluster->nodes[count]; 390 391 if (model == RTE_GRAPH_MODEL_MCORE_DISPATCH) { 392 sched_objs += node->dispatch.total_sched_objs; 393 sched_fail += node->dispatch.total_sched_fail; 394 } 395 396 calls += node->total_calls; 397 objs += node->total_objs; 398 cycles += node->total_cycles; 399 realloc_count += node->realloc_count; 400 } 401 402 stat->calls = calls; 403 stat->objs = objs; 404 stat->cycles = cycles; 405 406 if (model == RTE_GRAPH_MODEL_MCORE_DISPATCH) { 407 stat->dispatch.sched_objs = sched_objs; 408 stat->dispatch.sched_fail = sched_fail; 409 } 410 411 stat->ts = rte_get_timer_cycles(); 412 stat->realloc_count = realloc_count; 413 } 414 415 static inline void 416 cluster_node_store_prev_stats(struct cluster_node *cluster) 417 { 418 struct rte_graph_cluster_node_stats *stat = &cluster->stat; 419 420 stat->prev_ts = stat->ts; 421 stat->prev_calls = stat->calls; 422 stat->prev_objs = stat->objs; 423 stat->prev_cycles = stat->cycles; 424 } 425 426 void 427 rte_graph_cluster_stats_get(struct rte_graph_cluster_stats *stat, bool skip_cb) 428 { 429 struct cluster_node *cluster; 430 rte_node_t count; 431 int rc = 0; 432 433 cluster = stat->clusters; 434 435 for (count = 0; count < stat->max_nodes; count++) { 436 cluster_node_arregate_stats(cluster); 437 if (!skip_cb) 438 rc = stat->fn(!count, (count == stat->max_nodes - 1), 439 stat->cookie, &cluster->stat); 440 cluster_node_store_prev_stats(cluster); 441 if (rc) 442 break; 443 cluster = RTE_PTR_ADD(cluster, stat->cluster_node_size); 444 } 445 } 446 447 void 448 rte_graph_cluster_stats_reset(struct rte_graph_cluster_stats *stat) 449 { 450 struct cluster_node *cluster; 451 rte_node_t count; 452 453 cluster = stat->clusters; 454 455 for (count = 0; count < stat->max_nodes; count++) { 456 struct rte_graph_cluster_node_stats *node = &cluster->stat; 457 458 node->ts = 0; 459 node->calls = 0; 460 node->objs = 0; 461 node->cycles = 0; 462 node->prev_ts = 0; 463 node->prev_calls = 0; 464 node->prev_objs = 0; 465 node->prev_cycles = 0; 466 node->realloc_count = 0; 467 cluster = RTE_PTR_ADD(cluster, stat->cluster_node_size); 468 } 469 } 470