1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(C) 2020 Marvell International Ltd. 3 */ 4 5 #include <fnmatch.h> 6 #include <stdbool.h> 7 #include <stdlib.h> 8 9 #include <rte_common.h> 10 #include <rte_debug.h> 11 #include <rte_errno.h> 12 #include <rte_malloc.h> 13 #include <rte_memzone.h> 14 #include <rte_spinlock.h> 15 #include <rte_string_fns.h> 16 17 #include "graph_private.h" 18 #include "graph_pcap_private.h" 19 20 static struct graph_head graph_list = STAILQ_HEAD_INITIALIZER(graph_list); 21 static rte_spinlock_t graph_lock = RTE_SPINLOCK_INITIALIZER; 22 static rte_graph_t graph_id; 23 24 #define GRAPH_ID_CHECK(id) ID_CHECK(id, graph_id) 25 26 /* Private functions */ 27 struct graph_head * 28 graph_list_head_get(void) 29 { 30 return &graph_list; 31 } 32 33 rte_spinlock_t * 34 graph_spinlock_get(void) 35 { 36 return &graph_lock; 37 } 38 39 void 40 graph_spinlock_lock(void) 41 { 42 rte_spinlock_lock(graph_spinlock_get()); 43 } 44 45 void 46 graph_spinlock_unlock(void) 47 { 48 rte_spinlock_unlock(graph_spinlock_get()); 49 } 50 51 static int 52 graph_node_add(struct graph *graph, struct node *node) 53 { 54 struct graph_node *graph_node; 55 size_t sz; 56 57 /* Skip the duplicate nodes */ 58 STAILQ_FOREACH(graph_node, &graph->node_list, next) 59 if (strncmp(node->name, graph_node->node->name, 60 RTE_NODE_NAMESIZE) == 0) 61 return 0; 62 63 /* Allocate new graph node object */ 64 sz = sizeof(*graph_node) + node->nb_edges * sizeof(struct node *); 65 graph_node = calloc(1, sz); 66 67 if (graph_node == NULL) 68 SET_ERR_JMP(ENOMEM, free, "Failed to calloc %s object", 69 node->name); 70 71 /* Initialize the graph node */ 72 graph_node->node = node; 73 74 /* Add to graph node list */ 75 STAILQ_INSERT_TAIL(&graph->node_list, graph_node, next); 76 return 0; 77 78 free: 79 free(graph_node); 80 return -rte_errno; 81 } 82 83 static struct graph_node * 84 node_to_graph_node(struct graph *graph, struct node *node) 85 { 86 struct graph_node *graph_node; 87 88 STAILQ_FOREACH(graph_node, &graph->node_list, next) 89 if (graph_node->node == node) 90 return graph_node; 91 92 SET_ERR_JMP(ENODEV, fail, "Found isolated node %s", node->name); 93 fail: 94 return NULL; 95 } 96 97 static int 98 graph_node_edges_add(struct graph *graph) 99 { 100 struct graph_node *graph_node; 101 struct node *adjacency; 102 const char *next; 103 rte_edge_t i; 104 105 STAILQ_FOREACH(graph_node, &graph->node_list, next) { 106 for (i = 0; i < graph_node->node->nb_edges; i++) { 107 next = graph_node->node->next_nodes[i]; 108 adjacency = node_from_name(next); 109 if (adjacency == NULL) 110 SET_ERR_JMP(EINVAL, fail, 111 "Node %s not registered", next); 112 if (graph_node_add(graph, adjacency)) 113 goto fail; 114 } 115 } 116 return 0; 117 fail: 118 return -rte_errno; 119 } 120 121 static int 122 graph_adjacency_list_update(struct graph *graph) 123 { 124 struct graph_node *graph_node, *tmp; 125 struct node *adjacency; 126 const char *next; 127 rte_edge_t i; 128 129 STAILQ_FOREACH(graph_node, &graph->node_list, next) { 130 for (i = 0; i < graph_node->node->nb_edges; i++) { 131 next = graph_node->node->next_nodes[i]; 132 adjacency = node_from_name(next); 133 if (adjacency == NULL) 134 SET_ERR_JMP(EINVAL, fail, 135 "Node %s not registered", next); 136 tmp = node_to_graph_node(graph, adjacency); 137 if (tmp == NULL) 138 goto fail; 139 graph_node->adjacency_list[i] = tmp; 140 } 141 } 142 143 return 0; 144 fail: 145 return -rte_errno; 146 } 147 148 static int 149 expand_pattern_to_node(struct graph *graph, const char *pattern) 150 { 151 struct node_head *node_head = node_list_head_get(); 152 bool found = false; 153 struct node *node; 154 155 /* Check for pattern match */ 156 STAILQ_FOREACH(node, node_head, next) { 157 if (fnmatch(pattern, node->name, 0) == 0) { 158 if (graph_node_add(graph, node)) 159 goto fail; 160 found = true; 161 } 162 } 163 if (found == false) 164 SET_ERR_JMP(EFAULT, fail, "Pattern %s node not found", pattern); 165 166 return 0; 167 fail: 168 return -rte_errno; 169 } 170 171 static void 172 graph_cleanup(struct graph *graph) 173 { 174 struct graph_node *graph_node; 175 176 while (!STAILQ_EMPTY(&graph->node_list)) { 177 graph_node = STAILQ_FIRST(&graph->node_list); 178 STAILQ_REMOVE_HEAD(&graph->node_list, next); 179 free(graph_node); 180 } 181 } 182 183 static int 184 graph_node_init(struct graph *graph) 185 { 186 struct graph_node *graph_node; 187 const char *name; 188 int rc; 189 190 STAILQ_FOREACH(graph_node, &graph->node_list, next) { 191 if (graph_node->node->init) { 192 name = graph_node->node->name; 193 rc = graph_node->node->init( 194 graph->graph, 195 graph_node_name_to_ptr(graph->graph, name)); 196 if (rc) 197 SET_ERR_JMP(rc, err, "Node %s init() failed", 198 name); 199 } 200 } 201 202 return 0; 203 err: 204 return -rte_errno; 205 } 206 207 static void 208 graph_node_fini(struct graph *graph) 209 { 210 struct graph_node *graph_node; 211 212 STAILQ_FOREACH(graph_node, &graph->node_list, next) 213 if (graph_node->node->fini) 214 graph_node->node->fini( 215 graph->graph, 216 graph_node_name_to_ptr(graph->graph, 217 graph_node->node->name)); 218 } 219 220 static struct rte_graph * 221 graph_mem_fixup_node_ctx(struct rte_graph *graph) 222 { 223 struct rte_node *node; 224 struct node *node_db; 225 rte_graph_off_t off; 226 rte_node_t count; 227 const char *name; 228 229 rte_graph_foreach_node(count, off, graph, node) { 230 if (node->parent_id == RTE_NODE_ID_INVALID) /* Static node */ 231 name = node->name; 232 else /* Cloned node */ 233 name = node->parent; 234 235 node_db = node_from_name(name); 236 if (node_db == NULL) 237 SET_ERR_JMP(ENOLINK, fail, "Node %s not found", name); 238 239 if (graph->pcap_enable) { 240 node->process = graph_pcap_dispatch; 241 node->original_process = node_db->process; 242 } else 243 node->process = node_db->process; 244 } 245 246 return graph; 247 fail: 248 return NULL; 249 } 250 251 static struct rte_graph * 252 graph_mem_fixup_secondary(struct rte_graph *graph) 253 { 254 if (graph == NULL || rte_eal_process_type() == RTE_PROC_PRIMARY) 255 return graph; 256 257 if (graph_pcap_file_open(graph->pcap_filename) || graph_pcap_mp_init()) 258 graph_pcap_exit(graph); 259 260 return graph_mem_fixup_node_ctx(graph); 261 } 262 263 static bool 264 graph_src_node_avail(struct graph *graph) 265 { 266 struct graph_node *graph_node; 267 268 STAILQ_FOREACH(graph_node, &graph->node_list, next) 269 if ((graph_node->node->flags & RTE_NODE_SOURCE_F) && 270 (graph_node->node->lcore_id == RTE_MAX_LCORE || 271 graph->lcore_id == graph_node->node->lcore_id)) 272 return true; 273 274 return false; 275 } 276 277 int 278 rte_graph_model_mcore_dispatch_core_bind(rte_graph_t id, int lcore) 279 { 280 struct graph *graph; 281 282 GRAPH_ID_CHECK(id); 283 if (!rte_lcore_is_enabled(lcore)) 284 SET_ERR_JMP(ENOLINK, fail, "lcore %d not enabled", lcore); 285 286 STAILQ_FOREACH(graph, &graph_list, next) 287 if (graph->id == id) 288 break; 289 290 if (graph->graph->model == RTE_GRAPH_MODEL_MCORE_DISPATCH) 291 goto fail; 292 293 graph->lcore_id = lcore; 294 graph->graph->dispatch.lcore_id = graph->lcore_id; 295 graph->socket = rte_lcore_to_socket_id(lcore); 296 297 /* check the availability of source node */ 298 if (!graph_src_node_avail(graph)) 299 graph->graph->head = 0; 300 301 return 0; 302 303 fail: 304 return -rte_errno; 305 } 306 307 void 308 rte_graph_model_mcore_dispatch_core_unbind(rte_graph_t id) 309 { 310 struct graph *graph; 311 312 GRAPH_ID_CHECK(id); 313 STAILQ_FOREACH(graph, &graph_list, next) 314 if (graph->id == id) 315 break; 316 317 graph->lcore_id = RTE_MAX_LCORE; 318 graph->graph->dispatch.lcore_id = RTE_MAX_LCORE; 319 320 fail: 321 return; 322 } 323 324 struct rte_graph * 325 rte_graph_lookup(const char *name) 326 { 327 const struct rte_memzone *mz; 328 struct rte_graph *rc = NULL; 329 330 mz = rte_memzone_lookup(name); 331 if (mz) 332 rc = mz->addr; 333 334 return graph_mem_fixup_secondary(rc); 335 } 336 337 rte_graph_t 338 rte_graph_create(const char *name, struct rte_graph_param *prm) 339 { 340 rte_node_t src_node_count; 341 struct graph *graph; 342 const char *pattern; 343 uint16_t i; 344 345 graph_spinlock_lock(); 346 347 /* Check arguments sanity */ 348 if (prm == NULL) 349 SET_ERR_JMP(EINVAL, fail, "Param should not be NULL"); 350 351 if (name == NULL) 352 SET_ERR_JMP(EINVAL, fail, "Graph name should not be NULL"); 353 354 /* Check for existence of duplicate graph */ 355 STAILQ_FOREACH(graph, &graph_list, next) 356 if (strncmp(name, graph->name, RTE_GRAPH_NAMESIZE) == 0) 357 SET_ERR_JMP(EEXIST, fail, "Found duplicate graph %s", 358 name); 359 360 /* Create graph object */ 361 graph = calloc(1, sizeof(*graph)); 362 if (graph == NULL) 363 SET_ERR_JMP(ENOMEM, fail, "Failed to calloc graph object"); 364 365 /* Initialize the graph object */ 366 STAILQ_INIT(&graph->node_list); 367 if (rte_strscpy(graph->name, name, RTE_GRAPH_NAMESIZE) < 0) 368 SET_ERR_JMP(E2BIG, free, "Too big name=%s", name); 369 370 /* Expand node pattern and add the nodes to the graph */ 371 for (i = 0; i < prm->nb_node_patterns; i++) { 372 pattern = prm->node_patterns[i]; 373 if (expand_pattern_to_node(graph, pattern)) 374 goto graph_cleanup; 375 } 376 377 /* Go over all the nodes edges and add them to the graph */ 378 if (graph_node_edges_add(graph)) 379 goto graph_cleanup; 380 381 /* Update adjacency list of all nodes in the graph */ 382 if (graph_adjacency_list_update(graph)) 383 goto graph_cleanup; 384 385 /* Make sure at least a source node present in the graph */ 386 src_node_count = graph_src_nodes_count(graph); 387 if (src_node_count == 0) 388 goto graph_cleanup; 389 390 /* Make sure no node is pointing to source node */ 391 if (graph_node_has_edge_to_src_node(graph)) 392 goto graph_cleanup; 393 394 /* Don't allow node has loop to self */ 395 if (graph_node_has_loop_edge(graph)) 396 goto graph_cleanup; 397 398 /* Do BFS from src nodes on the graph to find isolated nodes */ 399 if (graph_has_isolated_node(graph)) 400 goto graph_cleanup; 401 402 /* Initialize pcap config. */ 403 graph_pcap_enable(prm->pcap_enable); 404 405 /* Initialize graph object */ 406 graph->socket = prm->socket_id; 407 graph->src_node_count = src_node_count; 408 graph->node_count = graph_nodes_count(graph); 409 graph->id = graph_id; 410 graph->parent_id = RTE_GRAPH_ID_INVALID; 411 graph->lcore_id = RTE_MAX_LCORE; 412 graph->num_pkt_to_capture = prm->num_pkt_to_capture; 413 if (prm->pcap_filename) 414 rte_strscpy(graph->pcap_filename, prm->pcap_filename, RTE_GRAPH_PCAP_FILE_SZ); 415 416 /* Allocate the Graph fast path memory and populate the data */ 417 if (graph_fp_mem_create(graph)) 418 goto graph_cleanup; 419 420 /* Call init() of the all the nodes in the graph */ 421 if (graph_node_init(graph)) 422 goto graph_mem_destroy; 423 424 /* All good, Lets add the graph to the list */ 425 graph_id++; 426 STAILQ_INSERT_TAIL(&graph_list, graph, next); 427 428 graph_spinlock_unlock(); 429 return graph->id; 430 431 graph_mem_destroy: 432 graph_fp_mem_destroy(graph); 433 graph_cleanup: 434 graph_cleanup(graph); 435 free: 436 free(graph); 437 fail: 438 graph_spinlock_unlock(); 439 return RTE_GRAPH_ID_INVALID; 440 } 441 442 int 443 rte_graph_destroy(rte_graph_t id) 444 { 445 struct graph *graph, *tmp; 446 int rc = -ENOENT; 447 448 graph_spinlock_lock(); 449 450 graph = STAILQ_FIRST(&graph_list); 451 while (graph != NULL) { 452 tmp = STAILQ_NEXT(graph, next); 453 if (graph->id == id) { 454 /* Call fini() of the all the nodes in the graph */ 455 graph_node_fini(graph); 456 /* Destroy graph fast path memory */ 457 rc = graph_fp_mem_destroy(graph); 458 if (rc) 459 SET_ERR_JMP(rc, done, "Graph %s destroy failed", 460 graph->name); 461 462 graph_cleanup(graph); 463 STAILQ_REMOVE(&graph_list, graph, graph, next); 464 free(graph); 465 graph_id--; 466 goto done; 467 } 468 graph = tmp; 469 } 470 done: 471 graph_spinlock_unlock(); 472 return rc; 473 } 474 475 static rte_graph_t 476 graph_clone(struct graph *parent_graph, const char *name, struct rte_graph_param *prm) 477 { 478 struct graph_node *graph_node; 479 struct graph *graph; 480 RTE_SET_USED(prm); 481 482 graph_spinlock_lock(); 483 484 /* Don't allow to clone a node from a cloned graph */ 485 if (parent_graph->parent_id != RTE_GRAPH_ID_INVALID) 486 SET_ERR_JMP(EEXIST, fail, "A cloned graph is not allowed to be cloned"); 487 488 /* Create graph object */ 489 graph = calloc(1, sizeof(*graph)); 490 if (graph == NULL) 491 SET_ERR_JMP(ENOMEM, fail, "Failed to calloc cloned graph object"); 492 493 /* Naming ceremony of the new graph. name is node->name + "-" + name */ 494 if (clone_name(graph->name, parent_graph->name, name)) 495 goto free; 496 497 /* Check for existence of duplicate graph */ 498 if (rte_graph_from_name(graph->name) != RTE_GRAPH_ID_INVALID) 499 SET_ERR_JMP(EEXIST, free, "Found duplicate graph %s", 500 graph->name); 501 502 /* Clone nodes from parent graph firstly */ 503 STAILQ_INIT(&graph->node_list); 504 STAILQ_FOREACH(graph_node, &parent_graph->node_list, next) { 505 if (graph_node_add(graph, graph_node->node)) 506 goto graph_cleanup; 507 } 508 509 /* Just update adjacency list of all nodes in the graph */ 510 if (graph_adjacency_list_update(graph)) 511 goto graph_cleanup; 512 513 /* Initialize the graph object */ 514 graph->src_node_count = parent_graph->src_node_count; 515 graph->node_count = parent_graph->node_count; 516 graph->parent_id = parent_graph->id; 517 graph->lcore_id = parent_graph->lcore_id; 518 graph->socket = parent_graph->socket; 519 graph->id = graph_id; 520 521 /* Allocate the Graph fast path memory and populate the data */ 522 if (graph_fp_mem_create(graph)) 523 goto graph_cleanup; 524 525 /* Clone the graph model */ 526 graph->graph->model = parent_graph->graph->model; 527 528 /* Call init() of the all the nodes in the graph */ 529 if (graph_node_init(graph)) 530 goto graph_mem_destroy; 531 532 /* All good, Lets add the graph to the list */ 533 graph_id++; 534 STAILQ_INSERT_TAIL(&graph_list, graph, next); 535 536 graph_spinlock_unlock(); 537 return graph->id; 538 539 graph_mem_destroy: 540 graph_fp_mem_destroy(graph); 541 graph_cleanup: 542 graph_cleanup(graph); 543 free: 544 free(graph); 545 fail: 546 graph_spinlock_unlock(); 547 return RTE_GRAPH_ID_INVALID; 548 } 549 550 rte_graph_t 551 rte_graph_clone(rte_graph_t id, const char *name, struct rte_graph_param *prm) 552 { 553 struct graph *graph; 554 555 GRAPH_ID_CHECK(id); 556 STAILQ_FOREACH(graph, &graph_list, next) 557 if (graph->id == id) 558 return graph_clone(graph, name, prm); 559 560 fail: 561 return RTE_GRAPH_ID_INVALID; 562 } 563 564 rte_graph_t 565 rte_graph_from_name(const char *name) 566 { 567 struct graph *graph; 568 569 STAILQ_FOREACH(graph, &graph_list, next) 570 if (strncmp(graph->name, name, RTE_GRAPH_NAMESIZE) == 0) 571 return graph->id; 572 573 return RTE_GRAPH_ID_INVALID; 574 } 575 576 char * 577 rte_graph_id_to_name(rte_graph_t id) 578 { 579 struct graph *graph; 580 581 GRAPH_ID_CHECK(id); 582 STAILQ_FOREACH(graph, &graph_list, next) 583 if (graph->id == id) 584 return graph->name; 585 586 fail: 587 return NULL; 588 } 589 590 struct rte_node * 591 rte_graph_node_get(rte_graph_t gid, uint32_t nid) 592 { 593 struct rte_node *node; 594 struct graph *graph; 595 rte_graph_off_t off; 596 rte_node_t count; 597 598 GRAPH_ID_CHECK(gid); 599 STAILQ_FOREACH(graph, &graph_list, next) 600 if (graph->id == gid) { 601 rte_graph_foreach_node(count, off, graph->graph, 602 node) { 603 if (node->id == nid) 604 return node; 605 } 606 break; 607 } 608 fail: 609 return NULL; 610 } 611 612 struct rte_node * 613 rte_graph_node_get_by_name(const char *graph_name, const char *node_name) 614 { 615 struct rte_node *node; 616 struct graph *graph; 617 rte_graph_off_t off; 618 rte_node_t count; 619 620 STAILQ_FOREACH(graph, &graph_list, next) 621 if (!strncmp(graph->name, graph_name, RTE_GRAPH_NAMESIZE)) { 622 rte_graph_foreach_node(count, off, graph->graph, 623 node) { 624 if (!strncmp(node->name, node_name, 625 RTE_NODE_NAMESIZE)) 626 return node; 627 } 628 break; 629 } 630 631 return NULL; 632 } 633 634 void __rte_noinline 635 __rte_node_stream_alloc(struct rte_graph *graph, struct rte_node *node) 636 { 637 uint16_t size = node->size; 638 639 RTE_VERIFY(size != UINT16_MAX); 640 /* Allocate double amount of size to avoid immediate realloc */ 641 size = RTE_MIN(UINT16_MAX, RTE_MAX(RTE_GRAPH_BURST_SIZE, size * 2)); 642 node->objs = rte_realloc_socket(node->objs, size * sizeof(void *), 643 RTE_CACHE_LINE_SIZE, graph->socket); 644 RTE_VERIFY(node->objs); 645 node->size = size; 646 node->realloc_count++; 647 } 648 649 void __rte_noinline 650 __rte_node_stream_alloc_size(struct rte_graph *graph, struct rte_node *node, 651 uint16_t req_size) 652 { 653 uint16_t size = node->size; 654 655 RTE_VERIFY(size != UINT16_MAX); 656 /* Allocate double amount of size to avoid immediate realloc */ 657 size = RTE_MIN(UINT16_MAX, RTE_MAX(RTE_GRAPH_BURST_SIZE, req_size * 2)); 658 node->objs = rte_realloc_socket(node->objs, size * sizeof(void *), 659 RTE_CACHE_LINE_SIZE, graph->socket); 660 RTE_VERIFY(node->objs); 661 node->size = size; 662 node->realloc_count++; 663 } 664 665 static int 666 graph_to_dot(FILE *f, struct graph *graph) 667 { 668 const char *src_edge_color = " [color=blue]\n"; 669 const char *edge_color = "\n"; 670 struct graph_node *graph_node; 671 char *node_name; 672 rte_edge_t i; 673 int rc; 674 675 rc = fprintf(f, "Digraph %s {\n\trankdir=LR;\n", graph->name); 676 if (rc < 0) 677 goto end; 678 679 STAILQ_FOREACH(graph_node, &graph->node_list, next) { 680 node_name = graph_node->node->name; 681 for (i = 0; i < graph_node->node->nb_edges; i++) { 682 rc = fprintf(f, "\t\"%s\"->\"%s\"%s", node_name, 683 graph_node->adjacency_list[i]->node->name, 684 graph_node->node->flags & RTE_NODE_SOURCE_F 685 ? src_edge_color 686 : edge_color); 687 if (rc < 0) 688 goto end; 689 } 690 } 691 rc = fprintf(f, "}\n"); 692 if (rc < 0) 693 goto end; 694 695 return 0; 696 end: 697 rte_errno = EBADF; 698 return -rte_errno; 699 } 700 701 int 702 rte_graph_export(const char *name, FILE *f) 703 { 704 struct graph *graph; 705 int rc = ENOENT; 706 707 STAILQ_FOREACH(graph, &graph_list, next) { 708 if (strncmp(graph->name, name, RTE_GRAPH_NAMESIZE) == 0) { 709 rc = graph_to_dot(f, graph); 710 goto end; 711 } 712 } 713 end: 714 return -rc; 715 } 716 717 static void 718 graph_scan_dump(FILE *f, rte_graph_t id, bool all) 719 { 720 struct graph *graph; 721 722 RTE_VERIFY(f); 723 GRAPH_ID_CHECK(id); 724 725 STAILQ_FOREACH(graph, &graph_list, next) { 726 if (all == true) { 727 graph_dump(f, graph); 728 } else if (graph->id == id) { 729 graph_dump(f, graph); 730 return; 731 } 732 } 733 fail: 734 return; 735 } 736 737 void 738 rte_graph_dump(FILE *f, rte_graph_t id) 739 { 740 graph_scan_dump(f, id, false); 741 } 742 743 void 744 rte_graph_list_dump(FILE *f) 745 { 746 graph_scan_dump(f, 0, true); 747 } 748 749 rte_graph_t 750 rte_graph_max_count(void) 751 { 752 return graph_id; 753 } 754 755 RTE_LOG_REGISTER_DEFAULT(rte_graph_logtype, INFO); 756