1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(C) 2020 Marvell International Ltd. 3 */ 4 5 #include <fnmatch.h> 6 #include <stdbool.h> 7 #include <stdlib.h> 8 9 #include <rte_common.h> 10 #include <rte_debug.h> 11 #include <rte_errno.h> 12 #include <rte_malloc.h> 13 #include <rte_memzone.h> 14 #include <rte_spinlock.h> 15 #include <rte_string_fns.h> 16 17 #include "graph_private.h" 18 #include "graph_pcap_private.h" 19 20 static struct graph_head graph_list = STAILQ_HEAD_INITIALIZER(graph_list); 21 static rte_spinlock_t graph_lock = RTE_SPINLOCK_INITIALIZER; 22 static rte_graph_t graph_id; 23 24 #define GRAPH_ID_CHECK(id) ID_CHECK(id, graph_id) 25 26 /* Private functions */ 27 struct graph_head * 28 graph_list_head_get(void) 29 { 30 return &graph_list; 31 } 32 33 rte_spinlock_t * 34 graph_spinlock_get(void) 35 { 36 return &graph_lock; 37 } 38 39 void 40 graph_spinlock_lock(void) 41 { 42 rte_spinlock_lock(graph_spinlock_get()); 43 } 44 45 void 46 graph_spinlock_unlock(void) 47 { 48 rte_spinlock_unlock(graph_spinlock_get()); 49 } 50 51 static int 52 graph_node_add(struct graph *graph, struct node *node) 53 { 54 struct graph_node *graph_node; 55 size_t sz; 56 57 /* Skip the duplicate nodes */ 58 STAILQ_FOREACH(graph_node, &graph->node_list, next) 59 if (strncmp(node->name, graph_node->node->name, 60 RTE_NODE_NAMESIZE) == 0) 61 return 0; 62 63 /* Allocate new graph node object */ 64 sz = sizeof(*graph_node) + node->nb_edges * sizeof(struct node *); 65 graph_node = calloc(1, sz); 66 67 if (graph_node == NULL) 68 SET_ERR_JMP(ENOMEM, free, "Failed to calloc %s object", 69 node->name); 70 71 /* Initialize the graph node */ 72 graph_node->node = node; 73 74 /* Add to graph node list */ 75 STAILQ_INSERT_TAIL(&graph->node_list, graph_node, next); 76 return 0; 77 78 free: 79 free(graph_node); 80 return -rte_errno; 81 } 82 83 static struct graph_node * 84 node_to_graph_node(struct graph *graph, struct node *node) 85 { 86 struct graph_node *graph_node; 87 88 STAILQ_FOREACH(graph_node, &graph->node_list, next) 89 if (graph_node->node == node) 90 return graph_node; 91 92 SET_ERR_JMP(ENODEV, fail, "Found isolated node %s", node->name); 93 fail: 94 return NULL; 95 } 96 97 static int 98 graph_node_edges_add(struct graph *graph) 99 { 100 struct graph_node *graph_node; 101 struct node *adjacency; 102 const char *next; 103 rte_edge_t i; 104 105 STAILQ_FOREACH(graph_node, &graph->node_list, next) { 106 for (i = 0; i < graph_node->node->nb_edges; i++) { 107 next = graph_node->node->next_nodes[i]; 108 adjacency = node_from_name(next); 109 if (adjacency == NULL) 110 SET_ERR_JMP(EINVAL, fail, 111 "Node %s not registered", next); 112 if (graph_node_add(graph, adjacency)) 113 goto fail; 114 } 115 } 116 return 0; 117 fail: 118 return -rte_errno; 119 } 120 121 static int 122 graph_adjacency_list_update(struct graph *graph) 123 { 124 struct graph_node *graph_node, *tmp; 125 struct node *adjacency; 126 const char *next; 127 rte_edge_t i; 128 129 STAILQ_FOREACH(graph_node, &graph->node_list, next) { 130 for (i = 0; i < graph_node->node->nb_edges; i++) { 131 next = graph_node->node->next_nodes[i]; 132 adjacency = node_from_name(next); 133 if (adjacency == NULL) 134 SET_ERR_JMP(EINVAL, fail, 135 "Node %s not registered", next); 136 tmp = node_to_graph_node(graph, adjacency); 137 if (tmp == NULL) 138 goto fail; 139 graph_node->adjacency_list[i] = tmp; 140 } 141 } 142 143 return 0; 144 fail: 145 return -rte_errno; 146 } 147 148 static int 149 expand_pattern_to_node(struct graph *graph, const char *pattern) 150 { 151 struct node_head *node_head = node_list_head_get(); 152 bool found = false; 153 struct node *node; 154 155 /* Check for pattern match */ 156 STAILQ_FOREACH(node, node_head, next) { 157 if (fnmatch(pattern, node->name, 0) == 0) { 158 if (graph_node_add(graph, node)) 159 goto fail; 160 found = true; 161 } 162 } 163 if (found == false) 164 SET_ERR_JMP(EFAULT, fail, "Pattern %s node not found", pattern); 165 166 return 0; 167 fail: 168 return -rte_errno; 169 } 170 171 static void 172 graph_cleanup(struct graph *graph) 173 { 174 struct graph_node *graph_node; 175 176 while (!STAILQ_EMPTY(&graph->node_list)) { 177 graph_node = STAILQ_FIRST(&graph->node_list); 178 STAILQ_REMOVE_HEAD(&graph->node_list, next); 179 free(graph_node); 180 } 181 } 182 183 static int 184 graph_node_init(struct graph *graph) 185 { 186 struct graph_node *graph_node; 187 const char *name; 188 int rc; 189 190 STAILQ_FOREACH(graph_node, &graph->node_list, next) { 191 if (graph_node->node->init) { 192 name = graph_node->node->name; 193 rc = graph_node->node->init( 194 graph->graph, 195 graph_node_name_to_ptr(graph->graph, name)); 196 if (rc) 197 SET_ERR_JMP(rc, err, "Node %s init() failed", 198 name); 199 } 200 } 201 202 return 0; 203 err: 204 return -rte_errno; 205 } 206 207 static void 208 graph_node_fini(struct graph *graph) 209 { 210 struct graph_node *graph_node; 211 212 STAILQ_FOREACH(graph_node, &graph->node_list, next) 213 if (graph_node->node->fini) 214 graph_node->node->fini( 215 graph->graph, 216 graph_node_name_to_ptr(graph->graph, 217 graph_node->node->name)); 218 } 219 220 static struct rte_graph * 221 graph_mem_fixup_node_ctx(struct rte_graph *graph) 222 { 223 struct rte_node *node; 224 struct node *node_db; 225 rte_graph_off_t off; 226 rte_node_t count; 227 const char *name; 228 229 rte_graph_foreach_node(count, off, graph, node) { 230 if (node->parent_id == RTE_NODE_ID_INVALID) /* Static node */ 231 name = node->name; 232 else /* Cloned node */ 233 name = node->parent; 234 235 node_db = node_from_name(name); 236 if (node_db == NULL) 237 SET_ERR_JMP(ENOLINK, fail, "Node %s not found", name); 238 239 if (graph->pcap_enable) { 240 node->process = graph_pcap_dispatch; 241 node->original_process = node_db->process; 242 } else 243 node->process = node_db->process; 244 } 245 246 return graph; 247 fail: 248 return NULL; 249 } 250 251 static struct rte_graph * 252 graph_mem_fixup_secondary(struct rte_graph *graph) 253 { 254 if (graph == NULL || rte_eal_process_type() == RTE_PROC_PRIMARY) 255 return graph; 256 257 if (graph_pcap_file_open(graph->pcap_filename) || graph_pcap_mp_init()) 258 graph_pcap_exit(graph); 259 260 return graph_mem_fixup_node_ctx(graph); 261 } 262 263 static bool 264 graph_src_node_avail(struct graph *graph) 265 { 266 struct graph_node *graph_node; 267 268 STAILQ_FOREACH(graph_node, &graph->node_list, next) 269 if ((graph_node->node->flags & RTE_NODE_SOURCE_F) && 270 (graph_node->node->lcore_id == RTE_MAX_LCORE || 271 graph->lcore_id == graph_node->node->lcore_id)) 272 return true; 273 274 return false; 275 } 276 277 int 278 rte_graph_model_mcore_dispatch_core_bind(rte_graph_t id, int lcore) 279 { 280 struct graph *graph; 281 282 GRAPH_ID_CHECK(id); 283 if (!rte_lcore_is_enabled(lcore)) 284 SET_ERR_JMP(ENOLINK, fail, "lcore %d not enabled", lcore); 285 286 STAILQ_FOREACH(graph, &graph_list, next) 287 if (graph->id == id) 288 break; 289 290 if (graph->graph->model != RTE_GRAPH_MODEL_MCORE_DISPATCH) 291 goto fail; 292 293 graph->lcore_id = lcore; 294 graph->graph->dispatch.lcore_id = graph->lcore_id; 295 graph->socket = rte_lcore_to_socket_id(lcore); 296 297 /* check the availability of source node */ 298 if (!graph_src_node_avail(graph)) 299 graph->graph->head = 0; 300 301 return 0; 302 303 fail: 304 return -rte_errno; 305 } 306 307 void 308 rte_graph_model_mcore_dispatch_core_unbind(rte_graph_t id) 309 { 310 struct graph *graph; 311 312 GRAPH_ID_CHECK(id); 313 STAILQ_FOREACH(graph, &graph_list, next) 314 if (graph->id == id) 315 break; 316 317 graph->lcore_id = RTE_MAX_LCORE; 318 graph->graph->dispatch.lcore_id = RTE_MAX_LCORE; 319 320 fail: 321 return; 322 } 323 324 struct rte_graph * 325 rte_graph_lookup(const char *name) 326 { 327 const struct rte_memzone *mz; 328 struct rte_graph *rc = NULL; 329 330 mz = rte_memzone_lookup(name); 331 if (mz) 332 rc = mz->addr; 333 334 return graph_mem_fixup_secondary(rc); 335 } 336 337 rte_graph_t 338 rte_graph_create(const char *name, struct rte_graph_param *prm) 339 { 340 rte_node_t src_node_count; 341 struct graph *graph; 342 const char *pattern; 343 uint16_t i; 344 345 graph_spinlock_lock(); 346 347 /* Check arguments sanity */ 348 if (prm == NULL) 349 SET_ERR_JMP(EINVAL, fail, "Param should not be NULL"); 350 351 if (name == NULL) 352 SET_ERR_JMP(EINVAL, fail, "Graph name should not be NULL"); 353 354 /* Check for existence of duplicate graph */ 355 STAILQ_FOREACH(graph, &graph_list, next) 356 if (strncmp(name, graph->name, RTE_GRAPH_NAMESIZE) == 0) 357 SET_ERR_JMP(EEXIST, fail, "Found duplicate graph %s", 358 name); 359 360 /* Create graph object */ 361 graph = calloc(1, sizeof(*graph)); 362 if (graph == NULL) 363 SET_ERR_JMP(ENOMEM, fail, "Failed to calloc graph object"); 364 365 /* Initialize the graph object */ 366 STAILQ_INIT(&graph->node_list); 367 if (rte_strscpy(graph->name, name, RTE_GRAPH_NAMESIZE) < 0) 368 SET_ERR_JMP(E2BIG, free, "Too big name=%s", name); 369 370 /* Expand node pattern and add the nodes to the graph */ 371 for (i = 0; i < prm->nb_node_patterns; i++) { 372 pattern = prm->node_patterns[i]; 373 if (expand_pattern_to_node(graph, pattern)) 374 goto graph_cleanup; 375 } 376 377 /* Go over all the nodes edges and add them to the graph */ 378 if (graph_node_edges_add(graph)) 379 goto graph_cleanup; 380 381 /* Update adjacency list of all nodes in the graph */ 382 if (graph_adjacency_list_update(graph)) 383 goto graph_cleanup; 384 385 /* Make sure at least a source node present in the graph */ 386 src_node_count = graph_src_nodes_count(graph); 387 if (src_node_count == 0) 388 goto graph_cleanup; 389 390 /* Make sure no node is pointing to source node */ 391 if (graph_node_has_edge_to_src_node(graph)) 392 goto graph_cleanup; 393 394 /* Don't allow node has loop to self */ 395 if (graph_node_has_loop_edge(graph)) 396 goto graph_cleanup; 397 398 /* Do BFS from src nodes on the graph to find isolated nodes */ 399 if (graph_has_isolated_node(graph)) 400 goto graph_cleanup; 401 402 /* Initialize pcap config. */ 403 graph_pcap_enable(prm->pcap_enable); 404 405 /* Initialize graph object */ 406 graph->socket = prm->socket_id; 407 graph->src_node_count = src_node_count; 408 graph->node_count = graph_nodes_count(graph); 409 graph->id = graph_id; 410 graph->parent_id = RTE_GRAPH_ID_INVALID; 411 graph->lcore_id = RTE_MAX_LCORE; 412 graph->num_pkt_to_capture = prm->num_pkt_to_capture; 413 if (prm->pcap_filename) 414 rte_strscpy(graph->pcap_filename, prm->pcap_filename, RTE_GRAPH_PCAP_FILE_SZ); 415 416 /* Allocate the Graph fast path memory and populate the data */ 417 if (graph_fp_mem_create(graph)) 418 goto graph_cleanup; 419 420 /* Call init() of the all the nodes in the graph */ 421 if (graph_node_init(graph)) 422 goto graph_mem_destroy; 423 424 /* All good, Lets add the graph to the list */ 425 graph_id++; 426 STAILQ_INSERT_TAIL(&graph_list, graph, next); 427 428 graph_spinlock_unlock(); 429 return graph->id; 430 431 graph_mem_destroy: 432 graph_fp_mem_destroy(graph); 433 graph_cleanup: 434 graph_cleanup(graph); 435 free: 436 free(graph); 437 fail: 438 graph_spinlock_unlock(); 439 return RTE_GRAPH_ID_INVALID; 440 } 441 442 int 443 rte_graph_destroy(rte_graph_t id) 444 { 445 struct graph *graph, *tmp; 446 int rc = -ENOENT; 447 448 graph_spinlock_lock(); 449 450 graph = STAILQ_FIRST(&graph_list); 451 while (graph != NULL) { 452 tmp = STAILQ_NEXT(graph, next); 453 if (graph->id == id) { 454 /* Destroy the schedule work queue if has */ 455 if (rte_graph_worker_model_get(graph->graph) == 456 RTE_GRAPH_MODEL_MCORE_DISPATCH) 457 graph_sched_wq_destroy(graph); 458 459 /* Call fini() of the all the nodes in the graph */ 460 graph_node_fini(graph); 461 /* Destroy graph fast path memory */ 462 rc = graph_fp_mem_destroy(graph); 463 if (rc) 464 SET_ERR_JMP(rc, done, "Graph %s destroy failed", 465 graph->name); 466 467 graph_cleanup(graph); 468 STAILQ_REMOVE(&graph_list, graph, graph, next); 469 free(graph); 470 graph_id--; 471 goto done; 472 } 473 graph = tmp; 474 } 475 done: 476 graph_spinlock_unlock(); 477 return rc; 478 } 479 480 static rte_graph_t 481 graph_clone(struct graph *parent_graph, const char *name, struct rte_graph_param *prm) 482 { 483 struct graph_node *graph_node; 484 struct graph *graph; 485 486 graph_spinlock_lock(); 487 488 /* Don't allow to clone a node from a cloned graph */ 489 if (parent_graph->parent_id != RTE_GRAPH_ID_INVALID) 490 SET_ERR_JMP(EEXIST, fail, "A cloned graph is not allowed to be cloned"); 491 492 /* Create graph object */ 493 graph = calloc(1, sizeof(*graph)); 494 if (graph == NULL) 495 SET_ERR_JMP(ENOMEM, fail, "Failed to calloc cloned graph object"); 496 497 /* Naming ceremony of the new graph. name is node->name + "-" + name */ 498 if (clone_name(graph->name, parent_graph->name, name)) 499 goto free; 500 501 /* Check for existence of duplicate graph */ 502 if (rte_graph_from_name(graph->name) != RTE_GRAPH_ID_INVALID) 503 SET_ERR_JMP(EEXIST, free, "Found duplicate graph %s", 504 graph->name); 505 506 /* Clone nodes from parent graph firstly */ 507 STAILQ_INIT(&graph->node_list); 508 STAILQ_FOREACH(graph_node, &parent_graph->node_list, next) { 509 if (graph_node_add(graph, graph_node->node)) 510 goto graph_cleanup; 511 } 512 513 /* Just update adjacency list of all nodes in the graph */ 514 if (graph_adjacency_list_update(graph)) 515 goto graph_cleanup; 516 517 /* Initialize the graph object */ 518 graph->src_node_count = parent_graph->src_node_count; 519 graph->node_count = parent_graph->node_count; 520 graph->parent_id = parent_graph->id; 521 graph->lcore_id = parent_graph->lcore_id; 522 graph->socket = parent_graph->socket; 523 graph->id = graph_id; 524 525 /* Allocate the Graph fast path memory and populate the data */ 526 if (graph_fp_mem_create(graph)) 527 goto graph_cleanup; 528 529 /* Clone the graph model */ 530 graph->graph->model = parent_graph->graph->model; 531 532 /* Create the graph schedule work queue */ 533 if (rte_graph_worker_model_get(graph->graph) == RTE_GRAPH_MODEL_MCORE_DISPATCH && 534 graph_sched_wq_create(graph, parent_graph, prm)) 535 goto graph_mem_destroy; 536 537 /* Call init() of the all the nodes in the graph */ 538 if (graph_node_init(graph)) 539 goto graph_mem_destroy; 540 541 /* All good, Lets add the graph to the list */ 542 graph_id++; 543 STAILQ_INSERT_TAIL(&graph_list, graph, next); 544 545 graph_spinlock_unlock(); 546 return graph->id; 547 548 graph_mem_destroy: 549 graph_fp_mem_destroy(graph); 550 graph_cleanup: 551 graph_cleanup(graph); 552 free: 553 free(graph); 554 fail: 555 graph_spinlock_unlock(); 556 return RTE_GRAPH_ID_INVALID; 557 } 558 559 rte_graph_t 560 rte_graph_clone(rte_graph_t id, const char *name, struct rte_graph_param *prm) 561 { 562 struct graph *graph; 563 564 GRAPH_ID_CHECK(id); 565 STAILQ_FOREACH(graph, &graph_list, next) 566 if (graph->id == id) 567 return graph_clone(graph, name, prm); 568 569 fail: 570 return RTE_GRAPH_ID_INVALID; 571 } 572 573 rte_graph_t 574 rte_graph_from_name(const char *name) 575 { 576 struct graph *graph; 577 578 STAILQ_FOREACH(graph, &graph_list, next) 579 if (strncmp(graph->name, name, RTE_GRAPH_NAMESIZE) == 0) 580 return graph->id; 581 582 return RTE_GRAPH_ID_INVALID; 583 } 584 585 char * 586 rte_graph_id_to_name(rte_graph_t id) 587 { 588 struct graph *graph; 589 590 GRAPH_ID_CHECK(id); 591 STAILQ_FOREACH(graph, &graph_list, next) 592 if (graph->id == id) 593 return graph->name; 594 595 fail: 596 return NULL; 597 } 598 599 struct rte_node * 600 rte_graph_node_get(rte_graph_t gid, uint32_t nid) 601 { 602 struct rte_node *node; 603 struct graph *graph; 604 rte_graph_off_t off; 605 rte_node_t count; 606 607 GRAPH_ID_CHECK(gid); 608 STAILQ_FOREACH(graph, &graph_list, next) 609 if (graph->id == gid) { 610 rte_graph_foreach_node(count, off, graph->graph, 611 node) { 612 if (node->id == nid) 613 return node; 614 } 615 break; 616 } 617 fail: 618 return NULL; 619 } 620 621 struct rte_node * 622 rte_graph_node_get_by_name(const char *graph_name, const char *node_name) 623 { 624 struct rte_node *node; 625 struct graph *graph; 626 rte_graph_off_t off; 627 rte_node_t count; 628 629 STAILQ_FOREACH(graph, &graph_list, next) 630 if (!strncmp(graph->name, graph_name, RTE_GRAPH_NAMESIZE)) { 631 rte_graph_foreach_node(count, off, graph->graph, 632 node) { 633 if (!strncmp(node->name, node_name, 634 RTE_NODE_NAMESIZE)) 635 return node; 636 } 637 break; 638 } 639 640 return NULL; 641 } 642 643 void __rte_noinline 644 __rte_node_stream_alloc(struct rte_graph *graph, struct rte_node *node) 645 { 646 uint16_t size = node->size; 647 648 RTE_VERIFY(size != UINT16_MAX); 649 /* Allocate double amount of size to avoid immediate realloc */ 650 size = RTE_MIN(UINT16_MAX, RTE_MAX(RTE_GRAPH_BURST_SIZE, size * 2)); 651 node->objs = rte_realloc_socket(node->objs, size * sizeof(void *), 652 RTE_CACHE_LINE_SIZE, graph->socket); 653 RTE_VERIFY(node->objs); 654 node->size = size; 655 node->realloc_count++; 656 } 657 658 void __rte_noinline 659 __rte_node_stream_alloc_size(struct rte_graph *graph, struct rte_node *node, 660 uint16_t req_size) 661 { 662 uint16_t size = node->size; 663 664 RTE_VERIFY(size != UINT16_MAX); 665 /* Allocate double amount of size to avoid immediate realloc */ 666 size = RTE_MIN(UINT16_MAX, RTE_MAX(RTE_GRAPH_BURST_SIZE, req_size * 2)); 667 node->objs = rte_realloc_socket(node->objs, size * sizeof(void *), 668 RTE_CACHE_LINE_SIZE, graph->socket); 669 RTE_VERIFY(node->objs); 670 node->size = size; 671 node->realloc_count++; 672 } 673 674 static int 675 graph_to_dot(FILE *f, struct graph *graph) 676 { 677 const char *src_edge_color = " [color=blue]\n"; 678 const char *edge_color = "\n"; 679 struct graph_node *graph_node; 680 char *node_name; 681 rte_edge_t i; 682 int rc; 683 684 rc = fprintf(f, "Digraph %s {\n\trankdir=LR;\n", graph->name); 685 if (rc < 0) 686 goto end; 687 688 STAILQ_FOREACH(graph_node, &graph->node_list, next) { 689 node_name = graph_node->node->name; 690 for (i = 0; i < graph_node->node->nb_edges; i++) { 691 rc = fprintf(f, "\t\"%s\"->\"%s\"%s", node_name, 692 graph_node->adjacency_list[i]->node->name, 693 graph_node->node->flags & RTE_NODE_SOURCE_F 694 ? src_edge_color 695 : edge_color); 696 if (rc < 0) 697 goto end; 698 } 699 } 700 rc = fprintf(f, "}\n"); 701 if (rc < 0) 702 goto end; 703 704 return 0; 705 end: 706 rte_errno = EBADF; 707 return -rte_errno; 708 } 709 710 int 711 rte_graph_export(const char *name, FILE *f) 712 { 713 struct graph *graph; 714 int rc = ENOENT; 715 716 STAILQ_FOREACH(graph, &graph_list, next) { 717 if (strncmp(graph->name, name, RTE_GRAPH_NAMESIZE) == 0) { 718 rc = graph_to_dot(f, graph); 719 goto end; 720 } 721 } 722 end: 723 return -rc; 724 } 725 726 static void 727 graph_scan_dump(FILE *f, rte_graph_t id, bool all) 728 { 729 struct graph *graph; 730 731 RTE_VERIFY(f); 732 GRAPH_ID_CHECK(id); 733 734 STAILQ_FOREACH(graph, &graph_list, next) { 735 if (all == true) { 736 graph_dump(f, graph); 737 } else if (graph->id == id) { 738 graph_dump(f, graph); 739 return; 740 } 741 } 742 fail: 743 return; 744 } 745 746 void 747 rte_graph_dump(FILE *f, rte_graph_t id) 748 { 749 graph_scan_dump(f, id, false); 750 } 751 752 void 753 rte_graph_list_dump(FILE *f) 754 { 755 graph_scan_dump(f, 0, true); 756 } 757 758 rte_graph_t 759 rte_graph_max_count(void) 760 { 761 return graph_id; 762 } 763 764 RTE_LOG_REGISTER_DEFAULT(rte_graph_logtype, INFO); 765