1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017 Intel Corporation 3 */ 4 5 #include <inttypes.h> 6 #include <string.h> 7 8 #include <rte_bus_vdev.h> 9 #include <rte_lcore.h> 10 #include <rte_memzone.h> 11 #include <rte_kvargs.h> 12 #include <rte_errno.h> 13 #include <rte_cycles.h> 14 15 #include "opdl_evdev.h" 16 #include "opdl_ring.h" 17 #include "opdl_log.h" 18 19 #define EVENTDEV_NAME_OPDL_PMD event_opdl 20 #define NUMA_NODE_ARG "numa_node" 21 #define DO_VALIDATION_ARG "do_validation" 22 #define DO_TEST_ARG "self_test" 23 24 25 static void 26 opdl_info_get(struct rte_eventdev *dev, struct rte_event_dev_info *info); 27 28 uint16_t 29 opdl_event_enqueue_burst(void *port, 30 const struct rte_event ev[], 31 uint16_t num) 32 { 33 struct opdl_port *p = port; 34 35 if (unlikely(!p->opdl->data->dev_started)) 36 return 0; 37 38 39 /* either rx_enqueue or disclaim*/ 40 return p->enq(p, ev, num); 41 } 42 43 uint16_t 44 opdl_event_enqueue(void *port, const struct rte_event *ev) 45 { 46 struct opdl_port *p = port; 47 48 if (unlikely(!p->opdl->data->dev_started)) 49 return 0; 50 51 52 return p->enq(p, ev, 1); 53 } 54 55 uint16_t 56 opdl_event_dequeue_burst(void *port, 57 struct rte_event *ev, 58 uint16_t num, 59 uint64_t wait) 60 { 61 struct opdl_port *p = (void *)port; 62 63 RTE_SET_USED(wait); 64 65 if (unlikely(!p->opdl->data->dev_started)) 66 return 0; 67 68 /* This function pointer can point to tx_dequeue or claim*/ 69 return p->deq(p, ev, num); 70 } 71 72 uint16_t 73 opdl_event_dequeue(void *port, 74 struct rte_event *ev, 75 uint64_t wait) 76 { 77 struct opdl_port *p = (void *)port; 78 79 if (unlikely(!p->opdl->data->dev_started)) 80 return 0; 81 82 RTE_SET_USED(wait); 83 84 return p->deq(p, ev, 1); 85 } 86 87 static int 88 opdl_port_link(struct rte_eventdev *dev, 89 void *port, 90 const uint8_t queues[], 91 const uint8_t priorities[], 92 uint16_t num) 93 { 94 struct opdl_port *p = port; 95 96 RTE_SET_USED(priorities); 97 RTE_SET_USED(dev); 98 99 if (unlikely(dev->data->dev_started)) { 100 PMD_DRV_LOG(ERR, "DEV_ID:[%02d] : " 101 "Attempt to link queue (%u) to port %d while device started\n", 102 dev->data->dev_id, 103 queues[0], 104 p->id); 105 rte_errno = EINVAL; 106 return 0; 107 } 108 109 /* Max of 1 queue per port */ 110 if (num > 1) { 111 PMD_DRV_LOG(ERR, "DEV_ID:[%02d] : " 112 "Attempt to link more than one queue (%u) to port %d requested\n", 113 dev->data->dev_id, 114 num, 115 p->id); 116 rte_errno = EDQUOT; 117 return 0; 118 } 119 120 if (!p->configured) { 121 PMD_DRV_LOG(ERR, "DEV_ID:[%02d] : " 122 "port %d not configured, cannot link to %u\n", 123 dev->data->dev_id, 124 p->id, 125 queues[0]); 126 rte_errno = EINVAL; 127 return 0; 128 } 129 130 if (p->external_qid != OPDL_INVALID_QID) { 131 PMD_DRV_LOG(ERR, "DEV_ID:[%02d] : " 132 "port %d already linked to queue %u, cannot link to %u\n", 133 dev->data->dev_id, 134 p->id, 135 p->external_qid, 136 queues[0]); 137 rte_errno = EINVAL; 138 return 0; 139 } 140 141 p->external_qid = queues[0]; 142 143 return 1; 144 } 145 146 static int 147 opdl_port_unlink(struct rte_eventdev *dev, 148 void *port, 149 uint8_t queues[], 150 uint16_t nb_unlinks) 151 { 152 struct opdl_port *p = port; 153 154 RTE_SET_USED(queues); 155 RTE_SET_USED(nb_unlinks); 156 157 if (unlikely(dev->data->dev_started)) { 158 PMD_DRV_LOG(ERR, "DEV_ID:[%02d] : " 159 "Attempt to unlink queue (%u) to port %d while device started\n", 160 dev->data->dev_id, 161 queues[0], 162 p->id); 163 rte_errno = EINVAL; 164 return 0; 165 } 166 RTE_SET_USED(nb_unlinks); 167 168 /* Port Stuff */ 169 p->queue_id = OPDL_INVALID_QID; 170 p->p_type = OPDL_INVALID_PORT; 171 p->external_qid = OPDL_INVALID_QID; 172 173 /* always unlink 0 queue due to statice pipeline */ 174 return 0; 175 } 176 177 static int 178 opdl_port_setup(struct rte_eventdev *dev, 179 uint8_t port_id, 180 const struct rte_event_port_conf *conf) 181 { 182 struct opdl_evdev *device = opdl_pmd_priv(dev); 183 struct opdl_port *p = &device->ports[port_id]; 184 185 RTE_SET_USED(conf); 186 187 /* Check if port already configured */ 188 if (p->configured) { 189 PMD_DRV_LOG(ERR, "DEV_ID:[%02d] : " 190 "Attempt to setup port %d which is already setup\n", 191 dev->data->dev_id, 192 p->id); 193 return -EDQUOT; 194 } 195 196 *p = (struct opdl_port){0}; /* zero entire structure */ 197 p->id = port_id; 198 p->opdl = device; 199 p->queue_id = OPDL_INVALID_QID; 200 p->external_qid = OPDL_INVALID_QID; 201 dev->data->ports[port_id] = p; 202 rte_smp_wmb(); 203 p->configured = 1; 204 device->nb_ports++; 205 return 0; 206 } 207 208 static void 209 opdl_port_release(void *port) 210 { 211 struct opdl_port *p = (void *)port; 212 213 if (p == NULL || 214 p->opdl->data->dev_started) { 215 return; 216 } 217 218 p->configured = 0; 219 p->initialized = 0; 220 } 221 222 static void 223 opdl_port_def_conf(struct rte_eventdev *dev, uint8_t port_id, 224 struct rte_event_port_conf *port_conf) 225 { 226 RTE_SET_USED(dev); 227 RTE_SET_USED(port_id); 228 229 port_conf->new_event_threshold = MAX_OPDL_CONS_Q_DEPTH; 230 port_conf->dequeue_depth = MAX_OPDL_CONS_Q_DEPTH; 231 port_conf->enqueue_depth = MAX_OPDL_CONS_Q_DEPTH; 232 } 233 234 static int 235 opdl_queue_setup(struct rte_eventdev *dev, 236 uint8_t queue_id, 237 const struct rte_event_queue_conf *conf) 238 { 239 enum queue_type type; 240 241 struct opdl_evdev *device = opdl_pmd_priv(dev); 242 243 /* Extra sanity check, probably not needed */ 244 if (queue_id == OPDL_INVALID_QID) { 245 PMD_DRV_LOG(ERR, "DEV_ID:[%02d] : " 246 "Invalid queue id %u requested\n", 247 dev->data->dev_id, 248 queue_id); 249 return -EINVAL; 250 } 251 252 if (device->nb_q_md > device->max_queue_nb) { 253 PMD_DRV_LOG(ERR, "DEV_ID:[%02d] : " 254 "Max number of queues %u exceeded by request %u\n", 255 dev->data->dev_id, 256 device->max_queue_nb, 257 device->nb_q_md); 258 return -EINVAL; 259 } 260 261 if (RTE_EVENT_QUEUE_CFG_ALL_TYPES 262 & conf->event_queue_cfg) { 263 PMD_DRV_LOG(ERR, "DEV_ID:[%02d] : " 264 "QUEUE_CFG_ALL_TYPES not supported\n", 265 dev->data->dev_id); 266 return -ENOTSUP; 267 } else if (RTE_EVENT_QUEUE_CFG_SINGLE_LINK 268 & conf->event_queue_cfg) { 269 type = OPDL_Q_TYPE_SINGLE_LINK; 270 } else { 271 switch (conf->schedule_type) { 272 case RTE_SCHED_TYPE_ORDERED: 273 type = OPDL_Q_TYPE_ORDERED; 274 break; 275 case RTE_SCHED_TYPE_ATOMIC: 276 type = OPDL_Q_TYPE_ATOMIC; 277 break; 278 case RTE_SCHED_TYPE_PARALLEL: 279 type = OPDL_Q_TYPE_ORDERED; 280 break; 281 default: 282 PMD_DRV_LOG(ERR, "DEV_ID:[%02d] : " 283 "Unknown queue type %d requested\n", 284 dev->data->dev_id, 285 conf->event_queue_cfg); 286 return -EINVAL; 287 } 288 } 289 /* Check if queue id has been setup already */ 290 uint32_t i; 291 for (i = 0; i < device->nb_q_md; i++) { 292 if (device->q_md[i].ext_id == queue_id) { 293 PMD_DRV_LOG(ERR, "DEV_ID:[%02d] : " 294 "queue id %u already setup\n", 295 dev->data->dev_id, 296 queue_id); 297 return -EINVAL; 298 } 299 } 300 301 device->q_md[device->nb_q_md].ext_id = queue_id; 302 device->q_md[device->nb_q_md].type = type; 303 device->q_md[device->nb_q_md].setup = 1; 304 device->nb_q_md++; 305 306 return 1; 307 } 308 309 static void 310 opdl_queue_release(struct rte_eventdev *dev, uint8_t queue_id) 311 { 312 struct opdl_evdev *device = opdl_pmd_priv(dev); 313 314 RTE_SET_USED(queue_id); 315 316 if (device->data->dev_started) 317 return; 318 319 } 320 321 static void 322 opdl_queue_def_conf(struct rte_eventdev *dev, 323 uint8_t queue_id, 324 struct rte_event_queue_conf *conf) 325 { 326 RTE_SET_USED(dev); 327 RTE_SET_USED(queue_id); 328 329 static const struct rte_event_queue_conf default_conf = { 330 .nb_atomic_flows = 1024, 331 .nb_atomic_order_sequences = 1, 332 .event_queue_cfg = 0, 333 .schedule_type = RTE_SCHED_TYPE_ORDERED, 334 .priority = RTE_EVENT_DEV_PRIORITY_NORMAL, 335 }; 336 337 *conf = default_conf; 338 } 339 340 341 static int 342 opdl_dev_configure(const struct rte_eventdev *dev) 343 { 344 struct opdl_evdev *opdl = opdl_pmd_priv(dev); 345 const struct rte_eventdev_data *data = dev->data; 346 const struct rte_event_dev_config *conf = &data->dev_conf; 347 348 opdl->max_queue_nb = conf->nb_event_queues; 349 opdl->max_port_nb = conf->nb_event_ports; 350 opdl->nb_events_limit = conf->nb_events_limit; 351 352 if (conf->event_dev_cfg & RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT) { 353 PMD_DRV_LOG(ERR, "DEV_ID:[%02d] : " 354 "DEQUEUE_TIMEOUT not supported\n", 355 dev->data->dev_id); 356 return -ENOTSUP; 357 } 358 359 return 0; 360 } 361 362 static void 363 opdl_info_get(struct rte_eventdev *dev, struct rte_event_dev_info *info) 364 { 365 RTE_SET_USED(dev); 366 367 static const struct rte_event_dev_info evdev_opdl_info = { 368 .driver_name = OPDL_PMD_NAME, 369 .max_event_queues = RTE_EVENT_MAX_QUEUES_PER_DEV, 370 .max_event_queue_flows = OPDL_QID_NUM_FIDS, 371 .max_event_queue_priority_levels = OPDL_Q_PRIORITY_MAX, 372 .max_event_priority_levels = OPDL_IQS_MAX, 373 .max_event_ports = OPDL_PORTS_MAX, 374 .max_event_port_dequeue_depth = MAX_OPDL_CONS_Q_DEPTH, 375 .max_event_port_enqueue_depth = MAX_OPDL_CONS_Q_DEPTH, 376 .max_num_events = OPDL_INFLIGHT_EVENTS_TOTAL, 377 .event_dev_cap = RTE_EVENT_DEV_CAP_BURST_MODE, 378 }; 379 380 *info = evdev_opdl_info; 381 } 382 383 static void 384 opdl_dump(struct rte_eventdev *dev, FILE *f) 385 { 386 struct opdl_evdev *device = opdl_pmd_priv(dev); 387 388 if (!device->do_validation) 389 return; 390 391 fprintf(f, 392 "\n\n -- RING STATISTICS --\n"); 393 uint32_t i; 394 for (i = 0; i < device->nb_opdls; i++) 395 opdl_ring_dump(device->opdl[i], f); 396 397 fprintf(f, 398 "\n\n -- PORT STATISTICS --\n" 399 "Type Port Index Port Id Queue Id Av. Req Size " 400 "Av. Grant Size Av. Cycles PP" 401 " Empty DEQs Non Empty DEQs Pkts Processed\n"); 402 403 for (i = 0; i < device->max_port_nb; i++) { 404 char queue_id[64]; 405 char total_cyc[64]; 406 const char *p_type; 407 408 uint64_t cne, cpg; 409 struct opdl_port *port = &device->ports[i]; 410 411 if (port->initialized) { 412 cne = port->port_stat[claim_non_empty]; 413 cpg = port->port_stat[claim_pkts_granted]; 414 if (port->p_type == OPDL_REGULAR_PORT) 415 p_type = "REG"; 416 else if (port->p_type == OPDL_PURE_RX_PORT) 417 p_type = " RX"; 418 else if (port->p_type == OPDL_PURE_TX_PORT) 419 p_type = " TX"; 420 else if (port->p_type == OPDL_ASYNC_PORT) 421 p_type = "SYNC"; 422 else 423 p_type = "????"; 424 425 snprintf(queue_id, sizeof(queue_id), "%02u", 426 port->external_qid); 427 if (port->p_type == OPDL_REGULAR_PORT || 428 port->p_type == OPDL_ASYNC_PORT) 429 snprintf(total_cyc, sizeof(total_cyc), 430 " %'16"PRIu64"", 431 (cpg != 0 ? 432 port->port_stat[total_cycles] / cpg 433 : 0)); 434 else 435 snprintf(total_cyc, sizeof(total_cyc), 436 " ----"); 437 fprintf(f, 438 "%4s %10u %8u %9s %'16"PRIu64" %'16"PRIu64" %s " 439 "%'16"PRIu64" %'16"PRIu64" %'16"PRIu64"\n", 440 p_type, 441 i, 442 port->id, 443 (port->external_qid == OPDL_INVALID_QID ? "---" 444 : queue_id), 445 (cne != 0 ? 446 port->port_stat[claim_pkts_requested] / cne 447 : 0), 448 (cne != 0 ? 449 port->port_stat[claim_pkts_granted] / cne 450 : 0), 451 total_cyc, 452 port->port_stat[claim_empty], 453 port->port_stat[claim_non_empty], 454 port->port_stat[claim_pkts_granted]); 455 } 456 } 457 fprintf(f, "\n"); 458 } 459 460 461 static void 462 opdl_stop(struct rte_eventdev *dev) 463 { 464 struct opdl_evdev *device = opdl_pmd_priv(dev); 465 466 opdl_xstats_uninit(dev); 467 468 destroy_queues_and_rings(dev); 469 470 471 device->started = 0; 472 473 rte_smp_wmb(); 474 } 475 476 static int 477 opdl_start(struct rte_eventdev *dev) 478 { 479 int err = 0; 480 481 if (!err) 482 err = create_queues_and_rings(dev); 483 484 485 if (!err) 486 err = assign_internal_queue_ids(dev); 487 488 489 if (!err) 490 err = initialise_queue_zero_ports(dev); 491 492 493 if (!err) 494 err = initialise_all_other_ports(dev); 495 496 497 if (!err) 498 err = check_queues_linked(dev); 499 500 501 if (!err) 502 err = opdl_add_event_handlers(dev); 503 504 505 if (!err) 506 err = build_all_dependencies(dev); 507 508 if (!err) { 509 opdl_xstats_init(dev); 510 511 struct opdl_evdev *device = opdl_pmd_priv(dev); 512 513 PMD_DRV_LOG(INFO, "DEV_ID:[%02d] : " 514 "SUCCESS : Created %u total queues (%u ex, %u in)," 515 " %u opdls, %u event_dev ports, %u input ports", 516 opdl_pmd_dev_id(device), 517 device->nb_queues, 518 (device->nb_queues - device->nb_opdls), 519 device->nb_opdls, 520 device->nb_opdls, 521 device->nb_ports, 522 device->queue[0].nb_ports); 523 } else 524 opdl_stop(dev); 525 526 return err; 527 } 528 529 static int 530 opdl_close(struct rte_eventdev *dev) 531 { 532 struct opdl_evdev *device = opdl_pmd_priv(dev); 533 uint32_t i; 534 535 for (i = 0; i < device->max_port_nb; i++) { 536 memset(&device->ports[i], 537 0, 538 sizeof(struct opdl_port)); 539 } 540 541 memset(&device->s_md, 542 0x0, 543 sizeof(struct opdl_stage_meta_data)*OPDL_PORTS_MAX); 544 545 memset(&device->q_md, 546 0xFF, 547 sizeof(struct opdl_queue_meta_data)*OPDL_MAX_QUEUES); 548 549 550 memset(device->q_map_ex_to_in, 551 0, 552 sizeof(uint8_t)*OPDL_INVALID_QID); 553 554 opdl_xstats_uninit(dev); 555 556 device->max_port_nb = 0; 557 558 device->max_queue_nb = 0; 559 560 device->nb_opdls = 0; 561 562 device->nb_queues = 0; 563 564 device->nb_ports = 0; 565 566 device->nb_q_md = 0; 567 568 dev->data->nb_queues = 0; 569 570 dev->data->nb_ports = 0; 571 572 573 return 0; 574 } 575 576 static int 577 assign_numa_node(const char *key __rte_unused, const char *value, void *opaque) 578 { 579 int *socket_id = opaque; 580 *socket_id = atoi(value); 581 if (*socket_id >= RTE_MAX_NUMA_NODES) 582 return -1; 583 return 0; 584 } 585 586 static int 587 set_do_validation(const char *key __rte_unused, const char *value, void *opaque) 588 { 589 int *do_val = opaque; 590 *do_val = atoi(value); 591 if (*do_val != 0) 592 *do_val = 1; 593 594 return 0; 595 } 596 static int 597 set_do_test(const char *key __rte_unused, const char *value, void *opaque) 598 { 599 int *do_test = opaque; 600 601 *do_test = atoi(value); 602 603 if (*do_test != 0) 604 *do_test = 1; 605 return 0; 606 } 607 608 static int 609 opdl_probe(struct rte_vdev_device *vdev) 610 { 611 static struct rte_eventdev_ops evdev_opdl_ops = { 612 .dev_configure = opdl_dev_configure, 613 .dev_infos_get = opdl_info_get, 614 .dev_close = opdl_close, 615 .dev_start = opdl_start, 616 .dev_stop = opdl_stop, 617 .dump = opdl_dump, 618 619 .queue_def_conf = opdl_queue_def_conf, 620 .queue_setup = opdl_queue_setup, 621 .queue_release = opdl_queue_release, 622 .port_def_conf = opdl_port_def_conf, 623 .port_setup = opdl_port_setup, 624 .port_release = opdl_port_release, 625 .port_link = opdl_port_link, 626 .port_unlink = opdl_port_unlink, 627 628 629 .xstats_get = opdl_xstats_get, 630 .xstats_get_names = opdl_xstats_get_names, 631 .xstats_get_by_name = opdl_xstats_get_by_name, 632 .xstats_reset = opdl_xstats_reset, 633 }; 634 635 static const char *const args[] = { 636 NUMA_NODE_ARG, 637 DO_VALIDATION_ARG, 638 DO_TEST_ARG, 639 NULL 640 }; 641 const char *name; 642 const char *params; 643 struct rte_eventdev *dev; 644 struct opdl_evdev *opdl; 645 int socket_id = rte_socket_id(); 646 int do_validation = 0; 647 int do_test = 0; 648 int str_len; 649 int test_result = 0; 650 651 name = rte_vdev_device_name(vdev); 652 params = rte_vdev_device_args(vdev); 653 if (params != NULL && params[0] != '\0') { 654 struct rte_kvargs *kvlist = rte_kvargs_parse(params, args); 655 656 if (!kvlist) { 657 PMD_DRV_LOG(INFO, 658 "Ignoring unsupported parameters when creating device '%s'\n", 659 name); 660 } else { 661 int ret = rte_kvargs_process(kvlist, NUMA_NODE_ARG, 662 assign_numa_node, &socket_id); 663 if (ret != 0) { 664 PMD_DRV_LOG(ERR, 665 "%s: Error parsing numa node parameter", 666 name); 667 668 rte_kvargs_free(kvlist); 669 return ret; 670 } 671 672 ret = rte_kvargs_process(kvlist, DO_VALIDATION_ARG, 673 set_do_validation, &do_validation); 674 if (ret != 0) { 675 PMD_DRV_LOG(ERR, 676 "%s: Error parsing do validation parameter", 677 name); 678 rte_kvargs_free(kvlist); 679 return ret; 680 } 681 682 ret = rte_kvargs_process(kvlist, DO_TEST_ARG, 683 set_do_test, &do_test); 684 if (ret != 0) { 685 PMD_DRV_LOG(ERR, 686 "%s: Error parsing do test parameter", 687 name); 688 rte_kvargs_free(kvlist); 689 return ret; 690 } 691 692 rte_kvargs_free(kvlist); 693 } 694 } 695 dev = rte_event_pmd_vdev_init(name, 696 sizeof(struct opdl_evdev), socket_id); 697 698 if (dev == NULL) { 699 PMD_DRV_LOG(ERR, "eventdev vdev init() failed"); 700 return -EFAULT; 701 } 702 703 PMD_DRV_LOG(INFO, "DEV_ID:[%02d] : " 704 "Success - creating eventdev device %s, numa_node:[%d], do_valdation:[%s]" 705 " , self_test:[%s]\n", 706 dev->data->dev_id, 707 name, 708 socket_id, 709 (do_validation ? "true" : "false"), 710 (do_test ? "true" : "false")); 711 712 dev->dev_ops = &evdev_opdl_ops; 713 714 dev->enqueue = opdl_event_enqueue; 715 dev->enqueue_burst = opdl_event_enqueue_burst; 716 dev->enqueue_new_burst = opdl_event_enqueue_burst; 717 dev->enqueue_forward_burst = opdl_event_enqueue_burst; 718 dev->dequeue = opdl_event_dequeue; 719 dev->dequeue_burst = opdl_event_dequeue_burst; 720 721 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 722 return 0; 723 724 opdl = dev->data->dev_private; 725 opdl->data = dev->data; 726 opdl->socket = socket_id; 727 opdl->do_validation = do_validation; 728 opdl->do_test = do_test; 729 str_len = strlen(name); 730 memcpy(opdl->service_name, name, str_len); 731 732 if (do_test == 1) 733 test_result = opdl_selftest(); 734 735 return test_result; 736 } 737 738 static int 739 opdl_remove(struct rte_vdev_device *vdev) 740 { 741 const char *name; 742 743 name = rte_vdev_device_name(vdev); 744 if (name == NULL) 745 return -EINVAL; 746 747 PMD_DRV_LOG(INFO, "Closing eventdev opdl device %s\n", name); 748 749 return rte_event_pmd_vdev_uninit(name); 750 } 751 752 static struct rte_vdev_driver evdev_opdl_pmd_drv = { 753 .probe = opdl_probe, 754 .remove = opdl_remove 755 }; 756 757 RTE_INIT(opdl_init_log) 758 { 759 opdl_logtype_driver = rte_log_register("pmd.event.opdl.driver"); 760 if (opdl_logtype_driver >= 0) 761 rte_log_set_level(opdl_logtype_driver, RTE_LOG_INFO); 762 } 763 764 765 RTE_PMD_REGISTER_VDEV(EVENTDEV_NAME_OPDL_PMD, evdev_opdl_pmd_drv); 766 RTE_PMD_REGISTER_PARAM_STRING(event_opdl, NUMA_NODE_ARG "=<int>" 767 DO_VALIDATION_ARG "=<int>" DO_TEST_ARG "=<int>"); 768