1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2016-2017 Intel Corporation 3 */ 4 5 #include <inttypes.h> 6 #include <string.h> 7 8 #include <rte_bus_vdev.h> 9 #include <rte_kvargs.h> 10 #include <rte_ring.h> 11 #include <rte_errno.h> 12 #include <rte_event_ring.h> 13 #include <rte_service_component.h> 14 15 #include "sw_evdev.h" 16 #include "iq_chunk.h" 17 #include "event_ring.h" 18 19 #define EVENTDEV_NAME_SW_PMD event_sw 20 #define NUMA_NODE_ARG "numa_node" 21 #define SCHED_QUANTA_ARG "sched_quanta" 22 #define CREDIT_QUANTA_ARG "credit_quanta" 23 #define MIN_BURST_SIZE_ARG "min_burst" 24 #define DEQ_BURST_SIZE_ARG "deq_burst" 25 #define REFIL_ONCE_ARG "refill_once" 26 27 static void 28 sw_info_get(struct rte_eventdev *dev, struct rte_event_dev_info *info); 29 30 static int 31 sw_port_link(struct rte_eventdev *dev, void *port, const uint8_t queues[], 32 const uint8_t priorities[], uint16_t num) 33 { 34 struct sw_port *p = port; 35 struct sw_evdev *sw = sw_pmd_priv(dev); 36 int i; 37 38 RTE_SET_USED(priorities); 39 for (i = 0; i < num; i++) { 40 struct sw_qid *q = &sw->qids[queues[i]]; 41 unsigned int j; 42 43 /* check for qid map overflow */ 44 if (q->cq_num_mapped_cqs >= RTE_DIM(q->cq_map)) { 45 rte_errno = EDQUOT; 46 break; 47 } 48 49 if (p->is_directed && p->num_qids_mapped > 0) { 50 rte_errno = EDQUOT; 51 break; 52 } 53 54 for (j = 0; j < q->cq_num_mapped_cqs; j++) { 55 if (q->cq_map[j] == p->id) 56 break; 57 } 58 59 /* check if port is already linked */ 60 if (j < q->cq_num_mapped_cqs) 61 continue; 62 63 if (q->type == SW_SCHED_TYPE_DIRECT) { 64 /* check directed qids only map to one port */ 65 if (p->num_qids_mapped > 0) { 66 rte_errno = EDQUOT; 67 break; 68 } 69 /* check port only takes a directed flow */ 70 if (num > 1) { 71 rte_errno = EDQUOT; 72 break; 73 } 74 75 p->is_directed = 1; 76 p->num_qids_mapped = 1; 77 } else if (q->type == RTE_SCHED_TYPE_ORDERED) { 78 p->num_ordered_qids++; 79 p->num_qids_mapped++; 80 } else if (q->type == RTE_SCHED_TYPE_ATOMIC || 81 q->type == RTE_SCHED_TYPE_PARALLEL) { 82 p->num_qids_mapped++; 83 } 84 85 q->cq_map[q->cq_num_mapped_cqs] = p->id; 86 rte_smp_wmb(); 87 q->cq_num_mapped_cqs++; 88 } 89 return i; 90 } 91 92 static int 93 sw_port_unlink(struct rte_eventdev *dev, void *port, uint8_t queues[], 94 uint16_t nb_unlinks) 95 { 96 struct sw_port *p = port; 97 struct sw_evdev *sw = sw_pmd_priv(dev); 98 unsigned int i, j; 99 100 int unlinked = 0; 101 for (i = 0; i < nb_unlinks; i++) { 102 struct sw_qid *q = &sw->qids[queues[i]]; 103 for (j = 0; j < q->cq_num_mapped_cqs; j++) { 104 if (q->cq_map[j] == p->id) { 105 q->cq_map[j] = 106 q->cq_map[q->cq_num_mapped_cqs - 1]; 107 rte_smp_wmb(); 108 q->cq_num_mapped_cqs--; 109 unlinked++; 110 111 p->num_qids_mapped--; 112 113 if (q->type == RTE_SCHED_TYPE_ORDERED) 114 p->num_ordered_qids--; 115 116 continue; 117 } 118 } 119 } 120 121 p->unlinks_in_progress += unlinked; 122 rte_smp_mb(); 123 124 return unlinked; 125 } 126 127 static int 128 sw_port_unlinks_in_progress(struct rte_eventdev *dev, void *port) 129 { 130 RTE_SET_USED(dev); 131 struct sw_port *p = port; 132 return p->unlinks_in_progress; 133 } 134 135 static int 136 sw_port_setup(struct rte_eventdev *dev, uint8_t port_id, 137 const struct rte_event_port_conf *conf) 138 { 139 struct sw_evdev *sw = sw_pmd_priv(dev); 140 struct sw_port *p = &sw->ports[port_id]; 141 char buf[RTE_RING_NAMESIZE]; 142 unsigned int i; 143 144 struct rte_event_dev_info info; 145 sw_info_get(dev, &info); 146 147 /* detect re-configuring and return credits to instance if needed */ 148 if (p->initialized) { 149 /* taking credits from pool is done one quanta at a time, and 150 * credits may be spend (counted in p->inflights) or still 151 * available in the port (p->inflight_credits). We must return 152 * the sum to no leak credits 153 */ 154 int possible_inflights = p->inflight_credits + p->inflights; 155 rte_atomic32_sub(&sw->inflights, possible_inflights); 156 } 157 158 *p = (struct sw_port){0}; /* zero entire structure */ 159 p->id = port_id; 160 p->sw = sw; 161 162 /* check to see if rings exists - port_setup() can be called multiple 163 * times legally (assuming device is stopped). If ring exists, free it 164 * to so it gets re-created with the correct size 165 */ 166 snprintf(buf, sizeof(buf), "sw%d_p%u_%s", dev->data->dev_id, 167 port_id, "rx_worker_ring"); 168 struct rte_event_ring *existing_ring = rte_event_ring_lookup(buf); 169 if (existing_ring) 170 rte_event_ring_free(existing_ring); 171 172 p->rx_worker_ring = rte_event_ring_create(buf, MAX_SW_PROD_Q_DEPTH, 173 dev->data->socket_id, 174 RING_F_SP_ENQ | RING_F_SC_DEQ | RING_F_EXACT_SZ); 175 if (p->rx_worker_ring == NULL) { 176 SW_LOG_ERR("Error creating RX worker ring for port %d\n", 177 port_id); 178 return -1; 179 } 180 181 p->inflight_max = conf->new_event_threshold; 182 p->implicit_release = !(conf->event_port_cfg & 183 RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL); 184 185 /* check if ring exists, same as rx_worker above */ 186 snprintf(buf, sizeof(buf), "sw%d_p%u, %s", dev->data->dev_id, 187 port_id, "cq_worker_ring"); 188 existing_ring = rte_event_ring_lookup(buf); 189 if (existing_ring) 190 rte_event_ring_free(existing_ring); 191 192 p->cq_worker_ring = rte_event_ring_create(buf, conf->dequeue_depth, 193 dev->data->socket_id, 194 RING_F_SP_ENQ | RING_F_SC_DEQ | RING_F_EXACT_SZ); 195 if (p->cq_worker_ring == NULL) { 196 rte_event_ring_free(p->rx_worker_ring); 197 SW_LOG_ERR("Error creating CQ worker ring for port %d\n", 198 port_id); 199 return -1; 200 } 201 sw->cq_ring_space[port_id] = conf->dequeue_depth; 202 203 /* set hist list contents to empty */ 204 for (i = 0; i < SW_PORT_HIST_LIST; i++) { 205 p->hist_list[i].fid = -1; 206 p->hist_list[i].qid = -1; 207 } 208 dev->data->ports[port_id] = p; 209 210 rte_smp_wmb(); 211 p->initialized = 1; 212 return 0; 213 } 214 215 static void 216 sw_port_release(void *port) 217 { 218 struct sw_port *p = (void *)port; 219 if (p == NULL) 220 return; 221 222 rte_event_ring_free(p->rx_worker_ring); 223 rte_event_ring_free(p->cq_worker_ring); 224 memset(p, 0, sizeof(*p)); 225 } 226 227 static int32_t 228 qid_init(struct sw_evdev *sw, unsigned int idx, int type, 229 const struct rte_event_queue_conf *queue_conf) 230 { 231 unsigned int i; 232 int dev_id = sw->data->dev_id; 233 int socket_id = sw->data->socket_id; 234 char buf[IQ_ROB_NAMESIZE]; 235 struct sw_qid *qid = &sw->qids[idx]; 236 237 /* Initialize the FID structures to no pinning (-1), and zero packets */ 238 const struct sw_fid_t fid = {.cq = -1, .pcount = 0}; 239 for (i = 0; i < RTE_DIM(qid->fids); i++) 240 qid->fids[i] = fid; 241 242 qid->id = idx; 243 qid->type = type; 244 qid->priority = queue_conf->priority; 245 246 if (qid->type == RTE_SCHED_TYPE_ORDERED) { 247 uint32_t window_size; 248 249 /* rte_ring and window_size_mask require require window_size to 250 * be a power-of-2. 251 */ 252 window_size = rte_align32pow2( 253 queue_conf->nb_atomic_order_sequences); 254 255 qid->window_size = window_size - 1; 256 257 if (!window_size) { 258 SW_LOG_DBG( 259 "invalid reorder_window_size for ordered queue\n" 260 ); 261 goto cleanup; 262 } 263 264 snprintf(buf, sizeof(buf), "sw%d_iq_%d_rob", dev_id, i); 265 qid->reorder_buffer = rte_zmalloc_socket(buf, 266 window_size * sizeof(qid->reorder_buffer[0]), 267 0, socket_id); 268 if (!qid->reorder_buffer) { 269 SW_LOG_DBG("reorder_buffer malloc failed\n"); 270 goto cleanup; 271 } 272 273 memset(&qid->reorder_buffer[0], 274 0, 275 window_size * sizeof(qid->reorder_buffer[0])); 276 277 qid->reorder_buffer_freelist = rob_ring_create(window_size, 278 socket_id); 279 if (!qid->reorder_buffer_freelist) { 280 SW_LOG_DBG("freelist ring create failed"); 281 goto cleanup; 282 } 283 284 /* Populate the freelist with reorder buffer entries. Enqueue 285 * 'window_size - 1' entries because the rte_ring holds only 286 * that many. 287 */ 288 for (i = 0; i < window_size - 1; i++) { 289 if (rob_ring_enqueue(qid->reorder_buffer_freelist, 290 &qid->reorder_buffer[i]) != 1) 291 goto cleanup; 292 } 293 294 qid->reorder_buffer_index = 0; 295 qid->cq_next_tx = 0; 296 } 297 298 qid->initialized = 1; 299 300 return 0; 301 302 cleanup: 303 if (qid->reorder_buffer) { 304 rte_free(qid->reorder_buffer); 305 qid->reorder_buffer = NULL; 306 } 307 308 if (qid->reorder_buffer_freelist) { 309 rob_ring_free(qid->reorder_buffer_freelist); 310 qid->reorder_buffer_freelist = NULL; 311 } 312 313 return -EINVAL; 314 } 315 316 static void 317 sw_queue_release(struct rte_eventdev *dev, uint8_t id) 318 { 319 struct sw_evdev *sw = sw_pmd_priv(dev); 320 struct sw_qid *qid = &sw->qids[id]; 321 322 if (qid->type == RTE_SCHED_TYPE_ORDERED) { 323 rte_free(qid->reorder_buffer); 324 rob_ring_free(qid->reorder_buffer_freelist); 325 } 326 memset(qid, 0, sizeof(*qid)); 327 } 328 329 static int 330 sw_queue_setup(struct rte_eventdev *dev, uint8_t queue_id, 331 const struct rte_event_queue_conf *conf) 332 { 333 int type; 334 335 type = conf->schedule_type; 336 337 if (RTE_EVENT_QUEUE_CFG_SINGLE_LINK & conf->event_queue_cfg) { 338 type = SW_SCHED_TYPE_DIRECT; 339 } else if (RTE_EVENT_QUEUE_CFG_ALL_TYPES 340 & conf->event_queue_cfg) { 341 SW_LOG_ERR("QUEUE_CFG_ALL_TYPES not supported\n"); 342 return -ENOTSUP; 343 } 344 345 struct sw_evdev *sw = sw_pmd_priv(dev); 346 347 if (sw->qids[queue_id].initialized) 348 sw_queue_release(dev, queue_id); 349 350 return qid_init(sw, queue_id, type, conf); 351 } 352 353 static void 354 sw_init_qid_iqs(struct sw_evdev *sw) 355 { 356 int i, j; 357 358 /* Initialize the IQ memory of all configured qids */ 359 for (i = 0; i < RTE_EVENT_MAX_QUEUES_PER_DEV; i++) { 360 struct sw_qid *qid = &sw->qids[i]; 361 362 if (!qid->initialized) 363 continue; 364 365 for (j = 0; j < SW_IQS_MAX; j++) 366 iq_init(sw, &qid->iq[j]); 367 } 368 } 369 370 static int 371 sw_qids_empty(struct sw_evdev *sw) 372 { 373 unsigned int i, j; 374 375 for (i = 0; i < sw->qid_count; i++) { 376 for (j = 0; j < SW_IQS_MAX; j++) { 377 if (iq_count(&sw->qids[i].iq[j])) 378 return 0; 379 } 380 } 381 382 return 1; 383 } 384 385 static int 386 sw_ports_empty(struct sw_evdev *sw) 387 { 388 unsigned int i; 389 390 for (i = 0; i < sw->port_count; i++) { 391 if ((rte_event_ring_count(sw->ports[i].rx_worker_ring)) || 392 rte_event_ring_count(sw->ports[i].cq_worker_ring)) 393 return 0; 394 } 395 396 return 1; 397 } 398 399 static void 400 sw_drain_ports(struct rte_eventdev *dev) 401 { 402 struct sw_evdev *sw = sw_pmd_priv(dev); 403 eventdev_stop_flush_t flush; 404 unsigned int i; 405 uint8_t dev_id; 406 void *arg; 407 408 flush = dev->dev_ops->dev_stop_flush; 409 dev_id = dev->data->dev_id; 410 arg = dev->data->dev_stop_flush_arg; 411 412 for (i = 0; i < sw->port_count; i++) { 413 struct rte_event ev; 414 415 while (rte_event_dequeue_burst(dev_id, i, &ev, 1, 0)) { 416 if (flush) 417 flush(dev_id, ev, arg); 418 419 ev.op = RTE_EVENT_OP_RELEASE; 420 rte_event_enqueue_burst(dev_id, i, &ev, 1); 421 } 422 } 423 } 424 425 static void 426 sw_drain_queue(struct rte_eventdev *dev, struct sw_iq *iq) 427 { 428 struct sw_evdev *sw = sw_pmd_priv(dev); 429 eventdev_stop_flush_t flush; 430 uint8_t dev_id; 431 void *arg; 432 433 flush = dev->dev_ops->dev_stop_flush; 434 dev_id = dev->data->dev_id; 435 arg = dev->data->dev_stop_flush_arg; 436 437 while (iq_count(iq) > 0) { 438 struct rte_event ev; 439 440 iq_dequeue_burst(sw, iq, &ev, 1); 441 442 if (flush) 443 flush(dev_id, ev, arg); 444 } 445 } 446 447 static void 448 sw_drain_queues(struct rte_eventdev *dev) 449 { 450 struct sw_evdev *sw = sw_pmd_priv(dev); 451 unsigned int i, j; 452 453 for (i = 0; i < sw->qid_count; i++) { 454 for (j = 0; j < SW_IQS_MAX; j++) 455 sw_drain_queue(dev, &sw->qids[i].iq[j]); 456 } 457 } 458 459 static void 460 sw_clean_qid_iqs(struct rte_eventdev *dev) 461 { 462 struct sw_evdev *sw = sw_pmd_priv(dev); 463 int i, j; 464 465 /* Release the IQ memory of all configured qids */ 466 for (i = 0; i < RTE_EVENT_MAX_QUEUES_PER_DEV; i++) { 467 struct sw_qid *qid = &sw->qids[i]; 468 469 for (j = 0; j < SW_IQS_MAX; j++) { 470 if (!qid->iq[j].head) 471 continue; 472 iq_free_chunk_list(sw, qid->iq[j].head); 473 qid->iq[j].head = NULL; 474 } 475 } 476 } 477 478 static void 479 sw_queue_def_conf(struct rte_eventdev *dev, uint8_t queue_id, 480 struct rte_event_queue_conf *conf) 481 { 482 RTE_SET_USED(dev); 483 RTE_SET_USED(queue_id); 484 485 static const struct rte_event_queue_conf default_conf = { 486 .nb_atomic_flows = 4096, 487 .nb_atomic_order_sequences = 1, 488 .schedule_type = RTE_SCHED_TYPE_ATOMIC, 489 .priority = RTE_EVENT_DEV_PRIORITY_NORMAL, 490 }; 491 492 *conf = default_conf; 493 } 494 495 static void 496 sw_port_def_conf(struct rte_eventdev *dev, uint8_t port_id, 497 struct rte_event_port_conf *port_conf) 498 { 499 RTE_SET_USED(dev); 500 RTE_SET_USED(port_id); 501 502 port_conf->new_event_threshold = 1024; 503 port_conf->dequeue_depth = 16; 504 port_conf->enqueue_depth = 16; 505 port_conf->event_port_cfg = 0; 506 } 507 508 static int 509 sw_dev_configure(const struct rte_eventdev *dev) 510 { 511 struct sw_evdev *sw = sw_pmd_priv(dev); 512 const struct rte_eventdev_data *data = dev->data; 513 const struct rte_event_dev_config *conf = &data->dev_conf; 514 int num_chunks, i; 515 516 sw->qid_count = conf->nb_event_queues; 517 sw->port_count = conf->nb_event_ports; 518 sw->nb_events_limit = conf->nb_events_limit; 519 rte_atomic32_set(&sw->inflights, 0); 520 521 /* Number of chunks sized for worst-case spread of events across IQs */ 522 num_chunks = ((SW_INFLIGHT_EVENTS_TOTAL/SW_EVS_PER_Q_CHUNK)+1) + 523 sw->qid_count*SW_IQS_MAX*2; 524 525 /* If this is a reconfiguration, free the previous IQ allocation. All 526 * IQ chunk references were cleaned out of the QIDs in sw_stop(), and 527 * will be reinitialized in sw_start(). 528 */ 529 if (sw->chunks) 530 rte_free(sw->chunks); 531 532 sw->chunks = rte_malloc_socket(NULL, 533 sizeof(struct sw_queue_chunk) * 534 num_chunks, 535 0, 536 sw->data->socket_id); 537 if (!sw->chunks) 538 return -ENOMEM; 539 540 sw->chunk_list_head = NULL; 541 for (i = 0; i < num_chunks; i++) 542 iq_free_chunk(sw, &sw->chunks[i]); 543 544 if (conf->event_dev_cfg & RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT) 545 return -ENOTSUP; 546 547 return 0; 548 } 549 550 struct rte_eth_dev; 551 552 static int 553 sw_eth_rx_adapter_caps_get(const struct rte_eventdev *dev, 554 const struct rte_eth_dev *eth_dev, 555 uint32_t *caps) 556 { 557 RTE_SET_USED(dev); 558 RTE_SET_USED(eth_dev); 559 *caps = RTE_EVENT_ETH_RX_ADAPTER_SW_CAP; 560 return 0; 561 } 562 563 static int 564 sw_timer_adapter_caps_get(const struct rte_eventdev *dev, uint64_t flags, 565 uint32_t *caps, 566 const struct event_timer_adapter_ops **ops) 567 { 568 RTE_SET_USED(dev); 569 RTE_SET_USED(flags); 570 *caps = 0; 571 572 /* Use default SW ops */ 573 *ops = NULL; 574 575 return 0; 576 } 577 578 static int 579 sw_crypto_adapter_caps_get(const struct rte_eventdev *dev, 580 const struct rte_cryptodev *cdev, 581 uint32_t *caps) 582 { 583 RTE_SET_USED(dev); 584 RTE_SET_USED(cdev); 585 *caps = RTE_EVENT_CRYPTO_ADAPTER_SW_CAP; 586 return 0; 587 } 588 589 static void 590 sw_info_get(struct rte_eventdev *dev, struct rte_event_dev_info *info) 591 { 592 RTE_SET_USED(dev); 593 594 static const struct rte_event_dev_info evdev_sw_info = { 595 .driver_name = SW_PMD_NAME, 596 .max_event_queues = RTE_EVENT_MAX_QUEUES_PER_DEV, 597 .max_event_queue_flows = SW_QID_NUM_FIDS, 598 .max_event_queue_priority_levels = SW_Q_PRIORITY_MAX, 599 .max_event_priority_levels = SW_IQS_MAX, 600 .max_event_ports = SW_PORTS_MAX, 601 .max_event_port_dequeue_depth = MAX_SW_CONS_Q_DEPTH, 602 .max_event_port_enqueue_depth = MAX_SW_PROD_Q_DEPTH, 603 .max_num_events = SW_INFLIGHT_EVENTS_TOTAL, 604 .event_dev_cap = ( 605 RTE_EVENT_DEV_CAP_QUEUE_QOS | 606 RTE_EVENT_DEV_CAP_BURST_MODE | 607 RTE_EVENT_DEV_CAP_EVENT_QOS | 608 RTE_EVENT_DEV_CAP_IMPLICIT_RELEASE_DISABLE| 609 RTE_EVENT_DEV_CAP_RUNTIME_PORT_LINK | 610 RTE_EVENT_DEV_CAP_MULTIPLE_QUEUE_PORT | 611 RTE_EVENT_DEV_CAP_NONSEQ_MODE | 612 RTE_EVENT_DEV_CAP_CARRY_FLOW_ID), 613 }; 614 615 *info = evdev_sw_info; 616 } 617 618 static void 619 sw_dump(struct rte_eventdev *dev, FILE *f) 620 { 621 const struct sw_evdev *sw = sw_pmd_priv(dev); 622 623 static const char * const q_type_strings[] = { 624 "Ordered", "Atomic", "Parallel", "Directed" 625 }; 626 uint32_t i; 627 fprintf(f, "EventDev %s: ports %d, qids %d\n", "todo-fix-name", 628 sw->port_count, sw->qid_count); 629 630 fprintf(f, "\trx %"PRIu64"\n\tdrop %"PRIu64"\n\ttx %"PRIu64"\n", 631 sw->stats.rx_pkts, sw->stats.rx_dropped, sw->stats.tx_pkts); 632 fprintf(f, "\tsched calls: %"PRIu64"\n", sw->sched_called); 633 fprintf(f, "\tsched cq/qid call: %"PRIu64"\n", sw->sched_cq_qid_called); 634 fprintf(f, "\tsched no IQ enq: %"PRIu64"\n", sw->sched_no_iq_enqueues); 635 fprintf(f, "\tsched no CQ enq: %"PRIu64"\n", sw->sched_no_cq_enqueues); 636 uint32_t inflights = rte_atomic32_read(&sw->inflights); 637 uint32_t credits = sw->nb_events_limit - inflights; 638 fprintf(f, "\tinflight %d, credits: %d\n", inflights, credits); 639 640 #define COL_RED "\x1b[31m" 641 #define COL_RESET "\x1b[0m" 642 643 for (i = 0; i < sw->port_count; i++) { 644 int max, j; 645 const struct sw_port *p = &sw->ports[i]; 646 if (!p->initialized) { 647 fprintf(f, " %sPort %d not initialized.%s\n", 648 COL_RED, i, COL_RESET); 649 continue; 650 } 651 fprintf(f, " Port %d %s\n", i, 652 p->is_directed ? " (SingleCons)" : ""); 653 fprintf(f, "\trx %"PRIu64"\tdrop %"PRIu64"\ttx %"PRIu64 654 "\t%sinflight %d%s\n", sw->ports[i].stats.rx_pkts, 655 sw->ports[i].stats.rx_dropped, 656 sw->ports[i].stats.tx_pkts, 657 (p->inflights == p->inflight_max) ? 658 COL_RED : COL_RESET, 659 sw->ports[i].inflights, COL_RESET); 660 661 fprintf(f, "\tMax New: %u" 662 "\tAvg cycles PP: %"PRIu64"\tCredits: %u\n", 663 sw->ports[i].inflight_max, 664 sw->ports[i].avg_pkt_ticks, 665 sw->ports[i].inflight_credits); 666 fprintf(f, "\tReceive burst distribution:\n"); 667 float zp_percent = p->zero_polls * 100.0 / p->total_polls; 668 fprintf(f, zp_percent < 10 ? "\t\t0:%.02f%% " : "\t\t0:%.0f%% ", 669 zp_percent); 670 for (max = (int)RTE_DIM(p->poll_buckets); max-- > 0;) 671 if (p->poll_buckets[max] != 0) 672 break; 673 for (j = 0; j <= max; j++) { 674 if (p->poll_buckets[j] != 0) { 675 float poll_pc = p->poll_buckets[j] * 100.0 / 676 p->total_polls; 677 fprintf(f, "%u-%u:%.02f%% ", 678 ((j << SW_DEQ_STAT_BUCKET_SHIFT) + 1), 679 ((j+1) << SW_DEQ_STAT_BUCKET_SHIFT), 680 poll_pc); 681 } 682 } 683 fprintf(f, "\n"); 684 685 if (p->rx_worker_ring) { 686 uint64_t used = rte_event_ring_count(p->rx_worker_ring); 687 uint64_t space = rte_event_ring_free_count( 688 p->rx_worker_ring); 689 const char *col = (space == 0) ? COL_RED : COL_RESET; 690 fprintf(f, "\t%srx ring used: %4"PRIu64"\tfree: %4" 691 PRIu64 COL_RESET"\n", col, used, space); 692 } else 693 fprintf(f, "\trx ring not initialized.\n"); 694 695 if (p->cq_worker_ring) { 696 uint64_t used = rte_event_ring_count(p->cq_worker_ring); 697 uint64_t space = rte_event_ring_free_count( 698 p->cq_worker_ring); 699 const char *col = (space == 0) ? COL_RED : COL_RESET; 700 fprintf(f, "\t%scq ring used: %4"PRIu64"\tfree: %4" 701 PRIu64 COL_RESET"\n", col, used, space); 702 } else 703 fprintf(f, "\tcq ring not initialized.\n"); 704 } 705 706 for (i = 0; i < sw->qid_count; i++) { 707 const struct sw_qid *qid = &sw->qids[i]; 708 if (!qid->initialized) { 709 fprintf(f, " %sQueue %d not initialized.%s\n", 710 COL_RED, i, COL_RESET); 711 continue; 712 } 713 int affinities_per_port[SW_PORTS_MAX] = {0}; 714 uint32_t inflights = 0; 715 716 fprintf(f, " Queue %d (%s)\n", i, q_type_strings[qid->type]); 717 fprintf(f, "\trx %"PRIu64"\tdrop %"PRIu64"\ttx %"PRIu64"\n", 718 qid->stats.rx_pkts, qid->stats.rx_dropped, 719 qid->stats.tx_pkts); 720 if (qid->type == RTE_SCHED_TYPE_ORDERED) { 721 struct rob_ring *rob_buf_free = 722 qid->reorder_buffer_freelist; 723 if (rob_buf_free) 724 fprintf(f, "\tReorder entries in use: %u\n", 725 rob_ring_free_count(rob_buf_free)); 726 else 727 fprintf(f, 728 "\tReorder buffer not initialized\n"); 729 } 730 731 uint32_t flow; 732 for (flow = 0; flow < RTE_DIM(qid->fids); flow++) 733 if (qid->fids[flow].cq != -1) { 734 affinities_per_port[qid->fids[flow].cq]++; 735 inflights += qid->fids[flow].pcount; 736 } 737 738 uint32_t port; 739 fprintf(f, "\tPer Port Stats:\n"); 740 for (port = 0; port < sw->port_count; port++) { 741 fprintf(f, "\t Port %d: Pkts: %"PRIu64, port, 742 qid->to_port[port]); 743 fprintf(f, "\tFlows: %d\n", affinities_per_port[port]); 744 } 745 746 uint32_t iq; 747 uint32_t iq_printed = 0; 748 for (iq = 0; iq < SW_IQS_MAX; iq++) { 749 if (!qid->iq[iq].head) { 750 fprintf(f, "\tiq %d is not initialized.\n", iq); 751 iq_printed = 1; 752 continue; 753 } 754 uint32_t used = iq_count(&qid->iq[iq]); 755 const char *col = COL_RESET; 756 if (used > 0) { 757 fprintf(f, "\t%siq %d: Used %d" 758 COL_RESET"\n", col, iq, used); 759 iq_printed = 1; 760 } 761 } 762 if (iq_printed == 0) 763 fprintf(f, "\t-- iqs empty --\n"); 764 } 765 } 766 767 static int 768 sw_start(struct rte_eventdev *dev) 769 { 770 unsigned int i, j; 771 struct sw_evdev *sw = sw_pmd_priv(dev); 772 773 rte_service_component_runstate_set(sw->service_id, 1); 774 775 /* check a service core is mapped to this service */ 776 if (!rte_service_runstate_get(sw->service_id)) { 777 SW_LOG_ERR("Warning: No Service core enabled on service %s\n", 778 sw->service_name); 779 return -ENOENT; 780 } 781 782 /* check all ports are set up */ 783 for (i = 0; i < sw->port_count; i++) 784 if (sw->ports[i].rx_worker_ring == NULL) { 785 SW_LOG_ERR("Port %d not configured\n", i); 786 return -ESTALE; 787 } 788 789 /* check all queues are configured and mapped to ports*/ 790 for (i = 0; i < sw->qid_count; i++) 791 if (!sw->qids[i].initialized || 792 sw->qids[i].cq_num_mapped_cqs == 0) { 793 SW_LOG_ERR("Queue %d not configured\n", i); 794 return -ENOLINK; 795 } 796 797 /* build up our prioritized array of qids */ 798 /* We don't use qsort here, as if all/multiple entries have the same 799 * priority, the result is non-deterministic. From "man 3 qsort": 800 * "If two members compare as equal, their order in the sorted 801 * array is undefined." 802 */ 803 uint32_t qidx = 0; 804 for (j = 0; j <= RTE_EVENT_DEV_PRIORITY_LOWEST; j++) { 805 for (i = 0; i < sw->qid_count; i++) { 806 if (sw->qids[i].priority == j) { 807 sw->qids_prioritized[qidx] = &sw->qids[i]; 808 qidx++; 809 } 810 } 811 } 812 813 sw_init_qid_iqs(sw); 814 815 if (sw_xstats_init(sw) < 0) 816 return -EINVAL; 817 818 rte_smp_wmb(); 819 sw->started = 1; 820 821 return 0; 822 } 823 824 static void 825 sw_stop(struct rte_eventdev *dev) 826 { 827 struct sw_evdev *sw = sw_pmd_priv(dev); 828 int32_t runstate; 829 830 /* Stop the scheduler if it's running */ 831 runstate = rte_service_runstate_get(sw->service_id); 832 if (runstate == 1) 833 rte_service_runstate_set(sw->service_id, 0); 834 835 while (rte_service_may_be_active(sw->service_id)) 836 rte_pause(); 837 838 /* Flush all events out of the device */ 839 while (!(sw_qids_empty(sw) && sw_ports_empty(sw))) { 840 sw_event_schedule(dev); 841 sw_drain_ports(dev); 842 sw_drain_queues(dev); 843 } 844 845 sw_clean_qid_iqs(dev); 846 sw_xstats_uninit(sw); 847 sw->started = 0; 848 rte_smp_wmb(); 849 850 if (runstate == 1) 851 rte_service_runstate_set(sw->service_id, 1); 852 } 853 854 static int 855 sw_close(struct rte_eventdev *dev) 856 { 857 struct sw_evdev *sw = sw_pmd_priv(dev); 858 uint32_t i; 859 860 for (i = 0; i < sw->qid_count; i++) 861 sw_queue_release(dev, i); 862 sw->qid_count = 0; 863 864 for (i = 0; i < sw->port_count; i++) 865 sw_port_release(&sw->ports[i]); 866 sw->port_count = 0; 867 868 memset(&sw->stats, 0, sizeof(sw->stats)); 869 sw->sched_called = 0; 870 sw->sched_no_iq_enqueues = 0; 871 sw->sched_no_cq_enqueues = 0; 872 sw->sched_cq_qid_called = 0; 873 874 return 0; 875 } 876 877 static int 878 assign_numa_node(const char *key __rte_unused, const char *value, void *opaque) 879 { 880 int *socket_id = opaque; 881 *socket_id = atoi(value); 882 if (*socket_id >= RTE_MAX_NUMA_NODES) 883 return -1; 884 return 0; 885 } 886 887 static int 888 set_sched_quanta(const char *key __rte_unused, const char *value, void *opaque) 889 { 890 int *quanta = opaque; 891 *quanta = atoi(value); 892 if (*quanta < 0 || *quanta >= 4096) 893 return -1; 894 return 0; 895 } 896 897 static int 898 set_credit_quanta(const char *key __rte_unused, const char *value, void *opaque) 899 { 900 int *credit = opaque; 901 *credit = atoi(value); 902 if (*credit < 0 || *credit >= 128) 903 return -1; 904 return 0; 905 } 906 907 static int 908 set_deq_burst_sz(const char *key __rte_unused, const char *value, void *opaque) 909 { 910 int *deq_burst_sz = opaque; 911 *deq_burst_sz = atoi(value); 912 if (*deq_burst_sz < 0 || *deq_burst_sz > SCHED_DEQUEUE_MAX_BURST_SIZE) 913 return -1; 914 return 0; 915 } 916 917 static int 918 set_min_burst_sz(const char *key __rte_unused, const char *value, void *opaque) 919 { 920 int *min_burst_sz = opaque; 921 *min_burst_sz = atoi(value); 922 if (*min_burst_sz < 0 || *min_burst_sz > SCHED_DEQUEUE_MAX_BURST_SIZE) 923 return -1; 924 return 0; 925 } 926 927 static int 928 set_refill_once(const char *key __rte_unused, const char *value, void *opaque) 929 { 930 int *refill_once_per_call = opaque; 931 *refill_once_per_call = atoi(value); 932 if (*refill_once_per_call < 0 || *refill_once_per_call > 1) 933 return -1; 934 return 0; 935 } 936 937 static int32_t sw_sched_service_func(void *args) 938 { 939 struct rte_eventdev *dev = args; 940 sw_event_schedule(dev); 941 return 0; 942 } 943 944 static int 945 sw_probe(struct rte_vdev_device *vdev) 946 { 947 static struct eventdev_ops evdev_sw_ops = { 948 .dev_configure = sw_dev_configure, 949 .dev_infos_get = sw_info_get, 950 .dev_close = sw_close, 951 .dev_start = sw_start, 952 .dev_stop = sw_stop, 953 .dump = sw_dump, 954 955 .queue_def_conf = sw_queue_def_conf, 956 .queue_setup = sw_queue_setup, 957 .queue_release = sw_queue_release, 958 .port_def_conf = sw_port_def_conf, 959 .port_setup = sw_port_setup, 960 .port_release = sw_port_release, 961 .port_link = sw_port_link, 962 .port_unlink = sw_port_unlink, 963 .port_unlinks_in_progress = sw_port_unlinks_in_progress, 964 965 .eth_rx_adapter_caps_get = sw_eth_rx_adapter_caps_get, 966 967 .timer_adapter_caps_get = sw_timer_adapter_caps_get, 968 969 .crypto_adapter_caps_get = sw_crypto_adapter_caps_get, 970 971 .xstats_get = sw_xstats_get, 972 .xstats_get_names = sw_xstats_get_names, 973 .xstats_get_by_name = sw_xstats_get_by_name, 974 .xstats_reset = sw_xstats_reset, 975 976 .dev_selftest = test_sw_eventdev, 977 }; 978 979 static const char *const args[] = { 980 NUMA_NODE_ARG, 981 SCHED_QUANTA_ARG, 982 CREDIT_QUANTA_ARG, 983 MIN_BURST_SIZE_ARG, 984 DEQ_BURST_SIZE_ARG, 985 REFIL_ONCE_ARG, 986 NULL 987 }; 988 const char *name; 989 const char *params; 990 struct rte_eventdev *dev; 991 struct sw_evdev *sw; 992 int socket_id = rte_socket_id(); 993 int sched_quanta = SW_DEFAULT_SCHED_QUANTA; 994 int credit_quanta = SW_DEFAULT_CREDIT_QUANTA; 995 int min_burst_size = 1; 996 int deq_burst_size = SCHED_DEQUEUE_DEFAULT_BURST_SIZE; 997 int refill_once = 0; 998 999 name = rte_vdev_device_name(vdev); 1000 params = rte_vdev_device_args(vdev); 1001 if (params != NULL && params[0] != '\0') { 1002 struct rte_kvargs *kvlist = rte_kvargs_parse(params, args); 1003 1004 if (!kvlist) { 1005 SW_LOG_INFO( 1006 "Ignoring unsupported parameters when creating device '%s'\n", 1007 name); 1008 } else { 1009 int ret = rte_kvargs_process(kvlist, NUMA_NODE_ARG, 1010 assign_numa_node, &socket_id); 1011 if (ret != 0) { 1012 SW_LOG_ERR( 1013 "%s: Error parsing numa node parameter", 1014 name); 1015 rte_kvargs_free(kvlist); 1016 return ret; 1017 } 1018 1019 ret = rte_kvargs_process(kvlist, SCHED_QUANTA_ARG, 1020 set_sched_quanta, &sched_quanta); 1021 if (ret != 0) { 1022 SW_LOG_ERR( 1023 "%s: Error parsing sched quanta parameter", 1024 name); 1025 rte_kvargs_free(kvlist); 1026 return ret; 1027 } 1028 1029 ret = rte_kvargs_process(kvlist, CREDIT_QUANTA_ARG, 1030 set_credit_quanta, &credit_quanta); 1031 if (ret != 0) { 1032 SW_LOG_ERR( 1033 "%s: Error parsing credit quanta parameter", 1034 name); 1035 rte_kvargs_free(kvlist); 1036 return ret; 1037 } 1038 1039 ret = rte_kvargs_process(kvlist, MIN_BURST_SIZE_ARG, 1040 set_min_burst_sz, &min_burst_size); 1041 if (ret != 0) { 1042 SW_LOG_ERR( 1043 "%s: Error parsing minimum burst size parameter", 1044 name); 1045 rte_kvargs_free(kvlist); 1046 return ret; 1047 } 1048 1049 ret = rte_kvargs_process(kvlist, DEQ_BURST_SIZE_ARG, 1050 set_deq_burst_sz, &deq_burst_size); 1051 if (ret != 0) { 1052 SW_LOG_ERR( 1053 "%s: Error parsing dequeue burst size parameter", 1054 name); 1055 rte_kvargs_free(kvlist); 1056 return ret; 1057 } 1058 1059 ret = rte_kvargs_process(kvlist, REFIL_ONCE_ARG, 1060 set_refill_once, &refill_once); 1061 if (ret != 0) { 1062 SW_LOG_ERR( 1063 "%s: Error parsing refill once per call switch", 1064 name); 1065 rte_kvargs_free(kvlist); 1066 return ret; 1067 } 1068 1069 rte_kvargs_free(kvlist); 1070 } 1071 } 1072 1073 SW_LOG_INFO( 1074 "Creating eventdev sw device %s, numa_node=%d, " 1075 "sched_quanta=%d, credit_quanta=%d " 1076 "min_burst=%d, deq_burst=%d, refill_once=%d\n", 1077 name, socket_id, sched_quanta, credit_quanta, 1078 min_burst_size, deq_burst_size, refill_once); 1079 1080 dev = rte_event_pmd_vdev_init(name, 1081 sizeof(struct sw_evdev), socket_id); 1082 if (dev == NULL) { 1083 SW_LOG_ERR("eventdev vdev init() failed"); 1084 return -EFAULT; 1085 } 1086 dev->dev_ops = &evdev_sw_ops; 1087 dev->enqueue = sw_event_enqueue; 1088 dev->enqueue_burst = sw_event_enqueue_burst; 1089 dev->enqueue_new_burst = sw_event_enqueue_burst; 1090 dev->enqueue_forward_burst = sw_event_enqueue_burst; 1091 dev->dequeue = sw_event_dequeue; 1092 dev->dequeue_burst = sw_event_dequeue_burst; 1093 1094 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 1095 return 0; 1096 1097 sw = dev->data->dev_private; 1098 sw->data = dev->data; 1099 1100 /* copy values passed from vdev command line to instance */ 1101 sw->credit_update_quanta = credit_quanta; 1102 sw->sched_quanta = sched_quanta; 1103 sw->sched_min_burst_size = min_burst_size; 1104 sw->sched_deq_burst_size = deq_burst_size; 1105 sw->refill_once_per_iter = refill_once; 1106 1107 /* register service with EAL */ 1108 struct rte_service_spec service; 1109 memset(&service, 0, sizeof(struct rte_service_spec)); 1110 snprintf(service.name, sizeof(service.name), "%s_service", name); 1111 snprintf(sw->service_name, sizeof(sw->service_name), "%s_service", 1112 name); 1113 service.socket_id = socket_id; 1114 service.callback = sw_sched_service_func; 1115 service.callback_userdata = (void *)dev; 1116 1117 int32_t ret = rte_service_component_register(&service, &sw->service_id); 1118 if (ret) { 1119 SW_LOG_ERR("service register() failed"); 1120 return -ENOEXEC; 1121 } 1122 1123 dev->data->service_inited = 1; 1124 dev->data->service_id = sw->service_id; 1125 1126 event_dev_probing_finish(dev); 1127 1128 return 0; 1129 } 1130 1131 static int 1132 sw_remove(struct rte_vdev_device *vdev) 1133 { 1134 const char *name; 1135 1136 name = rte_vdev_device_name(vdev); 1137 if (name == NULL) 1138 return -EINVAL; 1139 1140 SW_LOG_INFO("Closing eventdev sw device %s\n", name); 1141 1142 return rte_event_pmd_vdev_uninit(name); 1143 } 1144 1145 static struct rte_vdev_driver evdev_sw_pmd_drv = { 1146 .probe = sw_probe, 1147 .remove = sw_remove 1148 }; 1149 1150 RTE_PMD_REGISTER_VDEV(EVENTDEV_NAME_SW_PMD, evdev_sw_pmd_drv); 1151 RTE_PMD_REGISTER_PARAM_STRING(event_sw, NUMA_NODE_ARG "=<int> " 1152 SCHED_QUANTA_ARG "=<int>" CREDIT_QUANTA_ARG "=<int>" 1153 MIN_BURST_SIZE_ARG "=<int>" DEQ_BURST_SIZE_ARG "=<int>" 1154 REFIL_ONCE_ARG "=<int>"); 1155 RTE_LOG_REGISTER_DEFAULT(eventdev_sw_log_level, NOTICE); 1156