1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2016-2017 Intel Corporation 3 */ 4 5 #include <inttypes.h> 6 #include <string.h> 7 8 #include <rte_bus_vdev.h> 9 #include <rte_kvargs.h> 10 #include <rte_ring.h> 11 #include <rte_errno.h> 12 #include <rte_event_ring.h> 13 #include <rte_service_component.h> 14 15 #include "sw_evdev.h" 16 #include "iq_chunk.h" 17 18 #define EVENTDEV_NAME_SW_PMD event_sw 19 #define NUMA_NODE_ARG "numa_node" 20 #define SCHED_QUANTA_ARG "sched_quanta" 21 #define CREDIT_QUANTA_ARG "credit_quanta" 22 23 static void 24 sw_info_get(struct rte_eventdev *dev, struct rte_event_dev_info *info); 25 26 static int 27 sw_port_link(struct rte_eventdev *dev, void *port, const uint8_t queues[], 28 const uint8_t priorities[], uint16_t num) 29 { 30 struct sw_port *p = port; 31 struct sw_evdev *sw = sw_pmd_priv(dev); 32 int i; 33 34 RTE_SET_USED(priorities); 35 for (i = 0; i < num; i++) { 36 struct sw_qid *q = &sw->qids[queues[i]]; 37 unsigned int j; 38 39 /* check for qid map overflow */ 40 if (q->cq_num_mapped_cqs >= RTE_DIM(q->cq_map)) { 41 rte_errno = -EDQUOT; 42 break; 43 } 44 45 if (p->is_directed && p->num_qids_mapped > 0) { 46 rte_errno = -EDQUOT; 47 break; 48 } 49 50 for (j = 0; j < q->cq_num_mapped_cqs; j++) { 51 if (q->cq_map[j] == p->id) 52 break; 53 } 54 55 /* check if port is already linked */ 56 if (j < q->cq_num_mapped_cqs) 57 continue; 58 59 if (q->type == SW_SCHED_TYPE_DIRECT) { 60 /* check directed qids only map to one port */ 61 if (p->num_qids_mapped > 0) { 62 rte_errno = -EDQUOT; 63 break; 64 } 65 /* check port only takes a directed flow */ 66 if (num > 1) { 67 rte_errno = -EDQUOT; 68 break; 69 } 70 71 p->is_directed = 1; 72 p->num_qids_mapped = 1; 73 } else if (q->type == RTE_SCHED_TYPE_ORDERED) { 74 p->num_ordered_qids++; 75 p->num_qids_mapped++; 76 } else if (q->type == RTE_SCHED_TYPE_ATOMIC || 77 q->type == RTE_SCHED_TYPE_PARALLEL) { 78 p->num_qids_mapped++; 79 } 80 81 q->cq_map[q->cq_num_mapped_cqs] = p->id; 82 rte_smp_wmb(); 83 q->cq_num_mapped_cqs++; 84 } 85 return i; 86 } 87 88 static int 89 sw_port_unlink(struct rte_eventdev *dev, void *port, uint8_t queues[], 90 uint16_t nb_unlinks) 91 { 92 struct sw_port *p = port; 93 struct sw_evdev *sw = sw_pmd_priv(dev); 94 unsigned int i, j; 95 96 int unlinked = 0; 97 for (i = 0; i < nb_unlinks; i++) { 98 struct sw_qid *q = &sw->qids[queues[i]]; 99 for (j = 0; j < q->cq_num_mapped_cqs; j++) { 100 if (q->cq_map[j] == p->id) { 101 q->cq_map[j] = 102 q->cq_map[q->cq_num_mapped_cqs - 1]; 103 rte_smp_wmb(); 104 q->cq_num_mapped_cqs--; 105 unlinked++; 106 107 p->num_qids_mapped--; 108 109 if (q->type == RTE_SCHED_TYPE_ORDERED) 110 p->num_ordered_qids--; 111 112 continue; 113 } 114 } 115 } 116 return unlinked; 117 } 118 119 static int 120 sw_port_setup(struct rte_eventdev *dev, uint8_t port_id, 121 const struct rte_event_port_conf *conf) 122 { 123 struct sw_evdev *sw = sw_pmd_priv(dev); 124 struct sw_port *p = &sw->ports[port_id]; 125 char buf[RTE_RING_NAMESIZE]; 126 unsigned int i; 127 128 struct rte_event_dev_info info; 129 sw_info_get(dev, &info); 130 131 /* detect re-configuring and return credits to instance if needed */ 132 if (p->initialized) { 133 /* taking credits from pool is done one quanta at a time, and 134 * credits may be spend (counted in p->inflights) or still 135 * available in the port (p->inflight_credits). We must return 136 * the sum to no leak credits 137 */ 138 int possible_inflights = p->inflight_credits + p->inflights; 139 rte_atomic32_sub(&sw->inflights, possible_inflights); 140 } 141 142 *p = (struct sw_port){0}; /* zero entire structure */ 143 p->id = port_id; 144 p->sw = sw; 145 146 /* check to see if rings exists - port_setup() can be called multiple 147 * times legally (assuming device is stopped). If ring exists, free it 148 * to so it gets re-created with the correct size 149 */ 150 snprintf(buf, sizeof(buf), "sw%d_p%u_%s", dev->data->dev_id, 151 port_id, "rx_worker_ring"); 152 struct rte_event_ring *existing_ring = rte_event_ring_lookup(buf); 153 if (existing_ring) 154 rte_event_ring_free(existing_ring); 155 156 p->rx_worker_ring = rte_event_ring_create(buf, MAX_SW_PROD_Q_DEPTH, 157 dev->data->socket_id, 158 RING_F_SP_ENQ | RING_F_SC_DEQ | RING_F_EXACT_SZ); 159 if (p->rx_worker_ring == NULL) { 160 SW_LOG_ERR("Error creating RX worker ring for port %d\n", 161 port_id); 162 return -1; 163 } 164 165 p->inflight_max = conf->new_event_threshold; 166 p->implicit_release = !conf->disable_implicit_release; 167 168 /* check if ring exists, same as rx_worker above */ 169 snprintf(buf, sizeof(buf), "sw%d_p%u, %s", dev->data->dev_id, 170 port_id, "cq_worker_ring"); 171 existing_ring = rte_event_ring_lookup(buf); 172 if (existing_ring) 173 rte_event_ring_free(existing_ring); 174 175 p->cq_worker_ring = rte_event_ring_create(buf, conf->dequeue_depth, 176 dev->data->socket_id, 177 RING_F_SP_ENQ | RING_F_SC_DEQ | RING_F_EXACT_SZ); 178 if (p->cq_worker_ring == NULL) { 179 rte_event_ring_free(p->rx_worker_ring); 180 SW_LOG_ERR("Error creating CQ worker ring for port %d\n", 181 port_id); 182 return -1; 183 } 184 sw->cq_ring_space[port_id] = conf->dequeue_depth; 185 186 /* set hist list contents to empty */ 187 for (i = 0; i < SW_PORT_HIST_LIST; i++) { 188 p->hist_list[i].fid = -1; 189 p->hist_list[i].qid = -1; 190 } 191 dev->data->ports[port_id] = p; 192 193 rte_smp_wmb(); 194 p->initialized = 1; 195 return 0; 196 } 197 198 static void 199 sw_port_release(void *port) 200 { 201 struct sw_port *p = (void *)port; 202 if (p == NULL) 203 return; 204 205 rte_event_ring_free(p->rx_worker_ring); 206 rte_event_ring_free(p->cq_worker_ring); 207 memset(p, 0, sizeof(*p)); 208 } 209 210 static int32_t 211 qid_init(struct sw_evdev *sw, unsigned int idx, int type, 212 const struct rte_event_queue_conf *queue_conf) 213 { 214 unsigned int i; 215 int dev_id = sw->data->dev_id; 216 int socket_id = sw->data->socket_id; 217 char buf[IQ_ROB_NAMESIZE]; 218 struct sw_qid *qid = &sw->qids[idx]; 219 220 /* Initialize the FID structures to no pinning (-1), and zero packets */ 221 const struct sw_fid_t fid = {.cq = -1, .pcount = 0}; 222 for (i = 0; i < RTE_DIM(qid->fids); i++) 223 qid->fids[i] = fid; 224 225 qid->id = idx; 226 qid->type = type; 227 qid->priority = queue_conf->priority; 228 229 if (qid->type == RTE_SCHED_TYPE_ORDERED) { 230 char ring_name[RTE_RING_NAMESIZE]; 231 uint32_t window_size; 232 233 /* rte_ring and window_size_mask require require window_size to 234 * be a power-of-2. 235 */ 236 window_size = rte_align32pow2( 237 queue_conf->nb_atomic_order_sequences); 238 239 qid->window_size = window_size - 1; 240 241 if (!window_size) { 242 SW_LOG_DBG( 243 "invalid reorder_window_size for ordered queue\n" 244 ); 245 goto cleanup; 246 } 247 248 snprintf(buf, sizeof(buf), "sw%d_iq_%d_rob", dev_id, i); 249 qid->reorder_buffer = rte_zmalloc_socket(buf, 250 window_size * sizeof(qid->reorder_buffer[0]), 251 0, socket_id); 252 if (!qid->reorder_buffer) { 253 SW_LOG_DBG("reorder_buffer malloc failed\n"); 254 goto cleanup; 255 } 256 257 memset(&qid->reorder_buffer[0], 258 0, 259 window_size * sizeof(qid->reorder_buffer[0])); 260 261 snprintf(ring_name, sizeof(ring_name), "sw%d_q%d_freelist", 262 dev_id, idx); 263 264 /* lookup the ring, and if it already exists, free it */ 265 struct rte_ring *cleanup = rte_ring_lookup(ring_name); 266 if (cleanup) 267 rte_ring_free(cleanup); 268 269 qid->reorder_buffer_freelist = rte_ring_create(ring_name, 270 window_size, 271 socket_id, 272 RING_F_SP_ENQ | RING_F_SC_DEQ); 273 if (!qid->reorder_buffer_freelist) { 274 SW_LOG_DBG("freelist ring create failed"); 275 goto cleanup; 276 } 277 278 /* Populate the freelist with reorder buffer entries. Enqueue 279 * 'window_size - 1' entries because the rte_ring holds only 280 * that many. 281 */ 282 for (i = 0; i < window_size - 1; i++) { 283 if (rte_ring_sp_enqueue(qid->reorder_buffer_freelist, 284 &qid->reorder_buffer[i]) < 0) 285 goto cleanup; 286 } 287 288 qid->reorder_buffer_index = 0; 289 qid->cq_next_tx = 0; 290 } 291 292 qid->initialized = 1; 293 294 return 0; 295 296 cleanup: 297 if (qid->reorder_buffer) { 298 rte_free(qid->reorder_buffer); 299 qid->reorder_buffer = NULL; 300 } 301 302 if (qid->reorder_buffer_freelist) { 303 rte_ring_free(qid->reorder_buffer_freelist); 304 qid->reorder_buffer_freelist = NULL; 305 } 306 307 return -EINVAL; 308 } 309 310 static void 311 sw_queue_release(struct rte_eventdev *dev, uint8_t id) 312 { 313 struct sw_evdev *sw = sw_pmd_priv(dev); 314 struct sw_qid *qid = &sw->qids[id]; 315 316 if (qid->type == RTE_SCHED_TYPE_ORDERED) { 317 rte_free(qid->reorder_buffer); 318 rte_ring_free(qid->reorder_buffer_freelist); 319 } 320 memset(qid, 0, sizeof(*qid)); 321 } 322 323 static int 324 sw_queue_setup(struct rte_eventdev *dev, uint8_t queue_id, 325 const struct rte_event_queue_conf *conf) 326 { 327 int type; 328 329 type = conf->schedule_type; 330 331 if (RTE_EVENT_QUEUE_CFG_SINGLE_LINK & conf->event_queue_cfg) { 332 type = SW_SCHED_TYPE_DIRECT; 333 } else if (RTE_EVENT_QUEUE_CFG_ALL_TYPES 334 & conf->event_queue_cfg) { 335 SW_LOG_ERR("QUEUE_CFG_ALL_TYPES not supported\n"); 336 return -ENOTSUP; 337 } 338 339 struct sw_evdev *sw = sw_pmd_priv(dev); 340 341 if (sw->qids[queue_id].initialized) 342 sw_queue_release(dev, queue_id); 343 344 return qid_init(sw, queue_id, type, conf); 345 } 346 347 static void 348 sw_init_qid_iqs(struct sw_evdev *sw) 349 { 350 int i, j; 351 352 /* Initialize the IQ memory of all configured qids */ 353 for (i = 0; i < RTE_EVENT_MAX_QUEUES_PER_DEV; i++) { 354 struct sw_qid *qid = &sw->qids[i]; 355 356 if (!qid->initialized) 357 continue; 358 359 for (j = 0; j < SW_IQS_MAX; j++) 360 iq_init(sw, &qid->iq[j]); 361 } 362 } 363 364 static void 365 sw_clean_qid_iqs(struct sw_evdev *sw) 366 { 367 int i, j; 368 369 /* Release the IQ memory of all configured qids */ 370 for (i = 0; i < RTE_EVENT_MAX_QUEUES_PER_DEV; i++) { 371 struct sw_qid *qid = &sw->qids[i]; 372 373 for (j = 0; j < SW_IQS_MAX; j++) { 374 if (!qid->iq[j].head) 375 continue; 376 iq_free_chunk_list(sw, qid->iq[j].head); 377 qid->iq[j].head = NULL; 378 } 379 } 380 } 381 382 static void 383 sw_queue_def_conf(struct rte_eventdev *dev, uint8_t queue_id, 384 struct rte_event_queue_conf *conf) 385 { 386 RTE_SET_USED(dev); 387 RTE_SET_USED(queue_id); 388 389 static const struct rte_event_queue_conf default_conf = { 390 .nb_atomic_flows = 4096, 391 .nb_atomic_order_sequences = 1, 392 .schedule_type = RTE_SCHED_TYPE_ATOMIC, 393 .priority = RTE_EVENT_DEV_PRIORITY_NORMAL, 394 }; 395 396 *conf = default_conf; 397 } 398 399 static void 400 sw_port_def_conf(struct rte_eventdev *dev, uint8_t port_id, 401 struct rte_event_port_conf *port_conf) 402 { 403 RTE_SET_USED(dev); 404 RTE_SET_USED(port_id); 405 406 port_conf->new_event_threshold = 1024; 407 port_conf->dequeue_depth = 16; 408 port_conf->enqueue_depth = 16; 409 port_conf->disable_implicit_release = 0; 410 } 411 412 static int 413 sw_dev_configure(const struct rte_eventdev *dev) 414 { 415 struct sw_evdev *sw = sw_pmd_priv(dev); 416 const struct rte_eventdev_data *data = dev->data; 417 const struct rte_event_dev_config *conf = &data->dev_conf; 418 int num_chunks, i; 419 420 sw->qid_count = conf->nb_event_queues; 421 sw->port_count = conf->nb_event_ports; 422 sw->nb_events_limit = conf->nb_events_limit; 423 rte_atomic32_set(&sw->inflights, 0); 424 425 /* Number of chunks sized for worst-case spread of events across IQs */ 426 num_chunks = ((SW_INFLIGHT_EVENTS_TOTAL/SW_EVS_PER_Q_CHUNK)+1) + 427 sw->qid_count*SW_IQS_MAX*2; 428 429 /* If this is a reconfiguration, free the previous IQ allocation. All 430 * IQ chunk references were cleaned out of the QIDs in sw_stop(), and 431 * will be reinitialized in sw_start(). 432 */ 433 if (sw->chunks) 434 rte_free(sw->chunks); 435 436 sw->chunks = rte_malloc_socket(NULL, 437 sizeof(struct sw_queue_chunk) * 438 num_chunks, 439 0, 440 sw->data->socket_id); 441 if (!sw->chunks) 442 return -ENOMEM; 443 444 sw->chunk_list_head = NULL; 445 for (i = 0; i < num_chunks; i++) 446 iq_free_chunk(sw, &sw->chunks[i]); 447 448 if (conf->event_dev_cfg & RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT) 449 return -ENOTSUP; 450 451 return 0; 452 } 453 454 struct rte_eth_dev; 455 456 static int 457 sw_eth_rx_adapter_caps_get(const struct rte_eventdev *dev, 458 const struct rte_eth_dev *eth_dev, 459 uint32_t *caps) 460 { 461 RTE_SET_USED(dev); 462 RTE_SET_USED(eth_dev); 463 *caps = RTE_EVENT_ETH_RX_ADAPTER_SW_CAP; 464 return 0; 465 } 466 467 static void 468 sw_info_get(struct rte_eventdev *dev, struct rte_event_dev_info *info) 469 { 470 RTE_SET_USED(dev); 471 472 static const struct rte_event_dev_info evdev_sw_info = { 473 .driver_name = SW_PMD_NAME, 474 .max_event_queues = RTE_EVENT_MAX_QUEUES_PER_DEV, 475 .max_event_queue_flows = SW_QID_NUM_FIDS, 476 .max_event_queue_priority_levels = SW_Q_PRIORITY_MAX, 477 .max_event_priority_levels = SW_IQS_MAX, 478 .max_event_ports = SW_PORTS_MAX, 479 .max_event_port_dequeue_depth = MAX_SW_CONS_Q_DEPTH, 480 .max_event_port_enqueue_depth = MAX_SW_PROD_Q_DEPTH, 481 .max_num_events = SW_INFLIGHT_EVENTS_TOTAL, 482 .event_dev_cap = ( 483 RTE_EVENT_DEV_CAP_QUEUE_QOS | 484 RTE_EVENT_DEV_CAP_BURST_MODE | 485 RTE_EVENT_DEV_CAP_EVENT_QOS | 486 RTE_EVENT_DEV_CAP_IMPLICIT_RELEASE_DISABLE| 487 RTE_EVENT_DEV_CAP_RUNTIME_PORT_LINK | 488 RTE_EVENT_DEV_CAP_MULTIPLE_QUEUE_PORT | 489 RTE_EVENT_DEV_CAP_NONSEQ_MODE), 490 }; 491 492 *info = evdev_sw_info; 493 } 494 495 static void 496 sw_dump(struct rte_eventdev *dev, FILE *f) 497 { 498 const struct sw_evdev *sw = sw_pmd_priv(dev); 499 500 static const char * const q_type_strings[] = { 501 "Ordered", "Atomic", "Parallel", "Directed" 502 }; 503 uint32_t i; 504 fprintf(f, "EventDev %s: ports %d, qids %d\n", "todo-fix-name", 505 sw->port_count, sw->qid_count); 506 507 fprintf(f, "\trx %"PRIu64"\n\tdrop %"PRIu64"\n\ttx %"PRIu64"\n", 508 sw->stats.rx_pkts, sw->stats.rx_dropped, sw->stats.tx_pkts); 509 fprintf(f, "\tsched calls: %"PRIu64"\n", sw->sched_called); 510 fprintf(f, "\tsched cq/qid call: %"PRIu64"\n", sw->sched_cq_qid_called); 511 fprintf(f, "\tsched no IQ enq: %"PRIu64"\n", sw->sched_no_iq_enqueues); 512 fprintf(f, "\tsched no CQ enq: %"PRIu64"\n", sw->sched_no_cq_enqueues); 513 uint32_t inflights = rte_atomic32_read(&sw->inflights); 514 uint32_t credits = sw->nb_events_limit - inflights; 515 fprintf(f, "\tinflight %d, credits: %d\n", inflights, credits); 516 517 #define COL_RED "\x1b[31m" 518 #define COL_RESET "\x1b[0m" 519 520 for (i = 0; i < sw->port_count; i++) { 521 int max, j; 522 const struct sw_port *p = &sw->ports[i]; 523 if (!p->initialized) { 524 fprintf(f, " %sPort %d not initialized.%s\n", 525 COL_RED, i, COL_RESET); 526 continue; 527 } 528 fprintf(f, " Port %d %s\n", i, 529 p->is_directed ? " (SingleCons)" : ""); 530 fprintf(f, "\trx %"PRIu64"\tdrop %"PRIu64"\ttx %"PRIu64 531 "\t%sinflight %d%s\n", sw->ports[i].stats.rx_pkts, 532 sw->ports[i].stats.rx_dropped, 533 sw->ports[i].stats.tx_pkts, 534 (p->inflights == p->inflight_max) ? 535 COL_RED : COL_RESET, 536 sw->ports[i].inflights, COL_RESET); 537 538 fprintf(f, "\tMax New: %u" 539 "\tAvg cycles PP: %"PRIu64"\tCredits: %u\n", 540 sw->ports[i].inflight_max, 541 sw->ports[i].avg_pkt_ticks, 542 sw->ports[i].inflight_credits); 543 fprintf(f, "\tReceive burst distribution:\n"); 544 float zp_percent = p->zero_polls * 100.0 / p->total_polls; 545 fprintf(f, zp_percent < 10 ? "\t\t0:%.02f%% " : "\t\t0:%.0f%% ", 546 zp_percent); 547 for (max = (int)RTE_DIM(p->poll_buckets); max-- > 0;) 548 if (p->poll_buckets[max] != 0) 549 break; 550 for (j = 0; j <= max; j++) { 551 if (p->poll_buckets[j] != 0) { 552 float poll_pc = p->poll_buckets[j] * 100.0 / 553 p->total_polls; 554 fprintf(f, "%u-%u:%.02f%% ", 555 ((j << SW_DEQ_STAT_BUCKET_SHIFT) + 1), 556 ((j+1) << SW_DEQ_STAT_BUCKET_SHIFT), 557 poll_pc); 558 } 559 } 560 fprintf(f, "\n"); 561 562 if (p->rx_worker_ring) { 563 uint64_t used = rte_event_ring_count(p->rx_worker_ring); 564 uint64_t space = rte_event_ring_free_count( 565 p->rx_worker_ring); 566 const char *col = (space == 0) ? COL_RED : COL_RESET; 567 fprintf(f, "\t%srx ring used: %4"PRIu64"\tfree: %4" 568 PRIu64 COL_RESET"\n", col, used, space); 569 } else 570 fprintf(f, "\trx ring not initialized.\n"); 571 572 if (p->cq_worker_ring) { 573 uint64_t used = rte_event_ring_count(p->cq_worker_ring); 574 uint64_t space = rte_event_ring_free_count( 575 p->cq_worker_ring); 576 const char *col = (space == 0) ? COL_RED : COL_RESET; 577 fprintf(f, "\t%scq ring used: %4"PRIu64"\tfree: %4" 578 PRIu64 COL_RESET"\n", col, used, space); 579 } else 580 fprintf(f, "\tcq ring not initialized.\n"); 581 } 582 583 for (i = 0; i < sw->qid_count; i++) { 584 const struct sw_qid *qid = &sw->qids[i]; 585 if (!qid->initialized) { 586 fprintf(f, " %sQueue %d not initialized.%s\n", 587 COL_RED, i, COL_RESET); 588 continue; 589 } 590 int affinities_per_port[SW_PORTS_MAX] = {0}; 591 uint32_t inflights = 0; 592 593 fprintf(f, " Queue %d (%s)\n", i, q_type_strings[qid->type]); 594 fprintf(f, "\trx %"PRIu64"\tdrop %"PRIu64"\ttx %"PRIu64"\n", 595 qid->stats.rx_pkts, qid->stats.rx_dropped, 596 qid->stats.tx_pkts); 597 if (qid->type == RTE_SCHED_TYPE_ORDERED) { 598 struct rte_ring *rob_buf_free = 599 qid->reorder_buffer_freelist; 600 if (rob_buf_free) 601 fprintf(f, "\tReorder entries in use: %u\n", 602 rte_ring_free_count(rob_buf_free)); 603 else 604 fprintf(f, 605 "\tReorder buffer not initialized\n"); 606 } 607 608 uint32_t flow; 609 for (flow = 0; flow < RTE_DIM(qid->fids); flow++) 610 if (qid->fids[flow].cq != -1) { 611 affinities_per_port[qid->fids[flow].cq]++; 612 inflights += qid->fids[flow].pcount; 613 } 614 615 uint32_t port; 616 fprintf(f, "\tPer Port Stats:\n"); 617 for (port = 0; port < sw->port_count; port++) { 618 fprintf(f, "\t Port %d: Pkts: %"PRIu64, port, 619 qid->to_port[port]); 620 fprintf(f, "\tFlows: %d\n", affinities_per_port[port]); 621 } 622 623 uint32_t iq; 624 uint32_t iq_printed = 0; 625 for (iq = 0; iq < SW_IQS_MAX; iq++) { 626 if (!qid->iq[iq].head) { 627 fprintf(f, "\tiq %d is not initialized.\n", iq); 628 iq_printed = 1; 629 continue; 630 } 631 uint32_t used = iq_count(&qid->iq[iq]); 632 const char *col = COL_RESET; 633 if (used > 0) { 634 fprintf(f, "\t%siq %d: Used %d" 635 COL_RESET"\n", col, iq, used); 636 iq_printed = 1; 637 } 638 } 639 if (iq_printed == 0) 640 fprintf(f, "\t-- iqs empty --\n"); 641 } 642 } 643 644 static int 645 sw_start(struct rte_eventdev *dev) 646 { 647 unsigned int i, j; 648 struct sw_evdev *sw = sw_pmd_priv(dev); 649 650 rte_service_component_runstate_set(sw->service_id, 1); 651 652 /* check a service core is mapped to this service */ 653 if (!rte_service_runstate_get(sw->service_id)) { 654 SW_LOG_ERR("Warning: No Service core enabled on service %s\n", 655 sw->service_name); 656 return -ENOENT; 657 } 658 659 /* check all ports are set up */ 660 for (i = 0; i < sw->port_count; i++) 661 if (sw->ports[i].rx_worker_ring == NULL) { 662 SW_LOG_ERR("Port %d not configured\n", i); 663 return -ESTALE; 664 } 665 666 /* check all queues are configured and mapped to ports*/ 667 for (i = 0; i < sw->qid_count; i++) 668 if (!sw->qids[i].initialized || 669 sw->qids[i].cq_num_mapped_cqs == 0) { 670 SW_LOG_ERR("Queue %d not configured\n", i); 671 return -ENOLINK; 672 } 673 674 /* build up our prioritized array of qids */ 675 /* We don't use qsort here, as if all/multiple entries have the same 676 * priority, the result is non-deterministic. From "man 3 qsort": 677 * "If two members compare as equal, their order in the sorted 678 * array is undefined." 679 */ 680 uint32_t qidx = 0; 681 for (j = 0; j <= RTE_EVENT_DEV_PRIORITY_LOWEST; j++) { 682 for (i = 0; i < sw->qid_count; i++) { 683 if (sw->qids[i].priority == j) { 684 sw->qids_prioritized[qidx] = &sw->qids[i]; 685 qidx++; 686 } 687 } 688 } 689 690 sw_init_qid_iqs(sw); 691 692 if (sw_xstats_init(sw) < 0) 693 return -EINVAL; 694 695 rte_smp_wmb(); 696 sw->started = 1; 697 698 return 0; 699 } 700 701 static void 702 sw_stop(struct rte_eventdev *dev) 703 { 704 struct sw_evdev *sw = sw_pmd_priv(dev); 705 sw_clean_qid_iqs(sw); 706 sw_xstats_uninit(sw); 707 sw->started = 0; 708 rte_smp_wmb(); 709 } 710 711 static int 712 sw_close(struct rte_eventdev *dev) 713 { 714 struct sw_evdev *sw = sw_pmd_priv(dev); 715 uint32_t i; 716 717 for (i = 0; i < sw->qid_count; i++) 718 sw_queue_release(dev, i); 719 sw->qid_count = 0; 720 721 for (i = 0; i < sw->port_count; i++) 722 sw_port_release(&sw->ports[i]); 723 sw->port_count = 0; 724 725 memset(&sw->stats, 0, sizeof(sw->stats)); 726 sw->sched_called = 0; 727 sw->sched_no_iq_enqueues = 0; 728 sw->sched_no_cq_enqueues = 0; 729 sw->sched_cq_qid_called = 0; 730 731 return 0; 732 } 733 734 static int 735 assign_numa_node(const char *key __rte_unused, const char *value, void *opaque) 736 { 737 int *socket_id = opaque; 738 *socket_id = atoi(value); 739 if (*socket_id >= RTE_MAX_NUMA_NODES) 740 return -1; 741 return 0; 742 } 743 744 static int 745 set_sched_quanta(const char *key __rte_unused, const char *value, void *opaque) 746 { 747 int *quanta = opaque; 748 *quanta = atoi(value); 749 if (*quanta < 0 || *quanta >= 4096) 750 return -1; 751 return 0; 752 } 753 754 static int 755 set_credit_quanta(const char *key __rte_unused, const char *value, void *opaque) 756 { 757 int *credit = opaque; 758 *credit = atoi(value); 759 if (*credit < 0 || *credit >= 128) 760 return -1; 761 return 0; 762 } 763 764 765 static int32_t sw_sched_service_func(void *args) 766 { 767 struct rte_eventdev *dev = args; 768 sw_event_schedule(dev); 769 return 0; 770 } 771 772 static int 773 sw_probe(struct rte_vdev_device *vdev) 774 { 775 static const struct rte_eventdev_ops evdev_sw_ops = { 776 .dev_configure = sw_dev_configure, 777 .dev_infos_get = sw_info_get, 778 .dev_close = sw_close, 779 .dev_start = sw_start, 780 .dev_stop = sw_stop, 781 .dump = sw_dump, 782 783 .queue_def_conf = sw_queue_def_conf, 784 .queue_setup = sw_queue_setup, 785 .queue_release = sw_queue_release, 786 .port_def_conf = sw_port_def_conf, 787 .port_setup = sw_port_setup, 788 .port_release = sw_port_release, 789 .port_link = sw_port_link, 790 .port_unlink = sw_port_unlink, 791 792 .eth_rx_adapter_caps_get = sw_eth_rx_adapter_caps_get, 793 794 .xstats_get = sw_xstats_get, 795 .xstats_get_names = sw_xstats_get_names, 796 .xstats_get_by_name = sw_xstats_get_by_name, 797 .xstats_reset = sw_xstats_reset, 798 799 .dev_selftest = test_sw_eventdev, 800 }; 801 802 static const char *const args[] = { 803 NUMA_NODE_ARG, 804 SCHED_QUANTA_ARG, 805 CREDIT_QUANTA_ARG, 806 NULL 807 }; 808 const char *name; 809 const char *params; 810 struct rte_eventdev *dev; 811 struct sw_evdev *sw; 812 int socket_id = rte_socket_id(); 813 int sched_quanta = SW_DEFAULT_SCHED_QUANTA; 814 int credit_quanta = SW_DEFAULT_CREDIT_QUANTA; 815 816 name = rte_vdev_device_name(vdev); 817 params = rte_vdev_device_args(vdev); 818 if (params != NULL && params[0] != '\0') { 819 struct rte_kvargs *kvlist = rte_kvargs_parse(params, args); 820 821 if (!kvlist) { 822 SW_LOG_INFO( 823 "Ignoring unsupported parameters when creating device '%s'\n", 824 name); 825 } else { 826 int ret = rte_kvargs_process(kvlist, NUMA_NODE_ARG, 827 assign_numa_node, &socket_id); 828 if (ret != 0) { 829 SW_LOG_ERR( 830 "%s: Error parsing numa node parameter", 831 name); 832 rte_kvargs_free(kvlist); 833 return ret; 834 } 835 836 ret = rte_kvargs_process(kvlist, SCHED_QUANTA_ARG, 837 set_sched_quanta, &sched_quanta); 838 if (ret != 0) { 839 SW_LOG_ERR( 840 "%s: Error parsing sched quanta parameter", 841 name); 842 rte_kvargs_free(kvlist); 843 return ret; 844 } 845 846 ret = rte_kvargs_process(kvlist, CREDIT_QUANTA_ARG, 847 set_credit_quanta, &credit_quanta); 848 if (ret != 0) { 849 SW_LOG_ERR( 850 "%s: Error parsing credit quanta parameter", 851 name); 852 rte_kvargs_free(kvlist); 853 return ret; 854 } 855 856 rte_kvargs_free(kvlist); 857 } 858 } 859 860 SW_LOG_INFO( 861 "Creating eventdev sw device %s, numa_node=%d, sched_quanta=%d, credit_quanta=%d\n", 862 name, socket_id, sched_quanta, credit_quanta); 863 864 dev = rte_event_pmd_vdev_init(name, 865 sizeof(struct sw_evdev), socket_id); 866 if (dev == NULL) { 867 SW_LOG_ERR("eventdev vdev init() failed"); 868 return -EFAULT; 869 } 870 dev->dev_ops = &evdev_sw_ops; 871 dev->enqueue = sw_event_enqueue; 872 dev->enqueue_burst = sw_event_enqueue_burst; 873 dev->enqueue_new_burst = sw_event_enqueue_burst; 874 dev->enqueue_forward_burst = sw_event_enqueue_burst; 875 dev->dequeue = sw_event_dequeue; 876 dev->dequeue_burst = sw_event_dequeue_burst; 877 878 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 879 return 0; 880 881 sw = dev->data->dev_private; 882 sw->data = dev->data; 883 884 /* copy values passed from vdev command line to instance */ 885 sw->credit_update_quanta = credit_quanta; 886 sw->sched_quanta = sched_quanta; 887 888 /* register service with EAL */ 889 struct rte_service_spec service; 890 memset(&service, 0, sizeof(struct rte_service_spec)); 891 snprintf(service.name, sizeof(service.name), "%s_service", name); 892 snprintf(sw->service_name, sizeof(sw->service_name), "%s_service", 893 name); 894 service.socket_id = socket_id; 895 service.callback = sw_sched_service_func; 896 service.callback_userdata = (void *)dev; 897 898 int32_t ret = rte_service_component_register(&service, &sw->service_id); 899 if (ret) { 900 SW_LOG_ERR("service register() failed"); 901 return -ENOEXEC; 902 } 903 904 dev->data->service_inited = 1; 905 dev->data->service_id = sw->service_id; 906 907 return 0; 908 } 909 910 static int 911 sw_remove(struct rte_vdev_device *vdev) 912 { 913 const char *name; 914 915 name = rte_vdev_device_name(vdev); 916 if (name == NULL) 917 return -EINVAL; 918 919 SW_LOG_INFO("Closing eventdev sw device %s\n", name); 920 921 return rte_event_pmd_vdev_uninit(name); 922 } 923 924 static struct rte_vdev_driver evdev_sw_pmd_drv = { 925 .probe = sw_probe, 926 .remove = sw_remove 927 }; 928 929 RTE_PMD_REGISTER_VDEV(EVENTDEV_NAME_SW_PMD, evdev_sw_pmd_drv); 930 RTE_PMD_REGISTER_PARAM_STRING(event_sw, NUMA_NODE_ARG "=<int> " 931 SCHED_QUANTA_ARG "=<int>" CREDIT_QUANTA_ARG "=<int>"); 932