1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017 Intel Corporation 3 */ 4 5 #include <inttypes.h> 6 #include <string.h> 7 8 #include <rte_bus_vdev.h> 9 #include <rte_errno.h> 10 #include <rte_cycles.h> 11 #include <rte_memzone.h> 12 13 #include "opdl_evdev.h" 14 #include "opdl_ring.h" 15 #include "opdl_log.h" 16 17 18 static __rte_always_inline uint32_t 19 enqueue_check(struct opdl_port *p, 20 const struct rte_event ev[], 21 uint16_t num, 22 uint16_t num_events) 23 { 24 uint16_t i; 25 26 if (p->opdl->do_validation) { 27 28 for (i = 0; i < num; i++) { 29 if (ev[i].queue_id != p->next_external_qid) { 30 PMD_DRV_LOG(ERR, "DEV_ID:[%02d] : " 31 "ERROR - port:[%u] - event wants" 32 " to enq to q_id[%u]," 33 " but should be [%u]", 34 opdl_pmd_dev_id(p->opdl), 35 p->id, 36 ev[i].queue_id, 37 p->next_external_qid); 38 rte_errno = EINVAL; 39 return 0; 40 } 41 } 42 43 /* Stats */ 44 if (p->p_type == OPDL_PURE_RX_PORT || 45 p->p_type == OPDL_ASYNC_PORT) { 46 /* Stats */ 47 if (num_events) { 48 p->port_stat[claim_pkts_requested] += num; 49 p->port_stat[claim_pkts_granted] += num_events; 50 p->port_stat[claim_non_empty]++; 51 p->start_cycles = rte_rdtsc(); 52 } else { 53 p->port_stat[claim_empty]++; 54 p->start_cycles = 0; 55 } 56 } else { 57 if (p->start_cycles) { 58 uint64_t end_cycles = rte_rdtsc(); 59 p->port_stat[total_cycles] += 60 end_cycles - p->start_cycles; 61 } 62 } 63 } else { 64 if (num > 0 && 65 ev[0].queue_id != p->next_external_qid) { 66 rte_errno = EINVAL; 67 return 0; 68 } 69 } 70 71 return num; 72 } 73 74 static __rte_always_inline void 75 update_on_dequeue(struct opdl_port *p, 76 struct rte_event ev[], 77 uint16_t num, 78 uint16_t num_events) 79 { 80 if (p->opdl->do_validation) { 81 int16_t i; 82 for (i = 0; i < num; i++) 83 ev[i].queue_id = 84 p->opdl->queue[p->queue_id].external_qid; 85 86 /* Stats */ 87 if (num_events) { 88 p->port_stat[claim_pkts_requested] += num; 89 p->port_stat[claim_pkts_granted] += num_events; 90 p->port_stat[claim_non_empty]++; 91 p->start_cycles = rte_rdtsc(); 92 } else { 93 p->port_stat[claim_empty]++; 94 p->start_cycles = 0; 95 } 96 } else { 97 if (num > 0) 98 ev[0].queue_id = 99 p->opdl->queue[p->queue_id].external_qid; 100 } 101 } 102 103 104 /* 105 * Error RX enqueue: 106 * 107 * 108 */ 109 110 static uint16_t 111 opdl_rx_error_enqueue(struct opdl_port *p, 112 const struct rte_event ev[], 113 uint16_t num) 114 { 115 RTE_SET_USED(p); 116 RTE_SET_USED(ev); 117 RTE_SET_USED(num); 118 119 rte_errno = ENOSPC; 120 121 return 0; 122 } 123 124 /* 125 * RX enqueue: 126 * 127 * This function handles enqueue for a single input stage_inst with 128 * threadsafe disabled or enabled. eg 1 thread using a stage_inst or 129 * multiple threads sharing a stage_inst 130 */ 131 132 static uint16_t 133 opdl_rx_enqueue(struct opdl_port *p, 134 const struct rte_event ev[], 135 uint16_t num) 136 { 137 uint16_t enqueued = 0; 138 139 enqueued = opdl_ring_input(opdl_stage_get_opdl_ring(p->enq_stage_inst), 140 ev, 141 num, 142 false); 143 if (!enqueue_check(p, ev, num, enqueued)) 144 return 0; 145 146 147 if (enqueued < num) 148 rte_errno = ENOSPC; 149 150 return enqueued; 151 } 152 153 /* 154 * Error TX handler 155 * 156 */ 157 158 static uint16_t 159 opdl_tx_error_dequeue(struct opdl_port *p, 160 struct rte_event ev[], 161 uint16_t num) 162 { 163 RTE_SET_USED(p); 164 RTE_SET_USED(ev); 165 RTE_SET_USED(num); 166 167 rte_errno = ENOSPC; 168 169 return 0; 170 } 171 172 /* 173 * TX single threaded claim 174 * 175 * This function handles dequeue for a single worker stage_inst with 176 * threadsafe disabled. eg 1 thread using an stage_inst 177 */ 178 179 static uint16_t 180 opdl_tx_dequeue_single_thread(struct opdl_port *p, 181 struct rte_event ev[], 182 uint16_t num) 183 { 184 uint16_t returned; 185 186 struct opdl_ring *ring; 187 188 ring = opdl_stage_get_opdl_ring(p->deq_stage_inst); 189 190 returned = opdl_ring_copy_to_burst(ring, 191 p->deq_stage_inst, 192 ev, 193 num, 194 false); 195 196 update_on_dequeue(p, ev, num, returned); 197 198 return returned; 199 } 200 201 /* 202 * TX multi threaded claim 203 * 204 * This function handles dequeue for multiple worker stage_inst with 205 * threadsafe disabled. eg multiple stage_inst each with its own instance 206 */ 207 208 static uint16_t 209 opdl_tx_dequeue_multi_inst(struct opdl_port *p, 210 struct rte_event ev[], 211 uint16_t num) 212 { 213 uint32_t num_events = 0; 214 215 num_events = opdl_stage_claim(p->deq_stage_inst, 216 (void *)ev, 217 num, 218 NULL, 219 false, 220 false); 221 222 update_on_dequeue(p, ev, num, num_events); 223 224 return opdl_stage_disclaim(p->deq_stage_inst, num_events, false); 225 } 226 227 228 /* 229 * Worker thread claim 230 * 231 */ 232 233 static uint16_t 234 opdl_claim(struct opdl_port *p, struct rte_event ev[], uint16_t num) 235 { 236 uint32_t num_events = 0; 237 238 if (unlikely(num > MAX_OPDL_CONS_Q_DEPTH)) { 239 PMD_DRV_LOG(ERR, "DEV_ID:[%02d] : " 240 "Attempt to dequeue num of events larger than port (%d) max", 241 opdl_pmd_dev_id(p->opdl), 242 p->id); 243 rte_errno = EINVAL; 244 return 0; 245 } 246 247 248 num_events = opdl_stage_claim(p->deq_stage_inst, 249 (void *)ev, 250 num, 251 NULL, 252 false, 253 p->atomic_claim); 254 255 256 update_on_dequeue(p, ev, num, num_events); 257 258 return num_events; 259 } 260 261 /* 262 * Worker thread disclaim 263 */ 264 265 static uint16_t 266 opdl_disclaim(struct opdl_port *p, const struct rte_event ev[], uint16_t num) 267 { 268 uint16_t enqueued = 0; 269 270 uint32_t i = 0; 271 272 for (i = 0; i < num; i++) 273 opdl_ring_cas_slot(p->enq_stage_inst, &ev[i], 274 i, p->atomic_claim); 275 276 enqueued = opdl_stage_disclaim(p->enq_stage_inst, 277 num, 278 false); 279 280 return enqueue_check(p, ev, num, enqueued); 281 } 282 283 static __rte_always_inline struct opdl_stage * 284 stage_for_port(struct opdl_queue *q, unsigned int i) 285 { 286 if (q->q_pos == OPDL_Q_POS_START || q->q_pos == OPDL_Q_POS_MIDDLE) 287 return q->ports[i]->enq_stage_inst; 288 else 289 return q->ports[i]->deq_stage_inst; 290 } 291 292 static int opdl_add_deps(struct opdl_evdev *device, 293 int q_id, 294 int deps_q_id) 295 { 296 unsigned int i, j; 297 int status; 298 struct opdl_ring *ring; 299 struct opdl_queue *queue = &device->queue[q_id]; 300 struct opdl_queue *queue_deps = &device->queue[deps_q_id]; 301 struct opdl_stage *dep_stages[OPDL_PORTS_MAX]; 302 303 /* sanity check that all stages are for same opdl ring */ 304 for (i = 0; i < queue->nb_ports; i++) { 305 struct opdl_ring *r = 306 opdl_stage_get_opdl_ring(stage_for_port(queue, i)); 307 for (j = 0; j < queue_deps->nb_ports; j++) { 308 struct opdl_ring *rj = 309 opdl_stage_get_opdl_ring( 310 stage_for_port(queue_deps, j)); 311 if (r != rj) { 312 PMD_DRV_LOG(ERR, "DEV_ID:[%02d] : " 313 "Stages and dependents" 314 " are not for same opdl ring", 315 opdl_pmd_dev_id(device)); 316 uint32_t k; 317 for (k = 0; k < device->nb_opdls; k++) { 318 opdl_ring_dump(device->opdl[k], 319 stdout); 320 } 321 return -EINVAL; 322 } 323 } 324 } 325 326 /* Gather all stages instance in deps */ 327 for (i = 0; i < queue_deps->nb_ports; i++) 328 dep_stages[i] = stage_for_port(queue_deps, i); 329 330 331 /* Add all deps for each port->stage_inst in this queue */ 332 for (i = 0; i < queue->nb_ports; i++) { 333 334 ring = opdl_stage_get_opdl_ring(stage_for_port(queue, i)); 335 336 status = opdl_stage_deps_add(ring, 337 stage_for_port(queue, i), 338 queue->ports[i]->num_instance, 339 queue->ports[i]->instance_id, 340 dep_stages, 341 queue_deps->nb_ports); 342 if (status < 0) 343 return -EINVAL; 344 } 345 346 return 0; 347 } 348 349 int 350 opdl_add_event_handlers(struct rte_eventdev *dev) 351 { 352 int err = 0; 353 354 struct opdl_evdev *device = opdl_pmd_priv(dev); 355 unsigned int i; 356 357 for (i = 0; i < device->max_port_nb; i++) { 358 359 struct opdl_port *port = &device->ports[i]; 360 361 if (port->configured) { 362 if (port->p_type == OPDL_PURE_RX_PORT) { 363 port->enq = opdl_rx_enqueue; 364 port->deq = opdl_tx_error_dequeue; 365 366 } else if (port->p_type == OPDL_PURE_TX_PORT) { 367 368 port->enq = opdl_rx_error_enqueue; 369 370 if (port->num_instance == 1) 371 port->deq = 372 opdl_tx_dequeue_single_thread; 373 else 374 port->deq = opdl_tx_dequeue_multi_inst; 375 376 } else if (port->p_type == OPDL_REGULAR_PORT) { 377 378 port->enq = opdl_disclaim; 379 port->deq = opdl_claim; 380 381 } else if (port->p_type == OPDL_ASYNC_PORT) { 382 383 port->enq = opdl_rx_enqueue; 384 385 /* Always single instance */ 386 port->deq = opdl_tx_dequeue_single_thread; 387 } else { 388 PMD_DRV_LOG(ERR, "DEV_ID:[%02d] : " 389 "port:[%u] has invalid port type - ", 390 opdl_pmd_dev_id(port->opdl), 391 port->id); 392 err = -EINVAL; 393 break; 394 } 395 port->initialized = 1; 396 } 397 } 398 399 if (!err) 400 fprintf(stdout, "Success - enqueue/dequeue handler(s) added\n"); 401 return err; 402 } 403 404 int 405 build_all_dependencies(struct rte_eventdev *dev) 406 { 407 408 int err = 0; 409 unsigned int i; 410 struct opdl_evdev *device = opdl_pmd_priv(dev); 411 412 uint8_t start_qid = 0; 413 414 for (i = 0; i < RTE_EVENT_MAX_QUEUES_PER_DEV; i++) { 415 struct opdl_queue *queue = &device->queue[i]; 416 if (!queue->initialized) 417 break; 418 419 if (queue->q_pos == OPDL_Q_POS_START) { 420 start_qid = i; 421 continue; 422 } 423 424 if (queue->q_pos == OPDL_Q_POS_MIDDLE) { 425 err = opdl_add_deps(device, i, i-1); 426 if (err < 0) { 427 PMD_DRV_LOG(ERR, "DEV_ID:[%02d] : " 428 "dependency addition for queue:[%u] - FAILED", 429 dev->data->dev_id, 430 queue->external_qid); 431 break; 432 } 433 } 434 435 if (queue->q_pos == OPDL_Q_POS_END) { 436 /* Add this dependency */ 437 err = opdl_add_deps(device, i, i-1); 438 if (err < 0) { 439 PMD_DRV_LOG(ERR, "DEV_ID:[%02d] : " 440 "dependency addition for queue:[%u] - FAILED", 441 dev->data->dev_id, 442 queue->external_qid); 443 break; 444 } 445 /* Add dependency for rx on tx */ 446 err = opdl_add_deps(device, start_qid, i); 447 if (err < 0) { 448 PMD_DRV_LOG(ERR, "DEV_ID:[%02d] : " 449 "dependency addition for queue:[%u] - FAILED", 450 dev->data->dev_id, 451 queue->external_qid); 452 break; 453 } 454 } 455 } 456 457 if (!err) 458 fprintf(stdout, "Success - dependencies built\n"); 459 460 return err; 461 } 462 int 463 check_queues_linked(struct rte_eventdev *dev) 464 { 465 466 int err = 0; 467 unsigned int i; 468 struct opdl_evdev *device = opdl_pmd_priv(dev); 469 uint32_t nb_iq = 0; 470 471 for (i = 0; i < RTE_EVENT_MAX_QUEUES_PER_DEV; i++) { 472 struct opdl_queue *queue = &device->queue[i]; 473 474 if (!queue->initialized) 475 break; 476 477 if (queue->external_qid == OPDL_INVALID_QID) 478 nb_iq++; 479 480 if (queue->nb_ports == 0) { 481 PMD_DRV_LOG(ERR, "DEV_ID:[%02d] : " 482 "queue:[%u] has no associated ports", 483 dev->data->dev_id, 484 i); 485 err = -EINVAL; 486 break; 487 } 488 } 489 if (!err) { 490 if ((i - nb_iq) != device->max_queue_nb) { 491 PMD_DRV_LOG(ERR, "DEV_ID:[%02d] : " 492 "%u queues counted but should be %u", 493 dev->data->dev_id, 494 i - nb_iq, 495 device->max_queue_nb); 496 err = -1; 497 } 498 499 } 500 return err; 501 } 502 503 void 504 destroy_queues_and_rings(struct rte_eventdev *dev) 505 { 506 struct opdl_evdev *device = opdl_pmd_priv(dev); 507 uint32_t i; 508 509 for (i = 0; i < device->nb_opdls; i++) { 510 if (device->opdl[i]) 511 opdl_ring_free(device->opdl[i]); 512 } 513 514 memset(&device->queue, 515 0, 516 sizeof(struct opdl_queue) 517 * RTE_EVENT_MAX_QUEUES_PER_DEV); 518 } 519 520 #define OPDL_ID(d)(d->nb_opdls - 1) 521 522 static __rte_always_inline void 523 initialise_queue(struct opdl_evdev *device, 524 enum queue_pos pos, 525 int32_t i) 526 { 527 struct opdl_queue *queue = &device->queue[device->nb_queues]; 528 529 if (i == -1) { 530 queue->q_type = OPDL_Q_TYPE_ORDERED; 531 queue->external_qid = OPDL_INVALID_QID; 532 } else { 533 queue->q_type = device->q_md[i].type; 534 queue->external_qid = device->q_md[i].ext_id; 535 /* Add ex->in for queues setup */ 536 device->q_map_ex_to_in[queue->external_qid] = device->nb_queues; 537 } 538 queue->opdl_id = OPDL_ID(device); 539 queue->q_pos = pos; 540 queue->nb_ports = 0; 541 queue->configured = 1; 542 543 device->nb_queues++; 544 } 545 546 547 static __rte_always_inline int 548 create_opdl(struct opdl_evdev *device) 549 { 550 int err = 0; 551 552 char name[RTE_MEMZONE_NAMESIZE]; 553 554 snprintf(name, RTE_MEMZONE_NAMESIZE, 555 "%s_%u", device->service_name, device->nb_opdls); 556 557 device->opdl[device->nb_opdls] = 558 opdl_ring_create(name, 559 device->nb_events_limit, 560 sizeof(struct rte_event), 561 device->max_port_nb * 2, 562 device->socket); 563 564 if (!device->opdl[device->nb_opdls]) { 565 PMD_DRV_LOG(ERR, "DEV_ID:[%02d] : " 566 "opdl ring %u creation - FAILED", 567 opdl_pmd_dev_id(device), 568 device->nb_opdls); 569 err = -EINVAL; 570 } else { 571 device->nb_opdls++; 572 } 573 return err; 574 } 575 576 static __rte_always_inline int 577 create_link_opdl(struct opdl_evdev *device, uint32_t index) 578 { 579 580 int err = 0; 581 582 if (device->q_md[index + 1].type != 583 OPDL_Q_TYPE_SINGLE_LINK) { 584 585 /* async queue with regular 586 * queue following it 587 */ 588 589 /* create a new opdl ring */ 590 err = create_opdl(device); 591 if (!err) { 592 /* create an initial 593 * dummy queue for new opdl 594 */ 595 initialise_queue(device, 596 OPDL_Q_POS_START, 597 -1); 598 } else { 599 err = -EINVAL; 600 } 601 } else { 602 PMD_DRV_LOG(ERR, "DEV_ID:[%02d] : " 603 "queue %u, two consecutive" 604 " SINGLE_LINK queues, not allowed", 605 opdl_pmd_dev_id(device), 606 index); 607 err = -EINVAL; 608 } 609 610 return err; 611 } 612 613 int 614 create_queues_and_rings(struct rte_eventdev *dev) 615 { 616 int err = 0; 617 618 struct opdl_evdev *device = opdl_pmd_priv(dev); 619 620 device->nb_queues = 0; 621 622 if (device->nb_ports != device->max_port_nb) { 623 PMD_DRV_LOG(ERR, "Number ports setup:%u NOT EQUAL to max port" 624 " number:%u for this device", 625 device->nb_ports, 626 device->max_port_nb); 627 err = -1; 628 } 629 630 if (!err) { 631 /* We will have at least one opdl so create it now */ 632 err = create_opdl(device); 633 } 634 635 if (!err) { 636 637 /* Create 1st "dummy" queue */ 638 initialise_queue(device, 639 OPDL_Q_POS_START, 640 -1); 641 642 uint32_t i; 643 for (i = 0; i < device->nb_q_md; i++) { 644 645 /* Check */ 646 if (!device->q_md[i].setup) { 647 648 PMD_DRV_LOG(ERR, "DEV_ID:[%02d] : " 649 "queue meta data slot %u" 650 " not setup - FAILING", 651 dev->data->dev_id, 652 i); 653 err = -EINVAL; 654 break; 655 } else if (device->q_md[i].type != 656 OPDL_Q_TYPE_SINGLE_LINK) { 657 658 if (!device->q_md[i + 1].setup) { 659 /* Create a simple ORDERED/ATOMIC 660 * queue at the end 661 */ 662 initialise_queue(device, 663 OPDL_Q_POS_END, 664 i); 665 666 } else { 667 /* Create a simple ORDERED/ATOMIC 668 * queue in the middle 669 */ 670 initialise_queue(device, 671 OPDL_Q_POS_MIDDLE, 672 i); 673 } 674 } else if (device->q_md[i].type == 675 OPDL_Q_TYPE_SINGLE_LINK) { 676 677 /* create last queue for this opdl */ 678 initialise_queue(device, 679 OPDL_Q_POS_END, 680 i); 681 682 err = create_link_opdl(device, i); 683 684 if (err) 685 break; 686 687 688 } 689 } 690 } 691 if (err) 692 destroy_queues_and_rings(dev); 693 694 return err; 695 } 696 697 698 int 699 initialise_all_other_ports(struct rte_eventdev *dev) 700 { 701 int err = 0; 702 struct opdl_stage *stage_inst = NULL; 703 704 struct opdl_evdev *device = opdl_pmd_priv(dev); 705 706 uint32_t i; 707 for (i = 0; i < device->nb_ports; i++) { 708 struct opdl_port *port = &device->ports[i]; 709 struct opdl_queue *queue = &device->queue[port->queue_id]; 710 711 if (port->queue_id == 0) { 712 continue; 713 } else if (queue->q_type != OPDL_Q_TYPE_SINGLE_LINK) { 714 715 if (queue->q_pos == OPDL_Q_POS_MIDDLE) { 716 717 /* Regular port with claim/disclaim */ 718 stage_inst = opdl_stage_add( 719 device->opdl[queue->opdl_id], 720 false, 721 false); 722 port->deq_stage_inst = stage_inst; 723 port->enq_stage_inst = stage_inst; 724 725 if (queue->q_type == OPDL_Q_TYPE_ATOMIC) 726 port->atomic_claim = true; 727 else 728 port->atomic_claim = false; 729 730 port->p_type = OPDL_REGULAR_PORT; 731 732 /* Add the port to the queue array of ports */ 733 queue->ports[queue->nb_ports] = port; 734 port->instance_id = queue->nb_ports; 735 queue->nb_ports++; 736 opdl_stage_set_queue_id(stage_inst, 737 port->queue_id); 738 739 } else if (queue->q_pos == OPDL_Q_POS_END) { 740 741 /* tx port */ 742 stage_inst = opdl_stage_add( 743 device->opdl[queue->opdl_id], 744 false, 745 false); 746 port->deq_stage_inst = stage_inst; 747 port->enq_stage_inst = NULL; 748 port->p_type = OPDL_PURE_TX_PORT; 749 750 /* Add the port to the queue array of ports */ 751 queue->ports[queue->nb_ports] = port; 752 port->instance_id = queue->nb_ports; 753 queue->nb_ports++; 754 } else { 755 756 PMD_DRV_LOG(ERR, "DEV_ID:[%02d] : " 757 "port %u:, linked incorrectly" 758 " to a q_pos START/INVALID %u", 759 opdl_pmd_dev_id(port->opdl), 760 port->id, 761 queue->q_pos); 762 err = -EINVAL; 763 break; 764 } 765 766 } else if (queue->q_type == OPDL_Q_TYPE_SINGLE_LINK) { 767 768 port->p_type = OPDL_ASYNC_PORT; 769 770 /* -- tx -- */ 771 stage_inst = opdl_stage_add( 772 device->opdl[queue->opdl_id], 773 false, 774 false); /* First stage */ 775 port->deq_stage_inst = stage_inst; 776 777 /* Add the port to the queue array of ports */ 778 queue->ports[queue->nb_ports] = port; 779 port->instance_id = queue->nb_ports; 780 queue->nb_ports++; 781 782 if (queue->nb_ports > 1) { 783 PMD_DRV_LOG(ERR, "DEV_ID:[%02d] : " 784 "queue %u:, setup as SINGLE_LINK" 785 " but has more than one port linked", 786 opdl_pmd_dev_id(port->opdl), 787 queue->external_qid); 788 err = -EINVAL; 789 break; 790 } 791 792 /* -- single instance rx for next opdl -- */ 793 uint8_t next_qid = 794 device->q_map_ex_to_in[queue->external_qid] + 1; 795 if (next_qid < RTE_EVENT_MAX_QUEUES_PER_DEV && 796 device->queue[next_qid].configured) { 797 798 /* Remap the queue */ 799 queue = &device->queue[next_qid]; 800 801 stage_inst = opdl_stage_add( 802 device->opdl[queue->opdl_id], 803 false, 804 true); 805 port->enq_stage_inst = stage_inst; 806 807 /* Add the port to the queue array of ports */ 808 queue->ports[queue->nb_ports] = port; 809 port->instance_id = queue->nb_ports; 810 queue->nb_ports++; 811 if (queue->nb_ports > 1) { 812 PMD_DRV_LOG(ERR, "DEV_ID:[%02d] : " 813 "dummy queue %u: for " 814 "port %u, " 815 "SINGLE_LINK but has more " 816 "than one port linked", 817 opdl_pmd_dev_id(port->opdl), 818 next_qid, 819 port->id); 820 err = -EINVAL; 821 break; 822 } 823 /* Set this queue to initialized as it is never 824 * referenced by any ports 825 */ 826 queue->initialized = 1; 827 } 828 } 829 } 830 831 /* Now that all ports are initialised we need to 832 * setup the last bit of stage md 833 */ 834 if (!err) { 835 for (i = 0; i < device->nb_ports; i++) { 836 struct opdl_port *port = &device->ports[i]; 837 struct opdl_queue *queue = 838 &device->queue[port->queue_id]; 839 840 if (port->configured && 841 (port->queue_id != OPDL_INVALID_QID)) { 842 if (queue->nb_ports == 0) { 843 PMD_DRV_LOG(ERR, "DEV_ID:[%02d] : " 844 "queue:[%u] has no ports" 845 " linked to it", 846 opdl_pmd_dev_id(port->opdl), 847 port->id); 848 err = -EINVAL; 849 break; 850 } 851 852 port->num_instance = queue->nb_ports; 853 port->initialized = 1; 854 queue->initialized = 1; 855 } else { 856 PMD_DRV_LOG(ERR, "DEV_ID:[%02d] : " 857 "Port:[%u] not configured invalid" 858 " queue configuration", 859 opdl_pmd_dev_id(port->opdl), 860 port->id); 861 err = -EINVAL; 862 break; 863 } 864 } 865 } 866 return err; 867 } 868 869 int 870 initialise_queue_zero_ports(struct rte_eventdev *dev) 871 { 872 int err = 0; 873 uint8_t mt_rx = 0; 874 struct opdl_stage *stage_inst = NULL; 875 struct opdl_queue *queue = NULL; 876 877 struct opdl_evdev *device = opdl_pmd_priv(dev); 878 879 /* Assign queue zero and figure out how many Q0 ports we have */ 880 uint32_t i; 881 for (i = 0; i < device->nb_ports; i++) { 882 struct opdl_port *port = &device->ports[i]; 883 if (port->queue_id == OPDL_INVALID_QID) { 884 port->queue_id = 0; 885 port->external_qid = OPDL_INVALID_QID; 886 port->p_type = OPDL_PURE_RX_PORT; 887 mt_rx++; 888 } 889 } 890 891 /* Create the stage */ 892 stage_inst = opdl_stage_add(device->opdl[0], 893 (mt_rx > 1 ? true : false), 894 true); 895 if (stage_inst) { 896 897 /* Assign the new created input stage to all relevant ports */ 898 for (i = 0; i < device->nb_ports; i++) { 899 struct opdl_port *port = &device->ports[i]; 900 if (port->queue_id == 0) { 901 queue = &device->queue[port->queue_id]; 902 port->enq_stage_inst = stage_inst; 903 port->deq_stage_inst = NULL; 904 port->configured = 1; 905 port->initialized = 1; 906 907 queue->ports[queue->nb_ports] = port; 908 port->instance_id = queue->nb_ports; 909 queue->nb_ports++; 910 } 911 } 912 } else { 913 err = -1; 914 } 915 return err; 916 } 917 918 int 919 assign_internal_queue_ids(struct rte_eventdev *dev) 920 { 921 int err = 0; 922 struct opdl_evdev *device = opdl_pmd_priv(dev); 923 uint32_t i; 924 925 for (i = 0; i < device->nb_ports; i++) { 926 struct opdl_port *port = &device->ports[i]; 927 if (port->external_qid != OPDL_INVALID_QID) { 928 port->queue_id = 929 device->q_map_ex_to_in[port->external_qid]; 930 931 /* Now do the external_qid of the next queue */ 932 struct opdl_queue *queue = 933 &device->queue[port->queue_id]; 934 if (queue->q_pos == OPDL_Q_POS_END) 935 port->next_external_qid = 936 device->queue[port->queue_id + 2].external_qid; 937 else 938 port->next_external_qid = 939 device->queue[port->queue_id + 1].external_qid; 940 } 941 } 942 return err; 943 } 944