1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017 Intel Corporation 3 */ 4 5 #include <stdio.h> 6 #include <inttypes.h> 7 8 #include <rte_eal.h> 9 #include <rte_common.h> 10 #include <rte_dev.h> 11 #include <rte_launch.h> 12 #include <rte_bbdev.h> 13 #include <rte_cycles.h> 14 #include <rte_lcore.h> 15 #include <rte_malloc.h> 16 #include <rte_random.h> 17 #include <rte_hexdump.h> 18 19 #include "main.h" 20 #include "test_bbdev_vector.h" 21 22 #define GET_SOCKET(socket_id) (((socket_id) == SOCKET_ID_ANY) ? 0 : (socket_id)) 23 24 #define MAX_QUEUES RTE_MAX_LCORE 25 26 #define OPS_CACHE_SIZE 256U 27 #define OPS_POOL_SIZE_MIN 511U /* 0.5K per queue */ 28 29 #define SYNC_WAIT 0 30 #define SYNC_START 1 31 32 #define INVALID_QUEUE_ID -1 33 34 static struct test_bbdev_vector test_vector; 35 36 /* Switch between PMD and Interrupt for throughput TC */ 37 static bool intr_enabled; 38 39 /* Represents tested active devices */ 40 static struct active_device { 41 const char *driver_name; 42 uint8_t dev_id; 43 uint16_t supported_ops; 44 uint16_t queue_ids[MAX_QUEUES]; 45 uint16_t nb_queues; 46 struct rte_mempool *ops_mempool; 47 struct rte_mempool *in_mbuf_pool; 48 struct rte_mempool *hard_out_mbuf_pool; 49 struct rte_mempool *soft_out_mbuf_pool; 50 } active_devs[RTE_BBDEV_MAX_DEVS]; 51 52 static uint8_t nb_active_devs; 53 54 /* Data buffers used by BBDEV ops */ 55 struct test_buffers { 56 struct rte_bbdev_op_data *inputs; 57 struct rte_bbdev_op_data *hard_outputs; 58 struct rte_bbdev_op_data *soft_outputs; 59 }; 60 61 /* Operation parameters specific for given test case */ 62 struct test_op_params { 63 struct rte_mempool *mp; 64 struct rte_bbdev_dec_op *ref_dec_op; 65 struct rte_bbdev_enc_op *ref_enc_op; 66 uint16_t burst_sz; 67 uint16_t num_to_process; 68 uint16_t num_lcores; 69 int vector_mask; 70 rte_atomic16_t sync; 71 struct test_buffers q_bufs[RTE_MAX_NUMA_NODES][MAX_QUEUES]; 72 }; 73 74 /* Contains per lcore params */ 75 struct thread_params { 76 uint8_t dev_id; 77 uint16_t queue_id; 78 uint64_t start_time; 79 double mops; 80 double mbps; 81 rte_atomic16_t nb_dequeued; 82 rte_atomic16_t processing_status; 83 struct test_op_params *op_params; 84 }; 85 86 typedef int (test_case_function)(struct active_device *ad, 87 struct test_op_params *op_params); 88 89 static inline void 90 set_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type) 91 { 92 ad->supported_ops |= (1 << op_type); 93 } 94 95 static inline bool 96 is_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type) 97 { 98 return ad->supported_ops & (1 << op_type); 99 } 100 101 static inline bool 102 flags_match(uint32_t flags_req, uint32_t flags_present) 103 { 104 return (flags_req & flags_present) == flags_req; 105 } 106 107 static void 108 clear_soft_out_cap(uint32_t *op_flags) 109 { 110 *op_flags &= ~RTE_BBDEV_TURBO_SOFT_OUTPUT; 111 *op_flags &= ~RTE_BBDEV_TURBO_POS_LLR_1_BIT_SOFT_OUT; 112 *op_flags &= ~RTE_BBDEV_TURBO_NEG_LLR_1_BIT_SOFT_OUT; 113 } 114 115 static int 116 check_dev_cap(const struct rte_bbdev_info *dev_info) 117 { 118 unsigned int i; 119 unsigned int nb_inputs, nb_soft_outputs, nb_hard_outputs; 120 const struct rte_bbdev_op_cap *op_cap = dev_info->drv.capabilities; 121 122 nb_inputs = test_vector.entries[DATA_INPUT].nb_segments; 123 nb_soft_outputs = test_vector.entries[DATA_SOFT_OUTPUT].nb_segments; 124 nb_hard_outputs = test_vector.entries[DATA_HARD_OUTPUT].nb_segments; 125 126 for (i = 0; op_cap->type != RTE_BBDEV_OP_NONE; ++i, ++op_cap) { 127 if (op_cap->type != test_vector.op_type) 128 continue; 129 130 if (op_cap->type == RTE_BBDEV_OP_TURBO_DEC) { 131 const struct rte_bbdev_op_cap_turbo_dec *cap = 132 &op_cap->cap.turbo_dec; 133 /* Ignore lack of soft output capability, just skip 134 * checking if soft output is valid. 135 */ 136 if ((test_vector.turbo_dec.op_flags & 137 RTE_BBDEV_TURBO_SOFT_OUTPUT) && 138 !(cap->capability_flags & 139 RTE_BBDEV_TURBO_SOFT_OUTPUT)) { 140 printf( 141 "WARNING: Device \"%s\" does not support soft output - soft output flags will be ignored.\n", 142 dev_info->dev_name); 143 clear_soft_out_cap( 144 &test_vector.turbo_dec.op_flags); 145 } 146 147 if (!flags_match(test_vector.turbo_dec.op_flags, 148 cap->capability_flags)) 149 return TEST_FAILED; 150 if (nb_inputs > cap->num_buffers_src) { 151 printf("Too many inputs defined: %u, max: %u\n", 152 nb_inputs, cap->num_buffers_src); 153 return TEST_FAILED; 154 } 155 if (nb_soft_outputs > cap->num_buffers_soft_out && 156 (test_vector.turbo_dec.op_flags & 157 RTE_BBDEV_TURBO_SOFT_OUTPUT)) { 158 printf( 159 "Too many soft outputs defined: %u, max: %u\n", 160 nb_soft_outputs, 161 cap->num_buffers_soft_out); 162 return TEST_FAILED; 163 } 164 if (nb_hard_outputs > cap->num_buffers_hard_out) { 165 printf( 166 "Too many hard outputs defined: %u, max: %u\n", 167 nb_hard_outputs, 168 cap->num_buffers_hard_out); 169 return TEST_FAILED; 170 } 171 if (intr_enabled && !(cap->capability_flags & 172 RTE_BBDEV_TURBO_DEC_INTERRUPTS)) { 173 printf( 174 "Dequeue interrupts are not supported!\n"); 175 return TEST_FAILED; 176 } 177 178 return TEST_SUCCESS; 179 } else if (op_cap->type == RTE_BBDEV_OP_TURBO_ENC) { 180 const struct rte_bbdev_op_cap_turbo_enc *cap = 181 &op_cap->cap.turbo_enc; 182 183 if (!flags_match(test_vector.turbo_enc.op_flags, 184 cap->capability_flags)) 185 return TEST_FAILED; 186 if (nb_inputs > cap->num_buffers_src) { 187 printf("Too many inputs defined: %u, max: %u\n", 188 nb_inputs, cap->num_buffers_src); 189 return TEST_FAILED; 190 } 191 if (nb_hard_outputs > cap->num_buffers_dst) { 192 printf( 193 "Too many hard outputs defined: %u, max: %u\n", 194 nb_hard_outputs, cap->num_buffers_src); 195 return TEST_FAILED; 196 } 197 if (intr_enabled && !(cap->capability_flags & 198 RTE_BBDEV_TURBO_ENC_INTERRUPTS)) { 199 printf( 200 "Dequeue interrupts are not supported!\n"); 201 return TEST_FAILED; 202 } 203 204 return TEST_SUCCESS; 205 } 206 } 207 208 if ((i == 0) && (test_vector.op_type == RTE_BBDEV_OP_NONE)) 209 return TEST_SUCCESS; /* Special case for NULL device */ 210 211 return TEST_FAILED; 212 } 213 214 /* calculates optimal mempool size not smaller than the val */ 215 static unsigned int 216 optimal_mempool_size(unsigned int val) 217 { 218 return rte_align32pow2(val + 1) - 1; 219 } 220 221 /* allocates mbuf mempool for inputs and outputs */ 222 static struct rte_mempool * 223 create_mbuf_pool(struct op_data_entries *entries, uint8_t dev_id, 224 int socket_id, unsigned int mbuf_pool_size, 225 const char *op_type_str) 226 { 227 unsigned int i; 228 uint32_t max_seg_sz = 0; 229 char pool_name[RTE_MEMPOOL_NAMESIZE]; 230 231 /* find max input segment size */ 232 for (i = 0; i < entries->nb_segments; ++i) 233 if (entries->segments[i].length > max_seg_sz) 234 max_seg_sz = entries->segments[i].length; 235 236 snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str, 237 dev_id); 238 return rte_pktmbuf_pool_create(pool_name, mbuf_pool_size, 0, 0, 239 RTE_MAX(max_seg_sz + RTE_PKTMBUF_HEADROOM, 240 (unsigned int)RTE_MBUF_DEFAULT_BUF_SIZE), socket_id); 241 } 242 243 static int 244 create_mempools(struct active_device *ad, int socket_id, 245 enum rte_bbdev_op_type op_type, uint16_t num_ops) 246 { 247 struct rte_mempool *mp; 248 unsigned int ops_pool_size, mbuf_pool_size = 0; 249 char pool_name[RTE_MEMPOOL_NAMESIZE]; 250 const char *op_type_str; 251 252 struct op_data_entries *in = &test_vector.entries[DATA_INPUT]; 253 struct op_data_entries *hard_out = 254 &test_vector.entries[DATA_HARD_OUTPUT]; 255 struct op_data_entries *soft_out = 256 &test_vector.entries[DATA_SOFT_OUTPUT]; 257 258 /* allocate ops mempool */ 259 ops_pool_size = optimal_mempool_size(RTE_MAX( 260 /* Ops used plus 1 reference op */ 261 RTE_MAX((unsigned int)(ad->nb_queues * num_ops + 1), 262 /* Minimal cache size plus 1 reference op */ 263 (unsigned int)(1.5 * rte_lcore_count() * 264 OPS_CACHE_SIZE + 1)), 265 OPS_POOL_SIZE_MIN)); 266 267 op_type_str = rte_bbdev_op_type_str(op_type); 268 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type); 269 270 snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str, 271 ad->dev_id); 272 mp = rte_bbdev_op_pool_create(pool_name, op_type, 273 ops_pool_size, OPS_CACHE_SIZE, socket_id); 274 TEST_ASSERT_NOT_NULL(mp, 275 "ERROR Failed to create %u items ops pool for dev %u on socket %u.", 276 ops_pool_size, 277 ad->dev_id, 278 socket_id); 279 ad->ops_mempool = mp; 280 281 /* Inputs */ 282 mbuf_pool_size = optimal_mempool_size(ops_pool_size * in->nb_segments); 283 mp = create_mbuf_pool(in, ad->dev_id, socket_id, mbuf_pool_size, "in"); 284 TEST_ASSERT_NOT_NULL(mp, 285 "ERROR Failed to create %u items input pktmbuf pool for dev %u on socket %u.", 286 mbuf_pool_size, 287 ad->dev_id, 288 socket_id); 289 ad->in_mbuf_pool = mp; 290 291 /* Hard outputs */ 292 mbuf_pool_size = optimal_mempool_size(ops_pool_size * 293 hard_out->nb_segments); 294 mp = create_mbuf_pool(hard_out, ad->dev_id, socket_id, mbuf_pool_size, 295 "hard_out"); 296 TEST_ASSERT_NOT_NULL(mp, 297 "ERROR Failed to create %u items hard output pktmbuf pool for dev %u on socket %u.", 298 mbuf_pool_size, 299 ad->dev_id, 300 socket_id); 301 ad->hard_out_mbuf_pool = mp; 302 303 if (soft_out->nb_segments == 0) 304 return TEST_SUCCESS; 305 306 /* Soft outputs */ 307 mbuf_pool_size = optimal_mempool_size(ops_pool_size * 308 soft_out->nb_segments); 309 mp = create_mbuf_pool(soft_out, ad->dev_id, socket_id, mbuf_pool_size, 310 "soft_out"); 311 TEST_ASSERT_NOT_NULL(mp, 312 "ERROR Failed to create %uB soft output pktmbuf pool for dev %u on socket %u.", 313 mbuf_pool_size, 314 ad->dev_id, 315 socket_id); 316 ad->soft_out_mbuf_pool = mp; 317 318 return 0; 319 } 320 321 static int 322 add_bbdev_dev(uint8_t dev_id, struct rte_bbdev_info *info, 323 struct test_bbdev_vector *vector) 324 { 325 int ret; 326 unsigned int queue_id; 327 struct rte_bbdev_queue_conf qconf; 328 struct active_device *ad = &active_devs[nb_active_devs]; 329 unsigned int nb_queues; 330 enum rte_bbdev_op_type op_type = vector->op_type; 331 332 nb_queues = RTE_MIN(rte_lcore_count(), info->drv.max_num_queues); 333 /* setup device */ 334 ret = rte_bbdev_setup_queues(dev_id, nb_queues, info->socket_id); 335 if (ret < 0) { 336 printf("rte_bbdev_setup_queues(%u, %u, %d) ret %i\n", 337 dev_id, nb_queues, info->socket_id, ret); 338 return TEST_FAILED; 339 } 340 341 /* configure interrupts if needed */ 342 if (intr_enabled) { 343 ret = rte_bbdev_intr_enable(dev_id); 344 if (ret < 0) { 345 printf("rte_bbdev_intr_enable(%u) ret %i\n", dev_id, 346 ret); 347 return TEST_FAILED; 348 } 349 } 350 351 /* setup device queues */ 352 qconf.socket = info->socket_id; 353 qconf.queue_size = info->drv.default_queue_conf.queue_size; 354 qconf.priority = 0; 355 qconf.deferred_start = 0; 356 qconf.op_type = op_type; 357 358 for (queue_id = 0; queue_id < nb_queues; ++queue_id) { 359 ret = rte_bbdev_queue_configure(dev_id, queue_id, &qconf); 360 if (ret != 0) { 361 printf( 362 "Allocated all queues (id=%u) at prio%u on dev%u\n", 363 queue_id, qconf.priority, dev_id); 364 qconf.priority++; 365 ret = rte_bbdev_queue_configure(ad->dev_id, queue_id, 366 &qconf); 367 } 368 if (ret != 0) { 369 printf("All queues on dev %u allocated: %u\n", 370 dev_id, queue_id); 371 break; 372 } 373 ad->queue_ids[queue_id] = queue_id; 374 } 375 TEST_ASSERT(queue_id != 0, 376 "ERROR Failed to configure any queues on dev %u", 377 dev_id); 378 ad->nb_queues = queue_id; 379 380 set_avail_op(ad, op_type); 381 382 return TEST_SUCCESS; 383 } 384 385 static int 386 add_active_device(uint8_t dev_id, struct rte_bbdev_info *info, 387 struct test_bbdev_vector *vector) 388 { 389 int ret; 390 391 active_devs[nb_active_devs].driver_name = info->drv.driver_name; 392 active_devs[nb_active_devs].dev_id = dev_id; 393 394 ret = add_bbdev_dev(dev_id, info, vector); 395 if (ret == TEST_SUCCESS) 396 ++nb_active_devs; 397 return ret; 398 } 399 400 static uint8_t 401 populate_active_devices(void) 402 { 403 int ret; 404 uint8_t dev_id; 405 uint8_t nb_devs_added = 0; 406 struct rte_bbdev_info info; 407 408 RTE_BBDEV_FOREACH(dev_id) { 409 rte_bbdev_info_get(dev_id, &info); 410 411 if (check_dev_cap(&info)) { 412 printf( 413 "Device %d (%s) does not support specified capabilities\n", 414 dev_id, info.dev_name); 415 continue; 416 } 417 418 ret = add_active_device(dev_id, &info, &test_vector); 419 if (ret != 0) { 420 printf("Adding active bbdev %s skipped\n", 421 info.dev_name); 422 continue; 423 } 424 nb_devs_added++; 425 } 426 427 return nb_devs_added; 428 } 429 430 static int 431 read_test_vector(void) 432 { 433 int ret; 434 435 memset(&test_vector, 0, sizeof(test_vector)); 436 printf("Test vector file = %s\n", get_vector_filename()); 437 ret = test_bbdev_vector_read(get_vector_filename(), &test_vector); 438 TEST_ASSERT_SUCCESS(ret, "Failed to parse file %s\n", 439 get_vector_filename()); 440 441 return TEST_SUCCESS; 442 } 443 444 static int 445 testsuite_setup(void) 446 { 447 TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n"); 448 449 if (populate_active_devices() == 0) { 450 printf("No suitable devices found!\n"); 451 return TEST_SKIPPED; 452 } 453 454 return TEST_SUCCESS; 455 } 456 457 static int 458 interrupt_testsuite_setup(void) 459 { 460 TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n"); 461 462 /* Enable interrupts */ 463 intr_enabled = true; 464 465 /* Special case for NULL device (RTE_BBDEV_OP_NONE) */ 466 if (populate_active_devices() == 0 || 467 test_vector.op_type == RTE_BBDEV_OP_NONE) { 468 intr_enabled = false; 469 printf("No suitable devices found!\n"); 470 return TEST_SKIPPED; 471 } 472 473 return TEST_SUCCESS; 474 } 475 476 static void 477 testsuite_teardown(void) 478 { 479 uint8_t dev_id; 480 481 /* Unconfigure devices */ 482 RTE_BBDEV_FOREACH(dev_id) 483 rte_bbdev_close(dev_id); 484 485 /* Clear active devices structs. */ 486 memset(active_devs, 0, sizeof(active_devs)); 487 nb_active_devs = 0; 488 } 489 490 static int 491 ut_setup(void) 492 { 493 uint8_t i, dev_id; 494 495 for (i = 0; i < nb_active_devs; i++) { 496 dev_id = active_devs[i].dev_id; 497 /* reset bbdev stats */ 498 TEST_ASSERT_SUCCESS(rte_bbdev_stats_reset(dev_id), 499 "Failed to reset stats of bbdev %u", dev_id); 500 /* start the device */ 501 TEST_ASSERT_SUCCESS(rte_bbdev_start(dev_id), 502 "Failed to start bbdev %u", dev_id); 503 } 504 505 return TEST_SUCCESS; 506 } 507 508 static void 509 ut_teardown(void) 510 { 511 uint8_t i, dev_id; 512 struct rte_bbdev_stats stats; 513 514 for (i = 0; i < nb_active_devs; i++) { 515 dev_id = active_devs[i].dev_id; 516 /* read stats and print */ 517 rte_bbdev_stats_get(dev_id, &stats); 518 /* Stop the device */ 519 rte_bbdev_stop(dev_id); 520 } 521 } 522 523 static int 524 init_op_data_objs(struct rte_bbdev_op_data *bufs, 525 struct op_data_entries *ref_entries, 526 struct rte_mempool *mbuf_pool, const uint16_t n, 527 enum op_data_type op_type, uint16_t min_alignment) 528 { 529 int ret; 530 unsigned int i, j; 531 532 for (i = 0; i < n; ++i) { 533 char *data; 534 struct op_data_buf *seg = &ref_entries->segments[0]; 535 struct rte_mbuf *m_head = rte_pktmbuf_alloc(mbuf_pool); 536 TEST_ASSERT_NOT_NULL(m_head, 537 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)", 538 op_type, n * ref_entries->nb_segments, 539 mbuf_pool->size); 540 541 bufs[i].data = m_head; 542 bufs[i].offset = 0; 543 bufs[i].length = 0; 544 545 if (op_type == DATA_INPUT) { 546 data = rte_pktmbuf_append(m_head, seg->length); 547 TEST_ASSERT_NOT_NULL(data, 548 "Couldn't append %u bytes to mbuf from %d data type mbuf pool", 549 seg->length, op_type); 550 551 TEST_ASSERT(data == RTE_PTR_ALIGN(data, min_alignment), 552 "Data addr in mbuf (%p) is not aligned to device min alignment (%u)", 553 data, min_alignment); 554 rte_memcpy(data, seg->addr, seg->length); 555 bufs[i].length += seg->length; 556 557 558 for (j = 1; j < ref_entries->nb_segments; ++j) { 559 struct rte_mbuf *m_tail = 560 rte_pktmbuf_alloc(mbuf_pool); 561 TEST_ASSERT_NOT_NULL(m_tail, 562 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)", 563 op_type, 564 n * ref_entries->nb_segments, 565 mbuf_pool->size); 566 seg += 1; 567 568 data = rte_pktmbuf_append(m_tail, seg->length); 569 TEST_ASSERT_NOT_NULL(data, 570 "Couldn't append %u bytes to mbuf from %d data type mbuf pool", 571 seg->length, op_type); 572 573 TEST_ASSERT(data == RTE_PTR_ALIGN(data, 574 min_alignment), 575 "Data addr in mbuf (%p) is not aligned to device min alignment (%u)", 576 data, min_alignment); 577 rte_memcpy(data, seg->addr, seg->length); 578 bufs[i].length += seg->length; 579 580 ret = rte_pktmbuf_chain(m_head, m_tail); 581 TEST_ASSERT_SUCCESS(ret, 582 "Couldn't chain mbufs from %d data type mbuf pool", 583 op_type); 584 } 585 } 586 } 587 588 return 0; 589 } 590 591 static int 592 allocate_buffers_on_socket(struct rte_bbdev_op_data **buffers, const int len, 593 const int socket) 594 { 595 int i; 596 597 *buffers = rte_zmalloc_socket(NULL, len, 0, socket); 598 if (*buffers == NULL) { 599 printf("WARNING: Failed to allocate op_data on socket %d\n", 600 socket); 601 /* try to allocate memory on other detected sockets */ 602 for (i = 0; i < socket; i++) { 603 *buffers = rte_zmalloc_socket(NULL, len, 0, i); 604 if (*buffers != NULL) 605 break; 606 } 607 } 608 609 return (*buffers == NULL) ? TEST_FAILED : TEST_SUCCESS; 610 } 611 612 static int 613 fill_queue_buffers(struct test_op_params *op_params, 614 struct rte_mempool *in_mp, struct rte_mempool *hard_out_mp, 615 struct rte_mempool *soft_out_mp, uint16_t queue_id, 616 uint16_t min_alignment, const int socket_id) 617 { 618 int ret; 619 enum op_data_type type; 620 const uint16_t n = op_params->num_to_process; 621 622 struct rte_mempool *mbuf_pools[DATA_NUM_TYPES] = { 623 in_mp, 624 soft_out_mp, 625 hard_out_mp, 626 }; 627 628 struct rte_bbdev_op_data **queue_ops[DATA_NUM_TYPES] = { 629 &op_params->q_bufs[socket_id][queue_id].inputs, 630 &op_params->q_bufs[socket_id][queue_id].soft_outputs, 631 &op_params->q_bufs[socket_id][queue_id].hard_outputs, 632 }; 633 634 for (type = DATA_INPUT; type < DATA_NUM_TYPES; ++type) { 635 struct op_data_entries *ref_entries = 636 &test_vector.entries[type]; 637 if (ref_entries->nb_segments == 0) 638 continue; 639 640 ret = allocate_buffers_on_socket(queue_ops[type], 641 n * sizeof(struct rte_bbdev_op_data), 642 socket_id); 643 TEST_ASSERT_SUCCESS(ret, 644 "Couldn't allocate memory for rte_bbdev_op_data structs"); 645 646 ret = init_op_data_objs(*queue_ops[type], ref_entries, 647 mbuf_pools[type], n, type, min_alignment); 648 TEST_ASSERT_SUCCESS(ret, 649 "Couldn't init rte_bbdev_op_data structs"); 650 } 651 652 return 0; 653 } 654 655 static void 656 free_buffers(struct active_device *ad, struct test_op_params *op_params) 657 { 658 unsigned int i, j; 659 660 rte_mempool_free(ad->ops_mempool); 661 rte_mempool_free(ad->in_mbuf_pool); 662 rte_mempool_free(ad->hard_out_mbuf_pool); 663 rte_mempool_free(ad->soft_out_mbuf_pool); 664 665 for (i = 0; i < rte_lcore_count(); ++i) { 666 for (j = 0; j < RTE_MAX_NUMA_NODES; ++j) { 667 rte_free(op_params->q_bufs[j][i].inputs); 668 rte_free(op_params->q_bufs[j][i].hard_outputs); 669 rte_free(op_params->q_bufs[j][i].soft_outputs); 670 } 671 } 672 } 673 674 static void 675 copy_reference_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n, 676 unsigned int start_idx, 677 struct rte_bbdev_op_data *inputs, 678 struct rte_bbdev_op_data *hard_outputs, 679 struct rte_bbdev_op_data *soft_outputs, 680 struct rte_bbdev_dec_op *ref_op) 681 { 682 unsigned int i; 683 struct rte_bbdev_op_turbo_dec *turbo_dec = &ref_op->turbo_dec; 684 685 for (i = 0; i < n; ++i) { 686 if (turbo_dec->code_block_mode == 0) { 687 ops[i]->turbo_dec.tb_params.ea = 688 turbo_dec->tb_params.ea; 689 ops[i]->turbo_dec.tb_params.eb = 690 turbo_dec->tb_params.eb; 691 ops[i]->turbo_dec.tb_params.k_pos = 692 turbo_dec->tb_params.k_pos; 693 ops[i]->turbo_dec.tb_params.k_neg = 694 turbo_dec->tb_params.k_neg; 695 ops[i]->turbo_dec.tb_params.c = 696 turbo_dec->tb_params.c; 697 ops[i]->turbo_dec.tb_params.c_neg = 698 turbo_dec->tb_params.c_neg; 699 ops[i]->turbo_dec.tb_params.cab = 700 turbo_dec->tb_params.cab; 701 } else { 702 ops[i]->turbo_dec.cb_params.e = turbo_dec->cb_params.e; 703 ops[i]->turbo_dec.cb_params.k = turbo_dec->cb_params.k; 704 } 705 706 ops[i]->turbo_dec.ext_scale = turbo_dec->ext_scale; 707 ops[i]->turbo_dec.iter_max = turbo_dec->iter_max; 708 ops[i]->turbo_dec.iter_min = turbo_dec->iter_min; 709 ops[i]->turbo_dec.op_flags = turbo_dec->op_flags; 710 ops[i]->turbo_dec.rv_index = turbo_dec->rv_index; 711 ops[i]->turbo_dec.num_maps = turbo_dec->num_maps; 712 ops[i]->turbo_dec.code_block_mode = turbo_dec->code_block_mode; 713 714 ops[i]->turbo_dec.hard_output = hard_outputs[start_idx + i]; 715 ops[i]->turbo_dec.input = inputs[start_idx + i]; 716 if (soft_outputs != NULL) 717 ops[i]->turbo_dec.soft_output = 718 soft_outputs[start_idx + i]; 719 } 720 } 721 722 static void 723 copy_reference_enc_op(struct rte_bbdev_enc_op **ops, unsigned int n, 724 unsigned int start_idx, 725 struct rte_bbdev_op_data *inputs, 726 struct rte_bbdev_op_data *outputs, 727 struct rte_bbdev_enc_op *ref_op) 728 { 729 unsigned int i; 730 struct rte_bbdev_op_turbo_enc *turbo_enc = &ref_op->turbo_enc; 731 for (i = 0; i < n; ++i) { 732 if (turbo_enc->code_block_mode == 0) { 733 ops[i]->turbo_enc.tb_params.ea = 734 turbo_enc->tb_params.ea; 735 ops[i]->turbo_enc.tb_params.eb = 736 turbo_enc->tb_params.eb; 737 ops[i]->turbo_enc.tb_params.k_pos = 738 turbo_enc->tb_params.k_pos; 739 ops[i]->turbo_enc.tb_params.k_neg = 740 turbo_enc->tb_params.k_neg; 741 ops[i]->turbo_enc.tb_params.c = 742 turbo_enc->tb_params.c; 743 ops[i]->turbo_enc.tb_params.c_neg = 744 turbo_enc->tb_params.c_neg; 745 ops[i]->turbo_enc.tb_params.cab = 746 turbo_enc->tb_params.cab; 747 ops[i]->turbo_enc.tb_params.ncb_pos = 748 turbo_enc->tb_params.ncb_pos; 749 ops[i]->turbo_enc.tb_params.ncb_neg = 750 turbo_enc->tb_params.ncb_neg; 751 ops[i]->turbo_enc.tb_params.r = turbo_enc->tb_params.r; 752 } else { 753 ops[i]->turbo_enc.cb_params.e = turbo_enc->cb_params.e; 754 ops[i]->turbo_enc.cb_params.k = turbo_enc->cb_params.k; 755 ops[i]->turbo_enc.cb_params.ncb = 756 turbo_enc->cb_params.ncb; 757 } 758 ops[i]->turbo_enc.rv_index = turbo_enc->rv_index; 759 ops[i]->turbo_enc.op_flags = turbo_enc->op_flags; 760 ops[i]->turbo_enc.code_block_mode = turbo_enc->code_block_mode; 761 762 ops[i]->turbo_enc.output = outputs[start_idx + i]; 763 ops[i]->turbo_enc.input = inputs[start_idx + i]; 764 } 765 } 766 767 static int 768 check_dec_status_and_ordering(struct rte_bbdev_dec_op *op, 769 unsigned int order_idx, const int expected_status) 770 { 771 TEST_ASSERT(op->status == expected_status, 772 "op_status (%d) != expected_status (%d)", 773 op->status, expected_status); 774 775 TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data, 776 "Ordering error, expected %p, got %p", 777 (void *)(uintptr_t)order_idx, op->opaque_data); 778 779 return TEST_SUCCESS; 780 } 781 782 static int 783 check_enc_status_and_ordering(struct rte_bbdev_enc_op *op, 784 unsigned int order_idx, const int expected_status) 785 { 786 TEST_ASSERT(op->status == expected_status, 787 "op_status (%d) != expected_status (%d)", 788 op->status, expected_status); 789 790 TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data, 791 "Ordering error, expected %p, got %p", 792 (void *)(uintptr_t)order_idx, op->opaque_data); 793 794 return TEST_SUCCESS; 795 } 796 797 static inline int 798 validate_op_chain(struct rte_bbdev_op_data *op, 799 struct op_data_entries *orig_op) 800 { 801 uint8_t i; 802 struct rte_mbuf *m = op->data; 803 uint8_t nb_dst_segments = orig_op->nb_segments; 804 805 TEST_ASSERT(nb_dst_segments == m->nb_segs, 806 "Number of segments differ in original (%u) and filled (%u) op", 807 nb_dst_segments, m->nb_segs); 808 809 for (i = 0; i < nb_dst_segments; ++i) { 810 /* Apply offset to the first mbuf segment */ 811 uint16_t offset = (i == 0) ? op->offset : 0; 812 uint16_t data_len = m->data_len - offset; 813 814 TEST_ASSERT(orig_op->segments[i].length == data_len, 815 "Length of segment differ in original (%u) and filled (%u) op", 816 orig_op->segments[i].length, data_len); 817 TEST_ASSERT_BUFFERS_ARE_EQUAL(orig_op->segments[i].addr, 818 rte_pktmbuf_mtod_offset(m, uint32_t *, offset), 819 data_len, 820 "Output buffers (CB=%u) are not equal", i); 821 m = m->next; 822 } 823 824 return TEST_SUCCESS; 825 } 826 827 static int 828 validate_dec_buffers(struct rte_bbdev_dec_op *ref_op, struct test_buffers *bufs, 829 const uint16_t num_to_process) 830 { 831 int i; 832 833 struct op_data_entries *hard_data_orig = 834 &test_vector.entries[DATA_HARD_OUTPUT]; 835 struct op_data_entries *soft_data_orig = 836 &test_vector.entries[DATA_SOFT_OUTPUT]; 837 838 for (i = 0; i < num_to_process; i++) { 839 TEST_ASSERT_SUCCESS(validate_op_chain(&bufs->hard_outputs[i], 840 hard_data_orig), 841 "Hard output buffers are not equal"); 842 if (ref_op->turbo_dec.op_flags & 843 RTE_BBDEV_TURBO_SOFT_OUTPUT) 844 TEST_ASSERT_SUCCESS(validate_op_chain( 845 &bufs->soft_outputs[i], 846 soft_data_orig), 847 "Soft output buffers are not equal"); 848 } 849 850 return TEST_SUCCESS; 851 } 852 853 static int 854 validate_enc_buffers(struct test_buffers *bufs, const uint16_t num_to_process) 855 { 856 int i; 857 858 struct op_data_entries *hard_data_orig = 859 &test_vector.entries[DATA_HARD_OUTPUT]; 860 861 for (i = 0; i < num_to_process; i++) 862 TEST_ASSERT_SUCCESS(validate_op_chain(&bufs->hard_outputs[i], 863 hard_data_orig), ""); 864 865 return TEST_SUCCESS; 866 } 867 868 static int 869 validate_dec_op(struct rte_bbdev_dec_op **ops, const uint16_t n, 870 struct rte_bbdev_dec_op *ref_op, const int vector_mask) 871 { 872 unsigned int i; 873 int ret; 874 struct op_data_entries *hard_data_orig = 875 &test_vector.entries[DATA_HARD_OUTPUT]; 876 struct op_data_entries *soft_data_orig = 877 &test_vector.entries[DATA_SOFT_OUTPUT]; 878 struct rte_bbdev_op_turbo_dec *ops_td; 879 struct rte_bbdev_op_data *hard_output; 880 struct rte_bbdev_op_data *soft_output; 881 struct rte_bbdev_op_turbo_dec *ref_td = &ref_op->turbo_dec; 882 883 for (i = 0; i < n; ++i) { 884 ops_td = &ops[i]->turbo_dec; 885 hard_output = &ops_td->hard_output; 886 soft_output = &ops_td->soft_output; 887 888 if (vector_mask & TEST_BBDEV_VF_EXPECTED_ITER_COUNT) 889 TEST_ASSERT(ops_td->iter_count <= ref_td->iter_count, 890 "Returned iter_count (%d) > expected iter_count (%d)", 891 ops_td->iter_count, ref_td->iter_count); 892 ret = check_dec_status_and_ordering(ops[i], i, ref_op->status); 893 TEST_ASSERT_SUCCESS(ret, 894 "Checking status and ordering for decoder failed"); 895 896 TEST_ASSERT_SUCCESS(validate_op_chain(hard_output, 897 hard_data_orig), 898 "Hard output buffers (CB=%u) are not equal", 899 i); 900 901 if (ref_op->turbo_dec.op_flags & RTE_BBDEV_TURBO_SOFT_OUTPUT) 902 TEST_ASSERT_SUCCESS(validate_op_chain(soft_output, 903 soft_data_orig), 904 "Soft output buffers (CB=%u) are not equal", 905 i); 906 } 907 908 return TEST_SUCCESS; 909 } 910 911 static int 912 validate_enc_op(struct rte_bbdev_enc_op **ops, const uint16_t n, 913 struct rte_bbdev_enc_op *ref_op) 914 { 915 unsigned int i; 916 int ret; 917 struct op_data_entries *hard_data_orig = 918 &test_vector.entries[DATA_HARD_OUTPUT]; 919 920 for (i = 0; i < n; ++i) { 921 ret = check_enc_status_and_ordering(ops[i], i, ref_op->status); 922 TEST_ASSERT_SUCCESS(ret, 923 "Checking status and ordering for encoder failed"); 924 TEST_ASSERT_SUCCESS(validate_op_chain( 925 &ops[i]->turbo_enc.output, 926 hard_data_orig), 927 "Output buffers (CB=%u) are not equal", 928 i); 929 } 930 931 return TEST_SUCCESS; 932 } 933 934 static void 935 create_reference_dec_op(struct rte_bbdev_dec_op *op) 936 { 937 unsigned int i; 938 struct op_data_entries *entry; 939 940 op->turbo_dec = test_vector.turbo_dec; 941 entry = &test_vector.entries[DATA_INPUT]; 942 for (i = 0; i < entry->nb_segments; ++i) 943 op->turbo_dec.input.length += 944 entry->segments[i].length; 945 } 946 947 static void 948 create_reference_enc_op(struct rte_bbdev_enc_op *op) 949 { 950 unsigned int i; 951 struct op_data_entries *entry; 952 953 op->turbo_enc = test_vector.turbo_enc; 954 entry = &test_vector.entries[DATA_INPUT]; 955 for (i = 0; i < entry->nb_segments; ++i) 956 op->turbo_enc.input.length += 957 entry->segments[i].length; 958 } 959 960 static int 961 init_test_op_params(struct test_op_params *op_params, 962 enum rte_bbdev_op_type op_type, const int expected_status, 963 const int vector_mask, struct rte_mempool *ops_mp, 964 uint16_t burst_sz, uint16_t num_to_process, uint16_t num_lcores) 965 { 966 int ret = 0; 967 if (op_type == RTE_BBDEV_OP_TURBO_DEC) 968 ret = rte_bbdev_dec_op_alloc_bulk(ops_mp, 969 &op_params->ref_dec_op, 1); 970 else 971 ret = rte_bbdev_enc_op_alloc_bulk(ops_mp, 972 &op_params->ref_enc_op, 1); 973 974 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_op_alloc_bulk() failed"); 975 976 op_params->mp = ops_mp; 977 op_params->burst_sz = burst_sz; 978 op_params->num_to_process = num_to_process; 979 op_params->num_lcores = num_lcores; 980 op_params->vector_mask = vector_mask; 981 if (op_type == RTE_BBDEV_OP_TURBO_DEC) 982 op_params->ref_dec_op->status = expected_status; 983 else if (op_type == RTE_BBDEV_OP_TURBO_ENC) 984 op_params->ref_enc_op->status = expected_status; 985 986 return 0; 987 } 988 989 static int 990 run_test_case_on_device(test_case_function *test_case_func, uint8_t dev_id, 991 struct test_op_params *op_params) 992 { 993 int t_ret, f_ret, socket_id = SOCKET_ID_ANY; 994 unsigned int i; 995 struct active_device *ad; 996 unsigned int burst_sz = get_burst_sz(); 997 enum rte_bbdev_op_type op_type = test_vector.op_type; 998 999 ad = &active_devs[dev_id]; 1000 1001 /* Check if device supports op_type */ 1002 if (!is_avail_op(ad, test_vector.op_type)) 1003 return TEST_SUCCESS; 1004 1005 struct rte_bbdev_info info; 1006 rte_bbdev_info_get(ad->dev_id, &info); 1007 socket_id = GET_SOCKET(info.socket_id); 1008 1009 if (op_type == RTE_BBDEV_OP_NONE) 1010 op_type = RTE_BBDEV_OP_TURBO_ENC; 1011 f_ret = create_mempools(ad, socket_id, op_type, 1012 get_num_ops()); 1013 if (f_ret != TEST_SUCCESS) { 1014 printf("Couldn't create mempools"); 1015 goto fail; 1016 } 1017 1018 f_ret = init_test_op_params(op_params, test_vector.op_type, 1019 test_vector.expected_status, 1020 test_vector.mask, 1021 ad->ops_mempool, 1022 burst_sz, 1023 get_num_ops(), 1024 get_num_lcores()); 1025 if (f_ret != TEST_SUCCESS) { 1026 printf("Couldn't init test op params"); 1027 goto fail; 1028 } 1029 1030 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) 1031 create_reference_dec_op(op_params->ref_dec_op); 1032 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) 1033 create_reference_enc_op(op_params->ref_enc_op); 1034 1035 for (i = 0; i < ad->nb_queues; ++i) { 1036 f_ret = fill_queue_buffers(op_params, 1037 ad->in_mbuf_pool, 1038 ad->hard_out_mbuf_pool, 1039 ad->soft_out_mbuf_pool, 1040 ad->queue_ids[i], 1041 info.drv.min_alignment, 1042 socket_id); 1043 if (f_ret != TEST_SUCCESS) { 1044 printf("Couldn't init queue buffers"); 1045 goto fail; 1046 } 1047 } 1048 1049 /* Run test case function */ 1050 t_ret = test_case_func(ad, op_params); 1051 1052 /* Free active device resources and return */ 1053 free_buffers(ad, op_params); 1054 return t_ret; 1055 1056 fail: 1057 free_buffers(ad, op_params); 1058 return TEST_FAILED; 1059 } 1060 1061 /* Run given test function per active device per supported op type 1062 * per burst size. 1063 */ 1064 static int 1065 run_test_case(test_case_function *test_case_func) 1066 { 1067 int ret = 0; 1068 uint8_t dev; 1069 1070 /* Alloc op_params */ 1071 struct test_op_params *op_params = rte_zmalloc(NULL, 1072 sizeof(struct test_op_params), RTE_CACHE_LINE_SIZE); 1073 TEST_ASSERT_NOT_NULL(op_params, "Failed to alloc %zuB for op_params", 1074 RTE_ALIGN(sizeof(struct test_op_params), 1075 RTE_CACHE_LINE_SIZE)); 1076 1077 /* For each device run test case function */ 1078 for (dev = 0; dev < nb_active_devs; ++dev) 1079 ret |= run_test_case_on_device(test_case_func, dev, op_params); 1080 1081 rte_free(op_params); 1082 1083 return ret; 1084 } 1085 1086 static void 1087 dequeue_event_callback(uint16_t dev_id, 1088 enum rte_bbdev_event_type event, void *cb_arg, 1089 void *ret_param) 1090 { 1091 int ret; 1092 uint16_t i; 1093 uint64_t total_time; 1094 uint16_t deq, burst_sz, num_to_process; 1095 uint16_t queue_id = INVALID_QUEUE_ID; 1096 struct rte_bbdev_dec_op *dec_ops[MAX_BURST]; 1097 struct rte_bbdev_enc_op *enc_ops[MAX_BURST]; 1098 struct test_buffers *bufs; 1099 struct rte_bbdev_info info; 1100 1101 /* Input length in bytes, million operations per second, 1102 * million bits per second. 1103 */ 1104 double in_len; 1105 1106 struct thread_params *tp = cb_arg; 1107 1108 RTE_SET_USED(ret_param); 1109 queue_id = tp->queue_id; 1110 1111 /* Find matching thread params using queue_id */ 1112 for (i = 0; i < MAX_QUEUES; ++i, ++tp) 1113 if (tp->queue_id == queue_id) 1114 break; 1115 1116 if (i == MAX_QUEUES) { 1117 printf("%s: Queue_id from interrupt details was not found!\n", 1118 __func__); 1119 return; 1120 } 1121 1122 if (unlikely(event != RTE_BBDEV_EVENT_DEQUEUE)) { 1123 rte_atomic16_set(&tp->processing_status, TEST_FAILED); 1124 printf( 1125 "Dequeue interrupt handler called for incorrect event!\n"); 1126 return; 1127 } 1128 1129 burst_sz = tp->op_params->burst_sz; 1130 num_to_process = tp->op_params->num_to_process; 1131 1132 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) 1133 deq = rte_bbdev_dequeue_dec_ops(dev_id, queue_id, dec_ops, 1134 burst_sz); 1135 else 1136 deq = rte_bbdev_dequeue_enc_ops(dev_id, queue_id, enc_ops, 1137 burst_sz); 1138 1139 if (deq < burst_sz) { 1140 printf( 1141 "After receiving the interrupt all operations should be dequeued. Expected: %u, got: %u\n", 1142 burst_sz, deq); 1143 rte_atomic16_set(&tp->processing_status, TEST_FAILED); 1144 return; 1145 } 1146 1147 if (rte_atomic16_read(&tp->nb_dequeued) + deq < num_to_process) { 1148 rte_atomic16_add(&tp->nb_dequeued, deq); 1149 return; 1150 } 1151 1152 total_time = rte_rdtsc_precise() - tp->start_time; 1153 1154 rte_bbdev_info_get(dev_id, &info); 1155 1156 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 1157 1158 ret = TEST_SUCCESS; 1159 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) 1160 ret = validate_dec_buffers(tp->op_params->ref_dec_op, bufs, 1161 num_to_process); 1162 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) 1163 ret = validate_enc_buffers(bufs, num_to_process); 1164 1165 if (ret) { 1166 printf("Buffers validation failed\n"); 1167 rte_atomic16_set(&tp->processing_status, TEST_FAILED); 1168 } 1169 1170 switch (test_vector.op_type) { 1171 case RTE_BBDEV_OP_TURBO_DEC: 1172 in_len = tp->op_params->ref_dec_op->turbo_dec.input.length; 1173 break; 1174 case RTE_BBDEV_OP_TURBO_ENC: 1175 in_len = tp->op_params->ref_enc_op->turbo_enc.input.length; 1176 break; 1177 case RTE_BBDEV_OP_NONE: 1178 in_len = 0.0; 1179 break; 1180 default: 1181 printf("Unknown op type: %d\n", test_vector.op_type); 1182 rte_atomic16_set(&tp->processing_status, TEST_FAILED); 1183 return; 1184 } 1185 1186 tp->mops = ((double)num_to_process / 1000000.0) / 1187 ((double)total_time / (double)rte_get_tsc_hz()); 1188 tp->mbps = ((double)num_to_process * in_len * 8 / 1000000.0) / 1189 ((double)total_time / (double)rte_get_tsc_hz()); 1190 1191 rte_atomic16_add(&tp->nb_dequeued, deq); 1192 } 1193 1194 static int 1195 throughput_intr_lcore_dec(void *arg) 1196 { 1197 struct thread_params *tp = arg; 1198 unsigned int enqueued; 1199 struct rte_bbdev_dec_op *ops[MAX_BURST]; 1200 const uint16_t queue_id = tp->queue_id; 1201 const uint16_t burst_sz = tp->op_params->burst_sz; 1202 const uint16_t num_to_process = tp->op_params->num_to_process; 1203 struct test_buffers *bufs = NULL; 1204 unsigned int allocs_failed = 0; 1205 struct rte_bbdev_info info; 1206 int ret; 1207 1208 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 1209 "BURST_SIZE should be <= %u", MAX_BURST); 1210 1211 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id), 1212 "Failed to enable interrupts for dev: %u, queue_id: %u", 1213 tp->dev_id, queue_id); 1214 1215 rte_bbdev_info_get(tp->dev_id, &info); 1216 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 1217 1218 rte_atomic16_clear(&tp->processing_status); 1219 rte_atomic16_clear(&tp->nb_dequeued); 1220 1221 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT) 1222 rte_pause(); 1223 1224 tp->start_time = rte_rdtsc_precise(); 1225 for (enqueued = 0; enqueued < num_to_process;) { 1226 1227 uint16_t num_to_enq = burst_sz; 1228 1229 if (unlikely(num_to_process - enqueued < num_to_enq)) 1230 num_to_enq = num_to_process - enqueued; 1231 1232 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops, 1233 num_to_enq); 1234 if (ret != 0) { 1235 allocs_failed++; 1236 continue; 1237 } 1238 1239 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 1240 copy_reference_dec_op(ops, num_to_enq, enqueued, 1241 bufs->inputs, 1242 bufs->hard_outputs, 1243 bufs->soft_outputs, 1244 tp->op_params->ref_dec_op); 1245 1246 enqueued += rte_bbdev_enqueue_dec_ops(tp->dev_id, queue_id, ops, 1247 num_to_enq); 1248 1249 rte_bbdev_dec_op_free_bulk(ops, num_to_enq); 1250 } 1251 1252 if (allocs_failed > 0) 1253 printf("WARNING: op allocations failed: %u times\n", 1254 allocs_failed); 1255 1256 return TEST_SUCCESS; 1257 } 1258 1259 static int 1260 throughput_intr_lcore_enc(void *arg) 1261 { 1262 struct thread_params *tp = arg; 1263 unsigned int enqueued; 1264 struct rte_bbdev_enc_op *ops[MAX_BURST]; 1265 const uint16_t queue_id = tp->queue_id; 1266 const uint16_t burst_sz = tp->op_params->burst_sz; 1267 const uint16_t num_to_process = tp->op_params->num_to_process; 1268 struct test_buffers *bufs = NULL; 1269 unsigned int allocs_failed = 0; 1270 struct rte_bbdev_info info; 1271 int ret; 1272 1273 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 1274 "BURST_SIZE should be <= %u", MAX_BURST); 1275 1276 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id), 1277 "Failed to enable interrupts for dev: %u, queue_id: %u", 1278 tp->dev_id, queue_id); 1279 1280 rte_bbdev_info_get(tp->dev_id, &info); 1281 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 1282 1283 rte_atomic16_clear(&tp->processing_status); 1284 rte_atomic16_clear(&tp->nb_dequeued); 1285 1286 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT) 1287 rte_pause(); 1288 1289 tp->start_time = rte_rdtsc_precise(); 1290 for (enqueued = 0; enqueued < num_to_process;) { 1291 1292 uint16_t num_to_enq = burst_sz; 1293 1294 if (unlikely(num_to_process - enqueued < num_to_enq)) 1295 num_to_enq = num_to_process - enqueued; 1296 1297 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops, 1298 num_to_enq); 1299 if (ret != 0) { 1300 allocs_failed++; 1301 continue; 1302 } 1303 1304 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 1305 copy_reference_enc_op(ops, num_to_enq, enqueued, 1306 bufs->inputs, 1307 bufs->hard_outputs, 1308 tp->op_params->ref_enc_op); 1309 1310 enqueued += rte_bbdev_enqueue_enc_ops(tp->dev_id, queue_id, ops, 1311 num_to_enq); 1312 1313 rte_bbdev_enc_op_free_bulk(ops, num_to_enq); 1314 } 1315 1316 if (allocs_failed > 0) 1317 printf("WARNING: op allocations failed: %u times\n", 1318 allocs_failed); 1319 1320 return TEST_SUCCESS; 1321 } 1322 1323 static int 1324 throughput_pmd_lcore_dec(void *arg) 1325 { 1326 struct thread_params *tp = arg; 1327 unsigned int enqueued, dequeued; 1328 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 1329 uint64_t total_time, start_time; 1330 const uint16_t queue_id = tp->queue_id; 1331 const uint16_t burst_sz = tp->op_params->burst_sz; 1332 const uint16_t num_to_process = tp->op_params->num_to_process; 1333 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 1334 struct test_buffers *bufs = NULL; 1335 unsigned int allocs_failed = 0; 1336 int ret; 1337 struct rte_bbdev_info info; 1338 1339 /* Input length in bytes, million operations per second, million bits 1340 * per second. 1341 */ 1342 double in_len; 1343 1344 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 1345 "BURST_SIZE should be <= %u", MAX_BURST); 1346 1347 rte_bbdev_info_get(tp->dev_id, &info); 1348 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 1349 1350 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT) 1351 rte_pause(); 1352 1353 start_time = rte_rdtsc_precise(); 1354 for (enqueued = 0, dequeued = 0; dequeued < num_to_process;) { 1355 uint16_t deq; 1356 1357 if (likely(enqueued < num_to_process)) { 1358 1359 uint16_t num_to_enq = burst_sz; 1360 1361 if (unlikely(num_to_process - enqueued < num_to_enq)) 1362 num_to_enq = num_to_process - enqueued; 1363 1364 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, 1365 ops_enq, num_to_enq); 1366 if (ret != 0) { 1367 allocs_failed++; 1368 goto do_dequeue; 1369 } 1370 1371 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 1372 copy_reference_dec_op(ops_enq, num_to_enq, 1373 enqueued, 1374 bufs->inputs, 1375 bufs->hard_outputs, 1376 bufs->soft_outputs, 1377 ref_op); 1378 1379 enqueued += rte_bbdev_enqueue_dec_ops(tp->dev_id, 1380 queue_id, ops_enq, num_to_enq); 1381 } 1382 do_dequeue: 1383 deq = rte_bbdev_dequeue_dec_ops(tp->dev_id, queue_id, ops_deq, 1384 burst_sz); 1385 dequeued += deq; 1386 rte_bbdev_dec_op_free_bulk(ops_enq, deq); 1387 } 1388 total_time = rte_rdtsc_precise() - start_time; 1389 1390 if (allocs_failed > 0) 1391 printf("WARNING: op allocations failed: %u times\n", 1392 allocs_failed); 1393 1394 TEST_ASSERT(enqueued == dequeued, "enqueued (%u) != dequeued (%u)", 1395 enqueued, dequeued); 1396 1397 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 1398 ret = validate_dec_buffers(ref_op, bufs, num_to_process); 1399 TEST_ASSERT_SUCCESS(ret, "Buffers validation failed"); 1400 } 1401 1402 in_len = ref_op->turbo_dec.input.length; 1403 tp->mops = ((double)num_to_process / 1000000.0) / 1404 ((double)total_time / (double)rte_get_tsc_hz()); 1405 tp->mbps = ((double)num_to_process * in_len * 8 / 1000000.0) / 1406 ((double)total_time / (double)rte_get_tsc_hz()); 1407 1408 return TEST_SUCCESS; 1409 } 1410 1411 static int 1412 throughput_pmd_lcore_enc(void *arg) 1413 { 1414 struct thread_params *tp = arg; 1415 unsigned int enqueued, dequeued; 1416 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 1417 uint64_t total_time, start_time; 1418 const uint16_t queue_id = tp->queue_id; 1419 const uint16_t burst_sz = tp->op_params->burst_sz; 1420 const uint16_t num_to_process = tp->op_params->num_to_process; 1421 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op; 1422 struct test_buffers *bufs = NULL; 1423 unsigned int allocs_failed = 0; 1424 int ret; 1425 struct rte_bbdev_info info; 1426 1427 /* Input length in bytes, million operations per second, million bits 1428 * per second. 1429 */ 1430 double in_len; 1431 1432 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 1433 "BURST_SIZE should be <= %u", MAX_BURST); 1434 1435 rte_bbdev_info_get(tp->dev_id, &info); 1436 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 1437 1438 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT) 1439 rte_pause(); 1440 1441 start_time = rte_rdtsc_precise(); 1442 for (enqueued = 0, dequeued = 0; dequeued < num_to_process;) { 1443 uint16_t deq; 1444 1445 if (likely(enqueued < num_to_process)) { 1446 1447 uint16_t num_to_enq = burst_sz; 1448 1449 if (unlikely(num_to_process - enqueued < num_to_enq)) 1450 num_to_enq = num_to_process - enqueued; 1451 1452 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, 1453 ops_enq, num_to_enq); 1454 if (ret != 0) { 1455 allocs_failed++; 1456 goto do_dequeue; 1457 } 1458 1459 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 1460 copy_reference_enc_op(ops_enq, num_to_enq, 1461 enqueued, 1462 bufs->inputs, 1463 bufs->hard_outputs, 1464 ref_op); 1465 1466 enqueued += rte_bbdev_enqueue_enc_ops(tp->dev_id, 1467 queue_id, ops_enq, num_to_enq); 1468 } 1469 do_dequeue: 1470 deq = rte_bbdev_dequeue_enc_ops(tp->dev_id, queue_id, ops_deq, 1471 burst_sz); 1472 dequeued += deq; 1473 rte_bbdev_enc_op_free_bulk(ops_enq, deq); 1474 } 1475 total_time = rte_rdtsc_precise() - start_time; 1476 1477 if (allocs_failed > 0) 1478 printf("WARNING: op allocations failed: %u times\n", 1479 allocs_failed); 1480 1481 TEST_ASSERT(enqueued == dequeued, "enqueued (%u) != dequeued (%u)", 1482 enqueued, dequeued); 1483 1484 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 1485 ret = validate_enc_buffers(bufs, num_to_process); 1486 TEST_ASSERT_SUCCESS(ret, "Buffers validation failed"); 1487 } 1488 1489 in_len = ref_op->turbo_enc.input.length; 1490 1491 tp->mops = ((double)num_to_process / 1000000.0) / 1492 ((double)total_time / (double)rte_get_tsc_hz()); 1493 tp->mbps = ((double)num_to_process * in_len * 8 / 1000000.0) / 1494 ((double)total_time / (double)rte_get_tsc_hz()); 1495 1496 return TEST_SUCCESS; 1497 } 1498 static void 1499 print_throughput(struct thread_params *t_params, unsigned int used_cores) 1500 { 1501 unsigned int lcore_id, iter = 0; 1502 double total_mops = 0, total_mbps = 0; 1503 1504 RTE_LCORE_FOREACH(lcore_id) { 1505 if (iter++ >= used_cores) 1506 break; 1507 printf("\tlcore_id: %u, throughput: %.8lg MOPS, %.8lg Mbps\n", 1508 lcore_id, t_params[lcore_id].mops, t_params[lcore_id].mbps); 1509 total_mops += t_params[lcore_id].mops; 1510 total_mbps += t_params[lcore_id].mbps; 1511 } 1512 printf( 1513 "\n\tTotal stats for %u cores: throughput: %.8lg MOPS, %.8lg Mbps\n", 1514 used_cores, total_mops, total_mbps); 1515 } 1516 1517 /* 1518 * Test function that determines how long an enqueue + dequeue of a burst 1519 * takes on available lcores. 1520 */ 1521 static int 1522 throughput_test(struct active_device *ad, 1523 struct test_op_params *op_params) 1524 { 1525 int ret; 1526 unsigned int lcore_id, used_cores = 0; 1527 struct thread_params t_params[MAX_QUEUES]; 1528 struct rte_bbdev_info info; 1529 lcore_function_t *throughput_function; 1530 struct thread_params *tp; 1531 uint16_t num_lcores; 1532 const char *op_type_str; 1533 1534 rte_bbdev_info_get(ad->dev_id, &info); 1535 1536 op_type_str = rte_bbdev_op_type_str(test_vector.op_type); 1537 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", 1538 test_vector.op_type); 1539 1540 printf( 1541 "Throughput test: dev: %s, nb_queues: %u, burst size: %u, num ops: %u, num_lcores: %u, op type: %s, int mode: %s, GHz: %lg\n", 1542 info.dev_name, ad->nb_queues, op_params->burst_sz, 1543 op_params->num_to_process, op_params->num_lcores, 1544 op_type_str, 1545 intr_enabled ? "Interrupt mode" : "PMD mode", 1546 (double)rte_get_tsc_hz() / 1000000000.0); 1547 1548 /* Set number of lcores */ 1549 num_lcores = (ad->nb_queues < (op_params->num_lcores)) 1550 ? ad->nb_queues 1551 : op_params->num_lcores; 1552 1553 if (intr_enabled) { 1554 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) 1555 throughput_function = throughput_intr_lcore_dec; 1556 else 1557 throughput_function = throughput_intr_lcore_enc; 1558 1559 /* Dequeue interrupt callback registration */ 1560 ret = rte_bbdev_callback_register(ad->dev_id, 1561 RTE_BBDEV_EVENT_DEQUEUE, dequeue_event_callback, 1562 &t_params); 1563 if (ret < 0) 1564 return ret; 1565 } else { 1566 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) 1567 throughput_function = throughput_pmd_lcore_dec; 1568 else 1569 throughput_function = throughput_pmd_lcore_enc; 1570 } 1571 1572 rte_atomic16_set(&op_params->sync, SYNC_WAIT); 1573 1574 t_params[rte_lcore_id()].dev_id = ad->dev_id; 1575 t_params[rte_lcore_id()].op_params = op_params; 1576 t_params[rte_lcore_id()].queue_id = 1577 ad->queue_ids[used_cores++]; 1578 1579 RTE_LCORE_FOREACH_SLAVE(lcore_id) { 1580 if (used_cores >= num_lcores) 1581 break; 1582 1583 t_params[lcore_id].dev_id = ad->dev_id; 1584 t_params[lcore_id].op_params = op_params; 1585 t_params[lcore_id].queue_id = ad->queue_ids[used_cores++]; 1586 1587 rte_eal_remote_launch(throughput_function, &t_params[lcore_id], 1588 lcore_id); 1589 } 1590 1591 rte_atomic16_set(&op_params->sync, SYNC_START); 1592 ret = throughput_function(&t_params[rte_lcore_id()]); 1593 1594 /* Master core is always used */ 1595 used_cores = 1; 1596 RTE_LCORE_FOREACH_SLAVE(lcore_id) { 1597 if (used_cores++ >= num_lcores) 1598 break; 1599 1600 ret |= rte_eal_wait_lcore(lcore_id); 1601 } 1602 1603 /* Return if test failed */ 1604 if (ret) 1605 return ret; 1606 1607 /* Print throughput if interrupts are disabled and test passed */ 1608 if (!intr_enabled) { 1609 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 1610 print_throughput(t_params, num_lcores); 1611 return ret; 1612 } 1613 1614 /* In interrupt TC we need to wait for the interrupt callback to deqeue 1615 * all pending operations. Skip waiting for queues which reported an 1616 * error using processing_status variable. 1617 * Wait for master lcore operations. 1618 */ 1619 tp = &t_params[rte_lcore_id()]; 1620 while ((rte_atomic16_read(&tp->nb_dequeued) < 1621 op_params->num_to_process) && 1622 (rte_atomic16_read(&tp->processing_status) != 1623 TEST_FAILED)) 1624 rte_pause(); 1625 1626 ret |= rte_atomic16_read(&tp->processing_status); 1627 1628 /* Wait for slave lcores operations */ 1629 used_cores = 1; 1630 RTE_LCORE_FOREACH_SLAVE(lcore_id) { 1631 tp = &t_params[lcore_id]; 1632 if (used_cores++ >= num_lcores) 1633 break; 1634 1635 while ((rte_atomic16_read(&tp->nb_dequeued) < 1636 op_params->num_to_process) && 1637 (rte_atomic16_read(&tp->processing_status) != 1638 TEST_FAILED)) 1639 rte_pause(); 1640 1641 ret |= rte_atomic16_read(&tp->processing_status); 1642 } 1643 1644 /* Print throughput if test passed */ 1645 if (!ret && test_vector.op_type != RTE_BBDEV_OP_NONE) 1646 print_throughput(t_params, num_lcores); 1647 1648 return ret; 1649 } 1650 1651 static int 1652 operation_latency_test_dec(struct rte_mempool *mempool, 1653 struct test_buffers *bufs, struct rte_bbdev_dec_op *ref_op, 1654 int vector_mask, uint16_t dev_id, uint16_t queue_id, 1655 const uint16_t num_to_process, uint16_t burst_sz, 1656 uint64_t *total_time) 1657 { 1658 int ret = TEST_SUCCESS; 1659 uint16_t i, j, dequeued; 1660 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 1661 uint64_t start_time = 0; 1662 1663 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 1664 uint16_t enq = 0, deq = 0; 1665 bool first_time = true; 1666 1667 if (unlikely(num_to_process - dequeued < burst_sz)) 1668 burst_sz = num_to_process - dequeued; 1669 1670 rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz); 1671 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 1672 copy_reference_dec_op(ops_enq, burst_sz, dequeued, 1673 bufs->inputs, 1674 bufs->hard_outputs, 1675 bufs->soft_outputs, 1676 ref_op); 1677 1678 /* Set counter to validate the ordering */ 1679 for (j = 0; j < burst_sz; ++j) 1680 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 1681 1682 start_time = rte_rdtsc_precise(); 1683 1684 enq = rte_bbdev_enqueue_dec_ops(dev_id, queue_id, &ops_enq[enq], 1685 burst_sz); 1686 TEST_ASSERT(enq == burst_sz, 1687 "Error enqueueing burst, expected %u, got %u", 1688 burst_sz, enq); 1689 1690 /* Dequeue */ 1691 do { 1692 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id, 1693 &ops_deq[deq], burst_sz - deq); 1694 if (likely(first_time && (deq > 0))) { 1695 *total_time += rte_rdtsc_precise() - start_time; 1696 first_time = false; 1697 } 1698 } while (unlikely(burst_sz != deq)); 1699 1700 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 1701 ret = validate_dec_op(ops_deq, burst_sz, ref_op, 1702 vector_mask); 1703 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 1704 } 1705 1706 rte_bbdev_dec_op_free_bulk(ops_enq, deq); 1707 dequeued += deq; 1708 } 1709 1710 return i; 1711 } 1712 1713 static int 1714 operation_latency_test_enc(struct rte_mempool *mempool, 1715 struct test_buffers *bufs, struct rte_bbdev_enc_op *ref_op, 1716 uint16_t dev_id, uint16_t queue_id, 1717 const uint16_t num_to_process, uint16_t burst_sz, 1718 uint64_t *total_time) 1719 { 1720 int ret = TEST_SUCCESS; 1721 uint16_t i, j, dequeued; 1722 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 1723 uint64_t start_time = 0; 1724 1725 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 1726 uint16_t enq = 0, deq = 0; 1727 bool first_time = true; 1728 1729 if (unlikely(num_to_process - dequeued < burst_sz)) 1730 burst_sz = num_to_process - dequeued; 1731 1732 rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz); 1733 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 1734 copy_reference_enc_op(ops_enq, burst_sz, dequeued, 1735 bufs->inputs, 1736 bufs->hard_outputs, 1737 ref_op); 1738 1739 /* Set counter to validate the ordering */ 1740 for (j = 0; j < burst_sz; ++j) 1741 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 1742 1743 start_time = rte_rdtsc_precise(); 1744 1745 enq = rte_bbdev_enqueue_enc_ops(dev_id, queue_id, &ops_enq[enq], 1746 burst_sz); 1747 TEST_ASSERT(enq == burst_sz, 1748 "Error enqueueing burst, expected %u, got %u", 1749 burst_sz, enq); 1750 1751 /* Dequeue */ 1752 do { 1753 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id, 1754 &ops_deq[deq], burst_sz - deq); 1755 if (likely(first_time && (deq > 0))) { 1756 *total_time += rte_rdtsc_precise() - start_time; 1757 first_time = false; 1758 } 1759 } while (unlikely(burst_sz != deq)); 1760 1761 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 1762 ret = validate_enc_op(ops_deq, burst_sz, ref_op); 1763 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 1764 } 1765 1766 rte_bbdev_enc_op_free_bulk(ops_enq, deq); 1767 dequeued += deq; 1768 } 1769 1770 return i; 1771 } 1772 1773 static int 1774 operation_latency_test(struct active_device *ad, 1775 struct test_op_params *op_params) 1776 { 1777 int iter; 1778 uint16_t burst_sz = op_params->burst_sz; 1779 const uint16_t num_to_process = op_params->num_to_process; 1780 const enum rte_bbdev_op_type op_type = test_vector.op_type; 1781 const uint16_t queue_id = ad->queue_ids[0]; 1782 struct test_buffers *bufs = NULL; 1783 struct rte_bbdev_info info; 1784 uint64_t total_time = 0; 1785 const char *op_type_str; 1786 1787 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 1788 "BURST_SIZE should be <= %u", MAX_BURST); 1789 1790 rte_bbdev_info_get(ad->dev_id, &info); 1791 bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 1792 1793 op_type_str = rte_bbdev_op_type_str(op_type); 1794 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type); 1795 1796 printf( 1797 "Validation/Latency test: dev: %s, burst size: %u, num ops: %u, op type: %s\n", 1798 info.dev_name, burst_sz, num_to_process, op_type_str); 1799 1800 if (op_type == RTE_BBDEV_OP_TURBO_DEC) 1801 iter = operation_latency_test_dec(op_params->mp, bufs, 1802 op_params->ref_dec_op, op_params->vector_mask, 1803 ad->dev_id, queue_id, num_to_process, 1804 burst_sz, &total_time); 1805 else 1806 iter = operation_latency_test_enc(op_params->mp, bufs, 1807 op_params->ref_enc_op, ad->dev_id, queue_id, 1808 num_to_process, burst_sz, &total_time); 1809 1810 if (iter <= 0) 1811 return TEST_FAILED; 1812 1813 printf("\toperation avg. latency: %lg cycles, %lg us\n", 1814 (double)total_time / (double)iter, 1815 (double)(total_time * 1000000) / (double)iter / 1816 (double)rte_get_tsc_hz()); 1817 1818 return TEST_SUCCESS; 1819 } 1820 1821 static int 1822 offload_latency_test_dec(struct rte_mempool *mempool, struct test_buffers *bufs, 1823 struct rte_bbdev_dec_op *ref_op, uint16_t dev_id, 1824 uint16_t queue_id, const uint16_t num_to_process, 1825 uint16_t burst_sz, uint64_t *enq_total_time, 1826 uint64_t *deq_total_time) 1827 { 1828 int i, dequeued; 1829 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 1830 uint64_t enq_start_time, deq_start_time; 1831 1832 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 1833 uint16_t enq = 0, deq = 0; 1834 1835 if (unlikely(num_to_process - dequeued < burst_sz)) 1836 burst_sz = num_to_process - dequeued; 1837 1838 rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz); 1839 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 1840 copy_reference_dec_op(ops_enq, burst_sz, dequeued, 1841 bufs->inputs, 1842 bufs->hard_outputs, 1843 bufs->soft_outputs, 1844 ref_op); 1845 1846 /* Start time measurment for enqueue function offload latency */ 1847 enq_start_time = rte_rdtsc(); 1848 do { 1849 enq += rte_bbdev_enqueue_dec_ops(dev_id, queue_id, 1850 &ops_enq[enq], burst_sz - enq); 1851 } while (unlikely(burst_sz != enq)); 1852 *enq_total_time += rte_rdtsc() - enq_start_time; 1853 1854 /* ensure enqueue has been completed */ 1855 rte_delay_ms(10); 1856 1857 /* Start time measurment for dequeue function offload latency */ 1858 deq_start_time = rte_rdtsc(); 1859 do { 1860 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id, 1861 &ops_deq[deq], burst_sz - deq); 1862 } while (unlikely(burst_sz != deq)); 1863 *deq_total_time += rte_rdtsc() - deq_start_time; 1864 1865 rte_bbdev_dec_op_free_bulk(ops_enq, deq); 1866 dequeued += deq; 1867 } 1868 1869 return i; 1870 } 1871 1872 static int 1873 offload_latency_test_enc(struct rte_mempool *mempool, struct test_buffers *bufs, 1874 struct rte_bbdev_enc_op *ref_op, uint16_t dev_id, 1875 uint16_t queue_id, const uint16_t num_to_process, 1876 uint16_t burst_sz, uint64_t *enq_total_time, 1877 uint64_t *deq_total_time) 1878 { 1879 int i, dequeued; 1880 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 1881 uint64_t enq_start_time, deq_start_time; 1882 1883 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 1884 uint16_t enq = 0, deq = 0; 1885 1886 if (unlikely(num_to_process - dequeued < burst_sz)) 1887 burst_sz = num_to_process - dequeued; 1888 1889 rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz); 1890 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 1891 copy_reference_enc_op(ops_enq, burst_sz, dequeued, 1892 bufs->inputs, 1893 bufs->hard_outputs, 1894 ref_op); 1895 1896 /* Start time measurment for enqueue function offload latency */ 1897 enq_start_time = rte_rdtsc(); 1898 do { 1899 enq += rte_bbdev_enqueue_enc_ops(dev_id, queue_id, 1900 &ops_enq[enq], burst_sz - enq); 1901 } while (unlikely(burst_sz != enq)); 1902 *enq_total_time += rte_rdtsc() - enq_start_time; 1903 1904 /* ensure enqueue has been completed */ 1905 rte_delay_ms(10); 1906 1907 /* Start time measurment for dequeue function offload latency */ 1908 deq_start_time = rte_rdtsc(); 1909 do { 1910 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id, 1911 &ops_deq[deq], burst_sz - deq); 1912 } while (unlikely(burst_sz != deq)); 1913 *deq_total_time += rte_rdtsc() - deq_start_time; 1914 1915 rte_bbdev_enc_op_free_bulk(ops_enq, deq); 1916 dequeued += deq; 1917 } 1918 1919 return i; 1920 } 1921 1922 static int 1923 offload_latency_test(struct active_device *ad, 1924 struct test_op_params *op_params) 1925 { 1926 int iter; 1927 uint64_t enq_total_time = 0, deq_total_time = 0; 1928 uint16_t burst_sz = op_params->burst_sz; 1929 const uint16_t num_to_process = op_params->num_to_process; 1930 const enum rte_bbdev_op_type op_type = test_vector.op_type; 1931 const uint16_t queue_id = ad->queue_ids[0]; 1932 struct test_buffers *bufs = NULL; 1933 struct rte_bbdev_info info; 1934 const char *op_type_str; 1935 1936 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 1937 "BURST_SIZE should be <= %u", MAX_BURST); 1938 1939 rte_bbdev_info_get(ad->dev_id, &info); 1940 bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 1941 1942 op_type_str = rte_bbdev_op_type_str(op_type); 1943 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type); 1944 1945 printf( 1946 "Offload latency test: dev: %s, burst size: %u, num ops: %u, op type: %s\n", 1947 info.dev_name, burst_sz, num_to_process, op_type_str); 1948 1949 if (op_type == RTE_BBDEV_OP_TURBO_DEC) 1950 iter = offload_latency_test_dec(op_params->mp, bufs, 1951 op_params->ref_dec_op, ad->dev_id, queue_id, 1952 num_to_process, burst_sz, &enq_total_time, 1953 &deq_total_time); 1954 else 1955 iter = offload_latency_test_enc(op_params->mp, bufs, 1956 op_params->ref_enc_op, ad->dev_id, queue_id, 1957 num_to_process, burst_sz, &enq_total_time, 1958 &deq_total_time); 1959 1960 if (iter <= 0) 1961 return TEST_FAILED; 1962 1963 printf("\tenq offload avg. latency: %lg cycles, %lg us\n", 1964 (double)enq_total_time / (double)iter, 1965 (double)(enq_total_time * 1000000) / (double)iter / 1966 (double)rte_get_tsc_hz()); 1967 1968 printf("\tdeq offload avg. latency: %lg cycles, %lg us\n", 1969 (double)deq_total_time / (double)iter, 1970 (double)(deq_total_time * 1000000) / (double)iter / 1971 (double)rte_get_tsc_hz()); 1972 1973 return TEST_SUCCESS; 1974 } 1975 1976 static int 1977 offload_latency_empty_q_test_dec(uint16_t dev_id, uint16_t queue_id, 1978 const uint16_t num_to_process, uint16_t burst_sz, 1979 uint64_t *deq_total_time) 1980 { 1981 int i, deq_total; 1982 struct rte_bbdev_dec_op *ops[MAX_BURST]; 1983 uint64_t deq_start_time; 1984 1985 /* Test deq offload latency from an empty queue */ 1986 deq_start_time = rte_rdtsc_precise(); 1987 for (i = 0, deq_total = 0; deq_total < num_to_process; 1988 ++i, deq_total += burst_sz) { 1989 if (unlikely(num_to_process - deq_total < burst_sz)) 1990 burst_sz = num_to_process - deq_total; 1991 rte_bbdev_dequeue_dec_ops(dev_id, queue_id, ops, burst_sz); 1992 } 1993 *deq_total_time = rte_rdtsc_precise() - deq_start_time; 1994 1995 return i; 1996 } 1997 1998 static int 1999 offload_latency_empty_q_test_enc(uint16_t dev_id, uint16_t queue_id, 2000 const uint16_t num_to_process, uint16_t burst_sz, 2001 uint64_t *deq_total_time) 2002 { 2003 int i, deq_total; 2004 struct rte_bbdev_enc_op *ops[MAX_BURST]; 2005 uint64_t deq_start_time; 2006 2007 /* Test deq offload latency from an empty queue */ 2008 deq_start_time = rte_rdtsc_precise(); 2009 for (i = 0, deq_total = 0; deq_total < num_to_process; 2010 ++i, deq_total += burst_sz) { 2011 if (unlikely(num_to_process - deq_total < burst_sz)) 2012 burst_sz = num_to_process - deq_total; 2013 rte_bbdev_dequeue_enc_ops(dev_id, queue_id, ops, burst_sz); 2014 } 2015 *deq_total_time = rte_rdtsc_precise() - deq_start_time; 2016 2017 return i; 2018 } 2019 2020 static int 2021 offload_latency_empty_q_test(struct active_device *ad, 2022 struct test_op_params *op_params) 2023 { 2024 int iter; 2025 uint64_t deq_total_time = 0; 2026 uint16_t burst_sz = op_params->burst_sz; 2027 const uint16_t num_to_process = op_params->num_to_process; 2028 const enum rte_bbdev_op_type op_type = test_vector.op_type; 2029 const uint16_t queue_id = ad->queue_ids[0]; 2030 struct rte_bbdev_info info; 2031 const char *op_type_str; 2032 2033 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 2034 "BURST_SIZE should be <= %u", MAX_BURST); 2035 2036 rte_bbdev_info_get(ad->dev_id, &info); 2037 2038 op_type_str = rte_bbdev_op_type_str(op_type); 2039 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type); 2040 2041 printf( 2042 "Offload latency empty dequeue test: dev: %s, burst size: %u, num ops: %u, op type: %s\n", 2043 info.dev_name, burst_sz, num_to_process, op_type_str); 2044 2045 if (op_type == RTE_BBDEV_OP_TURBO_DEC) 2046 iter = offload_latency_empty_q_test_dec(ad->dev_id, queue_id, 2047 num_to_process, burst_sz, &deq_total_time); 2048 else 2049 iter = offload_latency_empty_q_test_enc(ad->dev_id, queue_id, 2050 num_to_process, burst_sz, &deq_total_time); 2051 2052 if (iter <= 0) 2053 return TEST_FAILED; 2054 2055 printf("\tempty deq offload avg. latency: %lg cycles, %lg us\n", 2056 (double)deq_total_time / (double)iter, 2057 (double)(deq_total_time * 1000000) / (double)iter / 2058 (double)rte_get_tsc_hz()); 2059 2060 return TEST_SUCCESS; 2061 } 2062 2063 static int 2064 throughput_tc(void) 2065 { 2066 return run_test_case(throughput_test); 2067 } 2068 2069 static int 2070 offload_latency_tc(void) 2071 { 2072 return run_test_case(offload_latency_test); 2073 } 2074 2075 static int 2076 offload_latency_empty_q_tc(void) 2077 { 2078 return run_test_case(offload_latency_empty_q_test); 2079 } 2080 2081 static int 2082 operation_latency_tc(void) 2083 { 2084 return run_test_case(operation_latency_test); 2085 } 2086 2087 static int 2088 interrupt_tc(void) 2089 { 2090 return run_test_case(throughput_test); 2091 } 2092 2093 static struct unit_test_suite bbdev_throughput_testsuite = { 2094 .suite_name = "BBdev Throughput Tests", 2095 .setup = testsuite_setup, 2096 .teardown = testsuite_teardown, 2097 .unit_test_cases = { 2098 TEST_CASE_ST(ut_setup, ut_teardown, throughput_tc), 2099 TEST_CASES_END() /**< NULL terminate unit test array */ 2100 } 2101 }; 2102 2103 static struct unit_test_suite bbdev_validation_testsuite = { 2104 .suite_name = "BBdev Validation Tests", 2105 .setup = testsuite_setup, 2106 .teardown = testsuite_teardown, 2107 .unit_test_cases = { 2108 TEST_CASE_ST(ut_setup, ut_teardown, operation_latency_tc), 2109 TEST_CASES_END() /**< NULL terminate unit test array */ 2110 } 2111 }; 2112 2113 static struct unit_test_suite bbdev_latency_testsuite = { 2114 .suite_name = "BBdev Latency Tests", 2115 .setup = testsuite_setup, 2116 .teardown = testsuite_teardown, 2117 .unit_test_cases = { 2118 TEST_CASE_ST(ut_setup, ut_teardown, offload_latency_tc), 2119 TEST_CASE_ST(ut_setup, ut_teardown, offload_latency_empty_q_tc), 2120 TEST_CASE_ST(ut_setup, ut_teardown, operation_latency_tc), 2121 TEST_CASES_END() /**< NULL terminate unit test array */ 2122 } 2123 }; 2124 2125 static struct unit_test_suite bbdev_interrupt_testsuite = { 2126 .suite_name = "BBdev Interrupt Tests", 2127 .setup = interrupt_testsuite_setup, 2128 .teardown = testsuite_teardown, 2129 .unit_test_cases = { 2130 TEST_CASE_ST(ut_setup, ut_teardown, interrupt_tc), 2131 TEST_CASES_END() /**< NULL terminate unit test array */ 2132 } 2133 }; 2134 2135 REGISTER_TEST_COMMAND(throughput, bbdev_throughput_testsuite); 2136 REGISTER_TEST_COMMAND(validation, bbdev_validation_testsuite); 2137 REGISTER_TEST_COMMAND(latency, bbdev_latency_testsuite); 2138 REGISTER_TEST_COMMAND(interrupt, bbdev_interrupt_testsuite); 2139