1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017 Intel Corporation 3 */ 4 5 #include <stdio.h> 6 #include <inttypes.h> 7 8 #include <rte_eal.h> 9 #include <rte_common.h> 10 #include <rte_dev.h> 11 #include <rte_launch.h> 12 #include <rte_bbdev.h> 13 #include <rte_cycles.h> 14 #include <rte_lcore.h> 15 #include <rte_malloc.h> 16 #include <rte_random.h> 17 #include <rte_hexdump.h> 18 19 #include "main.h" 20 #include "test_bbdev_vector.h" 21 22 #define GET_SOCKET(socket_id) (((socket_id) == SOCKET_ID_ANY) ? 0 : (socket_id)) 23 24 #define MAX_QUEUES RTE_MAX_LCORE 25 26 #define OPS_CACHE_SIZE 256U 27 #define OPS_POOL_SIZE_MIN 511U /* 0.5K per queue */ 28 29 #define SYNC_WAIT 0 30 #define SYNC_START 1 31 32 #define INVALID_QUEUE_ID -1 33 34 static struct test_bbdev_vector test_vector; 35 36 /* Switch between PMD and Interrupt for throughput TC */ 37 static bool intr_enabled; 38 39 /* Represents tested active devices */ 40 static struct active_device { 41 const char *driver_name; 42 uint8_t dev_id; 43 uint16_t supported_ops; 44 uint16_t queue_ids[MAX_QUEUES]; 45 uint16_t nb_queues; 46 struct rte_mempool *ops_mempool; 47 struct rte_mempool *in_mbuf_pool; 48 struct rte_mempool *hard_out_mbuf_pool; 49 struct rte_mempool *soft_out_mbuf_pool; 50 } active_devs[RTE_BBDEV_MAX_DEVS]; 51 52 static uint8_t nb_active_devs; 53 54 /* Data buffers used by BBDEV ops */ 55 struct test_buffers { 56 struct rte_bbdev_op_data *inputs; 57 struct rte_bbdev_op_data *hard_outputs; 58 struct rte_bbdev_op_data *soft_outputs; 59 }; 60 61 /* Operation parameters specific for given test case */ 62 struct test_op_params { 63 struct rte_mempool *mp; 64 struct rte_bbdev_dec_op *ref_dec_op; 65 struct rte_bbdev_enc_op *ref_enc_op; 66 uint16_t burst_sz; 67 uint16_t num_to_process; 68 uint16_t num_lcores; 69 int vector_mask; 70 rte_atomic16_t sync; 71 struct test_buffers q_bufs[RTE_MAX_NUMA_NODES][MAX_QUEUES]; 72 }; 73 74 /* Contains per lcore params */ 75 struct thread_params { 76 uint8_t dev_id; 77 uint16_t queue_id; 78 uint64_t start_time; 79 double mops; 80 double mbps; 81 rte_atomic16_t nb_dequeued; 82 rte_atomic16_t processing_status; 83 struct test_op_params *op_params; 84 }; 85 86 typedef int (test_case_function)(struct active_device *ad, 87 struct test_op_params *op_params); 88 89 static inline void 90 set_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type) 91 { 92 ad->supported_ops |= (1 << op_type); 93 } 94 95 static inline bool 96 is_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type) 97 { 98 return ad->supported_ops & (1 << op_type); 99 } 100 101 static inline bool 102 flags_match(uint32_t flags_req, uint32_t flags_present) 103 { 104 return (flags_req & flags_present) == flags_req; 105 } 106 107 static void 108 clear_soft_out_cap(uint32_t *op_flags) 109 { 110 *op_flags &= ~RTE_BBDEV_TURBO_SOFT_OUTPUT; 111 *op_flags &= ~RTE_BBDEV_TURBO_POS_LLR_1_BIT_SOFT_OUT; 112 *op_flags &= ~RTE_BBDEV_TURBO_NEG_LLR_1_BIT_SOFT_OUT; 113 } 114 115 static int 116 check_dev_cap(const struct rte_bbdev_info *dev_info) 117 { 118 unsigned int i; 119 unsigned int nb_inputs, nb_soft_outputs, nb_hard_outputs; 120 const struct rte_bbdev_op_cap *op_cap = dev_info->drv.capabilities; 121 122 nb_inputs = test_vector.entries[DATA_INPUT].nb_segments; 123 nb_soft_outputs = test_vector.entries[DATA_SOFT_OUTPUT].nb_segments; 124 nb_hard_outputs = test_vector.entries[DATA_HARD_OUTPUT].nb_segments; 125 126 for (i = 0; op_cap->type != RTE_BBDEV_OP_NONE; ++i, ++op_cap) { 127 if (op_cap->type != test_vector.op_type) 128 continue; 129 130 if (op_cap->type == RTE_BBDEV_OP_TURBO_DEC) { 131 const struct rte_bbdev_op_cap_turbo_dec *cap = 132 &op_cap->cap.turbo_dec; 133 /* Ignore lack of soft output capability, just skip 134 * checking if soft output is valid. 135 */ 136 if ((test_vector.turbo_dec.op_flags & 137 RTE_BBDEV_TURBO_SOFT_OUTPUT) && 138 !(cap->capability_flags & 139 RTE_BBDEV_TURBO_SOFT_OUTPUT)) { 140 printf( 141 "WARNING: Device \"%s\" does not support soft output - soft output flags will be ignored.\n", 142 dev_info->dev_name); 143 clear_soft_out_cap( 144 &test_vector.turbo_dec.op_flags); 145 } 146 147 if (!flags_match(test_vector.turbo_dec.op_flags, 148 cap->capability_flags)) 149 return TEST_FAILED; 150 if (nb_inputs > cap->num_buffers_src) { 151 printf("Too many inputs defined: %u, max: %u\n", 152 nb_inputs, cap->num_buffers_src); 153 return TEST_FAILED; 154 } 155 if (nb_soft_outputs > cap->num_buffers_soft_out && 156 (test_vector.turbo_dec.op_flags & 157 RTE_BBDEV_TURBO_SOFT_OUTPUT)) { 158 printf( 159 "Too many soft outputs defined: %u, max: %u\n", 160 nb_soft_outputs, 161 cap->num_buffers_soft_out); 162 return TEST_FAILED; 163 } 164 if (nb_hard_outputs > cap->num_buffers_hard_out) { 165 printf( 166 "Too many hard outputs defined: %u, max: %u\n", 167 nb_hard_outputs, 168 cap->num_buffers_hard_out); 169 return TEST_FAILED; 170 } 171 if (intr_enabled && !(cap->capability_flags & 172 RTE_BBDEV_TURBO_DEC_INTERRUPTS)) { 173 printf( 174 "Dequeue interrupts are not supported!\n"); 175 return TEST_FAILED; 176 } 177 178 return TEST_SUCCESS; 179 } else if (op_cap->type == RTE_BBDEV_OP_TURBO_ENC) { 180 const struct rte_bbdev_op_cap_turbo_enc *cap = 181 &op_cap->cap.turbo_enc; 182 183 if (!flags_match(test_vector.turbo_enc.op_flags, 184 cap->capability_flags)) 185 return TEST_FAILED; 186 if (nb_inputs > cap->num_buffers_src) { 187 printf("Too many inputs defined: %u, max: %u\n", 188 nb_inputs, cap->num_buffers_src); 189 return TEST_FAILED; 190 } 191 if (nb_hard_outputs > cap->num_buffers_dst) { 192 printf( 193 "Too many hard outputs defined: %u, max: %u\n", 194 nb_hard_outputs, cap->num_buffers_src); 195 return TEST_FAILED; 196 } 197 if (intr_enabled && !(cap->capability_flags & 198 RTE_BBDEV_TURBO_ENC_INTERRUPTS)) { 199 printf( 200 "Dequeue interrupts are not supported!\n"); 201 return TEST_FAILED; 202 } 203 204 return TEST_SUCCESS; 205 } 206 } 207 208 if ((i == 0) && (test_vector.op_type == RTE_BBDEV_OP_NONE)) 209 return TEST_SUCCESS; /* Special case for NULL device */ 210 211 return TEST_FAILED; 212 } 213 214 /* calculates optimal mempool size not smaller than the val */ 215 static unsigned int 216 optimal_mempool_size(unsigned int val) 217 { 218 return rte_align32pow2(val + 1) - 1; 219 } 220 221 /* allocates mbuf mempool for inputs and outputs */ 222 static struct rte_mempool * 223 create_mbuf_pool(struct op_data_entries *entries, uint8_t dev_id, 224 int socket_id, unsigned int mbuf_pool_size, 225 const char *op_type_str) 226 { 227 unsigned int i; 228 uint32_t max_seg_sz = 0; 229 char pool_name[RTE_MEMPOOL_NAMESIZE]; 230 231 /* find max input segment size */ 232 for (i = 0; i < entries->nb_segments; ++i) 233 if (entries->segments[i].length > max_seg_sz) 234 max_seg_sz = entries->segments[i].length; 235 236 snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str, 237 dev_id); 238 return rte_pktmbuf_pool_create(pool_name, mbuf_pool_size, 0, 0, 239 RTE_MAX(max_seg_sz + RTE_PKTMBUF_HEADROOM, 240 (unsigned int)RTE_MBUF_DEFAULT_BUF_SIZE), socket_id); 241 } 242 243 static int 244 create_mempools(struct active_device *ad, int socket_id, 245 enum rte_bbdev_op_type op_type, uint16_t num_ops) 246 { 247 struct rte_mempool *mp; 248 unsigned int ops_pool_size, mbuf_pool_size = 0; 249 char pool_name[RTE_MEMPOOL_NAMESIZE]; 250 const char *op_type_str; 251 252 struct op_data_entries *in = &test_vector.entries[DATA_INPUT]; 253 struct op_data_entries *hard_out = 254 &test_vector.entries[DATA_HARD_OUTPUT]; 255 struct op_data_entries *soft_out = 256 &test_vector.entries[DATA_SOFT_OUTPUT]; 257 258 /* allocate ops mempool */ 259 ops_pool_size = optimal_mempool_size(RTE_MAX( 260 /* Ops used plus 1 reference op */ 261 RTE_MAX((unsigned int)(ad->nb_queues * num_ops + 1), 262 /* Minimal cache size plus 1 reference op */ 263 (unsigned int)(1.5 * rte_lcore_count() * 264 OPS_CACHE_SIZE + 1)), 265 OPS_POOL_SIZE_MIN)); 266 267 op_type_str = rte_bbdev_op_type_str(op_type); 268 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type); 269 270 snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str, 271 ad->dev_id); 272 mp = rte_bbdev_op_pool_create(pool_name, op_type, 273 ops_pool_size, OPS_CACHE_SIZE, socket_id); 274 TEST_ASSERT_NOT_NULL(mp, 275 "ERROR Failed to create %u items ops pool for dev %u on socket %u.", 276 ops_pool_size, 277 ad->dev_id, 278 socket_id); 279 ad->ops_mempool = mp; 280 281 /* Inputs */ 282 mbuf_pool_size = optimal_mempool_size(ops_pool_size * in->nb_segments); 283 mp = create_mbuf_pool(in, ad->dev_id, socket_id, mbuf_pool_size, "in"); 284 TEST_ASSERT_NOT_NULL(mp, 285 "ERROR Failed to create %u items input pktmbuf pool for dev %u on socket %u.", 286 mbuf_pool_size, 287 ad->dev_id, 288 socket_id); 289 ad->in_mbuf_pool = mp; 290 291 /* Hard outputs */ 292 mbuf_pool_size = optimal_mempool_size(ops_pool_size * 293 hard_out->nb_segments); 294 mp = create_mbuf_pool(hard_out, ad->dev_id, socket_id, mbuf_pool_size, 295 "hard_out"); 296 TEST_ASSERT_NOT_NULL(mp, 297 "ERROR Failed to create %u items hard output pktmbuf pool for dev %u on socket %u.", 298 mbuf_pool_size, 299 ad->dev_id, 300 socket_id); 301 ad->hard_out_mbuf_pool = mp; 302 303 if (soft_out->nb_segments == 0) 304 return TEST_SUCCESS; 305 306 /* Soft outputs */ 307 mbuf_pool_size = optimal_mempool_size(ops_pool_size * 308 soft_out->nb_segments); 309 mp = create_mbuf_pool(soft_out, ad->dev_id, socket_id, mbuf_pool_size, 310 "soft_out"); 311 TEST_ASSERT_NOT_NULL(mp, 312 "ERROR Failed to create %uB soft output pktmbuf pool for dev %u on socket %u.", 313 mbuf_pool_size, 314 ad->dev_id, 315 socket_id); 316 ad->soft_out_mbuf_pool = mp; 317 318 return 0; 319 } 320 321 static int 322 add_bbdev_dev(uint8_t dev_id, struct rte_bbdev_info *info, 323 struct test_bbdev_vector *vector) 324 { 325 int ret; 326 unsigned int queue_id; 327 struct rte_bbdev_queue_conf qconf; 328 struct active_device *ad = &active_devs[nb_active_devs]; 329 unsigned int nb_queues; 330 enum rte_bbdev_op_type op_type = vector->op_type; 331 332 nb_queues = RTE_MIN(rte_lcore_count(), info->drv.max_num_queues); 333 /* setup device */ 334 ret = rte_bbdev_setup_queues(dev_id, nb_queues, info->socket_id); 335 if (ret < 0) { 336 printf("rte_bbdev_setup_queues(%u, %u, %d) ret %i\n", 337 dev_id, nb_queues, info->socket_id, ret); 338 return TEST_FAILED; 339 } 340 341 /* configure interrupts if needed */ 342 if (intr_enabled) { 343 ret = rte_bbdev_intr_enable(dev_id); 344 if (ret < 0) { 345 printf("rte_bbdev_intr_enable(%u) ret %i\n", dev_id, 346 ret); 347 return TEST_FAILED; 348 } 349 } 350 351 /* setup device queues */ 352 qconf.socket = info->socket_id; 353 qconf.queue_size = info->drv.default_queue_conf.queue_size; 354 qconf.priority = 0; 355 qconf.deferred_start = 0; 356 qconf.op_type = op_type; 357 358 for (queue_id = 0; queue_id < nb_queues; ++queue_id) { 359 ret = rte_bbdev_queue_configure(dev_id, queue_id, &qconf); 360 if (ret != 0) { 361 printf( 362 "Allocated all queues (id=%u) at prio%u on dev%u\n", 363 queue_id, qconf.priority, dev_id); 364 qconf.priority++; 365 ret = rte_bbdev_queue_configure(ad->dev_id, queue_id, 366 &qconf); 367 } 368 if (ret != 0) { 369 printf("All queues on dev %u allocated: %u\n", 370 dev_id, queue_id); 371 break; 372 } 373 ad->queue_ids[queue_id] = queue_id; 374 } 375 TEST_ASSERT(queue_id != 0, 376 "ERROR Failed to configure any queues on dev %u", 377 dev_id); 378 ad->nb_queues = queue_id; 379 380 set_avail_op(ad, op_type); 381 382 return TEST_SUCCESS; 383 } 384 385 static int 386 add_active_device(uint8_t dev_id, struct rte_bbdev_info *info, 387 struct test_bbdev_vector *vector) 388 { 389 int ret; 390 391 active_devs[nb_active_devs].driver_name = info->drv.driver_name; 392 active_devs[nb_active_devs].dev_id = dev_id; 393 394 ret = add_bbdev_dev(dev_id, info, vector); 395 if (ret == TEST_SUCCESS) 396 ++nb_active_devs; 397 return ret; 398 } 399 400 static uint8_t 401 populate_active_devices(void) 402 { 403 int ret; 404 uint8_t dev_id; 405 uint8_t nb_devs_added = 0; 406 struct rte_bbdev_info info; 407 408 RTE_BBDEV_FOREACH(dev_id) { 409 rte_bbdev_info_get(dev_id, &info); 410 411 if (check_dev_cap(&info)) { 412 printf( 413 "Device %d (%s) does not support specified capabilities\n", 414 dev_id, info.dev_name); 415 continue; 416 } 417 418 ret = add_active_device(dev_id, &info, &test_vector); 419 if (ret != 0) { 420 printf("Adding active bbdev %s skipped\n", 421 info.dev_name); 422 continue; 423 } 424 nb_devs_added++; 425 } 426 427 return nb_devs_added; 428 } 429 430 static int 431 read_test_vector(void) 432 { 433 int ret; 434 435 memset(&test_vector, 0, sizeof(test_vector)); 436 printf("Test vector file = %s\n", get_vector_filename()); 437 ret = test_bbdev_vector_read(get_vector_filename(), &test_vector); 438 TEST_ASSERT_SUCCESS(ret, "Failed to parse file %s\n", 439 get_vector_filename()); 440 441 return TEST_SUCCESS; 442 } 443 444 static int 445 testsuite_setup(void) 446 { 447 TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n"); 448 449 if (populate_active_devices() == 0) { 450 printf("No suitable devices found!\n"); 451 return TEST_SKIPPED; 452 } 453 454 return TEST_SUCCESS; 455 } 456 457 static int 458 interrupt_testsuite_setup(void) 459 { 460 TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n"); 461 462 /* Enable interrupts */ 463 intr_enabled = true; 464 465 /* Special case for NULL device (RTE_BBDEV_OP_NONE) */ 466 if (populate_active_devices() == 0 || 467 test_vector.op_type == RTE_BBDEV_OP_NONE) { 468 intr_enabled = false; 469 printf("No suitable devices found!\n"); 470 return TEST_SKIPPED; 471 } 472 473 return TEST_SUCCESS; 474 } 475 476 static void 477 testsuite_teardown(void) 478 { 479 uint8_t dev_id; 480 481 /* Unconfigure devices */ 482 RTE_BBDEV_FOREACH(dev_id) 483 rte_bbdev_close(dev_id); 484 485 /* Clear active devices structs. */ 486 memset(active_devs, 0, sizeof(active_devs)); 487 nb_active_devs = 0; 488 } 489 490 static int 491 ut_setup(void) 492 { 493 uint8_t i, dev_id; 494 495 for (i = 0; i < nb_active_devs; i++) { 496 dev_id = active_devs[i].dev_id; 497 /* reset bbdev stats */ 498 TEST_ASSERT_SUCCESS(rte_bbdev_stats_reset(dev_id), 499 "Failed to reset stats of bbdev %u", dev_id); 500 /* start the device */ 501 TEST_ASSERT_SUCCESS(rte_bbdev_start(dev_id), 502 "Failed to start bbdev %u", dev_id); 503 } 504 505 return TEST_SUCCESS; 506 } 507 508 static void 509 ut_teardown(void) 510 { 511 uint8_t i, dev_id; 512 struct rte_bbdev_stats stats; 513 514 for (i = 0; i < nb_active_devs; i++) { 515 dev_id = active_devs[i].dev_id; 516 /* read stats and print */ 517 rte_bbdev_stats_get(dev_id, &stats); 518 /* Stop the device */ 519 rte_bbdev_stop(dev_id); 520 } 521 } 522 523 static int 524 init_op_data_objs(struct rte_bbdev_op_data *bufs, 525 struct op_data_entries *ref_entries, 526 struct rte_mempool *mbuf_pool, const uint16_t n, 527 enum op_data_type op_type, uint16_t min_alignment) 528 { 529 int ret; 530 unsigned int i, j; 531 532 for (i = 0; i < n; ++i) { 533 char *data; 534 struct op_data_buf *seg = &ref_entries->segments[0]; 535 struct rte_mbuf *m_head = rte_pktmbuf_alloc(mbuf_pool); 536 TEST_ASSERT_NOT_NULL(m_head, 537 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)", 538 op_type, n * ref_entries->nb_segments, 539 mbuf_pool->size); 540 541 bufs[i].data = m_head; 542 bufs[i].offset = 0; 543 bufs[i].length = 0; 544 545 if (op_type == DATA_INPUT) { 546 data = rte_pktmbuf_append(m_head, seg->length); 547 TEST_ASSERT_NOT_NULL(data, 548 "Couldn't append %u bytes to mbuf from %d data type mbuf pool", 549 seg->length, op_type); 550 551 TEST_ASSERT(data == RTE_PTR_ALIGN(data, min_alignment), 552 "Data addr in mbuf (%p) is not aligned to device min alignment (%u)", 553 data, min_alignment); 554 rte_memcpy(data, seg->addr, seg->length); 555 bufs[i].length += seg->length; 556 557 558 for (j = 1; j < ref_entries->nb_segments; ++j) { 559 struct rte_mbuf *m_tail = 560 rte_pktmbuf_alloc(mbuf_pool); 561 TEST_ASSERT_NOT_NULL(m_tail, 562 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)", 563 op_type, 564 n * ref_entries->nb_segments, 565 mbuf_pool->size); 566 seg += 1; 567 568 data = rte_pktmbuf_append(m_tail, seg->length); 569 TEST_ASSERT_NOT_NULL(data, 570 "Couldn't append %u bytes to mbuf from %d data type mbuf pool", 571 seg->length, op_type); 572 573 TEST_ASSERT(data == RTE_PTR_ALIGN(data, 574 min_alignment), 575 "Data addr in mbuf (%p) is not aligned to device min alignment (%u)", 576 data, min_alignment); 577 rte_memcpy(data, seg->addr, seg->length); 578 bufs[i].length += seg->length; 579 580 ret = rte_pktmbuf_chain(m_head, m_tail); 581 TEST_ASSERT_SUCCESS(ret, 582 "Couldn't chain mbufs from %d data type mbuf pool", 583 op_type); 584 } 585 } 586 } 587 588 return 0; 589 } 590 591 static int 592 allocate_buffers_on_socket(struct rte_bbdev_op_data **buffers, const int len, 593 const int socket) 594 { 595 int i; 596 597 *buffers = rte_zmalloc_socket(NULL, len, 0, socket); 598 if (*buffers == NULL) { 599 printf("WARNING: Failed to allocate op_data on socket %d\n", 600 socket); 601 /* try to allocate memory on other detected sockets */ 602 for (i = 0; i < socket; i++) { 603 *buffers = rte_zmalloc_socket(NULL, len, 0, i); 604 if (*buffers != NULL) 605 break; 606 } 607 } 608 609 return (*buffers == NULL) ? TEST_FAILED : TEST_SUCCESS; 610 } 611 612 static int 613 fill_queue_buffers(struct test_op_params *op_params, 614 struct rte_mempool *in_mp, struct rte_mempool *hard_out_mp, 615 struct rte_mempool *soft_out_mp, uint16_t queue_id, 616 uint16_t min_alignment, const int socket_id) 617 { 618 int ret; 619 enum op_data_type type; 620 const uint16_t n = op_params->num_to_process; 621 622 struct rte_mempool *mbuf_pools[DATA_NUM_TYPES] = { 623 in_mp, 624 soft_out_mp, 625 hard_out_mp, 626 }; 627 628 struct rte_bbdev_op_data **queue_ops[DATA_NUM_TYPES] = { 629 &op_params->q_bufs[socket_id][queue_id].inputs, 630 &op_params->q_bufs[socket_id][queue_id].soft_outputs, 631 &op_params->q_bufs[socket_id][queue_id].hard_outputs, 632 }; 633 634 for (type = DATA_INPUT; type < DATA_NUM_TYPES; ++type) { 635 struct op_data_entries *ref_entries = 636 &test_vector.entries[type]; 637 if (ref_entries->nb_segments == 0) 638 continue; 639 640 ret = allocate_buffers_on_socket(queue_ops[type], 641 n * sizeof(struct rte_bbdev_op_data), 642 socket_id); 643 TEST_ASSERT_SUCCESS(ret, 644 "Couldn't allocate memory for rte_bbdev_op_data structs"); 645 646 ret = init_op_data_objs(*queue_ops[type], ref_entries, 647 mbuf_pools[type], n, type, min_alignment); 648 TEST_ASSERT_SUCCESS(ret, 649 "Couldn't init rte_bbdev_op_data structs"); 650 } 651 652 return 0; 653 } 654 655 static void 656 free_buffers(struct active_device *ad, struct test_op_params *op_params) 657 { 658 unsigned int i, j; 659 660 rte_mempool_free(ad->ops_mempool); 661 rte_mempool_free(ad->in_mbuf_pool); 662 rte_mempool_free(ad->hard_out_mbuf_pool); 663 rte_mempool_free(ad->soft_out_mbuf_pool); 664 665 for (i = 0; i < rte_lcore_count(); ++i) { 666 for (j = 0; j < RTE_MAX_NUMA_NODES; ++j) { 667 rte_free(op_params->q_bufs[j][i].inputs); 668 rte_free(op_params->q_bufs[j][i].hard_outputs); 669 rte_free(op_params->q_bufs[j][i].soft_outputs); 670 } 671 } 672 } 673 674 static void 675 copy_reference_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n, 676 unsigned int start_idx, 677 struct rte_bbdev_op_data *inputs, 678 struct rte_bbdev_op_data *hard_outputs, 679 struct rte_bbdev_op_data *soft_outputs, 680 struct rte_bbdev_dec_op *ref_op) 681 { 682 unsigned int i; 683 struct rte_bbdev_op_turbo_dec *turbo_dec = &ref_op->turbo_dec; 684 685 for (i = 0; i < n; ++i) { 686 if (turbo_dec->code_block_mode == 0) { 687 ops[i]->turbo_dec.tb_params.ea = 688 turbo_dec->tb_params.ea; 689 ops[i]->turbo_dec.tb_params.eb = 690 turbo_dec->tb_params.eb; 691 ops[i]->turbo_dec.tb_params.k_pos = 692 turbo_dec->tb_params.k_pos; 693 ops[i]->turbo_dec.tb_params.k_neg = 694 turbo_dec->tb_params.k_neg; 695 ops[i]->turbo_dec.tb_params.c = 696 turbo_dec->tb_params.c; 697 ops[i]->turbo_dec.tb_params.c_neg = 698 turbo_dec->tb_params.c_neg; 699 ops[i]->turbo_dec.tb_params.cab = 700 turbo_dec->tb_params.cab; 701 } else { 702 ops[i]->turbo_dec.cb_params.e = turbo_dec->cb_params.e; 703 ops[i]->turbo_dec.cb_params.k = turbo_dec->cb_params.k; 704 } 705 706 ops[i]->turbo_dec.ext_scale = turbo_dec->ext_scale; 707 ops[i]->turbo_dec.iter_max = turbo_dec->iter_max; 708 ops[i]->turbo_dec.iter_min = turbo_dec->iter_min; 709 ops[i]->turbo_dec.op_flags = turbo_dec->op_flags; 710 ops[i]->turbo_dec.rv_index = turbo_dec->rv_index; 711 ops[i]->turbo_dec.num_maps = turbo_dec->num_maps; 712 ops[i]->turbo_dec.code_block_mode = turbo_dec->code_block_mode; 713 714 ops[i]->turbo_dec.hard_output = hard_outputs[start_idx + i]; 715 ops[i]->turbo_dec.input = inputs[start_idx + i]; 716 if (soft_outputs != NULL) 717 ops[i]->turbo_dec.soft_output = 718 soft_outputs[start_idx + i]; 719 } 720 } 721 722 static void 723 copy_reference_enc_op(struct rte_bbdev_enc_op **ops, unsigned int n, 724 unsigned int start_idx, 725 struct rte_bbdev_op_data *inputs, 726 struct rte_bbdev_op_data *outputs, 727 struct rte_bbdev_enc_op *ref_op) 728 { 729 unsigned int i; 730 struct rte_bbdev_op_turbo_enc *turbo_enc = &ref_op->turbo_enc; 731 for (i = 0; i < n; ++i) { 732 if (turbo_enc->code_block_mode == 0) { 733 ops[i]->turbo_enc.tb_params.ea = 734 turbo_enc->tb_params.ea; 735 ops[i]->turbo_enc.tb_params.eb = 736 turbo_enc->tb_params.eb; 737 ops[i]->turbo_enc.tb_params.k_pos = 738 turbo_enc->tb_params.k_pos; 739 ops[i]->turbo_enc.tb_params.k_neg = 740 turbo_enc->tb_params.k_neg; 741 ops[i]->turbo_enc.tb_params.c = 742 turbo_enc->tb_params.c; 743 ops[i]->turbo_enc.tb_params.c_neg = 744 turbo_enc->tb_params.c_neg; 745 ops[i]->turbo_enc.tb_params.cab = 746 turbo_enc->tb_params.cab; 747 ops[i]->turbo_enc.tb_params.ncb_pos = 748 turbo_enc->tb_params.ncb_pos; 749 ops[i]->turbo_enc.tb_params.ncb_neg = 750 turbo_enc->tb_params.ncb_neg; 751 ops[i]->turbo_enc.tb_params.r = turbo_enc->tb_params.r; 752 } else { 753 ops[i]->turbo_enc.cb_params.e = turbo_enc->cb_params.e; 754 ops[i]->turbo_enc.cb_params.k = turbo_enc->cb_params.k; 755 ops[i]->turbo_enc.cb_params.ncb = 756 turbo_enc->cb_params.ncb; 757 } 758 ops[i]->turbo_enc.rv_index = turbo_enc->rv_index; 759 ops[i]->turbo_enc.op_flags = turbo_enc->op_flags; 760 ops[i]->turbo_enc.code_block_mode = turbo_enc->code_block_mode; 761 762 ops[i]->turbo_enc.output = outputs[start_idx + i]; 763 ops[i]->turbo_enc.input = inputs[start_idx + i]; 764 } 765 } 766 767 static int 768 check_dec_status_and_ordering(struct rte_bbdev_dec_op *op, 769 unsigned int order_idx, const int expected_status) 770 { 771 TEST_ASSERT(op->status == expected_status, 772 "op_status (%d) != expected_status (%d)", 773 op->status, expected_status); 774 775 TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data, 776 "Ordering error, expected %p, got %p", 777 (void *)(uintptr_t)order_idx, op->opaque_data); 778 779 return TEST_SUCCESS; 780 } 781 782 static int 783 check_enc_status_and_ordering(struct rte_bbdev_enc_op *op, 784 unsigned int order_idx, const int expected_status) 785 { 786 TEST_ASSERT(op->status == expected_status, 787 "op_status (%d) != expected_status (%d)", 788 op->status, expected_status); 789 790 TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data, 791 "Ordering error, expected %p, got %p", 792 (void *)(uintptr_t)order_idx, op->opaque_data); 793 794 return TEST_SUCCESS; 795 } 796 797 static inline int 798 validate_op_chain(struct rte_bbdev_op_data *op, 799 struct op_data_entries *orig_op) 800 { 801 uint8_t i; 802 struct rte_mbuf *m = op->data; 803 uint8_t nb_dst_segments = orig_op->nb_segments; 804 805 TEST_ASSERT(nb_dst_segments == m->nb_segs, 806 "Number of segments differ in original (%u) and filled (%u) op", 807 nb_dst_segments, m->nb_segs); 808 809 for (i = 0; i < nb_dst_segments; ++i) { 810 /* Apply offset to the first mbuf segment */ 811 uint16_t offset = (i == 0) ? op->offset : 0; 812 uint16_t data_len = m->data_len - offset; 813 814 TEST_ASSERT(orig_op->segments[i].length == data_len, 815 "Length of segment differ in original (%u) and filled (%u) op", 816 orig_op->segments[i].length, data_len); 817 TEST_ASSERT_BUFFERS_ARE_EQUAL(orig_op->segments[i].addr, 818 rte_pktmbuf_mtod_offset(m, uint32_t *, offset), 819 data_len, 820 "Output buffers (CB=%u) are not equal", i); 821 m = m->next; 822 } 823 824 return TEST_SUCCESS; 825 } 826 827 static int 828 validate_dec_buffers(struct rte_bbdev_dec_op *ref_op, struct test_buffers *bufs, 829 const uint16_t num_to_process) 830 { 831 int i; 832 833 struct op_data_entries *hard_data_orig = 834 &test_vector.entries[DATA_HARD_OUTPUT]; 835 struct op_data_entries *soft_data_orig = 836 &test_vector.entries[DATA_SOFT_OUTPUT]; 837 838 for (i = 0; i < num_to_process; i++) { 839 TEST_ASSERT_SUCCESS(validate_op_chain(&bufs->hard_outputs[i], 840 hard_data_orig), 841 "Hard output buffers are not equal"); 842 if (ref_op->turbo_dec.op_flags & 843 RTE_BBDEV_TURBO_SOFT_OUTPUT) 844 TEST_ASSERT_SUCCESS(validate_op_chain( 845 &bufs->soft_outputs[i], 846 soft_data_orig), 847 "Soft output buffers are not equal"); 848 } 849 850 return TEST_SUCCESS; 851 } 852 853 static int 854 validate_enc_buffers(struct test_buffers *bufs, const uint16_t num_to_process) 855 { 856 int i; 857 858 struct op_data_entries *hard_data_orig = 859 &test_vector.entries[DATA_HARD_OUTPUT]; 860 861 for (i = 0; i < num_to_process; i++) 862 TEST_ASSERT_SUCCESS(validate_op_chain(&bufs->hard_outputs[i], 863 hard_data_orig), ""); 864 865 return TEST_SUCCESS; 866 } 867 868 static int 869 validate_dec_op(struct rte_bbdev_dec_op **ops, const uint16_t n, 870 struct rte_bbdev_dec_op *ref_op, const int vector_mask) 871 { 872 unsigned int i; 873 int ret; 874 struct op_data_entries *hard_data_orig = 875 &test_vector.entries[DATA_HARD_OUTPUT]; 876 struct op_data_entries *soft_data_orig = 877 &test_vector.entries[DATA_SOFT_OUTPUT]; 878 struct rte_bbdev_op_turbo_dec *ops_td; 879 struct rte_bbdev_op_data *hard_output; 880 struct rte_bbdev_op_data *soft_output; 881 struct rte_bbdev_op_turbo_dec *ref_td = &ref_op->turbo_dec; 882 883 for (i = 0; i < n; ++i) { 884 ops_td = &ops[i]->turbo_dec; 885 hard_output = &ops_td->hard_output; 886 soft_output = &ops_td->soft_output; 887 888 if (vector_mask & TEST_BBDEV_VF_EXPECTED_ITER_COUNT) 889 TEST_ASSERT(ops_td->iter_count <= ref_td->iter_count, 890 "Returned iter_count (%d) > expected iter_count (%d)", 891 ops_td->iter_count, ref_td->iter_count); 892 ret = check_dec_status_and_ordering(ops[i], i, ref_op->status); 893 TEST_ASSERT_SUCCESS(ret, 894 "Checking status and ordering for decoder failed"); 895 896 TEST_ASSERT_SUCCESS(validate_op_chain(hard_output, 897 hard_data_orig), 898 "Hard output buffers (CB=%u) are not equal", 899 i); 900 901 if (ref_op->turbo_dec.op_flags & RTE_BBDEV_TURBO_SOFT_OUTPUT) 902 TEST_ASSERT_SUCCESS(validate_op_chain(soft_output, 903 soft_data_orig), 904 "Soft output buffers (CB=%u) are not equal", 905 i); 906 } 907 908 return TEST_SUCCESS; 909 } 910 911 static int 912 validate_enc_op(struct rte_bbdev_enc_op **ops, const uint16_t n, 913 struct rte_bbdev_enc_op *ref_op) 914 { 915 unsigned int i; 916 int ret; 917 struct op_data_entries *hard_data_orig = 918 &test_vector.entries[DATA_HARD_OUTPUT]; 919 920 for (i = 0; i < n; ++i) { 921 ret = check_enc_status_and_ordering(ops[i], i, ref_op->status); 922 TEST_ASSERT_SUCCESS(ret, 923 "Checking status and ordering for encoder failed"); 924 TEST_ASSERT_SUCCESS(validate_op_chain( 925 &ops[i]->turbo_enc.output, 926 hard_data_orig), 927 "Output buffers (CB=%u) are not equal", 928 i); 929 } 930 931 return TEST_SUCCESS; 932 } 933 934 static void 935 create_reference_dec_op(struct rte_bbdev_dec_op *op) 936 { 937 unsigned int i; 938 struct op_data_entries *entry; 939 940 op->turbo_dec = test_vector.turbo_dec; 941 entry = &test_vector.entries[DATA_INPUT]; 942 for (i = 0; i < entry->nb_segments; ++i) 943 op->turbo_dec.input.length += 944 entry->segments[i].length; 945 } 946 947 static void 948 create_reference_enc_op(struct rte_bbdev_enc_op *op) 949 { 950 unsigned int i; 951 struct op_data_entries *entry; 952 953 op->turbo_enc = test_vector.turbo_enc; 954 entry = &test_vector.entries[DATA_INPUT]; 955 for (i = 0; i < entry->nb_segments; ++i) 956 op->turbo_enc.input.length += 957 entry->segments[i].length; 958 } 959 960 static int 961 init_test_op_params(struct test_op_params *op_params, 962 enum rte_bbdev_op_type op_type, const int expected_status, 963 const int vector_mask, struct rte_mempool *ops_mp, 964 uint16_t burst_sz, uint16_t num_to_process, uint16_t num_lcores) 965 { 966 int ret = 0; 967 if (op_type == RTE_BBDEV_OP_TURBO_DEC) 968 ret = rte_bbdev_dec_op_alloc_bulk(ops_mp, 969 &op_params->ref_dec_op, 1); 970 else 971 ret = rte_bbdev_enc_op_alloc_bulk(ops_mp, 972 &op_params->ref_enc_op, 1); 973 974 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_op_alloc_bulk() failed"); 975 976 op_params->mp = ops_mp; 977 op_params->burst_sz = burst_sz; 978 op_params->num_to_process = num_to_process; 979 op_params->num_lcores = num_lcores; 980 op_params->vector_mask = vector_mask; 981 if (op_type == RTE_BBDEV_OP_TURBO_DEC) 982 op_params->ref_dec_op->status = expected_status; 983 else if (op_type == RTE_BBDEV_OP_TURBO_ENC) 984 op_params->ref_enc_op->status = expected_status; 985 986 return 0; 987 } 988 989 static int 990 run_test_case_on_device(test_case_function *test_case_func, uint8_t dev_id, 991 struct test_op_params *op_params) 992 { 993 int t_ret, f_ret, socket_id = SOCKET_ID_ANY; 994 unsigned int i; 995 struct active_device *ad; 996 unsigned int burst_sz = get_burst_sz(); 997 enum rte_bbdev_op_type op_type = test_vector.op_type; 998 999 ad = &active_devs[dev_id]; 1000 1001 /* Check if device supports op_type */ 1002 if (!is_avail_op(ad, test_vector.op_type)) 1003 return TEST_SUCCESS; 1004 1005 struct rte_bbdev_info info; 1006 rte_bbdev_info_get(ad->dev_id, &info); 1007 socket_id = GET_SOCKET(info.socket_id); 1008 1009 if (op_type == RTE_BBDEV_OP_NONE) 1010 op_type = RTE_BBDEV_OP_TURBO_ENC; 1011 f_ret = create_mempools(ad, socket_id, op_type, 1012 get_num_ops()); 1013 if (f_ret != TEST_SUCCESS) { 1014 printf("Couldn't create mempools"); 1015 goto fail; 1016 } 1017 1018 f_ret = init_test_op_params(op_params, test_vector.op_type, 1019 test_vector.expected_status, 1020 test_vector.mask, 1021 ad->ops_mempool, 1022 burst_sz, 1023 get_num_ops(), 1024 get_num_lcores()); 1025 if (f_ret != TEST_SUCCESS) { 1026 printf("Couldn't init test op params"); 1027 goto fail; 1028 } 1029 1030 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) 1031 create_reference_dec_op(op_params->ref_dec_op); 1032 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) 1033 create_reference_enc_op(op_params->ref_enc_op); 1034 1035 for (i = 0; i < ad->nb_queues; ++i) { 1036 f_ret = fill_queue_buffers(op_params, 1037 ad->in_mbuf_pool, 1038 ad->hard_out_mbuf_pool, 1039 ad->soft_out_mbuf_pool, 1040 ad->queue_ids[i], 1041 info.drv.min_alignment, 1042 socket_id); 1043 if (f_ret != TEST_SUCCESS) { 1044 printf("Couldn't init queue buffers"); 1045 goto fail; 1046 } 1047 } 1048 1049 /* Run test case function */ 1050 t_ret = test_case_func(ad, op_params); 1051 1052 /* Free active device resources and return */ 1053 free_buffers(ad, op_params); 1054 return t_ret; 1055 1056 fail: 1057 free_buffers(ad, op_params); 1058 return TEST_FAILED; 1059 } 1060 1061 /* Run given test function per active device per supported op type 1062 * per burst size. 1063 */ 1064 static int 1065 run_test_case(test_case_function *test_case_func) 1066 { 1067 int ret = 0; 1068 uint8_t dev; 1069 1070 /* Alloc op_params */ 1071 struct test_op_params *op_params = rte_zmalloc(NULL, 1072 sizeof(struct test_op_params), RTE_CACHE_LINE_SIZE); 1073 TEST_ASSERT_NOT_NULL(op_params, "Failed to alloc %zuB for op_params", 1074 RTE_ALIGN(sizeof(struct test_op_params), 1075 RTE_CACHE_LINE_SIZE)); 1076 1077 /* For each device run test case function */ 1078 for (dev = 0; dev < nb_active_devs; ++dev) 1079 ret |= run_test_case_on_device(test_case_func, dev, op_params); 1080 1081 rte_free(op_params); 1082 1083 return ret; 1084 } 1085 1086 static void 1087 dequeue_event_callback(uint16_t dev_id, 1088 enum rte_bbdev_event_type event, void *cb_arg, 1089 void *ret_param) 1090 { 1091 int ret; 1092 uint16_t i; 1093 uint64_t total_time; 1094 uint16_t deq, burst_sz, num_to_process; 1095 uint16_t queue_id = INVALID_QUEUE_ID; 1096 struct rte_bbdev_dec_op *dec_ops[MAX_BURST]; 1097 struct rte_bbdev_enc_op *enc_ops[MAX_BURST]; 1098 struct test_buffers *bufs; 1099 struct rte_bbdev_info info; 1100 1101 /* Input length in bytes, million operations per second, 1102 * million bits per second. 1103 */ 1104 double in_len; 1105 1106 struct thread_params *tp = cb_arg; 1107 1108 RTE_SET_USED(ret_param); 1109 queue_id = tp->queue_id; 1110 1111 /* Find matching thread params using queue_id */ 1112 for (i = 0; i < MAX_QUEUES; ++i, ++tp) 1113 if (tp->queue_id == queue_id) 1114 break; 1115 1116 if (i == MAX_QUEUES) { 1117 printf("%s: Queue_id from interrupt details was not found!\n", 1118 __func__); 1119 return; 1120 } 1121 1122 if (unlikely(event != RTE_BBDEV_EVENT_DEQUEUE)) { 1123 rte_atomic16_set(&tp->processing_status, TEST_FAILED); 1124 printf( 1125 "Dequeue interrupt handler called for incorrect event!\n"); 1126 return; 1127 } 1128 1129 burst_sz = tp->op_params->burst_sz; 1130 num_to_process = tp->op_params->num_to_process; 1131 1132 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) 1133 deq = rte_bbdev_dequeue_dec_ops(dev_id, queue_id, dec_ops, 1134 burst_sz); 1135 else 1136 deq = rte_bbdev_dequeue_enc_ops(dev_id, queue_id, enc_ops, 1137 burst_sz); 1138 1139 if (deq < burst_sz) { 1140 printf( 1141 "After receiving the interrupt all operations should be dequeued. Expected: %u, got: %u\n", 1142 burst_sz, deq); 1143 rte_atomic16_set(&tp->processing_status, TEST_FAILED); 1144 return; 1145 } 1146 1147 if (rte_atomic16_read(&tp->nb_dequeued) + deq < num_to_process) { 1148 rte_atomic16_add(&tp->nb_dequeued, deq); 1149 return; 1150 } 1151 1152 total_time = rte_rdtsc_precise() - tp->start_time; 1153 1154 rte_bbdev_info_get(dev_id, &info); 1155 1156 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 1157 1158 ret = TEST_SUCCESS; 1159 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) 1160 ret = validate_dec_buffers(tp->op_params->ref_dec_op, bufs, 1161 num_to_process); 1162 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) 1163 ret = validate_enc_buffers(bufs, num_to_process); 1164 1165 if (ret) { 1166 printf("Buffers validation failed\n"); 1167 rte_atomic16_set(&tp->processing_status, TEST_FAILED); 1168 } 1169 1170 switch (test_vector.op_type) { 1171 case RTE_BBDEV_OP_TURBO_DEC: 1172 in_len = tp->op_params->ref_dec_op->turbo_dec.input.length; 1173 break; 1174 case RTE_BBDEV_OP_TURBO_ENC: 1175 in_len = tp->op_params->ref_enc_op->turbo_enc.input.length; 1176 break; 1177 case RTE_BBDEV_OP_NONE: 1178 in_len = 0.0; 1179 break; 1180 default: 1181 printf("Unknown op type: %d\n", test_vector.op_type); 1182 rte_atomic16_set(&tp->processing_status, TEST_FAILED); 1183 return; 1184 } 1185 1186 tp->mops = ((double)num_to_process / 1000000.0) / 1187 ((double)total_time / (double)rte_get_tsc_hz()); 1188 tp->mbps = ((double)num_to_process * in_len * 8 / 1000000.0) / 1189 ((double)total_time / (double)rte_get_tsc_hz()); 1190 1191 rte_atomic16_add(&tp->nb_dequeued, deq); 1192 } 1193 1194 static int 1195 throughput_intr_lcore_dec(void *arg) 1196 { 1197 struct thread_params *tp = arg; 1198 unsigned int enqueued; 1199 struct rte_bbdev_dec_op *ops[MAX_BURST]; 1200 const uint16_t queue_id = tp->queue_id; 1201 const uint16_t burst_sz = tp->op_params->burst_sz; 1202 const uint16_t num_to_process = tp->op_params->num_to_process; 1203 struct test_buffers *bufs = NULL; 1204 unsigned int allocs_failed = 0; 1205 struct rte_bbdev_info info; 1206 int ret; 1207 1208 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 1209 "BURST_SIZE should be <= %u", MAX_BURST); 1210 1211 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id), 1212 "Failed to enable interrupts for dev: %u, queue_id: %u", 1213 tp->dev_id, queue_id); 1214 1215 rte_bbdev_info_get(tp->dev_id, &info); 1216 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 1217 1218 rte_atomic16_clear(&tp->processing_status); 1219 rte_atomic16_clear(&tp->nb_dequeued); 1220 1221 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT) 1222 rte_pause(); 1223 1224 tp->start_time = rte_rdtsc_precise(); 1225 for (enqueued = 0; enqueued < num_to_process;) { 1226 1227 uint16_t num_to_enq = burst_sz; 1228 1229 if (unlikely(num_to_process - enqueued < num_to_enq)) 1230 num_to_enq = num_to_process - enqueued; 1231 1232 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops, 1233 num_to_enq); 1234 if (ret != 0) { 1235 allocs_failed++; 1236 continue; 1237 } 1238 1239 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 1240 copy_reference_dec_op(ops, num_to_enq, enqueued, 1241 bufs->inputs, 1242 bufs->hard_outputs, 1243 bufs->soft_outputs, 1244 tp->op_params->ref_dec_op); 1245 1246 enqueued += rte_bbdev_enqueue_dec_ops(tp->dev_id, queue_id, ops, 1247 num_to_enq); 1248 1249 rte_bbdev_dec_op_free_bulk(ops, num_to_enq); 1250 } 1251 1252 if (allocs_failed > 0) 1253 printf("WARNING: op allocations failed: %u times\n", 1254 allocs_failed); 1255 1256 return TEST_SUCCESS; 1257 } 1258 1259 static int 1260 throughput_intr_lcore_enc(void *arg) 1261 { 1262 struct thread_params *tp = arg; 1263 unsigned int enqueued; 1264 struct rte_bbdev_enc_op *ops[MAX_BURST]; 1265 const uint16_t queue_id = tp->queue_id; 1266 const uint16_t burst_sz = tp->op_params->burst_sz; 1267 const uint16_t num_to_process = tp->op_params->num_to_process; 1268 struct test_buffers *bufs = NULL; 1269 unsigned int allocs_failed = 0; 1270 struct rte_bbdev_info info; 1271 int ret; 1272 1273 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 1274 "BURST_SIZE should be <= %u", MAX_BURST); 1275 1276 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id), 1277 "Failed to enable interrupts for dev: %u, queue_id: %u", 1278 tp->dev_id, queue_id); 1279 1280 rte_bbdev_info_get(tp->dev_id, &info); 1281 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 1282 1283 rte_atomic16_clear(&tp->processing_status); 1284 rte_atomic16_clear(&tp->nb_dequeued); 1285 1286 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT) 1287 rte_pause(); 1288 1289 tp->start_time = rte_rdtsc_precise(); 1290 for (enqueued = 0; enqueued < num_to_process;) { 1291 1292 uint16_t num_to_enq = burst_sz; 1293 1294 if (unlikely(num_to_process - enqueued < num_to_enq)) 1295 num_to_enq = num_to_process - enqueued; 1296 1297 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops, 1298 num_to_enq); 1299 if (ret != 0) { 1300 allocs_failed++; 1301 continue; 1302 } 1303 1304 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 1305 copy_reference_enc_op(ops, num_to_enq, enqueued, 1306 bufs->inputs, 1307 bufs->hard_outputs, 1308 tp->op_params->ref_enc_op); 1309 1310 enqueued += rte_bbdev_enqueue_enc_ops(tp->dev_id, queue_id, ops, 1311 num_to_enq); 1312 1313 rte_bbdev_enc_op_free_bulk(ops, num_to_enq); 1314 } 1315 1316 if (allocs_failed > 0) 1317 printf("WARNING: op allocations failed: %u times\n", 1318 allocs_failed); 1319 1320 return TEST_SUCCESS; 1321 } 1322 1323 static int 1324 throughput_pmd_lcore_dec(void *arg) 1325 { 1326 struct thread_params *tp = arg; 1327 unsigned int enqueued, dequeued; 1328 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 1329 uint64_t total_time, start_time; 1330 const uint16_t queue_id = tp->queue_id; 1331 const uint16_t burst_sz = tp->op_params->burst_sz; 1332 const uint16_t num_to_process = tp->op_params->num_to_process; 1333 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 1334 struct test_buffers *bufs = NULL; 1335 unsigned int allocs_failed = 0; 1336 int ret; 1337 struct rte_bbdev_info info; 1338 1339 /* Input length in bytes, million operations per second, million bits 1340 * per second. 1341 */ 1342 double in_len; 1343 1344 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 1345 "BURST_SIZE should be <= %u", MAX_BURST); 1346 1347 rte_bbdev_info_get(tp->dev_id, &info); 1348 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 1349 1350 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT) 1351 rte_pause(); 1352 1353 start_time = rte_rdtsc_precise(); 1354 for (enqueued = 0, dequeued = 0; dequeued < num_to_process;) { 1355 uint16_t deq; 1356 1357 if (likely(enqueued < num_to_process)) { 1358 1359 uint16_t num_to_enq = burst_sz; 1360 1361 if (unlikely(num_to_process - enqueued < num_to_enq)) 1362 num_to_enq = num_to_process - enqueued; 1363 1364 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, 1365 ops_enq, num_to_enq); 1366 if (ret != 0) { 1367 allocs_failed++; 1368 goto do_dequeue; 1369 } 1370 1371 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 1372 copy_reference_dec_op(ops_enq, num_to_enq, 1373 enqueued, 1374 bufs->inputs, 1375 bufs->hard_outputs, 1376 bufs->soft_outputs, 1377 ref_op); 1378 1379 enqueued += rte_bbdev_enqueue_dec_ops(tp->dev_id, 1380 queue_id, ops_enq, num_to_enq); 1381 } 1382 do_dequeue: 1383 deq = rte_bbdev_dequeue_dec_ops(tp->dev_id, queue_id, ops_deq, 1384 burst_sz); 1385 dequeued += deq; 1386 rte_bbdev_dec_op_free_bulk(ops_enq, deq); 1387 } 1388 total_time = rte_rdtsc_precise() - start_time; 1389 1390 if (allocs_failed > 0) 1391 printf("WARNING: op allocations failed: %u times\n", 1392 allocs_failed); 1393 1394 TEST_ASSERT(enqueued == dequeued, "enqueued (%u) != dequeued (%u)", 1395 enqueued, dequeued); 1396 1397 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 1398 ret = validate_dec_buffers(ref_op, bufs, num_to_process); 1399 TEST_ASSERT_SUCCESS(ret, "Buffers validation failed"); 1400 } 1401 1402 in_len = ref_op->turbo_dec.input.length; 1403 tp->mops = ((double)num_to_process / 1000000.0) / 1404 ((double)total_time / (double)rte_get_tsc_hz()); 1405 tp->mbps = ((double)num_to_process * in_len * 8 / 1000000.0) / 1406 ((double)total_time / (double)rte_get_tsc_hz()); 1407 1408 return TEST_SUCCESS; 1409 } 1410 1411 static int 1412 throughput_pmd_lcore_enc(void *arg) 1413 { 1414 struct thread_params *tp = arg; 1415 unsigned int enqueued, dequeued; 1416 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 1417 uint64_t total_time, start_time; 1418 const uint16_t queue_id = tp->queue_id; 1419 const uint16_t burst_sz = tp->op_params->burst_sz; 1420 const uint16_t num_to_process = tp->op_params->num_to_process; 1421 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op; 1422 struct test_buffers *bufs = NULL; 1423 unsigned int allocs_failed = 0; 1424 int ret; 1425 struct rte_bbdev_info info; 1426 1427 /* Input length in bytes, million operations per second, million bits 1428 * per second. 1429 */ 1430 double in_len; 1431 1432 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 1433 "BURST_SIZE should be <= %u", MAX_BURST); 1434 1435 rte_bbdev_info_get(tp->dev_id, &info); 1436 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 1437 1438 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT) 1439 rte_pause(); 1440 1441 start_time = rte_rdtsc_precise(); 1442 for (enqueued = 0, dequeued = 0; dequeued < num_to_process;) { 1443 uint16_t deq; 1444 1445 if (likely(enqueued < num_to_process)) { 1446 1447 uint16_t num_to_enq = burst_sz; 1448 1449 if (unlikely(num_to_process - enqueued < num_to_enq)) 1450 num_to_enq = num_to_process - enqueued; 1451 1452 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, 1453 ops_enq, num_to_enq); 1454 if (ret != 0) { 1455 allocs_failed++; 1456 goto do_dequeue; 1457 } 1458 1459 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 1460 copy_reference_enc_op(ops_enq, num_to_enq, 1461 enqueued, 1462 bufs->inputs, 1463 bufs->hard_outputs, 1464 ref_op); 1465 1466 enqueued += rte_bbdev_enqueue_enc_ops(tp->dev_id, 1467 queue_id, ops_enq, num_to_enq); 1468 } 1469 do_dequeue: 1470 deq = rte_bbdev_dequeue_enc_ops(tp->dev_id, queue_id, ops_deq, 1471 burst_sz); 1472 dequeued += deq; 1473 rte_bbdev_enc_op_free_bulk(ops_enq, deq); 1474 } 1475 total_time = rte_rdtsc_precise() - start_time; 1476 1477 if (allocs_failed > 0) 1478 printf("WARNING: op allocations failed: %u times\n", 1479 allocs_failed); 1480 1481 TEST_ASSERT(enqueued == dequeued, "enqueued (%u) != dequeued (%u)", 1482 enqueued, dequeued); 1483 1484 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 1485 ret = validate_enc_buffers(bufs, num_to_process); 1486 TEST_ASSERT_SUCCESS(ret, "Buffers validation failed"); 1487 } 1488 1489 in_len = ref_op->turbo_enc.input.length; 1490 1491 tp->mops = ((double)num_to_process / 1000000.0) / 1492 ((double)total_time / (double)rte_get_tsc_hz()); 1493 tp->mbps = ((double)num_to_process * in_len * 8 / 1000000.0) / 1494 ((double)total_time / (double)rte_get_tsc_hz()); 1495 1496 return TEST_SUCCESS; 1497 } 1498 static void 1499 print_throughput(struct thread_params *t_params, unsigned int used_cores) 1500 { 1501 unsigned int lcore_id, iter = 0; 1502 double total_mops = 0, total_mbps = 0; 1503 1504 RTE_LCORE_FOREACH(lcore_id) { 1505 if (iter++ >= used_cores) 1506 break; 1507 printf("\tlcore_id: %u, throughput: %.8lg MOPS, %.8lg Mbps\n", 1508 lcore_id, t_params[lcore_id].mops, t_params[lcore_id].mbps); 1509 total_mops += t_params[lcore_id].mops; 1510 total_mbps += t_params[lcore_id].mbps; 1511 } 1512 printf( 1513 "\n\tTotal stats for %u cores: throughput: %.8lg MOPS, %.8lg Mbps\n", 1514 used_cores, total_mops, total_mbps); 1515 } 1516 1517 /* 1518 * Test function that determines how long an enqueue + dequeue of a burst 1519 * takes on available lcores. 1520 */ 1521 static int 1522 throughput_test(struct active_device *ad, 1523 struct test_op_params *op_params) 1524 { 1525 int ret; 1526 unsigned int lcore_id, used_cores = 0; 1527 struct thread_params t_params[MAX_QUEUES]; 1528 struct rte_bbdev_info info; 1529 lcore_function_t *throughput_function; 1530 struct thread_params *tp; 1531 uint16_t num_lcores; 1532 const char *op_type_str; 1533 1534 rte_bbdev_info_get(ad->dev_id, &info); 1535 1536 op_type_str = rte_bbdev_op_type_str(test_vector.op_type); 1537 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", 1538 test_vector.op_type); 1539 1540 printf( 1541 "Throughput test: dev: %s, nb_queues: %u, burst size: %u, num ops: %u, num_lcores: %u, op type: %s, int mode: %s, GHz: %lg\n", 1542 info.dev_name, ad->nb_queues, op_params->burst_sz, 1543 op_params->num_to_process, op_params->num_lcores, 1544 op_type_str, 1545 intr_enabled ? "Interrupt mode" : "PMD mode", 1546 (double)rte_get_tsc_hz() / 1000000000.0); 1547 1548 /* Set number of lcores */ 1549 num_lcores = (ad->nb_queues < (op_params->num_lcores)) 1550 ? ad->nb_queues 1551 : op_params->num_lcores; 1552 1553 if (intr_enabled) { 1554 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) 1555 throughput_function = throughput_intr_lcore_dec; 1556 else 1557 throughput_function = throughput_intr_lcore_enc; 1558 1559 /* Dequeue interrupt callback registration */ 1560 rte_bbdev_callback_register(ad->dev_id, RTE_BBDEV_EVENT_DEQUEUE, 1561 dequeue_event_callback, 1562 &t_params); 1563 } else { 1564 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) 1565 throughput_function = throughput_pmd_lcore_dec; 1566 else 1567 throughput_function = throughput_pmd_lcore_enc; 1568 } 1569 1570 rte_atomic16_set(&op_params->sync, SYNC_WAIT); 1571 1572 t_params[rte_lcore_id()].dev_id = ad->dev_id; 1573 t_params[rte_lcore_id()].op_params = op_params; 1574 t_params[rte_lcore_id()].queue_id = 1575 ad->queue_ids[used_cores++]; 1576 1577 RTE_LCORE_FOREACH_SLAVE(lcore_id) { 1578 if (used_cores >= num_lcores) 1579 break; 1580 1581 t_params[lcore_id].dev_id = ad->dev_id; 1582 t_params[lcore_id].op_params = op_params; 1583 t_params[lcore_id].queue_id = ad->queue_ids[used_cores++]; 1584 1585 rte_eal_remote_launch(throughput_function, &t_params[lcore_id], 1586 lcore_id); 1587 } 1588 1589 rte_atomic16_set(&op_params->sync, SYNC_START); 1590 ret = throughput_function(&t_params[rte_lcore_id()]); 1591 1592 /* Master core is always used */ 1593 used_cores = 1; 1594 RTE_LCORE_FOREACH_SLAVE(lcore_id) { 1595 if (used_cores++ >= num_lcores) 1596 break; 1597 1598 ret |= rte_eal_wait_lcore(lcore_id); 1599 } 1600 1601 /* Return if test failed */ 1602 if (ret) 1603 return ret; 1604 1605 /* Print throughput if interrupts are disabled and test passed */ 1606 if (!intr_enabled) { 1607 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 1608 print_throughput(t_params, num_lcores); 1609 return ret; 1610 } 1611 1612 /* In interrupt TC we need to wait for the interrupt callback to deqeue 1613 * all pending operations. Skip waiting for queues which reported an 1614 * error using processing_status variable. 1615 * Wait for master lcore operations. 1616 */ 1617 tp = &t_params[rte_lcore_id()]; 1618 while ((rte_atomic16_read(&tp->nb_dequeued) < 1619 op_params->num_to_process) && 1620 (rte_atomic16_read(&tp->processing_status) != 1621 TEST_FAILED)) 1622 rte_pause(); 1623 1624 ret |= rte_atomic16_read(&tp->processing_status); 1625 1626 /* Wait for slave lcores operations */ 1627 used_cores = 1; 1628 RTE_LCORE_FOREACH_SLAVE(lcore_id) { 1629 tp = &t_params[lcore_id]; 1630 if (used_cores++ >= num_lcores) 1631 break; 1632 1633 while ((rte_atomic16_read(&tp->nb_dequeued) < 1634 op_params->num_to_process) && 1635 (rte_atomic16_read(&tp->processing_status) != 1636 TEST_FAILED)) 1637 rte_pause(); 1638 1639 ret |= rte_atomic16_read(&tp->processing_status); 1640 } 1641 1642 /* Print throughput if test passed */ 1643 if (!ret && test_vector.op_type != RTE_BBDEV_OP_NONE) 1644 print_throughput(t_params, num_lcores); 1645 1646 return ret; 1647 } 1648 1649 static int 1650 operation_latency_test_dec(struct rte_mempool *mempool, 1651 struct test_buffers *bufs, struct rte_bbdev_dec_op *ref_op, 1652 int vector_mask, uint16_t dev_id, uint16_t queue_id, 1653 const uint16_t num_to_process, uint16_t burst_sz, 1654 uint64_t *total_time) 1655 { 1656 int ret = TEST_SUCCESS; 1657 uint16_t i, j, dequeued; 1658 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 1659 uint64_t start_time = 0; 1660 1661 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 1662 uint16_t enq = 0, deq = 0; 1663 bool first_time = true; 1664 1665 if (unlikely(num_to_process - dequeued < burst_sz)) 1666 burst_sz = num_to_process - dequeued; 1667 1668 rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz); 1669 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 1670 copy_reference_dec_op(ops_enq, burst_sz, dequeued, 1671 bufs->inputs, 1672 bufs->hard_outputs, 1673 bufs->soft_outputs, 1674 ref_op); 1675 1676 /* Set counter to validate the ordering */ 1677 for (j = 0; j < burst_sz; ++j) 1678 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 1679 1680 start_time = rte_rdtsc_precise(); 1681 1682 enq = rte_bbdev_enqueue_dec_ops(dev_id, queue_id, &ops_enq[enq], 1683 burst_sz); 1684 TEST_ASSERT(enq == burst_sz, 1685 "Error enqueueing burst, expected %u, got %u", 1686 burst_sz, enq); 1687 1688 /* Dequeue */ 1689 do { 1690 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id, 1691 &ops_deq[deq], burst_sz - deq); 1692 if (likely(first_time && (deq > 0))) { 1693 *total_time += rte_rdtsc_precise() - start_time; 1694 first_time = false; 1695 } 1696 } while (unlikely(burst_sz != deq)); 1697 1698 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 1699 ret = validate_dec_op(ops_deq, burst_sz, ref_op, 1700 vector_mask); 1701 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 1702 } 1703 1704 rte_bbdev_dec_op_free_bulk(ops_enq, deq); 1705 dequeued += deq; 1706 } 1707 1708 return i; 1709 } 1710 1711 static int 1712 operation_latency_test_enc(struct rte_mempool *mempool, 1713 struct test_buffers *bufs, struct rte_bbdev_enc_op *ref_op, 1714 uint16_t dev_id, uint16_t queue_id, 1715 const uint16_t num_to_process, uint16_t burst_sz, 1716 uint64_t *total_time) 1717 { 1718 int ret = TEST_SUCCESS; 1719 uint16_t i, j, dequeued; 1720 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 1721 uint64_t start_time = 0; 1722 1723 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 1724 uint16_t enq = 0, deq = 0; 1725 bool first_time = true; 1726 1727 if (unlikely(num_to_process - dequeued < burst_sz)) 1728 burst_sz = num_to_process - dequeued; 1729 1730 rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz); 1731 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 1732 copy_reference_enc_op(ops_enq, burst_sz, dequeued, 1733 bufs->inputs, 1734 bufs->hard_outputs, 1735 ref_op); 1736 1737 /* Set counter to validate the ordering */ 1738 for (j = 0; j < burst_sz; ++j) 1739 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 1740 1741 start_time = rte_rdtsc_precise(); 1742 1743 enq = rte_bbdev_enqueue_enc_ops(dev_id, queue_id, &ops_enq[enq], 1744 burst_sz); 1745 TEST_ASSERT(enq == burst_sz, 1746 "Error enqueueing burst, expected %u, got %u", 1747 burst_sz, enq); 1748 1749 /* Dequeue */ 1750 do { 1751 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id, 1752 &ops_deq[deq], burst_sz - deq); 1753 if (likely(first_time && (deq > 0))) { 1754 *total_time += rte_rdtsc_precise() - start_time; 1755 first_time = false; 1756 } 1757 } while (unlikely(burst_sz != deq)); 1758 1759 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 1760 ret = validate_enc_op(ops_deq, burst_sz, ref_op); 1761 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 1762 } 1763 1764 rte_bbdev_enc_op_free_bulk(ops_enq, deq); 1765 dequeued += deq; 1766 } 1767 1768 return i; 1769 } 1770 1771 static int 1772 operation_latency_test(struct active_device *ad, 1773 struct test_op_params *op_params) 1774 { 1775 int iter; 1776 uint16_t burst_sz = op_params->burst_sz; 1777 const uint16_t num_to_process = op_params->num_to_process; 1778 const enum rte_bbdev_op_type op_type = test_vector.op_type; 1779 const uint16_t queue_id = ad->queue_ids[0]; 1780 struct test_buffers *bufs = NULL; 1781 struct rte_bbdev_info info; 1782 uint64_t total_time = 0; 1783 const char *op_type_str; 1784 1785 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 1786 "BURST_SIZE should be <= %u", MAX_BURST); 1787 1788 rte_bbdev_info_get(ad->dev_id, &info); 1789 bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 1790 1791 op_type_str = rte_bbdev_op_type_str(op_type); 1792 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type); 1793 1794 printf( 1795 "Validation/Latency test: dev: %s, burst size: %u, num ops: %u, op type: %s\n", 1796 info.dev_name, burst_sz, num_to_process, op_type_str); 1797 1798 if (op_type == RTE_BBDEV_OP_TURBO_DEC) 1799 iter = operation_latency_test_dec(op_params->mp, bufs, 1800 op_params->ref_dec_op, op_params->vector_mask, 1801 ad->dev_id, queue_id, num_to_process, 1802 burst_sz, &total_time); 1803 else 1804 iter = operation_latency_test_enc(op_params->mp, bufs, 1805 op_params->ref_enc_op, ad->dev_id, queue_id, 1806 num_to_process, burst_sz, &total_time); 1807 1808 if (iter < 0) 1809 return TEST_FAILED; 1810 1811 printf("\toperation avg. latency: %lg cycles, %lg us\n", 1812 (double)total_time / (double)iter, 1813 (double)(total_time * 1000000) / (double)iter / 1814 (double)rte_get_tsc_hz()); 1815 1816 return TEST_SUCCESS; 1817 } 1818 1819 static int 1820 offload_latency_test_dec(struct rte_mempool *mempool, struct test_buffers *bufs, 1821 struct rte_bbdev_dec_op *ref_op, uint16_t dev_id, 1822 uint16_t queue_id, const uint16_t num_to_process, 1823 uint16_t burst_sz, uint64_t *enq_total_time, 1824 uint64_t *deq_total_time) 1825 { 1826 int i, dequeued; 1827 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 1828 uint64_t enq_start_time, deq_start_time; 1829 1830 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 1831 uint16_t enq = 0, deq = 0; 1832 1833 if (unlikely(num_to_process - dequeued < burst_sz)) 1834 burst_sz = num_to_process - dequeued; 1835 1836 rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz); 1837 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 1838 copy_reference_dec_op(ops_enq, burst_sz, dequeued, 1839 bufs->inputs, 1840 bufs->hard_outputs, 1841 bufs->soft_outputs, 1842 ref_op); 1843 1844 /* Start time measurment for enqueue function offload latency */ 1845 enq_start_time = rte_rdtsc(); 1846 do { 1847 enq += rte_bbdev_enqueue_dec_ops(dev_id, queue_id, 1848 &ops_enq[enq], burst_sz - enq); 1849 } while (unlikely(burst_sz != enq)); 1850 *enq_total_time += rte_rdtsc() - enq_start_time; 1851 1852 /* ensure enqueue has been completed */ 1853 rte_delay_ms(10); 1854 1855 /* Start time measurment for dequeue function offload latency */ 1856 deq_start_time = rte_rdtsc(); 1857 do { 1858 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id, 1859 &ops_deq[deq], burst_sz - deq); 1860 } while (unlikely(burst_sz != deq)); 1861 *deq_total_time += rte_rdtsc() - deq_start_time; 1862 1863 rte_bbdev_dec_op_free_bulk(ops_enq, deq); 1864 dequeued += deq; 1865 } 1866 1867 return i; 1868 } 1869 1870 static int 1871 offload_latency_test_enc(struct rte_mempool *mempool, struct test_buffers *bufs, 1872 struct rte_bbdev_enc_op *ref_op, uint16_t dev_id, 1873 uint16_t queue_id, const uint16_t num_to_process, 1874 uint16_t burst_sz, uint64_t *enq_total_time, 1875 uint64_t *deq_total_time) 1876 { 1877 int i, dequeued; 1878 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 1879 uint64_t enq_start_time, deq_start_time; 1880 1881 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 1882 uint16_t enq = 0, deq = 0; 1883 1884 if (unlikely(num_to_process - dequeued < burst_sz)) 1885 burst_sz = num_to_process - dequeued; 1886 1887 rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz); 1888 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 1889 copy_reference_enc_op(ops_enq, burst_sz, dequeued, 1890 bufs->inputs, 1891 bufs->hard_outputs, 1892 ref_op); 1893 1894 /* Start time measurment for enqueue function offload latency */ 1895 enq_start_time = rte_rdtsc(); 1896 do { 1897 enq += rte_bbdev_enqueue_enc_ops(dev_id, queue_id, 1898 &ops_enq[enq], burst_sz - enq); 1899 } while (unlikely(burst_sz != enq)); 1900 *enq_total_time += rte_rdtsc() - enq_start_time; 1901 1902 /* ensure enqueue has been completed */ 1903 rte_delay_ms(10); 1904 1905 /* Start time measurment for dequeue function offload latency */ 1906 deq_start_time = rte_rdtsc(); 1907 do { 1908 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id, 1909 &ops_deq[deq], burst_sz - deq); 1910 } while (unlikely(burst_sz != deq)); 1911 *deq_total_time += rte_rdtsc() - deq_start_time; 1912 1913 rte_bbdev_enc_op_free_bulk(ops_enq, deq); 1914 dequeued += deq; 1915 } 1916 1917 return i; 1918 } 1919 1920 static int 1921 offload_latency_test(struct active_device *ad, 1922 struct test_op_params *op_params) 1923 { 1924 int iter; 1925 uint64_t enq_total_time = 0, deq_total_time = 0; 1926 uint16_t burst_sz = op_params->burst_sz; 1927 const uint16_t num_to_process = op_params->num_to_process; 1928 const enum rte_bbdev_op_type op_type = test_vector.op_type; 1929 const uint16_t queue_id = ad->queue_ids[0]; 1930 struct test_buffers *bufs = NULL; 1931 struct rte_bbdev_info info; 1932 const char *op_type_str; 1933 1934 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 1935 "BURST_SIZE should be <= %u", MAX_BURST); 1936 1937 rte_bbdev_info_get(ad->dev_id, &info); 1938 bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 1939 1940 op_type_str = rte_bbdev_op_type_str(op_type); 1941 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type); 1942 1943 printf( 1944 "Offload latency test: dev: %s, burst size: %u, num ops: %u, op type: %s\n", 1945 info.dev_name, burst_sz, num_to_process, op_type_str); 1946 1947 if (op_type == RTE_BBDEV_OP_TURBO_DEC) 1948 iter = offload_latency_test_dec(op_params->mp, bufs, 1949 op_params->ref_dec_op, ad->dev_id, queue_id, 1950 num_to_process, burst_sz, &enq_total_time, 1951 &deq_total_time); 1952 else 1953 iter = offload_latency_test_enc(op_params->mp, bufs, 1954 op_params->ref_enc_op, ad->dev_id, queue_id, 1955 num_to_process, burst_sz, &enq_total_time, 1956 &deq_total_time); 1957 1958 if (iter < 0) 1959 return TEST_FAILED; 1960 1961 printf("\tenq offload avg. latency: %lg cycles, %lg us\n", 1962 (double)enq_total_time / (double)iter, 1963 (double)(enq_total_time * 1000000) / (double)iter / 1964 (double)rte_get_tsc_hz()); 1965 1966 printf("\tdeq offload avg. latency: %lg cycles, %lg us\n", 1967 (double)deq_total_time / (double)iter, 1968 (double)(deq_total_time * 1000000) / (double)iter / 1969 (double)rte_get_tsc_hz()); 1970 1971 return TEST_SUCCESS; 1972 } 1973 1974 static int 1975 offload_latency_empty_q_test_dec(uint16_t dev_id, uint16_t queue_id, 1976 const uint16_t num_to_process, uint16_t burst_sz, 1977 uint64_t *deq_total_time) 1978 { 1979 int i, deq_total; 1980 struct rte_bbdev_dec_op *ops[MAX_BURST]; 1981 uint64_t deq_start_time; 1982 1983 /* Test deq offload latency from an empty queue */ 1984 deq_start_time = rte_rdtsc_precise(); 1985 for (i = 0, deq_total = 0; deq_total < num_to_process; 1986 ++i, deq_total += burst_sz) { 1987 if (unlikely(num_to_process - deq_total < burst_sz)) 1988 burst_sz = num_to_process - deq_total; 1989 rte_bbdev_dequeue_dec_ops(dev_id, queue_id, ops, burst_sz); 1990 } 1991 *deq_total_time = rte_rdtsc_precise() - deq_start_time; 1992 1993 return i; 1994 } 1995 1996 static int 1997 offload_latency_empty_q_test_enc(uint16_t dev_id, uint16_t queue_id, 1998 const uint16_t num_to_process, uint16_t burst_sz, 1999 uint64_t *deq_total_time) 2000 { 2001 int i, deq_total; 2002 struct rte_bbdev_enc_op *ops[MAX_BURST]; 2003 uint64_t deq_start_time; 2004 2005 /* Test deq offload latency from an empty queue */ 2006 deq_start_time = rte_rdtsc_precise(); 2007 for (i = 0, deq_total = 0; deq_total < num_to_process; 2008 ++i, deq_total += burst_sz) { 2009 if (unlikely(num_to_process - deq_total < burst_sz)) 2010 burst_sz = num_to_process - deq_total; 2011 rte_bbdev_dequeue_enc_ops(dev_id, queue_id, ops, burst_sz); 2012 } 2013 *deq_total_time = rte_rdtsc_precise() - deq_start_time; 2014 2015 return i; 2016 } 2017 2018 static int 2019 offload_latency_empty_q_test(struct active_device *ad, 2020 struct test_op_params *op_params) 2021 { 2022 int iter; 2023 uint64_t deq_total_time = 0; 2024 uint16_t burst_sz = op_params->burst_sz; 2025 const uint16_t num_to_process = op_params->num_to_process; 2026 const enum rte_bbdev_op_type op_type = test_vector.op_type; 2027 const uint16_t queue_id = ad->queue_ids[0]; 2028 struct rte_bbdev_info info; 2029 const char *op_type_str; 2030 2031 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 2032 "BURST_SIZE should be <= %u", MAX_BURST); 2033 2034 rte_bbdev_info_get(ad->dev_id, &info); 2035 2036 op_type_str = rte_bbdev_op_type_str(op_type); 2037 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type); 2038 2039 printf( 2040 "Offload latency empty dequeue test: dev: %s, burst size: %u, num ops: %u, op type: %s\n", 2041 info.dev_name, burst_sz, num_to_process, op_type_str); 2042 2043 if (op_type == RTE_BBDEV_OP_TURBO_DEC) 2044 iter = offload_latency_empty_q_test_dec(ad->dev_id, queue_id, 2045 num_to_process, burst_sz, &deq_total_time); 2046 else 2047 iter = offload_latency_empty_q_test_enc(ad->dev_id, queue_id, 2048 num_to_process, burst_sz, &deq_total_time); 2049 2050 if (iter < 0) 2051 return TEST_FAILED; 2052 2053 printf("\tempty deq offload avg. latency: %lg cycles, %lg us\n", 2054 (double)deq_total_time / (double)iter, 2055 (double)(deq_total_time * 1000000) / (double)iter / 2056 (double)rte_get_tsc_hz()); 2057 2058 return TEST_SUCCESS; 2059 } 2060 2061 static int 2062 throughput_tc(void) 2063 { 2064 return run_test_case(throughput_test); 2065 } 2066 2067 static int 2068 offload_latency_tc(void) 2069 { 2070 return run_test_case(offload_latency_test); 2071 } 2072 2073 static int 2074 offload_latency_empty_q_tc(void) 2075 { 2076 return run_test_case(offload_latency_empty_q_test); 2077 } 2078 2079 static int 2080 operation_latency_tc(void) 2081 { 2082 return run_test_case(operation_latency_test); 2083 } 2084 2085 static int 2086 interrupt_tc(void) 2087 { 2088 return run_test_case(throughput_test); 2089 } 2090 2091 static struct unit_test_suite bbdev_throughput_testsuite = { 2092 .suite_name = "BBdev Throughput Tests", 2093 .setup = testsuite_setup, 2094 .teardown = testsuite_teardown, 2095 .unit_test_cases = { 2096 TEST_CASE_ST(ut_setup, ut_teardown, throughput_tc), 2097 TEST_CASES_END() /**< NULL terminate unit test array */ 2098 } 2099 }; 2100 2101 static struct unit_test_suite bbdev_validation_testsuite = { 2102 .suite_name = "BBdev Validation Tests", 2103 .setup = testsuite_setup, 2104 .teardown = testsuite_teardown, 2105 .unit_test_cases = { 2106 TEST_CASE_ST(ut_setup, ut_teardown, operation_latency_tc), 2107 TEST_CASES_END() /**< NULL terminate unit test array */ 2108 } 2109 }; 2110 2111 static struct unit_test_suite bbdev_latency_testsuite = { 2112 .suite_name = "BBdev Latency Tests", 2113 .setup = testsuite_setup, 2114 .teardown = testsuite_teardown, 2115 .unit_test_cases = { 2116 TEST_CASE_ST(ut_setup, ut_teardown, offload_latency_tc), 2117 TEST_CASE_ST(ut_setup, ut_teardown, offload_latency_empty_q_tc), 2118 TEST_CASE_ST(ut_setup, ut_teardown, operation_latency_tc), 2119 TEST_CASES_END() /**< NULL terminate unit test array */ 2120 } 2121 }; 2122 2123 static struct unit_test_suite bbdev_interrupt_testsuite = { 2124 .suite_name = "BBdev Interrupt Tests", 2125 .setup = interrupt_testsuite_setup, 2126 .teardown = testsuite_teardown, 2127 .unit_test_cases = { 2128 TEST_CASE_ST(ut_setup, ut_teardown, interrupt_tc), 2129 TEST_CASES_END() /**< NULL terminate unit test array */ 2130 } 2131 }; 2132 2133 REGISTER_TEST_COMMAND(throughput, bbdev_throughput_testsuite); 2134 REGISTER_TEST_COMMAND(validation, bbdev_validation_testsuite); 2135 REGISTER_TEST_COMMAND(latency, bbdev_latency_testsuite); 2136 REGISTER_TEST_COMMAND(interrupt, bbdev_interrupt_testsuite); 2137