1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017 Intel Corporation 3 */ 4 5 #include <stdio.h> 6 #include <inttypes.h> 7 #include <math.h> 8 9 #include <rte_eal.h> 10 #include <rte_common.h> 11 #include <rte_dev.h> 12 #include <rte_launch.h> 13 #include <rte_bbdev.h> 14 #include <rte_cycles.h> 15 #include <rte_lcore.h> 16 #include <rte_malloc.h> 17 #include <rte_random.h> 18 #include <rte_hexdump.h> 19 20 #include "main.h" 21 #include "test_bbdev_vector.h" 22 23 #define GET_SOCKET(socket_id) (((socket_id) == SOCKET_ID_ANY) ? 0 : (socket_id)) 24 25 #define MAX_QUEUES RTE_MAX_LCORE 26 27 #define OPS_CACHE_SIZE 256U 28 #define OPS_POOL_SIZE_MIN 511U /* 0.5K per queue */ 29 30 #define SYNC_WAIT 0 31 #define SYNC_START 1 32 33 #define INVALID_QUEUE_ID -1 34 35 static struct test_bbdev_vector test_vector; 36 37 /* Switch between PMD and Interrupt for throughput TC */ 38 static bool intr_enabled; 39 40 /* Represents tested active devices */ 41 static struct active_device { 42 const char *driver_name; 43 uint8_t dev_id; 44 uint16_t supported_ops; 45 uint16_t queue_ids[MAX_QUEUES]; 46 uint16_t nb_queues; 47 struct rte_mempool *ops_mempool; 48 struct rte_mempool *in_mbuf_pool; 49 struct rte_mempool *hard_out_mbuf_pool; 50 struct rte_mempool *soft_out_mbuf_pool; 51 } active_devs[RTE_BBDEV_MAX_DEVS]; 52 53 static uint8_t nb_active_devs; 54 55 /* Data buffers used by BBDEV ops */ 56 struct test_buffers { 57 struct rte_bbdev_op_data *inputs; 58 struct rte_bbdev_op_data *hard_outputs; 59 struct rte_bbdev_op_data *soft_outputs; 60 }; 61 62 /* Operation parameters specific for given test case */ 63 struct test_op_params { 64 struct rte_mempool *mp; 65 struct rte_bbdev_dec_op *ref_dec_op; 66 struct rte_bbdev_enc_op *ref_enc_op; 67 uint16_t burst_sz; 68 uint16_t num_to_process; 69 uint16_t num_lcores; 70 int vector_mask; 71 rte_atomic16_t sync; 72 struct test_buffers q_bufs[RTE_MAX_NUMA_NODES][MAX_QUEUES]; 73 }; 74 75 /* Contains per lcore params */ 76 struct thread_params { 77 uint8_t dev_id; 78 uint16_t queue_id; 79 uint64_t start_time; 80 double mops; 81 double mbps; 82 rte_atomic16_t nb_dequeued; 83 rte_atomic16_t processing_status; 84 struct test_op_params *op_params; 85 }; 86 87 #ifdef RTE_BBDEV_OFFLOAD_COST 88 /* Stores time statistics */ 89 struct test_time_stats { 90 /* Stores software enqueue total working time */ 91 uint64_t enq_sw_tot_time; 92 /* Stores minimum value of software enqueue working time */ 93 uint64_t enq_sw_min_time; 94 /* Stores maximum value of software enqueue working time */ 95 uint64_t enq_sw_max_time; 96 /* Stores turbo enqueue total working time */ 97 uint64_t enq_tur_tot_time; 98 /* Stores minimum value of turbo enqueue working time */ 99 uint64_t enq_tur_min_time; 100 /* Stores maximum value of turbo enqueue working time */ 101 uint64_t enq_tur_max_time; 102 /* Stores dequeue total working time */ 103 uint64_t deq_tot_time; 104 /* Stores minimum value of dequeue working time */ 105 uint64_t deq_min_time; 106 /* Stores maximum value of dequeue working time */ 107 uint64_t deq_max_time; 108 }; 109 #endif 110 111 typedef int (test_case_function)(struct active_device *ad, 112 struct test_op_params *op_params); 113 114 static inline void 115 set_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type) 116 { 117 ad->supported_ops |= (1 << op_type); 118 } 119 120 static inline bool 121 is_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type) 122 { 123 return ad->supported_ops & (1 << op_type); 124 } 125 126 static inline bool 127 flags_match(uint32_t flags_req, uint32_t flags_present) 128 { 129 return (flags_req & flags_present) == flags_req; 130 } 131 132 static void 133 clear_soft_out_cap(uint32_t *op_flags) 134 { 135 *op_flags &= ~RTE_BBDEV_TURBO_SOFT_OUTPUT; 136 *op_flags &= ~RTE_BBDEV_TURBO_POS_LLR_1_BIT_SOFT_OUT; 137 *op_flags &= ~RTE_BBDEV_TURBO_NEG_LLR_1_BIT_SOFT_OUT; 138 } 139 140 static int 141 check_dev_cap(const struct rte_bbdev_info *dev_info) 142 { 143 unsigned int i; 144 unsigned int nb_inputs, nb_soft_outputs, nb_hard_outputs; 145 const struct rte_bbdev_op_cap *op_cap = dev_info->drv.capabilities; 146 147 nb_inputs = test_vector.entries[DATA_INPUT].nb_segments; 148 nb_soft_outputs = test_vector.entries[DATA_SOFT_OUTPUT].nb_segments; 149 nb_hard_outputs = test_vector.entries[DATA_HARD_OUTPUT].nb_segments; 150 151 for (i = 0; op_cap->type != RTE_BBDEV_OP_NONE; ++i, ++op_cap) { 152 if (op_cap->type != test_vector.op_type) 153 continue; 154 155 if (op_cap->type == RTE_BBDEV_OP_TURBO_DEC) { 156 const struct rte_bbdev_op_cap_turbo_dec *cap = 157 &op_cap->cap.turbo_dec; 158 /* Ignore lack of soft output capability, just skip 159 * checking if soft output is valid. 160 */ 161 if ((test_vector.turbo_dec.op_flags & 162 RTE_BBDEV_TURBO_SOFT_OUTPUT) && 163 !(cap->capability_flags & 164 RTE_BBDEV_TURBO_SOFT_OUTPUT)) { 165 printf( 166 "WARNING: Device \"%s\" does not support soft output - soft output flags will be ignored.\n", 167 dev_info->dev_name); 168 clear_soft_out_cap( 169 &test_vector.turbo_dec.op_flags); 170 } 171 172 if (!flags_match(test_vector.turbo_dec.op_flags, 173 cap->capability_flags)) 174 return TEST_FAILED; 175 if (nb_inputs > cap->num_buffers_src) { 176 printf("Too many inputs defined: %u, max: %u\n", 177 nb_inputs, cap->num_buffers_src); 178 return TEST_FAILED; 179 } 180 if (nb_soft_outputs > cap->num_buffers_soft_out && 181 (test_vector.turbo_dec.op_flags & 182 RTE_BBDEV_TURBO_SOFT_OUTPUT)) { 183 printf( 184 "Too many soft outputs defined: %u, max: %u\n", 185 nb_soft_outputs, 186 cap->num_buffers_soft_out); 187 return TEST_FAILED; 188 } 189 if (nb_hard_outputs > cap->num_buffers_hard_out) { 190 printf( 191 "Too many hard outputs defined: %u, max: %u\n", 192 nb_hard_outputs, 193 cap->num_buffers_hard_out); 194 return TEST_FAILED; 195 } 196 if (intr_enabled && !(cap->capability_flags & 197 RTE_BBDEV_TURBO_DEC_INTERRUPTS)) { 198 printf( 199 "Dequeue interrupts are not supported!\n"); 200 return TEST_FAILED; 201 } 202 203 return TEST_SUCCESS; 204 } else if (op_cap->type == RTE_BBDEV_OP_TURBO_ENC) { 205 const struct rte_bbdev_op_cap_turbo_enc *cap = 206 &op_cap->cap.turbo_enc; 207 208 if (!flags_match(test_vector.turbo_enc.op_flags, 209 cap->capability_flags)) 210 return TEST_FAILED; 211 if (nb_inputs > cap->num_buffers_src) { 212 printf("Too many inputs defined: %u, max: %u\n", 213 nb_inputs, cap->num_buffers_src); 214 return TEST_FAILED; 215 } 216 if (nb_hard_outputs > cap->num_buffers_dst) { 217 printf( 218 "Too many hard outputs defined: %u, max: %u\n", 219 nb_hard_outputs, cap->num_buffers_src); 220 return TEST_FAILED; 221 } 222 if (intr_enabled && !(cap->capability_flags & 223 RTE_BBDEV_TURBO_ENC_INTERRUPTS)) { 224 printf( 225 "Dequeue interrupts are not supported!\n"); 226 return TEST_FAILED; 227 } 228 229 return TEST_SUCCESS; 230 } 231 } 232 233 if ((i == 0) && (test_vector.op_type == RTE_BBDEV_OP_NONE)) 234 return TEST_SUCCESS; /* Special case for NULL device */ 235 236 return TEST_FAILED; 237 } 238 239 /* calculates optimal mempool size not smaller than the val */ 240 static unsigned int 241 optimal_mempool_size(unsigned int val) 242 { 243 return rte_align32pow2(val + 1) - 1; 244 } 245 246 /* allocates mbuf mempool for inputs and outputs */ 247 static struct rte_mempool * 248 create_mbuf_pool(struct op_data_entries *entries, uint8_t dev_id, 249 int socket_id, unsigned int mbuf_pool_size, 250 const char *op_type_str) 251 { 252 unsigned int i; 253 uint32_t max_seg_sz = 0; 254 char pool_name[RTE_MEMPOOL_NAMESIZE]; 255 256 /* find max input segment size */ 257 for (i = 0; i < entries->nb_segments; ++i) 258 if (entries->segments[i].length > max_seg_sz) 259 max_seg_sz = entries->segments[i].length; 260 261 snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str, 262 dev_id); 263 return rte_pktmbuf_pool_create(pool_name, mbuf_pool_size, 0, 0, 264 RTE_MAX(max_seg_sz + RTE_PKTMBUF_HEADROOM, 265 (unsigned int)RTE_MBUF_DEFAULT_BUF_SIZE), socket_id); 266 } 267 268 static int 269 create_mempools(struct active_device *ad, int socket_id, 270 enum rte_bbdev_op_type op_type, uint16_t num_ops) 271 { 272 struct rte_mempool *mp; 273 unsigned int ops_pool_size, mbuf_pool_size = 0; 274 char pool_name[RTE_MEMPOOL_NAMESIZE]; 275 const char *op_type_str; 276 277 struct op_data_entries *in = &test_vector.entries[DATA_INPUT]; 278 struct op_data_entries *hard_out = 279 &test_vector.entries[DATA_HARD_OUTPUT]; 280 struct op_data_entries *soft_out = 281 &test_vector.entries[DATA_SOFT_OUTPUT]; 282 283 /* allocate ops mempool */ 284 ops_pool_size = optimal_mempool_size(RTE_MAX( 285 /* Ops used plus 1 reference op */ 286 RTE_MAX((unsigned int)(ad->nb_queues * num_ops + 1), 287 /* Minimal cache size plus 1 reference op */ 288 (unsigned int)(1.5 * rte_lcore_count() * 289 OPS_CACHE_SIZE + 1)), 290 OPS_POOL_SIZE_MIN)); 291 292 op_type_str = rte_bbdev_op_type_str(op_type); 293 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type); 294 295 snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str, 296 ad->dev_id); 297 mp = rte_bbdev_op_pool_create(pool_name, op_type, 298 ops_pool_size, OPS_CACHE_SIZE, socket_id); 299 TEST_ASSERT_NOT_NULL(mp, 300 "ERROR Failed to create %u items ops pool for dev %u on socket %u.", 301 ops_pool_size, 302 ad->dev_id, 303 socket_id); 304 ad->ops_mempool = mp; 305 306 /* Inputs */ 307 mbuf_pool_size = optimal_mempool_size(ops_pool_size * in->nb_segments); 308 mp = create_mbuf_pool(in, ad->dev_id, socket_id, mbuf_pool_size, "in"); 309 TEST_ASSERT_NOT_NULL(mp, 310 "ERROR Failed to create %u items input pktmbuf pool for dev %u on socket %u.", 311 mbuf_pool_size, 312 ad->dev_id, 313 socket_id); 314 ad->in_mbuf_pool = mp; 315 316 /* Hard outputs */ 317 mbuf_pool_size = optimal_mempool_size(ops_pool_size * 318 hard_out->nb_segments); 319 mp = create_mbuf_pool(hard_out, ad->dev_id, socket_id, mbuf_pool_size, 320 "hard_out"); 321 TEST_ASSERT_NOT_NULL(mp, 322 "ERROR Failed to create %u items hard output pktmbuf pool for dev %u on socket %u.", 323 mbuf_pool_size, 324 ad->dev_id, 325 socket_id); 326 ad->hard_out_mbuf_pool = mp; 327 328 if (soft_out->nb_segments == 0) 329 return TEST_SUCCESS; 330 331 /* Soft outputs */ 332 mbuf_pool_size = optimal_mempool_size(ops_pool_size * 333 soft_out->nb_segments); 334 mp = create_mbuf_pool(soft_out, ad->dev_id, socket_id, mbuf_pool_size, 335 "soft_out"); 336 TEST_ASSERT_NOT_NULL(mp, 337 "ERROR Failed to create %uB soft output pktmbuf pool for dev %u on socket %u.", 338 mbuf_pool_size, 339 ad->dev_id, 340 socket_id); 341 ad->soft_out_mbuf_pool = mp; 342 343 return 0; 344 } 345 346 static int 347 add_bbdev_dev(uint8_t dev_id, struct rte_bbdev_info *info, 348 struct test_bbdev_vector *vector) 349 { 350 int ret; 351 unsigned int queue_id; 352 struct rte_bbdev_queue_conf qconf; 353 struct active_device *ad = &active_devs[nb_active_devs]; 354 unsigned int nb_queues; 355 enum rte_bbdev_op_type op_type = vector->op_type; 356 357 nb_queues = RTE_MIN(rte_lcore_count(), info->drv.max_num_queues); 358 /* setup device */ 359 ret = rte_bbdev_setup_queues(dev_id, nb_queues, info->socket_id); 360 if (ret < 0) { 361 printf("rte_bbdev_setup_queues(%u, %u, %d) ret %i\n", 362 dev_id, nb_queues, info->socket_id, ret); 363 return TEST_FAILED; 364 } 365 366 /* configure interrupts if needed */ 367 if (intr_enabled) { 368 ret = rte_bbdev_intr_enable(dev_id); 369 if (ret < 0) { 370 printf("rte_bbdev_intr_enable(%u) ret %i\n", dev_id, 371 ret); 372 return TEST_FAILED; 373 } 374 } 375 376 /* setup device queues */ 377 qconf.socket = info->socket_id; 378 qconf.queue_size = info->drv.default_queue_conf.queue_size; 379 qconf.priority = 0; 380 qconf.deferred_start = 0; 381 qconf.op_type = op_type; 382 383 for (queue_id = 0; queue_id < nb_queues; ++queue_id) { 384 ret = rte_bbdev_queue_configure(dev_id, queue_id, &qconf); 385 if (ret != 0) { 386 printf( 387 "Allocated all queues (id=%u) at prio%u on dev%u\n", 388 queue_id, qconf.priority, dev_id); 389 qconf.priority++; 390 ret = rte_bbdev_queue_configure(ad->dev_id, queue_id, 391 &qconf); 392 } 393 if (ret != 0) { 394 printf("All queues on dev %u allocated: %u\n", 395 dev_id, queue_id); 396 break; 397 } 398 ad->queue_ids[queue_id] = queue_id; 399 } 400 TEST_ASSERT(queue_id != 0, 401 "ERROR Failed to configure any queues on dev %u", 402 dev_id); 403 ad->nb_queues = queue_id; 404 405 set_avail_op(ad, op_type); 406 407 return TEST_SUCCESS; 408 } 409 410 static int 411 add_active_device(uint8_t dev_id, struct rte_bbdev_info *info, 412 struct test_bbdev_vector *vector) 413 { 414 int ret; 415 416 active_devs[nb_active_devs].driver_name = info->drv.driver_name; 417 active_devs[nb_active_devs].dev_id = dev_id; 418 419 ret = add_bbdev_dev(dev_id, info, vector); 420 if (ret == TEST_SUCCESS) 421 ++nb_active_devs; 422 return ret; 423 } 424 425 static uint8_t 426 populate_active_devices(void) 427 { 428 int ret; 429 uint8_t dev_id; 430 uint8_t nb_devs_added = 0; 431 struct rte_bbdev_info info; 432 433 RTE_BBDEV_FOREACH(dev_id) { 434 rte_bbdev_info_get(dev_id, &info); 435 436 if (check_dev_cap(&info)) { 437 printf( 438 "Device %d (%s) does not support specified capabilities\n", 439 dev_id, info.dev_name); 440 continue; 441 } 442 443 ret = add_active_device(dev_id, &info, &test_vector); 444 if (ret != 0) { 445 printf("Adding active bbdev %s skipped\n", 446 info.dev_name); 447 continue; 448 } 449 nb_devs_added++; 450 } 451 452 return nb_devs_added; 453 } 454 455 static int 456 read_test_vector(void) 457 { 458 int ret; 459 460 memset(&test_vector, 0, sizeof(test_vector)); 461 printf("Test vector file = %s\n", get_vector_filename()); 462 ret = test_bbdev_vector_read(get_vector_filename(), &test_vector); 463 TEST_ASSERT_SUCCESS(ret, "Failed to parse file %s\n", 464 get_vector_filename()); 465 466 return TEST_SUCCESS; 467 } 468 469 static int 470 testsuite_setup(void) 471 { 472 TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n"); 473 474 if (populate_active_devices() == 0) { 475 printf("No suitable devices found!\n"); 476 return TEST_SKIPPED; 477 } 478 479 return TEST_SUCCESS; 480 } 481 482 static int 483 interrupt_testsuite_setup(void) 484 { 485 TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n"); 486 487 /* Enable interrupts */ 488 intr_enabled = true; 489 490 /* Special case for NULL device (RTE_BBDEV_OP_NONE) */ 491 if (populate_active_devices() == 0 || 492 test_vector.op_type == RTE_BBDEV_OP_NONE) { 493 intr_enabled = false; 494 printf("No suitable devices found!\n"); 495 return TEST_SKIPPED; 496 } 497 498 return TEST_SUCCESS; 499 } 500 501 static void 502 testsuite_teardown(void) 503 { 504 uint8_t dev_id; 505 506 /* Unconfigure devices */ 507 RTE_BBDEV_FOREACH(dev_id) 508 rte_bbdev_close(dev_id); 509 510 /* Clear active devices structs. */ 511 memset(active_devs, 0, sizeof(active_devs)); 512 nb_active_devs = 0; 513 } 514 515 static int 516 ut_setup(void) 517 { 518 uint8_t i, dev_id; 519 520 for (i = 0; i < nb_active_devs; i++) { 521 dev_id = active_devs[i].dev_id; 522 /* reset bbdev stats */ 523 TEST_ASSERT_SUCCESS(rte_bbdev_stats_reset(dev_id), 524 "Failed to reset stats of bbdev %u", dev_id); 525 /* start the device */ 526 TEST_ASSERT_SUCCESS(rte_bbdev_start(dev_id), 527 "Failed to start bbdev %u", dev_id); 528 } 529 530 return TEST_SUCCESS; 531 } 532 533 static void 534 ut_teardown(void) 535 { 536 uint8_t i, dev_id; 537 struct rte_bbdev_stats stats; 538 539 for (i = 0; i < nb_active_devs; i++) { 540 dev_id = active_devs[i].dev_id; 541 /* read stats and print */ 542 rte_bbdev_stats_get(dev_id, &stats); 543 /* Stop the device */ 544 rte_bbdev_stop(dev_id); 545 } 546 } 547 548 static int 549 init_op_data_objs(struct rte_bbdev_op_data *bufs, 550 struct op_data_entries *ref_entries, 551 struct rte_mempool *mbuf_pool, const uint16_t n, 552 enum op_data_type op_type, uint16_t min_alignment) 553 { 554 int ret; 555 unsigned int i, j; 556 557 for (i = 0; i < n; ++i) { 558 char *data; 559 struct op_data_buf *seg = &ref_entries->segments[0]; 560 struct rte_mbuf *m_head = rte_pktmbuf_alloc(mbuf_pool); 561 TEST_ASSERT_NOT_NULL(m_head, 562 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)", 563 op_type, n * ref_entries->nb_segments, 564 mbuf_pool->size); 565 566 bufs[i].data = m_head; 567 bufs[i].offset = 0; 568 bufs[i].length = 0; 569 570 if (op_type == DATA_INPUT) { 571 data = rte_pktmbuf_append(m_head, seg->length); 572 TEST_ASSERT_NOT_NULL(data, 573 "Couldn't append %u bytes to mbuf from %d data type mbuf pool", 574 seg->length, op_type); 575 576 TEST_ASSERT(data == RTE_PTR_ALIGN(data, min_alignment), 577 "Data addr in mbuf (%p) is not aligned to device min alignment (%u)", 578 data, min_alignment); 579 rte_memcpy(data, seg->addr, seg->length); 580 bufs[i].length += seg->length; 581 582 583 for (j = 1; j < ref_entries->nb_segments; ++j) { 584 struct rte_mbuf *m_tail = 585 rte_pktmbuf_alloc(mbuf_pool); 586 TEST_ASSERT_NOT_NULL(m_tail, 587 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)", 588 op_type, 589 n * ref_entries->nb_segments, 590 mbuf_pool->size); 591 seg += 1; 592 593 data = rte_pktmbuf_append(m_tail, seg->length); 594 TEST_ASSERT_NOT_NULL(data, 595 "Couldn't append %u bytes to mbuf from %d data type mbuf pool", 596 seg->length, op_type); 597 598 TEST_ASSERT(data == RTE_PTR_ALIGN(data, 599 min_alignment), 600 "Data addr in mbuf (%p) is not aligned to device min alignment (%u)", 601 data, min_alignment); 602 rte_memcpy(data, seg->addr, seg->length); 603 bufs[i].length += seg->length; 604 605 ret = rte_pktmbuf_chain(m_head, m_tail); 606 TEST_ASSERT_SUCCESS(ret, 607 "Couldn't chain mbufs from %d data type mbuf pool", 608 op_type); 609 } 610 } 611 } 612 613 return 0; 614 } 615 616 static int 617 allocate_buffers_on_socket(struct rte_bbdev_op_data **buffers, const int len, 618 const int socket) 619 { 620 int i; 621 622 *buffers = rte_zmalloc_socket(NULL, len, 0, socket); 623 if (*buffers == NULL) { 624 printf("WARNING: Failed to allocate op_data on socket %d\n", 625 socket); 626 /* try to allocate memory on other detected sockets */ 627 for (i = 0; i < socket; i++) { 628 *buffers = rte_zmalloc_socket(NULL, len, 0, i); 629 if (*buffers != NULL) 630 break; 631 } 632 } 633 634 return (*buffers == NULL) ? TEST_FAILED : TEST_SUCCESS; 635 } 636 637 static void 638 limit_input_llr_val_range(struct rte_bbdev_op_data *input_ops, 639 uint16_t n, int8_t max_llr_modulus) 640 { 641 uint16_t i, byte_idx; 642 643 for (i = 0; i < n; ++i) { 644 struct rte_mbuf *m = input_ops[i].data; 645 while (m != NULL) { 646 int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *, 647 input_ops[i].offset); 648 for (byte_idx = 0; byte_idx < input_ops[i].length; 649 ++byte_idx) 650 llr[byte_idx] = round((double)max_llr_modulus * 651 llr[byte_idx] / INT8_MAX); 652 653 m = m->next; 654 } 655 } 656 } 657 658 static int 659 fill_queue_buffers(struct test_op_params *op_params, 660 struct rte_mempool *in_mp, struct rte_mempool *hard_out_mp, 661 struct rte_mempool *soft_out_mp, uint16_t queue_id, 662 const struct rte_bbdev_op_cap *capabilities, 663 uint16_t min_alignment, const int socket_id) 664 { 665 int ret; 666 enum op_data_type type; 667 const uint16_t n = op_params->num_to_process; 668 669 struct rte_mempool *mbuf_pools[DATA_NUM_TYPES] = { 670 in_mp, 671 soft_out_mp, 672 hard_out_mp, 673 }; 674 675 struct rte_bbdev_op_data **queue_ops[DATA_NUM_TYPES] = { 676 &op_params->q_bufs[socket_id][queue_id].inputs, 677 &op_params->q_bufs[socket_id][queue_id].soft_outputs, 678 &op_params->q_bufs[socket_id][queue_id].hard_outputs, 679 }; 680 681 for (type = DATA_INPUT; type < DATA_NUM_TYPES; ++type) { 682 struct op_data_entries *ref_entries = 683 &test_vector.entries[type]; 684 if (ref_entries->nb_segments == 0) 685 continue; 686 687 ret = allocate_buffers_on_socket(queue_ops[type], 688 n * sizeof(struct rte_bbdev_op_data), 689 socket_id); 690 TEST_ASSERT_SUCCESS(ret, 691 "Couldn't allocate memory for rte_bbdev_op_data structs"); 692 693 ret = init_op_data_objs(*queue_ops[type], ref_entries, 694 mbuf_pools[type], n, type, min_alignment); 695 TEST_ASSERT_SUCCESS(ret, 696 "Couldn't init rte_bbdev_op_data structs"); 697 } 698 699 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) 700 limit_input_llr_val_range(*queue_ops[DATA_INPUT], n, 701 capabilities->cap.turbo_dec.max_llr_modulus); 702 703 return 0; 704 } 705 706 static void 707 free_buffers(struct active_device *ad, struct test_op_params *op_params) 708 { 709 unsigned int i, j; 710 711 rte_mempool_free(ad->ops_mempool); 712 rte_mempool_free(ad->in_mbuf_pool); 713 rte_mempool_free(ad->hard_out_mbuf_pool); 714 rte_mempool_free(ad->soft_out_mbuf_pool); 715 716 for (i = 0; i < rte_lcore_count(); ++i) { 717 for (j = 0; j < RTE_MAX_NUMA_NODES; ++j) { 718 rte_free(op_params->q_bufs[j][i].inputs); 719 rte_free(op_params->q_bufs[j][i].hard_outputs); 720 rte_free(op_params->q_bufs[j][i].soft_outputs); 721 } 722 } 723 } 724 725 static void 726 copy_reference_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n, 727 unsigned int start_idx, 728 struct rte_bbdev_op_data *inputs, 729 struct rte_bbdev_op_data *hard_outputs, 730 struct rte_bbdev_op_data *soft_outputs, 731 struct rte_bbdev_dec_op *ref_op) 732 { 733 unsigned int i; 734 struct rte_bbdev_op_turbo_dec *turbo_dec = &ref_op->turbo_dec; 735 736 for (i = 0; i < n; ++i) { 737 if (turbo_dec->code_block_mode == 0) { 738 ops[i]->turbo_dec.tb_params.ea = 739 turbo_dec->tb_params.ea; 740 ops[i]->turbo_dec.tb_params.eb = 741 turbo_dec->tb_params.eb; 742 ops[i]->turbo_dec.tb_params.k_pos = 743 turbo_dec->tb_params.k_pos; 744 ops[i]->turbo_dec.tb_params.k_neg = 745 turbo_dec->tb_params.k_neg; 746 ops[i]->turbo_dec.tb_params.c = 747 turbo_dec->tb_params.c; 748 ops[i]->turbo_dec.tb_params.c_neg = 749 turbo_dec->tb_params.c_neg; 750 ops[i]->turbo_dec.tb_params.cab = 751 turbo_dec->tb_params.cab; 752 } else { 753 ops[i]->turbo_dec.cb_params.e = turbo_dec->cb_params.e; 754 ops[i]->turbo_dec.cb_params.k = turbo_dec->cb_params.k; 755 } 756 757 ops[i]->turbo_dec.ext_scale = turbo_dec->ext_scale; 758 ops[i]->turbo_dec.iter_max = turbo_dec->iter_max; 759 ops[i]->turbo_dec.iter_min = turbo_dec->iter_min; 760 ops[i]->turbo_dec.op_flags = turbo_dec->op_flags; 761 ops[i]->turbo_dec.rv_index = turbo_dec->rv_index; 762 ops[i]->turbo_dec.num_maps = turbo_dec->num_maps; 763 ops[i]->turbo_dec.code_block_mode = turbo_dec->code_block_mode; 764 765 ops[i]->turbo_dec.hard_output = hard_outputs[start_idx + i]; 766 ops[i]->turbo_dec.input = inputs[start_idx + i]; 767 if (soft_outputs != NULL) 768 ops[i]->turbo_dec.soft_output = 769 soft_outputs[start_idx + i]; 770 } 771 } 772 773 static void 774 copy_reference_enc_op(struct rte_bbdev_enc_op **ops, unsigned int n, 775 unsigned int start_idx, 776 struct rte_bbdev_op_data *inputs, 777 struct rte_bbdev_op_data *outputs, 778 struct rte_bbdev_enc_op *ref_op) 779 { 780 unsigned int i; 781 struct rte_bbdev_op_turbo_enc *turbo_enc = &ref_op->turbo_enc; 782 for (i = 0; i < n; ++i) { 783 if (turbo_enc->code_block_mode == 0) { 784 ops[i]->turbo_enc.tb_params.ea = 785 turbo_enc->tb_params.ea; 786 ops[i]->turbo_enc.tb_params.eb = 787 turbo_enc->tb_params.eb; 788 ops[i]->turbo_enc.tb_params.k_pos = 789 turbo_enc->tb_params.k_pos; 790 ops[i]->turbo_enc.tb_params.k_neg = 791 turbo_enc->tb_params.k_neg; 792 ops[i]->turbo_enc.tb_params.c = 793 turbo_enc->tb_params.c; 794 ops[i]->turbo_enc.tb_params.c_neg = 795 turbo_enc->tb_params.c_neg; 796 ops[i]->turbo_enc.tb_params.cab = 797 turbo_enc->tb_params.cab; 798 ops[i]->turbo_enc.tb_params.ncb_pos = 799 turbo_enc->tb_params.ncb_pos; 800 ops[i]->turbo_enc.tb_params.ncb_neg = 801 turbo_enc->tb_params.ncb_neg; 802 ops[i]->turbo_enc.tb_params.r = turbo_enc->tb_params.r; 803 } else { 804 ops[i]->turbo_enc.cb_params.e = turbo_enc->cb_params.e; 805 ops[i]->turbo_enc.cb_params.k = turbo_enc->cb_params.k; 806 ops[i]->turbo_enc.cb_params.ncb = 807 turbo_enc->cb_params.ncb; 808 } 809 ops[i]->turbo_enc.rv_index = turbo_enc->rv_index; 810 ops[i]->turbo_enc.op_flags = turbo_enc->op_flags; 811 ops[i]->turbo_enc.code_block_mode = turbo_enc->code_block_mode; 812 813 ops[i]->turbo_enc.output = outputs[start_idx + i]; 814 ops[i]->turbo_enc.input = inputs[start_idx + i]; 815 } 816 } 817 818 static int 819 check_dec_status_and_ordering(struct rte_bbdev_dec_op *op, 820 unsigned int order_idx, const int expected_status) 821 { 822 TEST_ASSERT(op->status == expected_status, 823 "op_status (%d) != expected_status (%d)", 824 op->status, expected_status); 825 826 TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data, 827 "Ordering error, expected %p, got %p", 828 (void *)(uintptr_t)order_idx, op->opaque_data); 829 830 return TEST_SUCCESS; 831 } 832 833 static int 834 check_enc_status_and_ordering(struct rte_bbdev_enc_op *op, 835 unsigned int order_idx, const int expected_status) 836 { 837 TEST_ASSERT(op->status == expected_status, 838 "op_status (%d) != expected_status (%d)", 839 op->status, expected_status); 840 841 TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data, 842 "Ordering error, expected %p, got %p", 843 (void *)(uintptr_t)order_idx, op->opaque_data); 844 845 return TEST_SUCCESS; 846 } 847 848 static inline int 849 validate_op_chain(struct rte_bbdev_op_data *op, 850 struct op_data_entries *orig_op) 851 { 852 uint8_t i; 853 struct rte_mbuf *m = op->data; 854 uint8_t nb_dst_segments = orig_op->nb_segments; 855 856 TEST_ASSERT(nb_dst_segments == m->nb_segs, 857 "Number of segments differ in original (%u) and filled (%u) op", 858 nb_dst_segments, m->nb_segs); 859 860 for (i = 0; i < nb_dst_segments; ++i) { 861 /* Apply offset to the first mbuf segment */ 862 uint16_t offset = (i == 0) ? op->offset : 0; 863 uint16_t data_len = m->data_len - offset; 864 865 TEST_ASSERT(orig_op->segments[i].length == data_len, 866 "Length of segment differ in original (%u) and filled (%u) op", 867 orig_op->segments[i].length, data_len); 868 TEST_ASSERT_BUFFERS_ARE_EQUAL(orig_op->segments[i].addr, 869 rte_pktmbuf_mtod_offset(m, uint32_t *, offset), 870 data_len, 871 "Output buffers (CB=%u) are not equal", i); 872 m = m->next; 873 } 874 875 return TEST_SUCCESS; 876 } 877 878 static int 879 validate_dec_buffers(struct rte_bbdev_dec_op *ref_op, struct test_buffers *bufs, 880 const uint16_t num_to_process) 881 { 882 int i; 883 884 struct op_data_entries *hard_data_orig = 885 &test_vector.entries[DATA_HARD_OUTPUT]; 886 struct op_data_entries *soft_data_orig = 887 &test_vector.entries[DATA_SOFT_OUTPUT]; 888 889 for (i = 0; i < num_to_process; i++) { 890 TEST_ASSERT_SUCCESS(validate_op_chain(&bufs->hard_outputs[i], 891 hard_data_orig), 892 "Hard output buffers are not equal"); 893 if (ref_op->turbo_dec.op_flags & 894 RTE_BBDEV_TURBO_SOFT_OUTPUT) 895 TEST_ASSERT_SUCCESS(validate_op_chain( 896 &bufs->soft_outputs[i], 897 soft_data_orig), 898 "Soft output buffers are not equal"); 899 } 900 901 return TEST_SUCCESS; 902 } 903 904 static int 905 validate_enc_buffers(struct test_buffers *bufs, const uint16_t num_to_process) 906 { 907 int i; 908 909 struct op_data_entries *hard_data_orig = 910 &test_vector.entries[DATA_HARD_OUTPUT]; 911 912 for (i = 0; i < num_to_process; i++) 913 TEST_ASSERT_SUCCESS(validate_op_chain(&bufs->hard_outputs[i], 914 hard_data_orig), ""); 915 916 return TEST_SUCCESS; 917 } 918 919 static int 920 validate_dec_op(struct rte_bbdev_dec_op **ops, const uint16_t n, 921 struct rte_bbdev_dec_op *ref_op, const int vector_mask) 922 { 923 unsigned int i; 924 int ret; 925 struct op_data_entries *hard_data_orig = 926 &test_vector.entries[DATA_HARD_OUTPUT]; 927 struct op_data_entries *soft_data_orig = 928 &test_vector.entries[DATA_SOFT_OUTPUT]; 929 struct rte_bbdev_op_turbo_dec *ops_td; 930 struct rte_bbdev_op_data *hard_output; 931 struct rte_bbdev_op_data *soft_output; 932 struct rte_bbdev_op_turbo_dec *ref_td = &ref_op->turbo_dec; 933 934 for (i = 0; i < n; ++i) { 935 ops_td = &ops[i]->turbo_dec; 936 hard_output = &ops_td->hard_output; 937 soft_output = &ops_td->soft_output; 938 939 if (vector_mask & TEST_BBDEV_VF_EXPECTED_ITER_COUNT) 940 TEST_ASSERT(ops_td->iter_count <= ref_td->iter_count, 941 "Returned iter_count (%d) > expected iter_count (%d)", 942 ops_td->iter_count, ref_td->iter_count); 943 ret = check_dec_status_and_ordering(ops[i], i, ref_op->status); 944 TEST_ASSERT_SUCCESS(ret, 945 "Checking status and ordering for decoder failed"); 946 947 TEST_ASSERT_SUCCESS(validate_op_chain(hard_output, 948 hard_data_orig), 949 "Hard output buffers (CB=%u) are not equal", 950 i); 951 952 if (ref_op->turbo_dec.op_flags & RTE_BBDEV_TURBO_SOFT_OUTPUT) 953 TEST_ASSERT_SUCCESS(validate_op_chain(soft_output, 954 soft_data_orig), 955 "Soft output buffers (CB=%u) are not equal", 956 i); 957 } 958 959 return TEST_SUCCESS; 960 } 961 962 static int 963 validate_enc_op(struct rte_bbdev_enc_op **ops, const uint16_t n, 964 struct rte_bbdev_enc_op *ref_op) 965 { 966 unsigned int i; 967 int ret; 968 struct op_data_entries *hard_data_orig = 969 &test_vector.entries[DATA_HARD_OUTPUT]; 970 971 for (i = 0; i < n; ++i) { 972 ret = check_enc_status_and_ordering(ops[i], i, ref_op->status); 973 TEST_ASSERT_SUCCESS(ret, 974 "Checking status and ordering for encoder failed"); 975 TEST_ASSERT_SUCCESS(validate_op_chain( 976 &ops[i]->turbo_enc.output, 977 hard_data_orig), 978 "Output buffers (CB=%u) are not equal", 979 i); 980 } 981 982 return TEST_SUCCESS; 983 } 984 985 static void 986 create_reference_dec_op(struct rte_bbdev_dec_op *op) 987 { 988 unsigned int i; 989 struct op_data_entries *entry; 990 991 op->turbo_dec = test_vector.turbo_dec; 992 entry = &test_vector.entries[DATA_INPUT]; 993 for (i = 0; i < entry->nb_segments; ++i) 994 op->turbo_dec.input.length += 995 entry->segments[i].length; 996 } 997 998 static void 999 create_reference_enc_op(struct rte_bbdev_enc_op *op) 1000 { 1001 unsigned int i; 1002 struct op_data_entries *entry; 1003 1004 op->turbo_enc = test_vector.turbo_enc; 1005 entry = &test_vector.entries[DATA_INPUT]; 1006 for (i = 0; i < entry->nb_segments; ++i) 1007 op->turbo_enc.input.length += 1008 entry->segments[i].length; 1009 } 1010 1011 static int 1012 init_test_op_params(struct test_op_params *op_params, 1013 enum rte_bbdev_op_type op_type, const int expected_status, 1014 const int vector_mask, struct rte_mempool *ops_mp, 1015 uint16_t burst_sz, uint16_t num_to_process, uint16_t num_lcores) 1016 { 1017 int ret = 0; 1018 if (op_type == RTE_BBDEV_OP_TURBO_DEC) 1019 ret = rte_bbdev_dec_op_alloc_bulk(ops_mp, 1020 &op_params->ref_dec_op, 1); 1021 else 1022 ret = rte_bbdev_enc_op_alloc_bulk(ops_mp, 1023 &op_params->ref_enc_op, 1); 1024 1025 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_op_alloc_bulk() failed"); 1026 1027 op_params->mp = ops_mp; 1028 op_params->burst_sz = burst_sz; 1029 op_params->num_to_process = num_to_process; 1030 op_params->num_lcores = num_lcores; 1031 op_params->vector_mask = vector_mask; 1032 if (op_type == RTE_BBDEV_OP_TURBO_DEC) 1033 op_params->ref_dec_op->status = expected_status; 1034 else if (op_type == RTE_BBDEV_OP_TURBO_ENC) 1035 op_params->ref_enc_op->status = expected_status; 1036 1037 return 0; 1038 } 1039 1040 static int 1041 run_test_case_on_device(test_case_function *test_case_func, uint8_t dev_id, 1042 struct test_op_params *op_params) 1043 { 1044 int t_ret, f_ret, socket_id = SOCKET_ID_ANY; 1045 unsigned int i; 1046 struct active_device *ad; 1047 unsigned int burst_sz = get_burst_sz(); 1048 enum rte_bbdev_op_type op_type = test_vector.op_type; 1049 const struct rte_bbdev_op_cap *capabilities = NULL; 1050 1051 ad = &active_devs[dev_id]; 1052 1053 /* Check if device supports op_type */ 1054 if (!is_avail_op(ad, test_vector.op_type)) 1055 return TEST_SUCCESS; 1056 1057 struct rte_bbdev_info info; 1058 rte_bbdev_info_get(ad->dev_id, &info); 1059 socket_id = GET_SOCKET(info.socket_id); 1060 1061 if (op_type == RTE_BBDEV_OP_NONE) 1062 op_type = RTE_BBDEV_OP_TURBO_ENC; 1063 f_ret = create_mempools(ad, socket_id, op_type, 1064 get_num_ops()); 1065 if (f_ret != TEST_SUCCESS) { 1066 printf("Couldn't create mempools"); 1067 goto fail; 1068 } 1069 1070 f_ret = init_test_op_params(op_params, test_vector.op_type, 1071 test_vector.expected_status, 1072 test_vector.mask, 1073 ad->ops_mempool, 1074 burst_sz, 1075 get_num_ops(), 1076 get_num_lcores()); 1077 if (f_ret != TEST_SUCCESS) { 1078 printf("Couldn't init test op params"); 1079 goto fail; 1080 } 1081 1082 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) { 1083 /* Find Decoder capabilities */ 1084 const struct rte_bbdev_op_cap *cap = info.drv.capabilities; 1085 while (cap->type != RTE_BBDEV_OP_NONE) { 1086 if (cap->type == RTE_BBDEV_OP_TURBO_DEC) { 1087 capabilities = cap; 1088 break; 1089 } 1090 } 1091 TEST_ASSERT_NOT_NULL(capabilities, 1092 "Couldn't find Decoder capabilities"); 1093 1094 create_reference_dec_op(op_params->ref_dec_op); 1095 } else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) 1096 create_reference_enc_op(op_params->ref_enc_op); 1097 1098 for (i = 0; i < ad->nb_queues; ++i) { 1099 f_ret = fill_queue_buffers(op_params, 1100 ad->in_mbuf_pool, 1101 ad->hard_out_mbuf_pool, 1102 ad->soft_out_mbuf_pool, 1103 ad->queue_ids[i], 1104 capabilities, 1105 info.drv.min_alignment, 1106 socket_id); 1107 if (f_ret != TEST_SUCCESS) { 1108 printf("Couldn't init queue buffers"); 1109 goto fail; 1110 } 1111 } 1112 1113 /* Run test case function */ 1114 t_ret = test_case_func(ad, op_params); 1115 1116 /* Free active device resources and return */ 1117 free_buffers(ad, op_params); 1118 return t_ret; 1119 1120 fail: 1121 free_buffers(ad, op_params); 1122 return TEST_FAILED; 1123 } 1124 1125 /* Run given test function per active device per supported op type 1126 * per burst size. 1127 */ 1128 static int 1129 run_test_case(test_case_function *test_case_func) 1130 { 1131 int ret = 0; 1132 uint8_t dev; 1133 1134 /* Alloc op_params */ 1135 struct test_op_params *op_params = rte_zmalloc(NULL, 1136 sizeof(struct test_op_params), RTE_CACHE_LINE_SIZE); 1137 TEST_ASSERT_NOT_NULL(op_params, "Failed to alloc %zuB for op_params", 1138 RTE_ALIGN(sizeof(struct test_op_params), 1139 RTE_CACHE_LINE_SIZE)); 1140 1141 /* For each device run test case function */ 1142 for (dev = 0; dev < nb_active_devs; ++dev) 1143 ret |= run_test_case_on_device(test_case_func, dev, op_params); 1144 1145 rte_free(op_params); 1146 1147 return ret; 1148 } 1149 1150 static void 1151 dequeue_event_callback(uint16_t dev_id, 1152 enum rte_bbdev_event_type event, void *cb_arg, 1153 void *ret_param) 1154 { 1155 int ret; 1156 uint16_t i; 1157 uint64_t total_time; 1158 uint16_t deq, burst_sz, num_to_process; 1159 uint16_t queue_id = INVALID_QUEUE_ID; 1160 struct rte_bbdev_dec_op *dec_ops[MAX_BURST]; 1161 struct rte_bbdev_enc_op *enc_ops[MAX_BURST]; 1162 struct test_buffers *bufs; 1163 struct rte_bbdev_info info; 1164 1165 /* Input length in bytes, million operations per second, 1166 * million bits per second. 1167 */ 1168 double in_len; 1169 1170 struct thread_params *tp = cb_arg; 1171 RTE_SET_USED(ret_param); 1172 queue_id = tp->queue_id; 1173 1174 /* Find matching thread params using queue_id */ 1175 for (i = 0; i < MAX_QUEUES; ++i, ++tp) 1176 if (tp->queue_id == queue_id) 1177 break; 1178 1179 if (i == MAX_QUEUES) { 1180 printf("%s: Queue_id from interrupt details was not found!\n", 1181 __func__); 1182 return; 1183 } 1184 1185 if (unlikely(event != RTE_BBDEV_EVENT_DEQUEUE)) { 1186 rte_atomic16_set(&tp->processing_status, TEST_FAILED); 1187 printf( 1188 "Dequeue interrupt handler called for incorrect event!\n"); 1189 return; 1190 } 1191 1192 burst_sz = tp->op_params->burst_sz; 1193 num_to_process = tp->op_params->num_to_process; 1194 1195 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) 1196 deq = rte_bbdev_dequeue_dec_ops(dev_id, queue_id, dec_ops, 1197 burst_sz); 1198 else 1199 deq = rte_bbdev_dequeue_enc_ops(dev_id, queue_id, enc_ops, 1200 burst_sz); 1201 1202 if (deq < burst_sz) { 1203 printf( 1204 "After receiving the interrupt all operations should be dequeued. Expected: %u, got: %u\n", 1205 burst_sz, deq); 1206 rte_atomic16_set(&tp->processing_status, TEST_FAILED); 1207 return; 1208 } 1209 1210 if (rte_atomic16_read(&tp->nb_dequeued) + deq < num_to_process) { 1211 rte_atomic16_add(&tp->nb_dequeued, deq); 1212 return; 1213 } 1214 1215 total_time = rte_rdtsc_precise() - tp->start_time; 1216 1217 rte_bbdev_info_get(dev_id, &info); 1218 1219 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 1220 1221 ret = TEST_SUCCESS; 1222 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) 1223 ret = validate_dec_buffers(tp->op_params->ref_dec_op, bufs, 1224 num_to_process); 1225 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) 1226 ret = validate_enc_buffers(bufs, num_to_process); 1227 1228 if (ret) { 1229 printf("Buffers validation failed\n"); 1230 rte_atomic16_set(&tp->processing_status, TEST_FAILED); 1231 } 1232 1233 switch (test_vector.op_type) { 1234 case RTE_BBDEV_OP_TURBO_DEC: 1235 in_len = tp->op_params->ref_dec_op->turbo_dec.input.length; 1236 break; 1237 case RTE_BBDEV_OP_TURBO_ENC: 1238 in_len = tp->op_params->ref_enc_op->turbo_enc.input.length; 1239 break; 1240 case RTE_BBDEV_OP_NONE: 1241 in_len = 0.0; 1242 break; 1243 default: 1244 printf("Unknown op type: %d\n", test_vector.op_type); 1245 rte_atomic16_set(&tp->processing_status, TEST_FAILED); 1246 return; 1247 } 1248 1249 tp->mops = ((double)num_to_process / 1000000.0) / 1250 ((double)total_time / (double)rte_get_tsc_hz()); 1251 tp->mbps = ((double)num_to_process * in_len * 8 / 1000000.0) / 1252 ((double)total_time / (double)rte_get_tsc_hz()); 1253 1254 rte_atomic16_add(&tp->nb_dequeued, deq); 1255 } 1256 1257 static int 1258 throughput_intr_lcore_dec(void *arg) 1259 { 1260 struct thread_params *tp = arg; 1261 unsigned int enqueued; 1262 struct rte_bbdev_dec_op *ops[MAX_BURST]; 1263 const uint16_t queue_id = tp->queue_id; 1264 const uint16_t burst_sz = tp->op_params->burst_sz; 1265 const uint16_t num_to_process = tp->op_params->num_to_process; 1266 struct test_buffers *bufs = NULL; 1267 unsigned int allocs_failed = 0; 1268 struct rte_bbdev_info info; 1269 int ret; 1270 1271 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 1272 "BURST_SIZE should be <= %u", MAX_BURST); 1273 1274 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id), 1275 "Failed to enable interrupts for dev: %u, queue_id: %u", 1276 tp->dev_id, queue_id); 1277 1278 rte_bbdev_info_get(tp->dev_id, &info); 1279 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 1280 1281 rte_atomic16_clear(&tp->processing_status); 1282 rte_atomic16_clear(&tp->nb_dequeued); 1283 1284 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT) 1285 rte_pause(); 1286 1287 tp->start_time = rte_rdtsc_precise(); 1288 for (enqueued = 0; enqueued < num_to_process;) { 1289 1290 uint16_t num_to_enq = burst_sz; 1291 1292 if (unlikely(num_to_process - enqueued < num_to_enq)) 1293 num_to_enq = num_to_process - enqueued; 1294 1295 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops, 1296 num_to_enq); 1297 if (ret != 0) { 1298 allocs_failed++; 1299 continue; 1300 } 1301 1302 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 1303 copy_reference_dec_op(ops, num_to_enq, enqueued, 1304 bufs->inputs, 1305 bufs->hard_outputs, 1306 bufs->soft_outputs, 1307 tp->op_params->ref_dec_op); 1308 1309 enqueued += rte_bbdev_enqueue_dec_ops(tp->dev_id, queue_id, ops, 1310 num_to_enq); 1311 1312 rte_bbdev_dec_op_free_bulk(ops, num_to_enq); 1313 } 1314 1315 if (allocs_failed > 0) 1316 printf("WARNING: op allocations failed: %u times\n", 1317 allocs_failed); 1318 1319 return TEST_SUCCESS; 1320 } 1321 1322 static int 1323 throughput_intr_lcore_enc(void *arg) 1324 { 1325 struct thread_params *tp = arg; 1326 unsigned int enqueued; 1327 struct rte_bbdev_enc_op *ops[MAX_BURST]; 1328 const uint16_t queue_id = tp->queue_id; 1329 const uint16_t burst_sz = tp->op_params->burst_sz; 1330 const uint16_t num_to_process = tp->op_params->num_to_process; 1331 struct test_buffers *bufs = NULL; 1332 unsigned int allocs_failed = 0; 1333 struct rte_bbdev_info info; 1334 int ret; 1335 1336 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 1337 "BURST_SIZE should be <= %u", MAX_BURST); 1338 1339 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id), 1340 "Failed to enable interrupts for dev: %u, queue_id: %u", 1341 tp->dev_id, queue_id); 1342 1343 rte_bbdev_info_get(tp->dev_id, &info); 1344 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 1345 1346 rte_atomic16_clear(&tp->processing_status); 1347 rte_atomic16_clear(&tp->nb_dequeued); 1348 1349 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT) 1350 rte_pause(); 1351 1352 tp->start_time = rte_rdtsc_precise(); 1353 for (enqueued = 0; enqueued < num_to_process;) { 1354 1355 uint16_t num_to_enq = burst_sz; 1356 1357 if (unlikely(num_to_process - enqueued < num_to_enq)) 1358 num_to_enq = num_to_process - enqueued; 1359 1360 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops, 1361 num_to_enq); 1362 if (ret != 0) { 1363 allocs_failed++; 1364 continue; 1365 } 1366 1367 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 1368 copy_reference_enc_op(ops, num_to_enq, enqueued, 1369 bufs->inputs, 1370 bufs->hard_outputs, 1371 tp->op_params->ref_enc_op); 1372 1373 enqueued += rte_bbdev_enqueue_enc_ops(tp->dev_id, queue_id, ops, 1374 num_to_enq); 1375 1376 rte_bbdev_enc_op_free_bulk(ops, num_to_enq); 1377 } 1378 1379 if (allocs_failed > 0) 1380 printf("WARNING: op allocations failed: %u times\n", 1381 allocs_failed); 1382 1383 return TEST_SUCCESS; 1384 } 1385 1386 static int 1387 throughput_pmd_lcore_dec(void *arg) 1388 { 1389 struct thread_params *tp = arg; 1390 unsigned int enqueued, dequeued; 1391 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 1392 uint64_t total_time, start_time; 1393 const uint16_t queue_id = tp->queue_id; 1394 const uint16_t burst_sz = tp->op_params->burst_sz; 1395 const uint16_t num_to_process = tp->op_params->num_to_process; 1396 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 1397 struct test_buffers *bufs = NULL; 1398 unsigned int allocs_failed = 0; 1399 int ret; 1400 struct rte_bbdev_info info; 1401 1402 /* Input length in bytes, million operations per second, million bits 1403 * per second. 1404 */ 1405 double in_len; 1406 1407 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 1408 "BURST_SIZE should be <= %u", MAX_BURST); 1409 1410 rte_bbdev_info_get(tp->dev_id, &info); 1411 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 1412 1413 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT) 1414 rte_pause(); 1415 1416 start_time = rte_rdtsc_precise(); 1417 for (enqueued = 0, dequeued = 0; dequeued < num_to_process;) { 1418 uint16_t deq; 1419 1420 if (likely(enqueued < num_to_process)) { 1421 1422 uint16_t num_to_enq = burst_sz; 1423 1424 if (unlikely(num_to_process - enqueued < num_to_enq)) 1425 num_to_enq = num_to_process - enqueued; 1426 1427 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, 1428 ops_enq, num_to_enq); 1429 if (ret != 0) { 1430 allocs_failed++; 1431 goto do_dequeue; 1432 } 1433 1434 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 1435 copy_reference_dec_op(ops_enq, num_to_enq, 1436 enqueued, 1437 bufs->inputs, 1438 bufs->hard_outputs, 1439 bufs->soft_outputs, 1440 ref_op); 1441 1442 enqueued += rte_bbdev_enqueue_dec_ops(tp->dev_id, 1443 queue_id, ops_enq, num_to_enq); 1444 } 1445 do_dequeue: 1446 deq = rte_bbdev_dequeue_dec_ops(tp->dev_id, queue_id, ops_deq, 1447 burst_sz); 1448 dequeued += deq; 1449 rte_bbdev_dec_op_free_bulk(ops_enq, deq); 1450 } 1451 total_time = rte_rdtsc_precise() - start_time; 1452 1453 if (allocs_failed > 0) 1454 printf("WARNING: op allocations failed: %u times\n", 1455 allocs_failed); 1456 1457 TEST_ASSERT(enqueued == dequeued, "enqueued (%u) != dequeued (%u)", 1458 enqueued, dequeued); 1459 1460 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 1461 ret = validate_dec_buffers(ref_op, bufs, num_to_process); 1462 TEST_ASSERT_SUCCESS(ret, "Buffers validation failed"); 1463 } 1464 1465 in_len = ref_op->turbo_dec.input.length; 1466 tp->mops = ((double)num_to_process / 1000000.0) / 1467 ((double)total_time / (double)rte_get_tsc_hz()); 1468 tp->mbps = ((double)num_to_process * in_len * 8 / 1000000.0) / 1469 ((double)total_time / (double)rte_get_tsc_hz()); 1470 1471 return TEST_SUCCESS; 1472 } 1473 1474 static int 1475 throughput_pmd_lcore_enc(void *arg) 1476 { 1477 struct thread_params *tp = arg; 1478 unsigned int enqueued, dequeued; 1479 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 1480 uint64_t total_time, start_time; 1481 const uint16_t queue_id = tp->queue_id; 1482 const uint16_t burst_sz = tp->op_params->burst_sz; 1483 const uint16_t num_to_process = tp->op_params->num_to_process; 1484 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op; 1485 struct test_buffers *bufs = NULL; 1486 unsigned int allocs_failed = 0; 1487 int ret; 1488 struct rte_bbdev_info info; 1489 1490 /* Input length in bytes, million operations per second, million bits 1491 * per second. 1492 */ 1493 double in_len; 1494 1495 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 1496 "BURST_SIZE should be <= %u", MAX_BURST); 1497 1498 rte_bbdev_info_get(tp->dev_id, &info); 1499 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 1500 1501 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT) 1502 rte_pause(); 1503 1504 start_time = rte_rdtsc_precise(); 1505 for (enqueued = 0, dequeued = 0; dequeued < num_to_process;) { 1506 uint16_t deq; 1507 1508 if (likely(enqueued < num_to_process)) { 1509 1510 uint16_t num_to_enq = burst_sz; 1511 1512 if (unlikely(num_to_process - enqueued < num_to_enq)) 1513 num_to_enq = num_to_process - enqueued; 1514 1515 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, 1516 ops_enq, num_to_enq); 1517 if (ret != 0) { 1518 allocs_failed++; 1519 goto do_dequeue; 1520 } 1521 1522 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 1523 copy_reference_enc_op(ops_enq, num_to_enq, 1524 enqueued, 1525 bufs->inputs, 1526 bufs->hard_outputs, 1527 ref_op); 1528 1529 enqueued += rte_bbdev_enqueue_enc_ops(tp->dev_id, 1530 queue_id, ops_enq, num_to_enq); 1531 } 1532 do_dequeue: 1533 deq = rte_bbdev_dequeue_enc_ops(tp->dev_id, queue_id, ops_deq, 1534 burst_sz); 1535 dequeued += deq; 1536 rte_bbdev_enc_op_free_bulk(ops_enq, deq); 1537 } 1538 total_time = rte_rdtsc_precise() - start_time; 1539 1540 if (allocs_failed > 0) 1541 printf("WARNING: op allocations failed: %u times\n", 1542 allocs_failed); 1543 1544 TEST_ASSERT(enqueued == dequeued, "enqueued (%u) != dequeued (%u)", 1545 enqueued, dequeued); 1546 1547 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 1548 ret = validate_enc_buffers(bufs, num_to_process); 1549 TEST_ASSERT_SUCCESS(ret, "Buffers validation failed"); 1550 } 1551 1552 in_len = ref_op->turbo_enc.input.length; 1553 1554 tp->mops = ((double)num_to_process / 1000000.0) / 1555 ((double)total_time / (double)rte_get_tsc_hz()); 1556 tp->mbps = ((double)num_to_process * in_len * 8 / 1000000.0) / 1557 ((double)total_time / (double)rte_get_tsc_hz()); 1558 1559 return TEST_SUCCESS; 1560 } 1561 static void 1562 print_throughput(struct thread_params *t_params, unsigned int used_cores) 1563 { 1564 unsigned int lcore_id, iter = 0; 1565 double total_mops = 0, total_mbps = 0; 1566 1567 RTE_LCORE_FOREACH(lcore_id) { 1568 if (iter++ >= used_cores) 1569 break; 1570 printf("\tlcore_id: %u, throughput: %.8lg MOPS, %.8lg Mbps\n", 1571 lcore_id, t_params[lcore_id].mops, t_params[lcore_id].mbps); 1572 total_mops += t_params[lcore_id].mops; 1573 total_mbps += t_params[lcore_id].mbps; 1574 } 1575 printf( 1576 "\n\tTotal stats for %u cores: throughput: %.8lg MOPS, %.8lg Mbps\n", 1577 used_cores, total_mops, total_mbps); 1578 } 1579 1580 /* 1581 * Test function that determines how long an enqueue + dequeue of a burst 1582 * takes on available lcores. 1583 */ 1584 static int 1585 throughput_test(struct active_device *ad, 1586 struct test_op_params *op_params) 1587 { 1588 int ret; 1589 unsigned int lcore_id, used_cores = 0; 1590 struct thread_params t_params[MAX_QUEUES]; 1591 struct rte_bbdev_info info; 1592 lcore_function_t *throughput_function; 1593 struct thread_params *tp; 1594 uint16_t num_lcores; 1595 const char *op_type_str; 1596 1597 rte_bbdev_info_get(ad->dev_id, &info); 1598 1599 op_type_str = rte_bbdev_op_type_str(test_vector.op_type); 1600 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", 1601 test_vector.op_type); 1602 1603 printf( 1604 "Throughput test: dev: %s, nb_queues: %u, burst size: %u, num ops: %u, num_lcores: %u, op type: %s, int mode: %s, GHz: %lg\n", 1605 info.dev_name, ad->nb_queues, op_params->burst_sz, 1606 op_params->num_to_process, op_params->num_lcores, 1607 op_type_str, 1608 intr_enabled ? "Interrupt mode" : "PMD mode", 1609 (double)rte_get_tsc_hz() / 1000000000.0); 1610 1611 /* Set number of lcores */ 1612 num_lcores = (ad->nb_queues < (op_params->num_lcores)) 1613 ? ad->nb_queues 1614 : op_params->num_lcores; 1615 1616 if (intr_enabled) { 1617 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) 1618 throughput_function = throughput_intr_lcore_dec; 1619 else 1620 throughput_function = throughput_intr_lcore_enc; 1621 1622 /* Dequeue interrupt callback registration */ 1623 ret = rte_bbdev_callback_register(ad->dev_id, 1624 RTE_BBDEV_EVENT_DEQUEUE, dequeue_event_callback, 1625 &t_params); 1626 if (ret < 0) 1627 return ret; 1628 } else { 1629 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) 1630 throughput_function = throughput_pmd_lcore_dec; 1631 else 1632 throughput_function = throughput_pmd_lcore_enc; 1633 } 1634 1635 rte_atomic16_set(&op_params->sync, SYNC_WAIT); 1636 1637 t_params[rte_lcore_id()].dev_id = ad->dev_id; 1638 t_params[rte_lcore_id()].op_params = op_params; 1639 t_params[rte_lcore_id()].queue_id = 1640 ad->queue_ids[used_cores++]; 1641 1642 RTE_LCORE_FOREACH_SLAVE(lcore_id) { 1643 if (used_cores >= num_lcores) 1644 break; 1645 1646 t_params[lcore_id].dev_id = ad->dev_id; 1647 t_params[lcore_id].op_params = op_params; 1648 t_params[lcore_id].queue_id = ad->queue_ids[used_cores++]; 1649 1650 rte_eal_remote_launch(throughput_function, &t_params[lcore_id], 1651 lcore_id); 1652 } 1653 1654 rte_atomic16_set(&op_params->sync, SYNC_START); 1655 ret = throughput_function(&t_params[rte_lcore_id()]); 1656 1657 /* Master core is always used */ 1658 used_cores = 1; 1659 RTE_LCORE_FOREACH_SLAVE(lcore_id) { 1660 if (used_cores++ >= num_lcores) 1661 break; 1662 1663 ret |= rte_eal_wait_lcore(lcore_id); 1664 } 1665 1666 /* Return if test failed */ 1667 if (ret) 1668 return ret; 1669 1670 /* Print throughput if interrupts are disabled and test passed */ 1671 if (!intr_enabled) { 1672 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 1673 print_throughput(t_params, num_lcores); 1674 return ret; 1675 } 1676 1677 /* In interrupt TC we need to wait for the interrupt callback to deqeue 1678 * all pending operations. Skip waiting for queues which reported an 1679 * error using processing_status variable. 1680 * Wait for master lcore operations. 1681 */ 1682 tp = &t_params[rte_lcore_id()]; 1683 while ((rte_atomic16_read(&tp->nb_dequeued) < 1684 op_params->num_to_process) && 1685 (rte_atomic16_read(&tp->processing_status) != 1686 TEST_FAILED)) 1687 rte_pause(); 1688 1689 ret |= rte_atomic16_read(&tp->processing_status); 1690 1691 /* Wait for slave lcores operations */ 1692 used_cores = 1; 1693 RTE_LCORE_FOREACH_SLAVE(lcore_id) { 1694 tp = &t_params[lcore_id]; 1695 if (used_cores++ >= num_lcores) 1696 break; 1697 1698 while ((rte_atomic16_read(&tp->nb_dequeued) < 1699 op_params->num_to_process) && 1700 (rte_atomic16_read(&tp->processing_status) != 1701 TEST_FAILED)) 1702 rte_pause(); 1703 1704 ret |= rte_atomic16_read(&tp->processing_status); 1705 } 1706 1707 /* Print throughput if test passed */ 1708 if (!ret && test_vector.op_type != RTE_BBDEV_OP_NONE) 1709 print_throughput(t_params, num_lcores); 1710 1711 return ret; 1712 } 1713 1714 static int 1715 latency_test_dec(struct rte_mempool *mempool, 1716 struct test_buffers *bufs, struct rte_bbdev_dec_op *ref_op, 1717 int vector_mask, uint16_t dev_id, uint16_t queue_id, 1718 const uint16_t num_to_process, uint16_t burst_sz, 1719 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time) 1720 { 1721 int ret = TEST_SUCCESS; 1722 uint16_t i, j, dequeued; 1723 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 1724 uint64_t start_time = 0, last_time = 0; 1725 1726 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 1727 uint16_t enq = 0, deq = 0; 1728 bool first_time = true; 1729 last_time = 0; 1730 1731 if (unlikely(num_to_process - dequeued < burst_sz)) 1732 burst_sz = num_to_process - dequeued; 1733 1734 ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz); 1735 TEST_ASSERT_SUCCESS(ret, 1736 "rte_bbdev_dec_op_alloc_bulk() failed"); 1737 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 1738 copy_reference_dec_op(ops_enq, burst_sz, dequeued, 1739 bufs->inputs, 1740 bufs->hard_outputs, 1741 bufs->soft_outputs, 1742 ref_op); 1743 1744 /* Set counter to validate the ordering */ 1745 for (j = 0; j < burst_sz; ++j) 1746 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 1747 1748 start_time = rte_rdtsc_precise(); 1749 1750 enq = rte_bbdev_enqueue_dec_ops(dev_id, queue_id, &ops_enq[enq], 1751 burst_sz); 1752 TEST_ASSERT(enq == burst_sz, 1753 "Error enqueueing burst, expected %u, got %u", 1754 burst_sz, enq); 1755 1756 /* Dequeue */ 1757 do { 1758 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id, 1759 &ops_deq[deq], burst_sz - deq); 1760 if (likely(first_time && (deq > 0))) { 1761 last_time = rte_rdtsc_precise() - start_time; 1762 first_time = false; 1763 } 1764 } while (unlikely(burst_sz != deq)); 1765 1766 *max_time = RTE_MAX(*max_time, last_time); 1767 *min_time = RTE_MIN(*min_time, last_time); 1768 *total_time += last_time; 1769 1770 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 1771 ret = validate_dec_op(ops_deq, burst_sz, ref_op, 1772 vector_mask); 1773 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 1774 } 1775 1776 rte_bbdev_dec_op_free_bulk(ops_enq, deq); 1777 dequeued += deq; 1778 } 1779 1780 return i; 1781 } 1782 1783 static int 1784 latency_test_enc(struct rte_mempool *mempool, 1785 struct test_buffers *bufs, struct rte_bbdev_enc_op *ref_op, 1786 uint16_t dev_id, uint16_t queue_id, 1787 const uint16_t num_to_process, uint16_t burst_sz, 1788 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time) 1789 { 1790 int ret = TEST_SUCCESS; 1791 uint16_t i, j, dequeued; 1792 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 1793 uint64_t start_time = 0, last_time = 0; 1794 1795 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 1796 uint16_t enq = 0, deq = 0; 1797 bool first_time = true; 1798 last_time = 0; 1799 1800 if (unlikely(num_to_process - dequeued < burst_sz)) 1801 burst_sz = num_to_process - dequeued; 1802 1803 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz); 1804 TEST_ASSERT_SUCCESS(ret, 1805 "rte_bbdev_enc_op_alloc_bulk() failed"); 1806 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 1807 copy_reference_enc_op(ops_enq, burst_sz, dequeued, 1808 bufs->inputs, 1809 bufs->hard_outputs, 1810 ref_op); 1811 1812 /* Set counter to validate the ordering */ 1813 for (j = 0; j < burst_sz; ++j) 1814 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 1815 1816 start_time = rte_rdtsc_precise(); 1817 1818 enq = rte_bbdev_enqueue_enc_ops(dev_id, queue_id, &ops_enq[enq], 1819 burst_sz); 1820 TEST_ASSERT(enq == burst_sz, 1821 "Error enqueueing burst, expected %u, got %u", 1822 burst_sz, enq); 1823 1824 /* Dequeue */ 1825 do { 1826 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id, 1827 &ops_deq[deq], burst_sz - deq); 1828 if (likely(first_time && (deq > 0))) { 1829 last_time += rte_rdtsc_precise() - start_time; 1830 first_time = false; 1831 } 1832 } while (unlikely(burst_sz != deq)); 1833 1834 *max_time = RTE_MAX(*max_time, last_time); 1835 *min_time = RTE_MIN(*min_time, last_time); 1836 *total_time += last_time; 1837 1838 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 1839 ret = validate_enc_op(ops_deq, burst_sz, ref_op); 1840 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 1841 } 1842 1843 rte_bbdev_enc_op_free_bulk(ops_enq, deq); 1844 dequeued += deq; 1845 } 1846 1847 return i; 1848 } 1849 1850 static int 1851 latency_test(struct active_device *ad, 1852 struct test_op_params *op_params) 1853 { 1854 int iter; 1855 uint16_t burst_sz = op_params->burst_sz; 1856 const uint16_t num_to_process = op_params->num_to_process; 1857 const enum rte_bbdev_op_type op_type = test_vector.op_type; 1858 const uint16_t queue_id = ad->queue_ids[0]; 1859 struct test_buffers *bufs = NULL; 1860 struct rte_bbdev_info info; 1861 uint64_t total_time, min_time, max_time; 1862 const char *op_type_str; 1863 1864 total_time = max_time = 0; 1865 min_time = UINT64_MAX; 1866 1867 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 1868 "BURST_SIZE should be <= %u", MAX_BURST); 1869 1870 rte_bbdev_info_get(ad->dev_id, &info); 1871 bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 1872 1873 op_type_str = rte_bbdev_op_type_str(op_type); 1874 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type); 1875 1876 printf( 1877 "Validation/Latency test: dev: %s, burst size: %u, num ops: %u, op type: %s\n", 1878 info.dev_name, burst_sz, num_to_process, op_type_str); 1879 1880 if (op_type == RTE_BBDEV_OP_TURBO_DEC) 1881 iter = latency_test_dec(op_params->mp, bufs, 1882 op_params->ref_dec_op, op_params->vector_mask, 1883 ad->dev_id, queue_id, num_to_process, 1884 burst_sz, &total_time, &min_time, &max_time); 1885 else 1886 iter = latency_test_enc(op_params->mp, bufs, 1887 op_params->ref_enc_op, ad->dev_id, queue_id, 1888 num_to_process, burst_sz, &total_time, 1889 &min_time, &max_time); 1890 1891 if (iter <= 0) 1892 return TEST_FAILED; 1893 1894 printf("\toperation latency:\n" 1895 "\t\tavg latency: %lg cycles, %lg us\n" 1896 "\t\tmin latency: %lg cycles, %lg us\n" 1897 "\t\tmax latency: %lg cycles, %lg us\n", 1898 (double)total_time / (double)iter, 1899 (double)(total_time * 1000000) / (double)iter / 1900 (double)rte_get_tsc_hz(), (double)min_time, 1901 (double)(min_time * 1000000) / (double)rte_get_tsc_hz(), 1902 (double)max_time, (double)(max_time * 1000000) / 1903 (double)rte_get_tsc_hz()); 1904 1905 return TEST_SUCCESS; 1906 } 1907 1908 #ifdef RTE_BBDEV_OFFLOAD_COST 1909 static int 1910 get_bbdev_queue_stats(uint16_t dev_id, uint16_t queue_id, 1911 struct rte_bbdev_stats *stats) 1912 { 1913 struct rte_bbdev *dev = &rte_bbdev_devices[dev_id]; 1914 struct rte_bbdev_stats *q_stats; 1915 1916 if (queue_id >= dev->data->num_queues) 1917 return -1; 1918 1919 q_stats = &dev->data->queues[queue_id].queue_stats; 1920 1921 stats->enqueued_count = q_stats->enqueued_count; 1922 stats->dequeued_count = q_stats->dequeued_count; 1923 stats->enqueue_err_count = q_stats->enqueue_err_count; 1924 stats->dequeue_err_count = q_stats->dequeue_err_count; 1925 stats->offload_time = q_stats->offload_time; 1926 1927 return 0; 1928 } 1929 1930 static int 1931 offload_latency_test_dec(struct rte_mempool *mempool, struct test_buffers *bufs, 1932 struct rte_bbdev_dec_op *ref_op, uint16_t dev_id, 1933 uint16_t queue_id, const uint16_t num_to_process, 1934 uint16_t burst_sz, struct test_time_stats *time_st) 1935 { 1936 int i, dequeued, ret; 1937 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 1938 uint64_t enq_start_time, deq_start_time; 1939 uint64_t enq_sw_last_time, deq_last_time; 1940 struct rte_bbdev_stats stats; 1941 1942 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 1943 uint16_t enq = 0, deq = 0; 1944 1945 if (unlikely(num_to_process - dequeued < burst_sz)) 1946 burst_sz = num_to_process - dequeued; 1947 1948 rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz); 1949 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 1950 copy_reference_dec_op(ops_enq, burst_sz, dequeued, 1951 bufs->inputs, 1952 bufs->hard_outputs, 1953 bufs->soft_outputs, 1954 ref_op); 1955 1956 /* Start time meas for enqueue function offload latency */ 1957 enq_start_time = rte_rdtsc_precise(); 1958 do { 1959 enq += rte_bbdev_enqueue_dec_ops(dev_id, queue_id, 1960 &ops_enq[enq], burst_sz - enq); 1961 } while (unlikely(burst_sz != enq)); 1962 1963 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats); 1964 TEST_ASSERT_SUCCESS(ret, 1965 "Failed to get stats for queue (%u) of device (%u)", 1966 queue_id, dev_id); 1967 1968 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time - 1969 stats.offload_time; 1970 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time, 1971 enq_sw_last_time); 1972 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time, 1973 enq_sw_last_time); 1974 time_st->enq_sw_tot_time += enq_sw_last_time; 1975 1976 time_st->enq_tur_max_time = RTE_MAX(time_st->enq_tur_max_time, 1977 stats.offload_time); 1978 time_st->enq_tur_min_time = RTE_MIN(time_st->enq_tur_min_time, 1979 stats.offload_time); 1980 time_st->enq_tur_tot_time += stats.offload_time; 1981 1982 /* ensure enqueue has been completed */ 1983 rte_delay_ms(10); 1984 1985 /* Start time meas for dequeue function offload latency */ 1986 deq_start_time = rte_rdtsc_precise(); 1987 /* Dequeue one operation */ 1988 do { 1989 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id, 1990 &ops_deq[deq], 1); 1991 } while (unlikely(deq != 1)); 1992 1993 deq_last_time = rte_rdtsc_precise() - deq_start_time; 1994 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time, 1995 deq_last_time); 1996 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time, 1997 deq_last_time); 1998 time_st->deq_tot_time += deq_last_time; 1999 2000 /* Dequeue remaining operations if needed*/ 2001 while (burst_sz != deq) 2002 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id, 2003 &ops_deq[deq], burst_sz - deq); 2004 2005 rte_bbdev_dec_op_free_bulk(ops_enq, deq); 2006 dequeued += deq; 2007 } 2008 2009 return i; 2010 } 2011 2012 static int 2013 offload_latency_test_enc(struct rte_mempool *mempool, struct test_buffers *bufs, 2014 struct rte_bbdev_enc_op *ref_op, uint16_t dev_id, 2015 uint16_t queue_id, const uint16_t num_to_process, 2016 uint16_t burst_sz, struct test_time_stats *time_st) 2017 { 2018 int i, dequeued, ret; 2019 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 2020 uint64_t enq_start_time, deq_start_time; 2021 uint64_t enq_sw_last_time, deq_last_time; 2022 struct rte_bbdev_stats stats; 2023 2024 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 2025 uint16_t enq = 0, deq = 0; 2026 2027 if (unlikely(num_to_process - dequeued < burst_sz)) 2028 burst_sz = num_to_process - dequeued; 2029 2030 rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz); 2031 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 2032 copy_reference_enc_op(ops_enq, burst_sz, dequeued, 2033 bufs->inputs, 2034 bufs->hard_outputs, 2035 ref_op); 2036 2037 /* Start time meas for enqueue function offload latency */ 2038 enq_start_time = rte_rdtsc_precise(); 2039 do { 2040 enq += rte_bbdev_enqueue_enc_ops(dev_id, queue_id, 2041 &ops_enq[enq], burst_sz - enq); 2042 } while (unlikely(burst_sz != enq)); 2043 2044 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats); 2045 TEST_ASSERT_SUCCESS(ret, 2046 "Failed to get stats for queue (%u) of device (%u)", 2047 queue_id, dev_id); 2048 2049 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time - 2050 stats.offload_time; 2051 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time, 2052 enq_sw_last_time); 2053 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time, 2054 enq_sw_last_time); 2055 time_st->enq_sw_tot_time += enq_sw_last_time; 2056 2057 time_st->enq_tur_max_time = RTE_MAX(time_st->enq_tur_max_time, 2058 stats.offload_time); 2059 time_st->enq_tur_min_time = RTE_MIN(time_st->enq_tur_min_time, 2060 stats.offload_time); 2061 time_st->enq_tur_tot_time += stats.offload_time; 2062 2063 /* ensure enqueue has been completed */ 2064 rte_delay_ms(10); 2065 2066 /* Start time meas for dequeue function offload latency */ 2067 deq_start_time = rte_rdtsc_precise(); 2068 /* Dequeue one operation */ 2069 do { 2070 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id, 2071 &ops_deq[deq], 1); 2072 } while (unlikely(deq != 1)); 2073 2074 deq_last_time = rte_rdtsc_precise() - deq_start_time; 2075 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time, 2076 deq_last_time); 2077 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time, 2078 deq_last_time); 2079 time_st->deq_tot_time += deq_last_time; 2080 2081 while (burst_sz != deq) 2082 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id, 2083 &ops_deq[deq], burst_sz - deq); 2084 2085 rte_bbdev_enc_op_free_bulk(ops_enq, deq); 2086 dequeued += deq; 2087 } 2088 2089 return i; 2090 } 2091 #endif 2092 2093 static int 2094 offload_cost_test(struct active_device *ad, 2095 struct test_op_params *op_params) 2096 { 2097 #ifndef RTE_BBDEV_OFFLOAD_COST 2098 RTE_SET_USED(ad); 2099 RTE_SET_USED(op_params); 2100 printf("Offload latency test is disabled.\n"); 2101 printf("Set RTE_BBDEV_OFFLOAD_COST to 'y' to turn the test on.\n"); 2102 return TEST_SKIPPED; 2103 #else 2104 int iter; 2105 uint16_t burst_sz = op_params->burst_sz; 2106 const uint16_t num_to_process = op_params->num_to_process; 2107 const enum rte_bbdev_op_type op_type = test_vector.op_type; 2108 const uint16_t queue_id = ad->queue_ids[0]; 2109 struct test_buffers *bufs = NULL; 2110 struct rte_bbdev_info info; 2111 const char *op_type_str; 2112 struct test_time_stats time_st; 2113 2114 memset(&time_st, 0, sizeof(struct test_time_stats)); 2115 time_st.enq_sw_min_time = UINT64_MAX; 2116 time_st.enq_tur_min_time = UINT64_MAX; 2117 time_st.deq_min_time = UINT64_MAX; 2118 2119 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 2120 "BURST_SIZE should be <= %u", MAX_BURST); 2121 2122 rte_bbdev_info_get(ad->dev_id, &info); 2123 bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 2124 2125 op_type_str = rte_bbdev_op_type_str(op_type); 2126 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type); 2127 2128 printf( 2129 "Offload latency test: dev: %s, burst size: %u, num ops: %u, op type: %s\n", 2130 info.dev_name, burst_sz, num_to_process, op_type_str); 2131 2132 if (op_type == RTE_BBDEV_OP_TURBO_DEC) 2133 iter = offload_latency_test_dec(op_params->mp, bufs, 2134 op_params->ref_dec_op, ad->dev_id, queue_id, 2135 num_to_process, burst_sz, &time_st); 2136 else 2137 iter = offload_latency_test_enc(op_params->mp, bufs, 2138 op_params->ref_enc_op, ad->dev_id, queue_id, 2139 num_to_process, burst_sz, &time_st); 2140 2141 if (iter <= 0) 2142 return TEST_FAILED; 2143 2144 printf("\tenq offload cost latency:\n" 2145 "\t\tsoftware avg %lg cycles, %lg us\n" 2146 "\t\tsoftware min %lg cycles, %lg us\n" 2147 "\t\tsoftware max %lg cycles, %lg us\n" 2148 "\t\tturbo avg %lg cycles, %lg us\n" 2149 "\t\tturbo min %lg cycles, %lg us\n" 2150 "\t\tturbo max %lg cycles, %lg us\n", 2151 (double)time_st.enq_sw_tot_time / (double)iter, 2152 (double)(time_st.enq_sw_tot_time * 1000000) / 2153 (double)iter / (double)rte_get_tsc_hz(), 2154 (double)time_st.enq_sw_min_time, 2155 (double)(time_st.enq_sw_min_time * 1000000) / 2156 rte_get_tsc_hz(), (double)time_st.enq_sw_max_time, 2157 (double)(time_st.enq_sw_max_time * 1000000) / 2158 rte_get_tsc_hz(), (double)time_st.enq_tur_tot_time / 2159 (double)iter, 2160 (double)(time_st.enq_tur_tot_time * 1000000) / 2161 (double)iter / (double)rte_get_tsc_hz(), 2162 (double)time_st.enq_tur_min_time, 2163 (double)(time_st.enq_tur_min_time * 1000000) / 2164 rte_get_tsc_hz(), (double)time_st.enq_tur_max_time, 2165 (double)(time_st.enq_tur_max_time * 1000000) / 2166 rte_get_tsc_hz()); 2167 2168 printf("\tdeq offload cost latency - one op:\n" 2169 "\t\tavg %lg cycles, %lg us\n" 2170 "\t\tmin %lg cycles, %lg us\n" 2171 "\t\tmax %lg cycles, %lg us\n", 2172 (double)time_st.deq_tot_time / (double)iter, 2173 (double)(time_st.deq_tot_time * 1000000) / 2174 (double)iter / (double)rte_get_tsc_hz(), 2175 (double)time_st.deq_min_time, 2176 (double)(time_st.deq_min_time * 1000000) / 2177 rte_get_tsc_hz(), (double)time_st.deq_max_time, 2178 (double)(time_st.deq_max_time * 1000000) / 2179 rte_get_tsc_hz()); 2180 2181 return TEST_SUCCESS; 2182 #endif 2183 } 2184 2185 #ifdef RTE_BBDEV_OFFLOAD_COST 2186 static int 2187 offload_latency_empty_q_test_dec(uint16_t dev_id, uint16_t queue_id, 2188 const uint16_t num_to_process, uint16_t burst_sz, 2189 uint64_t *deq_tot_time, uint64_t *deq_min_time, 2190 uint64_t *deq_max_time) 2191 { 2192 int i, deq_total; 2193 struct rte_bbdev_dec_op *ops[MAX_BURST]; 2194 uint64_t deq_start_time, deq_last_time; 2195 2196 /* Test deq offload latency from an empty queue */ 2197 2198 for (i = 0, deq_total = 0; deq_total < num_to_process; 2199 ++i, deq_total += burst_sz) { 2200 deq_start_time = rte_rdtsc_precise(); 2201 2202 if (unlikely(num_to_process - deq_total < burst_sz)) 2203 burst_sz = num_to_process - deq_total; 2204 rte_bbdev_dequeue_dec_ops(dev_id, queue_id, ops, burst_sz); 2205 2206 deq_last_time = rte_rdtsc_precise() - deq_start_time; 2207 *deq_max_time = RTE_MAX(*deq_max_time, deq_last_time); 2208 *deq_min_time = RTE_MIN(*deq_min_time, deq_last_time); 2209 *deq_tot_time += deq_last_time; 2210 } 2211 2212 return i; 2213 } 2214 2215 static int 2216 offload_latency_empty_q_test_enc(uint16_t dev_id, uint16_t queue_id, 2217 const uint16_t num_to_process, uint16_t burst_sz, 2218 uint64_t *deq_tot_time, uint64_t *deq_min_time, 2219 uint64_t *deq_max_time) 2220 { 2221 int i, deq_total; 2222 struct rte_bbdev_enc_op *ops[MAX_BURST]; 2223 uint64_t deq_start_time, deq_last_time; 2224 2225 /* Test deq offload latency from an empty queue */ 2226 for (i = 0, deq_total = 0; deq_total < num_to_process; 2227 ++i, deq_total += burst_sz) { 2228 deq_start_time = rte_rdtsc_precise(); 2229 2230 if (unlikely(num_to_process - deq_total < burst_sz)) 2231 burst_sz = num_to_process - deq_total; 2232 rte_bbdev_dequeue_enc_ops(dev_id, queue_id, ops, burst_sz); 2233 2234 deq_last_time = rte_rdtsc_precise() - deq_start_time; 2235 *deq_max_time = RTE_MAX(*deq_max_time, deq_last_time); 2236 *deq_min_time = RTE_MIN(*deq_min_time, deq_last_time); 2237 *deq_tot_time += deq_last_time; 2238 } 2239 2240 return i; 2241 } 2242 #endif 2243 2244 static int 2245 offload_latency_empty_q_test(struct active_device *ad, 2246 struct test_op_params *op_params) 2247 { 2248 #ifndef RTE_BBDEV_OFFLOAD_COST 2249 RTE_SET_USED(ad); 2250 RTE_SET_USED(op_params); 2251 printf("Offload latency empty dequeue test is disabled.\n"); 2252 printf("Set RTE_BBDEV_OFFLOAD_COST to 'y' to turn the test on.\n"); 2253 return TEST_SKIPPED; 2254 #else 2255 int iter; 2256 uint64_t deq_tot_time, deq_min_time, deq_max_time; 2257 uint16_t burst_sz = op_params->burst_sz; 2258 const uint16_t num_to_process = op_params->num_to_process; 2259 const enum rte_bbdev_op_type op_type = test_vector.op_type; 2260 const uint16_t queue_id = ad->queue_ids[0]; 2261 struct rte_bbdev_info info; 2262 const char *op_type_str; 2263 2264 deq_tot_time = deq_max_time = 0; 2265 deq_min_time = UINT64_MAX; 2266 2267 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 2268 "BURST_SIZE should be <= %u", MAX_BURST); 2269 2270 rte_bbdev_info_get(ad->dev_id, &info); 2271 2272 op_type_str = rte_bbdev_op_type_str(op_type); 2273 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type); 2274 2275 printf( 2276 "Offload latency empty dequeue test: dev: %s, burst size: %u, num ops: %u, op type: %s\n", 2277 info.dev_name, burst_sz, num_to_process, op_type_str); 2278 2279 if (op_type == RTE_BBDEV_OP_TURBO_DEC) 2280 iter = offload_latency_empty_q_test_dec(ad->dev_id, queue_id, 2281 num_to_process, burst_sz, &deq_tot_time, 2282 &deq_min_time, &deq_max_time); 2283 else 2284 iter = offload_latency_empty_q_test_enc(ad->dev_id, queue_id, 2285 num_to_process, burst_sz, &deq_tot_time, 2286 &deq_min_time, &deq_max_time); 2287 2288 if (iter <= 0) 2289 return TEST_FAILED; 2290 2291 printf("\tempty deq offload\n" 2292 "\t\tavg. latency: %lg cycles, %lg us\n" 2293 "\t\tmin. latency: %lg cycles, %lg us\n" 2294 "\t\tmax. latency: %lg cycles, %lg us\n", 2295 (double)deq_tot_time / (double)iter, 2296 (double)(deq_tot_time * 1000000) / (double)iter / 2297 (double)rte_get_tsc_hz(), (double)deq_min_time, 2298 (double)(deq_min_time * 1000000) / rte_get_tsc_hz(), 2299 (double)deq_max_time, (double)(deq_max_time * 1000000) / 2300 rte_get_tsc_hz()); 2301 2302 return TEST_SUCCESS; 2303 #endif 2304 } 2305 2306 static int 2307 throughput_tc(void) 2308 { 2309 return run_test_case(throughput_test); 2310 } 2311 2312 static int 2313 offload_cost_tc(void) 2314 { 2315 return run_test_case(offload_cost_test); 2316 } 2317 2318 static int 2319 offload_latency_empty_q_tc(void) 2320 { 2321 return run_test_case(offload_latency_empty_q_test); 2322 } 2323 2324 static int 2325 latency_tc(void) 2326 { 2327 return run_test_case(latency_test); 2328 } 2329 2330 static int 2331 interrupt_tc(void) 2332 { 2333 return run_test_case(throughput_test); 2334 } 2335 2336 static struct unit_test_suite bbdev_throughput_testsuite = { 2337 .suite_name = "BBdev Throughput Tests", 2338 .setup = testsuite_setup, 2339 .teardown = testsuite_teardown, 2340 .unit_test_cases = { 2341 TEST_CASE_ST(ut_setup, ut_teardown, throughput_tc), 2342 TEST_CASES_END() /**< NULL terminate unit test array */ 2343 } 2344 }; 2345 2346 static struct unit_test_suite bbdev_validation_testsuite = { 2347 .suite_name = "BBdev Validation Tests", 2348 .setup = testsuite_setup, 2349 .teardown = testsuite_teardown, 2350 .unit_test_cases = { 2351 TEST_CASE_ST(ut_setup, ut_teardown, latency_tc), 2352 TEST_CASES_END() /**< NULL terminate unit test array */ 2353 } 2354 }; 2355 2356 static struct unit_test_suite bbdev_latency_testsuite = { 2357 .suite_name = "BBdev Latency Tests", 2358 .setup = testsuite_setup, 2359 .teardown = testsuite_teardown, 2360 .unit_test_cases = { 2361 TEST_CASE_ST(ut_setup, ut_teardown, latency_tc), 2362 TEST_CASES_END() /**< NULL terminate unit test array */ 2363 } 2364 }; 2365 2366 static struct unit_test_suite bbdev_offload_cost_testsuite = { 2367 .suite_name = "BBdev Offload Cost Tests", 2368 .setup = testsuite_setup, 2369 .teardown = testsuite_teardown, 2370 .unit_test_cases = { 2371 TEST_CASE_ST(ut_setup, ut_teardown, offload_cost_tc), 2372 TEST_CASE_ST(ut_setup, ut_teardown, offload_latency_empty_q_tc), 2373 TEST_CASES_END() /**< NULL terminate unit test array */ 2374 } 2375 }; 2376 2377 static struct unit_test_suite bbdev_interrupt_testsuite = { 2378 .suite_name = "BBdev Interrupt Tests", 2379 .setup = interrupt_testsuite_setup, 2380 .teardown = testsuite_teardown, 2381 .unit_test_cases = { 2382 TEST_CASE_ST(ut_setup, ut_teardown, interrupt_tc), 2383 TEST_CASES_END() /**< NULL terminate unit test array */ 2384 } 2385 }; 2386 2387 REGISTER_TEST_COMMAND(throughput, bbdev_throughput_testsuite); 2388 REGISTER_TEST_COMMAND(validation, bbdev_validation_testsuite); 2389 REGISTER_TEST_COMMAND(latency, bbdev_latency_testsuite); 2390 REGISTER_TEST_COMMAND(offload, bbdev_offload_cost_testsuite); 2391 REGISTER_TEST_COMMAND(interrupt, bbdev_interrupt_testsuite); 2392