1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017 Intel Corporation 3 */ 4 5 #include <stdio.h> 6 #include <inttypes.h> 7 #include <math.h> 8 9 #include <rte_eal.h> 10 #include <rte_common.h> 11 #include <rte_dev.h> 12 #include <rte_launch.h> 13 #include <rte_bbdev.h> 14 #include <rte_cycles.h> 15 #include <rte_lcore.h> 16 #include <rte_malloc.h> 17 #include <rte_random.h> 18 #include <rte_hexdump.h> 19 #include <rte_interrupts.h> 20 21 #ifdef RTE_LIBRTE_PMD_BBDEV_FPGA_LTE_FEC 22 #include <fpga_lte_fec.h> 23 #endif 24 25 #include "main.h" 26 #include "test_bbdev_vector.h" 27 28 #define GET_SOCKET(socket_id) (((socket_id) == SOCKET_ID_ANY) ? 0 : (socket_id)) 29 30 #define MAX_QUEUES RTE_MAX_LCORE 31 #define TEST_REPETITIONS 1000 32 33 #ifdef RTE_LIBRTE_PMD_BBDEV_FPGA_LTE_FEC 34 #define FPGA_PF_DRIVER_NAME ("intel_fpga_lte_fec_pf") 35 #define FPGA_VF_DRIVER_NAME ("intel_fpga_lte_fec_vf") 36 #define VF_UL_QUEUE_VALUE 4 37 #define VF_DL_QUEUE_VALUE 4 38 #define UL_BANDWIDTH 3 39 #define DL_BANDWIDTH 3 40 #define UL_LOAD_BALANCE 128 41 #define DL_LOAD_BALANCE 128 42 #define FLR_TIMEOUT 610 43 #endif 44 45 #define OPS_CACHE_SIZE 256U 46 #define OPS_POOL_SIZE_MIN 511U /* 0.5K per queue */ 47 48 #define SYNC_WAIT 0 49 #define SYNC_START 1 50 51 #define INVALID_QUEUE_ID -1 52 53 static struct test_bbdev_vector test_vector; 54 55 /* Switch between PMD and Interrupt for throughput TC */ 56 static bool intr_enabled; 57 58 /* Represents tested active devices */ 59 static struct active_device { 60 const char *driver_name; 61 uint8_t dev_id; 62 uint16_t supported_ops; 63 uint16_t queue_ids[MAX_QUEUES]; 64 uint16_t nb_queues; 65 struct rte_mempool *ops_mempool; 66 struct rte_mempool *in_mbuf_pool; 67 struct rte_mempool *hard_out_mbuf_pool; 68 struct rte_mempool *soft_out_mbuf_pool; 69 struct rte_mempool *harq_in_mbuf_pool; 70 struct rte_mempool *harq_out_mbuf_pool; 71 } active_devs[RTE_BBDEV_MAX_DEVS]; 72 73 static uint8_t nb_active_devs; 74 75 /* Data buffers used by BBDEV ops */ 76 struct test_buffers { 77 struct rte_bbdev_op_data *inputs; 78 struct rte_bbdev_op_data *hard_outputs; 79 struct rte_bbdev_op_data *soft_outputs; 80 struct rte_bbdev_op_data *harq_inputs; 81 struct rte_bbdev_op_data *harq_outputs; 82 }; 83 84 /* Operation parameters specific for given test case */ 85 struct test_op_params { 86 struct rte_mempool *mp; 87 struct rte_bbdev_dec_op *ref_dec_op; 88 struct rte_bbdev_enc_op *ref_enc_op; 89 uint16_t burst_sz; 90 uint16_t num_to_process; 91 uint16_t num_lcores; 92 int vector_mask; 93 rte_atomic16_t sync; 94 struct test_buffers q_bufs[RTE_MAX_NUMA_NODES][MAX_QUEUES]; 95 }; 96 97 /* Contains per lcore params */ 98 struct thread_params { 99 uint8_t dev_id; 100 uint16_t queue_id; 101 uint32_t lcore_id; 102 uint64_t start_time; 103 double ops_per_sec; 104 double mbps; 105 uint8_t iter_count; 106 rte_atomic16_t nb_dequeued; 107 rte_atomic16_t processing_status; 108 rte_atomic16_t burst_sz; 109 struct test_op_params *op_params; 110 struct rte_bbdev_dec_op *dec_ops[MAX_BURST]; 111 struct rte_bbdev_enc_op *enc_ops[MAX_BURST]; 112 }; 113 114 #ifdef RTE_BBDEV_OFFLOAD_COST 115 /* Stores time statistics */ 116 struct test_time_stats { 117 /* Stores software enqueue total working time */ 118 uint64_t enq_sw_total_time; 119 /* Stores minimum value of software enqueue working time */ 120 uint64_t enq_sw_min_time; 121 /* Stores maximum value of software enqueue working time */ 122 uint64_t enq_sw_max_time; 123 /* Stores turbo enqueue total working time */ 124 uint64_t enq_acc_total_time; 125 /* Stores minimum value of accelerator enqueue working time */ 126 uint64_t enq_acc_min_time; 127 /* Stores maximum value of accelerator enqueue working time */ 128 uint64_t enq_acc_max_time; 129 /* Stores dequeue total working time */ 130 uint64_t deq_total_time; 131 /* Stores minimum value of dequeue working time */ 132 uint64_t deq_min_time; 133 /* Stores maximum value of dequeue working time */ 134 uint64_t deq_max_time; 135 }; 136 #endif 137 138 typedef int (test_case_function)(struct active_device *ad, 139 struct test_op_params *op_params); 140 141 static inline void 142 mbuf_reset(struct rte_mbuf *m) 143 { 144 m->pkt_len = 0; 145 146 do { 147 m->data_len = 0; 148 m = m->next; 149 } while (m != NULL); 150 } 151 152 /* Read flag value 0/1 from bitmap */ 153 static inline bool 154 check_bit(uint32_t bitmap, uint32_t bitmask) 155 { 156 return bitmap & bitmask; 157 } 158 159 static inline void 160 set_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type) 161 { 162 ad->supported_ops |= (1 << op_type); 163 } 164 165 static inline bool 166 is_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type) 167 { 168 return ad->supported_ops & (1 << op_type); 169 } 170 171 static inline bool 172 flags_match(uint32_t flags_req, uint32_t flags_present) 173 { 174 return (flags_req & flags_present) == flags_req; 175 } 176 177 static void 178 clear_soft_out_cap(uint32_t *op_flags) 179 { 180 *op_flags &= ~RTE_BBDEV_TURBO_SOFT_OUTPUT; 181 *op_flags &= ~RTE_BBDEV_TURBO_POS_LLR_1_BIT_SOFT_OUT; 182 *op_flags &= ~RTE_BBDEV_TURBO_NEG_LLR_1_BIT_SOFT_OUT; 183 } 184 185 static int 186 check_dev_cap(const struct rte_bbdev_info *dev_info) 187 { 188 unsigned int i; 189 unsigned int nb_inputs, nb_soft_outputs, nb_hard_outputs, 190 nb_harq_inputs, nb_harq_outputs; 191 const struct rte_bbdev_op_cap *op_cap = dev_info->drv.capabilities; 192 193 nb_inputs = test_vector.entries[DATA_INPUT].nb_segments; 194 nb_soft_outputs = test_vector.entries[DATA_SOFT_OUTPUT].nb_segments; 195 nb_hard_outputs = test_vector.entries[DATA_HARD_OUTPUT].nb_segments; 196 nb_harq_inputs = test_vector.entries[DATA_HARQ_INPUT].nb_segments; 197 nb_harq_outputs = test_vector.entries[DATA_HARQ_OUTPUT].nb_segments; 198 199 for (i = 0; op_cap->type != RTE_BBDEV_OP_NONE; ++i, ++op_cap) { 200 if (op_cap->type != test_vector.op_type) 201 continue; 202 203 if (op_cap->type == RTE_BBDEV_OP_TURBO_DEC) { 204 const struct rte_bbdev_op_cap_turbo_dec *cap = 205 &op_cap->cap.turbo_dec; 206 /* Ignore lack of soft output capability, just skip 207 * checking if soft output is valid. 208 */ 209 if ((test_vector.turbo_dec.op_flags & 210 RTE_BBDEV_TURBO_SOFT_OUTPUT) && 211 !(cap->capability_flags & 212 RTE_BBDEV_TURBO_SOFT_OUTPUT)) { 213 printf( 214 "INFO: Device \"%s\" does not support soft output - soft output flags will be ignored.\n", 215 dev_info->dev_name); 216 clear_soft_out_cap( 217 &test_vector.turbo_dec.op_flags); 218 } 219 220 if (!flags_match(test_vector.turbo_dec.op_flags, 221 cap->capability_flags)) 222 return TEST_FAILED; 223 if (nb_inputs > cap->num_buffers_src) { 224 printf("Too many inputs defined: %u, max: %u\n", 225 nb_inputs, cap->num_buffers_src); 226 return TEST_FAILED; 227 } 228 if (nb_soft_outputs > cap->num_buffers_soft_out && 229 (test_vector.turbo_dec.op_flags & 230 RTE_BBDEV_TURBO_SOFT_OUTPUT)) { 231 printf( 232 "Too many soft outputs defined: %u, max: %u\n", 233 nb_soft_outputs, 234 cap->num_buffers_soft_out); 235 return TEST_FAILED; 236 } 237 if (nb_hard_outputs > cap->num_buffers_hard_out) { 238 printf( 239 "Too many hard outputs defined: %u, max: %u\n", 240 nb_hard_outputs, 241 cap->num_buffers_hard_out); 242 return TEST_FAILED; 243 } 244 if (intr_enabled && !(cap->capability_flags & 245 RTE_BBDEV_TURBO_DEC_INTERRUPTS)) { 246 printf( 247 "Dequeue interrupts are not supported!\n"); 248 return TEST_FAILED; 249 } 250 251 return TEST_SUCCESS; 252 } else if (op_cap->type == RTE_BBDEV_OP_TURBO_ENC) { 253 const struct rte_bbdev_op_cap_turbo_enc *cap = 254 &op_cap->cap.turbo_enc; 255 256 if (!flags_match(test_vector.turbo_enc.op_flags, 257 cap->capability_flags)) 258 return TEST_FAILED; 259 if (nb_inputs > cap->num_buffers_src) { 260 printf("Too many inputs defined: %u, max: %u\n", 261 nb_inputs, cap->num_buffers_src); 262 return TEST_FAILED; 263 } 264 if (nb_hard_outputs > cap->num_buffers_dst) { 265 printf( 266 "Too many hard outputs defined: %u, max: %u\n", 267 nb_hard_outputs, cap->num_buffers_dst); 268 return TEST_FAILED; 269 } 270 if (intr_enabled && !(cap->capability_flags & 271 RTE_BBDEV_TURBO_ENC_INTERRUPTS)) { 272 printf( 273 "Dequeue interrupts are not supported!\n"); 274 return TEST_FAILED; 275 } 276 277 return TEST_SUCCESS; 278 } else if (op_cap->type == RTE_BBDEV_OP_LDPC_ENC) { 279 const struct rte_bbdev_op_cap_ldpc_enc *cap = 280 &op_cap->cap.ldpc_enc; 281 282 if (!flags_match(test_vector.ldpc_enc.op_flags, 283 cap->capability_flags)){ 284 printf("Flag Mismatch\n"); 285 return TEST_FAILED; 286 } 287 if (nb_inputs > cap->num_buffers_src) { 288 printf("Too many inputs defined: %u, max: %u\n", 289 nb_inputs, cap->num_buffers_src); 290 return TEST_FAILED; 291 } 292 if (nb_hard_outputs > cap->num_buffers_dst) { 293 printf( 294 "Too many hard outputs defined: %u, max: %u\n", 295 nb_hard_outputs, cap->num_buffers_dst); 296 return TEST_FAILED; 297 } 298 if (intr_enabled && !(cap->capability_flags & 299 RTE_BBDEV_TURBO_ENC_INTERRUPTS)) { 300 printf( 301 "Dequeue interrupts are not supported!\n"); 302 return TEST_FAILED; 303 } 304 305 return TEST_SUCCESS; 306 } else if (op_cap->type == RTE_BBDEV_OP_LDPC_DEC) { 307 const struct rte_bbdev_op_cap_ldpc_dec *cap = 308 &op_cap->cap.ldpc_dec; 309 310 if (!flags_match(test_vector.ldpc_dec.op_flags, 311 cap->capability_flags)){ 312 printf("Flag Mismatch\n"); 313 return TEST_FAILED; 314 } 315 if (nb_inputs > cap->num_buffers_src) { 316 printf("Too many inputs defined: %u, max: %u\n", 317 nb_inputs, cap->num_buffers_src); 318 return TEST_FAILED; 319 } 320 if (nb_hard_outputs > cap->num_buffers_hard_out) { 321 printf( 322 "Too many hard outputs defined: %u, max: %u\n", 323 nb_hard_outputs, 324 cap->num_buffers_hard_out); 325 return TEST_FAILED; 326 } 327 if (nb_harq_inputs > cap->num_buffers_hard_out) { 328 printf( 329 "Too many HARQ inputs defined: %u, max: %u\n", 330 nb_hard_outputs, 331 cap->num_buffers_hard_out); 332 return TEST_FAILED; 333 } 334 if (nb_harq_outputs > cap->num_buffers_hard_out) { 335 printf( 336 "Too many HARQ outputs defined: %u, max: %u\n", 337 nb_hard_outputs, 338 cap->num_buffers_hard_out); 339 return TEST_FAILED; 340 } 341 if (intr_enabled && !(cap->capability_flags & 342 RTE_BBDEV_TURBO_DEC_INTERRUPTS)) { 343 printf( 344 "Dequeue interrupts are not supported!\n"); 345 return TEST_FAILED; 346 } 347 348 return TEST_SUCCESS; 349 } 350 } 351 352 if ((i == 0) && (test_vector.op_type == RTE_BBDEV_OP_NONE)) 353 return TEST_SUCCESS; /* Special case for NULL device */ 354 355 return TEST_FAILED; 356 } 357 358 /* calculates optimal mempool size not smaller than the val */ 359 static unsigned int 360 optimal_mempool_size(unsigned int val) 361 { 362 return rte_align32pow2(val + 1) - 1; 363 } 364 365 /* allocates mbuf mempool for inputs and outputs */ 366 static struct rte_mempool * 367 create_mbuf_pool(struct op_data_entries *entries, uint8_t dev_id, 368 int socket_id, unsigned int mbuf_pool_size, 369 const char *op_type_str) 370 { 371 unsigned int i; 372 uint32_t max_seg_sz = 0; 373 char pool_name[RTE_MEMPOOL_NAMESIZE]; 374 375 /* find max input segment size */ 376 for (i = 0; i < entries->nb_segments; ++i) 377 if (entries->segments[i].length > max_seg_sz) 378 max_seg_sz = entries->segments[i].length; 379 380 snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str, 381 dev_id); 382 return rte_pktmbuf_pool_create(pool_name, mbuf_pool_size, 0, 0, 383 RTE_MAX(max_seg_sz + RTE_PKTMBUF_HEADROOM, 384 (unsigned int)RTE_MBUF_DEFAULT_BUF_SIZE), socket_id); 385 } 386 387 static int 388 create_mempools(struct active_device *ad, int socket_id, 389 enum rte_bbdev_op_type org_op_type, uint16_t num_ops) 390 { 391 struct rte_mempool *mp; 392 unsigned int ops_pool_size, mbuf_pool_size = 0; 393 char pool_name[RTE_MEMPOOL_NAMESIZE]; 394 const char *op_type_str; 395 enum rte_bbdev_op_type op_type = org_op_type; 396 397 struct op_data_entries *in = &test_vector.entries[DATA_INPUT]; 398 struct op_data_entries *hard_out = 399 &test_vector.entries[DATA_HARD_OUTPUT]; 400 struct op_data_entries *soft_out = 401 &test_vector.entries[DATA_SOFT_OUTPUT]; 402 struct op_data_entries *harq_in = 403 &test_vector.entries[DATA_HARQ_INPUT]; 404 struct op_data_entries *harq_out = 405 &test_vector.entries[DATA_HARQ_OUTPUT]; 406 407 /* allocate ops mempool */ 408 ops_pool_size = optimal_mempool_size(RTE_MAX( 409 /* Ops used plus 1 reference op */ 410 RTE_MAX((unsigned int)(ad->nb_queues * num_ops + 1), 411 /* Minimal cache size plus 1 reference op */ 412 (unsigned int)(1.5 * rte_lcore_count() * 413 OPS_CACHE_SIZE + 1)), 414 OPS_POOL_SIZE_MIN)); 415 416 if (org_op_type == RTE_BBDEV_OP_NONE) 417 op_type = RTE_BBDEV_OP_TURBO_ENC; 418 419 op_type_str = rte_bbdev_op_type_str(op_type); 420 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type); 421 422 snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str, 423 ad->dev_id); 424 mp = rte_bbdev_op_pool_create(pool_name, op_type, 425 ops_pool_size, OPS_CACHE_SIZE, socket_id); 426 TEST_ASSERT_NOT_NULL(mp, 427 "ERROR Failed to create %u items ops pool for dev %u on socket %u.", 428 ops_pool_size, 429 ad->dev_id, 430 socket_id); 431 ad->ops_mempool = mp; 432 433 /* Do not create inputs and outputs mbufs for BaseBand Null Device */ 434 if (org_op_type == RTE_BBDEV_OP_NONE) 435 return TEST_SUCCESS; 436 437 /* Inputs */ 438 mbuf_pool_size = optimal_mempool_size(ops_pool_size * in->nb_segments); 439 mp = create_mbuf_pool(in, ad->dev_id, socket_id, mbuf_pool_size, "in"); 440 TEST_ASSERT_NOT_NULL(mp, 441 "ERROR Failed to create %u items input pktmbuf pool for dev %u on socket %u.", 442 mbuf_pool_size, 443 ad->dev_id, 444 socket_id); 445 ad->in_mbuf_pool = mp; 446 447 /* Hard outputs */ 448 mbuf_pool_size = optimal_mempool_size(ops_pool_size * 449 hard_out->nb_segments); 450 mp = create_mbuf_pool(hard_out, ad->dev_id, socket_id, mbuf_pool_size, 451 "hard_out"); 452 TEST_ASSERT_NOT_NULL(mp, 453 "ERROR Failed to create %u items hard output pktmbuf pool for dev %u on socket %u.", 454 mbuf_pool_size, 455 ad->dev_id, 456 socket_id); 457 ad->hard_out_mbuf_pool = mp; 458 459 460 /* Soft outputs */ 461 if (soft_out->nb_segments > 0) { 462 mbuf_pool_size = optimal_mempool_size(ops_pool_size * 463 soft_out->nb_segments); 464 mp = create_mbuf_pool(soft_out, ad->dev_id, socket_id, 465 mbuf_pool_size, 466 "soft_out"); 467 TEST_ASSERT_NOT_NULL(mp, 468 "ERROR Failed to create %uB soft output pktmbuf pool for dev %u on socket %u.", 469 mbuf_pool_size, 470 ad->dev_id, 471 socket_id); 472 ad->soft_out_mbuf_pool = mp; 473 } 474 475 /* HARQ inputs */ 476 if (harq_in->nb_segments > 0) { 477 mbuf_pool_size = optimal_mempool_size(ops_pool_size * 478 harq_in->nb_segments); 479 mp = create_mbuf_pool(harq_in, ad->dev_id, socket_id, 480 mbuf_pool_size, 481 "harq_in"); 482 TEST_ASSERT_NOT_NULL(mp, 483 "ERROR Failed to create %uB harq input pktmbuf pool for dev %u on socket %u.", 484 mbuf_pool_size, 485 ad->dev_id, 486 socket_id); 487 ad->harq_in_mbuf_pool = mp; 488 } 489 490 /* HARQ outputs */ 491 if (harq_out->nb_segments > 0) { 492 mbuf_pool_size = optimal_mempool_size(ops_pool_size * 493 harq_out->nb_segments); 494 mp = create_mbuf_pool(harq_out, ad->dev_id, socket_id, 495 mbuf_pool_size, 496 "harq_out"); 497 TEST_ASSERT_NOT_NULL(mp, 498 "ERROR Failed to create %uB harq output pktmbuf pool for dev %u on socket %u.", 499 mbuf_pool_size, 500 ad->dev_id, 501 socket_id); 502 ad->harq_out_mbuf_pool = mp; 503 } 504 505 return TEST_SUCCESS; 506 } 507 508 static int 509 add_bbdev_dev(uint8_t dev_id, struct rte_bbdev_info *info, 510 struct test_bbdev_vector *vector) 511 { 512 int ret; 513 unsigned int queue_id; 514 struct rte_bbdev_queue_conf qconf; 515 struct active_device *ad = &active_devs[nb_active_devs]; 516 unsigned int nb_queues; 517 enum rte_bbdev_op_type op_type = vector->op_type; 518 519 /* Configure fpga lte fec with PF & VF values 520 * if '-i' flag is set and using fpga device 521 */ 522 #ifdef RTE_LIBRTE_PMD_BBDEV_FPGA_LTE_FEC 523 if ((get_init_device() == true) && 524 (!strcmp(info->drv.driver_name, FPGA_PF_DRIVER_NAME))) { 525 struct fpga_lte_fec_conf conf; 526 unsigned int i; 527 528 printf("Configure FPGA FEC Driver %s with default values\n", 529 info->drv.driver_name); 530 531 /* clear default configuration before initialization */ 532 memset(&conf, 0, sizeof(struct fpga_lte_fec_conf)); 533 534 /* Set PF mode : 535 * true if PF is used for data plane 536 * false for VFs 537 */ 538 conf.pf_mode_en = true; 539 540 for (i = 0; i < FPGA_LTE_FEC_NUM_VFS; ++i) { 541 /* Number of UL queues per VF (fpga supports 8 VFs) */ 542 conf.vf_ul_queues_number[i] = VF_UL_QUEUE_VALUE; 543 /* Number of DL queues per VF (fpga supports 8 VFs) */ 544 conf.vf_dl_queues_number[i] = VF_DL_QUEUE_VALUE; 545 } 546 547 /* UL bandwidth. Needed for schedule algorithm */ 548 conf.ul_bandwidth = UL_BANDWIDTH; 549 /* DL bandwidth */ 550 conf.dl_bandwidth = DL_BANDWIDTH; 551 552 /* UL & DL load Balance Factor to 64 */ 553 conf.ul_load_balance = UL_LOAD_BALANCE; 554 conf.dl_load_balance = DL_LOAD_BALANCE; 555 556 /**< FLR timeout value */ 557 conf.flr_time_out = FLR_TIMEOUT; 558 559 /* setup FPGA PF with configuration information */ 560 ret = fpga_lte_fec_configure(info->dev_name, &conf); 561 TEST_ASSERT_SUCCESS(ret, 562 "Failed to configure 4G FPGA PF for bbdev %s", 563 info->dev_name); 564 } 565 #endif 566 nb_queues = RTE_MIN(rte_lcore_count(), info->drv.max_num_queues); 567 nb_queues = RTE_MIN(nb_queues, (unsigned int) MAX_QUEUES); 568 569 /* setup device */ 570 ret = rte_bbdev_setup_queues(dev_id, nb_queues, info->socket_id); 571 if (ret < 0) { 572 printf("rte_bbdev_setup_queues(%u, %u, %d) ret %i\n", 573 dev_id, nb_queues, info->socket_id, ret); 574 return TEST_FAILED; 575 } 576 577 /* configure interrupts if needed */ 578 if (intr_enabled) { 579 ret = rte_bbdev_intr_enable(dev_id); 580 if (ret < 0) { 581 printf("rte_bbdev_intr_enable(%u) ret %i\n", dev_id, 582 ret); 583 return TEST_FAILED; 584 } 585 } 586 587 /* setup device queues */ 588 qconf.socket = info->socket_id; 589 qconf.queue_size = info->drv.default_queue_conf.queue_size; 590 qconf.priority = 0; 591 qconf.deferred_start = 0; 592 qconf.op_type = op_type; 593 594 for (queue_id = 0; queue_id < nb_queues; ++queue_id) { 595 ret = rte_bbdev_queue_configure(dev_id, queue_id, &qconf); 596 if (ret != 0) { 597 printf( 598 "Allocated all queues (id=%u) at prio%u on dev%u\n", 599 queue_id, qconf.priority, dev_id); 600 qconf.priority++; 601 ret = rte_bbdev_queue_configure(ad->dev_id, queue_id, 602 &qconf); 603 } 604 if (ret != 0) { 605 printf("All queues on dev %u allocated: %u\n", 606 dev_id, queue_id); 607 break; 608 } 609 ad->queue_ids[queue_id] = queue_id; 610 } 611 TEST_ASSERT(queue_id != 0, 612 "ERROR Failed to configure any queues on dev %u", 613 dev_id); 614 ad->nb_queues = queue_id; 615 616 set_avail_op(ad, op_type); 617 618 return TEST_SUCCESS; 619 } 620 621 static int 622 add_active_device(uint8_t dev_id, struct rte_bbdev_info *info, 623 struct test_bbdev_vector *vector) 624 { 625 int ret; 626 627 active_devs[nb_active_devs].driver_name = info->drv.driver_name; 628 active_devs[nb_active_devs].dev_id = dev_id; 629 630 ret = add_bbdev_dev(dev_id, info, vector); 631 if (ret == TEST_SUCCESS) 632 ++nb_active_devs; 633 return ret; 634 } 635 636 static uint8_t 637 populate_active_devices(void) 638 { 639 int ret; 640 uint8_t dev_id; 641 uint8_t nb_devs_added = 0; 642 struct rte_bbdev_info info; 643 644 RTE_BBDEV_FOREACH(dev_id) { 645 rte_bbdev_info_get(dev_id, &info); 646 647 if (check_dev_cap(&info)) { 648 printf( 649 "Device %d (%s) does not support specified capabilities\n", 650 dev_id, info.dev_name); 651 continue; 652 } 653 654 ret = add_active_device(dev_id, &info, &test_vector); 655 if (ret != 0) { 656 printf("Adding active bbdev %s skipped\n", 657 info.dev_name); 658 continue; 659 } 660 nb_devs_added++; 661 } 662 663 return nb_devs_added; 664 } 665 666 static int 667 read_test_vector(void) 668 { 669 int ret; 670 671 memset(&test_vector, 0, sizeof(test_vector)); 672 printf("Test vector file = %s\n", get_vector_filename()); 673 ret = test_bbdev_vector_read(get_vector_filename(), &test_vector); 674 TEST_ASSERT_SUCCESS(ret, "Failed to parse file %s\n", 675 get_vector_filename()); 676 677 return TEST_SUCCESS; 678 } 679 680 static int 681 testsuite_setup(void) 682 { 683 TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n"); 684 685 if (populate_active_devices() == 0) { 686 printf("No suitable devices found!\n"); 687 return TEST_SKIPPED; 688 } 689 690 return TEST_SUCCESS; 691 } 692 693 static int 694 interrupt_testsuite_setup(void) 695 { 696 TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n"); 697 698 /* Enable interrupts */ 699 intr_enabled = true; 700 701 /* Special case for NULL device (RTE_BBDEV_OP_NONE) */ 702 if (populate_active_devices() == 0 || 703 test_vector.op_type == RTE_BBDEV_OP_NONE) { 704 intr_enabled = false; 705 printf("No suitable devices found!\n"); 706 return TEST_SKIPPED; 707 } 708 709 return TEST_SUCCESS; 710 } 711 712 static void 713 testsuite_teardown(void) 714 { 715 uint8_t dev_id; 716 717 /* Unconfigure devices */ 718 RTE_BBDEV_FOREACH(dev_id) 719 rte_bbdev_close(dev_id); 720 721 /* Clear active devices structs. */ 722 memset(active_devs, 0, sizeof(active_devs)); 723 nb_active_devs = 0; 724 } 725 726 static int 727 ut_setup(void) 728 { 729 uint8_t i, dev_id; 730 731 for (i = 0; i < nb_active_devs; i++) { 732 dev_id = active_devs[i].dev_id; 733 /* reset bbdev stats */ 734 TEST_ASSERT_SUCCESS(rte_bbdev_stats_reset(dev_id), 735 "Failed to reset stats of bbdev %u", dev_id); 736 /* start the device */ 737 TEST_ASSERT_SUCCESS(rte_bbdev_start(dev_id), 738 "Failed to start bbdev %u", dev_id); 739 } 740 741 return TEST_SUCCESS; 742 } 743 744 static void 745 ut_teardown(void) 746 { 747 uint8_t i, dev_id; 748 struct rte_bbdev_stats stats; 749 750 for (i = 0; i < nb_active_devs; i++) { 751 dev_id = active_devs[i].dev_id; 752 /* read stats and print */ 753 rte_bbdev_stats_get(dev_id, &stats); 754 /* Stop the device */ 755 rte_bbdev_stop(dev_id); 756 } 757 } 758 759 static int 760 init_op_data_objs(struct rte_bbdev_op_data *bufs, 761 struct op_data_entries *ref_entries, 762 struct rte_mempool *mbuf_pool, const uint16_t n, 763 enum op_data_type op_type, uint16_t min_alignment) 764 { 765 int ret; 766 unsigned int i, j; 767 768 for (i = 0; i < n; ++i) { 769 char *data; 770 struct op_data_buf *seg = &ref_entries->segments[0]; 771 struct rte_mbuf *m_head = rte_pktmbuf_alloc(mbuf_pool); 772 TEST_ASSERT_NOT_NULL(m_head, 773 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)", 774 op_type, n * ref_entries->nb_segments, 775 mbuf_pool->size); 776 777 TEST_ASSERT_SUCCESS(((seg->length + RTE_PKTMBUF_HEADROOM) > 778 (uint32_t)UINT16_MAX), 779 "Given data is bigger than allowed mbuf segment size"); 780 781 bufs[i].data = m_head; 782 bufs[i].offset = 0; 783 bufs[i].length = 0; 784 785 if ((op_type == DATA_INPUT) || (op_type == DATA_HARQ_INPUT)) { 786 data = rte_pktmbuf_append(m_head, seg->length); 787 TEST_ASSERT_NOT_NULL(data, 788 "Couldn't append %u bytes to mbuf from %d data type mbuf pool", 789 seg->length, op_type); 790 791 TEST_ASSERT(data == RTE_PTR_ALIGN(data, min_alignment), 792 "Data addr in mbuf (%p) is not aligned to device min alignment (%u)", 793 data, min_alignment); 794 rte_memcpy(data, seg->addr, seg->length); 795 bufs[i].length += seg->length; 796 797 for (j = 1; j < ref_entries->nb_segments; ++j) { 798 struct rte_mbuf *m_tail = 799 rte_pktmbuf_alloc(mbuf_pool); 800 TEST_ASSERT_NOT_NULL(m_tail, 801 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)", 802 op_type, 803 n * ref_entries->nb_segments, 804 mbuf_pool->size); 805 seg += 1; 806 807 data = rte_pktmbuf_append(m_tail, seg->length); 808 TEST_ASSERT_NOT_NULL(data, 809 "Couldn't append %u bytes to mbuf from %d data type mbuf pool", 810 seg->length, op_type); 811 812 TEST_ASSERT(data == RTE_PTR_ALIGN(data, 813 min_alignment), 814 "Data addr in mbuf (%p) is not aligned to device min alignment (%u)", 815 data, min_alignment); 816 rte_memcpy(data, seg->addr, seg->length); 817 bufs[i].length += seg->length; 818 819 ret = rte_pktmbuf_chain(m_head, m_tail); 820 TEST_ASSERT_SUCCESS(ret, 821 "Couldn't chain mbufs from %d data type mbuf pool", 822 op_type); 823 } 824 } else { 825 826 /* allocate chained-mbuf for output buffer */ 827 for (j = 1; j < ref_entries->nb_segments; ++j) { 828 struct rte_mbuf *m_tail = 829 rte_pktmbuf_alloc(mbuf_pool); 830 TEST_ASSERT_NOT_NULL(m_tail, 831 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)", 832 op_type, 833 n * ref_entries->nb_segments, 834 mbuf_pool->size); 835 836 ret = rte_pktmbuf_chain(m_head, m_tail); 837 TEST_ASSERT_SUCCESS(ret, 838 "Couldn't chain mbufs from %d data type mbuf pool", 839 op_type); 840 } 841 } 842 } 843 844 return 0; 845 } 846 847 static int 848 allocate_buffers_on_socket(struct rte_bbdev_op_data **buffers, const int len, 849 const int socket) 850 { 851 int i; 852 853 *buffers = rte_zmalloc_socket(NULL, len, 0, socket); 854 if (*buffers == NULL) { 855 printf("WARNING: Failed to allocate op_data on socket %d\n", 856 socket); 857 /* try to allocate memory on other detected sockets */ 858 for (i = 0; i < socket; i++) { 859 *buffers = rte_zmalloc_socket(NULL, len, 0, i); 860 if (*buffers != NULL) 861 break; 862 } 863 } 864 865 return (*buffers == NULL) ? TEST_FAILED : TEST_SUCCESS; 866 } 867 868 static void 869 limit_input_llr_val_range(struct rte_bbdev_op_data *input_ops, 870 const uint16_t n, const int8_t max_llr_modulus) 871 { 872 uint16_t i, byte_idx; 873 874 for (i = 0; i < n; ++i) { 875 struct rte_mbuf *m = input_ops[i].data; 876 while (m != NULL) { 877 int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *, 878 input_ops[i].offset); 879 for (byte_idx = 0; byte_idx < rte_pktmbuf_data_len(m); 880 ++byte_idx) 881 llr[byte_idx] = round((double)max_llr_modulus * 882 llr[byte_idx] / INT8_MAX); 883 884 m = m->next; 885 } 886 } 887 } 888 889 static void 890 ldpc_input_llr_scaling(struct rte_bbdev_op_data *input_ops, 891 const uint16_t n, const int8_t llr_size, 892 const int8_t llr_decimals) 893 { 894 if (input_ops == NULL) 895 return; 896 897 uint16_t i, byte_idx; 898 899 int16_t llr_max, llr_min, llr_tmp; 900 llr_max = (1 << (llr_size - 1)) - 1; 901 llr_min = -llr_max; 902 for (i = 0; i < n; ++i) { 903 struct rte_mbuf *m = input_ops[i].data; 904 while (m != NULL) { 905 int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *, 906 input_ops[i].offset); 907 for (byte_idx = 0; byte_idx < rte_pktmbuf_data_len(m); 908 ++byte_idx) { 909 910 llr_tmp = llr[byte_idx]; 911 if (llr_decimals == 2) 912 llr_tmp *= 2; 913 else if (llr_decimals == 0) 914 llr_tmp /= 2; 915 llr_tmp = RTE_MIN(llr_max, 916 RTE_MAX(llr_min, llr_tmp)); 917 llr[byte_idx] = (int8_t) llr_tmp; 918 } 919 920 m = m->next; 921 } 922 } 923 } 924 925 926 927 static int 928 fill_queue_buffers(struct test_op_params *op_params, 929 struct rte_mempool *in_mp, struct rte_mempool *hard_out_mp, 930 struct rte_mempool *soft_out_mp, 931 struct rte_mempool *harq_in_mp, struct rte_mempool *harq_out_mp, 932 uint16_t queue_id, 933 const struct rte_bbdev_op_cap *capabilities, 934 uint16_t min_alignment, const int socket_id) 935 { 936 int ret; 937 enum op_data_type type; 938 const uint16_t n = op_params->num_to_process; 939 940 struct rte_mempool *mbuf_pools[DATA_NUM_TYPES] = { 941 in_mp, 942 soft_out_mp, 943 hard_out_mp, 944 harq_in_mp, 945 harq_out_mp, 946 }; 947 948 struct rte_bbdev_op_data **queue_ops[DATA_NUM_TYPES] = { 949 &op_params->q_bufs[socket_id][queue_id].inputs, 950 &op_params->q_bufs[socket_id][queue_id].soft_outputs, 951 &op_params->q_bufs[socket_id][queue_id].hard_outputs, 952 &op_params->q_bufs[socket_id][queue_id].harq_inputs, 953 &op_params->q_bufs[socket_id][queue_id].harq_outputs, 954 }; 955 956 for (type = DATA_INPUT; type < DATA_NUM_TYPES; ++type) { 957 struct op_data_entries *ref_entries = 958 &test_vector.entries[type]; 959 if (ref_entries->nb_segments == 0) 960 continue; 961 962 ret = allocate_buffers_on_socket(queue_ops[type], 963 n * sizeof(struct rte_bbdev_op_data), 964 socket_id); 965 TEST_ASSERT_SUCCESS(ret, 966 "Couldn't allocate memory for rte_bbdev_op_data structs"); 967 968 ret = init_op_data_objs(*queue_ops[type], ref_entries, 969 mbuf_pools[type], n, type, min_alignment); 970 TEST_ASSERT_SUCCESS(ret, 971 "Couldn't init rte_bbdev_op_data structs"); 972 } 973 974 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) 975 limit_input_llr_val_range(*queue_ops[DATA_INPUT], n, 976 capabilities->cap.turbo_dec.max_llr_modulus); 977 978 if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) { 979 ldpc_input_llr_scaling(*queue_ops[DATA_INPUT], n, 980 capabilities->cap.ldpc_dec.llr_size, 981 capabilities->cap.ldpc_dec.llr_decimals); 982 ldpc_input_llr_scaling(*queue_ops[DATA_HARQ_INPUT], n, 983 capabilities->cap.ldpc_dec.llr_size, 984 capabilities->cap.ldpc_dec.llr_decimals); 985 } 986 987 return 0; 988 } 989 990 static void 991 free_buffers(struct active_device *ad, struct test_op_params *op_params) 992 { 993 unsigned int i, j; 994 995 rte_mempool_free(ad->ops_mempool); 996 rte_mempool_free(ad->in_mbuf_pool); 997 rte_mempool_free(ad->hard_out_mbuf_pool); 998 rte_mempool_free(ad->soft_out_mbuf_pool); 999 rte_mempool_free(ad->harq_in_mbuf_pool); 1000 rte_mempool_free(ad->harq_out_mbuf_pool); 1001 1002 for (i = 0; i < rte_lcore_count(); ++i) { 1003 for (j = 0; j < RTE_MAX_NUMA_NODES; ++j) { 1004 rte_free(op_params->q_bufs[j][i].inputs); 1005 rte_free(op_params->q_bufs[j][i].hard_outputs); 1006 rte_free(op_params->q_bufs[j][i].soft_outputs); 1007 rte_free(op_params->q_bufs[j][i].harq_inputs); 1008 rte_free(op_params->q_bufs[j][i].harq_outputs); 1009 } 1010 } 1011 } 1012 1013 static void 1014 copy_reference_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n, 1015 unsigned int start_idx, 1016 struct rte_bbdev_op_data *inputs, 1017 struct rte_bbdev_op_data *hard_outputs, 1018 struct rte_bbdev_op_data *soft_outputs, 1019 struct rte_bbdev_dec_op *ref_op) 1020 { 1021 unsigned int i; 1022 struct rte_bbdev_op_turbo_dec *turbo_dec = &ref_op->turbo_dec; 1023 1024 for (i = 0; i < n; ++i) { 1025 if (turbo_dec->code_block_mode == 0) { 1026 ops[i]->turbo_dec.tb_params.ea = 1027 turbo_dec->tb_params.ea; 1028 ops[i]->turbo_dec.tb_params.eb = 1029 turbo_dec->tb_params.eb; 1030 ops[i]->turbo_dec.tb_params.k_pos = 1031 turbo_dec->tb_params.k_pos; 1032 ops[i]->turbo_dec.tb_params.k_neg = 1033 turbo_dec->tb_params.k_neg; 1034 ops[i]->turbo_dec.tb_params.c = 1035 turbo_dec->tb_params.c; 1036 ops[i]->turbo_dec.tb_params.c_neg = 1037 turbo_dec->tb_params.c_neg; 1038 ops[i]->turbo_dec.tb_params.cab = 1039 turbo_dec->tb_params.cab; 1040 ops[i]->turbo_dec.tb_params.r = 1041 turbo_dec->tb_params.r; 1042 } else { 1043 ops[i]->turbo_dec.cb_params.e = turbo_dec->cb_params.e; 1044 ops[i]->turbo_dec.cb_params.k = turbo_dec->cb_params.k; 1045 } 1046 1047 ops[i]->turbo_dec.ext_scale = turbo_dec->ext_scale; 1048 ops[i]->turbo_dec.iter_max = turbo_dec->iter_max; 1049 ops[i]->turbo_dec.iter_min = turbo_dec->iter_min; 1050 ops[i]->turbo_dec.op_flags = turbo_dec->op_flags; 1051 ops[i]->turbo_dec.rv_index = turbo_dec->rv_index; 1052 ops[i]->turbo_dec.num_maps = turbo_dec->num_maps; 1053 ops[i]->turbo_dec.code_block_mode = turbo_dec->code_block_mode; 1054 1055 ops[i]->turbo_dec.hard_output = hard_outputs[start_idx + i]; 1056 ops[i]->turbo_dec.input = inputs[start_idx + i]; 1057 if (soft_outputs != NULL) 1058 ops[i]->turbo_dec.soft_output = 1059 soft_outputs[start_idx + i]; 1060 } 1061 } 1062 1063 static void 1064 copy_reference_enc_op(struct rte_bbdev_enc_op **ops, unsigned int n, 1065 unsigned int start_idx, 1066 struct rte_bbdev_op_data *inputs, 1067 struct rte_bbdev_op_data *outputs, 1068 struct rte_bbdev_enc_op *ref_op) 1069 { 1070 unsigned int i; 1071 struct rte_bbdev_op_turbo_enc *turbo_enc = &ref_op->turbo_enc; 1072 for (i = 0; i < n; ++i) { 1073 if (turbo_enc->code_block_mode == 0) { 1074 ops[i]->turbo_enc.tb_params.ea = 1075 turbo_enc->tb_params.ea; 1076 ops[i]->turbo_enc.tb_params.eb = 1077 turbo_enc->tb_params.eb; 1078 ops[i]->turbo_enc.tb_params.k_pos = 1079 turbo_enc->tb_params.k_pos; 1080 ops[i]->turbo_enc.tb_params.k_neg = 1081 turbo_enc->tb_params.k_neg; 1082 ops[i]->turbo_enc.tb_params.c = 1083 turbo_enc->tb_params.c; 1084 ops[i]->turbo_enc.tb_params.c_neg = 1085 turbo_enc->tb_params.c_neg; 1086 ops[i]->turbo_enc.tb_params.cab = 1087 turbo_enc->tb_params.cab; 1088 ops[i]->turbo_enc.tb_params.ncb_pos = 1089 turbo_enc->tb_params.ncb_pos; 1090 ops[i]->turbo_enc.tb_params.ncb_neg = 1091 turbo_enc->tb_params.ncb_neg; 1092 ops[i]->turbo_enc.tb_params.r = turbo_enc->tb_params.r; 1093 } else { 1094 ops[i]->turbo_enc.cb_params.e = turbo_enc->cb_params.e; 1095 ops[i]->turbo_enc.cb_params.k = turbo_enc->cb_params.k; 1096 ops[i]->turbo_enc.cb_params.ncb = 1097 turbo_enc->cb_params.ncb; 1098 } 1099 ops[i]->turbo_enc.rv_index = turbo_enc->rv_index; 1100 ops[i]->turbo_enc.op_flags = turbo_enc->op_flags; 1101 ops[i]->turbo_enc.code_block_mode = turbo_enc->code_block_mode; 1102 1103 ops[i]->turbo_enc.output = outputs[start_idx + i]; 1104 ops[i]->turbo_enc.input = inputs[start_idx + i]; 1105 } 1106 } 1107 1108 static void 1109 copy_reference_ldpc_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n, 1110 unsigned int start_idx, 1111 struct rte_bbdev_op_data *inputs, 1112 struct rte_bbdev_op_data *hard_outputs, 1113 struct rte_bbdev_op_data *soft_outputs, 1114 struct rte_bbdev_op_data *harq_inputs, 1115 struct rte_bbdev_op_data *harq_outputs, 1116 struct rte_bbdev_dec_op *ref_op) 1117 { 1118 unsigned int i; 1119 struct rte_bbdev_op_ldpc_dec *ldpc_dec = &ref_op->ldpc_dec; 1120 1121 for (i = 0; i < n; ++i) { 1122 if (ldpc_dec->code_block_mode == 0) { 1123 ops[i]->ldpc_dec.tb_params.ea = 1124 ldpc_dec->tb_params.ea; 1125 ops[i]->ldpc_dec.tb_params.eb = 1126 ldpc_dec->tb_params.eb; 1127 ops[i]->ldpc_dec.tb_params.c = 1128 ldpc_dec->tb_params.c; 1129 ops[i]->ldpc_dec.tb_params.cab = 1130 ldpc_dec->tb_params.cab; 1131 ops[i]->ldpc_dec.tb_params.r = 1132 ldpc_dec->tb_params.r; 1133 } else { 1134 ops[i]->ldpc_dec.cb_params.e = ldpc_dec->cb_params.e; 1135 } 1136 1137 ops[i]->ldpc_dec.basegraph = ldpc_dec->basegraph; 1138 ops[i]->ldpc_dec.z_c = ldpc_dec->z_c; 1139 ops[i]->ldpc_dec.q_m = ldpc_dec->q_m; 1140 ops[i]->ldpc_dec.n_filler = ldpc_dec->n_filler; 1141 ops[i]->ldpc_dec.n_cb = ldpc_dec->n_cb; 1142 ops[i]->ldpc_dec.iter_max = ldpc_dec->iter_max; 1143 ops[i]->ldpc_dec.rv_index = ldpc_dec->rv_index; 1144 ops[i]->ldpc_dec.op_flags = ldpc_dec->op_flags; 1145 ops[i]->ldpc_dec.code_block_mode = ldpc_dec->code_block_mode; 1146 1147 ops[i]->ldpc_dec.hard_output = hard_outputs[start_idx + i]; 1148 ops[i]->ldpc_dec.input = inputs[start_idx + i]; 1149 if (soft_outputs != NULL) 1150 ops[i]->ldpc_dec.soft_output = 1151 soft_outputs[start_idx + i]; 1152 if (harq_inputs != NULL) 1153 ops[i]->ldpc_dec.harq_combined_input = 1154 harq_inputs[start_idx + i]; 1155 if (harq_outputs != NULL) 1156 ops[i]->ldpc_dec.harq_combined_output = 1157 harq_outputs[start_idx + i]; 1158 } 1159 } 1160 1161 1162 static void 1163 copy_reference_ldpc_enc_op(struct rte_bbdev_enc_op **ops, unsigned int n, 1164 unsigned int start_idx, 1165 struct rte_bbdev_op_data *inputs, 1166 struct rte_bbdev_op_data *outputs, 1167 struct rte_bbdev_enc_op *ref_op) 1168 { 1169 unsigned int i; 1170 struct rte_bbdev_op_ldpc_enc *ldpc_enc = &ref_op->ldpc_enc; 1171 for (i = 0; i < n; ++i) { 1172 if (ldpc_enc->code_block_mode == 0) { 1173 ops[i]->ldpc_enc.tb_params.ea = ldpc_enc->tb_params.ea; 1174 ops[i]->ldpc_enc.tb_params.eb = ldpc_enc->tb_params.eb; 1175 ops[i]->ldpc_enc.tb_params.cab = 1176 ldpc_enc->tb_params.cab; 1177 ops[i]->ldpc_enc.tb_params.c = ldpc_enc->tb_params.c; 1178 ops[i]->ldpc_enc.tb_params.r = ldpc_enc->tb_params.r; 1179 } else { 1180 ops[i]->ldpc_enc.cb_params.e = ldpc_enc->cb_params.e; 1181 } 1182 ops[i]->ldpc_enc.basegraph = ldpc_enc->basegraph; 1183 ops[i]->ldpc_enc.z_c = ldpc_enc->z_c; 1184 ops[i]->ldpc_enc.q_m = ldpc_enc->q_m; 1185 ops[i]->ldpc_enc.n_filler = ldpc_enc->n_filler; 1186 ops[i]->ldpc_enc.n_cb = ldpc_enc->n_cb; 1187 ops[i]->ldpc_enc.rv_index = ldpc_enc->rv_index; 1188 ops[i]->ldpc_enc.op_flags = ldpc_enc->op_flags; 1189 ops[i]->ldpc_enc.code_block_mode = ldpc_enc->code_block_mode; 1190 ops[i]->ldpc_enc.output = outputs[start_idx + i]; 1191 ops[i]->ldpc_enc.input = inputs[start_idx + i]; 1192 } 1193 } 1194 1195 static int 1196 check_dec_status_and_ordering(struct rte_bbdev_dec_op *op, 1197 unsigned int order_idx, const int expected_status) 1198 { 1199 TEST_ASSERT(op->status == expected_status, 1200 "op_status (%d) != expected_status (%d)", 1201 op->status, expected_status); 1202 1203 TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data, 1204 "Ordering error, expected %p, got %p", 1205 (void *)(uintptr_t)order_idx, op->opaque_data); 1206 1207 return TEST_SUCCESS; 1208 } 1209 1210 static int 1211 check_enc_status_and_ordering(struct rte_bbdev_enc_op *op, 1212 unsigned int order_idx, const int expected_status) 1213 { 1214 TEST_ASSERT(op->status == expected_status, 1215 "op_status (%d) != expected_status (%d)", 1216 op->status, expected_status); 1217 1218 TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data, 1219 "Ordering error, expected %p, got %p", 1220 (void *)(uintptr_t)order_idx, op->opaque_data); 1221 1222 return TEST_SUCCESS; 1223 } 1224 1225 static inline int 1226 validate_op_chain(struct rte_bbdev_op_data *op, 1227 struct op_data_entries *orig_op) 1228 { 1229 uint8_t i; 1230 struct rte_mbuf *m = op->data; 1231 uint8_t nb_dst_segments = orig_op->nb_segments; 1232 uint32_t total_data_size = 0; 1233 1234 TEST_ASSERT(nb_dst_segments == m->nb_segs, 1235 "Number of segments differ in original (%u) and filled (%u) op", 1236 nb_dst_segments, m->nb_segs); 1237 1238 /* Validate each mbuf segment length */ 1239 for (i = 0; i < nb_dst_segments; ++i) { 1240 /* Apply offset to the first mbuf segment */ 1241 uint16_t offset = (i == 0) ? op->offset : 0; 1242 uint16_t data_len = rte_pktmbuf_data_len(m) - offset; 1243 total_data_size += orig_op->segments[i].length; 1244 1245 TEST_ASSERT(orig_op->segments[i].length == data_len, 1246 "Length of segment differ in original (%u) and filled (%u) op", 1247 orig_op->segments[i].length, data_len); 1248 TEST_ASSERT_BUFFERS_ARE_EQUAL(orig_op->segments[i].addr, 1249 rte_pktmbuf_mtod_offset(m, uint32_t *, offset), 1250 data_len, 1251 "Output buffers (CB=%u) are not equal", i); 1252 m = m->next; 1253 } 1254 1255 /* Validate total mbuf pkt length */ 1256 uint32_t pkt_len = rte_pktmbuf_pkt_len(op->data) - op->offset; 1257 TEST_ASSERT(total_data_size == pkt_len, 1258 "Length of data differ in original (%u) and filled (%u) op", 1259 total_data_size, pkt_len); 1260 1261 return TEST_SUCCESS; 1262 } 1263 1264 static int 1265 validate_dec_op(struct rte_bbdev_dec_op **ops, const uint16_t n, 1266 struct rte_bbdev_dec_op *ref_op, const int vector_mask) 1267 { 1268 unsigned int i; 1269 int ret; 1270 struct op_data_entries *hard_data_orig = 1271 &test_vector.entries[DATA_HARD_OUTPUT]; 1272 struct op_data_entries *soft_data_orig = 1273 &test_vector.entries[DATA_SOFT_OUTPUT]; 1274 struct rte_bbdev_op_turbo_dec *ops_td; 1275 struct rte_bbdev_op_data *hard_output; 1276 struct rte_bbdev_op_data *soft_output; 1277 struct rte_bbdev_op_turbo_dec *ref_td = &ref_op->turbo_dec; 1278 1279 for (i = 0; i < n; ++i) { 1280 ops_td = &ops[i]->turbo_dec; 1281 hard_output = &ops_td->hard_output; 1282 soft_output = &ops_td->soft_output; 1283 1284 if (vector_mask & TEST_BBDEV_VF_EXPECTED_ITER_COUNT) 1285 TEST_ASSERT(ops_td->iter_count <= ref_td->iter_count, 1286 "Returned iter_count (%d) > expected iter_count (%d)", 1287 ops_td->iter_count, ref_td->iter_count); 1288 ret = check_dec_status_and_ordering(ops[i], i, ref_op->status); 1289 TEST_ASSERT_SUCCESS(ret, 1290 "Checking status and ordering for decoder failed"); 1291 1292 TEST_ASSERT_SUCCESS(validate_op_chain(hard_output, 1293 hard_data_orig), 1294 "Hard output buffers (CB=%u) are not equal", 1295 i); 1296 1297 if (ref_op->turbo_dec.op_flags & RTE_BBDEV_TURBO_SOFT_OUTPUT) 1298 TEST_ASSERT_SUCCESS(validate_op_chain(soft_output, 1299 soft_data_orig), 1300 "Soft output buffers (CB=%u) are not equal", 1301 i); 1302 } 1303 1304 return TEST_SUCCESS; 1305 } 1306 1307 1308 static int 1309 validate_ldpc_dec_op(struct rte_bbdev_dec_op **ops, const uint16_t n, 1310 struct rte_bbdev_dec_op *ref_op, const int vector_mask) 1311 { 1312 unsigned int i; 1313 int ret; 1314 struct op_data_entries *hard_data_orig = 1315 &test_vector.entries[DATA_HARD_OUTPUT]; 1316 struct op_data_entries *soft_data_orig = 1317 &test_vector.entries[DATA_SOFT_OUTPUT]; 1318 struct op_data_entries *harq_data_orig = 1319 &test_vector.entries[DATA_HARQ_OUTPUT]; 1320 struct rte_bbdev_op_ldpc_dec *ops_td; 1321 struct rte_bbdev_op_data *hard_output; 1322 struct rte_bbdev_op_data *harq_output; 1323 struct rte_bbdev_op_data *soft_output; 1324 struct rte_bbdev_op_ldpc_dec *ref_td = &ref_op->ldpc_dec; 1325 1326 for (i = 0; i < n; ++i) { 1327 ops_td = &ops[i]->ldpc_dec; 1328 hard_output = &ops_td->hard_output; 1329 harq_output = &ops_td->harq_combined_output; 1330 soft_output = &ops_td->soft_output; 1331 1332 ret = check_dec_status_and_ordering(ops[i], i, ref_op->status); 1333 TEST_ASSERT_SUCCESS(ret, 1334 "Checking status and ordering for decoder failed"); 1335 if (vector_mask & TEST_BBDEV_VF_EXPECTED_ITER_COUNT) 1336 TEST_ASSERT(ops_td->iter_count <= ref_td->iter_count, 1337 "Returned iter_count (%d) > expected iter_count (%d)", 1338 ops_td->iter_count, ref_td->iter_count); 1339 /* We can ignore data when the decoding failed to converge */ 1340 if ((ops[i]->status & (1 << RTE_BBDEV_SYNDROME_ERROR)) == 0) 1341 TEST_ASSERT_SUCCESS(validate_op_chain(hard_output, 1342 hard_data_orig), 1343 "Hard output buffers (CB=%u) are not equal", 1344 i); 1345 1346 if (ref_op->ldpc_dec.op_flags & RTE_BBDEV_LDPC_SOFT_OUT_ENABLE) 1347 TEST_ASSERT_SUCCESS(validate_op_chain(soft_output, 1348 soft_data_orig), 1349 "Soft output buffers (CB=%u) are not equal", 1350 i); 1351 if (ref_op->ldpc_dec.op_flags & 1352 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE) { 1353 ldpc_input_llr_scaling(harq_output, 1, 8, 0); 1354 TEST_ASSERT_SUCCESS(validate_op_chain(harq_output, 1355 harq_data_orig), 1356 "HARQ output buffers (CB=%u) are not equal", 1357 i); 1358 } 1359 } 1360 1361 return TEST_SUCCESS; 1362 } 1363 1364 1365 static int 1366 validate_enc_op(struct rte_bbdev_enc_op **ops, const uint16_t n, 1367 struct rte_bbdev_enc_op *ref_op) 1368 { 1369 unsigned int i; 1370 int ret; 1371 struct op_data_entries *hard_data_orig = 1372 &test_vector.entries[DATA_HARD_OUTPUT]; 1373 1374 for (i = 0; i < n; ++i) { 1375 ret = check_enc_status_and_ordering(ops[i], i, ref_op->status); 1376 TEST_ASSERT_SUCCESS(ret, 1377 "Checking status and ordering for encoder failed"); 1378 TEST_ASSERT_SUCCESS(validate_op_chain( 1379 &ops[i]->turbo_enc.output, 1380 hard_data_orig), 1381 "Output buffers (CB=%u) are not equal", 1382 i); 1383 } 1384 1385 return TEST_SUCCESS; 1386 } 1387 1388 static int 1389 validate_ldpc_enc_op(struct rte_bbdev_enc_op **ops, const uint16_t n, 1390 struct rte_bbdev_enc_op *ref_op) 1391 { 1392 unsigned int i; 1393 int ret; 1394 struct op_data_entries *hard_data_orig = 1395 &test_vector.entries[DATA_HARD_OUTPUT]; 1396 1397 for (i = 0; i < n; ++i) { 1398 ret = check_enc_status_and_ordering(ops[i], i, ref_op->status); 1399 TEST_ASSERT_SUCCESS(ret, 1400 "Checking status and ordering for encoder failed"); 1401 TEST_ASSERT_SUCCESS(validate_op_chain( 1402 &ops[i]->ldpc_enc.output, 1403 hard_data_orig), 1404 "Output buffers (CB=%u) are not equal", 1405 i); 1406 } 1407 1408 return TEST_SUCCESS; 1409 } 1410 1411 static void 1412 create_reference_dec_op(struct rte_bbdev_dec_op *op) 1413 { 1414 unsigned int i; 1415 struct op_data_entries *entry; 1416 1417 op->turbo_dec = test_vector.turbo_dec; 1418 entry = &test_vector.entries[DATA_INPUT]; 1419 for (i = 0; i < entry->nb_segments; ++i) 1420 op->turbo_dec.input.length += 1421 entry->segments[i].length; 1422 } 1423 1424 static void 1425 create_reference_ldpc_dec_op(struct rte_bbdev_dec_op *op) 1426 { 1427 unsigned int i; 1428 struct op_data_entries *entry; 1429 1430 op->ldpc_dec = test_vector.ldpc_dec; 1431 entry = &test_vector.entries[DATA_INPUT]; 1432 for (i = 0; i < entry->nb_segments; ++i) 1433 op->ldpc_dec.input.length += 1434 entry->segments[i].length; 1435 if (test_vector.ldpc_dec.op_flags & 1436 RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE) { 1437 entry = &test_vector.entries[DATA_HARQ_INPUT]; 1438 for (i = 0; i < entry->nb_segments; ++i) 1439 op->ldpc_dec.harq_combined_input.length += 1440 entry->segments[i].length; 1441 } 1442 } 1443 1444 1445 static void 1446 create_reference_enc_op(struct rte_bbdev_enc_op *op) 1447 { 1448 unsigned int i; 1449 struct op_data_entries *entry; 1450 1451 op->turbo_enc = test_vector.turbo_enc; 1452 entry = &test_vector.entries[DATA_INPUT]; 1453 for (i = 0; i < entry->nb_segments; ++i) 1454 op->turbo_enc.input.length += 1455 entry->segments[i].length; 1456 } 1457 1458 static void 1459 create_reference_ldpc_enc_op(struct rte_bbdev_enc_op *op) 1460 { 1461 unsigned int i; 1462 struct op_data_entries *entry; 1463 1464 op->ldpc_enc = test_vector.ldpc_enc; 1465 entry = &test_vector.entries[DATA_INPUT]; 1466 for (i = 0; i < entry->nb_segments; ++i) 1467 op->ldpc_enc.input.length += 1468 entry->segments[i].length; 1469 } 1470 1471 static uint32_t 1472 calc_dec_TB_size(struct rte_bbdev_dec_op *op) 1473 { 1474 uint8_t i; 1475 uint32_t c, r, tb_size = 0; 1476 1477 if (op->turbo_dec.code_block_mode) { 1478 tb_size = op->turbo_dec.tb_params.k_neg; 1479 } else { 1480 c = op->turbo_dec.tb_params.c; 1481 r = op->turbo_dec.tb_params.r; 1482 for (i = 0; i < c-r; i++) 1483 tb_size += (r < op->turbo_dec.tb_params.c_neg) ? 1484 op->turbo_dec.tb_params.k_neg : 1485 op->turbo_dec.tb_params.k_pos; 1486 } 1487 return tb_size; 1488 } 1489 1490 static uint32_t 1491 calc_ldpc_dec_TB_size(struct rte_bbdev_dec_op *op) 1492 { 1493 uint8_t i; 1494 uint32_t c, r, tb_size = 0; 1495 uint16_t sys_cols = (op->ldpc_dec.basegraph == 1) ? 22 : 10; 1496 1497 if (op->ldpc_dec.code_block_mode) { 1498 tb_size = sys_cols * op->ldpc_dec.z_c - op->ldpc_dec.n_filler; 1499 } else { 1500 c = op->ldpc_dec.tb_params.c; 1501 r = op->ldpc_dec.tb_params.r; 1502 for (i = 0; i < c-r; i++) 1503 tb_size += sys_cols * op->ldpc_dec.z_c 1504 - op->ldpc_dec.n_filler; 1505 } 1506 return tb_size; 1507 } 1508 1509 static uint32_t 1510 calc_enc_TB_size(struct rte_bbdev_enc_op *op) 1511 { 1512 uint8_t i; 1513 uint32_t c, r, tb_size = 0; 1514 1515 if (op->turbo_enc.code_block_mode) { 1516 tb_size = op->turbo_enc.tb_params.k_neg; 1517 } else { 1518 c = op->turbo_enc.tb_params.c; 1519 r = op->turbo_enc.tb_params.r; 1520 for (i = 0; i < c-r; i++) 1521 tb_size += (r < op->turbo_enc.tb_params.c_neg) ? 1522 op->turbo_enc.tb_params.k_neg : 1523 op->turbo_enc.tb_params.k_pos; 1524 } 1525 return tb_size; 1526 } 1527 1528 static uint32_t 1529 calc_ldpc_enc_TB_size(struct rte_bbdev_enc_op *op) 1530 { 1531 uint8_t i; 1532 uint32_t c, r, tb_size = 0; 1533 uint16_t sys_cols = (op->ldpc_enc.basegraph == 1) ? 22 : 10; 1534 1535 if (op->turbo_enc.code_block_mode) { 1536 tb_size = sys_cols * op->ldpc_enc.z_c - op->ldpc_enc.n_filler; 1537 } else { 1538 c = op->turbo_enc.tb_params.c; 1539 r = op->turbo_enc.tb_params.r; 1540 for (i = 0; i < c-r; i++) 1541 tb_size += sys_cols * op->ldpc_enc.z_c 1542 - op->ldpc_enc.n_filler; 1543 } 1544 return tb_size; 1545 } 1546 1547 1548 static int 1549 init_test_op_params(struct test_op_params *op_params, 1550 enum rte_bbdev_op_type op_type, const int expected_status, 1551 const int vector_mask, struct rte_mempool *ops_mp, 1552 uint16_t burst_sz, uint16_t num_to_process, uint16_t num_lcores) 1553 { 1554 int ret = 0; 1555 if (op_type == RTE_BBDEV_OP_TURBO_DEC || 1556 op_type == RTE_BBDEV_OP_LDPC_DEC) 1557 ret = rte_bbdev_dec_op_alloc_bulk(ops_mp, 1558 &op_params->ref_dec_op, 1); 1559 else 1560 ret = rte_bbdev_enc_op_alloc_bulk(ops_mp, 1561 &op_params->ref_enc_op, 1); 1562 1563 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_op_alloc_bulk() failed"); 1564 1565 op_params->mp = ops_mp; 1566 op_params->burst_sz = burst_sz; 1567 op_params->num_to_process = num_to_process; 1568 op_params->num_lcores = num_lcores; 1569 op_params->vector_mask = vector_mask; 1570 if (op_type == RTE_BBDEV_OP_TURBO_DEC || 1571 op_type == RTE_BBDEV_OP_LDPC_DEC) 1572 op_params->ref_dec_op->status = expected_status; 1573 else if (op_type == RTE_BBDEV_OP_TURBO_ENC 1574 || op_type == RTE_BBDEV_OP_LDPC_ENC) 1575 op_params->ref_enc_op->status = expected_status; 1576 return 0; 1577 } 1578 1579 static int 1580 run_test_case_on_device(test_case_function *test_case_func, uint8_t dev_id, 1581 struct test_op_params *op_params) 1582 { 1583 int t_ret, f_ret, socket_id = SOCKET_ID_ANY; 1584 unsigned int i; 1585 struct active_device *ad; 1586 unsigned int burst_sz = get_burst_sz(); 1587 enum rte_bbdev_op_type op_type = test_vector.op_type; 1588 const struct rte_bbdev_op_cap *capabilities = NULL; 1589 1590 ad = &active_devs[dev_id]; 1591 1592 /* Check if device supports op_type */ 1593 if (!is_avail_op(ad, test_vector.op_type)) 1594 return TEST_SUCCESS; 1595 1596 struct rte_bbdev_info info; 1597 rte_bbdev_info_get(ad->dev_id, &info); 1598 socket_id = GET_SOCKET(info.socket_id); 1599 1600 f_ret = create_mempools(ad, socket_id, op_type, 1601 get_num_ops()); 1602 if (f_ret != TEST_SUCCESS) { 1603 printf("Couldn't create mempools"); 1604 goto fail; 1605 } 1606 if (op_type == RTE_BBDEV_OP_NONE) 1607 op_type = RTE_BBDEV_OP_TURBO_ENC; 1608 1609 f_ret = init_test_op_params(op_params, test_vector.op_type, 1610 test_vector.expected_status, 1611 test_vector.mask, 1612 ad->ops_mempool, 1613 burst_sz, 1614 get_num_ops(), 1615 get_num_lcores()); 1616 if (f_ret != TEST_SUCCESS) { 1617 printf("Couldn't init test op params"); 1618 goto fail; 1619 } 1620 1621 1622 /* Find capabilities */ 1623 const struct rte_bbdev_op_cap *cap = info.drv.capabilities; 1624 for (i = 0; i < RTE_BBDEV_OP_TYPE_COUNT; i++) { 1625 if (cap->type == test_vector.op_type) { 1626 capabilities = cap; 1627 break; 1628 } 1629 cap++; 1630 } 1631 TEST_ASSERT_NOT_NULL(capabilities, 1632 "Couldn't find capabilities"); 1633 1634 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) { 1635 create_reference_dec_op(op_params->ref_dec_op); 1636 } else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) 1637 create_reference_enc_op(op_params->ref_enc_op); 1638 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) 1639 create_reference_ldpc_enc_op(op_params->ref_enc_op); 1640 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 1641 create_reference_ldpc_dec_op(op_params->ref_dec_op); 1642 1643 for (i = 0; i < ad->nb_queues; ++i) { 1644 f_ret = fill_queue_buffers(op_params, 1645 ad->in_mbuf_pool, 1646 ad->hard_out_mbuf_pool, 1647 ad->soft_out_mbuf_pool, 1648 ad->harq_in_mbuf_pool, 1649 ad->harq_out_mbuf_pool, 1650 ad->queue_ids[i], 1651 capabilities, 1652 info.drv.min_alignment, 1653 socket_id); 1654 if (f_ret != TEST_SUCCESS) { 1655 printf("Couldn't init queue buffers"); 1656 goto fail; 1657 } 1658 } 1659 1660 /* Run test case function */ 1661 t_ret = test_case_func(ad, op_params); 1662 1663 /* Free active device resources and return */ 1664 free_buffers(ad, op_params); 1665 return t_ret; 1666 1667 fail: 1668 free_buffers(ad, op_params); 1669 return TEST_FAILED; 1670 } 1671 1672 /* Run given test function per active device per supported op type 1673 * per burst size. 1674 */ 1675 static int 1676 run_test_case(test_case_function *test_case_func) 1677 { 1678 int ret = 0; 1679 uint8_t dev; 1680 1681 /* Alloc op_params */ 1682 struct test_op_params *op_params = rte_zmalloc(NULL, 1683 sizeof(struct test_op_params), RTE_CACHE_LINE_SIZE); 1684 TEST_ASSERT_NOT_NULL(op_params, "Failed to alloc %zuB for op_params", 1685 RTE_ALIGN(sizeof(struct test_op_params), 1686 RTE_CACHE_LINE_SIZE)); 1687 1688 /* For each device run test case function */ 1689 for (dev = 0; dev < nb_active_devs; ++dev) 1690 ret |= run_test_case_on_device(test_case_func, dev, op_params); 1691 1692 rte_free(op_params); 1693 1694 return ret; 1695 } 1696 1697 static void 1698 dequeue_event_callback(uint16_t dev_id, 1699 enum rte_bbdev_event_type event, void *cb_arg, 1700 void *ret_param) 1701 { 1702 int ret; 1703 uint16_t i; 1704 uint64_t total_time; 1705 uint16_t deq, burst_sz, num_ops; 1706 uint16_t queue_id = *(uint16_t *) ret_param; 1707 struct rte_bbdev_info info; 1708 double tb_len_bits; 1709 struct thread_params *tp = cb_arg; 1710 1711 /* Find matching thread params using queue_id */ 1712 for (i = 0; i < MAX_QUEUES; ++i, ++tp) 1713 if (tp->queue_id == queue_id) 1714 break; 1715 1716 if (i == MAX_QUEUES) { 1717 printf("%s: Queue_id from interrupt details was not found!\n", 1718 __func__); 1719 return; 1720 } 1721 1722 if (unlikely(event != RTE_BBDEV_EVENT_DEQUEUE)) { 1723 rte_atomic16_set(&tp->processing_status, TEST_FAILED); 1724 printf( 1725 "Dequeue interrupt handler called for incorrect event!\n"); 1726 return; 1727 } 1728 1729 burst_sz = rte_atomic16_read(&tp->burst_sz); 1730 num_ops = tp->op_params->num_to_process; 1731 1732 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC || 1733 test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 1734 deq = rte_bbdev_dequeue_dec_ops(dev_id, queue_id, 1735 &tp->dec_ops[ 1736 rte_atomic16_read(&tp->nb_dequeued)], 1737 burst_sz); 1738 else 1739 deq = rte_bbdev_dequeue_enc_ops(dev_id, queue_id, 1740 &tp->enc_ops[ 1741 rte_atomic16_read(&tp->nb_dequeued)], 1742 burst_sz); 1743 1744 if (deq < burst_sz) { 1745 printf( 1746 "After receiving the interrupt all operations should be dequeued. Expected: %u, got: %u\n", 1747 burst_sz, deq); 1748 rte_atomic16_set(&tp->processing_status, TEST_FAILED); 1749 return; 1750 } 1751 1752 if (rte_atomic16_read(&tp->nb_dequeued) + deq < num_ops) { 1753 rte_atomic16_add(&tp->nb_dequeued, deq); 1754 return; 1755 } 1756 1757 total_time = rte_rdtsc_precise() - tp->start_time; 1758 1759 rte_bbdev_info_get(dev_id, &info); 1760 1761 ret = TEST_SUCCESS; 1762 1763 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) { 1764 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 1765 ret = validate_dec_op(tp->dec_ops, num_ops, ref_op, 1766 tp->op_params->vector_mask); 1767 /* get the max of iter_count for all dequeued ops */ 1768 for (i = 0; i < num_ops; ++i) 1769 tp->iter_count = RTE_MAX( 1770 tp->dec_ops[i]->turbo_dec.iter_count, 1771 tp->iter_count); 1772 rte_bbdev_dec_op_free_bulk(tp->dec_ops, deq); 1773 } else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) { 1774 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op; 1775 ret = validate_enc_op(tp->enc_ops, num_ops, ref_op); 1776 rte_bbdev_enc_op_free_bulk(tp->enc_ops, deq); 1777 } else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) { 1778 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op; 1779 ret = validate_ldpc_enc_op(tp->enc_ops, num_ops, ref_op); 1780 rte_bbdev_enc_op_free_bulk(tp->enc_ops, deq); 1781 } else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) { 1782 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 1783 ret = validate_ldpc_dec_op(tp->dec_ops, num_ops, ref_op, 1784 tp->op_params->vector_mask); 1785 rte_bbdev_dec_op_free_bulk(tp->dec_ops, deq); 1786 } 1787 1788 if (ret) { 1789 printf("Buffers validation failed\n"); 1790 rte_atomic16_set(&tp->processing_status, TEST_FAILED); 1791 } 1792 1793 switch (test_vector.op_type) { 1794 case RTE_BBDEV_OP_TURBO_DEC: 1795 tb_len_bits = calc_dec_TB_size(tp->op_params->ref_dec_op); 1796 break; 1797 case RTE_BBDEV_OP_TURBO_ENC: 1798 tb_len_bits = calc_enc_TB_size(tp->op_params->ref_enc_op); 1799 break; 1800 case RTE_BBDEV_OP_LDPC_DEC: 1801 tb_len_bits = calc_ldpc_dec_TB_size(tp->op_params->ref_dec_op); 1802 break; 1803 case RTE_BBDEV_OP_LDPC_ENC: 1804 tb_len_bits = calc_ldpc_enc_TB_size(tp->op_params->ref_enc_op); 1805 break; 1806 case RTE_BBDEV_OP_NONE: 1807 tb_len_bits = 0.0; 1808 break; 1809 default: 1810 printf("Unknown op type: %d\n", test_vector.op_type); 1811 rte_atomic16_set(&tp->processing_status, TEST_FAILED); 1812 return; 1813 } 1814 1815 tp->ops_per_sec += ((double)num_ops) / 1816 ((double)total_time / (double)rte_get_tsc_hz()); 1817 tp->mbps += (((double)(num_ops * tb_len_bits)) / 1000000.0) / 1818 ((double)total_time / (double)rte_get_tsc_hz()); 1819 1820 rte_atomic16_add(&tp->nb_dequeued, deq); 1821 } 1822 1823 static int 1824 throughput_intr_lcore_dec(void *arg) 1825 { 1826 struct thread_params *tp = arg; 1827 unsigned int enqueued; 1828 const uint16_t queue_id = tp->queue_id; 1829 const uint16_t burst_sz = tp->op_params->burst_sz; 1830 const uint16_t num_to_process = tp->op_params->num_to_process; 1831 struct rte_bbdev_dec_op *ops[num_to_process]; 1832 struct test_buffers *bufs = NULL; 1833 struct rte_bbdev_info info; 1834 int ret, i, j; 1835 uint16_t num_to_enq, enq; 1836 1837 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 1838 "BURST_SIZE should be <= %u", MAX_BURST); 1839 1840 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id), 1841 "Failed to enable interrupts for dev: %u, queue_id: %u", 1842 tp->dev_id, queue_id); 1843 1844 rte_bbdev_info_get(tp->dev_id, &info); 1845 1846 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim), 1847 "NUM_OPS cannot exceed %u for this device", 1848 info.drv.queue_size_lim); 1849 1850 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 1851 1852 rte_atomic16_clear(&tp->processing_status); 1853 rte_atomic16_clear(&tp->nb_dequeued); 1854 1855 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT) 1856 rte_pause(); 1857 1858 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops, 1859 num_to_process); 1860 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", 1861 num_to_process); 1862 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 1863 copy_reference_dec_op(ops, num_to_process, 0, bufs->inputs, 1864 bufs->hard_outputs, bufs->soft_outputs, 1865 tp->op_params->ref_dec_op); 1866 1867 /* Set counter to validate the ordering */ 1868 for (j = 0; j < num_to_process; ++j) 1869 ops[j]->opaque_data = (void *)(uintptr_t)j; 1870 1871 for (j = 0; j < TEST_REPETITIONS; ++j) { 1872 for (i = 0; i < num_to_process; ++i) 1873 rte_pktmbuf_reset(ops[i]->turbo_dec.hard_output.data); 1874 1875 tp->start_time = rte_rdtsc_precise(); 1876 for (enqueued = 0; enqueued < num_to_process;) { 1877 num_to_enq = burst_sz; 1878 1879 if (unlikely(num_to_process - enqueued < num_to_enq)) 1880 num_to_enq = num_to_process - enqueued; 1881 1882 enq = 0; 1883 do { 1884 enq += rte_bbdev_enqueue_dec_ops(tp->dev_id, 1885 queue_id, &ops[enqueued], 1886 num_to_enq); 1887 } while (unlikely(num_to_enq != enq)); 1888 enqueued += enq; 1889 1890 /* Write to thread burst_sz current number of enqueued 1891 * descriptors. It ensures that proper number of 1892 * descriptors will be dequeued in callback 1893 * function - needed for last batch in case where 1894 * the number of operations is not a multiple of 1895 * burst size. 1896 */ 1897 rte_atomic16_set(&tp->burst_sz, num_to_enq); 1898 1899 /* Wait until processing of previous batch is 1900 * completed 1901 */ 1902 while (rte_atomic16_read(&tp->nb_dequeued) != 1903 (int16_t) enqueued) 1904 rte_pause(); 1905 } 1906 if (j != TEST_REPETITIONS - 1) 1907 rte_atomic16_clear(&tp->nb_dequeued); 1908 } 1909 1910 return TEST_SUCCESS; 1911 } 1912 1913 static int 1914 throughput_intr_lcore_enc(void *arg) 1915 { 1916 struct thread_params *tp = arg; 1917 unsigned int enqueued; 1918 const uint16_t queue_id = tp->queue_id; 1919 const uint16_t burst_sz = tp->op_params->burst_sz; 1920 const uint16_t num_to_process = tp->op_params->num_to_process; 1921 struct rte_bbdev_enc_op *ops[num_to_process]; 1922 struct test_buffers *bufs = NULL; 1923 struct rte_bbdev_info info; 1924 int ret, i, j; 1925 uint16_t num_to_enq, enq; 1926 1927 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 1928 "BURST_SIZE should be <= %u", MAX_BURST); 1929 1930 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id), 1931 "Failed to enable interrupts for dev: %u, queue_id: %u", 1932 tp->dev_id, queue_id); 1933 1934 rte_bbdev_info_get(tp->dev_id, &info); 1935 1936 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim), 1937 "NUM_OPS cannot exceed %u for this device", 1938 info.drv.queue_size_lim); 1939 1940 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 1941 1942 rte_atomic16_clear(&tp->processing_status); 1943 rte_atomic16_clear(&tp->nb_dequeued); 1944 1945 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT) 1946 rte_pause(); 1947 1948 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops, 1949 num_to_process); 1950 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", 1951 num_to_process); 1952 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 1953 copy_reference_enc_op(ops, num_to_process, 0, bufs->inputs, 1954 bufs->hard_outputs, tp->op_params->ref_enc_op); 1955 1956 /* Set counter to validate the ordering */ 1957 for (j = 0; j < num_to_process; ++j) 1958 ops[j]->opaque_data = (void *)(uintptr_t)j; 1959 1960 for (j = 0; j < TEST_REPETITIONS; ++j) { 1961 for (i = 0; i < num_to_process; ++i) 1962 rte_pktmbuf_reset(ops[i]->turbo_enc.output.data); 1963 1964 tp->start_time = rte_rdtsc_precise(); 1965 for (enqueued = 0; enqueued < num_to_process;) { 1966 num_to_enq = burst_sz; 1967 1968 if (unlikely(num_to_process - enqueued < num_to_enq)) 1969 num_to_enq = num_to_process - enqueued; 1970 1971 enq = 0; 1972 do { 1973 enq += rte_bbdev_enqueue_enc_ops(tp->dev_id, 1974 queue_id, &ops[enqueued], 1975 num_to_enq); 1976 } while (unlikely(enq != num_to_enq)); 1977 enqueued += enq; 1978 1979 /* Write to thread burst_sz current number of enqueued 1980 * descriptors. It ensures that proper number of 1981 * descriptors will be dequeued in callback 1982 * function - needed for last batch in case where 1983 * the number of operations is not a multiple of 1984 * burst size. 1985 */ 1986 rte_atomic16_set(&tp->burst_sz, num_to_enq); 1987 1988 /* Wait until processing of previous batch is 1989 * completed 1990 */ 1991 while (rte_atomic16_read(&tp->nb_dequeued) != 1992 (int16_t) enqueued) 1993 rte_pause(); 1994 } 1995 if (j != TEST_REPETITIONS - 1) 1996 rte_atomic16_clear(&tp->nb_dequeued); 1997 } 1998 1999 return TEST_SUCCESS; 2000 } 2001 2002 static int 2003 throughput_pmd_lcore_dec(void *arg) 2004 { 2005 struct thread_params *tp = arg; 2006 uint16_t enq, deq; 2007 uint64_t total_time = 0, start_time; 2008 const uint16_t queue_id = tp->queue_id; 2009 const uint16_t burst_sz = tp->op_params->burst_sz; 2010 const uint16_t num_ops = tp->op_params->num_to_process; 2011 struct rte_bbdev_dec_op *ops_enq[num_ops]; 2012 struct rte_bbdev_dec_op *ops_deq[num_ops]; 2013 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 2014 struct test_buffers *bufs = NULL; 2015 int i, j, ret; 2016 struct rte_bbdev_info info; 2017 uint16_t num_to_enq; 2018 2019 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 2020 "BURST_SIZE should be <= %u", MAX_BURST); 2021 2022 rte_bbdev_info_get(tp->dev_id, &info); 2023 2024 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 2025 "NUM_OPS cannot exceed %u for this device", 2026 info.drv.queue_size_lim); 2027 2028 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 2029 2030 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT) 2031 rte_pause(); 2032 2033 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops); 2034 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops); 2035 2036 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 2037 copy_reference_dec_op(ops_enq, num_ops, 0, bufs->inputs, 2038 bufs->hard_outputs, bufs->soft_outputs, ref_op); 2039 2040 /* Set counter to validate the ordering */ 2041 for (j = 0; j < num_ops; ++j) 2042 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 2043 2044 for (i = 0; i < TEST_REPETITIONS; ++i) { 2045 2046 for (j = 0; j < num_ops; ++j) 2047 mbuf_reset(ops_enq[j]->turbo_dec.hard_output.data); 2048 2049 start_time = rte_rdtsc_precise(); 2050 2051 for (enq = 0, deq = 0; enq < num_ops;) { 2052 num_to_enq = burst_sz; 2053 2054 if (unlikely(num_ops - enq < num_to_enq)) 2055 num_to_enq = num_ops - enq; 2056 2057 enq += rte_bbdev_enqueue_dec_ops(tp->dev_id, 2058 queue_id, &ops_enq[enq], num_to_enq); 2059 2060 deq += rte_bbdev_dequeue_dec_ops(tp->dev_id, 2061 queue_id, &ops_deq[deq], enq - deq); 2062 } 2063 2064 /* dequeue the remaining */ 2065 while (deq < enq) { 2066 deq += rte_bbdev_dequeue_dec_ops(tp->dev_id, 2067 queue_id, &ops_deq[deq], enq - deq); 2068 } 2069 2070 total_time += rte_rdtsc_precise() - start_time; 2071 } 2072 2073 tp->iter_count = 0; 2074 /* get the max of iter_count for all dequeued ops */ 2075 for (i = 0; i < num_ops; ++i) { 2076 tp->iter_count = RTE_MAX(ops_enq[i]->turbo_dec.iter_count, 2077 tp->iter_count); 2078 } 2079 2080 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 2081 ret = validate_dec_op(ops_deq, num_ops, ref_op, 2082 tp->op_params->vector_mask); 2083 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 2084 } 2085 2086 rte_bbdev_dec_op_free_bulk(ops_enq, num_ops); 2087 2088 double tb_len_bits = calc_dec_TB_size(ref_op); 2089 2090 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) / 2091 ((double)total_time / (double)rte_get_tsc_hz()); 2092 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) / 2093 1000000.0) / ((double)total_time / 2094 (double)rte_get_tsc_hz()); 2095 2096 return TEST_SUCCESS; 2097 } 2098 2099 static int 2100 throughput_pmd_lcore_ldpc_dec(void *arg) 2101 { 2102 struct thread_params *tp = arg; 2103 uint16_t enq, deq; 2104 uint64_t total_time = 0, start_time; 2105 const uint16_t queue_id = tp->queue_id; 2106 const uint16_t burst_sz = tp->op_params->burst_sz; 2107 const uint16_t num_ops = tp->op_params->num_to_process; 2108 struct rte_bbdev_dec_op *ops_enq[num_ops]; 2109 struct rte_bbdev_dec_op *ops_deq[num_ops]; 2110 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 2111 struct test_buffers *bufs = NULL; 2112 int i, j, ret; 2113 struct rte_bbdev_info info; 2114 uint16_t num_to_enq; 2115 2116 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 2117 "BURST_SIZE should be <= %u", MAX_BURST); 2118 2119 rte_bbdev_info_get(tp->dev_id, &info); 2120 2121 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 2122 "NUM_OPS cannot exceed %u for this device", 2123 info.drv.queue_size_lim); 2124 2125 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 2126 2127 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT) 2128 rte_pause(); 2129 2130 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops); 2131 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops); 2132 2133 /* For throughput tests we need to disable early termination */ 2134 if (check_bit(ref_op->ldpc_dec.op_flags, 2135 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE)) 2136 ref_op->ldpc_dec.op_flags -= 2137 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE; 2138 ref_op->ldpc_dec.iter_max = 6; 2139 ref_op->ldpc_dec.iter_count = ref_op->ldpc_dec.iter_max; 2140 2141 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 2142 copy_reference_ldpc_dec_op(ops_enq, num_ops, 0, bufs->inputs, 2143 bufs->hard_outputs, bufs->soft_outputs, 2144 bufs->harq_inputs, bufs->harq_outputs, ref_op); 2145 2146 /* Set counter to validate the ordering */ 2147 for (j = 0; j < num_ops; ++j) 2148 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 2149 2150 for (i = 0; i < TEST_REPETITIONS; ++i) { 2151 for (j = 0; j < num_ops; ++j) { 2152 mbuf_reset(ops_enq[j]->ldpc_dec.hard_output.data); 2153 if (check_bit(ref_op->ldpc_dec.op_flags, 2154 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE)) 2155 mbuf_reset( 2156 ops_enq[j]->ldpc_dec.harq_combined_output.data); 2157 } 2158 2159 start_time = rte_rdtsc_precise(); 2160 2161 for (enq = 0, deq = 0; enq < num_ops;) { 2162 num_to_enq = burst_sz; 2163 2164 if (unlikely(num_ops - enq < num_to_enq)) 2165 num_to_enq = num_ops - enq; 2166 2167 enq += rte_bbdev_enqueue_ldpc_dec_ops(tp->dev_id, 2168 queue_id, &ops_enq[enq], num_to_enq); 2169 2170 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id, 2171 queue_id, &ops_deq[deq], enq - deq); 2172 } 2173 2174 /* dequeue the remaining */ 2175 while (deq < enq) { 2176 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id, 2177 queue_id, &ops_deq[deq], enq - deq); 2178 } 2179 2180 total_time += rte_rdtsc_precise() - start_time; 2181 } 2182 2183 tp->iter_count = 0; 2184 /* get the max of iter_count for all dequeued ops */ 2185 for (i = 0; i < num_ops; ++i) { 2186 tp->iter_count = RTE_MAX(ops_enq[i]->ldpc_dec.iter_count, 2187 tp->iter_count); 2188 } 2189 2190 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 2191 ret = validate_ldpc_dec_op(ops_deq, num_ops, ref_op, 2192 tp->op_params->vector_mask); 2193 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 2194 } 2195 2196 rte_bbdev_dec_op_free_bulk(ops_enq, num_ops); 2197 2198 double tb_len_bits = calc_ldpc_dec_TB_size(ref_op); 2199 2200 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) / 2201 ((double)total_time / (double)rte_get_tsc_hz()); 2202 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) / 2203 1000000.0) / ((double)total_time / 2204 (double)rte_get_tsc_hz()); 2205 2206 return TEST_SUCCESS; 2207 } 2208 2209 static int 2210 throughput_pmd_lcore_enc(void *arg) 2211 { 2212 struct thread_params *tp = arg; 2213 uint16_t enq, deq; 2214 uint64_t total_time = 0, start_time; 2215 const uint16_t queue_id = tp->queue_id; 2216 const uint16_t burst_sz = tp->op_params->burst_sz; 2217 const uint16_t num_ops = tp->op_params->num_to_process; 2218 struct rte_bbdev_enc_op *ops_enq[num_ops]; 2219 struct rte_bbdev_enc_op *ops_deq[num_ops]; 2220 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op; 2221 struct test_buffers *bufs = NULL; 2222 int i, j, ret; 2223 struct rte_bbdev_info info; 2224 uint16_t num_to_enq; 2225 2226 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 2227 "BURST_SIZE should be <= %u", MAX_BURST); 2228 2229 rte_bbdev_info_get(tp->dev_id, &info); 2230 2231 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 2232 "NUM_OPS cannot exceed %u for this device", 2233 info.drv.queue_size_lim); 2234 2235 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 2236 2237 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT) 2238 rte_pause(); 2239 2240 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops_enq, 2241 num_ops); 2242 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", 2243 num_ops); 2244 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 2245 copy_reference_enc_op(ops_enq, num_ops, 0, bufs->inputs, 2246 bufs->hard_outputs, ref_op); 2247 2248 /* Set counter to validate the ordering */ 2249 for (j = 0; j < num_ops; ++j) 2250 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 2251 2252 for (i = 0; i < TEST_REPETITIONS; ++i) { 2253 2254 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 2255 for (j = 0; j < num_ops; ++j) 2256 mbuf_reset(ops_enq[j]->turbo_enc.output.data); 2257 2258 start_time = rte_rdtsc_precise(); 2259 2260 for (enq = 0, deq = 0; enq < num_ops;) { 2261 num_to_enq = burst_sz; 2262 2263 if (unlikely(num_ops - enq < num_to_enq)) 2264 num_to_enq = num_ops - enq; 2265 2266 enq += rte_bbdev_enqueue_enc_ops(tp->dev_id, 2267 queue_id, &ops_enq[enq], num_to_enq); 2268 2269 deq += rte_bbdev_dequeue_enc_ops(tp->dev_id, 2270 queue_id, &ops_deq[deq], enq - deq); 2271 } 2272 2273 /* dequeue the remaining */ 2274 while (deq < enq) { 2275 deq += rte_bbdev_dequeue_enc_ops(tp->dev_id, 2276 queue_id, &ops_deq[deq], enq - deq); 2277 } 2278 2279 total_time += rte_rdtsc_precise() - start_time; 2280 } 2281 2282 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 2283 ret = validate_enc_op(ops_deq, num_ops, ref_op); 2284 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 2285 } 2286 2287 rte_bbdev_enc_op_free_bulk(ops_enq, num_ops); 2288 2289 double tb_len_bits = calc_enc_TB_size(ref_op); 2290 2291 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) / 2292 ((double)total_time / (double)rte_get_tsc_hz()); 2293 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) 2294 / 1000000.0) / ((double)total_time / 2295 (double)rte_get_tsc_hz()); 2296 2297 return TEST_SUCCESS; 2298 } 2299 2300 static int 2301 throughput_pmd_lcore_ldpc_enc(void *arg) 2302 { 2303 struct thread_params *tp = arg; 2304 uint16_t enq, deq; 2305 uint64_t total_time = 0, start_time; 2306 const uint16_t queue_id = tp->queue_id; 2307 const uint16_t burst_sz = tp->op_params->burst_sz; 2308 const uint16_t num_ops = tp->op_params->num_to_process; 2309 struct rte_bbdev_enc_op *ops_enq[num_ops]; 2310 struct rte_bbdev_enc_op *ops_deq[num_ops]; 2311 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op; 2312 struct test_buffers *bufs = NULL; 2313 int i, j, ret; 2314 struct rte_bbdev_info info; 2315 uint16_t num_to_enq; 2316 2317 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 2318 "BURST_SIZE should be <= %u", MAX_BURST); 2319 2320 rte_bbdev_info_get(tp->dev_id, &info); 2321 2322 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 2323 "NUM_OPS cannot exceed %u for this device", 2324 info.drv.queue_size_lim); 2325 2326 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 2327 2328 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT) 2329 rte_pause(); 2330 2331 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops_enq, 2332 num_ops); 2333 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", 2334 num_ops); 2335 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 2336 copy_reference_ldpc_enc_op(ops_enq, num_ops, 0, bufs->inputs, 2337 bufs->hard_outputs, ref_op); 2338 2339 /* Set counter to validate the ordering */ 2340 for (j = 0; j < num_ops; ++j) 2341 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 2342 2343 for (i = 0; i < TEST_REPETITIONS; ++i) { 2344 2345 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 2346 for (j = 0; j < num_ops; ++j) 2347 mbuf_reset(ops_enq[j]->turbo_enc.output.data); 2348 2349 start_time = rte_rdtsc_precise(); 2350 2351 for (enq = 0, deq = 0; enq < num_ops;) { 2352 num_to_enq = burst_sz; 2353 2354 if (unlikely(num_ops - enq < num_to_enq)) 2355 num_to_enq = num_ops - enq; 2356 2357 enq += rte_bbdev_enqueue_ldpc_enc_ops(tp->dev_id, 2358 queue_id, &ops_enq[enq], num_to_enq); 2359 2360 deq += rte_bbdev_dequeue_ldpc_enc_ops(tp->dev_id, 2361 queue_id, &ops_deq[deq], enq - deq); 2362 } 2363 2364 /* dequeue the remaining */ 2365 while (deq < enq) { 2366 deq += rte_bbdev_dequeue_ldpc_enc_ops(tp->dev_id, 2367 queue_id, &ops_deq[deq], enq - deq); 2368 } 2369 2370 total_time += rte_rdtsc_precise() - start_time; 2371 } 2372 2373 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 2374 ret = validate_ldpc_enc_op(ops_deq, num_ops, ref_op); 2375 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 2376 } 2377 2378 rte_bbdev_enc_op_free_bulk(ops_enq, num_ops); 2379 2380 double tb_len_bits = calc_ldpc_enc_TB_size(ref_op); 2381 2382 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) / 2383 ((double)total_time / (double)rte_get_tsc_hz()); 2384 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) 2385 / 1000000.0) / ((double)total_time / 2386 (double)rte_get_tsc_hz()); 2387 2388 return TEST_SUCCESS; 2389 } 2390 2391 static void 2392 print_enc_throughput(struct thread_params *t_params, unsigned int used_cores) 2393 { 2394 unsigned int iter = 0; 2395 double total_mops = 0, total_mbps = 0; 2396 2397 for (iter = 0; iter < used_cores; iter++) { 2398 printf( 2399 "Throughput for core (%u): %.8lg Ops/s, %.8lg Mbps\n", 2400 t_params[iter].lcore_id, t_params[iter].ops_per_sec, 2401 t_params[iter].mbps); 2402 total_mops += t_params[iter].ops_per_sec; 2403 total_mbps += t_params[iter].mbps; 2404 } 2405 printf( 2406 "\nTotal throughput for %u cores: %.8lg MOPS, %.8lg Mbps\n", 2407 used_cores, total_mops, total_mbps); 2408 } 2409 2410 static void 2411 print_dec_throughput(struct thread_params *t_params, unsigned int used_cores) 2412 { 2413 unsigned int iter = 0; 2414 double total_mops = 0, total_mbps = 0; 2415 uint8_t iter_count = 0; 2416 2417 for (iter = 0; iter < used_cores; iter++) { 2418 printf( 2419 "Throughput for core (%u): %.8lg Ops/s, %.8lg Mbps @ max %u iterations\n", 2420 t_params[iter].lcore_id, t_params[iter].ops_per_sec, 2421 t_params[iter].mbps, t_params[iter].iter_count); 2422 total_mops += t_params[iter].ops_per_sec; 2423 total_mbps += t_params[iter].mbps; 2424 iter_count = RTE_MAX(iter_count, t_params[iter].iter_count); 2425 } 2426 printf( 2427 "\nTotal throughput for %u cores: %.8lg MOPS, %.8lg Mbps @ max %u iterations\n", 2428 used_cores, total_mops, total_mbps, iter_count); 2429 } 2430 2431 /* 2432 * Test function that determines how long an enqueue + dequeue of a burst 2433 * takes on available lcores. 2434 */ 2435 static int 2436 throughput_test(struct active_device *ad, 2437 struct test_op_params *op_params) 2438 { 2439 int ret; 2440 unsigned int lcore_id, used_cores = 0; 2441 struct thread_params *t_params, *tp; 2442 struct rte_bbdev_info info; 2443 lcore_function_t *throughput_function; 2444 uint16_t num_lcores; 2445 const char *op_type_str; 2446 2447 rte_bbdev_info_get(ad->dev_id, &info); 2448 2449 op_type_str = rte_bbdev_op_type_str(test_vector.op_type); 2450 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", 2451 test_vector.op_type); 2452 2453 printf("+ ------------------------------------------------------- +\n"); 2454 printf("== test: throughput\ndev: %s, nb_queues: %u, burst size: %u, num ops: %u, num_lcores: %u, op type: %s, itr mode: %s, GHz: %lg\n", 2455 info.dev_name, ad->nb_queues, op_params->burst_sz, 2456 op_params->num_to_process, op_params->num_lcores, 2457 op_type_str, 2458 intr_enabled ? "Interrupt mode" : "PMD mode", 2459 (double)rte_get_tsc_hz() / 1000000000.0); 2460 2461 /* Set number of lcores */ 2462 num_lcores = (ad->nb_queues < (op_params->num_lcores)) 2463 ? ad->nb_queues 2464 : op_params->num_lcores; 2465 2466 /* Allocate memory for thread parameters structure */ 2467 t_params = rte_zmalloc(NULL, num_lcores * sizeof(struct thread_params), 2468 RTE_CACHE_LINE_SIZE); 2469 TEST_ASSERT_NOT_NULL(t_params, "Failed to alloc %zuB for t_params", 2470 RTE_ALIGN(sizeof(struct thread_params) * num_lcores, 2471 RTE_CACHE_LINE_SIZE)); 2472 2473 if (intr_enabled) { 2474 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) 2475 throughput_function = throughput_intr_lcore_dec; 2476 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 2477 throughput_function = throughput_intr_lcore_dec; 2478 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) 2479 throughput_function = throughput_intr_lcore_enc; 2480 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) 2481 throughput_function = throughput_intr_lcore_enc; 2482 else 2483 throughput_function = throughput_intr_lcore_enc; 2484 2485 /* Dequeue interrupt callback registration */ 2486 ret = rte_bbdev_callback_register(ad->dev_id, 2487 RTE_BBDEV_EVENT_DEQUEUE, dequeue_event_callback, 2488 t_params); 2489 if (ret < 0) { 2490 rte_free(t_params); 2491 return ret; 2492 } 2493 } else { 2494 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) 2495 throughput_function = throughput_pmd_lcore_dec; 2496 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 2497 throughput_function = throughput_pmd_lcore_ldpc_dec; 2498 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) 2499 throughput_function = throughput_pmd_lcore_enc; 2500 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) 2501 throughput_function = throughput_pmd_lcore_ldpc_enc; 2502 else 2503 throughput_function = throughput_pmd_lcore_enc; 2504 } 2505 2506 rte_atomic16_set(&op_params->sync, SYNC_WAIT); 2507 2508 /* Master core is set at first entry */ 2509 t_params[0].dev_id = ad->dev_id; 2510 t_params[0].lcore_id = rte_lcore_id(); 2511 t_params[0].op_params = op_params; 2512 t_params[0].queue_id = ad->queue_ids[used_cores++]; 2513 t_params[0].iter_count = 0; 2514 2515 RTE_LCORE_FOREACH_SLAVE(lcore_id) { 2516 if (used_cores >= num_lcores) 2517 break; 2518 2519 t_params[used_cores].dev_id = ad->dev_id; 2520 t_params[used_cores].lcore_id = lcore_id; 2521 t_params[used_cores].op_params = op_params; 2522 t_params[used_cores].queue_id = ad->queue_ids[used_cores]; 2523 t_params[used_cores].iter_count = 0; 2524 2525 rte_eal_remote_launch(throughput_function, 2526 &t_params[used_cores++], lcore_id); 2527 } 2528 2529 rte_atomic16_set(&op_params->sync, SYNC_START); 2530 ret = throughput_function(&t_params[0]); 2531 2532 /* Master core is always used */ 2533 for (used_cores = 1; used_cores < num_lcores; used_cores++) 2534 ret |= rte_eal_wait_lcore(t_params[used_cores].lcore_id); 2535 2536 /* Return if test failed */ 2537 if (ret) { 2538 rte_free(t_params); 2539 return ret; 2540 } 2541 2542 /* Print throughput if interrupts are disabled and test passed */ 2543 if (!intr_enabled) { 2544 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC || 2545 test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 2546 print_dec_throughput(t_params, num_lcores); 2547 else 2548 print_enc_throughput(t_params, num_lcores); 2549 rte_free(t_params); 2550 return ret; 2551 } 2552 2553 /* In interrupt TC we need to wait for the interrupt callback to deqeue 2554 * all pending operations. Skip waiting for queues which reported an 2555 * error using processing_status variable. 2556 * Wait for master lcore operations. 2557 */ 2558 tp = &t_params[0]; 2559 while ((rte_atomic16_read(&tp->nb_dequeued) < 2560 op_params->num_to_process) && 2561 (rte_atomic16_read(&tp->processing_status) != 2562 TEST_FAILED)) 2563 rte_pause(); 2564 2565 tp->ops_per_sec /= TEST_REPETITIONS; 2566 tp->mbps /= TEST_REPETITIONS; 2567 ret |= (int)rte_atomic16_read(&tp->processing_status); 2568 2569 /* Wait for slave lcores operations */ 2570 for (used_cores = 1; used_cores < num_lcores; used_cores++) { 2571 tp = &t_params[used_cores]; 2572 2573 while ((rte_atomic16_read(&tp->nb_dequeued) < 2574 op_params->num_to_process) && 2575 (rte_atomic16_read(&tp->processing_status) != 2576 TEST_FAILED)) 2577 rte_pause(); 2578 2579 tp->ops_per_sec /= TEST_REPETITIONS; 2580 tp->mbps /= TEST_REPETITIONS; 2581 ret |= (int)rte_atomic16_read(&tp->processing_status); 2582 } 2583 2584 /* Print throughput if test passed */ 2585 if (!ret) { 2586 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC || 2587 test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 2588 print_dec_throughput(t_params, num_lcores); 2589 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC || 2590 test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) 2591 print_enc_throughput(t_params, num_lcores); 2592 } 2593 2594 rte_free(t_params); 2595 return ret; 2596 } 2597 2598 static int 2599 latency_test_dec(struct rte_mempool *mempool, 2600 struct test_buffers *bufs, struct rte_bbdev_dec_op *ref_op, 2601 int vector_mask, uint16_t dev_id, uint16_t queue_id, 2602 const uint16_t num_to_process, uint16_t burst_sz, 2603 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time) 2604 { 2605 int ret = TEST_SUCCESS; 2606 uint16_t i, j, dequeued; 2607 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 2608 uint64_t start_time = 0, last_time = 0; 2609 2610 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 2611 uint16_t enq = 0, deq = 0; 2612 bool first_time = true; 2613 last_time = 0; 2614 2615 if (unlikely(num_to_process - dequeued < burst_sz)) 2616 burst_sz = num_to_process - dequeued; 2617 2618 ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz); 2619 TEST_ASSERT_SUCCESS(ret, 2620 "rte_bbdev_dec_op_alloc_bulk() failed"); 2621 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 2622 copy_reference_dec_op(ops_enq, burst_sz, dequeued, 2623 bufs->inputs, 2624 bufs->hard_outputs, 2625 bufs->soft_outputs, 2626 ref_op); 2627 2628 /* Set counter to validate the ordering */ 2629 for (j = 0; j < burst_sz; ++j) 2630 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 2631 2632 start_time = rte_rdtsc_precise(); 2633 2634 enq = rte_bbdev_enqueue_dec_ops(dev_id, queue_id, &ops_enq[enq], 2635 burst_sz); 2636 TEST_ASSERT(enq == burst_sz, 2637 "Error enqueueing burst, expected %u, got %u", 2638 burst_sz, enq); 2639 2640 /* Dequeue */ 2641 do { 2642 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id, 2643 &ops_deq[deq], burst_sz - deq); 2644 if (likely(first_time && (deq > 0))) { 2645 last_time = rte_rdtsc_precise() - start_time; 2646 first_time = false; 2647 } 2648 } while (unlikely(burst_sz != deq)); 2649 2650 *max_time = RTE_MAX(*max_time, last_time); 2651 *min_time = RTE_MIN(*min_time, last_time); 2652 *total_time += last_time; 2653 2654 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 2655 ret = validate_dec_op(ops_deq, burst_sz, ref_op, 2656 vector_mask); 2657 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 2658 } 2659 2660 rte_bbdev_dec_op_free_bulk(ops_enq, deq); 2661 dequeued += deq; 2662 } 2663 2664 return i; 2665 } 2666 2667 static int 2668 latency_test_ldpc_dec(struct rte_mempool *mempool, 2669 struct test_buffers *bufs, struct rte_bbdev_dec_op *ref_op, 2670 int vector_mask, uint16_t dev_id, uint16_t queue_id, 2671 const uint16_t num_to_process, uint16_t burst_sz, 2672 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time) 2673 { 2674 int ret = TEST_SUCCESS; 2675 uint16_t i, j, dequeued; 2676 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 2677 uint64_t start_time = 0, last_time = 0; 2678 2679 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 2680 uint16_t enq = 0, deq = 0; 2681 bool first_time = true; 2682 last_time = 0; 2683 2684 if (unlikely(num_to_process - dequeued < burst_sz)) 2685 burst_sz = num_to_process - dequeued; 2686 2687 ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz); 2688 TEST_ASSERT_SUCCESS(ret, 2689 "rte_bbdev_dec_op_alloc_bulk() failed"); 2690 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 2691 copy_reference_ldpc_dec_op(ops_enq, burst_sz, dequeued, 2692 bufs->inputs, 2693 bufs->hard_outputs, 2694 bufs->soft_outputs, 2695 bufs->harq_inputs, 2696 bufs->harq_outputs, 2697 ref_op); 2698 2699 /* Set counter to validate the ordering */ 2700 for (j = 0; j < burst_sz; ++j) 2701 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 2702 2703 start_time = rte_rdtsc_precise(); 2704 2705 enq = rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id, 2706 &ops_enq[enq], burst_sz); 2707 TEST_ASSERT(enq == burst_sz, 2708 "Error enqueueing burst, expected %u, got %u", 2709 burst_sz, enq); 2710 2711 /* Dequeue */ 2712 do { 2713 deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id, 2714 &ops_deq[deq], burst_sz - deq); 2715 if (likely(first_time && (deq > 0))) { 2716 last_time = rte_rdtsc_precise() - start_time; 2717 first_time = false; 2718 } 2719 } while (unlikely(burst_sz != deq)); 2720 2721 *max_time = RTE_MAX(*max_time, last_time); 2722 *min_time = RTE_MIN(*min_time, last_time); 2723 *total_time += last_time; 2724 2725 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 2726 ret = validate_ldpc_dec_op(ops_deq, burst_sz, ref_op, 2727 vector_mask); 2728 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 2729 } 2730 2731 rte_bbdev_dec_op_free_bulk(ops_enq, deq); 2732 dequeued += deq; 2733 } 2734 2735 return i; 2736 } 2737 2738 static int 2739 latency_test_enc(struct rte_mempool *mempool, 2740 struct test_buffers *bufs, struct rte_bbdev_enc_op *ref_op, 2741 uint16_t dev_id, uint16_t queue_id, 2742 const uint16_t num_to_process, uint16_t burst_sz, 2743 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time) 2744 { 2745 int ret = TEST_SUCCESS; 2746 uint16_t i, j, dequeued; 2747 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 2748 uint64_t start_time = 0, last_time = 0; 2749 2750 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 2751 uint16_t enq = 0, deq = 0; 2752 bool first_time = true; 2753 last_time = 0; 2754 2755 if (unlikely(num_to_process - dequeued < burst_sz)) 2756 burst_sz = num_to_process - dequeued; 2757 2758 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz); 2759 TEST_ASSERT_SUCCESS(ret, 2760 "rte_bbdev_enc_op_alloc_bulk() failed"); 2761 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 2762 copy_reference_enc_op(ops_enq, burst_sz, dequeued, 2763 bufs->inputs, 2764 bufs->hard_outputs, 2765 ref_op); 2766 2767 /* Set counter to validate the ordering */ 2768 for (j = 0; j < burst_sz; ++j) 2769 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 2770 2771 start_time = rte_rdtsc_precise(); 2772 2773 enq = rte_bbdev_enqueue_enc_ops(dev_id, queue_id, &ops_enq[enq], 2774 burst_sz); 2775 TEST_ASSERT(enq == burst_sz, 2776 "Error enqueueing burst, expected %u, got %u", 2777 burst_sz, enq); 2778 2779 /* Dequeue */ 2780 do { 2781 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id, 2782 &ops_deq[deq], burst_sz - deq); 2783 if (likely(first_time && (deq > 0))) { 2784 last_time += rte_rdtsc_precise() - start_time; 2785 first_time = false; 2786 } 2787 } while (unlikely(burst_sz != deq)); 2788 2789 *max_time = RTE_MAX(*max_time, last_time); 2790 *min_time = RTE_MIN(*min_time, last_time); 2791 *total_time += last_time; 2792 2793 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 2794 ret = validate_enc_op(ops_deq, burst_sz, ref_op); 2795 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 2796 } 2797 2798 rte_bbdev_enc_op_free_bulk(ops_enq, deq); 2799 dequeued += deq; 2800 } 2801 2802 return i; 2803 } 2804 2805 static int 2806 latency_test_ldpc_enc(struct rte_mempool *mempool, 2807 struct test_buffers *bufs, struct rte_bbdev_enc_op *ref_op, 2808 uint16_t dev_id, uint16_t queue_id, 2809 const uint16_t num_to_process, uint16_t burst_sz, 2810 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time) 2811 { 2812 int ret = TEST_SUCCESS; 2813 uint16_t i, j, dequeued; 2814 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 2815 uint64_t start_time = 0, last_time = 0; 2816 2817 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 2818 uint16_t enq = 0, deq = 0; 2819 bool first_time = true; 2820 last_time = 0; 2821 2822 if (unlikely(num_to_process - dequeued < burst_sz)) 2823 burst_sz = num_to_process - dequeued; 2824 2825 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz); 2826 2827 TEST_ASSERT_SUCCESS(ret, 2828 "rte_bbdev_enc_op_alloc_bulk() failed"); 2829 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 2830 copy_reference_ldpc_enc_op(ops_enq, burst_sz, dequeued, 2831 bufs->inputs, 2832 bufs->hard_outputs, 2833 ref_op); 2834 2835 /* Set counter to validate the ordering */ 2836 for (j = 0; j < burst_sz; ++j) 2837 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 2838 2839 start_time = rte_rdtsc_precise(); 2840 2841 /* 2842 * printf("Latency Debug %d\n", 2843 * ops_enq[0]->ldpc_enc.cb_params.z_c); REMOVEME 2844 */ 2845 2846 enq = rte_bbdev_enqueue_ldpc_enc_ops(dev_id, queue_id, 2847 &ops_enq[enq], burst_sz); 2848 TEST_ASSERT(enq == burst_sz, 2849 "Error enqueueing burst, expected %u, got %u", 2850 burst_sz, enq); 2851 2852 /* Dequeue */ 2853 do { 2854 deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id, 2855 &ops_deq[deq], burst_sz - deq); 2856 if (likely(first_time && (deq > 0))) { 2857 last_time += rte_rdtsc_precise() - start_time; 2858 first_time = false; 2859 } 2860 } while (unlikely(burst_sz != deq)); 2861 2862 *max_time = RTE_MAX(*max_time, last_time); 2863 *min_time = RTE_MIN(*min_time, last_time); 2864 *total_time += last_time; 2865 2866 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 2867 ret = validate_enc_op(ops_deq, burst_sz, ref_op); 2868 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 2869 } 2870 2871 /* 2872 * printf("Ready to free - deq %d num_to_process %d\n", FIXME 2873 * deq, num_to_process); 2874 * printf("cache %d\n", ops_enq[0]->mempool->cache_size); 2875 */ 2876 rte_bbdev_enc_op_free_bulk(ops_enq, deq); 2877 dequeued += deq; 2878 } 2879 2880 return i; 2881 } 2882 2883 static int 2884 latency_test(struct active_device *ad, 2885 struct test_op_params *op_params) 2886 { 2887 int iter; 2888 uint16_t burst_sz = op_params->burst_sz; 2889 const uint16_t num_to_process = op_params->num_to_process; 2890 const enum rte_bbdev_op_type op_type = test_vector.op_type; 2891 const uint16_t queue_id = ad->queue_ids[0]; 2892 struct test_buffers *bufs = NULL; 2893 struct rte_bbdev_info info; 2894 uint64_t total_time, min_time, max_time; 2895 const char *op_type_str; 2896 2897 total_time = max_time = 0; 2898 min_time = UINT64_MAX; 2899 2900 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 2901 "BURST_SIZE should be <= %u", MAX_BURST); 2902 2903 rte_bbdev_info_get(ad->dev_id, &info); 2904 bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 2905 2906 op_type_str = rte_bbdev_op_type_str(op_type); 2907 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type); 2908 2909 printf("+ ------------------------------------------------------- +\n"); 2910 printf("== test: validation/latency\ndev: %s, burst size: %u, num ops: %u, op type: %s\n", 2911 info.dev_name, burst_sz, num_to_process, op_type_str); 2912 2913 if (op_type == RTE_BBDEV_OP_TURBO_DEC) 2914 iter = latency_test_dec(op_params->mp, bufs, 2915 op_params->ref_dec_op, op_params->vector_mask, 2916 ad->dev_id, queue_id, num_to_process, 2917 burst_sz, &total_time, &min_time, &max_time); 2918 else if (op_type == RTE_BBDEV_OP_TURBO_ENC) 2919 iter = latency_test_enc(op_params->mp, bufs, 2920 op_params->ref_enc_op, ad->dev_id, queue_id, 2921 num_to_process, burst_sz, &total_time, 2922 &min_time, &max_time); 2923 else if (op_type == RTE_BBDEV_OP_LDPC_ENC) 2924 iter = latency_test_ldpc_enc(op_params->mp, bufs, 2925 op_params->ref_enc_op, ad->dev_id, queue_id, 2926 num_to_process, burst_sz, &total_time, 2927 &min_time, &max_time); 2928 else if (op_type == RTE_BBDEV_OP_LDPC_DEC) 2929 iter = latency_test_ldpc_dec(op_params->mp, bufs, 2930 op_params->ref_dec_op, op_params->vector_mask, 2931 ad->dev_id, queue_id, num_to_process, 2932 burst_sz, &total_time, &min_time, &max_time); 2933 else 2934 iter = latency_test_enc(op_params->mp, bufs, 2935 op_params->ref_enc_op, 2936 ad->dev_id, queue_id, 2937 num_to_process, burst_sz, &total_time, 2938 &min_time, &max_time); 2939 2940 if (iter <= 0) 2941 return TEST_FAILED; 2942 2943 printf("Operation latency:\n" 2944 "\tavg: %lg cycles, %lg us\n" 2945 "\tmin: %lg cycles, %lg us\n" 2946 "\tmax: %lg cycles, %lg us\n", 2947 (double)total_time / (double)iter, 2948 (double)(total_time * 1000000) / (double)iter / 2949 (double)rte_get_tsc_hz(), (double)min_time, 2950 (double)(min_time * 1000000) / (double)rte_get_tsc_hz(), 2951 (double)max_time, (double)(max_time * 1000000) / 2952 (double)rte_get_tsc_hz()); 2953 2954 return TEST_SUCCESS; 2955 } 2956 2957 #ifdef RTE_BBDEV_OFFLOAD_COST 2958 static int 2959 get_bbdev_queue_stats(uint16_t dev_id, uint16_t queue_id, 2960 struct rte_bbdev_stats *stats) 2961 { 2962 struct rte_bbdev *dev = &rte_bbdev_devices[dev_id]; 2963 struct rte_bbdev_stats *q_stats; 2964 2965 if (queue_id >= dev->data->num_queues) 2966 return -1; 2967 2968 q_stats = &dev->data->queues[queue_id].queue_stats; 2969 2970 stats->enqueued_count = q_stats->enqueued_count; 2971 stats->dequeued_count = q_stats->dequeued_count; 2972 stats->enqueue_err_count = q_stats->enqueue_err_count; 2973 stats->dequeue_err_count = q_stats->dequeue_err_count; 2974 stats->acc_offload_cycles = q_stats->acc_offload_cycles; 2975 2976 return 0; 2977 } 2978 2979 static int 2980 offload_latency_test_dec(struct rte_mempool *mempool, struct test_buffers *bufs, 2981 struct rte_bbdev_dec_op *ref_op, uint16_t dev_id, 2982 uint16_t queue_id, const uint16_t num_to_process, 2983 uint16_t burst_sz, struct test_time_stats *time_st) 2984 { 2985 int i, dequeued, ret; 2986 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 2987 uint64_t enq_start_time, deq_start_time; 2988 uint64_t enq_sw_last_time, deq_last_time; 2989 struct rte_bbdev_stats stats; 2990 2991 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 2992 uint16_t enq = 0, deq = 0; 2993 2994 if (unlikely(num_to_process - dequeued < burst_sz)) 2995 burst_sz = num_to_process - dequeued; 2996 2997 rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz); 2998 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 2999 copy_reference_dec_op(ops_enq, burst_sz, dequeued, 3000 bufs->inputs, 3001 bufs->hard_outputs, 3002 bufs->soft_outputs, 3003 ref_op); 3004 3005 /* Start time meas for enqueue function offload latency */ 3006 enq_start_time = rte_rdtsc_precise(); 3007 do { 3008 enq += rte_bbdev_enqueue_dec_ops(dev_id, queue_id, 3009 &ops_enq[enq], burst_sz - enq); 3010 } while (unlikely(burst_sz != enq)); 3011 3012 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats); 3013 TEST_ASSERT_SUCCESS(ret, 3014 "Failed to get stats for queue (%u) of device (%u)", 3015 queue_id, dev_id); 3016 3017 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time - 3018 stats.acc_offload_cycles; 3019 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time, 3020 enq_sw_last_time); 3021 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time, 3022 enq_sw_last_time); 3023 time_st->enq_sw_total_time += enq_sw_last_time; 3024 3025 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time, 3026 stats.acc_offload_cycles); 3027 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time, 3028 stats.acc_offload_cycles); 3029 time_st->enq_acc_total_time += stats.acc_offload_cycles; 3030 3031 /* give time for device to process ops */ 3032 rte_delay_us(200); 3033 3034 /* Start time meas for dequeue function offload latency */ 3035 deq_start_time = rte_rdtsc_precise(); 3036 /* Dequeue one operation */ 3037 do { 3038 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id, 3039 &ops_deq[deq], 1); 3040 } while (unlikely(deq != 1)); 3041 3042 deq_last_time = rte_rdtsc_precise() - deq_start_time; 3043 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time, 3044 deq_last_time); 3045 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time, 3046 deq_last_time); 3047 time_st->deq_total_time += deq_last_time; 3048 3049 /* Dequeue remaining operations if needed*/ 3050 while (burst_sz != deq) 3051 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id, 3052 &ops_deq[deq], burst_sz - deq); 3053 3054 rte_bbdev_dec_op_free_bulk(ops_enq, deq); 3055 dequeued += deq; 3056 } 3057 3058 return i; 3059 } 3060 3061 static int 3062 offload_latency_test_ldpc_dec(struct rte_mempool *mempool, 3063 struct test_buffers *bufs, 3064 struct rte_bbdev_dec_op *ref_op, uint16_t dev_id, 3065 uint16_t queue_id, const uint16_t num_to_process, 3066 uint16_t burst_sz, struct test_time_stats *time_st) 3067 { 3068 int i, dequeued, ret; 3069 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 3070 uint64_t enq_start_time, deq_start_time; 3071 uint64_t enq_sw_last_time, deq_last_time; 3072 struct rte_bbdev_stats stats; 3073 3074 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 3075 uint16_t enq = 0, deq = 0; 3076 3077 if (unlikely(num_to_process - dequeued < burst_sz)) 3078 burst_sz = num_to_process - dequeued; 3079 3080 rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz); 3081 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3082 copy_reference_ldpc_dec_op(ops_enq, burst_sz, dequeued, 3083 bufs->inputs, 3084 bufs->hard_outputs, 3085 bufs->soft_outputs, 3086 bufs->harq_inputs, 3087 bufs->harq_outputs, 3088 ref_op); 3089 3090 /* Start time meas for enqueue function offload latency */ 3091 enq_start_time = rte_rdtsc_precise(); 3092 do { 3093 enq += rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id, 3094 &ops_enq[enq], burst_sz - enq); 3095 } while (unlikely(burst_sz != enq)); 3096 3097 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats); 3098 TEST_ASSERT_SUCCESS(ret, 3099 "Failed to get stats for queue (%u) of device (%u)", 3100 queue_id, dev_id); 3101 3102 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time - 3103 stats.acc_offload_cycles; 3104 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time, 3105 enq_sw_last_time); 3106 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time, 3107 enq_sw_last_time); 3108 time_st->enq_sw_total_time += enq_sw_last_time; 3109 3110 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time, 3111 stats.acc_offload_cycles); 3112 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time, 3113 stats.acc_offload_cycles); 3114 time_st->enq_acc_total_time += stats.acc_offload_cycles; 3115 3116 /* give time for device to process ops */ 3117 rte_delay_us(200); 3118 3119 /* Start time meas for dequeue function offload latency */ 3120 deq_start_time = rte_rdtsc_precise(); 3121 /* Dequeue one operation */ 3122 do { 3123 deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id, 3124 &ops_deq[deq], 1); 3125 } while (unlikely(deq != 1)); 3126 3127 deq_last_time = rte_rdtsc_precise() - deq_start_time; 3128 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time, 3129 deq_last_time); 3130 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time, 3131 deq_last_time); 3132 time_st->deq_total_time += deq_last_time; 3133 3134 /* Dequeue remaining operations if needed*/ 3135 while (burst_sz != deq) 3136 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id, 3137 &ops_deq[deq], burst_sz - deq); 3138 3139 rte_bbdev_dec_op_free_bulk(ops_enq, deq); 3140 dequeued += deq; 3141 } 3142 3143 return i; 3144 } 3145 3146 static int 3147 offload_latency_test_enc(struct rte_mempool *mempool, struct test_buffers *bufs, 3148 struct rte_bbdev_enc_op *ref_op, uint16_t dev_id, 3149 uint16_t queue_id, const uint16_t num_to_process, 3150 uint16_t burst_sz, struct test_time_stats *time_st) 3151 { 3152 int i, dequeued, ret; 3153 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 3154 uint64_t enq_start_time, deq_start_time; 3155 uint64_t enq_sw_last_time, deq_last_time; 3156 struct rte_bbdev_stats stats; 3157 3158 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 3159 uint16_t enq = 0, deq = 0; 3160 3161 if (unlikely(num_to_process - dequeued < burst_sz)) 3162 burst_sz = num_to_process - dequeued; 3163 3164 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz); 3165 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_op_alloc_bulk() failed"); 3166 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3167 copy_reference_enc_op(ops_enq, burst_sz, dequeued, 3168 bufs->inputs, 3169 bufs->hard_outputs, 3170 ref_op); 3171 3172 /* Start time meas for enqueue function offload latency */ 3173 enq_start_time = rte_rdtsc_precise(); 3174 do { 3175 enq += rte_bbdev_enqueue_enc_ops(dev_id, queue_id, 3176 &ops_enq[enq], burst_sz - enq); 3177 } while (unlikely(burst_sz != enq)); 3178 3179 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats); 3180 TEST_ASSERT_SUCCESS(ret, 3181 "Failed to get stats for queue (%u) of device (%u)", 3182 queue_id, dev_id); 3183 3184 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time - 3185 stats.acc_offload_cycles; 3186 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time, 3187 enq_sw_last_time); 3188 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time, 3189 enq_sw_last_time); 3190 time_st->enq_sw_total_time += enq_sw_last_time; 3191 3192 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time, 3193 stats.acc_offload_cycles); 3194 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time, 3195 stats.acc_offload_cycles); 3196 time_st->enq_acc_total_time += stats.acc_offload_cycles; 3197 3198 /* give time for device to process ops */ 3199 rte_delay_us(200); 3200 3201 /* Start time meas for dequeue function offload latency */ 3202 deq_start_time = rte_rdtsc_precise(); 3203 /* Dequeue one operation */ 3204 do { 3205 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id, 3206 &ops_deq[deq], 1); 3207 } while (unlikely(deq != 1)); 3208 3209 deq_last_time = rte_rdtsc_precise() - deq_start_time; 3210 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time, 3211 deq_last_time); 3212 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time, 3213 deq_last_time); 3214 time_st->deq_total_time += deq_last_time; 3215 3216 while (burst_sz != deq) 3217 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id, 3218 &ops_deq[deq], burst_sz - deq); 3219 3220 rte_bbdev_enc_op_free_bulk(ops_enq, deq); 3221 dequeued += deq; 3222 } 3223 3224 return i; 3225 } 3226 3227 static int 3228 offload_latency_test_ldpc_enc(struct rte_mempool *mempool, 3229 struct test_buffers *bufs, 3230 struct rte_bbdev_enc_op *ref_op, uint16_t dev_id, 3231 uint16_t queue_id, const uint16_t num_to_process, 3232 uint16_t burst_sz, struct test_time_stats *time_st) 3233 { 3234 int i, dequeued, ret; 3235 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 3236 uint64_t enq_start_time, deq_start_time; 3237 uint64_t enq_sw_last_time, deq_last_time; 3238 struct rte_bbdev_stats stats; 3239 3240 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 3241 uint16_t enq = 0, deq = 0; 3242 3243 if (unlikely(num_to_process - dequeued < burst_sz)) 3244 burst_sz = num_to_process - dequeued; 3245 3246 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz); 3247 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_op_alloc_bulk() failed"); 3248 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3249 copy_reference_ldpc_enc_op(ops_enq, burst_sz, dequeued, 3250 bufs->inputs, 3251 bufs->hard_outputs, 3252 ref_op); 3253 3254 /* Start time meas for enqueue function offload latency */ 3255 enq_start_time = rte_rdtsc_precise(); 3256 do { 3257 enq += rte_bbdev_enqueue_ldpc_enc_ops(dev_id, queue_id, 3258 &ops_enq[enq], burst_sz - enq); 3259 } while (unlikely(burst_sz != enq)); 3260 3261 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats); 3262 TEST_ASSERT_SUCCESS(ret, 3263 "Failed to get stats for queue (%u) of device (%u)", 3264 queue_id, dev_id); 3265 3266 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time - 3267 stats.acc_offload_cycles; 3268 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time, 3269 enq_sw_last_time); 3270 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time, 3271 enq_sw_last_time); 3272 time_st->enq_sw_total_time += enq_sw_last_time; 3273 3274 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time, 3275 stats.acc_offload_cycles); 3276 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time, 3277 stats.acc_offload_cycles); 3278 time_st->enq_acc_total_time += stats.acc_offload_cycles; 3279 3280 /* give time for device to process ops */ 3281 rte_delay_us(200); 3282 3283 /* Start time meas for dequeue function offload latency */ 3284 deq_start_time = rte_rdtsc_precise(); 3285 /* Dequeue one operation */ 3286 do { 3287 deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id, 3288 &ops_deq[deq], 1); 3289 } while (unlikely(deq != 1)); 3290 3291 deq_last_time = rte_rdtsc_precise() - deq_start_time; 3292 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time, 3293 deq_last_time); 3294 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time, 3295 deq_last_time); 3296 time_st->deq_total_time += deq_last_time; 3297 3298 while (burst_sz != deq) 3299 deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id, 3300 &ops_deq[deq], burst_sz - deq); 3301 3302 rte_bbdev_enc_op_free_bulk(ops_enq, deq); 3303 dequeued += deq; 3304 } 3305 3306 return i; 3307 } 3308 #endif 3309 3310 static int 3311 offload_cost_test(struct active_device *ad, 3312 struct test_op_params *op_params) 3313 { 3314 #ifndef RTE_BBDEV_OFFLOAD_COST 3315 RTE_SET_USED(ad); 3316 RTE_SET_USED(op_params); 3317 printf("Offload latency test is disabled.\n"); 3318 printf("Set RTE_BBDEV_OFFLOAD_COST to 'y' to turn the test on.\n"); 3319 return TEST_SKIPPED; 3320 #else 3321 int iter; 3322 uint16_t burst_sz = op_params->burst_sz; 3323 const uint16_t num_to_process = op_params->num_to_process; 3324 const enum rte_bbdev_op_type op_type = test_vector.op_type; 3325 const uint16_t queue_id = ad->queue_ids[0]; 3326 struct test_buffers *bufs = NULL; 3327 struct rte_bbdev_info info; 3328 const char *op_type_str; 3329 struct test_time_stats time_st; 3330 3331 memset(&time_st, 0, sizeof(struct test_time_stats)); 3332 time_st.enq_sw_min_time = UINT64_MAX; 3333 time_st.enq_acc_min_time = UINT64_MAX; 3334 time_st.deq_min_time = UINT64_MAX; 3335 3336 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 3337 "BURST_SIZE should be <= %u", MAX_BURST); 3338 3339 rte_bbdev_info_get(ad->dev_id, &info); 3340 bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 3341 3342 op_type_str = rte_bbdev_op_type_str(op_type); 3343 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type); 3344 3345 printf("+ ------------------------------------------------------- +\n"); 3346 printf("== test: offload latency test\ndev: %s, burst size: %u, num ops: %u, op type: %s\n", 3347 info.dev_name, burst_sz, num_to_process, op_type_str); 3348 3349 if (op_type == RTE_BBDEV_OP_TURBO_DEC) 3350 iter = offload_latency_test_dec(op_params->mp, bufs, 3351 op_params->ref_dec_op, ad->dev_id, queue_id, 3352 num_to_process, burst_sz, &time_st); 3353 else if (op_type == RTE_BBDEV_OP_TURBO_ENC) 3354 iter = offload_latency_test_enc(op_params->mp, bufs, 3355 op_params->ref_enc_op, ad->dev_id, queue_id, 3356 num_to_process, burst_sz, &time_st); 3357 else if (op_type == RTE_BBDEV_OP_LDPC_ENC) 3358 iter = offload_latency_test_ldpc_enc(op_params->mp, bufs, 3359 op_params->ref_enc_op, ad->dev_id, queue_id, 3360 num_to_process, burst_sz, &time_st); 3361 else if (op_type == RTE_BBDEV_OP_LDPC_DEC) 3362 iter = offload_latency_test_ldpc_dec(op_params->mp, bufs, 3363 op_params->ref_dec_op, ad->dev_id, queue_id, 3364 num_to_process, burst_sz, &time_st); 3365 else 3366 iter = offload_latency_test_enc(op_params->mp, bufs, 3367 op_params->ref_enc_op, ad->dev_id, queue_id, 3368 num_to_process, burst_sz, &time_st); 3369 3370 if (iter <= 0) 3371 return TEST_FAILED; 3372 3373 printf("Enqueue driver offload cost latency:\n" 3374 "\tavg: %lg cycles, %lg us\n" 3375 "\tmin: %lg cycles, %lg us\n" 3376 "\tmax: %lg cycles, %lg us\n" 3377 "Enqueue accelerator offload cost latency:\n" 3378 "\tavg: %lg cycles, %lg us\n" 3379 "\tmin: %lg cycles, %lg us\n" 3380 "\tmax: %lg cycles, %lg us\n", 3381 (double)time_st.enq_sw_total_time / (double)iter, 3382 (double)(time_st.enq_sw_total_time * 1000000) / 3383 (double)iter / (double)rte_get_tsc_hz(), 3384 (double)time_st.enq_sw_min_time, 3385 (double)(time_st.enq_sw_min_time * 1000000) / 3386 rte_get_tsc_hz(), (double)time_st.enq_sw_max_time, 3387 (double)(time_st.enq_sw_max_time * 1000000) / 3388 rte_get_tsc_hz(), (double)time_st.enq_acc_total_time / 3389 (double)iter, 3390 (double)(time_st.enq_acc_total_time * 1000000) / 3391 (double)iter / (double)rte_get_tsc_hz(), 3392 (double)time_st.enq_acc_min_time, 3393 (double)(time_st.enq_acc_min_time * 1000000) / 3394 rte_get_tsc_hz(), (double)time_st.enq_acc_max_time, 3395 (double)(time_st.enq_acc_max_time * 1000000) / 3396 rte_get_tsc_hz()); 3397 3398 printf("Dequeue offload cost latency - one op:\n" 3399 "\tavg: %lg cycles, %lg us\n" 3400 "\tmin: %lg cycles, %lg us\n" 3401 "\tmax: %lg cycles, %lg us\n", 3402 (double)time_st.deq_total_time / (double)iter, 3403 (double)(time_st.deq_total_time * 1000000) / 3404 (double)iter / (double)rte_get_tsc_hz(), 3405 (double)time_st.deq_min_time, 3406 (double)(time_st.deq_min_time * 1000000) / 3407 rte_get_tsc_hz(), (double)time_st.deq_max_time, 3408 (double)(time_st.deq_max_time * 1000000) / 3409 rte_get_tsc_hz()); 3410 3411 return TEST_SUCCESS; 3412 #endif 3413 } 3414 3415 #ifdef RTE_BBDEV_OFFLOAD_COST 3416 static int 3417 offload_latency_empty_q_test_dec(uint16_t dev_id, uint16_t queue_id, 3418 const uint16_t num_to_process, uint16_t burst_sz, 3419 uint64_t *deq_total_time, uint64_t *deq_min_time, 3420 uint64_t *deq_max_time) 3421 { 3422 int i, deq_total; 3423 struct rte_bbdev_dec_op *ops[MAX_BURST]; 3424 uint64_t deq_start_time, deq_last_time; 3425 3426 /* Test deq offload latency from an empty queue */ 3427 3428 for (i = 0, deq_total = 0; deq_total < num_to_process; 3429 ++i, deq_total += burst_sz) { 3430 deq_start_time = rte_rdtsc_precise(); 3431 3432 if (unlikely(num_to_process - deq_total < burst_sz)) 3433 burst_sz = num_to_process - deq_total; 3434 rte_bbdev_dequeue_dec_ops(dev_id, queue_id, ops, burst_sz); 3435 3436 deq_last_time = rte_rdtsc_precise() - deq_start_time; 3437 *deq_max_time = RTE_MAX(*deq_max_time, deq_last_time); 3438 *deq_min_time = RTE_MIN(*deq_min_time, deq_last_time); 3439 *deq_total_time += deq_last_time; 3440 } 3441 3442 return i; 3443 } 3444 3445 static int 3446 offload_latency_empty_q_test_enc(uint16_t dev_id, uint16_t queue_id, 3447 const uint16_t num_to_process, uint16_t burst_sz, 3448 uint64_t *deq_total_time, uint64_t *deq_min_time, 3449 uint64_t *deq_max_time) 3450 { 3451 int i, deq_total; 3452 struct rte_bbdev_enc_op *ops[MAX_BURST]; 3453 uint64_t deq_start_time, deq_last_time; 3454 3455 /* Test deq offload latency from an empty queue */ 3456 for (i = 0, deq_total = 0; deq_total < num_to_process; 3457 ++i, deq_total += burst_sz) { 3458 deq_start_time = rte_rdtsc_precise(); 3459 3460 if (unlikely(num_to_process - deq_total < burst_sz)) 3461 burst_sz = num_to_process - deq_total; 3462 rte_bbdev_dequeue_enc_ops(dev_id, queue_id, ops, burst_sz); 3463 3464 deq_last_time = rte_rdtsc_precise() - deq_start_time; 3465 *deq_max_time = RTE_MAX(*deq_max_time, deq_last_time); 3466 *deq_min_time = RTE_MIN(*deq_min_time, deq_last_time); 3467 *deq_total_time += deq_last_time; 3468 } 3469 3470 return i; 3471 } 3472 #endif 3473 3474 static int 3475 offload_latency_empty_q_test(struct active_device *ad, 3476 struct test_op_params *op_params) 3477 { 3478 #ifndef RTE_BBDEV_OFFLOAD_COST 3479 RTE_SET_USED(ad); 3480 RTE_SET_USED(op_params); 3481 printf("Offload latency empty dequeue test is disabled.\n"); 3482 printf("Set RTE_BBDEV_OFFLOAD_COST to 'y' to turn the test on.\n"); 3483 return TEST_SKIPPED; 3484 #else 3485 int iter; 3486 uint64_t deq_total_time, deq_min_time, deq_max_time; 3487 uint16_t burst_sz = op_params->burst_sz; 3488 const uint16_t num_to_process = op_params->num_to_process; 3489 const enum rte_bbdev_op_type op_type = test_vector.op_type; 3490 const uint16_t queue_id = ad->queue_ids[0]; 3491 struct rte_bbdev_info info; 3492 const char *op_type_str; 3493 3494 deq_total_time = deq_max_time = 0; 3495 deq_min_time = UINT64_MAX; 3496 3497 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 3498 "BURST_SIZE should be <= %u", MAX_BURST); 3499 3500 rte_bbdev_info_get(ad->dev_id, &info); 3501 3502 op_type_str = rte_bbdev_op_type_str(op_type); 3503 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type); 3504 3505 printf("+ ------------------------------------------------------- +\n"); 3506 printf("== test: offload latency empty dequeue\ndev: %s, burst size: %u, num ops: %u, op type: %s\n", 3507 info.dev_name, burst_sz, num_to_process, op_type_str); 3508 3509 if (op_type == RTE_BBDEV_OP_TURBO_DEC) 3510 iter = offload_latency_empty_q_test_dec(ad->dev_id, queue_id, 3511 num_to_process, burst_sz, &deq_total_time, 3512 &deq_min_time, &deq_max_time); 3513 else 3514 iter = offload_latency_empty_q_test_enc(ad->dev_id, queue_id, 3515 num_to_process, burst_sz, &deq_total_time, 3516 &deq_min_time, &deq_max_time); 3517 3518 if (iter <= 0) 3519 return TEST_FAILED; 3520 3521 printf("Empty dequeue offload:\n" 3522 "\tavg: %lg cycles, %lg us\n" 3523 "\tmin: %lg cycles, %lg us\n" 3524 "\tmax: %lg cycles, %lg us\n", 3525 (double)deq_total_time / (double)iter, 3526 (double)(deq_total_time * 1000000) / (double)iter / 3527 (double)rte_get_tsc_hz(), (double)deq_min_time, 3528 (double)(deq_min_time * 1000000) / rte_get_tsc_hz(), 3529 (double)deq_max_time, (double)(deq_max_time * 1000000) / 3530 rte_get_tsc_hz()); 3531 3532 return TEST_SUCCESS; 3533 #endif 3534 } 3535 3536 static int 3537 throughput_tc(void) 3538 { 3539 return run_test_case(throughput_test); 3540 } 3541 3542 static int 3543 offload_cost_tc(void) 3544 { 3545 return run_test_case(offload_cost_test); 3546 } 3547 3548 static int 3549 offload_latency_empty_q_tc(void) 3550 { 3551 return run_test_case(offload_latency_empty_q_test); 3552 } 3553 3554 static int 3555 latency_tc(void) 3556 { 3557 return run_test_case(latency_test); 3558 } 3559 3560 static int 3561 interrupt_tc(void) 3562 { 3563 return run_test_case(throughput_test); 3564 } 3565 3566 static struct unit_test_suite bbdev_throughput_testsuite = { 3567 .suite_name = "BBdev Throughput Tests", 3568 .setup = testsuite_setup, 3569 .teardown = testsuite_teardown, 3570 .unit_test_cases = { 3571 TEST_CASE_ST(ut_setup, ut_teardown, throughput_tc), 3572 TEST_CASES_END() /**< NULL terminate unit test array */ 3573 } 3574 }; 3575 3576 static struct unit_test_suite bbdev_validation_testsuite = { 3577 .suite_name = "BBdev Validation Tests", 3578 .setup = testsuite_setup, 3579 .teardown = testsuite_teardown, 3580 .unit_test_cases = { 3581 TEST_CASE_ST(ut_setup, ut_teardown, latency_tc), 3582 TEST_CASES_END() /**< NULL terminate unit test array */ 3583 } 3584 }; 3585 3586 static struct unit_test_suite bbdev_latency_testsuite = { 3587 .suite_name = "BBdev Latency Tests", 3588 .setup = testsuite_setup, 3589 .teardown = testsuite_teardown, 3590 .unit_test_cases = { 3591 TEST_CASE_ST(ut_setup, ut_teardown, latency_tc), 3592 TEST_CASES_END() /**< NULL terminate unit test array */ 3593 } 3594 }; 3595 3596 static struct unit_test_suite bbdev_offload_cost_testsuite = { 3597 .suite_name = "BBdev Offload Cost Tests", 3598 .setup = testsuite_setup, 3599 .teardown = testsuite_teardown, 3600 .unit_test_cases = { 3601 TEST_CASE_ST(ut_setup, ut_teardown, offload_cost_tc), 3602 TEST_CASE_ST(ut_setup, ut_teardown, offload_latency_empty_q_tc), 3603 TEST_CASES_END() /**< NULL terminate unit test array */ 3604 } 3605 }; 3606 3607 static struct unit_test_suite bbdev_interrupt_testsuite = { 3608 .suite_name = "BBdev Interrupt Tests", 3609 .setup = interrupt_testsuite_setup, 3610 .teardown = testsuite_teardown, 3611 .unit_test_cases = { 3612 TEST_CASE_ST(ut_setup, ut_teardown, interrupt_tc), 3613 TEST_CASES_END() /**< NULL terminate unit test array */ 3614 } 3615 }; 3616 3617 REGISTER_TEST_COMMAND(throughput, bbdev_throughput_testsuite); 3618 REGISTER_TEST_COMMAND(validation, bbdev_validation_testsuite); 3619 REGISTER_TEST_COMMAND(latency, bbdev_latency_testsuite); 3620 REGISTER_TEST_COMMAND(offload, bbdev_offload_cost_testsuite); 3621 REGISTER_TEST_COMMAND(interrupt, bbdev_interrupt_testsuite); 3622