1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017 Intel Corporation 3 */ 4 5 #include <stdio.h> 6 #include <inttypes.h> 7 #include <math.h> 8 9 #include <rte_eal.h> 10 #include <rte_common.h> 11 #include <rte_dev.h> 12 #include <rte_launch.h> 13 #include <rte_bbdev.h> 14 #include <rte_cycles.h> 15 #include <rte_lcore.h> 16 #include <rte_malloc.h> 17 #include <rte_random.h> 18 #include <rte_hexdump.h> 19 #include <rte_interrupts.h> 20 21 #include "main.h" 22 #include "test_bbdev_vector.h" 23 24 #define GET_SOCKET(socket_id) (((socket_id) == SOCKET_ID_ANY) ? 0 : (socket_id)) 25 26 #define MAX_QUEUES RTE_MAX_LCORE 27 #define TEST_REPETITIONS 100 28 #define WAIT_OFFLOAD_US 1000 29 30 #ifdef RTE_BASEBAND_FPGA_LTE_FEC 31 #include <fpga_lte_fec.h> 32 #define FPGA_LTE_PF_DRIVER_NAME ("intel_fpga_lte_fec_pf") 33 #define FPGA_LTE_VF_DRIVER_NAME ("intel_fpga_lte_fec_vf") 34 #define VF_UL_4G_QUEUE_VALUE 4 35 #define VF_DL_4G_QUEUE_VALUE 4 36 #define UL_4G_BANDWIDTH 3 37 #define DL_4G_BANDWIDTH 3 38 #define UL_4G_LOAD_BALANCE 128 39 #define DL_4G_LOAD_BALANCE 128 40 #define FLR_4G_TIMEOUT 610 41 #endif 42 43 #ifdef RTE_BASEBAND_FPGA_5GNR_FEC 44 #include <rte_pmd_fpga_5gnr_fec.h> 45 #define FPGA_5GNR_PF_DRIVER_NAME ("intel_fpga_5gnr_fec_pf") 46 #define FPGA_5GNR_VF_DRIVER_NAME ("intel_fpga_5gnr_fec_vf") 47 #define VF_UL_5G_QUEUE_VALUE 4 48 #define VF_DL_5G_QUEUE_VALUE 4 49 #define UL_5G_BANDWIDTH 3 50 #define DL_5G_BANDWIDTH 3 51 #define UL_5G_LOAD_BALANCE 128 52 #define DL_5G_LOAD_BALANCE 128 53 #define FLR_5G_TIMEOUT 610 54 #endif 55 56 #ifdef RTE_BASEBAND_ACC100 57 #include <rte_acc100_cfg.h> 58 #define ACC100PF_DRIVER_NAME ("intel_acc100_pf") 59 #define ACC100VF_DRIVER_NAME ("intel_acc100_vf") 60 #define ACC100_QMGR_NUM_AQS 16 61 #define ACC100_QMGR_NUM_QGS 2 62 #define ACC100_QMGR_AQ_DEPTH 5 63 #define ACC100_QMGR_INVALID_IDX -1 64 #define ACC100_QMGR_RR 1 65 #define ACC100_QOS_GBR 0 66 #endif 67 68 #define OPS_CACHE_SIZE 256U 69 #define OPS_POOL_SIZE_MIN 511U /* 0.5K per queue */ 70 71 #define SYNC_WAIT 0 72 #define SYNC_START 1 73 #define INVALID_OPAQUE -1 74 75 #define INVALID_QUEUE_ID -1 76 /* Increment for next code block in external HARQ memory */ 77 #define HARQ_INCR 32768 78 /* Headroom for filler LLRs insertion in HARQ buffer */ 79 #define FILLER_HEADROOM 1024 80 /* Constants from K0 computation from 3GPP 38.212 Table 5.4.2.1-2 */ 81 #define N_ZC_1 66 /* N = 66 Zc for BG 1 */ 82 #define N_ZC_2 50 /* N = 50 Zc for BG 2 */ 83 #define K0_1_1 17 /* K0 fraction numerator for rv 1 and BG 1 */ 84 #define K0_1_2 13 /* K0 fraction numerator for rv 1 and BG 2 */ 85 #define K0_2_1 33 /* K0 fraction numerator for rv 2 and BG 1 */ 86 #define K0_2_2 25 /* K0 fraction numerator for rv 2 and BG 2 */ 87 #define K0_3_1 56 /* K0 fraction numerator for rv 3 and BG 1 */ 88 #define K0_3_2 43 /* K0 fraction numerator for rv 3 and BG 2 */ 89 90 static struct test_bbdev_vector test_vector; 91 92 /* Switch between PMD and Interrupt for throughput TC */ 93 static bool intr_enabled; 94 95 /* LLR arithmetic representation for numerical conversion */ 96 static int ldpc_llr_decimals; 97 static int ldpc_llr_size; 98 /* Keep track of the LDPC decoder device capability flag */ 99 static uint32_t ldpc_cap_flags; 100 101 /* Represents tested active devices */ 102 static struct active_device { 103 const char *driver_name; 104 uint8_t dev_id; 105 uint16_t supported_ops; 106 uint16_t queue_ids[MAX_QUEUES]; 107 uint16_t nb_queues; 108 struct rte_mempool *ops_mempool; 109 struct rte_mempool *in_mbuf_pool; 110 struct rte_mempool *hard_out_mbuf_pool; 111 struct rte_mempool *soft_out_mbuf_pool; 112 struct rte_mempool *harq_in_mbuf_pool; 113 struct rte_mempool *harq_out_mbuf_pool; 114 } active_devs[RTE_BBDEV_MAX_DEVS]; 115 116 static uint8_t nb_active_devs; 117 118 /* Data buffers used by BBDEV ops */ 119 struct test_buffers { 120 struct rte_bbdev_op_data *inputs; 121 struct rte_bbdev_op_data *hard_outputs; 122 struct rte_bbdev_op_data *soft_outputs; 123 struct rte_bbdev_op_data *harq_inputs; 124 struct rte_bbdev_op_data *harq_outputs; 125 }; 126 127 /* Operation parameters specific for given test case */ 128 struct test_op_params { 129 struct rte_mempool *mp; 130 struct rte_bbdev_dec_op *ref_dec_op; 131 struct rte_bbdev_enc_op *ref_enc_op; 132 uint16_t burst_sz; 133 uint16_t num_to_process; 134 uint16_t num_lcores; 135 int vector_mask; 136 rte_atomic16_t sync; 137 struct test_buffers q_bufs[RTE_MAX_NUMA_NODES][MAX_QUEUES]; 138 }; 139 140 /* Contains per lcore params */ 141 struct thread_params { 142 uint8_t dev_id; 143 uint16_t queue_id; 144 uint32_t lcore_id; 145 uint64_t start_time; 146 double ops_per_sec; 147 double mbps; 148 uint8_t iter_count; 149 double iter_average; 150 double bler; 151 rte_atomic16_t nb_dequeued; 152 rte_atomic16_t processing_status; 153 rte_atomic16_t burst_sz; 154 struct test_op_params *op_params; 155 struct rte_bbdev_dec_op *dec_ops[MAX_BURST]; 156 struct rte_bbdev_enc_op *enc_ops[MAX_BURST]; 157 }; 158 159 #ifdef RTE_BBDEV_OFFLOAD_COST 160 /* Stores time statistics */ 161 struct test_time_stats { 162 /* Stores software enqueue total working time */ 163 uint64_t enq_sw_total_time; 164 /* Stores minimum value of software enqueue working time */ 165 uint64_t enq_sw_min_time; 166 /* Stores maximum value of software enqueue working time */ 167 uint64_t enq_sw_max_time; 168 /* Stores turbo enqueue total working time */ 169 uint64_t enq_acc_total_time; 170 /* Stores minimum value of accelerator enqueue working time */ 171 uint64_t enq_acc_min_time; 172 /* Stores maximum value of accelerator enqueue working time */ 173 uint64_t enq_acc_max_time; 174 /* Stores dequeue total working time */ 175 uint64_t deq_total_time; 176 /* Stores minimum value of dequeue working time */ 177 uint64_t deq_min_time; 178 /* Stores maximum value of dequeue working time */ 179 uint64_t deq_max_time; 180 }; 181 #endif 182 183 typedef int (test_case_function)(struct active_device *ad, 184 struct test_op_params *op_params); 185 186 static inline void 187 mbuf_reset(struct rte_mbuf *m) 188 { 189 m->pkt_len = 0; 190 191 do { 192 m->data_len = 0; 193 m = m->next; 194 } while (m != NULL); 195 } 196 197 /* Read flag value 0/1 from bitmap */ 198 static inline bool 199 check_bit(uint32_t bitmap, uint32_t bitmask) 200 { 201 return bitmap & bitmask; 202 } 203 204 static inline void 205 set_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type) 206 { 207 ad->supported_ops |= (1 << op_type); 208 } 209 210 static inline bool 211 is_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type) 212 { 213 return ad->supported_ops & (1 << op_type); 214 } 215 216 static inline bool 217 flags_match(uint32_t flags_req, uint32_t flags_present) 218 { 219 return (flags_req & flags_present) == flags_req; 220 } 221 222 static void 223 clear_soft_out_cap(uint32_t *op_flags) 224 { 225 *op_flags &= ~RTE_BBDEV_TURBO_SOFT_OUTPUT; 226 *op_flags &= ~RTE_BBDEV_TURBO_POS_LLR_1_BIT_SOFT_OUT; 227 *op_flags &= ~RTE_BBDEV_TURBO_NEG_LLR_1_BIT_SOFT_OUT; 228 } 229 230 static int 231 check_dev_cap(const struct rte_bbdev_info *dev_info) 232 { 233 unsigned int i; 234 unsigned int nb_inputs, nb_soft_outputs, nb_hard_outputs, 235 nb_harq_inputs, nb_harq_outputs; 236 const struct rte_bbdev_op_cap *op_cap = dev_info->drv.capabilities; 237 238 nb_inputs = test_vector.entries[DATA_INPUT].nb_segments; 239 nb_soft_outputs = test_vector.entries[DATA_SOFT_OUTPUT].nb_segments; 240 nb_hard_outputs = test_vector.entries[DATA_HARD_OUTPUT].nb_segments; 241 nb_harq_inputs = test_vector.entries[DATA_HARQ_INPUT].nb_segments; 242 nb_harq_outputs = test_vector.entries[DATA_HARQ_OUTPUT].nb_segments; 243 244 for (i = 0; op_cap->type != RTE_BBDEV_OP_NONE; ++i, ++op_cap) { 245 if (op_cap->type != test_vector.op_type) 246 continue; 247 248 if (op_cap->type == RTE_BBDEV_OP_TURBO_DEC) { 249 const struct rte_bbdev_op_cap_turbo_dec *cap = 250 &op_cap->cap.turbo_dec; 251 /* Ignore lack of soft output capability, just skip 252 * checking if soft output is valid. 253 */ 254 if ((test_vector.turbo_dec.op_flags & 255 RTE_BBDEV_TURBO_SOFT_OUTPUT) && 256 !(cap->capability_flags & 257 RTE_BBDEV_TURBO_SOFT_OUTPUT)) { 258 printf( 259 "INFO: Device \"%s\" does not support soft output - soft output flags will be ignored.\n", 260 dev_info->dev_name); 261 clear_soft_out_cap( 262 &test_vector.turbo_dec.op_flags); 263 } 264 265 if (!flags_match(test_vector.turbo_dec.op_flags, 266 cap->capability_flags)) 267 return TEST_FAILED; 268 if (nb_inputs > cap->num_buffers_src) { 269 printf("Too many inputs defined: %u, max: %u\n", 270 nb_inputs, cap->num_buffers_src); 271 return TEST_FAILED; 272 } 273 if (nb_soft_outputs > cap->num_buffers_soft_out && 274 (test_vector.turbo_dec.op_flags & 275 RTE_BBDEV_TURBO_SOFT_OUTPUT)) { 276 printf( 277 "Too many soft outputs defined: %u, max: %u\n", 278 nb_soft_outputs, 279 cap->num_buffers_soft_out); 280 return TEST_FAILED; 281 } 282 if (nb_hard_outputs > cap->num_buffers_hard_out) { 283 printf( 284 "Too many hard outputs defined: %u, max: %u\n", 285 nb_hard_outputs, 286 cap->num_buffers_hard_out); 287 return TEST_FAILED; 288 } 289 if (intr_enabled && !(cap->capability_flags & 290 RTE_BBDEV_TURBO_DEC_INTERRUPTS)) { 291 printf( 292 "Dequeue interrupts are not supported!\n"); 293 return TEST_FAILED; 294 } 295 296 return TEST_SUCCESS; 297 } else if (op_cap->type == RTE_BBDEV_OP_TURBO_ENC) { 298 const struct rte_bbdev_op_cap_turbo_enc *cap = 299 &op_cap->cap.turbo_enc; 300 301 if (!flags_match(test_vector.turbo_enc.op_flags, 302 cap->capability_flags)) 303 return TEST_FAILED; 304 if (nb_inputs > cap->num_buffers_src) { 305 printf("Too many inputs defined: %u, max: %u\n", 306 nb_inputs, cap->num_buffers_src); 307 return TEST_FAILED; 308 } 309 if (nb_hard_outputs > cap->num_buffers_dst) { 310 printf( 311 "Too many hard outputs defined: %u, max: %u\n", 312 nb_hard_outputs, cap->num_buffers_dst); 313 return TEST_FAILED; 314 } 315 if (intr_enabled && !(cap->capability_flags & 316 RTE_BBDEV_TURBO_ENC_INTERRUPTS)) { 317 printf( 318 "Dequeue interrupts are not supported!\n"); 319 return TEST_FAILED; 320 } 321 322 return TEST_SUCCESS; 323 } else if (op_cap->type == RTE_BBDEV_OP_LDPC_ENC) { 324 const struct rte_bbdev_op_cap_ldpc_enc *cap = 325 &op_cap->cap.ldpc_enc; 326 327 if (!flags_match(test_vector.ldpc_enc.op_flags, 328 cap->capability_flags)){ 329 printf("Flag Mismatch\n"); 330 return TEST_FAILED; 331 } 332 if (nb_inputs > cap->num_buffers_src) { 333 printf("Too many inputs defined: %u, max: %u\n", 334 nb_inputs, cap->num_buffers_src); 335 return TEST_FAILED; 336 } 337 if (nb_hard_outputs > cap->num_buffers_dst) { 338 printf( 339 "Too many hard outputs defined: %u, max: %u\n", 340 nb_hard_outputs, cap->num_buffers_dst); 341 return TEST_FAILED; 342 } 343 if (intr_enabled && !(cap->capability_flags & 344 RTE_BBDEV_LDPC_ENC_INTERRUPTS)) { 345 printf( 346 "Dequeue interrupts are not supported!\n"); 347 return TEST_FAILED; 348 } 349 350 return TEST_SUCCESS; 351 } else if (op_cap->type == RTE_BBDEV_OP_LDPC_DEC) { 352 const struct rte_bbdev_op_cap_ldpc_dec *cap = 353 &op_cap->cap.ldpc_dec; 354 355 if (!flags_match(test_vector.ldpc_dec.op_flags, 356 cap->capability_flags)){ 357 printf("Flag Mismatch\n"); 358 return TEST_FAILED; 359 } 360 if (nb_inputs > cap->num_buffers_src) { 361 printf("Too many inputs defined: %u, max: %u\n", 362 nb_inputs, cap->num_buffers_src); 363 return TEST_FAILED; 364 } 365 if (nb_hard_outputs > cap->num_buffers_hard_out) { 366 printf( 367 "Too many hard outputs defined: %u, max: %u\n", 368 nb_hard_outputs, 369 cap->num_buffers_hard_out); 370 return TEST_FAILED; 371 } 372 if (nb_harq_inputs > cap->num_buffers_hard_out) { 373 printf( 374 "Too many HARQ inputs defined: %u, max: %u\n", 375 nb_hard_outputs, 376 cap->num_buffers_hard_out); 377 return TEST_FAILED; 378 } 379 if (nb_harq_outputs > cap->num_buffers_hard_out) { 380 printf( 381 "Too many HARQ outputs defined: %u, max: %u\n", 382 nb_hard_outputs, 383 cap->num_buffers_hard_out); 384 return TEST_FAILED; 385 } 386 if (intr_enabled && !(cap->capability_flags & 387 RTE_BBDEV_LDPC_DEC_INTERRUPTS)) { 388 printf( 389 "Dequeue interrupts are not supported!\n"); 390 return TEST_FAILED; 391 } 392 if (intr_enabled && (test_vector.ldpc_dec.op_flags & 393 (RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE | 394 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE | 395 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK 396 ))) { 397 printf("Skip loop-back with interrupt\n"); 398 return TEST_FAILED; 399 } 400 return TEST_SUCCESS; 401 } 402 } 403 404 if ((i == 0) && (test_vector.op_type == RTE_BBDEV_OP_NONE)) 405 return TEST_SUCCESS; /* Special case for NULL device */ 406 407 return TEST_FAILED; 408 } 409 410 /* calculates optimal mempool size not smaller than the val */ 411 static unsigned int 412 optimal_mempool_size(unsigned int val) 413 { 414 return rte_align32pow2(val + 1) - 1; 415 } 416 417 /* allocates mbuf mempool for inputs and outputs */ 418 static struct rte_mempool * 419 create_mbuf_pool(struct op_data_entries *entries, uint8_t dev_id, 420 int socket_id, unsigned int mbuf_pool_size, 421 const char *op_type_str) 422 { 423 unsigned int i; 424 uint32_t max_seg_sz = 0; 425 char pool_name[RTE_MEMPOOL_NAMESIZE]; 426 427 /* find max input segment size */ 428 for (i = 0; i < entries->nb_segments; ++i) 429 if (entries->segments[i].length > max_seg_sz) 430 max_seg_sz = entries->segments[i].length; 431 432 snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str, 433 dev_id); 434 return rte_pktmbuf_pool_create(pool_name, mbuf_pool_size, 0, 0, 435 RTE_MAX(max_seg_sz + RTE_PKTMBUF_HEADROOM 436 + FILLER_HEADROOM, 437 (unsigned int)RTE_MBUF_DEFAULT_BUF_SIZE), socket_id); 438 } 439 440 static int 441 create_mempools(struct active_device *ad, int socket_id, 442 enum rte_bbdev_op_type org_op_type, uint16_t num_ops) 443 { 444 struct rte_mempool *mp; 445 unsigned int ops_pool_size, mbuf_pool_size = 0; 446 char pool_name[RTE_MEMPOOL_NAMESIZE]; 447 const char *op_type_str; 448 enum rte_bbdev_op_type op_type = org_op_type; 449 450 struct op_data_entries *in = &test_vector.entries[DATA_INPUT]; 451 struct op_data_entries *hard_out = 452 &test_vector.entries[DATA_HARD_OUTPUT]; 453 struct op_data_entries *soft_out = 454 &test_vector.entries[DATA_SOFT_OUTPUT]; 455 struct op_data_entries *harq_in = 456 &test_vector.entries[DATA_HARQ_INPUT]; 457 struct op_data_entries *harq_out = 458 &test_vector.entries[DATA_HARQ_OUTPUT]; 459 460 /* allocate ops mempool */ 461 ops_pool_size = optimal_mempool_size(RTE_MAX( 462 /* Ops used plus 1 reference op */ 463 RTE_MAX((unsigned int)(ad->nb_queues * num_ops + 1), 464 /* Minimal cache size plus 1 reference op */ 465 (unsigned int)(1.5 * rte_lcore_count() * 466 OPS_CACHE_SIZE + 1)), 467 OPS_POOL_SIZE_MIN)); 468 469 if (org_op_type == RTE_BBDEV_OP_NONE) 470 op_type = RTE_BBDEV_OP_TURBO_ENC; 471 472 op_type_str = rte_bbdev_op_type_str(op_type); 473 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type); 474 475 snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str, 476 ad->dev_id); 477 mp = rte_bbdev_op_pool_create(pool_name, op_type, 478 ops_pool_size, OPS_CACHE_SIZE, socket_id); 479 TEST_ASSERT_NOT_NULL(mp, 480 "ERROR Failed to create %u items ops pool for dev %u on socket %u.", 481 ops_pool_size, 482 ad->dev_id, 483 socket_id); 484 ad->ops_mempool = mp; 485 486 /* Do not create inputs and outputs mbufs for BaseBand Null Device */ 487 if (org_op_type == RTE_BBDEV_OP_NONE) 488 return TEST_SUCCESS; 489 490 /* Inputs */ 491 if (in->nb_segments > 0) { 492 mbuf_pool_size = optimal_mempool_size(ops_pool_size * 493 in->nb_segments); 494 mp = create_mbuf_pool(in, ad->dev_id, socket_id, 495 mbuf_pool_size, "in"); 496 TEST_ASSERT_NOT_NULL(mp, 497 "ERROR Failed to create %u items input pktmbuf pool for dev %u on socket %u.", 498 mbuf_pool_size, 499 ad->dev_id, 500 socket_id); 501 ad->in_mbuf_pool = mp; 502 } 503 504 /* Hard outputs */ 505 if (hard_out->nb_segments > 0) { 506 mbuf_pool_size = optimal_mempool_size(ops_pool_size * 507 hard_out->nb_segments); 508 mp = create_mbuf_pool(hard_out, ad->dev_id, socket_id, 509 mbuf_pool_size, 510 "hard_out"); 511 TEST_ASSERT_NOT_NULL(mp, 512 "ERROR Failed to create %u items hard output pktmbuf pool for dev %u on socket %u.", 513 mbuf_pool_size, 514 ad->dev_id, 515 socket_id); 516 ad->hard_out_mbuf_pool = mp; 517 } 518 519 /* Soft outputs */ 520 if (soft_out->nb_segments > 0) { 521 mbuf_pool_size = optimal_mempool_size(ops_pool_size * 522 soft_out->nb_segments); 523 mp = create_mbuf_pool(soft_out, ad->dev_id, socket_id, 524 mbuf_pool_size, 525 "soft_out"); 526 TEST_ASSERT_NOT_NULL(mp, 527 "ERROR Failed to create %uB soft output pktmbuf pool for dev %u on socket %u.", 528 mbuf_pool_size, 529 ad->dev_id, 530 socket_id); 531 ad->soft_out_mbuf_pool = mp; 532 } 533 534 /* HARQ inputs */ 535 if (harq_in->nb_segments > 0) { 536 mbuf_pool_size = optimal_mempool_size(ops_pool_size * 537 harq_in->nb_segments); 538 mp = create_mbuf_pool(harq_in, ad->dev_id, socket_id, 539 mbuf_pool_size, 540 "harq_in"); 541 TEST_ASSERT_NOT_NULL(mp, 542 "ERROR Failed to create %uB harq input pktmbuf pool for dev %u on socket %u.", 543 mbuf_pool_size, 544 ad->dev_id, 545 socket_id); 546 ad->harq_in_mbuf_pool = mp; 547 } 548 549 /* HARQ outputs */ 550 if (harq_out->nb_segments > 0) { 551 mbuf_pool_size = optimal_mempool_size(ops_pool_size * 552 harq_out->nb_segments); 553 mp = create_mbuf_pool(harq_out, ad->dev_id, socket_id, 554 mbuf_pool_size, 555 "harq_out"); 556 TEST_ASSERT_NOT_NULL(mp, 557 "ERROR Failed to create %uB harq output pktmbuf pool for dev %u on socket %u.", 558 mbuf_pool_size, 559 ad->dev_id, 560 socket_id); 561 ad->harq_out_mbuf_pool = mp; 562 } 563 564 return TEST_SUCCESS; 565 } 566 567 static int 568 add_bbdev_dev(uint8_t dev_id, struct rte_bbdev_info *info, 569 struct test_bbdev_vector *vector) 570 { 571 int ret; 572 unsigned int queue_id; 573 struct rte_bbdev_queue_conf qconf; 574 struct active_device *ad = &active_devs[nb_active_devs]; 575 unsigned int nb_queues; 576 enum rte_bbdev_op_type op_type = vector->op_type; 577 578 /* Configure fpga lte fec with PF & VF values 579 * if '-i' flag is set and using fpga device 580 */ 581 #ifdef RTE_BASEBAND_FPGA_LTE_FEC 582 if ((get_init_device() == true) && 583 (!strcmp(info->drv.driver_name, FPGA_LTE_PF_DRIVER_NAME))) { 584 struct rte_fpga_lte_fec_conf conf; 585 unsigned int i; 586 587 printf("Configure FPGA LTE FEC Driver %s with default values\n", 588 info->drv.driver_name); 589 590 /* clear default configuration before initialization */ 591 memset(&conf, 0, sizeof(struct rte_fpga_lte_fec_conf)); 592 593 /* Set PF mode : 594 * true if PF is used for data plane 595 * false for VFs 596 */ 597 conf.pf_mode_en = true; 598 599 for (i = 0; i < FPGA_LTE_FEC_NUM_VFS; ++i) { 600 /* Number of UL queues per VF (fpga supports 8 VFs) */ 601 conf.vf_ul_queues_number[i] = VF_UL_4G_QUEUE_VALUE; 602 /* Number of DL queues per VF (fpga supports 8 VFs) */ 603 conf.vf_dl_queues_number[i] = VF_DL_4G_QUEUE_VALUE; 604 } 605 606 /* UL bandwidth. Needed for schedule algorithm */ 607 conf.ul_bandwidth = UL_4G_BANDWIDTH; 608 /* DL bandwidth */ 609 conf.dl_bandwidth = DL_4G_BANDWIDTH; 610 611 /* UL & DL load Balance Factor to 64 */ 612 conf.ul_load_balance = UL_4G_LOAD_BALANCE; 613 conf.dl_load_balance = DL_4G_LOAD_BALANCE; 614 615 /**< FLR timeout value */ 616 conf.flr_time_out = FLR_4G_TIMEOUT; 617 618 /* setup FPGA PF with configuration information */ 619 ret = rte_fpga_lte_fec_configure(info->dev_name, &conf); 620 TEST_ASSERT_SUCCESS(ret, 621 "Failed to configure 4G FPGA PF for bbdev %s", 622 info->dev_name); 623 } 624 #endif 625 #ifdef RTE_BASEBAND_FPGA_5GNR_FEC 626 if ((get_init_device() == true) && 627 (!strcmp(info->drv.driver_name, FPGA_5GNR_PF_DRIVER_NAME))) { 628 struct rte_fpga_5gnr_fec_conf conf; 629 unsigned int i; 630 631 printf("Configure FPGA 5GNR FEC Driver %s with default values\n", 632 info->drv.driver_name); 633 634 /* clear default configuration before initialization */ 635 memset(&conf, 0, sizeof(struct rte_fpga_5gnr_fec_conf)); 636 637 /* Set PF mode : 638 * true if PF is used for data plane 639 * false for VFs 640 */ 641 conf.pf_mode_en = true; 642 643 for (i = 0; i < FPGA_5GNR_FEC_NUM_VFS; ++i) { 644 /* Number of UL queues per VF (fpga supports 8 VFs) */ 645 conf.vf_ul_queues_number[i] = VF_UL_5G_QUEUE_VALUE; 646 /* Number of DL queues per VF (fpga supports 8 VFs) */ 647 conf.vf_dl_queues_number[i] = VF_DL_5G_QUEUE_VALUE; 648 } 649 650 /* UL bandwidth. Needed for schedule algorithm */ 651 conf.ul_bandwidth = UL_5G_BANDWIDTH; 652 /* DL bandwidth */ 653 conf.dl_bandwidth = DL_5G_BANDWIDTH; 654 655 /* UL & DL load Balance Factor to 64 */ 656 conf.ul_load_balance = UL_5G_LOAD_BALANCE; 657 conf.dl_load_balance = DL_5G_LOAD_BALANCE; 658 659 /**< FLR timeout value */ 660 conf.flr_time_out = FLR_5G_TIMEOUT; 661 662 /* setup FPGA PF with configuration information */ 663 ret = rte_fpga_5gnr_fec_configure(info->dev_name, &conf); 664 TEST_ASSERT_SUCCESS(ret, 665 "Failed to configure 5G FPGA PF for bbdev %s", 666 info->dev_name); 667 } 668 #endif 669 #ifdef RTE_BASEBAND_ACC100 670 if ((get_init_device() == true) && 671 (!strcmp(info->drv.driver_name, ACC100PF_DRIVER_NAME))) { 672 struct rte_acc100_conf conf; 673 unsigned int i; 674 675 printf("Configure ACC100 FEC Driver %s with default values\n", 676 info->drv.driver_name); 677 678 /* clear default configuration before initialization */ 679 memset(&conf, 0, sizeof(struct rte_acc100_conf)); 680 681 /* Always set in PF mode for built-in configuration */ 682 conf.pf_mode_en = true; 683 for (i = 0; i < RTE_ACC100_NUM_VFS; ++i) { 684 conf.arb_dl_4g[i].gbr_threshold1 = ACC100_QOS_GBR; 685 conf.arb_dl_4g[i].gbr_threshold1 = ACC100_QOS_GBR; 686 conf.arb_dl_4g[i].round_robin_weight = ACC100_QMGR_RR; 687 conf.arb_ul_4g[i].gbr_threshold1 = ACC100_QOS_GBR; 688 conf.arb_ul_4g[i].gbr_threshold1 = ACC100_QOS_GBR; 689 conf.arb_ul_4g[i].round_robin_weight = ACC100_QMGR_RR; 690 conf.arb_dl_5g[i].gbr_threshold1 = ACC100_QOS_GBR; 691 conf.arb_dl_5g[i].gbr_threshold1 = ACC100_QOS_GBR; 692 conf.arb_dl_5g[i].round_robin_weight = ACC100_QMGR_RR; 693 conf.arb_ul_5g[i].gbr_threshold1 = ACC100_QOS_GBR; 694 conf.arb_ul_5g[i].gbr_threshold1 = ACC100_QOS_GBR; 695 conf.arb_ul_5g[i].round_robin_weight = ACC100_QMGR_RR; 696 } 697 698 conf.input_pos_llr_1_bit = true; 699 conf.output_pos_llr_1_bit = true; 700 conf.num_vf_bundles = 1; /**< Number of VF bundles to setup */ 701 702 conf.q_ul_4g.num_qgroups = ACC100_QMGR_NUM_QGS; 703 conf.q_ul_4g.first_qgroup_index = ACC100_QMGR_INVALID_IDX; 704 conf.q_ul_4g.num_aqs_per_groups = ACC100_QMGR_NUM_AQS; 705 conf.q_ul_4g.aq_depth_log2 = ACC100_QMGR_AQ_DEPTH; 706 conf.q_dl_4g.num_qgroups = ACC100_QMGR_NUM_QGS; 707 conf.q_dl_4g.first_qgroup_index = ACC100_QMGR_INVALID_IDX; 708 conf.q_dl_4g.num_aqs_per_groups = ACC100_QMGR_NUM_AQS; 709 conf.q_dl_4g.aq_depth_log2 = ACC100_QMGR_AQ_DEPTH; 710 conf.q_ul_5g.num_qgroups = ACC100_QMGR_NUM_QGS; 711 conf.q_ul_5g.first_qgroup_index = ACC100_QMGR_INVALID_IDX; 712 conf.q_ul_5g.num_aqs_per_groups = ACC100_QMGR_NUM_AQS; 713 conf.q_ul_5g.aq_depth_log2 = ACC100_QMGR_AQ_DEPTH; 714 conf.q_dl_5g.num_qgroups = ACC100_QMGR_NUM_QGS; 715 conf.q_dl_5g.first_qgroup_index = ACC100_QMGR_INVALID_IDX; 716 conf.q_dl_5g.num_aqs_per_groups = ACC100_QMGR_NUM_AQS; 717 conf.q_dl_5g.aq_depth_log2 = ACC100_QMGR_AQ_DEPTH; 718 719 /* setup PF with configuration information */ 720 ret = rte_acc100_configure(info->dev_name, &conf); 721 TEST_ASSERT_SUCCESS(ret, 722 "Failed to configure ACC100 PF for bbdev %s", 723 info->dev_name); 724 } 725 #endif 726 /* Let's refresh this now this is configured */ 727 rte_bbdev_info_get(dev_id, info); 728 nb_queues = RTE_MIN(rte_lcore_count(), info->drv.max_num_queues); 729 nb_queues = RTE_MIN(nb_queues, (unsigned int) MAX_QUEUES); 730 731 /* setup device */ 732 ret = rte_bbdev_setup_queues(dev_id, nb_queues, info->socket_id); 733 if (ret < 0) { 734 printf("rte_bbdev_setup_queues(%u, %u, %d) ret %i\n", 735 dev_id, nb_queues, info->socket_id, ret); 736 return TEST_FAILED; 737 } 738 739 /* configure interrupts if needed */ 740 if (intr_enabled) { 741 ret = rte_bbdev_intr_enable(dev_id); 742 if (ret < 0) { 743 printf("rte_bbdev_intr_enable(%u) ret %i\n", dev_id, 744 ret); 745 return TEST_FAILED; 746 } 747 } 748 749 /* setup device queues */ 750 qconf.socket = info->socket_id; 751 qconf.queue_size = info->drv.default_queue_conf.queue_size; 752 qconf.priority = 0; 753 qconf.deferred_start = 0; 754 qconf.op_type = op_type; 755 756 for (queue_id = 0; queue_id < nb_queues; ++queue_id) { 757 ret = rte_bbdev_queue_configure(dev_id, queue_id, &qconf); 758 if (ret != 0) { 759 printf( 760 "Allocated all queues (id=%u) at prio%u on dev%u\n", 761 queue_id, qconf.priority, dev_id); 762 qconf.priority++; 763 ret = rte_bbdev_queue_configure(ad->dev_id, queue_id, 764 &qconf); 765 } 766 if (ret != 0) { 767 printf("All queues on dev %u allocated: %u\n", 768 dev_id, queue_id); 769 break; 770 } 771 ad->queue_ids[queue_id] = queue_id; 772 } 773 TEST_ASSERT(queue_id != 0, 774 "ERROR Failed to configure any queues on dev %u", 775 dev_id); 776 ad->nb_queues = queue_id; 777 778 set_avail_op(ad, op_type); 779 780 return TEST_SUCCESS; 781 } 782 783 static int 784 add_active_device(uint8_t dev_id, struct rte_bbdev_info *info, 785 struct test_bbdev_vector *vector) 786 { 787 int ret; 788 789 active_devs[nb_active_devs].driver_name = info->drv.driver_name; 790 active_devs[nb_active_devs].dev_id = dev_id; 791 792 ret = add_bbdev_dev(dev_id, info, vector); 793 if (ret == TEST_SUCCESS) 794 ++nb_active_devs; 795 return ret; 796 } 797 798 static uint8_t 799 populate_active_devices(void) 800 { 801 int ret; 802 uint8_t dev_id; 803 uint8_t nb_devs_added = 0; 804 struct rte_bbdev_info info; 805 806 RTE_BBDEV_FOREACH(dev_id) { 807 rte_bbdev_info_get(dev_id, &info); 808 809 if (check_dev_cap(&info)) { 810 printf( 811 "Device %d (%s) does not support specified capabilities\n", 812 dev_id, info.dev_name); 813 continue; 814 } 815 816 ret = add_active_device(dev_id, &info, &test_vector); 817 if (ret != 0) { 818 printf("Adding active bbdev %s skipped\n", 819 info.dev_name); 820 continue; 821 } 822 nb_devs_added++; 823 } 824 825 return nb_devs_added; 826 } 827 828 static int 829 read_test_vector(void) 830 { 831 int ret; 832 833 memset(&test_vector, 0, sizeof(test_vector)); 834 printf("Test vector file = %s\n", get_vector_filename()); 835 ret = test_bbdev_vector_read(get_vector_filename(), &test_vector); 836 TEST_ASSERT_SUCCESS(ret, "Failed to parse file %s\n", 837 get_vector_filename()); 838 839 return TEST_SUCCESS; 840 } 841 842 static int 843 testsuite_setup(void) 844 { 845 TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n"); 846 847 if (populate_active_devices() == 0) { 848 printf("No suitable devices found!\n"); 849 return TEST_SKIPPED; 850 } 851 852 return TEST_SUCCESS; 853 } 854 855 static int 856 interrupt_testsuite_setup(void) 857 { 858 TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n"); 859 860 /* Enable interrupts */ 861 intr_enabled = true; 862 863 /* Special case for NULL device (RTE_BBDEV_OP_NONE) */ 864 if (populate_active_devices() == 0 || 865 test_vector.op_type == RTE_BBDEV_OP_NONE) { 866 intr_enabled = false; 867 printf("No suitable devices found!\n"); 868 return TEST_SKIPPED; 869 } 870 871 return TEST_SUCCESS; 872 } 873 874 static void 875 testsuite_teardown(void) 876 { 877 uint8_t dev_id; 878 879 /* Unconfigure devices */ 880 RTE_BBDEV_FOREACH(dev_id) 881 rte_bbdev_close(dev_id); 882 883 /* Clear active devices structs. */ 884 memset(active_devs, 0, sizeof(active_devs)); 885 nb_active_devs = 0; 886 887 /* Disable interrupts */ 888 intr_enabled = false; 889 } 890 891 static int 892 ut_setup(void) 893 { 894 uint8_t i, dev_id; 895 896 for (i = 0; i < nb_active_devs; i++) { 897 dev_id = active_devs[i].dev_id; 898 /* reset bbdev stats */ 899 TEST_ASSERT_SUCCESS(rte_bbdev_stats_reset(dev_id), 900 "Failed to reset stats of bbdev %u", dev_id); 901 /* start the device */ 902 TEST_ASSERT_SUCCESS(rte_bbdev_start(dev_id), 903 "Failed to start bbdev %u", dev_id); 904 } 905 906 return TEST_SUCCESS; 907 } 908 909 static void 910 ut_teardown(void) 911 { 912 uint8_t i, dev_id; 913 struct rte_bbdev_stats stats; 914 915 for (i = 0; i < nb_active_devs; i++) { 916 dev_id = active_devs[i].dev_id; 917 /* read stats and print */ 918 rte_bbdev_stats_get(dev_id, &stats); 919 /* Stop the device */ 920 rte_bbdev_stop(dev_id); 921 } 922 } 923 924 static int 925 init_op_data_objs(struct rte_bbdev_op_data *bufs, 926 struct op_data_entries *ref_entries, 927 struct rte_mempool *mbuf_pool, const uint16_t n, 928 enum op_data_type op_type, uint16_t min_alignment) 929 { 930 int ret; 931 unsigned int i, j; 932 bool large_input = false; 933 934 for (i = 0; i < n; ++i) { 935 char *data; 936 struct op_data_buf *seg = &ref_entries->segments[0]; 937 struct rte_mbuf *m_head = rte_pktmbuf_alloc(mbuf_pool); 938 TEST_ASSERT_NOT_NULL(m_head, 939 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)", 940 op_type, n * ref_entries->nb_segments, 941 mbuf_pool->size); 942 943 if (seg->length > RTE_BBDEV_LDPC_E_MAX_MBUF) { 944 /* 945 * Special case when DPDK mbuf cannot handle 946 * the required input size 947 */ 948 printf("Warning: Larger input size than DPDK mbuf %d\n", 949 seg->length); 950 large_input = true; 951 } 952 bufs[i].data = m_head; 953 bufs[i].offset = 0; 954 bufs[i].length = 0; 955 956 if ((op_type == DATA_INPUT) || (op_type == DATA_HARQ_INPUT)) { 957 if ((op_type == DATA_INPUT) && large_input) { 958 /* Allocate a fake overused mbuf */ 959 data = rte_malloc(NULL, seg->length, 0); 960 memcpy(data, seg->addr, seg->length); 961 m_head->buf_addr = data; 962 m_head->buf_iova = rte_malloc_virt2iova(data); 963 m_head->data_off = 0; 964 m_head->data_len = seg->length; 965 } else { 966 data = rte_pktmbuf_append(m_head, seg->length); 967 TEST_ASSERT_NOT_NULL(data, 968 "Couldn't append %u bytes to mbuf from %d data type mbuf pool", 969 seg->length, op_type); 970 971 TEST_ASSERT(data == RTE_PTR_ALIGN( 972 data, min_alignment), 973 "Data addr in mbuf (%p) is not aligned to device min alignment (%u)", 974 data, min_alignment); 975 rte_memcpy(data, seg->addr, seg->length); 976 } 977 978 bufs[i].length += seg->length; 979 980 for (j = 1; j < ref_entries->nb_segments; ++j) { 981 struct rte_mbuf *m_tail = 982 rte_pktmbuf_alloc(mbuf_pool); 983 TEST_ASSERT_NOT_NULL(m_tail, 984 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)", 985 op_type, 986 n * ref_entries->nb_segments, 987 mbuf_pool->size); 988 seg += 1; 989 990 data = rte_pktmbuf_append(m_tail, seg->length); 991 TEST_ASSERT_NOT_NULL(data, 992 "Couldn't append %u bytes to mbuf from %d data type mbuf pool", 993 seg->length, op_type); 994 995 TEST_ASSERT(data == RTE_PTR_ALIGN(data, 996 min_alignment), 997 "Data addr in mbuf (%p) is not aligned to device min alignment (%u)", 998 data, min_alignment); 999 rte_memcpy(data, seg->addr, seg->length); 1000 bufs[i].length += seg->length; 1001 1002 ret = rte_pktmbuf_chain(m_head, m_tail); 1003 TEST_ASSERT_SUCCESS(ret, 1004 "Couldn't chain mbufs from %d data type mbuf pool", 1005 op_type); 1006 } 1007 } else { 1008 1009 /* allocate chained-mbuf for output buffer */ 1010 for (j = 1; j < ref_entries->nb_segments; ++j) { 1011 struct rte_mbuf *m_tail = 1012 rte_pktmbuf_alloc(mbuf_pool); 1013 TEST_ASSERT_NOT_NULL(m_tail, 1014 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)", 1015 op_type, 1016 n * ref_entries->nb_segments, 1017 mbuf_pool->size); 1018 1019 ret = rte_pktmbuf_chain(m_head, m_tail); 1020 TEST_ASSERT_SUCCESS(ret, 1021 "Couldn't chain mbufs from %d data type mbuf pool", 1022 op_type); 1023 } 1024 } 1025 } 1026 1027 return 0; 1028 } 1029 1030 static int 1031 allocate_buffers_on_socket(struct rte_bbdev_op_data **buffers, const int len, 1032 const int socket) 1033 { 1034 int i; 1035 1036 *buffers = rte_zmalloc_socket(NULL, len, 0, socket); 1037 if (*buffers == NULL) { 1038 printf("WARNING: Failed to allocate op_data on socket %d\n", 1039 socket); 1040 /* try to allocate memory on other detected sockets */ 1041 for (i = 0; i < socket; i++) { 1042 *buffers = rte_zmalloc_socket(NULL, len, 0, i); 1043 if (*buffers != NULL) 1044 break; 1045 } 1046 } 1047 1048 return (*buffers == NULL) ? TEST_FAILED : TEST_SUCCESS; 1049 } 1050 1051 static void 1052 limit_input_llr_val_range(struct rte_bbdev_op_data *input_ops, 1053 const uint16_t n, const int8_t max_llr_modulus) 1054 { 1055 uint16_t i, byte_idx; 1056 1057 for (i = 0; i < n; ++i) { 1058 struct rte_mbuf *m = input_ops[i].data; 1059 while (m != NULL) { 1060 int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *, 1061 input_ops[i].offset); 1062 for (byte_idx = 0; byte_idx < rte_pktmbuf_data_len(m); 1063 ++byte_idx) 1064 llr[byte_idx] = round((double)max_llr_modulus * 1065 llr[byte_idx] / INT8_MAX); 1066 1067 m = m->next; 1068 } 1069 } 1070 } 1071 1072 /* 1073 * We may have to insert filler bits 1074 * when they are required by the HARQ assumption 1075 */ 1076 static void 1077 ldpc_add_filler(struct rte_bbdev_op_data *input_ops, 1078 const uint16_t n, struct test_op_params *op_params) 1079 { 1080 struct rte_bbdev_op_ldpc_dec dec = op_params->ref_dec_op->ldpc_dec; 1081 1082 if (input_ops == NULL) 1083 return; 1084 /* No need to add filler if not required by device */ 1085 if (!(ldpc_cap_flags & 1086 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_FILLERS)) 1087 return; 1088 /* No need to add filler for loopback operation */ 1089 if (dec.op_flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK) 1090 return; 1091 1092 uint16_t i, j, parity_offset; 1093 for (i = 0; i < n; ++i) { 1094 struct rte_mbuf *m = input_ops[i].data; 1095 int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *, 1096 input_ops[i].offset); 1097 parity_offset = (dec.basegraph == 1 ? 20 : 8) 1098 * dec.z_c - dec.n_filler; 1099 uint16_t new_hin_size = input_ops[i].length + dec.n_filler; 1100 m->data_len = new_hin_size; 1101 input_ops[i].length = new_hin_size; 1102 for (j = new_hin_size - 1; j >= parity_offset + dec.n_filler; 1103 j--) 1104 llr[j] = llr[j - dec.n_filler]; 1105 uint16_t llr_max_pre_scaling = (1 << (ldpc_llr_size - 1)) - 1; 1106 for (j = 0; j < dec.n_filler; j++) 1107 llr[parity_offset + j] = llr_max_pre_scaling; 1108 } 1109 } 1110 1111 static void 1112 ldpc_input_llr_scaling(struct rte_bbdev_op_data *input_ops, 1113 const uint16_t n, const int8_t llr_size, 1114 const int8_t llr_decimals) 1115 { 1116 if (input_ops == NULL) 1117 return; 1118 1119 uint16_t i, byte_idx; 1120 1121 int16_t llr_max, llr_min, llr_tmp; 1122 llr_max = (1 << (llr_size - 1)) - 1; 1123 llr_min = -llr_max; 1124 for (i = 0; i < n; ++i) { 1125 struct rte_mbuf *m = input_ops[i].data; 1126 while (m != NULL) { 1127 int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *, 1128 input_ops[i].offset); 1129 for (byte_idx = 0; byte_idx < rte_pktmbuf_data_len(m); 1130 ++byte_idx) { 1131 1132 llr_tmp = llr[byte_idx]; 1133 if (llr_decimals == 4) 1134 llr_tmp *= 8; 1135 else if (llr_decimals == 2) 1136 llr_tmp *= 2; 1137 else if (llr_decimals == 0) 1138 llr_tmp /= 2; 1139 llr_tmp = RTE_MIN(llr_max, 1140 RTE_MAX(llr_min, llr_tmp)); 1141 llr[byte_idx] = (int8_t) llr_tmp; 1142 } 1143 1144 m = m->next; 1145 } 1146 } 1147 } 1148 1149 1150 1151 static int 1152 fill_queue_buffers(struct test_op_params *op_params, 1153 struct rte_mempool *in_mp, struct rte_mempool *hard_out_mp, 1154 struct rte_mempool *soft_out_mp, 1155 struct rte_mempool *harq_in_mp, struct rte_mempool *harq_out_mp, 1156 uint16_t queue_id, 1157 const struct rte_bbdev_op_cap *capabilities, 1158 uint16_t min_alignment, const int socket_id) 1159 { 1160 int ret; 1161 enum op_data_type type; 1162 const uint16_t n = op_params->num_to_process; 1163 1164 struct rte_mempool *mbuf_pools[DATA_NUM_TYPES] = { 1165 in_mp, 1166 soft_out_mp, 1167 hard_out_mp, 1168 harq_in_mp, 1169 harq_out_mp, 1170 }; 1171 1172 struct rte_bbdev_op_data **queue_ops[DATA_NUM_TYPES] = { 1173 &op_params->q_bufs[socket_id][queue_id].inputs, 1174 &op_params->q_bufs[socket_id][queue_id].soft_outputs, 1175 &op_params->q_bufs[socket_id][queue_id].hard_outputs, 1176 &op_params->q_bufs[socket_id][queue_id].harq_inputs, 1177 &op_params->q_bufs[socket_id][queue_id].harq_outputs, 1178 }; 1179 1180 for (type = DATA_INPUT; type < DATA_NUM_TYPES; ++type) { 1181 struct op_data_entries *ref_entries = 1182 &test_vector.entries[type]; 1183 if (ref_entries->nb_segments == 0) 1184 continue; 1185 1186 ret = allocate_buffers_on_socket(queue_ops[type], 1187 n * sizeof(struct rte_bbdev_op_data), 1188 socket_id); 1189 TEST_ASSERT_SUCCESS(ret, 1190 "Couldn't allocate memory for rte_bbdev_op_data structs"); 1191 1192 ret = init_op_data_objs(*queue_ops[type], ref_entries, 1193 mbuf_pools[type], n, type, min_alignment); 1194 TEST_ASSERT_SUCCESS(ret, 1195 "Couldn't init rte_bbdev_op_data structs"); 1196 } 1197 1198 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) 1199 limit_input_llr_val_range(*queue_ops[DATA_INPUT], n, 1200 capabilities->cap.turbo_dec.max_llr_modulus); 1201 1202 if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) { 1203 bool loopback = op_params->ref_dec_op->ldpc_dec.op_flags & 1204 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK; 1205 bool llr_comp = op_params->ref_dec_op->ldpc_dec.op_flags & 1206 RTE_BBDEV_LDPC_LLR_COMPRESSION; 1207 bool harq_comp = op_params->ref_dec_op->ldpc_dec.op_flags & 1208 RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION; 1209 ldpc_llr_decimals = capabilities->cap.ldpc_dec.llr_decimals; 1210 ldpc_llr_size = capabilities->cap.ldpc_dec.llr_size; 1211 ldpc_cap_flags = capabilities->cap.ldpc_dec.capability_flags; 1212 if (!loopback && !llr_comp) 1213 ldpc_input_llr_scaling(*queue_ops[DATA_INPUT], n, 1214 ldpc_llr_size, ldpc_llr_decimals); 1215 if (!loopback && !harq_comp) 1216 ldpc_input_llr_scaling(*queue_ops[DATA_HARQ_INPUT], n, 1217 ldpc_llr_size, ldpc_llr_decimals); 1218 if (!loopback) 1219 ldpc_add_filler(*queue_ops[DATA_HARQ_INPUT], n, 1220 op_params); 1221 } 1222 1223 return 0; 1224 } 1225 1226 static void 1227 free_buffers(struct active_device *ad, struct test_op_params *op_params) 1228 { 1229 unsigned int i, j; 1230 1231 rte_mempool_free(ad->ops_mempool); 1232 rte_mempool_free(ad->in_mbuf_pool); 1233 rte_mempool_free(ad->hard_out_mbuf_pool); 1234 rte_mempool_free(ad->soft_out_mbuf_pool); 1235 rte_mempool_free(ad->harq_in_mbuf_pool); 1236 rte_mempool_free(ad->harq_out_mbuf_pool); 1237 1238 for (i = 0; i < rte_lcore_count(); ++i) { 1239 for (j = 0; j < RTE_MAX_NUMA_NODES; ++j) { 1240 rte_free(op_params->q_bufs[j][i].inputs); 1241 rte_free(op_params->q_bufs[j][i].hard_outputs); 1242 rte_free(op_params->q_bufs[j][i].soft_outputs); 1243 rte_free(op_params->q_bufs[j][i].harq_inputs); 1244 rte_free(op_params->q_bufs[j][i].harq_outputs); 1245 } 1246 } 1247 } 1248 1249 static void 1250 copy_reference_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n, 1251 unsigned int start_idx, 1252 struct rte_bbdev_op_data *inputs, 1253 struct rte_bbdev_op_data *hard_outputs, 1254 struct rte_bbdev_op_data *soft_outputs, 1255 struct rte_bbdev_dec_op *ref_op) 1256 { 1257 unsigned int i; 1258 struct rte_bbdev_op_turbo_dec *turbo_dec = &ref_op->turbo_dec; 1259 1260 for (i = 0; i < n; ++i) { 1261 if (turbo_dec->code_block_mode == 0) { 1262 ops[i]->turbo_dec.tb_params.ea = 1263 turbo_dec->tb_params.ea; 1264 ops[i]->turbo_dec.tb_params.eb = 1265 turbo_dec->tb_params.eb; 1266 ops[i]->turbo_dec.tb_params.k_pos = 1267 turbo_dec->tb_params.k_pos; 1268 ops[i]->turbo_dec.tb_params.k_neg = 1269 turbo_dec->tb_params.k_neg; 1270 ops[i]->turbo_dec.tb_params.c = 1271 turbo_dec->tb_params.c; 1272 ops[i]->turbo_dec.tb_params.c_neg = 1273 turbo_dec->tb_params.c_neg; 1274 ops[i]->turbo_dec.tb_params.cab = 1275 turbo_dec->tb_params.cab; 1276 ops[i]->turbo_dec.tb_params.r = 1277 turbo_dec->tb_params.r; 1278 } else { 1279 ops[i]->turbo_dec.cb_params.e = turbo_dec->cb_params.e; 1280 ops[i]->turbo_dec.cb_params.k = turbo_dec->cb_params.k; 1281 } 1282 1283 ops[i]->turbo_dec.ext_scale = turbo_dec->ext_scale; 1284 ops[i]->turbo_dec.iter_max = turbo_dec->iter_max; 1285 ops[i]->turbo_dec.iter_min = turbo_dec->iter_min; 1286 ops[i]->turbo_dec.op_flags = turbo_dec->op_flags; 1287 ops[i]->turbo_dec.rv_index = turbo_dec->rv_index; 1288 ops[i]->turbo_dec.num_maps = turbo_dec->num_maps; 1289 ops[i]->turbo_dec.code_block_mode = turbo_dec->code_block_mode; 1290 1291 ops[i]->turbo_dec.hard_output = hard_outputs[start_idx + i]; 1292 ops[i]->turbo_dec.input = inputs[start_idx + i]; 1293 if (soft_outputs != NULL) 1294 ops[i]->turbo_dec.soft_output = 1295 soft_outputs[start_idx + i]; 1296 } 1297 } 1298 1299 static void 1300 copy_reference_enc_op(struct rte_bbdev_enc_op **ops, unsigned int n, 1301 unsigned int start_idx, 1302 struct rte_bbdev_op_data *inputs, 1303 struct rte_bbdev_op_data *outputs, 1304 struct rte_bbdev_enc_op *ref_op) 1305 { 1306 unsigned int i; 1307 struct rte_bbdev_op_turbo_enc *turbo_enc = &ref_op->turbo_enc; 1308 for (i = 0; i < n; ++i) { 1309 if (turbo_enc->code_block_mode == 0) { 1310 ops[i]->turbo_enc.tb_params.ea = 1311 turbo_enc->tb_params.ea; 1312 ops[i]->turbo_enc.tb_params.eb = 1313 turbo_enc->tb_params.eb; 1314 ops[i]->turbo_enc.tb_params.k_pos = 1315 turbo_enc->tb_params.k_pos; 1316 ops[i]->turbo_enc.tb_params.k_neg = 1317 turbo_enc->tb_params.k_neg; 1318 ops[i]->turbo_enc.tb_params.c = 1319 turbo_enc->tb_params.c; 1320 ops[i]->turbo_enc.tb_params.c_neg = 1321 turbo_enc->tb_params.c_neg; 1322 ops[i]->turbo_enc.tb_params.cab = 1323 turbo_enc->tb_params.cab; 1324 ops[i]->turbo_enc.tb_params.ncb_pos = 1325 turbo_enc->tb_params.ncb_pos; 1326 ops[i]->turbo_enc.tb_params.ncb_neg = 1327 turbo_enc->tb_params.ncb_neg; 1328 ops[i]->turbo_enc.tb_params.r = turbo_enc->tb_params.r; 1329 } else { 1330 ops[i]->turbo_enc.cb_params.e = turbo_enc->cb_params.e; 1331 ops[i]->turbo_enc.cb_params.k = turbo_enc->cb_params.k; 1332 ops[i]->turbo_enc.cb_params.ncb = 1333 turbo_enc->cb_params.ncb; 1334 } 1335 ops[i]->turbo_enc.rv_index = turbo_enc->rv_index; 1336 ops[i]->turbo_enc.op_flags = turbo_enc->op_flags; 1337 ops[i]->turbo_enc.code_block_mode = turbo_enc->code_block_mode; 1338 1339 ops[i]->turbo_enc.output = outputs[start_idx + i]; 1340 ops[i]->turbo_enc.input = inputs[start_idx + i]; 1341 } 1342 } 1343 1344 1345 /* Returns a random number drawn from a normal distribution 1346 * with mean of 0 and variance of 1 1347 * Marsaglia algorithm 1348 */ 1349 static double 1350 randn(int n) 1351 { 1352 double S, Z, U1, U2, u, v, fac; 1353 1354 do { 1355 U1 = (double)rand() / RAND_MAX; 1356 U2 = (double)rand() / RAND_MAX; 1357 u = 2. * U1 - 1.; 1358 v = 2. * U2 - 1.; 1359 S = u * u + v * v; 1360 } while (S >= 1 || S == 0); 1361 fac = sqrt(-2. * log(S) / S); 1362 Z = (n % 2) ? u * fac : v * fac; 1363 return Z; 1364 } 1365 1366 static inline double 1367 maxstar(double A, double B) 1368 { 1369 if (fabs(A - B) > 5) 1370 return RTE_MAX(A, B); 1371 else 1372 return RTE_MAX(A, B) + log1p(exp(-fabs(A - B))); 1373 } 1374 1375 /* 1376 * Generate Qm LLRS for Qm==8 1377 * Modulation, AWGN and LLR estimation from max log development 1378 */ 1379 static void 1380 gen_qm8_llr(int8_t *llrs, uint32_t i, double N0, double llr_max) 1381 { 1382 int qm = 8; 1383 int qam = 256; 1384 int m, k; 1385 double I, Q, p0, p1, llr_, b[qm], log_syml_prob[qam]; 1386 /* 5.1.4 of TS38.211 */ 1387 const double symbols_I[256] = { 1388 5, 5, 7, 7, 5, 5, 7, 7, 3, 3, 1, 1, 3, 3, 1, 1, 5, 1389 5, 7, 7, 5, 5, 7, 7, 3, 3, 1, 1, 3, 3, 1, 1, 11, 1390 11, 9, 9, 11, 11, 9, 9, 13, 13, 15, 15, 13, 13, 1391 15, 15, 11, 11, 9, 9, 11, 11, 9, 9, 13, 13, 15, 1392 15, 13, 13, 15, 15, 5, 5, 7, 7, 5, 5, 7, 7, 3, 3, 1393 1, 1, 3, 3, 1, 1, 5, 5, 7, 7, 5, 5, 7, 7, 3, 3, 1, 1394 1, 3, 3, 1, 1, 11, 11, 9, 9, 11, 11, 9, 9, 13, 13, 1395 15, 15, 13, 13, 15, 15, 11, 11, 9, 9, 11, 11, 9, 9, 1396 13, 13, 15, 15, 13, 13, 15, 15, -5, -5, -7, -7, -5, 1397 -5, -7, -7, -3, -3, -1, -1, -3, -3, -1, -1, -5, -5, 1398 -7, -7, -5, -5, -7, -7, -3, -3, -1, -1, -3, -3, 1399 -1, -1, -11, -11, -9, -9, -11, -11, -9, -9, -13, 1400 -13, -15, -15, -13, -13, -15, -15, -11, -11, -9, 1401 -9, -11, -11, -9, -9, -13, -13, -15, -15, -13, 1402 -13, -15, -15, -5, -5, -7, -7, -5, -5, -7, -7, -3, 1403 -3, -1, -1, -3, -3, -1, -1, -5, -5, -7, -7, -5, -5, 1404 -7, -7, -3, -3, -1, -1, -3, -3, -1, -1, -11, -11, 1405 -9, -9, -11, -11, -9, -9, -13, -13, -15, -15, -13, 1406 -13, -15, -15, -11, -11, -9, -9, -11, -11, -9, -9, 1407 -13, -13, -15, -15, -13, -13, -15, -15}; 1408 const double symbols_Q[256] = { 1409 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 11, 1410 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9, 13, 15, 13, 1411 15, 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 1412 11, 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9, 13, 1413 15, 13, 15, -5, -7, -5, -7, -3, -1, -3, -1, -5, 1414 -7, -5, -7, -3, -1, -3, -1, -11, -9, -11, -9, -13, 1415 -15, -13, -15, -11, -9, -11, -9, -13, -15, -13, 1416 -15, -5, -7, -5, -7, -3, -1, -3, -1, -5, -7, -5, 1417 -7, -3, -1, -3, -1, -11, -9, -11, -9, -13, -15, 1418 -13, -15, -11, -9, -11, -9, -13, -15, -13, -15, 5, 1419 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 11, 1420 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9, 13, 15, 1421 13, 15, 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 1422 3, 1, 11, 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9, 1423 13, 15, 13, 15, -5, -7, -5, -7, -3, -1, -3, -1, 1424 -5, -7, -5, -7, -3, -1, -3, -1, -11, -9, -11, -9, 1425 -13, -15, -13, -15, -11, -9, -11, -9, -13, -15, 1426 -13, -15, -5, -7, -5, -7, -3, -1, -3, -1, -5, -7, 1427 -5, -7, -3, -1, -3, -1, -11, -9, -11, -9, -13, -15, 1428 -13, -15, -11, -9, -11, -9, -13, -15, -13, -15}; 1429 /* Average constellation point energy */ 1430 N0 *= 170.0; 1431 for (k = 0; k < qm; k++) 1432 b[k] = llrs[qm * i + k] < 0 ? 1.0 : 0.0; 1433 /* 5.1.4 of TS38.211 */ 1434 I = (1 - 2 * b[0]) * (8 - (1 - 2 * b[2]) * 1435 (4 - (1 - 2 * b[4]) * (2 - (1 - 2 * b[6])))); 1436 Q = (1 - 2 * b[1]) * (8 - (1 - 2 * b[3]) * 1437 (4 - (1 - 2 * b[5]) * (2 - (1 - 2 * b[7])))); 1438 /* AWGN channel */ 1439 I += sqrt(N0 / 2) * randn(0); 1440 Q += sqrt(N0 / 2) * randn(1); 1441 /* 1442 * Calculate the log of the probability that each of 1443 * the constellation points was transmitted 1444 */ 1445 for (m = 0; m < qam; m++) 1446 log_syml_prob[m] = -(pow(I - symbols_I[m], 2.0) 1447 + pow(Q - symbols_Q[m], 2.0)) / N0; 1448 /* Calculate an LLR for each of the k_64QAM bits in the set */ 1449 for (k = 0; k < qm; k++) { 1450 p0 = -999999; 1451 p1 = -999999; 1452 /* For each constellation point */ 1453 for (m = 0; m < qam; m++) { 1454 if ((m >> (qm - k - 1)) & 1) 1455 p1 = maxstar(p1, log_syml_prob[m]); 1456 else 1457 p0 = maxstar(p0, log_syml_prob[m]); 1458 } 1459 /* Calculate the LLR */ 1460 llr_ = p0 - p1; 1461 llr_ *= (1 << ldpc_llr_decimals); 1462 llr_ = round(llr_); 1463 if (llr_ > llr_max) 1464 llr_ = llr_max; 1465 if (llr_ < -llr_max) 1466 llr_ = -llr_max; 1467 llrs[qm * i + k] = (int8_t) llr_; 1468 } 1469 } 1470 1471 1472 /* 1473 * Generate Qm LLRS for Qm==6 1474 * Modulation, AWGN and LLR estimation from max log development 1475 */ 1476 static void 1477 gen_qm6_llr(int8_t *llrs, uint32_t i, double N0, double llr_max) 1478 { 1479 int qm = 6; 1480 int qam = 64; 1481 int m, k; 1482 double I, Q, p0, p1, llr_, b[qm], log_syml_prob[qam]; 1483 /* 5.1.4 of TS38.211 */ 1484 const double symbols_I[64] = { 1485 3, 3, 1, 1, 3, 3, 1, 1, 5, 5, 7, 7, 5, 5, 7, 7, 1486 3, 3, 1, 1, 3, 3, 1, 1, 5, 5, 7, 7, 5, 5, 7, 7, 1487 -3, -3, -1, -1, -3, -3, -1, -1, -5, -5, -7, -7, 1488 -5, -5, -7, -7, -3, -3, -1, -1, -3, -3, -1, -1, 1489 -5, -5, -7, -7, -5, -5, -7, -7}; 1490 const double symbols_Q[64] = { 1491 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 1492 -3, -1, -3, -1, -5, -7, -5, -7, -3, -1, -3, -1, 1493 -5, -7, -5, -7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 1494 5, 7, 5, 7, -3, -1, -3, -1, -5, -7, -5, -7, 1495 -3, -1, -3, -1, -5, -7, -5, -7}; 1496 /* Average constellation point energy */ 1497 N0 *= 42.0; 1498 for (k = 0; k < qm; k++) 1499 b[k] = llrs[qm * i + k] < 0 ? 1.0 : 0.0; 1500 /* 5.1.4 of TS38.211 */ 1501 I = (1 - 2 * b[0])*(4 - (1 - 2 * b[2]) * (2 - (1 - 2 * b[4]))); 1502 Q = (1 - 2 * b[1])*(4 - (1 - 2 * b[3]) * (2 - (1 - 2 * b[5]))); 1503 /* AWGN channel */ 1504 I += sqrt(N0 / 2) * randn(0); 1505 Q += sqrt(N0 / 2) * randn(1); 1506 /* 1507 * Calculate the log of the probability that each of 1508 * the constellation points was transmitted 1509 */ 1510 for (m = 0; m < qam; m++) 1511 log_syml_prob[m] = -(pow(I - symbols_I[m], 2.0) 1512 + pow(Q - symbols_Q[m], 2.0)) / N0; 1513 /* Calculate an LLR for each of the k_64QAM bits in the set */ 1514 for (k = 0; k < qm; k++) { 1515 p0 = -999999; 1516 p1 = -999999; 1517 /* For each constellation point */ 1518 for (m = 0; m < qam; m++) { 1519 if ((m >> (qm - k - 1)) & 1) 1520 p1 = maxstar(p1, log_syml_prob[m]); 1521 else 1522 p0 = maxstar(p0, log_syml_prob[m]); 1523 } 1524 /* Calculate the LLR */ 1525 llr_ = p0 - p1; 1526 llr_ *= (1 << ldpc_llr_decimals); 1527 llr_ = round(llr_); 1528 if (llr_ > llr_max) 1529 llr_ = llr_max; 1530 if (llr_ < -llr_max) 1531 llr_ = -llr_max; 1532 llrs[qm * i + k] = (int8_t) llr_; 1533 } 1534 } 1535 1536 /* 1537 * Generate Qm LLRS for Qm==4 1538 * Modulation, AWGN and LLR estimation from max log development 1539 */ 1540 static void 1541 gen_qm4_llr(int8_t *llrs, uint32_t i, double N0, double llr_max) 1542 { 1543 int qm = 4; 1544 int qam = 16; 1545 int m, k; 1546 double I, Q, p0, p1, llr_, b[qm], log_syml_prob[qam]; 1547 /* 5.1.4 of TS38.211 */ 1548 const double symbols_I[16] = {1, 1, 3, 3, 1, 1, 3, 3, 1549 -1, -1, -3, -3, -1, -1, -3, -3}; 1550 const double symbols_Q[16] = {1, 3, 1, 3, -1, -3, -1, -3, 1551 1, 3, 1, 3, -1, -3, -1, -3}; 1552 /* Average constellation point energy */ 1553 N0 *= 10.0; 1554 for (k = 0; k < qm; k++) 1555 b[k] = llrs[qm * i + k] < 0 ? 1.0 : 0.0; 1556 /* 5.1.4 of TS38.211 */ 1557 I = (1 - 2 * b[0]) * (2 - (1 - 2 * b[2])); 1558 Q = (1 - 2 * b[1]) * (2 - (1 - 2 * b[3])); 1559 /* AWGN channel */ 1560 I += sqrt(N0 / 2) * randn(0); 1561 Q += sqrt(N0 / 2) * randn(1); 1562 /* 1563 * Calculate the log of the probability that each of 1564 * the constellation points was transmitted 1565 */ 1566 for (m = 0; m < qam; m++) 1567 log_syml_prob[m] = -(pow(I - symbols_I[m], 2.0) 1568 + pow(Q - symbols_Q[m], 2.0)) / N0; 1569 /* Calculate an LLR for each of the k_64QAM bits in the set */ 1570 for (k = 0; k < qm; k++) { 1571 p0 = -999999; 1572 p1 = -999999; 1573 /* For each constellation point */ 1574 for (m = 0; m < qam; m++) { 1575 if ((m >> (qm - k - 1)) & 1) 1576 p1 = maxstar(p1, log_syml_prob[m]); 1577 else 1578 p0 = maxstar(p0, log_syml_prob[m]); 1579 } 1580 /* Calculate the LLR */ 1581 llr_ = p0 - p1; 1582 llr_ *= (1 << ldpc_llr_decimals); 1583 llr_ = round(llr_); 1584 if (llr_ > llr_max) 1585 llr_ = llr_max; 1586 if (llr_ < -llr_max) 1587 llr_ = -llr_max; 1588 llrs[qm * i + k] = (int8_t) llr_; 1589 } 1590 } 1591 1592 static void 1593 gen_qm2_llr(int8_t *llrs, uint32_t j, double N0, double llr_max) 1594 { 1595 double b, b1, n; 1596 double coeff = 2.0 * sqrt(N0); 1597 1598 /* Ignore in vectors rare quasi null LLRs not to be saturated */ 1599 if (llrs[j] < 8 && llrs[j] > -8) 1600 return; 1601 1602 /* Note don't change sign here */ 1603 n = randn(j % 2); 1604 b1 = ((llrs[j] > 0 ? 2.0 : -2.0) 1605 + coeff * n) / N0; 1606 b = b1 * (1 << ldpc_llr_decimals); 1607 b = round(b); 1608 if (b > llr_max) 1609 b = llr_max; 1610 if (b < -llr_max) 1611 b = -llr_max; 1612 llrs[j] = (int8_t) b; 1613 } 1614 1615 /* Generate LLR for a given SNR */ 1616 static void 1617 generate_llr_input(uint16_t n, struct rte_bbdev_op_data *inputs, 1618 struct rte_bbdev_dec_op *ref_op) 1619 { 1620 struct rte_mbuf *m; 1621 uint16_t qm; 1622 uint32_t i, j, e, range; 1623 double N0, llr_max; 1624 1625 e = ref_op->ldpc_dec.cb_params.e; 1626 qm = ref_op->ldpc_dec.q_m; 1627 llr_max = (1 << (ldpc_llr_size - 1)) - 1; 1628 range = e / qm; 1629 N0 = 1.0 / pow(10.0, get_snr() / 10.0); 1630 1631 for (i = 0; i < n; ++i) { 1632 m = inputs[i].data; 1633 int8_t *llrs = rte_pktmbuf_mtod_offset(m, int8_t *, 0); 1634 if (qm == 8) { 1635 for (j = 0; j < range; ++j) 1636 gen_qm8_llr(llrs, j, N0, llr_max); 1637 } else if (qm == 6) { 1638 for (j = 0; j < range; ++j) 1639 gen_qm6_llr(llrs, j, N0, llr_max); 1640 } else if (qm == 4) { 1641 for (j = 0; j < range; ++j) 1642 gen_qm4_llr(llrs, j, N0, llr_max); 1643 } else { 1644 for (j = 0; j < e; ++j) 1645 gen_qm2_llr(llrs, j, N0, llr_max); 1646 } 1647 } 1648 } 1649 1650 static void 1651 copy_reference_ldpc_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n, 1652 unsigned int start_idx, 1653 struct rte_bbdev_op_data *inputs, 1654 struct rte_bbdev_op_data *hard_outputs, 1655 struct rte_bbdev_op_data *soft_outputs, 1656 struct rte_bbdev_op_data *harq_inputs, 1657 struct rte_bbdev_op_data *harq_outputs, 1658 struct rte_bbdev_dec_op *ref_op) 1659 { 1660 unsigned int i; 1661 struct rte_bbdev_op_ldpc_dec *ldpc_dec = &ref_op->ldpc_dec; 1662 1663 for (i = 0; i < n; ++i) { 1664 if (ldpc_dec->code_block_mode == 0) { 1665 ops[i]->ldpc_dec.tb_params.ea = 1666 ldpc_dec->tb_params.ea; 1667 ops[i]->ldpc_dec.tb_params.eb = 1668 ldpc_dec->tb_params.eb; 1669 ops[i]->ldpc_dec.tb_params.c = 1670 ldpc_dec->tb_params.c; 1671 ops[i]->ldpc_dec.tb_params.cab = 1672 ldpc_dec->tb_params.cab; 1673 ops[i]->ldpc_dec.tb_params.r = 1674 ldpc_dec->tb_params.r; 1675 } else { 1676 ops[i]->ldpc_dec.cb_params.e = ldpc_dec->cb_params.e; 1677 } 1678 1679 ops[i]->ldpc_dec.basegraph = ldpc_dec->basegraph; 1680 ops[i]->ldpc_dec.z_c = ldpc_dec->z_c; 1681 ops[i]->ldpc_dec.q_m = ldpc_dec->q_m; 1682 ops[i]->ldpc_dec.n_filler = ldpc_dec->n_filler; 1683 ops[i]->ldpc_dec.n_cb = ldpc_dec->n_cb; 1684 ops[i]->ldpc_dec.iter_max = ldpc_dec->iter_max; 1685 ops[i]->ldpc_dec.rv_index = ldpc_dec->rv_index; 1686 ops[i]->ldpc_dec.op_flags = ldpc_dec->op_flags; 1687 ops[i]->ldpc_dec.code_block_mode = ldpc_dec->code_block_mode; 1688 1689 if (hard_outputs != NULL) 1690 ops[i]->ldpc_dec.hard_output = 1691 hard_outputs[start_idx + i]; 1692 if (inputs != NULL) 1693 ops[i]->ldpc_dec.input = 1694 inputs[start_idx + i]; 1695 if (soft_outputs != NULL) 1696 ops[i]->ldpc_dec.soft_output = 1697 soft_outputs[start_idx + i]; 1698 if (harq_inputs != NULL) 1699 ops[i]->ldpc_dec.harq_combined_input = 1700 harq_inputs[start_idx + i]; 1701 if (harq_outputs != NULL) 1702 ops[i]->ldpc_dec.harq_combined_output = 1703 harq_outputs[start_idx + i]; 1704 } 1705 } 1706 1707 1708 static void 1709 copy_reference_ldpc_enc_op(struct rte_bbdev_enc_op **ops, unsigned int n, 1710 unsigned int start_idx, 1711 struct rte_bbdev_op_data *inputs, 1712 struct rte_bbdev_op_data *outputs, 1713 struct rte_bbdev_enc_op *ref_op) 1714 { 1715 unsigned int i; 1716 struct rte_bbdev_op_ldpc_enc *ldpc_enc = &ref_op->ldpc_enc; 1717 for (i = 0; i < n; ++i) { 1718 if (ldpc_enc->code_block_mode == 0) { 1719 ops[i]->ldpc_enc.tb_params.ea = ldpc_enc->tb_params.ea; 1720 ops[i]->ldpc_enc.tb_params.eb = ldpc_enc->tb_params.eb; 1721 ops[i]->ldpc_enc.tb_params.cab = 1722 ldpc_enc->tb_params.cab; 1723 ops[i]->ldpc_enc.tb_params.c = ldpc_enc->tb_params.c; 1724 ops[i]->ldpc_enc.tb_params.r = ldpc_enc->tb_params.r; 1725 } else { 1726 ops[i]->ldpc_enc.cb_params.e = ldpc_enc->cb_params.e; 1727 } 1728 ops[i]->ldpc_enc.basegraph = ldpc_enc->basegraph; 1729 ops[i]->ldpc_enc.z_c = ldpc_enc->z_c; 1730 ops[i]->ldpc_enc.q_m = ldpc_enc->q_m; 1731 ops[i]->ldpc_enc.n_filler = ldpc_enc->n_filler; 1732 ops[i]->ldpc_enc.n_cb = ldpc_enc->n_cb; 1733 ops[i]->ldpc_enc.rv_index = ldpc_enc->rv_index; 1734 ops[i]->ldpc_enc.op_flags = ldpc_enc->op_flags; 1735 ops[i]->ldpc_enc.code_block_mode = ldpc_enc->code_block_mode; 1736 ops[i]->ldpc_enc.output = outputs[start_idx + i]; 1737 ops[i]->ldpc_enc.input = inputs[start_idx + i]; 1738 } 1739 } 1740 1741 static int 1742 check_dec_status_and_ordering(struct rte_bbdev_dec_op *op, 1743 unsigned int order_idx, const int expected_status) 1744 { 1745 int status = op->status; 1746 /* ignore parity mismatch false alarms for long iterations */ 1747 if (get_iter_max() >= 10) { 1748 if (!(expected_status & (1 << RTE_BBDEV_SYNDROME_ERROR)) && 1749 (status & (1 << RTE_BBDEV_SYNDROME_ERROR))) { 1750 printf("WARNING: Ignore Syndrome Check mismatch\n"); 1751 status -= (1 << RTE_BBDEV_SYNDROME_ERROR); 1752 } 1753 if ((expected_status & (1 << RTE_BBDEV_SYNDROME_ERROR)) && 1754 !(status & (1 << RTE_BBDEV_SYNDROME_ERROR))) { 1755 printf("WARNING: Ignore Syndrome Check mismatch\n"); 1756 status += (1 << RTE_BBDEV_SYNDROME_ERROR); 1757 } 1758 } 1759 1760 TEST_ASSERT(status == expected_status, 1761 "op_status (%d) != expected_status (%d)", 1762 op->status, expected_status); 1763 1764 TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data, 1765 "Ordering error, expected %p, got %p", 1766 (void *)(uintptr_t)order_idx, op->opaque_data); 1767 1768 return TEST_SUCCESS; 1769 } 1770 1771 static int 1772 check_enc_status_and_ordering(struct rte_bbdev_enc_op *op, 1773 unsigned int order_idx, const int expected_status) 1774 { 1775 TEST_ASSERT(op->status == expected_status, 1776 "op_status (%d) != expected_status (%d)", 1777 op->status, expected_status); 1778 1779 if (op->opaque_data != (void *)(uintptr_t)INVALID_OPAQUE) 1780 TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data, 1781 "Ordering error, expected %p, got %p", 1782 (void *)(uintptr_t)order_idx, op->opaque_data); 1783 1784 return TEST_SUCCESS; 1785 } 1786 1787 static inline int 1788 validate_op_chain(struct rte_bbdev_op_data *op, 1789 struct op_data_entries *orig_op) 1790 { 1791 uint8_t i; 1792 struct rte_mbuf *m = op->data; 1793 uint8_t nb_dst_segments = orig_op->nb_segments; 1794 uint32_t total_data_size = 0; 1795 1796 TEST_ASSERT(nb_dst_segments == m->nb_segs, 1797 "Number of segments differ in original (%u) and filled (%u) op", 1798 nb_dst_segments, m->nb_segs); 1799 1800 /* Validate each mbuf segment length */ 1801 for (i = 0; i < nb_dst_segments; ++i) { 1802 /* Apply offset to the first mbuf segment */ 1803 uint16_t offset = (i == 0) ? op->offset : 0; 1804 uint16_t data_len = rte_pktmbuf_data_len(m) - offset; 1805 total_data_size += orig_op->segments[i].length; 1806 1807 TEST_ASSERT(orig_op->segments[i].length == data_len, 1808 "Length of segment differ in original (%u) and filled (%u) op", 1809 orig_op->segments[i].length, data_len); 1810 TEST_ASSERT_BUFFERS_ARE_EQUAL(orig_op->segments[i].addr, 1811 rte_pktmbuf_mtod_offset(m, uint32_t *, offset), 1812 data_len, 1813 "Output buffers (CB=%u) are not equal", i); 1814 m = m->next; 1815 } 1816 1817 /* Validate total mbuf pkt length */ 1818 uint32_t pkt_len = rte_pktmbuf_pkt_len(op->data) - op->offset; 1819 TEST_ASSERT(total_data_size == pkt_len, 1820 "Length of data differ in original (%u) and filled (%u) op", 1821 total_data_size, pkt_len); 1822 1823 return TEST_SUCCESS; 1824 } 1825 1826 /* 1827 * Compute K0 for a given configuration for HARQ output length computation 1828 * As per definition in 3GPP 38.212 Table 5.4.2.1-2 1829 */ 1830 static inline uint16_t 1831 get_k0(uint16_t n_cb, uint16_t z_c, uint8_t bg, uint8_t rv_index) 1832 { 1833 if (rv_index == 0) 1834 return 0; 1835 uint16_t n = (bg == 1 ? N_ZC_1 : N_ZC_2) * z_c; 1836 if (n_cb == n) { 1837 if (rv_index == 1) 1838 return (bg == 1 ? K0_1_1 : K0_1_2) * z_c; 1839 else if (rv_index == 2) 1840 return (bg == 1 ? K0_2_1 : K0_2_2) * z_c; 1841 else 1842 return (bg == 1 ? K0_3_1 : K0_3_2) * z_c; 1843 } 1844 /* LBRM case - includes a division by N */ 1845 if (rv_index == 1) 1846 return (((bg == 1 ? K0_1_1 : K0_1_2) * n_cb) 1847 / n) * z_c; 1848 else if (rv_index == 2) 1849 return (((bg == 1 ? K0_2_1 : K0_2_2) * n_cb) 1850 / n) * z_c; 1851 else 1852 return (((bg == 1 ? K0_3_1 : K0_3_2) * n_cb) 1853 / n) * z_c; 1854 } 1855 1856 /* HARQ output length including the Filler bits */ 1857 static inline uint16_t 1858 compute_harq_len(struct rte_bbdev_op_ldpc_dec *ops_ld) 1859 { 1860 uint16_t k0 = 0; 1861 uint8_t max_rv = (ops_ld->rv_index == 1) ? 3 : ops_ld->rv_index; 1862 k0 = get_k0(ops_ld->n_cb, ops_ld->z_c, ops_ld->basegraph, max_rv); 1863 /* Compute RM out size and number of rows */ 1864 uint16_t parity_offset = (ops_ld->basegraph == 1 ? 20 : 8) 1865 * ops_ld->z_c - ops_ld->n_filler; 1866 uint16_t deRmOutSize = RTE_MIN( 1867 k0 + ops_ld->cb_params.e + 1868 ((k0 > parity_offset) ? 1869 0 : ops_ld->n_filler), 1870 ops_ld->n_cb); 1871 uint16_t numRows = ((deRmOutSize + ops_ld->z_c - 1) 1872 / ops_ld->z_c); 1873 uint16_t harq_output_len = numRows * ops_ld->z_c; 1874 return harq_output_len; 1875 } 1876 1877 static inline int 1878 validate_op_harq_chain(struct rte_bbdev_op_data *op, 1879 struct op_data_entries *orig_op, 1880 struct rte_bbdev_op_ldpc_dec *ops_ld) 1881 { 1882 uint8_t i; 1883 uint32_t j, jj, k; 1884 struct rte_mbuf *m = op->data; 1885 uint8_t nb_dst_segments = orig_op->nb_segments; 1886 uint32_t total_data_size = 0; 1887 int8_t *harq_orig, *harq_out, abs_harq_origin; 1888 uint32_t byte_error = 0, cum_error = 0, error; 1889 int16_t llr_max = (1 << (ldpc_llr_size - ldpc_llr_decimals)) - 1; 1890 int16_t llr_max_pre_scaling = (1 << (ldpc_llr_size - 1)) - 1; 1891 uint16_t parity_offset; 1892 1893 TEST_ASSERT(nb_dst_segments == m->nb_segs, 1894 "Number of segments differ in original (%u) and filled (%u) op", 1895 nb_dst_segments, m->nb_segs); 1896 1897 /* Validate each mbuf segment length */ 1898 for (i = 0; i < nb_dst_segments; ++i) { 1899 /* Apply offset to the first mbuf segment */ 1900 uint16_t offset = (i == 0) ? op->offset : 0; 1901 uint16_t data_len = rte_pktmbuf_data_len(m) - offset; 1902 total_data_size += orig_op->segments[i].length; 1903 1904 TEST_ASSERT(orig_op->segments[i].length < 1905 (uint32_t)(data_len + 64), 1906 "Length of segment differ in original (%u) and filled (%u) op", 1907 orig_op->segments[i].length, data_len); 1908 harq_orig = (int8_t *) orig_op->segments[i].addr; 1909 harq_out = rte_pktmbuf_mtod_offset(m, int8_t *, offset); 1910 1911 if (!(ldpc_cap_flags & 1912 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_FILLERS 1913 ) || (ops_ld->op_flags & 1914 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)) { 1915 data_len -= ops_ld->z_c; 1916 parity_offset = data_len; 1917 } else { 1918 /* Compute RM out size and number of rows */ 1919 parity_offset = (ops_ld->basegraph == 1 ? 20 : 8) 1920 * ops_ld->z_c - ops_ld->n_filler; 1921 uint16_t deRmOutSize = compute_harq_len(ops_ld) - 1922 ops_ld->n_filler; 1923 if (data_len > deRmOutSize) 1924 data_len = deRmOutSize; 1925 if (data_len > orig_op->segments[i].length) 1926 data_len = orig_op->segments[i].length; 1927 } 1928 /* 1929 * HARQ output can have minor differences 1930 * due to integer representation and related scaling 1931 */ 1932 for (j = 0, jj = 0; j < data_len; j++, jj++) { 1933 if (j == parity_offset) { 1934 /* Special Handling of the filler bits */ 1935 for (k = 0; k < ops_ld->n_filler; k++) { 1936 if (harq_out[jj] != 1937 llr_max_pre_scaling) { 1938 printf("HARQ Filler issue %d: %d %d\n", 1939 jj, harq_out[jj], 1940 llr_max); 1941 byte_error++; 1942 } 1943 jj++; 1944 } 1945 } 1946 if (!(ops_ld->op_flags & 1947 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)) { 1948 if (ldpc_llr_decimals > 1) 1949 harq_out[jj] = (harq_out[jj] + 1) 1950 >> (ldpc_llr_decimals - 1); 1951 /* Saturated to S7 */ 1952 if (harq_orig[j] > llr_max) 1953 harq_orig[j] = llr_max; 1954 if (harq_orig[j] < -llr_max) 1955 harq_orig[j] = -llr_max; 1956 } 1957 if (harq_orig[j] != harq_out[jj]) { 1958 error = (harq_orig[j] > harq_out[jj]) ? 1959 harq_orig[j] - harq_out[jj] : 1960 harq_out[jj] - harq_orig[j]; 1961 abs_harq_origin = harq_orig[j] > 0 ? 1962 harq_orig[j] : 1963 -harq_orig[j]; 1964 /* Residual quantization error */ 1965 if ((error > 8 && (abs_harq_origin < 1966 (llr_max - 16))) || 1967 (error > 16)) { 1968 printf("HARQ mismatch %d: exp %d act %d => %d\n", 1969 j, harq_orig[j], 1970 harq_out[jj], error); 1971 byte_error++; 1972 cum_error += error; 1973 } 1974 } 1975 } 1976 m = m->next; 1977 } 1978 1979 if (byte_error) 1980 TEST_ASSERT(byte_error <= 1, 1981 "HARQ output mismatch (%d) %d", 1982 byte_error, cum_error); 1983 1984 /* Validate total mbuf pkt length */ 1985 uint32_t pkt_len = rte_pktmbuf_pkt_len(op->data) - op->offset; 1986 TEST_ASSERT(total_data_size < pkt_len + 64, 1987 "Length of data differ in original (%u) and filled (%u) op", 1988 total_data_size, pkt_len); 1989 1990 return TEST_SUCCESS; 1991 } 1992 1993 static int 1994 validate_dec_op(struct rte_bbdev_dec_op **ops, const uint16_t n, 1995 struct rte_bbdev_dec_op *ref_op, const int vector_mask) 1996 { 1997 unsigned int i; 1998 int ret; 1999 struct op_data_entries *hard_data_orig = 2000 &test_vector.entries[DATA_HARD_OUTPUT]; 2001 struct op_data_entries *soft_data_orig = 2002 &test_vector.entries[DATA_SOFT_OUTPUT]; 2003 struct rte_bbdev_op_turbo_dec *ops_td; 2004 struct rte_bbdev_op_data *hard_output; 2005 struct rte_bbdev_op_data *soft_output; 2006 struct rte_bbdev_op_turbo_dec *ref_td = &ref_op->turbo_dec; 2007 2008 for (i = 0; i < n; ++i) { 2009 ops_td = &ops[i]->turbo_dec; 2010 hard_output = &ops_td->hard_output; 2011 soft_output = &ops_td->soft_output; 2012 2013 if (vector_mask & TEST_BBDEV_VF_EXPECTED_ITER_COUNT) 2014 TEST_ASSERT(ops_td->iter_count <= ref_td->iter_count, 2015 "Returned iter_count (%d) > expected iter_count (%d)", 2016 ops_td->iter_count, ref_td->iter_count); 2017 ret = check_dec_status_and_ordering(ops[i], i, ref_op->status); 2018 TEST_ASSERT_SUCCESS(ret, 2019 "Checking status and ordering for decoder failed"); 2020 2021 TEST_ASSERT_SUCCESS(validate_op_chain(hard_output, 2022 hard_data_orig), 2023 "Hard output buffers (CB=%u) are not equal", 2024 i); 2025 2026 if (ref_op->turbo_dec.op_flags & RTE_BBDEV_TURBO_SOFT_OUTPUT) 2027 TEST_ASSERT_SUCCESS(validate_op_chain(soft_output, 2028 soft_data_orig), 2029 "Soft output buffers (CB=%u) are not equal", 2030 i); 2031 } 2032 2033 return TEST_SUCCESS; 2034 } 2035 2036 /* Check Number of code blocks errors */ 2037 static int 2038 validate_ldpc_bler(struct rte_bbdev_dec_op **ops, const uint16_t n) 2039 { 2040 unsigned int i; 2041 struct op_data_entries *hard_data_orig = 2042 &test_vector.entries[DATA_HARD_OUTPUT]; 2043 struct rte_bbdev_op_ldpc_dec *ops_td; 2044 struct rte_bbdev_op_data *hard_output; 2045 int errors = 0; 2046 struct rte_mbuf *m; 2047 2048 for (i = 0; i < n; ++i) { 2049 ops_td = &ops[i]->ldpc_dec; 2050 hard_output = &ops_td->hard_output; 2051 m = hard_output->data; 2052 if (memcmp(rte_pktmbuf_mtod_offset(m, uint32_t *, 0), 2053 hard_data_orig->segments[0].addr, 2054 hard_data_orig->segments[0].length)) 2055 errors++; 2056 } 2057 return errors; 2058 } 2059 2060 static int 2061 validate_ldpc_dec_op(struct rte_bbdev_dec_op **ops, const uint16_t n, 2062 struct rte_bbdev_dec_op *ref_op, const int vector_mask) 2063 { 2064 unsigned int i; 2065 int ret; 2066 struct op_data_entries *hard_data_orig = 2067 &test_vector.entries[DATA_HARD_OUTPUT]; 2068 struct op_data_entries *soft_data_orig = 2069 &test_vector.entries[DATA_SOFT_OUTPUT]; 2070 struct op_data_entries *harq_data_orig = 2071 &test_vector.entries[DATA_HARQ_OUTPUT]; 2072 struct rte_bbdev_op_ldpc_dec *ops_td; 2073 struct rte_bbdev_op_data *hard_output; 2074 struct rte_bbdev_op_data *harq_output; 2075 struct rte_bbdev_op_data *soft_output; 2076 struct rte_bbdev_op_ldpc_dec *ref_td = &ref_op->ldpc_dec; 2077 2078 for (i = 0; i < n; ++i) { 2079 ops_td = &ops[i]->ldpc_dec; 2080 hard_output = &ops_td->hard_output; 2081 harq_output = &ops_td->harq_combined_output; 2082 soft_output = &ops_td->soft_output; 2083 2084 ret = check_dec_status_and_ordering(ops[i], i, ref_op->status); 2085 TEST_ASSERT_SUCCESS(ret, 2086 "Checking status and ordering for decoder failed"); 2087 if (vector_mask & TEST_BBDEV_VF_EXPECTED_ITER_COUNT) 2088 TEST_ASSERT(ops_td->iter_count <= ref_td->iter_count, 2089 "Returned iter_count (%d) > expected iter_count (%d)", 2090 ops_td->iter_count, ref_td->iter_count); 2091 /* 2092 * We can ignore output data when the decoding failed to 2093 * converge or for loop-back cases 2094 */ 2095 if (!check_bit(ops[i]->ldpc_dec.op_flags, 2096 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK 2097 ) && ( 2098 ops[i]->status & (1 << RTE_BBDEV_SYNDROME_ERROR 2099 )) == 0) 2100 TEST_ASSERT_SUCCESS(validate_op_chain(hard_output, 2101 hard_data_orig), 2102 "Hard output buffers (CB=%u) are not equal", 2103 i); 2104 2105 if (ref_op->ldpc_dec.op_flags & RTE_BBDEV_LDPC_SOFT_OUT_ENABLE) 2106 TEST_ASSERT_SUCCESS(validate_op_chain(soft_output, 2107 soft_data_orig), 2108 "Soft output buffers (CB=%u) are not equal", 2109 i); 2110 if (ref_op->ldpc_dec.op_flags & 2111 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE) { 2112 TEST_ASSERT_SUCCESS(validate_op_harq_chain(harq_output, 2113 harq_data_orig, ops_td), 2114 "HARQ output buffers (CB=%u) are not equal", 2115 i); 2116 } 2117 if (ref_op->ldpc_dec.op_flags & 2118 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK) 2119 TEST_ASSERT_SUCCESS(validate_op_harq_chain(harq_output, 2120 harq_data_orig, ops_td), 2121 "HARQ output buffers (CB=%u) are not equal", 2122 i); 2123 2124 } 2125 2126 return TEST_SUCCESS; 2127 } 2128 2129 2130 static int 2131 validate_enc_op(struct rte_bbdev_enc_op **ops, const uint16_t n, 2132 struct rte_bbdev_enc_op *ref_op) 2133 { 2134 unsigned int i; 2135 int ret; 2136 struct op_data_entries *hard_data_orig = 2137 &test_vector.entries[DATA_HARD_OUTPUT]; 2138 2139 for (i = 0; i < n; ++i) { 2140 ret = check_enc_status_and_ordering(ops[i], i, ref_op->status); 2141 TEST_ASSERT_SUCCESS(ret, 2142 "Checking status and ordering for encoder failed"); 2143 TEST_ASSERT_SUCCESS(validate_op_chain( 2144 &ops[i]->turbo_enc.output, 2145 hard_data_orig), 2146 "Output buffers (CB=%u) are not equal", 2147 i); 2148 } 2149 2150 return TEST_SUCCESS; 2151 } 2152 2153 static int 2154 validate_ldpc_enc_op(struct rte_bbdev_enc_op **ops, const uint16_t n, 2155 struct rte_bbdev_enc_op *ref_op) 2156 { 2157 unsigned int i; 2158 int ret; 2159 struct op_data_entries *hard_data_orig = 2160 &test_vector.entries[DATA_HARD_OUTPUT]; 2161 2162 for (i = 0; i < n; ++i) { 2163 ret = check_enc_status_and_ordering(ops[i], i, ref_op->status); 2164 TEST_ASSERT_SUCCESS(ret, 2165 "Checking status and ordering for encoder failed"); 2166 TEST_ASSERT_SUCCESS(validate_op_chain( 2167 &ops[i]->ldpc_enc.output, 2168 hard_data_orig), 2169 "Output buffers (CB=%u) are not equal", 2170 i); 2171 } 2172 2173 return TEST_SUCCESS; 2174 } 2175 2176 static void 2177 create_reference_dec_op(struct rte_bbdev_dec_op *op) 2178 { 2179 unsigned int i; 2180 struct op_data_entries *entry; 2181 2182 op->turbo_dec = test_vector.turbo_dec; 2183 entry = &test_vector.entries[DATA_INPUT]; 2184 for (i = 0; i < entry->nb_segments; ++i) 2185 op->turbo_dec.input.length += 2186 entry->segments[i].length; 2187 } 2188 2189 static void 2190 create_reference_ldpc_dec_op(struct rte_bbdev_dec_op *op) 2191 { 2192 unsigned int i; 2193 struct op_data_entries *entry; 2194 2195 op->ldpc_dec = test_vector.ldpc_dec; 2196 entry = &test_vector.entries[DATA_INPUT]; 2197 for (i = 0; i < entry->nb_segments; ++i) 2198 op->ldpc_dec.input.length += 2199 entry->segments[i].length; 2200 if (test_vector.ldpc_dec.op_flags & 2201 RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE) { 2202 entry = &test_vector.entries[DATA_HARQ_INPUT]; 2203 for (i = 0; i < entry->nb_segments; ++i) 2204 op->ldpc_dec.harq_combined_input.length += 2205 entry->segments[i].length; 2206 } 2207 } 2208 2209 2210 static void 2211 create_reference_enc_op(struct rte_bbdev_enc_op *op) 2212 { 2213 unsigned int i; 2214 struct op_data_entries *entry; 2215 2216 op->turbo_enc = test_vector.turbo_enc; 2217 entry = &test_vector.entries[DATA_INPUT]; 2218 for (i = 0; i < entry->nb_segments; ++i) 2219 op->turbo_enc.input.length += 2220 entry->segments[i].length; 2221 } 2222 2223 static void 2224 create_reference_ldpc_enc_op(struct rte_bbdev_enc_op *op) 2225 { 2226 unsigned int i; 2227 struct op_data_entries *entry; 2228 2229 op->ldpc_enc = test_vector.ldpc_enc; 2230 entry = &test_vector.entries[DATA_INPUT]; 2231 for (i = 0; i < entry->nb_segments; ++i) 2232 op->ldpc_enc.input.length += 2233 entry->segments[i].length; 2234 } 2235 2236 static uint32_t 2237 calc_dec_TB_size(struct rte_bbdev_dec_op *op) 2238 { 2239 uint8_t i; 2240 uint32_t c, r, tb_size = 0; 2241 2242 if (op->turbo_dec.code_block_mode) { 2243 tb_size = op->turbo_dec.tb_params.k_neg; 2244 } else { 2245 c = op->turbo_dec.tb_params.c; 2246 r = op->turbo_dec.tb_params.r; 2247 for (i = 0; i < c-r; i++) 2248 tb_size += (r < op->turbo_dec.tb_params.c_neg) ? 2249 op->turbo_dec.tb_params.k_neg : 2250 op->turbo_dec.tb_params.k_pos; 2251 } 2252 return tb_size; 2253 } 2254 2255 static uint32_t 2256 calc_ldpc_dec_TB_size(struct rte_bbdev_dec_op *op) 2257 { 2258 uint8_t i; 2259 uint32_t c, r, tb_size = 0; 2260 uint16_t sys_cols = (op->ldpc_dec.basegraph == 1) ? 22 : 10; 2261 2262 if (op->ldpc_dec.code_block_mode) { 2263 tb_size = sys_cols * op->ldpc_dec.z_c - op->ldpc_dec.n_filler; 2264 } else { 2265 c = op->ldpc_dec.tb_params.c; 2266 r = op->ldpc_dec.tb_params.r; 2267 for (i = 0; i < c-r; i++) 2268 tb_size += sys_cols * op->ldpc_dec.z_c 2269 - op->ldpc_dec.n_filler; 2270 } 2271 return tb_size; 2272 } 2273 2274 static uint32_t 2275 calc_enc_TB_size(struct rte_bbdev_enc_op *op) 2276 { 2277 uint8_t i; 2278 uint32_t c, r, tb_size = 0; 2279 2280 if (op->turbo_enc.code_block_mode) { 2281 tb_size = op->turbo_enc.tb_params.k_neg; 2282 } else { 2283 c = op->turbo_enc.tb_params.c; 2284 r = op->turbo_enc.tb_params.r; 2285 for (i = 0; i < c-r; i++) 2286 tb_size += (r < op->turbo_enc.tb_params.c_neg) ? 2287 op->turbo_enc.tb_params.k_neg : 2288 op->turbo_enc.tb_params.k_pos; 2289 } 2290 return tb_size; 2291 } 2292 2293 static uint32_t 2294 calc_ldpc_enc_TB_size(struct rte_bbdev_enc_op *op) 2295 { 2296 uint8_t i; 2297 uint32_t c, r, tb_size = 0; 2298 uint16_t sys_cols = (op->ldpc_enc.basegraph == 1) ? 22 : 10; 2299 2300 if (op->turbo_enc.code_block_mode) { 2301 tb_size = sys_cols * op->ldpc_enc.z_c - op->ldpc_enc.n_filler; 2302 } else { 2303 c = op->turbo_enc.tb_params.c; 2304 r = op->turbo_enc.tb_params.r; 2305 for (i = 0; i < c-r; i++) 2306 tb_size += sys_cols * op->ldpc_enc.z_c 2307 - op->ldpc_enc.n_filler; 2308 } 2309 return tb_size; 2310 } 2311 2312 2313 static int 2314 init_test_op_params(struct test_op_params *op_params, 2315 enum rte_bbdev_op_type op_type, const int expected_status, 2316 const int vector_mask, struct rte_mempool *ops_mp, 2317 uint16_t burst_sz, uint16_t num_to_process, uint16_t num_lcores) 2318 { 2319 int ret = 0; 2320 if (op_type == RTE_BBDEV_OP_TURBO_DEC || 2321 op_type == RTE_BBDEV_OP_LDPC_DEC) 2322 ret = rte_bbdev_dec_op_alloc_bulk(ops_mp, 2323 &op_params->ref_dec_op, 1); 2324 else 2325 ret = rte_bbdev_enc_op_alloc_bulk(ops_mp, 2326 &op_params->ref_enc_op, 1); 2327 2328 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_op_alloc_bulk() failed"); 2329 2330 op_params->mp = ops_mp; 2331 op_params->burst_sz = burst_sz; 2332 op_params->num_to_process = num_to_process; 2333 op_params->num_lcores = num_lcores; 2334 op_params->vector_mask = vector_mask; 2335 if (op_type == RTE_BBDEV_OP_TURBO_DEC || 2336 op_type == RTE_BBDEV_OP_LDPC_DEC) 2337 op_params->ref_dec_op->status = expected_status; 2338 else if (op_type == RTE_BBDEV_OP_TURBO_ENC 2339 || op_type == RTE_BBDEV_OP_LDPC_ENC) 2340 op_params->ref_enc_op->status = expected_status; 2341 return 0; 2342 } 2343 2344 static int 2345 run_test_case_on_device(test_case_function *test_case_func, uint8_t dev_id, 2346 struct test_op_params *op_params) 2347 { 2348 int t_ret, f_ret, socket_id = SOCKET_ID_ANY; 2349 unsigned int i; 2350 struct active_device *ad; 2351 unsigned int burst_sz = get_burst_sz(); 2352 enum rte_bbdev_op_type op_type = test_vector.op_type; 2353 const struct rte_bbdev_op_cap *capabilities = NULL; 2354 2355 ad = &active_devs[dev_id]; 2356 2357 /* Check if device supports op_type */ 2358 if (!is_avail_op(ad, test_vector.op_type)) 2359 return TEST_SUCCESS; 2360 2361 struct rte_bbdev_info info; 2362 rte_bbdev_info_get(ad->dev_id, &info); 2363 socket_id = GET_SOCKET(info.socket_id); 2364 2365 f_ret = create_mempools(ad, socket_id, op_type, 2366 get_num_ops()); 2367 if (f_ret != TEST_SUCCESS) { 2368 printf("Couldn't create mempools"); 2369 goto fail; 2370 } 2371 if (op_type == RTE_BBDEV_OP_NONE) 2372 op_type = RTE_BBDEV_OP_TURBO_ENC; 2373 2374 f_ret = init_test_op_params(op_params, test_vector.op_type, 2375 test_vector.expected_status, 2376 test_vector.mask, 2377 ad->ops_mempool, 2378 burst_sz, 2379 get_num_ops(), 2380 get_num_lcores()); 2381 if (f_ret != TEST_SUCCESS) { 2382 printf("Couldn't init test op params"); 2383 goto fail; 2384 } 2385 2386 2387 /* Find capabilities */ 2388 const struct rte_bbdev_op_cap *cap = info.drv.capabilities; 2389 for (i = 0; i < RTE_BBDEV_OP_TYPE_COUNT; i++) { 2390 if (cap->type == test_vector.op_type) { 2391 capabilities = cap; 2392 break; 2393 } 2394 cap++; 2395 } 2396 TEST_ASSERT_NOT_NULL(capabilities, 2397 "Couldn't find capabilities"); 2398 2399 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) { 2400 create_reference_dec_op(op_params->ref_dec_op); 2401 } else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) 2402 create_reference_enc_op(op_params->ref_enc_op); 2403 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) 2404 create_reference_ldpc_enc_op(op_params->ref_enc_op); 2405 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 2406 create_reference_ldpc_dec_op(op_params->ref_dec_op); 2407 2408 for (i = 0; i < ad->nb_queues; ++i) { 2409 f_ret = fill_queue_buffers(op_params, 2410 ad->in_mbuf_pool, 2411 ad->hard_out_mbuf_pool, 2412 ad->soft_out_mbuf_pool, 2413 ad->harq_in_mbuf_pool, 2414 ad->harq_out_mbuf_pool, 2415 ad->queue_ids[i], 2416 capabilities, 2417 info.drv.min_alignment, 2418 socket_id); 2419 if (f_ret != TEST_SUCCESS) { 2420 printf("Couldn't init queue buffers"); 2421 goto fail; 2422 } 2423 } 2424 2425 /* Run test case function */ 2426 t_ret = test_case_func(ad, op_params); 2427 2428 /* Free active device resources and return */ 2429 free_buffers(ad, op_params); 2430 return t_ret; 2431 2432 fail: 2433 free_buffers(ad, op_params); 2434 return TEST_FAILED; 2435 } 2436 2437 /* Run given test function per active device per supported op type 2438 * per burst size. 2439 */ 2440 static int 2441 run_test_case(test_case_function *test_case_func) 2442 { 2443 int ret = 0; 2444 uint8_t dev; 2445 2446 /* Alloc op_params */ 2447 struct test_op_params *op_params = rte_zmalloc(NULL, 2448 sizeof(struct test_op_params), RTE_CACHE_LINE_SIZE); 2449 TEST_ASSERT_NOT_NULL(op_params, "Failed to alloc %zuB for op_params", 2450 RTE_ALIGN(sizeof(struct test_op_params), 2451 RTE_CACHE_LINE_SIZE)); 2452 2453 /* For each device run test case function */ 2454 for (dev = 0; dev < nb_active_devs; ++dev) 2455 ret |= run_test_case_on_device(test_case_func, dev, op_params); 2456 2457 rte_free(op_params); 2458 2459 return ret; 2460 } 2461 2462 2463 /* Push back the HARQ output from DDR to host */ 2464 static void 2465 retrieve_harq_ddr(uint16_t dev_id, uint16_t queue_id, 2466 struct rte_bbdev_dec_op **ops, 2467 const uint16_t n) 2468 { 2469 uint16_t j; 2470 int save_status, ret; 2471 uint32_t harq_offset = (uint32_t) queue_id * HARQ_INCR * MAX_OPS; 2472 struct rte_bbdev_dec_op *ops_deq[MAX_BURST]; 2473 uint32_t flags = ops[0]->ldpc_dec.op_flags; 2474 bool loopback = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK; 2475 bool mem_out = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE; 2476 bool hc_out = flags & RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE; 2477 bool h_comp = flags & RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION; 2478 for (j = 0; j < n; ++j) { 2479 if ((loopback && mem_out) || hc_out) { 2480 save_status = ops[j]->status; 2481 ops[j]->ldpc_dec.op_flags = 2482 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK + 2483 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE; 2484 if (h_comp) 2485 ops[j]->ldpc_dec.op_flags += 2486 RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION; 2487 ops[j]->ldpc_dec.harq_combined_input.offset = 2488 harq_offset; 2489 ops[j]->ldpc_dec.harq_combined_output.offset = 0; 2490 harq_offset += HARQ_INCR; 2491 if (!loopback) 2492 ops[j]->ldpc_dec.harq_combined_input.length = 2493 ops[j]->ldpc_dec.harq_combined_output.length; 2494 rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id, 2495 &ops[j], 1); 2496 ret = 0; 2497 while (ret == 0) 2498 ret = rte_bbdev_dequeue_ldpc_dec_ops( 2499 dev_id, queue_id, 2500 &ops_deq[j], 1); 2501 ops[j]->ldpc_dec.op_flags = flags; 2502 ops[j]->status = save_status; 2503 } 2504 } 2505 } 2506 2507 /* 2508 * Push back the HARQ output from HW DDR to Host 2509 * Preload HARQ memory input and adjust HARQ offset 2510 */ 2511 static void 2512 preload_harq_ddr(uint16_t dev_id, uint16_t queue_id, 2513 struct rte_bbdev_dec_op **ops, const uint16_t n, 2514 bool preload) 2515 { 2516 uint16_t j; 2517 int deq; 2518 uint32_t harq_offset = (uint32_t) queue_id * HARQ_INCR * MAX_OPS; 2519 struct rte_bbdev_op_data save_hc_in[MAX_OPS], save_hc_out[MAX_OPS]; 2520 struct rte_bbdev_dec_op *ops_deq[MAX_OPS]; 2521 uint32_t flags = ops[0]->ldpc_dec.op_flags; 2522 bool mem_in = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE; 2523 bool hc_in = flags & RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE; 2524 bool mem_out = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE; 2525 bool hc_out = flags & RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE; 2526 bool h_comp = flags & RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION; 2527 if ((mem_in || hc_in) && preload) { 2528 for (j = 0; j < n; ++j) { 2529 save_hc_in[j] = ops[j]->ldpc_dec.harq_combined_input; 2530 save_hc_out[j] = ops[j]->ldpc_dec.harq_combined_output; 2531 ops[j]->ldpc_dec.op_flags = 2532 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK + 2533 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE; 2534 if (h_comp) 2535 ops[j]->ldpc_dec.op_flags += 2536 RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION; 2537 ops[j]->ldpc_dec.harq_combined_output.offset = 2538 harq_offset; 2539 ops[j]->ldpc_dec.harq_combined_input.offset = 0; 2540 harq_offset += HARQ_INCR; 2541 } 2542 rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id, &ops[0], n); 2543 deq = 0; 2544 while (deq != n) 2545 deq += rte_bbdev_dequeue_ldpc_dec_ops( 2546 dev_id, queue_id, &ops_deq[deq], 2547 n - deq); 2548 /* Restore the operations */ 2549 for (j = 0; j < n; ++j) { 2550 ops[j]->ldpc_dec.op_flags = flags; 2551 ops[j]->ldpc_dec.harq_combined_input = save_hc_in[j]; 2552 ops[j]->ldpc_dec.harq_combined_output = save_hc_out[j]; 2553 } 2554 } 2555 harq_offset = (uint32_t) queue_id * HARQ_INCR * MAX_OPS; 2556 for (j = 0; j < n; ++j) { 2557 /* Adjust HARQ offset when we reach external DDR */ 2558 if (mem_in || hc_in) 2559 ops[j]->ldpc_dec.harq_combined_input.offset 2560 = harq_offset; 2561 if (mem_out || hc_out) 2562 ops[j]->ldpc_dec.harq_combined_output.offset 2563 = harq_offset; 2564 harq_offset += HARQ_INCR; 2565 } 2566 } 2567 2568 static void 2569 dequeue_event_callback(uint16_t dev_id, 2570 enum rte_bbdev_event_type event, void *cb_arg, 2571 void *ret_param) 2572 { 2573 int ret; 2574 uint16_t i; 2575 uint64_t total_time; 2576 uint16_t deq, burst_sz, num_ops; 2577 uint16_t queue_id = *(uint16_t *) ret_param; 2578 struct rte_bbdev_info info; 2579 double tb_len_bits; 2580 struct thread_params *tp = cb_arg; 2581 2582 /* Find matching thread params using queue_id */ 2583 for (i = 0; i < MAX_QUEUES; ++i, ++tp) 2584 if (tp->queue_id == queue_id) 2585 break; 2586 2587 if (i == MAX_QUEUES) { 2588 printf("%s: Queue_id from interrupt details was not found!\n", 2589 __func__); 2590 return; 2591 } 2592 2593 if (unlikely(event != RTE_BBDEV_EVENT_DEQUEUE)) { 2594 rte_atomic16_set(&tp->processing_status, TEST_FAILED); 2595 printf( 2596 "Dequeue interrupt handler called for incorrect event!\n"); 2597 return; 2598 } 2599 2600 burst_sz = rte_atomic16_read(&tp->burst_sz); 2601 num_ops = tp->op_params->num_to_process; 2602 2603 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) 2604 deq = rte_bbdev_dequeue_dec_ops(dev_id, queue_id, 2605 &tp->dec_ops[ 2606 rte_atomic16_read(&tp->nb_dequeued)], 2607 burst_sz); 2608 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 2609 deq = rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id, 2610 &tp->dec_ops[ 2611 rte_atomic16_read(&tp->nb_dequeued)], 2612 burst_sz); 2613 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) 2614 deq = rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id, 2615 &tp->enc_ops[ 2616 rte_atomic16_read(&tp->nb_dequeued)], 2617 burst_sz); 2618 else /*RTE_BBDEV_OP_TURBO_ENC*/ 2619 deq = rte_bbdev_dequeue_enc_ops(dev_id, queue_id, 2620 &tp->enc_ops[ 2621 rte_atomic16_read(&tp->nb_dequeued)], 2622 burst_sz); 2623 2624 if (deq < burst_sz) { 2625 printf( 2626 "After receiving the interrupt all operations should be dequeued. Expected: %u, got: %u\n", 2627 burst_sz, deq); 2628 rte_atomic16_set(&tp->processing_status, TEST_FAILED); 2629 return; 2630 } 2631 2632 if (rte_atomic16_read(&tp->nb_dequeued) + deq < num_ops) { 2633 rte_atomic16_add(&tp->nb_dequeued, deq); 2634 return; 2635 } 2636 2637 total_time = rte_rdtsc_precise() - tp->start_time; 2638 2639 rte_bbdev_info_get(dev_id, &info); 2640 2641 ret = TEST_SUCCESS; 2642 2643 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) { 2644 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 2645 ret = validate_dec_op(tp->dec_ops, num_ops, ref_op, 2646 tp->op_params->vector_mask); 2647 /* get the max of iter_count for all dequeued ops */ 2648 for (i = 0; i < num_ops; ++i) 2649 tp->iter_count = RTE_MAX( 2650 tp->dec_ops[i]->turbo_dec.iter_count, 2651 tp->iter_count); 2652 rte_bbdev_dec_op_free_bulk(tp->dec_ops, deq); 2653 } else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) { 2654 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op; 2655 ret = validate_enc_op(tp->enc_ops, num_ops, ref_op); 2656 rte_bbdev_enc_op_free_bulk(tp->enc_ops, deq); 2657 } else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) { 2658 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op; 2659 ret = validate_ldpc_enc_op(tp->enc_ops, num_ops, ref_op); 2660 rte_bbdev_enc_op_free_bulk(tp->enc_ops, deq); 2661 } else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) { 2662 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 2663 ret = validate_ldpc_dec_op(tp->dec_ops, num_ops, ref_op, 2664 tp->op_params->vector_mask); 2665 rte_bbdev_dec_op_free_bulk(tp->dec_ops, deq); 2666 } 2667 2668 if (ret) { 2669 printf("Buffers validation failed\n"); 2670 rte_atomic16_set(&tp->processing_status, TEST_FAILED); 2671 } 2672 2673 switch (test_vector.op_type) { 2674 case RTE_BBDEV_OP_TURBO_DEC: 2675 tb_len_bits = calc_dec_TB_size(tp->op_params->ref_dec_op); 2676 break; 2677 case RTE_BBDEV_OP_TURBO_ENC: 2678 tb_len_bits = calc_enc_TB_size(tp->op_params->ref_enc_op); 2679 break; 2680 case RTE_BBDEV_OP_LDPC_DEC: 2681 tb_len_bits = calc_ldpc_dec_TB_size(tp->op_params->ref_dec_op); 2682 break; 2683 case RTE_BBDEV_OP_LDPC_ENC: 2684 tb_len_bits = calc_ldpc_enc_TB_size(tp->op_params->ref_enc_op); 2685 break; 2686 case RTE_BBDEV_OP_NONE: 2687 tb_len_bits = 0.0; 2688 break; 2689 default: 2690 printf("Unknown op type: %d\n", test_vector.op_type); 2691 rte_atomic16_set(&tp->processing_status, TEST_FAILED); 2692 return; 2693 } 2694 2695 tp->ops_per_sec += ((double)num_ops) / 2696 ((double)total_time / (double)rte_get_tsc_hz()); 2697 tp->mbps += (((double)(num_ops * tb_len_bits)) / 1000000.0) / 2698 ((double)total_time / (double)rte_get_tsc_hz()); 2699 2700 rte_atomic16_add(&tp->nb_dequeued, deq); 2701 } 2702 2703 static int 2704 throughput_intr_lcore_ldpc_dec(void *arg) 2705 { 2706 struct thread_params *tp = arg; 2707 unsigned int enqueued; 2708 const uint16_t queue_id = tp->queue_id; 2709 const uint16_t burst_sz = tp->op_params->burst_sz; 2710 const uint16_t num_to_process = tp->op_params->num_to_process; 2711 struct rte_bbdev_dec_op *ops[num_to_process]; 2712 struct test_buffers *bufs = NULL; 2713 struct rte_bbdev_info info; 2714 int ret, i, j; 2715 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 2716 uint16_t num_to_enq, enq; 2717 2718 bool loopback = check_bit(ref_op->ldpc_dec.op_flags, 2719 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK); 2720 bool hc_out = check_bit(ref_op->ldpc_dec.op_flags, 2721 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE); 2722 2723 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 2724 "BURST_SIZE should be <= %u", MAX_BURST); 2725 2726 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id), 2727 "Failed to enable interrupts for dev: %u, queue_id: %u", 2728 tp->dev_id, queue_id); 2729 2730 rte_bbdev_info_get(tp->dev_id, &info); 2731 2732 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim), 2733 "NUM_OPS cannot exceed %u for this device", 2734 info.drv.queue_size_lim); 2735 2736 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 2737 2738 rte_atomic16_clear(&tp->processing_status); 2739 rte_atomic16_clear(&tp->nb_dequeued); 2740 2741 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT) 2742 rte_pause(); 2743 2744 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops, 2745 num_to_process); 2746 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", 2747 num_to_process); 2748 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 2749 copy_reference_ldpc_dec_op(ops, num_to_process, 0, bufs->inputs, 2750 bufs->hard_outputs, bufs->soft_outputs, 2751 bufs->harq_inputs, bufs->harq_outputs, ref_op); 2752 2753 /* Set counter to validate the ordering */ 2754 for (j = 0; j < num_to_process; ++j) 2755 ops[j]->opaque_data = (void *)(uintptr_t)j; 2756 2757 for (j = 0; j < TEST_REPETITIONS; ++j) { 2758 for (i = 0; i < num_to_process; ++i) { 2759 if (!loopback) 2760 rte_pktmbuf_reset( 2761 ops[i]->ldpc_dec.hard_output.data); 2762 if (hc_out || loopback) 2763 mbuf_reset( 2764 ops[i]->ldpc_dec.harq_combined_output.data); 2765 } 2766 2767 tp->start_time = rte_rdtsc_precise(); 2768 for (enqueued = 0; enqueued < num_to_process;) { 2769 num_to_enq = burst_sz; 2770 2771 if (unlikely(num_to_process - enqueued < num_to_enq)) 2772 num_to_enq = num_to_process - enqueued; 2773 2774 enq = 0; 2775 do { 2776 enq += rte_bbdev_enqueue_ldpc_dec_ops( 2777 tp->dev_id, 2778 queue_id, &ops[enqueued], 2779 num_to_enq); 2780 } while (unlikely(num_to_enq != enq)); 2781 enqueued += enq; 2782 2783 /* Write to thread burst_sz current number of enqueued 2784 * descriptors. It ensures that proper number of 2785 * descriptors will be dequeued in callback 2786 * function - needed for last batch in case where 2787 * the number of operations is not a multiple of 2788 * burst size. 2789 */ 2790 rte_atomic16_set(&tp->burst_sz, num_to_enq); 2791 2792 /* Wait until processing of previous batch is 2793 * completed 2794 */ 2795 while (rte_atomic16_read(&tp->nb_dequeued) != 2796 (int16_t) enqueued) 2797 rte_pause(); 2798 } 2799 if (j != TEST_REPETITIONS - 1) 2800 rte_atomic16_clear(&tp->nb_dequeued); 2801 } 2802 2803 return TEST_SUCCESS; 2804 } 2805 2806 static int 2807 throughput_intr_lcore_dec(void *arg) 2808 { 2809 struct thread_params *tp = arg; 2810 unsigned int enqueued; 2811 const uint16_t queue_id = tp->queue_id; 2812 const uint16_t burst_sz = tp->op_params->burst_sz; 2813 const uint16_t num_to_process = tp->op_params->num_to_process; 2814 struct rte_bbdev_dec_op *ops[num_to_process]; 2815 struct test_buffers *bufs = NULL; 2816 struct rte_bbdev_info info; 2817 int ret, i, j; 2818 uint16_t num_to_enq, enq; 2819 2820 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 2821 "BURST_SIZE should be <= %u", MAX_BURST); 2822 2823 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id), 2824 "Failed to enable interrupts for dev: %u, queue_id: %u", 2825 tp->dev_id, queue_id); 2826 2827 rte_bbdev_info_get(tp->dev_id, &info); 2828 2829 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim), 2830 "NUM_OPS cannot exceed %u for this device", 2831 info.drv.queue_size_lim); 2832 2833 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 2834 2835 rte_atomic16_clear(&tp->processing_status); 2836 rte_atomic16_clear(&tp->nb_dequeued); 2837 2838 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT) 2839 rte_pause(); 2840 2841 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops, 2842 num_to_process); 2843 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", 2844 num_to_process); 2845 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 2846 copy_reference_dec_op(ops, num_to_process, 0, bufs->inputs, 2847 bufs->hard_outputs, bufs->soft_outputs, 2848 tp->op_params->ref_dec_op); 2849 2850 /* Set counter to validate the ordering */ 2851 for (j = 0; j < num_to_process; ++j) 2852 ops[j]->opaque_data = (void *)(uintptr_t)j; 2853 2854 for (j = 0; j < TEST_REPETITIONS; ++j) { 2855 for (i = 0; i < num_to_process; ++i) 2856 rte_pktmbuf_reset(ops[i]->turbo_dec.hard_output.data); 2857 2858 tp->start_time = rte_rdtsc_precise(); 2859 for (enqueued = 0; enqueued < num_to_process;) { 2860 num_to_enq = burst_sz; 2861 2862 if (unlikely(num_to_process - enqueued < num_to_enq)) 2863 num_to_enq = num_to_process - enqueued; 2864 2865 enq = 0; 2866 do { 2867 enq += rte_bbdev_enqueue_dec_ops(tp->dev_id, 2868 queue_id, &ops[enqueued], 2869 num_to_enq); 2870 } while (unlikely(num_to_enq != enq)); 2871 enqueued += enq; 2872 2873 /* Write to thread burst_sz current number of enqueued 2874 * descriptors. It ensures that proper number of 2875 * descriptors will be dequeued in callback 2876 * function - needed for last batch in case where 2877 * the number of operations is not a multiple of 2878 * burst size. 2879 */ 2880 rte_atomic16_set(&tp->burst_sz, num_to_enq); 2881 2882 /* Wait until processing of previous batch is 2883 * completed 2884 */ 2885 while (rte_atomic16_read(&tp->nb_dequeued) != 2886 (int16_t) enqueued) 2887 rte_pause(); 2888 } 2889 if (j != TEST_REPETITIONS - 1) 2890 rte_atomic16_clear(&tp->nb_dequeued); 2891 } 2892 2893 return TEST_SUCCESS; 2894 } 2895 2896 static int 2897 throughput_intr_lcore_enc(void *arg) 2898 { 2899 struct thread_params *tp = arg; 2900 unsigned int enqueued; 2901 const uint16_t queue_id = tp->queue_id; 2902 const uint16_t burst_sz = tp->op_params->burst_sz; 2903 const uint16_t num_to_process = tp->op_params->num_to_process; 2904 struct rte_bbdev_enc_op *ops[num_to_process]; 2905 struct test_buffers *bufs = NULL; 2906 struct rte_bbdev_info info; 2907 int ret, i, j; 2908 uint16_t num_to_enq, enq; 2909 2910 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 2911 "BURST_SIZE should be <= %u", MAX_BURST); 2912 2913 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id), 2914 "Failed to enable interrupts for dev: %u, queue_id: %u", 2915 tp->dev_id, queue_id); 2916 2917 rte_bbdev_info_get(tp->dev_id, &info); 2918 2919 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim), 2920 "NUM_OPS cannot exceed %u for this device", 2921 info.drv.queue_size_lim); 2922 2923 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 2924 2925 rte_atomic16_clear(&tp->processing_status); 2926 rte_atomic16_clear(&tp->nb_dequeued); 2927 2928 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT) 2929 rte_pause(); 2930 2931 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops, 2932 num_to_process); 2933 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", 2934 num_to_process); 2935 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 2936 copy_reference_enc_op(ops, num_to_process, 0, bufs->inputs, 2937 bufs->hard_outputs, tp->op_params->ref_enc_op); 2938 2939 /* Set counter to validate the ordering */ 2940 for (j = 0; j < num_to_process; ++j) 2941 ops[j]->opaque_data = (void *)(uintptr_t)j; 2942 2943 for (j = 0; j < TEST_REPETITIONS; ++j) { 2944 for (i = 0; i < num_to_process; ++i) 2945 rte_pktmbuf_reset(ops[i]->turbo_enc.output.data); 2946 2947 tp->start_time = rte_rdtsc_precise(); 2948 for (enqueued = 0; enqueued < num_to_process;) { 2949 num_to_enq = burst_sz; 2950 2951 if (unlikely(num_to_process - enqueued < num_to_enq)) 2952 num_to_enq = num_to_process - enqueued; 2953 2954 enq = 0; 2955 do { 2956 enq += rte_bbdev_enqueue_enc_ops(tp->dev_id, 2957 queue_id, &ops[enqueued], 2958 num_to_enq); 2959 } while (unlikely(enq != num_to_enq)); 2960 enqueued += enq; 2961 2962 /* Write to thread burst_sz current number of enqueued 2963 * descriptors. It ensures that proper number of 2964 * descriptors will be dequeued in callback 2965 * function - needed for last batch in case where 2966 * the number of operations is not a multiple of 2967 * burst size. 2968 */ 2969 rte_atomic16_set(&tp->burst_sz, num_to_enq); 2970 2971 /* Wait until processing of previous batch is 2972 * completed 2973 */ 2974 while (rte_atomic16_read(&tp->nb_dequeued) != 2975 (int16_t) enqueued) 2976 rte_pause(); 2977 } 2978 if (j != TEST_REPETITIONS - 1) 2979 rte_atomic16_clear(&tp->nb_dequeued); 2980 } 2981 2982 return TEST_SUCCESS; 2983 } 2984 2985 2986 static int 2987 throughput_intr_lcore_ldpc_enc(void *arg) 2988 { 2989 struct thread_params *tp = arg; 2990 unsigned int enqueued; 2991 const uint16_t queue_id = tp->queue_id; 2992 const uint16_t burst_sz = tp->op_params->burst_sz; 2993 const uint16_t num_to_process = tp->op_params->num_to_process; 2994 struct rte_bbdev_enc_op *ops[num_to_process]; 2995 struct test_buffers *bufs = NULL; 2996 struct rte_bbdev_info info; 2997 int ret, i, j; 2998 uint16_t num_to_enq, enq; 2999 3000 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 3001 "BURST_SIZE should be <= %u", MAX_BURST); 3002 3003 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id), 3004 "Failed to enable interrupts for dev: %u, queue_id: %u", 3005 tp->dev_id, queue_id); 3006 3007 rte_bbdev_info_get(tp->dev_id, &info); 3008 3009 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim), 3010 "NUM_OPS cannot exceed %u for this device", 3011 info.drv.queue_size_lim); 3012 3013 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 3014 3015 rte_atomic16_clear(&tp->processing_status); 3016 rte_atomic16_clear(&tp->nb_dequeued); 3017 3018 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT) 3019 rte_pause(); 3020 3021 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops, 3022 num_to_process); 3023 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", 3024 num_to_process); 3025 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3026 copy_reference_ldpc_enc_op(ops, num_to_process, 0, 3027 bufs->inputs, bufs->hard_outputs, 3028 tp->op_params->ref_enc_op); 3029 3030 /* Set counter to validate the ordering */ 3031 for (j = 0; j < num_to_process; ++j) 3032 ops[j]->opaque_data = (void *)(uintptr_t)j; 3033 3034 for (j = 0; j < TEST_REPETITIONS; ++j) { 3035 for (i = 0; i < num_to_process; ++i) 3036 rte_pktmbuf_reset(ops[i]->turbo_enc.output.data); 3037 3038 tp->start_time = rte_rdtsc_precise(); 3039 for (enqueued = 0; enqueued < num_to_process;) { 3040 num_to_enq = burst_sz; 3041 3042 if (unlikely(num_to_process - enqueued < num_to_enq)) 3043 num_to_enq = num_to_process - enqueued; 3044 3045 enq = 0; 3046 do { 3047 enq += rte_bbdev_enqueue_ldpc_enc_ops( 3048 tp->dev_id, 3049 queue_id, &ops[enqueued], 3050 num_to_enq); 3051 } while (unlikely(enq != num_to_enq)); 3052 enqueued += enq; 3053 3054 /* Write to thread burst_sz current number of enqueued 3055 * descriptors. It ensures that proper number of 3056 * descriptors will be dequeued in callback 3057 * function - needed for last batch in case where 3058 * the number of operations is not a multiple of 3059 * burst size. 3060 */ 3061 rte_atomic16_set(&tp->burst_sz, num_to_enq); 3062 3063 /* Wait until processing of previous batch is 3064 * completed 3065 */ 3066 while (rte_atomic16_read(&tp->nb_dequeued) != 3067 (int16_t) enqueued) 3068 rte_pause(); 3069 } 3070 if (j != TEST_REPETITIONS - 1) 3071 rte_atomic16_clear(&tp->nb_dequeued); 3072 } 3073 3074 return TEST_SUCCESS; 3075 } 3076 3077 static int 3078 throughput_pmd_lcore_dec(void *arg) 3079 { 3080 struct thread_params *tp = arg; 3081 uint16_t enq, deq; 3082 uint64_t total_time = 0, start_time; 3083 const uint16_t queue_id = tp->queue_id; 3084 const uint16_t burst_sz = tp->op_params->burst_sz; 3085 const uint16_t num_ops = tp->op_params->num_to_process; 3086 struct rte_bbdev_dec_op *ops_enq[num_ops]; 3087 struct rte_bbdev_dec_op *ops_deq[num_ops]; 3088 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 3089 struct test_buffers *bufs = NULL; 3090 int i, j, ret; 3091 struct rte_bbdev_info info; 3092 uint16_t num_to_enq; 3093 3094 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 3095 "BURST_SIZE should be <= %u", MAX_BURST); 3096 3097 rte_bbdev_info_get(tp->dev_id, &info); 3098 3099 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 3100 "NUM_OPS cannot exceed %u for this device", 3101 info.drv.queue_size_lim); 3102 3103 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 3104 3105 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT) 3106 rte_pause(); 3107 3108 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops); 3109 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops); 3110 3111 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3112 copy_reference_dec_op(ops_enq, num_ops, 0, bufs->inputs, 3113 bufs->hard_outputs, bufs->soft_outputs, ref_op); 3114 3115 /* Set counter to validate the ordering */ 3116 for (j = 0; j < num_ops; ++j) 3117 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 3118 3119 for (i = 0; i < TEST_REPETITIONS; ++i) { 3120 3121 for (j = 0; j < num_ops; ++j) 3122 mbuf_reset(ops_enq[j]->turbo_dec.hard_output.data); 3123 3124 start_time = rte_rdtsc_precise(); 3125 3126 for (enq = 0, deq = 0; enq < num_ops;) { 3127 num_to_enq = burst_sz; 3128 3129 if (unlikely(num_ops - enq < num_to_enq)) 3130 num_to_enq = num_ops - enq; 3131 3132 enq += rte_bbdev_enqueue_dec_ops(tp->dev_id, 3133 queue_id, &ops_enq[enq], num_to_enq); 3134 3135 deq += rte_bbdev_dequeue_dec_ops(tp->dev_id, 3136 queue_id, &ops_deq[deq], enq - deq); 3137 } 3138 3139 /* dequeue the remaining */ 3140 while (deq < enq) { 3141 deq += rte_bbdev_dequeue_dec_ops(tp->dev_id, 3142 queue_id, &ops_deq[deq], enq - deq); 3143 } 3144 3145 total_time += rte_rdtsc_precise() - start_time; 3146 } 3147 3148 tp->iter_count = 0; 3149 /* get the max of iter_count for all dequeued ops */ 3150 for (i = 0; i < num_ops; ++i) { 3151 tp->iter_count = RTE_MAX(ops_enq[i]->turbo_dec.iter_count, 3152 tp->iter_count); 3153 } 3154 3155 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 3156 ret = validate_dec_op(ops_deq, num_ops, ref_op, 3157 tp->op_params->vector_mask); 3158 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 3159 } 3160 3161 rte_bbdev_dec_op_free_bulk(ops_enq, num_ops); 3162 3163 double tb_len_bits = calc_dec_TB_size(ref_op); 3164 3165 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) / 3166 ((double)total_time / (double)rte_get_tsc_hz()); 3167 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) / 3168 1000000.0) / ((double)total_time / 3169 (double)rte_get_tsc_hz()); 3170 3171 return TEST_SUCCESS; 3172 } 3173 3174 static int 3175 bler_pmd_lcore_ldpc_dec(void *arg) 3176 { 3177 struct thread_params *tp = arg; 3178 uint16_t enq, deq; 3179 uint64_t total_time = 0, start_time; 3180 const uint16_t queue_id = tp->queue_id; 3181 const uint16_t burst_sz = tp->op_params->burst_sz; 3182 const uint16_t num_ops = tp->op_params->num_to_process; 3183 struct rte_bbdev_dec_op *ops_enq[num_ops]; 3184 struct rte_bbdev_dec_op *ops_deq[num_ops]; 3185 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 3186 struct test_buffers *bufs = NULL; 3187 int i, j, ret; 3188 float parity_bler = 0; 3189 struct rte_bbdev_info info; 3190 uint16_t num_to_enq; 3191 bool extDdr = check_bit(ldpc_cap_flags, 3192 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE); 3193 bool loopback = check_bit(ref_op->ldpc_dec.op_flags, 3194 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK); 3195 bool hc_out = check_bit(ref_op->ldpc_dec.op_flags, 3196 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE); 3197 3198 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 3199 "BURST_SIZE should be <= %u", MAX_BURST); 3200 3201 rte_bbdev_info_get(tp->dev_id, &info); 3202 3203 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 3204 "NUM_OPS cannot exceed %u for this device", 3205 info.drv.queue_size_lim); 3206 3207 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 3208 3209 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT) 3210 rte_pause(); 3211 3212 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops); 3213 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops); 3214 3215 /* For BLER tests we need to enable early termination */ 3216 if (!check_bit(ref_op->ldpc_dec.op_flags, 3217 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE)) 3218 ref_op->ldpc_dec.op_flags += 3219 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE; 3220 ref_op->ldpc_dec.iter_max = get_iter_max(); 3221 ref_op->ldpc_dec.iter_count = ref_op->ldpc_dec.iter_max; 3222 3223 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3224 copy_reference_ldpc_dec_op(ops_enq, num_ops, 0, bufs->inputs, 3225 bufs->hard_outputs, bufs->soft_outputs, 3226 bufs->harq_inputs, bufs->harq_outputs, ref_op); 3227 generate_llr_input(num_ops, bufs->inputs, ref_op); 3228 3229 /* Set counter to validate the ordering */ 3230 for (j = 0; j < num_ops; ++j) 3231 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 3232 3233 for (i = 0; i < 1; ++i) { /* Could add more iterations */ 3234 for (j = 0; j < num_ops; ++j) { 3235 if (!loopback) 3236 mbuf_reset( 3237 ops_enq[j]->ldpc_dec.hard_output.data); 3238 if (hc_out || loopback) 3239 mbuf_reset( 3240 ops_enq[j]->ldpc_dec.harq_combined_output.data); 3241 } 3242 if (extDdr) 3243 preload_harq_ddr(tp->dev_id, queue_id, ops_enq, 3244 num_ops, true); 3245 start_time = rte_rdtsc_precise(); 3246 3247 for (enq = 0, deq = 0; enq < num_ops;) { 3248 num_to_enq = burst_sz; 3249 3250 if (unlikely(num_ops - enq < num_to_enq)) 3251 num_to_enq = num_ops - enq; 3252 3253 enq += rte_bbdev_enqueue_ldpc_dec_ops(tp->dev_id, 3254 queue_id, &ops_enq[enq], num_to_enq); 3255 3256 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id, 3257 queue_id, &ops_deq[deq], enq - deq); 3258 } 3259 3260 /* dequeue the remaining */ 3261 while (deq < enq) { 3262 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id, 3263 queue_id, &ops_deq[deq], enq - deq); 3264 } 3265 3266 total_time += rte_rdtsc_precise() - start_time; 3267 } 3268 3269 tp->iter_count = 0; 3270 tp->iter_average = 0; 3271 /* get the max of iter_count for all dequeued ops */ 3272 for (i = 0; i < num_ops; ++i) { 3273 tp->iter_count = RTE_MAX(ops_enq[i]->ldpc_dec.iter_count, 3274 tp->iter_count); 3275 tp->iter_average += (double) ops_enq[i]->ldpc_dec.iter_count; 3276 if (ops_enq[i]->status & (1 << RTE_BBDEV_SYNDROME_ERROR)) 3277 parity_bler += 1.0; 3278 } 3279 3280 parity_bler /= num_ops; /* This one is based on SYND */ 3281 tp->iter_average /= num_ops; 3282 tp->bler = (double) validate_ldpc_bler(ops_deq, num_ops) / num_ops; 3283 3284 if (test_vector.op_type != RTE_BBDEV_OP_NONE 3285 && tp->bler == 0 3286 && parity_bler == 0 3287 && !hc_out) { 3288 ret = validate_ldpc_dec_op(ops_deq, num_ops, ref_op, 3289 tp->op_params->vector_mask); 3290 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 3291 } 3292 3293 rte_bbdev_dec_op_free_bulk(ops_enq, num_ops); 3294 3295 double tb_len_bits = calc_ldpc_dec_TB_size(ref_op); 3296 tp->ops_per_sec = ((double)num_ops * 1) / 3297 ((double)total_time / (double)rte_get_tsc_hz()); 3298 tp->mbps = (((double)(num_ops * 1 * tb_len_bits)) / 3299 1000000.0) / ((double)total_time / 3300 (double)rte_get_tsc_hz()); 3301 3302 return TEST_SUCCESS; 3303 } 3304 3305 static int 3306 throughput_pmd_lcore_ldpc_dec(void *arg) 3307 { 3308 struct thread_params *tp = arg; 3309 uint16_t enq, deq; 3310 uint64_t total_time = 0, start_time; 3311 const uint16_t queue_id = tp->queue_id; 3312 const uint16_t burst_sz = tp->op_params->burst_sz; 3313 const uint16_t num_ops = tp->op_params->num_to_process; 3314 struct rte_bbdev_dec_op *ops_enq[num_ops]; 3315 struct rte_bbdev_dec_op *ops_deq[num_ops]; 3316 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 3317 struct test_buffers *bufs = NULL; 3318 int i, j, ret; 3319 struct rte_bbdev_info info; 3320 uint16_t num_to_enq; 3321 bool extDdr = check_bit(ldpc_cap_flags, 3322 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE); 3323 bool loopback = check_bit(ref_op->ldpc_dec.op_flags, 3324 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK); 3325 bool hc_out = check_bit(ref_op->ldpc_dec.op_flags, 3326 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE); 3327 3328 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 3329 "BURST_SIZE should be <= %u", MAX_BURST); 3330 3331 rte_bbdev_info_get(tp->dev_id, &info); 3332 3333 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 3334 "NUM_OPS cannot exceed %u for this device", 3335 info.drv.queue_size_lim); 3336 3337 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 3338 3339 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT) 3340 rte_pause(); 3341 3342 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops); 3343 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops); 3344 3345 /* For throughput tests we need to disable early termination */ 3346 if (check_bit(ref_op->ldpc_dec.op_flags, 3347 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE)) 3348 ref_op->ldpc_dec.op_flags -= 3349 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE; 3350 ref_op->ldpc_dec.iter_max = get_iter_max(); 3351 ref_op->ldpc_dec.iter_count = ref_op->ldpc_dec.iter_max; 3352 3353 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3354 copy_reference_ldpc_dec_op(ops_enq, num_ops, 0, bufs->inputs, 3355 bufs->hard_outputs, bufs->soft_outputs, 3356 bufs->harq_inputs, bufs->harq_outputs, ref_op); 3357 3358 /* Set counter to validate the ordering */ 3359 for (j = 0; j < num_ops; ++j) 3360 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 3361 3362 for (i = 0; i < TEST_REPETITIONS; ++i) { 3363 for (j = 0; j < num_ops; ++j) { 3364 if (!loopback) 3365 mbuf_reset( 3366 ops_enq[j]->ldpc_dec.hard_output.data); 3367 if (hc_out || loopback) 3368 mbuf_reset( 3369 ops_enq[j]->ldpc_dec.harq_combined_output.data); 3370 } 3371 if (extDdr) 3372 preload_harq_ddr(tp->dev_id, queue_id, ops_enq, 3373 num_ops, true); 3374 start_time = rte_rdtsc_precise(); 3375 3376 for (enq = 0, deq = 0; enq < num_ops;) { 3377 num_to_enq = burst_sz; 3378 3379 if (unlikely(num_ops - enq < num_to_enq)) 3380 num_to_enq = num_ops - enq; 3381 3382 enq += rte_bbdev_enqueue_ldpc_dec_ops(tp->dev_id, 3383 queue_id, &ops_enq[enq], num_to_enq); 3384 3385 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id, 3386 queue_id, &ops_deq[deq], enq - deq); 3387 } 3388 3389 /* dequeue the remaining */ 3390 while (deq < enq) { 3391 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id, 3392 queue_id, &ops_deq[deq], enq - deq); 3393 } 3394 3395 total_time += rte_rdtsc_precise() - start_time; 3396 } 3397 3398 tp->iter_count = 0; 3399 /* get the max of iter_count for all dequeued ops */ 3400 for (i = 0; i < num_ops; ++i) { 3401 tp->iter_count = RTE_MAX(ops_enq[i]->ldpc_dec.iter_count, 3402 tp->iter_count); 3403 } 3404 if (extDdr) { 3405 /* Read loopback is not thread safe */ 3406 retrieve_harq_ddr(tp->dev_id, queue_id, ops_enq, num_ops); 3407 } 3408 3409 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 3410 ret = validate_ldpc_dec_op(ops_deq, num_ops, ref_op, 3411 tp->op_params->vector_mask); 3412 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 3413 } 3414 3415 rte_bbdev_dec_op_free_bulk(ops_enq, num_ops); 3416 3417 double tb_len_bits = calc_ldpc_dec_TB_size(ref_op); 3418 3419 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) / 3420 ((double)total_time / (double)rte_get_tsc_hz()); 3421 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) / 3422 1000000.0) / ((double)total_time / 3423 (double)rte_get_tsc_hz()); 3424 3425 return TEST_SUCCESS; 3426 } 3427 3428 static int 3429 throughput_pmd_lcore_enc(void *arg) 3430 { 3431 struct thread_params *tp = arg; 3432 uint16_t enq, deq; 3433 uint64_t total_time = 0, start_time; 3434 const uint16_t queue_id = tp->queue_id; 3435 const uint16_t burst_sz = tp->op_params->burst_sz; 3436 const uint16_t num_ops = tp->op_params->num_to_process; 3437 struct rte_bbdev_enc_op *ops_enq[num_ops]; 3438 struct rte_bbdev_enc_op *ops_deq[num_ops]; 3439 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op; 3440 struct test_buffers *bufs = NULL; 3441 int i, j, ret; 3442 struct rte_bbdev_info info; 3443 uint16_t num_to_enq; 3444 3445 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 3446 "BURST_SIZE should be <= %u", MAX_BURST); 3447 3448 rte_bbdev_info_get(tp->dev_id, &info); 3449 3450 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 3451 "NUM_OPS cannot exceed %u for this device", 3452 info.drv.queue_size_lim); 3453 3454 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 3455 3456 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT) 3457 rte_pause(); 3458 3459 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops_enq, 3460 num_ops); 3461 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", 3462 num_ops); 3463 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3464 copy_reference_enc_op(ops_enq, num_ops, 0, bufs->inputs, 3465 bufs->hard_outputs, ref_op); 3466 3467 /* Set counter to validate the ordering */ 3468 for (j = 0; j < num_ops; ++j) 3469 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 3470 3471 for (i = 0; i < TEST_REPETITIONS; ++i) { 3472 3473 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3474 for (j = 0; j < num_ops; ++j) 3475 mbuf_reset(ops_enq[j]->turbo_enc.output.data); 3476 3477 start_time = rte_rdtsc_precise(); 3478 3479 for (enq = 0, deq = 0; enq < num_ops;) { 3480 num_to_enq = burst_sz; 3481 3482 if (unlikely(num_ops - enq < num_to_enq)) 3483 num_to_enq = num_ops - enq; 3484 3485 enq += rte_bbdev_enqueue_enc_ops(tp->dev_id, 3486 queue_id, &ops_enq[enq], num_to_enq); 3487 3488 deq += rte_bbdev_dequeue_enc_ops(tp->dev_id, 3489 queue_id, &ops_deq[deq], enq - deq); 3490 } 3491 3492 /* dequeue the remaining */ 3493 while (deq < enq) { 3494 deq += rte_bbdev_dequeue_enc_ops(tp->dev_id, 3495 queue_id, &ops_deq[deq], enq - deq); 3496 } 3497 3498 total_time += rte_rdtsc_precise() - start_time; 3499 } 3500 3501 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 3502 ret = validate_enc_op(ops_deq, num_ops, ref_op); 3503 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 3504 } 3505 3506 rte_bbdev_enc_op_free_bulk(ops_enq, num_ops); 3507 3508 double tb_len_bits = calc_enc_TB_size(ref_op); 3509 3510 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) / 3511 ((double)total_time / (double)rte_get_tsc_hz()); 3512 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) 3513 / 1000000.0) / ((double)total_time / 3514 (double)rte_get_tsc_hz()); 3515 3516 return TEST_SUCCESS; 3517 } 3518 3519 static int 3520 throughput_pmd_lcore_ldpc_enc(void *arg) 3521 { 3522 struct thread_params *tp = arg; 3523 uint16_t enq, deq; 3524 uint64_t total_time = 0, start_time; 3525 const uint16_t queue_id = tp->queue_id; 3526 const uint16_t burst_sz = tp->op_params->burst_sz; 3527 const uint16_t num_ops = tp->op_params->num_to_process; 3528 struct rte_bbdev_enc_op *ops_enq[num_ops]; 3529 struct rte_bbdev_enc_op *ops_deq[num_ops]; 3530 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op; 3531 struct test_buffers *bufs = NULL; 3532 int i, j, ret; 3533 struct rte_bbdev_info info; 3534 uint16_t num_to_enq; 3535 3536 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 3537 "BURST_SIZE should be <= %u", MAX_BURST); 3538 3539 rte_bbdev_info_get(tp->dev_id, &info); 3540 3541 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 3542 "NUM_OPS cannot exceed %u for this device", 3543 info.drv.queue_size_lim); 3544 3545 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 3546 3547 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT) 3548 rte_pause(); 3549 3550 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops_enq, 3551 num_ops); 3552 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", 3553 num_ops); 3554 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3555 copy_reference_ldpc_enc_op(ops_enq, num_ops, 0, bufs->inputs, 3556 bufs->hard_outputs, ref_op); 3557 3558 /* Set counter to validate the ordering */ 3559 for (j = 0; j < num_ops; ++j) 3560 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 3561 3562 for (i = 0; i < TEST_REPETITIONS; ++i) { 3563 3564 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3565 for (j = 0; j < num_ops; ++j) 3566 mbuf_reset(ops_enq[j]->turbo_enc.output.data); 3567 3568 start_time = rte_rdtsc_precise(); 3569 3570 for (enq = 0, deq = 0; enq < num_ops;) { 3571 num_to_enq = burst_sz; 3572 3573 if (unlikely(num_ops - enq < num_to_enq)) 3574 num_to_enq = num_ops - enq; 3575 3576 enq += rte_bbdev_enqueue_ldpc_enc_ops(tp->dev_id, 3577 queue_id, &ops_enq[enq], num_to_enq); 3578 3579 deq += rte_bbdev_dequeue_ldpc_enc_ops(tp->dev_id, 3580 queue_id, &ops_deq[deq], enq - deq); 3581 } 3582 3583 /* dequeue the remaining */ 3584 while (deq < enq) { 3585 deq += rte_bbdev_dequeue_ldpc_enc_ops(tp->dev_id, 3586 queue_id, &ops_deq[deq], enq - deq); 3587 } 3588 3589 total_time += rte_rdtsc_precise() - start_time; 3590 } 3591 3592 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 3593 ret = validate_ldpc_enc_op(ops_deq, num_ops, ref_op); 3594 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 3595 } 3596 3597 rte_bbdev_enc_op_free_bulk(ops_enq, num_ops); 3598 3599 double tb_len_bits = calc_ldpc_enc_TB_size(ref_op); 3600 3601 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) / 3602 ((double)total_time / (double)rte_get_tsc_hz()); 3603 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) 3604 / 1000000.0) / ((double)total_time / 3605 (double)rte_get_tsc_hz()); 3606 3607 return TEST_SUCCESS; 3608 } 3609 3610 static void 3611 print_enc_throughput(struct thread_params *t_params, unsigned int used_cores) 3612 { 3613 unsigned int iter = 0; 3614 double total_mops = 0, total_mbps = 0; 3615 3616 for (iter = 0; iter < used_cores; iter++) { 3617 printf( 3618 "Throughput for core (%u): %.8lg Ops/s, %.8lg Mbps\n", 3619 t_params[iter].lcore_id, t_params[iter].ops_per_sec, 3620 t_params[iter].mbps); 3621 total_mops += t_params[iter].ops_per_sec; 3622 total_mbps += t_params[iter].mbps; 3623 } 3624 printf( 3625 "\nTotal throughput for %u cores: %.8lg MOPS, %.8lg Mbps\n", 3626 used_cores, total_mops, total_mbps); 3627 } 3628 3629 /* Aggregate the performance results over the number of cores used */ 3630 static void 3631 print_dec_throughput(struct thread_params *t_params, unsigned int used_cores) 3632 { 3633 unsigned int core_idx = 0; 3634 double total_mops = 0, total_mbps = 0; 3635 uint8_t iter_count = 0; 3636 3637 for (core_idx = 0; core_idx < used_cores; core_idx++) { 3638 printf( 3639 "Throughput for core (%u): %.8lg Ops/s, %.8lg Mbps @ max %u iterations\n", 3640 t_params[core_idx].lcore_id, 3641 t_params[core_idx].ops_per_sec, 3642 t_params[core_idx].mbps, 3643 t_params[core_idx].iter_count); 3644 total_mops += t_params[core_idx].ops_per_sec; 3645 total_mbps += t_params[core_idx].mbps; 3646 iter_count = RTE_MAX(iter_count, 3647 t_params[core_idx].iter_count); 3648 } 3649 printf( 3650 "\nTotal throughput for %u cores: %.8lg MOPS, %.8lg Mbps @ max %u iterations\n", 3651 used_cores, total_mops, total_mbps, iter_count); 3652 } 3653 3654 /* Aggregate the performance results over the number of cores used */ 3655 static void 3656 print_dec_bler(struct thread_params *t_params, unsigned int used_cores) 3657 { 3658 unsigned int core_idx = 0; 3659 double total_mbps = 0, total_bler = 0, total_iter = 0; 3660 double snr = get_snr(); 3661 3662 for (core_idx = 0; core_idx < used_cores; core_idx++) { 3663 printf("Core%u BLER %.1f %% - Iters %.1f - Tp %.1f Mbps %s\n", 3664 t_params[core_idx].lcore_id, 3665 t_params[core_idx].bler * 100, 3666 t_params[core_idx].iter_average, 3667 t_params[core_idx].mbps, 3668 get_vector_filename()); 3669 total_mbps += t_params[core_idx].mbps; 3670 total_bler += t_params[core_idx].bler; 3671 total_iter += t_params[core_idx].iter_average; 3672 } 3673 total_bler /= used_cores; 3674 total_iter /= used_cores; 3675 3676 printf("SNR %.2f BLER %.1f %% - Iterations %.1f %d - Tp %.1f Mbps %s\n", 3677 snr, total_bler * 100, total_iter, get_iter_max(), 3678 total_mbps, get_vector_filename()); 3679 } 3680 3681 /* 3682 * Test function that determines BLER wireless performance 3683 */ 3684 static int 3685 bler_test(struct active_device *ad, 3686 struct test_op_params *op_params) 3687 { 3688 int ret; 3689 unsigned int lcore_id, used_cores = 0; 3690 struct thread_params *t_params; 3691 struct rte_bbdev_info info; 3692 lcore_function_t *bler_function; 3693 uint16_t num_lcores; 3694 const char *op_type_str; 3695 3696 rte_bbdev_info_get(ad->dev_id, &info); 3697 3698 op_type_str = rte_bbdev_op_type_str(test_vector.op_type); 3699 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", 3700 test_vector.op_type); 3701 3702 printf("+ ------------------------------------------------------- +\n"); 3703 printf("== test: bler\ndev: %s, nb_queues: %u, burst size: %u, num ops: %u, num_lcores: %u, op type: %s, itr mode: %s, GHz: %lg\n", 3704 info.dev_name, ad->nb_queues, op_params->burst_sz, 3705 op_params->num_to_process, op_params->num_lcores, 3706 op_type_str, 3707 intr_enabled ? "Interrupt mode" : "PMD mode", 3708 (double)rte_get_tsc_hz() / 1000000000.0); 3709 3710 /* Set number of lcores */ 3711 num_lcores = (ad->nb_queues < (op_params->num_lcores)) 3712 ? ad->nb_queues 3713 : op_params->num_lcores; 3714 3715 /* Allocate memory for thread parameters structure */ 3716 t_params = rte_zmalloc(NULL, num_lcores * sizeof(struct thread_params), 3717 RTE_CACHE_LINE_SIZE); 3718 TEST_ASSERT_NOT_NULL(t_params, "Failed to alloc %zuB for t_params", 3719 RTE_ALIGN(sizeof(struct thread_params) * num_lcores, 3720 RTE_CACHE_LINE_SIZE)); 3721 3722 if ((test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) && 3723 !check_bit(test_vector.ldpc_dec.op_flags, 3724 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK) 3725 && !check_bit(test_vector.ldpc_dec.op_flags, 3726 RTE_BBDEV_LDPC_LLR_COMPRESSION)) 3727 bler_function = bler_pmd_lcore_ldpc_dec; 3728 else 3729 return TEST_SKIPPED; 3730 3731 rte_atomic16_set(&op_params->sync, SYNC_WAIT); 3732 3733 /* Main core is set at first entry */ 3734 t_params[0].dev_id = ad->dev_id; 3735 t_params[0].lcore_id = rte_lcore_id(); 3736 t_params[0].op_params = op_params; 3737 t_params[0].queue_id = ad->queue_ids[used_cores++]; 3738 t_params[0].iter_count = 0; 3739 3740 RTE_LCORE_FOREACH_WORKER(lcore_id) { 3741 if (used_cores >= num_lcores) 3742 break; 3743 3744 t_params[used_cores].dev_id = ad->dev_id; 3745 t_params[used_cores].lcore_id = lcore_id; 3746 t_params[used_cores].op_params = op_params; 3747 t_params[used_cores].queue_id = ad->queue_ids[used_cores]; 3748 t_params[used_cores].iter_count = 0; 3749 3750 rte_eal_remote_launch(bler_function, 3751 &t_params[used_cores++], lcore_id); 3752 } 3753 3754 rte_atomic16_set(&op_params->sync, SYNC_START); 3755 ret = bler_function(&t_params[0]); 3756 3757 /* Main core is always used */ 3758 for (used_cores = 1; used_cores < num_lcores; used_cores++) 3759 ret |= rte_eal_wait_lcore(t_params[used_cores].lcore_id); 3760 3761 print_dec_bler(t_params, num_lcores); 3762 3763 /* Return if test failed */ 3764 if (ret) { 3765 rte_free(t_params); 3766 return ret; 3767 } 3768 3769 /* Function to print something here*/ 3770 rte_free(t_params); 3771 return ret; 3772 } 3773 3774 /* 3775 * Test function that determines how long an enqueue + dequeue of a burst 3776 * takes on available lcores. 3777 */ 3778 static int 3779 throughput_test(struct active_device *ad, 3780 struct test_op_params *op_params) 3781 { 3782 int ret; 3783 unsigned int lcore_id, used_cores = 0; 3784 struct thread_params *t_params, *tp; 3785 struct rte_bbdev_info info; 3786 lcore_function_t *throughput_function; 3787 uint16_t num_lcores; 3788 const char *op_type_str; 3789 3790 rte_bbdev_info_get(ad->dev_id, &info); 3791 3792 op_type_str = rte_bbdev_op_type_str(test_vector.op_type); 3793 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", 3794 test_vector.op_type); 3795 3796 printf("+ ------------------------------------------------------- +\n"); 3797 printf("== test: throughput\ndev: %s, nb_queues: %u, burst size: %u, num ops: %u, num_lcores: %u, op type: %s, itr mode: %s, GHz: %lg\n", 3798 info.dev_name, ad->nb_queues, op_params->burst_sz, 3799 op_params->num_to_process, op_params->num_lcores, 3800 op_type_str, 3801 intr_enabled ? "Interrupt mode" : "PMD mode", 3802 (double)rte_get_tsc_hz() / 1000000000.0); 3803 3804 /* Set number of lcores */ 3805 num_lcores = (ad->nb_queues < (op_params->num_lcores)) 3806 ? ad->nb_queues 3807 : op_params->num_lcores; 3808 3809 /* Allocate memory for thread parameters structure */ 3810 t_params = rte_zmalloc(NULL, num_lcores * sizeof(struct thread_params), 3811 RTE_CACHE_LINE_SIZE); 3812 TEST_ASSERT_NOT_NULL(t_params, "Failed to alloc %zuB for t_params", 3813 RTE_ALIGN(sizeof(struct thread_params) * num_lcores, 3814 RTE_CACHE_LINE_SIZE)); 3815 3816 if (intr_enabled) { 3817 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) 3818 throughput_function = throughput_intr_lcore_dec; 3819 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 3820 throughput_function = throughput_intr_lcore_ldpc_dec; 3821 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) 3822 throughput_function = throughput_intr_lcore_enc; 3823 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) 3824 throughput_function = throughput_intr_lcore_ldpc_enc; 3825 else 3826 throughput_function = throughput_intr_lcore_enc; 3827 3828 /* Dequeue interrupt callback registration */ 3829 ret = rte_bbdev_callback_register(ad->dev_id, 3830 RTE_BBDEV_EVENT_DEQUEUE, dequeue_event_callback, 3831 t_params); 3832 if (ret < 0) { 3833 rte_free(t_params); 3834 return ret; 3835 } 3836 } else { 3837 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) 3838 throughput_function = throughput_pmd_lcore_dec; 3839 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 3840 throughput_function = throughput_pmd_lcore_ldpc_dec; 3841 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) 3842 throughput_function = throughput_pmd_lcore_enc; 3843 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) 3844 throughput_function = throughput_pmd_lcore_ldpc_enc; 3845 else 3846 throughput_function = throughput_pmd_lcore_enc; 3847 } 3848 3849 rte_atomic16_set(&op_params->sync, SYNC_WAIT); 3850 3851 /* Main core is set at first entry */ 3852 t_params[0].dev_id = ad->dev_id; 3853 t_params[0].lcore_id = rte_lcore_id(); 3854 t_params[0].op_params = op_params; 3855 t_params[0].queue_id = ad->queue_ids[used_cores++]; 3856 t_params[0].iter_count = 0; 3857 3858 RTE_LCORE_FOREACH_WORKER(lcore_id) { 3859 if (used_cores >= num_lcores) 3860 break; 3861 3862 t_params[used_cores].dev_id = ad->dev_id; 3863 t_params[used_cores].lcore_id = lcore_id; 3864 t_params[used_cores].op_params = op_params; 3865 t_params[used_cores].queue_id = ad->queue_ids[used_cores]; 3866 t_params[used_cores].iter_count = 0; 3867 3868 rte_eal_remote_launch(throughput_function, 3869 &t_params[used_cores++], lcore_id); 3870 } 3871 3872 rte_atomic16_set(&op_params->sync, SYNC_START); 3873 ret = throughput_function(&t_params[0]); 3874 3875 /* Main core is always used */ 3876 for (used_cores = 1; used_cores < num_lcores; used_cores++) 3877 ret |= rte_eal_wait_lcore(t_params[used_cores].lcore_id); 3878 3879 /* Return if test failed */ 3880 if (ret) { 3881 rte_free(t_params); 3882 return ret; 3883 } 3884 3885 /* Print throughput if interrupts are disabled and test passed */ 3886 if (!intr_enabled) { 3887 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC || 3888 test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 3889 print_dec_throughput(t_params, num_lcores); 3890 else 3891 print_enc_throughput(t_params, num_lcores); 3892 rte_free(t_params); 3893 return ret; 3894 } 3895 3896 /* In interrupt TC we need to wait for the interrupt callback to deqeue 3897 * all pending operations. Skip waiting for queues which reported an 3898 * error using processing_status variable. 3899 * Wait for main lcore operations. 3900 */ 3901 tp = &t_params[0]; 3902 while ((rte_atomic16_read(&tp->nb_dequeued) < 3903 op_params->num_to_process) && 3904 (rte_atomic16_read(&tp->processing_status) != 3905 TEST_FAILED)) 3906 rte_pause(); 3907 3908 tp->ops_per_sec /= TEST_REPETITIONS; 3909 tp->mbps /= TEST_REPETITIONS; 3910 ret |= (int)rte_atomic16_read(&tp->processing_status); 3911 3912 /* Wait for worker lcores operations */ 3913 for (used_cores = 1; used_cores < num_lcores; used_cores++) { 3914 tp = &t_params[used_cores]; 3915 3916 while ((rte_atomic16_read(&tp->nb_dequeued) < 3917 op_params->num_to_process) && 3918 (rte_atomic16_read(&tp->processing_status) != 3919 TEST_FAILED)) 3920 rte_pause(); 3921 3922 tp->ops_per_sec /= TEST_REPETITIONS; 3923 tp->mbps /= TEST_REPETITIONS; 3924 ret |= (int)rte_atomic16_read(&tp->processing_status); 3925 } 3926 3927 /* Print throughput if test passed */ 3928 if (!ret) { 3929 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC || 3930 test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 3931 print_dec_throughput(t_params, num_lcores); 3932 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC || 3933 test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) 3934 print_enc_throughput(t_params, num_lcores); 3935 } 3936 3937 rte_free(t_params); 3938 return ret; 3939 } 3940 3941 static int 3942 latency_test_dec(struct rte_mempool *mempool, 3943 struct test_buffers *bufs, struct rte_bbdev_dec_op *ref_op, 3944 int vector_mask, uint16_t dev_id, uint16_t queue_id, 3945 const uint16_t num_to_process, uint16_t burst_sz, 3946 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time) 3947 { 3948 int ret = TEST_SUCCESS; 3949 uint16_t i, j, dequeued; 3950 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 3951 uint64_t start_time = 0, last_time = 0; 3952 3953 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 3954 uint16_t enq = 0, deq = 0; 3955 bool first_time = true; 3956 last_time = 0; 3957 3958 if (unlikely(num_to_process - dequeued < burst_sz)) 3959 burst_sz = num_to_process - dequeued; 3960 3961 ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz); 3962 TEST_ASSERT_SUCCESS(ret, 3963 "rte_bbdev_dec_op_alloc_bulk() failed"); 3964 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3965 copy_reference_dec_op(ops_enq, burst_sz, dequeued, 3966 bufs->inputs, 3967 bufs->hard_outputs, 3968 bufs->soft_outputs, 3969 ref_op); 3970 3971 /* Set counter to validate the ordering */ 3972 for (j = 0; j < burst_sz; ++j) 3973 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 3974 3975 start_time = rte_rdtsc_precise(); 3976 3977 enq = rte_bbdev_enqueue_dec_ops(dev_id, queue_id, &ops_enq[enq], 3978 burst_sz); 3979 TEST_ASSERT(enq == burst_sz, 3980 "Error enqueueing burst, expected %u, got %u", 3981 burst_sz, enq); 3982 3983 /* Dequeue */ 3984 do { 3985 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id, 3986 &ops_deq[deq], burst_sz - deq); 3987 if (likely(first_time && (deq > 0))) { 3988 last_time = rte_rdtsc_precise() - start_time; 3989 first_time = false; 3990 } 3991 } while (unlikely(burst_sz != deq)); 3992 3993 *max_time = RTE_MAX(*max_time, last_time); 3994 *min_time = RTE_MIN(*min_time, last_time); 3995 *total_time += last_time; 3996 3997 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 3998 ret = validate_dec_op(ops_deq, burst_sz, ref_op, 3999 vector_mask); 4000 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 4001 } 4002 4003 rte_bbdev_dec_op_free_bulk(ops_enq, deq); 4004 dequeued += deq; 4005 } 4006 4007 return i; 4008 } 4009 4010 /* Test case for latency/validation for LDPC Decoder */ 4011 static int 4012 latency_test_ldpc_dec(struct rte_mempool *mempool, 4013 struct test_buffers *bufs, struct rte_bbdev_dec_op *ref_op, 4014 int vector_mask, uint16_t dev_id, uint16_t queue_id, 4015 const uint16_t num_to_process, uint16_t burst_sz, 4016 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time, 4017 bool disable_et) 4018 { 4019 int ret = TEST_SUCCESS; 4020 uint16_t i, j, dequeued; 4021 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 4022 uint64_t start_time = 0, last_time = 0; 4023 bool extDdr = ldpc_cap_flags & 4024 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE; 4025 4026 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 4027 uint16_t enq = 0, deq = 0; 4028 bool first_time = true; 4029 last_time = 0; 4030 4031 if (unlikely(num_to_process - dequeued < burst_sz)) 4032 burst_sz = num_to_process - dequeued; 4033 4034 ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz); 4035 TEST_ASSERT_SUCCESS(ret, 4036 "rte_bbdev_dec_op_alloc_bulk() failed"); 4037 4038 /* For latency tests we need to disable early termination */ 4039 if (disable_et && check_bit(ref_op->ldpc_dec.op_flags, 4040 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE)) 4041 ref_op->ldpc_dec.op_flags -= 4042 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE; 4043 ref_op->ldpc_dec.iter_max = get_iter_max(); 4044 ref_op->ldpc_dec.iter_count = ref_op->ldpc_dec.iter_max; 4045 4046 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4047 copy_reference_ldpc_dec_op(ops_enq, burst_sz, dequeued, 4048 bufs->inputs, 4049 bufs->hard_outputs, 4050 bufs->soft_outputs, 4051 bufs->harq_inputs, 4052 bufs->harq_outputs, 4053 ref_op); 4054 4055 if (extDdr) 4056 preload_harq_ddr(dev_id, queue_id, ops_enq, 4057 burst_sz, true); 4058 4059 /* Set counter to validate the ordering */ 4060 for (j = 0; j < burst_sz; ++j) 4061 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 4062 4063 start_time = rte_rdtsc_precise(); 4064 4065 enq = rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id, 4066 &ops_enq[enq], burst_sz); 4067 TEST_ASSERT(enq == burst_sz, 4068 "Error enqueueing burst, expected %u, got %u", 4069 burst_sz, enq); 4070 4071 /* Dequeue */ 4072 do { 4073 deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id, 4074 &ops_deq[deq], burst_sz - deq); 4075 if (likely(first_time && (deq > 0))) { 4076 last_time = rte_rdtsc_precise() - start_time; 4077 first_time = false; 4078 } 4079 } while (unlikely(burst_sz != deq)); 4080 4081 *max_time = RTE_MAX(*max_time, last_time); 4082 *min_time = RTE_MIN(*min_time, last_time); 4083 *total_time += last_time; 4084 4085 if (extDdr) 4086 retrieve_harq_ddr(dev_id, queue_id, ops_enq, burst_sz); 4087 4088 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 4089 ret = validate_ldpc_dec_op(ops_deq, burst_sz, ref_op, 4090 vector_mask); 4091 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 4092 } 4093 4094 rte_bbdev_dec_op_free_bulk(ops_enq, deq); 4095 dequeued += deq; 4096 } 4097 return i; 4098 } 4099 4100 static int 4101 latency_test_enc(struct rte_mempool *mempool, 4102 struct test_buffers *bufs, struct rte_bbdev_enc_op *ref_op, 4103 uint16_t dev_id, uint16_t queue_id, 4104 const uint16_t num_to_process, uint16_t burst_sz, 4105 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time) 4106 { 4107 int ret = TEST_SUCCESS; 4108 uint16_t i, j, dequeued; 4109 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 4110 uint64_t start_time = 0, last_time = 0; 4111 4112 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 4113 uint16_t enq = 0, deq = 0; 4114 bool first_time = true; 4115 last_time = 0; 4116 4117 if (unlikely(num_to_process - dequeued < burst_sz)) 4118 burst_sz = num_to_process - dequeued; 4119 4120 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz); 4121 TEST_ASSERT_SUCCESS(ret, 4122 "rte_bbdev_enc_op_alloc_bulk() failed"); 4123 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4124 copy_reference_enc_op(ops_enq, burst_sz, dequeued, 4125 bufs->inputs, 4126 bufs->hard_outputs, 4127 ref_op); 4128 4129 /* Set counter to validate the ordering */ 4130 for (j = 0; j < burst_sz; ++j) 4131 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 4132 4133 start_time = rte_rdtsc_precise(); 4134 4135 enq = rte_bbdev_enqueue_enc_ops(dev_id, queue_id, &ops_enq[enq], 4136 burst_sz); 4137 TEST_ASSERT(enq == burst_sz, 4138 "Error enqueueing burst, expected %u, got %u", 4139 burst_sz, enq); 4140 4141 /* Dequeue */ 4142 do { 4143 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id, 4144 &ops_deq[deq], burst_sz - deq); 4145 if (likely(first_time && (deq > 0))) { 4146 last_time += rte_rdtsc_precise() - start_time; 4147 first_time = false; 4148 } 4149 } while (unlikely(burst_sz != deq)); 4150 4151 *max_time = RTE_MAX(*max_time, last_time); 4152 *min_time = RTE_MIN(*min_time, last_time); 4153 *total_time += last_time; 4154 4155 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 4156 ret = validate_enc_op(ops_deq, burst_sz, ref_op); 4157 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 4158 } 4159 4160 rte_bbdev_enc_op_free_bulk(ops_enq, deq); 4161 dequeued += deq; 4162 } 4163 4164 return i; 4165 } 4166 4167 static int 4168 latency_test_ldpc_enc(struct rte_mempool *mempool, 4169 struct test_buffers *bufs, struct rte_bbdev_enc_op *ref_op, 4170 uint16_t dev_id, uint16_t queue_id, 4171 const uint16_t num_to_process, uint16_t burst_sz, 4172 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time) 4173 { 4174 int ret = TEST_SUCCESS; 4175 uint16_t i, j, dequeued; 4176 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 4177 uint64_t start_time = 0, last_time = 0; 4178 4179 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 4180 uint16_t enq = 0, deq = 0; 4181 bool first_time = true; 4182 last_time = 0; 4183 4184 if (unlikely(num_to_process - dequeued < burst_sz)) 4185 burst_sz = num_to_process - dequeued; 4186 4187 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz); 4188 TEST_ASSERT_SUCCESS(ret, 4189 "rte_bbdev_enc_op_alloc_bulk() failed"); 4190 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4191 copy_reference_ldpc_enc_op(ops_enq, burst_sz, dequeued, 4192 bufs->inputs, 4193 bufs->hard_outputs, 4194 ref_op); 4195 4196 /* Set counter to validate the ordering */ 4197 for (j = 0; j < burst_sz; ++j) 4198 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 4199 4200 start_time = rte_rdtsc_precise(); 4201 4202 enq = rte_bbdev_enqueue_ldpc_enc_ops(dev_id, queue_id, 4203 &ops_enq[enq], burst_sz); 4204 TEST_ASSERT(enq == burst_sz, 4205 "Error enqueueing burst, expected %u, got %u", 4206 burst_sz, enq); 4207 4208 /* Dequeue */ 4209 do { 4210 deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id, 4211 &ops_deq[deq], burst_sz - deq); 4212 if (likely(first_time && (deq > 0))) { 4213 last_time += rte_rdtsc_precise() - start_time; 4214 first_time = false; 4215 } 4216 } while (unlikely(burst_sz != deq)); 4217 4218 *max_time = RTE_MAX(*max_time, last_time); 4219 *min_time = RTE_MIN(*min_time, last_time); 4220 *total_time += last_time; 4221 4222 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 4223 ret = validate_enc_op(ops_deq, burst_sz, ref_op); 4224 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 4225 } 4226 4227 rte_bbdev_enc_op_free_bulk(ops_enq, deq); 4228 dequeued += deq; 4229 } 4230 4231 return i; 4232 } 4233 4234 /* Common function for running validation and latency test cases */ 4235 static int 4236 validation_latency_test(struct active_device *ad, 4237 struct test_op_params *op_params, bool latency_flag) 4238 { 4239 int iter; 4240 uint16_t burst_sz = op_params->burst_sz; 4241 const uint16_t num_to_process = op_params->num_to_process; 4242 const enum rte_bbdev_op_type op_type = test_vector.op_type; 4243 const uint16_t queue_id = ad->queue_ids[0]; 4244 struct test_buffers *bufs = NULL; 4245 struct rte_bbdev_info info; 4246 uint64_t total_time, min_time, max_time; 4247 const char *op_type_str; 4248 4249 total_time = max_time = 0; 4250 min_time = UINT64_MAX; 4251 4252 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 4253 "BURST_SIZE should be <= %u", MAX_BURST); 4254 4255 rte_bbdev_info_get(ad->dev_id, &info); 4256 bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 4257 4258 op_type_str = rte_bbdev_op_type_str(op_type); 4259 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type); 4260 4261 printf("+ ------------------------------------------------------- +\n"); 4262 if (latency_flag) 4263 printf("== test: latency\ndev:"); 4264 else 4265 printf("== test: validation\ndev:"); 4266 printf("%s, burst size: %u, num ops: %u, op type: %s\n", 4267 info.dev_name, burst_sz, num_to_process, op_type_str); 4268 4269 if (op_type == RTE_BBDEV_OP_TURBO_DEC) 4270 iter = latency_test_dec(op_params->mp, bufs, 4271 op_params->ref_dec_op, op_params->vector_mask, 4272 ad->dev_id, queue_id, num_to_process, 4273 burst_sz, &total_time, &min_time, &max_time); 4274 else if (op_type == RTE_BBDEV_OP_LDPC_ENC) 4275 iter = latency_test_ldpc_enc(op_params->mp, bufs, 4276 op_params->ref_enc_op, ad->dev_id, queue_id, 4277 num_to_process, burst_sz, &total_time, 4278 &min_time, &max_time); 4279 else if (op_type == RTE_BBDEV_OP_LDPC_DEC) 4280 iter = latency_test_ldpc_dec(op_params->mp, bufs, 4281 op_params->ref_dec_op, op_params->vector_mask, 4282 ad->dev_id, queue_id, num_to_process, 4283 burst_sz, &total_time, &min_time, &max_time, 4284 latency_flag); 4285 else /* RTE_BBDEV_OP_TURBO_ENC */ 4286 iter = latency_test_enc(op_params->mp, bufs, 4287 op_params->ref_enc_op, 4288 ad->dev_id, queue_id, 4289 num_to_process, burst_sz, &total_time, 4290 &min_time, &max_time); 4291 4292 if (iter <= 0) 4293 return TEST_FAILED; 4294 4295 printf("Operation latency:\n" 4296 "\tavg: %lg cycles, %lg us\n" 4297 "\tmin: %lg cycles, %lg us\n" 4298 "\tmax: %lg cycles, %lg us\n", 4299 (double)total_time / (double)iter, 4300 (double)(total_time * 1000000) / (double)iter / 4301 (double)rte_get_tsc_hz(), (double)min_time, 4302 (double)(min_time * 1000000) / (double)rte_get_tsc_hz(), 4303 (double)max_time, (double)(max_time * 1000000) / 4304 (double)rte_get_tsc_hz()); 4305 4306 return TEST_SUCCESS; 4307 } 4308 4309 static int 4310 latency_test(struct active_device *ad, struct test_op_params *op_params) 4311 { 4312 return validation_latency_test(ad, op_params, true); 4313 } 4314 4315 static int 4316 validation_test(struct active_device *ad, struct test_op_params *op_params) 4317 { 4318 return validation_latency_test(ad, op_params, false); 4319 } 4320 4321 #ifdef RTE_BBDEV_OFFLOAD_COST 4322 static int 4323 get_bbdev_queue_stats(uint16_t dev_id, uint16_t queue_id, 4324 struct rte_bbdev_stats *stats) 4325 { 4326 struct rte_bbdev *dev = &rte_bbdev_devices[dev_id]; 4327 struct rte_bbdev_stats *q_stats; 4328 4329 if (queue_id >= dev->data->num_queues) 4330 return -1; 4331 4332 q_stats = &dev->data->queues[queue_id].queue_stats; 4333 4334 stats->enqueued_count = q_stats->enqueued_count; 4335 stats->dequeued_count = q_stats->dequeued_count; 4336 stats->enqueue_err_count = q_stats->enqueue_err_count; 4337 stats->dequeue_err_count = q_stats->dequeue_err_count; 4338 stats->acc_offload_cycles = q_stats->acc_offload_cycles; 4339 4340 return 0; 4341 } 4342 4343 static int 4344 offload_latency_test_dec(struct rte_mempool *mempool, struct test_buffers *bufs, 4345 struct rte_bbdev_dec_op *ref_op, uint16_t dev_id, 4346 uint16_t queue_id, const uint16_t num_to_process, 4347 uint16_t burst_sz, struct test_time_stats *time_st) 4348 { 4349 int i, dequeued, ret; 4350 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 4351 uint64_t enq_start_time, deq_start_time; 4352 uint64_t enq_sw_last_time, deq_last_time; 4353 struct rte_bbdev_stats stats; 4354 4355 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 4356 uint16_t enq = 0, deq = 0; 4357 4358 if (unlikely(num_to_process - dequeued < burst_sz)) 4359 burst_sz = num_to_process - dequeued; 4360 4361 rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz); 4362 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4363 copy_reference_dec_op(ops_enq, burst_sz, dequeued, 4364 bufs->inputs, 4365 bufs->hard_outputs, 4366 bufs->soft_outputs, 4367 ref_op); 4368 4369 /* Start time meas for enqueue function offload latency */ 4370 enq_start_time = rte_rdtsc_precise(); 4371 do { 4372 enq += rte_bbdev_enqueue_dec_ops(dev_id, queue_id, 4373 &ops_enq[enq], burst_sz - enq); 4374 } while (unlikely(burst_sz != enq)); 4375 4376 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats); 4377 TEST_ASSERT_SUCCESS(ret, 4378 "Failed to get stats for queue (%u) of device (%u)", 4379 queue_id, dev_id); 4380 4381 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time - 4382 stats.acc_offload_cycles; 4383 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time, 4384 enq_sw_last_time); 4385 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time, 4386 enq_sw_last_time); 4387 time_st->enq_sw_total_time += enq_sw_last_time; 4388 4389 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time, 4390 stats.acc_offload_cycles); 4391 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time, 4392 stats.acc_offload_cycles); 4393 time_st->enq_acc_total_time += stats.acc_offload_cycles; 4394 4395 /* give time for device to process ops */ 4396 rte_delay_us(WAIT_OFFLOAD_US); 4397 4398 /* Start time meas for dequeue function offload latency */ 4399 deq_start_time = rte_rdtsc_precise(); 4400 /* Dequeue one operation */ 4401 do { 4402 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id, 4403 &ops_deq[deq], enq); 4404 } while (unlikely(deq == 0)); 4405 4406 deq_last_time = rte_rdtsc_precise() - deq_start_time; 4407 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time, 4408 deq_last_time); 4409 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time, 4410 deq_last_time); 4411 time_st->deq_total_time += deq_last_time; 4412 4413 /* Dequeue remaining operations if needed*/ 4414 while (burst_sz != deq) 4415 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id, 4416 &ops_deq[deq], burst_sz - deq); 4417 4418 rte_bbdev_dec_op_free_bulk(ops_enq, deq); 4419 dequeued += deq; 4420 } 4421 4422 return i; 4423 } 4424 4425 static int 4426 offload_latency_test_ldpc_dec(struct rte_mempool *mempool, 4427 struct test_buffers *bufs, 4428 struct rte_bbdev_dec_op *ref_op, uint16_t dev_id, 4429 uint16_t queue_id, const uint16_t num_to_process, 4430 uint16_t burst_sz, struct test_time_stats *time_st) 4431 { 4432 int i, dequeued, ret; 4433 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 4434 uint64_t enq_start_time, deq_start_time; 4435 uint64_t enq_sw_last_time, deq_last_time; 4436 struct rte_bbdev_stats stats; 4437 bool extDdr = ldpc_cap_flags & 4438 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE; 4439 4440 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 4441 uint16_t enq = 0, deq = 0; 4442 4443 if (unlikely(num_to_process - dequeued < burst_sz)) 4444 burst_sz = num_to_process - dequeued; 4445 4446 rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz); 4447 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4448 copy_reference_ldpc_dec_op(ops_enq, burst_sz, dequeued, 4449 bufs->inputs, 4450 bufs->hard_outputs, 4451 bufs->soft_outputs, 4452 bufs->harq_inputs, 4453 bufs->harq_outputs, 4454 ref_op); 4455 4456 if (extDdr) 4457 preload_harq_ddr(dev_id, queue_id, ops_enq, 4458 burst_sz, true); 4459 4460 /* Start time meas for enqueue function offload latency */ 4461 enq_start_time = rte_rdtsc_precise(); 4462 do { 4463 enq += rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id, 4464 &ops_enq[enq], burst_sz - enq); 4465 } while (unlikely(burst_sz != enq)); 4466 4467 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time; 4468 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats); 4469 TEST_ASSERT_SUCCESS(ret, 4470 "Failed to get stats for queue (%u) of device (%u)", 4471 queue_id, dev_id); 4472 4473 enq_sw_last_time -= stats.acc_offload_cycles; 4474 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time, 4475 enq_sw_last_time); 4476 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time, 4477 enq_sw_last_time); 4478 time_st->enq_sw_total_time += enq_sw_last_time; 4479 4480 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time, 4481 stats.acc_offload_cycles); 4482 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time, 4483 stats.acc_offload_cycles); 4484 time_st->enq_acc_total_time += stats.acc_offload_cycles; 4485 4486 /* give time for device to process ops */ 4487 rte_delay_us(WAIT_OFFLOAD_US); 4488 4489 /* Start time meas for dequeue function offload latency */ 4490 deq_start_time = rte_rdtsc_precise(); 4491 /* Dequeue one operation */ 4492 do { 4493 deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id, 4494 &ops_deq[deq], enq); 4495 } while (unlikely(deq == 0)); 4496 4497 deq_last_time = rte_rdtsc_precise() - deq_start_time; 4498 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time, 4499 deq_last_time); 4500 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time, 4501 deq_last_time); 4502 time_st->deq_total_time += deq_last_time; 4503 4504 /* Dequeue remaining operations if needed*/ 4505 while (burst_sz != deq) 4506 deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id, 4507 &ops_deq[deq], burst_sz - deq); 4508 4509 if (extDdr) { 4510 /* Read loopback is not thread safe */ 4511 retrieve_harq_ddr(dev_id, queue_id, ops_enq, burst_sz); 4512 } 4513 4514 rte_bbdev_dec_op_free_bulk(ops_enq, deq); 4515 dequeued += deq; 4516 } 4517 4518 return i; 4519 } 4520 4521 static int 4522 offload_latency_test_enc(struct rte_mempool *mempool, struct test_buffers *bufs, 4523 struct rte_bbdev_enc_op *ref_op, uint16_t dev_id, 4524 uint16_t queue_id, const uint16_t num_to_process, 4525 uint16_t burst_sz, struct test_time_stats *time_st) 4526 { 4527 int i, dequeued, ret; 4528 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 4529 uint64_t enq_start_time, deq_start_time; 4530 uint64_t enq_sw_last_time, deq_last_time; 4531 struct rte_bbdev_stats stats; 4532 4533 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 4534 uint16_t enq = 0, deq = 0; 4535 4536 if (unlikely(num_to_process - dequeued < burst_sz)) 4537 burst_sz = num_to_process - dequeued; 4538 4539 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz); 4540 TEST_ASSERT_SUCCESS(ret, 4541 "rte_bbdev_enc_op_alloc_bulk() failed"); 4542 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4543 copy_reference_enc_op(ops_enq, burst_sz, dequeued, 4544 bufs->inputs, 4545 bufs->hard_outputs, 4546 ref_op); 4547 4548 /* Start time meas for enqueue function offload latency */ 4549 enq_start_time = rte_rdtsc_precise(); 4550 do { 4551 enq += rte_bbdev_enqueue_enc_ops(dev_id, queue_id, 4552 &ops_enq[enq], burst_sz - enq); 4553 } while (unlikely(burst_sz != enq)); 4554 4555 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time; 4556 4557 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats); 4558 TEST_ASSERT_SUCCESS(ret, 4559 "Failed to get stats for queue (%u) of device (%u)", 4560 queue_id, dev_id); 4561 enq_sw_last_time -= stats.acc_offload_cycles; 4562 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time, 4563 enq_sw_last_time); 4564 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time, 4565 enq_sw_last_time); 4566 time_st->enq_sw_total_time += enq_sw_last_time; 4567 4568 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time, 4569 stats.acc_offload_cycles); 4570 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time, 4571 stats.acc_offload_cycles); 4572 time_st->enq_acc_total_time += stats.acc_offload_cycles; 4573 4574 /* give time for device to process ops */ 4575 rte_delay_us(WAIT_OFFLOAD_US); 4576 4577 /* Start time meas for dequeue function offload latency */ 4578 deq_start_time = rte_rdtsc_precise(); 4579 /* Dequeue one operation */ 4580 do { 4581 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id, 4582 &ops_deq[deq], enq); 4583 } while (unlikely(deq == 0)); 4584 4585 deq_last_time = rte_rdtsc_precise() - deq_start_time; 4586 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time, 4587 deq_last_time); 4588 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time, 4589 deq_last_time); 4590 time_st->deq_total_time += deq_last_time; 4591 4592 while (burst_sz != deq) 4593 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id, 4594 &ops_deq[deq], burst_sz - deq); 4595 4596 rte_bbdev_enc_op_free_bulk(ops_enq, deq); 4597 dequeued += deq; 4598 } 4599 4600 return i; 4601 } 4602 4603 static int 4604 offload_latency_test_ldpc_enc(struct rte_mempool *mempool, 4605 struct test_buffers *bufs, 4606 struct rte_bbdev_enc_op *ref_op, uint16_t dev_id, 4607 uint16_t queue_id, const uint16_t num_to_process, 4608 uint16_t burst_sz, struct test_time_stats *time_st) 4609 { 4610 int i, dequeued, ret; 4611 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 4612 uint64_t enq_start_time, deq_start_time; 4613 uint64_t enq_sw_last_time, deq_last_time; 4614 struct rte_bbdev_stats stats; 4615 4616 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 4617 uint16_t enq = 0, deq = 0; 4618 4619 if (unlikely(num_to_process - dequeued < burst_sz)) 4620 burst_sz = num_to_process - dequeued; 4621 4622 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz); 4623 TEST_ASSERT_SUCCESS(ret, 4624 "rte_bbdev_enc_op_alloc_bulk() failed"); 4625 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4626 copy_reference_ldpc_enc_op(ops_enq, burst_sz, dequeued, 4627 bufs->inputs, 4628 bufs->hard_outputs, 4629 ref_op); 4630 4631 /* Start time meas for enqueue function offload latency */ 4632 enq_start_time = rte_rdtsc_precise(); 4633 do { 4634 enq += rte_bbdev_enqueue_ldpc_enc_ops(dev_id, queue_id, 4635 &ops_enq[enq], burst_sz - enq); 4636 } while (unlikely(burst_sz != enq)); 4637 4638 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time; 4639 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats); 4640 TEST_ASSERT_SUCCESS(ret, 4641 "Failed to get stats for queue (%u) of device (%u)", 4642 queue_id, dev_id); 4643 4644 enq_sw_last_time -= stats.acc_offload_cycles; 4645 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time, 4646 enq_sw_last_time); 4647 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time, 4648 enq_sw_last_time); 4649 time_st->enq_sw_total_time += enq_sw_last_time; 4650 4651 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time, 4652 stats.acc_offload_cycles); 4653 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time, 4654 stats.acc_offload_cycles); 4655 time_st->enq_acc_total_time += stats.acc_offload_cycles; 4656 4657 /* give time for device to process ops */ 4658 rte_delay_us(WAIT_OFFLOAD_US); 4659 4660 /* Start time meas for dequeue function offload latency */ 4661 deq_start_time = rte_rdtsc_precise(); 4662 /* Dequeue one operation */ 4663 do { 4664 deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id, 4665 &ops_deq[deq], enq); 4666 } while (unlikely(deq == 0)); 4667 4668 deq_last_time = rte_rdtsc_precise() - deq_start_time; 4669 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time, 4670 deq_last_time); 4671 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time, 4672 deq_last_time); 4673 time_st->deq_total_time += deq_last_time; 4674 4675 while (burst_sz != deq) 4676 deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id, 4677 &ops_deq[deq], burst_sz - deq); 4678 4679 rte_bbdev_enc_op_free_bulk(ops_enq, deq); 4680 dequeued += deq; 4681 } 4682 4683 return i; 4684 } 4685 #endif 4686 4687 static int 4688 offload_cost_test(struct active_device *ad, 4689 struct test_op_params *op_params) 4690 { 4691 #ifndef RTE_BBDEV_OFFLOAD_COST 4692 RTE_SET_USED(ad); 4693 RTE_SET_USED(op_params); 4694 printf("Offload latency test is disabled.\n"); 4695 printf("Set RTE_BBDEV_OFFLOAD_COST to 'y' to turn the test on.\n"); 4696 return TEST_SKIPPED; 4697 #else 4698 int iter; 4699 uint16_t burst_sz = op_params->burst_sz; 4700 const uint16_t num_to_process = op_params->num_to_process; 4701 const enum rte_bbdev_op_type op_type = test_vector.op_type; 4702 const uint16_t queue_id = ad->queue_ids[0]; 4703 struct test_buffers *bufs = NULL; 4704 struct rte_bbdev_info info; 4705 const char *op_type_str; 4706 struct test_time_stats time_st; 4707 4708 memset(&time_st, 0, sizeof(struct test_time_stats)); 4709 time_st.enq_sw_min_time = UINT64_MAX; 4710 time_st.enq_acc_min_time = UINT64_MAX; 4711 time_st.deq_min_time = UINT64_MAX; 4712 4713 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 4714 "BURST_SIZE should be <= %u", MAX_BURST); 4715 4716 rte_bbdev_info_get(ad->dev_id, &info); 4717 bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 4718 4719 op_type_str = rte_bbdev_op_type_str(op_type); 4720 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type); 4721 4722 printf("+ ------------------------------------------------------- +\n"); 4723 printf("== test: offload latency test\ndev: %s, burst size: %u, num ops: %u, op type: %s\n", 4724 info.dev_name, burst_sz, num_to_process, op_type_str); 4725 4726 if (op_type == RTE_BBDEV_OP_TURBO_DEC) 4727 iter = offload_latency_test_dec(op_params->mp, bufs, 4728 op_params->ref_dec_op, ad->dev_id, queue_id, 4729 num_to_process, burst_sz, &time_st); 4730 else if (op_type == RTE_BBDEV_OP_TURBO_ENC) 4731 iter = offload_latency_test_enc(op_params->mp, bufs, 4732 op_params->ref_enc_op, ad->dev_id, queue_id, 4733 num_to_process, burst_sz, &time_st); 4734 else if (op_type == RTE_BBDEV_OP_LDPC_ENC) 4735 iter = offload_latency_test_ldpc_enc(op_params->mp, bufs, 4736 op_params->ref_enc_op, ad->dev_id, queue_id, 4737 num_to_process, burst_sz, &time_st); 4738 else if (op_type == RTE_BBDEV_OP_LDPC_DEC) 4739 iter = offload_latency_test_ldpc_dec(op_params->mp, bufs, 4740 op_params->ref_dec_op, ad->dev_id, queue_id, 4741 num_to_process, burst_sz, &time_st); 4742 else 4743 iter = offload_latency_test_enc(op_params->mp, bufs, 4744 op_params->ref_enc_op, ad->dev_id, queue_id, 4745 num_to_process, burst_sz, &time_st); 4746 4747 if (iter <= 0) 4748 return TEST_FAILED; 4749 4750 printf("Enqueue driver offload cost latency:\n" 4751 "\tavg: %lg cycles, %lg us\n" 4752 "\tmin: %lg cycles, %lg us\n" 4753 "\tmax: %lg cycles, %lg us\n" 4754 "Enqueue accelerator offload cost latency:\n" 4755 "\tavg: %lg cycles, %lg us\n" 4756 "\tmin: %lg cycles, %lg us\n" 4757 "\tmax: %lg cycles, %lg us\n", 4758 (double)time_st.enq_sw_total_time / (double)iter, 4759 (double)(time_st.enq_sw_total_time * 1000000) / 4760 (double)iter / (double)rte_get_tsc_hz(), 4761 (double)time_st.enq_sw_min_time, 4762 (double)(time_st.enq_sw_min_time * 1000000) / 4763 rte_get_tsc_hz(), (double)time_st.enq_sw_max_time, 4764 (double)(time_st.enq_sw_max_time * 1000000) / 4765 rte_get_tsc_hz(), (double)time_st.enq_acc_total_time / 4766 (double)iter, 4767 (double)(time_st.enq_acc_total_time * 1000000) / 4768 (double)iter / (double)rte_get_tsc_hz(), 4769 (double)time_st.enq_acc_min_time, 4770 (double)(time_st.enq_acc_min_time * 1000000) / 4771 rte_get_tsc_hz(), (double)time_st.enq_acc_max_time, 4772 (double)(time_st.enq_acc_max_time * 1000000) / 4773 rte_get_tsc_hz()); 4774 4775 printf("Dequeue offload cost latency - one op:\n" 4776 "\tavg: %lg cycles, %lg us\n" 4777 "\tmin: %lg cycles, %lg us\n" 4778 "\tmax: %lg cycles, %lg us\n", 4779 (double)time_st.deq_total_time / (double)iter, 4780 (double)(time_st.deq_total_time * 1000000) / 4781 (double)iter / (double)rte_get_tsc_hz(), 4782 (double)time_st.deq_min_time, 4783 (double)(time_st.deq_min_time * 1000000) / 4784 rte_get_tsc_hz(), (double)time_st.deq_max_time, 4785 (double)(time_st.deq_max_time * 1000000) / 4786 rte_get_tsc_hz()); 4787 4788 struct rte_bbdev_stats stats = {0}; 4789 get_bbdev_queue_stats(ad->dev_id, queue_id, &stats); 4790 if (op_type != RTE_BBDEV_OP_LDPC_DEC) { 4791 TEST_ASSERT_SUCCESS(stats.enqueued_count != num_to_process, 4792 "Mismatch in enqueue count %10"PRIu64" %d", 4793 stats.enqueued_count, num_to_process); 4794 TEST_ASSERT_SUCCESS(stats.dequeued_count != num_to_process, 4795 "Mismatch in dequeue count %10"PRIu64" %d", 4796 stats.dequeued_count, num_to_process); 4797 } 4798 TEST_ASSERT_SUCCESS(stats.enqueue_err_count != 0, 4799 "Enqueue count Error %10"PRIu64"", 4800 stats.enqueue_err_count); 4801 TEST_ASSERT_SUCCESS(stats.dequeue_err_count != 0, 4802 "Dequeue count Error (%10"PRIu64"", 4803 stats.dequeue_err_count); 4804 4805 return TEST_SUCCESS; 4806 #endif 4807 } 4808 4809 #ifdef RTE_BBDEV_OFFLOAD_COST 4810 static int 4811 offload_latency_empty_q_test_dec(uint16_t dev_id, uint16_t queue_id, 4812 const uint16_t num_to_process, uint16_t burst_sz, 4813 uint64_t *deq_total_time, uint64_t *deq_min_time, 4814 uint64_t *deq_max_time, const enum rte_bbdev_op_type op_type) 4815 { 4816 int i, deq_total; 4817 struct rte_bbdev_dec_op *ops[MAX_BURST]; 4818 uint64_t deq_start_time, deq_last_time; 4819 4820 /* Test deq offload latency from an empty queue */ 4821 4822 for (i = 0, deq_total = 0; deq_total < num_to_process; 4823 ++i, deq_total += burst_sz) { 4824 deq_start_time = rte_rdtsc_precise(); 4825 4826 if (unlikely(num_to_process - deq_total < burst_sz)) 4827 burst_sz = num_to_process - deq_total; 4828 if (op_type == RTE_BBDEV_OP_LDPC_DEC) 4829 rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id, ops, 4830 burst_sz); 4831 else 4832 rte_bbdev_dequeue_dec_ops(dev_id, queue_id, ops, 4833 burst_sz); 4834 4835 deq_last_time = rte_rdtsc_precise() - deq_start_time; 4836 *deq_max_time = RTE_MAX(*deq_max_time, deq_last_time); 4837 *deq_min_time = RTE_MIN(*deq_min_time, deq_last_time); 4838 *deq_total_time += deq_last_time; 4839 } 4840 4841 return i; 4842 } 4843 4844 static int 4845 offload_latency_empty_q_test_enc(uint16_t dev_id, uint16_t queue_id, 4846 const uint16_t num_to_process, uint16_t burst_sz, 4847 uint64_t *deq_total_time, uint64_t *deq_min_time, 4848 uint64_t *deq_max_time, const enum rte_bbdev_op_type op_type) 4849 { 4850 int i, deq_total; 4851 struct rte_bbdev_enc_op *ops[MAX_BURST]; 4852 uint64_t deq_start_time, deq_last_time; 4853 4854 /* Test deq offload latency from an empty queue */ 4855 for (i = 0, deq_total = 0; deq_total < num_to_process; 4856 ++i, deq_total += burst_sz) { 4857 deq_start_time = rte_rdtsc_precise(); 4858 4859 if (unlikely(num_to_process - deq_total < burst_sz)) 4860 burst_sz = num_to_process - deq_total; 4861 if (op_type == RTE_BBDEV_OP_LDPC_ENC) 4862 rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id, ops, 4863 burst_sz); 4864 else 4865 rte_bbdev_dequeue_enc_ops(dev_id, queue_id, ops, 4866 burst_sz); 4867 4868 deq_last_time = rte_rdtsc_precise() - deq_start_time; 4869 *deq_max_time = RTE_MAX(*deq_max_time, deq_last_time); 4870 *deq_min_time = RTE_MIN(*deq_min_time, deq_last_time); 4871 *deq_total_time += deq_last_time; 4872 } 4873 4874 return i; 4875 } 4876 4877 #endif 4878 4879 static int 4880 offload_latency_empty_q_test(struct active_device *ad, 4881 struct test_op_params *op_params) 4882 { 4883 #ifndef RTE_BBDEV_OFFLOAD_COST 4884 RTE_SET_USED(ad); 4885 RTE_SET_USED(op_params); 4886 printf("Offload latency empty dequeue test is disabled.\n"); 4887 printf("Set RTE_BBDEV_OFFLOAD_COST to 'y' to turn the test on.\n"); 4888 return TEST_SKIPPED; 4889 #else 4890 int iter; 4891 uint64_t deq_total_time, deq_min_time, deq_max_time; 4892 uint16_t burst_sz = op_params->burst_sz; 4893 const uint16_t num_to_process = op_params->num_to_process; 4894 const enum rte_bbdev_op_type op_type = test_vector.op_type; 4895 const uint16_t queue_id = ad->queue_ids[0]; 4896 struct rte_bbdev_info info; 4897 const char *op_type_str; 4898 4899 deq_total_time = deq_max_time = 0; 4900 deq_min_time = UINT64_MAX; 4901 4902 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 4903 "BURST_SIZE should be <= %u", MAX_BURST); 4904 4905 rte_bbdev_info_get(ad->dev_id, &info); 4906 4907 op_type_str = rte_bbdev_op_type_str(op_type); 4908 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type); 4909 4910 printf("+ ------------------------------------------------------- +\n"); 4911 printf("== test: offload latency empty dequeue\ndev: %s, burst size: %u, num ops: %u, op type: %s\n", 4912 info.dev_name, burst_sz, num_to_process, op_type_str); 4913 4914 if (op_type == RTE_BBDEV_OP_TURBO_DEC || 4915 op_type == RTE_BBDEV_OP_LDPC_DEC) 4916 iter = offload_latency_empty_q_test_dec(ad->dev_id, queue_id, 4917 num_to_process, burst_sz, &deq_total_time, 4918 &deq_min_time, &deq_max_time, op_type); 4919 else 4920 iter = offload_latency_empty_q_test_enc(ad->dev_id, queue_id, 4921 num_to_process, burst_sz, &deq_total_time, 4922 &deq_min_time, &deq_max_time, op_type); 4923 4924 if (iter <= 0) 4925 return TEST_FAILED; 4926 4927 printf("Empty dequeue offload:\n" 4928 "\tavg: %lg cycles, %lg us\n" 4929 "\tmin: %lg cycles, %lg us\n" 4930 "\tmax: %lg cycles, %lg us\n", 4931 (double)deq_total_time / (double)iter, 4932 (double)(deq_total_time * 1000000) / (double)iter / 4933 (double)rte_get_tsc_hz(), (double)deq_min_time, 4934 (double)(deq_min_time * 1000000) / rte_get_tsc_hz(), 4935 (double)deq_max_time, (double)(deq_max_time * 1000000) / 4936 rte_get_tsc_hz()); 4937 4938 return TEST_SUCCESS; 4939 #endif 4940 } 4941 4942 static int 4943 bler_tc(void) 4944 { 4945 return run_test_case(bler_test); 4946 } 4947 4948 static int 4949 throughput_tc(void) 4950 { 4951 return run_test_case(throughput_test); 4952 } 4953 4954 static int 4955 offload_cost_tc(void) 4956 { 4957 return run_test_case(offload_cost_test); 4958 } 4959 4960 static int 4961 offload_latency_empty_q_tc(void) 4962 { 4963 return run_test_case(offload_latency_empty_q_test); 4964 } 4965 4966 static int 4967 latency_tc(void) 4968 { 4969 return run_test_case(latency_test); 4970 } 4971 4972 static int 4973 validation_tc(void) 4974 { 4975 return run_test_case(validation_test); 4976 } 4977 4978 static int 4979 interrupt_tc(void) 4980 { 4981 return run_test_case(throughput_test); 4982 } 4983 4984 static struct unit_test_suite bbdev_bler_testsuite = { 4985 .suite_name = "BBdev BLER Tests", 4986 .setup = testsuite_setup, 4987 .teardown = testsuite_teardown, 4988 .unit_test_cases = { 4989 TEST_CASE_ST(ut_setup, ut_teardown, bler_tc), 4990 TEST_CASES_END() /**< NULL terminate unit test array */ 4991 } 4992 }; 4993 4994 static struct unit_test_suite bbdev_throughput_testsuite = { 4995 .suite_name = "BBdev Throughput Tests", 4996 .setup = testsuite_setup, 4997 .teardown = testsuite_teardown, 4998 .unit_test_cases = { 4999 TEST_CASE_ST(ut_setup, ut_teardown, throughput_tc), 5000 TEST_CASES_END() /**< NULL terminate unit test array */ 5001 } 5002 }; 5003 5004 static struct unit_test_suite bbdev_validation_testsuite = { 5005 .suite_name = "BBdev Validation Tests", 5006 .setup = testsuite_setup, 5007 .teardown = testsuite_teardown, 5008 .unit_test_cases = { 5009 TEST_CASE_ST(ut_setup, ut_teardown, validation_tc), 5010 TEST_CASES_END() /**< NULL terminate unit test array */ 5011 } 5012 }; 5013 5014 static struct unit_test_suite bbdev_latency_testsuite = { 5015 .suite_name = "BBdev Latency Tests", 5016 .setup = testsuite_setup, 5017 .teardown = testsuite_teardown, 5018 .unit_test_cases = { 5019 TEST_CASE_ST(ut_setup, ut_teardown, latency_tc), 5020 TEST_CASES_END() /**< NULL terminate unit test array */ 5021 } 5022 }; 5023 5024 static struct unit_test_suite bbdev_offload_cost_testsuite = { 5025 .suite_name = "BBdev Offload Cost Tests", 5026 .setup = testsuite_setup, 5027 .teardown = testsuite_teardown, 5028 .unit_test_cases = { 5029 TEST_CASE_ST(ut_setup, ut_teardown, offload_cost_tc), 5030 TEST_CASE_ST(ut_setup, ut_teardown, offload_latency_empty_q_tc), 5031 TEST_CASES_END() /**< NULL terminate unit test array */ 5032 } 5033 }; 5034 5035 static struct unit_test_suite bbdev_interrupt_testsuite = { 5036 .suite_name = "BBdev Interrupt Tests", 5037 .setup = interrupt_testsuite_setup, 5038 .teardown = testsuite_teardown, 5039 .unit_test_cases = { 5040 TEST_CASE_ST(ut_setup, ut_teardown, interrupt_tc), 5041 TEST_CASES_END() /**< NULL terminate unit test array */ 5042 } 5043 }; 5044 5045 REGISTER_TEST_COMMAND(bler, bbdev_bler_testsuite); 5046 REGISTER_TEST_COMMAND(throughput, bbdev_throughput_testsuite); 5047 REGISTER_TEST_COMMAND(validation, bbdev_validation_testsuite); 5048 REGISTER_TEST_COMMAND(latency, bbdev_latency_testsuite); 5049 REGISTER_TEST_COMMAND(offload, bbdev_offload_cost_testsuite); 5050 REGISTER_TEST_COMMAND(interrupt, bbdev_interrupt_testsuite); 5051