1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017 Intel Corporation 3 */ 4 5 #include <stdio.h> 6 #include <stdlib.h> 7 #include <inttypes.h> 8 #include <math.h> 9 10 #include <rte_eal.h> 11 #include <rte_common.h> 12 #include <rte_dev.h> 13 #include <rte_launch.h> 14 #include <rte_bbdev.h> 15 #include <rte_cycles.h> 16 #include <rte_lcore.h> 17 #include <rte_malloc.h> 18 #include <rte_random.h> 19 #include <rte_hexdump.h> 20 #include <rte_interrupts.h> 21 22 #include "main.h" 23 #include "test_bbdev_vector.h" 24 25 #define GET_SOCKET(socket_id) (((socket_id) == SOCKET_ID_ANY) ? 0 : (socket_id)) 26 27 #define MAX_QUEUES RTE_MAX_LCORE 28 #define TEST_REPETITIONS 100 29 #define TIME_OUT_POLL 1e9 30 #define WAIT_OFFLOAD_US 1000 31 32 #ifdef RTE_BASEBAND_FPGA_LTE_FEC 33 #include <fpga_lte_fec.h> 34 #define FPGA_LTE_PF_DRIVER_NAME ("intel_fpga_lte_fec_pf") 35 #define FPGA_LTE_VF_DRIVER_NAME ("intel_fpga_lte_fec_vf") 36 #define VF_UL_4G_QUEUE_VALUE 4 37 #define VF_DL_4G_QUEUE_VALUE 4 38 #define UL_4G_BANDWIDTH 3 39 #define DL_4G_BANDWIDTH 3 40 #define UL_4G_LOAD_BALANCE 128 41 #define DL_4G_LOAD_BALANCE 128 42 #define FLR_4G_TIMEOUT 610 43 #endif 44 45 #ifdef RTE_BASEBAND_FPGA_5GNR_FEC 46 #include <rte_pmd_fpga_5gnr_fec.h> 47 #define FPGA_5GNR_PF_DRIVER_NAME ("intel_fpga_5gnr_fec_pf") 48 #define FPGA_5GNR_VF_DRIVER_NAME ("intel_fpga_5gnr_fec_vf") 49 #define VF_UL_5G_QUEUE_VALUE 4 50 #define VF_DL_5G_QUEUE_VALUE 4 51 #define UL_5G_BANDWIDTH 3 52 #define DL_5G_BANDWIDTH 3 53 #define UL_5G_LOAD_BALANCE 128 54 #define DL_5G_LOAD_BALANCE 128 55 #endif 56 57 #ifdef RTE_BASEBAND_ACC 58 #include <rte_acc_cfg.h> 59 #define ACC100PF_DRIVER_NAME ("intel_acc100_pf") 60 #define ACC100VF_DRIVER_NAME ("intel_acc100_vf") 61 #define ACC100_QMGR_NUM_AQS 16 62 #define ACC100_QMGR_NUM_QGS 2 63 #define ACC100_QMGR_AQ_DEPTH 5 64 #define ACC100_QMGR_INVALID_IDX -1 65 #define ACC100_QMGR_RR 1 66 #define ACC100_QOS_GBR 0 67 #define VRBPF_DRIVER_NAME ("intel_vran_boost_pf") 68 #define VRBVF_DRIVER_NAME ("intel_vran_boost_vf") 69 #define VRB_QMGR_NUM_AQS 16 70 #define VRB_QMGR_NUM_QGS 2 71 #define VRB_QMGR_AQ_DEPTH 5 72 #define VRB_QMGR_INVALID_IDX -1 73 #define VRB_QMGR_RR 1 74 #define VRB_QOS_GBR 0 75 #endif 76 77 #define OPS_CACHE_SIZE 256U 78 #define OPS_POOL_SIZE_MIN 511U /* 0.5K per queue */ 79 80 #define SYNC_WAIT 0 81 #define SYNC_START 1 82 83 #define INVALID_QUEUE_ID -1 84 /* Increment for next code block in external HARQ memory */ 85 #define HARQ_INCR 32768 86 /* Headroom for filler LLRs insertion in HARQ buffer */ 87 #define FILLER_HEADROOM 2048 88 /* Constants from K0 computation from 3GPP 38.212 Table 5.4.2.1-2 */ 89 #define N_ZC_1 66 /* N = 66 Zc for BG 1 */ 90 #define N_ZC_2 50 /* N = 50 Zc for BG 2 */ 91 #define K0_1_1 17 /* K0 fraction numerator for rv 1 and BG 1 */ 92 #define K0_1_2 13 /* K0 fraction numerator for rv 1 and BG 2 */ 93 #define K0_2_1 33 /* K0 fraction numerator for rv 2 and BG 1 */ 94 #define K0_2_2 25 /* K0 fraction numerator for rv 2 and BG 2 */ 95 #define K0_3_1 56 /* K0 fraction numerator for rv 3 and BG 1 */ 96 #define K0_3_2 43 /* K0 fraction numerator for rv 3 and BG 2 */ 97 98 #define HARQ_MEM_TOLERANCE 256 99 static struct test_bbdev_vector test_vector; 100 101 /* Switch between PMD and Interrupt for throughput TC */ 102 static bool intr_enabled; 103 104 /* LLR arithmetic representation for numerical conversion */ 105 static int ldpc_llr_decimals; 106 static int ldpc_llr_size; 107 /* Keep track of the LDPC decoder device capability flag */ 108 static uint32_t ldpc_cap_flags; 109 110 /* Represents tested active devices */ 111 static struct active_device { 112 const char *driver_name; 113 uint8_t dev_id; 114 uint16_t supported_ops; 115 uint16_t queue_ids[MAX_QUEUES]; 116 uint16_t nb_queues; 117 struct rte_mempool *ops_mempool; 118 struct rte_mempool *in_mbuf_pool; 119 struct rte_mempool *hard_out_mbuf_pool; 120 struct rte_mempool *soft_out_mbuf_pool; 121 struct rte_mempool *harq_in_mbuf_pool; 122 struct rte_mempool *harq_out_mbuf_pool; 123 } active_devs[RTE_BBDEV_MAX_DEVS]; 124 125 static uint8_t nb_active_devs; 126 127 /* Data buffers used by BBDEV ops */ 128 struct test_buffers { 129 struct rte_bbdev_op_data *inputs; 130 struct rte_bbdev_op_data *hard_outputs; 131 struct rte_bbdev_op_data *soft_outputs; 132 struct rte_bbdev_op_data *harq_inputs; 133 struct rte_bbdev_op_data *harq_outputs; 134 }; 135 136 /* Operation parameters specific for given test case */ 137 struct test_op_params { 138 struct rte_mempool *mp; 139 struct rte_bbdev_dec_op *ref_dec_op; 140 struct rte_bbdev_enc_op *ref_enc_op; 141 struct rte_bbdev_fft_op *ref_fft_op; 142 struct rte_bbdev_mldts_op *ref_mldts_op; 143 uint16_t burst_sz; 144 uint16_t num_to_process; 145 uint16_t num_lcores; 146 int vector_mask; 147 uint16_t sync; 148 struct test_buffers q_bufs[RTE_MAX_NUMA_NODES][MAX_QUEUES]; 149 }; 150 151 /* Contains per lcore params */ 152 struct thread_params { 153 uint8_t dev_id; 154 uint16_t queue_id; 155 uint32_t lcore_id; 156 uint64_t start_time; 157 double ops_per_sec; 158 double mbps; 159 uint8_t iter_count; 160 double iter_average; 161 double bler; 162 uint16_t nb_dequeued; 163 int16_t processing_status; 164 uint16_t burst_sz; 165 struct test_op_params *op_params; 166 struct rte_bbdev_dec_op *dec_ops[MAX_BURST]; 167 struct rte_bbdev_enc_op *enc_ops[MAX_BURST]; 168 struct rte_bbdev_fft_op *fft_ops[MAX_BURST]; 169 struct rte_bbdev_mldts_op *mldts_ops[MAX_BURST]; 170 }; 171 172 /* Stores time statistics */ 173 struct test_time_stats { 174 /* Stores software enqueue total working time */ 175 uint64_t enq_sw_total_time; 176 /* Stores minimum value of software enqueue working time */ 177 uint64_t enq_sw_min_time; 178 /* Stores maximum value of software enqueue working time */ 179 uint64_t enq_sw_max_time; 180 /* Stores turbo enqueue total working time */ 181 uint64_t enq_acc_total_time; 182 /* Stores minimum value of accelerator enqueue working time */ 183 uint64_t enq_acc_min_time; 184 /* Stores maximum value of accelerator enqueue working time */ 185 uint64_t enq_acc_max_time; 186 /* Stores dequeue total working time */ 187 uint64_t deq_total_time; 188 /* Stores minimum value of dequeue working time */ 189 uint64_t deq_min_time; 190 /* Stores maximum value of dequeue working time */ 191 uint64_t deq_max_time; 192 }; 193 194 typedef int (test_case_function)(struct active_device *ad, 195 struct test_op_params *op_params); 196 197 /* Get device status before timeout exit */ 198 static inline void 199 timeout_exit(uint8_t dev_id) 200 { 201 struct rte_bbdev_info info; 202 rte_bbdev_info_get(dev_id, &info); 203 printf("Device Status %s\n", rte_bbdev_device_status_str(info.drv.device_status)); 204 } 205 206 static inline void 207 mbuf_reset(struct rte_mbuf *m) 208 { 209 m->pkt_len = 0; 210 211 do { 212 m->data_len = 0; 213 m = m->next; 214 } while (m != NULL); 215 } 216 217 /* Read flag value 0/1 from bitmap */ 218 static inline bool 219 check_bit(uint32_t bitmap, uint32_t bitmask) 220 { 221 return bitmap & bitmask; 222 } 223 224 static inline void 225 set_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type) 226 { 227 ad->supported_ops |= (1 << op_type); 228 } 229 230 static inline bool 231 is_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type) 232 { 233 return ad->supported_ops & (1 << op_type); 234 } 235 236 static inline bool 237 flags_match(uint32_t flags_req, uint32_t flags_present) 238 { 239 return (flags_req & flags_present) == flags_req; 240 } 241 242 static void 243 clear_soft_out_cap(uint32_t *op_flags) 244 { 245 *op_flags &= ~RTE_BBDEV_TURBO_SOFT_OUTPUT; 246 *op_flags &= ~RTE_BBDEV_TURBO_POS_LLR_1_BIT_SOFT_OUT; 247 *op_flags &= ~RTE_BBDEV_TURBO_NEG_LLR_1_BIT_SOFT_OUT; 248 } 249 250 /* This API is to convert all the test vector op data entries 251 * to big endian format. It is used when the device supports 252 * the input in the big endian format. 253 */ 254 static inline void 255 convert_op_data_to_be(void) 256 { 257 struct op_data_entries *op; 258 enum op_data_type type; 259 uint8_t nb_segs, *rem_data, temp; 260 uint32_t *data, len; 261 int complete, rem, i, j; 262 263 for (type = DATA_INPUT; type < DATA_NUM_TYPES; ++type) { 264 nb_segs = test_vector.entries[type].nb_segments; 265 op = &test_vector.entries[type]; 266 267 /* Invert byte endianness for all the segments */ 268 for (i = 0; i < nb_segs; ++i) { 269 len = op->segments[i].length; 270 data = op->segments[i].addr; 271 272 /* Swap complete u32 bytes */ 273 complete = len / 4; 274 for (j = 0; j < complete; j++) 275 data[j] = rte_bswap32(data[j]); 276 277 /* Swap any remaining bytes */ 278 rem = len % 4; 279 rem_data = (uint8_t *)&data[j]; 280 for (j = 0; j < rem/2; j++) { 281 temp = rem_data[j]; 282 rem_data[j] = rem_data[rem - j - 1]; 283 rem_data[rem - j - 1] = temp; 284 } 285 } 286 } 287 } 288 289 static int 290 check_dev_cap(const struct rte_bbdev_info *dev_info) 291 { 292 unsigned int i; 293 unsigned int nb_inputs, nb_soft_outputs, nb_hard_outputs, 294 nb_harq_inputs, nb_harq_outputs; 295 const struct rte_bbdev_op_cap *op_cap = dev_info->drv.capabilities; 296 uint8_t dev_data_endianness = dev_info->drv.data_endianness; 297 298 nb_inputs = test_vector.entries[DATA_INPUT].nb_segments; 299 nb_soft_outputs = test_vector.entries[DATA_SOFT_OUTPUT].nb_segments; 300 nb_hard_outputs = test_vector.entries[DATA_HARD_OUTPUT].nb_segments; 301 nb_harq_inputs = test_vector.entries[DATA_HARQ_INPUT].nb_segments; 302 nb_harq_outputs = test_vector.entries[DATA_HARQ_OUTPUT].nb_segments; 303 304 for (i = 0; op_cap->type != RTE_BBDEV_OP_NONE; ++i, ++op_cap) { 305 if (op_cap->type != test_vector.op_type) 306 continue; 307 308 if (dev_data_endianness == RTE_BIG_ENDIAN) 309 convert_op_data_to_be(); 310 311 if (op_cap->type == RTE_BBDEV_OP_TURBO_DEC) { 312 const struct rte_bbdev_op_cap_turbo_dec *cap = 313 &op_cap->cap.turbo_dec; 314 /* Ignore lack of soft output capability, just skip 315 * checking if soft output is valid. 316 */ 317 if ((test_vector.turbo_dec.op_flags & 318 RTE_BBDEV_TURBO_SOFT_OUTPUT) && 319 !(cap->capability_flags & 320 RTE_BBDEV_TURBO_SOFT_OUTPUT)) { 321 printf( 322 "INFO: Device \"%s\" does not support soft output - soft output flags will be ignored.\n", 323 dev_info->dev_name); 324 clear_soft_out_cap( 325 &test_vector.turbo_dec.op_flags); 326 } 327 328 if (!flags_match(test_vector.turbo_dec.op_flags, 329 cap->capability_flags)) 330 return TEST_FAILED; 331 if (nb_inputs > cap->num_buffers_src) { 332 printf("Too many inputs defined: %u, max: %u\n", 333 nb_inputs, cap->num_buffers_src); 334 return TEST_FAILED; 335 } 336 if (nb_soft_outputs > cap->num_buffers_soft_out && 337 (test_vector.turbo_dec.op_flags & 338 RTE_BBDEV_TURBO_SOFT_OUTPUT)) { 339 printf( 340 "Too many soft outputs defined: %u, max: %u\n", 341 nb_soft_outputs, 342 cap->num_buffers_soft_out); 343 return TEST_FAILED; 344 } 345 if (nb_hard_outputs > cap->num_buffers_hard_out) { 346 printf( 347 "Too many hard outputs defined: %u, max: %u\n", 348 nb_hard_outputs, 349 cap->num_buffers_hard_out); 350 return TEST_FAILED; 351 } 352 if (intr_enabled && !(cap->capability_flags & 353 RTE_BBDEV_TURBO_DEC_INTERRUPTS)) { 354 printf( 355 "Dequeue interrupts are not supported!\n"); 356 return TEST_FAILED; 357 } 358 359 return TEST_SUCCESS; 360 } else if (op_cap->type == RTE_BBDEV_OP_TURBO_ENC) { 361 const struct rte_bbdev_op_cap_turbo_enc *cap = 362 &op_cap->cap.turbo_enc; 363 364 if (!flags_match(test_vector.turbo_enc.op_flags, 365 cap->capability_flags)) 366 return TEST_FAILED; 367 if (nb_inputs > cap->num_buffers_src) { 368 printf("Too many inputs defined: %u, max: %u\n", 369 nb_inputs, cap->num_buffers_src); 370 return TEST_FAILED; 371 } 372 if (nb_hard_outputs > cap->num_buffers_dst) { 373 printf( 374 "Too many hard outputs defined: %u, max: %u\n", 375 nb_hard_outputs, cap->num_buffers_dst); 376 return TEST_FAILED; 377 } 378 if (intr_enabled && !(cap->capability_flags & 379 RTE_BBDEV_TURBO_ENC_INTERRUPTS)) { 380 printf( 381 "Dequeue interrupts are not supported!\n"); 382 return TEST_FAILED; 383 } 384 385 return TEST_SUCCESS; 386 } else if (op_cap->type == RTE_BBDEV_OP_LDPC_ENC) { 387 const struct rte_bbdev_op_cap_ldpc_enc *cap = 388 &op_cap->cap.ldpc_enc; 389 390 if (!flags_match(test_vector.ldpc_enc.op_flags, 391 cap->capability_flags)){ 392 printf("Flag Mismatch\n"); 393 return TEST_FAILED; 394 } 395 if (nb_inputs > cap->num_buffers_src) { 396 printf("Too many inputs defined: %u, max: %u\n", 397 nb_inputs, cap->num_buffers_src); 398 return TEST_FAILED; 399 } 400 if (nb_hard_outputs > cap->num_buffers_dst) { 401 printf( 402 "Too many hard outputs defined: %u, max: %u\n", 403 nb_hard_outputs, cap->num_buffers_dst); 404 return TEST_FAILED; 405 } 406 if (intr_enabled && !(cap->capability_flags & 407 RTE_BBDEV_LDPC_ENC_INTERRUPTS)) { 408 printf( 409 "Dequeue interrupts are not supported!\n"); 410 return TEST_FAILED; 411 } 412 413 return TEST_SUCCESS; 414 } else if (op_cap->type == RTE_BBDEV_OP_LDPC_DEC) { 415 const struct rte_bbdev_op_cap_ldpc_dec *cap = 416 &op_cap->cap.ldpc_dec; 417 418 if (!flags_match(test_vector.ldpc_dec.op_flags, 419 cap->capability_flags)){ 420 printf("Flag Mismatch\n"); 421 return TEST_FAILED; 422 } 423 if (nb_inputs > cap->num_buffers_src) { 424 printf("Too many inputs defined: %u, max: %u\n", 425 nb_inputs, cap->num_buffers_src); 426 return TEST_FAILED; 427 } 428 if (nb_hard_outputs > cap->num_buffers_hard_out) { 429 printf( 430 "Too many hard outputs defined: %u, max: %u\n", 431 nb_hard_outputs, 432 cap->num_buffers_hard_out); 433 return TEST_FAILED; 434 } 435 if (nb_harq_inputs > cap->num_buffers_hard_out) { 436 printf( 437 "Too many HARQ inputs defined: %u, max: %u\n", 438 nb_harq_inputs, 439 cap->num_buffers_hard_out); 440 return TEST_FAILED; 441 } 442 if (nb_harq_outputs > cap->num_buffers_hard_out) { 443 printf( 444 "Too many HARQ outputs defined: %u, max: %u\n", 445 nb_harq_outputs, 446 cap->num_buffers_hard_out); 447 return TEST_FAILED; 448 } 449 if (intr_enabled && !(cap->capability_flags & 450 RTE_BBDEV_LDPC_DEC_INTERRUPTS)) { 451 printf( 452 "Dequeue interrupts are not supported!\n"); 453 return TEST_FAILED; 454 } 455 if (intr_enabled && (test_vector.ldpc_dec.op_flags & 456 (RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE | 457 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE | 458 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK 459 ))) { 460 printf("Skip loop-back with interrupt\n"); 461 return TEST_FAILED; 462 } 463 return TEST_SUCCESS; 464 } else if (op_cap->type == RTE_BBDEV_OP_FFT) { 465 const struct rte_bbdev_op_cap_fft *cap = &op_cap->cap.fft; 466 467 if (!flags_match(test_vector.fft.op_flags, cap->capability_flags)) { 468 printf("Flag Mismatch\n"); 469 return TEST_FAILED; 470 } 471 if (nb_inputs > cap->num_buffers_src) { 472 printf("Too many inputs defined: %u, max: %u\n", 473 nb_inputs, cap->num_buffers_src); 474 return TEST_FAILED; 475 } 476 return TEST_SUCCESS; 477 } else if (op_cap->type == RTE_BBDEV_OP_MLDTS) { 478 const struct rte_bbdev_op_cap_mld *cap = &op_cap->cap.mld; 479 if (!flags_match(test_vector.mldts.op_flags, cap->capability_flags)) { 480 printf("Flag Mismatch\n"); 481 return TEST_FAILED; 482 } 483 if (nb_inputs > cap->num_buffers_src) { 484 printf("Too many inputs defined: %u, max: %u\n", 485 nb_inputs, cap->num_buffers_src); 486 return TEST_FAILED; 487 } 488 return TEST_SUCCESS; 489 } 490 } 491 492 if ((i == 0) && (test_vector.op_type == RTE_BBDEV_OP_NONE)) 493 return TEST_SUCCESS; /* Special case for NULL device */ 494 495 return TEST_FAILED; 496 } 497 498 /* calculates optimal mempool size not smaller than the val */ 499 static unsigned int 500 optimal_mempool_size(unsigned int val) 501 { 502 return rte_align32pow2(val + 1) - 1; 503 } 504 505 /* allocates mbuf mempool for inputs and outputs */ 506 static struct rte_mempool * 507 create_mbuf_pool(struct op_data_entries *entries, uint8_t dev_id, 508 int socket_id, unsigned int mbuf_pool_size, 509 const char *op_type_str) 510 { 511 unsigned int i; 512 uint32_t max_seg_sz = 0; 513 char pool_name[RTE_MEMPOOL_NAMESIZE]; 514 515 /* find max input segment size */ 516 for (i = 0; i < entries->nb_segments; ++i) 517 if (entries->segments[i].length > max_seg_sz) 518 max_seg_sz = entries->segments[i].length; 519 520 snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str, 521 dev_id); 522 return rte_pktmbuf_pool_create(pool_name, mbuf_pool_size, 0, 0, 523 RTE_MAX(max_seg_sz + RTE_PKTMBUF_HEADROOM 524 + FILLER_HEADROOM, 525 (unsigned int)RTE_MBUF_DEFAULT_BUF_SIZE), socket_id); 526 } 527 528 static int 529 create_mempools(struct active_device *ad, int socket_id, 530 enum rte_bbdev_op_type org_op_type, uint16_t num_ops) 531 { 532 struct rte_mempool *mp; 533 unsigned int ops_pool_size, mbuf_pool_size = 0; 534 char pool_name[RTE_MEMPOOL_NAMESIZE]; 535 const char *op_type_str; 536 enum rte_bbdev_op_type op_type = org_op_type; 537 538 struct op_data_entries *in = &test_vector.entries[DATA_INPUT]; 539 struct op_data_entries *hard_out = 540 &test_vector.entries[DATA_HARD_OUTPUT]; 541 struct op_data_entries *soft_out = 542 &test_vector.entries[DATA_SOFT_OUTPUT]; 543 struct op_data_entries *harq_in = 544 &test_vector.entries[DATA_HARQ_INPUT]; 545 struct op_data_entries *harq_out = 546 &test_vector.entries[DATA_HARQ_OUTPUT]; 547 548 /* allocate ops mempool */ 549 ops_pool_size = optimal_mempool_size(RTE_MAX( 550 /* Ops used plus 1 reference op */ 551 RTE_MAX((unsigned int)(ad->nb_queues * num_ops + 1), 552 /* Minimal cache size plus 1 reference op */ 553 (unsigned int)(1.5 * rte_lcore_count() * 554 OPS_CACHE_SIZE + 1)), 555 OPS_POOL_SIZE_MIN)); 556 557 if (org_op_type == RTE_BBDEV_OP_NONE) 558 op_type = RTE_BBDEV_OP_TURBO_ENC; 559 560 op_type_str = rte_bbdev_op_type_str(op_type); 561 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type); 562 563 snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str, 564 ad->dev_id); 565 mp = rte_bbdev_op_pool_create(pool_name, op_type, 566 ops_pool_size, OPS_CACHE_SIZE, socket_id); 567 TEST_ASSERT_NOT_NULL(mp, 568 "ERROR Failed to create %u items ops pool for dev %u on socket %u.", 569 ops_pool_size, 570 ad->dev_id, 571 socket_id); 572 ad->ops_mempool = mp; 573 574 /* Do not create inputs and outputs mbufs for BaseBand Null Device */ 575 if (org_op_type == RTE_BBDEV_OP_NONE) 576 return TEST_SUCCESS; 577 578 /* Inputs */ 579 if (in->nb_segments > 0) { 580 mbuf_pool_size = optimal_mempool_size(ops_pool_size * 581 in->nb_segments); 582 mp = create_mbuf_pool(in, ad->dev_id, socket_id, 583 mbuf_pool_size, "in"); 584 TEST_ASSERT_NOT_NULL(mp, 585 "ERROR Failed to create %u items input pktmbuf pool for dev %u on socket %u.", 586 mbuf_pool_size, 587 ad->dev_id, 588 socket_id); 589 ad->in_mbuf_pool = mp; 590 } 591 592 /* Hard outputs */ 593 if (hard_out->nb_segments > 0) { 594 mbuf_pool_size = optimal_mempool_size(ops_pool_size * 595 hard_out->nb_segments); 596 mp = create_mbuf_pool(hard_out, ad->dev_id, socket_id, 597 mbuf_pool_size, 598 "hard_out"); 599 TEST_ASSERT_NOT_NULL(mp, 600 "ERROR Failed to create %u items hard output pktmbuf pool for dev %u on socket %u.", 601 mbuf_pool_size, 602 ad->dev_id, 603 socket_id); 604 ad->hard_out_mbuf_pool = mp; 605 } 606 607 /* Soft outputs */ 608 if (soft_out->nb_segments > 0) { 609 mbuf_pool_size = optimal_mempool_size(ops_pool_size * 610 soft_out->nb_segments); 611 mp = create_mbuf_pool(soft_out, ad->dev_id, socket_id, 612 mbuf_pool_size, 613 "soft_out"); 614 TEST_ASSERT_NOT_NULL(mp, 615 "ERROR Failed to create %uB soft output pktmbuf pool for dev %u on socket %u.", 616 mbuf_pool_size, 617 ad->dev_id, 618 socket_id); 619 ad->soft_out_mbuf_pool = mp; 620 } 621 622 /* HARQ inputs */ 623 if (harq_in->nb_segments > 0) { 624 mbuf_pool_size = optimal_mempool_size(ops_pool_size * 625 harq_in->nb_segments); 626 mp = create_mbuf_pool(harq_in, ad->dev_id, socket_id, 627 mbuf_pool_size, 628 "harq_in"); 629 TEST_ASSERT_NOT_NULL(mp, 630 "ERROR Failed to create %uB harq input pktmbuf pool for dev %u on socket %u.", 631 mbuf_pool_size, 632 ad->dev_id, 633 socket_id); 634 ad->harq_in_mbuf_pool = mp; 635 } 636 637 /* HARQ outputs */ 638 if (harq_out->nb_segments > 0) { 639 mbuf_pool_size = optimal_mempool_size(ops_pool_size * 640 harq_out->nb_segments); 641 mp = create_mbuf_pool(harq_out, ad->dev_id, socket_id, 642 mbuf_pool_size, 643 "harq_out"); 644 TEST_ASSERT_NOT_NULL(mp, 645 "ERROR Failed to create %uB harq output pktmbuf pool for dev %u on socket %u.", 646 mbuf_pool_size, 647 ad->dev_id, 648 socket_id); 649 ad->harq_out_mbuf_pool = mp; 650 } 651 652 return TEST_SUCCESS; 653 } 654 655 static int 656 add_bbdev_dev(uint8_t dev_id, struct rte_bbdev_info *info, 657 struct test_bbdev_vector *vector) 658 { 659 int ret; 660 unsigned int queue_id; 661 struct rte_bbdev_queue_conf qconf; 662 struct active_device *ad = &active_devs[nb_active_devs]; 663 unsigned int nb_queues; 664 enum rte_bbdev_op_type op_type = vector->op_type; 665 666 /* Configure fpga lte fec with PF & VF values 667 * if '-i' flag is set and using fpga device 668 */ 669 #ifdef RTE_BASEBAND_FPGA_LTE_FEC 670 if ((get_init_device() == true) && 671 (!strcmp(info->drv.driver_name, FPGA_LTE_PF_DRIVER_NAME))) { 672 struct rte_fpga_lte_fec_conf conf; 673 unsigned int i; 674 675 printf("Configure FPGA LTE FEC Driver %s with default values\n", 676 info->drv.driver_name); 677 678 /* clear default configuration before initialization */ 679 memset(&conf, 0, sizeof(struct rte_fpga_lte_fec_conf)); 680 681 /* Set PF mode : 682 * true if PF is used for data plane 683 * false for VFs 684 */ 685 conf.pf_mode_en = true; 686 687 for (i = 0; i < FPGA_LTE_FEC_NUM_VFS; ++i) { 688 /* Number of UL queues per VF (fpga supports 8 VFs) */ 689 conf.vf_ul_queues_number[i] = VF_UL_4G_QUEUE_VALUE; 690 /* Number of DL queues per VF (fpga supports 8 VFs) */ 691 conf.vf_dl_queues_number[i] = VF_DL_4G_QUEUE_VALUE; 692 } 693 694 /* UL bandwidth. Needed for schedule algorithm */ 695 conf.ul_bandwidth = UL_4G_BANDWIDTH; 696 /* DL bandwidth */ 697 conf.dl_bandwidth = DL_4G_BANDWIDTH; 698 699 /* UL & DL load Balance Factor to 64 */ 700 conf.ul_load_balance = UL_4G_LOAD_BALANCE; 701 conf.dl_load_balance = DL_4G_LOAD_BALANCE; 702 703 /**< FLR timeout value */ 704 conf.flr_time_out = FLR_4G_TIMEOUT; 705 706 /* setup FPGA PF with configuration information */ 707 ret = rte_fpga_lte_fec_configure(info->dev_name, &conf); 708 TEST_ASSERT_SUCCESS(ret, 709 "Failed to configure 4G FPGA PF for bbdev %s", 710 info->dev_name); 711 } 712 #endif 713 #ifdef RTE_BASEBAND_FPGA_5GNR_FEC 714 if ((get_init_device() == true) && 715 (!strcmp(info->drv.driver_name, FPGA_5GNR_PF_DRIVER_NAME))) { 716 struct rte_fpga_5gnr_fec_conf conf; 717 unsigned int i; 718 719 printf("Configure FPGA 5GNR FEC Driver %s with default values\n", 720 info->drv.driver_name); 721 722 /* clear default configuration before initialization */ 723 memset(&conf, 0, sizeof(struct rte_fpga_5gnr_fec_conf)); 724 725 /* Set PF mode : 726 * true if PF is used for data plane 727 * false for VFs 728 */ 729 conf.pf_mode_en = true; 730 731 for (i = 0; i < FPGA_5GNR_FEC_NUM_VFS; ++i) { 732 /* Number of UL queues per VF (fpga supports 8 VFs) */ 733 conf.vf_ul_queues_number[i] = VF_UL_5G_QUEUE_VALUE; 734 /* Number of DL queues per VF (fpga supports 8 VFs) */ 735 conf.vf_dl_queues_number[i] = VF_DL_5G_QUEUE_VALUE; 736 } 737 738 /* UL bandwidth. Needed only for Vista Creek 5GNR schedule algorithm */ 739 conf.ul_bandwidth = UL_5G_BANDWIDTH; 740 /* DL bandwidth. Needed only for Vista Creek 5GNR schedule algorithm */ 741 conf.dl_bandwidth = DL_5G_BANDWIDTH; 742 743 /* UL & DL load Balance Factor to 64 */ 744 conf.ul_load_balance = UL_5G_LOAD_BALANCE; 745 conf.dl_load_balance = DL_5G_LOAD_BALANCE; 746 747 /* setup FPGA PF with configuration information */ 748 ret = rte_fpga_5gnr_fec_configure(info->dev_name, &conf); 749 TEST_ASSERT_SUCCESS(ret, 750 "Failed to configure 5G FPGA PF for bbdev %s", 751 info->dev_name); 752 } 753 #endif 754 #ifdef RTE_BASEBAND_ACC 755 if ((get_init_device() == true) && 756 (!strcmp(info->drv.driver_name, ACC100PF_DRIVER_NAME))) { 757 struct rte_acc_conf conf; 758 unsigned int i; 759 760 printf("Configure ACC100 FEC device %s with default values\n", 761 info->drv.driver_name); 762 763 /* clear default configuration before initialization */ 764 memset(&conf, 0, sizeof(struct rte_acc_conf)); 765 766 /* Always set in PF mode for built-in configuration */ 767 conf.pf_mode_en = true; 768 for (i = 0; i < RTE_ACC_NUM_VFS; ++i) { 769 conf.arb_dl_4g[i].gbr_threshold1 = ACC100_QOS_GBR; 770 conf.arb_dl_4g[i].gbr_threshold1 = ACC100_QOS_GBR; 771 conf.arb_dl_4g[i].round_robin_weight = ACC100_QMGR_RR; 772 conf.arb_ul_4g[i].gbr_threshold1 = ACC100_QOS_GBR; 773 conf.arb_ul_4g[i].gbr_threshold1 = ACC100_QOS_GBR; 774 conf.arb_ul_4g[i].round_robin_weight = ACC100_QMGR_RR; 775 conf.arb_dl_5g[i].gbr_threshold1 = ACC100_QOS_GBR; 776 conf.arb_dl_5g[i].gbr_threshold1 = ACC100_QOS_GBR; 777 conf.arb_dl_5g[i].round_robin_weight = ACC100_QMGR_RR; 778 conf.arb_ul_5g[i].gbr_threshold1 = ACC100_QOS_GBR; 779 conf.arb_ul_5g[i].gbr_threshold1 = ACC100_QOS_GBR; 780 conf.arb_ul_5g[i].round_robin_weight = ACC100_QMGR_RR; 781 } 782 783 conf.input_pos_llr_1_bit = true; 784 conf.output_pos_llr_1_bit = true; 785 conf.num_vf_bundles = 1; /**< Number of VF bundles to setup */ 786 787 conf.q_ul_4g.num_qgroups = ACC100_QMGR_NUM_QGS; 788 conf.q_ul_4g.first_qgroup_index = ACC100_QMGR_INVALID_IDX; 789 conf.q_ul_4g.num_aqs_per_groups = ACC100_QMGR_NUM_AQS; 790 conf.q_ul_4g.aq_depth_log2 = ACC100_QMGR_AQ_DEPTH; 791 conf.q_dl_4g.num_qgroups = ACC100_QMGR_NUM_QGS; 792 conf.q_dl_4g.first_qgroup_index = ACC100_QMGR_INVALID_IDX; 793 conf.q_dl_4g.num_aqs_per_groups = ACC100_QMGR_NUM_AQS; 794 conf.q_dl_4g.aq_depth_log2 = ACC100_QMGR_AQ_DEPTH; 795 conf.q_ul_5g.num_qgroups = ACC100_QMGR_NUM_QGS; 796 conf.q_ul_5g.first_qgroup_index = ACC100_QMGR_INVALID_IDX; 797 conf.q_ul_5g.num_aqs_per_groups = ACC100_QMGR_NUM_AQS; 798 conf.q_ul_5g.aq_depth_log2 = ACC100_QMGR_AQ_DEPTH; 799 conf.q_dl_5g.num_qgroups = ACC100_QMGR_NUM_QGS; 800 conf.q_dl_5g.first_qgroup_index = ACC100_QMGR_INVALID_IDX; 801 conf.q_dl_5g.num_aqs_per_groups = ACC100_QMGR_NUM_AQS; 802 conf.q_dl_5g.aq_depth_log2 = ACC100_QMGR_AQ_DEPTH; 803 804 /* setup PF with configuration information */ 805 ret = rte_acc_configure(info->dev_name, &conf); 806 TEST_ASSERT_SUCCESS(ret, 807 "Failed to configure ACC100 PF for bbdev %s", 808 info->dev_name); 809 } 810 if ((get_init_device() == true) && 811 (!strcmp(info->drv.driver_name, VRBPF_DRIVER_NAME))) { 812 struct rte_acc_conf conf; 813 unsigned int i; 814 815 printf("Configure Driver %s with default values\n", 816 info->drv.driver_name); 817 818 /* clear default configuration before initialization */ 819 memset(&conf, 0, sizeof(struct rte_acc_conf)); 820 821 /* Always set in PF mode for built-in configuration */ 822 conf.pf_mode_en = true; 823 for (i = 0; i < RTE_ACC_NUM_VFS; ++i) { 824 conf.arb_dl_4g[i].gbr_threshold1 = VRB_QOS_GBR; 825 conf.arb_dl_4g[i].gbr_threshold1 = VRB_QOS_GBR; 826 conf.arb_dl_4g[i].round_robin_weight = VRB_QMGR_RR; 827 conf.arb_ul_4g[i].gbr_threshold1 = VRB_QOS_GBR; 828 conf.arb_ul_4g[i].gbr_threshold1 = VRB_QOS_GBR; 829 conf.arb_ul_4g[i].round_robin_weight = VRB_QMGR_RR; 830 conf.arb_dl_5g[i].gbr_threshold1 = VRB_QOS_GBR; 831 conf.arb_dl_5g[i].gbr_threshold1 = VRB_QOS_GBR; 832 conf.arb_dl_5g[i].round_robin_weight = VRB_QMGR_RR; 833 conf.arb_ul_5g[i].gbr_threshold1 = VRB_QOS_GBR; 834 conf.arb_ul_5g[i].gbr_threshold1 = VRB_QOS_GBR; 835 conf.arb_ul_5g[i].round_robin_weight = VRB_QMGR_RR; 836 conf.arb_fft[i].gbr_threshold1 = VRB_QOS_GBR; 837 conf.arb_fft[i].gbr_threshold1 = VRB_QOS_GBR; 838 conf.arb_fft[i].round_robin_weight = VRB_QMGR_RR; 839 conf.arb_mld[i].gbr_threshold1 = VRB_QOS_GBR; 840 conf.arb_mld[i].gbr_threshold1 = VRB_QOS_GBR; 841 conf.arb_mld[i].round_robin_weight = VRB_QMGR_RR; 842 } 843 844 conf.input_pos_llr_1_bit = true; 845 conf.output_pos_llr_1_bit = true; 846 conf.num_vf_bundles = 1; /**< Number of VF bundles to setup */ 847 848 conf.q_ul_4g.num_qgroups = VRB_QMGR_NUM_QGS; 849 conf.q_ul_4g.first_qgroup_index = VRB_QMGR_INVALID_IDX; 850 conf.q_ul_4g.num_aqs_per_groups = VRB_QMGR_NUM_AQS; 851 conf.q_ul_4g.aq_depth_log2 = VRB_QMGR_AQ_DEPTH; 852 conf.q_dl_4g.num_qgroups = VRB_QMGR_NUM_QGS; 853 conf.q_dl_4g.first_qgroup_index = VRB_QMGR_INVALID_IDX; 854 conf.q_dl_4g.num_aqs_per_groups = VRB_QMGR_NUM_AQS; 855 conf.q_dl_4g.aq_depth_log2 = VRB_QMGR_AQ_DEPTH; 856 conf.q_ul_5g.num_qgroups = VRB_QMGR_NUM_QGS; 857 conf.q_ul_5g.first_qgroup_index = VRB_QMGR_INVALID_IDX; 858 conf.q_ul_5g.num_aqs_per_groups = VRB_QMGR_NUM_AQS; 859 conf.q_ul_5g.aq_depth_log2 = VRB_QMGR_AQ_DEPTH; 860 conf.q_dl_5g.num_qgroups = VRB_QMGR_NUM_QGS; 861 conf.q_dl_5g.first_qgroup_index = VRB_QMGR_INVALID_IDX; 862 conf.q_dl_5g.num_aqs_per_groups = VRB_QMGR_NUM_AQS; 863 conf.q_dl_5g.aq_depth_log2 = VRB_QMGR_AQ_DEPTH; 864 conf.q_fft.num_qgroups = VRB_QMGR_NUM_QGS; 865 conf.q_fft.first_qgroup_index = VRB_QMGR_INVALID_IDX; 866 conf.q_fft.num_aqs_per_groups = VRB_QMGR_NUM_AQS; 867 conf.q_fft.aq_depth_log2 = VRB_QMGR_AQ_DEPTH; 868 conf.q_mld.num_qgroups = VRB_QMGR_NUM_QGS; 869 conf.q_mld.first_qgroup_index = VRB_QMGR_INVALID_IDX; 870 conf.q_mld.num_aqs_per_groups = VRB_QMGR_NUM_AQS; 871 conf.q_mld.aq_depth_log2 = VRB_QMGR_AQ_DEPTH; 872 873 /* setup PF with configuration information */ 874 ret = rte_acc_configure(info->dev_name, &conf); 875 TEST_ASSERT_SUCCESS(ret, 876 "Failed to configure PF for bbdev %s", 877 info->dev_name); 878 } 879 #endif 880 /* Let's refresh this now this is configured */ 881 rte_bbdev_info_get(dev_id, info); 882 if (info->drv.device_status == RTE_BBDEV_DEV_FATAL_ERR) 883 printf("Device Status %s\n", rte_bbdev_device_status_str(info->drv.device_status)); 884 nb_queues = RTE_MIN(rte_lcore_count(), info->drv.max_num_queues); 885 nb_queues = RTE_MIN(nb_queues, (unsigned int) MAX_QUEUES); 886 887 /* setup device */ 888 ret = rte_bbdev_setup_queues(dev_id, nb_queues, info->socket_id); 889 if (ret < 0) { 890 printf("rte_bbdev_setup_queues(%u, %u, %d) ret %i\n", 891 dev_id, nb_queues, info->socket_id, ret); 892 return TEST_FAILED; 893 } 894 895 /* configure interrupts if needed */ 896 if (intr_enabled) { 897 ret = rte_bbdev_intr_enable(dev_id); 898 if (ret < 0) { 899 printf("rte_bbdev_intr_enable(%u) ret %i\n", dev_id, 900 ret); 901 return TEST_FAILED; 902 } 903 } 904 905 /* setup device queues */ 906 qconf.socket = info->socket_id; 907 qconf.queue_size = info->drv.default_queue_conf.queue_size; 908 qconf.priority = 0; 909 qconf.deferred_start = 0; 910 qconf.op_type = op_type; 911 912 for (queue_id = 0; queue_id < nb_queues; ++queue_id) { 913 ret = rte_bbdev_queue_configure(dev_id, queue_id, &qconf); 914 if (ret != 0) { 915 printf( 916 "Allocated all queues (id=%u) at prio%u on dev%u\n", 917 queue_id, qconf.priority, dev_id); 918 qconf.priority++; 919 ret = rte_bbdev_queue_configure(ad->dev_id, queue_id, &qconf); 920 } 921 if (ret != 0) { 922 printf("All queues on dev %u allocated: %u\n", dev_id, queue_id); 923 break; 924 } 925 ret = rte_bbdev_queue_start(ad->dev_id, queue_id); 926 if (ret != 0) { 927 printf("Failed to start queue on dev %u q_id: %u\n", dev_id, queue_id); 928 break; 929 } 930 ad->queue_ids[queue_id] = queue_id; 931 } 932 TEST_ASSERT(queue_id != 0, 933 "ERROR Failed to configure any queues on dev %u\n" 934 "\tthe device may not support the related operation capability\n" 935 "\tor the device may not have been configured yet", dev_id); 936 ad->nb_queues = queue_id; 937 938 set_avail_op(ad, op_type); 939 940 return TEST_SUCCESS; 941 } 942 943 static int 944 add_active_device(uint8_t dev_id, struct rte_bbdev_info *info, 945 struct test_bbdev_vector *vector) 946 { 947 int ret; 948 949 active_devs[nb_active_devs].driver_name = info->drv.driver_name; 950 active_devs[nb_active_devs].dev_id = dev_id; 951 952 ret = add_bbdev_dev(dev_id, info, vector); 953 if (ret == TEST_SUCCESS) 954 ++nb_active_devs; 955 return ret; 956 } 957 958 static uint8_t 959 populate_active_devices(void) 960 { 961 int ret; 962 uint8_t dev_id; 963 uint8_t nb_devs_added = 0; 964 struct rte_bbdev_info info; 965 966 RTE_BBDEV_FOREACH(dev_id) { 967 rte_bbdev_info_get(dev_id, &info); 968 969 if (check_dev_cap(&info)) { 970 printf( 971 "Device %d (%s) does not support specified capabilities\n", 972 dev_id, info.dev_name); 973 continue; 974 } 975 976 ret = add_active_device(dev_id, &info, &test_vector); 977 if (ret != 0) { 978 printf("Adding active bbdev %s skipped\n", 979 info.dev_name); 980 continue; 981 } 982 nb_devs_added++; 983 } 984 985 return nb_devs_added; 986 } 987 988 static int 989 read_test_vector(void) 990 { 991 int ret; 992 993 memset(&test_vector, 0, sizeof(test_vector)); 994 printf("Test vector file = %s\n", get_vector_filename()); 995 ret = test_bbdev_vector_read(get_vector_filename(), &test_vector); 996 TEST_ASSERT_SUCCESS(ret, "Failed to parse file %s\n", 997 get_vector_filename()); 998 999 return TEST_SUCCESS; 1000 } 1001 1002 static int 1003 testsuite_setup(void) 1004 { 1005 TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n"); 1006 1007 if (populate_active_devices() == 0) { 1008 printf("No suitable devices found!\n"); 1009 return TEST_SKIPPED; 1010 } 1011 1012 return TEST_SUCCESS; 1013 } 1014 1015 static int 1016 interrupt_testsuite_setup(void) 1017 { 1018 TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n"); 1019 1020 /* Enable interrupts */ 1021 intr_enabled = true; 1022 1023 /* Special case for NULL device (RTE_BBDEV_OP_NONE) */ 1024 if (populate_active_devices() == 0 || 1025 test_vector.op_type == RTE_BBDEV_OP_NONE) { 1026 intr_enabled = false; 1027 printf("No suitable devices found!\n"); 1028 return TEST_SKIPPED; 1029 } 1030 1031 return TEST_SUCCESS; 1032 } 1033 1034 static void 1035 testsuite_teardown(void) 1036 { 1037 uint8_t dev_id; 1038 1039 /* Unconfigure devices */ 1040 RTE_BBDEV_FOREACH(dev_id) 1041 rte_bbdev_close(dev_id); 1042 1043 /* Clear active devices structs. */ 1044 memset(active_devs, 0, sizeof(active_devs)); 1045 nb_active_devs = 0; 1046 1047 /* Disable interrupts */ 1048 intr_enabled = false; 1049 } 1050 1051 static int 1052 ut_setup(void) 1053 { 1054 uint8_t i, dev_id; 1055 1056 for (i = 0; i < nb_active_devs; i++) { 1057 dev_id = active_devs[i].dev_id; 1058 /* reset bbdev stats */ 1059 TEST_ASSERT_SUCCESS(rte_bbdev_stats_reset(dev_id), 1060 "Failed to reset stats of bbdev %u", dev_id); 1061 /* start the device */ 1062 TEST_ASSERT_SUCCESS(rte_bbdev_start(dev_id), 1063 "Failed to start bbdev %u", dev_id); 1064 } 1065 1066 return TEST_SUCCESS; 1067 } 1068 1069 static void 1070 ut_teardown(void) 1071 { 1072 uint8_t i, dev_id, ret; 1073 struct rte_bbdev_stats stats; 1074 1075 for (i = 0; i < nb_active_devs; i++) { 1076 dev_id = active_devs[i].dev_id; 1077 /* read stats and print */ 1078 ret = rte_bbdev_stats_get(dev_id, &stats); 1079 if (ret != 0) 1080 printf("Failed to get stats on bbdev %u\n", dev_id); 1081 /* Stop the device */ 1082 rte_bbdev_stop(dev_id); 1083 } 1084 } 1085 1086 static int 1087 init_op_data_objs(struct rte_bbdev_op_data *bufs, 1088 struct op_data_entries *ref_entries, 1089 struct rte_mempool *mbuf_pool, const uint16_t n, 1090 enum op_data_type op_type, uint16_t min_alignment) 1091 { 1092 int ret; 1093 unsigned int i, j; 1094 bool large_input = false; 1095 1096 for (i = 0; i < n; ++i) { 1097 char *data; 1098 struct op_data_buf *seg = &ref_entries->segments[0]; 1099 struct rte_mbuf *m_head = rte_pktmbuf_alloc(mbuf_pool); 1100 TEST_ASSERT_NOT_NULL(m_head, 1101 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)", 1102 op_type, n * ref_entries->nb_segments, 1103 mbuf_pool->size); 1104 1105 if ((seg->length + RTE_PKTMBUF_HEADROOM) > RTE_BBDEV_LDPC_E_MAX_MBUF) { 1106 /* 1107 * Special case when DPDK mbuf cannot handle 1108 * the required input size 1109 */ 1110 large_input = true; 1111 } 1112 bufs[i].data = m_head; 1113 bufs[i].offset = 0; 1114 bufs[i].length = 0; 1115 1116 if ((op_type == DATA_INPUT) || (op_type == DATA_HARQ_INPUT)) { 1117 if (large_input) { 1118 /* Allocate a fake overused mbuf */ 1119 data = rte_malloc(NULL, seg->length, 0); 1120 TEST_ASSERT_NOT_NULL(data, 1121 "rte malloc failed with %u bytes", 1122 seg->length); 1123 memcpy(data, seg->addr, seg->length); 1124 m_head->buf_addr = data; 1125 rte_mbuf_iova_set(m_head, rte_malloc_virt2iova(data)); 1126 m_head->data_off = 0; 1127 m_head->data_len = seg->length; 1128 } else { 1129 data = rte_pktmbuf_append(m_head, seg->length); 1130 TEST_ASSERT_NOT_NULL(data, 1131 "Couldn't append %u bytes to mbuf from %d data type mbuf pool", 1132 seg->length, op_type); 1133 1134 TEST_ASSERT(data == RTE_PTR_ALIGN( 1135 data, min_alignment), 1136 "Data addr in mbuf (%p) is not aligned to device min alignment (%u)", 1137 data, min_alignment); 1138 rte_memcpy(data, seg->addr, seg->length); 1139 } 1140 1141 bufs[i].length += seg->length; 1142 1143 for (j = 1; j < ref_entries->nb_segments; ++j) { 1144 struct rte_mbuf *m_tail = 1145 rte_pktmbuf_alloc(mbuf_pool); 1146 TEST_ASSERT_NOT_NULL(m_tail, 1147 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)", 1148 op_type, 1149 n * ref_entries->nb_segments, 1150 mbuf_pool->size); 1151 seg += 1; 1152 1153 data = rte_pktmbuf_append(m_tail, seg->length); 1154 TEST_ASSERT_NOT_NULL(data, 1155 "Couldn't append %u bytes to mbuf from %d data type mbuf pool", 1156 seg->length, op_type); 1157 1158 TEST_ASSERT(data == RTE_PTR_ALIGN(data, 1159 min_alignment), 1160 "Data addr in mbuf (%p) is not aligned to device min alignment (%u)", 1161 data, min_alignment); 1162 rte_memcpy(data, seg->addr, seg->length); 1163 bufs[i].length += seg->length; 1164 1165 ret = rte_pktmbuf_chain(m_head, m_tail); 1166 TEST_ASSERT_SUCCESS(ret, 1167 "Couldn't chain mbufs from %d data type mbuf pool", 1168 op_type); 1169 } 1170 } else { 1171 if (((op_type == DATA_HARD_OUTPUT) || (op_type == DATA_SOFT_OUTPUT)) 1172 && ((seg->length + RTE_PKTMBUF_HEADROOM) 1173 > RTE_BBDEV_LDPC_E_MAX_MBUF)) { 1174 /* Allocate a fake overused mbuf + margin */ 1175 data = rte_malloc(NULL, seg->length + 1024, 0); 1176 TEST_ASSERT_NOT_NULL(data, 1177 "rte malloc failed with %u bytes", 1178 seg->length + 1024); 1179 m_head->buf_addr = data; 1180 rte_mbuf_iova_set(m_head, rte_malloc_virt2iova(data)); 1181 m_head->data_off = 0; 1182 m_head->data_len = seg->length; 1183 } else { 1184 /* allocate chained-mbuf for output buffer */ 1185 for (j = 1; j < ref_entries->nb_segments; ++j) { 1186 struct rte_mbuf *m_tail = 1187 rte_pktmbuf_alloc(mbuf_pool); 1188 TEST_ASSERT_NOT_NULL(m_tail, 1189 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)", 1190 op_type, 1191 n * ref_entries->nb_segments, 1192 mbuf_pool->size); 1193 1194 ret = rte_pktmbuf_chain(m_head, m_tail); 1195 TEST_ASSERT_SUCCESS(ret, 1196 "Couldn't chain mbufs from %d data type mbuf pool", 1197 op_type); 1198 } 1199 } 1200 bufs[i].length += seg->length; 1201 } 1202 } 1203 1204 return 0; 1205 } 1206 1207 static int 1208 allocate_buffers_on_socket(struct rte_bbdev_op_data **buffers, const int len, 1209 const int socket) 1210 { 1211 int i; 1212 1213 *buffers = rte_zmalloc_socket(NULL, len, 0, socket); 1214 if (*buffers == NULL) { 1215 printf("WARNING: Failed to allocate op_data on socket %d\n", 1216 socket); 1217 /* try to allocate memory on other detected sockets */ 1218 for (i = 0; i < socket; i++) { 1219 *buffers = rte_zmalloc_socket(NULL, len, 0, i); 1220 if (*buffers != NULL) 1221 break; 1222 } 1223 } 1224 1225 return (*buffers == NULL) ? TEST_FAILED : TEST_SUCCESS; 1226 } 1227 1228 static void 1229 limit_input_llr_val_range(struct rte_bbdev_op_data *input_ops, 1230 const uint16_t n, const int8_t max_llr_modulus) 1231 { 1232 uint16_t i, byte_idx; 1233 1234 for (i = 0; i < n; ++i) { 1235 struct rte_mbuf *m = input_ops[i].data; 1236 while (m != NULL) { 1237 int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *, 1238 input_ops[i].offset); 1239 for (byte_idx = 0; byte_idx < rte_pktmbuf_data_len(m); 1240 ++byte_idx) 1241 llr[byte_idx] = round((double)max_llr_modulus * 1242 llr[byte_idx] / INT8_MAX); 1243 1244 m = m->next; 1245 } 1246 } 1247 } 1248 1249 /* 1250 * We may have to insert filler bits 1251 * when they are required by the HARQ assumption 1252 */ 1253 static void 1254 ldpc_add_filler(struct rte_bbdev_op_data *input_ops, 1255 const uint16_t n, struct test_op_params *op_params) 1256 { 1257 struct rte_bbdev_op_ldpc_dec dec = op_params->ref_dec_op->ldpc_dec; 1258 1259 if (input_ops == NULL) 1260 return; 1261 /* No need to add filler if not required by device */ 1262 if (!(ldpc_cap_flags & 1263 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_FILLERS)) 1264 return; 1265 /* No need to add filler for loopback operation */ 1266 if (dec.op_flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK) 1267 return; 1268 1269 uint16_t i, j, parity_offset; 1270 for (i = 0; i < n; ++i) { 1271 struct rte_mbuf *m = input_ops[i].data; 1272 int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *, 1273 input_ops[i].offset); 1274 parity_offset = (dec.basegraph == 1 ? 20 : 8) 1275 * dec.z_c - dec.n_filler; 1276 uint16_t new_hin_size = input_ops[i].length + dec.n_filler; 1277 m->data_len = new_hin_size; 1278 input_ops[i].length = new_hin_size; 1279 for (j = new_hin_size - 1; j >= parity_offset + dec.n_filler; 1280 j--) 1281 llr[j] = llr[j - dec.n_filler]; 1282 uint16_t llr_max_pre_scaling = (1 << (ldpc_llr_size - 1)) - 1; 1283 for (j = 0; j < dec.n_filler; j++) 1284 llr[parity_offset + j] = llr_max_pre_scaling; 1285 } 1286 } 1287 1288 static void 1289 ldpc_input_llr_scaling(struct rte_bbdev_op_data *input_ops, 1290 const uint16_t n, const int8_t llr_size, 1291 const int8_t llr_decimals) 1292 { 1293 if (input_ops == NULL) 1294 return; 1295 1296 uint16_t i, byte_idx; 1297 1298 int16_t llr_max, llr_min, llr_tmp; 1299 llr_max = (1 << (llr_size - 1)) - 1; 1300 llr_min = -llr_max; 1301 for (i = 0; i < n; ++i) { 1302 struct rte_mbuf *m = input_ops[i].data; 1303 while (m != NULL) { 1304 int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *, 1305 input_ops[i].offset); 1306 for (byte_idx = 0; byte_idx < rte_pktmbuf_data_len(m); 1307 ++byte_idx) { 1308 1309 llr_tmp = llr[byte_idx]; 1310 if (llr_decimals == 4) 1311 llr_tmp *= 8; 1312 else if (llr_decimals == 2) 1313 llr_tmp *= 2; 1314 else if (llr_decimals == 0) 1315 llr_tmp /= 2; 1316 llr_tmp = RTE_MIN(llr_max, 1317 RTE_MAX(llr_min, llr_tmp)); 1318 llr[byte_idx] = (int8_t) llr_tmp; 1319 } 1320 1321 m = m->next; 1322 } 1323 } 1324 } 1325 1326 1327 1328 static int 1329 fill_queue_buffers(struct test_op_params *op_params, 1330 struct rte_mempool *in_mp, struct rte_mempool *hard_out_mp, 1331 struct rte_mempool *soft_out_mp, 1332 struct rte_mempool *harq_in_mp, struct rte_mempool *harq_out_mp, 1333 uint16_t queue_id, 1334 const struct rte_bbdev_op_cap *capabilities, 1335 uint16_t min_alignment, const int socket_id) 1336 { 1337 int ret; 1338 enum op_data_type type; 1339 const uint16_t n = op_params->num_to_process; 1340 1341 struct rte_mempool *mbuf_pools[DATA_NUM_TYPES] = { 1342 in_mp, 1343 soft_out_mp, 1344 hard_out_mp, 1345 harq_in_mp, 1346 harq_out_mp, 1347 }; 1348 1349 struct rte_bbdev_op_data **queue_ops[DATA_NUM_TYPES] = { 1350 &op_params->q_bufs[socket_id][queue_id].inputs, 1351 &op_params->q_bufs[socket_id][queue_id].soft_outputs, 1352 &op_params->q_bufs[socket_id][queue_id].hard_outputs, 1353 &op_params->q_bufs[socket_id][queue_id].harq_inputs, 1354 &op_params->q_bufs[socket_id][queue_id].harq_outputs, 1355 }; 1356 1357 for (type = DATA_INPUT; type < DATA_NUM_TYPES; ++type) { 1358 struct op_data_entries *ref_entries = 1359 &test_vector.entries[type]; 1360 if (ref_entries->nb_segments == 0) 1361 continue; 1362 1363 ret = allocate_buffers_on_socket(queue_ops[type], 1364 n * sizeof(struct rte_bbdev_op_data), 1365 socket_id); 1366 TEST_ASSERT_SUCCESS(ret, 1367 "Couldn't allocate memory for rte_bbdev_op_data structs"); 1368 1369 ret = init_op_data_objs(*queue_ops[type], ref_entries, 1370 mbuf_pools[type], n, type, min_alignment); 1371 TEST_ASSERT_SUCCESS(ret, 1372 "Couldn't init rte_bbdev_op_data structs"); 1373 } 1374 1375 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) 1376 limit_input_llr_val_range(*queue_ops[DATA_INPUT], n, 1377 capabilities->cap.turbo_dec.max_llr_modulus); 1378 1379 if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) { 1380 bool loopback = op_params->ref_dec_op->ldpc_dec.op_flags & 1381 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK; 1382 bool llr_comp = op_params->ref_dec_op->ldpc_dec.op_flags & 1383 RTE_BBDEV_LDPC_LLR_COMPRESSION; 1384 bool harq_comp = op_params->ref_dec_op->ldpc_dec.op_flags & 1385 RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION; 1386 1387 ldpc_llr_decimals = capabilities->cap.ldpc_dec.llr_decimals; 1388 ldpc_llr_size = capabilities->cap.ldpc_dec.llr_size; 1389 ldpc_cap_flags = capabilities->cap.ldpc_dec.capability_flags; 1390 if (!loopback && !llr_comp) 1391 ldpc_input_llr_scaling(*queue_ops[DATA_INPUT], n, 1392 ldpc_llr_size, ldpc_llr_decimals); 1393 if (!loopback && !harq_comp) 1394 ldpc_input_llr_scaling(*queue_ops[DATA_HARQ_INPUT], n, 1395 ldpc_llr_size, ldpc_llr_decimals); 1396 if (!loopback) 1397 ldpc_add_filler(*queue_ops[DATA_HARQ_INPUT], n, 1398 op_params); 1399 } 1400 1401 return 0; 1402 } 1403 1404 static void 1405 free_buffers(struct active_device *ad, struct test_op_params *op_params) 1406 { 1407 unsigned int i, j; 1408 1409 rte_mempool_free(ad->ops_mempool); 1410 rte_mempool_free(ad->in_mbuf_pool); 1411 rte_mempool_free(ad->hard_out_mbuf_pool); 1412 rte_mempool_free(ad->soft_out_mbuf_pool); 1413 rte_mempool_free(ad->harq_in_mbuf_pool); 1414 rte_mempool_free(ad->harq_out_mbuf_pool); 1415 1416 for (i = 0; i < rte_lcore_count(); ++i) { 1417 for (j = 0; j < RTE_MAX_NUMA_NODES; ++j) { 1418 rte_free(op_params->q_bufs[j][i].inputs); 1419 rte_free(op_params->q_bufs[j][i].hard_outputs); 1420 rte_free(op_params->q_bufs[j][i].soft_outputs); 1421 rte_free(op_params->q_bufs[j][i].harq_inputs); 1422 rte_free(op_params->q_bufs[j][i].harq_outputs); 1423 } 1424 } 1425 } 1426 1427 static void 1428 copy_reference_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n, 1429 unsigned int start_idx, 1430 struct rte_bbdev_op_data *inputs, 1431 struct rte_bbdev_op_data *hard_outputs, 1432 struct rte_bbdev_op_data *soft_outputs, 1433 struct rte_bbdev_dec_op *ref_op) 1434 { 1435 unsigned int i; 1436 struct rte_bbdev_op_turbo_dec *turbo_dec = &ref_op->turbo_dec; 1437 1438 for (i = 0; i < n; ++i) { 1439 if (turbo_dec->code_block_mode == RTE_BBDEV_TRANSPORT_BLOCK) { 1440 ops[i]->turbo_dec.tb_params.ea = 1441 turbo_dec->tb_params.ea; 1442 ops[i]->turbo_dec.tb_params.eb = 1443 turbo_dec->tb_params.eb; 1444 ops[i]->turbo_dec.tb_params.k_pos = 1445 turbo_dec->tb_params.k_pos; 1446 ops[i]->turbo_dec.tb_params.k_neg = 1447 turbo_dec->tb_params.k_neg; 1448 ops[i]->turbo_dec.tb_params.c = 1449 turbo_dec->tb_params.c; 1450 ops[i]->turbo_dec.tb_params.c_neg = 1451 turbo_dec->tb_params.c_neg; 1452 ops[i]->turbo_dec.tb_params.cab = 1453 turbo_dec->tb_params.cab; 1454 ops[i]->turbo_dec.tb_params.r = 1455 turbo_dec->tb_params.r; 1456 } else { 1457 ops[i]->turbo_dec.cb_params.e = turbo_dec->cb_params.e; 1458 ops[i]->turbo_dec.cb_params.k = turbo_dec->cb_params.k; 1459 } 1460 1461 ops[i]->turbo_dec.ext_scale = turbo_dec->ext_scale; 1462 ops[i]->turbo_dec.iter_max = turbo_dec->iter_max; 1463 ops[i]->turbo_dec.iter_min = turbo_dec->iter_min; 1464 ops[i]->turbo_dec.op_flags = turbo_dec->op_flags; 1465 ops[i]->turbo_dec.rv_index = turbo_dec->rv_index; 1466 ops[i]->turbo_dec.num_maps = turbo_dec->num_maps; 1467 ops[i]->turbo_dec.code_block_mode = turbo_dec->code_block_mode; 1468 1469 ops[i]->turbo_dec.hard_output = hard_outputs[start_idx + i]; 1470 ops[i]->turbo_dec.input = inputs[start_idx + i]; 1471 if (soft_outputs != NULL) 1472 ops[i]->turbo_dec.soft_output = 1473 soft_outputs[start_idx + i]; 1474 } 1475 } 1476 1477 static void 1478 copy_reference_enc_op(struct rte_bbdev_enc_op **ops, unsigned int n, 1479 unsigned int start_idx, 1480 struct rte_bbdev_op_data *inputs, 1481 struct rte_bbdev_op_data *outputs, 1482 struct rte_bbdev_enc_op *ref_op) 1483 { 1484 unsigned int i; 1485 struct rte_bbdev_op_turbo_enc *turbo_enc = &ref_op->turbo_enc; 1486 for (i = 0; i < n; ++i) { 1487 if (turbo_enc->code_block_mode == RTE_BBDEV_TRANSPORT_BLOCK) { 1488 ops[i]->turbo_enc.tb_params.ea = 1489 turbo_enc->tb_params.ea; 1490 ops[i]->turbo_enc.tb_params.eb = 1491 turbo_enc->tb_params.eb; 1492 ops[i]->turbo_enc.tb_params.k_pos = 1493 turbo_enc->tb_params.k_pos; 1494 ops[i]->turbo_enc.tb_params.k_neg = 1495 turbo_enc->tb_params.k_neg; 1496 ops[i]->turbo_enc.tb_params.c = 1497 turbo_enc->tb_params.c; 1498 ops[i]->turbo_enc.tb_params.c_neg = 1499 turbo_enc->tb_params.c_neg; 1500 ops[i]->turbo_enc.tb_params.cab = 1501 turbo_enc->tb_params.cab; 1502 ops[i]->turbo_enc.tb_params.ncb_pos = 1503 turbo_enc->tb_params.ncb_pos; 1504 ops[i]->turbo_enc.tb_params.ncb_neg = 1505 turbo_enc->tb_params.ncb_neg; 1506 ops[i]->turbo_enc.tb_params.r = turbo_enc->tb_params.r; 1507 } else { 1508 ops[i]->turbo_enc.cb_params.e = turbo_enc->cb_params.e; 1509 ops[i]->turbo_enc.cb_params.k = turbo_enc->cb_params.k; 1510 ops[i]->turbo_enc.cb_params.ncb = 1511 turbo_enc->cb_params.ncb; 1512 } 1513 ops[i]->turbo_enc.rv_index = turbo_enc->rv_index; 1514 ops[i]->turbo_enc.op_flags = turbo_enc->op_flags; 1515 ops[i]->turbo_enc.code_block_mode = turbo_enc->code_block_mode; 1516 1517 ops[i]->turbo_enc.output = outputs[start_idx + i]; 1518 ops[i]->turbo_enc.input = inputs[start_idx + i]; 1519 } 1520 } 1521 1522 1523 /* Returns a random number drawn from a normal distribution 1524 * with mean of 0 and variance of 1 1525 * Marsaglia algorithm 1526 */ 1527 static double 1528 randn(int n) 1529 { 1530 double S, Z, U1, U2, u, v, fac; 1531 1532 do { 1533 U1 = (double)rand() / RAND_MAX; 1534 U2 = (double)rand() / RAND_MAX; 1535 u = 2. * U1 - 1.; 1536 v = 2. * U2 - 1.; 1537 S = u * u + v * v; 1538 } while (S >= 1 || S == 0); 1539 fac = sqrt(-2. * log(S) / S); 1540 Z = (n % 2) ? u * fac : v * fac; 1541 return Z; 1542 } 1543 1544 static inline double 1545 maxstar(double A, double B) 1546 { 1547 if (fabs(A - B) > 5) 1548 return RTE_MAX(A, B); 1549 else 1550 return RTE_MAX(A, B) + log1p(exp(-fabs(A - B))); 1551 } 1552 1553 /* 1554 * Generate Qm LLRS for Qm==8 1555 * Modulation, AWGN and LLR estimation from max log development 1556 */ 1557 static void 1558 gen_qm8_llr(int8_t *llrs, uint32_t i, double N0, double llr_max) 1559 { 1560 int qm = 8; 1561 int qam = 256; 1562 int m, k; 1563 double I, Q, p0, p1, llr_, b[qm], log_syml_prob[qam]; 1564 /* 5.1.4 of TS38.211 */ 1565 const double symbols_I[256] = { 1566 5, 5, 7, 7, 5, 5, 7, 7, 3, 3, 1, 1, 3, 3, 1, 1, 5, 1567 5, 7, 7, 5, 5, 7, 7, 3, 3, 1, 1, 3, 3, 1, 1, 11, 1568 11, 9, 9, 11, 11, 9, 9, 13, 13, 15, 15, 13, 13, 1569 15, 15, 11, 11, 9, 9, 11, 11, 9, 9, 13, 13, 15, 1570 15, 13, 13, 15, 15, 5, 5, 7, 7, 5, 5, 7, 7, 3, 3, 1571 1, 1, 3, 3, 1, 1, 5, 5, 7, 7, 5, 5, 7, 7, 3, 3, 1, 1572 1, 3, 3, 1, 1, 11, 11, 9, 9, 11, 11, 9, 9, 13, 13, 1573 15, 15, 13, 13, 15, 15, 11, 11, 9, 9, 11, 11, 9, 9, 1574 13, 13, 15, 15, 13, 13, 15, 15, -5, -5, -7, -7, -5, 1575 -5, -7, -7, -3, -3, -1, -1, -3, -3, -1, -1, -5, -5, 1576 -7, -7, -5, -5, -7, -7, -3, -3, -1, -1, -3, -3, 1577 -1, -1, -11, -11, -9, -9, -11, -11, -9, -9, -13, 1578 -13, -15, -15, -13, -13, -15, -15, -11, -11, -9, 1579 -9, -11, -11, -9, -9, -13, -13, -15, -15, -13, 1580 -13, -15, -15, -5, -5, -7, -7, -5, -5, -7, -7, -3, 1581 -3, -1, -1, -3, -3, -1, -1, -5, -5, -7, -7, -5, -5, 1582 -7, -7, -3, -3, -1, -1, -3, -3, -1, -1, -11, -11, 1583 -9, -9, -11, -11, -9, -9, -13, -13, -15, -15, -13, 1584 -13, -15, -15, -11, -11, -9, -9, -11, -11, -9, -9, 1585 -13, -13, -15, -15, -13, -13, -15, -15}; 1586 const double symbols_Q[256] = { 1587 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 11, 1588 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9, 13, 15, 13, 1589 15, 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 1590 11, 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9, 13, 1591 15, 13, 15, -5, -7, -5, -7, -3, -1, -3, -1, -5, 1592 -7, -5, -7, -3, -1, -3, -1, -11, -9, -11, -9, -13, 1593 -15, -13, -15, -11, -9, -11, -9, -13, -15, -13, 1594 -15, -5, -7, -5, -7, -3, -1, -3, -1, -5, -7, -5, 1595 -7, -3, -1, -3, -1, -11, -9, -11, -9, -13, -15, 1596 -13, -15, -11, -9, -11, -9, -13, -15, -13, -15, 5, 1597 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 11, 1598 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9, 13, 15, 1599 13, 15, 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 1600 3, 1, 11, 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9, 1601 13, 15, 13, 15, -5, -7, -5, -7, -3, -1, -3, -1, 1602 -5, -7, -5, -7, -3, -1, -3, -1, -11, -9, -11, -9, 1603 -13, -15, -13, -15, -11, -9, -11, -9, -13, -15, 1604 -13, -15, -5, -7, -5, -7, -3, -1, -3, -1, -5, -7, 1605 -5, -7, -3, -1, -3, -1, -11, -9, -11, -9, -13, -15, 1606 -13, -15, -11, -9, -11, -9, -13, -15, -13, -15}; 1607 /* Average constellation point energy */ 1608 N0 *= 170.0; 1609 for (k = 0; k < qm; k++) 1610 b[k] = llrs[qm * i + k] < 0 ? 1.0 : 0.0; 1611 /* 5.1.4 of TS38.211 */ 1612 I = (1 - 2 * b[0]) * (8 - (1 - 2 * b[2]) * 1613 (4 - (1 - 2 * b[4]) * (2 - (1 - 2 * b[6])))); 1614 Q = (1 - 2 * b[1]) * (8 - (1 - 2 * b[3]) * 1615 (4 - (1 - 2 * b[5]) * (2 - (1 - 2 * b[7])))); 1616 /* AWGN channel */ 1617 I += sqrt(N0 / 2) * randn(0); 1618 Q += sqrt(N0 / 2) * randn(1); 1619 /* 1620 * Calculate the log of the probability that each of 1621 * the constellation points was transmitted 1622 */ 1623 for (m = 0; m < qam; m++) 1624 log_syml_prob[m] = -(pow(I - symbols_I[m], 2.0) 1625 + pow(Q - symbols_Q[m], 2.0)) / N0; 1626 /* Calculate an LLR for each of the k_64QAM bits in the set */ 1627 for (k = 0; k < qm; k++) { 1628 p0 = -999999; 1629 p1 = -999999; 1630 /* For each constellation point */ 1631 for (m = 0; m < qam; m++) { 1632 if ((m >> (qm - k - 1)) & 1) 1633 p1 = maxstar(p1, log_syml_prob[m]); 1634 else 1635 p0 = maxstar(p0, log_syml_prob[m]); 1636 } 1637 /* Calculate the LLR */ 1638 llr_ = p0 - p1; 1639 llr_ *= (1 << ldpc_llr_decimals); 1640 llr_ = round(llr_); 1641 if (llr_ > llr_max) 1642 llr_ = llr_max; 1643 if (llr_ < -llr_max) 1644 llr_ = -llr_max; 1645 llrs[qm * i + k] = (int8_t) llr_; 1646 } 1647 } 1648 1649 1650 /* 1651 * Generate Qm LLRS for Qm==6 1652 * Modulation, AWGN and LLR estimation from max log development 1653 */ 1654 static void 1655 gen_qm6_llr(int8_t *llrs, uint32_t i, double N0, double llr_max) 1656 { 1657 int qm = 6; 1658 int qam = 64; 1659 int m, k; 1660 double I, Q, p0, p1, llr_, b[qm], log_syml_prob[qam]; 1661 /* 5.1.4 of TS38.211 */ 1662 const double symbols_I[64] = { 1663 3, 3, 1, 1, 3, 3, 1, 1, 5, 5, 7, 7, 5, 5, 7, 7, 1664 3, 3, 1, 1, 3, 3, 1, 1, 5, 5, 7, 7, 5, 5, 7, 7, 1665 -3, -3, -1, -1, -3, -3, -1, -1, -5, -5, -7, -7, 1666 -5, -5, -7, -7, -3, -3, -1, -1, -3, -3, -1, -1, 1667 -5, -5, -7, -7, -5, -5, -7, -7}; 1668 const double symbols_Q[64] = { 1669 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 1670 -3, -1, -3, -1, -5, -7, -5, -7, -3, -1, -3, -1, 1671 -5, -7, -5, -7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 1672 5, 7, 5, 7, -3, -1, -3, -1, -5, -7, -5, -7, 1673 -3, -1, -3, -1, -5, -7, -5, -7}; 1674 /* Average constellation point energy */ 1675 N0 *= 42.0; 1676 for (k = 0; k < qm; k++) 1677 b[k] = llrs[qm * i + k] < 0 ? 1.0 : 0.0; 1678 /* 5.1.4 of TS38.211 */ 1679 I = (1 - 2 * b[0])*(4 - (1 - 2 * b[2]) * (2 - (1 - 2 * b[4]))); 1680 Q = (1 - 2 * b[1])*(4 - (1 - 2 * b[3]) * (2 - (1 - 2 * b[5]))); 1681 /* AWGN channel */ 1682 I += sqrt(N0 / 2) * randn(0); 1683 Q += sqrt(N0 / 2) * randn(1); 1684 /* 1685 * Calculate the log of the probability that each of 1686 * the constellation points was transmitted 1687 */ 1688 for (m = 0; m < qam; m++) 1689 log_syml_prob[m] = -(pow(I - symbols_I[m], 2.0) 1690 + pow(Q - symbols_Q[m], 2.0)) / N0; 1691 /* Calculate an LLR for each of the k_64QAM bits in the set */ 1692 for (k = 0; k < qm; k++) { 1693 p0 = -999999; 1694 p1 = -999999; 1695 /* For each constellation point */ 1696 for (m = 0; m < qam; m++) { 1697 if ((m >> (qm - k - 1)) & 1) 1698 p1 = maxstar(p1, log_syml_prob[m]); 1699 else 1700 p0 = maxstar(p0, log_syml_prob[m]); 1701 } 1702 /* Calculate the LLR */ 1703 llr_ = p0 - p1; 1704 llr_ *= (1 << ldpc_llr_decimals); 1705 llr_ = round(llr_); 1706 if (llr_ > llr_max) 1707 llr_ = llr_max; 1708 if (llr_ < -llr_max) 1709 llr_ = -llr_max; 1710 llrs[qm * i + k] = (int8_t) llr_; 1711 } 1712 } 1713 1714 /* 1715 * Generate Qm LLRS for Qm==4 1716 * Modulation, AWGN and LLR estimation from max log development 1717 */ 1718 static void 1719 gen_qm4_llr(int8_t *llrs, uint32_t i, double N0, double llr_max) 1720 { 1721 int qm = 4; 1722 int qam = 16; 1723 int m, k; 1724 double I, Q, p0, p1, llr_, b[qm], log_syml_prob[qam]; 1725 /* 5.1.4 of TS38.211 */ 1726 const double symbols_I[16] = {1, 1, 3, 3, 1, 1, 3, 3, 1727 -1, -1, -3, -3, -1, -1, -3, -3}; 1728 const double symbols_Q[16] = {1, 3, 1, 3, -1, -3, -1, -3, 1729 1, 3, 1, 3, -1, -3, -1, -3}; 1730 /* Average constellation point energy */ 1731 N0 *= 10.0; 1732 for (k = 0; k < qm; k++) 1733 b[k] = llrs[qm * i + k] < 0 ? 1.0 : 0.0; 1734 /* 5.1.4 of TS38.211 */ 1735 I = (1 - 2 * b[0]) * (2 - (1 - 2 * b[2])); 1736 Q = (1 - 2 * b[1]) * (2 - (1 - 2 * b[3])); 1737 /* AWGN channel */ 1738 I += sqrt(N0 / 2) * randn(0); 1739 Q += sqrt(N0 / 2) * randn(1); 1740 /* 1741 * Calculate the log of the probability that each of 1742 * the constellation points was transmitted 1743 */ 1744 for (m = 0; m < qam; m++) 1745 log_syml_prob[m] = -(pow(I - symbols_I[m], 2.0) 1746 + pow(Q - symbols_Q[m], 2.0)) / N0; 1747 /* Calculate an LLR for each of the k_64QAM bits in the set */ 1748 for (k = 0; k < qm; k++) { 1749 p0 = -999999; 1750 p1 = -999999; 1751 /* For each constellation point */ 1752 for (m = 0; m < qam; m++) { 1753 if ((m >> (qm - k - 1)) & 1) 1754 p1 = maxstar(p1, log_syml_prob[m]); 1755 else 1756 p0 = maxstar(p0, log_syml_prob[m]); 1757 } 1758 /* Calculate the LLR */ 1759 llr_ = p0 - p1; 1760 llr_ *= (1 << ldpc_llr_decimals); 1761 llr_ = round(llr_); 1762 if (llr_ > llr_max) 1763 llr_ = llr_max; 1764 if (llr_ < -llr_max) 1765 llr_ = -llr_max; 1766 llrs[qm * i + k] = (int8_t) llr_; 1767 } 1768 } 1769 1770 static void 1771 gen_qm2_llr(int8_t *llrs, uint32_t j, double N0, double llr_max) 1772 { 1773 double b, b1, n; 1774 double coeff = 2.0 * sqrt(N0); 1775 1776 /* Ignore in vectors rare quasi null LLRs not to be saturated */ 1777 if (llrs[j] < 8 && llrs[j] > -8) 1778 return; 1779 1780 /* Note don't change sign here */ 1781 n = randn(j % 2); 1782 b1 = ((llrs[j] > 0 ? 2.0 : -2.0) 1783 + coeff * n) / N0; 1784 b = b1 * (1 << ldpc_llr_decimals); 1785 b = round(b); 1786 if (b > llr_max) 1787 b = llr_max; 1788 if (b < -llr_max) 1789 b = -llr_max; 1790 llrs[j] = (int8_t) b; 1791 } 1792 1793 /* Simple LLR generation assuming AWGN and QPSK */ 1794 static void 1795 gen_turbo_llr(int8_t *llrs, uint32_t j, double N0, double llr_max) 1796 { 1797 double b, b1, n; 1798 double coeff = 2.0 * sqrt(N0); 1799 1800 /* Ignore in vectors null LLRs not to be saturated */ 1801 if (llrs[j] == 0) 1802 return; 1803 1804 /* Note don't change sign here */ 1805 n = randn(j % 2); 1806 b1 = ((llrs[j] > 0 ? 2.0 : -2.0) 1807 + coeff * n) / N0; 1808 b = b1 * (1 << 4); 1809 b = round(b); 1810 if (b > llr_max) 1811 b = llr_max; 1812 if (b < -llr_max) 1813 b = -llr_max; 1814 llrs[j] = (int8_t) b; 1815 } 1816 1817 /* Generate LLR for a given SNR */ 1818 static void 1819 generate_llr_input(uint16_t n, struct rte_bbdev_op_data *inputs, 1820 struct rte_bbdev_dec_op *ref_op) 1821 { 1822 struct rte_mbuf *m; 1823 uint16_t qm; 1824 uint32_t i, j, e, range; 1825 double N0, llr_max; 1826 1827 e = ref_op->ldpc_dec.cb_params.e; 1828 qm = ref_op->ldpc_dec.q_m; 1829 llr_max = (1 << (ldpc_llr_size - 1)) - 1; 1830 range = e / qm; 1831 N0 = 1.0 / pow(10.0, get_snr() / 10.0); 1832 1833 for (i = 0; i < n; ++i) { 1834 m = inputs[i].data; 1835 int8_t *llrs = rte_pktmbuf_mtod_offset(m, int8_t *, 0); 1836 if (qm == 8) { 1837 for (j = 0; j < range; ++j) 1838 gen_qm8_llr(llrs, j, N0, llr_max); 1839 } else if (qm == 6) { 1840 for (j = 0; j < range; ++j) 1841 gen_qm6_llr(llrs, j, N0, llr_max); 1842 } else if (qm == 4) { 1843 for (j = 0; j < range; ++j) 1844 gen_qm4_llr(llrs, j, N0, llr_max); 1845 } else { 1846 for (j = 0; j < e; ++j) 1847 gen_qm2_llr(llrs, j, N0, llr_max); 1848 } 1849 } 1850 } 1851 1852 /* Generate LLR for turbo decoder for a given SNR */ 1853 static void 1854 generate_turbo_llr_input(uint16_t n, struct rte_bbdev_op_data *inputs, 1855 struct rte_bbdev_dec_op *ref_op) 1856 { 1857 struct rte_mbuf *m; 1858 uint32_t i, j, range; 1859 double N0, llr_max; 1860 1861 llr_max = 127; 1862 range = ref_op->turbo_dec.input.length; 1863 N0 = 1.0 / pow(10.0, get_snr() / 10.0); 1864 1865 if (range > inputs[0].data->data_len) { 1866 printf("Warning: Limiting LLR generation to first segment (%d from %d)\n", 1867 inputs[0].data->data_len, range); 1868 range = inputs[0].data->data_len; 1869 } 1870 1871 for (i = 0; i < n; ++i) { 1872 m = inputs[i].data; 1873 int8_t *llrs = rte_pktmbuf_mtod_offset(m, int8_t *, 0); 1874 for (j = 0; j < range; ++j) 1875 gen_turbo_llr(llrs, j, N0, llr_max); 1876 } 1877 } 1878 1879 static void 1880 copy_reference_ldpc_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n, 1881 unsigned int start_idx, 1882 struct rte_bbdev_op_data *inputs, 1883 struct rte_bbdev_op_data *hard_outputs, 1884 struct rte_bbdev_op_data *soft_outputs, 1885 struct rte_bbdev_op_data *harq_inputs, 1886 struct rte_bbdev_op_data *harq_outputs, 1887 struct rte_bbdev_dec_op *ref_op) 1888 { 1889 unsigned int i; 1890 struct rte_bbdev_op_ldpc_dec *ldpc_dec = &ref_op->ldpc_dec; 1891 1892 for (i = 0; i < n; ++i) { 1893 if (ldpc_dec->code_block_mode == RTE_BBDEV_TRANSPORT_BLOCK) { 1894 ops[i]->ldpc_dec.tb_params.ea = 1895 ldpc_dec->tb_params.ea; 1896 ops[i]->ldpc_dec.tb_params.eb = 1897 ldpc_dec->tb_params.eb; 1898 ops[i]->ldpc_dec.tb_params.c = 1899 ldpc_dec->tb_params.c; 1900 ops[i]->ldpc_dec.tb_params.cab = 1901 ldpc_dec->tb_params.cab; 1902 ops[i]->ldpc_dec.tb_params.r = 1903 ldpc_dec->tb_params.r; 1904 } else { 1905 ops[i]->ldpc_dec.cb_params.e = ldpc_dec->cb_params.e; 1906 } 1907 1908 ops[i]->ldpc_dec.basegraph = ldpc_dec->basegraph; 1909 ops[i]->ldpc_dec.z_c = ldpc_dec->z_c; 1910 ops[i]->ldpc_dec.q_m = ldpc_dec->q_m; 1911 ops[i]->ldpc_dec.n_filler = ldpc_dec->n_filler; 1912 ops[i]->ldpc_dec.n_cb = ldpc_dec->n_cb; 1913 ops[i]->ldpc_dec.iter_max = ldpc_dec->iter_max; 1914 ops[i]->ldpc_dec.rv_index = ldpc_dec->rv_index; 1915 ops[i]->ldpc_dec.op_flags = ldpc_dec->op_flags; 1916 ops[i]->ldpc_dec.code_block_mode = ldpc_dec->code_block_mode; 1917 1918 if (hard_outputs != NULL) 1919 ops[i]->ldpc_dec.hard_output = 1920 hard_outputs[start_idx + i]; 1921 if (inputs != NULL) 1922 ops[i]->ldpc_dec.input = 1923 inputs[start_idx + i]; 1924 if (soft_outputs != NULL) 1925 ops[i]->ldpc_dec.soft_output = 1926 soft_outputs[start_idx + i]; 1927 if (harq_inputs != NULL) 1928 ops[i]->ldpc_dec.harq_combined_input = 1929 harq_inputs[start_idx + i]; 1930 if (harq_outputs != NULL) 1931 ops[i]->ldpc_dec.harq_combined_output = 1932 harq_outputs[start_idx + i]; 1933 } 1934 } 1935 1936 1937 static void 1938 copy_reference_ldpc_enc_op(struct rte_bbdev_enc_op **ops, unsigned int n, 1939 unsigned int start_idx, 1940 struct rte_bbdev_op_data *inputs, 1941 struct rte_bbdev_op_data *outputs, 1942 struct rte_bbdev_enc_op *ref_op) 1943 { 1944 unsigned int i; 1945 struct rte_bbdev_op_ldpc_enc *ldpc_enc = &ref_op->ldpc_enc; 1946 for (i = 0; i < n; ++i) { 1947 if (ldpc_enc->code_block_mode == RTE_BBDEV_TRANSPORT_BLOCK) { 1948 ops[i]->ldpc_enc.tb_params.ea = ldpc_enc->tb_params.ea; 1949 ops[i]->ldpc_enc.tb_params.eb = ldpc_enc->tb_params.eb; 1950 ops[i]->ldpc_enc.tb_params.cab = 1951 ldpc_enc->tb_params.cab; 1952 ops[i]->ldpc_enc.tb_params.c = ldpc_enc->tb_params.c; 1953 ops[i]->ldpc_enc.tb_params.r = ldpc_enc->tb_params.r; 1954 } else { 1955 ops[i]->ldpc_enc.cb_params.e = ldpc_enc->cb_params.e; 1956 } 1957 ops[i]->ldpc_enc.basegraph = ldpc_enc->basegraph; 1958 ops[i]->ldpc_enc.z_c = ldpc_enc->z_c; 1959 ops[i]->ldpc_enc.q_m = ldpc_enc->q_m; 1960 ops[i]->ldpc_enc.n_filler = ldpc_enc->n_filler; 1961 ops[i]->ldpc_enc.n_cb = ldpc_enc->n_cb; 1962 ops[i]->ldpc_enc.rv_index = ldpc_enc->rv_index; 1963 ops[i]->ldpc_enc.op_flags = ldpc_enc->op_flags; 1964 ops[i]->ldpc_enc.code_block_mode = ldpc_enc->code_block_mode; 1965 ops[i]->ldpc_enc.output = outputs[start_idx + i]; 1966 ops[i]->ldpc_enc.input = inputs[start_idx + i]; 1967 } 1968 } 1969 1970 static void 1971 copy_reference_fft_op(struct rte_bbdev_fft_op **ops, unsigned int n, 1972 unsigned int start_idx, struct rte_bbdev_op_data *inputs, 1973 struct rte_bbdev_op_data *outputs, struct rte_bbdev_op_data *pwrouts, 1974 struct rte_bbdev_op_data *win_inputs, struct rte_bbdev_fft_op *ref_op) 1975 { 1976 unsigned int i, j; 1977 struct rte_bbdev_op_fft *fft = &ref_op->fft; 1978 for (i = 0; i < n; i++) { 1979 ops[i]->fft.input_sequence_size = fft->input_sequence_size; 1980 ops[i]->fft.input_leading_padding = fft->input_leading_padding; 1981 ops[i]->fft.output_sequence_size = fft->output_sequence_size; 1982 ops[i]->fft.output_leading_depadding = 1983 fft->output_leading_depadding; 1984 for (j = 0; j < RTE_BBDEV_MAX_CS_2; j++) 1985 ops[i]->fft.window_index[j] = fft->window_index[j]; 1986 for (j = 0; j < RTE_BBDEV_MAX_CS; j++) { 1987 ops[i]->fft.cs_theta_0[j] = fft->cs_theta_0[j]; 1988 ops[i]->fft.cs_theta_d[j] = fft->cs_theta_d[j]; 1989 ops[i]->fft.time_offset[j] = fft->time_offset[j]; 1990 } 1991 ops[i]->fft.cs_bitmap = fft->cs_bitmap; 1992 ops[i]->fft.num_antennas_log2 = fft->num_antennas_log2; 1993 ops[i]->fft.idft_log2 = fft->idft_log2; 1994 ops[i]->fft.dft_log2 = fft->dft_log2; 1995 ops[i]->fft.cs_time_adjustment = fft->cs_time_adjustment; 1996 ops[i]->fft.idft_shift = fft->idft_shift; 1997 ops[i]->fft.dft_shift = fft->dft_shift; 1998 ops[i]->fft.ncs_reciprocal = fft->ncs_reciprocal; 1999 ops[i]->fft.power_shift = fft->power_shift; 2000 ops[i]->fft.fp16_exp_adjust = fft->fp16_exp_adjust; 2001 ops[i]->fft.output_depadded_size = fft->output_depadded_size; 2002 ops[i]->fft.freq_resample_mode = fft->freq_resample_mode; 2003 ops[i]->fft.base_output = outputs[start_idx + i]; 2004 ops[i]->fft.base_input = inputs[start_idx + i]; 2005 if (win_inputs != NULL) 2006 ops[i]->fft.dewindowing_input = win_inputs[start_idx + i]; 2007 if (pwrouts != NULL) 2008 ops[i]->fft.power_meas_output = pwrouts[start_idx + i]; 2009 ops[i]->fft.op_flags = fft->op_flags; 2010 } 2011 } 2012 2013 static void 2014 copy_reference_mldts_op(struct rte_bbdev_mldts_op **ops, unsigned int n, 2015 unsigned int start_idx, 2016 struct rte_bbdev_op_data *q_inputs, 2017 struct rte_bbdev_op_data *r_inputs, 2018 struct rte_bbdev_op_data *outputs, 2019 struct rte_bbdev_mldts_op *ref_op) 2020 { 2021 unsigned int i, j; 2022 struct rte_bbdev_op_mldts *mldts = &ref_op->mldts; 2023 for (i = 0; i < n; i++) { 2024 ops[i]->mldts.c_rep = mldts->c_rep; 2025 ops[i]->mldts.num_layers = mldts->num_layers; 2026 ops[i]->mldts.num_rbs = mldts->num_rbs; 2027 ops[i]->mldts.op_flags = mldts->op_flags; 2028 for (j = 0; j < RTE_BBDEV_MAX_MLD_LAYERS; j++) 2029 ops[i]->mldts.q_m[j] = mldts->q_m[j]; 2030 ops[i]->mldts.r_rep = mldts->r_rep; 2031 ops[i]->mldts.c_rep = mldts->c_rep; 2032 ops[i]->mldts.r_input = r_inputs[start_idx + i]; 2033 ops[i]->mldts.qhy_input = q_inputs[start_idx + i]; 2034 ops[i]->mldts.output = outputs[start_idx + i]; 2035 } 2036 } 2037 2038 static int 2039 check_dec_status_and_ordering(struct rte_bbdev_dec_op *op, 2040 unsigned int order_idx, const int expected_status) 2041 { 2042 int status = op->status; 2043 /* ignore parity mismatch false alarms for long iterations */ 2044 if (get_iter_max() >= 10) { 2045 if (!(expected_status & (1 << RTE_BBDEV_SYNDROME_ERROR)) && 2046 (status & (1 << RTE_BBDEV_SYNDROME_ERROR))) { 2047 printf("WARNING: Ignore Syndrome Check mismatch\n"); 2048 status -= (1 << RTE_BBDEV_SYNDROME_ERROR); 2049 } 2050 if ((expected_status & (1 << RTE_BBDEV_SYNDROME_ERROR)) && 2051 !(status & (1 << RTE_BBDEV_SYNDROME_ERROR))) { 2052 printf("WARNING: Ignore Syndrome Check mismatch\n"); 2053 status += (1 << RTE_BBDEV_SYNDROME_ERROR); 2054 } 2055 } 2056 2057 TEST_ASSERT(status == expected_status, 2058 "op_status (%d) != expected_status (%d)", 2059 op->status, expected_status); 2060 2061 TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data, 2062 "Ordering error, expected %p, got %p", 2063 (void *)(uintptr_t)order_idx, op->opaque_data); 2064 2065 return TEST_SUCCESS; 2066 } 2067 2068 static int 2069 check_enc_status_and_ordering(struct rte_bbdev_enc_op *op, 2070 unsigned int order_idx, const int expected_status) 2071 { 2072 TEST_ASSERT(op->status == expected_status, 2073 "op_status (%d) != expected_status (%d)", 2074 op->status, expected_status); 2075 2076 TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data, 2077 "Ordering error, expected %p, got %p", 2078 (void *)(uintptr_t)order_idx, op->opaque_data); 2079 2080 return TEST_SUCCESS; 2081 } 2082 2083 static int 2084 check_fft_status_and_ordering(struct rte_bbdev_fft_op *op, 2085 unsigned int order_idx, const int expected_status) 2086 { 2087 TEST_ASSERT(op->status == expected_status, 2088 "op_status (%d) != expected_status (%d)", 2089 op->status, expected_status); 2090 2091 TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data, 2092 "Ordering error, expected %p, got %p", 2093 (void *)(uintptr_t)order_idx, op->opaque_data); 2094 2095 return TEST_SUCCESS; 2096 } 2097 2098 static int 2099 check_mldts_status_and_ordering(struct rte_bbdev_mldts_op *op, 2100 unsigned int order_idx, const int expected_status) 2101 { 2102 TEST_ASSERT(op->status == expected_status, 2103 "op_status (%d) != expected_status (%d)", 2104 op->status, expected_status); 2105 2106 TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data, 2107 "Ordering error, expected %p, got %p", 2108 (void *)(uintptr_t)order_idx, op->opaque_data); 2109 2110 return TEST_SUCCESS; 2111 } 2112 2113 static inline int 2114 validate_op_chain(struct rte_bbdev_op_data *op, 2115 struct op_data_entries *orig_op) 2116 { 2117 uint8_t i; 2118 struct rte_mbuf *m = op->data; 2119 uint8_t nb_dst_segments = orig_op->nb_segments; 2120 uint32_t total_data_size = 0; 2121 bool ignore_mbuf = false; /* ignore mbuf limitations */ 2122 2123 TEST_ASSERT(nb_dst_segments == m->nb_segs, 2124 "Number of segments differ in original (%u) and filled (%u) op", 2125 nb_dst_segments, m->nb_segs); 2126 2127 /* Validate each mbuf segment length */ 2128 for (i = 0; i < nb_dst_segments; ++i) { 2129 /* Apply offset to the first mbuf segment */ 2130 uint16_t offset = (i == 0) ? op->offset : 0; 2131 uint16_t data_len = rte_pktmbuf_data_len(m) - offset; 2132 total_data_size += orig_op->segments[i].length; 2133 2134 if (orig_op->segments[i].length > RTE_BBDEV_LDPC_E_MAX_MBUF) 2135 ignore_mbuf = true; 2136 if (!ignore_mbuf) 2137 TEST_ASSERT(orig_op->segments[i].length == data_len, 2138 "Length of segment differ in original (%u) and filled (%u) op", 2139 orig_op->segments[i].length, data_len); 2140 TEST_ASSERT_BUFFERS_ARE_EQUAL(orig_op->segments[i].addr, 2141 rte_pktmbuf_mtod_offset(m, uint32_t *, offset), 2142 orig_op->segments[i].length, 2143 "Output buffers (CB=%u) are not equal", i); 2144 m = m->next; 2145 } 2146 2147 /* Validate total mbuf pkt length */ 2148 uint32_t pkt_len = rte_pktmbuf_pkt_len(op->data) - op->offset; 2149 if (!ignore_mbuf) 2150 TEST_ASSERT(total_data_size == pkt_len, 2151 "Length of data differ in original (%u) and filled (%u) op", 2152 total_data_size, pkt_len); 2153 2154 return TEST_SUCCESS; 2155 } 2156 2157 /* 2158 * Compute K0 for a given configuration for HARQ output length computation 2159 * As per definition in 3GPP 38.212 Table 5.4.2.1-2 2160 */ 2161 static inline uint16_t 2162 get_k0(uint16_t n_cb, uint16_t z_c, uint8_t bg, uint8_t rv_index) 2163 { 2164 if (rv_index == 0) 2165 return 0; 2166 uint16_t n = (bg == 1 ? N_ZC_1 : N_ZC_2) * z_c; 2167 if (n_cb == n) { 2168 if (rv_index == 1) 2169 return (bg == 1 ? K0_1_1 : K0_1_2) * z_c; 2170 else if (rv_index == 2) 2171 return (bg == 1 ? K0_2_1 : K0_2_2) * z_c; 2172 else 2173 return (bg == 1 ? K0_3_1 : K0_3_2) * z_c; 2174 } 2175 /* LBRM case - includes a division by N */ 2176 if (rv_index == 1) 2177 return (((bg == 1 ? K0_1_1 : K0_1_2) * n_cb) 2178 / n) * z_c; 2179 else if (rv_index == 2) 2180 return (((bg == 1 ? K0_2_1 : K0_2_2) * n_cb) 2181 / n) * z_c; 2182 else 2183 return (((bg == 1 ? K0_3_1 : K0_3_2) * n_cb) 2184 / n) * z_c; 2185 } 2186 2187 /* HARQ output length including the Filler bits */ 2188 static inline uint16_t 2189 compute_harq_len(struct rte_bbdev_op_ldpc_dec *ops_ld) 2190 { 2191 uint16_t k0 = 0; 2192 uint8_t max_rv = (ops_ld->rv_index == 1) ? 3 : ops_ld->rv_index; 2193 k0 = get_k0(ops_ld->n_cb, ops_ld->z_c, ops_ld->basegraph, max_rv); 2194 /* Compute RM out size and number of rows */ 2195 uint16_t parity_offset = (ops_ld->basegraph == 1 ? 20 : 8) 2196 * ops_ld->z_c - ops_ld->n_filler; 2197 uint16_t deRmOutSize = RTE_MIN( 2198 k0 + ops_ld->cb_params.e + 2199 ((k0 > parity_offset) ? 2200 0 : ops_ld->n_filler), 2201 ops_ld->n_cb); 2202 uint16_t numRows = ((deRmOutSize + ops_ld->z_c - 1) 2203 / ops_ld->z_c); 2204 uint16_t harq_output_len = numRows * ops_ld->z_c; 2205 return harq_output_len; 2206 } 2207 2208 static inline int 2209 validate_op_harq_chain(struct rte_bbdev_op_data *op, 2210 struct op_data_entries *orig_op, 2211 struct rte_bbdev_op_ldpc_dec *ops_ld) 2212 { 2213 uint8_t i; 2214 uint32_t j, jj, k; 2215 struct rte_mbuf *m = op->data; 2216 uint8_t nb_dst_segments = orig_op->nb_segments; 2217 uint32_t total_data_size = 0; 2218 int8_t *harq_orig, *harq_out, abs_harq_origin; 2219 uint32_t byte_error = 0, cum_error = 0, error; 2220 int16_t llr_max = (1 << (ldpc_llr_size - ldpc_llr_decimals)) - 1; 2221 int16_t llr_max_pre_scaling = (1 << (ldpc_llr_size - 1)) - 1; 2222 uint16_t parity_offset; 2223 2224 TEST_ASSERT(nb_dst_segments == m->nb_segs, 2225 "Number of segments differ in original (%u) and filled (%u) op", 2226 nb_dst_segments, m->nb_segs); 2227 2228 /* Validate each mbuf segment length */ 2229 for (i = 0; i < nb_dst_segments; ++i) { 2230 /* Apply offset to the first mbuf segment */ 2231 uint16_t offset = (i == 0) ? op->offset : 0; 2232 uint16_t data_len = rte_pktmbuf_data_len(m) - offset; 2233 total_data_size += orig_op->segments[i].length; 2234 2235 TEST_ASSERT(orig_op->segments[i].length < (uint32_t)(data_len + HARQ_MEM_TOLERANCE), 2236 "Length of segment differ in original (%u) and filled (%u) op", 2237 orig_op->segments[i].length, data_len); 2238 harq_orig = (int8_t *) orig_op->segments[i].addr; 2239 harq_out = rte_pktmbuf_mtod_offset(m, int8_t *, offset); 2240 2241 /* Cannot compare HARQ output data for such cases */ 2242 if ((ldpc_llr_decimals > 1) && ((ops_ld->op_flags & RTE_BBDEV_LDPC_LLR_COMPRESSION) 2243 || (ops_ld->op_flags & RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION) 2244 || (ops_ld->op_flags & RTE_BBDEV_LDPC_HARQ_4BIT_COMPRESSION))) 2245 break; 2246 2247 if (!(ldpc_cap_flags & 2248 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_FILLERS 2249 ) || (ops_ld->op_flags & 2250 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)) { 2251 data_len -= ops_ld->z_c; 2252 parity_offset = data_len; 2253 } else { 2254 /* Compute RM out size and number of rows */ 2255 parity_offset = (ops_ld->basegraph == 1 ? 20 : 8) 2256 * ops_ld->z_c - ops_ld->n_filler; 2257 uint16_t deRmOutSize = compute_harq_len(ops_ld) - 2258 ops_ld->n_filler; 2259 if (data_len > deRmOutSize) 2260 data_len = deRmOutSize; 2261 } 2262 if (data_len > orig_op->segments[i].length) 2263 data_len = orig_op->segments[i].length; 2264 /* 2265 * HARQ output can have minor differences 2266 * due to integer representation and related scaling 2267 */ 2268 for (j = 0, jj = 0; j < data_len; j++, jj++) { 2269 if (j == parity_offset) { 2270 /* Special Handling of the filler bits */ 2271 for (k = 0; k < ops_ld->n_filler; k++) { 2272 if (harq_out[jj] != 2273 llr_max_pre_scaling) { 2274 printf("HARQ Filler issue %d: %d %d\n", 2275 jj, harq_out[jj], 2276 llr_max); 2277 byte_error++; 2278 } 2279 jj++; 2280 } 2281 } 2282 if (!(ops_ld->op_flags & 2283 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)) { 2284 if (ldpc_llr_decimals > 1) 2285 harq_out[jj] = (harq_out[jj] + 1) 2286 >> (ldpc_llr_decimals - 1); 2287 /* Saturated to S7 */ 2288 if (harq_orig[j] > llr_max) 2289 harq_orig[j] = llr_max; 2290 if (harq_orig[j] < -llr_max) 2291 harq_orig[j] = -llr_max; 2292 } 2293 if (harq_orig[j] != harq_out[jj]) { 2294 error = (harq_orig[j] > harq_out[jj]) ? 2295 harq_orig[j] - harq_out[jj] : 2296 harq_out[jj] - harq_orig[j]; 2297 abs_harq_origin = harq_orig[j] > 0 ? 2298 harq_orig[j] : 2299 -harq_orig[j]; 2300 /* Residual quantization error */ 2301 if ((error > 8 && (abs_harq_origin < 2302 (llr_max - 16))) || 2303 (error > 16)) { 2304 /* 2305 printf("HARQ mismatch %d: exp %d act %d => %d\n", 2306 j, harq_orig[j], 2307 harq_out[jj], error); 2308 */ 2309 byte_error++; 2310 cum_error += error; 2311 } 2312 } 2313 } 2314 m = m->next; 2315 } 2316 2317 if (byte_error) 2318 TEST_ASSERT(byte_error <= 1, 2319 "HARQ output mismatch (%d) %d", 2320 byte_error, cum_error); 2321 2322 /* Validate total mbuf pkt length */ 2323 uint32_t pkt_len = rte_pktmbuf_pkt_len(op->data) - op->offset; 2324 TEST_ASSERT(total_data_size < pkt_len + HARQ_MEM_TOLERANCE, 2325 "Length of data differ in original (%u) and filled (%u) op", 2326 total_data_size, pkt_len); 2327 2328 return TEST_SUCCESS; 2329 } 2330 2331 2332 static inline int 2333 validate_op_so_chain(struct rte_bbdev_op_data *op, 2334 struct op_data_entries *orig_op) 2335 { 2336 struct rte_mbuf *m = op->data; 2337 uint8_t i, nb_dst_segments = orig_op->nb_segments; 2338 uint32_t j, jj; 2339 int8_t *so_orig, *so_out; 2340 uint32_t byte_error = 0, error, margin_error = 0; 2341 2342 TEST_ASSERT(nb_dst_segments == m->nb_segs, 2343 "Number of segments differ in original (%u) and filled (%u) op", 2344 nb_dst_segments, m->nb_segs); 2345 2346 /* Validate each mbuf segment length. */ 2347 for (i = 0; i < nb_dst_segments; ++i) { 2348 /* Apply offset to the first mbuf segment. */ 2349 uint16_t offset = (i == 0) ? op->offset : 0; 2350 uint16_t data_len = rte_pktmbuf_data_len(m) - offset; 2351 2352 TEST_ASSERT(orig_op->segments[i].length == data_len, 2353 "Length of segment differ in original (%u) and filled (%u) op", 2354 orig_op->segments[i].length, data_len); 2355 so_orig = (int8_t *) orig_op->segments[i].addr; 2356 so_out = rte_pktmbuf_mtod_offset(m, int8_t *, offset); 2357 margin_error += data_len / 8; /* Allow for few % errors. */ 2358 2359 /* SO output can have minor differences due to algorithm variations. */ 2360 for (j = 0, jj = 0; j < data_len; j++, jj++) { 2361 if (so_orig[j] != so_out[jj]) { 2362 error = (so_orig[j] > so_out[jj]) ? so_orig[j] - so_out[jj] : 2363 so_out[jj] - so_orig[j]; 2364 /* Residual quantization error. */ 2365 if (error > 32) { 2366 printf("Warning: Soft mismatch %d: exp %d act %d => %d\n", 2367 j, so_orig[j], so_out[jj], error); 2368 byte_error++; 2369 } 2370 } 2371 } 2372 m = m->next; 2373 } 2374 2375 if (byte_error > margin_error) 2376 TEST_ASSERT(byte_error <= 1, "Soft output mismatch (%d) %d", 2377 byte_error, margin_error); 2378 2379 return TEST_SUCCESS; 2380 } 2381 2382 static int 2383 validate_dec_op(struct rte_bbdev_dec_op **ops, const uint16_t n, 2384 struct rte_bbdev_dec_op *ref_op) 2385 { 2386 unsigned int i; 2387 int ret; 2388 struct op_data_entries *hard_data_orig = 2389 &test_vector.entries[DATA_HARD_OUTPUT]; 2390 struct op_data_entries *soft_data_orig = 2391 &test_vector.entries[DATA_SOFT_OUTPUT]; 2392 struct rte_bbdev_op_turbo_dec *ops_td; 2393 struct rte_bbdev_op_data *hard_output; 2394 struct rte_bbdev_op_data *soft_output; 2395 2396 for (i = 0; i < n; ++i) { 2397 ops_td = &ops[i]->turbo_dec; 2398 hard_output = &ops_td->hard_output; 2399 soft_output = &ops_td->soft_output; 2400 2401 ret = check_dec_status_and_ordering(ops[i], i, ref_op->status); 2402 TEST_ASSERT_SUCCESS(ret, 2403 "Checking status and ordering for decoder failed"); 2404 2405 TEST_ASSERT_SUCCESS(validate_op_chain(hard_output, 2406 hard_data_orig), 2407 "Hard output buffers (CB=%u) are not equal", 2408 i); 2409 2410 if (ref_op->turbo_dec.op_flags & RTE_BBDEV_TURBO_SOFT_OUTPUT) 2411 TEST_ASSERT_SUCCESS(validate_op_so_chain(soft_output, 2412 soft_data_orig), 2413 "Soft output buffers (CB=%u) are not equal", 2414 i); 2415 } 2416 2417 return TEST_SUCCESS; 2418 } 2419 2420 /* Check Number of code blocks errors */ 2421 static int 2422 validate_ldpc_bler(struct rte_bbdev_dec_op **ops, const uint16_t n) 2423 { 2424 unsigned int i; 2425 struct op_data_entries *hard_data_orig = 2426 &test_vector.entries[DATA_HARD_OUTPUT]; 2427 struct rte_bbdev_op_ldpc_dec *ops_td; 2428 struct rte_bbdev_op_data *hard_output; 2429 int errors = 0; 2430 struct rte_mbuf *m; 2431 2432 for (i = 0; i < n; ++i) { 2433 ops_td = &ops[i]->ldpc_dec; 2434 hard_output = &ops_td->hard_output; 2435 m = hard_output->data; 2436 if (memcmp(rte_pktmbuf_mtod_offset(m, uint32_t *, 0), 2437 hard_data_orig->segments[0].addr, 2438 hard_data_orig->segments[0].length)) 2439 errors++; 2440 } 2441 return errors; 2442 } 2443 2444 /* Check Number of code blocks errors */ 2445 static int 2446 validate_turbo_bler(struct rte_bbdev_dec_op **ops, const uint16_t n) 2447 { 2448 unsigned int i; 2449 struct op_data_entries *hard_data_orig = &test_vector.entries[DATA_HARD_OUTPUT]; 2450 struct rte_bbdev_op_turbo_dec *ops_td; 2451 struct rte_bbdev_op_data *hard_output; 2452 int errors = 0; 2453 struct rte_mbuf *m; 2454 2455 for (i = 0; i < n; ++i) { 2456 ops_td = &ops[i]->turbo_dec; 2457 hard_output = &ops_td->hard_output; 2458 m = hard_output->data; 2459 if (memcmp(rte_pktmbuf_mtod_offset(m, uint32_t *, 0), 2460 hard_data_orig->segments[0].addr, 2461 hard_data_orig->segments[0].length)) 2462 errors++; 2463 } 2464 return errors; 2465 } 2466 2467 2468 static int 2469 validate_ldpc_dec_op(struct rte_bbdev_dec_op **ops, const uint16_t n, 2470 struct rte_bbdev_dec_op *ref_op, const int vector_mask) 2471 { 2472 unsigned int i; 2473 int ret; 2474 struct op_data_entries *hard_data_orig = 2475 &test_vector.entries[DATA_HARD_OUTPUT]; 2476 struct op_data_entries *soft_data_orig = 2477 &test_vector.entries[DATA_SOFT_OUTPUT]; 2478 struct op_data_entries *harq_data_orig = 2479 &test_vector.entries[DATA_HARQ_OUTPUT]; 2480 struct rte_bbdev_op_ldpc_dec *ops_td; 2481 struct rte_bbdev_op_data *hard_output; 2482 struct rte_bbdev_op_data *harq_output; 2483 struct rte_bbdev_op_data *soft_output; 2484 struct rte_bbdev_op_ldpc_dec *ref_td = &ref_op->ldpc_dec; 2485 2486 for (i = 0; i < n; ++i) { 2487 ops_td = &ops[i]->ldpc_dec; 2488 hard_output = &ops_td->hard_output; 2489 harq_output = &ops_td->harq_combined_output; 2490 soft_output = &ops_td->soft_output; 2491 2492 ret = check_dec_status_and_ordering(ops[i], i, ref_op->status); 2493 TEST_ASSERT_SUCCESS(ret, 2494 "Checking status and ordering for decoder failed"); 2495 if (vector_mask & TEST_BBDEV_VF_EXPECTED_ITER_COUNT) 2496 TEST_ASSERT(ops_td->iter_count <= ref_td->iter_count, 2497 "Returned iter_count (%d) > expected iter_count (%d)", 2498 ops_td->iter_count, ref_td->iter_count); 2499 /* 2500 * We can ignore output data when the decoding failed to 2501 * converge or for loop-back cases 2502 */ 2503 if (!check_bit(ops[i]->ldpc_dec.op_flags, 2504 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK 2505 ) && ( 2506 ops[i]->status & (1 << RTE_BBDEV_SYNDROME_ERROR 2507 )) == 0) 2508 TEST_ASSERT_SUCCESS(validate_op_chain(hard_output, 2509 hard_data_orig), 2510 "Hard output buffers (CB=%u) are not equal", 2511 i); 2512 2513 if (ref_op->ldpc_dec.op_flags & RTE_BBDEV_LDPC_SOFT_OUT_ENABLE) 2514 TEST_ASSERT_SUCCESS(validate_op_so_chain(soft_output, 2515 soft_data_orig), 2516 "Soft output buffers (CB=%u) are not equal", 2517 i); 2518 if (ref_op->ldpc_dec.op_flags & 2519 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE) { 2520 TEST_ASSERT_SUCCESS(validate_op_harq_chain(harq_output, 2521 harq_data_orig, ops_td), 2522 "HARQ output buffers (CB=%u) are not equal", 2523 i); 2524 } 2525 if (ref_op->ldpc_dec.op_flags & 2526 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK) 2527 TEST_ASSERT_SUCCESS(validate_op_harq_chain(harq_output, 2528 harq_data_orig, ops_td), 2529 "HARQ output buffers (CB=%u) are not equal", 2530 i); 2531 2532 } 2533 2534 return TEST_SUCCESS; 2535 } 2536 2537 2538 static int 2539 validate_enc_op(struct rte_bbdev_enc_op **ops, const uint16_t n, 2540 struct rte_bbdev_enc_op *ref_op) 2541 { 2542 unsigned int i; 2543 int ret; 2544 struct op_data_entries *hard_data_orig = 2545 &test_vector.entries[DATA_HARD_OUTPUT]; 2546 2547 for (i = 0; i < n; ++i) { 2548 ret = check_enc_status_and_ordering(ops[i], i, ref_op->status); 2549 TEST_ASSERT_SUCCESS(ret, 2550 "Checking status and ordering for encoder failed"); 2551 TEST_ASSERT_SUCCESS(validate_op_chain( 2552 &ops[i]->turbo_enc.output, 2553 hard_data_orig), 2554 "Output buffers (CB=%u) are not equal", 2555 i); 2556 } 2557 2558 return TEST_SUCCESS; 2559 } 2560 2561 static int 2562 validate_ldpc_enc_op(struct rte_bbdev_enc_op **ops, const uint16_t n, 2563 struct rte_bbdev_enc_op *ref_op) 2564 { 2565 unsigned int i; 2566 int ret; 2567 struct op_data_entries *hard_data_orig = 2568 &test_vector.entries[DATA_HARD_OUTPUT]; 2569 2570 for (i = 0; i < n; ++i) { 2571 ret = check_enc_status_and_ordering(ops[i], i, ref_op->status); 2572 TEST_ASSERT_SUCCESS(ret, 2573 "Checking status and ordering for encoder failed"); 2574 TEST_ASSERT_SUCCESS(validate_op_chain( 2575 &ops[i]->ldpc_enc.output, 2576 hard_data_orig), 2577 "Output buffers (CB=%u) are not equal", 2578 i); 2579 } 2580 2581 return TEST_SUCCESS; 2582 } 2583 2584 static inline int 2585 validate_op_fft_chain(struct rte_bbdev_op_data *op, struct op_data_entries *orig_op) 2586 { 2587 struct rte_mbuf *m = op->data; 2588 uint8_t i, nb_dst_segments = orig_op->nb_segments; 2589 int16_t delt, abs_delt, thres_hold = 4; 2590 uint32_t j, data_len_iq, error_num; 2591 int16_t *ref_out, *op_out; 2592 2593 TEST_ASSERT(nb_dst_segments == m->nb_segs, 2594 "Number of segments differ in original (%u) and filled (%u) op fft", 2595 nb_dst_segments, m->nb_segs); 2596 2597 /* Due to size limitation of mbuf, FFT doesn't use real mbuf. */ 2598 for (i = 0; i < nb_dst_segments; ++i) { 2599 uint16_t offset = (i == 0) ? op->offset : 0; 2600 uint32_t data_len = op->length; 2601 2602 TEST_ASSERT(orig_op->segments[i].length == data_len, 2603 "Length of segment differ in original (%u) and filled (%u) op fft", 2604 orig_op->segments[i].length, data_len); 2605 /* Divided by 2 to get the number of 16bits data. */ 2606 data_len_iq = data_len >> 1; 2607 ref_out = (int16_t *)(orig_op->segments[i].addr); 2608 op_out = rte_pktmbuf_mtod_offset(m, int16_t *, offset); 2609 error_num = 0; 2610 for (j = 0; j < data_len_iq; j++) { 2611 delt = ref_out[j] - op_out[j]; 2612 abs_delt = delt > 0 ? delt : -delt; 2613 error_num += (abs_delt > thres_hold ? 1 : 0); 2614 } 2615 if (error_num > 0) { 2616 rte_memdump(stdout, "Buffer A", ref_out, data_len); 2617 rte_memdump(stdout, "Buffer B", op_out, data_len); 2618 TEST_ASSERT(error_num == 0, 2619 "FFT Output are not matched total (%u) errors (%u)", 2620 data_len_iq, error_num); 2621 } 2622 2623 m = m->next; 2624 } 2625 2626 return TEST_SUCCESS; 2627 } 2628 2629 static inline int 2630 validate_op_mldts_chain(struct rte_bbdev_op_data *op, 2631 struct op_data_entries *orig_op) 2632 { 2633 uint8_t i; 2634 struct rte_mbuf *m = op->data; 2635 uint8_t nb_dst_segments = orig_op->nb_segments; 2636 /*the result is not bit exact*/ 2637 int16_t thres_hold = 3; 2638 int16_t delt, abs_delt; 2639 uint32_t j, data_len_iq; 2640 uint32_t error_num; 2641 int8_t *ref_out; 2642 int8_t *op_out; 2643 2644 TEST_ASSERT(nb_dst_segments == m->nb_segs, 2645 "Number of segments differ in original (%u) and filled (%u) op mldts", 2646 nb_dst_segments, m->nb_segs); 2647 2648 /* Due to size limitation of mbuf, MLDTS doesn't use real mbuf. */ 2649 for (i = 0; i < nb_dst_segments; ++i) { 2650 uint16_t offset = (i == 0) ? op->offset : 0; 2651 uint32_t data_len = op->length; 2652 2653 TEST_ASSERT(orig_op->segments[i].length == data_len, 2654 "Length of segment differ in original (%u) and filled (%u) op mldts", 2655 orig_op->segments[i].length, data_len); 2656 data_len_iq = data_len; 2657 ref_out = (int8_t *)(orig_op->segments[i].addr); 2658 op_out = rte_pktmbuf_mtod_offset(m, int8_t *, offset); 2659 error_num = 0; 2660 for (j = 0; j < data_len_iq; j++) { 2661 2662 delt = ref_out[j] - op_out[j]; 2663 abs_delt = delt > 0 ? delt : -delt; 2664 error_num += (abs_delt > thres_hold ? 1 : 0); 2665 if (error_num > 0) 2666 printf("MLD Error %d: Exp %x %d Actual %x %d Diff %d\n", 2667 j, ref_out[j], ref_out[j], op_out[j], op_out[j], 2668 delt); 2669 } 2670 TEST_ASSERT(error_num == 0, 2671 "MLDTS Output are not matched total (%u) errors (%u)", 2672 data_len_iq, error_num); 2673 2674 m = m->next; 2675 } 2676 2677 return TEST_SUCCESS; 2678 } 2679 2680 static int 2681 validate_fft_op(struct rte_bbdev_fft_op **ops, const uint16_t n, 2682 struct rte_bbdev_fft_op *ref_op) 2683 { 2684 unsigned int i; 2685 int ret; 2686 struct op_data_entries *fft_data_orig = &test_vector.entries[DATA_HARD_OUTPUT]; 2687 struct op_data_entries *fft_pwr_orig = &test_vector.entries[DATA_SOFT_OUTPUT]; 2688 2689 for (i = 0; i < n; ++i) { 2690 ret = check_fft_status_and_ordering(ops[i], i, ref_op->status); 2691 TEST_ASSERT_SUCCESS(ret, "Checking status and ordering for FFT failed"); 2692 TEST_ASSERT_SUCCESS(validate_op_fft_chain( 2693 &ops[i]->fft.base_output, fft_data_orig), 2694 "FFT Output buffers (op=%u) are not matched", i); 2695 if (check_bit(ops[i]->fft.op_flags, RTE_BBDEV_FFT_POWER_MEAS)) 2696 TEST_ASSERT_SUCCESS(validate_op_fft_chain( 2697 &ops[i]->fft.power_meas_output, fft_pwr_orig), 2698 "FFT Power Output buffers (op=%u) are not matched", i); 2699 } 2700 2701 return TEST_SUCCESS; 2702 } 2703 2704 static int 2705 validate_mldts_op(struct rte_bbdev_mldts_op **ops, const uint16_t n, 2706 struct rte_bbdev_mldts_op *ref_op) 2707 { 2708 unsigned int i; 2709 int ret; 2710 struct op_data_entries *mldts_data_orig = 2711 &test_vector.entries[DATA_HARD_OUTPUT]; 2712 for (i = 0; i < n; ++i) { 2713 ret = check_mldts_status_and_ordering(ops[i], i, ref_op->status); 2714 TEST_ASSERT_SUCCESS(ret, 2715 "Checking status and ordering for MLDTS failed"); 2716 TEST_ASSERT_SUCCESS(validate_op_mldts_chain( 2717 &ops[i]->mldts.output, 2718 mldts_data_orig), 2719 "MLDTS Output buffers (op=%u) are not matched", 2720 i); 2721 } 2722 2723 return TEST_SUCCESS; 2724 } 2725 2726 static void 2727 create_reference_dec_op(struct rte_bbdev_dec_op *op) 2728 { 2729 unsigned int i; 2730 struct op_data_entries *entry; 2731 2732 op->turbo_dec = test_vector.turbo_dec; 2733 entry = &test_vector.entries[DATA_INPUT]; 2734 for (i = 0; i < entry->nb_segments; ++i) 2735 op->turbo_dec.input.length += 2736 entry->segments[i].length; 2737 } 2738 2739 static void 2740 create_reference_ldpc_dec_op(struct rte_bbdev_dec_op *op) 2741 { 2742 unsigned int i; 2743 struct op_data_entries *entry; 2744 2745 op->ldpc_dec = test_vector.ldpc_dec; 2746 entry = &test_vector.entries[DATA_INPUT]; 2747 for (i = 0; i < entry->nb_segments; ++i) 2748 op->ldpc_dec.input.length += 2749 entry->segments[i].length; 2750 if (test_vector.ldpc_dec.op_flags & 2751 RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE) { 2752 entry = &test_vector.entries[DATA_HARQ_INPUT]; 2753 for (i = 0; i < entry->nb_segments; ++i) 2754 op->ldpc_dec.harq_combined_input.length += 2755 entry->segments[i].length; 2756 } 2757 } 2758 2759 static void 2760 create_reference_fft_op(struct rte_bbdev_fft_op *op) 2761 { 2762 unsigned int i; 2763 struct op_data_entries *entry; 2764 op->fft = test_vector.fft; 2765 entry = &test_vector.entries[DATA_INPUT]; 2766 for (i = 0; i < entry->nb_segments; ++i) 2767 op->fft.base_input.length += entry->segments[i].length; 2768 entry = &test_vector.entries[DATA_HARQ_INPUT]; 2769 for (i = 0; i < entry->nb_segments; ++i) 2770 op->fft.dewindowing_input.length += entry->segments[i].length; 2771 } 2772 2773 static void 2774 create_reference_mldts_op(struct rte_bbdev_mldts_op *op) 2775 { 2776 unsigned int i; 2777 struct op_data_entries *entry; 2778 op->mldts = test_vector.mldts; 2779 entry = &test_vector.entries[DATA_INPUT]; 2780 for (i = 0; i < entry->nb_segments; ++i) 2781 op->mldts.qhy_input.length += entry->segments[i].length; 2782 entry = &test_vector.entries[DATA_HARQ_INPUT]; 2783 for (i = 0; i < entry->nb_segments; ++i) 2784 op->mldts.r_input.length += entry->segments[i].length; 2785 } 2786 2787 static void 2788 create_reference_enc_op(struct rte_bbdev_enc_op *op) 2789 { 2790 unsigned int i; 2791 struct op_data_entries *entry; 2792 2793 op->turbo_enc = test_vector.turbo_enc; 2794 entry = &test_vector.entries[DATA_INPUT]; 2795 for (i = 0; i < entry->nb_segments; ++i) 2796 op->turbo_enc.input.length += 2797 entry->segments[i].length; 2798 } 2799 2800 static void 2801 create_reference_ldpc_enc_op(struct rte_bbdev_enc_op *op) 2802 { 2803 unsigned int i; 2804 struct op_data_entries *entry; 2805 2806 op->ldpc_enc = test_vector.ldpc_enc; 2807 entry = &test_vector.entries[DATA_INPUT]; 2808 for (i = 0; i < entry->nb_segments; ++i) 2809 op->ldpc_enc.input.length += 2810 entry->segments[i].length; 2811 } 2812 2813 static uint32_t 2814 calc_dec_TB_size(struct rte_bbdev_dec_op *op) 2815 { 2816 uint8_t i; 2817 uint32_t c, r, tb_size = 0; 2818 2819 if (op->turbo_dec.code_block_mode == RTE_BBDEV_CODE_BLOCK) { 2820 tb_size = op->turbo_dec.tb_params.k_neg; 2821 } else { 2822 c = op->turbo_dec.tb_params.c; 2823 r = op->turbo_dec.tb_params.r; 2824 for (i = 0; i < c-r; i++) 2825 tb_size += (r < op->turbo_dec.tb_params.c_neg) ? 2826 op->turbo_dec.tb_params.k_neg : 2827 op->turbo_dec.tb_params.k_pos; 2828 } 2829 return tb_size; 2830 } 2831 2832 static uint32_t 2833 calc_ldpc_dec_TB_size(struct rte_bbdev_dec_op *op) 2834 { 2835 uint8_t num_cbs = 0; 2836 uint32_t tb_size = 0; 2837 uint16_t sys_cols = (op->ldpc_dec.basegraph == 1) ? 22 : 10; 2838 2839 if (op->ldpc_dec.code_block_mode == RTE_BBDEV_CODE_BLOCK) 2840 num_cbs = 1; 2841 else 2842 num_cbs = op->ldpc_dec.tb_params.c - op->ldpc_dec.tb_params.r; 2843 2844 tb_size = (sys_cols * op->ldpc_dec.z_c - op->ldpc_dec.n_filler) * num_cbs; 2845 return tb_size; 2846 } 2847 2848 static uint32_t 2849 calc_enc_TB_size(struct rte_bbdev_enc_op *op) 2850 { 2851 uint8_t i; 2852 uint32_t c, r, tb_size = 0; 2853 2854 if (op->turbo_enc.code_block_mode == RTE_BBDEV_CODE_BLOCK) { 2855 tb_size = op->turbo_enc.tb_params.k_neg; 2856 } else { 2857 c = op->turbo_enc.tb_params.c; 2858 r = op->turbo_enc.tb_params.r; 2859 for (i = 0; i < c-r; i++) 2860 tb_size += (r < op->turbo_enc.tb_params.c_neg) ? 2861 op->turbo_enc.tb_params.k_neg : 2862 op->turbo_enc.tb_params.k_pos; 2863 } 2864 return tb_size; 2865 } 2866 2867 static uint32_t 2868 calc_ldpc_enc_TB_size(struct rte_bbdev_enc_op *op) 2869 { 2870 uint8_t num_cbs = 0; 2871 uint32_t tb_size = 0; 2872 uint16_t sys_cols = (op->ldpc_enc.basegraph == 1) ? 22 : 10; 2873 2874 if (op->ldpc_enc.code_block_mode == RTE_BBDEV_CODE_BLOCK) 2875 num_cbs = 1; 2876 else 2877 num_cbs = op->ldpc_enc.tb_params.c - op->ldpc_enc.tb_params.r; 2878 2879 tb_size = (sys_cols * op->ldpc_enc.z_c - op->ldpc_enc.n_filler) * num_cbs; 2880 return tb_size; 2881 } 2882 2883 static uint32_t 2884 calc_fft_size(struct rte_bbdev_fft_op *op) 2885 { 2886 uint32_t output_size; 2887 int num_cs = 0, i; 2888 for (i = 0; i < 12; i++) 2889 if (check_bit(op->fft.cs_bitmap, 1 << i)) 2890 num_cs++; 2891 output_size = (num_cs * op->fft.output_sequence_size * 4) << op->fft.num_antennas_log2; 2892 return output_size; 2893 } 2894 2895 static uint32_t 2896 calc_mldts_size(struct rte_bbdev_mldts_op *op) 2897 { 2898 uint32_t output_size; 2899 output_size = op->mldts.num_layers * op->mldts.num_rbs * op->mldts.c_rep; 2900 return output_size; 2901 } 2902 2903 static int 2904 init_test_op_params(struct test_op_params *op_params, 2905 enum rte_bbdev_op_type op_type, const int expected_status, 2906 const int vector_mask, struct rte_mempool *ops_mp, 2907 uint16_t burst_sz, uint16_t num_to_process, uint16_t num_lcores) 2908 { 2909 int ret = 0; 2910 if (op_type == RTE_BBDEV_OP_TURBO_DEC || 2911 op_type == RTE_BBDEV_OP_LDPC_DEC) 2912 ret = rte_bbdev_dec_op_alloc_bulk(ops_mp, 2913 &op_params->ref_dec_op, 1); 2914 else if (op_type == RTE_BBDEV_OP_FFT) 2915 ret = rte_bbdev_fft_op_alloc_bulk(ops_mp, 2916 &op_params->ref_fft_op, 1); 2917 else if (op_type == RTE_BBDEV_OP_MLDTS) 2918 ret = rte_bbdev_mldts_op_alloc_bulk(ops_mp, 2919 &op_params->ref_mldts_op, 1); 2920 else 2921 ret = rte_bbdev_enc_op_alloc_bulk(ops_mp, 2922 &op_params->ref_enc_op, 1); 2923 2924 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_op_alloc_bulk() failed"); 2925 2926 op_params->mp = ops_mp; 2927 op_params->burst_sz = burst_sz; 2928 op_params->num_to_process = num_to_process; 2929 op_params->num_lcores = num_lcores; 2930 op_params->vector_mask = vector_mask; 2931 if (op_type == RTE_BBDEV_OP_TURBO_DEC || 2932 op_type == RTE_BBDEV_OP_LDPC_DEC) 2933 op_params->ref_dec_op->status = expected_status; 2934 else if (op_type == RTE_BBDEV_OP_TURBO_ENC 2935 || op_type == RTE_BBDEV_OP_LDPC_ENC) 2936 op_params->ref_enc_op->status = expected_status; 2937 else if (op_type == RTE_BBDEV_OP_FFT) 2938 op_params->ref_fft_op->status = expected_status; 2939 else if (op_type == RTE_BBDEV_OP_MLDTS) 2940 op_params->ref_mldts_op->status = expected_status; 2941 return 0; 2942 } 2943 2944 static int 2945 run_test_case_on_device(test_case_function *test_case_func, uint8_t dev_id, 2946 struct test_op_params *op_params) 2947 { 2948 int t_ret, f_ret, socket_id = SOCKET_ID_ANY; 2949 unsigned int i; 2950 struct active_device *ad; 2951 unsigned int burst_sz = get_burst_sz(); 2952 enum rte_bbdev_op_type op_type = test_vector.op_type; 2953 const struct rte_bbdev_op_cap *capabilities = NULL; 2954 2955 ad = &active_devs[dev_id]; 2956 2957 /* Check if device supports op_type */ 2958 if (!is_avail_op(ad, test_vector.op_type)) 2959 return TEST_SUCCESS; 2960 2961 struct rte_bbdev_info info; 2962 rte_bbdev_info_get(ad->dev_id, &info); 2963 socket_id = GET_SOCKET(info.socket_id); 2964 2965 f_ret = create_mempools(ad, socket_id, op_type, 2966 get_num_ops()); 2967 if (f_ret != TEST_SUCCESS) { 2968 printf("Couldn't create mempools"); 2969 goto fail; 2970 } 2971 if (op_type == RTE_BBDEV_OP_NONE) 2972 op_type = RTE_BBDEV_OP_TURBO_ENC; 2973 2974 f_ret = init_test_op_params(op_params, test_vector.op_type, 2975 test_vector.expected_status, 2976 test_vector.mask, 2977 ad->ops_mempool, 2978 burst_sz, 2979 get_num_ops(), 2980 get_num_lcores()); 2981 if (f_ret != TEST_SUCCESS) { 2982 printf("Couldn't init test op params"); 2983 goto fail; 2984 } 2985 2986 2987 /* Find capabilities */ 2988 const struct rte_bbdev_op_cap *cap = info.drv.capabilities; 2989 do { 2990 if (cap->type == test_vector.op_type) { 2991 capabilities = cap; 2992 break; 2993 } 2994 cap++; 2995 } while (cap->type != RTE_BBDEV_OP_NONE); 2996 TEST_ASSERT_NOT_NULL(capabilities, 2997 "Couldn't find capabilities"); 2998 2999 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) { 3000 create_reference_dec_op(op_params->ref_dec_op); 3001 } else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) 3002 create_reference_enc_op(op_params->ref_enc_op); 3003 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) 3004 create_reference_ldpc_enc_op(op_params->ref_enc_op); 3005 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 3006 create_reference_ldpc_dec_op(op_params->ref_dec_op); 3007 else if (test_vector.op_type == RTE_BBDEV_OP_FFT) 3008 create_reference_fft_op(op_params->ref_fft_op); 3009 else if (test_vector.op_type == RTE_BBDEV_OP_MLDTS) 3010 create_reference_mldts_op(op_params->ref_mldts_op); 3011 3012 for (i = 0; i < ad->nb_queues; ++i) { 3013 f_ret = fill_queue_buffers(op_params, 3014 ad->in_mbuf_pool, 3015 ad->hard_out_mbuf_pool, 3016 ad->soft_out_mbuf_pool, 3017 ad->harq_in_mbuf_pool, 3018 ad->harq_out_mbuf_pool, 3019 ad->queue_ids[i], 3020 capabilities, 3021 info.drv.min_alignment, 3022 socket_id); 3023 if (f_ret != TEST_SUCCESS) { 3024 printf("Couldn't init queue buffers"); 3025 goto fail; 3026 } 3027 } 3028 3029 /* Run test case function */ 3030 t_ret = test_case_func(ad, op_params); 3031 3032 /* Free active device resources and return */ 3033 free_buffers(ad, op_params); 3034 return t_ret; 3035 3036 fail: 3037 free_buffers(ad, op_params); 3038 return TEST_FAILED; 3039 } 3040 3041 /* Run given test function per active device per supported op type 3042 * per burst size. 3043 */ 3044 static int 3045 run_test_case(test_case_function *test_case_func) 3046 { 3047 int ret = 0; 3048 uint8_t dev; 3049 3050 /* Alloc op_params */ 3051 struct test_op_params *op_params = rte_zmalloc(NULL, 3052 sizeof(struct test_op_params), RTE_CACHE_LINE_SIZE); 3053 TEST_ASSERT_NOT_NULL(op_params, "Failed to alloc %zuB for op_params", 3054 RTE_ALIGN(sizeof(struct test_op_params), 3055 RTE_CACHE_LINE_SIZE)); 3056 3057 /* For each device run test case function */ 3058 for (dev = 0; dev < nb_active_devs; ++dev) 3059 ret |= run_test_case_on_device(test_case_func, dev, op_params); 3060 3061 rte_free(op_params); 3062 3063 return ret; 3064 } 3065 3066 3067 /* Push back the HARQ output from DDR to host */ 3068 static void 3069 retrieve_harq_ddr(uint16_t dev_id, uint16_t queue_id, 3070 struct rte_bbdev_dec_op **ops, 3071 const uint16_t n) 3072 { 3073 uint16_t j; 3074 int save_status, ret; 3075 uint32_t harq_offset = (uint32_t) queue_id * HARQ_INCR * MAX_OPS; 3076 struct rte_bbdev_dec_op *ops_deq[MAX_BURST]; 3077 uint32_t flags = ops[0]->ldpc_dec.op_flags; 3078 bool loopback = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK; 3079 bool mem_out = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE; 3080 bool hc_out = flags & RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE; 3081 bool h_comp = flags & RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION; 3082 for (j = 0; j < n; ++j) { 3083 if ((loopback && mem_out) || hc_out) { 3084 save_status = ops[j]->status; 3085 ops[j]->ldpc_dec.op_flags = 3086 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK + 3087 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE; 3088 if (h_comp) 3089 ops[j]->ldpc_dec.op_flags += 3090 RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION; 3091 ops[j]->ldpc_dec.harq_combined_input.offset = 3092 harq_offset; 3093 ops[j]->ldpc_dec.harq_combined_output.offset = 0; 3094 harq_offset += HARQ_INCR; 3095 if (!loopback) 3096 ops[j]->ldpc_dec.harq_combined_input.length = 3097 ops[j]->ldpc_dec.harq_combined_output.length; 3098 rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id, 3099 &ops[j], 1); 3100 ret = 0; 3101 while (ret == 0) 3102 ret = rte_bbdev_dequeue_ldpc_dec_ops( 3103 dev_id, queue_id, 3104 &ops_deq[j], 1); 3105 ops[j]->ldpc_dec.op_flags = flags; 3106 ops[j]->status = save_status; 3107 } 3108 } 3109 } 3110 3111 /* 3112 * Push back the HARQ output from HW DDR to Host 3113 * Preload HARQ memory input and adjust HARQ offset 3114 */ 3115 static void 3116 preload_harq_ddr(uint16_t dev_id, uint16_t queue_id, 3117 struct rte_bbdev_dec_op **ops, const uint16_t n, 3118 bool preload) 3119 { 3120 uint16_t j; 3121 int deq; 3122 uint32_t harq_offset = (uint32_t) queue_id * HARQ_INCR * MAX_OPS; 3123 struct rte_bbdev_op_data save_hc_in[MAX_OPS], save_hc_out[MAX_OPS]; 3124 struct rte_bbdev_dec_op *ops_deq[MAX_OPS]; 3125 uint32_t flags = ops[0]->ldpc_dec.op_flags; 3126 bool mem_in = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE; 3127 bool hc_in = flags & RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE; 3128 bool mem_out = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE; 3129 bool hc_out = flags & RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE; 3130 bool h_comp = flags & RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION; 3131 if ((mem_in || hc_in) && preload) { 3132 for (j = 0; j < n; ++j) { 3133 save_hc_in[j] = ops[j]->ldpc_dec.harq_combined_input; 3134 save_hc_out[j] = ops[j]->ldpc_dec.harq_combined_output; 3135 ops[j]->ldpc_dec.op_flags = 3136 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK + 3137 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE; 3138 if (h_comp) 3139 ops[j]->ldpc_dec.op_flags += 3140 RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION; 3141 ops[j]->ldpc_dec.harq_combined_output.offset = 3142 harq_offset; 3143 ops[j]->ldpc_dec.harq_combined_input.offset = 0; 3144 harq_offset += HARQ_INCR; 3145 } 3146 rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id, &ops[0], n); 3147 deq = 0; 3148 while (deq != n) 3149 deq += rte_bbdev_dequeue_ldpc_dec_ops( 3150 dev_id, queue_id, &ops_deq[deq], 3151 n - deq); 3152 /* Restore the operations */ 3153 for (j = 0; j < n; ++j) { 3154 ops[j]->ldpc_dec.op_flags = flags; 3155 ops[j]->ldpc_dec.harq_combined_input = save_hc_in[j]; 3156 ops[j]->ldpc_dec.harq_combined_output = save_hc_out[j]; 3157 } 3158 } 3159 harq_offset = (uint32_t) queue_id * HARQ_INCR * MAX_OPS; 3160 for (j = 0; j < n; ++j) { 3161 /* Adjust HARQ offset when we reach external DDR */ 3162 if (mem_in || hc_in) 3163 ops[j]->ldpc_dec.harq_combined_input.offset 3164 = harq_offset; 3165 if (mem_out || hc_out) 3166 ops[j]->ldpc_dec.harq_combined_output.offset 3167 = harq_offset; 3168 harq_offset += HARQ_INCR; 3169 } 3170 } 3171 3172 static void 3173 dequeue_event_callback(uint16_t dev_id, 3174 enum rte_bbdev_event_type event, void *cb_arg, 3175 void *ret_param) 3176 { 3177 int ret; 3178 uint16_t i; 3179 uint64_t total_time; 3180 uint16_t deq, burst_sz, num_ops; 3181 uint16_t queue_id = *(uint16_t *) ret_param; 3182 struct rte_bbdev_info info; 3183 double tb_len_bits; 3184 struct thread_params *tp = cb_arg; 3185 3186 /* Find matching thread params using queue_id */ 3187 for (i = 0; i < MAX_QUEUES; ++i, ++tp) 3188 if (tp->queue_id == queue_id) 3189 break; 3190 3191 if (i == MAX_QUEUES) { 3192 printf("%s: Queue_id from interrupt details was not found!\n", 3193 __func__); 3194 return; 3195 } 3196 3197 if (unlikely(event != RTE_BBDEV_EVENT_DEQUEUE)) { 3198 __atomic_store_n(&tp->processing_status, TEST_FAILED, __ATOMIC_RELAXED); 3199 printf( 3200 "Dequeue interrupt handler called for incorrect event!\n"); 3201 return; 3202 } 3203 3204 burst_sz = __atomic_load_n(&tp->burst_sz, __ATOMIC_RELAXED); 3205 num_ops = tp->op_params->num_to_process; 3206 3207 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) 3208 deq = rte_bbdev_dequeue_dec_ops(dev_id, queue_id, 3209 &tp->dec_ops[ 3210 __atomic_load_n(&tp->nb_dequeued, __ATOMIC_RELAXED)], 3211 burst_sz); 3212 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 3213 deq = rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id, 3214 &tp->dec_ops[ 3215 __atomic_load_n(&tp->nb_dequeued, __ATOMIC_RELAXED)], 3216 burst_sz); 3217 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) 3218 deq = rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id, 3219 &tp->enc_ops[ 3220 __atomic_load_n(&tp->nb_dequeued, __ATOMIC_RELAXED)], 3221 burst_sz); 3222 else if (test_vector.op_type == RTE_BBDEV_OP_FFT) 3223 deq = rte_bbdev_dequeue_fft_ops(dev_id, queue_id, 3224 &tp->fft_ops[ 3225 __atomic_load_n(&tp->nb_dequeued, __ATOMIC_RELAXED)], 3226 burst_sz); 3227 else if (test_vector.op_type == RTE_BBDEV_OP_MLDTS) 3228 deq = rte_bbdev_dequeue_mldts_ops(dev_id, queue_id, 3229 &tp->mldts_ops[ 3230 __atomic_load_n(&tp->nb_dequeued, __ATOMIC_RELAXED)], 3231 burst_sz); 3232 else /*RTE_BBDEV_OP_TURBO_ENC*/ 3233 deq = rte_bbdev_dequeue_enc_ops(dev_id, queue_id, 3234 &tp->enc_ops[ 3235 __atomic_load_n(&tp->nb_dequeued, __ATOMIC_RELAXED)], 3236 burst_sz); 3237 3238 if (deq < burst_sz) { 3239 printf( 3240 "After receiving the interrupt all operations should be dequeued. Expected: %u, got: %u\n", 3241 burst_sz, deq); 3242 __atomic_store_n(&tp->processing_status, TEST_FAILED, __ATOMIC_RELAXED); 3243 return; 3244 } 3245 3246 if (__atomic_load_n(&tp->nb_dequeued, __ATOMIC_RELAXED) + deq < num_ops) { 3247 __atomic_fetch_add(&tp->nb_dequeued, deq, __ATOMIC_RELAXED); 3248 return; 3249 } 3250 3251 total_time = rte_rdtsc_precise() - tp->start_time; 3252 3253 rte_bbdev_info_get(dev_id, &info); 3254 3255 ret = TEST_SUCCESS; 3256 3257 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) { 3258 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 3259 ret = validate_dec_op(tp->dec_ops, num_ops, ref_op); 3260 /* get the max of iter_count for all dequeued ops */ 3261 for (i = 0; i < num_ops; ++i) 3262 tp->iter_count = RTE_MAX( 3263 tp->dec_ops[i]->turbo_dec.iter_count, 3264 tp->iter_count); 3265 rte_bbdev_dec_op_free_bulk(tp->dec_ops, deq); 3266 } else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) { 3267 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op; 3268 ret = validate_enc_op(tp->enc_ops, num_ops, ref_op); 3269 rte_bbdev_enc_op_free_bulk(tp->enc_ops, deq); 3270 } else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) { 3271 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op; 3272 ret = validate_ldpc_enc_op(tp->enc_ops, num_ops, ref_op); 3273 rte_bbdev_enc_op_free_bulk(tp->enc_ops, deq); 3274 } else if (test_vector.op_type == RTE_BBDEV_OP_FFT) { 3275 struct rte_bbdev_fft_op *ref_op = tp->op_params->ref_fft_op; 3276 ret = validate_fft_op(tp->fft_ops, num_ops, ref_op); 3277 rte_bbdev_fft_op_free_bulk(tp->fft_ops, deq); 3278 } else if (test_vector.op_type == RTE_BBDEV_OP_MLDTS) { 3279 struct rte_bbdev_mldts_op *ref_op = tp->op_params->ref_mldts_op; 3280 ret = validate_mldts_op(tp->mldts_ops, num_ops, ref_op); 3281 rte_bbdev_mldts_op_free_bulk(tp->mldts_ops, deq); 3282 } else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) { 3283 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 3284 ret = validate_ldpc_dec_op(tp->dec_ops, num_ops, ref_op, 3285 tp->op_params->vector_mask); 3286 rte_bbdev_dec_op_free_bulk(tp->dec_ops, deq); 3287 } 3288 3289 if (ret) { 3290 printf("Buffers validation failed\n"); 3291 __atomic_store_n(&tp->processing_status, TEST_FAILED, __ATOMIC_RELAXED); 3292 } 3293 3294 switch (test_vector.op_type) { 3295 case RTE_BBDEV_OP_TURBO_DEC: 3296 tb_len_bits = calc_dec_TB_size(tp->op_params->ref_dec_op); 3297 break; 3298 case RTE_BBDEV_OP_TURBO_ENC: 3299 tb_len_bits = calc_enc_TB_size(tp->op_params->ref_enc_op); 3300 break; 3301 case RTE_BBDEV_OP_LDPC_DEC: 3302 tb_len_bits = calc_ldpc_dec_TB_size(tp->op_params->ref_dec_op); 3303 break; 3304 case RTE_BBDEV_OP_FFT: 3305 tb_len_bits = calc_fft_size(tp->op_params->ref_fft_op); 3306 break; 3307 case RTE_BBDEV_OP_MLDTS: 3308 tb_len_bits = calc_mldts_size(tp->op_params->ref_mldts_op); 3309 break; 3310 case RTE_BBDEV_OP_LDPC_ENC: 3311 tb_len_bits = calc_ldpc_enc_TB_size(tp->op_params->ref_enc_op); 3312 break; 3313 case RTE_BBDEV_OP_NONE: 3314 tb_len_bits = 0.0; 3315 break; 3316 default: 3317 printf("Unknown op type: %d\n", test_vector.op_type); 3318 __atomic_store_n(&tp->processing_status, TEST_FAILED, __ATOMIC_RELAXED); 3319 return; 3320 } 3321 3322 tp->ops_per_sec += ((double)num_ops) / 3323 ((double)total_time / (double)rte_get_tsc_hz()); 3324 tp->mbps += (((double)(num_ops * tb_len_bits)) / 1000000.0) / 3325 ((double)total_time / (double)rte_get_tsc_hz()); 3326 3327 __atomic_fetch_add(&tp->nb_dequeued, deq, __ATOMIC_RELAXED); 3328 } 3329 3330 static int 3331 throughput_intr_lcore_ldpc_dec(void *arg) 3332 { 3333 struct thread_params *tp = arg; 3334 unsigned int enqueued; 3335 const uint16_t queue_id = tp->queue_id; 3336 const uint16_t burst_sz = tp->op_params->burst_sz; 3337 const uint16_t num_to_process = tp->op_params->num_to_process; 3338 struct rte_bbdev_dec_op *ops[num_to_process]; 3339 struct test_buffers *bufs = NULL; 3340 struct rte_bbdev_info info; 3341 int ret, i, j; 3342 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 3343 uint16_t num_to_enq, enq; 3344 3345 bool loopback = check_bit(ref_op->ldpc_dec.op_flags, 3346 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK); 3347 bool hc_out = check_bit(ref_op->ldpc_dec.op_flags, 3348 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE); 3349 3350 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 3351 "BURST_SIZE should be <= %u", MAX_BURST); 3352 3353 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id), 3354 "Failed to enable interrupts for dev: %u, queue_id: %u", 3355 tp->dev_id, queue_id); 3356 3357 rte_bbdev_info_get(tp->dev_id, &info); 3358 3359 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim), 3360 "NUM_OPS cannot exceed %u for this device", 3361 info.drv.queue_size_lim); 3362 3363 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 3364 3365 __atomic_store_n(&tp->processing_status, 0, __ATOMIC_RELAXED); 3366 __atomic_store_n(&tp->nb_dequeued, 0, __ATOMIC_RELAXED); 3367 3368 rte_wait_until_equal_16(&tp->op_params->sync, SYNC_START, __ATOMIC_RELAXED); 3369 3370 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops, 3371 num_to_process); 3372 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", 3373 num_to_process); 3374 ref_op->ldpc_dec.iter_max = get_iter_max(); 3375 3376 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3377 copy_reference_ldpc_dec_op(ops, num_to_process, 0, bufs->inputs, 3378 bufs->hard_outputs, bufs->soft_outputs, 3379 bufs->harq_inputs, bufs->harq_outputs, ref_op); 3380 3381 /* Set counter to validate the ordering */ 3382 for (j = 0; j < num_to_process; ++j) 3383 ops[j]->opaque_data = (void *)(uintptr_t)j; 3384 3385 for (j = 0; j < TEST_REPETITIONS; ++j) { 3386 for (i = 0; i < num_to_process; ++i) { 3387 if (!loopback) 3388 mbuf_reset(ops[i]->ldpc_dec.hard_output.data); 3389 if (hc_out || loopback) 3390 mbuf_reset(ops[i]->ldpc_dec.harq_combined_output.data); 3391 if (ops[i]->ldpc_dec.soft_output.data != NULL) 3392 mbuf_reset(ops[i]->ldpc_dec.soft_output.data); 3393 } 3394 3395 tp->start_time = rte_rdtsc_precise(); 3396 for (enqueued = 0; enqueued < num_to_process;) { 3397 num_to_enq = burst_sz; 3398 3399 if (unlikely(num_to_process - enqueued < num_to_enq)) 3400 num_to_enq = num_to_process - enqueued; 3401 3402 enq = 0; 3403 do { 3404 enq += rte_bbdev_enqueue_ldpc_dec_ops( 3405 tp->dev_id, 3406 queue_id, &ops[enqueued], 3407 num_to_enq); 3408 } while (unlikely(num_to_enq != enq)); 3409 enqueued += enq; 3410 3411 /* Write to thread burst_sz current number of enqueued 3412 * descriptors. It ensures that proper number of 3413 * descriptors will be dequeued in callback 3414 * function - needed for last batch in case where 3415 * the number of operations is not a multiple of 3416 * burst size. 3417 */ 3418 __atomic_store_n(&tp->burst_sz, num_to_enq, __ATOMIC_RELAXED); 3419 3420 /* Wait until processing of previous batch is 3421 * completed 3422 */ 3423 rte_wait_until_equal_16(&tp->nb_dequeued, enqueued, __ATOMIC_RELAXED); 3424 } 3425 if (j != TEST_REPETITIONS - 1) 3426 __atomic_store_n(&tp->nb_dequeued, 0, __ATOMIC_RELAXED); 3427 } 3428 3429 return TEST_SUCCESS; 3430 } 3431 3432 static int 3433 throughput_intr_lcore_dec(void *arg) 3434 { 3435 struct thread_params *tp = arg; 3436 unsigned int enqueued; 3437 const uint16_t queue_id = tp->queue_id; 3438 const uint16_t burst_sz = tp->op_params->burst_sz; 3439 const uint16_t num_to_process = tp->op_params->num_to_process; 3440 struct rte_bbdev_dec_op *ops[num_to_process]; 3441 struct test_buffers *bufs = NULL; 3442 struct rte_bbdev_info info; 3443 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 3444 int ret, i, j; 3445 uint16_t num_to_enq, enq; 3446 3447 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 3448 "BURST_SIZE should be <= %u", MAX_BURST); 3449 3450 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id), 3451 "Failed to enable interrupts for dev: %u, queue_id: %u", 3452 tp->dev_id, queue_id); 3453 3454 rte_bbdev_info_get(tp->dev_id, &info); 3455 3456 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim), 3457 "NUM_OPS cannot exceed %u for this device", 3458 info.drv.queue_size_lim); 3459 3460 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 3461 3462 __atomic_store_n(&tp->processing_status, 0, __ATOMIC_RELAXED); 3463 __atomic_store_n(&tp->nb_dequeued, 0, __ATOMIC_RELAXED); 3464 3465 rte_wait_until_equal_16(&tp->op_params->sync, SYNC_START, __ATOMIC_RELAXED); 3466 3467 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops, 3468 num_to_process); 3469 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_to_process); 3470 ref_op->turbo_dec.iter_max = get_iter_max(); 3471 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3472 copy_reference_dec_op(ops, num_to_process, 0, bufs->inputs, 3473 bufs->hard_outputs, bufs->soft_outputs, 3474 tp->op_params->ref_dec_op); 3475 3476 /* Set counter to validate the ordering. */ 3477 for (j = 0; j < num_to_process; ++j) 3478 ops[j]->opaque_data = (void *)(uintptr_t)j; 3479 3480 for (j = 0; j < TEST_REPETITIONS; ++j) { 3481 for (i = 0; i < num_to_process; ++i) { 3482 mbuf_reset(ops[i]->turbo_dec.hard_output.data); 3483 if (ops[i]->turbo_dec.soft_output.data != NULL) 3484 mbuf_reset(ops[i]->turbo_dec.soft_output.data); 3485 } 3486 3487 tp->start_time = rte_rdtsc_precise(); 3488 for (enqueued = 0; enqueued < num_to_process;) { 3489 num_to_enq = burst_sz; 3490 3491 if (unlikely(num_to_process - enqueued < num_to_enq)) 3492 num_to_enq = num_to_process - enqueued; 3493 3494 enq = 0; 3495 do { 3496 enq += rte_bbdev_enqueue_dec_ops(tp->dev_id, 3497 queue_id, &ops[enqueued], 3498 num_to_enq); 3499 } while (unlikely(num_to_enq != enq)); 3500 enqueued += enq; 3501 3502 /* Write to thread burst_sz current number of enqueued 3503 * descriptors. It ensures that proper number of 3504 * descriptors will be dequeued in callback 3505 * function - needed for last batch in case where 3506 * the number of operations is not a multiple of 3507 * burst size. 3508 */ 3509 __atomic_store_n(&tp->burst_sz, num_to_enq, __ATOMIC_RELAXED); 3510 3511 /* Wait until processing of previous batch is 3512 * completed 3513 */ 3514 rte_wait_until_equal_16(&tp->nb_dequeued, enqueued, __ATOMIC_RELAXED); 3515 } 3516 if (j != TEST_REPETITIONS - 1) 3517 __atomic_store_n(&tp->nb_dequeued, 0, __ATOMIC_RELAXED); 3518 } 3519 3520 return TEST_SUCCESS; 3521 } 3522 3523 static int 3524 throughput_intr_lcore_enc(void *arg) 3525 { 3526 struct thread_params *tp = arg; 3527 unsigned int enqueued; 3528 const uint16_t queue_id = tp->queue_id; 3529 const uint16_t burst_sz = tp->op_params->burst_sz; 3530 const uint16_t num_to_process = tp->op_params->num_to_process; 3531 struct rte_bbdev_enc_op *ops[num_to_process]; 3532 struct test_buffers *bufs = NULL; 3533 struct rte_bbdev_info info; 3534 int ret, i, j; 3535 uint16_t num_to_enq, enq; 3536 3537 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 3538 "BURST_SIZE should be <= %u", MAX_BURST); 3539 3540 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id), 3541 "Failed to enable interrupts for dev: %u, queue_id: %u", 3542 tp->dev_id, queue_id); 3543 3544 rte_bbdev_info_get(tp->dev_id, &info); 3545 3546 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim), 3547 "NUM_OPS cannot exceed %u for this device", 3548 info.drv.queue_size_lim); 3549 3550 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 3551 3552 __atomic_store_n(&tp->processing_status, 0, __ATOMIC_RELAXED); 3553 __atomic_store_n(&tp->nb_dequeued, 0, __ATOMIC_RELAXED); 3554 3555 rte_wait_until_equal_16(&tp->op_params->sync, SYNC_START, __ATOMIC_RELAXED); 3556 3557 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops, 3558 num_to_process); 3559 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", 3560 num_to_process); 3561 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3562 copy_reference_enc_op(ops, num_to_process, 0, bufs->inputs, 3563 bufs->hard_outputs, tp->op_params->ref_enc_op); 3564 3565 /* Set counter to validate the ordering */ 3566 for (j = 0; j < num_to_process; ++j) 3567 ops[j]->opaque_data = (void *)(uintptr_t)j; 3568 3569 for (j = 0; j < TEST_REPETITIONS; ++j) { 3570 for (i = 0; i < num_to_process; ++i) 3571 mbuf_reset(ops[i]->turbo_enc.output.data); 3572 3573 tp->start_time = rte_rdtsc_precise(); 3574 for (enqueued = 0; enqueued < num_to_process;) { 3575 num_to_enq = burst_sz; 3576 3577 if (unlikely(num_to_process - enqueued < num_to_enq)) 3578 num_to_enq = num_to_process - enqueued; 3579 3580 enq = 0; 3581 do { 3582 enq += rte_bbdev_enqueue_enc_ops(tp->dev_id, 3583 queue_id, &ops[enqueued], 3584 num_to_enq); 3585 } while (unlikely(enq != num_to_enq)); 3586 enqueued += enq; 3587 3588 /* Write to thread burst_sz current number of enqueued 3589 * descriptors. It ensures that proper number of 3590 * descriptors will be dequeued in callback 3591 * function - needed for last batch in case where 3592 * the number of operations is not a multiple of 3593 * burst size. 3594 */ 3595 __atomic_store_n(&tp->burst_sz, num_to_enq, __ATOMIC_RELAXED); 3596 3597 /* Wait until processing of previous batch is 3598 * completed 3599 */ 3600 rte_wait_until_equal_16(&tp->nb_dequeued, enqueued, __ATOMIC_RELAXED); 3601 } 3602 if (j != TEST_REPETITIONS - 1) 3603 __atomic_store_n(&tp->nb_dequeued, 0, __ATOMIC_RELAXED); 3604 } 3605 3606 return TEST_SUCCESS; 3607 } 3608 3609 3610 static int 3611 throughput_intr_lcore_ldpc_enc(void *arg) 3612 { 3613 struct thread_params *tp = arg; 3614 unsigned int enqueued; 3615 const uint16_t queue_id = tp->queue_id; 3616 const uint16_t burst_sz = tp->op_params->burst_sz; 3617 const uint16_t num_to_process = tp->op_params->num_to_process; 3618 struct rte_bbdev_enc_op *ops[num_to_process]; 3619 struct test_buffers *bufs = NULL; 3620 struct rte_bbdev_info info; 3621 int ret, i, j; 3622 uint16_t num_to_enq, enq; 3623 3624 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 3625 "BURST_SIZE should be <= %u", MAX_BURST); 3626 3627 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id), 3628 "Failed to enable interrupts for dev: %u, queue_id: %u", 3629 tp->dev_id, queue_id); 3630 3631 rte_bbdev_info_get(tp->dev_id, &info); 3632 3633 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim), 3634 "NUM_OPS cannot exceed %u for this device", 3635 info.drv.queue_size_lim); 3636 3637 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 3638 3639 __atomic_store_n(&tp->processing_status, 0, __ATOMIC_RELAXED); 3640 __atomic_store_n(&tp->nb_dequeued, 0, __ATOMIC_RELAXED); 3641 3642 rte_wait_until_equal_16(&tp->op_params->sync, SYNC_START, __ATOMIC_RELAXED); 3643 3644 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops, 3645 num_to_process); 3646 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", 3647 num_to_process); 3648 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3649 copy_reference_ldpc_enc_op(ops, num_to_process, 0, 3650 bufs->inputs, bufs->hard_outputs, 3651 tp->op_params->ref_enc_op); 3652 3653 /* Set counter to validate the ordering */ 3654 for (j = 0; j < num_to_process; ++j) 3655 ops[j]->opaque_data = (void *)(uintptr_t)j; 3656 3657 for (j = 0; j < TEST_REPETITIONS; ++j) { 3658 for (i = 0; i < num_to_process; ++i) 3659 mbuf_reset(ops[i]->turbo_enc.output.data); 3660 3661 tp->start_time = rte_rdtsc_precise(); 3662 for (enqueued = 0; enqueued < num_to_process;) { 3663 num_to_enq = burst_sz; 3664 3665 if (unlikely(num_to_process - enqueued < num_to_enq)) 3666 num_to_enq = num_to_process - enqueued; 3667 3668 enq = 0; 3669 do { 3670 enq += rte_bbdev_enqueue_ldpc_enc_ops( 3671 tp->dev_id, 3672 queue_id, &ops[enqueued], 3673 num_to_enq); 3674 } while (unlikely(enq != num_to_enq)); 3675 enqueued += enq; 3676 3677 /* Write to thread burst_sz current number of enqueued 3678 * descriptors. It ensures that proper number of 3679 * descriptors will be dequeued in callback 3680 * function - needed for last batch in case where 3681 * the number of operations is not a multiple of 3682 * burst size. 3683 */ 3684 __atomic_store_n(&tp->burst_sz, num_to_enq, __ATOMIC_RELAXED); 3685 3686 /* Wait until processing of previous batch is 3687 * completed 3688 */ 3689 rte_wait_until_equal_16(&tp->nb_dequeued, enqueued, __ATOMIC_RELAXED); 3690 } 3691 if (j != TEST_REPETITIONS - 1) 3692 __atomic_store_n(&tp->nb_dequeued, 0, __ATOMIC_RELAXED); 3693 } 3694 3695 return TEST_SUCCESS; 3696 } 3697 3698 3699 static int 3700 throughput_intr_lcore_fft(void *arg) 3701 { 3702 struct thread_params *tp = arg; 3703 unsigned int enqueued; 3704 const uint16_t queue_id = tp->queue_id; 3705 const uint16_t burst_sz = tp->op_params->burst_sz; 3706 const uint16_t num_to_process = tp->op_params->num_to_process; 3707 struct rte_bbdev_fft_op *ops[num_to_process]; 3708 struct test_buffers *bufs = NULL; 3709 struct rte_bbdev_info info; 3710 int ret, i, j; 3711 uint16_t num_to_enq, enq; 3712 3713 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 3714 "BURST_SIZE should be <= %u", MAX_BURST); 3715 3716 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id), 3717 "Failed to enable interrupts for dev: %u, queue_id: %u", 3718 tp->dev_id, queue_id); 3719 3720 rte_bbdev_info_get(tp->dev_id, &info); 3721 3722 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim), 3723 "NUM_OPS cannot exceed %u for this device", 3724 info.drv.queue_size_lim); 3725 3726 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 3727 3728 __atomic_store_n(&tp->processing_status, 0, __ATOMIC_RELAXED); 3729 __atomic_store_n(&tp->nb_dequeued, 0, __ATOMIC_RELAXED); 3730 3731 rte_wait_until_equal_16(&tp->op_params->sync, SYNC_START, __ATOMIC_RELAXED); 3732 3733 ret = rte_bbdev_fft_op_alloc_bulk(tp->op_params->mp, ops, 3734 num_to_process); 3735 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", 3736 num_to_process); 3737 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3738 copy_reference_fft_op(ops, num_to_process, 0, bufs->inputs, 3739 bufs->hard_outputs, bufs->soft_outputs, bufs->harq_inputs, 3740 tp->op_params->ref_fft_op); 3741 3742 /* Set counter to validate the ordering */ 3743 for (j = 0; j < num_to_process; ++j) 3744 ops[j]->opaque_data = (void *)(uintptr_t)j; 3745 3746 for (j = 0; j < TEST_REPETITIONS; ++j) { 3747 for (i = 0; i < num_to_process; ++i) 3748 mbuf_reset(ops[i]->fft.base_output.data); 3749 3750 tp->start_time = rte_rdtsc_precise(); 3751 for (enqueued = 0; enqueued < num_to_process;) { 3752 num_to_enq = burst_sz; 3753 3754 if (unlikely(num_to_process - enqueued < num_to_enq)) 3755 num_to_enq = num_to_process - enqueued; 3756 3757 enq = 0; 3758 do { 3759 enq += rte_bbdev_enqueue_fft_ops(tp->dev_id, 3760 queue_id, &ops[enqueued], 3761 num_to_enq); 3762 } while (unlikely(enq != num_to_enq)); 3763 enqueued += enq; 3764 3765 /* Write to thread burst_sz current number of enqueued 3766 * descriptors. It ensures that proper number of 3767 * descriptors will be dequeued in callback 3768 * function - needed for last batch in case where 3769 * the number of operations is not a multiple of 3770 * burst size. 3771 */ 3772 __atomic_store_n(&tp->burst_sz, num_to_enq, __ATOMIC_RELAXED); 3773 3774 /* Wait until processing of previous batch is 3775 * completed 3776 */ 3777 rte_wait_until_equal_16(&tp->nb_dequeued, enqueued, __ATOMIC_RELAXED); 3778 } 3779 if (j != TEST_REPETITIONS - 1) 3780 __atomic_store_n(&tp->nb_dequeued, 0, __ATOMIC_RELAXED); 3781 } 3782 3783 return TEST_SUCCESS; 3784 } 3785 3786 static int 3787 throughput_intr_lcore_mldts(void *arg) 3788 { 3789 struct thread_params *tp = arg; 3790 unsigned int enqueued; 3791 const uint16_t queue_id = tp->queue_id; 3792 const uint16_t burst_sz = tp->op_params->burst_sz; 3793 const uint16_t num_to_process = tp->op_params->num_to_process; 3794 struct rte_bbdev_mldts_op *ops[num_to_process]; 3795 struct test_buffers *bufs = NULL; 3796 struct rte_bbdev_info info; 3797 int ret, i, j; 3798 uint16_t num_to_enq, enq; 3799 3800 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), "BURST_SIZE should be <= %u", MAX_BURST); 3801 3802 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id), 3803 "Failed to enable interrupts for dev: %u, queue_id: %u", 3804 tp->dev_id, queue_id); 3805 3806 rte_bbdev_info_get(tp->dev_id, &info); 3807 3808 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim), 3809 "NUM_OPS cannot exceed %u for this device", 3810 info.drv.queue_size_lim); 3811 3812 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 3813 3814 __atomic_store_n(&tp->processing_status, 0, __ATOMIC_RELAXED); 3815 __atomic_store_n(&tp->nb_dequeued, 0, __ATOMIC_RELAXED); 3816 3817 rte_wait_until_equal_16(&tp->op_params->sync, SYNC_START, __ATOMIC_RELAXED); 3818 3819 ret = rte_bbdev_mldts_op_alloc_bulk(tp->op_params->mp, ops, num_to_process); 3820 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_to_process); 3821 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3822 copy_reference_mldts_op(ops, num_to_process, 0, bufs->inputs, bufs->harq_inputs, 3823 bufs->hard_outputs, tp->op_params->ref_mldts_op); 3824 3825 /* Set counter to validate the ordering */ 3826 for (j = 0; j < num_to_process; ++j) 3827 ops[j]->opaque_data = (void *)(uintptr_t)j; 3828 3829 for (j = 0; j < TEST_REPETITIONS; ++j) { 3830 for (i = 0; i < num_to_process; ++i) 3831 mbuf_reset(ops[i]->mldts.output.data); 3832 3833 tp->start_time = rte_rdtsc_precise(); 3834 for (enqueued = 0; enqueued < num_to_process;) { 3835 num_to_enq = burst_sz; 3836 3837 if (unlikely(num_to_process - enqueued < num_to_enq)) 3838 num_to_enq = num_to_process - enqueued; 3839 3840 enq = 0; 3841 do { 3842 enq += rte_bbdev_enqueue_mldts_ops(tp->dev_id, 3843 queue_id, &ops[enqueued], num_to_enq); 3844 } while (unlikely(enq != num_to_enq)); 3845 enqueued += enq; 3846 3847 /* Write to thread burst_sz current number of enqueued 3848 * descriptors. It ensures that proper number of 3849 * descriptors will be dequeued in callback 3850 * function - needed for last batch in case where 3851 * the number of operations is not a multiple of 3852 * burst size. 3853 */ 3854 __atomic_store_n(&tp->burst_sz, num_to_enq, __ATOMIC_RELAXED); 3855 3856 /* Wait until processing of previous batch is 3857 * completed 3858 */ 3859 rte_wait_until_equal_16(&tp->nb_dequeued, enqueued, __ATOMIC_RELAXED); 3860 } 3861 if (j != TEST_REPETITIONS - 1) 3862 __atomic_store_n(&tp->nb_dequeued, 0, __ATOMIC_RELAXED); 3863 } 3864 3865 return TEST_SUCCESS; 3866 } 3867 3868 static int 3869 throughput_pmd_lcore_dec(void *arg) 3870 { 3871 struct thread_params *tp = arg; 3872 uint16_t enq, deq; 3873 uint64_t total_time = 0, start_time; 3874 const uint16_t queue_id = tp->queue_id; 3875 const uint16_t burst_sz = tp->op_params->burst_sz; 3876 const uint16_t num_ops = tp->op_params->num_to_process; 3877 struct rte_bbdev_dec_op *ops_enq[num_ops]; 3878 struct rte_bbdev_dec_op *ops_deq[num_ops]; 3879 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 3880 struct test_buffers *bufs = NULL; 3881 int i, j, ret; 3882 struct rte_bbdev_info info; 3883 uint16_t num_to_enq; 3884 bool so_enable; 3885 3886 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 3887 "BURST_SIZE should be <= %u", MAX_BURST); 3888 3889 rte_bbdev_info_get(tp->dev_id, &info); 3890 3891 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 3892 "NUM_OPS cannot exceed %u for this device", 3893 info.drv.queue_size_lim); 3894 3895 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 3896 3897 rte_wait_until_equal_16(&tp->op_params->sync, SYNC_START, __ATOMIC_RELAXED); 3898 3899 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops); 3900 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops); 3901 ref_op->turbo_dec.iter_max = get_iter_max(); 3902 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3903 copy_reference_dec_op(ops_enq, num_ops, 0, bufs->inputs, 3904 bufs->hard_outputs, bufs->soft_outputs, ref_op); 3905 3906 so_enable = check_bit(ops_enq[0]->turbo_dec.op_flags, RTE_BBDEV_TURBO_SOFT_OUTPUT); 3907 3908 /* Set counter to validate the ordering */ 3909 for (j = 0; j < num_ops; ++j) 3910 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 3911 3912 for (i = 0; i < TEST_REPETITIONS; ++i) { 3913 uint32_t time_out = 0; 3914 for (j = 0; j < num_ops; ++j) 3915 mbuf_reset(ops_enq[j]->turbo_dec.hard_output.data); 3916 if (so_enable) 3917 for (j = 0; j < num_ops; ++j) 3918 mbuf_reset(ops_enq[j]->turbo_dec.soft_output.data); 3919 3920 start_time = rte_rdtsc_precise(); 3921 3922 for (enq = 0, deq = 0; enq < num_ops;) { 3923 num_to_enq = burst_sz; 3924 3925 if (unlikely(num_ops - enq < num_to_enq)) 3926 num_to_enq = num_ops - enq; 3927 3928 enq += rte_bbdev_enqueue_dec_ops(tp->dev_id, 3929 queue_id, &ops_enq[enq], num_to_enq); 3930 3931 deq += rte_bbdev_dequeue_dec_ops(tp->dev_id, 3932 queue_id, &ops_deq[deq], enq - deq); 3933 time_out++; 3934 if (time_out >= TIME_OUT_POLL) { 3935 timeout_exit(tp->dev_id); 3936 TEST_ASSERT_SUCCESS(TEST_FAILED, "Enqueue timeout!"); 3937 } 3938 } 3939 3940 /* dequeue the remaining */ 3941 time_out = 0; 3942 while (deq < enq) { 3943 deq += rte_bbdev_dequeue_dec_ops(tp->dev_id, 3944 queue_id, &ops_deq[deq], enq - deq); 3945 time_out++; 3946 if (time_out >= TIME_OUT_POLL) { 3947 timeout_exit(tp->dev_id); 3948 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 3949 } 3950 } 3951 3952 total_time += rte_rdtsc_precise() - start_time; 3953 } 3954 3955 tp->iter_count = 0; 3956 /* get the max of iter_count for all dequeued ops */ 3957 for (i = 0; i < num_ops; ++i) { 3958 tp->iter_count = RTE_MAX(ops_enq[i]->turbo_dec.iter_count, 3959 tp->iter_count); 3960 } 3961 3962 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 3963 ret = validate_dec_op(ops_deq, num_ops, ref_op); 3964 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 3965 } 3966 3967 rte_bbdev_dec_op_free_bulk(ops_enq, num_ops); 3968 3969 double tb_len_bits = calc_dec_TB_size(ref_op); 3970 3971 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) / 3972 ((double)total_time / (double)rte_get_tsc_hz()); 3973 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) / 3974 1000000.0) / ((double)total_time / 3975 (double)rte_get_tsc_hz()); 3976 3977 return TEST_SUCCESS; 3978 } 3979 3980 static int 3981 bler_pmd_lcore_ldpc_dec(void *arg) 3982 { 3983 struct thread_params *tp = arg; 3984 uint16_t enq, deq; 3985 uint64_t total_time = 0, start_time; 3986 const uint16_t queue_id = tp->queue_id; 3987 const uint16_t burst_sz = tp->op_params->burst_sz; 3988 const uint16_t num_ops = tp->op_params->num_to_process; 3989 struct rte_bbdev_dec_op *ops_enq[num_ops]; 3990 struct rte_bbdev_dec_op *ops_deq[num_ops]; 3991 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 3992 struct test_buffers *bufs = NULL; 3993 int i, j, ret; 3994 float parity_bler = 0; 3995 struct rte_bbdev_info info; 3996 uint16_t num_to_enq; 3997 bool extDdr = check_bit(ldpc_cap_flags, 3998 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE); 3999 bool loopback = check_bit(ref_op->ldpc_dec.op_flags, 4000 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK); 4001 bool hc_out = check_bit(ref_op->ldpc_dec.op_flags, 4002 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE); 4003 4004 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 4005 "BURST_SIZE should be <= %u", MAX_BURST); 4006 TEST_ASSERT_SUCCESS((num_ops == 0), "NUM_OPS must be greater than 0"); 4007 4008 rte_bbdev_info_get(tp->dev_id, &info); 4009 4010 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 4011 "NUM_OPS cannot exceed %u for this device", 4012 info.drv.queue_size_lim); 4013 4014 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 4015 4016 rte_wait_until_equal_16(&tp->op_params->sync, SYNC_START, __ATOMIC_RELAXED); 4017 4018 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops); 4019 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops); 4020 4021 /* For BLER tests we need to enable early termination */ 4022 if (!check_bit(ref_op->ldpc_dec.op_flags, RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE)) 4023 ref_op->ldpc_dec.op_flags += RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE; 4024 4025 ref_op->ldpc_dec.iter_max = get_iter_max(); 4026 4027 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4028 copy_reference_ldpc_dec_op(ops_enq, num_ops, 0, bufs->inputs, 4029 bufs->hard_outputs, bufs->soft_outputs, 4030 bufs->harq_inputs, bufs->harq_outputs, ref_op); 4031 generate_llr_input(num_ops, bufs->inputs, ref_op); 4032 4033 /* Set counter to validate the ordering */ 4034 for (j = 0; j < num_ops; ++j) 4035 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 4036 4037 for (i = 0; i < 1; ++i) { /* Could add more iterations */ 4038 uint32_t time_out = 0; 4039 for (j = 0; j < num_ops; ++j) { 4040 if (!loopback) 4041 mbuf_reset(ops_enq[j]->ldpc_dec.hard_output.data); 4042 if (hc_out || loopback) 4043 mbuf_reset(ops_enq[j]->ldpc_dec.harq_combined_output.data); 4044 if (ops_enq[j]->ldpc_dec.soft_output.data != NULL) 4045 mbuf_reset(ops_enq[j]->ldpc_dec.soft_output.data); 4046 } 4047 if (extDdr) 4048 preload_harq_ddr(tp->dev_id, queue_id, ops_enq, 4049 num_ops, true); 4050 start_time = rte_rdtsc_precise(); 4051 4052 for (enq = 0, deq = 0; enq < num_ops;) { 4053 num_to_enq = burst_sz; 4054 4055 if (unlikely(num_ops - enq < num_to_enq)) 4056 num_to_enq = num_ops - enq; 4057 4058 enq += rte_bbdev_enqueue_ldpc_dec_ops(tp->dev_id, 4059 queue_id, &ops_enq[enq], num_to_enq); 4060 4061 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id, 4062 queue_id, &ops_deq[deq], enq - deq); 4063 time_out++; 4064 if (time_out >= TIME_OUT_POLL) { 4065 timeout_exit(tp->dev_id); 4066 TEST_ASSERT_SUCCESS(TEST_FAILED, "Enqueue timeout!"); 4067 } 4068 } 4069 4070 /* dequeue the remaining */ 4071 time_out = 0; 4072 while (deq < enq) { 4073 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id, 4074 queue_id, &ops_deq[deq], enq - deq); 4075 time_out++; 4076 if (time_out >= TIME_OUT_POLL) { 4077 timeout_exit(tp->dev_id); 4078 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 4079 } 4080 } 4081 4082 total_time += rte_rdtsc_precise() - start_time; 4083 } 4084 4085 tp->iter_count = 0; 4086 tp->iter_average = 0; 4087 /* get the max of iter_count for all dequeued ops */ 4088 for (i = 0; i < num_ops; ++i) { 4089 tp->iter_count = RTE_MAX(ops_enq[i]->ldpc_dec.iter_count, 4090 tp->iter_count); 4091 tp->iter_average += (double) ops_enq[i]->ldpc_dec.iter_count; 4092 if (ops_enq[i]->status & (1 << RTE_BBDEV_SYNDROME_ERROR)) 4093 parity_bler += 1.0; 4094 } 4095 4096 parity_bler /= num_ops; /* This one is based on SYND */ 4097 tp->iter_average /= num_ops; 4098 tp->bler = (double) validate_ldpc_bler(ops_deq, num_ops) / num_ops; 4099 4100 if (test_vector.op_type != RTE_BBDEV_OP_NONE 4101 && tp->bler == 0 4102 && parity_bler == 0 4103 && !hc_out) { 4104 ret = validate_ldpc_dec_op(ops_deq, num_ops, ref_op, 4105 tp->op_params->vector_mask); 4106 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 4107 } 4108 4109 rte_bbdev_dec_op_free_bulk(ops_enq, num_ops); 4110 4111 double tb_len_bits = calc_ldpc_dec_TB_size(ref_op); 4112 tp->ops_per_sec = ((double)num_ops * 1) / 4113 ((double)total_time / (double)rte_get_tsc_hz()); 4114 tp->mbps = (((double)(num_ops * 1 * tb_len_bits)) / 4115 1000000.0) / ((double)total_time / 4116 (double)rte_get_tsc_hz()); 4117 4118 return TEST_SUCCESS; 4119 } 4120 4121 4122 static int 4123 bler_pmd_lcore_turbo_dec(void *arg) 4124 { 4125 struct thread_params *tp = arg; 4126 uint16_t enq, deq; 4127 uint64_t total_time = 0, start_time; 4128 const uint16_t queue_id = tp->queue_id; 4129 const uint16_t burst_sz = tp->op_params->burst_sz; 4130 const uint16_t num_ops = tp->op_params->num_to_process; 4131 struct rte_bbdev_dec_op *ops_enq[num_ops]; 4132 struct rte_bbdev_dec_op *ops_deq[num_ops]; 4133 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 4134 struct test_buffers *bufs = NULL; 4135 int i, j, ret; 4136 struct rte_bbdev_info info; 4137 uint16_t num_to_enq; 4138 4139 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 4140 "BURST_SIZE should be <= %u", MAX_BURST); 4141 TEST_ASSERT_SUCCESS((num_ops == 0), "NUM_OPS must be greater than 0"); 4142 4143 rte_bbdev_info_get(tp->dev_id, &info); 4144 4145 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 4146 "NUM_OPS cannot exceed %u for this device", 4147 info.drv.queue_size_lim); 4148 4149 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 4150 4151 rte_wait_until_equal_16(&tp->op_params->sync, SYNC_START, __ATOMIC_RELAXED); 4152 4153 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops); 4154 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops); 4155 4156 /* For BLER tests we need to enable early termination */ 4157 if (!check_bit(ref_op->turbo_dec.op_flags, RTE_BBDEV_TURBO_EARLY_TERMINATION)) 4158 ref_op->turbo_dec.op_flags += RTE_BBDEV_TURBO_EARLY_TERMINATION; 4159 4160 ref_op->turbo_dec.iter_max = get_iter_max(); 4161 4162 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4163 copy_reference_dec_op(ops_enq, num_ops, 0, bufs->inputs, 4164 bufs->hard_outputs, bufs->soft_outputs, 4165 ref_op); 4166 generate_turbo_llr_input(num_ops, bufs->inputs, ref_op); 4167 4168 /* Set counter to validate the ordering */ 4169 for (j = 0; j < num_ops; ++j) 4170 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 4171 4172 for (i = 0; i < 1; ++i) { /* Could add more iterations */ 4173 uint32_t time_out = 0; 4174 for (j = 0; j < num_ops; ++j) { 4175 mbuf_reset( 4176 ops_enq[j]->turbo_dec.hard_output.data); 4177 } 4178 4179 start_time = rte_rdtsc_precise(); 4180 4181 for (enq = 0, deq = 0; enq < num_ops;) { 4182 num_to_enq = burst_sz; 4183 4184 if (unlikely(num_ops - enq < num_to_enq)) 4185 num_to_enq = num_ops - enq; 4186 4187 enq += rte_bbdev_enqueue_dec_ops(tp->dev_id, 4188 queue_id, &ops_enq[enq], num_to_enq); 4189 4190 deq += rte_bbdev_dequeue_dec_ops(tp->dev_id, 4191 queue_id, &ops_deq[deq], enq - deq); 4192 time_out++; 4193 if (time_out >= TIME_OUT_POLL) { 4194 timeout_exit(tp->dev_id); 4195 TEST_ASSERT_SUCCESS(TEST_FAILED, "Enqueue timeout!"); 4196 } 4197 } 4198 4199 /* dequeue the remaining */ 4200 time_out = 0; 4201 while (deq < enq) { 4202 deq += rte_bbdev_dequeue_dec_ops(tp->dev_id, 4203 queue_id, &ops_deq[deq], enq - deq); 4204 time_out++; 4205 if (time_out >= TIME_OUT_POLL) { 4206 timeout_exit(tp->dev_id); 4207 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 4208 } 4209 } 4210 4211 total_time += rte_rdtsc_precise() - start_time; 4212 } 4213 4214 tp->iter_count = 0; 4215 tp->iter_average = 0; 4216 /* get the max of iter_count for all dequeued ops */ 4217 for (i = 0; i < num_ops; ++i) { 4218 tp->iter_count = RTE_MAX(ops_enq[i]->turbo_dec.iter_count, 4219 tp->iter_count); 4220 tp->iter_average += (double) ops_enq[i]->turbo_dec.iter_count; 4221 } 4222 4223 tp->iter_average /= num_ops; 4224 tp->bler = (double) validate_turbo_bler(ops_deq, num_ops) / num_ops; 4225 4226 rte_bbdev_dec_op_free_bulk(ops_enq, num_ops); 4227 4228 double tb_len_bits = calc_dec_TB_size(ref_op); 4229 tp->ops_per_sec = ((double)num_ops * 1) / 4230 ((double)total_time / (double)rte_get_tsc_hz()); 4231 tp->mbps = (((double)(num_ops * 1 * tb_len_bits)) / 4232 1000000.0) / ((double)total_time / 4233 (double)rte_get_tsc_hz()); 4234 printf("TBS %.0f Time %.0f\n", tb_len_bits, 1000000.0 * 4235 ((double)total_time / (double)rte_get_tsc_hz())); 4236 4237 return TEST_SUCCESS; 4238 } 4239 4240 static int 4241 throughput_pmd_lcore_ldpc_dec(void *arg) 4242 { 4243 struct thread_params *tp = arg; 4244 uint16_t enq, deq; 4245 uint64_t total_time = 0, start_time; 4246 const uint16_t queue_id = tp->queue_id; 4247 const uint16_t burst_sz = tp->op_params->burst_sz; 4248 const uint16_t num_ops = tp->op_params->num_to_process; 4249 struct rte_bbdev_dec_op *ops_enq[num_ops]; 4250 struct rte_bbdev_dec_op *ops_deq[num_ops]; 4251 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 4252 struct test_buffers *bufs = NULL; 4253 int i, j, ret; 4254 struct rte_bbdev_info info; 4255 uint16_t num_to_enq; 4256 bool extDdr = check_bit(ldpc_cap_flags, 4257 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE); 4258 bool loopback = check_bit(ref_op->ldpc_dec.op_flags, 4259 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK); 4260 bool hc_out = check_bit(ref_op->ldpc_dec.op_flags, 4261 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE); 4262 4263 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 4264 "BURST_SIZE should be <= %u", MAX_BURST); 4265 4266 rte_bbdev_info_get(tp->dev_id, &info); 4267 4268 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 4269 "NUM_OPS cannot exceed %u for this device", 4270 info.drv.queue_size_lim); 4271 4272 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 4273 4274 rte_wait_until_equal_16(&tp->op_params->sync, SYNC_START, __ATOMIC_RELAXED); 4275 4276 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops); 4277 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops); 4278 4279 /* For throughput tests we need to disable early termination */ 4280 if (check_bit(ref_op->ldpc_dec.op_flags, RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE)) 4281 ref_op->ldpc_dec.op_flags -= RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE; 4282 4283 ref_op->ldpc_dec.iter_max = get_iter_max(); 4284 /* Since ET is disabled, the expected iter_count is iter_max */ 4285 ref_op->ldpc_dec.iter_count = ref_op->ldpc_dec.iter_max; 4286 4287 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4288 copy_reference_ldpc_dec_op(ops_enq, num_ops, 0, bufs->inputs, 4289 bufs->hard_outputs, bufs->soft_outputs, 4290 bufs->harq_inputs, bufs->harq_outputs, ref_op); 4291 4292 /* Set counter to validate the ordering */ 4293 for (j = 0; j < num_ops; ++j) 4294 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 4295 4296 for (i = 0; i < TEST_REPETITIONS; ++i) { 4297 uint32_t time_out = 0; 4298 for (j = 0; j < num_ops; ++j) { 4299 if (!loopback) 4300 mbuf_reset(ops_enq[j]->ldpc_dec.hard_output.data); 4301 if (hc_out || loopback) 4302 mbuf_reset(ops_enq[j]->ldpc_dec.harq_combined_output.data); 4303 if (ops_enq[j]->ldpc_dec.soft_output.data != NULL) 4304 mbuf_reset(ops_enq[j]->ldpc_dec.soft_output.data); 4305 } 4306 if (extDdr) 4307 preload_harq_ddr(tp->dev_id, queue_id, ops_enq, 4308 num_ops, true); 4309 start_time = rte_rdtsc_precise(); 4310 4311 for (enq = 0, deq = 0; enq < num_ops;) { 4312 num_to_enq = burst_sz; 4313 4314 if (unlikely(num_ops - enq < num_to_enq)) 4315 num_to_enq = num_ops - enq; 4316 4317 enq += rte_bbdev_enqueue_ldpc_dec_ops(tp->dev_id, 4318 queue_id, &ops_enq[enq], num_to_enq); 4319 4320 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id, 4321 queue_id, &ops_deq[deq], enq - deq); 4322 time_out++; 4323 if (time_out >= TIME_OUT_POLL) { 4324 timeout_exit(tp->dev_id); 4325 TEST_ASSERT_SUCCESS(TEST_FAILED, "Enqueue timeout!"); 4326 } 4327 } 4328 4329 /* dequeue the remaining */ 4330 time_out = 0; 4331 while (deq < enq) { 4332 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id, 4333 queue_id, &ops_deq[deq], enq - deq); 4334 time_out++; 4335 if (time_out >= TIME_OUT_POLL) { 4336 timeout_exit(tp->dev_id); 4337 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 4338 } 4339 } 4340 4341 total_time += rte_rdtsc_precise() - start_time; 4342 } 4343 4344 tp->iter_count = 0; 4345 /* get the max of iter_count for all dequeued ops */ 4346 for (i = 0; i < num_ops; ++i) { 4347 tp->iter_count = RTE_MAX(ops_enq[i]->ldpc_dec.iter_count, 4348 tp->iter_count); 4349 } 4350 if (extDdr) { 4351 /* Read loopback is not thread safe */ 4352 retrieve_harq_ddr(tp->dev_id, queue_id, ops_enq, num_ops); 4353 } 4354 4355 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 4356 ret = validate_ldpc_dec_op(ops_deq, num_ops, ref_op, 4357 tp->op_params->vector_mask); 4358 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 4359 } 4360 4361 ret = rte_bbdev_queue_stop(tp->dev_id, queue_id); 4362 if (ret != 0) 4363 printf("Failed to stop queue on dev %u q_id: %u\n", tp->dev_id, queue_id); 4364 rte_bbdev_dec_op_free_bulk(ops_enq, num_ops); 4365 4366 double tb_len_bits = calc_ldpc_dec_TB_size(ref_op); 4367 4368 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) / 4369 ((double)total_time / (double)rte_get_tsc_hz()); 4370 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) / 4371 1000000.0) / ((double)total_time / 4372 (double)rte_get_tsc_hz()); 4373 4374 return TEST_SUCCESS; 4375 } 4376 4377 static int 4378 throughput_pmd_lcore_enc(void *arg) 4379 { 4380 struct thread_params *tp = arg; 4381 uint16_t enq, deq; 4382 uint64_t total_time = 0, start_time; 4383 const uint16_t queue_id = tp->queue_id; 4384 const uint16_t burst_sz = tp->op_params->burst_sz; 4385 const uint16_t num_ops = tp->op_params->num_to_process; 4386 struct rte_bbdev_enc_op *ops_enq[num_ops]; 4387 struct rte_bbdev_enc_op *ops_deq[num_ops]; 4388 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op; 4389 struct test_buffers *bufs = NULL; 4390 int i, j, ret; 4391 struct rte_bbdev_info info; 4392 uint16_t num_to_enq; 4393 4394 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 4395 "BURST_SIZE should be <= %u", MAX_BURST); 4396 4397 rte_bbdev_info_get(tp->dev_id, &info); 4398 4399 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 4400 "NUM_OPS cannot exceed %u for this device", 4401 info.drv.queue_size_lim); 4402 4403 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 4404 4405 rte_wait_until_equal_16(&tp->op_params->sync, SYNC_START, __ATOMIC_RELAXED); 4406 4407 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops_enq, 4408 num_ops); 4409 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", 4410 num_ops); 4411 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4412 copy_reference_enc_op(ops_enq, num_ops, 0, bufs->inputs, 4413 bufs->hard_outputs, ref_op); 4414 4415 /* Set counter to validate the ordering */ 4416 for (j = 0; j < num_ops; ++j) 4417 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 4418 4419 for (i = 0; i < TEST_REPETITIONS; ++i) { 4420 uint32_t time_out = 0; 4421 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4422 for (j = 0; j < num_ops; ++j) 4423 mbuf_reset(ops_enq[j]->turbo_enc.output.data); 4424 4425 start_time = rte_rdtsc_precise(); 4426 4427 for (enq = 0, deq = 0; enq < num_ops;) { 4428 num_to_enq = burst_sz; 4429 4430 if (unlikely(num_ops - enq < num_to_enq)) 4431 num_to_enq = num_ops - enq; 4432 4433 enq += rte_bbdev_enqueue_enc_ops(tp->dev_id, 4434 queue_id, &ops_enq[enq], num_to_enq); 4435 4436 deq += rte_bbdev_dequeue_enc_ops(tp->dev_id, 4437 queue_id, &ops_deq[deq], enq - deq); 4438 time_out++; 4439 if (time_out >= TIME_OUT_POLL) { 4440 timeout_exit(tp->dev_id); 4441 TEST_ASSERT_SUCCESS(TEST_FAILED, "Enqueue timeout!"); 4442 } 4443 } 4444 4445 /* dequeue the remaining */ 4446 time_out = 0; 4447 while (deq < enq) { 4448 deq += rte_bbdev_dequeue_enc_ops(tp->dev_id, 4449 queue_id, &ops_deq[deq], enq - deq); 4450 time_out++; 4451 if (time_out >= TIME_OUT_POLL) { 4452 timeout_exit(tp->dev_id); 4453 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 4454 } 4455 } 4456 4457 total_time += rte_rdtsc_precise() - start_time; 4458 } 4459 4460 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 4461 ret = validate_enc_op(ops_deq, num_ops, ref_op); 4462 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 4463 } 4464 4465 rte_bbdev_enc_op_free_bulk(ops_enq, num_ops); 4466 4467 double tb_len_bits = calc_enc_TB_size(ref_op); 4468 4469 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) / 4470 ((double)total_time / (double)rte_get_tsc_hz()); 4471 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) 4472 / 1000000.0) / ((double)total_time / 4473 (double)rte_get_tsc_hz()); 4474 4475 return TEST_SUCCESS; 4476 } 4477 4478 static int 4479 throughput_pmd_lcore_ldpc_enc(void *arg) 4480 { 4481 struct thread_params *tp = arg; 4482 uint16_t enq, deq; 4483 uint64_t total_time = 0, start_time; 4484 const uint16_t queue_id = tp->queue_id; 4485 const uint16_t burst_sz = tp->op_params->burst_sz; 4486 const uint16_t num_ops = tp->op_params->num_to_process; 4487 struct rte_bbdev_enc_op *ops_enq[num_ops]; 4488 struct rte_bbdev_enc_op *ops_deq[num_ops]; 4489 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op; 4490 struct test_buffers *bufs = NULL; 4491 int i, j, ret; 4492 struct rte_bbdev_info info; 4493 uint16_t num_to_enq; 4494 4495 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 4496 "BURST_SIZE should be <= %u", MAX_BURST); 4497 4498 rte_bbdev_info_get(tp->dev_id, &info); 4499 4500 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 4501 "NUM_OPS cannot exceed %u for this device", 4502 info.drv.queue_size_lim); 4503 4504 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 4505 4506 rte_wait_until_equal_16(&tp->op_params->sync, SYNC_START, __ATOMIC_RELAXED); 4507 4508 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops_enq, 4509 num_ops); 4510 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", 4511 num_ops); 4512 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4513 copy_reference_ldpc_enc_op(ops_enq, num_ops, 0, bufs->inputs, 4514 bufs->hard_outputs, ref_op); 4515 4516 /* Set counter to validate the ordering */ 4517 for (j = 0; j < num_ops; ++j) 4518 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 4519 4520 for (i = 0; i < TEST_REPETITIONS; ++i) { 4521 uint32_t time_out = 0; 4522 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4523 for (j = 0; j < num_ops; ++j) 4524 mbuf_reset(ops_enq[j]->turbo_enc.output.data); 4525 4526 start_time = rte_rdtsc_precise(); 4527 4528 for (enq = 0, deq = 0; enq < num_ops;) { 4529 num_to_enq = burst_sz; 4530 4531 if (unlikely(num_ops - enq < num_to_enq)) 4532 num_to_enq = num_ops - enq; 4533 4534 enq += rte_bbdev_enqueue_ldpc_enc_ops(tp->dev_id, 4535 queue_id, &ops_enq[enq], num_to_enq); 4536 4537 deq += rte_bbdev_dequeue_ldpc_enc_ops(tp->dev_id, 4538 queue_id, &ops_deq[deq], enq - deq); 4539 time_out++; 4540 if (time_out >= TIME_OUT_POLL) { 4541 timeout_exit(tp->dev_id); 4542 TEST_ASSERT_SUCCESS(TEST_FAILED, "Enqueue timeout!"); 4543 } 4544 } 4545 4546 /* dequeue the remaining */ 4547 time_out = 0; 4548 while (deq < enq) { 4549 deq += rte_bbdev_dequeue_ldpc_enc_ops(tp->dev_id, 4550 queue_id, &ops_deq[deq], enq - deq); 4551 time_out++; 4552 if (time_out >= TIME_OUT_POLL) { 4553 timeout_exit(tp->dev_id); 4554 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 4555 } 4556 } 4557 4558 total_time += rte_rdtsc_precise() - start_time; 4559 } 4560 4561 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 4562 ret = validate_ldpc_enc_op(ops_deq, num_ops, ref_op); 4563 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 4564 } 4565 4566 rte_bbdev_enc_op_free_bulk(ops_enq, num_ops); 4567 4568 double tb_len_bits = calc_ldpc_enc_TB_size(ref_op); 4569 4570 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) / 4571 ((double)total_time / (double)rte_get_tsc_hz()); 4572 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) 4573 / 1000000.0) / ((double)total_time / 4574 (double)rte_get_tsc_hz()); 4575 4576 return TEST_SUCCESS; 4577 } 4578 4579 static int 4580 throughput_pmd_lcore_fft(void *arg) 4581 { 4582 struct thread_params *tp = arg; 4583 uint16_t enq, deq; 4584 uint64_t total_time = 0, start_time; 4585 const uint16_t queue_id = tp->queue_id; 4586 const uint16_t burst_sz = tp->op_params->burst_sz; 4587 const uint16_t num_ops = tp->op_params->num_to_process; 4588 struct rte_bbdev_fft_op *ops_enq[num_ops]; 4589 struct rte_bbdev_fft_op *ops_deq[num_ops]; 4590 struct rte_bbdev_fft_op *ref_op = tp->op_params->ref_fft_op; 4591 struct test_buffers *bufs = NULL; 4592 int i, j, ret; 4593 struct rte_bbdev_info info; 4594 uint16_t num_to_enq; 4595 4596 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 4597 "BURST_SIZE should be <= %u", MAX_BURST); 4598 4599 rte_bbdev_info_get(tp->dev_id, &info); 4600 4601 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 4602 "NUM_OPS cannot exceed %u for this device", 4603 info.drv.queue_size_lim); 4604 4605 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 4606 4607 rte_wait_until_equal_16(&tp->op_params->sync, SYNC_START, __ATOMIC_RELAXED); 4608 4609 ret = rte_bbdev_fft_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops); 4610 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops); 4611 4612 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4613 copy_reference_fft_op(ops_enq, num_ops, 0, bufs->inputs, 4614 bufs->hard_outputs, bufs->soft_outputs, bufs->harq_inputs, ref_op); 4615 4616 /* Set counter to validate the ordering */ 4617 for (j = 0; j < num_ops; ++j) 4618 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 4619 4620 for (i = 0; i < TEST_REPETITIONS; ++i) { 4621 uint32_t time_out = 0; 4622 for (j = 0; j < num_ops; ++j) 4623 mbuf_reset(ops_enq[j]->fft.base_output.data); 4624 4625 start_time = rte_rdtsc_precise(); 4626 4627 for (enq = 0, deq = 0; enq < num_ops;) { 4628 num_to_enq = burst_sz; 4629 4630 if (unlikely(num_ops - enq < num_to_enq)) 4631 num_to_enq = num_ops - enq; 4632 4633 enq += rte_bbdev_enqueue_fft_ops(tp->dev_id, 4634 queue_id, &ops_enq[enq], num_to_enq); 4635 4636 deq += rte_bbdev_dequeue_fft_ops(tp->dev_id, 4637 queue_id, &ops_deq[deq], enq - deq); 4638 time_out++; 4639 if (time_out >= TIME_OUT_POLL) { 4640 timeout_exit(tp->dev_id); 4641 TEST_ASSERT_SUCCESS(TEST_FAILED, "Enqueue timeout!"); 4642 } 4643 } 4644 4645 /* dequeue the remaining */ 4646 time_out = 0; 4647 while (deq < enq) { 4648 deq += rte_bbdev_dequeue_fft_ops(tp->dev_id, 4649 queue_id, &ops_deq[deq], enq - deq); 4650 time_out++; 4651 if (time_out >= TIME_OUT_POLL) { 4652 timeout_exit(tp->dev_id); 4653 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 4654 } 4655 } 4656 4657 total_time += rte_rdtsc_precise() - start_time; 4658 } 4659 4660 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 4661 ret = validate_fft_op(ops_deq, num_ops, ref_op); 4662 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 4663 } 4664 4665 rte_bbdev_fft_op_free_bulk(ops_enq, num_ops); 4666 4667 double tb_len_bits = calc_fft_size(ref_op); 4668 4669 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) / 4670 ((double)total_time / (double)rte_get_tsc_hz()); 4671 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) / 4672 1000000.0) / ((double)total_time / 4673 (double)rte_get_tsc_hz()); 4674 4675 return TEST_SUCCESS; 4676 } 4677 4678 static int 4679 throughput_pmd_lcore_mldts(void *arg) 4680 { 4681 struct thread_params *tp = arg; 4682 uint16_t enq, deq; 4683 uint64_t total_time = 0, start_time; 4684 const uint16_t queue_id = tp->queue_id; 4685 const uint16_t burst_sz = tp->op_params->burst_sz; 4686 const uint16_t num_ops = tp->op_params->num_to_process; 4687 struct rte_bbdev_mldts_op *ops_enq[num_ops]; 4688 struct rte_bbdev_mldts_op *ops_deq[num_ops]; 4689 struct rte_bbdev_mldts_op *ref_op = tp->op_params->ref_mldts_op; 4690 struct test_buffers *bufs = NULL; 4691 int i, j, ret; 4692 struct rte_bbdev_info info; 4693 uint16_t num_to_enq; 4694 4695 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), "BURST_SIZE should be <= %u", MAX_BURST); 4696 4697 rte_bbdev_info_get(tp->dev_id, &info); 4698 4699 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 4700 "NUM_OPS cannot exceed %u for this device", 4701 info.drv.queue_size_lim); 4702 4703 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 4704 4705 rte_wait_until_equal_16(&tp->op_params->sync, SYNC_START, __ATOMIC_RELAXED); 4706 4707 ret = rte_bbdev_mldts_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops); 4708 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops); 4709 4710 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4711 copy_reference_mldts_op(ops_enq, num_ops, 0, bufs->inputs, bufs->harq_inputs, 4712 bufs->hard_outputs, ref_op); 4713 4714 /* Set counter to validate the ordering */ 4715 for (j = 0; j < num_ops; ++j) 4716 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 4717 4718 for (i = 0; i < TEST_REPETITIONS; ++i) { 4719 uint32_t time_out = 0; 4720 for (j = 0; j < num_ops; ++j) 4721 mbuf_reset(ops_enq[j]->mldts.output.data); 4722 4723 start_time = rte_rdtsc_precise(); 4724 4725 for (enq = 0, deq = 0; enq < num_ops;) { 4726 num_to_enq = burst_sz; 4727 4728 if (unlikely(num_ops - enq < num_to_enq)) 4729 num_to_enq = num_ops - enq; 4730 4731 enq += rte_bbdev_enqueue_mldts_ops(tp->dev_id, 4732 queue_id, &ops_enq[enq], num_to_enq); 4733 4734 deq += rte_bbdev_dequeue_mldts_ops(tp->dev_id, 4735 queue_id, &ops_deq[deq], enq - deq); 4736 time_out++; 4737 if (time_out >= TIME_OUT_POLL) { 4738 timeout_exit(tp->dev_id); 4739 TEST_ASSERT_SUCCESS(TEST_FAILED, "Enqueue timeout!"); 4740 } 4741 } 4742 4743 /* dequeue the remaining */ 4744 time_out = 0; 4745 while (deq < enq) { 4746 deq += rte_bbdev_dequeue_mldts_ops(tp->dev_id, 4747 queue_id, &ops_deq[deq], enq - deq); 4748 time_out++; 4749 if (time_out >= TIME_OUT_POLL) { 4750 timeout_exit(tp->dev_id); 4751 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 4752 } 4753 } 4754 4755 total_time += rte_rdtsc_precise() - start_time; 4756 } 4757 4758 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 4759 ret = validate_mldts_op(ops_deq, num_ops, ref_op); 4760 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 4761 } 4762 4763 rte_bbdev_mldts_op_free_bulk(ops_enq, num_ops); 4764 4765 double tb_len_bits = calc_mldts_size(ref_op); 4766 4767 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) / 4768 ((double)total_time / (double)rte_get_tsc_hz()); 4769 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) / 4770 1000000.0) / ((double)total_time / 4771 (double)rte_get_tsc_hz()); 4772 4773 return TEST_SUCCESS; 4774 } 4775 4776 static void 4777 print_enc_throughput(struct thread_params *t_params, unsigned int used_cores) 4778 { 4779 unsigned int iter = 0; 4780 double total_mops = 0, total_mbps = 0; 4781 4782 for (iter = 0; iter < used_cores; iter++) { 4783 printf( 4784 "Throughput for core (%u): %.8lg Ops/s, %.8lg Mbps\n", 4785 t_params[iter].lcore_id, t_params[iter].ops_per_sec, 4786 t_params[iter].mbps); 4787 total_mops += t_params[iter].ops_per_sec; 4788 total_mbps += t_params[iter].mbps; 4789 } 4790 printf( 4791 "\nTotal throughput for %u cores: %.8lg MOPS, %.8lg Mbps\n", 4792 used_cores, total_mops, total_mbps); 4793 } 4794 4795 /* Aggregate the performance results over the number of cores used */ 4796 static void 4797 print_dec_throughput(struct thread_params *t_params, unsigned int used_cores) 4798 { 4799 unsigned int core_idx = 0; 4800 double total_mops = 0, total_mbps = 0; 4801 uint8_t iter_count = 0; 4802 4803 for (core_idx = 0; core_idx < used_cores; core_idx++) { 4804 printf( 4805 "Throughput for core (%u): %.8lg Ops/s, %.8lg Mbps @ max %u iterations\n", 4806 t_params[core_idx].lcore_id, 4807 t_params[core_idx].ops_per_sec, 4808 t_params[core_idx].mbps, 4809 t_params[core_idx].iter_count); 4810 total_mops += t_params[core_idx].ops_per_sec; 4811 total_mbps += t_params[core_idx].mbps; 4812 iter_count = RTE_MAX(iter_count, 4813 t_params[core_idx].iter_count); 4814 } 4815 printf( 4816 "\nTotal throughput for %u cores: %.8lg MOPS, %.8lg Mbps @ max %u iterations\n", 4817 used_cores, total_mops, total_mbps, iter_count); 4818 } 4819 4820 /* Aggregate the performance results over the number of cores used */ 4821 static void 4822 print_dec_bler(struct thread_params *t_params, unsigned int used_cores) 4823 { 4824 unsigned int core_idx = 0; 4825 double total_mbps = 0, total_bler = 0, total_iter = 0; 4826 double snr = get_snr(); 4827 4828 for (core_idx = 0; core_idx < used_cores; core_idx++) { 4829 printf("Core%u BLER %.1f %% - Iters %.1f - Tp %.1f Mbps %s\n", 4830 t_params[core_idx].lcore_id, 4831 t_params[core_idx].bler * 100, 4832 t_params[core_idx].iter_average, 4833 t_params[core_idx].mbps, 4834 get_vector_filename()); 4835 total_mbps += t_params[core_idx].mbps; 4836 total_bler += t_params[core_idx].bler; 4837 total_iter += t_params[core_idx].iter_average; 4838 } 4839 total_bler /= used_cores; 4840 total_iter /= used_cores; 4841 4842 printf("SNR %.2f BLER %.1f %% - Iterations %.1f %d - Tp %.3f Mbps %s\n", 4843 snr, total_bler * 100, total_iter, get_iter_max(), 4844 total_mbps, get_vector_filename()); 4845 } 4846 4847 /* 4848 * Test function that determines BLER wireless performance 4849 */ 4850 static int 4851 bler_test(struct active_device *ad, 4852 struct test_op_params *op_params) 4853 { 4854 int ret; 4855 unsigned int lcore_id, used_cores = 0; 4856 struct thread_params *t_params; 4857 struct rte_bbdev_info info; 4858 lcore_function_t *bler_function; 4859 uint16_t num_lcores; 4860 const char *op_type_str; 4861 4862 rte_bbdev_info_get(ad->dev_id, &info); 4863 4864 op_type_str = rte_bbdev_op_type_str(test_vector.op_type); 4865 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", 4866 test_vector.op_type); 4867 4868 printf("+ ------------------------------------------------------- +\n"); 4869 printf("== test: bler\ndev: %s, nb_queues: %u, burst size: %u, num ops: %u, num_lcores: %u, op type: %s, itr mode: %s, GHz: %lg\n", 4870 info.dev_name, ad->nb_queues, op_params->burst_sz, 4871 op_params->num_to_process, op_params->num_lcores, 4872 op_type_str, 4873 intr_enabled ? "Interrupt mode" : "PMD mode", 4874 (double)rte_get_tsc_hz() / 1000000000.0); 4875 4876 /* Set number of lcores */ 4877 num_lcores = (ad->nb_queues < (op_params->num_lcores)) 4878 ? ad->nb_queues 4879 : op_params->num_lcores; 4880 4881 /* Allocate memory for thread parameters structure */ 4882 t_params = rte_zmalloc(NULL, num_lcores * sizeof(struct thread_params), 4883 RTE_CACHE_LINE_SIZE); 4884 TEST_ASSERT_NOT_NULL(t_params, "Failed to alloc %zuB for t_params", 4885 RTE_ALIGN(sizeof(struct thread_params) * num_lcores, 4886 RTE_CACHE_LINE_SIZE)); 4887 4888 if ((test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) && 4889 !check_bit(test_vector.ldpc_dec.op_flags, 4890 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK) 4891 && !check_bit(test_vector.ldpc_dec.op_flags, 4892 RTE_BBDEV_LDPC_LLR_COMPRESSION)) 4893 bler_function = bler_pmd_lcore_ldpc_dec; 4894 else if ((test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) && 4895 !check_bit(test_vector.turbo_dec.op_flags, 4896 RTE_BBDEV_TURBO_SOFT_OUTPUT)) 4897 bler_function = bler_pmd_lcore_turbo_dec; 4898 else 4899 return TEST_SKIPPED; 4900 4901 __atomic_store_n(&op_params->sync, SYNC_WAIT, __ATOMIC_RELAXED); 4902 4903 /* Main core is set at first entry */ 4904 t_params[0].dev_id = ad->dev_id; 4905 t_params[0].lcore_id = rte_lcore_id(); 4906 t_params[0].op_params = op_params; 4907 t_params[0].queue_id = ad->queue_ids[used_cores++]; 4908 t_params[0].iter_count = 0; 4909 4910 RTE_LCORE_FOREACH_WORKER(lcore_id) { 4911 if (used_cores >= num_lcores) 4912 break; 4913 4914 t_params[used_cores].dev_id = ad->dev_id; 4915 t_params[used_cores].lcore_id = lcore_id; 4916 t_params[used_cores].op_params = op_params; 4917 t_params[used_cores].queue_id = ad->queue_ids[used_cores]; 4918 t_params[used_cores].iter_count = 0; 4919 4920 rte_eal_remote_launch(bler_function, 4921 &t_params[used_cores++], lcore_id); 4922 } 4923 4924 __atomic_store_n(&op_params->sync, SYNC_START, __ATOMIC_RELAXED); 4925 ret = bler_function(&t_params[0]); 4926 4927 /* Main core is always used */ 4928 for (used_cores = 1; used_cores < num_lcores; used_cores++) 4929 ret |= rte_eal_wait_lcore(t_params[used_cores].lcore_id); 4930 4931 print_dec_bler(t_params, num_lcores); 4932 4933 /* Return if test failed */ 4934 if (ret) { 4935 rte_free(t_params); 4936 return ret; 4937 } 4938 4939 /* Function to print something here*/ 4940 rte_free(t_params); 4941 return ret; 4942 } 4943 4944 /* 4945 * Test function that determines how long an enqueue + dequeue of a burst 4946 * takes on available lcores. 4947 */ 4948 static int 4949 throughput_test(struct active_device *ad, 4950 struct test_op_params *op_params) 4951 { 4952 int ret; 4953 unsigned int lcore_id, used_cores = 0; 4954 struct thread_params *t_params, *tp; 4955 struct rte_bbdev_info info; 4956 lcore_function_t *throughput_function; 4957 uint16_t num_lcores; 4958 const char *op_type_str; 4959 4960 rte_bbdev_info_get(ad->dev_id, &info); 4961 4962 op_type_str = rte_bbdev_op_type_str(test_vector.op_type); 4963 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", 4964 test_vector.op_type); 4965 4966 printf("+ ------------------------------------------------------- +\n"); 4967 printf("== test: throughput\ndev: %s, nb_queues: %u, burst size: %u, num ops: %u, num_lcores: %u, op type: %s, itr mode: %s, GHz: %lg\n", 4968 info.dev_name, ad->nb_queues, op_params->burst_sz, 4969 op_params->num_to_process, op_params->num_lcores, 4970 op_type_str, 4971 intr_enabled ? "Interrupt mode" : "PMD mode", 4972 (double)rte_get_tsc_hz() / 1000000000.0); 4973 4974 /* Set number of lcores */ 4975 num_lcores = (ad->nb_queues < (op_params->num_lcores)) 4976 ? ad->nb_queues 4977 : op_params->num_lcores; 4978 4979 /* Allocate memory for thread parameters structure */ 4980 t_params = rte_zmalloc(NULL, num_lcores * sizeof(struct thread_params), 4981 RTE_CACHE_LINE_SIZE); 4982 TEST_ASSERT_NOT_NULL(t_params, "Failed to alloc %zuB for t_params", 4983 RTE_ALIGN(sizeof(struct thread_params) * num_lcores, 4984 RTE_CACHE_LINE_SIZE)); 4985 4986 if (intr_enabled) { 4987 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) 4988 throughput_function = throughput_intr_lcore_dec; 4989 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 4990 throughput_function = throughput_intr_lcore_ldpc_dec; 4991 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) 4992 throughput_function = throughput_intr_lcore_enc; 4993 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) 4994 throughput_function = throughput_intr_lcore_ldpc_enc; 4995 else if (test_vector.op_type == RTE_BBDEV_OP_FFT) 4996 throughput_function = throughput_intr_lcore_fft; 4997 else if (test_vector.op_type == RTE_BBDEV_OP_MLDTS) 4998 throughput_function = throughput_intr_lcore_mldts; 4999 else 5000 throughput_function = throughput_intr_lcore_enc; 5001 5002 /* Dequeue interrupt callback registration */ 5003 ret = rte_bbdev_callback_register(ad->dev_id, 5004 RTE_BBDEV_EVENT_DEQUEUE, dequeue_event_callback, 5005 t_params); 5006 if (ret < 0) { 5007 rte_free(t_params); 5008 return ret; 5009 } 5010 } else { 5011 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) 5012 throughput_function = throughput_pmd_lcore_dec; 5013 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 5014 throughput_function = throughput_pmd_lcore_ldpc_dec; 5015 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) 5016 throughput_function = throughput_pmd_lcore_enc; 5017 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) 5018 throughput_function = throughput_pmd_lcore_ldpc_enc; 5019 else if (test_vector.op_type == RTE_BBDEV_OP_FFT) 5020 throughput_function = throughput_pmd_lcore_fft; 5021 else if (test_vector.op_type == RTE_BBDEV_OP_MLDTS) 5022 throughput_function = throughput_pmd_lcore_mldts; 5023 else 5024 throughput_function = throughput_pmd_lcore_enc; 5025 } 5026 5027 __atomic_store_n(&op_params->sync, SYNC_WAIT, __ATOMIC_RELAXED); 5028 5029 /* Main core is set at first entry */ 5030 t_params[0].dev_id = ad->dev_id; 5031 t_params[0].lcore_id = rte_lcore_id(); 5032 t_params[0].op_params = op_params; 5033 t_params[0].queue_id = ad->queue_ids[used_cores++]; 5034 t_params[0].iter_count = 0; 5035 5036 RTE_LCORE_FOREACH_WORKER(lcore_id) { 5037 if (used_cores >= num_lcores) 5038 break; 5039 5040 t_params[used_cores].dev_id = ad->dev_id; 5041 t_params[used_cores].lcore_id = lcore_id; 5042 t_params[used_cores].op_params = op_params; 5043 t_params[used_cores].queue_id = ad->queue_ids[used_cores]; 5044 t_params[used_cores].iter_count = 0; 5045 5046 rte_eal_remote_launch(throughput_function, 5047 &t_params[used_cores++], lcore_id); 5048 } 5049 5050 __atomic_store_n(&op_params->sync, SYNC_START, __ATOMIC_RELAXED); 5051 ret = throughput_function(&t_params[0]); 5052 5053 /* Main core is always used */ 5054 for (used_cores = 1; used_cores < num_lcores; used_cores++) 5055 ret |= rte_eal_wait_lcore(t_params[used_cores].lcore_id); 5056 5057 /* Return if test failed */ 5058 if (ret) { 5059 rte_free(t_params); 5060 return ret; 5061 } 5062 5063 /* Print throughput if interrupts are disabled and test passed */ 5064 if (!intr_enabled) { 5065 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC || 5066 test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 5067 print_dec_throughput(t_params, num_lcores); 5068 else 5069 print_enc_throughput(t_params, num_lcores); 5070 rte_free(t_params); 5071 return ret; 5072 } 5073 5074 /* In interrupt TC we need to wait for the interrupt callback to deqeue 5075 * all pending operations. Skip waiting for queues which reported an 5076 * error using processing_status variable. 5077 * Wait for main lcore operations. 5078 */ 5079 tp = &t_params[0]; 5080 while ((__atomic_load_n(&tp->nb_dequeued, __ATOMIC_RELAXED) < 5081 op_params->num_to_process) && 5082 (__atomic_load_n(&tp->processing_status, __ATOMIC_RELAXED) != 5083 TEST_FAILED)) 5084 rte_pause(); 5085 5086 tp->ops_per_sec /= TEST_REPETITIONS; 5087 tp->mbps /= TEST_REPETITIONS; 5088 ret |= (int)__atomic_load_n(&tp->processing_status, __ATOMIC_RELAXED); 5089 5090 /* Wait for worker lcores operations */ 5091 for (used_cores = 1; used_cores < num_lcores; used_cores++) { 5092 tp = &t_params[used_cores]; 5093 5094 while ((__atomic_load_n(&tp->nb_dequeued, __ATOMIC_RELAXED) < 5095 op_params->num_to_process) && 5096 (__atomic_load_n(&tp->processing_status, __ATOMIC_RELAXED) != 5097 TEST_FAILED)) 5098 rte_pause(); 5099 5100 tp->ops_per_sec /= TEST_REPETITIONS; 5101 tp->mbps /= TEST_REPETITIONS; 5102 ret |= (int)__atomic_load_n(&tp->processing_status, __ATOMIC_RELAXED); 5103 } 5104 5105 /* Print throughput if test passed */ 5106 if (!ret) { 5107 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC || 5108 test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 5109 print_dec_throughput(t_params, num_lcores); 5110 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC || 5111 test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) 5112 print_enc_throughput(t_params, num_lcores); 5113 } 5114 5115 rte_free(t_params); 5116 return ret; 5117 } 5118 5119 static int 5120 latency_test_dec(struct rte_mempool *mempool, 5121 struct test_buffers *bufs, struct rte_bbdev_dec_op *ref_op, 5122 uint16_t dev_id, uint16_t queue_id, 5123 const uint16_t num_to_process, uint16_t burst_sz, 5124 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time, bool disable_et) 5125 { 5126 int ret = TEST_SUCCESS; 5127 uint16_t i, j, dequeued; 5128 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 5129 uint64_t start_time = 0, last_time = 0; 5130 5131 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 5132 uint16_t enq = 0, deq = 0; 5133 uint32_t time_out = 0; 5134 bool first_time = true; 5135 last_time = 0; 5136 5137 if (unlikely(num_to_process - dequeued < burst_sz)) 5138 burst_sz = num_to_process - dequeued; 5139 5140 ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz); 5141 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_dec_op_alloc_bulk() failed"); 5142 5143 ref_op->turbo_dec.iter_max = get_iter_max(); 5144 /* For validation tests we want to enable early termination */ 5145 if (!disable_et && !check_bit(ref_op->turbo_dec.op_flags, 5146 RTE_BBDEV_TURBO_EARLY_TERMINATION)) 5147 ref_op->turbo_dec.op_flags |= RTE_BBDEV_TURBO_EARLY_TERMINATION; 5148 5149 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 5150 copy_reference_dec_op(ops_enq, burst_sz, dequeued, 5151 bufs->inputs, 5152 bufs->hard_outputs, 5153 bufs->soft_outputs, 5154 ref_op); 5155 5156 /* Set counter to validate the ordering */ 5157 for (j = 0; j < burst_sz; ++j) 5158 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 5159 5160 start_time = rte_rdtsc_precise(); 5161 5162 enq = rte_bbdev_enqueue_dec_ops(dev_id, queue_id, &ops_enq[enq], 5163 burst_sz); 5164 TEST_ASSERT(enq == burst_sz, 5165 "Error enqueueing burst, expected %u, got %u", 5166 burst_sz, enq); 5167 5168 /* Dequeue */ 5169 do { 5170 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id, 5171 &ops_deq[deq], burst_sz - deq); 5172 if (likely(first_time && (deq > 0))) { 5173 last_time = rte_rdtsc_precise() - start_time; 5174 first_time = false; 5175 } 5176 time_out++; 5177 if (time_out >= TIME_OUT_POLL) { 5178 timeout_exit(dev_id); 5179 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 5180 } 5181 } while (unlikely(burst_sz != deq)); 5182 5183 *max_time = RTE_MAX(*max_time, last_time); 5184 *min_time = RTE_MIN(*min_time, last_time); 5185 *total_time += last_time; 5186 5187 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 5188 ret = validate_dec_op(ops_deq, burst_sz, ref_op); 5189 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 5190 } 5191 5192 rte_bbdev_dec_op_free_bulk(ops_enq, deq); 5193 dequeued += deq; 5194 } 5195 5196 return i; 5197 } 5198 5199 /* Test case for latency/validation for LDPC Decoder */ 5200 static int 5201 latency_test_ldpc_dec(struct rte_mempool *mempool, 5202 struct test_buffers *bufs, struct rte_bbdev_dec_op *ref_op, 5203 int vector_mask, uint16_t dev_id, uint16_t queue_id, 5204 const uint16_t num_to_process, uint16_t burst_sz, 5205 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time, 5206 bool disable_et) 5207 { 5208 int ret = TEST_SUCCESS; 5209 uint16_t i, j, dequeued; 5210 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 5211 uint64_t start_time = 0, last_time = 0; 5212 bool extDdr = ldpc_cap_flags & 5213 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE; 5214 5215 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 5216 uint16_t enq = 0, deq = 0; 5217 uint32_t time_out = 0; 5218 bool first_time = true; 5219 last_time = 0; 5220 5221 if (unlikely(num_to_process - dequeued < burst_sz)) 5222 burst_sz = num_to_process - dequeued; 5223 5224 ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz); 5225 TEST_ASSERT_SUCCESS(ret, 5226 "rte_bbdev_dec_op_alloc_bulk() failed"); 5227 5228 /* For latency tests we need to disable early termination */ 5229 if (disable_et && check_bit(ref_op->ldpc_dec.op_flags, 5230 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE)) 5231 ref_op->ldpc_dec.op_flags -= RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE; 5232 5233 ref_op->ldpc_dec.iter_max = get_iter_max(); 5234 /* When ET is disabled, the expected iter_count is iter_max */ 5235 if (disable_et) 5236 ref_op->ldpc_dec.iter_count = ref_op->ldpc_dec.iter_max; 5237 5238 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 5239 copy_reference_ldpc_dec_op(ops_enq, burst_sz, dequeued, 5240 bufs->inputs, 5241 bufs->hard_outputs, 5242 bufs->soft_outputs, 5243 bufs->harq_inputs, 5244 bufs->harq_outputs, 5245 ref_op); 5246 5247 if (extDdr) 5248 preload_harq_ddr(dev_id, queue_id, ops_enq, 5249 burst_sz, true); 5250 5251 /* Set counter to validate the ordering */ 5252 for (j = 0; j < burst_sz; ++j) 5253 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 5254 5255 start_time = rte_rdtsc_precise(); 5256 5257 enq = rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id, 5258 &ops_enq[enq], burst_sz); 5259 TEST_ASSERT(enq == burst_sz, 5260 "Error enqueueing burst, expected %u, got %u", 5261 burst_sz, enq); 5262 5263 /* Dequeue */ 5264 do { 5265 deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id, 5266 &ops_deq[deq], burst_sz - deq); 5267 if (likely(first_time && (deq > 0))) { 5268 last_time = rte_rdtsc_precise() - start_time; 5269 first_time = false; 5270 } 5271 time_out++; 5272 if (time_out >= TIME_OUT_POLL) { 5273 timeout_exit(dev_id); 5274 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 5275 } 5276 } while (unlikely(burst_sz != deq)); 5277 5278 *max_time = RTE_MAX(*max_time, last_time); 5279 *min_time = RTE_MIN(*min_time, last_time); 5280 *total_time += last_time; 5281 5282 if (extDdr) 5283 retrieve_harq_ddr(dev_id, queue_id, ops_enq, burst_sz); 5284 5285 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 5286 ret = validate_ldpc_dec_op(ops_deq, burst_sz, ref_op, vector_mask); 5287 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 5288 } 5289 5290 rte_bbdev_dec_op_free_bulk(ops_enq, deq); 5291 dequeued += deq; 5292 } 5293 return i; 5294 } 5295 5296 static int 5297 latency_test_enc(struct rte_mempool *mempool, 5298 struct test_buffers *bufs, struct rte_bbdev_enc_op *ref_op, 5299 uint16_t dev_id, uint16_t queue_id, 5300 const uint16_t num_to_process, uint16_t burst_sz, 5301 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time) 5302 { 5303 int ret = TEST_SUCCESS; 5304 uint16_t i, j, dequeued; 5305 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 5306 uint64_t start_time = 0, last_time = 0; 5307 5308 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 5309 uint16_t enq = 0, deq = 0; 5310 uint32_t time_out = 0; 5311 bool first_time = true; 5312 last_time = 0; 5313 5314 if (unlikely(num_to_process - dequeued < burst_sz)) 5315 burst_sz = num_to_process - dequeued; 5316 5317 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz); 5318 TEST_ASSERT_SUCCESS(ret, 5319 "rte_bbdev_enc_op_alloc_bulk() failed"); 5320 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 5321 copy_reference_enc_op(ops_enq, burst_sz, dequeued, 5322 bufs->inputs, 5323 bufs->hard_outputs, 5324 ref_op); 5325 5326 /* Set counter to validate the ordering */ 5327 for (j = 0; j < burst_sz; ++j) 5328 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 5329 5330 start_time = rte_rdtsc_precise(); 5331 5332 enq = rte_bbdev_enqueue_enc_ops(dev_id, queue_id, &ops_enq[enq], 5333 burst_sz); 5334 TEST_ASSERT(enq == burst_sz, 5335 "Error enqueueing burst, expected %u, got %u", 5336 burst_sz, enq); 5337 5338 /* Dequeue */ 5339 do { 5340 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id, 5341 &ops_deq[deq], burst_sz - deq); 5342 if (likely(first_time && (deq > 0))) { 5343 last_time += rte_rdtsc_precise() - start_time; 5344 first_time = false; 5345 } 5346 time_out++; 5347 if (time_out >= TIME_OUT_POLL) { 5348 timeout_exit(dev_id); 5349 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 5350 } 5351 } while (unlikely(burst_sz != deq)); 5352 5353 *max_time = RTE_MAX(*max_time, last_time); 5354 *min_time = RTE_MIN(*min_time, last_time); 5355 *total_time += last_time; 5356 5357 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 5358 ret = validate_enc_op(ops_deq, burst_sz, ref_op); 5359 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 5360 } 5361 5362 rte_bbdev_enc_op_free_bulk(ops_enq, deq); 5363 dequeued += deq; 5364 } 5365 5366 return i; 5367 } 5368 5369 static int 5370 latency_test_ldpc_enc(struct rte_mempool *mempool, 5371 struct test_buffers *bufs, struct rte_bbdev_enc_op *ref_op, 5372 uint16_t dev_id, uint16_t queue_id, 5373 const uint16_t num_to_process, uint16_t burst_sz, 5374 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time) 5375 { 5376 int ret = TEST_SUCCESS; 5377 uint16_t i, j, dequeued; 5378 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 5379 uint64_t start_time = 0, last_time = 0; 5380 5381 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 5382 uint16_t enq = 0, deq = 0; 5383 uint32_t time_out = 0; 5384 bool first_time = true; 5385 last_time = 0; 5386 5387 if (unlikely(num_to_process - dequeued < burst_sz)) 5388 burst_sz = num_to_process - dequeued; 5389 5390 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz); 5391 TEST_ASSERT_SUCCESS(ret, 5392 "rte_bbdev_enc_op_alloc_bulk() failed"); 5393 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 5394 copy_reference_ldpc_enc_op(ops_enq, burst_sz, dequeued, 5395 bufs->inputs, 5396 bufs->hard_outputs, 5397 ref_op); 5398 5399 /* Set counter to validate the ordering */ 5400 for (j = 0; j < burst_sz; ++j) 5401 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 5402 5403 start_time = rte_rdtsc_precise(); 5404 5405 enq = rte_bbdev_enqueue_ldpc_enc_ops(dev_id, queue_id, 5406 &ops_enq[enq], burst_sz); 5407 TEST_ASSERT(enq == burst_sz, 5408 "Error enqueueing burst, expected %u, got %u", 5409 burst_sz, enq); 5410 5411 /* Dequeue */ 5412 do { 5413 deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id, 5414 &ops_deq[deq], burst_sz - deq); 5415 if (likely(first_time && (deq > 0))) { 5416 last_time += rte_rdtsc_precise() - start_time; 5417 first_time = false; 5418 } 5419 time_out++; 5420 if (time_out >= TIME_OUT_POLL) { 5421 timeout_exit(dev_id); 5422 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 5423 } 5424 } while (unlikely(burst_sz != deq)); 5425 5426 *max_time = RTE_MAX(*max_time, last_time); 5427 *min_time = RTE_MIN(*min_time, last_time); 5428 *total_time += last_time; 5429 5430 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 5431 ret = validate_enc_op(ops_deq, burst_sz, ref_op); 5432 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 5433 } 5434 5435 rte_bbdev_enc_op_free_bulk(ops_enq, deq); 5436 dequeued += deq; 5437 } 5438 5439 return i; 5440 } 5441 5442 5443 static int 5444 latency_test_fft(struct rte_mempool *mempool, 5445 struct test_buffers *bufs, struct rte_bbdev_fft_op *ref_op, 5446 uint16_t dev_id, uint16_t queue_id, 5447 const uint16_t num_to_process, uint16_t burst_sz, 5448 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time) 5449 { 5450 int ret = TEST_SUCCESS; 5451 uint16_t i, j, dequeued; 5452 struct rte_bbdev_fft_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 5453 uint64_t start_time = 0, last_time = 0; 5454 5455 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 5456 uint16_t enq = 0, deq = 0; 5457 uint32_t time_out = 0; 5458 bool first_time = true; 5459 last_time = 0; 5460 5461 if (unlikely(num_to_process - dequeued < burst_sz)) 5462 burst_sz = num_to_process - dequeued; 5463 5464 ret = rte_bbdev_fft_op_alloc_bulk(mempool, ops_enq, burst_sz); 5465 TEST_ASSERT_SUCCESS(ret, 5466 "rte_bbdev_fft_op_alloc_bulk() failed"); 5467 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 5468 copy_reference_fft_op(ops_enq, burst_sz, dequeued, 5469 bufs->inputs, 5470 bufs->hard_outputs, bufs->soft_outputs, bufs->harq_inputs, 5471 ref_op); 5472 5473 /* Set counter to validate the ordering */ 5474 for (j = 0; j < burst_sz; ++j) 5475 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 5476 5477 start_time = rte_rdtsc_precise(); 5478 5479 enq = rte_bbdev_enqueue_fft_ops(dev_id, queue_id, 5480 &ops_enq[enq], burst_sz); 5481 TEST_ASSERT(enq == burst_sz, 5482 "Error enqueueing burst, expected %u, got %u", 5483 burst_sz, enq); 5484 5485 /* Dequeue */ 5486 do { 5487 deq += rte_bbdev_dequeue_fft_ops(dev_id, queue_id, 5488 &ops_deq[deq], burst_sz - deq); 5489 if (likely(first_time && (deq > 0))) { 5490 last_time += rte_rdtsc_precise() - start_time; 5491 first_time = false; 5492 } 5493 time_out++; 5494 if (time_out >= TIME_OUT_POLL) { 5495 timeout_exit(dev_id); 5496 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 5497 } 5498 } while (unlikely(burst_sz != deq)); 5499 5500 *max_time = RTE_MAX(*max_time, last_time); 5501 *min_time = RTE_MIN(*min_time, last_time); 5502 *total_time += last_time; 5503 5504 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 5505 ret = validate_fft_op(ops_deq, burst_sz, ref_op); 5506 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 5507 } 5508 5509 rte_bbdev_fft_op_free_bulk(ops_enq, deq); 5510 dequeued += deq; 5511 } 5512 5513 return i; 5514 } 5515 5516 static int 5517 latency_test_mldts(struct rte_mempool *mempool, 5518 struct test_buffers *bufs, struct rte_bbdev_mldts_op *ref_op, 5519 uint16_t dev_id, uint16_t queue_id, 5520 const uint16_t num_to_process, uint16_t burst_sz, 5521 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time) 5522 { 5523 int ret = TEST_SUCCESS; 5524 uint16_t i, j, dequeued; 5525 struct rte_bbdev_mldts_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 5526 uint64_t start_time = 0, last_time = 0; 5527 5528 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 5529 uint16_t enq = 0, deq = 0; 5530 uint32_t time_out = 0; 5531 bool first_time = true; 5532 last_time = 0; 5533 5534 if (unlikely(num_to_process - dequeued < burst_sz)) 5535 burst_sz = num_to_process - dequeued; 5536 5537 ret = rte_bbdev_mldts_op_alloc_bulk(mempool, ops_enq, burst_sz); 5538 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_mldts_op_alloc_bulk() failed"); 5539 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 5540 copy_reference_mldts_op(ops_enq, burst_sz, dequeued, 5541 bufs->inputs, bufs->harq_inputs, 5542 bufs->hard_outputs, 5543 ref_op); 5544 5545 /* Set counter to validate the ordering */ 5546 for (j = 0; j < burst_sz; ++j) 5547 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 5548 5549 start_time = rte_rdtsc_precise(); 5550 5551 enq = rte_bbdev_enqueue_mldts_ops(dev_id, queue_id, &ops_enq[enq], burst_sz); 5552 TEST_ASSERT(enq == burst_sz, 5553 "Error enqueueing burst, expected %u, got %u", 5554 burst_sz, enq); 5555 5556 /* Dequeue */ 5557 do { 5558 deq += rte_bbdev_dequeue_mldts_ops(dev_id, queue_id, 5559 &ops_deq[deq], burst_sz - deq); 5560 if (likely(first_time && (deq > 0))) { 5561 last_time += rte_rdtsc_precise() - start_time; 5562 first_time = false; 5563 } 5564 time_out++; 5565 if (time_out >= TIME_OUT_POLL) { 5566 timeout_exit(dev_id); 5567 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 5568 } 5569 } while (unlikely(burst_sz != deq)); 5570 5571 *max_time = RTE_MAX(*max_time, last_time); 5572 *min_time = RTE_MIN(*min_time, last_time); 5573 *total_time += last_time; 5574 5575 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 5576 ret = validate_mldts_op(ops_deq, burst_sz, ref_op); 5577 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 5578 } 5579 5580 rte_bbdev_mldts_op_free_bulk(ops_enq, deq); 5581 dequeued += deq; 5582 } 5583 5584 return i; 5585 } 5586 5587 /* Common function for running validation and latency test cases */ 5588 static int 5589 validation_latency_test(struct active_device *ad, 5590 struct test_op_params *op_params, bool latency_flag) 5591 { 5592 int iter; 5593 uint16_t burst_sz = op_params->burst_sz; 5594 const uint16_t num_to_process = op_params->num_to_process; 5595 const enum rte_bbdev_op_type op_type = test_vector.op_type; 5596 const uint16_t queue_id = ad->queue_ids[0]; 5597 struct test_buffers *bufs = NULL; 5598 struct rte_bbdev_info info; 5599 uint64_t total_time, min_time, max_time; 5600 const char *op_type_str; 5601 5602 total_time = max_time = 0; 5603 min_time = UINT64_MAX; 5604 5605 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 5606 "BURST_SIZE should be <= %u", MAX_BURST); 5607 5608 rte_bbdev_info_get(ad->dev_id, &info); 5609 bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 5610 5611 op_type_str = rte_bbdev_op_type_str(op_type); 5612 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type); 5613 5614 printf("+ ------------------------------------------------------- +\n"); 5615 if (latency_flag) 5616 printf("== test: latency\ndev:"); 5617 else 5618 printf("== test: validation\ndev:"); 5619 printf("%s, burst size: %u, num ops: %u, op type: %s\n", 5620 info.dev_name, burst_sz, num_to_process, op_type_str); 5621 5622 if (op_type == RTE_BBDEV_OP_TURBO_DEC) 5623 iter = latency_test_dec(op_params->mp, bufs, 5624 op_params->ref_dec_op, ad->dev_id, queue_id, 5625 num_to_process, burst_sz, &total_time, 5626 &min_time, &max_time, latency_flag); 5627 else if (op_type == RTE_BBDEV_OP_LDPC_ENC) 5628 iter = latency_test_ldpc_enc(op_params->mp, bufs, 5629 op_params->ref_enc_op, ad->dev_id, queue_id, 5630 num_to_process, burst_sz, &total_time, 5631 &min_time, &max_time); 5632 else if (op_type == RTE_BBDEV_OP_LDPC_DEC) 5633 iter = latency_test_ldpc_dec(op_params->mp, bufs, 5634 op_params->ref_dec_op, op_params->vector_mask, 5635 ad->dev_id, queue_id, num_to_process, 5636 burst_sz, &total_time, &min_time, &max_time, 5637 latency_flag); 5638 else if (op_type == RTE_BBDEV_OP_FFT) 5639 iter = latency_test_fft(op_params->mp, bufs, 5640 op_params->ref_fft_op, 5641 ad->dev_id, queue_id, 5642 num_to_process, burst_sz, &total_time, 5643 &min_time, &max_time); 5644 else if (op_type == RTE_BBDEV_OP_MLDTS) 5645 iter = latency_test_mldts(op_params->mp, bufs, 5646 op_params->ref_mldts_op, 5647 ad->dev_id, queue_id, 5648 num_to_process, burst_sz, &total_time, 5649 &min_time, &max_time); 5650 else /* RTE_BBDEV_OP_TURBO_ENC */ 5651 iter = latency_test_enc(op_params->mp, bufs, 5652 op_params->ref_enc_op, 5653 ad->dev_id, queue_id, 5654 num_to_process, burst_sz, &total_time, 5655 &min_time, &max_time); 5656 5657 if (iter <= 0) 5658 return TEST_FAILED; 5659 5660 printf("Operation latency:\n" 5661 "\tavg: %lg cycles, %lg us\n" 5662 "\tmin: %lg cycles, %lg us\n" 5663 "\tmax: %lg cycles, %lg us\n", 5664 (double)total_time / (double)iter, 5665 (double)(total_time * 1000000) / (double)iter / 5666 (double)rte_get_tsc_hz(), (double)min_time, 5667 (double)(min_time * 1000000) / (double)rte_get_tsc_hz(), 5668 (double)max_time, (double)(max_time * 1000000) / 5669 (double)rte_get_tsc_hz()); 5670 5671 return TEST_SUCCESS; 5672 } 5673 5674 static int 5675 latency_test(struct active_device *ad, struct test_op_params *op_params) 5676 { 5677 return validation_latency_test(ad, op_params, true); 5678 } 5679 5680 static int 5681 validation_test(struct active_device *ad, struct test_op_params *op_params) 5682 { 5683 return validation_latency_test(ad, op_params, false); 5684 } 5685 5686 static int 5687 get_bbdev_queue_stats(uint16_t dev_id, uint16_t queue_id, 5688 struct rte_bbdev_stats *stats) 5689 { 5690 struct rte_bbdev *dev = &rte_bbdev_devices[dev_id]; 5691 struct rte_bbdev_stats *q_stats; 5692 5693 if (queue_id >= dev->data->num_queues) 5694 return -1; 5695 5696 q_stats = &dev->data->queues[queue_id].queue_stats; 5697 5698 stats->enqueued_count = q_stats->enqueued_count; 5699 stats->dequeued_count = q_stats->dequeued_count; 5700 stats->enqueue_err_count = q_stats->enqueue_err_count; 5701 stats->dequeue_err_count = q_stats->dequeue_err_count; 5702 stats->enqueue_warn_count = q_stats->enqueue_warn_count; 5703 stats->dequeue_warn_count = q_stats->dequeue_warn_count; 5704 stats->acc_offload_cycles = q_stats->acc_offload_cycles; 5705 5706 return 0; 5707 } 5708 5709 static int 5710 offload_latency_test_fft(struct rte_mempool *mempool, struct test_buffers *bufs, 5711 struct rte_bbdev_fft_op *ref_op, uint16_t dev_id, 5712 uint16_t queue_id, const uint16_t num_to_process, 5713 uint16_t burst_sz, struct test_time_stats *time_st) 5714 { 5715 int i, dequeued, ret; 5716 struct rte_bbdev_fft_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 5717 uint64_t enq_start_time, deq_start_time; 5718 uint64_t enq_sw_last_time, deq_last_time; 5719 struct rte_bbdev_stats stats; 5720 5721 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 5722 uint16_t enq = 0, deq = 0; 5723 5724 if (unlikely(num_to_process - dequeued < burst_sz)) 5725 burst_sz = num_to_process - dequeued; 5726 5727 ret = rte_bbdev_fft_op_alloc_bulk(mempool, ops_enq, burst_sz); 5728 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_fft_op_alloc_bulk() failed"); 5729 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 5730 copy_reference_fft_op(ops_enq, burst_sz, dequeued, 5731 bufs->inputs, 5732 bufs->hard_outputs, bufs->soft_outputs, bufs->harq_inputs, 5733 ref_op); 5734 5735 /* Start time meas for enqueue function offload latency */ 5736 enq_start_time = rte_rdtsc_precise(); 5737 do { 5738 enq += rte_bbdev_enqueue_fft_ops(dev_id, queue_id, 5739 &ops_enq[enq], burst_sz - enq); 5740 } while (unlikely(burst_sz != enq)); 5741 5742 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats); 5743 TEST_ASSERT_SUCCESS(ret, 5744 "Failed to get stats for queue (%u) of device (%u)", 5745 queue_id, dev_id); 5746 5747 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time - 5748 stats.acc_offload_cycles; 5749 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time, 5750 enq_sw_last_time); 5751 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time, 5752 enq_sw_last_time); 5753 time_st->enq_sw_total_time += enq_sw_last_time; 5754 5755 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time, 5756 stats.acc_offload_cycles); 5757 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time, 5758 stats.acc_offload_cycles); 5759 time_st->enq_acc_total_time += stats.acc_offload_cycles; 5760 5761 /* give time for device to process ops */ 5762 rte_delay_us(WAIT_OFFLOAD_US); 5763 5764 /* Start time meas for dequeue function offload latency */ 5765 deq_start_time = rte_rdtsc_precise(); 5766 /* Dequeue one operation */ 5767 do { 5768 deq += rte_bbdev_dequeue_fft_ops(dev_id, queue_id, 5769 &ops_deq[deq], enq); 5770 } while (unlikely(deq == 0)); 5771 5772 deq_last_time = rte_rdtsc_precise() - deq_start_time; 5773 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time, 5774 deq_last_time); 5775 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time, 5776 deq_last_time); 5777 time_st->deq_total_time += deq_last_time; 5778 5779 /* Dequeue remaining operations if needed*/ 5780 while (burst_sz != deq) 5781 deq += rte_bbdev_dequeue_fft_ops(dev_id, queue_id, 5782 &ops_deq[deq], burst_sz - deq); 5783 5784 rte_bbdev_fft_op_free_bulk(ops_enq, deq); 5785 dequeued += deq; 5786 } 5787 5788 return i; 5789 } 5790 5791 static int 5792 offload_latency_test_mldts(struct rte_mempool *mempool, struct test_buffers *bufs, 5793 struct rte_bbdev_mldts_op *ref_op, uint16_t dev_id, 5794 uint16_t queue_id, const uint16_t num_to_process, 5795 uint16_t burst_sz, struct test_time_stats *time_st) 5796 { 5797 int i, dequeued, ret; 5798 struct rte_bbdev_mldts_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 5799 uint64_t enq_start_time, deq_start_time; 5800 uint64_t enq_sw_last_time, deq_last_time; 5801 struct rte_bbdev_stats stats; 5802 5803 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 5804 uint16_t enq = 0, deq = 0; 5805 5806 if (unlikely(num_to_process - dequeued < burst_sz)) 5807 burst_sz = num_to_process - dequeued; 5808 5809 ret = rte_bbdev_mldts_op_alloc_bulk(mempool, ops_enq, burst_sz); 5810 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_mldts_op_alloc_bulk() failed"); 5811 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 5812 copy_reference_mldts_op(ops_enq, burst_sz, dequeued, 5813 bufs->inputs, bufs->harq_inputs, 5814 bufs->hard_outputs, 5815 ref_op); 5816 5817 /* Start time meas for enqueue function offload latency */ 5818 enq_start_time = rte_rdtsc_precise(); 5819 do { 5820 enq += rte_bbdev_enqueue_mldts_ops(dev_id, queue_id, 5821 &ops_enq[enq], burst_sz - enq); 5822 } while (unlikely(burst_sz != enq)); 5823 5824 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats); 5825 TEST_ASSERT_SUCCESS(ret, 5826 "Failed to get stats for queue (%u) of device (%u)", 5827 queue_id, dev_id); 5828 5829 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time - 5830 stats.acc_offload_cycles; 5831 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time, 5832 enq_sw_last_time); 5833 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time, 5834 enq_sw_last_time); 5835 time_st->enq_sw_total_time += enq_sw_last_time; 5836 5837 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time, 5838 stats.acc_offload_cycles); 5839 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time, 5840 stats.acc_offload_cycles); 5841 time_st->enq_acc_total_time += stats.acc_offload_cycles; 5842 5843 /* give time for device to process ops */ 5844 rte_delay_us(WAIT_OFFLOAD_US); 5845 5846 /* Start time meas for dequeue function offload latency */ 5847 deq_start_time = rte_rdtsc_precise(); 5848 /* Dequeue one operation */ 5849 do { 5850 deq += rte_bbdev_dequeue_mldts_ops(dev_id, queue_id, &ops_deq[deq], enq); 5851 } while (unlikely(deq == 0)); 5852 5853 deq_last_time = rte_rdtsc_precise() - deq_start_time; 5854 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time, deq_last_time); 5855 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time, deq_last_time); 5856 time_st->deq_total_time += deq_last_time; 5857 5858 /* Dequeue remaining operations if needed*/ 5859 while (burst_sz != deq) 5860 deq += rte_bbdev_dequeue_mldts_ops(dev_id, queue_id, 5861 &ops_deq[deq], burst_sz - deq); 5862 5863 rte_bbdev_mldts_op_free_bulk(ops_enq, deq); 5864 dequeued += deq; 5865 } 5866 5867 return i; 5868 } 5869 5870 static int 5871 offload_latency_test_dec(struct rte_mempool *mempool, struct test_buffers *bufs, 5872 struct rte_bbdev_dec_op *ref_op, uint16_t dev_id, 5873 uint16_t queue_id, const uint16_t num_to_process, 5874 uint16_t burst_sz, struct test_time_stats *time_st) 5875 { 5876 int i, dequeued, ret; 5877 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 5878 uint64_t enq_start_time, deq_start_time; 5879 uint64_t enq_sw_last_time, deq_last_time; 5880 struct rte_bbdev_stats stats; 5881 5882 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 5883 uint16_t enq = 0, deq = 0; 5884 5885 if (unlikely(num_to_process - dequeued < burst_sz)) 5886 burst_sz = num_to_process - dequeued; 5887 5888 ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz); 5889 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_dec_op_alloc_bulk() failed"); 5890 ref_op->turbo_dec.iter_max = get_iter_max(); 5891 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 5892 copy_reference_dec_op(ops_enq, burst_sz, dequeued, 5893 bufs->inputs, 5894 bufs->hard_outputs, 5895 bufs->soft_outputs, 5896 ref_op); 5897 5898 /* Start time meas for enqueue function offload latency */ 5899 enq_start_time = rte_rdtsc_precise(); 5900 do { 5901 enq += rte_bbdev_enqueue_dec_ops(dev_id, queue_id, 5902 &ops_enq[enq], burst_sz - enq); 5903 } while (unlikely(burst_sz != enq)); 5904 5905 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats); 5906 TEST_ASSERT_SUCCESS(ret, 5907 "Failed to get stats for queue (%u) of device (%u)", 5908 queue_id, dev_id); 5909 5910 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time - 5911 stats.acc_offload_cycles; 5912 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time, 5913 enq_sw_last_time); 5914 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time, 5915 enq_sw_last_time); 5916 time_st->enq_sw_total_time += enq_sw_last_time; 5917 5918 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time, 5919 stats.acc_offload_cycles); 5920 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time, 5921 stats.acc_offload_cycles); 5922 time_st->enq_acc_total_time += stats.acc_offload_cycles; 5923 5924 /* give time for device to process ops */ 5925 rte_delay_us(WAIT_OFFLOAD_US); 5926 5927 /* Start time meas for dequeue function offload latency */ 5928 deq_start_time = rte_rdtsc_precise(); 5929 /* Dequeue one operation */ 5930 do { 5931 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id, 5932 &ops_deq[deq], enq); 5933 } while (unlikely(deq == 0)); 5934 5935 deq_last_time = rte_rdtsc_precise() - deq_start_time; 5936 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time, 5937 deq_last_time); 5938 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time, 5939 deq_last_time); 5940 time_st->deq_total_time += deq_last_time; 5941 5942 /* Dequeue remaining operations if needed*/ 5943 while (burst_sz != deq) 5944 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id, 5945 &ops_deq[deq], burst_sz - deq); 5946 5947 rte_bbdev_dec_op_free_bulk(ops_enq, deq); 5948 dequeued += deq; 5949 } 5950 5951 return i; 5952 } 5953 5954 static int 5955 offload_latency_test_ldpc_dec(struct rte_mempool *mempool, 5956 struct test_buffers *bufs, 5957 struct rte_bbdev_dec_op *ref_op, uint16_t dev_id, 5958 uint16_t queue_id, const uint16_t num_to_process, 5959 uint16_t burst_sz, struct test_time_stats *time_st) 5960 { 5961 int i, dequeued, ret; 5962 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 5963 uint64_t enq_start_time, deq_start_time; 5964 uint64_t enq_sw_last_time, deq_last_time; 5965 struct rte_bbdev_stats stats; 5966 bool extDdr = ldpc_cap_flags & 5967 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE; 5968 5969 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 5970 uint16_t enq = 0, deq = 0; 5971 5972 if (unlikely(num_to_process - dequeued < burst_sz)) 5973 burst_sz = num_to_process - dequeued; 5974 5975 ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz); 5976 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_dec_op_alloc_bulk() failed"); 5977 ref_op->ldpc_dec.iter_max = get_iter_max(); 5978 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 5979 copy_reference_ldpc_dec_op(ops_enq, burst_sz, dequeued, 5980 bufs->inputs, 5981 bufs->hard_outputs, 5982 bufs->soft_outputs, 5983 bufs->harq_inputs, 5984 bufs->harq_outputs, 5985 ref_op); 5986 5987 if (extDdr) 5988 preload_harq_ddr(dev_id, queue_id, ops_enq, 5989 burst_sz, true); 5990 5991 /* Start time meas for enqueue function offload latency */ 5992 enq_start_time = rte_rdtsc_precise(); 5993 do { 5994 enq += rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id, 5995 &ops_enq[enq], burst_sz - enq); 5996 } while (unlikely(burst_sz != enq)); 5997 5998 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time; 5999 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats); 6000 TEST_ASSERT_SUCCESS(ret, 6001 "Failed to get stats for queue (%u) of device (%u)", 6002 queue_id, dev_id); 6003 6004 enq_sw_last_time -= stats.acc_offload_cycles; 6005 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time, 6006 enq_sw_last_time); 6007 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time, 6008 enq_sw_last_time); 6009 time_st->enq_sw_total_time += enq_sw_last_time; 6010 6011 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time, 6012 stats.acc_offload_cycles); 6013 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time, 6014 stats.acc_offload_cycles); 6015 time_st->enq_acc_total_time += stats.acc_offload_cycles; 6016 6017 /* give time for device to process ops */ 6018 rte_delay_us(WAIT_OFFLOAD_US); 6019 6020 /* Start time meas for dequeue function offload latency */ 6021 deq_start_time = rte_rdtsc_precise(); 6022 /* Dequeue one operation */ 6023 do { 6024 deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id, 6025 &ops_deq[deq], enq); 6026 } while (unlikely(deq == 0)); 6027 6028 deq_last_time = rte_rdtsc_precise() - deq_start_time; 6029 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time, 6030 deq_last_time); 6031 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time, 6032 deq_last_time); 6033 time_st->deq_total_time += deq_last_time; 6034 6035 /* Dequeue remaining operations if needed*/ 6036 while (burst_sz != deq) 6037 deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id, 6038 &ops_deq[deq], burst_sz - deq); 6039 6040 if (extDdr) { 6041 /* Read loopback is not thread safe */ 6042 retrieve_harq_ddr(dev_id, queue_id, ops_enq, burst_sz); 6043 } 6044 6045 rte_bbdev_dec_op_free_bulk(ops_enq, deq); 6046 dequeued += deq; 6047 } 6048 6049 return i; 6050 } 6051 6052 static int 6053 offload_latency_test_enc(struct rte_mempool *mempool, struct test_buffers *bufs, 6054 struct rte_bbdev_enc_op *ref_op, uint16_t dev_id, 6055 uint16_t queue_id, const uint16_t num_to_process, 6056 uint16_t burst_sz, struct test_time_stats *time_st) 6057 { 6058 int i, dequeued, ret; 6059 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 6060 uint64_t enq_start_time, deq_start_time; 6061 uint64_t enq_sw_last_time, deq_last_time; 6062 struct rte_bbdev_stats stats; 6063 6064 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 6065 uint16_t enq = 0, deq = 0; 6066 6067 if (unlikely(num_to_process - dequeued < burst_sz)) 6068 burst_sz = num_to_process - dequeued; 6069 6070 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz); 6071 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_enc_op_alloc_bulk() failed"); 6072 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 6073 copy_reference_enc_op(ops_enq, burst_sz, dequeued, 6074 bufs->inputs, 6075 bufs->hard_outputs, 6076 ref_op); 6077 6078 /* Start time meas for enqueue function offload latency */ 6079 enq_start_time = rte_rdtsc_precise(); 6080 do { 6081 enq += rte_bbdev_enqueue_enc_ops(dev_id, queue_id, 6082 &ops_enq[enq], burst_sz - enq); 6083 } while (unlikely(burst_sz != enq)); 6084 6085 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time; 6086 6087 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats); 6088 TEST_ASSERT_SUCCESS(ret, 6089 "Failed to get stats for queue (%u) of device (%u)", 6090 queue_id, dev_id); 6091 enq_sw_last_time -= stats.acc_offload_cycles; 6092 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time, 6093 enq_sw_last_time); 6094 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time, 6095 enq_sw_last_time); 6096 time_st->enq_sw_total_time += enq_sw_last_time; 6097 6098 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time, 6099 stats.acc_offload_cycles); 6100 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time, 6101 stats.acc_offload_cycles); 6102 time_st->enq_acc_total_time += stats.acc_offload_cycles; 6103 6104 /* give time for device to process ops */ 6105 rte_delay_us(WAIT_OFFLOAD_US); 6106 6107 /* Start time meas for dequeue function offload latency */ 6108 deq_start_time = rte_rdtsc_precise(); 6109 /* Dequeue one operation */ 6110 do { 6111 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id, 6112 &ops_deq[deq], enq); 6113 } while (unlikely(deq == 0)); 6114 6115 deq_last_time = rte_rdtsc_precise() - deq_start_time; 6116 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time, 6117 deq_last_time); 6118 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time, 6119 deq_last_time); 6120 time_st->deq_total_time += deq_last_time; 6121 6122 while (burst_sz != deq) 6123 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id, 6124 &ops_deq[deq], burst_sz - deq); 6125 6126 rte_bbdev_enc_op_free_bulk(ops_enq, deq); 6127 dequeued += deq; 6128 } 6129 6130 return i; 6131 } 6132 6133 static int 6134 offload_latency_test_ldpc_enc(struct rte_mempool *mempool, 6135 struct test_buffers *bufs, 6136 struct rte_bbdev_enc_op *ref_op, uint16_t dev_id, 6137 uint16_t queue_id, const uint16_t num_to_process, 6138 uint16_t burst_sz, struct test_time_stats *time_st) 6139 { 6140 int i, dequeued, ret; 6141 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 6142 uint64_t enq_start_time, deq_start_time; 6143 uint64_t enq_sw_last_time, deq_last_time; 6144 struct rte_bbdev_stats stats; 6145 6146 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 6147 uint16_t enq = 0, deq = 0; 6148 6149 if (unlikely(num_to_process - dequeued < burst_sz)) 6150 burst_sz = num_to_process - dequeued; 6151 6152 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz); 6153 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_enc_op_alloc_bulk() failed"); 6154 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 6155 copy_reference_ldpc_enc_op(ops_enq, burst_sz, dequeued, 6156 bufs->inputs, 6157 bufs->hard_outputs, 6158 ref_op); 6159 6160 /* Start time meas for enqueue function offload latency */ 6161 enq_start_time = rte_rdtsc_precise(); 6162 do { 6163 enq += rte_bbdev_enqueue_ldpc_enc_ops(dev_id, queue_id, 6164 &ops_enq[enq], burst_sz - enq); 6165 } while (unlikely(burst_sz != enq)); 6166 6167 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time; 6168 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats); 6169 TEST_ASSERT_SUCCESS(ret, 6170 "Failed to get stats for queue (%u) of device (%u)", 6171 queue_id, dev_id); 6172 6173 enq_sw_last_time -= stats.acc_offload_cycles; 6174 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time, 6175 enq_sw_last_time); 6176 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time, 6177 enq_sw_last_time); 6178 time_st->enq_sw_total_time += enq_sw_last_time; 6179 6180 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time, 6181 stats.acc_offload_cycles); 6182 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time, 6183 stats.acc_offload_cycles); 6184 time_st->enq_acc_total_time += stats.acc_offload_cycles; 6185 6186 /* give time for device to process ops */ 6187 rte_delay_us(WAIT_OFFLOAD_US); 6188 6189 /* Start time meas for dequeue function offload latency */ 6190 deq_start_time = rte_rdtsc_precise(); 6191 /* Dequeue one operation */ 6192 do { 6193 deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id, 6194 &ops_deq[deq], enq); 6195 } while (unlikely(deq == 0)); 6196 6197 deq_last_time = rte_rdtsc_precise() - deq_start_time; 6198 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time, 6199 deq_last_time); 6200 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time, 6201 deq_last_time); 6202 time_st->deq_total_time += deq_last_time; 6203 6204 while (burst_sz != deq) 6205 deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id, 6206 &ops_deq[deq], burst_sz - deq); 6207 6208 rte_bbdev_enc_op_free_bulk(ops_enq, deq); 6209 dequeued += deq; 6210 } 6211 6212 return i; 6213 } 6214 6215 static int 6216 offload_cost_test(struct active_device *ad, 6217 struct test_op_params *op_params) 6218 { 6219 int iter, ret; 6220 uint16_t burst_sz = op_params->burst_sz; 6221 const uint16_t num_to_process = op_params->num_to_process; 6222 const enum rte_bbdev_op_type op_type = test_vector.op_type; 6223 const uint16_t queue_id = ad->queue_ids[0]; 6224 struct test_buffers *bufs = NULL; 6225 struct rte_bbdev_info info; 6226 const char *op_type_str; 6227 struct test_time_stats time_st; 6228 6229 memset(&time_st, 0, sizeof(struct test_time_stats)); 6230 time_st.enq_sw_min_time = UINT64_MAX; 6231 time_st.enq_acc_min_time = UINT64_MAX; 6232 time_st.deq_min_time = UINT64_MAX; 6233 6234 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 6235 "BURST_SIZE should be <= %u", MAX_BURST); 6236 6237 rte_bbdev_info_get(ad->dev_id, &info); 6238 bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 6239 6240 op_type_str = rte_bbdev_op_type_str(op_type); 6241 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type); 6242 6243 printf("+ ------------------------------------------------------- +\n"); 6244 printf("== test: offload latency test\ndev: %s, burst size: %u, num ops: %u, op type: %s\n", 6245 info.dev_name, burst_sz, num_to_process, op_type_str); 6246 6247 if (op_type == RTE_BBDEV_OP_TURBO_DEC) 6248 iter = offload_latency_test_dec(op_params->mp, bufs, 6249 op_params->ref_dec_op, ad->dev_id, queue_id, 6250 num_to_process, burst_sz, &time_st); 6251 else if (op_type == RTE_BBDEV_OP_TURBO_ENC) 6252 iter = offload_latency_test_enc(op_params->mp, bufs, 6253 op_params->ref_enc_op, ad->dev_id, queue_id, 6254 num_to_process, burst_sz, &time_st); 6255 else if (op_type == RTE_BBDEV_OP_LDPC_ENC) 6256 iter = offload_latency_test_ldpc_enc(op_params->mp, bufs, 6257 op_params->ref_enc_op, ad->dev_id, queue_id, 6258 num_to_process, burst_sz, &time_st); 6259 else if (op_type == RTE_BBDEV_OP_LDPC_DEC) 6260 iter = offload_latency_test_ldpc_dec(op_params->mp, bufs, 6261 op_params->ref_dec_op, ad->dev_id, queue_id, 6262 num_to_process, burst_sz, &time_st); 6263 else if (op_type == RTE_BBDEV_OP_FFT) 6264 iter = offload_latency_test_fft(op_params->mp, bufs, 6265 op_params->ref_fft_op, ad->dev_id, queue_id, 6266 num_to_process, burst_sz, &time_st); 6267 else if (op_type == RTE_BBDEV_OP_MLDTS) 6268 iter = offload_latency_test_mldts(op_params->mp, bufs, 6269 op_params->ref_mldts_op, ad->dev_id, queue_id, 6270 num_to_process, burst_sz, &time_st); 6271 else 6272 iter = offload_latency_test_enc(op_params->mp, bufs, 6273 op_params->ref_enc_op, ad->dev_id, queue_id, 6274 num_to_process, burst_sz, &time_st); 6275 6276 if (iter <= 0) 6277 return TEST_FAILED; 6278 6279 printf("Enqueue driver offload cost latency:\n" 6280 "\tavg: %lg cycles, %lg us\n" 6281 "\tmin: %lg cycles, %lg us\n" 6282 "\tmax: %lg cycles, %lg us\n" 6283 "Enqueue accelerator offload cost latency:\n" 6284 "\tavg: %lg cycles, %lg us\n" 6285 "\tmin: %lg cycles, %lg us\n" 6286 "\tmax: %lg cycles, %lg us\n", 6287 (double)time_st.enq_sw_total_time / (double)iter, 6288 (double)(time_st.enq_sw_total_time * 1000000) / 6289 (double)iter / (double)rte_get_tsc_hz(), 6290 (double)time_st.enq_sw_min_time, 6291 (double)(time_st.enq_sw_min_time * 1000000) / 6292 rte_get_tsc_hz(), (double)time_st.enq_sw_max_time, 6293 (double)(time_st.enq_sw_max_time * 1000000) / 6294 rte_get_tsc_hz(), (double)time_st.enq_acc_total_time / 6295 (double)iter, 6296 (double)(time_st.enq_acc_total_time * 1000000) / 6297 (double)iter / (double)rte_get_tsc_hz(), 6298 (double)time_st.enq_acc_min_time, 6299 (double)(time_st.enq_acc_min_time * 1000000) / 6300 rte_get_tsc_hz(), (double)time_st.enq_acc_max_time, 6301 (double)(time_st.enq_acc_max_time * 1000000) / 6302 rte_get_tsc_hz()); 6303 6304 printf("Dequeue offload cost latency - one op:\n" 6305 "\tavg: %lg cycles, %lg us\n" 6306 "\tmin: %lg cycles, %lg us\n" 6307 "\tmax: %lg cycles, %lg us\n", 6308 (double)time_st.deq_total_time / (double)iter, 6309 (double)(time_st.deq_total_time * 1000000) / 6310 (double)iter / (double)rte_get_tsc_hz(), 6311 (double)time_st.deq_min_time, 6312 (double)(time_st.deq_min_time * 1000000) / 6313 rte_get_tsc_hz(), (double)time_st.deq_max_time, 6314 (double)(time_st.deq_max_time * 1000000) / 6315 rte_get_tsc_hz()); 6316 6317 struct rte_bbdev_stats stats = {0}; 6318 ret = get_bbdev_queue_stats(ad->dev_id, queue_id, &stats); 6319 TEST_ASSERT_SUCCESS(ret, 6320 "Failed to get stats for queue (%u) of device (%u)", 6321 queue_id, ad->dev_id); 6322 if (stats.enqueue_warn_count > 0) 6323 printf("Warning reported on the queue : %10"PRIu64"\n", 6324 stats.enqueue_warn_count); 6325 if (op_type != RTE_BBDEV_OP_LDPC_DEC) { 6326 TEST_ASSERT_SUCCESS(stats.enqueued_count != num_to_process, 6327 "Mismatch in enqueue count %10"PRIu64" %d", 6328 stats.enqueued_count, num_to_process); 6329 TEST_ASSERT_SUCCESS(stats.dequeued_count != num_to_process, 6330 "Mismatch in dequeue count %10"PRIu64" %d", 6331 stats.dequeued_count, num_to_process); 6332 } 6333 TEST_ASSERT_SUCCESS(stats.enqueue_err_count != 0, 6334 "Enqueue count Error %10"PRIu64"", 6335 stats.enqueue_err_count); 6336 TEST_ASSERT_SUCCESS(stats.dequeue_err_count != 0, 6337 "Dequeue count Error (%10"PRIu64"", 6338 stats.dequeue_err_count); 6339 6340 return TEST_SUCCESS; 6341 } 6342 6343 static int 6344 offload_latency_empty_q_test_dec(uint16_t dev_id, uint16_t queue_id, 6345 const uint16_t num_to_process, uint16_t burst_sz, 6346 uint64_t *deq_total_time, uint64_t *deq_min_time, 6347 uint64_t *deq_max_time, const enum rte_bbdev_op_type op_type) 6348 { 6349 int i, deq_total; 6350 struct rte_bbdev_dec_op *ops[MAX_BURST]; 6351 uint64_t deq_start_time, deq_last_time; 6352 6353 /* Test deq offload latency from an empty queue */ 6354 6355 for (i = 0, deq_total = 0; deq_total < num_to_process; 6356 ++i, deq_total += burst_sz) { 6357 deq_start_time = rte_rdtsc_precise(); 6358 6359 if (unlikely(num_to_process - deq_total < burst_sz)) 6360 burst_sz = num_to_process - deq_total; 6361 if (op_type == RTE_BBDEV_OP_LDPC_DEC) 6362 rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id, ops, 6363 burst_sz); 6364 else 6365 rte_bbdev_dequeue_dec_ops(dev_id, queue_id, ops, 6366 burst_sz); 6367 6368 deq_last_time = rte_rdtsc_precise() - deq_start_time; 6369 *deq_max_time = RTE_MAX(*deq_max_time, deq_last_time); 6370 *deq_min_time = RTE_MIN(*deq_min_time, deq_last_time); 6371 *deq_total_time += deq_last_time; 6372 } 6373 6374 return i; 6375 } 6376 6377 static int 6378 offload_latency_empty_q_test_enc(uint16_t dev_id, uint16_t queue_id, 6379 const uint16_t num_to_process, uint16_t burst_sz, 6380 uint64_t *deq_total_time, uint64_t *deq_min_time, 6381 uint64_t *deq_max_time, const enum rte_bbdev_op_type op_type) 6382 { 6383 int i, deq_total; 6384 struct rte_bbdev_enc_op *ops[MAX_BURST]; 6385 uint64_t deq_start_time, deq_last_time; 6386 6387 /* Test deq offload latency from an empty queue */ 6388 for (i = 0, deq_total = 0; deq_total < num_to_process; 6389 ++i, deq_total += burst_sz) { 6390 deq_start_time = rte_rdtsc_precise(); 6391 6392 if (unlikely(num_to_process - deq_total < burst_sz)) 6393 burst_sz = num_to_process - deq_total; 6394 if (op_type == RTE_BBDEV_OP_LDPC_ENC) 6395 rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id, ops, 6396 burst_sz); 6397 else 6398 rte_bbdev_dequeue_enc_ops(dev_id, queue_id, ops, 6399 burst_sz); 6400 6401 deq_last_time = rte_rdtsc_precise() - deq_start_time; 6402 *deq_max_time = RTE_MAX(*deq_max_time, deq_last_time); 6403 *deq_min_time = RTE_MIN(*deq_min_time, deq_last_time); 6404 *deq_total_time += deq_last_time; 6405 } 6406 6407 return i; 6408 } 6409 6410 static int 6411 offload_latency_empty_q_test(struct active_device *ad, 6412 struct test_op_params *op_params) 6413 { 6414 int iter; 6415 uint64_t deq_total_time, deq_min_time, deq_max_time; 6416 uint16_t burst_sz = op_params->burst_sz; 6417 const uint16_t num_to_process = op_params->num_to_process; 6418 const enum rte_bbdev_op_type op_type = test_vector.op_type; 6419 const uint16_t queue_id = ad->queue_ids[0]; 6420 struct rte_bbdev_info info; 6421 const char *op_type_str; 6422 6423 deq_total_time = deq_max_time = 0; 6424 deq_min_time = UINT64_MAX; 6425 6426 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 6427 "BURST_SIZE should be <= %u", MAX_BURST); 6428 6429 rte_bbdev_info_get(ad->dev_id, &info); 6430 6431 op_type_str = rte_bbdev_op_type_str(op_type); 6432 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type); 6433 6434 printf("+ ------------------------------------------------------- +\n"); 6435 printf("== test: offload latency empty dequeue\ndev: %s, burst size: %u, num ops: %u, op type: %s\n", 6436 info.dev_name, burst_sz, num_to_process, op_type_str); 6437 6438 if (op_type == RTE_BBDEV_OP_TURBO_DEC || 6439 op_type == RTE_BBDEV_OP_LDPC_DEC) 6440 iter = offload_latency_empty_q_test_dec(ad->dev_id, queue_id, 6441 num_to_process, burst_sz, &deq_total_time, 6442 &deq_min_time, &deq_max_time, op_type); 6443 else 6444 iter = offload_latency_empty_q_test_enc(ad->dev_id, queue_id, 6445 num_to_process, burst_sz, &deq_total_time, 6446 &deq_min_time, &deq_max_time, op_type); 6447 6448 if (iter <= 0) 6449 return TEST_FAILED; 6450 6451 printf("Empty dequeue offload:\n" 6452 "\tavg: %lg cycles, %lg us\n" 6453 "\tmin: %lg cycles, %lg us\n" 6454 "\tmax: %lg cycles, %lg us\n", 6455 (double)deq_total_time / (double)iter, 6456 (double)(deq_total_time * 1000000) / (double)iter / 6457 (double)rte_get_tsc_hz(), (double)deq_min_time, 6458 (double)(deq_min_time * 1000000) / rte_get_tsc_hz(), 6459 (double)deq_max_time, (double)(deq_max_time * 1000000) / 6460 rte_get_tsc_hz()); 6461 6462 return TEST_SUCCESS; 6463 } 6464 6465 static int 6466 bler_tc(void) 6467 { 6468 return run_test_case(bler_test); 6469 } 6470 6471 static int 6472 throughput_tc(void) 6473 { 6474 return run_test_case(throughput_test); 6475 } 6476 6477 static int 6478 offload_cost_tc(void) 6479 { 6480 return run_test_case(offload_cost_test); 6481 } 6482 6483 static int 6484 offload_latency_empty_q_tc(void) 6485 { 6486 return run_test_case(offload_latency_empty_q_test); 6487 } 6488 6489 static int 6490 latency_tc(void) 6491 { 6492 return run_test_case(latency_test); 6493 } 6494 6495 static int 6496 validation_tc(void) 6497 { 6498 return run_test_case(validation_test); 6499 } 6500 6501 static int 6502 interrupt_tc(void) 6503 { 6504 return run_test_case(throughput_test); 6505 } 6506 6507 static struct unit_test_suite bbdev_bler_testsuite = { 6508 .suite_name = "BBdev BLER Tests", 6509 .setup = testsuite_setup, 6510 .teardown = testsuite_teardown, 6511 .unit_test_cases = { 6512 TEST_CASE_ST(ut_setup, ut_teardown, bler_tc), 6513 TEST_CASES_END() /**< NULL terminate unit test array */ 6514 } 6515 }; 6516 6517 static struct unit_test_suite bbdev_throughput_testsuite = { 6518 .suite_name = "BBdev Throughput Tests", 6519 .setup = testsuite_setup, 6520 .teardown = testsuite_teardown, 6521 .unit_test_cases = { 6522 TEST_CASE_ST(ut_setup, ut_teardown, throughput_tc), 6523 TEST_CASES_END() /**< NULL terminate unit test array */ 6524 } 6525 }; 6526 6527 static struct unit_test_suite bbdev_validation_testsuite = { 6528 .suite_name = "BBdev Validation Tests", 6529 .setup = testsuite_setup, 6530 .teardown = testsuite_teardown, 6531 .unit_test_cases = { 6532 TEST_CASE_ST(ut_setup, ut_teardown, validation_tc), 6533 TEST_CASES_END() /**< NULL terminate unit test array */ 6534 } 6535 }; 6536 6537 static struct unit_test_suite bbdev_latency_testsuite = { 6538 .suite_name = "BBdev Latency Tests", 6539 .setup = testsuite_setup, 6540 .teardown = testsuite_teardown, 6541 .unit_test_cases = { 6542 TEST_CASE_ST(ut_setup, ut_teardown, latency_tc), 6543 TEST_CASES_END() /**< NULL terminate unit test array */ 6544 } 6545 }; 6546 6547 static struct unit_test_suite bbdev_offload_cost_testsuite = { 6548 .suite_name = "BBdev Offload Cost Tests", 6549 .setup = testsuite_setup, 6550 .teardown = testsuite_teardown, 6551 .unit_test_cases = { 6552 TEST_CASE_ST(ut_setup, ut_teardown, offload_cost_tc), 6553 TEST_CASE_ST(ut_setup, ut_teardown, offload_latency_empty_q_tc), 6554 TEST_CASES_END() /**< NULL terminate unit test array */ 6555 } 6556 }; 6557 6558 static struct unit_test_suite bbdev_interrupt_testsuite = { 6559 .suite_name = "BBdev Interrupt Tests", 6560 .setup = interrupt_testsuite_setup, 6561 .teardown = testsuite_teardown, 6562 .unit_test_cases = { 6563 TEST_CASE_ST(ut_setup, ut_teardown, interrupt_tc), 6564 TEST_CASES_END() /**< NULL terminate unit test array */ 6565 } 6566 }; 6567 6568 REGISTER_TEST_COMMAND(bler, bbdev_bler_testsuite); 6569 REGISTER_TEST_COMMAND(throughput, bbdev_throughput_testsuite); 6570 REGISTER_TEST_COMMAND(validation, bbdev_validation_testsuite); 6571 REGISTER_TEST_COMMAND(latency, bbdev_latency_testsuite); 6572 REGISTER_TEST_COMMAND(offload, bbdev_offload_cost_testsuite); 6573 REGISTER_TEST_COMMAND(interrupt, bbdev_interrupt_testsuite); 6574