1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017 Intel Corporation 3 */ 4 5 #include <stdio.h> 6 #include <stdlib.h> 7 #include <inttypes.h> 8 #include <math.h> 9 10 #include <rte_eal.h> 11 #include <rte_common.h> 12 #include <rte_dev.h> 13 #include <rte_launch.h> 14 #include <rte_bbdev.h> 15 #include <rte_cycles.h> 16 #include <rte_lcore.h> 17 #include <rte_malloc.h> 18 #include <rte_random.h> 19 #include <rte_hexdump.h> 20 #include <rte_interrupts.h> 21 22 #include "main.h" 23 #include "test_bbdev_vector.h" 24 25 #define GET_SOCKET(socket_id) (((socket_id) == SOCKET_ID_ANY) ? 0 : (socket_id)) 26 27 #define MAX_QUEUES RTE_MAX_LCORE 28 #define TEST_REPETITIONS 100 29 #define TIME_OUT_POLL 1e9 30 #define WAIT_OFFLOAD_US 1000 31 32 #ifdef RTE_BASEBAND_FPGA_LTE_FEC 33 #include <fpga_lte_fec.h> 34 #define FPGA_LTE_PF_DRIVER_NAME ("intel_fpga_lte_fec_pf") 35 #define FPGA_LTE_VF_DRIVER_NAME ("intel_fpga_lte_fec_vf") 36 #define VF_UL_4G_QUEUE_VALUE 4 37 #define VF_DL_4G_QUEUE_VALUE 4 38 #define UL_4G_BANDWIDTH 3 39 #define DL_4G_BANDWIDTH 3 40 #define UL_4G_LOAD_BALANCE 128 41 #define DL_4G_LOAD_BALANCE 128 42 #define FLR_4G_TIMEOUT 610 43 #endif 44 45 #ifdef RTE_BASEBAND_FPGA_5GNR_FEC 46 #include <rte_pmd_fpga_5gnr_fec.h> 47 #define FPGA_5GNR_PF_DRIVER_NAME ("intel_fpga_5gnr_fec_pf") 48 #define FPGA_5GNR_VF_DRIVER_NAME ("intel_fpga_5gnr_fec_vf") 49 #define VF_UL_5G_QUEUE_VALUE 4 50 #define VF_DL_5G_QUEUE_VALUE 4 51 #define UL_5G_BANDWIDTH 3 52 #define DL_5G_BANDWIDTH 3 53 #define UL_5G_LOAD_BALANCE 128 54 #define DL_5G_LOAD_BALANCE 128 55 #endif 56 57 #ifdef RTE_BASEBAND_ACC 58 #include <rte_acc_cfg.h> 59 #define ACC100PF_DRIVER_NAME ("intel_acc100_pf") 60 #define ACC100VF_DRIVER_NAME ("intel_acc100_vf") 61 #define ACC100_QMGR_NUM_AQS 16 62 #define ACC100_QMGR_NUM_QGS 2 63 #define ACC100_QMGR_AQ_DEPTH 5 64 #define ACC100_QMGR_INVALID_IDX -1 65 #define ACC100_QMGR_RR 1 66 #define ACC100_QOS_GBR 0 67 #define VRBPF_DRIVER_NAME ("intel_vran_boost_pf") 68 #define VRBVF_DRIVER_NAME ("intel_vran_boost_vf") 69 #define VRB_QMGR_NUM_AQS 16 70 #define VRB_QMGR_NUM_QGS 2 71 #define VRB_QMGR_AQ_DEPTH 5 72 #define VRB_QMGR_INVALID_IDX -1 73 #define VRB_QMGR_RR 1 74 #define VRB_QOS_GBR 0 75 #endif 76 77 #define OPS_CACHE_SIZE 256U 78 #define OPS_POOL_SIZE_MIN 511U /* 0.5K per queue */ 79 80 #define SYNC_WAIT 0 81 #define SYNC_START 1 82 83 #define INVALID_QUEUE_ID -1 84 /* Increment for next code block in external HARQ memory */ 85 #define HARQ_INCR 32768 86 /* Headroom for filler LLRs insertion in HARQ buffer */ 87 #define FILLER_HEADROOM 2048 88 /* Constants from K0 computation from 3GPP 38.212 Table 5.4.2.1-2 */ 89 #define N_ZC_1 66 /* N = 66 Zc for BG 1 */ 90 #define N_ZC_2 50 /* N = 50 Zc for BG 2 */ 91 #define K0_1_1 17 /* K0 fraction numerator for rv 1 and BG 1 */ 92 #define K0_1_2 13 /* K0 fraction numerator for rv 1 and BG 2 */ 93 #define K0_2_1 33 /* K0 fraction numerator for rv 2 and BG 1 */ 94 #define K0_2_2 25 /* K0 fraction numerator for rv 2 and BG 2 */ 95 #define K0_3_1 56 /* K0 fraction numerator for rv 3 and BG 1 */ 96 #define K0_3_2 43 /* K0 fraction numerator for rv 3 and BG 2 */ 97 #define NUM_SC_PER_RB (12) /* Number of subcarriers in a RB in 3GPP. */ 98 #define BITS_PER_LLR (8) /* Number of bits in a LLR. */ 99 100 #define HARQ_MEM_TOLERANCE 256 101 static struct test_bbdev_vector test_vector; 102 103 /* Switch between PMD and Interrupt for throughput TC */ 104 static bool intr_enabled; 105 106 /* LLR arithmetic representation for numerical conversion */ 107 static int ldpc_llr_decimals; 108 static int ldpc_llr_size; 109 /* Keep track of the LDPC decoder device capability flag */ 110 static uint32_t ldpc_cap_flags; 111 /* FFT window width predefined on device and on vector. */ 112 static int fft_window_width_dev; 113 114 /* Represents tested active devices */ 115 static struct active_device { 116 const char *driver_name; 117 uint8_t dev_id; 118 uint16_t supported_ops; 119 uint16_t queue_ids[MAX_QUEUES]; 120 uint16_t nb_queues; 121 struct rte_mempool *ops_mempool; 122 struct rte_mempool *in_mbuf_pool; 123 struct rte_mempool *hard_out_mbuf_pool; 124 struct rte_mempool *soft_out_mbuf_pool; 125 struct rte_mempool *harq_in_mbuf_pool; 126 struct rte_mempool *harq_out_mbuf_pool; 127 } active_devs[RTE_BBDEV_MAX_DEVS]; 128 129 static uint8_t nb_active_devs; 130 131 /* Data buffers used by BBDEV ops */ 132 struct test_buffers { 133 struct rte_bbdev_op_data *inputs; 134 struct rte_bbdev_op_data *hard_outputs; 135 struct rte_bbdev_op_data *soft_outputs; 136 struct rte_bbdev_op_data *harq_inputs; 137 struct rte_bbdev_op_data *harq_outputs; 138 }; 139 140 /* Operation parameters specific for given test case */ 141 struct test_op_params { 142 struct rte_mempool *mp; 143 struct rte_bbdev_dec_op *ref_dec_op; 144 struct rte_bbdev_enc_op *ref_enc_op; 145 struct rte_bbdev_fft_op *ref_fft_op; 146 struct rte_bbdev_mldts_op *ref_mldts_op; 147 uint16_t burst_sz; 148 uint16_t num_to_process; 149 uint16_t num_lcores; 150 int vector_mask; 151 RTE_ATOMIC(uint16_t) sync; 152 struct test_buffers q_bufs[RTE_MAX_NUMA_NODES][MAX_QUEUES]; 153 }; 154 155 /* Contains per lcore params */ 156 struct thread_params { 157 uint8_t dev_id; 158 uint16_t queue_id; 159 uint32_t lcore_id; 160 uint64_t start_time; 161 double ops_per_sec; 162 double mbps; 163 uint8_t iter_count; 164 double iter_average; 165 double bler; 166 RTE_ATOMIC(uint16_t) nb_dequeued; 167 RTE_ATOMIC(int16_t) processing_status; 168 RTE_ATOMIC(uint16_t) burst_sz; 169 struct test_op_params *op_params; 170 struct rte_bbdev_dec_op *dec_ops[MAX_BURST]; 171 struct rte_bbdev_enc_op *enc_ops[MAX_BURST]; 172 struct rte_bbdev_fft_op *fft_ops[MAX_BURST]; 173 struct rte_bbdev_mldts_op *mldts_ops[MAX_BURST]; 174 }; 175 176 /* Stores time statistics */ 177 struct test_time_stats { 178 /* Stores software enqueue total working time */ 179 uint64_t enq_sw_total_time; 180 /* Stores minimum value of software enqueue working time */ 181 uint64_t enq_sw_min_time; 182 /* Stores maximum value of software enqueue working time */ 183 uint64_t enq_sw_max_time; 184 /* Stores turbo enqueue total working time */ 185 uint64_t enq_acc_total_time; 186 /* Stores minimum value of accelerator enqueue working time */ 187 uint64_t enq_acc_min_time; 188 /* Stores maximum value of accelerator enqueue working time */ 189 uint64_t enq_acc_max_time; 190 /* Stores dequeue total working time */ 191 uint64_t deq_total_time; 192 /* Stores minimum value of dequeue working time */ 193 uint64_t deq_min_time; 194 /* Stores maximum value of dequeue working time */ 195 uint64_t deq_max_time; 196 }; 197 198 typedef int (test_case_function)(struct active_device *ad, 199 struct test_op_params *op_params); 200 201 /* Get device status before timeout exit */ 202 static inline void 203 timeout_exit(uint8_t dev_id) 204 { 205 struct rte_bbdev_info info; 206 rte_bbdev_info_get(dev_id, &info); 207 printf("Device Status %s\n", rte_bbdev_device_status_str(info.drv.device_status)); 208 } 209 210 static inline void 211 mbuf_reset(struct rte_mbuf *m) 212 { 213 m->pkt_len = 0; 214 215 do { 216 m->data_len = 0; 217 m = m->next; 218 } while (m != NULL); 219 } 220 221 /* Read flag value 0/1 from bitmap */ 222 static inline bool 223 check_bit(uint32_t bitmap, uint32_t bitmask) 224 { 225 return bitmap & bitmask; 226 } 227 228 static inline void 229 set_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type) 230 { 231 ad->supported_ops |= (1 << op_type); 232 } 233 234 static inline bool 235 is_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type) 236 { 237 return ad->supported_ops & (1 << op_type); 238 } 239 240 static inline bool 241 flags_match(uint32_t flags_req, uint32_t flags_present) 242 { 243 return (flags_req & flags_present) == flags_req; 244 } 245 246 static void 247 clear_soft_out_cap(uint32_t *op_flags) 248 { 249 *op_flags &= ~RTE_BBDEV_TURBO_SOFT_OUTPUT; 250 *op_flags &= ~RTE_BBDEV_TURBO_POS_LLR_1_BIT_SOFT_OUT; 251 *op_flags &= ~RTE_BBDEV_TURBO_NEG_LLR_1_BIT_SOFT_OUT; 252 } 253 254 /* This API is to convert all the test vector op data entries 255 * to big endian format. It is used when the device supports 256 * the input in the big endian format. 257 */ 258 static inline void 259 convert_op_data_to_be(void) 260 { 261 struct op_data_entries *op; 262 enum op_data_type type; 263 uint8_t nb_segs, *rem_data, temp; 264 uint32_t *data, len; 265 int complete, rem, i, j; 266 267 for (type = DATA_INPUT; type < DATA_NUM_TYPES; ++type) { 268 nb_segs = test_vector.entries[type].nb_segments; 269 op = &test_vector.entries[type]; 270 271 /* Invert byte endianness for all the segments */ 272 for (i = 0; i < nb_segs; ++i) { 273 len = op->segments[i].length; 274 data = op->segments[i].addr; 275 276 /* Swap complete u32 bytes */ 277 complete = len / 4; 278 for (j = 0; j < complete; j++) 279 data[j] = rte_bswap32(data[j]); 280 281 /* Swap any remaining bytes */ 282 rem = len % 4; 283 rem_data = (uint8_t *)&data[j]; 284 for (j = 0; j < rem/2; j++) { 285 temp = rem_data[j]; 286 rem_data[j] = rem_data[rem - j - 1]; 287 rem_data[rem - j - 1] = temp; 288 } 289 } 290 } 291 } 292 293 static int 294 check_dev_cap(const struct rte_bbdev_info *dev_info) 295 { 296 unsigned int i; 297 unsigned int nb_inputs, nb_soft_outputs, nb_hard_outputs, 298 nb_harq_inputs, nb_harq_outputs; 299 const struct rte_bbdev_op_cap *op_cap = dev_info->drv.capabilities; 300 uint8_t dev_data_endianness = dev_info->drv.data_endianness; 301 302 nb_inputs = test_vector.entries[DATA_INPUT].nb_segments; 303 nb_soft_outputs = test_vector.entries[DATA_SOFT_OUTPUT].nb_segments; 304 nb_hard_outputs = test_vector.entries[DATA_HARD_OUTPUT].nb_segments; 305 nb_harq_inputs = test_vector.entries[DATA_HARQ_INPUT].nb_segments; 306 nb_harq_outputs = test_vector.entries[DATA_HARQ_OUTPUT].nb_segments; 307 308 for (i = 0; op_cap->type != RTE_BBDEV_OP_NONE; ++i, ++op_cap) { 309 if (op_cap->type != test_vector.op_type) 310 continue; 311 312 if (dev_data_endianness == RTE_BIG_ENDIAN) 313 convert_op_data_to_be(); 314 315 if (op_cap->type == RTE_BBDEV_OP_TURBO_DEC) { 316 const struct rte_bbdev_op_cap_turbo_dec *cap = 317 &op_cap->cap.turbo_dec; 318 /* Ignore lack of soft output capability, just skip 319 * checking if soft output is valid. 320 */ 321 if ((test_vector.turbo_dec.op_flags & 322 RTE_BBDEV_TURBO_SOFT_OUTPUT) && 323 !(cap->capability_flags & 324 RTE_BBDEV_TURBO_SOFT_OUTPUT)) { 325 printf( 326 "INFO: Device \"%s\" does not support soft output - soft output flags will be ignored.\n", 327 dev_info->dev_name); 328 clear_soft_out_cap( 329 &test_vector.turbo_dec.op_flags); 330 } 331 332 if (!flags_match(test_vector.turbo_dec.op_flags, 333 cap->capability_flags)) 334 return TEST_FAILED; 335 if (nb_inputs > cap->num_buffers_src) { 336 printf("Too many inputs defined: %u, max: %u\n", 337 nb_inputs, cap->num_buffers_src); 338 return TEST_FAILED; 339 } 340 if (nb_soft_outputs > cap->num_buffers_soft_out && 341 (test_vector.turbo_dec.op_flags & 342 RTE_BBDEV_TURBO_SOFT_OUTPUT)) { 343 printf( 344 "Too many soft outputs defined: %u, max: %u\n", 345 nb_soft_outputs, 346 cap->num_buffers_soft_out); 347 return TEST_FAILED; 348 } 349 if (nb_hard_outputs > cap->num_buffers_hard_out) { 350 printf( 351 "Too many hard outputs defined: %u, max: %u\n", 352 nb_hard_outputs, 353 cap->num_buffers_hard_out); 354 return TEST_FAILED; 355 } 356 if (intr_enabled && !(cap->capability_flags & 357 RTE_BBDEV_TURBO_DEC_INTERRUPTS)) { 358 printf( 359 "Dequeue interrupts are not supported!\n"); 360 return TEST_FAILED; 361 } 362 363 return TEST_SUCCESS; 364 } else if (op_cap->type == RTE_BBDEV_OP_TURBO_ENC) { 365 const struct rte_bbdev_op_cap_turbo_enc *cap = 366 &op_cap->cap.turbo_enc; 367 368 if (!flags_match(test_vector.turbo_enc.op_flags, 369 cap->capability_flags)) 370 return TEST_FAILED; 371 if (nb_inputs > cap->num_buffers_src) { 372 printf("Too many inputs defined: %u, max: %u\n", 373 nb_inputs, cap->num_buffers_src); 374 return TEST_FAILED; 375 } 376 if (nb_hard_outputs > cap->num_buffers_dst) { 377 printf( 378 "Too many hard outputs defined: %u, max: %u\n", 379 nb_hard_outputs, cap->num_buffers_dst); 380 return TEST_FAILED; 381 } 382 if (intr_enabled && !(cap->capability_flags & 383 RTE_BBDEV_TURBO_ENC_INTERRUPTS)) { 384 printf( 385 "Dequeue interrupts are not supported!\n"); 386 return TEST_FAILED; 387 } 388 389 return TEST_SUCCESS; 390 } else if (op_cap->type == RTE_BBDEV_OP_LDPC_ENC) { 391 const struct rte_bbdev_op_cap_ldpc_enc *cap = 392 &op_cap->cap.ldpc_enc; 393 394 if (!flags_match(test_vector.ldpc_enc.op_flags, 395 cap->capability_flags)){ 396 printf("Flag Mismatch\n"); 397 return TEST_FAILED; 398 } 399 if (nb_inputs > cap->num_buffers_src) { 400 printf("Too many inputs defined: %u, max: %u\n", 401 nb_inputs, cap->num_buffers_src); 402 return TEST_FAILED; 403 } 404 if (nb_hard_outputs > cap->num_buffers_dst) { 405 printf( 406 "Too many hard outputs defined: %u, max: %u\n", 407 nb_hard_outputs, cap->num_buffers_dst); 408 return TEST_FAILED; 409 } 410 if (intr_enabled && !(cap->capability_flags & 411 RTE_BBDEV_LDPC_ENC_INTERRUPTS)) { 412 printf( 413 "Dequeue interrupts are not supported!\n"); 414 return TEST_FAILED; 415 } 416 417 return TEST_SUCCESS; 418 } else if (op_cap->type == RTE_BBDEV_OP_LDPC_DEC) { 419 const struct rte_bbdev_op_cap_ldpc_dec *cap = 420 &op_cap->cap.ldpc_dec; 421 422 if (!flags_match(test_vector.ldpc_dec.op_flags, 423 cap->capability_flags)){ 424 printf("Flag Mismatch\n"); 425 return TEST_FAILED; 426 } 427 if (nb_inputs > cap->num_buffers_src) { 428 printf("Too many inputs defined: %u, max: %u\n", 429 nb_inputs, cap->num_buffers_src); 430 return TEST_FAILED; 431 } 432 if (nb_hard_outputs > cap->num_buffers_hard_out) { 433 printf( 434 "Too many hard outputs defined: %u, max: %u\n", 435 nb_hard_outputs, 436 cap->num_buffers_hard_out); 437 return TEST_FAILED; 438 } 439 if (nb_harq_inputs > cap->num_buffers_hard_out) { 440 printf( 441 "Too many HARQ inputs defined: %u, max: %u\n", 442 nb_harq_inputs, 443 cap->num_buffers_hard_out); 444 return TEST_FAILED; 445 } 446 if (nb_harq_outputs > cap->num_buffers_hard_out) { 447 printf( 448 "Too many HARQ outputs defined: %u, max: %u\n", 449 nb_harq_outputs, 450 cap->num_buffers_hard_out); 451 return TEST_FAILED; 452 } 453 if (intr_enabled && !(cap->capability_flags & 454 RTE_BBDEV_LDPC_DEC_INTERRUPTS)) { 455 printf( 456 "Dequeue interrupts are not supported!\n"); 457 return TEST_FAILED; 458 } 459 if (intr_enabled && (test_vector.ldpc_dec.op_flags & 460 (RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE | 461 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE | 462 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK 463 ))) { 464 printf("Skip loop-back with interrupt\n"); 465 return TEST_FAILED; 466 } 467 return TEST_SUCCESS; 468 } else if (op_cap->type == RTE_BBDEV_OP_FFT) { 469 const struct rte_bbdev_op_cap_fft *cap = &op_cap->cap.fft; 470 471 if (!flags_match(test_vector.fft.op_flags, cap->capability_flags)) { 472 printf("Flag Mismatch\n"); 473 return TEST_FAILED; 474 } 475 if (nb_inputs > cap->num_buffers_src) { 476 printf("Too many inputs defined: %u, max: %u\n", 477 nb_inputs, cap->num_buffers_src); 478 return TEST_FAILED; 479 } 480 return TEST_SUCCESS; 481 } else if (op_cap->type == RTE_BBDEV_OP_MLDTS) { 482 const struct rte_bbdev_op_cap_mld *cap = &op_cap->cap.mld; 483 if (!flags_match(test_vector.mldts.op_flags, cap->capability_flags)) { 484 printf("Flag Mismatch\n"); 485 return TEST_FAILED; 486 } 487 if (nb_inputs > cap->num_buffers_src) { 488 printf("Too many inputs defined: %u, max: %u\n", 489 nb_inputs, cap->num_buffers_src); 490 return TEST_FAILED; 491 } 492 return TEST_SUCCESS; 493 } 494 } 495 496 if ((i == 0) && (test_vector.op_type == RTE_BBDEV_OP_NONE)) 497 return TEST_SUCCESS; /* Special case for NULL device */ 498 499 return TEST_FAILED; 500 } 501 502 /* calculates optimal mempool size not smaller than the val */ 503 static unsigned int 504 optimal_mempool_size(unsigned int val) 505 { 506 return rte_align32pow2(val + 1) - 1; 507 } 508 509 /* allocates mbuf mempool for inputs and outputs */ 510 static struct rte_mempool * 511 create_mbuf_pool(struct op_data_entries *entries, uint8_t dev_id, 512 int socket_id, unsigned int mbuf_pool_size, 513 const char *op_type_str) 514 { 515 unsigned int i; 516 uint32_t max_seg_sz = 0; 517 char pool_name[RTE_MEMPOOL_NAMESIZE]; 518 519 /* find max input segment size */ 520 for (i = 0; i < entries->nb_segments; ++i) 521 if (entries->segments[i].length > max_seg_sz) 522 max_seg_sz = entries->segments[i].length; 523 524 snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str, 525 dev_id); 526 return rte_pktmbuf_pool_create(pool_name, mbuf_pool_size, 0, 0, 527 RTE_MAX(max_seg_sz + RTE_PKTMBUF_HEADROOM 528 + FILLER_HEADROOM, 529 (unsigned int)RTE_MBUF_DEFAULT_BUF_SIZE), socket_id); 530 } 531 532 static int 533 create_mempools(struct active_device *ad, int socket_id, 534 enum rte_bbdev_op_type org_op_type, uint16_t num_ops) 535 { 536 struct rte_mempool *mp; 537 unsigned int ops_pool_size, mbuf_pool_size = 0; 538 char pool_name[RTE_MEMPOOL_NAMESIZE]; 539 const char *op_type_str; 540 enum rte_bbdev_op_type op_type = org_op_type; 541 542 struct op_data_entries *in = &test_vector.entries[DATA_INPUT]; 543 struct op_data_entries *hard_out = 544 &test_vector.entries[DATA_HARD_OUTPUT]; 545 struct op_data_entries *soft_out = 546 &test_vector.entries[DATA_SOFT_OUTPUT]; 547 struct op_data_entries *harq_in = 548 &test_vector.entries[DATA_HARQ_INPUT]; 549 struct op_data_entries *harq_out = 550 &test_vector.entries[DATA_HARQ_OUTPUT]; 551 552 /* allocate ops mempool */ 553 ops_pool_size = optimal_mempool_size(RTE_MAX( 554 /* Ops used plus 1 reference op */ 555 RTE_MAX((unsigned int)(ad->nb_queues * num_ops + 1), 556 /* Minimal cache size plus 1 reference op */ 557 (unsigned int)(1.5 * rte_lcore_count() * 558 OPS_CACHE_SIZE + 1)), 559 OPS_POOL_SIZE_MIN)); 560 561 if (org_op_type == RTE_BBDEV_OP_NONE) 562 op_type = RTE_BBDEV_OP_TURBO_ENC; 563 564 op_type_str = rte_bbdev_op_type_str(op_type); 565 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type); 566 567 snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str, 568 ad->dev_id); 569 mp = rte_bbdev_op_pool_create(pool_name, op_type, 570 ops_pool_size, OPS_CACHE_SIZE, socket_id); 571 TEST_ASSERT_NOT_NULL(mp, 572 "ERROR Failed to create %u items ops pool for dev %u on socket %u.", 573 ops_pool_size, 574 ad->dev_id, 575 socket_id); 576 ad->ops_mempool = mp; 577 578 /* Do not create inputs and outputs mbufs for BaseBand Null Device */ 579 if (org_op_type == RTE_BBDEV_OP_NONE) 580 return TEST_SUCCESS; 581 582 /* Inputs */ 583 if (in->nb_segments > 0) { 584 mbuf_pool_size = optimal_mempool_size(ops_pool_size * 585 in->nb_segments); 586 mp = create_mbuf_pool(in, ad->dev_id, socket_id, 587 mbuf_pool_size, "in"); 588 TEST_ASSERT_NOT_NULL(mp, 589 "ERROR Failed to create %u items input pktmbuf pool for dev %u on socket %u.", 590 mbuf_pool_size, 591 ad->dev_id, 592 socket_id); 593 ad->in_mbuf_pool = mp; 594 } 595 596 /* Hard outputs */ 597 if (hard_out->nb_segments > 0) { 598 mbuf_pool_size = optimal_mempool_size(ops_pool_size * 599 hard_out->nb_segments); 600 mp = create_mbuf_pool(hard_out, ad->dev_id, socket_id, 601 mbuf_pool_size, 602 "hard_out"); 603 TEST_ASSERT_NOT_NULL(mp, 604 "ERROR Failed to create %u items hard output pktmbuf pool for dev %u on socket %u.", 605 mbuf_pool_size, 606 ad->dev_id, 607 socket_id); 608 ad->hard_out_mbuf_pool = mp; 609 } 610 611 /* Soft outputs */ 612 if (soft_out->nb_segments > 0) { 613 mbuf_pool_size = optimal_mempool_size(ops_pool_size * 614 soft_out->nb_segments); 615 mp = create_mbuf_pool(soft_out, ad->dev_id, socket_id, 616 mbuf_pool_size, 617 "soft_out"); 618 TEST_ASSERT_NOT_NULL(mp, 619 "ERROR Failed to create %uB soft output pktmbuf pool for dev %u on socket %u.", 620 mbuf_pool_size, 621 ad->dev_id, 622 socket_id); 623 ad->soft_out_mbuf_pool = mp; 624 } 625 626 /* HARQ inputs */ 627 if (harq_in->nb_segments > 0) { 628 mbuf_pool_size = optimal_mempool_size(ops_pool_size * 629 harq_in->nb_segments); 630 mp = create_mbuf_pool(harq_in, ad->dev_id, socket_id, 631 mbuf_pool_size, 632 "harq_in"); 633 TEST_ASSERT_NOT_NULL(mp, 634 "ERROR Failed to create %uB harq input pktmbuf pool for dev %u on socket %u.", 635 mbuf_pool_size, 636 ad->dev_id, 637 socket_id); 638 ad->harq_in_mbuf_pool = mp; 639 } 640 641 /* HARQ outputs */ 642 if (harq_out->nb_segments > 0) { 643 mbuf_pool_size = optimal_mempool_size(ops_pool_size * 644 harq_out->nb_segments); 645 mp = create_mbuf_pool(harq_out, ad->dev_id, socket_id, 646 mbuf_pool_size, 647 "harq_out"); 648 TEST_ASSERT_NOT_NULL(mp, 649 "ERROR Failed to create %uB harq output pktmbuf pool for dev %u on socket %u.", 650 mbuf_pool_size, 651 ad->dev_id, 652 socket_id); 653 ad->harq_out_mbuf_pool = mp; 654 } 655 656 return TEST_SUCCESS; 657 } 658 659 static int 660 add_bbdev_dev(uint8_t dev_id, struct rte_bbdev_info *info, 661 struct test_bbdev_vector *vector) 662 { 663 int ret; 664 unsigned int queue_id; 665 struct rte_bbdev_queue_conf qconf; 666 struct active_device *ad = &active_devs[nb_active_devs]; 667 unsigned int nb_queues; 668 enum rte_bbdev_op_type op_type = vector->op_type; 669 670 /* Configure fpga lte fec with PF & VF values 671 * if '-i' flag is set and using fpga device 672 */ 673 #ifdef RTE_BASEBAND_FPGA_LTE_FEC 674 if ((get_init_device() == true) && 675 (!strcmp(info->drv.driver_name, FPGA_LTE_PF_DRIVER_NAME))) { 676 struct rte_fpga_lte_fec_conf conf; 677 unsigned int i; 678 679 printf("Configure FPGA LTE FEC Driver %s with default values\n", 680 info->drv.driver_name); 681 682 /* clear default configuration before initialization */ 683 memset(&conf, 0, sizeof(struct rte_fpga_lte_fec_conf)); 684 685 /* Set PF mode : 686 * true if PF is used for data plane 687 * false for VFs 688 */ 689 conf.pf_mode_en = true; 690 691 for (i = 0; i < FPGA_LTE_FEC_NUM_VFS; ++i) { 692 /* Number of UL queues per VF (fpga supports 8 VFs) */ 693 conf.vf_ul_queues_number[i] = VF_UL_4G_QUEUE_VALUE; 694 /* Number of DL queues per VF (fpga supports 8 VFs) */ 695 conf.vf_dl_queues_number[i] = VF_DL_4G_QUEUE_VALUE; 696 } 697 698 /* UL bandwidth. Needed for schedule algorithm */ 699 conf.ul_bandwidth = UL_4G_BANDWIDTH; 700 /* DL bandwidth */ 701 conf.dl_bandwidth = DL_4G_BANDWIDTH; 702 703 /* UL & DL load Balance Factor to 64 */ 704 conf.ul_load_balance = UL_4G_LOAD_BALANCE; 705 conf.dl_load_balance = DL_4G_LOAD_BALANCE; 706 707 /**< FLR timeout value */ 708 conf.flr_time_out = FLR_4G_TIMEOUT; 709 710 /* setup FPGA PF with configuration information */ 711 ret = rte_fpga_lte_fec_configure(info->dev_name, &conf); 712 TEST_ASSERT_SUCCESS(ret, 713 "Failed to configure 4G FPGA PF for bbdev %s", 714 info->dev_name); 715 } 716 #endif 717 #ifdef RTE_BASEBAND_FPGA_5GNR_FEC 718 if ((get_init_device() == true) && 719 (!strcmp(info->drv.driver_name, FPGA_5GNR_PF_DRIVER_NAME))) { 720 struct rte_fpga_5gnr_fec_conf conf; 721 unsigned int i; 722 723 printf("Configure FPGA 5GNR FEC Driver %s with default values\n", 724 info->drv.driver_name); 725 726 /* clear default configuration before initialization */ 727 memset(&conf, 0, sizeof(struct rte_fpga_5gnr_fec_conf)); 728 729 /* Set PF mode : 730 * true if PF is used for data plane 731 * false for VFs 732 */ 733 conf.pf_mode_en = true; 734 735 for (i = 0; i < FPGA_5GNR_FEC_NUM_VFS; ++i) { 736 /* Number of UL queues per VF (fpga supports 8 VFs) */ 737 conf.vf_ul_queues_number[i] = VF_UL_5G_QUEUE_VALUE; 738 /* Number of DL queues per VF (fpga supports 8 VFs) */ 739 conf.vf_dl_queues_number[i] = VF_DL_5G_QUEUE_VALUE; 740 } 741 742 /* UL bandwidth. Needed only for Vista Creek 5GNR schedule algorithm */ 743 conf.ul_bandwidth = UL_5G_BANDWIDTH; 744 /* DL bandwidth. Needed only for Vista Creek 5GNR schedule algorithm */ 745 conf.dl_bandwidth = DL_5G_BANDWIDTH; 746 747 /* UL & DL load Balance Factor to 64 */ 748 conf.ul_load_balance = UL_5G_LOAD_BALANCE; 749 conf.dl_load_balance = DL_5G_LOAD_BALANCE; 750 751 /* setup FPGA PF with configuration information */ 752 ret = rte_fpga_5gnr_fec_configure(info->dev_name, &conf); 753 TEST_ASSERT_SUCCESS(ret, 754 "Failed to configure 5G FPGA PF for bbdev %s", 755 info->dev_name); 756 } 757 #endif 758 #ifdef RTE_BASEBAND_ACC 759 if ((get_init_device() == true) && 760 (!strcmp(info->drv.driver_name, ACC100PF_DRIVER_NAME))) { 761 struct rte_acc_conf conf; 762 unsigned int i; 763 764 printf("Configure ACC100 FEC device %s with default values\n", 765 info->drv.driver_name); 766 767 /* clear default configuration before initialization */ 768 memset(&conf, 0, sizeof(struct rte_acc_conf)); 769 770 /* Always set in PF mode for built-in configuration */ 771 conf.pf_mode_en = true; 772 for (i = 0; i < RTE_ACC_NUM_VFS; ++i) { 773 conf.arb_dl_4g[i].gbr_threshold1 = ACC100_QOS_GBR; 774 conf.arb_dl_4g[i].gbr_threshold1 = ACC100_QOS_GBR; 775 conf.arb_dl_4g[i].round_robin_weight = ACC100_QMGR_RR; 776 conf.arb_ul_4g[i].gbr_threshold1 = ACC100_QOS_GBR; 777 conf.arb_ul_4g[i].gbr_threshold1 = ACC100_QOS_GBR; 778 conf.arb_ul_4g[i].round_robin_weight = ACC100_QMGR_RR; 779 conf.arb_dl_5g[i].gbr_threshold1 = ACC100_QOS_GBR; 780 conf.arb_dl_5g[i].gbr_threshold1 = ACC100_QOS_GBR; 781 conf.arb_dl_5g[i].round_robin_weight = ACC100_QMGR_RR; 782 conf.arb_ul_5g[i].gbr_threshold1 = ACC100_QOS_GBR; 783 conf.arb_ul_5g[i].gbr_threshold1 = ACC100_QOS_GBR; 784 conf.arb_ul_5g[i].round_robin_weight = ACC100_QMGR_RR; 785 } 786 787 conf.input_pos_llr_1_bit = true; 788 conf.output_pos_llr_1_bit = true; 789 conf.num_vf_bundles = 1; /**< Number of VF bundles to setup */ 790 791 conf.q_ul_4g.num_qgroups = ACC100_QMGR_NUM_QGS; 792 conf.q_ul_4g.first_qgroup_index = ACC100_QMGR_INVALID_IDX; 793 conf.q_ul_4g.num_aqs_per_groups = ACC100_QMGR_NUM_AQS; 794 conf.q_ul_4g.aq_depth_log2 = ACC100_QMGR_AQ_DEPTH; 795 conf.q_dl_4g.num_qgroups = ACC100_QMGR_NUM_QGS; 796 conf.q_dl_4g.first_qgroup_index = ACC100_QMGR_INVALID_IDX; 797 conf.q_dl_4g.num_aqs_per_groups = ACC100_QMGR_NUM_AQS; 798 conf.q_dl_4g.aq_depth_log2 = ACC100_QMGR_AQ_DEPTH; 799 conf.q_ul_5g.num_qgroups = ACC100_QMGR_NUM_QGS; 800 conf.q_ul_5g.first_qgroup_index = ACC100_QMGR_INVALID_IDX; 801 conf.q_ul_5g.num_aqs_per_groups = ACC100_QMGR_NUM_AQS; 802 conf.q_ul_5g.aq_depth_log2 = ACC100_QMGR_AQ_DEPTH; 803 conf.q_dl_5g.num_qgroups = ACC100_QMGR_NUM_QGS; 804 conf.q_dl_5g.first_qgroup_index = ACC100_QMGR_INVALID_IDX; 805 conf.q_dl_5g.num_aqs_per_groups = ACC100_QMGR_NUM_AQS; 806 conf.q_dl_5g.aq_depth_log2 = ACC100_QMGR_AQ_DEPTH; 807 808 /* setup PF with configuration information */ 809 ret = rte_acc_configure(info->dev_name, &conf); 810 TEST_ASSERT_SUCCESS(ret, 811 "Failed to configure ACC100 PF for bbdev %s", 812 info->dev_name); 813 } 814 if ((get_init_device() == true) && 815 (!strcmp(info->drv.driver_name, VRBPF_DRIVER_NAME))) { 816 struct rte_acc_conf conf; 817 unsigned int i; 818 819 printf("Configure Driver %s with default values\n", 820 info->drv.driver_name); 821 822 /* clear default configuration before initialization */ 823 memset(&conf, 0, sizeof(struct rte_acc_conf)); 824 825 /* Always set in PF mode for built-in configuration */ 826 conf.pf_mode_en = true; 827 for (i = 0; i < RTE_ACC_NUM_VFS; ++i) { 828 conf.arb_dl_4g[i].gbr_threshold1 = VRB_QOS_GBR; 829 conf.arb_dl_4g[i].gbr_threshold1 = VRB_QOS_GBR; 830 conf.arb_dl_4g[i].round_robin_weight = VRB_QMGR_RR; 831 conf.arb_ul_4g[i].gbr_threshold1 = VRB_QOS_GBR; 832 conf.arb_ul_4g[i].gbr_threshold1 = VRB_QOS_GBR; 833 conf.arb_ul_4g[i].round_robin_weight = VRB_QMGR_RR; 834 conf.arb_dl_5g[i].gbr_threshold1 = VRB_QOS_GBR; 835 conf.arb_dl_5g[i].gbr_threshold1 = VRB_QOS_GBR; 836 conf.arb_dl_5g[i].round_robin_weight = VRB_QMGR_RR; 837 conf.arb_ul_5g[i].gbr_threshold1 = VRB_QOS_GBR; 838 conf.arb_ul_5g[i].gbr_threshold1 = VRB_QOS_GBR; 839 conf.arb_ul_5g[i].round_robin_weight = VRB_QMGR_RR; 840 conf.arb_fft[i].gbr_threshold1 = VRB_QOS_GBR; 841 conf.arb_fft[i].gbr_threshold1 = VRB_QOS_GBR; 842 conf.arb_fft[i].round_robin_weight = VRB_QMGR_RR; 843 conf.arb_mld[i].gbr_threshold1 = VRB_QOS_GBR; 844 conf.arb_mld[i].gbr_threshold1 = VRB_QOS_GBR; 845 conf.arb_mld[i].round_robin_weight = VRB_QMGR_RR; 846 } 847 848 conf.input_pos_llr_1_bit = true; 849 conf.output_pos_llr_1_bit = true; 850 conf.num_vf_bundles = 1; /**< Number of VF bundles to setup */ 851 852 conf.q_ul_4g.num_qgroups = VRB_QMGR_NUM_QGS; 853 conf.q_ul_4g.first_qgroup_index = VRB_QMGR_INVALID_IDX; 854 conf.q_ul_4g.num_aqs_per_groups = VRB_QMGR_NUM_AQS; 855 conf.q_ul_4g.aq_depth_log2 = VRB_QMGR_AQ_DEPTH; 856 conf.q_dl_4g.num_qgroups = VRB_QMGR_NUM_QGS; 857 conf.q_dl_4g.first_qgroup_index = VRB_QMGR_INVALID_IDX; 858 conf.q_dl_4g.num_aqs_per_groups = VRB_QMGR_NUM_AQS; 859 conf.q_dl_4g.aq_depth_log2 = VRB_QMGR_AQ_DEPTH; 860 conf.q_ul_5g.num_qgroups = VRB_QMGR_NUM_QGS; 861 conf.q_ul_5g.first_qgroup_index = VRB_QMGR_INVALID_IDX; 862 conf.q_ul_5g.num_aqs_per_groups = VRB_QMGR_NUM_AQS; 863 conf.q_ul_5g.aq_depth_log2 = VRB_QMGR_AQ_DEPTH; 864 conf.q_dl_5g.num_qgroups = VRB_QMGR_NUM_QGS; 865 conf.q_dl_5g.first_qgroup_index = VRB_QMGR_INVALID_IDX; 866 conf.q_dl_5g.num_aqs_per_groups = VRB_QMGR_NUM_AQS; 867 conf.q_dl_5g.aq_depth_log2 = VRB_QMGR_AQ_DEPTH; 868 conf.q_fft.num_qgroups = VRB_QMGR_NUM_QGS; 869 conf.q_fft.first_qgroup_index = VRB_QMGR_INVALID_IDX; 870 conf.q_fft.num_aqs_per_groups = VRB_QMGR_NUM_AQS; 871 conf.q_fft.aq_depth_log2 = VRB_QMGR_AQ_DEPTH; 872 conf.q_mld.num_qgroups = VRB_QMGR_NUM_QGS; 873 conf.q_mld.first_qgroup_index = VRB_QMGR_INVALID_IDX; 874 conf.q_mld.num_aqs_per_groups = VRB_QMGR_NUM_AQS; 875 conf.q_mld.aq_depth_log2 = VRB_QMGR_AQ_DEPTH; 876 877 /* setup PF with configuration information */ 878 ret = rte_acc_configure(info->dev_name, &conf); 879 TEST_ASSERT_SUCCESS(ret, 880 "Failed to configure PF for bbdev %s", 881 info->dev_name); 882 } 883 #endif 884 /* Let's refresh this now this is configured */ 885 rte_bbdev_info_get(dev_id, info); 886 if (info->drv.device_status == RTE_BBDEV_DEV_FATAL_ERR) 887 printf("Device Status %s\n", rte_bbdev_device_status_str(info->drv.device_status)); 888 if (info->drv.fft_window_width != NULL) 889 fft_window_width_dev = info->drv.fft_window_width[0]; 890 else 891 fft_window_width_dev = 0; 892 if (fft_window_width_dev != 0) 893 printf(" FFT Window0 width %d\n", fft_window_width_dev); 894 895 nb_queues = RTE_MIN(rte_lcore_count(), info->drv.max_num_queues); 896 nb_queues = RTE_MIN(nb_queues, (unsigned int) MAX_QUEUES); 897 898 /* setup device */ 899 ret = rte_bbdev_setup_queues(dev_id, nb_queues, info->socket_id); 900 if (ret < 0) { 901 printf("rte_bbdev_setup_queues(%u, %u, %d) ret %i\n", 902 dev_id, nb_queues, info->socket_id, ret); 903 return TEST_FAILED; 904 } 905 906 /* configure interrupts if needed */ 907 if (intr_enabled) { 908 ret = rte_bbdev_intr_enable(dev_id); 909 if (ret < 0) { 910 printf("rte_bbdev_intr_enable(%u) ret %i\n", dev_id, 911 ret); 912 return TEST_FAILED; 913 } 914 } 915 916 /* setup device queues */ 917 qconf.socket = info->socket_id; 918 qconf.queue_size = info->drv.default_queue_conf.queue_size; 919 qconf.priority = 0; 920 qconf.deferred_start = 0; 921 qconf.op_type = op_type; 922 923 for (queue_id = 0; queue_id < nb_queues; ++queue_id) { 924 ret = rte_bbdev_queue_configure(dev_id, queue_id, &qconf); 925 if (ret != 0) { 926 printf( 927 "Allocated all queues (id=%u) at prio%u on dev%u\n", 928 queue_id, qconf.priority, dev_id); 929 qconf.priority++; 930 ret = rte_bbdev_queue_configure(ad->dev_id, queue_id, &qconf); 931 } 932 if (ret != 0) { 933 printf("All queues on dev %u allocated: %u\n", dev_id, queue_id); 934 break; 935 } 936 ret = rte_bbdev_queue_start(ad->dev_id, queue_id); 937 if (ret != 0) { 938 printf("Failed to start queue on dev %u q_id: %u\n", dev_id, queue_id); 939 break; 940 } 941 ad->queue_ids[queue_id] = queue_id; 942 } 943 TEST_ASSERT(queue_id != 0, 944 "ERROR Failed to configure any queues on dev %u\n" 945 "\tthe device may not support the related operation capability\n" 946 "\tor the device may not have been configured yet", dev_id); 947 ad->nb_queues = queue_id; 948 949 set_avail_op(ad, op_type); 950 951 return TEST_SUCCESS; 952 } 953 954 static int 955 add_active_device(uint8_t dev_id, struct rte_bbdev_info *info, 956 struct test_bbdev_vector *vector) 957 { 958 int ret; 959 960 active_devs[nb_active_devs].driver_name = info->drv.driver_name; 961 active_devs[nb_active_devs].dev_id = dev_id; 962 963 ret = add_bbdev_dev(dev_id, info, vector); 964 if (ret == TEST_SUCCESS) 965 ++nb_active_devs; 966 return ret; 967 } 968 969 static uint8_t 970 populate_active_devices(void) 971 { 972 int ret; 973 uint8_t dev_id; 974 uint8_t nb_devs_added = 0; 975 struct rte_bbdev_info info; 976 977 RTE_BBDEV_FOREACH(dev_id) { 978 rte_bbdev_info_get(dev_id, &info); 979 980 if (check_dev_cap(&info)) { 981 printf( 982 "Device %d (%s) does not support specified capabilities\n", 983 dev_id, info.dev_name); 984 continue; 985 } 986 987 ret = add_active_device(dev_id, &info, &test_vector); 988 if (ret != 0) { 989 printf("Adding active bbdev %s skipped\n", 990 info.dev_name); 991 continue; 992 } 993 nb_devs_added++; 994 } 995 996 return nb_devs_added; 997 } 998 999 static int 1000 read_test_vector(void) 1001 { 1002 int ret; 1003 1004 memset(&test_vector, 0, sizeof(test_vector)); 1005 printf("Test vector file = %s\n", get_vector_filename()); 1006 ret = test_bbdev_vector_read(get_vector_filename(), &test_vector); 1007 TEST_ASSERT_SUCCESS(ret, "Failed to parse file %s\n", 1008 get_vector_filename()); 1009 1010 return TEST_SUCCESS; 1011 } 1012 1013 static int 1014 testsuite_setup(void) 1015 { 1016 TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n"); 1017 1018 if (populate_active_devices() == 0) { 1019 printf("No suitable devices found!\n"); 1020 return TEST_SKIPPED; 1021 } 1022 1023 return TEST_SUCCESS; 1024 } 1025 1026 static int 1027 interrupt_testsuite_setup(void) 1028 { 1029 TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n"); 1030 1031 /* Enable interrupts */ 1032 intr_enabled = true; 1033 1034 /* Special case for NULL device (RTE_BBDEV_OP_NONE) */ 1035 if (populate_active_devices() == 0 || 1036 test_vector.op_type == RTE_BBDEV_OP_NONE) { 1037 intr_enabled = false; 1038 printf("No suitable devices found!\n"); 1039 return TEST_SKIPPED; 1040 } 1041 1042 return TEST_SUCCESS; 1043 } 1044 1045 static void 1046 testsuite_teardown(void) 1047 { 1048 uint8_t dev_id; 1049 1050 /* Unconfigure devices */ 1051 RTE_BBDEV_FOREACH(dev_id) 1052 rte_bbdev_close(dev_id); 1053 1054 /* Clear active devices structs. */ 1055 memset(active_devs, 0, sizeof(active_devs)); 1056 nb_active_devs = 0; 1057 1058 /* Disable interrupts */ 1059 intr_enabled = false; 1060 } 1061 1062 static int 1063 ut_setup(void) 1064 { 1065 uint8_t i, dev_id; 1066 1067 for (i = 0; i < nb_active_devs; i++) { 1068 dev_id = active_devs[i].dev_id; 1069 /* reset bbdev stats */ 1070 TEST_ASSERT_SUCCESS(rte_bbdev_stats_reset(dev_id), 1071 "Failed to reset stats of bbdev %u", dev_id); 1072 /* start the device */ 1073 TEST_ASSERT_SUCCESS(rte_bbdev_start(dev_id), 1074 "Failed to start bbdev %u", dev_id); 1075 } 1076 1077 return TEST_SUCCESS; 1078 } 1079 1080 static void 1081 ut_teardown(void) 1082 { 1083 uint8_t i, dev_id, ret; 1084 struct rte_bbdev_stats stats; 1085 1086 for (i = 0; i < nb_active_devs; i++) { 1087 dev_id = active_devs[i].dev_id; 1088 /* read stats and print */ 1089 ret = rte_bbdev_stats_get(dev_id, &stats); 1090 if (ret != 0) 1091 printf("Failed to get stats on bbdev %u\n", dev_id); 1092 /* Stop the device */ 1093 rte_bbdev_stop(dev_id); 1094 } 1095 } 1096 1097 static int 1098 init_op_data_objs(struct rte_bbdev_op_data *bufs, 1099 struct op_data_entries *ref_entries, 1100 struct rte_mempool *mbuf_pool, const uint16_t n, 1101 enum op_data_type op_type, uint16_t min_alignment) 1102 { 1103 int ret; 1104 unsigned int i, j; 1105 bool large_input = false; 1106 1107 for (i = 0; i < n; ++i) { 1108 char *data; 1109 struct op_data_buf *seg = &ref_entries->segments[0]; 1110 struct rte_mbuf *m_head = rte_pktmbuf_alloc(mbuf_pool); 1111 TEST_ASSERT_NOT_NULL(m_head, 1112 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)", 1113 op_type, n * ref_entries->nb_segments, 1114 mbuf_pool->size); 1115 1116 if ((seg->length + RTE_PKTMBUF_HEADROOM) > RTE_BBDEV_LDPC_E_MAX_MBUF) { 1117 /* 1118 * Special case when DPDK mbuf cannot handle 1119 * the required input size 1120 */ 1121 large_input = true; 1122 } 1123 bufs[i].data = m_head; 1124 bufs[i].offset = 0; 1125 bufs[i].length = 0; 1126 1127 if ((op_type == DATA_INPUT) || (op_type == DATA_HARQ_INPUT)) { 1128 if (large_input) { 1129 /* Allocate a fake overused mbuf */ 1130 data = rte_malloc(NULL, seg->length, 0); 1131 TEST_ASSERT_NOT_NULL(data, 1132 "rte malloc failed with %u bytes", 1133 seg->length); 1134 memcpy(data, seg->addr, seg->length); 1135 m_head->buf_addr = data; 1136 rte_mbuf_iova_set(m_head, rte_malloc_virt2iova(data)); 1137 m_head->data_off = 0; 1138 m_head->data_len = seg->length; 1139 } else { 1140 data = rte_pktmbuf_append(m_head, seg->length); 1141 TEST_ASSERT_NOT_NULL(data, 1142 "Couldn't append %u bytes to mbuf from %d data type mbuf pool", 1143 seg->length, op_type); 1144 1145 TEST_ASSERT(data == RTE_PTR_ALIGN( 1146 data, min_alignment), 1147 "Data addr in mbuf (%p) is not aligned to device min alignment (%u)", 1148 data, min_alignment); 1149 rte_memcpy(data, seg->addr, seg->length); 1150 } 1151 1152 bufs[i].length += seg->length; 1153 1154 for (j = 1; j < ref_entries->nb_segments; ++j) { 1155 struct rte_mbuf *m_tail = 1156 rte_pktmbuf_alloc(mbuf_pool); 1157 TEST_ASSERT_NOT_NULL(m_tail, 1158 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)", 1159 op_type, 1160 n * ref_entries->nb_segments, 1161 mbuf_pool->size); 1162 seg += 1; 1163 1164 data = rte_pktmbuf_append(m_tail, seg->length); 1165 TEST_ASSERT_NOT_NULL(data, 1166 "Couldn't append %u bytes to mbuf from %d data type mbuf pool", 1167 seg->length, op_type); 1168 1169 TEST_ASSERT(data == RTE_PTR_ALIGN(data, 1170 min_alignment), 1171 "Data addr in mbuf (%p) is not aligned to device min alignment (%u)", 1172 data, min_alignment); 1173 rte_memcpy(data, seg->addr, seg->length); 1174 bufs[i].length += seg->length; 1175 1176 ret = rte_pktmbuf_chain(m_head, m_tail); 1177 TEST_ASSERT_SUCCESS(ret, 1178 "Couldn't chain mbufs from %d data type mbuf pool", 1179 op_type); 1180 } 1181 } else { 1182 if (((op_type == DATA_HARD_OUTPUT) || (op_type == DATA_SOFT_OUTPUT)) 1183 && ((seg->length + RTE_PKTMBUF_HEADROOM) 1184 > RTE_BBDEV_LDPC_E_MAX_MBUF)) { 1185 /* Allocate a fake overused mbuf + margin */ 1186 data = rte_malloc(NULL, seg->length + 1024, 0); 1187 TEST_ASSERT_NOT_NULL(data, 1188 "rte malloc failed with %u bytes", 1189 seg->length + 1024); 1190 m_head->buf_addr = data; 1191 rte_mbuf_iova_set(m_head, rte_malloc_virt2iova(data)); 1192 m_head->data_off = 0; 1193 m_head->data_len = seg->length; 1194 } else { 1195 /* allocate chained-mbuf for output buffer */ 1196 for (j = 1; j < ref_entries->nb_segments; ++j) { 1197 struct rte_mbuf *m_tail = 1198 rte_pktmbuf_alloc(mbuf_pool); 1199 TEST_ASSERT_NOT_NULL(m_tail, 1200 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)", 1201 op_type, 1202 n * ref_entries->nb_segments, 1203 mbuf_pool->size); 1204 1205 ret = rte_pktmbuf_chain(m_head, m_tail); 1206 TEST_ASSERT_SUCCESS(ret, 1207 "Couldn't chain mbufs from %d data type mbuf pool", 1208 op_type); 1209 } 1210 } 1211 bufs[i].length += seg->length; 1212 } 1213 } 1214 1215 return 0; 1216 } 1217 1218 static int 1219 allocate_buffers_on_socket(struct rte_bbdev_op_data **buffers, const int len, 1220 const int socket) 1221 { 1222 int i; 1223 1224 *buffers = rte_zmalloc_socket(NULL, len, 0, socket); 1225 if (*buffers == NULL) { 1226 printf("WARNING: Failed to allocate op_data on socket %d\n", 1227 socket); 1228 /* try to allocate memory on other detected sockets */ 1229 for (i = 0; i < socket; i++) { 1230 *buffers = rte_zmalloc_socket(NULL, len, 0, i); 1231 if (*buffers != NULL) 1232 break; 1233 } 1234 } 1235 1236 return (*buffers == NULL) ? TEST_FAILED : TEST_SUCCESS; 1237 } 1238 1239 static void 1240 limit_input_llr_val_range(struct rte_bbdev_op_data *input_ops, 1241 const uint16_t n, const int8_t max_llr_modulus) 1242 { 1243 uint16_t i, byte_idx; 1244 1245 for (i = 0; i < n; ++i) { 1246 struct rte_mbuf *m = input_ops[i].data; 1247 while (m != NULL) { 1248 int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *, 1249 input_ops[i].offset); 1250 for (byte_idx = 0; byte_idx < rte_pktmbuf_data_len(m); 1251 ++byte_idx) 1252 llr[byte_idx] = round((double)max_llr_modulus * 1253 llr[byte_idx] / INT8_MAX); 1254 1255 m = m->next; 1256 } 1257 } 1258 } 1259 1260 /* 1261 * We may have to insert filler bits 1262 * when they are required by the HARQ assumption 1263 */ 1264 static void 1265 ldpc_add_filler(struct rte_bbdev_op_data *input_ops, 1266 const uint16_t n, struct test_op_params *op_params) 1267 { 1268 struct rte_bbdev_op_ldpc_dec dec = op_params->ref_dec_op->ldpc_dec; 1269 1270 if (input_ops == NULL) 1271 return; 1272 /* No need to add filler if not required by device */ 1273 if (!(ldpc_cap_flags & 1274 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_FILLERS)) 1275 return; 1276 /* No need to add filler for loopback operation */ 1277 if (dec.op_flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK) 1278 return; 1279 1280 uint16_t i, j, parity_offset; 1281 for (i = 0; i < n; ++i) { 1282 struct rte_mbuf *m = input_ops[i].data; 1283 int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *, 1284 input_ops[i].offset); 1285 parity_offset = (dec.basegraph == 1 ? 20 : 8) 1286 * dec.z_c - dec.n_filler; 1287 uint16_t new_hin_size = input_ops[i].length + dec.n_filler; 1288 m->data_len = new_hin_size; 1289 input_ops[i].length = new_hin_size; 1290 for (j = new_hin_size - 1; j >= parity_offset + dec.n_filler; 1291 j--) 1292 llr[j] = llr[j - dec.n_filler]; 1293 uint16_t llr_max_pre_scaling = (1 << (ldpc_llr_size - 1)) - 1; 1294 for (j = 0; j < dec.n_filler; j++) 1295 llr[parity_offset + j] = llr_max_pre_scaling; 1296 } 1297 } 1298 1299 static void 1300 ldpc_input_llr_scaling(struct rte_bbdev_op_data *input_ops, 1301 const uint16_t n, const int8_t llr_size, 1302 const int8_t llr_decimals) 1303 { 1304 if (input_ops == NULL) 1305 return; 1306 1307 uint16_t i, byte_idx; 1308 1309 int16_t llr_max, llr_min, llr_tmp; 1310 llr_max = (1 << (llr_size - 1)) - 1; 1311 llr_min = -llr_max; 1312 for (i = 0; i < n; ++i) { 1313 struct rte_mbuf *m = input_ops[i].data; 1314 while (m != NULL) { 1315 int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *, 1316 input_ops[i].offset); 1317 for (byte_idx = 0; byte_idx < rte_pktmbuf_data_len(m); 1318 ++byte_idx) { 1319 1320 llr_tmp = llr[byte_idx]; 1321 if (llr_decimals == 4) 1322 llr_tmp *= 8; 1323 else if (llr_decimals == 2) 1324 llr_tmp *= 2; 1325 else if (llr_decimals == 0) 1326 llr_tmp /= 2; 1327 llr_tmp = RTE_MIN(llr_max, 1328 RTE_MAX(llr_min, llr_tmp)); 1329 llr[byte_idx] = (int8_t) llr_tmp; 1330 } 1331 1332 m = m->next; 1333 } 1334 } 1335 } 1336 1337 1338 1339 static int 1340 fill_queue_buffers(struct test_op_params *op_params, 1341 struct rte_mempool *in_mp, struct rte_mempool *hard_out_mp, 1342 struct rte_mempool *soft_out_mp, 1343 struct rte_mempool *harq_in_mp, struct rte_mempool *harq_out_mp, 1344 uint16_t queue_id, 1345 const struct rte_bbdev_op_cap *capabilities, 1346 uint16_t min_alignment, const int socket_id) 1347 { 1348 int ret; 1349 enum op_data_type type; 1350 const uint16_t n = op_params->num_to_process; 1351 1352 struct rte_mempool *mbuf_pools[DATA_NUM_TYPES] = { 1353 in_mp, 1354 soft_out_mp, 1355 hard_out_mp, 1356 harq_in_mp, 1357 harq_out_mp, 1358 }; 1359 1360 struct rte_bbdev_op_data **queue_ops[DATA_NUM_TYPES] = { 1361 &op_params->q_bufs[socket_id][queue_id].inputs, 1362 &op_params->q_bufs[socket_id][queue_id].soft_outputs, 1363 &op_params->q_bufs[socket_id][queue_id].hard_outputs, 1364 &op_params->q_bufs[socket_id][queue_id].harq_inputs, 1365 &op_params->q_bufs[socket_id][queue_id].harq_outputs, 1366 }; 1367 1368 for (type = DATA_INPUT; type < DATA_NUM_TYPES; ++type) { 1369 struct op_data_entries *ref_entries = 1370 &test_vector.entries[type]; 1371 if (ref_entries->nb_segments == 0) 1372 continue; 1373 1374 ret = allocate_buffers_on_socket(queue_ops[type], 1375 n * sizeof(struct rte_bbdev_op_data), 1376 socket_id); 1377 TEST_ASSERT_SUCCESS(ret, 1378 "Couldn't allocate memory for rte_bbdev_op_data structs"); 1379 1380 ret = init_op_data_objs(*queue_ops[type], ref_entries, 1381 mbuf_pools[type], n, type, min_alignment); 1382 TEST_ASSERT_SUCCESS(ret, 1383 "Couldn't init rte_bbdev_op_data structs"); 1384 } 1385 1386 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) 1387 limit_input_llr_val_range(*queue_ops[DATA_INPUT], n, 1388 capabilities->cap.turbo_dec.max_llr_modulus); 1389 1390 if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) { 1391 bool loopback = op_params->ref_dec_op->ldpc_dec.op_flags & 1392 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK; 1393 bool llr_comp = op_params->ref_dec_op->ldpc_dec.op_flags & 1394 RTE_BBDEV_LDPC_LLR_COMPRESSION; 1395 bool harq_comp = op_params->ref_dec_op->ldpc_dec.op_flags & 1396 RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION; 1397 1398 ldpc_llr_decimals = capabilities->cap.ldpc_dec.llr_decimals; 1399 ldpc_llr_size = capabilities->cap.ldpc_dec.llr_size; 1400 ldpc_cap_flags = capabilities->cap.ldpc_dec.capability_flags; 1401 if (!loopback && !llr_comp) 1402 ldpc_input_llr_scaling(*queue_ops[DATA_INPUT], n, 1403 ldpc_llr_size, ldpc_llr_decimals); 1404 if (!loopback && !harq_comp) 1405 ldpc_input_llr_scaling(*queue_ops[DATA_HARQ_INPUT], n, 1406 ldpc_llr_size, ldpc_llr_decimals); 1407 if (!loopback) 1408 ldpc_add_filler(*queue_ops[DATA_HARQ_INPUT], n, 1409 op_params); 1410 } 1411 1412 return 0; 1413 } 1414 1415 static void 1416 free_buffers(struct active_device *ad, struct test_op_params *op_params) 1417 { 1418 unsigned int i, j; 1419 1420 rte_mempool_free(ad->ops_mempool); 1421 rte_mempool_free(ad->in_mbuf_pool); 1422 rte_mempool_free(ad->hard_out_mbuf_pool); 1423 rte_mempool_free(ad->soft_out_mbuf_pool); 1424 rte_mempool_free(ad->harq_in_mbuf_pool); 1425 rte_mempool_free(ad->harq_out_mbuf_pool); 1426 1427 for (i = 0; i < rte_lcore_count(); ++i) { 1428 for (j = 0; j < RTE_MAX_NUMA_NODES; ++j) { 1429 rte_free(op_params->q_bufs[j][i].inputs); 1430 rte_free(op_params->q_bufs[j][i].hard_outputs); 1431 rte_free(op_params->q_bufs[j][i].soft_outputs); 1432 rte_free(op_params->q_bufs[j][i].harq_inputs); 1433 rte_free(op_params->q_bufs[j][i].harq_outputs); 1434 } 1435 } 1436 } 1437 1438 static void 1439 copy_reference_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n, 1440 unsigned int start_idx, 1441 struct rte_bbdev_op_data *inputs, 1442 struct rte_bbdev_op_data *hard_outputs, 1443 struct rte_bbdev_op_data *soft_outputs, 1444 struct rte_bbdev_dec_op *ref_op) 1445 { 1446 unsigned int i; 1447 struct rte_bbdev_op_turbo_dec *turbo_dec = &ref_op->turbo_dec; 1448 1449 for (i = 0; i < n; ++i) { 1450 if (turbo_dec->code_block_mode == RTE_BBDEV_TRANSPORT_BLOCK) { 1451 ops[i]->turbo_dec.tb_params.ea = 1452 turbo_dec->tb_params.ea; 1453 ops[i]->turbo_dec.tb_params.eb = 1454 turbo_dec->tb_params.eb; 1455 ops[i]->turbo_dec.tb_params.k_pos = 1456 turbo_dec->tb_params.k_pos; 1457 ops[i]->turbo_dec.tb_params.k_neg = 1458 turbo_dec->tb_params.k_neg; 1459 ops[i]->turbo_dec.tb_params.c = 1460 turbo_dec->tb_params.c; 1461 ops[i]->turbo_dec.tb_params.c_neg = 1462 turbo_dec->tb_params.c_neg; 1463 ops[i]->turbo_dec.tb_params.cab = 1464 turbo_dec->tb_params.cab; 1465 ops[i]->turbo_dec.tb_params.r = 1466 turbo_dec->tb_params.r; 1467 } else { 1468 ops[i]->turbo_dec.cb_params.e = turbo_dec->cb_params.e; 1469 ops[i]->turbo_dec.cb_params.k = turbo_dec->cb_params.k; 1470 } 1471 1472 ops[i]->turbo_dec.ext_scale = turbo_dec->ext_scale; 1473 ops[i]->turbo_dec.iter_max = turbo_dec->iter_max; 1474 ops[i]->turbo_dec.iter_min = turbo_dec->iter_min; 1475 ops[i]->turbo_dec.op_flags = turbo_dec->op_flags; 1476 ops[i]->turbo_dec.rv_index = turbo_dec->rv_index; 1477 ops[i]->turbo_dec.num_maps = turbo_dec->num_maps; 1478 ops[i]->turbo_dec.code_block_mode = turbo_dec->code_block_mode; 1479 1480 ops[i]->turbo_dec.hard_output = hard_outputs[start_idx + i]; 1481 ops[i]->turbo_dec.input = inputs[start_idx + i]; 1482 if (soft_outputs != NULL) 1483 ops[i]->turbo_dec.soft_output = 1484 soft_outputs[start_idx + i]; 1485 } 1486 } 1487 1488 static void 1489 copy_reference_enc_op(struct rte_bbdev_enc_op **ops, unsigned int n, 1490 unsigned int start_idx, 1491 struct rte_bbdev_op_data *inputs, 1492 struct rte_bbdev_op_data *outputs, 1493 struct rte_bbdev_enc_op *ref_op) 1494 { 1495 unsigned int i; 1496 struct rte_bbdev_op_turbo_enc *turbo_enc = &ref_op->turbo_enc; 1497 for (i = 0; i < n; ++i) { 1498 if (turbo_enc->code_block_mode == RTE_BBDEV_TRANSPORT_BLOCK) { 1499 ops[i]->turbo_enc.tb_params.ea = 1500 turbo_enc->tb_params.ea; 1501 ops[i]->turbo_enc.tb_params.eb = 1502 turbo_enc->tb_params.eb; 1503 ops[i]->turbo_enc.tb_params.k_pos = 1504 turbo_enc->tb_params.k_pos; 1505 ops[i]->turbo_enc.tb_params.k_neg = 1506 turbo_enc->tb_params.k_neg; 1507 ops[i]->turbo_enc.tb_params.c = 1508 turbo_enc->tb_params.c; 1509 ops[i]->turbo_enc.tb_params.c_neg = 1510 turbo_enc->tb_params.c_neg; 1511 ops[i]->turbo_enc.tb_params.cab = 1512 turbo_enc->tb_params.cab; 1513 ops[i]->turbo_enc.tb_params.ncb_pos = 1514 turbo_enc->tb_params.ncb_pos; 1515 ops[i]->turbo_enc.tb_params.ncb_neg = 1516 turbo_enc->tb_params.ncb_neg; 1517 ops[i]->turbo_enc.tb_params.r = turbo_enc->tb_params.r; 1518 } else { 1519 ops[i]->turbo_enc.cb_params.e = turbo_enc->cb_params.e; 1520 ops[i]->turbo_enc.cb_params.k = turbo_enc->cb_params.k; 1521 ops[i]->turbo_enc.cb_params.ncb = 1522 turbo_enc->cb_params.ncb; 1523 } 1524 ops[i]->turbo_enc.rv_index = turbo_enc->rv_index; 1525 ops[i]->turbo_enc.op_flags = turbo_enc->op_flags; 1526 ops[i]->turbo_enc.code_block_mode = turbo_enc->code_block_mode; 1527 1528 ops[i]->turbo_enc.output = outputs[start_idx + i]; 1529 ops[i]->turbo_enc.input = inputs[start_idx + i]; 1530 } 1531 } 1532 1533 1534 /* Returns a random number drawn from a normal distribution 1535 * with mean of 0 and variance of 1 1536 * Marsaglia algorithm 1537 */ 1538 static double 1539 randn(int n) 1540 { 1541 double S, Z, U1, U2, u, v, fac; 1542 1543 do { 1544 U1 = (double)rand() / RAND_MAX; 1545 U2 = (double)rand() / RAND_MAX; 1546 u = 2. * U1 - 1.; 1547 v = 2. * U2 - 1.; 1548 S = u * u + v * v; 1549 } while (S >= 1 || S == 0); 1550 fac = sqrt(-2. * log(S) / S); 1551 Z = (n % 2) ? u * fac : v * fac; 1552 return Z; 1553 } 1554 1555 static inline double 1556 maxstar(double A, double B) 1557 { 1558 if (fabs(A - B) > 5) 1559 return RTE_MAX(A, B); 1560 else 1561 return RTE_MAX(A, B) + log1p(exp(-fabs(A - B))); 1562 } 1563 1564 /* 1565 * Generate Qm LLRS for Qm==8 1566 * Modulation, AWGN and LLR estimation from max log development 1567 */ 1568 static void 1569 gen_qm8_llr(int8_t *llrs, uint32_t i, double N0, double llr_max) 1570 { 1571 int qm = 8; 1572 int qam = 256; 1573 int m, k; 1574 double I, Q, p0, p1, llr_, b[qm], log_syml_prob[qam]; 1575 /* 5.1.4 of TS38.211 */ 1576 const double symbols_I[256] = { 1577 5, 5, 7, 7, 5, 5, 7, 7, 3, 3, 1, 1, 3, 3, 1, 1, 5, 1578 5, 7, 7, 5, 5, 7, 7, 3, 3, 1, 1, 3, 3, 1, 1, 11, 1579 11, 9, 9, 11, 11, 9, 9, 13, 13, 15, 15, 13, 13, 1580 15, 15, 11, 11, 9, 9, 11, 11, 9, 9, 13, 13, 15, 1581 15, 13, 13, 15, 15, 5, 5, 7, 7, 5, 5, 7, 7, 3, 3, 1582 1, 1, 3, 3, 1, 1, 5, 5, 7, 7, 5, 5, 7, 7, 3, 3, 1, 1583 1, 3, 3, 1, 1, 11, 11, 9, 9, 11, 11, 9, 9, 13, 13, 1584 15, 15, 13, 13, 15, 15, 11, 11, 9, 9, 11, 11, 9, 9, 1585 13, 13, 15, 15, 13, 13, 15, 15, -5, -5, -7, -7, -5, 1586 -5, -7, -7, -3, -3, -1, -1, -3, -3, -1, -1, -5, -5, 1587 -7, -7, -5, -5, -7, -7, -3, -3, -1, -1, -3, -3, 1588 -1, -1, -11, -11, -9, -9, -11, -11, -9, -9, -13, 1589 -13, -15, -15, -13, -13, -15, -15, -11, -11, -9, 1590 -9, -11, -11, -9, -9, -13, -13, -15, -15, -13, 1591 -13, -15, -15, -5, -5, -7, -7, -5, -5, -7, -7, -3, 1592 -3, -1, -1, -3, -3, -1, -1, -5, -5, -7, -7, -5, -5, 1593 -7, -7, -3, -3, -1, -1, -3, -3, -1, -1, -11, -11, 1594 -9, -9, -11, -11, -9, -9, -13, -13, -15, -15, -13, 1595 -13, -15, -15, -11, -11, -9, -9, -11, -11, -9, -9, 1596 -13, -13, -15, -15, -13, -13, -15, -15}; 1597 const double symbols_Q[256] = { 1598 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 11, 1599 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9, 13, 15, 13, 1600 15, 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 1601 11, 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9, 13, 1602 15, 13, 15, -5, -7, -5, -7, -3, -1, -3, -1, -5, 1603 -7, -5, -7, -3, -1, -3, -1, -11, -9, -11, -9, -13, 1604 -15, -13, -15, -11, -9, -11, -9, -13, -15, -13, 1605 -15, -5, -7, -5, -7, -3, -1, -3, -1, -5, -7, -5, 1606 -7, -3, -1, -3, -1, -11, -9, -11, -9, -13, -15, 1607 -13, -15, -11, -9, -11, -9, -13, -15, -13, -15, 5, 1608 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 11, 1609 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9, 13, 15, 1610 13, 15, 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 1611 3, 1, 11, 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9, 1612 13, 15, 13, 15, -5, -7, -5, -7, -3, -1, -3, -1, 1613 -5, -7, -5, -7, -3, -1, -3, -1, -11, -9, -11, -9, 1614 -13, -15, -13, -15, -11, -9, -11, -9, -13, -15, 1615 -13, -15, -5, -7, -5, -7, -3, -1, -3, -1, -5, -7, 1616 -5, -7, -3, -1, -3, -1, -11, -9, -11, -9, -13, -15, 1617 -13, -15, -11, -9, -11, -9, -13, -15, -13, -15}; 1618 /* Average constellation point energy */ 1619 N0 *= 170.0; 1620 for (k = 0; k < qm; k++) 1621 b[k] = llrs[qm * i + k] < 0 ? 1.0 : 0.0; 1622 /* 5.1.4 of TS38.211 */ 1623 I = (1 - 2 * b[0]) * (8 - (1 - 2 * b[2]) * 1624 (4 - (1 - 2 * b[4]) * (2 - (1 - 2 * b[6])))); 1625 Q = (1 - 2 * b[1]) * (8 - (1 - 2 * b[3]) * 1626 (4 - (1 - 2 * b[5]) * (2 - (1 - 2 * b[7])))); 1627 /* AWGN channel */ 1628 I += sqrt(N0 / 2) * randn(0); 1629 Q += sqrt(N0 / 2) * randn(1); 1630 /* 1631 * Calculate the log of the probability that each of 1632 * the constellation points was transmitted 1633 */ 1634 for (m = 0; m < qam; m++) 1635 log_syml_prob[m] = -(pow(I - symbols_I[m], 2.0) 1636 + pow(Q - symbols_Q[m], 2.0)) / N0; 1637 /* Calculate an LLR for each of the k_64QAM bits in the set */ 1638 for (k = 0; k < qm; k++) { 1639 p0 = -999999; 1640 p1 = -999999; 1641 /* For each constellation point */ 1642 for (m = 0; m < qam; m++) { 1643 if ((m >> (qm - k - 1)) & 1) 1644 p1 = maxstar(p1, log_syml_prob[m]); 1645 else 1646 p0 = maxstar(p0, log_syml_prob[m]); 1647 } 1648 /* Calculate the LLR */ 1649 llr_ = p0 - p1; 1650 llr_ *= (1 << ldpc_llr_decimals); 1651 llr_ = round(llr_); 1652 if (llr_ > llr_max) 1653 llr_ = llr_max; 1654 if (llr_ < -llr_max) 1655 llr_ = -llr_max; 1656 llrs[qm * i + k] = (int8_t) llr_; 1657 } 1658 } 1659 1660 1661 /* 1662 * Generate Qm LLRS for Qm==6 1663 * Modulation, AWGN and LLR estimation from max log development 1664 */ 1665 static void 1666 gen_qm6_llr(int8_t *llrs, uint32_t i, double N0, double llr_max) 1667 { 1668 int qm = 6; 1669 int qam = 64; 1670 int m, k; 1671 double I, Q, p0, p1, llr_, b[qm], log_syml_prob[qam]; 1672 /* 5.1.4 of TS38.211 */ 1673 const double symbols_I[64] = { 1674 3, 3, 1, 1, 3, 3, 1, 1, 5, 5, 7, 7, 5, 5, 7, 7, 1675 3, 3, 1, 1, 3, 3, 1, 1, 5, 5, 7, 7, 5, 5, 7, 7, 1676 -3, -3, -1, -1, -3, -3, -1, -1, -5, -5, -7, -7, 1677 -5, -5, -7, -7, -3, -3, -1, -1, -3, -3, -1, -1, 1678 -5, -5, -7, -7, -5, -5, -7, -7}; 1679 const double symbols_Q[64] = { 1680 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 1681 -3, -1, -3, -1, -5, -7, -5, -7, -3, -1, -3, -1, 1682 -5, -7, -5, -7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 1683 5, 7, 5, 7, -3, -1, -3, -1, -5, -7, -5, -7, 1684 -3, -1, -3, -1, -5, -7, -5, -7}; 1685 /* Average constellation point energy */ 1686 N0 *= 42.0; 1687 for (k = 0; k < qm; k++) 1688 b[k] = llrs[qm * i + k] < 0 ? 1.0 : 0.0; 1689 /* 5.1.4 of TS38.211 */ 1690 I = (1 - 2 * b[0])*(4 - (1 - 2 * b[2]) * (2 - (1 - 2 * b[4]))); 1691 Q = (1 - 2 * b[1])*(4 - (1 - 2 * b[3]) * (2 - (1 - 2 * b[5]))); 1692 /* AWGN channel */ 1693 I += sqrt(N0 / 2) * randn(0); 1694 Q += sqrt(N0 / 2) * randn(1); 1695 /* 1696 * Calculate the log of the probability that each of 1697 * the constellation points was transmitted 1698 */ 1699 for (m = 0; m < qam; m++) 1700 log_syml_prob[m] = -(pow(I - symbols_I[m], 2.0) 1701 + pow(Q - symbols_Q[m], 2.0)) / N0; 1702 /* Calculate an LLR for each of the k_64QAM bits in the set */ 1703 for (k = 0; k < qm; k++) { 1704 p0 = -999999; 1705 p1 = -999999; 1706 /* For each constellation point */ 1707 for (m = 0; m < qam; m++) { 1708 if ((m >> (qm - k - 1)) & 1) 1709 p1 = maxstar(p1, log_syml_prob[m]); 1710 else 1711 p0 = maxstar(p0, log_syml_prob[m]); 1712 } 1713 /* Calculate the LLR */ 1714 llr_ = p0 - p1; 1715 llr_ *= (1 << ldpc_llr_decimals); 1716 llr_ = round(llr_); 1717 if (llr_ > llr_max) 1718 llr_ = llr_max; 1719 if (llr_ < -llr_max) 1720 llr_ = -llr_max; 1721 llrs[qm * i + k] = (int8_t) llr_; 1722 } 1723 } 1724 1725 /* 1726 * Generate Qm LLRS for Qm==4 1727 * Modulation, AWGN and LLR estimation from max log development 1728 */ 1729 static void 1730 gen_qm4_llr(int8_t *llrs, uint32_t i, double N0, double llr_max) 1731 { 1732 int qm = 4; 1733 int qam = 16; 1734 int m, k; 1735 double I, Q, p0, p1, llr_, b[qm], log_syml_prob[qam]; 1736 /* 5.1.4 of TS38.211 */ 1737 const double symbols_I[16] = {1, 1, 3, 3, 1, 1, 3, 3, 1738 -1, -1, -3, -3, -1, -1, -3, -3}; 1739 const double symbols_Q[16] = {1, 3, 1, 3, -1, -3, -1, -3, 1740 1, 3, 1, 3, -1, -3, -1, -3}; 1741 /* Average constellation point energy */ 1742 N0 *= 10.0; 1743 for (k = 0; k < qm; k++) 1744 b[k] = llrs[qm * i + k] < 0 ? 1.0 : 0.0; 1745 /* 5.1.4 of TS38.211 */ 1746 I = (1 - 2 * b[0]) * (2 - (1 - 2 * b[2])); 1747 Q = (1 - 2 * b[1]) * (2 - (1 - 2 * b[3])); 1748 /* AWGN channel */ 1749 I += sqrt(N0 / 2) * randn(0); 1750 Q += sqrt(N0 / 2) * randn(1); 1751 /* 1752 * Calculate the log of the probability that each of 1753 * the constellation points was transmitted 1754 */ 1755 for (m = 0; m < qam; m++) 1756 log_syml_prob[m] = -(pow(I - symbols_I[m], 2.0) 1757 + pow(Q - symbols_Q[m], 2.0)) / N0; 1758 /* Calculate an LLR for each of the k_64QAM bits in the set */ 1759 for (k = 0; k < qm; k++) { 1760 p0 = -999999; 1761 p1 = -999999; 1762 /* For each constellation point */ 1763 for (m = 0; m < qam; m++) { 1764 if ((m >> (qm - k - 1)) & 1) 1765 p1 = maxstar(p1, log_syml_prob[m]); 1766 else 1767 p0 = maxstar(p0, log_syml_prob[m]); 1768 } 1769 /* Calculate the LLR */ 1770 llr_ = p0 - p1; 1771 llr_ *= (1 << ldpc_llr_decimals); 1772 llr_ = round(llr_); 1773 if (llr_ > llr_max) 1774 llr_ = llr_max; 1775 if (llr_ < -llr_max) 1776 llr_ = -llr_max; 1777 llrs[qm * i + k] = (int8_t) llr_; 1778 } 1779 } 1780 1781 static void 1782 gen_qm2_llr(int8_t *llrs, uint32_t j, double N0, double llr_max) 1783 { 1784 double b, b1, n; 1785 double coeff = 2.0 * sqrt(N0); 1786 1787 /* Ignore in vectors rare quasi null LLRs not to be saturated */ 1788 if (llrs[j] < 8 && llrs[j] > -8) 1789 return; 1790 1791 /* Note don't change sign here */ 1792 n = randn(j % 2); 1793 b1 = ((llrs[j] > 0 ? 2.0 : -2.0) 1794 + coeff * n) / N0; 1795 b = b1 * (1 << ldpc_llr_decimals); 1796 b = round(b); 1797 if (b > llr_max) 1798 b = llr_max; 1799 if (b < -llr_max) 1800 b = -llr_max; 1801 llrs[j] = (int8_t) b; 1802 } 1803 1804 /* Simple LLR generation assuming AWGN and QPSK */ 1805 static void 1806 gen_turbo_llr(int8_t *llrs, uint32_t j, double N0, double llr_max) 1807 { 1808 double b, b1, n; 1809 double coeff = 2.0 * sqrt(N0); 1810 1811 /* Ignore in vectors null LLRs not to be saturated */ 1812 if (llrs[j] == 0) 1813 return; 1814 1815 /* Note don't change sign here */ 1816 n = randn(j % 2); 1817 b1 = ((llrs[j] > 0 ? 2.0 : -2.0) 1818 + coeff * n) / N0; 1819 b = b1 * (1 << 4); 1820 b = round(b); 1821 if (b > llr_max) 1822 b = llr_max; 1823 if (b < -llr_max) 1824 b = -llr_max; 1825 llrs[j] = (int8_t) b; 1826 } 1827 1828 /* Generate LLR for a given SNR */ 1829 static void 1830 generate_llr_input(uint16_t n, struct rte_bbdev_op_data *inputs, 1831 struct rte_bbdev_dec_op *ref_op) 1832 { 1833 struct rte_mbuf *m; 1834 uint16_t qm; 1835 uint32_t i, j, e, range; 1836 double N0, llr_max; 1837 1838 e = ref_op->ldpc_dec.cb_params.e; 1839 qm = ref_op->ldpc_dec.q_m; 1840 llr_max = (1 << (ldpc_llr_size - 1)) - 1; 1841 range = e / qm; 1842 N0 = 1.0 / pow(10.0, get_snr() / 10.0); 1843 1844 for (i = 0; i < n; ++i) { 1845 m = inputs[i].data; 1846 int8_t *llrs = rte_pktmbuf_mtod_offset(m, int8_t *, 0); 1847 if (qm == 8) { 1848 for (j = 0; j < range; ++j) 1849 gen_qm8_llr(llrs, j, N0, llr_max); 1850 } else if (qm == 6) { 1851 for (j = 0; j < range; ++j) 1852 gen_qm6_llr(llrs, j, N0, llr_max); 1853 } else if (qm == 4) { 1854 for (j = 0; j < range; ++j) 1855 gen_qm4_llr(llrs, j, N0, llr_max); 1856 } else { 1857 for (j = 0; j < e; ++j) 1858 gen_qm2_llr(llrs, j, N0, llr_max); 1859 } 1860 } 1861 } 1862 1863 /* Generate LLR for turbo decoder for a given SNR */ 1864 static void 1865 generate_turbo_llr_input(uint16_t n, struct rte_bbdev_op_data *inputs, 1866 struct rte_bbdev_dec_op *ref_op) 1867 { 1868 struct rte_mbuf *m; 1869 uint32_t i, j, range; 1870 double N0, llr_max; 1871 1872 llr_max = 127; 1873 range = ref_op->turbo_dec.input.length; 1874 N0 = 1.0 / pow(10.0, get_snr() / 10.0); 1875 1876 if (range > inputs[0].data->data_len) { 1877 printf("Warning: Limiting LLR generation to first segment (%d from %d)\n", 1878 inputs[0].data->data_len, range); 1879 range = inputs[0].data->data_len; 1880 } 1881 1882 for (i = 0; i < n; ++i) { 1883 m = inputs[i].data; 1884 int8_t *llrs = rte_pktmbuf_mtod_offset(m, int8_t *, 0); 1885 for (j = 0; j < range; ++j) 1886 gen_turbo_llr(llrs, j, N0, llr_max); 1887 } 1888 } 1889 1890 static void 1891 copy_reference_ldpc_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n, 1892 unsigned int start_idx, 1893 struct rte_bbdev_op_data *inputs, 1894 struct rte_bbdev_op_data *hard_outputs, 1895 struct rte_bbdev_op_data *soft_outputs, 1896 struct rte_bbdev_op_data *harq_inputs, 1897 struct rte_bbdev_op_data *harq_outputs, 1898 struct rte_bbdev_dec_op *ref_op) 1899 { 1900 unsigned int i; 1901 struct rte_bbdev_op_ldpc_dec *ldpc_dec = &ref_op->ldpc_dec; 1902 1903 for (i = 0; i < n; ++i) { 1904 if (ldpc_dec->code_block_mode == RTE_BBDEV_TRANSPORT_BLOCK) { 1905 ops[i]->ldpc_dec.tb_params.ea = 1906 ldpc_dec->tb_params.ea; 1907 ops[i]->ldpc_dec.tb_params.eb = 1908 ldpc_dec->tb_params.eb; 1909 ops[i]->ldpc_dec.tb_params.c = 1910 ldpc_dec->tb_params.c; 1911 ops[i]->ldpc_dec.tb_params.cab = 1912 ldpc_dec->tb_params.cab; 1913 ops[i]->ldpc_dec.tb_params.r = 1914 ldpc_dec->tb_params.r; 1915 } else { 1916 ops[i]->ldpc_dec.cb_params.e = ldpc_dec->cb_params.e; 1917 } 1918 1919 ops[i]->ldpc_dec.basegraph = ldpc_dec->basegraph; 1920 ops[i]->ldpc_dec.z_c = ldpc_dec->z_c; 1921 ops[i]->ldpc_dec.q_m = ldpc_dec->q_m; 1922 ops[i]->ldpc_dec.n_filler = ldpc_dec->n_filler; 1923 ops[i]->ldpc_dec.n_cb = ldpc_dec->n_cb; 1924 ops[i]->ldpc_dec.iter_max = ldpc_dec->iter_max; 1925 ops[i]->ldpc_dec.rv_index = ldpc_dec->rv_index; 1926 ops[i]->ldpc_dec.op_flags = ldpc_dec->op_flags; 1927 ops[i]->ldpc_dec.code_block_mode = ldpc_dec->code_block_mode; 1928 1929 if (hard_outputs != NULL) 1930 ops[i]->ldpc_dec.hard_output = 1931 hard_outputs[start_idx + i]; 1932 if (inputs != NULL) 1933 ops[i]->ldpc_dec.input = 1934 inputs[start_idx + i]; 1935 if (soft_outputs != NULL) 1936 ops[i]->ldpc_dec.soft_output = 1937 soft_outputs[start_idx + i]; 1938 if (harq_inputs != NULL) 1939 ops[i]->ldpc_dec.harq_combined_input = 1940 harq_inputs[start_idx + i]; 1941 if (harq_outputs != NULL) 1942 ops[i]->ldpc_dec.harq_combined_output = 1943 harq_outputs[start_idx + i]; 1944 } 1945 } 1946 1947 1948 static void 1949 copy_reference_ldpc_enc_op(struct rte_bbdev_enc_op **ops, unsigned int n, 1950 unsigned int start_idx, 1951 struct rte_bbdev_op_data *inputs, 1952 struct rte_bbdev_op_data *outputs, 1953 struct rte_bbdev_enc_op *ref_op) 1954 { 1955 unsigned int i; 1956 struct rte_bbdev_op_ldpc_enc *ldpc_enc = &ref_op->ldpc_enc; 1957 for (i = 0; i < n; ++i) { 1958 if (ldpc_enc->code_block_mode == RTE_BBDEV_TRANSPORT_BLOCK) { 1959 ops[i]->ldpc_enc.tb_params.ea = ldpc_enc->tb_params.ea; 1960 ops[i]->ldpc_enc.tb_params.eb = ldpc_enc->tb_params.eb; 1961 ops[i]->ldpc_enc.tb_params.cab = 1962 ldpc_enc->tb_params.cab; 1963 ops[i]->ldpc_enc.tb_params.c = ldpc_enc->tb_params.c; 1964 ops[i]->ldpc_enc.tb_params.r = ldpc_enc->tb_params.r; 1965 } else { 1966 ops[i]->ldpc_enc.cb_params.e = ldpc_enc->cb_params.e; 1967 } 1968 ops[i]->ldpc_enc.basegraph = ldpc_enc->basegraph; 1969 ops[i]->ldpc_enc.z_c = ldpc_enc->z_c; 1970 ops[i]->ldpc_enc.q_m = ldpc_enc->q_m; 1971 ops[i]->ldpc_enc.n_filler = ldpc_enc->n_filler; 1972 ops[i]->ldpc_enc.n_cb = ldpc_enc->n_cb; 1973 ops[i]->ldpc_enc.rv_index = ldpc_enc->rv_index; 1974 ops[i]->ldpc_enc.op_flags = ldpc_enc->op_flags; 1975 ops[i]->ldpc_enc.code_block_mode = ldpc_enc->code_block_mode; 1976 ops[i]->ldpc_enc.output = outputs[start_idx + i]; 1977 ops[i]->ldpc_enc.input = inputs[start_idx + i]; 1978 } 1979 } 1980 1981 static void 1982 copy_reference_fft_op(struct rte_bbdev_fft_op **ops, unsigned int n, 1983 unsigned int start_idx, struct rte_bbdev_op_data *inputs, 1984 struct rte_bbdev_op_data *outputs, struct rte_bbdev_op_data *pwrouts, 1985 struct rte_bbdev_op_data *win_inputs, struct rte_bbdev_fft_op *ref_op) 1986 { 1987 unsigned int i, j; 1988 struct rte_bbdev_op_fft *fft = &ref_op->fft; 1989 for (i = 0; i < n; i++) { 1990 ops[i]->fft.input_sequence_size = fft->input_sequence_size; 1991 ops[i]->fft.input_leading_padding = fft->input_leading_padding; 1992 ops[i]->fft.output_sequence_size = fft->output_sequence_size; 1993 ops[i]->fft.output_leading_depadding = 1994 fft->output_leading_depadding; 1995 for (j = 0; j < RTE_BBDEV_MAX_CS_2; j++) 1996 ops[i]->fft.window_index[j] = fft->window_index[j]; 1997 for (j = 0; j < RTE_BBDEV_MAX_CS; j++) { 1998 ops[i]->fft.cs_theta_0[j] = fft->cs_theta_0[j]; 1999 ops[i]->fft.cs_theta_d[j] = fft->cs_theta_d[j]; 2000 ops[i]->fft.time_offset[j] = fft->time_offset[j]; 2001 } 2002 ops[i]->fft.cs_bitmap = fft->cs_bitmap; 2003 ops[i]->fft.num_antennas_log2 = fft->num_antennas_log2; 2004 ops[i]->fft.idft_log2 = fft->idft_log2; 2005 ops[i]->fft.dft_log2 = fft->dft_log2; 2006 ops[i]->fft.cs_time_adjustment = fft->cs_time_adjustment; 2007 ops[i]->fft.idft_shift = fft->idft_shift; 2008 ops[i]->fft.dft_shift = fft->dft_shift; 2009 ops[i]->fft.ncs_reciprocal = fft->ncs_reciprocal; 2010 ops[i]->fft.power_shift = fft->power_shift; 2011 ops[i]->fft.fp16_exp_adjust = fft->fp16_exp_adjust; 2012 ops[i]->fft.output_depadded_size = fft->output_depadded_size; 2013 ops[i]->fft.freq_resample_mode = fft->freq_resample_mode; 2014 ops[i]->fft.base_output = outputs[start_idx + i]; 2015 ops[i]->fft.base_input = inputs[start_idx + i]; 2016 if (win_inputs != NULL) 2017 ops[i]->fft.dewindowing_input = win_inputs[start_idx + i]; 2018 if (pwrouts != NULL) 2019 ops[i]->fft.power_meas_output = pwrouts[start_idx + i]; 2020 ops[i]->fft.op_flags = fft->op_flags; 2021 } 2022 } 2023 2024 static void 2025 copy_reference_mldts_op(struct rte_bbdev_mldts_op **ops, unsigned int n, 2026 unsigned int start_idx, 2027 struct rte_bbdev_op_data *q_inputs, 2028 struct rte_bbdev_op_data *r_inputs, 2029 struct rte_bbdev_op_data *outputs, 2030 struct rte_bbdev_mldts_op *ref_op) 2031 { 2032 unsigned int i, j; 2033 struct rte_bbdev_op_mldts *mldts = &ref_op->mldts; 2034 for (i = 0; i < n; i++) { 2035 ops[i]->mldts.c_rep = mldts->c_rep; 2036 ops[i]->mldts.num_layers = mldts->num_layers; 2037 ops[i]->mldts.num_rbs = mldts->num_rbs; 2038 ops[i]->mldts.op_flags = mldts->op_flags; 2039 for (j = 0; j < RTE_BBDEV_MAX_MLD_LAYERS; j++) 2040 ops[i]->mldts.q_m[j] = mldts->q_m[j]; 2041 ops[i]->mldts.r_rep = mldts->r_rep; 2042 ops[i]->mldts.c_rep = mldts->c_rep; 2043 ops[i]->mldts.r_input = r_inputs[start_idx + i]; 2044 ops[i]->mldts.qhy_input = q_inputs[start_idx + i]; 2045 ops[i]->mldts.output = outputs[start_idx + i]; 2046 } 2047 } 2048 2049 static int 2050 check_dec_status_and_ordering(struct rte_bbdev_dec_op *op, 2051 unsigned int order_idx, const int expected_status) 2052 { 2053 int status = op->status; 2054 /* ignore parity mismatch false alarms for long iterations */ 2055 if (get_iter_max() >= 10) { 2056 if (!(expected_status & (1 << RTE_BBDEV_SYNDROME_ERROR)) && 2057 (status & (1 << RTE_BBDEV_SYNDROME_ERROR))) { 2058 printf("WARNING: Ignore Syndrome Check mismatch\n"); 2059 status -= (1 << RTE_BBDEV_SYNDROME_ERROR); 2060 } 2061 if ((expected_status & (1 << RTE_BBDEV_SYNDROME_ERROR)) && 2062 !(status & (1 << RTE_BBDEV_SYNDROME_ERROR))) { 2063 printf("WARNING: Ignore Syndrome Check mismatch\n"); 2064 status += (1 << RTE_BBDEV_SYNDROME_ERROR); 2065 } 2066 } 2067 2068 TEST_ASSERT(status == expected_status, 2069 "op_status (%d) != expected_status (%d)", 2070 op->status, expected_status); 2071 2072 TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data, 2073 "Ordering error, expected %p, got %p", 2074 (void *)(uintptr_t)order_idx, op->opaque_data); 2075 2076 return TEST_SUCCESS; 2077 } 2078 2079 static int 2080 check_enc_status_and_ordering(struct rte_bbdev_enc_op *op, 2081 unsigned int order_idx, const int expected_status) 2082 { 2083 TEST_ASSERT(op->status == expected_status, 2084 "op_status (%d) != expected_status (%d)", 2085 op->status, expected_status); 2086 2087 TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data, 2088 "Ordering error, expected %p, got %p", 2089 (void *)(uintptr_t)order_idx, op->opaque_data); 2090 2091 return TEST_SUCCESS; 2092 } 2093 2094 static int 2095 check_fft_status_and_ordering(struct rte_bbdev_fft_op *op, 2096 unsigned int order_idx, const int expected_status) 2097 { 2098 TEST_ASSERT(op->status == expected_status, 2099 "op_status (%d) != expected_status (%d)", 2100 op->status, expected_status); 2101 2102 TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data, 2103 "Ordering error, expected %p, got %p", 2104 (void *)(uintptr_t)order_idx, op->opaque_data); 2105 2106 return TEST_SUCCESS; 2107 } 2108 2109 static int 2110 check_mldts_status_and_ordering(struct rte_bbdev_mldts_op *op, 2111 unsigned int order_idx, const int expected_status) 2112 { 2113 TEST_ASSERT(op->status == expected_status, 2114 "op_status (%d) != expected_status (%d)", 2115 op->status, expected_status); 2116 2117 TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data, 2118 "Ordering error, expected %p, got %p", 2119 (void *)(uintptr_t)order_idx, op->opaque_data); 2120 2121 return TEST_SUCCESS; 2122 } 2123 2124 static inline int 2125 validate_op_chain(struct rte_bbdev_op_data *op, 2126 struct op_data_entries *orig_op) 2127 { 2128 uint8_t i; 2129 struct rte_mbuf *m = op->data; 2130 uint8_t nb_dst_segments = orig_op->nb_segments; 2131 uint32_t total_data_size = 0; 2132 bool ignore_mbuf = false; /* ignore mbuf limitations */ 2133 2134 TEST_ASSERT(nb_dst_segments == m->nb_segs, 2135 "Number of segments differ in original (%u) and filled (%u) op", 2136 nb_dst_segments, m->nb_segs); 2137 2138 /* Validate each mbuf segment length */ 2139 for (i = 0; i < nb_dst_segments; ++i) { 2140 /* Apply offset to the first mbuf segment */ 2141 uint16_t offset = (i == 0) ? op->offset : 0; 2142 uint16_t data_len = rte_pktmbuf_data_len(m) - offset; 2143 total_data_size += orig_op->segments[i].length; 2144 2145 if ((orig_op->segments[i].length + RTE_PKTMBUF_HEADROOM) 2146 > RTE_BBDEV_LDPC_E_MAX_MBUF) 2147 ignore_mbuf = true; 2148 if (!ignore_mbuf) 2149 TEST_ASSERT(orig_op->segments[i].length == data_len, 2150 "Length of segment differ in original (%u) and filled (%u) op", 2151 orig_op->segments[i].length, data_len); 2152 TEST_ASSERT_BUFFERS_ARE_EQUAL(orig_op->segments[i].addr, 2153 rte_pktmbuf_mtod_offset(m, uint32_t *, offset), 2154 orig_op->segments[i].length, 2155 "Output buffers (CB=%u) are not equal", i); 2156 m = m->next; 2157 } 2158 2159 /* Validate total mbuf pkt length */ 2160 uint32_t pkt_len = rte_pktmbuf_pkt_len(op->data) - op->offset; 2161 if (!ignore_mbuf) 2162 TEST_ASSERT(total_data_size == pkt_len, 2163 "Length of data differ in original (%u) and filled (%u) op", 2164 total_data_size, pkt_len); 2165 2166 return TEST_SUCCESS; 2167 } 2168 2169 /* 2170 * Compute K0 for a given configuration for HARQ output length computation 2171 * As per definition in 3GPP 38.212 Table 5.4.2.1-2 2172 */ 2173 static inline uint16_t 2174 get_k0(uint16_t n_cb, uint16_t z_c, uint8_t bg, uint8_t rv_index) 2175 { 2176 if (rv_index == 0) 2177 return 0; 2178 uint16_t n = (bg == 1 ? N_ZC_1 : N_ZC_2) * z_c; 2179 if (n_cb == n) { 2180 if (rv_index == 1) 2181 return (bg == 1 ? K0_1_1 : K0_1_2) * z_c; 2182 else if (rv_index == 2) 2183 return (bg == 1 ? K0_2_1 : K0_2_2) * z_c; 2184 else 2185 return (bg == 1 ? K0_3_1 : K0_3_2) * z_c; 2186 } 2187 /* LBRM case - includes a division by N */ 2188 if (rv_index == 1) 2189 return (((bg == 1 ? K0_1_1 : K0_1_2) * n_cb) 2190 / n) * z_c; 2191 else if (rv_index == 2) 2192 return (((bg == 1 ? K0_2_1 : K0_2_2) * n_cb) 2193 / n) * z_c; 2194 else 2195 return (((bg == 1 ? K0_3_1 : K0_3_2) * n_cb) 2196 / n) * z_c; 2197 } 2198 2199 /* HARQ output length including the Filler bits */ 2200 static inline uint16_t 2201 compute_harq_len(struct rte_bbdev_op_ldpc_dec *ops_ld) 2202 { 2203 uint16_t k0 = 0; 2204 uint8_t max_rv = (ops_ld->rv_index == 1) ? 3 : ops_ld->rv_index; 2205 k0 = get_k0(ops_ld->n_cb, ops_ld->z_c, ops_ld->basegraph, max_rv); 2206 /* Compute RM out size and number of rows */ 2207 uint16_t parity_offset = (ops_ld->basegraph == 1 ? 20 : 8) 2208 * ops_ld->z_c - ops_ld->n_filler; 2209 uint16_t deRmOutSize = RTE_MIN( 2210 k0 + ops_ld->cb_params.e + 2211 ((k0 > parity_offset) ? 2212 0 : ops_ld->n_filler), 2213 ops_ld->n_cb); 2214 uint16_t numRows = ((deRmOutSize + ops_ld->z_c - 1) 2215 / ops_ld->z_c); 2216 uint16_t harq_output_len = numRows * ops_ld->z_c; 2217 return harq_output_len; 2218 } 2219 2220 static inline int 2221 validate_op_harq_chain(struct rte_bbdev_op_data *op, 2222 struct op_data_entries *orig_op, 2223 struct rte_bbdev_op_ldpc_dec *ops_ld) 2224 { 2225 uint8_t i; 2226 uint32_t j, jj, k; 2227 struct rte_mbuf *m = op->data; 2228 uint8_t nb_dst_segments = orig_op->nb_segments; 2229 uint32_t total_data_size = 0; 2230 int8_t *harq_orig, *harq_out, abs_harq_origin; 2231 uint32_t byte_error = 0, cum_error = 0, error; 2232 int16_t llr_max = (1 << (ldpc_llr_size - ldpc_llr_decimals)) - 1; 2233 int16_t llr_max_pre_scaling = (1 << (ldpc_llr_size - 1)) - 1; 2234 uint16_t parity_offset; 2235 2236 TEST_ASSERT(nb_dst_segments == m->nb_segs, 2237 "Number of segments differ in original (%u) and filled (%u) op", 2238 nb_dst_segments, m->nb_segs); 2239 2240 /* Validate each mbuf segment length */ 2241 for (i = 0; i < nb_dst_segments; ++i) { 2242 /* Apply offset to the first mbuf segment */ 2243 uint16_t offset = (i == 0) ? op->offset : 0; 2244 uint16_t data_len = rte_pktmbuf_data_len(m) - offset; 2245 total_data_size += orig_op->segments[i].length; 2246 2247 TEST_ASSERT(orig_op->segments[i].length < (uint32_t)(data_len + HARQ_MEM_TOLERANCE), 2248 "Length of segment differ in original (%u) and filled (%u) op", 2249 orig_op->segments[i].length, data_len); 2250 harq_orig = (int8_t *) orig_op->segments[i].addr; 2251 harq_out = rte_pktmbuf_mtod_offset(m, int8_t *, offset); 2252 2253 /* Cannot compare HARQ output data for such cases */ 2254 if ((ldpc_llr_decimals > 1) && ((ops_ld->op_flags & RTE_BBDEV_LDPC_LLR_COMPRESSION) 2255 || (ops_ld->op_flags & RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION) 2256 || (ops_ld->op_flags & RTE_BBDEV_LDPC_HARQ_4BIT_COMPRESSION))) 2257 break; 2258 2259 if (!(ldpc_cap_flags & 2260 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_FILLERS 2261 ) || (ops_ld->op_flags & 2262 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)) { 2263 data_len -= ops_ld->z_c; 2264 parity_offset = data_len; 2265 } else { 2266 /* Compute RM out size and number of rows */ 2267 parity_offset = (ops_ld->basegraph == 1 ? 20 : 8) 2268 * ops_ld->z_c - ops_ld->n_filler; 2269 uint16_t deRmOutSize = compute_harq_len(ops_ld) - 2270 ops_ld->n_filler; 2271 if (data_len > deRmOutSize) 2272 data_len = deRmOutSize; 2273 } 2274 if (data_len > orig_op->segments[i].length) 2275 data_len = orig_op->segments[i].length; 2276 /* 2277 * HARQ output can have minor differences 2278 * due to integer representation and related scaling 2279 */ 2280 for (j = 0, jj = 0; j < data_len; j++, jj++) { 2281 if (j == parity_offset) { 2282 /* Special Handling of the filler bits */ 2283 for (k = 0; k < ops_ld->n_filler; k++) { 2284 if (harq_out[jj] != 2285 llr_max_pre_scaling) { 2286 printf("HARQ Filler issue %d: %d %d\n", 2287 jj, harq_out[jj], 2288 llr_max); 2289 byte_error++; 2290 } 2291 jj++; 2292 } 2293 } 2294 if (!(ops_ld->op_flags & 2295 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)) { 2296 if (ldpc_llr_decimals > 1) 2297 harq_out[jj] = (harq_out[jj] + 1) 2298 >> (ldpc_llr_decimals - 1); 2299 /* Saturated to S7 */ 2300 if (harq_orig[j] > llr_max) 2301 harq_orig[j] = llr_max; 2302 if (harq_orig[j] < -llr_max) 2303 harq_orig[j] = -llr_max; 2304 } 2305 if (harq_orig[j] != harq_out[jj]) { 2306 error = (harq_orig[j] > harq_out[jj]) ? 2307 harq_orig[j] - harq_out[jj] : 2308 harq_out[jj] - harq_orig[j]; 2309 abs_harq_origin = harq_orig[j] > 0 ? 2310 harq_orig[j] : 2311 -harq_orig[j]; 2312 /* Residual quantization error */ 2313 if ((error > 8 && (abs_harq_origin < 2314 (llr_max - 16))) || 2315 (error > 16)) { 2316 /* 2317 printf("HARQ mismatch %d: exp %d act %d => %d\n", 2318 j, harq_orig[j], 2319 harq_out[jj], error); 2320 */ 2321 byte_error++; 2322 cum_error += error; 2323 } 2324 } 2325 } 2326 m = m->next; 2327 } 2328 2329 if (byte_error) 2330 TEST_ASSERT(byte_error <= 1, 2331 "HARQ output mismatch (%d) %d", 2332 byte_error, cum_error); 2333 2334 /* Validate total mbuf pkt length */ 2335 uint32_t pkt_len = rte_pktmbuf_pkt_len(op->data) - op->offset; 2336 TEST_ASSERT(total_data_size < pkt_len + HARQ_MEM_TOLERANCE, 2337 "Length of data differ in original (%u) and filled (%u) op", 2338 total_data_size, pkt_len); 2339 2340 return TEST_SUCCESS; 2341 } 2342 2343 2344 static inline int 2345 validate_op_so_chain(struct rte_bbdev_op_data *op, 2346 struct op_data_entries *orig_op) 2347 { 2348 struct rte_mbuf *m = op->data; 2349 uint8_t i, nb_dst_segments = orig_op->nb_segments; 2350 uint32_t j, jj; 2351 int8_t *so_orig, *so_out; 2352 uint32_t byte_error = 0, error, margin_error = 0; 2353 2354 TEST_ASSERT(nb_dst_segments == m->nb_segs, 2355 "Number of segments differ in original (%u) and filled (%u) op", 2356 nb_dst_segments, m->nb_segs); 2357 2358 /* Validate each mbuf segment length. */ 2359 for (i = 0; i < nb_dst_segments; ++i) { 2360 /* Apply offset to the first mbuf segment. */ 2361 uint16_t offset = (i == 0) ? op->offset : 0; 2362 uint16_t data_len = rte_pktmbuf_data_len(m) - offset; 2363 2364 TEST_ASSERT(orig_op->segments[i].length == data_len, 2365 "Length of segment differ in original (%u) and filled (%u) op", 2366 orig_op->segments[i].length, data_len); 2367 so_orig = (int8_t *) orig_op->segments[i].addr; 2368 so_out = rte_pktmbuf_mtod_offset(m, int8_t *, offset); 2369 margin_error += data_len / 8; /* Allow for few % errors. */ 2370 2371 /* SO output can have minor differences due to algorithm variations. */ 2372 for (j = 0, jj = 0; j < data_len; j++, jj++) { 2373 if (so_orig[j] != so_out[jj]) { 2374 error = (so_orig[j] > so_out[jj]) ? so_orig[j] - so_out[jj] : 2375 so_out[jj] - so_orig[j]; 2376 /* Residual quantization error. */ 2377 if (error > 32) { 2378 printf("Warning: Soft mismatch %d: exp %d act %d => %d\n", 2379 j, so_orig[j], so_out[jj], error); 2380 byte_error++; 2381 } 2382 } 2383 } 2384 m = m->next; 2385 } 2386 2387 if (byte_error > margin_error) 2388 TEST_ASSERT(byte_error <= 1, "Soft output mismatch (%d) %d", 2389 byte_error, margin_error); 2390 2391 return TEST_SUCCESS; 2392 } 2393 2394 static int 2395 validate_dec_op(struct rte_bbdev_dec_op **ops, const uint16_t n, 2396 struct rte_bbdev_dec_op *ref_op) 2397 { 2398 unsigned int i; 2399 int ret; 2400 struct op_data_entries *hard_data_orig = 2401 &test_vector.entries[DATA_HARD_OUTPUT]; 2402 struct op_data_entries *soft_data_orig = 2403 &test_vector.entries[DATA_SOFT_OUTPUT]; 2404 struct rte_bbdev_op_turbo_dec *ops_td; 2405 struct rte_bbdev_op_data *hard_output; 2406 struct rte_bbdev_op_data *soft_output; 2407 2408 for (i = 0; i < n; ++i) { 2409 ops_td = &ops[i]->turbo_dec; 2410 hard_output = &ops_td->hard_output; 2411 soft_output = &ops_td->soft_output; 2412 2413 ret = check_dec_status_and_ordering(ops[i], i, ref_op->status); 2414 TEST_ASSERT_SUCCESS(ret, 2415 "Checking status and ordering for decoder failed"); 2416 2417 TEST_ASSERT_SUCCESS(validate_op_chain(hard_output, 2418 hard_data_orig), 2419 "Hard output buffers (CB=%u) are not equal", 2420 i); 2421 2422 if (ref_op->turbo_dec.op_flags & RTE_BBDEV_TURBO_SOFT_OUTPUT) 2423 TEST_ASSERT_SUCCESS(validate_op_so_chain(soft_output, 2424 soft_data_orig), 2425 "Soft output buffers (CB=%u) are not equal", 2426 i); 2427 } 2428 2429 return TEST_SUCCESS; 2430 } 2431 2432 /* Check Number of code blocks errors */ 2433 static int 2434 validate_ldpc_bler(struct rte_bbdev_dec_op **ops, const uint16_t n) 2435 { 2436 unsigned int i; 2437 struct op_data_entries *hard_data_orig = 2438 &test_vector.entries[DATA_HARD_OUTPUT]; 2439 struct rte_bbdev_op_ldpc_dec *ops_td; 2440 struct rte_bbdev_op_data *hard_output; 2441 int errors = 0; 2442 struct rte_mbuf *m; 2443 2444 for (i = 0; i < n; ++i) { 2445 ops_td = &ops[i]->ldpc_dec; 2446 hard_output = &ops_td->hard_output; 2447 m = hard_output->data; 2448 if (memcmp(rte_pktmbuf_mtod_offset(m, uint32_t *, 0), 2449 hard_data_orig->segments[0].addr, 2450 hard_data_orig->segments[0].length)) 2451 errors++; 2452 } 2453 return errors; 2454 } 2455 2456 /* Check Number of code blocks errors */ 2457 static int 2458 validate_turbo_bler(struct rte_bbdev_dec_op **ops, const uint16_t n) 2459 { 2460 unsigned int i; 2461 struct op_data_entries *hard_data_orig = &test_vector.entries[DATA_HARD_OUTPUT]; 2462 struct rte_bbdev_op_turbo_dec *ops_td; 2463 struct rte_bbdev_op_data *hard_output; 2464 int errors = 0; 2465 struct rte_mbuf *m; 2466 2467 for (i = 0; i < n; ++i) { 2468 ops_td = &ops[i]->turbo_dec; 2469 hard_output = &ops_td->hard_output; 2470 m = hard_output->data; 2471 if (memcmp(rte_pktmbuf_mtod_offset(m, uint32_t *, 0), 2472 hard_data_orig->segments[0].addr, 2473 hard_data_orig->segments[0].length)) 2474 errors++; 2475 } 2476 return errors; 2477 } 2478 2479 2480 static int 2481 validate_ldpc_dec_op(struct rte_bbdev_dec_op **ops, const uint16_t n, 2482 struct rte_bbdev_dec_op *ref_op, const int vector_mask) 2483 { 2484 unsigned int i; 2485 int ret; 2486 struct op_data_entries *hard_data_orig = 2487 &test_vector.entries[DATA_HARD_OUTPUT]; 2488 struct op_data_entries *soft_data_orig = 2489 &test_vector.entries[DATA_SOFT_OUTPUT]; 2490 struct op_data_entries *harq_data_orig = 2491 &test_vector.entries[DATA_HARQ_OUTPUT]; 2492 struct rte_bbdev_op_ldpc_dec *ops_td; 2493 struct rte_bbdev_op_data *hard_output; 2494 struct rte_bbdev_op_data *harq_output; 2495 struct rte_bbdev_op_data *soft_output; 2496 struct rte_bbdev_op_ldpc_dec *ref_td = &ref_op->ldpc_dec; 2497 2498 for (i = 0; i < n; ++i) { 2499 ops_td = &ops[i]->ldpc_dec; 2500 hard_output = &ops_td->hard_output; 2501 harq_output = &ops_td->harq_combined_output; 2502 soft_output = &ops_td->soft_output; 2503 2504 ret = check_dec_status_and_ordering(ops[i], i, ref_op->status); 2505 TEST_ASSERT_SUCCESS(ret, 2506 "Checking status and ordering for decoder failed"); 2507 if (vector_mask & TEST_BBDEV_VF_EXPECTED_ITER_COUNT) 2508 TEST_ASSERT(ops_td->iter_count <= ref_td->iter_count, 2509 "Returned iter_count (%d) > expected iter_count (%d)", 2510 ops_td->iter_count, ref_td->iter_count); 2511 /* 2512 * We can ignore output data when the decoding failed to 2513 * converge or for loop-back cases 2514 */ 2515 if (!check_bit(ops[i]->ldpc_dec.op_flags, 2516 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK 2517 ) && ( 2518 ops[i]->status & (1 << RTE_BBDEV_SYNDROME_ERROR 2519 )) == 0) 2520 TEST_ASSERT_SUCCESS(validate_op_chain(hard_output, 2521 hard_data_orig), 2522 "Hard output buffers (CB=%u) are not equal", 2523 i); 2524 2525 if (ref_op->ldpc_dec.op_flags & RTE_BBDEV_LDPC_SOFT_OUT_ENABLE) 2526 TEST_ASSERT_SUCCESS(validate_op_so_chain(soft_output, 2527 soft_data_orig), 2528 "Soft output buffers (CB=%u) are not equal", 2529 i); 2530 if (ref_op->ldpc_dec.op_flags & 2531 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE) { 2532 TEST_ASSERT_SUCCESS(validate_op_harq_chain(harq_output, 2533 harq_data_orig, ops_td), 2534 "HARQ output buffers (CB=%u) are not equal", 2535 i); 2536 } 2537 if (ref_op->ldpc_dec.op_flags & 2538 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK) 2539 TEST_ASSERT_SUCCESS(validate_op_harq_chain(harq_output, 2540 harq_data_orig, ops_td), 2541 "HARQ output buffers (CB=%u) are not equal", 2542 i); 2543 2544 } 2545 2546 return TEST_SUCCESS; 2547 } 2548 2549 2550 static int 2551 validate_enc_op(struct rte_bbdev_enc_op **ops, const uint16_t n, 2552 struct rte_bbdev_enc_op *ref_op) 2553 { 2554 unsigned int i; 2555 int ret; 2556 struct op_data_entries *hard_data_orig = 2557 &test_vector.entries[DATA_HARD_OUTPUT]; 2558 2559 for (i = 0; i < n; ++i) { 2560 ret = check_enc_status_and_ordering(ops[i], i, ref_op->status); 2561 TEST_ASSERT_SUCCESS(ret, 2562 "Checking status and ordering for encoder failed"); 2563 TEST_ASSERT_SUCCESS(validate_op_chain( 2564 &ops[i]->turbo_enc.output, 2565 hard_data_orig), 2566 "Output buffers (CB=%u) are not equal", 2567 i); 2568 } 2569 2570 return TEST_SUCCESS; 2571 } 2572 2573 static int 2574 validate_ldpc_enc_op(struct rte_bbdev_enc_op **ops, const uint16_t n, 2575 struct rte_bbdev_enc_op *ref_op) 2576 { 2577 unsigned int i; 2578 int ret; 2579 struct op_data_entries *hard_data_orig = 2580 &test_vector.entries[DATA_HARD_OUTPUT]; 2581 2582 for (i = 0; i < n; ++i) { 2583 ret = check_enc_status_and_ordering(ops[i], i, ref_op->status); 2584 TEST_ASSERT_SUCCESS(ret, 2585 "Checking status and ordering for encoder failed"); 2586 TEST_ASSERT_SUCCESS(validate_op_chain( 2587 &ops[i]->ldpc_enc.output, 2588 hard_data_orig), 2589 "Output buffers (CB=%u) are not equal", 2590 i); 2591 } 2592 2593 return TEST_SUCCESS; 2594 } 2595 2596 static inline int 2597 validate_op_fft_chain(struct rte_bbdev_op_data *op, struct op_data_entries *orig_op, 2598 bool skip_validate_output) 2599 { 2600 struct rte_mbuf *m = op->data; 2601 uint8_t i, nb_dst_segments = orig_op->nb_segments; 2602 int16_t delt, abs_delt, thres_hold = 4; 2603 uint32_t j, data_len_iq, error_num; 2604 int16_t *ref_out, *op_out; 2605 2606 TEST_ASSERT(nb_dst_segments == m->nb_segs, 2607 "Number of segments differ in original (%u) and filled (%u) op fft", 2608 nb_dst_segments, m->nb_segs); 2609 2610 /* Due to size limitation of mbuf, FFT doesn't use real mbuf. */ 2611 for (i = 0; i < nb_dst_segments; ++i) { 2612 uint16_t offset = (i == 0) ? op->offset : 0; 2613 uint32_t data_len = op->length; 2614 2615 TEST_ASSERT(orig_op->segments[i].length == data_len, 2616 "Length of segment differ in original (%u) and filled (%u) op fft", 2617 orig_op->segments[i].length, data_len); 2618 /* Divided by 2 to get the number of 16bits data. */ 2619 data_len_iq = data_len >> 1; 2620 ref_out = (int16_t *)(orig_op->segments[i].addr); 2621 op_out = rte_pktmbuf_mtod_offset(m, int16_t *, offset); 2622 error_num = 0; 2623 for (j = 0; j < data_len_iq; j++) { 2624 delt = ref_out[j] - op_out[j]; 2625 abs_delt = delt > 0 ? delt : -delt; 2626 error_num += (abs_delt > thres_hold ? 1 : 0); 2627 } 2628 if ((error_num > 0) && !skip_validate_output) { 2629 rte_memdump(stdout, "Buffer A", ref_out, data_len); 2630 rte_memdump(stdout, "Buffer B", op_out, data_len); 2631 TEST_ASSERT(error_num == 0, 2632 "FFT Output are not matched total (%u) errors (%u)", 2633 data_len_iq, error_num); 2634 } 2635 2636 m = m->next; 2637 } 2638 2639 return TEST_SUCCESS; 2640 } 2641 2642 static inline int 2643 validate_op_fft_meas_chain(struct rte_bbdev_op_data *op, struct op_data_entries *orig_op, 2644 bool skip_validate_output) 2645 { 2646 struct rte_mbuf *m = op->data; 2647 uint8_t i, nb_dst_segments = orig_op->nb_segments; 2648 double thres_hold = 1.0; 2649 uint32_t j, data_len_iq, error_num; 2650 int32_t *ref_out, *op_out; 2651 double estSNR, refSNR, delt, abs_delt; 2652 2653 TEST_ASSERT(nb_dst_segments == m->nb_segs, 2654 "Number of segments differ in original (%u) and filled (%u) op fft", 2655 nb_dst_segments, m->nb_segs); 2656 2657 /* Due to size limitation of mbuf, FFT doesn't use real mbuf. */ 2658 for (i = 0; i < nb_dst_segments; ++i) { 2659 uint16_t offset = (i == 0) ? op->offset : 0; 2660 uint32_t data_len = op->length; 2661 2662 TEST_ASSERT(orig_op->segments[i].length == data_len, 2663 "Length of segment differ in original (%u) and filled (%u) op fft", 2664 orig_op->segments[i].length, data_len); 2665 2666 /* Divided by 4 to get the number of 32 bits data. */ 2667 data_len_iq = data_len >> 2; 2668 ref_out = (int32_t *)(orig_op->segments[i].addr); 2669 op_out = rte_pktmbuf_mtod_offset(m, int32_t *, offset); 2670 error_num = 0; 2671 for (j = 0; j < data_len_iq; j++) { 2672 estSNR = 10*log10(op_out[j]); 2673 refSNR = 10*log10(ref_out[j]); 2674 delt = refSNR - estSNR; 2675 abs_delt = delt > 0 ? delt : -delt; 2676 error_num += (abs_delt > thres_hold ? 1 : 0); 2677 } 2678 if ((error_num > 0) && !skip_validate_output) { 2679 rte_memdump(stdout, "Buffer A", ref_out, data_len); 2680 rte_memdump(stdout, "Buffer B", op_out, data_len); 2681 TEST_ASSERT(error_num == 0, 2682 "FFT Output are not matched total (%u) errors (%u)", 2683 data_len_iq, error_num); 2684 } 2685 2686 m = m->next; 2687 } 2688 2689 return TEST_SUCCESS; 2690 } 2691 2692 static inline int 2693 validate_op_mldts_chain(struct rte_bbdev_op_data *op, 2694 struct op_data_entries *orig_op) 2695 { 2696 uint8_t i; 2697 struct rte_mbuf *m = op->data; 2698 uint8_t nb_dst_segments = orig_op->nb_segments; 2699 /*the result is not bit exact*/ 2700 int16_t thres_hold = 3; 2701 int16_t delt, abs_delt; 2702 uint32_t j, data_len_iq; 2703 uint32_t error_num; 2704 int8_t *ref_out; 2705 int8_t *op_out; 2706 2707 TEST_ASSERT(nb_dst_segments == m->nb_segs, 2708 "Number of segments differ in original (%u) and filled (%u) op mldts", 2709 nb_dst_segments, m->nb_segs); 2710 2711 /* Due to size limitation of mbuf, MLDTS doesn't use real mbuf. */ 2712 for (i = 0; i < nb_dst_segments; ++i) { 2713 uint16_t offset = (i == 0) ? op->offset : 0; 2714 uint32_t data_len = op->length; 2715 2716 TEST_ASSERT(orig_op->segments[i].length == data_len, 2717 "Length of segment differ in original (%u) and filled (%u) op mldts", 2718 orig_op->segments[i].length, data_len); 2719 data_len_iq = data_len; 2720 ref_out = (int8_t *)(orig_op->segments[i].addr); 2721 op_out = rte_pktmbuf_mtod_offset(m, int8_t *, offset); 2722 error_num = 0; 2723 for (j = 0; j < data_len_iq; j++) { 2724 2725 delt = ref_out[j] - op_out[j]; 2726 abs_delt = delt > 0 ? delt : -delt; 2727 error_num += (abs_delt > thres_hold ? 1 : 0); 2728 if (error_num > 0) 2729 printf("MLD Error %d: Exp %x %d Actual %x %d Diff %d\n", 2730 j, ref_out[j], ref_out[j], op_out[j], op_out[j], 2731 delt); 2732 } 2733 TEST_ASSERT(error_num == 0, 2734 "MLDTS Output are not matched total (%u) errors (%u)", 2735 data_len_iq, error_num); 2736 2737 m = m->next; 2738 } 2739 2740 return TEST_SUCCESS; 2741 } 2742 2743 static int 2744 validate_fft_op(struct rte_bbdev_fft_op **ops, const uint16_t n, 2745 struct rte_bbdev_fft_op *ref_op) 2746 { 2747 unsigned int i; 2748 int ret; 2749 struct op_data_entries *fft_data_orig = &test_vector.entries[DATA_HARD_OUTPUT]; 2750 struct op_data_entries *fft_pwr_orig = &test_vector.entries[DATA_SOFT_OUTPUT]; 2751 bool skip_validate_output = false; 2752 2753 if ((test_vector.fft_window_width_vec > 0) && 2754 (test_vector.fft_window_width_vec != fft_window_width_dev)) { 2755 printf("The vector FFT width doesn't match with device - skip %d %d\n", 2756 test_vector.fft_window_width_vec, fft_window_width_dev); 2757 skip_validate_output = true; 2758 } 2759 2760 for (i = 0; i < n; ++i) { 2761 ret = check_fft_status_and_ordering(ops[i], i, ref_op->status); 2762 TEST_ASSERT_SUCCESS(ret, "Checking status and ordering for FFT failed"); 2763 TEST_ASSERT_SUCCESS(validate_op_fft_chain( 2764 &ops[i]->fft.base_output, fft_data_orig, skip_validate_output), 2765 "FFT Output buffers (op=%u) are not matched", i); 2766 if (check_bit(ops[i]->fft.op_flags, RTE_BBDEV_FFT_POWER_MEAS)) 2767 TEST_ASSERT_SUCCESS(validate_op_fft_meas_chain( 2768 &ops[i]->fft.power_meas_output, fft_pwr_orig, skip_validate_output), 2769 "FFT Power Output buffers (op=%u) are not matched", i); 2770 } 2771 2772 return TEST_SUCCESS; 2773 } 2774 2775 static int 2776 validate_mldts_op(struct rte_bbdev_mldts_op **ops, const uint16_t n, 2777 struct rte_bbdev_mldts_op *ref_op) 2778 { 2779 unsigned int i; 2780 int ret; 2781 struct op_data_entries *mldts_data_orig = 2782 &test_vector.entries[DATA_HARD_OUTPUT]; 2783 for (i = 0; i < n; ++i) { 2784 ret = check_mldts_status_and_ordering(ops[i], i, ref_op->status); 2785 TEST_ASSERT_SUCCESS(ret, 2786 "Checking status and ordering for MLDTS failed"); 2787 TEST_ASSERT_SUCCESS(validate_op_mldts_chain( 2788 &ops[i]->mldts.output, 2789 mldts_data_orig), 2790 "MLDTS Output buffers (op=%u) are not matched", 2791 i); 2792 } 2793 2794 return TEST_SUCCESS; 2795 } 2796 2797 static void 2798 create_reference_dec_op(struct rte_bbdev_dec_op *op) 2799 { 2800 unsigned int i; 2801 struct op_data_entries *entry; 2802 2803 op->turbo_dec = test_vector.turbo_dec; 2804 entry = &test_vector.entries[DATA_INPUT]; 2805 for (i = 0; i < entry->nb_segments; ++i) 2806 op->turbo_dec.input.length += 2807 entry->segments[i].length; 2808 } 2809 2810 static void 2811 create_reference_ldpc_dec_op(struct rte_bbdev_dec_op *op) 2812 { 2813 unsigned int i; 2814 struct op_data_entries *entry; 2815 2816 op->ldpc_dec = test_vector.ldpc_dec; 2817 entry = &test_vector.entries[DATA_INPUT]; 2818 for (i = 0; i < entry->nb_segments; ++i) 2819 op->ldpc_dec.input.length += 2820 entry->segments[i].length; 2821 if (test_vector.ldpc_dec.op_flags & 2822 RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE) { 2823 entry = &test_vector.entries[DATA_HARQ_INPUT]; 2824 for (i = 0; i < entry->nb_segments; ++i) 2825 op->ldpc_dec.harq_combined_input.length += 2826 entry->segments[i].length; 2827 } 2828 } 2829 2830 static void 2831 create_reference_fft_op(struct rte_bbdev_fft_op *op) 2832 { 2833 unsigned int i; 2834 struct op_data_entries *entry; 2835 op->fft = test_vector.fft; 2836 entry = &test_vector.entries[DATA_INPUT]; 2837 for (i = 0; i < entry->nb_segments; ++i) 2838 op->fft.base_input.length += entry->segments[i].length; 2839 entry = &test_vector.entries[DATA_HARQ_INPUT]; 2840 for (i = 0; i < entry->nb_segments; ++i) 2841 op->fft.dewindowing_input.length += entry->segments[i].length; 2842 } 2843 2844 static void 2845 create_reference_mldts_op(struct rte_bbdev_mldts_op *op) 2846 { 2847 unsigned int i; 2848 struct op_data_entries *entry; 2849 op->mldts = test_vector.mldts; 2850 entry = &test_vector.entries[DATA_INPUT]; 2851 for (i = 0; i < entry->nb_segments; ++i) 2852 op->mldts.qhy_input.length += entry->segments[i].length; 2853 entry = &test_vector.entries[DATA_HARQ_INPUT]; 2854 for (i = 0; i < entry->nb_segments; ++i) 2855 op->mldts.r_input.length += entry->segments[i].length; 2856 } 2857 2858 static void 2859 create_reference_enc_op(struct rte_bbdev_enc_op *op) 2860 { 2861 unsigned int i; 2862 struct op_data_entries *entry; 2863 2864 op->turbo_enc = test_vector.turbo_enc; 2865 entry = &test_vector.entries[DATA_INPUT]; 2866 for (i = 0; i < entry->nb_segments; ++i) 2867 op->turbo_enc.input.length += 2868 entry->segments[i].length; 2869 } 2870 2871 static void 2872 create_reference_ldpc_enc_op(struct rte_bbdev_enc_op *op) 2873 { 2874 unsigned int i; 2875 struct op_data_entries *entry; 2876 2877 op->ldpc_enc = test_vector.ldpc_enc; 2878 entry = &test_vector.entries[DATA_INPUT]; 2879 for (i = 0; i < entry->nb_segments; ++i) 2880 op->ldpc_enc.input.length += 2881 entry->segments[i].length; 2882 } 2883 2884 static uint32_t 2885 calc_dec_TB_size(struct rte_bbdev_dec_op *op) 2886 { 2887 uint8_t i; 2888 uint32_t c, r, tb_size = 0; 2889 2890 if (op->turbo_dec.code_block_mode == RTE_BBDEV_CODE_BLOCK) { 2891 tb_size = op->turbo_dec.tb_params.k_neg; 2892 } else { 2893 c = op->turbo_dec.tb_params.c; 2894 r = op->turbo_dec.tb_params.r; 2895 for (i = 0; i < c-r; i++) 2896 tb_size += (r < op->turbo_dec.tb_params.c_neg) ? 2897 op->turbo_dec.tb_params.k_neg : 2898 op->turbo_dec.tb_params.k_pos; 2899 } 2900 return tb_size; 2901 } 2902 2903 static uint32_t 2904 calc_ldpc_dec_TB_size(struct rte_bbdev_dec_op *op) 2905 { 2906 uint8_t num_cbs = 0; 2907 uint32_t tb_size = 0; 2908 uint16_t sys_cols = (op->ldpc_dec.basegraph == 1) ? 22 : 10; 2909 2910 if (op->ldpc_dec.code_block_mode == RTE_BBDEV_CODE_BLOCK) 2911 num_cbs = 1; 2912 else 2913 num_cbs = op->ldpc_dec.tb_params.c - op->ldpc_dec.tb_params.r; 2914 2915 tb_size = (sys_cols * op->ldpc_dec.z_c - op->ldpc_dec.n_filler) * num_cbs; 2916 return tb_size; 2917 } 2918 2919 static uint32_t 2920 calc_enc_TB_size(struct rte_bbdev_enc_op *op) 2921 { 2922 uint8_t i; 2923 uint32_t c, r, tb_size = 0; 2924 2925 if (op->turbo_enc.code_block_mode == RTE_BBDEV_CODE_BLOCK) { 2926 tb_size = op->turbo_enc.tb_params.k_neg; 2927 } else { 2928 c = op->turbo_enc.tb_params.c; 2929 r = op->turbo_enc.tb_params.r; 2930 for (i = 0; i < c-r; i++) 2931 tb_size += (r < op->turbo_enc.tb_params.c_neg) ? 2932 op->turbo_enc.tb_params.k_neg : 2933 op->turbo_enc.tb_params.k_pos; 2934 } 2935 return tb_size; 2936 } 2937 2938 static uint32_t 2939 calc_ldpc_enc_TB_size(struct rte_bbdev_enc_op *op) 2940 { 2941 uint8_t num_cbs = 0; 2942 uint32_t tb_size = 0; 2943 uint16_t sys_cols = (op->ldpc_enc.basegraph == 1) ? 22 : 10; 2944 2945 if (op->ldpc_enc.code_block_mode == RTE_BBDEV_CODE_BLOCK) 2946 num_cbs = 1; 2947 else 2948 num_cbs = op->ldpc_enc.tb_params.c - op->ldpc_enc.tb_params.r; 2949 2950 tb_size = (sys_cols * op->ldpc_enc.z_c - op->ldpc_enc.n_filler) * num_cbs; 2951 return tb_size; 2952 } 2953 2954 static uint32_t 2955 calc_fft_size(struct rte_bbdev_fft_op *op) 2956 { 2957 uint32_t output_size; 2958 int num_cs = 0, i; 2959 for (i = 0; i < 12; i++) 2960 if (check_bit(op->fft.cs_bitmap, 1 << i)) 2961 num_cs++; 2962 output_size = (num_cs * op->fft.output_sequence_size * 4) << op->fft.num_antennas_log2; 2963 return output_size; 2964 } 2965 2966 static uint32_t 2967 calc_mldts_size(struct rte_bbdev_mldts_op *op) 2968 { 2969 uint32_t output_size = 0; 2970 uint16_t i; 2971 2972 for (i = 0; i < op->mldts.num_layers; i++) 2973 output_size += op->mldts.q_m[i]; 2974 2975 output_size *= NUM_SC_PER_RB * BITS_PER_LLR * op->mldts.num_rbs * (op->mldts.c_rep + 1); 2976 2977 return output_size; 2978 } 2979 2980 static int 2981 init_test_op_params(struct test_op_params *op_params, 2982 enum rte_bbdev_op_type op_type, const int expected_status, 2983 const int vector_mask, struct rte_mempool *ops_mp, 2984 uint16_t burst_sz, uint16_t num_to_process, uint16_t num_lcores) 2985 { 2986 int ret = 0; 2987 if (op_type == RTE_BBDEV_OP_TURBO_DEC || 2988 op_type == RTE_BBDEV_OP_LDPC_DEC) 2989 ret = rte_bbdev_dec_op_alloc_bulk(ops_mp, 2990 &op_params->ref_dec_op, 1); 2991 else if (op_type == RTE_BBDEV_OP_FFT) 2992 ret = rte_bbdev_fft_op_alloc_bulk(ops_mp, 2993 &op_params->ref_fft_op, 1); 2994 else if (op_type == RTE_BBDEV_OP_MLDTS) 2995 ret = rte_bbdev_mldts_op_alloc_bulk(ops_mp, 2996 &op_params->ref_mldts_op, 1); 2997 else 2998 ret = rte_bbdev_enc_op_alloc_bulk(ops_mp, 2999 &op_params->ref_enc_op, 1); 3000 3001 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_op_alloc_bulk() failed"); 3002 3003 op_params->mp = ops_mp; 3004 op_params->burst_sz = burst_sz; 3005 op_params->num_to_process = num_to_process; 3006 op_params->num_lcores = num_lcores; 3007 op_params->vector_mask = vector_mask; 3008 if (op_type == RTE_BBDEV_OP_TURBO_DEC || 3009 op_type == RTE_BBDEV_OP_LDPC_DEC) 3010 op_params->ref_dec_op->status = expected_status; 3011 else if (op_type == RTE_BBDEV_OP_TURBO_ENC 3012 || op_type == RTE_BBDEV_OP_LDPC_ENC) 3013 op_params->ref_enc_op->status = expected_status; 3014 else if (op_type == RTE_BBDEV_OP_FFT) 3015 op_params->ref_fft_op->status = expected_status; 3016 else if (op_type == RTE_BBDEV_OP_MLDTS) 3017 op_params->ref_mldts_op->status = expected_status; 3018 return 0; 3019 } 3020 3021 static int 3022 run_test_case_on_device(test_case_function *test_case_func, uint8_t dev_id, 3023 struct test_op_params *op_params) 3024 { 3025 int t_ret, f_ret, socket_id = SOCKET_ID_ANY; 3026 unsigned int i; 3027 struct active_device *ad; 3028 unsigned int burst_sz = get_burst_sz(); 3029 enum rte_bbdev_op_type op_type = test_vector.op_type; 3030 const struct rte_bbdev_op_cap *capabilities = NULL; 3031 3032 ad = &active_devs[dev_id]; 3033 3034 /* Check if device supports op_type */ 3035 if (!is_avail_op(ad, test_vector.op_type)) 3036 return TEST_SUCCESS; 3037 3038 struct rte_bbdev_info info; 3039 rte_bbdev_info_get(ad->dev_id, &info); 3040 socket_id = GET_SOCKET(info.socket_id); 3041 3042 f_ret = create_mempools(ad, socket_id, op_type, 3043 get_num_ops()); 3044 if (f_ret != TEST_SUCCESS) { 3045 printf("Couldn't create mempools"); 3046 goto fail; 3047 } 3048 if (op_type == RTE_BBDEV_OP_NONE) 3049 op_type = RTE_BBDEV_OP_TURBO_ENC; 3050 3051 f_ret = init_test_op_params(op_params, test_vector.op_type, 3052 test_vector.expected_status, 3053 test_vector.mask, 3054 ad->ops_mempool, 3055 burst_sz, 3056 get_num_ops(), 3057 get_num_lcores()); 3058 if (f_ret != TEST_SUCCESS) { 3059 printf("Couldn't init test op params"); 3060 goto fail; 3061 } 3062 3063 3064 /* Find capabilities */ 3065 const struct rte_bbdev_op_cap *cap = info.drv.capabilities; 3066 do { 3067 if (cap->type == test_vector.op_type) { 3068 capabilities = cap; 3069 break; 3070 } 3071 cap++; 3072 } while (cap->type != RTE_BBDEV_OP_NONE); 3073 TEST_ASSERT_NOT_NULL(capabilities, 3074 "Couldn't find capabilities"); 3075 3076 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) { 3077 create_reference_dec_op(op_params->ref_dec_op); 3078 } else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) 3079 create_reference_enc_op(op_params->ref_enc_op); 3080 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) 3081 create_reference_ldpc_enc_op(op_params->ref_enc_op); 3082 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 3083 create_reference_ldpc_dec_op(op_params->ref_dec_op); 3084 else if (test_vector.op_type == RTE_BBDEV_OP_FFT) 3085 create_reference_fft_op(op_params->ref_fft_op); 3086 else if (test_vector.op_type == RTE_BBDEV_OP_MLDTS) 3087 create_reference_mldts_op(op_params->ref_mldts_op); 3088 3089 for (i = 0; i < ad->nb_queues; ++i) { 3090 f_ret = fill_queue_buffers(op_params, 3091 ad->in_mbuf_pool, 3092 ad->hard_out_mbuf_pool, 3093 ad->soft_out_mbuf_pool, 3094 ad->harq_in_mbuf_pool, 3095 ad->harq_out_mbuf_pool, 3096 ad->queue_ids[i], 3097 capabilities, 3098 info.drv.min_alignment, 3099 socket_id); 3100 if (f_ret != TEST_SUCCESS) { 3101 printf("Couldn't init queue buffers"); 3102 goto fail; 3103 } 3104 } 3105 3106 /* Run test case function */ 3107 t_ret = test_case_func(ad, op_params); 3108 3109 /* Free active device resources and return */ 3110 free_buffers(ad, op_params); 3111 return t_ret; 3112 3113 fail: 3114 free_buffers(ad, op_params); 3115 return TEST_FAILED; 3116 } 3117 3118 /* Run given test function per active device per supported op type 3119 * per burst size. 3120 */ 3121 static int 3122 run_test_case(test_case_function *test_case_func) 3123 { 3124 int ret = 0; 3125 uint8_t dev; 3126 3127 /* Alloc op_params */ 3128 struct test_op_params *op_params = rte_zmalloc(NULL, 3129 sizeof(struct test_op_params), RTE_CACHE_LINE_SIZE); 3130 TEST_ASSERT_NOT_NULL(op_params, "Failed to alloc %zuB for op_params", 3131 RTE_ALIGN(sizeof(struct test_op_params), 3132 RTE_CACHE_LINE_SIZE)); 3133 3134 /* For each device run test case function */ 3135 for (dev = 0; dev < nb_active_devs; ++dev) 3136 ret |= run_test_case_on_device(test_case_func, dev, op_params); 3137 3138 rte_free(op_params); 3139 3140 return ret; 3141 } 3142 3143 3144 /* Push back the HARQ output from DDR to host */ 3145 static void 3146 retrieve_harq_ddr(uint16_t dev_id, uint16_t queue_id, 3147 struct rte_bbdev_dec_op **ops, 3148 const uint16_t n) 3149 { 3150 uint16_t j; 3151 int save_status, ret; 3152 uint32_t harq_offset = (uint32_t) queue_id * HARQ_INCR * MAX_OPS; 3153 struct rte_bbdev_dec_op *ops_deq[MAX_BURST]; 3154 uint32_t flags = ops[0]->ldpc_dec.op_flags; 3155 bool loopback = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK; 3156 bool mem_out = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE; 3157 bool hc_out = flags & RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE; 3158 bool h_comp = flags & RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION; 3159 for (j = 0; j < n; ++j) { 3160 if ((loopback && mem_out) || hc_out) { 3161 save_status = ops[j]->status; 3162 ops[j]->ldpc_dec.op_flags = 3163 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK + 3164 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE; 3165 if (h_comp) 3166 ops[j]->ldpc_dec.op_flags += 3167 RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION; 3168 ops[j]->ldpc_dec.harq_combined_input.offset = 3169 harq_offset; 3170 ops[j]->ldpc_dec.harq_combined_output.offset = 0; 3171 harq_offset += HARQ_INCR; 3172 if (!loopback) 3173 ops[j]->ldpc_dec.harq_combined_input.length = 3174 ops[j]->ldpc_dec.harq_combined_output.length; 3175 rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id, 3176 &ops[j], 1); 3177 ret = 0; 3178 while (ret == 0) 3179 ret = rte_bbdev_dequeue_ldpc_dec_ops( 3180 dev_id, queue_id, 3181 &ops_deq[j], 1); 3182 ops[j]->ldpc_dec.op_flags = flags; 3183 ops[j]->status = save_status; 3184 } 3185 } 3186 } 3187 3188 /* 3189 * Push back the HARQ output from HW DDR to Host 3190 * Preload HARQ memory input and adjust HARQ offset 3191 */ 3192 static void 3193 preload_harq_ddr(uint16_t dev_id, uint16_t queue_id, 3194 struct rte_bbdev_dec_op **ops, const uint16_t n, 3195 bool preload) 3196 { 3197 uint16_t j; 3198 int deq; 3199 uint32_t harq_offset = (uint32_t) queue_id * HARQ_INCR * MAX_OPS; 3200 struct rte_bbdev_op_data save_hc_in[MAX_OPS], save_hc_out[MAX_OPS]; 3201 struct rte_bbdev_dec_op *ops_deq[MAX_OPS]; 3202 uint32_t flags = ops[0]->ldpc_dec.op_flags; 3203 bool mem_in = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE; 3204 bool hc_in = flags & RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE; 3205 bool mem_out = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE; 3206 bool hc_out = flags & RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE; 3207 bool h_comp = flags & RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION; 3208 if ((mem_in || hc_in) && preload) { 3209 for (j = 0; j < n; ++j) { 3210 save_hc_in[j] = ops[j]->ldpc_dec.harq_combined_input; 3211 save_hc_out[j] = ops[j]->ldpc_dec.harq_combined_output; 3212 ops[j]->ldpc_dec.op_flags = 3213 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK + 3214 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE; 3215 if (h_comp) 3216 ops[j]->ldpc_dec.op_flags += 3217 RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION; 3218 ops[j]->ldpc_dec.harq_combined_output.offset = 3219 harq_offset; 3220 ops[j]->ldpc_dec.harq_combined_input.offset = 0; 3221 harq_offset += HARQ_INCR; 3222 } 3223 rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id, &ops[0], n); 3224 deq = 0; 3225 while (deq != n) 3226 deq += rte_bbdev_dequeue_ldpc_dec_ops( 3227 dev_id, queue_id, &ops_deq[deq], 3228 n - deq); 3229 /* Restore the operations */ 3230 for (j = 0; j < n; ++j) { 3231 ops[j]->ldpc_dec.op_flags = flags; 3232 ops[j]->ldpc_dec.harq_combined_input = save_hc_in[j]; 3233 ops[j]->ldpc_dec.harq_combined_output = save_hc_out[j]; 3234 } 3235 } 3236 harq_offset = (uint32_t) queue_id * HARQ_INCR * MAX_OPS; 3237 for (j = 0; j < n; ++j) { 3238 /* Adjust HARQ offset when we reach external DDR */ 3239 if (mem_in || hc_in) 3240 ops[j]->ldpc_dec.harq_combined_input.offset 3241 = harq_offset; 3242 if (mem_out || hc_out) 3243 ops[j]->ldpc_dec.harq_combined_output.offset 3244 = harq_offset; 3245 harq_offset += HARQ_INCR; 3246 } 3247 } 3248 3249 static void 3250 dequeue_event_callback(uint16_t dev_id, 3251 enum rte_bbdev_event_type event, void *cb_arg, 3252 void *ret_param) 3253 { 3254 int ret; 3255 uint16_t i; 3256 uint64_t total_time; 3257 uint16_t deq, burst_sz, num_ops; 3258 uint16_t queue_id = *(uint16_t *) ret_param; 3259 struct rte_bbdev_info info; 3260 double tb_len_bits; 3261 struct thread_params *tp = cb_arg; 3262 3263 /* Find matching thread params using queue_id */ 3264 for (i = 0; i < MAX_QUEUES; ++i, ++tp) 3265 if (tp->queue_id == queue_id) 3266 break; 3267 3268 if (i == MAX_QUEUES) { 3269 printf("%s: Queue_id from interrupt details was not found!\n", 3270 __func__); 3271 return; 3272 } 3273 3274 if (unlikely(event != RTE_BBDEV_EVENT_DEQUEUE)) { 3275 rte_atomic_store_explicit(&tp->processing_status, TEST_FAILED, 3276 rte_memory_order_relaxed); 3277 printf( 3278 "Dequeue interrupt handler called for incorrect event!\n"); 3279 return; 3280 } 3281 3282 burst_sz = rte_atomic_load_explicit(&tp->burst_sz, rte_memory_order_relaxed); 3283 num_ops = tp->op_params->num_to_process; 3284 3285 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) 3286 deq = rte_bbdev_dequeue_dec_ops(dev_id, queue_id, 3287 &tp->dec_ops[ 3288 rte_atomic_load_explicit(&tp->nb_dequeued, 3289 rte_memory_order_relaxed)], 3290 burst_sz); 3291 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 3292 deq = rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id, 3293 &tp->dec_ops[ 3294 rte_atomic_load_explicit(&tp->nb_dequeued, 3295 rte_memory_order_relaxed)], 3296 burst_sz); 3297 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) 3298 deq = rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id, 3299 &tp->enc_ops[ 3300 rte_atomic_load_explicit(&tp->nb_dequeued, 3301 rte_memory_order_relaxed)], 3302 burst_sz); 3303 else if (test_vector.op_type == RTE_BBDEV_OP_FFT) 3304 deq = rte_bbdev_dequeue_fft_ops(dev_id, queue_id, 3305 &tp->fft_ops[ 3306 rte_atomic_load_explicit(&tp->nb_dequeued, 3307 rte_memory_order_relaxed)], 3308 burst_sz); 3309 else if (test_vector.op_type == RTE_BBDEV_OP_MLDTS) 3310 deq = rte_bbdev_dequeue_mldts_ops(dev_id, queue_id, 3311 &tp->mldts_ops[ 3312 rte_atomic_load_explicit(&tp->nb_dequeued, 3313 rte_memory_order_relaxed)], 3314 burst_sz); 3315 else /*RTE_BBDEV_OP_TURBO_ENC*/ 3316 deq = rte_bbdev_dequeue_enc_ops(dev_id, queue_id, 3317 &tp->enc_ops[ 3318 rte_atomic_load_explicit(&tp->nb_dequeued, 3319 rte_memory_order_relaxed)], 3320 burst_sz); 3321 3322 if (deq < burst_sz) { 3323 printf( 3324 "After receiving the interrupt all operations should be dequeued. Expected: %u, got: %u\n", 3325 burst_sz, deq); 3326 rte_atomic_store_explicit(&tp->processing_status, TEST_FAILED, 3327 rte_memory_order_relaxed); 3328 return; 3329 } 3330 3331 if (rte_atomic_load_explicit(&tp->nb_dequeued, rte_memory_order_relaxed) + deq < num_ops) { 3332 rte_atomic_fetch_add_explicit(&tp->nb_dequeued, deq, rte_memory_order_relaxed); 3333 return; 3334 } 3335 3336 total_time = rte_rdtsc_precise() - tp->start_time; 3337 3338 rte_bbdev_info_get(dev_id, &info); 3339 3340 ret = TEST_SUCCESS; 3341 3342 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) { 3343 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 3344 ret = validate_dec_op(tp->dec_ops, num_ops, ref_op); 3345 /* get the max of iter_count for all dequeued ops */ 3346 for (i = 0; i < num_ops; ++i) 3347 tp->iter_count = RTE_MAX( 3348 tp->dec_ops[i]->turbo_dec.iter_count, 3349 tp->iter_count); 3350 rte_bbdev_dec_op_free_bulk(tp->dec_ops, deq); 3351 } else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) { 3352 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op; 3353 ret = validate_enc_op(tp->enc_ops, num_ops, ref_op); 3354 rte_bbdev_enc_op_free_bulk(tp->enc_ops, deq); 3355 } else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) { 3356 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op; 3357 ret = validate_ldpc_enc_op(tp->enc_ops, num_ops, ref_op); 3358 rte_bbdev_enc_op_free_bulk(tp->enc_ops, deq); 3359 } else if (test_vector.op_type == RTE_BBDEV_OP_FFT) { 3360 struct rte_bbdev_fft_op *ref_op = tp->op_params->ref_fft_op; 3361 ret = validate_fft_op(tp->fft_ops, num_ops, ref_op); 3362 rte_bbdev_fft_op_free_bulk(tp->fft_ops, deq); 3363 } else if (test_vector.op_type == RTE_BBDEV_OP_MLDTS) { 3364 struct rte_bbdev_mldts_op *ref_op = tp->op_params->ref_mldts_op; 3365 ret = validate_mldts_op(tp->mldts_ops, num_ops, ref_op); 3366 rte_bbdev_mldts_op_free_bulk(tp->mldts_ops, deq); 3367 } else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) { 3368 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 3369 ret = validate_ldpc_dec_op(tp->dec_ops, num_ops, ref_op, 3370 tp->op_params->vector_mask); 3371 rte_bbdev_dec_op_free_bulk(tp->dec_ops, deq); 3372 } 3373 3374 if (ret) { 3375 printf("Buffers validation failed\n"); 3376 rte_atomic_store_explicit(&tp->processing_status, TEST_FAILED, 3377 rte_memory_order_relaxed); 3378 } 3379 3380 switch (test_vector.op_type) { 3381 case RTE_BBDEV_OP_TURBO_DEC: 3382 tb_len_bits = calc_dec_TB_size(tp->op_params->ref_dec_op); 3383 break; 3384 case RTE_BBDEV_OP_TURBO_ENC: 3385 tb_len_bits = calc_enc_TB_size(tp->op_params->ref_enc_op); 3386 break; 3387 case RTE_BBDEV_OP_LDPC_DEC: 3388 tb_len_bits = calc_ldpc_dec_TB_size(tp->op_params->ref_dec_op); 3389 break; 3390 case RTE_BBDEV_OP_FFT: 3391 tb_len_bits = calc_fft_size(tp->op_params->ref_fft_op); 3392 break; 3393 case RTE_BBDEV_OP_MLDTS: 3394 tb_len_bits = calc_mldts_size(tp->op_params->ref_mldts_op); 3395 break; 3396 case RTE_BBDEV_OP_LDPC_ENC: 3397 tb_len_bits = calc_ldpc_enc_TB_size(tp->op_params->ref_enc_op); 3398 break; 3399 case RTE_BBDEV_OP_NONE: 3400 tb_len_bits = 0.0; 3401 break; 3402 default: 3403 printf("Unknown op type: %d\n", test_vector.op_type); 3404 rte_atomic_store_explicit(&tp->processing_status, TEST_FAILED, 3405 rte_memory_order_relaxed); 3406 return; 3407 } 3408 3409 tp->ops_per_sec += ((double)num_ops) / 3410 ((double)total_time / (double)rte_get_tsc_hz()); 3411 tp->mbps += (((double)(num_ops * tb_len_bits)) / 1000000.0) / 3412 ((double)total_time / (double)rte_get_tsc_hz()); 3413 3414 rte_atomic_fetch_add_explicit(&tp->nb_dequeued, deq, rte_memory_order_relaxed); 3415 } 3416 3417 static int 3418 throughput_intr_lcore_ldpc_dec(void *arg) 3419 { 3420 struct thread_params *tp = arg; 3421 unsigned int enqueued; 3422 const uint16_t queue_id = tp->queue_id; 3423 const uint16_t burst_sz = tp->op_params->burst_sz; 3424 const uint16_t num_to_process = tp->op_params->num_to_process; 3425 struct rte_bbdev_dec_op *ops[num_to_process]; 3426 struct test_buffers *bufs = NULL; 3427 struct rte_bbdev_info info; 3428 int ret, i, j; 3429 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 3430 uint16_t num_to_enq, enq; 3431 3432 bool loopback = check_bit(ref_op->ldpc_dec.op_flags, 3433 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK); 3434 bool hc_out = check_bit(ref_op->ldpc_dec.op_flags, 3435 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE); 3436 3437 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 3438 "BURST_SIZE should be <= %u", MAX_BURST); 3439 3440 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id), 3441 "Failed to enable interrupts for dev: %u, queue_id: %u", 3442 tp->dev_id, queue_id); 3443 3444 rte_bbdev_info_get(tp->dev_id, &info); 3445 3446 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim), 3447 "NUM_OPS cannot exceed %u for this device", 3448 info.drv.queue_size_lim); 3449 3450 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 3451 3452 rte_atomic_store_explicit(&tp->processing_status, 0, rte_memory_order_relaxed); 3453 rte_atomic_store_explicit(&tp->nb_dequeued, 0, rte_memory_order_relaxed); 3454 3455 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->op_params->sync, SYNC_START, 3456 rte_memory_order_relaxed); 3457 3458 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops, 3459 num_to_process); 3460 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", 3461 num_to_process); 3462 ref_op->ldpc_dec.iter_max = get_iter_max(); 3463 3464 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3465 copy_reference_ldpc_dec_op(ops, num_to_process, 0, bufs->inputs, 3466 bufs->hard_outputs, bufs->soft_outputs, 3467 bufs->harq_inputs, bufs->harq_outputs, ref_op); 3468 3469 /* Set counter to validate the ordering */ 3470 for (j = 0; j < num_to_process; ++j) 3471 ops[j]->opaque_data = (void *)(uintptr_t)j; 3472 3473 for (j = 0; j < TEST_REPETITIONS; ++j) { 3474 for (i = 0; i < num_to_process; ++i) { 3475 if (!loopback) 3476 mbuf_reset(ops[i]->ldpc_dec.hard_output.data); 3477 if (hc_out || loopback) 3478 mbuf_reset(ops[i]->ldpc_dec.harq_combined_output.data); 3479 if (ops[i]->ldpc_dec.soft_output.data != NULL) 3480 mbuf_reset(ops[i]->ldpc_dec.soft_output.data); 3481 } 3482 3483 tp->start_time = rte_rdtsc_precise(); 3484 for (enqueued = 0; enqueued < num_to_process;) { 3485 num_to_enq = burst_sz; 3486 3487 if (unlikely(num_to_process - enqueued < num_to_enq)) 3488 num_to_enq = num_to_process - enqueued; 3489 3490 /* Write to thread burst_sz current number of enqueued 3491 * descriptors. It ensures that proper number of 3492 * descriptors will be dequeued in callback 3493 * function - needed for last batch in case where 3494 * the number of operations is not a multiple of 3495 * burst size. 3496 */ 3497 rte_atomic_store_explicit(&tp->burst_sz, num_to_enq, 3498 rte_memory_order_relaxed); 3499 3500 enq = 0; 3501 do { 3502 enq += rte_bbdev_enqueue_ldpc_dec_ops( 3503 tp->dev_id, 3504 queue_id, &ops[enqueued], 3505 num_to_enq); 3506 } while (unlikely(num_to_enq != enq)); 3507 enqueued += enq; 3508 3509 /* Wait until processing of previous batch is 3510 * completed 3511 */ 3512 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->nb_dequeued, enqueued, 3513 rte_memory_order_relaxed); 3514 } 3515 if (j != TEST_REPETITIONS - 1) 3516 rte_atomic_store_explicit(&tp->nb_dequeued, 0, rte_memory_order_relaxed); 3517 } 3518 3519 return TEST_SUCCESS; 3520 } 3521 3522 static int 3523 throughput_intr_lcore_dec(void *arg) 3524 { 3525 struct thread_params *tp = arg; 3526 unsigned int enqueued; 3527 const uint16_t queue_id = tp->queue_id; 3528 const uint16_t burst_sz = tp->op_params->burst_sz; 3529 const uint16_t num_to_process = tp->op_params->num_to_process; 3530 struct rte_bbdev_dec_op *ops[num_to_process]; 3531 struct test_buffers *bufs = NULL; 3532 struct rte_bbdev_info info; 3533 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 3534 int ret, i, j; 3535 uint16_t num_to_enq, enq; 3536 3537 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 3538 "BURST_SIZE should be <= %u", MAX_BURST); 3539 3540 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id), 3541 "Failed to enable interrupts for dev: %u, queue_id: %u", 3542 tp->dev_id, queue_id); 3543 3544 rte_bbdev_info_get(tp->dev_id, &info); 3545 3546 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim), 3547 "NUM_OPS cannot exceed %u for this device", 3548 info.drv.queue_size_lim); 3549 3550 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 3551 3552 rte_atomic_store_explicit(&tp->processing_status, 0, rte_memory_order_relaxed); 3553 rte_atomic_store_explicit(&tp->nb_dequeued, 0, rte_memory_order_relaxed); 3554 3555 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->op_params->sync, SYNC_START, 3556 rte_memory_order_relaxed); 3557 3558 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops, 3559 num_to_process); 3560 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_to_process); 3561 ref_op->turbo_dec.iter_max = get_iter_max(); 3562 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3563 copy_reference_dec_op(ops, num_to_process, 0, bufs->inputs, 3564 bufs->hard_outputs, bufs->soft_outputs, 3565 tp->op_params->ref_dec_op); 3566 3567 /* Set counter to validate the ordering. */ 3568 for (j = 0; j < num_to_process; ++j) 3569 ops[j]->opaque_data = (void *)(uintptr_t)j; 3570 3571 for (j = 0; j < TEST_REPETITIONS; ++j) { 3572 for (i = 0; i < num_to_process; ++i) { 3573 mbuf_reset(ops[i]->turbo_dec.hard_output.data); 3574 if (ops[i]->turbo_dec.soft_output.data != NULL) 3575 mbuf_reset(ops[i]->turbo_dec.soft_output.data); 3576 } 3577 3578 tp->start_time = rte_rdtsc_precise(); 3579 for (enqueued = 0; enqueued < num_to_process;) { 3580 num_to_enq = burst_sz; 3581 3582 if (unlikely(num_to_process - enqueued < num_to_enq)) 3583 num_to_enq = num_to_process - enqueued; 3584 3585 /* Write to thread burst_sz current number of enqueued 3586 * descriptors. It ensures that proper number of 3587 * descriptors will be dequeued in callback 3588 * function - needed for last batch in case where 3589 * the number of operations is not a multiple of 3590 * burst size. 3591 */ 3592 rte_atomic_store_explicit(&tp->burst_sz, num_to_enq, 3593 rte_memory_order_relaxed); 3594 3595 enq = 0; 3596 do { 3597 enq += rte_bbdev_enqueue_dec_ops(tp->dev_id, 3598 queue_id, &ops[enqueued], 3599 num_to_enq); 3600 } while (unlikely(num_to_enq != enq)); 3601 enqueued += enq; 3602 3603 /* Wait until processing of previous batch is 3604 * completed 3605 */ 3606 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->nb_dequeued, enqueued, 3607 rte_memory_order_relaxed); 3608 } 3609 if (j != TEST_REPETITIONS - 1) 3610 rte_atomic_store_explicit(&tp->nb_dequeued, 0, rte_memory_order_relaxed); 3611 } 3612 3613 return TEST_SUCCESS; 3614 } 3615 3616 static int 3617 throughput_intr_lcore_enc(void *arg) 3618 { 3619 struct thread_params *tp = arg; 3620 unsigned int enqueued; 3621 const uint16_t queue_id = tp->queue_id; 3622 const uint16_t burst_sz = tp->op_params->burst_sz; 3623 const uint16_t num_to_process = tp->op_params->num_to_process; 3624 struct rte_bbdev_enc_op *ops[num_to_process]; 3625 struct test_buffers *bufs = NULL; 3626 struct rte_bbdev_info info; 3627 int ret, i, j; 3628 uint16_t num_to_enq, enq; 3629 3630 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 3631 "BURST_SIZE should be <= %u", MAX_BURST); 3632 3633 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id), 3634 "Failed to enable interrupts for dev: %u, queue_id: %u", 3635 tp->dev_id, queue_id); 3636 3637 rte_bbdev_info_get(tp->dev_id, &info); 3638 3639 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim), 3640 "NUM_OPS cannot exceed %u for this device", 3641 info.drv.queue_size_lim); 3642 3643 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 3644 3645 rte_atomic_store_explicit(&tp->processing_status, 0, rte_memory_order_relaxed); 3646 rte_atomic_store_explicit(&tp->nb_dequeued, 0, rte_memory_order_relaxed); 3647 3648 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->op_params->sync, SYNC_START, 3649 rte_memory_order_relaxed); 3650 3651 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops, 3652 num_to_process); 3653 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", 3654 num_to_process); 3655 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3656 copy_reference_enc_op(ops, num_to_process, 0, bufs->inputs, 3657 bufs->hard_outputs, tp->op_params->ref_enc_op); 3658 3659 /* Set counter to validate the ordering */ 3660 for (j = 0; j < num_to_process; ++j) 3661 ops[j]->opaque_data = (void *)(uintptr_t)j; 3662 3663 for (j = 0; j < TEST_REPETITIONS; ++j) { 3664 for (i = 0; i < num_to_process; ++i) 3665 mbuf_reset(ops[i]->turbo_enc.output.data); 3666 3667 tp->start_time = rte_rdtsc_precise(); 3668 for (enqueued = 0; enqueued < num_to_process;) { 3669 num_to_enq = burst_sz; 3670 3671 if (unlikely(num_to_process - enqueued < num_to_enq)) 3672 num_to_enq = num_to_process - enqueued; 3673 3674 /* Write to thread burst_sz current number of enqueued 3675 * descriptors. It ensures that proper number of 3676 * descriptors will be dequeued in callback 3677 * function - needed for last batch in case where 3678 * the number of operations is not a multiple of 3679 * burst size. 3680 */ 3681 rte_atomic_store_explicit(&tp->burst_sz, num_to_enq, 3682 rte_memory_order_relaxed); 3683 3684 enq = 0; 3685 do { 3686 enq += rte_bbdev_enqueue_enc_ops(tp->dev_id, 3687 queue_id, &ops[enqueued], 3688 num_to_enq); 3689 } while (unlikely(enq != num_to_enq)); 3690 enqueued += enq; 3691 3692 /* Wait until processing of previous batch is 3693 * completed 3694 */ 3695 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->nb_dequeued, enqueued, 3696 rte_memory_order_relaxed); 3697 } 3698 if (j != TEST_REPETITIONS - 1) 3699 rte_atomic_store_explicit(&tp->nb_dequeued, 0, rte_memory_order_relaxed); 3700 } 3701 3702 return TEST_SUCCESS; 3703 } 3704 3705 3706 static int 3707 throughput_intr_lcore_ldpc_enc(void *arg) 3708 { 3709 struct thread_params *tp = arg; 3710 unsigned int enqueued; 3711 const uint16_t queue_id = tp->queue_id; 3712 const uint16_t burst_sz = tp->op_params->burst_sz; 3713 const uint16_t num_to_process = tp->op_params->num_to_process; 3714 struct rte_bbdev_enc_op *ops[num_to_process]; 3715 struct test_buffers *bufs = NULL; 3716 struct rte_bbdev_info info; 3717 int ret, i, j; 3718 uint16_t num_to_enq, enq; 3719 3720 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 3721 "BURST_SIZE should be <= %u", MAX_BURST); 3722 3723 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id), 3724 "Failed to enable interrupts for dev: %u, queue_id: %u", 3725 tp->dev_id, queue_id); 3726 3727 rte_bbdev_info_get(tp->dev_id, &info); 3728 3729 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim), 3730 "NUM_OPS cannot exceed %u for this device", 3731 info.drv.queue_size_lim); 3732 3733 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 3734 3735 rte_atomic_store_explicit(&tp->processing_status, 0, rte_memory_order_relaxed); 3736 rte_atomic_store_explicit(&tp->nb_dequeued, 0, rte_memory_order_relaxed); 3737 3738 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->op_params->sync, SYNC_START, 3739 rte_memory_order_relaxed); 3740 3741 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops, 3742 num_to_process); 3743 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", 3744 num_to_process); 3745 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3746 copy_reference_ldpc_enc_op(ops, num_to_process, 0, 3747 bufs->inputs, bufs->hard_outputs, 3748 tp->op_params->ref_enc_op); 3749 3750 /* Set counter to validate the ordering */ 3751 for (j = 0; j < num_to_process; ++j) 3752 ops[j]->opaque_data = (void *)(uintptr_t)j; 3753 3754 for (j = 0; j < TEST_REPETITIONS; ++j) { 3755 for (i = 0; i < num_to_process; ++i) 3756 mbuf_reset(ops[i]->turbo_enc.output.data); 3757 3758 tp->start_time = rte_rdtsc_precise(); 3759 for (enqueued = 0; enqueued < num_to_process;) { 3760 num_to_enq = burst_sz; 3761 3762 if (unlikely(num_to_process - enqueued < num_to_enq)) 3763 num_to_enq = num_to_process - enqueued; 3764 3765 /* Write to thread burst_sz current number of enqueued 3766 * descriptors. It ensures that proper number of 3767 * descriptors will be dequeued in callback 3768 * function - needed for last batch in case where 3769 * the number of operations is not a multiple of 3770 * burst size. 3771 */ 3772 rte_atomic_store_explicit(&tp->burst_sz, num_to_enq, 3773 rte_memory_order_relaxed); 3774 3775 enq = 0; 3776 do { 3777 enq += rte_bbdev_enqueue_ldpc_enc_ops( 3778 tp->dev_id, 3779 queue_id, &ops[enqueued], 3780 num_to_enq); 3781 } while (unlikely(enq != num_to_enq)); 3782 enqueued += enq; 3783 3784 /* Wait until processing of previous batch is 3785 * completed 3786 */ 3787 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->nb_dequeued, enqueued, 3788 rte_memory_order_relaxed); 3789 } 3790 if (j != TEST_REPETITIONS - 1) 3791 rte_atomic_store_explicit(&tp->nb_dequeued, 0, rte_memory_order_relaxed); 3792 } 3793 3794 return TEST_SUCCESS; 3795 } 3796 3797 3798 static int 3799 throughput_intr_lcore_fft(void *arg) 3800 { 3801 struct thread_params *tp = arg; 3802 unsigned int enqueued; 3803 const uint16_t queue_id = tp->queue_id; 3804 const uint16_t burst_sz = tp->op_params->burst_sz; 3805 const uint16_t num_to_process = tp->op_params->num_to_process; 3806 struct rte_bbdev_fft_op *ops[num_to_process]; 3807 struct test_buffers *bufs = NULL; 3808 struct rte_bbdev_info info; 3809 int ret, i, j; 3810 uint16_t num_to_enq, enq; 3811 3812 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 3813 "BURST_SIZE should be <= %u", MAX_BURST); 3814 3815 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id), 3816 "Failed to enable interrupts for dev: %u, queue_id: %u", 3817 tp->dev_id, queue_id); 3818 3819 rte_bbdev_info_get(tp->dev_id, &info); 3820 3821 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim), 3822 "NUM_OPS cannot exceed %u for this device", 3823 info.drv.queue_size_lim); 3824 3825 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 3826 3827 rte_atomic_store_explicit(&tp->processing_status, 0, rte_memory_order_relaxed); 3828 rte_atomic_store_explicit(&tp->nb_dequeued, 0, rte_memory_order_relaxed); 3829 3830 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->op_params->sync, SYNC_START, 3831 rte_memory_order_relaxed); 3832 3833 ret = rte_bbdev_fft_op_alloc_bulk(tp->op_params->mp, ops, 3834 num_to_process); 3835 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", 3836 num_to_process); 3837 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3838 copy_reference_fft_op(ops, num_to_process, 0, bufs->inputs, 3839 bufs->hard_outputs, bufs->soft_outputs, bufs->harq_inputs, 3840 tp->op_params->ref_fft_op); 3841 3842 /* Set counter to validate the ordering */ 3843 for (j = 0; j < num_to_process; ++j) 3844 ops[j]->opaque_data = (void *)(uintptr_t)j; 3845 3846 for (j = 0; j < TEST_REPETITIONS; ++j) { 3847 for (i = 0; i < num_to_process; ++i) 3848 mbuf_reset(ops[i]->fft.base_output.data); 3849 3850 tp->start_time = rte_rdtsc_precise(); 3851 for (enqueued = 0; enqueued < num_to_process;) { 3852 num_to_enq = burst_sz; 3853 3854 if (unlikely(num_to_process - enqueued < num_to_enq)) 3855 num_to_enq = num_to_process - enqueued; 3856 3857 /* Write to thread burst_sz current number of enqueued 3858 * descriptors. It ensures that proper number of 3859 * descriptors will be dequeued in callback 3860 * function - needed for last batch in case where 3861 * the number of operations is not a multiple of 3862 * burst size. 3863 */ 3864 rte_atomic_store_explicit(&tp->burst_sz, num_to_enq, 3865 rte_memory_order_relaxed); 3866 3867 enq = 0; 3868 do { 3869 enq += rte_bbdev_enqueue_fft_ops(tp->dev_id, 3870 queue_id, &ops[enqueued], 3871 num_to_enq); 3872 } while (unlikely(enq != num_to_enq)); 3873 enqueued += enq; 3874 3875 /* Wait until processing of previous batch is 3876 * completed 3877 */ 3878 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->nb_dequeued, enqueued, 3879 rte_memory_order_relaxed); 3880 } 3881 if (j != TEST_REPETITIONS - 1) 3882 rte_atomic_store_explicit(&tp->nb_dequeued, 0, rte_memory_order_relaxed); 3883 } 3884 3885 return TEST_SUCCESS; 3886 } 3887 3888 static int 3889 throughput_intr_lcore_mldts(void *arg) 3890 { 3891 struct thread_params *tp = arg; 3892 unsigned int enqueued; 3893 const uint16_t queue_id = tp->queue_id; 3894 const uint16_t burst_sz = tp->op_params->burst_sz; 3895 const uint16_t num_to_process = tp->op_params->num_to_process; 3896 struct rte_bbdev_mldts_op *ops[num_to_process]; 3897 struct test_buffers *bufs = NULL; 3898 struct rte_bbdev_info info; 3899 int ret, i, j; 3900 uint16_t num_to_enq, enq; 3901 3902 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), "BURST_SIZE should be <= %u", MAX_BURST); 3903 3904 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id), 3905 "Failed to enable interrupts for dev: %u, queue_id: %u", 3906 tp->dev_id, queue_id); 3907 3908 rte_bbdev_info_get(tp->dev_id, &info); 3909 3910 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim), 3911 "NUM_OPS cannot exceed %u for this device", 3912 info.drv.queue_size_lim); 3913 3914 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 3915 3916 rte_atomic_store_explicit(&tp->processing_status, 0, rte_memory_order_relaxed); 3917 rte_atomic_store_explicit(&tp->nb_dequeued, 0, rte_memory_order_relaxed); 3918 3919 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->op_params->sync, SYNC_START, 3920 rte_memory_order_relaxed); 3921 3922 ret = rte_bbdev_mldts_op_alloc_bulk(tp->op_params->mp, ops, num_to_process); 3923 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_to_process); 3924 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3925 copy_reference_mldts_op(ops, num_to_process, 0, bufs->inputs, bufs->harq_inputs, 3926 bufs->hard_outputs, tp->op_params->ref_mldts_op); 3927 3928 /* Set counter to validate the ordering */ 3929 for (j = 0; j < num_to_process; ++j) 3930 ops[j]->opaque_data = (void *)(uintptr_t)j; 3931 3932 for (j = 0; j < TEST_REPETITIONS; ++j) { 3933 for (i = 0; i < num_to_process; ++i) 3934 mbuf_reset(ops[i]->mldts.output.data); 3935 3936 tp->start_time = rte_rdtsc_precise(); 3937 for (enqueued = 0; enqueued < num_to_process;) { 3938 num_to_enq = burst_sz; 3939 3940 if (unlikely(num_to_process - enqueued < num_to_enq)) 3941 num_to_enq = num_to_process - enqueued; 3942 3943 /* Write to thread burst_sz current number of enqueued 3944 * descriptors. It ensures that proper number of 3945 * descriptors will be dequeued in callback 3946 * function - needed for last batch in case where 3947 * the number of operations is not a multiple of 3948 * burst size. 3949 */ 3950 rte_atomic_store_explicit(&tp->burst_sz, num_to_enq, 3951 rte_memory_order_relaxed); 3952 3953 enq = 0; 3954 do { 3955 enq += rte_bbdev_enqueue_mldts_ops(tp->dev_id, 3956 queue_id, &ops[enqueued], num_to_enq); 3957 } while (unlikely(enq != num_to_enq)); 3958 enqueued += enq; 3959 3960 /* Wait until processing of previous batch is 3961 * completed 3962 */ 3963 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->nb_dequeued, enqueued, 3964 rte_memory_order_relaxed); 3965 } 3966 if (j != TEST_REPETITIONS - 1) 3967 rte_atomic_store_explicit(&tp->nb_dequeued, 0, rte_memory_order_relaxed); 3968 } 3969 3970 return TEST_SUCCESS; 3971 } 3972 3973 static int 3974 throughput_pmd_lcore_dec(void *arg) 3975 { 3976 struct thread_params *tp = arg; 3977 uint16_t enq, deq; 3978 uint64_t total_time = 0, start_time; 3979 const uint16_t queue_id = tp->queue_id; 3980 const uint16_t burst_sz = tp->op_params->burst_sz; 3981 const uint16_t num_ops = tp->op_params->num_to_process; 3982 struct rte_bbdev_dec_op *ops_enq[num_ops]; 3983 struct rte_bbdev_dec_op *ops_deq[num_ops]; 3984 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 3985 struct test_buffers *bufs = NULL; 3986 int i, j, ret; 3987 struct rte_bbdev_info info; 3988 uint16_t num_to_enq; 3989 bool so_enable; 3990 3991 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 3992 "BURST_SIZE should be <= %u", MAX_BURST); 3993 3994 rte_bbdev_info_get(tp->dev_id, &info); 3995 3996 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 3997 "NUM_OPS cannot exceed %u for this device", 3998 info.drv.queue_size_lim); 3999 4000 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 4001 4002 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->op_params->sync, SYNC_START, 4003 rte_memory_order_relaxed); 4004 4005 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops); 4006 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops); 4007 ref_op->turbo_dec.iter_max = get_iter_max(); 4008 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4009 copy_reference_dec_op(ops_enq, num_ops, 0, bufs->inputs, 4010 bufs->hard_outputs, bufs->soft_outputs, ref_op); 4011 4012 so_enable = check_bit(ops_enq[0]->turbo_dec.op_flags, RTE_BBDEV_TURBO_SOFT_OUTPUT); 4013 4014 /* Set counter to validate the ordering */ 4015 for (j = 0; j < num_ops; ++j) 4016 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 4017 4018 for (i = 0; i < TEST_REPETITIONS; ++i) { 4019 uint32_t time_out = 0; 4020 for (j = 0; j < num_ops; ++j) 4021 mbuf_reset(ops_enq[j]->turbo_dec.hard_output.data); 4022 if (so_enable) 4023 for (j = 0; j < num_ops; ++j) 4024 mbuf_reset(ops_enq[j]->turbo_dec.soft_output.data); 4025 4026 start_time = rte_rdtsc_precise(); 4027 4028 for (enq = 0, deq = 0; enq < num_ops;) { 4029 num_to_enq = burst_sz; 4030 4031 if (unlikely(num_ops - enq < num_to_enq)) 4032 num_to_enq = num_ops - enq; 4033 4034 enq += rte_bbdev_enqueue_dec_ops(tp->dev_id, 4035 queue_id, &ops_enq[enq], num_to_enq); 4036 4037 deq += rte_bbdev_dequeue_dec_ops(tp->dev_id, 4038 queue_id, &ops_deq[deq], enq - deq); 4039 time_out++; 4040 if (time_out >= TIME_OUT_POLL) { 4041 timeout_exit(tp->dev_id); 4042 TEST_ASSERT_SUCCESS(TEST_FAILED, "Enqueue timeout!"); 4043 } 4044 } 4045 4046 /* dequeue the remaining */ 4047 time_out = 0; 4048 while (deq < enq) { 4049 deq += rte_bbdev_dequeue_dec_ops(tp->dev_id, 4050 queue_id, &ops_deq[deq], enq - deq); 4051 time_out++; 4052 if (time_out >= TIME_OUT_POLL) { 4053 timeout_exit(tp->dev_id); 4054 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 4055 } 4056 } 4057 4058 total_time += rte_rdtsc_precise() - start_time; 4059 } 4060 4061 tp->iter_count = 0; 4062 /* get the max of iter_count for all dequeued ops */ 4063 for (i = 0; i < num_ops; ++i) { 4064 tp->iter_count = RTE_MAX(ops_enq[i]->turbo_dec.iter_count, 4065 tp->iter_count); 4066 } 4067 4068 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 4069 ret = validate_dec_op(ops_deq, num_ops, ref_op); 4070 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 4071 } 4072 4073 rte_bbdev_dec_op_free_bulk(ops_enq, num_ops); 4074 4075 double tb_len_bits = calc_dec_TB_size(ref_op); 4076 4077 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) / 4078 ((double)total_time / (double)rte_get_tsc_hz()); 4079 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) / 4080 1000000.0) / ((double)total_time / 4081 (double)rte_get_tsc_hz()); 4082 4083 return TEST_SUCCESS; 4084 } 4085 4086 static int 4087 bler_pmd_lcore_ldpc_dec(void *arg) 4088 { 4089 struct thread_params *tp = arg; 4090 uint16_t enq, deq; 4091 uint64_t total_time = 0, start_time; 4092 const uint16_t queue_id = tp->queue_id; 4093 const uint16_t burst_sz = tp->op_params->burst_sz; 4094 const uint16_t num_ops = tp->op_params->num_to_process; 4095 struct rte_bbdev_dec_op *ops_enq[num_ops]; 4096 struct rte_bbdev_dec_op *ops_deq[num_ops]; 4097 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 4098 struct test_buffers *bufs = NULL; 4099 int i, j, ret; 4100 float parity_bler = 0; 4101 struct rte_bbdev_info info; 4102 uint16_t num_to_enq; 4103 bool extDdr = check_bit(ldpc_cap_flags, 4104 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE); 4105 bool loopback = check_bit(ref_op->ldpc_dec.op_flags, 4106 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK); 4107 bool hc_out = check_bit(ref_op->ldpc_dec.op_flags, 4108 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE); 4109 4110 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 4111 "BURST_SIZE should be <= %u", MAX_BURST); 4112 TEST_ASSERT_SUCCESS((num_ops == 0), "NUM_OPS must be greater than 0"); 4113 4114 rte_bbdev_info_get(tp->dev_id, &info); 4115 4116 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 4117 "NUM_OPS cannot exceed %u for this device", 4118 info.drv.queue_size_lim); 4119 4120 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 4121 4122 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->op_params->sync, SYNC_START, 4123 rte_memory_order_relaxed); 4124 4125 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops); 4126 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops); 4127 4128 /* For BLER tests we need to enable early termination */ 4129 if (!check_bit(ref_op->ldpc_dec.op_flags, RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE)) 4130 ref_op->ldpc_dec.op_flags += RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE; 4131 4132 ref_op->ldpc_dec.iter_max = get_iter_max(); 4133 4134 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4135 copy_reference_ldpc_dec_op(ops_enq, num_ops, 0, bufs->inputs, 4136 bufs->hard_outputs, bufs->soft_outputs, 4137 bufs->harq_inputs, bufs->harq_outputs, ref_op); 4138 generate_llr_input(num_ops, bufs->inputs, ref_op); 4139 4140 /* Set counter to validate the ordering */ 4141 for (j = 0; j < num_ops; ++j) 4142 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 4143 4144 for (i = 0; i < 1; ++i) { /* Could add more iterations */ 4145 uint32_t time_out = 0; 4146 for (j = 0; j < num_ops; ++j) { 4147 if (!loopback) 4148 mbuf_reset(ops_enq[j]->ldpc_dec.hard_output.data); 4149 if (hc_out || loopback) 4150 mbuf_reset(ops_enq[j]->ldpc_dec.harq_combined_output.data); 4151 if (ops_enq[j]->ldpc_dec.soft_output.data != NULL) 4152 mbuf_reset(ops_enq[j]->ldpc_dec.soft_output.data); 4153 } 4154 if (extDdr) 4155 preload_harq_ddr(tp->dev_id, queue_id, ops_enq, 4156 num_ops, true); 4157 start_time = rte_rdtsc_precise(); 4158 4159 for (enq = 0, deq = 0; enq < num_ops;) { 4160 num_to_enq = burst_sz; 4161 4162 if (unlikely(num_ops - enq < num_to_enq)) 4163 num_to_enq = num_ops - enq; 4164 4165 enq += rte_bbdev_enqueue_ldpc_dec_ops(tp->dev_id, 4166 queue_id, &ops_enq[enq], num_to_enq); 4167 4168 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id, 4169 queue_id, &ops_deq[deq], enq - deq); 4170 time_out++; 4171 if (time_out >= TIME_OUT_POLL) { 4172 timeout_exit(tp->dev_id); 4173 TEST_ASSERT_SUCCESS(TEST_FAILED, "Enqueue timeout!"); 4174 } 4175 } 4176 4177 /* dequeue the remaining */ 4178 time_out = 0; 4179 while (deq < enq) { 4180 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id, 4181 queue_id, &ops_deq[deq], enq - deq); 4182 time_out++; 4183 if (time_out >= TIME_OUT_POLL) { 4184 timeout_exit(tp->dev_id); 4185 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 4186 } 4187 } 4188 4189 total_time += rte_rdtsc_precise() - start_time; 4190 } 4191 4192 tp->iter_count = 0; 4193 tp->iter_average = 0; 4194 /* get the max of iter_count for all dequeued ops */ 4195 for (i = 0; i < num_ops; ++i) { 4196 tp->iter_count = RTE_MAX(ops_enq[i]->ldpc_dec.iter_count, 4197 tp->iter_count); 4198 tp->iter_average += (double) ops_enq[i]->ldpc_dec.iter_count; 4199 if (ops_enq[i]->status & (1 << RTE_BBDEV_SYNDROME_ERROR)) 4200 parity_bler += 1.0; 4201 } 4202 4203 parity_bler /= num_ops; /* This one is based on SYND */ 4204 tp->iter_average /= num_ops; 4205 tp->bler = (double) validate_ldpc_bler(ops_deq, num_ops) / num_ops; 4206 4207 if (test_vector.op_type != RTE_BBDEV_OP_NONE 4208 && tp->bler == 0 4209 && parity_bler == 0 4210 && !hc_out) { 4211 ret = validate_ldpc_dec_op(ops_deq, num_ops, ref_op, 4212 tp->op_params->vector_mask); 4213 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 4214 } 4215 4216 rte_bbdev_dec_op_free_bulk(ops_enq, num_ops); 4217 4218 double tb_len_bits = calc_ldpc_dec_TB_size(ref_op); 4219 tp->ops_per_sec = ((double)num_ops * 1) / 4220 ((double)total_time / (double)rte_get_tsc_hz()); 4221 tp->mbps = (((double)(num_ops * 1 * tb_len_bits)) / 4222 1000000.0) / ((double)total_time / 4223 (double)rte_get_tsc_hz()); 4224 4225 return TEST_SUCCESS; 4226 } 4227 4228 4229 static int 4230 bler_pmd_lcore_turbo_dec(void *arg) 4231 { 4232 struct thread_params *tp = arg; 4233 uint16_t enq, deq; 4234 uint64_t total_time = 0, start_time; 4235 const uint16_t queue_id = tp->queue_id; 4236 const uint16_t burst_sz = tp->op_params->burst_sz; 4237 const uint16_t num_ops = tp->op_params->num_to_process; 4238 struct rte_bbdev_dec_op *ops_enq[num_ops]; 4239 struct rte_bbdev_dec_op *ops_deq[num_ops]; 4240 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 4241 struct test_buffers *bufs = NULL; 4242 int i, j, ret; 4243 struct rte_bbdev_info info; 4244 uint16_t num_to_enq; 4245 4246 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 4247 "BURST_SIZE should be <= %u", MAX_BURST); 4248 TEST_ASSERT_SUCCESS((num_ops == 0), "NUM_OPS must be greater than 0"); 4249 4250 rte_bbdev_info_get(tp->dev_id, &info); 4251 4252 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 4253 "NUM_OPS cannot exceed %u for this device", 4254 info.drv.queue_size_lim); 4255 4256 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 4257 4258 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->op_params->sync, SYNC_START, 4259 rte_memory_order_relaxed); 4260 4261 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops); 4262 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops); 4263 4264 /* For BLER tests we need to enable early termination */ 4265 if (!check_bit(ref_op->turbo_dec.op_flags, RTE_BBDEV_TURBO_EARLY_TERMINATION)) 4266 ref_op->turbo_dec.op_flags += RTE_BBDEV_TURBO_EARLY_TERMINATION; 4267 4268 ref_op->turbo_dec.iter_max = get_iter_max(); 4269 4270 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4271 copy_reference_dec_op(ops_enq, num_ops, 0, bufs->inputs, 4272 bufs->hard_outputs, bufs->soft_outputs, 4273 ref_op); 4274 generate_turbo_llr_input(num_ops, bufs->inputs, ref_op); 4275 4276 /* Set counter to validate the ordering */ 4277 for (j = 0; j < num_ops; ++j) 4278 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 4279 4280 for (i = 0; i < 1; ++i) { /* Could add more iterations */ 4281 uint32_t time_out = 0; 4282 for (j = 0; j < num_ops; ++j) { 4283 mbuf_reset( 4284 ops_enq[j]->turbo_dec.hard_output.data); 4285 } 4286 4287 start_time = rte_rdtsc_precise(); 4288 4289 for (enq = 0, deq = 0; enq < num_ops;) { 4290 num_to_enq = burst_sz; 4291 4292 if (unlikely(num_ops - enq < num_to_enq)) 4293 num_to_enq = num_ops - enq; 4294 4295 enq += rte_bbdev_enqueue_dec_ops(tp->dev_id, 4296 queue_id, &ops_enq[enq], num_to_enq); 4297 4298 deq += rte_bbdev_dequeue_dec_ops(tp->dev_id, 4299 queue_id, &ops_deq[deq], enq - deq); 4300 time_out++; 4301 if (time_out >= TIME_OUT_POLL) { 4302 timeout_exit(tp->dev_id); 4303 TEST_ASSERT_SUCCESS(TEST_FAILED, "Enqueue timeout!"); 4304 } 4305 } 4306 4307 /* dequeue the remaining */ 4308 time_out = 0; 4309 while (deq < enq) { 4310 deq += rte_bbdev_dequeue_dec_ops(tp->dev_id, 4311 queue_id, &ops_deq[deq], enq - deq); 4312 time_out++; 4313 if (time_out >= TIME_OUT_POLL) { 4314 timeout_exit(tp->dev_id); 4315 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 4316 } 4317 } 4318 4319 total_time += rte_rdtsc_precise() - start_time; 4320 } 4321 4322 tp->iter_count = 0; 4323 tp->iter_average = 0; 4324 /* get the max of iter_count for all dequeued ops */ 4325 for (i = 0; i < num_ops; ++i) { 4326 tp->iter_count = RTE_MAX(ops_enq[i]->turbo_dec.iter_count, 4327 tp->iter_count); 4328 tp->iter_average += (double) ops_enq[i]->turbo_dec.iter_count; 4329 } 4330 4331 tp->iter_average /= num_ops; 4332 tp->bler = (double) validate_turbo_bler(ops_deq, num_ops) / num_ops; 4333 4334 rte_bbdev_dec_op_free_bulk(ops_enq, num_ops); 4335 4336 double tb_len_bits = calc_dec_TB_size(ref_op); 4337 tp->ops_per_sec = ((double)num_ops * 1) / 4338 ((double)total_time / (double)rte_get_tsc_hz()); 4339 tp->mbps = (((double)(num_ops * 1 * tb_len_bits)) / 4340 1000000.0) / ((double)total_time / 4341 (double)rte_get_tsc_hz()); 4342 printf("TBS %.0f Time %.0f\n", tb_len_bits, 1000000.0 * 4343 ((double)total_time / (double)rte_get_tsc_hz())); 4344 4345 return TEST_SUCCESS; 4346 } 4347 4348 static int 4349 throughput_pmd_lcore_ldpc_dec(void *arg) 4350 { 4351 struct thread_params *tp = arg; 4352 uint16_t enq, deq; 4353 uint64_t total_time = 0, start_time; 4354 const uint16_t queue_id = tp->queue_id; 4355 const uint16_t burst_sz = tp->op_params->burst_sz; 4356 const uint16_t num_ops = tp->op_params->num_to_process; 4357 struct rte_bbdev_dec_op *ops_enq[num_ops]; 4358 struct rte_bbdev_dec_op *ops_deq[num_ops]; 4359 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 4360 struct test_buffers *bufs = NULL; 4361 int i, j, ret; 4362 struct rte_bbdev_info info; 4363 uint16_t num_to_enq; 4364 bool extDdr = check_bit(ldpc_cap_flags, 4365 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE); 4366 bool loopback = check_bit(ref_op->ldpc_dec.op_flags, 4367 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK); 4368 bool hc_out = check_bit(ref_op->ldpc_dec.op_flags, 4369 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE); 4370 4371 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 4372 "BURST_SIZE should be <= %u", MAX_BURST); 4373 4374 rte_bbdev_info_get(tp->dev_id, &info); 4375 4376 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 4377 "NUM_OPS cannot exceed %u for this device", 4378 info.drv.queue_size_lim); 4379 4380 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 4381 4382 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->op_params->sync, SYNC_START, 4383 rte_memory_order_relaxed); 4384 4385 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops); 4386 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops); 4387 4388 /* For throughput tests we need to disable early termination */ 4389 if (check_bit(ref_op->ldpc_dec.op_flags, RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE)) 4390 ref_op->ldpc_dec.op_flags -= RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE; 4391 4392 ref_op->ldpc_dec.iter_max = get_iter_max(); 4393 /* Since ET is disabled, the expected iter_count is iter_max */ 4394 ref_op->ldpc_dec.iter_count = ref_op->ldpc_dec.iter_max; 4395 4396 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4397 copy_reference_ldpc_dec_op(ops_enq, num_ops, 0, bufs->inputs, 4398 bufs->hard_outputs, bufs->soft_outputs, 4399 bufs->harq_inputs, bufs->harq_outputs, ref_op); 4400 4401 /* Set counter to validate the ordering */ 4402 for (j = 0; j < num_ops; ++j) 4403 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 4404 4405 for (i = 0; i < TEST_REPETITIONS; ++i) { 4406 uint32_t time_out = 0; 4407 for (j = 0; j < num_ops; ++j) { 4408 if (!loopback) 4409 mbuf_reset(ops_enq[j]->ldpc_dec.hard_output.data); 4410 if (hc_out || loopback) 4411 mbuf_reset(ops_enq[j]->ldpc_dec.harq_combined_output.data); 4412 if (ops_enq[j]->ldpc_dec.soft_output.data != NULL) 4413 mbuf_reset(ops_enq[j]->ldpc_dec.soft_output.data); 4414 } 4415 if (extDdr) 4416 preload_harq_ddr(tp->dev_id, queue_id, ops_enq, 4417 num_ops, true); 4418 start_time = rte_rdtsc_precise(); 4419 4420 for (enq = 0, deq = 0; enq < num_ops;) { 4421 num_to_enq = burst_sz; 4422 4423 if (unlikely(num_ops - enq < num_to_enq)) 4424 num_to_enq = num_ops - enq; 4425 4426 enq += rte_bbdev_enqueue_ldpc_dec_ops(tp->dev_id, 4427 queue_id, &ops_enq[enq], num_to_enq); 4428 4429 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id, 4430 queue_id, &ops_deq[deq], enq - deq); 4431 time_out++; 4432 if (time_out >= TIME_OUT_POLL) { 4433 timeout_exit(tp->dev_id); 4434 TEST_ASSERT_SUCCESS(TEST_FAILED, "Enqueue timeout!"); 4435 } 4436 } 4437 4438 /* dequeue the remaining */ 4439 time_out = 0; 4440 while (deq < enq) { 4441 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id, 4442 queue_id, &ops_deq[deq], enq - deq); 4443 time_out++; 4444 if (time_out >= TIME_OUT_POLL) { 4445 timeout_exit(tp->dev_id); 4446 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 4447 } 4448 } 4449 4450 total_time += rte_rdtsc_precise() - start_time; 4451 } 4452 4453 tp->iter_count = 0; 4454 /* get the max of iter_count for all dequeued ops */ 4455 for (i = 0; i < num_ops; ++i) { 4456 tp->iter_count = RTE_MAX(ops_enq[i]->ldpc_dec.iter_count, 4457 tp->iter_count); 4458 } 4459 if (extDdr) { 4460 /* Read loopback is not thread safe */ 4461 retrieve_harq_ddr(tp->dev_id, queue_id, ops_enq, num_ops); 4462 } 4463 4464 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 4465 ret = validate_ldpc_dec_op(ops_deq, num_ops, ref_op, 4466 tp->op_params->vector_mask); 4467 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 4468 } 4469 4470 ret = rte_bbdev_queue_stop(tp->dev_id, queue_id); 4471 if (ret != 0) 4472 printf("Failed to stop queue on dev %u q_id: %u\n", tp->dev_id, queue_id); 4473 rte_bbdev_dec_op_free_bulk(ops_enq, num_ops); 4474 4475 double tb_len_bits = calc_ldpc_dec_TB_size(ref_op); 4476 4477 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) / 4478 ((double)total_time / (double)rte_get_tsc_hz()); 4479 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) / 4480 1000000.0) / ((double)total_time / 4481 (double)rte_get_tsc_hz()); 4482 4483 return TEST_SUCCESS; 4484 } 4485 4486 static int 4487 throughput_pmd_lcore_enc(void *arg) 4488 { 4489 struct thread_params *tp = arg; 4490 uint16_t enq, deq; 4491 uint64_t total_time = 0, start_time; 4492 const uint16_t queue_id = tp->queue_id; 4493 const uint16_t burst_sz = tp->op_params->burst_sz; 4494 const uint16_t num_ops = tp->op_params->num_to_process; 4495 struct rte_bbdev_enc_op *ops_enq[num_ops]; 4496 struct rte_bbdev_enc_op *ops_deq[num_ops]; 4497 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op; 4498 struct test_buffers *bufs = NULL; 4499 int i, j, ret; 4500 struct rte_bbdev_info info; 4501 uint16_t num_to_enq; 4502 4503 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 4504 "BURST_SIZE should be <= %u", MAX_BURST); 4505 4506 rte_bbdev_info_get(tp->dev_id, &info); 4507 4508 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 4509 "NUM_OPS cannot exceed %u for this device", 4510 info.drv.queue_size_lim); 4511 4512 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 4513 4514 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->op_params->sync, SYNC_START, 4515 rte_memory_order_relaxed); 4516 4517 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops_enq, 4518 num_ops); 4519 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", 4520 num_ops); 4521 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4522 copy_reference_enc_op(ops_enq, num_ops, 0, bufs->inputs, 4523 bufs->hard_outputs, ref_op); 4524 4525 /* Set counter to validate the ordering */ 4526 for (j = 0; j < num_ops; ++j) 4527 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 4528 4529 for (i = 0; i < TEST_REPETITIONS; ++i) { 4530 uint32_t time_out = 0; 4531 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4532 for (j = 0; j < num_ops; ++j) 4533 mbuf_reset(ops_enq[j]->turbo_enc.output.data); 4534 4535 start_time = rte_rdtsc_precise(); 4536 4537 for (enq = 0, deq = 0; enq < num_ops;) { 4538 num_to_enq = burst_sz; 4539 4540 if (unlikely(num_ops - enq < num_to_enq)) 4541 num_to_enq = num_ops - enq; 4542 4543 enq += rte_bbdev_enqueue_enc_ops(tp->dev_id, 4544 queue_id, &ops_enq[enq], num_to_enq); 4545 4546 deq += rte_bbdev_dequeue_enc_ops(tp->dev_id, 4547 queue_id, &ops_deq[deq], enq - deq); 4548 time_out++; 4549 if (time_out >= TIME_OUT_POLL) { 4550 timeout_exit(tp->dev_id); 4551 TEST_ASSERT_SUCCESS(TEST_FAILED, "Enqueue timeout!"); 4552 } 4553 } 4554 4555 /* dequeue the remaining */ 4556 time_out = 0; 4557 while (deq < enq) { 4558 deq += rte_bbdev_dequeue_enc_ops(tp->dev_id, 4559 queue_id, &ops_deq[deq], enq - deq); 4560 time_out++; 4561 if (time_out >= TIME_OUT_POLL) { 4562 timeout_exit(tp->dev_id); 4563 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 4564 } 4565 } 4566 4567 total_time += rte_rdtsc_precise() - start_time; 4568 } 4569 4570 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 4571 ret = validate_enc_op(ops_deq, num_ops, ref_op); 4572 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 4573 } 4574 4575 rte_bbdev_enc_op_free_bulk(ops_enq, num_ops); 4576 4577 double tb_len_bits = calc_enc_TB_size(ref_op); 4578 4579 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) / 4580 ((double)total_time / (double)rte_get_tsc_hz()); 4581 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) 4582 / 1000000.0) / ((double)total_time / 4583 (double)rte_get_tsc_hz()); 4584 4585 return TEST_SUCCESS; 4586 } 4587 4588 static int 4589 throughput_pmd_lcore_ldpc_enc(void *arg) 4590 { 4591 struct thread_params *tp = arg; 4592 uint16_t enq, deq; 4593 uint64_t total_time = 0, start_time; 4594 const uint16_t queue_id = tp->queue_id; 4595 const uint16_t burst_sz = tp->op_params->burst_sz; 4596 const uint16_t num_ops = tp->op_params->num_to_process; 4597 struct rte_bbdev_enc_op *ops_enq[num_ops]; 4598 struct rte_bbdev_enc_op *ops_deq[num_ops]; 4599 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op; 4600 struct test_buffers *bufs = NULL; 4601 int i, j, ret; 4602 struct rte_bbdev_info info; 4603 uint16_t num_to_enq; 4604 4605 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 4606 "BURST_SIZE should be <= %u", MAX_BURST); 4607 4608 rte_bbdev_info_get(tp->dev_id, &info); 4609 4610 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 4611 "NUM_OPS cannot exceed %u for this device", 4612 info.drv.queue_size_lim); 4613 4614 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 4615 4616 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->op_params->sync, SYNC_START, 4617 rte_memory_order_relaxed); 4618 4619 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops_enq, 4620 num_ops); 4621 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", 4622 num_ops); 4623 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4624 copy_reference_ldpc_enc_op(ops_enq, num_ops, 0, bufs->inputs, 4625 bufs->hard_outputs, ref_op); 4626 4627 /* Set counter to validate the ordering */ 4628 for (j = 0; j < num_ops; ++j) 4629 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 4630 4631 for (i = 0; i < TEST_REPETITIONS; ++i) { 4632 uint32_t time_out = 0; 4633 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4634 for (j = 0; j < num_ops; ++j) 4635 mbuf_reset(ops_enq[j]->turbo_enc.output.data); 4636 4637 start_time = rte_rdtsc_precise(); 4638 4639 for (enq = 0, deq = 0; enq < num_ops;) { 4640 num_to_enq = burst_sz; 4641 4642 if (unlikely(num_ops - enq < num_to_enq)) 4643 num_to_enq = num_ops - enq; 4644 4645 enq += rte_bbdev_enqueue_ldpc_enc_ops(tp->dev_id, 4646 queue_id, &ops_enq[enq], num_to_enq); 4647 4648 deq += rte_bbdev_dequeue_ldpc_enc_ops(tp->dev_id, 4649 queue_id, &ops_deq[deq], enq - deq); 4650 time_out++; 4651 if (time_out >= TIME_OUT_POLL) { 4652 timeout_exit(tp->dev_id); 4653 TEST_ASSERT_SUCCESS(TEST_FAILED, "Enqueue timeout!"); 4654 } 4655 } 4656 4657 /* dequeue the remaining */ 4658 time_out = 0; 4659 while (deq < enq) { 4660 deq += rte_bbdev_dequeue_ldpc_enc_ops(tp->dev_id, 4661 queue_id, &ops_deq[deq], enq - deq); 4662 time_out++; 4663 if (time_out >= TIME_OUT_POLL) { 4664 timeout_exit(tp->dev_id); 4665 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 4666 } 4667 } 4668 4669 total_time += rte_rdtsc_precise() - start_time; 4670 } 4671 4672 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 4673 ret = validate_ldpc_enc_op(ops_deq, num_ops, ref_op); 4674 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 4675 } 4676 4677 rte_bbdev_enc_op_free_bulk(ops_enq, num_ops); 4678 4679 double tb_len_bits = calc_ldpc_enc_TB_size(ref_op); 4680 4681 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) / 4682 ((double)total_time / (double)rte_get_tsc_hz()); 4683 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) 4684 / 1000000.0) / ((double)total_time / 4685 (double)rte_get_tsc_hz()); 4686 4687 return TEST_SUCCESS; 4688 } 4689 4690 static int 4691 throughput_pmd_lcore_fft(void *arg) 4692 { 4693 struct thread_params *tp = arg; 4694 uint16_t enq, deq; 4695 uint64_t total_time = 0, start_time; 4696 const uint16_t queue_id = tp->queue_id; 4697 const uint16_t burst_sz = tp->op_params->burst_sz; 4698 const uint16_t num_ops = tp->op_params->num_to_process; 4699 struct rte_bbdev_fft_op *ops_enq[num_ops]; 4700 struct rte_bbdev_fft_op *ops_deq[num_ops]; 4701 struct rte_bbdev_fft_op *ref_op = tp->op_params->ref_fft_op; 4702 struct test_buffers *bufs = NULL; 4703 int i, j, ret; 4704 struct rte_bbdev_info info; 4705 uint16_t num_to_enq; 4706 4707 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 4708 "BURST_SIZE should be <= %u", MAX_BURST); 4709 4710 rte_bbdev_info_get(tp->dev_id, &info); 4711 4712 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 4713 "NUM_OPS cannot exceed %u for this device", 4714 info.drv.queue_size_lim); 4715 4716 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 4717 4718 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->op_params->sync, SYNC_START, 4719 rte_memory_order_relaxed); 4720 4721 ret = rte_bbdev_fft_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops); 4722 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops); 4723 4724 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4725 copy_reference_fft_op(ops_enq, num_ops, 0, bufs->inputs, 4726 bufs->hard_outputs, bufs->soft_outputs, bufs->harq_inputs, ref_op); 4727 4728 /* Set counter to validate the ordering */ 4729 for (j = 0; j < num_ops; ++j) 4730 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 4731 4732 for (i = 0; i < TEST_REPETITIONS; ++i) { 4733 uint32_t time_out = 0; 4734 for (j = 0; j < num_ops; ++j) 4735 mbuf_reset(ops_enq[j]->fft.base_output.data); 4736 4737 start_time = rte_rdtsc_precise(); 4738 4739 for (enq = 0, deq = 0; enq < num_ops;) { 4740 num_to_enq = burst_sz; 4741 4742 if (unlikely(num_ops - enq < num_to_enq)) 4743 num_to_enq = num_ops - enq; 4744 4745 enq += rte_bbdev_enqueue_fft_ops(tp->dev_id, 4746 queue_id, &ops_enq[enq], num_to_enq); 4747 4748 deq += rte_bbdev_dequeue_fft_ops(tp->dev_id, 4749 queue_id, &ops_deq[deq], enq - deq); 4750 time_out++; 4751 if (time_out >= TIME_OUT_POLL) { 4752 timeout_exit(tp->dev_id); 4753 TEST_ASSERT_SUCCESS(TEST_FAILED, "Enqueue timeout!"); 4754 } 4755 } 4756 4757 /* dequeue the remaining */ 4758 time_out = 0; 4759 while (deq < enq) { 4760 deq += rte_bbdev_dequeue_fft_ops(tp->dev_id, 4761 queue_id, &ops_deq[deq], enq - deq); 4762 time_out++; 4763 if (time_out >= TIME_OUT_POLL) { 4764 timeout_exit(tp->dev_id); 4765 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 4766 } 4767 } 4768 4769 total_time += rte_rdtsc_precise() - start_time; 4770 } 4771 4772 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 4773 ret = validate_fft_op(ops_deq, num_ops, ref_op); 4774 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 4775 } 4776 4777 rte_bbdev_fft_op_free_bulk(ops_enq, num_ops); 4778 4779 double tb_len_bits = calc_fft_size(ref_op); 4780 4781 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) / 4782 ((double)total_time / (double)rte_get_tsc_hz()); 4783 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) / 4784 1000000.0) / ((double)total_time / 4785 (double)rte_get_tsc_hz()); 4786 4787 return TEST_SUCCESS; 4788 } 4789 4790 static int 4791 throughput_pmd_lcore_mldts(void *arg) 4792 { 4793 struct thread_params *tp = arg; 4794 uint16_t enq, deq; 4795 uint64_t total_time = 0, start_time; 4796 const uint16_t queue_id = tp->queue_id; 4797 const uint16_t burst_sz = tp->op_params->burst_sz; 4798 const uint16_t num_ops = tp->op_params->num_to_process; 4799 struct rte_bbdev_mldts_op *ops_enq[num_ops]; 4800 struct rte_bbdev_mldts_op *ops_deq[num_ops]; 4801 struct rte_bbdev_mldts_op *ref_op = tp->op_params->ref_mldts_op; 4802 struct test_buffers *bufs = NULL; 4803 int i, j, ret; 4804 struct rte_bbdev_info info; 4805 uint16_t num_to_enq; 4806 4807 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), "BURST_SIZE should be <= %u", MAX_BURST); 4808 4809 rte_bbdev_info_get(tp->dev_id, &info); 4810 4811 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 4812 "NUM_OPS cannot exceed %u for this device", 4813 info.drv.queue_size_lim); 4814 4815 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 4816 4817 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->op_params->sync, SYNC_START, 4818 rte_memory_order_relaxed); 4819 4820 ret = rte_bbdev_mldts_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops); 4821 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops); 4822 4823 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4824 copy_reference_mldts_op(ops_enq, num_ops, 0, bufs->inputs, bufs->harq_inputs, 4825 bufs->hard_outputs, ref_op); 4826 4827 /* Set counter to validate the ordering */ 4828 for (j = 0; j < num_ops; ++j) 4829 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 4830 4831 for (i = 0; i < TEST_REPETITIONS; ++i) { 4832 uint32_t time_out = 0; 4833 for (j = 0; j < num_ops; ++j) 4834 mbuf_reset(ops_enq[j]->mldts.output.data); 4835 4836 start_time = rte_rdtsc_precise(); 4837 4838 for (enq = 0, deq = 0; enq < num_ops;) { 4839 num_to_enq = burst_sz; 4840 4841 if (unlikely(num_ops - enq < num_to_enq)) 4842 num_to_enq = num_ops - enq; 4843 4844 enq += rte_bbdev_enqueue_mldts_ops(tp->dev_id, 4845 queue_id, &ops_enq[enq], num_to_enq); 4846 4847 deq += rte_bbdev_dequeue_mldts_ops(tp->dev_id, 4848 queue_id, &ops_deq[deq], enq - deq); 4849 time_out++; 4850 if (time_out >= TIME_OUT_POLL) { 4851 timeout_exit(tp->dev_id); 4852 TEST_ASSERT_SUCCESS(TEST_FAILED, "Enqueue timeout!"); 4853 } 4854 } 4855 4856 /* dequeue the remaining */ 4857 time_out = 0; 4858 while (deq < enq) { 4859 deq += rte_bbdev_dequeue_mldts_ops(tp->dev_id, 4860 queue_id, &ops_deq[deq], enq - deq); 4861 time_out++; 4862 if (time_out >= TIME_OUT_POLL) { 4863 timeout_exit(tp->dev_id); 4864 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 4865 } 4866 } 4867 4868 total_time += rte_rdtsc_precise() - start_time; 4869 } 4870 4871 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 4872 ret = validate_mldts_op(ops_deq, num_ops, ref_op); 4873 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 4874 } 4875 4876 rte_bbdev_mldts_op_free_bulk(ops_enq, num_ops); 4877 4878 double tb_len_bits = calc_mldts_size(ref_op); 4879 4880 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) / 4881 ((double)total_time / (double)rte_get_tsc_hz()); 4882 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) / 4883 1000000.0) / ((double)total_time / 4884 (double)rte_get_tsc_hz()); 4885 4886 return TEST_SUCCESS; 4887 } 4888 4889 static void 4890 print_enc_throughput(struct thread_params *t_params, unsigned int used_cores) 4891 { 4892 unsigned int iter = 0; 4893 double total_mops = 0, total_mbps = 0; 4894 4895 for (iter = 0; iter < used_cores; iter++) { 4896 printf( 4897 "Throughput for core (%u): %.8lg Ops/s, %.8lg Mbps\n", 4898 t_params[iter].lcore_id, t_params[iter].ops_per_sec, 4899 t_params[iter].mbps); 4900 total_mops += t_params[iter].ops_per_sec; 4901 total_mbps += t_params[iter].mbps; 4902 } 4903 printf( 4904 "\nTotal throughput for %u cores: %.8lg MOPS, %.8lg Mbps\n", 4905 used_cores, total_mops, total_mbps); 4906 } 4907 4908 /* Aggregate the performance results over the number of cores used */ 4909 static void 4910 print_dec_throughput(struct thread_params *t_params, unsigned int used_cores) 4911 { 4912 unsigned int core_idx = 0; 4913 double total_mops = 0, total_mbps = 0; 4914 uint8_t iter_count = 0; 4915 4916 for (core_idx = 0; core_idx < used_cores; core_idx++) { 4917 printf( 4918 "Throughput for core (%u): %.8lg Ops/s, %.8lg Mbps @ max %u iterations\n", 4919 t_params[core_idx].lcore_id, 4920 t_params[core_idx].ops_per_sec, 4921 t_params[core_idx].mbps, 4922 t_params[core_idx].iter_count); 4923 total_mops += t_params[core_idx].ops_per_sec; 4924 total_mbps += t_params[core_idx].mbps; 4925 iter_count = RTE_MAX(iter_count, 4926 t_params[core_idx].iter_count); 4927 } 4928 printf( 4929 "\nTotal throughput for %u cores: %.8lg MOPS, %.8lg Mbps @ max %u iterations\n", 4930 used_cores, total_mops, total_mbps, iter_count); 4931 } 4932 4933 /* Aggregate the performance results over the number of cores used */ 4934 static void 4935 print_dec_bler(struct thread_params *t_params, unsigned int used_cores) 4936 { 4937 unsigned int core_idx = 0; 4938 double total_mbps = 0, total_bler = 0, total_iter = 0; 4939 double snr = get_snr(); 4940 4941 for (core_idx = 0; core_idx < used_cores; core_idx++) { 4942 printf("Core%u BLER %.1f %% - Iters %.1f - Tp %.1f Mbps %s\n", 4943 t_params[core_idx].lcore_id, 4944 t_params[core_idx].bler * 100, 4945 t_params[core_idx].iter_average, 4946 t_params[core_idx].mbps, 4947 get_vector_filename()); 4948 total_mbps += t_params[core_idx].mbps; 4949 total_bler += t_params[core_idx].bler; 4950 total_iter += t_params[core_idx].iter_average; 4951 } 4952 total_bler /= used_cores; 4953 total_iter /= used_cores; 4954 4955 printf("SNR %.2f BLER %.1f %% - Iterations %.1f %d - Tp %.3f Mbps %s\n", 4956 snr, total_bler * 100, total_iter, get_iter_max(), 4957 total_mbps, get_vector_filename()); 4958 } 4959 4960 /* 4961 * Test function that determines BLER wireless performance 4962 */ 4963 static int 4964 bler_test(struct active_device *ad, 4965 struct test_op_params *op_params) 4966 { 4967 int ret; 4968 unsigned int lcore_id, used_cores = 0; 4969 struct thread_params *t_params; 4970 struct rte_bbdev_info info; 4971 lcore_function_t *bler_function; 4972 uint16_t num_lcores; 4973 const char *op_type_str; 4974 4975 rte_bbdev_info_get(ad->dev_id, &info); 4976 4977 op_type_str = rte_bbdev_op_type_str(test_vector.op_type); 4978 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", 4979 test_vector.op_type); 4980 4981 printf("+ ------------------------------------------------------- +\n"); 4982 printf("== test: bler\ndev: %s, nb_queues: %u, burst size: %u, num ops: %u, num_lcores: %u, op type: %s, itr mode: %s, GHz: %lg\n", 4983 info.dev_name, ad->nb_queues, op_params->burst_sz, 4984 op_params->num_to_process, op_params->num_lcores, 4985 op_type_str, 4986 intr_enabled ? "Interrupt mode" : "PMD mode", 4987 (double)rte_get_tsc_hz() / 1000000000.0); 4988 4989 /* Set number of lcores */ 4990 num_lcores = (ad->nb_queues < (op_params->num_lcores)) 4991 ? ad->nb_queues 4992 : op_params->num_lcores; 4993 4994 /* Allocate memory for thread parameters structure */ 4995 t_params = rte_zmalloc(NULL, num_lcores * sizeof(struct thread_params), 4996 RTE_CACHE_LINE_SIZE); 4997 TEST_ASSERT_NOT_NULL(t_params, "Failed to alloc %zuB for t_params", 4998 RTE_ALIGN(sizeof(struct thread_params) * num_lcores, 4999 RTE_CACHE_LINE_SIZE)); 5000 5001 if ((test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) && 5002 !check_bit(test_vector.ldpc_dec.op_flags, 5003 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK) 5004 && !check_bit(test_vector.ldpc_dec.op_flags, 5005 RTE_BBDEV_LDPC_LLR_COMPRESSION)) 5006 bler_function = bler_pmd_lcore_ldpc_dec; 5007 else if ((test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) && 5008 !check_bit(test_vector.turbo_dec.op_flags, 5009 RTE_BBDEV_TURBO_SOFT_OUTPUT)) 5010 bler_function = bler_pmd_lcore_turbo_dec; 5011 else 5012 return TEST_SKIPPED; 5013 5014 rte_atomic_store_explicit(&op_params->sync, SYNC_WAIT, rte_memory_order_relaxed); 5015 5016 /* Main core is set at first entry */ 5017 t_params[0].dev_id = ad->dev_id; 5018 t_params[0].lcore_id = rte_lcore_id(); 5019 t_params[0].op_params = op_params; 5020 t_params[0].queue_id = ad->queue_ids[used_cores++]; 5021 t_params[0].iter_count = 0; 5022 5023 RTE_LCORE_FOREACH_WORKER(lcore_id) { 5024 if (used_cores >= num_lcores) 5025 break; 5026 5027 t_params[used_cores].dev_id = ad->dev_id; 5028 t_params[used_cores].lcore_id = lcore_id; 5029 t_params[used_cores].op_params = op_params; 5030 t_params[used_cores].queue_id = ad->queue_ids[used_cores]; 5031 t_params[used_cores].iter_count = 0; 5032 5033 rte_eal_remote_launch(bler_function, 5034 &t_params[used_cores++], lcore_id); 5035 } 5036 5037 rte_atomic_store_explicit(&op_params->sync, SYNC_START, rte_memory_order_relaxed); 5038 ret = bler_function(&t_params[0]); 5039 5040 /* Main core is always used */ 5041 for (used_cores = 1; used_cores < num_lcores; used_cores++) 5042 ret |= rte_eal_wait_lcore(t_params[used_cores].lcore_id); 5043 5044 print_dec_bler(t_params, num_lcores); 5045 5046 /* Return if test failed */ 5047 if (ret) { 5048 rte_free(t_params); 5049 return ret; 5050 } 5051 5052 /* Function to print something here*/ 5053 rte_free(t_params); 5054 return ret; 5055 } 5056 5057 /* 5058 * Test function that determines how long an enqueue + dequeue of a burst 5059 * takes on available lcores. 5060 */ 5061 static int 5062 throughput_test(struct active_device *ad, 5063 struct test_op_params *op_params) 5064 { 5065 int ret; 5066 unsigned int lcore_id, used_cores = 0; 5067 struct thread_params *t_params, *tp; 5068 struct rte_bbdev_info info; 5069 lcore_function_t *throughput_function; 5070 uint16_t num_lcores; 5071 const char *op_type_str; 5072 5073 rte_bbdev_info_get(ad->dev_id, &info); 5074 5075 op_type_str = rte_bbdev_op_type_str(test_vector.op_type); 5076 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", 5077 test_vector.op_type); 5078 5079 printf("+ ------------------------------------------------------- +\n"); 5080 printf("== test: throughput\ndev: %s, nb_queues: %u, burst size: %u, num ops: %u, num_lcores: %u, op type: %s, itr mode: %s, GHz: %lg\n", 5081 info.dev_name, ad->nb_queues, op_params->burst_sz, 5082 op_params->num_to_process, op_params->num_lcores, 5083 op_type_str, 5084 intr_enabled ? "Interrupt mode" : "PMD mode", 5085 (double)rte_get_tsc_hz() / 1000000000.0); 5086 5087 /* Set number of lcores */ 5088 num_lcores = (ad->nb_queues < (op_params->num_lcores)) 5089 ? ad->nb_queues 5090 : op_params->num_lcores; 5091 5092 /* Allocate memory for thread parameters structure */ 5093 t_params = rte_zmalloc(NULL, num_lcores * sizeof(struct thread_params), 5094 RTE_CACHE_LINE_SIZE); 5095 TEST_ASSERT_NOT_NULL(t_params, "Failed to alloc %zuB for t_params", 5096 RTE_ALIGN(sizeof(struct thread_params) * num_lcores, 5097 RTE_CACHE_LINE_SIZE)); 5098 5099 if (intr_enabled) { 5100 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) 5101 throughput_function = throughput_intr_lcore_dec; 5102 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 5103 throughput_function = throughput_intr_lcore_ldpc_dec; 5104 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) 5105 throughput_function = throughput_intr_lcore_enc; 5106 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) 5107 throughput_function = throughput_intr_lcore_ldpc_enc; 5108 else if (test_vector.op_type == RTE_BBDEV_OP_FFT) 5109 throughput_function = throughput_intr_lcore_fft; 5110 else if (test_vector.op_type == RTE_BBDEV_OP_MLDTS) 5111 throughput_function = throughput_intr_lcore_mldts; 5112 else 5113 throughput_function = throughput_intr_lcore_enc; 5114 5115 /* Dequeue interrupt callback registration */ 5116 ret = rte_bbdev_callback_register(ad->dev_id, 5117 RTE_BBDEV_EVENT_DEQUEUE, dequeue_event_callback, 5118 t_params); 5119 if (ret < 0) { 5120 rte_free(t_params); 5121 return ret; 5122 } 5123 } else { 5124 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) 5125 throughput_function = throughput_pmd_lcore_dec; 5126 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 5127 throughput_function = throughput_pmd_lcore_ldpc_dec; 5128 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) 5129 throughput_function = throughput_pmd_lcore_enc; 5130 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) 5131 throughput_function = throughput_pmd_lcore_ldpc_enc; 5132 else if (test_vector.op_type == RTE_BBDEV_OP_FFT) 5133 throughput_function = throughput_pmd_lcore_fft; 5134 else if (test_vector.op_type == RTE_BBDEV_OP_MLDTS) 5135 throughput_function = throughput_pmd_lcore_mldts; 5136 else 5137 throughput_function = throughput_pmd_lcore_enc; 5138 } 5139 5140 rte_atomic_store_explicit(&op_params->sync, SYNC_WAIT, rte_memory_order_relaxed); 5141 5142 /* Main core is set at first entry */ 5143 t_params[0].dev_id = ad->dev_id; 5144 t_params[0].lcore_id = rte_lcore_id(); 5145 t_params[0].op_params = op_params; 5146 t_params[0].queue_id = ad->queue_ids[used_cores++]; 5147 t_params[0].iter_count = 0; 5148 5149 RTE_LCORE_FOREACH_WORKER(lcore_id) { 5150 if (used_cores >= num_lcores) 5151 break; 5152 5153 t_params[used_cores].dev_id = ad->dev_id; 5154 t_params[used_cores].lcore_id = lcore_id; 5155 t_params[used_cores].op_params = op_params; 5156 t_params[used_cores].queue_id = ad->queue_ids[used_cores]; 5157 t_params[used_cores].iter_count = 0; 5158 5159 rte_eal_remote_launch(throughput_function, 5160 &t_params[used_cores++], lcore_id); 5161 } 5162 5163 rte_atomic_store_explicit(&op_params->sync, SYNC_START, rte_memory_order_relaxed); 5164 ret = throughput_function(&t_params[0]); 5165 5166 /* Main core is always used */ 5167 for (used_cores = 1; used_cores < num_lcores; used_cores++) 5168 ret |= rte_eal_wait_lcore(t_params[used_cores].lcore_id); 5169 5170 /* Return if test failed */ 5171 if (ret) { 5172 rte_free(t_params); 5173 return ret; 5174 } 5175 5176 /* Print throughput if interrupts are disabled and test passed */ 5177 if (!intr_enabled) { 5178 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC || 5179 test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 5180 print_dec_throughput(t_params, num_lcores); 5181 else 5182 print_enc_throughput(t_params, num_lcores); 5183 rte_free(t_params); 5184 return ret; 5185 } 5186 5187 /* In interrupt TC we need to wait for the interrupt callback to deqeue 5188 * all pending operations. Skip waiting for queues which reported an 5189 * error using processing_status variable. 5190 * Wait for main lcore operations. 5191 */ 5192 tp = &t_params[0]; 5193 while ((rte_atomic_load_explicit(&tp->nb_dequeued, rte_memory_order_relaxed) < 5194 op_params->num_to_process) && 5195 (rte_atomic_load_explicit(&tp->processing_status, rte_memory_order_relaxed) != 5196 TEST_FAILED)) 5197 rte_pause(); 5198 5199 tp->ops_per_sec /= TEST_REPETITIONS; 5200 tp->mbps /= TEST_REPETITIONS; 5201 ret |= (int)rte_atomic_load_explicit(&tp->processing_status, rte_memory_order_relaxed); 5202 5203 /* Wait for worker lcores operations */ 5204 for (used_cores = 1; used_cores < num_lcores; used_cores++) { 5205 tp = &t_params[used_cores]; 5206 5207 while ((rte_atomic_load_explicit(&tp->nb_dequeued, rte_memory_order_relaxed) < 5208 op_params->num_to_process) && 5209 (rte_atomic_load_explicit(&tp->processing_status, 5210 rte_memory_order_relaxed) != TEST_FAILED)) 5211 rte_pause(); 5212 5213 tp->ops_per_sec /= TEST_REPETITIONS; 5214 tp->mbps /= TEST_REPETITIONS; 5215 ret |= (int)rte_atomic_load_explicit(&tp->processing_status, 5216 rte_memory_order_relaxed); 5217 } 5218 5219 /* Print throughput if test passed */ 5220 if (!ret) { 5221 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC || 5222 test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 5223 print_dec_throughput(t_params, num_lcores); 5224 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC || 5225 test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) 5226 print_enc_throughput(t_params, num_lcores); 5227 } 5228 5229 rte_free(t_params); 5230 return ret; 5231 } 5232 5233 static int 5234 latency_test_dec(struct rte_mempool *mempool, 5235 struct test_buffers *bufs, struct rte_bbdev_dec_op *ref_op, 5236 uint16_t dev_id, uint16_t queue_id, 5237 const uint16_t num_to_process, uint16_t burst_sz, 5238 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time, bool disable_et) 5239 { 5240 int ret = TEST_SUCCESS; 5241 uint16_t i, j, dequeued; 5242 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 5243 uint64_t start_time = 0, last_time = 0; 5244 5245 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 5246 uint16_t enq = 0, deq = 0; 5247 uint32_t time_out = 0; 5248 bool first_time = true; 5249 last_time = 0; 5250 5251 if (unlikely(num_to_process - dequeued < burst_sz)) 5252 burst_sz = num_to_process - dequeued; 5253 5254 ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz); 5255 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_dec_op_alloc_bulk() failed"); 5256 5257 ref_op->turbo_dec.iter_max = get_iter_max(); 5258 /* For validation tests we want to enable early termination */ 5259 if (!disable_et && !check_bit(ref_op->turbo_dec.op_flags, 5260 RTE_BBDEV_TURBO_EARLY_TERMINATION)) 5261 ref_op->turbo_dec.op_flags |= RTE_BBDEV_TURBO_EARLY_TERMINATION; 5262 5263 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 5264 copy_reference_dec_op(ops_enq, burst_sz, dequeued, 5265 bufs->inputs, 5266 bufs->hard_outputs, 5267 bufs->soft_outputs, 5268 ref_op); 5269 5270 /* Set counter to validate the ordering */ 5271 for (j = 0; j < burst_sz; ++j) 5272 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 5273 5274 start_time = rte_rdtsc_precise(); 5275 5276 enq = rte_bbdev_enqueue_dec_ops(dev_id, queue_id, &ops_enq[enq], 5277 burst_sz); 5278 TEST_ASSERT(enq == burst_sz, 5279 "Error enqueueing burst, expected %u, got %u", 5280 burst_sz, enq); 5281 5282 /* Dequeue */ 5283 do { 5284 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id, 5285 &ops_deq[deq], burst_sz - deq); 5286 if (likely(first_time && (deq > 0))) { 5287 last_time = rte_rdtsc_precise() - start_time; 5288 first_time = false; 5289 } 5290 time_out++; 5291 if (time_out >= TIME_OUT_POLL) { 5292 timeout_exit(dev_id); 5293 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 5294 } 5295 } while (unlikely(burst_sz != deq)); 5296 5297 *max_time = RTE_MAX(*max_time, last_time); 5298 *min_time = RTE_MIN(*min_time, last_time); 5299 *total_time += last_time; 5300 5301 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 5302 ret = validate_dec_op(ops_deq, burst_sz, ref_op); 5303 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 5304 } 5305 5306 rte_bbdev_dec_op_free_bulk(ops_enq, deq); 5307 dequeued += deq; 5308 } 5309 5310 return i; 5311 } 5312 5313 /* Test case for latency/validation for LDPC Decoder */ 5314 static int 5315 latency_test_ldpc_dec(struct rte_mempool *mempool, 5316 struct test_buffers *bufs, struct rte_bbdev_dec_op *ref_op, 5317 int vector_mask, uint16_t dev_id, uint16_t queue_id, 5318 const uint16_t num_to_process, uint16_t burst_sz, 5319 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time, 5320 bool disable_et) 5321 { 5322 int ret = TEST_SUCCESS; 5323 uint16_t i, j, dequeued; 5324 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 5325 uint64_t start_time = 0, last_time = 0; 5326 bool extDdr = ldpc_cap_flags & 5327 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE; 5328 5329 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 5330 uint16_t enq = 0, deq = 0; 5331 uint32_t time_out = 0; 5332 bool first_time = true; 5333 last_time = 0; 5334 5335 if (unlikely(num_to_process - dequeued < burst_sz)) 5336 burst_sz = num_to_process - dequeued; 5337 5338 ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz); 5339 TEST_ASSERT_SUCCESS(ret, 5340 "rte_bbdev_dec_op_alloc_bulk() failed"); 5341 5342 /* For latency tests we need to disable early termination */ 5343 if (disable_et && check_bit(ref_op->ldpc_dec.op_flags, 5344 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE)) 5345 ref_op->ldpc_dec.op_flags -= RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE; 5346 5347 ref_op->ldpc_dec.iter_max = get_iter_max(); 5348 /* When ET is disabled, the expected iter_count is iter_max */ 5349 if (disable_et) 5350 ref_op->ldpc_dec.iter_count = ref_op->ldpc_dec.iter_max; 5351 5352 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 5353 copy_reference_ldpc_dec_op(ops_enq, burst_sz, dequeued, 5354 bufs->inputs, 5355 bufs->hard_outputs, 5356 bufs->soft_outputs, 5357 bufs->harq_inputs, 5358 bufs->harq_outputs, 5359 ref_op); 5360 5361 if (extDdr) 5362 preload_harq_ddr(dev_id, queue_id, ops_enq, 5363 burst_sz, true); 5364 5365 /* Set counter to validate the ordering */ 5366 for (j = 0; j < burst_sz; ++j) 5367 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 5368 5369 start_time = rte_rdtsc_precise(); 5370 5371 enq = rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id, 5372 &ops_enq[enq], burst_sz); 5373 TEST_ASSERT(enq == burst_sz, 5374 "Error enqueueing burst, expected %u, got %u", 5375 burst_sz, enq); 5376 5377 /* Dequeue */ 5378 do { 5379 deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id, 5380 &ops_deq[deq], burst_sz - deq); 5381 if (likely(first_time && (deq > 0))) { 5382 last_time = rte_rdtsc_precise() - start_time; 5383 first_time = false; 5384 } 5385 time_out++; 5386 if (time_out >= TIME_OUT_POLL) { 5387 timeout_exit(dev_id); 5388 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 5389 } 5390 } while (unlikely(burst_sz != deq)); 5391 5392 *max_time = RTE_MAX(*max_time, last_time); 5393 *min_time = RTE_MIN(*min_time, last_time); 5394 *total_time += last_time; 5395 5396 if (extDdr) 5397 retrieve_harq_ddr(dev_id, queue_id, ops_enq, burst_sz); 5398 5399 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 5400 ret = validate_ldpc_dec_op(ops_deq, burst_sz, ref_op, vector_mask); 5401 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 5402 } 5403 5404 rte_bbdev_dec_op_free_bulk(ops_enq, deq); 5405 dequeued += deq; 5406 } 5407 return i; 5408 } 5409 5410 static int 5411 latency_test_enc(struct rte_mempool *mempool, 5412 struct test_buffers *bufs, struct rte_bbdev_enc_op *ref_op, 5413 uint16_t dev_id, uint16_t queue_id, 5414 const uint16_t num_to_process, uint16_t burst_sz, 5415 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time) 5416 { 5417 int ret = TEST_SUCCESS; 5418 uint16_t i, j, dequeued; 5419 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 5420 uint64_t start_time = 0, last_time = 0; 5421 5422 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 5423 uint16_t enq = 0, deq = 0; 5424 uint32_t time_out = 0; 5425 bool first_time = true; 5426 last_time = 0; 5427 5428 if (unlikely(num_to_process - dequeued < burst_sz)) 5429 burst_sz = num_to_process - dequeued; 5430 5431 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz); 5432 TEST_ASSERT_SUCCESS(ret, 5433 "rte_bbdev_enc_op_alloc_bulk() failed"); 5434 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 5435 copy_reference_enc_op(ops_enq, burst_sz, dequeued, 5436 bufs->inputs, 5437 bufs->hard_outputs, 5438 ref_op); 5439 5440 /* Set counter to validate the ordering */ 5441 for (j = 0; j < burst_sz; ++j) 5442 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 5443 5444 start_time = rte_rdtsc_precise(); 5445 5446 enq = rte_bbdev_enqueue_enc_ops(dev_id, queue_id, &ops_enq[enq], 5447 burst_sz); 5448 TEST_ASSERT(enq == burst_sz, 5449 "Error enqueueing burst, expected %u, got %u", 5450 burst_sz, enq); 5451 5452 /* Dequeue */ 5453 do { 5454 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id, 5455 &ops_deq[deq], burst_sz - deq); 5456 if (likely(first_time && (deq > 0))) { 5457 last_time += rte_rdtsc_precise() - start_time; 5458 first_time = false; 5459 } 5460 time_out++; 5461 if (time_out >= TIME_OUT_POLL) { 5462 timeout_exit(dev_id); 5463 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 5464 } 5465 } while (unlikely(burst_sz != deq)); 5466 5467 *max_time = RTE_MAX(*max_time, last_time); 5468 *min_time = RTE_MIN(*min_time, last_time); 5469 *total_time += last_time; 5470 5471 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 5472 ret = validate_enc_op(ops_deq, burst_sz, ref_op); 5473 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 5474 } 5475 5476 rte_bbdev_enc_op_free_bulk(ops_enq, deq); 5477 dequeued += deq; 5478 } 5479 5480 return i; 5481 } 5482 5483 static int 5484 latency_test_ldpc_enc(struct rte_mempool *mempool, 5485 struct test_buffers *bufs, struct rte_bbdev_enc_op *ref_op, 5486 uint16_t dev_id, uint16_t queue_id, 5487 const uint16_t num_to_process, uint16_t burst_sz, 5488 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time) 5489 { 5490 int ret = TEST_SUCCESS; 5491 uint16_t i, j, dequeued; 5492 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 5493 uint64_t start_time = 0, last_time = 0; 5494 5495 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 5496 uint16_t enq = 0, deq = 0; 5497 uint32_t time_out = 0; 5498 bool first_time = true; 5499 last_time = 0; 5500 5501 if (unlikely(num_to_process - dequeued < burst_sz)) 5502 burst_sz = num_to_process - dequeued; 5503 5504 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz); 5505 TEST_ASSERT_SUCCESS(ret, 5506 "rte_bbdev_enc_op_alloc_bulk() failed"); 5507 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 5508 copy_reference_ldpc_enc_op(ops_enq, burst_sz, dequeued, 5509 bufs->inputs, 5510 bufs->hard_outputs, 5511 ref_op); 5512 5513 /* Set counter to validate the ordering */ 5514 for (j = 0; j < burst_sz; ++j) 5515 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 5516 5517 start_time = rte_rdtsc_precise(); 5518 5519 enq = rte_bbdev_enqueue_ldpc_enc_ops(dev_id, queue_id, 5520 &ops_enq[enq], burst_sz); 5521 TEST_ASSERT(enq == burst_sz, 5522 "Error enqueueing burst, expected %u, got %u", 5523 burst_sz, enq); 5524 5525 /* Dequeue */ 5526 do { 5527 deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id, 5528 &ops_deq[deq], burst_sz - deq); 5529 if (likely(first_time && (deq > 0))) { 5530 last_time += rte_rdtsc_precise() - start_time; 5531 first_time = false; 5532 } 5533 time_out++; 5534 if (time_out >= TIME_OUT_POLL) { 5535 timeout_exit(dev_id); 5536 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 5537 } 5538 } while (unlikely(burst_sz != deq)); 5539 5540 *max_time = RTE_MAX(*max_time, last_time); 5541 *min_time = RTE_MIN(*min_time, last_time); 5542 *total_time += last_time; 5543 5544 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 5545 ret = validate_enc_op(ops_deq, burst_sz, ref_op); 5546 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 5547 } 5548 5549 rte_bbdev_enc_op_free_bulk(ops_enq, deq); 5550 dequeued += deq; 5551 } 5552 5553 return i; 5554 } 5555 5556 5557 static int 5558 latency_test_fft(struct rte_mempool *mempool, 5559 struct test_buffers *bufs, struct rte_bbdev_fft_op *ref_op, 5560 uint16_t dev_id, uint16_t queue_id, 5561 const uint16_t num_to_process, uint16_t burst_sz, 5562 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time) 5563 { 5564 int ret = TEST_SUCCESS; 5565 uint16_t i, j, dequeued; 5566 struct rte_bbdev_fft_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 5567 uint64_t start_time = 0, last_time = 0; 5568 5569 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 5570 uint16_t enq = 0, deq = 0; 5571 uint32_t time_out = 0; 5572 bool first_time = true; 5573 last_time = 0; 5574 5575 if (unlikely(num_to_process - dequeued < burst_sz)) 5576 burst_sz = num_to_process - dequeued; 5577 5578 ret = rte_bbdev_fft_op_alloc_bulk(mempool, ops_enq, burst_sz); 5579 TEST_ASSERT_SUCCESS(ret, 5580 "rte_bbdev_fft_op_alloc_bulk() failed"); 5581 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 5582 copy_reference_fft_op(ops_enq, burst_sz, dequeued, 5583 bufs->inputs, 5584 bufs->hard_outputs, bufs->soft_outputs, bufs->harq_inputs, 5585 ref_op); 5586 5587 /* Set counter to validate the ordering */ 5588 for (j = 0; j < burst_sz; ++j) 5589 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 5590 5591 start_time = rte_rdtsc_precise(); 5592 5593 enq = rte_bbdev_enqueue_fft_ops(dev_id, queue_id, 5594 &ops_enq[enq], burst_sz); 5595 TEST_ASSERT(enq == burst_sz, 5596 "Error enqueueing burst, expected %u, got %u", 5597 burst_sz, enq); 5598 5599 /* Dequeue */ 5600 do { 5601 deq += rte_bbdev_dequeue_fft_ops(dev_id, queue_id, 5602 &ops_deq[deq], burst_sz - deq); 5603 if (likely(first_time && (deq > 0))) { 5604 last_time += rte_rdtsc_precise() - start_time; 5605 first_time = false; 5606 } 5607 time_out++; 5608 if (time_out >= TIME_OUT_POLL) { 5609 timeout_exit(dev_id); 5610 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 5611 } 5612 } while (unlikely(burst_sz != deq)); 5613 5614 *max_time = RTE_MAX(*max_time, last_time); 5615 *min_time = RTE_MIN(*min_time, last_time); 5616 *total_time += last_time; 5617 5618 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 5619 ret = validate_fft_op(ops_deq, burst_sz, ref_op); 5620 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 5621 } 5622 5623 rte_bbdev_fft_op_free_bulk(ops_enq, deq); 5624 dequeued += deq; 5625 } 5626 5627 return i; 5628 } 5629 5630 static int 5631 latency_test_mldts(struct rte_mempool *mempool, 5632 struct test_buffers *bufs, struct rte_bbdev_mldts_op *ref_op, 5633 uint16_t dev_id, uint16_t queue_id, 5634 const uint16_t num_to_process, uint16_t burst_sz, 5635 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time) 5636 { 5637 int ret = TEST_SUCCESS; 5638 uint16_t i, j, dequeued; 5639 struct rte_bbdev_mldts_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 5640 uint64_t start_time = 0, last_time = 0; 5641 5642 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 5643 uint16_t enq = 0, deq = 0; 5644 uint32_t time_out = 0; 5645 bool first_time = true; 5646 last_time = 0; 5647 5648 if (unlikely(num_to_process - dequeued < burst_sz)) 5649 burst_sz = num_to_process - dequeued; 5650 5651 ret = rte_bbdev_mldts_op_alloc_bulk(mempool, ops_enq, burst_sz); 5652 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_mldts_op_alloc_bulk() failed"); 5653 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 5654 copy_reference_mldts_op(ops_enq, burst_sz, dequeued, 5655 bufs->inputs, bufs->harq_inputs, 5656 bufs->hard_outputs, 5657 ref_op); 5658 5659 /* Set counter to validate the ordering */ 5660 for (j = 0; j < burst_sz; ++j) 5661 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 5662 5663 start_time = rte_rdtsc_precise(); 5664 5665 enq = rte_bbdev_enqueue_mldts_ops(dev_id, queue_id, &ops_enq[enq], burst_sz); 5666 TEST_ASSERT(enq == burst_sz, 5667 "Error enqueueing burst, expected %u, got %u", 5668 burst_sz, enq); 5669 5670 /* Dequeue */ 5671 do { 5672 deq += rte_bbdev_dequeue_mldts_ops(dev_id, queue_id, 5673 &ops_deq[deq], burst_sz - deq); 5674 if (likely(first_time && (deq > 0))) { 5675 last_time += rte_rdtsc_precise() - start_time; 5676 first_time = false; 5677 } 5678 time_out++; 5679 if (time_out >= TIME_OUT_POLL) { 5680 timeout_exit(dev_id); 5681 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 5682 } 5683 } while (unlikely(burst_sz != deq)); 5684 5685 *max_time = RTE_MAX(*max_time, last_time); 5686 *min_time = RTE_MIN(*min_time, last_time); 5687 *total_time += last_time; 5688 5689 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 5690 ret = validate_mldts_op(ops_deq, burst_sz, ref_op); 5691 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 5692 } 5693 5694 rte_bbdev_mldts_op_free_bulk(ops_enq, deq); 5695 dequeued += deq; 5696 } 5697 5698 return i; 5699 } 5700 5701 /* Common function for running validation and latency test cases */ 5702 static int 5703 validation_latency_test(struct active_device *ad, 5704 struct test_op_params *op_params, bool latency_flag) 5705 { 5706 int iter; 5707 uint16_t burst_sz = op_params->burst_sz; 5708 const uint16_t num_to_process = op_params->num_to_process; 5709 const enum rte_bbdev_op_type op_type = test_vector.op_type; 5710 const uint16_t queue_id = ad->queue_ids[0]; 5711 struct test_buffers *bufs = NULL; 5712 struct rte_bbdev_info info; 5713 uint64_t total_time, min_time, max_time; 5714 const char *op_type_str; 5715 5716 total_time = max_time = 0; 5717 min_time = UINT64_MAX; 5718 5719 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 5720 "BURST_SIZE should be <= %u", MAX_BURST); 5721 5722 rte_bbdev_info_get(ad->dev_id, &info); 5723 bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 5724 5725 op_type_str = rte_bbdev_op_type_str(op_type); 5726 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type); 5727 5728 printf("+ ------------------------------------------------------- +\n"); 5729 if (latency_flag) 5730 printf("== test: latency\ndev:"); 5731 else 5732 printf("== test: validation\ndev:"); 5733 printf("%s, burst size: %u, num ops: %u, op type: %s\n", 5734 info.dev_name, burst_sz, num_to_process, op_type_str); 5735 5736 if (op_type == RTE_BBDEV_OP_TURBO_DEC) 5737 iter = latency_test_dec(op_params->mp, bufs, 5738 op_params->ref_dec_op, ad->dev_id, queue_id, 5739 num_to_process, burst_sz, &total_time, 5740 &min_time, &max_time, latency_flag); 5741 else if (op_type == RTE_BBDEV_OP_LDPC_ENC) 5742 iter = latency_test_ldpc_enc(op_params->mp, bufs, 5743 op_params->ref_enc_op, ad->dev_id, queue_id, 5744 num_to_process, burst_sz, &total_time, 5745 &min_time, &max_time); 5746 else if (op_type == RTE_BBDEV_OP_LDPC_DEC) 5747 iter = latency_test_ldpc_dec(op_params->mp, bufs, 5748 op_params->ref_dec_op, op_params->vector_mask, 5749 ad->dev_id, queue_id, num_to_process, 5750 burst_sz, &total_time, &min_time, &max_time, 5751 latency_flag); 5752 else if (op_type == RTE_BBDEV_OP_FFT) 5753 iter = latency_test_fft(op_params->mp, bufs, 5754 op_params->ref_fft_op, 5755 ad->dev_id, queue_id, 5756 num_to_process, burst_sz, &total_time, 5757 &min_time, &max_time); 5758 else if (op_type == RTE_BBDEV_OP_MLDTS) 5759 iter = latency_test_mldts(op_params->mp, bufs, 5760 op_params->ref_mldts_op, 5761 ad->dev_id, queue_id, 5762 num_to_process, burst_sz, &total_time, 5763 &min_time, &max_time); 5764 else /* RTE_BBDEV_OP_TURBO_ENC */ 5765 iter = latency_test_enc(op_params->mp, bufs, 5766 op_params->ref_enc_op, 5767 ad->dev_id, queue_id, 5768 num_to_process, burst_sz, &total_time, 5769 &min_time, &max_time); 5770 5771 if (iter <= 0) 5772 return TEST_FAILED; 5773 5774 printf("Operation latency:\n" 5775 "\tavg: %lg cycles, %lg us\n" 5776 "\tmin: %lg cycles, %lg us\n" 5777 "\tmax: %lg cycles, %lg us\n", 5778 (double)total_time / (double)iter, 5779 (double)(total_time * 1000000) / (double)iter / 5780 (double)rte_get_tsc_hz(), (double)min_time, 5781 (double)(min_time * 1000000) / (double)rte_get_tsc_hz(), 5782 (double)max_time, (double)(max_time * 1000000) / 5783 (double)rte_get_tsc_hz()); 5784 5785 return TEST_SUCCESS; 5786 } 5787 5788 static int 5789 latency_test(struct active_device *ad, struct test_op_params *op_params) 5790 { 5791 return validation_latency_test(ad, op_params, true); 5792 } 5793 5794 static int 5795 validation_test(struct active_device *ad, struct test_op_params *op_params) 5796 { 5797 return validation_latency_test(ad, op_params, false); 5798 } 5799 5800 static int 5801 get_bbdev_queue_stats(uint16_t dev_id, uint16_t queue_id, 5802 struct rte_bbdev_stats *stats) 5803 { 5804 struct rte_bbdev *dev = &rte_bbdev_devices[dev_id]; 5805 struct rte_bbdev_stats *q_stats; 5806 5807 if (queue_id >= dev->data->num_queues) 5808 return -1; 5809 5810 q_stats = &dev->data->queues[queue_id].queue_stats; 5811 5812 stats->enqueued_count = q_stats->enqueued_count; 5813 stats->dequeued_count = q_stats->dequeued_count; 5814 stats->enqueue_err_count = q_stats->enqueue_err_count; 5815 stats->dequeue_err_count = q_stats->dequeue_err_count; 5816 stats->enqueue_warn_count = q_stats->enqueue_warn_count; 5817 stats->dequeue_warn_count = q_stats->dequeue_warn_count; 5818 stats->acc_offload_cycles = q_stats->acc_offload_cycles; 5819 5820 return 0; 5821 } 5822 5823 static int 5824 offload_latency_test_fft(struct rte_mempool *mempool, struct test_buffers *bufs, 5825 struct rte_bbdev_fft_op *ref_op, uint16_t dev_id, 5826 uint16_t queue_id, const uint16_t num_to_process, 5827 uint16_t burst_sz, struct test_time_stats *time_st) 5828 { 5829 int i, dequeued, ret; 5830 struct rte_bbdev_fft_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 5831 uint64_t enq_start_time, deq_start_time; 5832 uint64_t enq_sw_last_time, deq_last_time; 5833 struct rte_bbdev_stats stats; 5834 5835 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 5836 uint16_t enq = 0, deq = 0; 5837 5838 if (unlikely(num_to_process - dequeued < burst_sz)) 5839 burst_sz = num_to_process - dequeued; 5840 5841 ret = rte_bbdev_fft_op_alloc_bulk(mempool, ops_enq, burst_sz); 5842 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_fft_op_alloc_bulk() failed"); 5843 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 5844 copy_reference_fft_op(ops_enq, burst_sz, dequeued, 5845 bufs->inputs, 5846 bufs->hard_outputs, bufs->soft_outputs, bufs->harq_inputs, 5847 ref_op); 5848 5849 /* Start time meas for enqueue function offload latency */ 5850 enq_start_time = rte_rdtsc_precise(); 5851 do { 5852 enq += rte_bbdev_enqueue_fft_ops(dev_id, queue_id, 5853 &ops_enq[enq], burst_sz - enq); 5854 } while (unlikely(burst_sz != enq)); 5855 5856 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats); 5857 TEST_ASSERT_SUCCESS(ret, 5858 "Failed to get stats for queue (%u) of device (%u)", 5859 queue_id, dev_id); 5860 5861 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time - 5862 stats.acc_offload_cycles; 5863 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time, 5864 enq_sw_last_time); 5865 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time, 5866 enq_sw_last_time); 5867 time_st->enq_sw_total_time += enq_sw_last_time; 5868 5869 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time, 5870 stats.acc_offload_cycles); 5871 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time, 5872 stats.acc_offload_cycles); 5873 time_st->enq_acc_total_time += stats.acc_offload_cycles; 5874 5875 /* give time for device to process ops */ 5876 rte_delay_us(WAIT_OFFLOAD_US); 5877 5878 /* Start time meas for dequeue function offload latency */ 5879 deq_start_time = rte_rdtsc_precise(); 5880 /* Dequeue one operation */ 5881 do { 5882 deq += rte_bbdev_dequeue_fft_ops(dev_id, queue_id, 5883 &ops_deq[deq], enq); 5884 } while (unlikely(deq == 0)); 5885 5886 deq_last_time = rte_rdtsc_precise() - deq_start_time; 5887 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time, 5888 deq_last_time); 5889 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time, 5890 deq_last_time); 5891 time_st->deq_total_time += deq_last_time; 5892 5893 /* Dequeue remaining operations if needed*/ 5894 while (burst_sz != deq) 5895 deq += rte_bbdev_dequeue_fft_ops(dev_id, queue_id, 5896 &ops_deq[deq], burst_sz - deq); 5897 5898 rte_bbdev_fft_op_free_bulk(ops_enq, deq); 5899 dequeued += deq; 5900 } 5901 5902 return i; 5903 } 5904 5905 static int 5906 offload_latency_test_mldts(struct rte_mempool *mempool, struct test_buffers *bufs, 5907 struct rte_bbdev_mldts_op *ref_op, uint16_t dev_id, 5908 uint16_t queue_id, const uint16_t num_to_process, 5909 uint16_t burst_sz, struct test_time_stats *time_st) 5910 { 5911 int i, dequeued, ret; 5912 struct rte_bbdev_mldts_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 5913 uint64_t enq_start_time, deq_start_time; 5914 uint64_t enq_sw_last_time, deq_last_time; 5915 struct rte_bbdev_stats stats; 5916 5917 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 5918 uint16_t enq = 0, deq = 0; 5919 5920 if (unlikely(num_to_process - dequeued < burst_sz)) 5921 burst_sz = num_to_process - dequeued; 5922 5923 ret = rte_bbdev_mldts_op_alloc_bulk(mempool, ops_enq, burst_sz); 5924 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_mldts_op_alloc_bulk() failed"); 5925 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 5926 copy_reference_mldts_op(ops_enq, burst_sz, dequeued, 5927 bufs->inputs, bufs->harq_inputs, 5928 bufs->hard_outputs, 5929 ref_op); 5930 5931 /* Start time meas for enqueue function offload latency */ 5932 enq_start_time = rte_rdtsc_precise(); 5933 do { 5934 enq += rte_bbdev_enqueue_mldts_ops(dev_id, queue_id, 5935 &ops_enq[enq], burst_sz - enq); 5936 } while (unlikely(burst_sz != enq)); 5937 5938 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats); 5939 TEST_ASSERT_SUCCESS(ret, 5940 "Failed to get stats for queue (%u) of device (%u)", 5941 queue_id, dev_id); 5942 5943 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time - 5944 stats.acc_offload_cycles; 5945 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time, 5946 enq_sw_last_time); 5947 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time, 5948 enq_sw_last_time); 5949 time_st->enq_sw_total_time += enq_sw_last_time; 5950 5951 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time, 5952 stats.acc_offload_cycles); 5953 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time, 5954 stats.acc_offload_cycles); 5955 time_st->enq_acc_total_time += stats.acc_offload_cycles; 5956 5957 /* give time for device to process ops */ 5958 rte_delay_us(WAIT_OFFLOAD_US); 5959 5960 /* Start time meas for dequeue function offload latency */ 5961 deq_start_time = rte_rdtsc_precise(); 5962 /* Dequeue one operation */ 5963 do { 5964 deq += rte_bbdev_dequeue_mldts_ops(dev_id, queue_id, &ops_deq[deq], enq); 5965 } while (unlikely(deq == 0)); 5966 5967 deq_last_time = rte_rdtsc_precise() - deq_start_time; 5968 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time, deq_last_time); 5969 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time, deq_last_time); 5970 time_st->deq_total_time += deq_last_time; 5971 5972 /* Dequeue remaining operations if needed*/ 5973 while (burst_sz != deq) 5974 deq += rte_bbdev_dequeue_mldts_ops(dev_id, queue_id, 5975 &ops_deq[deq], burst_sz - deq); 5976 5977 rte_bbdev_mldts_op_free_bulk(ops_enq, deq); 5978 dequeued += deq; 5979 } 5980 5981 return i; 5982 } 5983 5984 static int 5985 offload_latency_test_dec(struct rte_mempool *mempool, struct test_buffers *bufs, 5986 struct rte_bbdev_dec_op *ref_op, uint16_t dev_id, 5987 uint16_t queue_id, const uint16_t num_to_process, 5988 uint16_t burst_sz, struct test_time_stats *time_st) 5989 { 5990 int i, dequeued, ret; 5991 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 5992 uint64_t enq_start_time, deq_start_time; 5993 uint64_t enq_sw_last_time, deq_last_time; 5994 struct rte_bbdev_stats stats; 5995 5996 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 5997 uint16_t enq = 0, deq = 0; 5998 5999 if (unlikely(num_to_process - dequeued < burst_sz)) 6000 burst_sz = num_to_process - dequeued; 6001 6002 ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz); 6003 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_dec_op_alloc_bulk() failed"); 6004 ref_op->turbo_dec.iter_max = get_iter_max(); 6005 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 6006 copy_reference_dec_op(ops_enq, burst_sz, dequeued, 6007 bufs->inputs, 6008 bufs->hard_outputs, 6009 bufs->soft_outputs, 6010 ref_op); 6011 6012 /* Start time meas for enqueue function offload latency */ 6013 enq_start_time = rte_rdtsc_precise(); 6014 do { 6015 enq += rte_bbdev_enqueue_dec_ops(dev_id, queue_id, 6016 &ops_enq[enq], burst_sz - enq); 6017 } while (unlikely(burst_sz != enq)); 6018 6019 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats); 6020 TEST_ASSERT_SUCCESS(ret, 6021 "Failed to get stats for queue (%u) of device (%u)", 6022 queue_id, dev_id); 6023 6024 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time - 6025 stats.acc_offload_cycles; 6026 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time, 6027 enq_sw_last_time); 6028 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time, 6029 enq_sw_last_time); 6030 time_st->enq_sw_total_time += enq_sw_last_time; 6031 6032 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time, 6033 stats.acc_offload_cycles); 6034 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time, 6035 stats.acc_offload_cycles); 6036 time_st->enq_acc_total_time += stats.acc_offload_cycles; 6037 6038 /* give time for device to process ops */ 6039 rte_delay_us(WAIT_OFFLOAD_US); 6040 6041 /* Start time meas for dequeue function offload latency */ 6042 deq_start_time = rte_rdtsc_precise(); 6043 /* Dequeue one operation */ 6044 do { 6045 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id, 6046 &ops_deq[deq], enq); 6047 } while (unlikely(deq == 0)); 6048 6049 deq_last_time = rte_rdtsc_precise() - deq_start_time; 6050 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time, 6051 deq_last_time); 6052 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time, 6053 deq_last_time); 6054 time_st->deq_total_time += deq_last_time; 6055 6056 /* Dequeue remaining operations if needed*/ 6057 while (burst_sz != deq) 6058 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id, 6059 &ops_deq[deq], burst_sz - deq); 6060 6061 rte_bbdev_dec_op_free_bulk(ops_enq, deq); 6062 dequeued += deq; 6063 } 6064 6065 return i; 6066 } 6067 6068 static int 6069 offload_latency_test_ldpc_dec(struct rte_mempool *mempool, 6070 struct test_buffers *bufs, 6071 struct rte_bbdev_dec_op *ref_op, uint16_t dev_id, 6072 uint16_t queue_id, const uint16_t num_to_process, 6073 uint16_t burst_sz, struct test_time_stats *time_st) 6074 { 6075 int i, dequeued, ret; 6076 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 6077 uint64_t enq_start_time, deq_start_time; 6078 uint64_t enq_sw_last_time, deq_last_time; 6079 struct rte_bbdev_stats stats; 6080 bool extDdr = ldpc_cap_flags & 6081 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE; 6082 6083 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 6084 uint16_t enq = 0, deq = 0; 6085 6086 if (unlikely(num_to_process - dequeued < burst_sz)) 6087 burst_sz = num_to_process - dequeued; 6088 6089 ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz); 6090 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_dec_op_alloc_bulk() failed"); 6091 ref_op->ldpc_dec.iter_max = get_iter_max(); 6092 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 6093 copy_reference_ldpc_dec_op(ops_enq, burst_sz, dequeued, 6094 bufs->inputs, 6095 bufs->hard_outputs, 6096 bufs->soft_outputs, 6097 bufs->harq_inputs, 6098 bufs->harq_outputs, 6099 ref_op); 6100 6101 if (extDdr) 6102 preload_harq_ddr(dev_id, queue_id, ops_enq, 6103 burst_sz, true); 6104 6105 /* Start time meas for enqueue function offload latency */ 6106 enq_start_time = rte_rdtsc_precise(); 6107 do { 6108 enq += rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id, 6109 &ops_enq[enq], burst_sz - enq); 6110 } while (unlikely(burst_sz != enq)); 6111 6112 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time; 6113 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats); 6114 TEST_ASSERT_SUCCESS(ret, 6115 "Failed to get stats for queue (%u) of device (%u)", 6116 queue_id, dev_id); 6117 6118 enq_sw_last_time -= stats.acc_offload_cycles; 6119 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time, 6120 enq_sw_last_time); 6121 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time, 6122 enq_sw_last_time); 6123 time_st->enq_sw_total_time += enq_sw_last_time; 6124 6125 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time, 6126 stats.acc_offload_cycles); 6127 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time, 6128 stats.acc_offload_cycles); 6129 time_st->enq_acc_total_time += stats.acc_offload_cycles; 6130 6131 /* give time for device to process ops */ 6132 rte_delay_us(WAIT_OFFLOAD_US); 6133 6134 /* Start time meas for dequeue function offload latency */ 6135 deq_start_time = rte_rdtsc_precise(); 6136 /* Dequeue one operation */ 6137 do { 6138 deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id, 6139 &ops_deq[deq], enq); 6140 } while (unlikely(deq == 0)); 6141 6142 deq_last_time = rte_rdtsc_precise() - deq_start_time; 6143 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time, 6144 deq_last_time); 6145 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time, 6146 deq_last_time); 6147 time_st->deq_total_time += deq_last_time; 6148 6149 /* Dequeue remaining operations if needed*/ 6150 while (burst_sz != deq) 6151 deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id, 6152 &ops_deq[deq], burst_sz - deq); 6153 6154 if (extDdr) { 6155 /* Read loopback is not thread safe */ 6156 retrieve_harq_ddr(dev_id, queue_id, ops_enq, burst_sz); 6157 } 6158 6159 rte_bbdev_dec_op_free_bulk(ops_enq, deq); 6160 dequeued += deq; 6161 } 6162 6163 return i; 6164 } 6165 6166 static int 6167 offload_latency_test_enc(struct rte_mempool *mempool, struct test_buffers *bufs, 6168 struct rte_bbdev_enc_op *ref_op, uint16_t dev_id, 6169 uint16_t queue_id, const uint16_t num_to_process, 6170 uint16_t burst_sz, struct test_time_stats *time_st) 6171 { 6172 int i, dequeued, ret; 6173 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 6174 uint64_t enq_start_time, deq_start_time; 6175 uint64_t enq_sw_last_time, deq_last_time; 6176 struct rte_bbdev_stats stats; 6177 6178 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 6179 uint16_t enq = 0, deq = 0; 6180 6181 if (unlikely(num_to_process - dequeued < burst_sz)) 6182 burst_sz = num_to_process - dequeued; 6183 6184 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz); 6185 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_enc_op_alloc_bulk() failed"); 6186 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 6187 copy_reference_enc_op(ops_enq, burst_sz, dequeued, 6188 bufs->inputs, 6189 bufs->hard_outputs, 6190 ref_op); 6191 6192 /* Start time meas for enqueue function offload latency */ 6193 enq_start_time = rte_rdtsc_precise(); 6194 do { 6195 enq += rte_bbdev_enqueue_enc_ops(dev_id, queue_id, 6196 &ops_enq[enq], burst_sz - enq); 6197 } while (unlikely(burst_sz != enq)); 6198 6199 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time; 6200 6201 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats); 6202 TEST_ASSERT_SUCCESS(ret, 6203 "Failed to get stats for queue (%u) of device (%u)", 6204 queue_id, dev_id); 6205 enq_sw_last_time -= stats.acc_offload_cycles; 6206 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time, 6207 enq_sw_last_time); 6208 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time, 6209 enq_sw_last_time); 6210 time_st->enq_sw_total_time += enq_sw_last_time; 6211 6212 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time, 6213 stats.acc_offload_cycles); 6214 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time, 6215 stats.acc_offload_cycles); 6216 time_st->enq_acc_total_time += stats.acc_offload_cycles; 6217 6218 /* give time for device to process ops */ 6219 rte_delay_us(WAIT_OFFLOAD_US); 6220 6221 /* Start time meas for dequeue function offload latency */ 6222 deq_start_time = rte_rdtsc_precise(); 6223 /* Dequeue one operation */ 6224 do { 6225 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id, 6226 &ops_deq[deq], enq); 6227 } while (unlikely(deq == 0)); 6228 6229 deq_last_time = rte_rdtsc_precise() - deq_start_time; 6230 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time, 6231 deq_last_time); 6232 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time, 6233 deq_last_time); 6234 time_st->deq_total_time += deq_last_time; 6235 6236 while (burst_sz != deq) 6237 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id, 6238 &ops_deq[deq], burst_sz - deq); 6239 6240 rte_bbdev_enc_op_free_bulk(ops_enq, deq); 6241 dequeued += deq; 6242 } 6243 6244 return i; 6245 } 6246 6247 static int 6248 offload_latency_test_ldpc_enc(struct rte_mempool *mempool, 6249 struct test_buffers *bufs, 6250 struct rte_bbdev_enc_op *ref_op, uint16_t dev_id, 6251 uint16_t queue_id, const uint16_t num_to_process, 6252 uint16_t burst_sz, struct test_time_stats *time_st) 6253 { 6254 int i, dequeued, ret; 6255 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 6256 uint64_t enq_start_time, deq_start_time; 6257 uint64_t enq_sw_last_time, deq_last_time; 6258 struct rte_bbdev_stats stats; 6259 6260 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 6261 uint16_t enq = 0, deq = 0; 6262 6263 if (unlikely(num_to_process - dequeued < burst_sz)) 6264 burst_sz = num_to_process - dequeued; 6265 6266 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz); 6267 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_enc_op_alloc_bulk() failed"); 6268 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 6269 copy_reference_ldpc_enc_op(ops_enq, burst_sz, dequeued, 6270 bufs->inputs, 6271 bufs->hard_outputs, 6272 ref_op); 6273 6274 /* Start time meas for enqueue function offload latency */ 6275 enq_start_time = rte_rdtsc_precise(); 6276 do { 6277 enq += rte_bbdev_enqueue_ldpc_enc_ops(dev_id, queue_id, 6278 &ops_enq[enq], burst_sz - enq); 6279 } while (unlikely(burst_sz != enq)); 6280 6281 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time; 6282 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats); 6283 TEST_ASSERT_SUCCESS(ret, 6284 "Failed to get stats for queue (%u) of device (%u)", 6285 queue_id, dev_id); 6286 6287 enq_sw_last_time -= stats.acc_offload_cycles; 6288 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time, 6289 enq_sw_last_time); 6290 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time, 6291 enq_sw_last_time); 6292 time_st->enq_sw_total_time += enq_sw_last_time; 6293 6294 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time, 6295 stats.acc_offload_cycles); 6296 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time, 6297 stats.acc_offload_cycles); 6298 time_st->enq_acc_total_time += stats.acc_offload_cycles; 6299 6300 /* give time for device to process ops */ 6301 rte_delay_us(WAIT_OFFLOAD_US); 6302 6303 /* Start time meas for dequeue function offload latency */ 6304 deq_start_time = rte_rdtsc_precise(); 6305 /* Dequeue one operation */ 6306 do { 6307 deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id, 6308 &ops_deq[deq], enq); 6309 } while (unlikely(deq == 0)); 6310 6311 deq_last_time = rte_rdtsc_precise() - deq_start_time; 6312 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time, 6313 deq_last_time); 6314 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time, 6315 deq_last_time); 6316 time_st->deq_total_time += deq_last_time; 6317 6318 while (burst_sz != deq) 6319 deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id, 6320 &ops_deq[deq], burst_sz - deq); 6321 6322 rte_bbdev_enc_op_free_bulk(ops_enq, deq); 6323 dequeued += deq; 6324 } 6325 6326 return i; 6327 } 6328 6329 static int 6330 offload_cost_test(struct active_device *ad, 6331 struct test_op_params *op_params) 6332 { 6333 int iter, ret; 6334 uint16_t burst_sz = op_params->burst_sz; 6335 const uint16_t num_to_process = op_params->num_to_process; 6336 const enum rte_bbdev_op_type op_type = test_vector.op_type; 6337 const uint16_t queue_id = ad->queue_ids[0]; 6338 struct test_buffers *bufs = NULL; 6339 struct rte_bbdev_info info; 6340 const char *op_type_str; 6341 struct test_time_stats time_st; 6342 6343 memset(&time_st, 0, sizeof(struct test_time_stats)); 6344 time_st.enq_sw_min_time = UINT64_MAX; 6345 time_st.enq_acc_min_time = UINT64_MAX; 6346 time_st.deq_min_time = UINT64_MAX; 6347 6348 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 6349 "BURST_SIZE should be <= %u", MAX_BURST); 6350 6351 rte_bbdev_info_get(ad->dev_id, &info); 6352 bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 6353 6354 op_type_str = rte_bbdev_op_type_str(op_type); 6355 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type); 6356 6357 printf("+ ------------------------------------------------------- +\n"); 6358 printf("== test: offload latency test\ndev: %s, burst size: %u, num ops: %u, op type: %s\n", 6359 info.dev_name, burst_sz, num_to_process, op_type_str); 6360 6361 if (op_type == RTE_BBDEV_OP_TURBO_DEC) 6362 iter = offload_latency_test_dec(op_params->mp, bufs, 6363 op_params->ref_dec_op, ad->dev_id, queue_id, 6364 num_to_process, burst_sz, &time_st); 6365 else if (op_type == RTE_BBDEV_OP_TURBO_ENC) 6366 iter = offload_latency_test_enc(op_params->mp, bufs, 6367 op_params->ref_enc_op, ad->dev_id, queue_id, 6368 num_to_process, burst_sz, &time_st); 6369 else if (op_type == RTE_BBDEV_OP_LDPC_ENC) 6370 iter = offload_latency_test_ldpc_enc(op_params->mp, bufs, 6371 op_params->ref_enc_op, ad->dev_id, queue_id, 6372 num_to_process, burst_sz, &time_st); 6373 else if (op_type == RTE_BBDEV_OP_LDPC_DEC) 6374 iter = offload_latency_test_ldpc_dec(op_params->mp, bufs, 6375 op_params->ref_dec_op, ad->dev_id, queue_id, 6376 num_to_process, burst_sz, &time_st); 6377 else if (op_type == RTE_BBDEV_OP_FFT) 6378 iter = offload_latency_test_fft(op_params->mp, bufs, 6379 op_params->ref_fft_op, ad->dev_id, queue_id, 6380 num_to_process, burst_sz, &time_st); 6381 else if (op_type == RTE_BBDEV_OP_MLDTS) 6382 iter = offload_latency_test_mldts(op_params->mp, bufs, 6383 op_params->ref_mldts_op, ad->dev_id, queue_id, 6384 num_to_process, burst_sz, &time_st); 6385 else 6386 iter = offload_latency_test_enc(op_params->mp, bufs, 6387 op_params->ref_enc_op, ad->dev_id, queue_id, 6388 num_to_process, burst_sz, &time_st); 6389 6390 if (iter <= 0) 6391 return TEST_FAILED; 6392 6393 printf("Enqueue driver offload cost latency:\n" 6394 "\tavg: %lg cycles, %lg us\n" 6395 "\tmin: %lg cycles, %lg us\n" 6396 "\tmax: %lg cycles, %lg us\n" 6397 "Enqueue accelerator offload cost latency:\n" 6398 "\tavg: %lg cycles, %lg us\n" 6399 "\tmin: %lg cycles, %lg us\n" 6400 "\tmax: %lg cycles, %lg us\n", 6401 (double)time_st.enq_sw_total_time / (double)iter, 6402 (double)(time_st.enq_sw_total_time * 1000000) / 6403 (double)iter / (double)rte_get_tsc_hz(), 6404 (double)time_st.enq_sw_min_time, 6405 (double)(time_st.enq_sw_min_time * 1000000) / 6406 rte_get_tsc_hz(), (double)time_st.enq_sw_max_time, 6407 (double)(time_st.enq_sw_max_time * 1000000) / 6408 rte_get_tsc_hz(), (double)time_st.enq_acc_total_time / 6409 (double)iter, 6410 (double)(time_st.enq_acc_total_time * 1000000) / 6411 (double)iter / (double)rte_get_tsc_hz(), 6412 (double)time_st.enq_acc_min_time, 6413 (double)(time_st.enq_acc_min_time * 1000000) / 6414 rte_get_tsc_hz(), (double)time_st.enq_acc_max_time, 6415 (double)(time_st.enq_acc_max_time * 1000000) / 6416 rte_get_tsc_hz()); 6417 6418 printf("Dequeue offload cost latency - one op:\n" 6419 "\tavg: %lg cycles, %lg us\n" 6420 "\tmin: %lg cycles, %lg us\n" 6421 "\tmax: %lg cycles, %lg us\n", 6422 (double)time_st.deq_total_time / (double)iter, 6423 (double)(time_st.deq_total_time * 1000000) / 6424 (double)iter / (double)rte_get_tsc_hz(), 6425 (double)time_st.deq_min_time, 6426 (double)(time_st.deq_min_time * 1000000) / 6427 rte_get_tsc_hz(), (double)time_st.deq_max_time, 6428 (double)(time_st.deq_max_time * 1000000) / 6429 rte_get_tsc_hz()); 6430 6431 struct rte_bbdev_stats stats = {0}; 6432 ret = get_bbdev_queue_stats(ad->dev_id, queue_id, &stats); 6433 TEST_ASSERT_SUCCESS(ret, 6434 "Failed to get stats for queue (%u) of device (%u)", 6435 queue_id, ad->dev_id); 6436 if (stats.enqueue_warn_count > 0) 6437 printf("Warning reported on the queue : %10"PRIu64"\n", 6438 stats.enqueue_warn_count); 6439 if (op_type != RTE_BBDEV_OP_LDPC_DEC) { 6440 TEST_ASSERT_SUCCESS(stats.enqueued_count != num_to_process, 6441 "Mismatch in enqueue count %10"PRIu64" %d", 6442 stats.enqueued_count, num_to_process); 6443 TEST_ASSERT_SUCCESS(stats.dequeued_count != num_to_process, 6444 "Mismatch in dequeue count %10"PRIu64" %d", 6445 stats.dequeued_count, num_to_process); 6446 } 6447 TEST_ASSERT_SUCCESS(stats.enqueue_err_count != 0, 6448 "Enqueue count Error %10"PRIu64"", 6449 stats.enqueue_err_count); 6450 TEST_ASSERT_SUCCESS(stats.dequeue_err_count != 0, 6451 "Dequeue count Error (%10"PRIu64"", 6452 stats.dequeue_err_count); 6453 6454 return TEST_SUCCESS; 6455 } 6456 6457 static int 6458 offload_latency_empty_q_test_dec(uint16_t dev_id, uint16_t queue_id, 6459 const uint16_t num_to_process, uint16_t burst_sz, 6460 uint64_t *deq_total_time, uint64_t *deq_min_time, 6461 uint64_t *deq_max_time, const enum rte_bbdev_op_type op_type) 6462 { 6463 int i, deq_total; 6464 struct rte_bbdev_dec_op *ops[MAX_BURST]; 6465 uint64_t deq_start_time, deq_last_time; 6466 6467 /* Test deq offload latency from an empty queue */ 6468 6469 for (i = 0, deq_total = 0; deq_total < num_to_process; 6470 ++i, deq_total += burst_sz) { 6471 deq_start_time = rte_rdtsc_precise(); 6472 6473 if (unlikely(num_to_process - deq_total < burst_sz)) 6474 burst_sz = num_to_process - deq_total; 6475 if (op_type == RTE_BBDEV_OP_LDPC_DEC) 6476 rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id, ops, 6477 burst_sz); 6478 else 6479 rte_bbdev_dequeue_dec_ops(dev_id, queue_id, ops, 6480 burst_sz); 6481 6482 deq_last_time = rte_rdtsc_precise() - deq_start_time; 6483 *deq_max_time = RTE_MAX(*deq_max_time, deq_last_time); 6484 *deq_min_time = RTE_MIN(*deq_min_time, deq_last_time); 6485 *deq_total_time += deq_last_time; 6486 } 6487 6488 return i; 6489 } 6490 6491 static int 6492 offload_latency_empty_q_test_enc(uint16_t dev_id, uint16_t queue_id, 6493 const uint16_t num_to_process, uint16_t burst_sz, 6494 uint64_t *deq_total_time, uint64_t *deq_min_time, 6495 uint64_t *deq_max_time, const enum rte_bbdev_op_type op_type) 6496 { 6497 int i, deq_total; 6498 struct rte_bbdev_enc_op *ops[MAX_BURST]; 6499 uint64_t deq_start_time, deq_last_time; 6500 6501 /* Test deq offload latency from an empty queue */ 6502 for (i = 0, deq_total = 0; deq_total < num_to_process; 6503 ++i, deq_total += burst_sz) { 6504 deq_start_time = rte_rdtsc_precise(); 6505 6506 if (unlikely(num_to_process - deq_total < burst_sz)) 6507 burst_sz = num_to_process - deq_total; 6508 if (op_type == RTE_BBDEV_OP_LDPC_ENC) 6509 rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id, ops, 6510 burst_sz); 6511 else 6512 rte_bbdev_dequeue_enc_ops(dev_id, queue_id, ops, 6513 burst_sz); 6514 6515 deq_last_time = rte_rdtsc_precise() - deq_start_time; 6516 *deq_max_time = RTE_MAX(*deq_max_time, deq_last_time); 6517 *deq_min_time = RTE_MIN(*deq_min_time, deq_last_time); 6518 *deq_total_time += deq_last_time; 6519 } 6520 6521 return i; 6522 } 6523 6524 static int 6525 offload_latency_empty_q_test(struct active_device *ad, 6526 struct test_op_params *op_params) 6527 { 6528 int iter; 6529 uint64_t deq_total_time, deq_min_time, deq_max_time; 6530 uint16_t burst_sz = op_params->burst_sz; 6531 const uint16_t num_to_process = op_params->num_to_process; 6532 const enum rte_bbdev_op_type op_type = test_vector.op_type; 6533 const uint16_t queue_id = ad->queue_ids[0]; 6534 struct rte_bbdev_info info; 6535 const char *op_type_str; 6536 6537 deq_total_time = deq_max_time = 0; 6538 deq_min_time = UINT64_MAX; 6539 6540 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 6541 "BURST_SIZE should be <= %u", MAX_BURST); 6542 6543 rte_bbdev_info_get(ad->dev_id, &info); 6544 6545 op_type_str = rte_bbdev_op_type_str(op_type); 6546 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type); 6547 6548 printf("+ ------------------------------------------------------- +\n"); 6549 printf("== test: offload latency empty dequeue\ndev: %s, burst size: %u, num ops: %u, op type: %s\n", 6550 info.dev_name, burst_sz, num_to_process, op_type_str); 6551 6552 if (op_type == RTE_BBDEV_OP_TURBO_DEC || 6553 op_type == RTE_BBDEV_OP_LDPC_DEC) 6554 iter = offload_latency_empty_q_test_dec(ad->dev_id, queue_id, 6555 num_to_process, burst_sz, &deq_total_time, 6556 &deq_min_time, &deq_max_time, op_type); 6557 else 6558 iter = offload_latency_empty_q_test_enc(ad->dev_id, queue_id, 6559 num_to_process, burst_sz, &deq_total_time, 6560 &deq_min_time, &deq_max_time, op_type); 6561 6562 if (iter <= 0) 6563 return TEST_FAILED; 6564 6565 printf("Empty dequeue offload:\n" 6566 "\tavg: %lg cycles, %lg us\n" 6567 "\tmin: %lg cycles, %lg us\n" 6568 "\tmax: %lg cycles, %lg us\n", 6569 (double)deq_total_time / (double)iter, 6570 (double)(deq_total_time * 1000000) / (double)iter / 6571 (double)rte_get_tsc_hz(), (double)deq_min_time, 6572 (double)(deq_min_time * 1000000) / rte_get_tsc_hz(), 6573 (double)deq_max_time, (double)(deq_max_time * 1000000) / 6574 rte_get_tsc_hz()); 6575 6576 return TEST_SUCCESS; 6577 } 6578 6579 static int 6580 bler_tc(void) 6581 { 6582 return run_test_case(bler_test); 6583 } 6584 6585 static int 6586 throughput_tc(void) 6587 { 6588 return run_test_case(throughput_test); 6589 } 6590 6591 static int 6592 offload_cost_tc(void) 6593 { 6594 return run_test_case(offload_cost_test); 6595 } 6596 6597 static int 6598 offload_latency_empty_q_tc(void) 6599 { 6600 return run_test_case(offload_latency_empty_q_test); 6601 } 6602 6603 static int 6604 latency_tc(void) 6605 { 6606 return run_test_case(latency_test); 6607 } 6608 6609 static int 6610 validation_tc(void) 6611 { 6612 return run_test_case(validation_test); 6613 } 6614 6615 static int 6616 interrupt_tc(void) 6617 { 6618 return run_test_case(throughput_test); 6619 } 6620 6621 static struct unit_test_suite bbdev_bler_testsuite = { 6622 .suite_name = "BBdev BLER Tests", 6623 .setup = testsuite_setup, 6624 .teardown = testsuite_teardown, 6625 .unit_test_cases = { 6626 TEST_CASE_ST(ut_setup, ut_teardown, bler_tc), 6627 TEST_CASES_END() /**< NULL terminate unit test array */ 6628 } 6629 }; 6630 6631 static struct unit_test_suite bbdev_throughput_testsuite = { 6632 .suite_name = "BBdev Throughput Tests", 6633 .setup = testsuite_setup, 6634 .teardown = testsuite_teardown, 6635 .unit_test_cases = { 6636 TEST_CASE_ST(ut_setup, ut_teardown, throughput_tc), 6637 TEST_CASES_END() /**< NULL terminate unit test array */ 6638 } 6639 }; 6640 6641 static struct unit_test_suite bbdev_validation_testsuite = { 6642 .suite_name = "BBdev Validation Tests", 6643 .setup = testsuite_setup, 6644 .teardown = testsuite_teardown, 6645 .unit_test_cases = { 6646 TEST_CASE_ST(ut_setup, ut_teardown, validation_tc), 6647 TEST_CASES_END() /**< NULL terminate unit test array */ 6648 } 6649 }; 6650 6651 static struct unit_test_suite bbdev_latency_testsuite = { 6652 .suite_name = "BBdev Latency Tests", 6653 .setup = testsuite_setup, 6654 .teardown = testsuite_teardown, 6655 .unit_test_cases = { 6656 TEST_CASE_ST(ut_setup, ut_teardown, latency_tc), 6657 TEST_CASES_END() /**< NULL terminate unit test array */ 6658 } 6659 }; 6660 6661 static struct unit_test_suite bbdev_offload_cost_testsuite = { 6662 .suite_name = "BBdev Offload Cost Tests", 6663 .setup = testsuite_setup, 6664 .teardown = testsuite_teardown, 6665 .unit_test_cases = { 6666 TEST_CASE_ST(ut_setup, ut_teardown, offload_cost_tc), 6667 TEST_CASE_ST(ut_setup, ut_teardown, offload_latency_empty_q_tc), 6668 TEST_CASES_END() /**< NULL terminate unit test array */ 6669 } 6670 }; 6671 6672 static struct unit_test_suite bbdev_interrupt_testsuite = { 6673 .suite_name = "BBdev Interrupt Tests", 6674 .setup = interrupt_testsuite_setup, 6675 .teardown = testsuite_teardown, 6676 .unit_test_cases = { 6677 TEST_CASE_ST(ut_setup, ut_teardown, interrupt_tc), 6678 TEST_CASES_END() /**< NULL terminate unit test array */ 6679 } 6680 }; 6681 6682 REGISTER_TEST_COMMAND(bler, bbdev_bler_testsuite); 6683 REGISTER_TEST_COMMAND(throughput, bbdev_throughput_testsuite); 6684 REGISTER_TEST_COMMAND(validation, bbdev_validation_testsuite); 6685 REGISTER_TEST_COMMAND(latency, bbdev_latency_testsuite); 6686 REGISTER_TEST_COMMAND(offload, bbdev_offload_cost_testsuite); 6687 REGISTER_TEST_COMMAND(interrupt, bbdev_interrupt_testsuite); 6688