1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017 Intel Corporation 3 */ 4 5 #include <stdio.h> 6 #include <stdlib.h> 7 #include <inttypes.h> 8 #include <math.h> 9 10 #include <rte_eal.h> 11 #include <rte_common.h> 12 #include <rte_dev.h> 13 #include <rte_launch.h> 14 #include <rte_bbdev.h> 15 #include <rte_cycles.h> 16 #include <rte_lcore.h> 17 #include <rte_malloc.h> 18 #include <rte_random.h> 19 #include <rte_hexdump.h> 20 #include <rte_interrupts.h> 21 22 #include "main.h" 23 #include "test_bbdev_vector.h" 24 25 #define GET_SOCKET(socket_id) (((socket_id) == SOCKET_ID_ANY) ? 0 : (socket_id)) 26 27 #define MAX_QUEUES RTE_MAX_LCORE 28 #define TEST_REPETITIONS 100 29 #define TIME_OUT_POLL 1e9 30 #define WAIT_OFFLOAD_US 1000 31 32 #ifdef RTE_BASEBAND_FPGA_LTE_FEC 33 #include <fpga_lte_fec.h> 34 #define FPGA_LTE_PF_DRIVER_NAME ("intel_fpga_lte_fec_pf") 35 #define FPGA_LTE_VF_DRIVER_NAME ("intel_fpga_lte_fec_vf") 36 #define VF_UL_4G_QUEUE_VALUE 4 37 #define VF_DL_4G_QUEUE_VALUE 4 38 #define UL_4G_BANDWIDTH 3 39 #define DL_4G_BANDWIDTH 3 40 #define UL_4G_LOAD_BALANCE 128 41 #define DL_4G_LOAD_BALANCE 128 42 #define FLR_4G_TIMEOUT 610 43 #endif 44 45 #ifdef RTE_BASEBAND_FPGA_5GNR_FEC 46 #include <rte_pmd_fpga_5gnr_fec.h> 47 #define FPGA_5GNR_PF_DRIVER_NAME ("intel_fpga_5gnr_fec_pf") 48 #define FPGA_5GNR_VF_DRIVER_NAME ("intel_fpga_5gnr_fec_vf") 49 #define VF_UL_5G_QUEUE_VALUE 4 50 #define VF_DL_5G_QUEUE_VALUE 4 51 #define UL_5G_BANDWIDTH 3 52 #define DL_5G_BANDWIDTH 3 53 #define UL_5G_LOAD_BALANCE 128 54 #define DL_5G_LOAD_BALANCE 128 55 #endif 56 57 #ifdef RTE_BASEBAND_ACC 58 #include <rte_acc_cfg.h> 59 #define ACC100PF_DRIVER_NAME ("intel_acc100_pf") 60 #define ACC100VF_DRIVER_NAME ("intel_acc100_vf") 61 #define ACC100_QMGR_NUM_AQS 16 62 #define ACC100_QMGR_NUM_QGS 2 63 #define ACC100_QMGR_AQ_DEPTH 5 64 #define ACC100_QMGR_INVALID_IDX -1 65 #define ACC100_QMGR_RR 1 66 #define ACC100_QOS_GBR 0 67 #define VRBPF_DRIVER_NAME ("intel_vran_boost_pf") 68 #define VRBVF_DRIVER_NAME ("intel_vran_boost_vf") 69 #define VRB_QMGR_NUM_AQS 16 70 #define VRB_QMGR_NUM_QGS 2 71 #define VRB_QMGR_AQ_DEPTH 5 72 #define VRB_QMGR_INVALID_IDX -1 73 #define VRB_QMGR_RR 1 74 #define VRB_QOS_GBR 0 75 #endif 76 77 #define OPS_CACHE_SIZE 256U 78 #define OPS_POOL_SIZE_MIN 511U /* 0.5K per queue */ 79 80 #define SYNC_WAIT 0 81 #define SYNC_START 1 82 83 #define INVALID_QUEUE_ID -1 84 /* Increment for next code block in external HARQ memory */ 85 #define HARQ_INCR 32768 86 /* Headroom for filler LLRs insertion in HARQ buffer */ 87 #define FILLER_HEADROOM 2048 88 /* Constants from K0 computation from 3GPP 38.212 Table 5.4.2.1-2 */ 89 #define N_ZC_1 66 /* N = 66 Zc for BG 1 */ 90 #define N_ZC_2 50 /* N = 50 Zc for BG 2 */ 91 #define K0_1_1 17 /* K0 fraction numerator for rv 1 and BG 1 */ 92 #define K0_1_2 13 /* K0 fraction numerator for rv 1 and BG 2 */ 93 #define K0_2_1 33 /* K0 fraction numerator for rv 2 and BG 1 */ 94 #define K0_2_2 25 /* K0 fraction numerator for rv 2 and BG 2 */ 95 #define K0_3_1 56 /* K0 fraction numerator for rv 3 and BG 1 */ 96 #define K0_3_2 43 /* K0 fraction numerator for rv 3 and BG 2 */ 97 #define NUM_SC_PER_RB (12) /* Number of subcarriers in a RB in 3GPP. */ 98 #define BITS_PER_LLR (8) /* Number of bits in a LLR. */ 99 100 #define HARQ_MEM_TOLERANCE 256 101 static struct test_bbdev_vector test_vector; 102 103 /* Switch between PMD and Interrupt for throughput TC */ 104 static bool intr_enabled; 105 106 /* LLR arithmetic representation for numerical conversion */ 107 static int ldpc_llr_decimals; 108 static int ldpc_llr_size; 109 /* Keep track of the LDPC decoder device capability flag */ 110 static uint32_t ldpc_cap_flags; 111 /* FFT window width predefined on device and on vector. */ 112 static int fft_window_width_dev; 113 114 bool dump_ops = true; 115 116 /* Represents tested active devices */ 117 static struct active_device { 118 const char *driver_name; 119 uint8_t dev_id; 120 uint16_t supported_ops; 121 uint16_t queue_ids[MAX_QUEUES]; 122 uint16_t nb_queues; 123 struct rte_mempool *ops_mempool; 124 struct rte_mempool *in_mbuf_pool; 125 struct rte_mempool *hard_out_mbuf_pool; 126 struct rte_mempool *soft_out_mbuf_pool; 127 struct rte_mempool *harq_in_mbuf_pool; 128 struct rte_mempool *harq_out_mbuf_pool; 129 } active_devs[RTE_BBDEV_MAX_DEVS]; 130 131 static uint8_t nb_active_devs; 132 133 /* Data buffers used by BBDEV ops */ 134 struct test_buffers { 135 struct rte_bbdev_op_data *inputs; 136 struct rte_bbdev_op_data *hard_outputs; 137 struct rte_bbdev_op_data *soft_outputs; 138 struct rte_bbdev_op_data *harq_inputs; 139 struct rte_bbdev_op_data *harq_outputs; 140 }; 141 142 /* Operation parameters specific for given test case */ 143 struct test_op_params { 144 struct rte_mempool *mp; 145 struct rte_bbdev_dec_op *ref_dec_op; 146 struct rte_bbdev_enc_op *ref_enc_op; 147 struct rte_bbdev_fft_op *ref_fft_op; 148 struct rte_bbdev_mldts_op *ref_mldts_op; 149 uint16_t burst_sz; 150 uint16_t num_to_process; 151 uint16_t num_lcores; 152 int vector_mask; 153 RTE_ATOMIC(uint16_t) sync; 154 struct test_buffers q_bufs[RTE_MAX_NUMA_NODES][MAX_QUEUES]; 155 }; 156 157 /* Contains per lcore params */ 158 struct thread_params { 159 uint8_t dev_id; 160 uint16_t queue_id; 161 uint32_t lcore_id; 162 uint64_t start_time; 163 double ops_per_sec; 164 double mbps; 165 uint8_t iter_count; 166 double iter_average; 167 double bler; 168 RTE_ATOMIC(uint16_t) nb_dequeued; 169 RTE_ATOMIC(int16_t) processing_status; 170 RTE_ATOMIC(uint16_t) burst_sz; 171 struct test_op_params *op_params; 172 struct rte_bbdev_dec_op *dec_ops[MAX_BURST]; 173 struct rte_bbdev_enc_op *enc_ops[MAX_BURST]; 174 struct rte_bbdev_fft_op *fft_ops[MAX_BURST]; 175 struct rte_bbdev_mldts_op *mldts_ops[MAX_BURST]; 176 }; 177 178 /* Stores time statistics */ 179 struct test_time_stats { 180 /* Stores software enqueue total working time */ 181 uint64_t enq_sw_total_time; 182 /* Stores minimum value of software enqueue working time */ 183 uint64_t enq_sw_min_time; 184 /* Stores maximum value of software enqueue working time */ 185 uint64_t enq_sw_max_time; 186 /* Stores turbo enqueue total working time */ 187 uint64_t enq_acc_total_time; 188 /* Stores minimum value of accelerator enqueue working time */ 189 uint64_t enq_acc_min_time; 190 /* Stores maximum value of accelerator enqueue working time */ 191 uint64_t enq_acc_max_time; 192 /* Stores dequeue total working time */ 193 uint64_t deq_total_time; 194 /* Stores minimum value of dequeue working time */ 195 uint64_t deq_min_time; 196 /* Stores maximum value of dequeue working time */ 197 uint64_t deq_max_time; 198 }; 199 200 typedef int (test_case_function)(struct active_device *ad, 201 struct test_op_params *op_params); 202 203 /* Get device status before timeout exit */ 204 static inline void 205 timeout_exit(uint8_t dev_id) 206 { 207 struct rte_bbdev_info info; 208 rte_bbdev_info_get(dev_id, &info); 209 printf("Device Status %s\n", rte_bbdev_device_status_str(info.drv.device_status)); 210 } 211 212 static inline void 213 mbuf_reset(struct rte_mbuf *m) 214 { 215 m->pkt_len = 0; 216 217 do { 218 m->data_len = 0; 219 m = m->next; 220 } while (m != NULL); 221 } 222 223 /* Read flag value 0/1 from bitmap */ 224 static inline bool 225 check_bit(uint32_t bitmap, uint32_t bitmask) 226 { 227 return bitmap & bitmask; 228 } 229 230 static inline void 231 set_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type) 232 { 233 ad->supported_ops |= (1 << op_type); 234 } 235 236 static inline bool 237 is_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type) 238 { 239 return ad->supported_ops & (1 << op_type); 240 } 241 242 static inline bool 243 flags_match(uint32_t flags_req, uint32_t flags_present) 244 { 245 return (flags_req & flags_present) == flags_req; 246 } 247 248 static void 249 clear_soft_out_cap(uint32_t *op_flags) 250 { 251 *op_flags &= ~RTE_BBDEV_TURBO_SOFT_OUTPUT; 252 *op_flags &= ~RTE_BBDEV_TURBO_POS_LLR_1_BIT_SOFT_OUT; 253 *op_flags &= ~RTE_BBDEV_TURBO_NEG_LLR_1_BIT_SOFT_OUT; 254 } 255 256 /* This API is to convert all the test vector op data entries 257 * to big endian format. It is used when the device supports 258 * the input in the big endian format. 259 */ 260 static inline void 261 convert_op_data_to_be(void) 262 { 263 struct op_data_entries *op; 264 enum op_data_type type; 265 uint8_t nb_segs, *rem_data, temp; 266 uint32_t *data, len; 267 int complete, rem, i, j; 268 269 for (type = DATA_INPUT; type < DATA_NUM_TYPES; ++type) { 270 nb_segs = test_vector.entries[type].nb_segments; 271 op = &test_vector.entries[type]; 272 273 /* Invert byte endianness for all the segments */ 274 for (i = 0; i < nb_segs; ++i) { 275 len = op->segments[i].length; 276 data = op->segments[i].addr; 277 278 /* Swap complete u32 bytes */ 279 complete = len / 4; 280 for (j = 0; j < complete; j++) 281 data[j] = rte_bswap32(data[j]); 282 283 /* Swap any remaining bytes */ 284 rem = len % 4; 285 rem_data = (uint8_t *)&data[j]; 286 for (j = 0; j < rem/2; j++) { 287 temp = rem_data[j]; 288 rem_data[j] = rem_data[rem - j - 1]; 289 rem_data[rem - j - 1] = temp; 290 } 291 } 292 } 293 } 294 295 static int 296 check_dev_cap(const struct rte_bbdev_info *dev_info) 297 { 298 unsigned int i; 299 unsigned int nb_inputs, nb_soft_outputs, nb_hard_outputs, 300 nb_harq_inputs, nb_harq_outputs; 301 const struct rte_bbdev_op_cap *op_cap = dev_info->drv.capabilities; 302 uint8_t dev_data_endianness = dev_info->drv.data_endianness; 303 304 nb_inputs = test_vector.entries[DATA_INPUT].nb_segments; 305 nb_soft_outputs = test_vector.entries[DATA_SOFT_OUTPUT].nb_segments; 306 nb_hard_outputs = test_vector.entries[DATA_HARD_OUTPUT].nb_segments; 307 nb_harq_inputs = test_vector.entries[DATA_HARQ_INPUT].nb_segments; 308 nb_harq_outputs = test_vector.entries[DATA_HARQ_OUTPUT].nb_segments; 309 310 for (i = 0; op_cap->type != RTE_BBDEV_OP_NONE; ++i, ++op_cap) { 311 if (op_cap->type != test_vector.op_type) 312 continue; 313 314 if (dev_data_endianness == RTE_BIG_ENDIAN) 315 convert_op_data_to_be(); 316 317 if (op_cap->type == RTE_BBDEV_OP_TURBO_DEC) { 318 const struct rte_bbdev_op_cap_turbo_dec *cap = 319 &op_cap->cap.turbo_dec; 320 /* Ignore lack of soft output capability, just skip 321 * checking if soft output is valid. 322 */ 323 if ((test_vector.turbo_dec.op_flags & 324 RTE_BBDEV_TURBO_SOFT_OUTPUT) && 325 !(cap->capability_flags & 326 RTE_BBDEV_TURBO_SOFT_OUTPUT)) { 327 printf( 328 "INFO: Device \"%s\" does not support soft output - soft output flags will be ignored.\n", 329 dev_info->dev_name); 330 clear_soft_out_cap( 331 &test_vector.turbo_dec.op_flags); 332 } 333 334 if (!flags_match(test_vector.turbo_dec.op_flags, 335 cap->capability_flags)) 336 return TEST_FAILED; 337 if (nb_inputs > cap->num_buffers_src) { 338 printf("Too many inputs defined: %u, max: %u\n", 339 nb_inputs, cap->num_buffers_src); 340 return TEST_FAILED; 341 } 342 if (nb_soft_outputs > cap->num_buffers_soft_out && 343 (test_vector.turbo_dec.op_flags & 344 RTE_BBDEV_TURBO_SOFT_OUTPUT)) { 345 printf( 346 "Too many soft outputs defined: %u, max: %u\n", 347 nb_soft_outputs, 348 cap->num_buffers_soft_out); 349 return TEST_FAILED; 350 } 351 if (nb_hard_outputs > cap->num_buffers_hard_out) { 352 printf( 353 "Too many hard outputs defined: %u, max: %u\n", 354 nb_hard_outputs, 355 cap->num_buffers_hard_out); 356 return TEST_FAILED; 357 } 358 if (intr_enabled && !(cap->capability_flags & 359 RTE_BBDEV_TURBO_DEC_INTERRUPTS)) { 360 printf( 361 "Dequeue interrupts are not supported!\n"); 362 return TEST_FAILED; 363 } 364 365 return TEST_SUCCESS; 366 } else if (op_cap->type == RTE_BBDEV_OP_TURBO_ENC) { 367 const struct rte_bbdev_op_cap_turbo_enc *cap = 368 &op_cap->cap.turbo_enc; 369 370 if (!flags_match(test_vector.turbo_enc.op_flags, 371 cap->capability_flags)) 372 return TEST_FAILED; 373 if (nb_inputs > cap->num_buffers_src) { 374 printf("Too many inputs defined: %u, max: %u\n", 375 nb_inputs, cap->num_buffers_src); 376 return TEST_FAILED; 377 } 378 if (nb_hard_outputs > cap->num_buffers_dst) { 379 printf( 380 "Too many hard outputs defined: %u, max: %u\n", 381 nb_hard_outputs, cap->num_buffers_dst); 382 return TEST_FAILED; 383 } 384 if (intr_enabled && !(cap->capability_flags & 385 RTE_BBDEV_TURBO_ENC_INTERRUPTS)) { 386 printf( 387 "Dequeue interrupts are not supported!\n"); 388 return TEST_FAILED; 389 } 390 391 return TEST_SUCCESS; 392 } else if (op_cap->type == RTE_BBDEV_OP_LDPC_ENC) { 393 const struct rte_bbdev_op_cap_ldpc_enc *cap = 394 &op_cap->cap.ldpc_enc; 395 396 if (!flags_match(test_vector.ldpc_enc.op_flags, 397 cap->capability_flags)){ 398 printf("Flag Mismatch\n"); 399 return TEST_FAILED; 400 } 401 if (nb_inputs > cap->num_buffers_src) { 402 printf("Too many inputs defined: %u, max: %u\n", 403 nb_inputs, cap->num_buffers_src); 404 return TEST_FAILED; 405 } 406 if (nb_hard_outputs > cap->num_buffers_dst) { 407 printf( 408 "Too many hard outputs defined: %u, max: %u\n", 409 nb_hard_outputs, cap->num_buffers_dst); 410 return TEST_FAILED; 411 } 412 if (intr_enabled && !(cap->capability_flags & 413 RTE_BBDEV_LDPC_ENC_INTERRUPTS)) { 414 printf( 415 "Dequeue interrupts are not supported!\n"); 416 return TEST_FAILED; 417 } 418 419 return TEST_SUCCESS; 420 } else if (op_cap->type == RTE_BBDEV_OP_LDPC_DEC) { 421 const struct rte_bbdev_op_cap_ldpc_dec *cap = 422 &op_cap->cap.ldpc_dec; 423 424 if (!flags_match(test_vector.ldpc_dec.op_flags, 425 cap->capability_flags)){ 426 printf("Flag Mismatch\n"); 427 return TEST_FAILED; 428 } 429 if (nb_inputs > cap->num_buffers_src) { 430 printf("Too many inputs defined: %u, max: %u\n", 431 nb_inputs, cap->num_buffers_src); 432 return TEST_FAILED; 433 } 434 if (nb_hard_outputs > cap->num_buffers_hard_out) { 435 printf( 436 "Too many hard outputs defined: %u, max: %u\n", 437 nb_hard_outputs, 438 cap->num_buffers_hard_out); 439 return TEST_FAILED; 440 } 441 if (nb_harq_inputs > cap->num_buffers_hard_out) { 442 printf( 443 "Too many HARQ inputs defined: %u, max: %u\n", 444 nb_harq_inputs, 445 cap->num_buffers_hard_out); 446 return TEST_FAILED; 447 } 448 if (nb_harq_outputs > cap->num_buffers_hard_out) { 449 printf( 450 "Too many HARQ outputs defined: %u, max: %u\n", 451 nb_harq_outputs, 452 cap->num_buffers_hard_out); 453 return TEST_FAILED; 454 } 455 if (intr_enabled && !(cap->capability_flags & 456 RTE_BBDEV_LDPC_DEC_INTERRUPTS)) { 457 printf( 458 "Dequeue interrupts are not supported!\n"); 459 return TEST_FAILED; 460 } 461 if (intr_enabled && (test_vector.ldpc_dec.op_flags & 462 (RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE | 463 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE | 464 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK 465 ))) { 466 printf("Skip loop-back with interrupt\n"); 467 return TEST_FAILED; 468 } 469 return TEST_SUCCESS; 470 } else if (op_cap->type == RTE_BBDEV_OP_FFT) { 471 const struct rte_bbdev_op_cap_fft *cap = &op_cap->cap.fft; 472 473 if (!flags_match(test_vector.fft.op_flags, cap->capability_flags)) { 474 printf("Flag Mismatch\n"); 475 return TEST_FAILED; 476 } 477 if (nb_inputs > cap->num_buffers_src) { 478 printf("Too many inputs defined: %u, max: %u\n", 479 nb_inputs, cap->num_buffers_src); 480 return TEST_FAILED; 481 } 482 return TEST_SUCCESS; 483 } else if (op_cap->type == RTE_BBDEV_OP_MLDTS) { 484 const struct rte_bbdev_op_cap_mld *cap = &op_cap->cap.mld; 485 if (!flags_match(test_vector.mldts.op_flags, cap->capability_flags)) { 486 printf("Flag Mismatch\n"); 487 return TEST_FAILED; 488 } 489 if (nb_inputs > cap->num_buffers_src) { 490 printf("Too many inputs defined: %u, max: %u\n", 491 nb_inputs, cap->num_buffers_src); 492 return TEST_FAILED; 493 } 494 return TEST_SUCCESS; 495 } 496 } 497 498 if ((i == 0) && (test_vector.op_type == RTE_BBDEV_OP_NONE)) 499 return TEST_SUCCESS; /* Special case for NULL device */ 500 501 return TEST_FAILED; 502 } 503 504 /* calculates optimal mempool size not smaller than the val */ 505 static unsigned int 506 optimal_mempool_size(unsigned int val) 507 { 508 return rte_align32pow2(val + 1) - 1; 509 } 510 511 /* allocates mbuf mempool for inputs and outputs */ 512 static struct rte_mempool * 513 create_mbuf_pool(struct op_data_entries *entries, uint8_t dev_id, 514 int socket_id, unsigned int mbuf_pool_size, 515 const char *op_type_str) 516 { 517 unsigned int i; 518 uint32_t max_seg_sz = 0; 519 char pool_name[RTE_MEMPOOL_NAMESIZE]; 520 521 /* find max input segment size */ 522 for (i = 0; i < entries->nb_segments; ++i) 523 if (entries->segments[i].length > max_seg_sz) 524 max_seg_sz = entries->segments[i].length; 525 526 snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str, 527 dev_id); 528 return rte_pktmbuf_pool_create(pool_name, mbuf_pool_size, 0, 0, 529 RTE_MAX(max_seg_sz + RTE_PKTMBUF_HEADROOM 530 + FILLER_HEADROOM, 531 (unsigned int)RTE_MBUF_DEFAULT_BUF_SIZE), socket_id); 532 } 533 534 static int 535 create_mempools(struct active_device *ad, int socket_id, 536 enum rte_bbdev_op_type org_op_type, uint16_t num_ops) 537 { 538 struct rte_mempool *mp; 539 unsigned int ops_pool_size, mbuf_pool_size = 0; 540 char pool_name[RTE_MEMPOOL_NAMESIZE]; 541 const char *op_type_str; 542 enum rte_bbdev_op_type op_type = org_op_type; 543 544 struct op_data_entries *in = &test_vector.entries[DATA_INPUT]; 545 struct op_data_entries *hard_out = 546 &test_vector.entries[DATA_HARD_OUTPUT]; 547 struct op_data_entries *soft_out = 548 &test_vector.entries[DATA_SOFT_OUTPUT]; 549 struct op_data_entries *harq_in = 550 &test_vector.entries[DATA_HARQ_INPUT]; 551 struct op_data_entries *harq_out = 552 &test_vector.entries[DATA_HARQ_OUTPUT]; 553 554 /* allocate ops mempool */ 555 ops_pool_size = optimal_mempool_size(RTE_MAX( 556 /* Ops used plus 1 reference op */ 557 RTE_MAX((unsigned int)(ad->nb_queues * num_ops + 1), 558 /* Minimal cache size plus 1 reference op */ 559 (unsigned int)(1.5 * rte_lcore_count() * 560 OPS_CACHE_SIZE + 1)), 561 OPS_POOL_SIZE_MIN)); 562 563 if (org_op_type == RTE_BBDEV_OP_NONE) 564 op_type = RTE_BBDEV_OP_TURBO_ENC; 565 566 op_type_str = rte_bbdev_op_type_str(op_type); 567 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type); 568 569 snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str, 570 ad->dev_id); 571 mp = rte_bbdev_op_pool_create(pool_name, op_type, 572 ops_pool_size, OPS_CACHE_SIZE, socket_id); 573 TEST_ASSERT_NOT_NULL(mp, 574 "ERROR Failed to create %u items ops pool for dev %u on socket %u.", 575 ops_pool_size, 576 ad->dev_id, 577 socket_id); 578 ad->ops_mempool = mp; 579 580 /* Do not create inputs and outputs mbufs for BaseBand Null Device */ 581 if (org_op_type == RTE_BBDEV_OP_NONE) 582 return TEST_SUCCESS; 583 584 /* Inputs */ 585 if (in->nb_segments > 0) { 586 mbuf_pool_size = optimal_mempool_size(ops_pool_size * 587 in->nb_segments); 588 mp = create_mbuf_pool(in, ad->dev_id, socket_id, 589 mbuf_pool_size, "in"); 590 TEST_ASSERT_NOT_NULL(mp, 591 "ERROR Failed to create %u items input pktmbuf pool for dev %u on socket %u.", 592 mbuf_pool_size, 593 ad->dev_id, 594 socket_id); 595 ad->in_mbuf_pool = mp; 596 } 597 598 /* Hard outputs */ 599 if (hard_out->nb_segments > 0) { 600 mbuf_pool_size = optimal_mempool_size(ops_pool_size * 601 hard_out->nb_segments); 602 mp = create_mbuf_pool(hard_out, ad->dev_id, socket_id, 603 mbuf_pool_size, 604 "hard_out"); 605 TEST_ASSERT_NOT_NULL(mp, 606 "ERROR Failed to create %u items hard output pktmbuf pool for dev %u on socket %u.", 607 mbuf_pool_size, 608 ad->dev_id, 609 socket_id); 610 ad->hard_out_mbuf_pool = mp; 611 } 612 613 /* Soft outputs */ 614 if (soft_out->nb_segments > 0) { 615 mbuf_pool_size = optimal_mempool_size(ops_pool_size * 616 soft_out->nb_segments); 617 mp = create_mbuf_pool(soft_out, ad->dev_id, socket_id, 618 mbuf_pool_size, 619 "soft_out"); 620 TEST_ASSERT_NOT_NULL(mp, 621 "ERROR Failed to create %uB soft output pktmbuf pool for dev %u on socket %u.", 622 mbuf_pool_size, 623 ad->dev_id, 624 socket_id); 625 ad->soft_out_mbuf_pool = mp; 626 } 627 628 /* HARQ inputs */ 629 if (harq_in->nb_segments > 0) { 630 mbuf_pool_size = optimal_mempool_size(ops_pool_size * 631 harq_in->nb_segments); 632 mp = create_mbuf_pool(harq_in, ad->dev_id, socket_id, 633 mbuf_pool_size, 634 "harq_in"); 635 TEST_ASSERT_NOT_NULL(mp, 636 "ERROR Failed to create %uB harq input pktmbuf pool for dev %u on socket %u.", 637 mbuf_pool_size, 638 ad->dev_id, 639 socket_id); 640 ad->harq_in_mbuf_pool = mp; 641 } 642 643 /* HARQ outputs */ 644 if (harq_out->nb_segments > 0) { 645 mbuf_pool_size = optimal_mempool_size(ops_pool_size * 646 harq_out->nb_segments); 647 mp = create_mbuf_pool(harq_out, ad->dev_id, socket_id, 648 mbuf_pool_size, 649 "harq_out"); 650 TEST_ASSERT_NOT_NULL(mp, 651 "ERROR Failed to create %uB harq output pktmbuf pool for dev %u on socket %u.", 652 mbuf_pool_size, 653 ad->dev_id, 654 socket_id); 655 ad->harq_out_mbuf_pool = mp; 656 } 657 658 return TEST_SUCCESS; 659 } 660 661 static int 662 add_bbdev_dev(uint8_t dev_id, struct rte_bbdev_info *info, 663 struct test_bbdev_vector *vector) 664 { 665 int ret; 666 unsigned int queue_id; 667 struct rte_bbdev_queue_conf qconf; 668 struct active_device *ad = &active_devs[nb_active_devs]; 669 unsigned int nb_queues; 670 enum rte_bbdev_op_type op_type = vector->op_type; 671 672 /* Configure fpga lte fec with PF & VF values 673 * if '-i' flag is set and using fpga device 674 */ 675 #ifdef RTE_BASEBAND_FPGA_LTE_FEC 676 if ((get_init_device() == true) && 677 (!strcmp(info->drv.driver_name, FPGA_LTE_PF_DRIVER_NAME))) { 678 struct rte_fpga_lte_fec_conf conf; 679 unsigned int i; 680 681 printf("Configure FPGA LTE FEC Driver %s with default values\n", 682 info->drv.driver_name); 683 684 /* clear default configuration before initialization */ 685 memset(&conf, 0, sizeof(struct rte_fpga_lte_fec_conf)); 686 687 /* Set PF mode : 688 * true if PF is used for data plane 689 * false for VFs 690 */ 691 conf.pf_mode_en = true; 692 693 for (i = 0; i < FPGA_LTE_FEC_NUM_VFS; ++i) { 694 /* Number of UL queues per VF (fpga supports 8 VFs) */ 695 conf.vf_ul_queues_number[i] = VF_UL_4G_QUEUE_VALUE; 696 /* Number of DL queues per VF (fpga supports 8 VFs) */ 697 conf.vf_dl_queues_number[i] = VF_DL_4G_QUEUE_VALUE; 698 } 699 700 /* UL bandwidth. Needed for schedule algorithm */ 701 conf.ul_bandwidth = UL_4G_BANDWIDTH; 702 /* DL bandwidth */ 703 conf.dl_bandwidth = DL_4G_BANDWIDTH; 704 705 /* UL & DL load Balance Factor to 64 */ 706 conf.ul_load_balance = UL_4G_LOAD_BALANCE; 707 conf.dl_load_balance = DL_4G_LOAD_BALANCE; 708 709 /**< FLR timeout value */ 710 conf.flr_time_out = FLR_4G_TIMEOUT; 711 712 /* setup FPGA PF with configuration information */ 713 ret = rte_fpga_lte_fec_configure(info->dev_name, &conf); 714 TEST_ASSERT_SUCCESS(ret, 715 "Failed to configure 4G FPGA PF for bbdev %s", 716 info->dev_name); 717 } 718 #endif 719 #ifdef RTE_BASEBAND_FPGA_5GNR_FEC 720 if ((get_init_device() == true) && 721 (!strcmp(info->drv.driver_name, FPGA_5GNR_PF_DRIVER_NAME))) { 722 struct rte_fpga_5gnr_fec_conf conf; 723 unsigned int i; 724 725 printf("Configure FPGA 5GNR FEC Driver %s with default values\n", 726 info->drv.driver_name); 727 728 /* clear default configuration before initialization */ 729 memset(&conf, 0, sizeof(struct rte_fpga_5gnr_fec_conf)); 730 731 /* Set PF mode : 732 * true if PF is used for data plane 733 * false for VFs 734 */ 735 conf.pf_mode_en = true; 736 737 for (i = 0; i < FPGA_5GNR_FEC_NUM_VFS; ++i) { 738 /* Number of UL queues per VF (fpga supports 8 VFs) */ 739 conf.vf_ul_queues_number[i] = VF_UL_5G_QUEUE_VALUE; 740 /* Number of DL queues per VF (fpga supports 8 VFs) */ 741 conf.vf_dl_queues_number[i] = VF_DL_5G_QUEUE_VALUE; 742 } 743 744 /* UL bandwidth. Needed only for Vista Creek 5GNR schedule algorithm */ 745 conf.ul_bandwidth = UL_5G_BANDWIDTH; 746 /* DL bandwidth. Needed only for Vista Creek 5GNR schedule algorithm */ 747 conf.dl_bandwidth = DL_5G_BANDWIDTH; 748 749 /* UL & DL load Balance Factor to 64 */ 750 conf.ul_load_balance = UL_5G_LOAD_BALANCE; 751 conf.dl_load_balance = DL_5G_LOAD_BALANCE; 752 753 /* setup FPGA PF with configuration information */ 754 ret = rte_fpga_5gnr_fec_configure(info->dev_name, &conf); 755 TEST_ASSERT_SUCCESS(ret, 756 "Failed to configure 5G FPGA PF for bbdev %s", 757 info->dev_name); 758 } 759 #endif 760 #ifdef RTE_BASEBAND_ACC 761 if ((get_init_device() == true) && 762 (!strcmp(info->drv.driver_name, ACC100PF_DRIVER_NAME))) { 763 struct rte_acc_conf conf; 764 unsigned int i; 765 766 printf("Configure ACC100 FEC device %s with default values\n", 767 info->drv.driver_name); 768 769 /* clear default configuration before initialization */ 770 memset(&conf, 0, sizeof(struct rte_acc_conf)); 771 772 /* Always set in PF mode for built-in configuration */ 773 conf.pf_mode_en = true; 774 for (i = 0; i < RTE_ACC_NUM_VFS; ++i) { 775 conf.arb_dl_4g[i].gbr_threshold1 = ACC100_QOS_GBR; 776 conf.arb_dl_4g[i].gbr_threshold1 = ACC100_QOS_GBR; 777 conf.arb_dl_4g[i].round_robin_weight = ACC100_QMGR_RR; 778 conf.arb_ul_4g[i].gbr_threshold1 = ACC100_QOS_GBR; 779 conf.arb_ul_4g[i].gbr_threshold1 = ACC100_QOS_GBR; 780 conf.arb_ul_4g[i].round_robin_weight = ACC100_QMGR_RR; 781 conf.arb_dl_5g[i].gbr_threshold1 = ACC100_QOS_GBR; 782 conf.arb_dl_5g[i].gbr_threshold1 = ACC100_QOS_GBR; 783 conf.arb_dl_5g[i].round_robin_weight = ACC100_QMGR_RR; 784 conf.arb_ul_5g[i].gbr_threshold1 = ACC100_QOS_GBR; 785 conf.arb_ul_5g[i].gbr_threshold1 = ACC100_QOS_GBR; 786 conf.arb_ul_5g[i].round_robin_weight = ACC100_QMGR_RR; 787 } 788 789 conf.input_pos_llr_1_bit = true; 790 conf.output_pos_llr_1_bit = true; 791 conf.num_vf_bundles = 1; /**< Number of VF bundles to setup */ 792 793 conf.q_ul_4g.num_qgroups = ACC100_QMGR_NUM_QGS; 794 conf.q_ul_4g.first_qgroup_index = ACC100_QMGR_INVALID_IDX; 795 conf.q_ul_4g.num_aqs_per_groups = ACC100_QMGR_NUM_AQS; 796 conf.q_ul_4g.aq_depth_log2 = ACC100_QMGR_AQ_DEPTH; 797 conf.q_dl_4g.num_qgroups = ACC100_QMGR_NUM_QGS; 798 conf.q_dl_4g.first_qgroup_index = ACC100_QMGR_INVALID_IDX; 799 conf.q_dl_4g.num_aqs_per_groups = ACC100_QMGR_NUM_AQS; 800 conf.q_dl_4g.aq_depth_log2 = ACC100_QMGR_AQ_DEPTH; 801 conf.q_ul_5g.num_qgroups = ACC100_QMGR_NUM_QGS; 802 conf.q_ul_5g.first_qgroup_index = ACC100_QMGR_INVALID_IDX; 803 conf.q_ul_5g.num_aqs_per_groups = ACC100_QMGR_NUM_AQS; 804 conf.q_ul_5g.aq_depth_log2 = ACC100_QMGR_AQ_DEPTH; 805 conf.q_dl_5g.num_qgroups = ACC100_QMGR_NUM_QGS; 806 conf.q_dl_5g.first_qgroup_index = ACC100_QMGR_INVALID_IDX; 807 conf.q_dl_5g.num_aqs_per_groups = ACC100_QMGR_NUM_AQS; 808 conf.q_dl_5g.aq_depth_log2 = ACC100_QMGR_AQ_DEPTH; 809 810 /* setup PF with configuration information */ 811 ret = rte_acc_configure(info->dev_name, &conf); 812 TEST_ASSERT_SUCCESS(ret, 813 "Failed to configure ACC100 PF for bbdev %s", 814 info->dev_name); 815 } 816 if ((get_init_device() == true) && 817 (!strcmp(info->drv.driver_name, VRBPF_DRIVER_NAME))) { 818 struct rte_acc_conf conf; 819 unsigned int i; 820 821 printf("Configure Driver %s with default values\n", 822 info->drv.driver_name); 823 824 /* clear default configuration before initialization */ 825 memset(&conf, 0, sizeof(struct rte_acc_conf)); 826 827 /* Always set in PF mode for built-in configuration */ 828 conf.pf_mode_en = true; 829 for (i = 0; i < RTE_ACC_NUM_VFS; ++i) { 830 conf.arb_dl_4g[i].gbr_threshold1 = VRB_QOS_GBR; 831 conf.arb_dl_4g[i].gbr_threshold1 = VRB_QOS_GBR; 832 conf.arb_dl_4g[i].round_robin_weight = VRB_QMGR_RR; 833 conf.arb_ul_4g[i].gbr_threshold1 = VRB_QOS_GBR; 834 conf.arb_ul_4g[i].gbr_threshold1 = VRB_QOS_GBR; 835 conf.arb_ul_4g[i].round_robin_weight = VRB_QMGR_RR; 836 conf.arb_dl_5g[i].gbr_threshold1 = VRB_QOS_GBR; 837 conf.arb_dl_5g[i].gbr_threshold1 = VRB_QOS_GBR; 838 conf.arb_dl_5g[i].round_robin_weight = VRB_QMGR_RR; 839 conf.arb_ul_5g[i].gbr_threshold1 = VRB_QOS_GBR; 840 conf.arb_ul_5g[i].gbr_threshold1 = VRB_QOS_GBR; 841 conf.arb_ul_5g[i].round_robin_weight = VRB_QMGR_RR; 842 conf.arb_fft[i].gbr_threshold1 = VRB_QOS_GBR; 843 conf.arb_fft[i].gbr_threshold1 = VRB_QOS_GBR; 844 conf.arb_fft[i].round_robin_weight = VRB_QMGR_RR; 845 conf.arb_mld[i].gbr_threshold1 = VRB_QOS_GBR; 846 conf.arb_mld[i].gbr_threshold1 = VRB_QOS_GBR; 847 conf.arb_mld[i].round_robin_weight = VRB_QMGR_RR; 848 } 849 850 conf.input_pos_llr_1_bit = true; 851 conf.output_pos_llr_1_bit = true; 852 conf.num_vf_bundles = 1; /**< Number of VF bundles to setup */ 853 854 conf.q_ul_4g.num_qgroups = VRB_QMGR_NUM_QGS; 855 conf.q_ul_4g.first_qgroup_index = VRB_QMGR_INVALID_IDX; 856 conf.q_ul_4g.num_aqs_per_groups = VRB_QMGR_NUM_AQS; 857 conf.q_ul_4g.aq_depth_log2 = VRB_QMGR_AQ_DEPTH; 858 conf.q_dl_4g.num_qgroups = VRB_QMGR_NUM_QGS; 859 conf.q_dl_4g.first_qgroup_index = VRB_QMGR_INVALID_IDX; 860 conf.q_dl_4g.num_aqs_per_groups = VRB_QMGR_NUM_AQS; 861 conf.q_dl_4g.aq_depth_log2 = VRB_QMGR_AQ_DEPTH; 862 conf.q_ul_5g.num_qgroups = VRB_QMGR_NUM_QGS; 863 conf.q_ul_5g.first_qgroup_index = VRB_QMGR_INVALID_IDX; 864 conf.q_ul_5g.num_aqs_per_groups = VRB_QMGR_NUM_AQS; 865 conf.q_ul_5g.aq_depth_log2 = VRB_QMGR_AQ_DEPTH; 866 conf.q_dl_5g.num_qgroups = VRB_QMGR_NUM_QGS; 867 conf.q_dl_5g.first_qgroup_index = VRB_QMGR_INVALID_IDX; 868 conf.q_dl_5g.num_aqs_per_groups = VRB_QMGR_NUM_AQS; 869 conf.q_dl_5g.aq_depth_log2 = VRB_QMGR_AQ_DEPTH; 870 conf.q_fft.num_qgroups = VRB_QMGR_NUM_QGS; 871 conf.q_fft.first_qgroup_index = VRB_QMGR_INVALID_IDX; 872 conf.q_fft.num_aqs_per_groups = VRB_QMGR_NUM_AQS; 873 conf.q_fft.aq_depth_log2 = VRB_QMGR_AQ_DEPTH; 874 conf.q_mld.num_qgroups = VRB_QMGR_NUM_QGS; 875 conf.q_mld.first_qgroup_index = VRB_QMGR_INVALID_IDX; 876 conf.q_mld.num_aqs_per_groups = VRB_QMGR_NUM_AQS; 877 conf.q_mld.aq_depth_log2 = VRB_QMGR_AQ_DEPTH; 878 879 /* setup PF with configuration information */ 880 ret = rte_acc_configure(info->dev_name, &conf); 881 TEST_ASSERT_SUCCESS(ret, 882 "Failed to configure PF for bbdev %s", 883 info->dev_name); 884 } 885 #endif 886 /* Let's refresh this now this is configured */ 887 rte_bbdev_info_get(dev_id, info); 888 if (info->drv.device_status == RTE_BBDEV_DEV_FATAL_ERR) 889 printf("Device Status %s\n", rte_bbdev_device_status_str(info->drv.device_status)); 890 if (info->drv.fft_window_width != NULL) 891 fft_window_width_dev = info->drv.fft_window_width[0]; 892 else 893 fft_window_width_dev = 0; 894 if (fft_window_width_dev != 0) 895 printf(" FFT Window0 width %d\n", fft_window_width_dev); 896 897 nb_queues = RTE_MIN(rte_lcore_count(), info->drv.max_num_queues); 898 nb_queues = RTE_MIN(nb_queues, (unsigned int) MAX_QUEUES); 899 900 /* setup device */ 901 ret = rte_bbdev_setup_queues(dev_id, nb_queues, info->socket_id); 902 if (ret < 0) { 903 printf("rte_bbdev_setup_queues(%u, %u, %d) ret %i\n", 904 dev_id, nb_queues, info->socket_id, ret); 905 return TEST_FAILED; 906 } 907 908 /* configure interrupts if needed */ 909 if (intr_enabled) { 910 ret = rte_bbdev_intr_enable(dev_id); 911 if (ret < 0) { 912 printf("rte_bbdev_intr_enable(%u) ret %i\n", dev_id, 913 ret); 914 return TEST_FAILED; 915 } 916 } 917 918 /* setup device queues */ 919 qconf.socket = info->socket_id; 920 qconf.queue_size = info->drv.default_queue_conf.queue_size; 921 qconf.priority = 0; 922 qconf.deferred_start = 0; 923 qconf.op_type = op_type; 924 925 for (queue_id = 0; queue_id < nb_queues; ++queue_id) { 926 ret = rte_bbdev_queue_configure(dev_id, queue_id, &qconf); 927 if (ret != 0) { 928 printf( 929 "Allocated all queues (id=%u) at prio%u on dev%u\n", 930 queue_id, qconf.priority, dev_id); 931 qconf.priority++; 932 ret = rte_bbdev_queue_configure(ad->dev_id, queue_id, &qconf); 933 } 934 if (ret != 0) { 935 printf("All queues on dev %u allocated: %u\n", dev_id, queue_id); 936 break; 937 } 938 ret = rte_bbdev_queue_start(ad->dev_id, queue_id); 939 if (ret != 0) { 940 printf("Failed to start queue on dev %u q_id: %u\n", dev_id, queue_id); 941 break; 942 } 943 ad->queue_ids[queue_id] = queue_id; 944 } 945 TEST_ASSERT(queue_id != 0, 946 "ERROR Failed to configure any queues on dev %u\n" 947 "\tthe device may not support the related operation capability\n" 948 "\tor the device may not have been configured yet", dev_id); 949 ad->nb_queues = queue_id; 950 951 set_avail_op(ad, op_type); 952 953 return TEST_SUCCESS; 954 } 955 956 static int 957 add_active_device(uint8_t dev_id, struct rte_bbdev_info *info, 958 struct test_bbdev_vector *vector) 959 { 960 int ret; 961 962 active_devs[nb_active_devs].driver_name = info->drv.driver_name; 963 active_devs[nb_active_devs].dev_id = dev_id; 964 965 ret = add_bbdev_dev(dev_id, info, vector); 966 if (ret == TEST_SUCCESS) 967 ++nb_active_devs; 968 return ret; 969 } 970 971 static uint8_t 972 populate_active_devices(void) 973 { 974 int ret; 975 uint8_t dev_id; 976 uint8_t nb_devs_added = 0; 977 struct rte_bbdev_info info; 978 979 RTE_BBDEV_FOREACH(dev_id) { 980 rte_bbdev_info_get(dev_id, &info); 981 982 if (check_dev_cap(&info)) { 983 printf( 984 "Device %d (%s) does not support specified capabilities\n", 985 dev_id, info.dev_name); 986 continue; 987 } 988 989 ret = add_active_device(dev_id, &info, &test_vector); 990 if (ret != 0) { 991 printf("Adding active bbdev %s skipped\n", 992 info.dev_name); 993 continue; 994 } 995 nb_devs_added++; 996 } 997 998 return nb_devs_added; 999 } 1000 1001 static int 1002 read_test_vector(void) 1003 { 1004 int ret; 1005 1006 memset(&test_vector, 0, sizeof(test_vector)); 1007 printf("Test vector file = %s\n", get_vector_filename()); 1008 ret = test_bbdev_vector_read(get_vector_filename(), &test_vector); 1009 TEST_ASSERT_SUCCESS(ret, "Failed to parse file %s\n", 1010 get_vector_filename()); 1011 1012 return TEST_SUCCESS; 1013 } 1014 1015 static int 1016 testsuite_setup(void) 1017 { 1018 TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n"); 1019 1020 if (populate_active_devices() == 0) { 1021 printf("No suitable devices found!\n"); 1022 return TEST_SKIPPED; 1023 } 1024 1025 return TEST_SUCCESS; 1026 } 1027 1028 static int 1029 interrupt_testsuite_setup(void) 1030 { 1031 TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n"); 1032 1033 /* Enable interrupts */ 1034 intr_enabled = true; 1035 1036 /* Special case for NULL device (RTE_BBDEV_OP_NONE) */ 1037 if (populate_active_devices() == 0 || 1038 test_vector.op_type == RTE_BBDEV_OP_NONE) { 1039 intr_enabled = false; 1040 printf("No suitable devices found!\n"); 1041 return TEST_SKIPPED; 1042 } 1043 1044 return TEST_SUCCESS; 1045 } 1046 1047 static void 1048 testsuite_teardown(void) 1049 { 1050 uint8_t dev_id; 1051 1052 /* Unconfigure devices */ 1053 RTE_BBDEV_FOREACH(dev_id) 1054 rte_bbdev_close(dev_id); 1055 1056 /* Clear active devices structs. */ 1057 memset(active_devs, 0, sizeof(active_devs)); 1058 nb_active_devs = 0; 1059 1060 /* Disable interrupts */ 1061 intr_enabled = false; 1062 } 1063 1064 static int 1065 ut_setup(void) 1066 { 1067 uint8_t i, dev_id; 1068 1069 for (i = 0; i < nb_active_devs; i++) { 1070 dev_id = active_devs[i].dev_id; 1071 /* reset bbdev stats */ 1072 TEST_ASSERT_SUCCESS(rte_bbdev_stats_reset(dev_id), 1073 "Failed to reset stats of bbdev %u", dev_id); 1074 /* start the device */ 1075 TEST_ASSERT_SUCCESS(rte_bbdev_start(dev_id), 1076 "Failed to start bbdev %u", dev_id); 1077 } 1078 1079 return TEST_SUCCESS; 1080 } 1081 1082 static void 1083 ut_teardown(void) 1084 { 1085 uint8_t i, dev_id, ret; 1086 struct rte_bbdev_stats stats; 1087 1088 for (i = 0; i < nb_active_devs; i++) { 1089 dev_id = active_devs[i].dev_id; 1090 /* read stats and print */ 1091 ret = rte_bbdev_stats_get(dev_id, &stats); 1092 if (ret != 0) 1093 printf("Failed to get stats on bbdev %u\n", dev_id); 1094 /* Stop the device */ 1095 rte_bbdev_stop(dev_id); 1096 } 1097 } 1098 1099 static int 1100 init_op_data_objs(struct rte_bbdev_op_data *bufs, 1101 struct op_data_entries *ref_entries, 1102 struct rte_mempool *mbuf_pool, const uint16_t n, 1103 enum op_data_type op_type, uint16_t min_alignment) 1104 { 1105 int ret; 1106 unsigned int i, j; 1107 bool large_input = false; 1108 1109 for (i = 0; i < n; ++i) { 1110 char *data; 1111 struct op_data_buf *seg = &ref_entries->segments[0]; 1112 struct rte_mbuf *m_head = rte_pktmbuf_alloc(mbuf_pool); 1113 TEST_ASSERT_NOT_NULL(m_head, 1114 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)", 1115 op_type, n * ref_entries->nb_segments, 1116 mbuf_pool->size); 1117 1118 if ((seg->length + RTE_PKTMBUF_HEADROOM) > RTE_BBDEV_LDPC_E_MAX_MBUF) { 1119 /* 1120 * Special case when DPDK mbuf cannot handle 1121 * the required input size 1122 */ 1123 large_input = true; 1124 } 1125 bufs[i].data = m_head; 1126 bufs[i].offset = 0; 1127 bufs[i].length = 0; 1128 1129 if ((op_type == DATA_INPUT) || (op_type == DATA_HARQ_INPUT)) { 1130 if (large_input) { 1131 /* Allocate a fake overused mbuf */ 1132 data = rte_malloc(NULL, seg->length, 0); 1133 TEST_ASSERT_NOT_NULL(data, 1134 "rte malloc failed with %u bytes", 1135 seg->length); 1136 memcpy(data, seg->addr, seg->length); 1137 m_head->buf_addr = data; 1138 rte_mbuf_iova_set(m_head, rte_malloc_virt2iova(data)); 1139 m_head->data_off = 0; 1140 m_head->data_len = seg->length; 1141 } else { 1142 data = rte_pktmbuf_append(m_head, seg->length); 1143 TEST_ASSERT_NOT_NULL(data, 1144 "Couldn't append %u bytes to mbuf from %d data type mbuf pool", 1145 seg->length, op_type); 1146 1147 TEST_ASSERT(data == RTE_PTR_ALIGN( 1148 data, min_alignment), 1149 "Data addr in mbuf (%p) is not aligned to device min alignment (%u)", 1150 data, min_alignment); 1151 rte_memcpy(data, seg->addr, seg->length); 1152 } 1153 1154 bufs[i].length += seg->length; 1155 1156 for (j = 1; j < ref_entries->nb_segments; ++j) { 1157 struct rte_mbuf *m_tail = 1158 rte_pktmbuf_alloc(mbuf_pool); 1159 TEST_ASSERT_NOT_NULL(m_tail, 1160 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)", 1161 op_type, 1162 n * ref_entries->nb_segments, 1163 mbuf_pool->size); 1164 seg += 1; 1165 1166 data = rte_pktmbuf_append(m_tail, seg->length); 1167 TEST_ASSERT_NOT_NULL(data, 1168 "Couldn't append %u bytes to mbuf from %d data type mbuf pool", 1169 seg->length, op_type); 1170 1171 TEST_ASSERT(data == RTE_PTR_ALIGN(data, 1172 min_alignment), 1173 "Data addr in mbuf (%p) is not aligned to device min alignment (%u)", 1174 data, min_alignment); 1175 rte_memcpy(data, seg->addr, seg->length); 1176 bufs[i].length += seg->length; 1177 1178 ret = rte_pktmbuf_chain(m_head, m_tail); 1179 TEST_ASSERT_SUCCESS(ret, 1180 "Couldn't chain mbufs from %d data type mbuf pool", 1181 op_type); 1182 } 1183 } else { 1184 if (((op_type == DATA_HARD_OUTPUT) || (op_type == DATA_SOFT_OUTPUT)) 1185 && ((seg->length + RTE_PKTMBUF_HEADROOM) 1186 > RTE_BBDEV_LDPC_E_MAX_MBUF)) { 1187 /* Allocate a fake overused mbuf + margin */ 1188 data = rte_malloc(NULL, seg->length + 1024, 0); 1189 TEST_ASSERT_NOT_NULL(data, 1190 "rte malloc failed with %u bytes", 1191 seg->length + 1024); 1192 m_head->buf_addr = data; 1193 rte_mbuf_iova_set(m_head, rte_malloc_virt2iova(data)); 1194 m_head->data_off = 0; 1195 m_head->data_len = seg->length; 1196 } else { 1197 /* allocate chained-mbuf for output buffer */ 1198 for (j = 1; j < ref_entries->nb_segments; ++j) { 1199 struct rte_mbuf *m_tail = 1200 rte_pktmbuf_alloc(mbuf_pool); 1201 TEST_ASSERT_NOT_NULL(m_tail, 1202 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)", 1203 op_type, 1204 n * ref_entries->nb_segments, 1205 mbuf_pool->size); 1206 1207 ret = rte_pktmbuf_chain(m_head, m_tail); 1208 TEST_ASSERT_SUCCESS(ret, 1209 "Couldn't chain mbufs from %d data type mbuf pool", 1210 op_type); 1211 } 1212 } 1213 bufs[i].length += seg->length; 1214 } 1215 } 1216 1217 return 0; 1218 } 1219 1220 static int 1221 allocate_buffers_on_socket(struct rte_bbdev_op_data **buffers, const int len, 1222 const int socket) 1223 { 1224 int i; 1225 1226 *buffers = rte_zmalloc_socket(NULL, len, 0, socket); 1227 if (*buffers == NULL) { 1228 printf("WARNING: Failed to allocate op_data on socket %d\n", 1229 socket); 1230 /* try to allocate memory on other detected sockets */ 1231 for (i = 0; i < socket; i++) { 1232 *buffers = rte_zmalloc_socket(NULL, len, 0, i); 1233 if (*buffers != NULL) 1234 break; 1235 } 1236 } 1237 1238 return (*buffers == NULL) ? TEST_FAILED : TEST_SUCCESS; 1239 } 1240 1241 static void 1242 limit_input_llr_val_range(struct rte_bbdev_op_data *input_ops, 1243 const uint16_t n, const int8_t max_llr_modulus) 1244 { 1245 uint16_t i, byte_idx; 1246 1247 for (i = 0; i < n; ++i) { 1248 struct rte_mbuf *m = input_ops[i].data; 1249 while (m != NULL) { 1250 int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *, 1251 input_ops[i].offset); 1252 for (byte_idx = 0; byte_idx < rte_pktmbuf_data_len(m); 1253 ++byte_idx) 1254 llr[byte_idx] = round((double)max_llr_modulus * 1255 llr[byte_idx] / INT8_MAX); 1256 1257 m = m->next; 1258 } 1259 } 1260 } 1261 1262 /* 1263 * We may have to insert filler bits 1264 * when they are required by the HARQ assumption 1265 */ 1266 static void 1267 ldpc_add_filler(struct rte_bbdev_op_data *input_ops, 1268 const uint16_t n, struct test_op_params *op_params) 1269 { 1270 struct rte_bbdev_op_ldpc_dec dec = op_params->ref_dec_op->ldpc_dec; 1271 1272 if (input_ops == NULL) 1273 return; 1274 /* No need to add filler if not required by device */ 1275 if (!(ldpc_cap_flags & 1276 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_FILLERS)) 1277 return; 1278 /* No need to add filler for loopback operation */ 1279 if (dec.op_flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK) 1280 return; 1281 1282 uint16_t i, j, parity_offset; 1283 for (i = 0; i < n; ++i) { 1284 struct rte_mbuf *m = input_ops[i].data; 1285 int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *, 1286 input_ops[i].offset); 1287 parity_offset = (dec.basegraph == 1 ? 20 : 8) 1288 * dec.z_c - dec.n_filler; 1289 uint16_t new_hin_size = input_ops[i].length + dec.n_filler; 1290 m->data_len = new_hin_size; 1291 input_ops[i].length = new_hin_size; 1292 for (j = new_hin_size - 1; j >= parity_offset + dec.n_filler; 1293 j--) 1294 llr[j] = llr[j - dec.n_filler]; 1295 uint16_t llr_max_pre_scaling = (1 << (ldpc_llr_size - 1)) - 1; 1296 for (j = 0; j < dec.n_filler; j++) 1297 llr[parity_offset + j] = llr_max_pre_scaling; 1298 } 1299 } 1300 1301 static void 1302 ldpc_input_llr_scaling(struct rte_bbdev_op_data *input_ops, 1303 const uint16_t n, const int8_t llr_size, 1304 const int8_t llr_decimals) 1305 { 1306 if (input_ops == NULL) 1307 return; 1308 1309 uint16_t i, byte_idx; 1310 1311 int16_t llr_max, llr_min, llr_tmp; 1312 llr_max = (1 << (llr_size - 1)) - 1; 1313 llr_min = -llr_max; 1314 for (i = 0; i < n; ++i) { 1315 struct rte_mbuf *m = input_ops[i].data; 1316 while (m != NULL) { 1317 int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *, 1318 input_ops[i].offset); 1319 for (byte_idx = 0; byte_idx < rte_pktmbuf_data_len(m); 1320 ++byte_idx) { 1321 1322 llr_tmp = llr[byte_idx]; 1323 if (llr_decimals == 4) 1324 llr_tmp *= 8; 1325 else if (llr_decimals == 2) 1326 llr_tmp *= 2; 1327 else if (llr_decimals == 0) 1328 llr_tmp /= 2; 1329 llr_tmp = RTE_MIN(llr_max, 1330 RTE_MAX(llr_min, llr_tmp)); 1331 llr[byte_idx] = (int8_t) llr_tmp; 1332 } 1333 1334 m = m->next; 1335 } 1336 } 1337 } 1338 1339 1340 1341 static int 1342 fill_queue_buffers(struct test_op_params *op_params, 1343 struct rte_mempool *in_mp, struct rte_mempool *hard_out_mp, 1344 struct rte_mempool *soft_out_mp, 1345 struct rte_mempool *harq_in_mp, struct rte_mempool *harq_out_mp, 1346 uint16_t queue_id, 1347 const struct rte_bbdev_op_cap *capabilities, 1348 uint16_t min_alignment, const int socket_id) 1349 { 1350 int ret; 1351 enum op_data_type type; 1352 const uint16_t n = op_params->num_to_process; 1353 1354 struct rte_mempool *mbuf_pools[DATA_NUM_TYPES] = { 1355 in_mp, 1356 soft_out_mp, 1357 hard_out_mp, 1358 harq_in_mp, 1359 harq_out_mp, 1360 }; 1361 1362 struct rte_bbdev_op_data **queue_ops[DATA_NUM_TYPES] = { 1363 &op_params->q_bufs[socket_id][queue_id].inputs, 1364 &op_params->q_bufs[socket_id][queue_id].soft_outputs, 1365 &op_params->q_bufs[socket_id][queue_id].hard_outputs, 1366 &op_params->q_bufs[socket_id][queue_id].harq_inputs, 1367 &op_params->q_bufs[socket_id][queue_id].harq_outputs, 1368 }; 1369 1370 for (type = DATA_INPUT; type < DATA_NUM_TYPES; ++type) { 1371 struct op_data_entries *ref_entries = 1372 &test_vector.entries[type]; 1373 if (ref_entries->nb_segments == 0) 1374 continue; 1375 1376 ret = allocate_buffers_on_socket(queue_ops[type], 1377 n * sizeof(struct rte_bbdev_op_data), 1378 socket_id); 1379 TEST_ASSERT_SUCCESS(ret, 1380 "Couldn't allocate memory for rte_bbdev_op_data structs"); 1381 1382 ret = init_op_data_objs(*queue_ops[type], ref_entries, 1383 mbuf_pools[type], n, type, min_alignment); 1384 TEST_ASSERT_SUCCESS(ret, 1385 "Couldn't init rte_bbdev_op_data structs"); 1386 } 1387 1388 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) 1389 limit_input_llr_val_range(*queue_ops[DATA_INPUT], n, 1390 capabilities->cap.turbo_dec.max_llr_modulus); 1391 1392 if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) { 1393 bool loopback = op_params->ref_dec_op->ldpc_dec.op_flags & 1394 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK; 1395 bool llr_comp = op_params->ref_dec_op->ldpc_dec.op_flags & 1396 RTE_BBDEV_LDPC_LLR_COMPRESSION; 1397 bool harq_comp = op_params->ref_dec_op->ldpc_dec.op_flags & 1398 RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION; 1399 1400 ldpc_llr_decimals = capabilities->cap.ldpc_dec.llr_decimals; 1401 ldpc_llr_size = capabilities->cap.ldpc_dec.llr_size; 1402 ldpc_cap_flags = capabilities->cap.ldpc_dec.capability_flags; 1403 if (!loopback && !llr_comp) 1404 ldpc_input_llr_scaling(*queue_ops[DATA_INPUT], n, 1405 ldpc_llr_size, ldpc_llr_decimals); 1406 if (!loopback && !harq_comp) 1407 ldpc_input_llr_scaling(*queue_ops[DATA_HARQ_INPUT], n, 1408 ldpc_llr_size, ldpc_llr_decimals); 1409 if (!loopback) 1410 ldpc_add_filler(*queue_ops[DATA_HARQ_INPUT], n, 1411 op_params); 1412 } 1413 1414 return 0; 1415 } 1416 1417 static void 1418 free_buffers(struct active_device *ad, struct test_op_params *op_params) 1419 { 1420 unsigned int i, j; 1421 1422 rte_mempool_free(ad->ops_mempool); 1423 rte_mempool_free(ad->in_mbuf_pool); 1424 rte_mempool_free(ad->hard_out_mbuf_pool); 1425 rte_mempool_free(ad->soft_out_mbuf_pool); 1426 rte_mempool_free(ad->harq_in_mbuf_pool); 1427 rte_mempool_free(ad->harq_out_mbuf_pool); 1428 1429 for (i = 0; i < rte_lcore_count(); ++i) { 1430 for (j = 0; j < RTE_MAX_NUMA_NODES; ++j) { 1431 rte_free(op_params->q_bufs[j][i].inputs); 1432 rte_free(op_params->q_bufs[j][i].hard_outputs); 1433 rte_free(op_params->q_bufs[j][i].soft_outputs); 1434 rte_free(op_params->q_bufs[j][i].harq_inputs); 1435 rte_free(op_params->q_bufs[j][i].harq_outputs); 1436 } 1437 } 1438 } 1439 1440 static void 1441 copy_reference_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n, 1442 unsigned int start_idx, 1443 struct rte_bbdev_op_data *inputs, 1444 struct rte_bbdev_op_data *hard_outputs, 1445 struct rte_bbdev_op_data *soft_outputs, 1446 struct rte_bbdev_dec_op *ref_op) 1447 { 1448 unsigned int i; 1449 struct rte_bbdev_op_turbo_dec *turbo_dec = &ref_op->turbo_dec; 1450 1451 for (i = 0; i < n; ++i) { 1452 if (turbo_dec->code_block_mode == RTE_BBDEV_TRANSPORT_BLOCK) { 1453 ops[i]->turbo_dec.tb_params.ea = 1454 turbo_dec->tb_params.ea; 1455 ops[i]->turbo_dec.tb_params.eb = 1456 turbo_dec->tb_params.eb; 1457 ops[i]->turbo_dec.tb_params.k_pos = 1458 turbo_dec->tb_params.k_pos; 1459 ops[i]->turbo_dec.tb_params.k_neg = 1460 turbo_dec->tb_params.k_neg; 1461 ops[i]->turbo_dec.tb_params.c = 1462 turbo_dec->tb_params.c; 1463 ops[i]->turbo_dec.tb_params.c_neg = 1464 turbo_dec->tb_params.c_neg; 1465 ops[i]->turbo_dec.tb_params.cab = 1466 turbo_dec->tb_params.cab; 1467 ops[i]->turbo_dec.tb_params.r = 1468 turbo_dec->tb_params.r; 1469 } else { 1470 ops[i]->turbo_dec.cb_params.e = turbo_dec->cb_params.e; 1471 ops[i]->turbo_dec.cb_params.k = turbo_dec->cb_params.k; 1472 } 1473 1474 ops[i]->turbo_dec.ext_scale = turbo_dec->ext_scale; 1475 ops[i]->turbo_dec.iter_max = turbo_dec->iter_max; 1476 ops[i]->turbo_dec.iter_min = turbo_dec->iter_min; 1477 ops[i]->turbo_dec.op_flags = turbo_dec->op_flags; 1478 ops[i]->turbo_dec.rv_index = turbo_dec->rv_index; 1479 ops[i]->turbo_dec.num_maps = turbo_dec->num_maps; 1480 ops[i]->turbo_dec.code_block_mode = turbo_dec->code_block_mode; 1481 1482 ops[i]->turbo_dec.hard_output = hard_outputs[start_idx + i]; 1483 ops[i]->turbo_dec.input = inputs[start_idx + i]; 1484 if (soft_outputs != NULL) 1485 ops[i]->turbo_dec.soft_output = 1486 soft_outputs[start_idx + i]; 1487 } 1488 } 1489 1490 static void 1491 copy_reference_enc_op(struct rte_bbdev_enc_op **ops, unsigned int n, 1492 unsigned int start_idx, 1493 struct rte_bbdev_op_data *inputs, 1494 struct rte_bbdev_op_data *outputs, 1495 struct rte_bbdev_enc_op *ref_op) 1496 { 1497 unsigned int i; 1498 struct rte_bbdev_op_turbo_enc *turbo_enc = &ref_op->turbo_enc; 1499 for (i = 0; i < n; ++i) { 1500 if (turbo_enc->code_block_mode == RTE_BBDEV_TRANSPORT_BLOCK) { 1501 ops[i]->turbo_enc.tb_params.ea = 1502 turbo_enc->tb_params.ea; 1503 ops[i]->turbo_enc.tb_params.eb = 1504 turbo_enc->tb_params.eb; 1505 ops[i]->turbo_enc.tb_params.k_pos = 1506 turbo_enc->tb_params.k_pos; 1507 ops[i]->turbo_enc.tb_params.k_neg = 1508 turbo_enc->tb_params.k_neg; 1509 ops[i]->turbo_enc.tb_params.c = 1510 turbo_enc->tb_params.c; 1511 ops[i]->turbo_enc.tb_params.c_neg = 1512 turbo_enc->tb_params.c_neg; 1513 ops[i]->turbo_enc.tb_params.cab = 1514 turbo_enc->tb_params.cab; 1515 ops[i]->turbo_enc.tb_params.ncb_pos = 1516 turbo_enc->tb_params.ncb_pos; 1517 ops[i]->turbo_enc.tb_params.ncb_neg = 1518 turbo_enc->tb_params.ncb_neg; 1519 ops[i]->turbo_enc.tb_params.r = turbo_enc->tb_params.r; 1520 } else { 1521 ops[i]->turbo_enc.cb_params.e = turbo_enc->cb_params.e; 1522 ops[i]->turbo_enc.cb_params.k = turbo_enc->cb_params.k; 1523 ops[i]->turbo_enc.cb_params.ncb = 1524 turbo_enc->cb_params.ncb; 1525 } 1526 ops[i]->turbo_enc.rv_index = turbo_enc->rv_index; 1527 ops[i]->turbo_enc.op_flags = turbo_enc->op_flags; 1528 ops[i]->turbo_enc.code_block_mode = turbo_enc->code_block_mode; 1529 1530 ops[i]->turbo_enc.output = outputs[start_idx + i]; 1531 ops[i]->turbo_enc.input = inputs[start_idx + i]; 1532 } 1533 } 1534 1535 1536 /* Returns a random number drawn from a normal distribution 1537 * with mean of 0 and variance of 1 1538 * Marsaglia algorithm 1539 */ 1540 static double 1541 randn(int n) 1542 { 1543 double S, Z, U1, U2, u, v, fac; 1544 1545 do { 1546 U1 = (double)rand() / RAND_MAX; 1547 U2 = (double)rand() / RAND_MAX; 1548 u = 2. * U1 - 1.; 1549 v = 2. * U2 - 1.; 1550 S = u * u + v * v; 1551 } while (S >= 1 || S == 0); 1552 fac = sqrt(-2. * log(S) / S); 1553 Z = (n % 2) ? u * fac : v * fac; 1554 return Z; 1555 } 1556 1557 static inline double 1558 maxstar(double A, double B) 1559 { 1560 if (fabs(A - B) > 5) 1561 return RTE_MAX(A, B); 1562 else 1563 return RTE_MAX(A, B) + log1p(exp(-fabs(A - B))); 1564 } 1565 1566 /* 1567 * Generate Qm LLRS for Qm==8 1568 * Modulation, AWGN and LLR estimation from max log development 1569 */ 1570 static void 1571 gen_qm8_llr(int8_t *llrs, uint32_t i, double N0, double llr_max) 1572 { 1573 int qm = 8; 1574 int qam = 256; 1575 int m, k; 1576 double I, Q, p0, p1, llr_, b[qm], log_syml_prob[qam]; 1577 /* 5.1.4 of TS38.211 */ 1578 const double symbols_I[256] = { 1579 5, 5, 7, 7, 5, 5, 7, 7, 3, 3, 1, 1, 3, 3, 1, 1, 5, 1580 5, 7, 7, 5, 5, 7, 7, 3, 3, 1, 1, 3, 3, 1, 1, 11, 1581 11, 9, 9, 11, 11, 9, 9, 13, 13, 15, 15, 13, 13, 1582 15, 15, 11, 11, 9, 9, 11, 11, 9, 9, 13, 13, 15, 1583 15, 13, 13, 15, 15, 5, 5, 7, 7, 5, 5, 7, 7, 3, 3, 1584 1, 1, 3, 3, 1, 1, 5, 5, 7, 7, 5, 5, 7, 7, 3, 3, 1, 1585 1, 3, 3, 1, 1, 11, 11, 9, 9, 11, 11, 9, 9, 13, 13, 1586 15, 15, 13, 13, 15, 15, 11, 11, 9, 9, 11, 11, 9, 9, 1587 13, 13, 15, 15, 13, 13, 15, 15, -5, -5, -7, -7, -5, 1588 -5, -7, -7, -3, -3, -1, -1, -3, -3, -1, -1, -5, -5, 1589 -7, -7, -5, -5, -7, -7, -3, -3, -1, -1, -3, -3, 1590 -1, -1, -11, -11, -9, -9, -11, -11, -9, -9, -13, 1591 -13, -15, -15, -13, -13, -15, -15, -11, -11, -9, 1592 -9, -11, -11, -9, -9, -13, -13, -15, -15, -13, 1593 -13, -15, -15, -5, -5, -7, -7, -5, -5, -7, -7, -3, 1594 -3, -1, -1, -3, -3, -1, -1, -5, -5, -7, -7, -5, -5, 1595 -7, -7, -3, -3, -1, -1, -3, -3, -1, -1, -11, -11, 1596 -9, -9, -11, -11, -9, -9, -13, -13, -15, -15, -13, 1597 -13, -15, -15, -11, -11, -9, -9, -11, -11, -9, -9, 1598 -13, -13, -15, -15, -13, -13, -15, -15}; 1599 const double symbols_Q[256] = { 1600 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 11, 1601 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9, 13, 15, 13, 1602 15, 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 1603 11, 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9, 13, 1604 15, 13, 15, -5, -7, -5, -7, -3, -1, -3, -1, -5, 1605 -7, -5, -7, -3, -1, -3, -1, -11, -9, -11, -9, -13, 1606 -15, -13, -15, -11, -9, -11, -9, -13, -15, -13, 1607 -15, -5, -7, -5, -7, -3, -1, -3, -1, -5, -7, -5, 1608 -7, -3, -1, -3, -1, -11, -9, -11, -9, -13, -15, 1609 -13, -15, -11, -9, -11, -9, -13, -15, -13, -15, 5, 1610 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 11, 1611 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9, 13, 15, 1612 13, 15, 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 1613 3, 1, 11, 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9, 1614 13, 15, 13, 15, -5, -7, -5, -7, -3, -1, -3, -1, 1615 -5, -7, -5, -7, -3, -1, -3, -1, -11, -9, -11, -9, 1616 -13, -15, -13, -15, -11, -9, -11, -9, -13, -15, 1617 -13, -15, -5, -7, -5, -7, -3, -1, -3, -1, -5, -7, 1618 -5, -7, -3, -1, -3, -1, -11, -9, -11, -9, -13, -15, 1619 -13, -15, -11, -9, -11, -9, -13, -15, -13, -15}; 1620 /* Average constellation point energy */ 1621 N0 *= 170.0; 1622 for (k = 0; k < qm; k++) 1623 b[k] = llrs[qm * i + k] < 0 ? 1.0 : 0.0; 1624 /* 5.1.4 of TS38.211 */ 1625 I = (1 - 2 * b[0]) * (8 - (1 - 2 * b[2]) * 1626 (4 - (1 - 2 * b[4]) * (2 - (1 - 2 * b[6])))); 1627 Q = (1 - 2 * b[1]) * (8 - (1 - 2 * b[3]) * 1628 (4 - (1 - 2 * b[5]) * (2 - (1 - 2 * b[7])))); 1629 /* AWGN channel */ 1630 I += sqrt(N0 / 2) * randn(0); 1631 Q += sqrt(N0 / 2) * randn(1); 1632 /* 1633 * Calculate the log of the probability that each of 1634 * the constellation points was transmitted 1635 */ 1636 for (m = 0; m < qam; m++) 1637 log_syml_prob[m] = -(pow(I - symbols_I[m], 2.0) 1638 + pow(Q - symbols_Q[m], 2.0)) / N0; 1639 /* Calculate an LLR for each of the k_64QAM bits in the set */ 1640 for (k = 0; k < qm; k++) { 1641 p0 = -999999; 1642 p1 = -999999; 1643 /* For each constellation point */ 1644 for (m = 0; m < qam; m++) { 1645 if ((m >> (qm - k - 1)) & 1) 1646 p1 = maxstar(p1, log_syml_prob[m]); 1647 else 1648 p0 = maxstar(p0, log_syml_prob[m]); 1649 } 1650 /* Calculate the LLR */ 1651 llr_ = p0 - p1; 1652 llr_ *= (1 << ldpc_llr_decimals); 1653 llr_ = round(llr_); 1654 if (llr_ > llr_max) 1655 llr_ = llr_max; 1656 if (llr_ < -llr_max) 1657 llr_ = -llr_max; 1658 llrs[qm * i + k] = (int8_t) llr_; 1659 } 1660 } 1661 1662 1663 /* 1664 * Generate Qm LLRS for Qm==6 1665 * Modulation, AWGN and LLR estimation from max log development 1666 */ 1667 static void 1668 gen_qm6_llr(int8_t *llrs, uint32_t i, double N0, double llr_max) 1669 { 1670 int qm = 6; 1671 int qam = 64; 1672 int m, k; 1673 double I, Q, p0, p1, llr_, b[qm], log_syml_prob[qam]; 1674 /* 5.1.4 of TS38.211 */ 1675 const double symbols_I[64] = { 1676 3, 3, 1, 1, 3, 3, 1, 1, 5, 5, 7, 7, 5, 5, 7, 7, 1677 3, 3, 1, 1, 3, 3, 1, 1, 5, 5, 7, 7, 5, 5, 7, 7, 1678 -3, -3, -1, -1, -3, -3, -1, -1, -5, -5, -7, -7, 1679 -5, -5, -7, -7, -3, -3, -1, -1, -3, -3, -1, -1, 1680 -5, -5, -7, -7, -5, -5, -7, -7}; 1681 const double symbols_Q[64] = { 1682 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 1683 -3, -1, -3, -1, -5, -7, -5, -7, -3, -1, -3, -1, 1684 -5, -7, -5, -7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 1685 5, 7, 5, 7, -3, -1, -3, -1, -5, -7, -5, -7, 1686 -3, -1, -3, -1, -5, -7, -5, -7}; 1687 /* Average constellation point energy */ 1688 N0 *= 42.0; 1689 for (k = 0; k < qm; k++) 1690 b[k] = llrs[qm * i + k] < 0 ? 1.0 : 0.0; 1691 /* 5.1.4 of TS38.211 */ 1692 I = (1 - 2 * b[0])*(4 - (1 - 2 * b[2]) * (2 - (1 - 2 * b[4]))); 1693 Q = (1 - 2 * b[1])*(4 - (1 - 2 * b[3]) * (2 - (1 - 2 * b[5]))); 1694 /* AWGN channel */ 1695 I += sqrt(N0 / 2) * randn(0); 1696 Q += sqrt(N0 / 2) * randn(1); 1697 /* 1698 * Calculate the log of the probability that each of 1699 * the constellation points was transmitted 1700 */ 1701 for (m = 0; m < qam; m++) 1702 log_syml_prob[m] = -(pow(I - symbols_I[m], 2.0) 1703 + pow(Q - symbols_Q[m], 2.0)) / N0; 1704 /* Calculate an LLR for each of the k_64QAM bits in the set */ 1705 for (k = 0; k < qm; k++) { 1706 p0 = -999999; 1707 p1 = -999999; 1708 /* For each constellation point */ 1709 for (m = 0; m < qam; m++) { 1710 if ((m >> (qm - k - 1)) & 1) 1711 p1 = maxstar(p1, log_syml_prob[m]); 1712 else 1713 p0 = maxstar(p0, log_syml_prob[m]); 1714 } 1715 /* Calculate the LLR */ 1716 llr_ = p0 - p1; 1717 llr_ *= (1 << ldpc_llr_decimals); 1718 llr_ = round(llr_); 1719 if (llr_ > llr_max) 1720 llr_ = llr_max; 1721 if (llr_ < -llr_max) 1722 llr_ = -llr_max; 1723 llrs[qm * i + k] = (int8_t) llr_; 1724 } 1725 } 1726 1727 /* 1728 * Generate Qm LLRS for Qm==4 1729 * Modulation, AWGN and LLR estimation from max log development 1730 */ 1731 static void 1732 gen_qm4_llr(int8_t *llrs, uint32_t i, double N0, double llr_max) 1733 { 1734 int qm = 4; 1735 int qam = 16; 1736 int m, k; 1737 double I, Q, p0, p1, llr_, b[qm], log_syml_prob[qam]; 1738 /* 5.1.4 of TS38.211 */ 1739 const double symbols_I[16] = {1, 1, 3, 3, 1, 1, 3, 3, 1740 -1, -1, -3, -3, -1, -1, -3, -3}; 1741 const double symbols_Q[16] = {1, 3, 1, 3, -1, -3, -1, -3, 1742 1, 3, 1, 3, -1, -3, -1, -3}; 1743 /* Average constellation point energy */ 1744 N0 *= 10.0; 1745 for (k = 0; k < qm; k++) 1746 b[k] = llrs[qm * i + k] < 0 ? 1.0 : 0.0; 1747 /* 5.1.4 of TS38.211 */ 1748 I = (1 - 2 * b[0]) * (2 - (1 - 2 * b[2])); 1749 Q = (1 - 2 * b[1]) * (2 - (1 - 2 * b[3])); 1750 /* AWGN channel */ 1751 I += sqrt(N0 / 2) * randn(0); 1752 Q += sqrt(N0 / 2) * randn(1); 1753 /* 1754 * Calculate the log of the probability that each of 1755 * the constellation points was transmitted 1756 */ 1757 for (m = 0; m < qam; m++) 1758 log_syml_prob[m] = -(pow(I - symbols_I[m], 2.0) 1759 + pow(Q - symbols_Q[m], 2.0)) / N0; 1760 /* Calculate an LLR for each of the k_64QAM bits in the set */ 1761 for (k = 0; k < qm; k++) { 1762 p0 = -999999; 1763 p1 = -999999; 1764 /* For each constellation point */ 1765 for (m = 0; m < qam; m++) { 1766 if ((m >> (qm - k - 1)) & 1) 1767 p1 = maxstar(p1, log_syml_prob[m]); 1768 else 1769 p0 = maxstar(p0, log_syml_prob[m]); 1770 } 1771 /* Calculate the LLR */ 1772 llr_ = p0 - p1; 1773 llr_ *= (1 << ldpc_llr_decimals); 1774 llr_ = round(llr_); 1775 if (llr_ > llr_max) 1776 llr_ = llr_max; 1777 if (llr_ < -llr_max) 1778 llr_ = -llr_max; 1779 llrs[qm * i + k] = (int8_t) llr_; 1780 } 1781 } 1782 1783 static void 1784 gen_qm2_llr(int8_t *llrs, uint32_t j, double N0, double llr_max) 1785 { 1786 double b, b1, n; 1787 double coeff = 2.0 * sqrt(N0); 1788 1789 /* Ignore in vectors rare quasi null LLRs not to be saturated */ 1790 if (llrs[j] < 8 && llrs[j] > -8) 1791 return; 1792 1793 /* Note don't change sign here */ 1794 n = randn(j % 2); 1795 b1 = ((llrs[j] > 0 ? 2.0 : -2.0) 1796 + coeff * n) / N0; 1797 b = b1 * (1 << ldpc_llr_decimals); 1798 b = round(b); 1799 if (b > llr_max) 1800 b = llr_max; 1801 if (b < -llr_max) 1802 b = -llr_max; 1803 llrs[j] = (int8_t) b; 1804 } 1805 1806 /* Simple LLR generation assuming AWGN and QPSK */ 1807 static void 1808 gen_turbo_llr(int8_t *llrs, uint32_t j, double N0, double llr_max) 1809 { 1810 double b, b1, n; 1811 double coeff = 2.0 * sqrt(N0); 1812 1813 /* Ignore in vectors null LLRs not to be saturated */ 1814 if (llrs[j] == 0) 1815 return; 1816 1817 /* Note don't change sign here */ 1818 n = randn(j % 2); 1819 b1 = ((llrs[j] > 0 ? 2.0 : -2.0) 1820 + coeff * n) / N0; 1821 b = b1 * (1 << 4); 1822 b = round(b); 1823 if (b > llr_max) 1824 b = llr_max; 1825 if (b < -llr_max) 1826 b = -llr_max; 1827 llrs[j] = (int8_t) b; 1828 } 1829 1830 /* Generate LLR for a given SNR */ 1831 static void 1832 generate_llr_input(uint16_t n, struct rte_bbdev_op_data *inputs, 1833 struct rte_bbdev_dec_op *ref_op) 1834 { 1835 struct rte_mbuf *m; 1836 uint16_t qm; 1837 uint32_t i, j, e, range; 1838 double N0, llr_max; 1839 1840 e = ref_op->ldpc_dec.cb_params.e; 1841 qm = ref_op->ldpc_dec.q_m; 1842 llr_max = (1 << (ldpc_llr_size - 1)) - 1; 1843 range = e / qm; 1844 N0 = 1.0 / pow(10.0, get_snr() / 10.0); 1845 1846 for (i = 0; i < n; ++i) { 1847 m = inputs[i].data; 1848 int8_t *llrs = rte_pktmbuf_mtod_offset(m, int8_t *, 0); 1849 if (qm == 8) { 1850 for (j = 0; j < range; ++j) 1851 gen_qm8_llr(llrs, j, N0, llr_max); 1852 } else if (qm == 6) { 1853 for (j = 0; j < range; ++j) 1854 gen_qm6_llr(llrs, j, N0, llr_max); 1855 } else if (qm == 4) { 1856 for (j = 0; j < range; ++j) 1857 gen_qm4_llr(llrs, j, N0, llr_max); 1858 } else { 1859 for (j = 0; j < e; ++j) 1860 gen_qm2_llr(llrs, j, N0, llr_max); 1861 } 1862 } 1863 } 1864 1865 /* Generate LLR for turbo decoder for a given SNR */ 1866 static void 1867 generate_turbo_llr_input(uint16_t n, struct rte_bbdev_op_data *inputs, 1868 struct rte_bbdev_dec_op *ref_op) 1869 { 1870 struct rte_mbuf *m; 1871 uint32_t i, j, range; 1872 double N0, llr_max; 1873 1874 llr_max = 127; 1875 range = ref_op->turbo_dec.input.length; 1876 N0 = 1.0 / pow(10.0, get_snr() / 10.0); 1877 1878 if (range > inputs[0].data->data_len) { 1879 printf("Warning: Limiting LLR generation to first segment (%d from %d)\n", 1880 inputs[0].data->data_len, range); 1881 range = inputs[0].data->data_len; 1882 } 1883 1884 for (i = 0; i < n; ++i) { 1885 m = inputs[i].data; 1886 int8_t *llrs = rte_pktmbuf_mtod_offset(m, int8_t *, 0); 1887 for (j = 0; j < range; ++j) 1888 gen_turbo_llr(llrs, j, N0, llr_max); 1889 } 1890 } 1891 1892 static void 1893 copy_reference_ldpc_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n, 1894 unsigned int start_idx, 1895 struct rte_bbdev_op_data *inputs, 1896 struct rte_bbdev_op_data *hard_outputs, 1897 struct rte_bbdev_op_data *soft_outputs, 1898 struct rte_bbdev_op_data *harq_inputs, 1899 struct rte_bbdev_op_data *harq_outputs, 1900 struct rte_bbdev_dec_op *ref_op) 1901 { 1902 unsigned int i; 1903 struct rte_bbdev_op_ldpc_dec *ldpc_dec = &ref_op->ldpc_dec; 1904 1905 for (i = 0; i < n; ++i) { 1906 if (ldpc_dec->code_block_mode == RTE_BBDEV_TRANSPORT_BLOCK) { 1907 ops[i]->ldpc_dec.tb_params.ea = 1908 ldpc_dec->tb_params.ea; 1909 ops[i]->ldpc_dec.tb_params.eb = 1910 ldpc_dec->tb_params.eb; 1911 ops[i]->ldpc_dec.tb_params.c = 1912 ldpc_dec->tb_params.c; 1913 ops[i]->ldpc_dec.tb_params.cab = 1914 ldpc_dec->tb_params.cab; 1915 ops[i]->ldpc_dec.tb_params.r = 1916 ldpc_dec->tb_params.r; 1917 } else { 1918 ops[i]->ldpc_dec.cb_params.e = ldpc_dec->cb_params.e; 1919 } 1920 1921 ops[i]->ldpc_dec.basegraph = ldpc_dec->basegraph; 1922 ops[i]->ldpc_dec.z_c = ldpc_dec->z_c; 1923 ops[i]->ldpc_dec.q_m = ldpc_dec->q_m; 1924 ops[i]->ldpc_dec.n_filler = ldpc_dec->n_filler; 1925 ops[i]->ldpc_dec.n_cb = ldpc_dec->n_cb; 1926 ops[i]->ldpc_dec.iter_max = ldpc_dec->iter_max; 1927 ops[i]->ldpc_dec.rv_index = ldpc_dec->rv_index; 1928 ops[i]->ldpc_dec.k0 = ldpc_dec->k0; 1929 ops[i]->ldpc_dec.op_flags = ldpc_dec->op_flags; 1930 ops[i]->ldpc_dec.code_block_mode = ldpc_dec->code_block_mode; 1931 1932 if (hard_outputs != NULL) 1933 ops[i]->ldpc_dec.hard_output = 1934 hard_outputs[start_idx + i]; 1935 if (inputs != NULL) 1936 ops[i]->ldpc_dec.input = 1937 inputs[start_idx + i]; 1938 if (soft_outputs != NULL) 1939 ops[i]->ldpc_dec.soft_output = 1940 soft_outputs[start_idx + i]; 1941 if (harq_inputs != NULL) 1942 ops[i]->ldpc_dec.harq_combined_input = 1943 harq_inputs[start_idx + i]; 1944 if (harq_outputs != NULL) 1945 ops[i]->ldpc_dec.harq_combined_output = 1946 harq_outputs[start_idx + i]; 1947 } 1948 } 1949 1950 1951 static void 1952 copy_reference_ldpc_enc_op(struct rte_bbdev_enc_op **ops, unsigned int n, 1953 unsigned int start_idx, 1954 struct rte_bbdev_op_data *inputs, 1955 struct rte_bbdev_op_data *outputs, 1956 struct rte_bbdev_enc_op *ref_op) 1957 { 1958 unsigned int i; 1959 struct rte_bbdev_op_ldpc_enc *ldpc_enc = &ref_op->ldpc_enc; 1960 for (i = 0; i < n; ++i) { 1961 if (ldpc_enc->code_block_mode == RTE_BBDEV_TRANSPORT_BLOCK) { 1962 ops[i]->ldpc_enc.tb_params.ea = ldpc_enc->tb_params.ea; 1963 ops[i]->ldpc_enc.tb_params.eb = ldpc_enc->tb_params.eb; 1964 ops[i]->ldpc_enc.tb_params.cab = 1965 ldpc_enc->tb_params.cab; 1966 ops[i]->ldpc_enc.tb_params.c = ldpc_enc->tb_params.c; 1967 ops[i]->ldpc_enc.tb_params.r = ldpc_enc->tb_params.r; 1968 } else { 1969 ops[i]->ldpc_enc.cb_params.e = ldpc_enc->cb_params.e; 1970 } 1971 ops[i]->ldpc_enc.basegraph = ldpc_enc->basegraph; 1972 ops[i]->ldpc_enc.z_c = ldpc_enc->z_c; 1973 ops[i]->ldpc_enc.q_m = ldpc_enc->q_m; 1974 ops[i]->ldpc_enc.n_filler = ldpc_enc->n_filler; 1975 ops[i]->ldpc_enc.n_cb = ldpc_enc->n_cb; 1976 ops[i]->ldpc_enc.rv_index = ldpc_enc->rv_index; 1977 ops[i]->ldpc_enc.op_flags = ldpc_enc->op_flags; 1978 ops[i]->ldpc_enc.code_block_mode = ldpc_enc->code_block_mode; 1979 ops[i]->ldpc_enc.output = outputs[start_idx + i]; 1980 ops[i]->ldpc_enc.input = inputs[start_idx + i]; 1981 } 1982 } 1983 1984 static void 1985 copy_reference_fft_op(struct rte_bbdev_fft_op **ops, unsigned int n, 1986 unsigned int start_idx, struct rte_bbdev_op_data *inputs, 1987 struct rte_bbdev_op_data *outputs, struct rte_bbdev_op_data *pwrouts, 1988 struct rte_bbdev_op_data *win_inputs, struct rte_bbdev_fft_op *ref_op) 1989 { 1990 unsigned int i, j; 1991 struct rte_bbdev_op_fft *fft = &ref_op->fft; 1992 for (i = 0; i < n; i++) { 1993 ops[i]->fft.input_sequence_size = fft->input_sequence_size; 1994 ops[i]->fft.input_leading_padding = fft->input_leading_padding; 1995 ops[i]->fft.output_sequence_size = fft->output_sequence_size; 1996 ops[i]->fft.output_leading_depadding = 1997 fft->output_leading_depadding; 1998 for (j = 0; j < RTE_BBDEV_MAX_CS_2; j++) 1999 ops[i]->fft.window_index[j] = fft->window_index[j]; 2000 for (j = 0; j < RTE_BBDEV_MAX_CS; j++) { 2001 ops[i]->fft.cs_theta_0[j] = fft->cs_theta_0[j]; 2002 ops[i]->fft.cs_theta_d[j] = fft->cs_theta_d[j]; 2003 ops[i]->fft.time_offset[j] = fft->time_offset[j]; 2004 } 2005 ops[i]->fft.cs_bitmap = fft->cs_bitmap; 2006 ops[i]->fft.num_antennas_log2 = fft->num_antennas_log2; 2007 ops[i]->fft.idft_log2 = fft->idft_log2; 2008 ops[i]->fft.dft_log2 = fft->dft_log2; 2009 ops[i]->fft.cs_time_adjustment = fft->cs_time_adjustment; 2010 ops[i]->fft.idft_shift = fft->idft_shift; 2011 ops[i]->fft.dft_shift = fft->dft_shift; 2012 ops[i]->fft.ncs_reciprocal = fft->ncs_reciprocal; 2013 ops[i]->fft.power_shift = fft->power_shift; 2014 ops[i]->fft.fp16_exp_adjust = fft->fp16_exp_adjust; 2015 ops[i]->fft.output_depadded_size = fft->output_depadded_size; 2016 ops[i]->fft.freq_resample_mode = fft->freq_resample_mode; 2017 ops[i]->fft.base_output = outputs[start_idx + i]; 2018 ops[i]->fft.base_input = inputs[start_idx + i]; 2019 if (win_inputs != NULL) 2020 ops[i]->fft.dewindowing_input = win_inputs[start_idx + i]; 2021 if (pwrouts != NULL) 2022 ops[i]->fft.power_meas_output = pwrouts[start_idx + i]; 2023 ops[i]->fft.op_flags = fft->op_flags; 2024 } 2025 } 2026 2027 static void 2028 copy_reference_mldts_op(struct rte_bbdev_mldts_op **ops, unsigned int n, 2029 unsigned int start_idx, 2030 struct rte_bbdev_op_data *q_inputs, 2031 struct rte_bbdev_op_data *r_inputs, 2032 struct rte_bbdev_op_data *outputs, 2033 struct rte_bbdev_mldts_op *ref_op) 2034 { 2035 unsigned int i, j; 2036 struct rte_bbdev_op_mldts *mldts = &ref_op->mldts; 2037 for (i = 0; i < n; i++) { 2038 ops[i]->mldts.c_rep = mldts->c_rep; 2039 ops[i]->mldts.num_layers = mldts->num_layers; 2040 ops[i]->mldts.num_rbs = mldts->num_rbs; 2041 ops[i]->mldts.op_flags = mldts->op_flags; 2042 for (j = 0; j < RTE_BBDEV_MAX_MLD_LAYERS; j++) 2043 ops[i]->mldts.q_m[j] = mldts->q_m[j]; 2044 ops[i]->mldts.r_rep = mldts->r_rep; 2045 ops[i]->mldts.c_rep = mldts->c_rep; 2046 ops[i]->mldts.r_input = r_inputs[start_idx + i]; 2047 ops[i]->mldts.qhy_input = q_inputs[start_idx + i]; 2048 ops[i]->mldts.output = outputs[start_idx + i]; 2049 } 2050 } 2051 2052 static int 2053 check_dec_status_and_ordering(struct rte_bbdev_dec_op *op, 2054 unsigned int order_idx, const int expected_status) 2055 { 2056 int status = op->status; 2057 /* ignore parity mismatch false alarms for long iterations */ 2058 if (get_iter_max() >= 10) { 2059 if (!(expected_status & (1 << RTE_BBDEV_SYNDROME_ERROR)) && 2060 (status & (1 << RTE_BBDEV_SYNDROME_ERROR))) { 2061 printf("WARNING: Ignore Syndrome Check mismatch\n"); 2062 status -= (1 << RTE_BBDEV_SYNDROME_ERROR); 2063 } 2064 if ((expected_status & (1 << RTE_BBDEV_SYNDROME_ERROR)) && 2065 !(status & (1 << RTE_BBDEV_SYNDROME_ERROR))) { 2066 printf("WARNING: Ignore Syndrome Check mismatch\n"); 2067 status += (1 << RTE_BBDEV_SYNDROME_ERROR); 2068 } 2069 } 2070 2071 TEST_ASSERT(status == expected_status, 2072 "op_status (%d) != expected_status (%d)", 2073 op->status, expected_status); 2074 2075 TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data, 2076 "Ordering error, expected %p, got %p", 2077 (void *)(uintptr_t)order_idx, op->opaque_data); 2078 2079 return TEST_SUCCESS; 2080 } 2081 2082 static int 2083 check_enc_status_and_ordering(struct rte_bbdev_enc_op *op, 2084 unsigned int order_idx, const int expected_status) 2085 { 2086 TEST_ASSERT(op->status == expected_status, 2087 "op_status (%d) != expected_status (%d)", 2088 op->status, expected_status); 2089 2090 TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data, 2091 "Ordering error, expected %p, got %p", 2092 (void *)(uintptr_t)order_idx, op->opaque_data); 2093 2094 return TEST_SUCCESS; 2095 } 2096 2097 static int 2098 check_fft_status_and_ordering(struct rte_bbdev_fft_op *op, 2099 unsigned int order_idx, const int expected_status) 2100 { 2101 TEST_ASSERT(op->status == expected_status, 2102 "op_status (%d) != expected_status (%d)", 2103 op->status, expected_status); 2104 2105 TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data, 2106 "Ordering error, expected %p, got %p", 2107 (void *)(uintptr_t)order_idx, op->opaque_data); 2108 2109 return TEST_SUCCESS; 2110 } 2111 2112 static int 2113 check_mldts_status_and_ordering(struct rte_bbdev_mldts_op *op, 2114 unsigned int order_idx, const int expected_status) 2115 { 2116 TEST_ASSERT(op->status == expected_status, 2117 "op_status (%d) != expected_status (%d)", 2118 op->status, expected_status); 2119 2120 TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data, 2121 "Ordering error, expected %p, got %p", 2122 (void *)(uintptr_t)order_idx, op->opaque_data); 2123 2124 return TEST_SUCCESS; 2125 } 2126 2127 static inline int 2128 validate_op_chain(struct rte_bbdev_op_data *op, 2129 struct op_data_entries *orig_op) 2130 { 2131 uint8_t i; 2132 struct rte_mbuf *m = op->data; 2133 uint8_t nb_dst_segments = orig_op->nb_segments; 2134 uint32_t total_data_size = 0; 2135 bool ignore_mbuf = false; /* ignore mbuf limitations */ 2136 2137 TEST_ASSERT(nb_dst_segments == m->nb_segs, 2138 "Number of segments differ in original (%u) and filled (%u) op", 2139 nb_dst_segments, m->nb_segs); 2140 2141 /* Validate each mbuf segment length */ 2142 for (i = 0; i < nb_dst_segments; ++i) { 2143 /* Apply offset to the first mbuf segment */ 2144 uint16_t offset = (i == 0) ? op->offset : 0; 2145 uint16_t data_len = rte_pktmbuf_data_len(m) - offset; 2146 total_data_size += orig_op->segments[i].length; 2147 2148 if ((orig_op->segments[i].length + RTE_PKTMBUF_HEADROOM) 2149 > RTE_BBDEV_LDPC_E_MAX_MBUF) 2150 ignore_mbuf = true; 2151 if (!ignore_mbuf) 2152 TEST_ASSERT(orig_op->segments[i].length == data_len, 2153 "Length of segment differ in original (%u) and filled (%u) op", 2154 orig_op->segments[i].length, data_len); 2155 TEST_ASSERT_BUFFERS_ARE_EQUAL(orig_op->segments[i].addr, 2156 rte_pktmbuf_mtod_offset(m, uint32_t *, offset), 2157 orig_op->segments[i].length, 2158 "Output buffers (CB=%u) are not equal", i); 2159 m = m->next; 2160 } 2161 2162 /* Validate total mbuf pkt length */ 2163 uint32_t pkt_len = rte_pktmbuf_pkt_len(op->data) - op->offset; 2164 if (!ignore_mbuf) 2165 TEST_ASSERT(total_data_size == pkt_len, 2166 "Length of data differ in original (%u) and filled (%u) op", 2167 total_data_size, pkt_len); 2168 2169 return TEST_SUCCESS; 2170 } 2171 2172 /* 2173 * Compute K0 for a given configuration for HARQ output length computation 2174 * As per definition in 3GPP 38.212 Table 5.4.2.1-2 2175 */ 2176 static inline uint16_t 2177 get_k0(uint16_t n_cb, uint16_t z_c, uint8_t bg, uint8_t rv_index, uint16_t k0) 2178 { 2179 if (k0 > 0) 2180 return k0; 2181 if (rv_index == 0) 2182 return 0; 2183 uint16_t n = (bg == 1 ? N_ZC_1 : N_ZC_2) * z_c; 2184 if (n_cb == n) { 2185 if (rv_index == 1) 2186 return (bg == 1 ? K0_1_1 : K0_1_2) * z_c; 2187 else if (rv_index == 2) 2188 return (bg == 1 ? K0_2_1 : K0_2_2) * z_c; 2189 else 2190 return (bg == 1 ? K0_3_1 : K0_3_2) * z_c; 2191 } 2192 /* LBRM case - includes a division by N */ 2193 if (rv_index == 1) 2194 return (((bg == 1 ? K0_1_1 : K0_1_2) * n_cb) 2195 / n) * z_c; 2196 else if (rv_index == 2) 2197 return (((bg == 1 ? K0_2_1 : K0_2_2) * n_cb) 2198 / n) * z_c; 2199 else 2200 return (((bg == 1 ? K0_3_1 : K0_3_2) * n_cb) 2201 / n) * z_c; 2202 } 2203 2204 /* HARQ output length including the Filler bits */ 2205 static inline uint16_t 2206 compute_harq_len(struct rte_bbdev_op_ldpc_dec *ops_ld) 2207 { 2208 uint16_t k0 = 0; 2209 uint8_t max_rv = (ops_ld->rv_index == 1) ? 3 : ops_ld->rv_index; 2210 k0 = get_k0(ops_ld->n_cb, ops_ld->z_c, ops_ld->basegraph, max_rv, ops_ld->k0); 2211 /* Compute RM out size and number of rows */ 2212 uint16_t parity_offset = (ops_ld->basegraph == 1 ? 20 : 8) 2213 * ops_ld->z_c - ops_ld->n_filler; 2214 uint16_t deRmOutSize = RTE_MIN( 2215 k0 + ops_ld->cb_params.e + 2216 ((k0 > parity_offset) ? 2217 0 : ops_ld->n_filler), 2218 ops_ld->n_cb); 2219 uint16_t numRows = ((deRmOutSize + ops_ld->z_c - 1) 2220 / ops_ld->z_c); 2221 uint16_t harq_output_len = numRows * ops_ld->z_c; 2222 return harq_output_len; 2223 } 2224 2225 static inline int 2226 validate_op_harq_chain(struct rte_bbdev_op_data *op, 2227 struct op_data_entries *orig_op, 2228 struct rte_bbdev_op_ldpc_dec *ops_ld) 2229 { 2230 uint8_t i; 2231 uint32_t j, jj, k; 2232 struct rte_mbuf *m = op->data; 2233 uint8_t nb_dst_segments = orig_op->nb_segments; 2234 uint32_t total_data_size = 0; 2235 int8_t *harq_orig, *harq_out, abs_harq_origin; 2236 uint32_t byte_error = 0, cum_error = 0, error; 2237 int16_t llr_max = (1 << (ldpc_llr_size - ldpc_llr_decimals)) - 1; 2238 int16_t llr_max_pre_scaling = (1 << (ldpc_llr_size - 1)) - 1; 2239 uint16_t parity_offset; 2240 2241 TEST_ASSERT(nb_dst_segments == m->nb_segs, 2242 "Number of segments differ in original (%u) and filled (%u) op", 2243 nb_dst_segments, m->nb_segs); 2244 2245 /* Validate each mbuf segment length */ 2246 for (i = 0; i < nb_dst_segments; ++i) { 2247 /* Apply offset to the first mbuf segment */ 2248 uint16_t offset = (i == 0) ? op->offset : 0; 2249 uint16_t data_len = rte_pktmbuf_data_len(m) - offset; 2250 total_data_size += orig_op->segments[i].length; 2251 2252 TEST_ASSERT(orig_op->segments[i].length < (uint32_t)(data_len + HARQ_MEM_TOLERANCE), 2253 "Length of segment differ in original (%u) and filled (%u) op", 2254 orig_op->segments[i].length, data_len); 2255 harq_orig = (int8_t *) orig_op->segments[i].addr; 2256 harq_out = rte_pktmbuf_mtod_offset(m, int8_t *, offset); 2257 2258 /* Cannot compare HARQ output data for such cases */ 2259 if ((ldpc_llr_decimals > 1) && ((ops_ld->op_flags & RTE_BBDEV_LDPC_LLR_COMPRESSION) 2260 || (ops_ld->op_flags & RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION) 2261 || (ops_ld->op_flags & RTE_BBDEV_LDPC_HARQ_4BIT_COMPRESSION))) 2262 break; 2263 2264 if (!(ldpc_cap_flags & 2265 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_FILLERS 2266 ) || (ops_ld->op_flags & 2267 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)) { 2268 data_len -= ops_ld->z_c; 2269 parity_offset = data_len; 2270 } else { 2271 /* Compute RM out size and number of rows */ 2272 parity_offset = (ops_ld->basegraph == 1 ? 20 : 8) 2273 * ops_ld->z_c - ops_ld->n_filler; 2274 uint16_t deRmOutSize = compute_harq_len(ops_ld) - 2275 ops_ld->n_filler; 2276 if (data_len > deRmOutSize) 2277 data_len = deRmOutSize; 2278 } 2279 if (data_len > orig_op->segments[i].length) 2280 data_len = orig_op->segments[i].length; 2281 /* 2282 * HARQ output can have minor differences 2283 * due to integer representation and related scaling 2284 */ 2285 for (j = 0, jj = 0; j < data_len; j++, jj++) { 2286 if (j == parity_offset) { 2287 /* Special Handling of the filler bits */ 2288 for (k = 0; k < ops_ld->n_filler; k++) { 2289 if (harq_out[jj] != 2290 llr_max_pre_scaling) { 2291 printf("HARQ Filler issue %d: %d %d\n", 2292 jj, harq_out[jj], 2293 llr_max); 2294 byte_error++; 2295 } 2296 jj++; 2297 } 2298 } 2299 if (!(ops_ld->op_flags & 2300 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)) { 2301 if (ldpc_llr_decimals > 1) 2302 harq_out[jj] = (harq_out[jj] + 1) 2303 >> (ldpc_llr_decimals - 1); 2304 /* Saturated to S7 */ 2305 if (harq_orig[j] > llr_max) 2306 harq_orig[j] = llr_max; 2307 if (harq_orig[j] < -llr_max) 2308 harq_orig[j] = -llr_max; 2309 } 2310 if (harq_orig[j] != harq_out[jj]) { 2311 error = (harq_orig[j] > harq_out[jj]) ? 2312 harq_orig[j] - harq_out[jj] : 2313 harq_out[jj] - harq_orig[j]; 2314 abs_harq_origin = harq_orig[j] > 0 ? 2315 harq_orig[j] : 2316 -harq_orig[j]; 2317 /* Residual quantization error */ 2318 if ((error > 8 && (abs_harq_origin < 2319 (llr_max - 16))) || 2320 (error > 16)) { 2321 /* 2322 printf("HARQ mismatch %d: exp %d act %d => %d\n", 2323 j, harq_orig[j], 2324 harq_out[jj], error); 2325 */ 2326 byte_error++; 2327 cum_error += error; 2328 } 2329 } 2330 } 2331 m = m->next; 2332 } 2333 2334 if (byte_error) 2335 TEST_ASSERT(byte_error <= 1, 2336 "HARQ output mismatch (%d) %d", 2337 byte_error, cum_error); 2338 2339 /* Validate total mbuf pkt length */ 2340 uint32_t pkt_len = rte_pktmbuf_pkt_len(op->data) - op->offset; 2341 TEST_ASSERT(total_data_size < pkt_len + HARQ_MEM_TOLERANCE, 2342 "Length of data differ in original (%u) and filled (%u) op", 2343 total_data_size, pkt_len); 2344 2345 return TEST_SUCCESS; 2346 } 2347 2348 2349 static inline int 2350 validate_op_so_chain(struct rte_bbdev_op_data *op, 2351 struct op_data_entries *orig_op) 2352 { 2353 struct rte_mbuf *m = op->data; 2354 uint8_t i, nb_dst_segments = orig_op->nb_segments; 2355 uint32_t j, jj; 2356 int8_t *so_orig, *so_out; 2357 uint32_t byte_error = 0, error, margin_error = 0; 2358 2359 TEST_ASSERT(nb_dst_segments == m->nb_segs, 2360 "Number of segments differ in original (%u) and filled (%u) op", 2361 nb_dst_segments, m->nb_segs); 2362 2363 /* Validate each mbuf segment length. */ 2364 for (i = 0; i < nb_dst_segments; ++i) { 2365 /* Apply offset to the first mbuf segment. */ 2366 uint16_t offset = (i == 0) ? op->offset : 0; 2367 uint16_t data_len = rte_pktmbuf_data_len(m) - offset; 2368 2369 TEST_ASSERT(orig_op->segments[i].length == data_len, 2370 "Length of segment differ in original (%u) and filled (%u) op", 2371 orig_op->segments[i].length, data_len); 2372 so_orig = (int8_t *) orig_op->segments[i].addr; 2373 so_out = rte_pktmbuf_mtod_offset(m, int8_t *, offset); 2374 margin_error += data_len / 8; /* Allow for few % errors. */ 2375 2376 /* SO output can have minor differences due to algorithm variations. */ 2377 for (j = 0, jj = 0; j < data_len; j++, jj++) { 2378 if (so_orig[j] != so_out[jj]) { 2379 error = (so_orig[j] > so_out[jj]) ? so_orig[j] - so_out[jj] : 2380 so_out[jj] - so_orig[j]; 2381 /* Residual quantization error. */ 2382 if (error > 32) { 2383 printf("Warning: Soft mismatch %d: exp %d act %d => %d\n", 2384 j, so_orig[j], so_out[jj], error); 2385 byte_error++; 2386 } 2387 } 2388 } 2389 m = m->next; 2390 } 2391 2392 if (byte_error > margin_error) 2393 TEST_ASSERT(byte_error <= 1, "Soft output mismatch (%d) %d", 2394 byte_error, margin_error); 2395 2396 return TEST_SUCCESS; 2397 } 2398 2399 static int 2400 validate_dec_op(struct rte_bbdev_dec_op **ops, const uint16_t n, 2401 struct rte_bbdev_dec_op *ref_op) 2402 { 2403 unsigned int i; 2404 int ret; 2405 struct op_data_entries *hard_data_orig = 2406 &test_vector.entries[DATA_HARD_OUTPUT]; 2407 struct op_data_entries *soft_data_orig = 2408 &test_vector.entries[DATA_SOFT_OUTPUT]; 2409 struct rte_bbdev_op_turbo_dec *ops_td; 2410 struct rte_bbdev_op_data *hard_output; 2411 struct rte_bbdev_op_data *soft_output; 2412 2413 for (i = 0; i < n; ++i) { 2414 ops_td = &ops[i]->turbo_dec; 2415 hard_output = &ops_td->hard_output; 2416 soft_output = &ops_td->soft_output; 2417 2418 ret = check_dec_status_and_ordering(ops[i], i, ref_op->status); 2419 TEST_ASSERT_SUCCESS(ret, 2420 "Checking status and ordering for decoder failed"); 2421 2422 TEST_ASSERT_SUCCESS(validate_op_chain(hard_output, 2423 hard_data_orig), 2424 "Hard output buffers (CB=%u) are not equal", 2425 i); 2426 2427 if (ref_op->turbo_dec.op_flags & RTE_BBDEV_TURBO_SOFT_OUTPUT) 2428 TEST_ASSERT_SUCCESS(validate_op_so_chain(soft_output, 2429 soft_data_orig), 2430 "Soft output buffers (CB=%u) are not equal", 2431 i); 2432 } 2433 2434 return TEST_SUCCESS; 2435 } 2436 2437 /* Check Number of code blocks errors */ 2438 static int 2439 validate_ldpc_bler(struct rte_bbdev_dec_op **ops, const uint16_t n) 2440 { 2441 unsigned int i; 2442 struct op_data_entries *hard_data_orig = 2443 &test_vector.entries[DATA_HARD_OUTPUT]; 2444 struct rte_bbdev_op_ldpc_dec *ops_td; 2445 struct rte_bbdev_op_data *hard_output; 2446 int errors = 0; 2447 struct rte_mbuf *m; 2448 2449 for (i = 0; i < n; ++i) { 2450 ops_td = &ops[i]->ldpc_dec; 2451 hard_output = &ops_td->hard_output; 2452 m = hard_output->data; 2453 if (memcmp(rte_pktmbuf_mtod_offset(m, uint32_t *, 0), 2454 hard_data_orig->segments[0].addr, 2455 hard_data_orig->segments[0].length)) 2456 errors++; 2457 } 2458 return errors; 2459 } 2460 2461 /* Check Number of code blocks errors */ 2462 static int 2463 validate_turbo_bler(struct rte_bbdev_dec_op **ops, const uint16_t n) 2464 { 2465 unsigned int i; 2466 struct op_data_entries *hard_data_orig = &test_vector.entries[DATA_HARD_OUTPUT]; 2467 struct rte_bbdev_op_turbo_dec *ops_td; 2468 struct rte_bbdev_op_data *hard_output; 2469 int errors = 0; 2470 struct rte_mbuf *m; 2471 2472 for (i = 0; i < n; ++i) { 2473 ops_td = &ops[i]->turbo_dec; 2474 hard_output = &ops_td->hard_output; 2475 m = hard_output->data; 2476 if (memcmp(rte_pktmbuf_mtod_offset(m, uint32_t *, 0), 2477 hard_data_orig->segments[0].addr, 2478 hard_data_orig->segments[0].length)) 2479 errors++; 2480 } 2481 return errors; 2482 } 2483 2484 2485 static int 2486 validate_ldpc_dec_op(struct rte_bbdev_dec_op **ops, const uint16_t n, 2487 struct rte_bbdev_dec_op *ref_op, const int vector_mask) 2488 { 2489 unsigned int i; 2490 int ret; 2491 struct op_data_entries *hard_data_orig = 2492 &test_vector.entries[DATA_HARD_OUTPUT]; 2493 struct op_data_entries *soft_data_orig = 2494 &test_vector.entries[DATA_SOFT_OUTPUT]; 2495 struct op_data_entries *harq_data_orig = 2496 &test_vector.entries[DATA_HARQ_OUTPUT]; 2497 struct rte_bbdev_op_ldpc_dec *ops_td; 2498 struct rte_bbdev_op_data *hard_output; 2499 struct rte_bbdev_op_data *harq_output; 2500 struct rte_bbdev_op_data *soft_output; 2501 struct rte_bbdev_op_ldpc_dec *ref_td = &ref_op->ldpc_dec; 2502 2503 for (i = 0; i < n; ++i) { 2504 ops_td = &ops[i]->ldpc_dec; 2505 hard_output = &ops_td->hard_output; 2506 harq_output = &ops_td->harq_combined_output; 2507 soft_output = &ops_td->soft_output; 2508 2509 ret = check_dec_status_and_ordering(ops[i], i, ref_op->status); 2510 TEST_ASSERT_SUCCESS(ret, 2511 "Checking status and ordering for decoder failed"); 2512 if (vector_mask & TEST_BBDEV_VF_EXPECTED_ITER_COUNT) 2513 TEST_ASSERT(ops_td->iter_count <= ref_td->iter_count, 2514 "Returned iter_count (%d) > expected iter_count (%d)", 2515 ops_td->iter_count, ref_td->iter_count); 2516 /* 2517 * We can ignore output data when the decoding failed to 2518 * converge or for loop-back cases 2519 */ 2520 if (!check_bit(ops[i]->ldpc_dec.op_flags, 2521 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK 2522 ) && ( 2523 ops[i]->status & (1 << RTE_BBDEV_SYNDROME_ERROR 2524 )) == 0) 2525 TEST_ASSERT_SUCCESS(validate_op_chain(hard_output, 2526 hard_data_orig), 2527 "Hard output buffers (CB=%u) are not equal", 2528 i); 2529 2530 if (ref_op->ldpc_dec.op_flags & RTE_BBDEV_LDPC_SOFT_OUT_ENABLE) 2531 TEST_ASSERT_SUCCESS(validate_op_so_chain(soft_output, 2532 soft_data_orig), 2533 "Soft output buffers (CB=%u) are not equal", 2534 i); 2535 if (ref_op->ldpc_dec.op_flags & 2536 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE) { 2537 TEST_ASSERT_SUCCESS(validate_op_harq_chain(harq_output, 2538 harq_data_orig, ops_td), 2539 "HARQ output buffers (CB=%u) are not equal", 2540 i); 2541 } 2542 if (ref_op->ldpc_dec.op_flags & 2543 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK) 2544 TEST_ASSERT_SUCCESS(validate_op_harq_chain(harq_output, 2545 harq_data_orig, ops_td), 2546 "HARQ output buffers (CB=%u) are not equal", 2547 i); 2548 2549 } 2550 2551 return TEST_SUCCESS; 2552 } 2553 2554 2555 static int 2556 validate_enc_op(struct rte_bbdev_enc_op **ops, const uint16_t n, 2557 struct rte_bbdev_enc_op *ref_op) 2558 { 2559 unsigned int i; 2560 int ret; 2561 struct op_data_entries *hard_data_orig = 2562 &test_vector.entries[DATA_HARD_OUTPUT]; 2563 2564 for (i = 0; i < n; ++i) { 2565 ret = check_enc_status_and_ordering(ops[i], i, ref_op->status); 2566 TEST_ASSERT_SUCCESS(ret, 2567 "Checking status and ordering for encoder failed"); 2568 TEST_ASSERT_SUCCESS(validate_op_chain( 2569 &ops[i]->turbo_enc.output, 2570 hard_data_orig), 2571 "Output buffers (CB=%u) are not equal", 2572 i); 2573 } 2574 2575 return TEST_SUCCESS; 2576 } 2577 2578 static int 2579 validate_ldpc_enc_op(struct rte_bbdev_enc_op **ops, const uint16_t n, 2580 struct rte_bbdev_enc_op *ref_op) 2581 { 2582 unsigned int i; 2583 int ret; 2584 struct op_data_entries *hard_data_orig = 2585 &test_vector.entries[DATA_HARD_OUTPUT]; 2586 2587 for (i = 0; i < n; ++i) { 2588 ret = check_enc_status_and_ordering(ops[i], i, ref_op->status); 2589 TEST_ASSERT_SUCCESS(ret, 2590 "Checking status and ordering for encoder failed"); 2591 TEST_ASSERT_SUCCESS(validate_op_chain( 2592 &ops[i]->ldpc_enc.output, 2593 hard_data_orig), 2594 "Output buffers (CB=%u) are not equal", 2595 i); 2596 } 2597 2598 return TEST_SUCCESS; 2599 } 2600 2601 static inline int 2602 validate_op_fft_chain(struct rte_bbdev_op_data *op, struct op_data_entries *orig_op, 2603 bool skip_validate_output) 2604 { 2605 struct rte_mbuf *m = op->data; 2606 uint8_t i, nb_dst_segments = orig_op->nb_segments; 2607 int16_t delt, abs_delt, thres_hold = 4; 2608 uint32_t j, data_len_iq, error_num; 2609 int16_t *ref_out, *op_out; 2610 2611 TEST_ASSERT(nb_dst_segments == m->nb_segs, 2612 "Number of segments differ in original (%u) and filled (%u) op fft", 2613 nb_dst_segments, m->nb_segs); 2614 2615 /* Due to size limitation of mbuf, FFT doesn't use real mbuf. */ 2616 for (i = 0; i < nb_dst_segments; ++i) { 2617 uint16_t offset = (i == 0) ? op->offset : 0; 2618 uint32_t data_len = op->length; 2619 2620 TEST_ASSERT(orig_op->segments[i].length == data_len, 2621 "Length of segment differ in original (%u) and filled (%u) op fft", 2622 orig_op->segments[i].length, data_len); 2623 /* Divided by 2 to get the number of 16bits data. */ 2624 data_len_iq = data_len >> 1; 2625 ref_out = (int16_t *)(orig_op->segments[i].addr); 2626 op_out = rte_pktmbuf_mtod_offset(m, int16_t *, offset); 2627 error_num = 0; 2628 for (j = 0; j < data_len_iq; j++) { 2629 delt = ref_out[j] - op_out[j]; 2630 abs_delt = delt > 0 ? delt : -delt; 2631 error_num += (abs_delt > thres_hold ? 1 : 0); 2632 } 2633 if ((error_num > 0) && !skip_validate_output) { 2634 rte_memdump(stdout, "Buffer A", ref_out, data_len); 2635 rte_memdump(stdout, "Buffer B", op_out, data_len); 2636 TEST_ASSERT(error_num == 0, 2637 "FFT Output are not matched total (%u) errors (%u)", 2638 data_len_iq, error_num); 2639 } 2640 2641 m = m->next; 2642 } 2643 2644 return TEST_SUCCESS; 2645 } 2646 2647 static inline int 2648 validate_op_fft_meas_chain(struct rte_bbdev_op_data *op, struct op_data_entries *orig_op, 2649 bool skip_validate_output) 2650 { 2651 struct rte_mbuf *m = op->data; 2652 uint8_t i, nb_dst_segments = orig_op->nb_segments; 2653 double thres_hold = 1.0; 2654 uint32_t j, data_len_iq, error_num; 2655 int32_t *ref_out, *op_out; 2656 double estSNR, refSNR, delt, abs_delt; 2657 2658 TEST_ASSERT(nb_dst_segments == m->nb_segs, 2659 "Number of segments differ in original (%u) and filled (%u) op fft", 2660 nb_dst_segments, m->nb_segs); 2661 2662 /* Due to size limitation of mbuf, FFT doesn't use real mbuf. */ 2663 for (i = 0; i < nb_dst_segments; ++i) { 2664 uint16_t offset = (i == 0) ? op->offset : 0; 2665 uint32_t data_len = op->length; 2666 2667 TEST_ASSERT(orig_op->segments[i].length == data_len, 2668 "Length of segment differ in original (%u) and filled (%u) op fft", 2669 orig_op->segments[i].length, data_len); 2670 2671 /* Divided by 4 to get the number of 32 bits data. */ 2672 data_len_iq = data_len >> 2; 2673 ref_out = (int32_t *)(orig_op->segments[i].addr); 2674 op_out = rte_pktmbuf_mtod_offset(m, int32_t *, offset); 2675 error_num = 0; 2676 for (j = 0; j < data_len_iq; j++) { 2677 estSNR = 10*log10(op_out[j]); 2678 refSNR = 10*log10(ref_out[j]); 2679 delt = refSNR - estSNR; 2680 abs_delt = delt > 0 ? delt : -delt; 2681 error_num += (abs_delt > thres_hold ? 1 : 0); 2682 } 2683 if ((error_num > 0) && !skip_validate_output) { 2684 rte_memdump(stdout, "Buffer A", ref_out, data_len); 2685 rte_memdump(stdout, "Buffer B", op_out, data_len); 2686 TEST_ASSERT(error_num == 0, 2687 "FFT Output are not matched total (%u) errors (%u)", 2688 data_len_iq, error_num); 2689 } 2690 2691 m = m->next; 2692 } 2693 2694 return TEST_SUCCESS; 2695 } 2696 2697 static inline int 2698 validate_op_mldts_chain(struct rte_bbdev_op_data *op, 2699 struct op_data_entries *orig_op) 2700 { 2701 uint8_t i; 2702 struct rte_mbuf *m = op->data; 2703 uint8_t nb_dst_segments = orig_op->nb_segments; 2704 /*the result is not bit exact*/ 2705 int16_t thres_hold = 3; 2706 int16_t delt, abs_delt; 2707 uint32_t j, data_len_iq; 2708 uint32_t error_num; 2709 int8_t *ref_out; 2710 int8_t *op_out; 2711 2712 TEST_ASSERT(nb_dst_segments == m->nb_segs, 2713 "Number of segments differ in original (%u) and filled (%u) op mldts", 2714 nb_dst_segments, m->nb_segs); 2715 2716 /* Due to size limitation of mbuf, MLDTS doesn't use real mbuf. */ 2717 for (i = 0; i < nb_dst_segments; ++i) { 2718 uint16_t offset = (i == 0) ? op->offset : 0; 2719 uint32_t data_len = op->length; 2720 2721 TEST_ASSERT(orig_op->segments[i].length == data_len, 2722 "Length of segment differ in original (%u) and filled (%u) op mldts", 2723 orig_op->segments[i].length, data_len); 2724 data_len_iq = data_len; 2725 ref_out = (int8_t *)(orig_op->segments[i].addr); 2726 op_out = rte_pktmbuf_mtod_offset(m, int8_t *, offset); 2727 error_num = 0; 2728 for (j = 0; j < data_len_iq; j++) { 2729 2730 delt = ref_out[j] - op_out[j]; 2731 abs_delt = delt > 0 ? delt : -delt; 2732 error_num += (abs_delt > thres_hold ? 1 : 0); 2733 if (error_num > 0) 2734 printf("MLD Error %d: Exp %x %d Actual %x %d Diff %d\n", 2735 j, ref_out[j], ref_out[j], op_out[j], op_out[j], 2736 delt); 2737 } 2738 TEST_ASSERT(error_num == 0, 2739 "MLDTS Output are not matched total (%u) errors (%u)", 2740 data_len_iq, error_num); 2741 2742 m = m->next; 2743 } 2744 2745 return TEST_SUCCESS; 2746 } 2747 2748 static int 2749 validate_fft_op(struct rte_bbdev_fft_op **ops, const uint16_t n, 2750 struct rte_bbdev_fft_op *ref_op) 2751 { 2752 unsigned int i; 2753 int ret; 2754 struct op_data_entries *fft_data_orig = &test_vector.entries[DATA_HARD_OUTPUT]; 2755 struct op_data_entries *fft_pwr_orig = &test_vector.entries[DATA_SOFT_OUTPUT]; 2756 bool skip_validate_output = false; 2757 2758 if ((test_vector.fft_window_width_vec > 0) && 2759 (test_vector.fft_window_width_vec != fft_window_width_dev)) { 2760 printf("The vector FFT width doesn't match with device - skip %d %d\n", 2761 test_vector.fft_window_width_vec, fft_window_width_dev); 2762 skip_validate_output = true; 2763 } 2764 2765 for (i = 0; i < n; ++i) { 2766 ret = check_fft_status_and_ordering(ops[i], i, ref_op->status); 2767 TEST_ASSERT_SUCCESS(ret, "Checking status and ordering for FFT failed"); 2768 TEST_ASSERT_SUCCESS(validate_op_fft_chain( 2769 &ops[i]->fft.base_output, fft_data_orig, skip_validate_output), 2770 "FFT Output buffers (op=%u) are not matched", i); 2771 if (check_bit(ops[i]->fft.op_flags, RTE_BBDEV_FFT_POWER_MEAS)) 2772 TEST_ASSERT_SUCCESS(validate_op_fft_meas_chain( 2773 &ops[i]->fft.power_meas_output, fft_pwr_orig, skip_validate_output), 2774 "FFT Power Output buffers (op=%u) are not matched", i); 2775 } 2776 2777 return TEST_SUCCESS; 2778 } 2779 2780 static int 2781 validate_mldts_op(struct rte_bbdev_mldts_op **ops, const uint16_t n, 2782 struct rte_bbdev_mldts_op *ref_op) 2783 { 2784 unsigned int i; 2785 int ret; 2786 struct op_data_entries *mldts_data_orig = 2787 &test_vector.entries[DATA_HARD_OUTPUT]; 2788 for (i = 0; i < n; ++i) { 2789 ret = check_mldts_status_and_ordering(ops[i], i, ref_op->status); 2790 TEST_ASSERT_SUCCESS(ret, 2791 "Checking status and ordering for MLDTS failed"); 2792 TEST_ASSERT_SUCCESS(validate_op_mldts_chain( 2793 &ops[i]->mldts.output, 2794 mldts_data_orig), 2795 "MLDTS Output buffers (op=%u) are not matched", 2796 i); 2797 } 2798 2799 return TEST_SUCCESS; 2800 } 2801 2802 static void 2803 create_reference_dec_op(struct rte_bbdev_dec_op *op) 2804 { 2805 unsigned int i; 2806 struct op_data_entries *entry; 2807 2808 op->turbo_dec = test_vector.turbo_dec; 2809 entry = &test_vector.entries[DATA_INPUT]; 2810 for (i = 0; i < entry->nb_segments; ++i) 2811 op->turbo_dec.input.length += 2812 entry->segments[i].length; 2813 } 2814 2815 static void 2816 create_reference_ldpc_dec_op(struct rte_bbdev_dec_op *op) 2817 { 2818 unsigned int i; 2819 struct op_data_entries *entry; 2820 2821 op->ldpc_dec = test_vector.ldpc_dec; 2822 entry = &test_vector.entries[DATA_INPUT]; 2823 for (i = 0; i < entry->nb_segments; ++i) 2824 op->ldpc_dec.input.length += 2825 entry->segments[i].length; 2826 if (test_vector.ldpc_dec.op_flags & 2827 RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE) { 2828 entry = &test_vector.entries[DATA_HARQ_INPUT]; 2829 for (i = 0; i < entry->nb_segments; ++i) 2830 op->ldpc_dec.harq_combined_input.length += 2831 entry->segments[i].length; 2832 } 2833 } 2834 2835 static void 2836 create_reference_fft_op(struct rte_bbdev_fft_op *op) 2837 { 2838 unsigned int i; 2839 struct op_data_entries *entry; 2840 op->fft = test_vector.fft; 2841 entry = &test_vector.entries[DATA_INPUT]; 2842 for (i = 0; i < entry->nb_segments; ++i) 2843 op->fft.base_input.length += entry->segments[i].length; 2844 entry = &test_vector.entries[DATA_HARQ_INPUT]; 2845 for (i = 0; i < entry->nb_segments; ++i) 2846 op->fft.dewindowing_input.length += entry->segments[i].length; 2847 } 2848 2849 static void 2850 create_reference_mldts_op(struct rte_bbdev_mldts_op *op) 2851 { 2852 unsigned int i; 2853 struct op_data_entries *entry; 2854 op->mldts = test_vector.mldts; 2855 entry = &test_vector.entries[DATA_INPUT]; 2856 for (i = 0; i < entry->nb_segments; ++i) 2857 op->mldts.qhy_input.length += entry->segments[i].length; 2858 entry = &test_vector.entries[DATA_HARQ_INPUT]; 2859 for (i = 0; i < entry->nb_segments; ++i) 2860 op->mldts.r_input.length += entry->segments[i].length; 2861 } 2862 2863 static void 2864 create_reference_enc_op(struct rte_bbdev_enc_op *op) 2865 { 2866 unsigned int i; 2867 struct op_data_entries *entry; 2868 2869 op->turbo_enc = test_vector.turbo_enc; 2870 entry = &test_vector.entries[DATA_INPUT]; 2871 for (i = 0; i < entry->nb_segments; ++i) 2872 op->turbo_enc.input.length += 2873 entry->segments[i].length; 2874 } 2875 2876 static void 2877 create_reference_ldpc_enc_op(struct rte_bbdev_enc_op *op) 2878 { 2879 unsigned int i; 2880 struct op_data_entries *entry; 2881 2882 op->ldpc_enc = test_vector.ldpc_enc; 2883 entry = &test_vector.entries[DATA_INPUT]; 2884 for (i = 0; i < entry->nb_segments; ++i) 2885 op->ldpc_enc.input.length += 2886 entry->segments[i].length; 2887 } 2888 2889 static uint32_t 2890 calc_dec_TB_size(struct rte_bbdev_dec_op *op) 2891 { 2892 uint8_t i; 2893 uint32_t c, r, tb_size = 0; 2894 2895 if (op->turbo_dec.code_block_mode == RTE_BBDEV_CODE_BLOCK) { 2896 tb_size = op->turbo_dec.tb_params.k_neg; 2897 } else { 2898 c = op->turbo_dec.tb_params.c; 2899 r = op->turbo_dec.tb_params.r; 2900 for (i = 0; i < c-r; i++) 2901 tb_size += (r < op->turbo_dec.tb_params.c_neg) ? 2902 op->turbo_dec.tb_params.k_neg : 2903 op->turbo_dec.tb_params.k_pos; 2904 } 2905 return tb_size; 2906 } 2907 2908 static uint32_t 2909 calc_ldpc_dec_TB_size(struct rte_bbdev_dec_op *op) 2910 { 2911 uint8_t num_cbs = 0; 2912 uint32_t tb_size = 0; 2913 uint16_t sys_cols = (op->ldpc_dec.basegraph == 1) ? 22 : 10; 2914 2915 if (op->ldpc_dec.code_block_mode == RTE_BBDEV_CODE_BLOCK) 2916 num_cbs = 1; 2917 else 2918 num_cbs = op->ldpc_dec.tb_params.c - op->ldpc_dec.tb_params.r; 2919 2920 tb_size = (sys_cols * op->ldpc_dec.z_c - op->ldpc_dec.n_filler) * num_cbs; 2921 return tb_size; 2922 } 2923 2924 static uint32_t 2925 calc_enc_TB_size(struct rte_bbdev_enc_op *op) 2926 { 2927 uint8_t i; 2928 uint32_t c, r, tb_size = 0; 2929 2930 if (op->turbo_enc.code_block_mode == RTE_BBDEV_CODE_BLOCK) { 2931 tb_size = op->turbo_enc.tb_params.k_neg; 2932 } else { 2933 c = op->turbo_enc.tb_params.c; 2934 r = op->turbo_enc.tb_params.r; 2935 for (i = 0; i < c-r; i++) 2936 tb_size += (r < op->turbo_enc.tb_params.c_neg) ? 2937 op->turbo_enc.tb_params.k_neg : 2938 op->turbo_enc.tb_params.k_pos; 2939 } 2940 return tb_size; 2941 } 2942 2943 static uint32_t 2944 calc_ldpc_enc_TB_size(struct rte_bbdev_enc_op *op) 2945 { 2946 uint8_t num_cbs = 0; 2947 uint32_t tb_size = 0; 2948 uint16_t sys_cols = (op->ldpc_enc.basegraph == 1) ? 22 : 10; 2949 2950 if (op->ldpc_enc.code_block_mode == RTE_BBDEV_CODE_BLOCK) 2951 num_cbs = 1; 2952 else 2953 num_cbs = op->ldpc_enc.tb_params.c - op->ldpc_enc.tb_params.r; 2954 2955 tb_size = (sys_cols * op->ldpc_enc.z_c - op->ldpc_enc.n_filler) * num_cbs; 2956 return tb_size; 2957 } 2958 2959 static uint32_t 2960 calc_fft_size(struct rte_bbdev_fft_op *op) 2961 { 2962 uint32_t output_size; 2963 int num_cs = 0, i; 2964 for (i = 0; i < 12; i++) 2965 if (check_bit(op->fft.cs_bitmap, 1 << i)) 2966 num_cs++; 2967 output_size = (num_cs * op->fft.output_sequence_size * 4) << op->fft.num_antennas_log2; 2968 return output_size; 2969 } 2970 2971 static uint32_t 2972 calc_mldts_size(struct rte_bbdev_mldts_op *op) 2973 { 2974 uint32_t output_size = 0; 2975 uint16_t i; 2976 2977 for (i = 0; i < op->mldts.num_layers; i++) 2978 output_size += op->mldts.q_m[i]; 2979 2980 output_size *= NUM_SC_PER_RB * BITS_PER_LLR * op->mldts.num_rbs * (op->mldts.c_rep + 1); 2981 2982 return output_size; 2983 } 2984 2985 static int 2986 init_test_op_params(struct test_op_params *op_params, 2987 enum rte_bbdev_op_type op_type, const int expected_status, 2988 const int vector_mask, struct rte_mempool *ops_mp, 2989 uint16_t burst_sz, uint16_t num_to_process, uint16_t num_lcores) 2990 { 2991 int ret = 0; 2992 if (op_type == RTE_BBDEV_OP_TURBO_DEC || 2993 op_type == RTE_BBDEV_OP_LDPC_DEC) 2994 ret = rte_bbdev_dec_op_alloc_bulk(ops_mp, 2995 &op_params->ref_dec_op, 1); 2996 else if (op_type == RTE_BBDEV_OP_FFT) 2997 ret = rte_bbdev_fft_op_alloc_bulk(ops_mp, 2998 &op_params->ref_fft_op, 1); 2999 else if (op_type == RTE_BBDEV_OP_MLDTS) 3000 ret = rte_bbdev_mldts_op_alloc_bulk(ops_mp, 3001 &op_params->ref_mldts_op, 1); 3002 else 3003 ret = rte_bbdev_enc_op_alloc_bulk(ops_mp, 3004 &op_params->ref_enc_op, 1); 3005 3006 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_op_alloc_bulk() failed"); 3007 3008 op_params->mp = ops_mp; 3009 op_params->burst_sz = burst_sz; 3010 op_params->num_to_process = num_to_process; 3011 op_params->num_lcores = num_lcores; 3012 op_params->vector_mask = vector_mask; 3013 if (op_type == RTE_BBDEV_OP_TURBO_DEC || 3014 op_type == RTE_BBDEV_OP_LDPC_DEC) 3015 op_params->ref_dec_op->status = expected_status; 3016 else if (op_type == RTE_BBDEV_OP_TURBO_ENC 3017 || op_type == RTE_BBDEV_OP_LDPC_ENC) 3018 op_params->ref_enc_op->status = expected_status; 3019 else if (op_type == RTE_BBDEV_OP_FFT) 3020 op_params->ref_fft_op->status = expected_status; 3021 else if (op_type == RTE_BBDEV_OP_MLDTS) 3022 op_params->ref_mldts_op->status = expected_status; 3023 return 0; 3024 } 3025 3026 static int 3027 run_test_case_on_device(test_case_function *test_case_func, uint8_t dev_id, 3028 struct test_op_params *op_params) 3029 { 3030 int t_ret, f_ret, socket_id = SOCKET_ID_ANY; 3031 unsigned int i; 3032 struct active_device *ad; 3033 unsigned int burst_sz = get_burst_sz(); 3034 enum rte_bbdev_op_type op_type = test_vector.op_type; 3035 const struct rte_bbdev_op_cap *capabilities = NULL; 3036 3037 ad = &active_devs[dev_id]; 3038 3039 /* Check if device supports op_type */ 3040 if (!is_avail_op(ad, test_vector.op_type)) 3041 return TEST_SUCCESS; 3042 3043 struct rte_bbdev_info info; 3044 rte_bbdev_info_get(ad->dev_id, &info); 3045 socket_id = GET_SOCKET(info.socket_id); 3046 3047 f_ret = create_mempools(ad, socket_id, op_type, 3048 get_num_ops()); 3049 if (f_ret != TEST_SUCCESS) { 3050 printf("Couldn't create mempools"); 3051 goto fail; 3052 } 3053 if (op_type == RTE_BBDEV_OP_NONE) 3054 op_type = RTE_BBDEV_OP_TURBO_ENC; 3055 3056 f_ret = init_test_op_params(op_params, test_vector.op_type, 3057 test_vector.expected_status, 3058 test_vector.mask, 3059 ad->ops_mempool, 3060 burst_sz, 3061 get_num_ops(), 3062 get_num_lcores()); 3063 if (f_ret != TEST_SUCCESS) { 3064 printf("Couldn't init test op params"); 3065 goto fail; 3066 } 3067 3068 3069 /* Find capabilities */ 3070 const struct rte_bbdev_op_cap *cap = info.drv.capabilities; 3071 do { 3072 if (cap->type == test_vector.op_type) { 3073 capabilities = cap; 3074 break; 3075 } 3076 cap++; 3077 } while (cap->type != RTE_BBDEV_OP_NONE); 3078 TEST_ASSERT_NOT_NULL(capabilities, 3079 "Couldn't find capabilities"); 3080 3081 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) { 3082 create_reference_dec_op(op_params->ref_dec_op); 3083 } else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) 3084 create_reference_enc_op(op_params->ref_enc_op); 3085 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) 3086 create_reference_ldpc_enc_op(op_params->ref_enc_op); 3087 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 3088 create_reference_ldpc_dec_op(op_params->ref_dec_op); 3089 else if (test_vector.op_type == RTE_BBDEV_OP_FFT) 3090 create_reference_fft_op(op_params->ref_fft_op); 3091 else if (test_vector.op_type == RTE_BBDEV_OP_MLDTS) 3092 create_reference_mldts_op(op_params->ref_mldts_op); 3093 3094 for (i = 0; i < ad->nb_queues; ++i) { 3095 f_ret = fill_queue_buffers(op_params, 3096 ad->in_mbuf_pool, 3097 ad->hard_out_mbuf_pool, 3098 ad->soft_out_mbuf_pool, 3099 ad->harq_in_mbuf_pool, 3100 ad->harq_out_mbuf_pool, 3101 ad->queue_ids[i], 3102 capabilities, 3103 info.drv.min_alignment, 3104 socket_id); 3105 if (f_ret != TEST_SUCCESS) { 3106 printf("Couldn't init queue buffers"); 3107 goto fail; 3108 } 3109 } 3110 3111 /* Run test case function */ 3112 t_ret = test_case_func(ad, op_params); 3113 3114 if (dump_ops) { 3115 /* Dump queue information in local file. */ 3116 static FILE *fd; 3117 fd = fopen("./dump_bbdev_queue_ops.txt", "w"); 3118 if (fd == NULL) { 3119 printf("Open dump file error.\n"); 3120 return -1; 3121 } 3122 rte_bbdev_queue_ops_dump(ad->dev_id, ad->queue_ids[i], fd); 3123 fclose(fd); 3124 /* Run it once only. */ 3125 dump_ops = false; 3126 } 3127 3128 /* Free active device resources and return */ 3129 free_buffers(ad, op_params); 3130 return t_ret; 3131 3132 fail: 3133 free_buffers(ad, op_params); 3134 return TEST_FAILED; 3135 } 3136 3137 /* Run given test function per active device per supported op type 3138 * per burst size. 3139 */ 3140 static int 3141 run_test_case(test_case_function *test_case_func) 3142 { 3143 int ret = 0; 3144 uint8_t dev; 3145 3146 /* Alloc op_params */ 3147 struct test_op_params *op_params = rte_zmalloc(NULL, 3148 sizeof(struct test_op_params), RTE_CACHE_LINE_SIZE); 3149 TEST_ASSERT_NOT_NULL(op_params, "Failed to alloc %zuB for op_params", 3150 RTE_ALIGN(sizeof(struct test_op_params), 3151 RTE_CACHE_LINE_SIZE)); 3152 3153 /* For each device run test case function */ 3154 for (dev = 0; dev < nb_active_devs; ++dev) 3155 ret |= run_test_case_on_device(test_case_func, dev, op_params); 3156 3157 rte_free(op_params); 3158 3159 return ret; 3160 } 3161 3162 3163 /* Push back the HARQ output from DDR to host */ 3164 static void 3165 retrieve_harq_ddr(uint16_t dev_id, uint16_t queue_id, 3166 struct rte_bbdev_dec_op **ops, 3167 const uint16_t n) 3168 { 3169 uint16_t j; 3170 int save_status, ret; 3171 uint32_t harq_offset = (uint32_t) queue_id * HARQ_INCR * MAX_OPS; 3172 struct rte_bbdev_dec_op *ops_deq[MAX_BURST]; 3173 uint32_t flags = ops[0]->ldpc_dec.op_flags; 3174 bool loopback = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK; 3175 bool mem_out = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE; 3176 bool hc_out = flags & RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE; 3177 bool h_comp = flags & RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION; 3178 for (j = 0; j < n; ++j) { 3179 if ((loopback && mem_out) || hc_out) { 3180 save_status = ops[j]->status; 3181 ops[j]->ldpc_dec.op_flags = 3182 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK + 3183 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE; 3184 if (h_comp) 3185 ops[j]->ldpc_dec.op_flags += 3186 RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION; 3187 ops[j]->ldpc_dec.harq_combined_input.offset = 3188 harq_offset; 3189 ops[j]->ldpc_dec.harq_combined_output.offset = 0; 3190 harq_offset += HARQ_INCR; 3191 if (!loopback) 3192 ops[j]->ldpc_dec.harq_combined_input.length = 3193 ops[j]->ldpc_dec.harq_combined_output.length; 3194 rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id, 3195 &ops[j], 1); 3196 ret = 0; 3197 while (ret == 0) 3198 ret = rte_bbdev_dequeue_ldpc_dec_ops( 3199 dev_id, queue_id, 3200 &ops_deq[j], 1); 3201 ops[j]->ldpc_dec.op_flags = flags; 3202 ops[j]->status = save_status; 3203 } 3204 } 3205 } 3206 3207 /* 3208 * Push back the HARQ output from HW DDR to Host 3209 * Preload HARQ memory input and adjust HARQ offset 3210 */ 3211 static void 3212 preload_harq_ddr(uint16_t dev_id, uint16_t queue_id, 3213 struct rte_bbdev_dec_op **ops, const uint16_t n, 3214 bool preload) 3215 { 3216 uint16_t j; 3217 int deq; 3218 uint32_t harq_offset = (uint32_t) queue_id * HARQ_INCR * MAX_OPS; 3219 struct rte_bbdev_op_data save_hc_in[MAX_OPS], save_hc_out[MAX_OPS]; 3220 struct rte_bbdev_dec_op *ops_deq[MAX_OPS]; 3221 uint32_t flags = ops[0]->ldpc_dec.op_flags; 3222 bool mem_in = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE; 3223 bool hc_in = flags & RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE; 3224 bool mem_out = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE; 3225 bool hc_out = flags & RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE; 3226 bool h_comp = flags & RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION; 3227 if ((mem_in || hc_in) && preload) { 3228 for (j = 0; j < n; ++j) { 3229 save_hc_in[j] = ops[j]->ldpc_dec.harq_combined_input; 3230 save_hc_out[j] = ops[j]->ldpc_dec.harq_combined_output; 3231 ops[j]->ldpc_dec.op_flags = 3232 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK + 3233 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE; 3234 if (h_comp) 3235 ops[j]->ldpc_dec.op_flags += 3236 RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION; 3237 ops[j]->ldpc_dec.harq_combined_output.offset = 3238 harq_offset; 3239 ops[j]->ldpc_dec.harq_combined_input.offset = 0; 3240 harq_offset += HARQ_INCR; 3241 } 3242 rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id, &ops[0], n); 3243 deq = 0; 3244 while (deq != n) 3245 deq += rte_bbdev_dequeue_ldpc_dec_ops( 3246 dev_id, queue_id, &ops_deq[deq], 3247 n - deq); 3248 /* Restore the operations */ 3249 for (j = 0; j < n; ++j) { 3250 ops[j]->ldpc_dec.op_flags = flags; 3251 ops[j]->ldpc_dec.harq_combined_input = save_hc_in[j]; 3252 ops[j]->ldpc_dec.harq_combined_output = save_hc_out[j]; 3253 } 3254 } 3255 harq_offset = (uint32_t) queue_id * HARQ_INCR * MAX_OPS; 3256 for (j = 0; j < n; ++j) { 3257 /* Adjust HARQ offset when we reach external DDR */ 3258 if (mem_in || hc_in) 3259 ops[j]->ldpc_dec.harq_combined_input.offset 3260 = harq_offset; 3261 if (mem_out || hc_out) 3262 ops[j]->ldpc_dec.harq_combined_output.offset 3263 = harq_offset; 3264 harq_offset += HARQ_INCR; 3265 } 3266 } 3267 3268 static void 3269 dequeue_event_callback(uint16_t dev_id, 3270 enum rte_bbdev_event_type event, void *cb_arg, 3271 void *ret_param) 3272 { 3273 int ret; 3274 uint16_t i; 3275 uint64_t total_time; 3276 uint16_t deq, burst_sz, num_ops; 3277 uint16_t queue_id = *(uint16_t *) ret_param; 3278 struct rte_bbdev_info info; 3279 double tb_len_bits; 3280 struct thread_params *tp = cb_arg; 3281 3282 /* Find matching thread params using queue_id */ 3283 for (i = 0; i < MAX_QUEUES; ++i, ++tp) 3284 if (tp->queue_id == queue_id) 3285 break; 3286 3287 if (i == MAX_QUEUES) { 3288 printf("%s: Queue_id from interrupt details was not found!\n", 3289 __func__); 3290 return; 3291 } 3292 3293 if (unlikely(event != RTE_BBDEV_EVENT_DEQUEUE)) { 3294 rte_atomic_store_explicit(&tp->processing_status, TEST_FAILED, 3295 rte_memory_order_relaxed); 3296 printf( 3297 "Dequeue interrupt handler called for incorrect event!\n"); 3298 return; 3299 } 3300 3301 burst_sz = rte_atomic_load_explicit(&tp->burst_sz, rte_memory_order_relaxed); 3302 num_ops = tp->op_params->num_to_process; 3303 3304 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) 3305 deq = rte_bbdev_dequeue_dec_ops(dev_id, queue_id, 3306 &tp->dec_ops[ 3307 rte_atomic_load_explicit(&tp->nb_dequeued, 3308 rte_memory_order_relaxed)], 3309 burst_sz); 3310 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 3311 deq = rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id, 3312 &tp->dec_ops[ 3313 rte_atomic_load_explicit(&tp->nb_dequeued, 3314 rte_memory_order_relaxed)], 3315 burst_sz); 3316 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) 3317 deq = rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id, 3318 &tp->enc_ops[ 3319 rte_atomic_load_explicit(&tp->nb_dequeued, 3320 rte_memory_order_relaxed)], 3321 burst_sz); 3322 else if (test_vector.op_type == RTE_BBDEV_OP_FFT) 3323 deq = rte_bbdev_dequeue_fft_ops(dev_id, queue_id, 3324 &tp->fft_ops[ 3325 rte_atomic_load_explicit(&tp->nb_dequeued, 3326 rte_memory_order_relaxed)], 3327 burst_sz); 3328 else if (test_vector.op_type == RTE_BBDEV_OP_MLDTS) 3329 deq = rte_bbdev_dequeue_mldts_ops(dev_id, queue_id, 3330 &tp->mldts_ops[ 3331 rte_atomic_load_explicit(&tp->nb_dequeued, 3332 rte_memory_order_relaxed)], 3333 burst_sz); 3334 else /*RTE_BBDEV_OP_TURBO_ENC*/ 3335 deq = rte_bbdev_dequeue_enc_ops(dev_id, queue_id, 3336 &tp->enc_ops[ 3337 rte_atomic_load_explicit(&tp->nb_dequeued, 3338 rte_memory_order_relaxed)], 3339 burst_sz); 3340 3341 if (deq < burst_sz) { 3342 printf( 3343 "After receiving the interrupt all operations should be dequeued. Expected: %u, got: %u\n", 3344 burst_sz, deq); 3345 rte_atomic_store_explicit(&tp->processing_status, TEST_FAILED, 3346 rte_memory_order_relaxed); 3347 return; 3348 } 3349 3350 if (rte_atomic_load_explicit(&tp->nb_dequeued, rte_memory_order_relaxed) + deq < num_ops) { 3351 rte_atomic_fetch_add_explicit(&tp->nb_dequeued, deq, rte_memory_order_relaxed); 3352 return; 3353 } 3354 3355 total_time = rte_rdtsc_precise() - tp->start_time; 3356 3357 rte_bbdev_info_get(dev_id, &info); 3358 3359 ret = TEST_SUCCESS; 3360 3361 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) { 3362 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 3363 ret = validate_dec_op(tp->dec_ops, num_ops, ref_op); 3364 /* get the max of iter_count for all dequeued ops */ 3365 for (i = 0; i < num_ops; ++i) 3366 tp->iter_count = RTE_MAX( 3367 tp->dec_ops[i]->turbo_dec.iter_count, 3368 tp->iter_count); 3369 rte_bbdev_dec_op_free_bulk(tp->dec_ops, deq); 3370 } else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) { 3371 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op; 3372 ret = validate_enc_op(tp->enc_ops, num_ops, ref_op); 3373 rte_bbdev_enc_op_free_bulk(tp->enc_ops, deq); 3374 } else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) { 3375 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op; 3376 ret = validate_ldpc_enc_op(tp->enc_ops, num_ops, ref_op); 3377 rte_bbdev_enc_op_free_bulk(tp->enc_ops, deq); 3378 } else if (test_vector.op_type == RTE_BBDEV_OP_FFT) { 3379 struct rte_bbdev_fft_op *ref_op = tp->op_params->ref_fft_op; 3380 ret = validate_fft_op(tp->fft_ops, num_ops, ref_op); 3381 rte_bbdev_fft_op_free_bulk(tp->fft_ops, deq); 3382 } else if (test_vector.op_type == RTE_BBDEV_OP_MLDTS) { 3383 struct rte_bbdev_mldts_op *ref_op = tp->op_params->ref_mldts_op; 3384 ret = validate_mldts_op(tp->mldts_ops, num_ops, ref_op); 3385 rte_bbdev_mldts_op_free_bulk(tp->mldts_ops, deq); 3386 } else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) { 3387 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 3388 ret = validate_ldpc_dec_op(tp->dec_ops, num_ops, ref_op, 3389 tp->op_params->vector_mask); 3390 rte_bbdev_dec_op_free_bulk(tp->dec_ops, deq); 3391 } 3392 3393 if (ret) { 3394 printf("Buffers validation failed\n"); 3395 rte_atomic_store_explicit(&tp->processing_status, TEST_FAILED, 3396 rte_memory_order_relaxed); 3397 } 3398 3399 switch (test_vector.op_type) { 3400 case RTE_BBDEV_OP_TURBO_DEC: 3401 tb_len_bits = calc_dec_TB_size(tp->op_params->ref_dec_op); 3402 break; 3403 case RTE_BBDEV_OP_TURBO_ENC: 3404 tb_len_bits = calc_enc_TB_size(tp->op_params->ref_enc_op); 3405 break; 3406 case RTE_BBDEV_OP_LDPC_DEC: 3407 tb_len_bits = calc_ldpc_dec_TB_size(tp->op_params->ref_dec_op); 3408 break; 3409 case RTE_BBDEV_OP_FFT: 3410 tb_len_bits = calc_fft_size(tp->op_params->ref_fft_op); 3411 break; 3412 case RTE_BBDEV_OP_MLDTS: 3413 tb_len_bits = calc_mldts_size(tp->op_params->ref_mldts_op); 3414 break; 3415 case RTE_BBDEV_OP_LDPC_ENC: 3416 tb_len_bits = calc_ldpc_enc_TB_size(tp->op_params->ref_enc_op); 3417 break; 3418 case RTE_BBDEV_OP_NONE: 3419 tb_len_bits = 0.0; 3420 break; 3421 default: 3422 printf("Unknown op type: %d\n", test_vector.op_type); 3423 rte_atomic_store_explicit(&tp->processing_status, TEST_FAILED, 3424 rte_memory_order_relaxed); 3425 return; 3426 } 3427 3428 tp->ops_per_sec += ((double)num_ops) / 3429 ((double)total_time / (double)rte_get_tsc_hz()); 3430 tp->mbps += (((double)(num_ops * tb_len_bits)) / 1000000.0) / 3431 ((double)total_time / (double)rte_get_tsc_hz()); 3432 3433 rte_atomic_fetch_add_explicit(&tp->nb_dequeued, deq, rte_memory_order_relaxed); 3434 } 3435 3436 static int 3437 throughput_intr_lcore_ldpc_dec(void *arg) 3438 { 3439 struct thread_params *tp = arg; 3440 unsigned int enqueued; 3441 const uint16_t queue_id = tp->queue_id; 3442 const uint16_t burst_sz = tp->op_params->burst_sz; 3443 const uint16_t num_to_process = tp->op_params->num_to_process; 3444 struct rte_bbdev_dec_op *ops[num_to_process]; 3445 struct test_buffers *bufs = NULL; 3446 struct rte_bbdev_info info; 3447 int ret, i, j; 3448 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 3449 uint16_t num_to_enq, enq; 3450 3451 bool loopback = check_bit(ref_op->ldpc_dec.op_flags, 3452 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK); 3453 bool hc_out = check_bit(ref_op->ldpc_dec.op_flags, 3454 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE); 3455 3456 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 3457 "BURST_SIZE should be <= %u", MAX_BURST); 3458 3459 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id), 3460 "Failed to enable interrupts for dev: %u, queue_id: %u", 3461 tp->dev_id, queue_id); 3462 3463 rte_bbdev_info_get(tp->dev_id, &info); 3464 3465 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim), 3466 "NUM_OPS cannot exceed %u for this device", 3467 info.drv.queue_size_lim); 3468 3469 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 3470 3471 rte_atomic_store_explicit(&tp->processing_status, 0, rte_memory_order_relaxed); 3472 rte_atomic_store_explicit(&tp->nb_dequeued, 0, rte_memory_order_relaxed); 3473 3474 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->op_params->sync, SYNC_START, 3475 rte_memory_order_relaxed); 3476 3477 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops, 3478 num_to_process); 3479 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", 3480 num_to_process); 3481 ref_op->ldpc_dec.iter_max = get_iter_max(); 3482 3483 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3484 copy_reference_ldpc_dec_op(ops, num_to_process, 0, bufs->inputs, 3485 bufs->hard_outputs, bufs->soft_outputs, 3486 bufs->harq_inputs, bufs->harq_outputs, ref_op); 3487 3488 /* Set counter to validate the ordering */ 3489 for (j = 0; j < num_to_process; ++j) 3490 ops[j]->opaque_data = (void *)(uintptr_t)j; 3491 3492 for (j = 0; j < TEST_REPETITIONS; ++j) { 3493 for (i = 0; i < num_to_process; ++i) { 3494 if (!loopback) 3495 mbuf_reset(ops[i]->ldpc_dec.hard_output.data); 3496 if (hc_out || loopback) 3497 mbuf_reset(ops[i]->ldpc_dec.harq_combined_output.data); 3498 if (ops[i]->ldpc_dec.soft_output.data != NULL) 3499 mbuf_reset(ops[i]->ldpc_dec.soft_output.data); 3500 } 3501 3502 tp->start_time = rte_rdtsc_precise(); 3503 for (enqueued = 0; enqueued < num_to_process;) { 3504 num_to_enq = burst_sz; 3505 3506 if (unlikely(num_to_process - enqueued < num_to_enq)) 3507 num_to_enq = num_to_process - enqueued; 3508 3509 /* Write to thread burst_sz current number of enqueued 3510 * descriptors. It ensures that proper number of 3511 * descriptors will be dequeued in callback 3512 * function - needed for last batch in case where 3513 * the number of operations is not a multiple of 3514 * burst size. 3515 */ 3516 rte_atomic_store_explicit(&tp->burst_sz, num_to_enq, 3517 rte_memory_order_relaxed); 3518 3519 enq = 0; 3520 do { 3521 enq += rte_bbdev_enqueue_ldpc_dec_ops( 3522 tp->dev_id, 3523 queue_id, &ops[enqueued], 3524 num_to_enq); 3525 } while (unlikely(num_to_enq != enq)); 3526 enqueued += enq; 3527 3528 /* Wait until processing of previous batch is 3529 * completed 3530 */ 3531 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->nb_dequeued, enqueued, 3532 rte_memory_order_relaxed); 3533 } 3534 if (j != TEST_REPETITIONS - 1) 3535 rte_atomic_store_explicit(&tp->nb_dequeued, 0, rte_memory_order_relaxed); 3536 } 3537 3538 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_disable(tp->dev_id, queue_id), 3539 "Failed to disable interrupts for dev: %u, queue_id: %u", 3540 tp->dev_id, queue_id); 3541 3542 return TEST_SUCCESS; 3543 } 3544 3545 static int 3546 throughput_intr_lcore_dec(void *arg) 3547 { 3548 struct thread_params *tp = arg; 3549 unsigned int enqueued; 3550 const uint16_t queue_id = tp->queue_id; 3551 const uint16_t burst_sz = tp->op_params->burst_sz; 3552 const uint16_t num_to_process = tp->op_params->num_to_process; 3553 struct rte_bbdev_dec_op *ops[num_to_process]; 3554 struct test_buffers *bufs = NULL; 3555 struct rte_bbdev_info info; 3556 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 3557 int ret, i, j; 3558 uint16_t num_to_enq, enq; 3559 3560 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 3561 "BURST_SIZE should be <= %u", MAX_BURST); 3562 3563 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id), 3564 "Failed to enable interrupts for dev: %u, queue_id: %u", 3565 tp->dev_id, queue_id); 3566 3567 rte_bbdev_info_get(tp->dev_id, &info); 3568 3569 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim), 3570 "NUM_OPS cannot exceed %u for this device", 3571 info.drv.queue_size_lim); 3572 3573 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 3574 3575 rte_atomic_store_explicit(&tp->processing_status, 0, rte_memory_order_relaxed); 3576 rte_atomic_store_explicit(&tp->nb_dequeued, 0, rte_memory_order_relaxed); 3577 3578 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->op_params->sync, SYNC_START, 3579 rte_memory_order_relaxed); 3580 3581 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops, 3582 num_to_process); 3583 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_to_process); 3584 ref_op->turbo_dec.iter_max = get_iter_max(); 3585 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3586 copy_reference_dec_op(ops, num_to_process, 0, bufs->inputs, 3587 bufs->hard_outputs, bufs->soft_outputs, 3588 tp->op_params->ref_dec_op); 3589 3590 /* Set counter to validate the ordering. */ 3591 for (j = 0; j < num_to_process; ++j) 3592 ops[j]->opaque_data = (void *)(uintptr_t)j; 3593 3594 for (j = 0; j < TEST_REPETITIONS; ++j) { 3595 for (i = 0; i < num_to_process; ++i) { 3596 mbuf_reset(ops[i]->turbo_dec.hard_output.data); 3597 if (ops[i]->turbo_dec.soft_output.data != NULL) 3598 mbuf_reset(ops[i]->turbo_dec.soft_output.data); 3599 } 3600 3601 tp->start_time = rte_rdtsc_precise(); 3602 for (enqueued = 0; enqueued < num_to_process;) { 3603 num_to_enq = burst_sz; 3604 3605 if (unlikely(num_to_process - enqueued < num_to_enq)) 3606 num_to_enq = num_to_process - enqueued; 3607 3608 /* Write to thread burst_sz current number of enqueued 3609 * descriptors. It ensures that proper number of 3610 * descriptors will be dequeued in callback 3611 * function - needed for last batch in case where 3612 * the number of operations is not a multiple of 3613 * burst size. 3614 */ 3615 rte_atomic_store_explicit(&tp->burst_sz, num_to_enq, 3616 rte_memory_order_relaxed); 3617 3618 enq = 0; 3619 do { 3620 enq += rte_bbdev_enqueue_dec_ops(tp->dev_id, 3621 queue_id, &ops[enqueued], 3622 num_to_enq); 3623 } while (unlikely(num_to_enq != enq)); 3624 enqueued += enq; 3625 3626 /* Wait until processing of previous batch is 3627 * completed 3628 */ 3629 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->nb_dequeued, enqueued, 3630 rte_memory_order_relaxed); 3631 } 3632 if (j != TEST_REPETITIONS - 1) 3633 rte_atomic_store_explicit(&tp->nb_dequeued, 0, rte_memory_order_relaxed); 3634 } 3635 3636 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_disable(tp->dev_id, queue_id), 3637 "Failed to disable interrupts for dev: %u, queue_id: %u", 3638 tp->dev_id, queue_id); 3639 3640 return TEST_SUCCESS; 3641 } 3642 3643 static int 3644 throughput_intr_lcore_enc(void *arg) 3645 { 3646 struct thread_params *tp = arg; 3647 unsigned int enqueued; 3648 const uint16_t queue_id = tp->queue_id; 3649 const uint16_t burst_sz = tp->op_params->burst_sz; 3650 const uint16_t num_to_process = tp->op_params->num_to_process; 3651 struct rte_bbdev_enc_op *ops[num_to_process]; 3652 struct test_buffers *bufs = NULL; 3653 struct rte_bbdev_info info; 3654 int ret, i, j; 3655 uint16_t num_to_enq, enq; 3656 3657 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 3658 "BURST_SIZE should be <= %u", MAX_BURST); 3659 3660 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id), 3661 "Failed to enable interrupts for dev: %u, queue_id: %u", 3662 tp->dev_id, queue_id); 3663 3664 rte_bbdev_info_get(tp->dev_id, &info); 3665 3666 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim), 3667 "NUM_OPS cannot exceed %u for this device", 3668 info.drv.queue_size_lim); 3669 3670 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 3671 3672 rte_atomic_store_explicit(&tp->processing_status, 0, rte_memory_order_relaxed); 3673 rte_atomic_store_explicit(&tp->nb_dequeued, 0, rte_memory_order_relaxed); 3674 3675 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->op_params->sync, SYNC_START, 3676 rte_memory_order_relaxed); 3677 3678 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops, 3679 num_to_process); 3680 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", 3681 num_to_process); 3682 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3683 copy_reference_enc_op(ops, num_to_process, 0, bufs->inputs, 3684 bufs->hard_outputs, tp->op_params->ref_enc_op); 3685 3686 /* Set counter to validate the ordering */ 3687 for (j = 0; j < num_to_process; ++j) 3688 ops[j]->opaque_data = (void *)(uintptr_t)j; 3689 3690 for (j = 0; j < TEST_REPETITIONS; ++j) { 3691 for (i = 0; i < num_to_process; ++i) 3692 mbuf_reset(ops[i]->turbo_enc.output.data); 3693 3694 tp->start_time = rte_rdtsc_precise(); 3695 for (enqueued = 0; enqueued < num_to_process;) { 3696 num_to_enq = burst_sz; 3697 3698 if (unlikely(num_to_process - enqueued < num_to_enq)) 3699 num_to_enq = num_to_process - enqueued; 3700 3701 /* Write to thread burst_sz current number of enqueued 3702 * descriptors. It ensures that proper number of 3703 * descriptors will be dequeued in callback 3704 * function - needed for last batch in case where 3705 * the number of operations is not a multiple of 3706 * burst size. 3707 */ 3708 rte_atomic_store_explicit(&tp->burst_sz, num_to_enq, 3709 rte_memory_order_relaxed); 3710 3711 enq = 0; 3712 do { 3713 enq += rte_bbdev_enqueue_enc_ops(tp->dev_id, 3714 queue_id, &ops[enqueued], 3715 num_to_enq); 3716 } while (unlikely(enq != num_to_enq)); 3717 enqueued += enq; 3718 3719 /* Wait until processing of previous batch is 3720 * completed 3721 */ 3722 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->nb_dequeued, enqueued, 3723 rte_memory_order_relaxed); 3724 } 3725 if (j != TEST_REPETITIONS - 1) 3726 rte_atomic_store_explicit(&tp->nb_dequeued, 0, rte_memory_order_relaxed); 3727 } 3728 3729 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_disable(tp->dev_id, queue_id), 3730 "Failed to disable interrupts for dev: %u, queue_id: %u", 3731 tp->dev_id, queue_id); 3732 3733 return TEST_SUCCESS; 3734 } 3735 3736 3737 static int 3738 throughput_intr_lcore_ldpc_enc(void *arg) 3739 { 3740 struct thread_params *tp = arg; 3741 unsigned int enqueued; 3742 const uint16_t queue_id = tp->queue_id; 3743 const uint16_t burst_sz = tp->op_params->burst_sz; 3744 const uint16_t num_to_process = tp->op_params->num_to_process; 3745 struct rte_bbdev_enc_op *ops[num_to_process]; 3746 struct test_buffers *bufs = NULL; 3747 struct rte_bbdev_info info; 3748 int ret, i, j; 3749 uint16_t num_to_enq, enq; 3750 3751 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 3752 "BURST_SIZE should be <= %u", MAX_BURST); 3753 3754 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id), 3755 "Failed to enable interrupts for dev: %u, queue_id: %u", 3756 tp->dev_id, queue_id); 3757 3758 rte_bbdev_info_get(tp->dev_id, &info); 3759 3760 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim), 3761 "NUM_OPS cannot exceed %u for this device", 3762 info.drv.queue_size_lim); 3763 3764 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 3765 3766 rte_atomic_store_explicit(&tp->processing_status, 0, rte_memory_order_relaxed); 3767 rte_atomic_store_explicit(&tp->nb_dequeued, 0, rte_memory_order_relaxed); 3768 3769 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->op_params->sync, SYNC_START, 3770 rte_memory_order_relaxed); 3771 3772 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops, 3773 num_to_process); 3774 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", 3775 num_to_process); 3776 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3777 copy_reference_ldpc_enc_op(ops, num_to_process, 0, 3778 bufs->inputs, bufs->hard_outputs, 3779 tp->op_params->ref_enc_op); 3780 3781 /* Set counter to validate the ordering */ 3782 for (j = 0; j < num_to_process; ++j) 3783 ops[j]->opaque_data = (void *)(uintptr_t)j; 3784 3785 for (j = 0; j < TEST_REPETITIONS; ++j) { 3786 for (i = 0; i < num_to_process; ++i) 3787 mbuf_reset(ops[i]->turbo_enc.output.data); 3788 3789 tp->start_time = rte_rdtsc_precise(); 3790 for (enqueued = 0; enqueued < num_to_process;) { 3791 num_to_enq = burst_sz; 3792 3793 if (unlikely(num_to_process - enqueued < num_to_enq)) 3794 num_to_enq = num_to_process - enqueued; 3795 3796 /* Write to thread burst_sz current number of enqueued 3797 * descriptors. It ensures that proper number of 3798 * descriptors will be dequeued in callback 3799 * function - needed for last batch in case where 3800 * the number of operations is not a multiple of 3801 * burst size. 3802 */ 3803 rte_atomic_store_explicit(&tp->burst_sz, num_to_enq, 3804 rte_memory_order_relaxed); 3805 3806 enq = 0; 3807 do { 3808 enq += rte_bbdev_enqueue_ldpc_enc_ops( 3809 tp->dev_id, 3810 queue_id, &ops[enqueued], 3811 num_to_enq); 3812 } while (unlikely(enq != num_to_enq)); 3813 enqueued += enq; 3814 3815 /* Wait until processing of previous batch is 3816 * completed 3817 */ 3818 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->nb_dequeued, enqueued, 3819 rte_memory_order_relaxed); 3820 } 3821 if (j != TEST_REPETITIONS - 1) 3822 rte_atomic_store_explicit(&tp->nb_dequeued, 0, rte_memory_order_relaxed); 3823 } 3824 3825 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_disable(tp->dev_id, queue_id), 3826 "Failed to disable interrupts for dev: %u, queue_id: %u", 3827 tp->dev_id, queue_id); 3828 3829 return TEST_SUCCESS; 3830 } 3831 3832 3833 static int 3834 throughput_intr_lcore_fft(void *arg) 3835 { 3836 struct thread_params *tp = arg; 3837 unsigned int enqueued; 3838 const uint16_t queue_id = tp->queue_id; 3839 const uint16_t burst_sz = tp->op_params->burst_sz; 3840 const uint16_t num_to_process = tp->op_params->num_to_process; 3841 struct rte_bbdev_fft_op *ops[num_to_process]; 3842 struct test_buffers *bufs = NULL; 3843 struct rte_bbdev_info info; 3844 int ret, i, j; 3845 uint16_t num_to_enq, enq; 3846 3847 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 3848 "BURST_SIZE should be <= %u", MAX_BURST); 3849 3850 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id), 3851 "Failed to enable interrupts for dev: %u, queue_id: %u", 3852 tp->dev_id, queue_id); 3853 3854 rte_bbdev_info_get(tp->dev_id, &info); 3855 3856 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim), 3857 "NUM_OPS cannot exceed %u for this device", 3858 info.drv.queue_size_lim); 3859 3860 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 3861 3862 rte_atomic_store_explicit(&tp->processing_status, 0, rte_memory_order_relaxed); 3863 rte_atomic_store_explicit(&tp->nb_dequeued, 0, rte_memory_order_relaxed); 3864 3865 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->op_params->sync, SYNC_START, 3866 rte_memory_order_relaxed); 3867 3868 ret = rte_bbdev_fft_op_alloc_bulk(tp->op_params->mp, ops, 3869 num_to_process); 3870 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", 3871 num_to_process); 3872 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3873 copy_reference_fft_op(ops, num_to_process, 0, bufs->inputs, 3874 bufs->hard_outputs, bufs->soft_outputs, bufs->harq_inputs, 3875 tp->op_params->ref_fft_op); 3876 3877 /* Set counter to validate the ordering */ 3878 for (j = 0; j < num_to_process; ++j) 3879 ops[j]->opaque_data = (void *)(uintptr_t)j; 3880 3881 for (j = 0; j < TEST_REPETITIONS; ++j) { 3882 for (i = 0; i < num_to_process; ++i) 3883 mbuf_reset(ops[i]->fft.base_output.data); 3884 3885 tp->start_time = rte_rdtsc_precise(); 3886 for (enqueued = 0; enqueued < num_to_process;) { 3887 num_to_enq = burst_sz; 3888 3889 if (unlikely(num_to_process - enqueued < num_to_enq)) 3890 num_to_enq = num_to_process - enqueued; 3891 3892 /* Write to thread burst_sz current number of enqueued 3893 * descriptors. It ensures that proper number of 3894 * descriptors will be dequeued in callback 3895 * function - needed for last batch in case where 3896 * the number of operations is not a multiple of 3897 * burst size. 3898 */ 3899 rte_atomic_store_explicit(&tp->burst_sz, num_to_enq, 3900 rte_memory_order_relaxed); 3901 3902 enq = 0; 3903 do { 3904 enq += rte_bbdev_enqueue_fft_ops(tp->dev_id, 3905 queue_id, &ops[enqueued], 3906 num_to_enq); 3907 } while (unlikely(enq != num_to_enq)); 3908 enqueued += enq; 3909 3910 /* Wait until processing of previous batch is 3911 * completed 3912 */ 3913 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->nb_dequeued, enqueued, 3914 rte_memory_order_relaxed); 3915 } 3916 if (j != TEST_REPETITIONS - 1) 3917 rte_atomic_store_explicit(&tp->nb_dequeued, 0, rte_memory_order_relaxed); 3918 } 3919 3920 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_disable(tp->dev_id, queue_id), 3921 "Failed to disable interrupts for dev: %u, queue_id: %u", 3922 tp->dev_id, queue_id); 3923 3924 return TEST_SUCCESS; 3925 } 3926 3927 static int 3928 throughput_intr_lcore_mldts(void *arg) 3929 { 3930 struct thread_params *tp = arg; 3931 unsigned int enqueued; 3932 const uint16_t queue_id = tp->queue_id; 3933 const uint16_t burst_sz = tp->op_params->burst_sz; 3934 const uint16_t num_to_process = tp->op_params->num_to_process; 3935 struct rte_bbdev_mldts_op *ops[num_to_process]; 3936 struct test_buffers *bufs = NULL; 3937 struct rte_bbdev_info info; 3938 int ret, i, j; 3939 uint16_t num_to_enq, enq; 3940 3941 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), "BURST_SIZE should be <= %u", MAX_BURST); 3942 3943 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id), 3944 "Failed to enable interrupts for dev: %u, queue_id: %u", 3945 tp->dev_id, queue_id); 3946 3947 rte_bbdev_info_get(tp->dev_id, &info); 3948 3949 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim), 3950 "NUM_OPS cannot exceed %u for this device", 3951 info.drv.queue_size_lim); 3952 3953 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 3954 3955 rte_atomic_store_explicit(&tp->processing_status, 0, rte_memory_order_relaxed); 3956 rte_atomic_store_explicit(&tp->nb_dequeued, 0, rte_memory_order_relaxed); 3957 3958 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->op_params->sync, SYNC_START, 3959 rte_memory_order_relaxed); 3960 3961 ret = rte_bbdev_mldts_op_alloc_bulk(tp->op_params->mp, ops, num_to_process); 3962 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_to_process); 3963 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3964 copy_reference_mldts_op(ops, num_to_process, 0, bufs->inputs, bufs->harq_inputs, 3965 bufs->hard_outputs, tp->op_params->ref_mldts_op); 3966 3967 /* Set counter to validate the ordering */ 3968 for (j = 0; j < num_to_process; ++j) 3969 ops[j]->opaque_data = (void *)(uintptr_t)j; 3970 3971 for (j = 0; j < TEST_REPETITIONS; ++j) { 3972 for (i = 0; i < num_to_process; ++i) 3973 mbuf_reset(ops[i]->mldts.output.data); 3974 3975 tp->start_time = rte_rdtsc_precise(); 3976 for (enqueued = 0; enqueued < num_to_process;) { 3977 num_to_enq = burst_sz; 3978 3979 if (unlikely(num_to_process - enqueued < num_to_enq)) 3980 num_to_enq = num_to_process - enqueued; 3981 3982 /* Write to thread burst_sz current number of enqueued 3983 * descriptors. It ensures that proper number of 3984 * descriptors will be dequeued in callback 3985 * function - needed for last batch in case where 3986 * the number of operations is not a multiple of 3987 * burst size. 3988 */ 3989 rte_atomic_store_explicit(&tp->burst_sz, num_to_enq, 3990 rte_memory_order_relaxed); 3991 3992 enq = 0; 3993 do { 3994 enq += rte_bbdev_enqueue_mldts_ops(tp->dev_id, 3995 queue_id, &ops[enqueued], num_to_enq); 3996 } while (unlikely(enq != num_to_enq)); 3997 enqueued += enq; 3998 3999 /* Wait until processing of previous batch is 4000 * completed 4001 */ 4002 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->nb_dequeued, enqueued, 4003 rte_memory_order_relaxed); 4004 } 4005 if (j != TEST_REPETITIONS - 1) 4006 rte_atomic_store_explicit(&tp->nb_dequeued, 0, rte_memory_order_relaxed); 4007 } 4008 4009 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_disable(tp->dev_id, queue_id), 4010 "Failed to disable interrupts for dev: %u, queue_id: %u", 4011 tp->dev_id, queue_id); 4012 4013 return TEST_SUCCESS; 4014 } 4015 4016 static int 4017 throughput_pmd_lcore_dec(void *arg) 4018 { 4019 struct thread_params *tp = arg; 4020 uint16_t enq, deq; 4021 uint64_t total_time = 0, start_time; 4022 const uint16_t queue_id = tp->queue_id; 4023 const uint16_t burst_sz = tp->op_params->burst_sz; 4024 const uint16_t num_ops = tp->op_params->num_to_process; 4025 struct rte_bbdev_dec_op *ops_enq[num_ops]; 4026 struct rte_bbdev_dec_op *ops_deq[num_ops]; 4027 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 4028 struct test_buffers *bufs = NULL; 4029 int i, j, ret; 4030 struct rte_bbdev_info info; 4031 uint16_t num_to_enq; 4032 bool so_enable; 4033 4034 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 4035 "BURST_SIZE should be <= %u", MAX_BURST); 4036 4037 rte_bbdev_info_get(tp->dev_id, &info); 4038 4039 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 4040 "NUM_OPS cannot exceed %u for this device", 4041 info.drv.queue_size_lim); 4042 4043 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 4044 4045 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->op_params->sync, SYNC_START, 4046 rte_memory_order_relaxed); 4047 4048 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops); 4049 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops); 4050 ref_op->turbo_dec.iter_max = get_iter_max(); 4051 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4052 copy_reference_dec_op(ops_enq, num_ops, 0, bufs->inputs, 4053 bufs->hard_outputs, bufs->soft_outputs, ref_op); 4054 4055 so_enable = check_bit(ops_enq[0]->turbo_dec.op_flags, RTE_BBDEV_TURBO_SOFT_OUTPUT); 4056 4057 /* Set counter to validate the ordering */ 4058 for (j = 0; j < num_ops; ++j) 4059 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 4060 4061 for (i = 0; i < TEST_REPETITIONS; ++i) { 4062 uint32_t time_out = 0; 4063 for (j = 0; j < num_ops; ++j) 4064 mbuf_reset(ops_enq[j]->turbo_dec.hard_output.data); 4065 if (so_enable) 4066 for (j = 0; j < num_ops; ++j) 4067 mbuf_reset(ops_enq[j]->turbo_dec.soft_output.data); 4068 4069 start_time = rte_rdtsc_precise(); 4070 4071 for (enq = 0, deq = 0; enq < num_ops;) { 4072 num_to_enq = burst_sz; 4073 4074 if (unlikely(num_ops - enq < num_to_enq)) 4075 num_to_enq = num_ops - enq; 4076 4077 enq += rte_bbdev_enqueue_dec_ops(tp->dev_id, 4078 queue_id, &ops_enq[enq], num_to_enq); 4079 4080 deq += rte_bbdev_dequeue_dec_ops(tp->dev_id, 4081 queue_id, &ops_deq[deq], enq - deq); 4082 time_out++; 4083 if (time_out >= TIME_OUT_POLL) { 4084 timeout_exit(tp->dev_id); 4085 TEST_ASSERT_SUCCESS(TEST_FAILED, "Enqueue timeout!"); 4086 } 4087 } 4088 4089 /* dequeue the remaining */ 4090 time_out = 0; 4091 while (deq < enq) { 4092 deq += rte_bbdev_dequeue_dec_ops(tp->dev_id, 4093 queue_id, &ops_deq[deq], enq - deq); 4094 time_out++; 4095 if (time_out >= TIME_OUT_POLL) { 4096 timeout_exit(tp->dev_id); 4097 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 4098 } 4099 } 4100 4101 total_time += rte_rdtsc_precise() - start_time; 4102 } 4103 4104 tp->iter_count = 0; 4105 /* get the max of iter_count for all dequeued ops */ 4106 for (i = 0; i < num_ops; ++i) { 4107 tp->iter_count = RTE_MAX(ops_enq[i]->turbo_dec.iter_count, 4108 tp->iter_count); 4109 } 4110 4111 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 4112 ret = validate_dec_op(ops_deq, num_ops, ref_op); 4113 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 4114 } 4115 4116 rte_bbdev_dec_op_free_bulk(ops_enq, num_ops); 4117 4118 double tb_len_bits = calc_dec_TB_size(ref_op); 4119 4120 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) / 4121 ((double)total_time / (double)rte_get_tsc_hz()); 4122 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) / 4123 1000000.0) / ((double)total_time / 4124 (double)rte_get_tsc_hz()); 4125 4126 return TEST_SUCCESS; 4127 } 4128 4129 static int 4130 bler_pmd_lcore_ldpc_dec(void *arg) 4131 { 4132 struct thread_params *tp = arg; 4133 uint16_t enq, deq; 4134 uint64_t total_time = 0, start_time; 4135 const uint16_t queue_id = tp->queue_id; 4136 const uint16_t burst_sz = tp->op_params->burst_sz; 4137 const uint16_t num_ops = tp->op_params->num_to_process; 4138 struct rte_bbdev_dec_op *ops_enq[num_ops]; 4139 struct rte_bbdev_dec_op *ops_deq[num_ops]; 4140 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 4141 struct test_buffers *bufs = NULL; 4142 int i, j, ret; 4143 float parity_bler = 0; 4144 struct rte_bbdev_info info; 4145 uint16_t num_to_enq; 4146 bool extDdr = check_bit(ldpc_cap_flags, 4147 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE); 4148 bool loopback = check_bit(ref_op->ldpc_dec.op_flags, 4149 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK); 4150 bool hc_out = check_bit(ref_op->ldpc_dec.op_flags, 4151 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE); 4152 4153 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 4154 "BURST_SIZE should be <= %u", MAX_BURST); 4155 TEST_ASSERT_SUCCESS((num_ops == 0), "NUM_OPS must be greater than 0"); 4156 4157 rte_bbdev_info_get(tp->dev_id, &info); 4158 4159 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 4160 "NUM_OPS cannot exceed %u for this device", 4161 info.drv.queue_size_lim); 4162 4163 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 4164 4165 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->op_params->sync, SYNC_START, 4166 rte_memory_order_relaxed); 4167 4168 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops); 4169 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops); 4170 4171 /* For BLER tests we need to enable early termination */ 4172 if (!check_bit(ref_op->ldpc_dec.op_flags, RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE)) 4173 ref_op->ldpc_dec.op_flags += RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE; 4174 4175 ref_op->ldpc_dec.iter_max = get_iter_max(); 4176 4177 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4178 copy_reference_ldpc_dec_op(ops_enq, num_ops, 0, bufs->inputs, 4179 bufs->hard_outputs, bufs->soft_outputs, 4180 bufs->harq_inputs, bufs->harq_outputs, ref_op); 4181 generate_llr_input(num_ops, bufs->inputs, ref_op); 4182 4183 /* Set counter to validate the ordering */ 4184 for (j = 0; j < num_ops; ++j) 4185 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 4186 4187 for (i = 0; i < 1; ++i) { /* Could add more iterations */ 4188 uint32_t time_out = 0; 4189 for (j = 0; j < num_ops; ++j) { 4190 if (!loopback) 4191 mbuf_reset(ops_enq[j]->ldpc_dec.hard_output.data); 4192 if (hc_out || loopback) 4193 mbuf_reset(ops_enq[j]->ldpc_dec.harq_combined_output.data); 4194 if (ops_enq[j]->ldpc_dec.soft_output.data != NULL) 4195 mbuf_reset(ops_enq[j]->ldpc_dec.soft_output.data); 4196 } 4197 if (extDdr) 4198 preload_harq_ddr(tp->dev_id, queue_id, ops_enq, 4199 num_ops, true); 4200 start_time = rte_rdtsc_precise(); 4201 4202 for (enq = 0, deq = 0; enq < num_ops;) { 4203 num_to_enq = burst_sz; 4204 4205 if (unlikely(num_ops - enq < num_to_enq)) 4206 num_to_enq = num_ops - enq; 4207 4208 enq += rte_bbdev_enqueue_ldpc_dec_ops(tp->dev_id, 4209 queue_id, &ops_enq[enq], num_to_enq); 4210 4211 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id, 4212 queue_id, &ops_deq[deq], enq - deq); 4213 time_out++; 4214 if (time_out >= TIME_OUT_POLL) { 4215 timeout_exit(tp->dev_id); 4216 TEST_ASSERT_SUCCESS(TEST_FAILED, "Enqueue timeout!"); 4217 } 4218 } 4219 4220 /* dequeue the remaining */ 4221 time_out = 0; 4222 while (deq < enq) { 4223 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id, 4224 queue_id, &ops_deq[deq], enq - deq); 4225 time_out++; 4226 if (time_out >= TIME_OUT_POLL) { 4227 timeout_exit(tp->dev_id); 4228 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 4229 } 4230 } 4231 4232 total_time += rte_rdtsc_precise() - start_time; 4233 } 4234 4235 tp->iter_count = 0; 4236 tp->iter_average = 0; 4237 /* get the max of iter_count for all dequeued ops */ 4238 for (i = 0; i < num_ops; ++i) { 4239 tp->iter_count = RTE_MAX(ops_enq[i]->ldpc_dec.iter_count, 4240 tp->iter_count); 4241 tp->iter_average += (double) ops_enq[i]->ldpc_dec.iter_count; 4242 if (ops_enq[i]->status & (1 << RTE_BBDEV_SYNDROME_ERROR)) 4243 parity_bler += 1.0; 4244 } 4245 4246 parity_bler /= num_ops; /* This one is based on SYND */ 4247 tp->iter_average /= num_ops; 4248 tp->bler = (double) validate_ldpc_bler(ops_deq, num_ops) / num_ops; 4249 4250 if (test_vector.op_type != RTE_BBDEV_OP_NONE 4251 && tp->bler == 0 4252 && parity_bler == 0 4253 && !hc_out) { 4254 ret = validate_ldpc_dec_op(ops_deq, num_ops, ref_op, 4255 tp->op_params->vector_mask); 4256 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 4257 } 4258 4259 rte_bbdev_dec_op_free_bulk(ops_enq, num_ops); 4260 4261 double tb_len_bits = calc_ldpc_dec_TB_size(ref_op); 4262 tp->ops_per_sec = ((double)num_ops * 1) / 4263 ((double)total_time / (double)rte_get_tsc_hz()); 4264 tp->mbps = (((double)(num_ops * 1 * tb_len_bits)) / 4265 1000000.0) / ((double)total_time / 4266 (double)rte_get_tsc_hz()); 4267 4268 return TEST_SUCCESS; 4269 } 4270 4271 4272 static int 4273 bler_pmd_lcore_turbo_dec(void *arg) 4274 { 4275 struct thread_params *tp = arg; 4276 uint16_t enq, deq; 4277 uint64_t total_time = 0, start_time; 4278 const uint16_t queue_id = tp->queue_id; 4279 const uint16_t burst_sz = tp->op_params->burst_sz; 4280 const uint16_t num_ops = tp->op_params->num_to_process; 4281 struct rte_bbdev_dec_op *ops_enq[num_ops]; 4282 struct rte_bbdev_dec_op *ops_deq[num_ops]; 4283 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 4284 struct test_buffers *bufs = NULL; 4285 int i, j, ret; 4286 struct rte_bbdev_info info; 4287 uint16_t num_to_enq; 4288 4289 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 4290 "BURST_SIZE should be <= %u", MAX_BURST); 4291 TEST_ASSERT_SUCCESS((num_ops == 0), "NUM_OPS must be greater than 0"); 4292 4293 rte_bbdev_info_get(tp->dev_id, &info); 4294 4295 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 4296 "NUM_OPS cannot exceed %u for this device", 4297 info.drv.queue_size_lim); 4298 4299 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 4300 4301 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->op_params->sync, SYNC_START, 4302 rte_memory_order_relaxed); 4303 4304 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops); 4305 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops); 4306 4307 /* For BLER tests we need to enable early termination */ 4308 if (!check_bit(ref_op->turbo_dec.op_flags, RTE_BBDEV_TURBO_EARLY_TERMINATION)) 4309 ref_op->turbo_dec.op_flags += RTE_BBDEV_TURBO_EARLY_TERMINATION; 4310 4311 ref_op->turbo_dec.iter_max = get_iter_max(); 4312 4313 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4314 copy_reference_dec_op(ops_enq, num_ops, 0, bufs->inputs, 4315 bufs->hard_outputs, bufs->soft_outputs, 4316 ref_op); 4317 generate_turbo_llr_input(num_ops, bufs->inputs, ref_op); 4318 4319 /* Set counter to validate the ordering */ 4320 for (j = 0; j < num_ops; ++j) 4321 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 4322 4323 for (i = 0; i < 1; ++i) { /* Could add more iterations */ 4324 uint32_t time_out = 0; 4325 for (j = 0; j < num_ops; ++j) { 4326 mbuf_reset( 4327 ops_enq[j]->turbo_dec.hard_output.data); 4328 } 4329 4330 start_time = rte_rdtsc_precise(); 4331 4332 for (enq = 0, deq = 0; enq < num_ops;) { 4333 num_to_enq = burst_sz; 4334 4335 if (unlikely(num_ops - enq < num_to_enq)) 4336 num_to_enq = num_ops - enq; 4337 4338 enq += rte_bbdev_enqueue_dec_ops(tp->dev_id, 4339 queue_id, &ops_enq[enq], num_to_enq); 4340 4341 deq += rte_bbdev_dequeue_dec_ops(tp->dev_id, 4342 queue_id, &ops_deq[deq], enq - deq); 4343 time_out++; 4344 if (time_out >= TIME_OUT_POLL) { 4345 timeout_exit(tp->dev_id); 4346 TEST_ASSERT_SUCCESS(TEST_FAILED, "Enqueue timeout!"); 4347 } 4348 } 4349 4350 /* dequeue the remaining */ 4351 time_out = 0; 4352 while (deq < enq) { 4353 deq += rte_bbdev_dequeue_dec_ops(tp->dev_id, 4354 queue_id, &ops_deq[deq], enq - deq); 4355 time_out++; 4356 if (time_out >= TIME_OUT_POLL) { 4357 timeout_exit(tp->dev_id); 4358 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 4359 } 4360 } 4361 4362 total_time += rte_rdtsc_precise() - start_time; 4363 } 4364 4365 tp->iter_count = 0; 4366 tp->iter_average = 0; 4367 /* get the max of iter_count for all dequeued ops */ 4368 for (i = 0; i < num_ops; ++i) { 4369 tp->iter_count = RTE_MAX(ops_enq[i]->turbo_dec.iter_count, 4370 tp->iter_count); 4371 tp->iter_average += (double) ops_enq[i]->turbo_dec.iter_count; 4372 } 4373 4374 tp->iter_average /= num_ops; 4375 tp->bler = (double) validate_turbo_bler(ops_deq, num_ops) / num_ops; 4376 4377 rte_bbdev_dec_op_free_bulk(ops_enq, num_ops); 4378 4379 double tb_len_bits = calc_dec_TB_size(ref_op); 4380 tp->ops_per_sec = ((double)num_ops * 1) / 4381 ((double)total_time / (double)rte_get_tsc_hz()); 4382 tp->mbps = (((double)(num_ops * 1 * tb_len_bits)) / 4383 1000000.0) / ((double)total_time / 4384 (double)rte_get_tsc_hz()); 4385 printf("TBS %.0f Time %.0f\n", tb_len_bits, 1000000.0 * 4386 ((double)total_time / (double)rte_get_tsc_hz())); 4387 4388 return TEST_SUCCESS; 4389 } 4390 4391 static int 4392 throughput_pmd_lcore_ldpc_dec(void *arg) 4393 { 4394 struct thread_params *tp = arg; 4395 uint16_t enq, deq; 4396 uint64_t total_time = 0, start_time; 4397 const uint16_t queue_id = tp->queue_id; 4398 const uint16_t burst_sz = tp->op_params->burst_sz; 4399 const uint16_t num_ops = tp->op_params->num_to_process; 4400 struct rte_bbdev_dec_op *ops_enq[num_ops]; 4401 struct rte_bbdev_dec_op *ops_deq[num_ops]; 4402 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 4403 struct test_buffers *bufs = NULL; 4404 int i, j, ret; 4405 struct rte_bbdev_info info; 4406 uint16_t num_to_enq; 4407 bool extDdr = check_bit(ldpc_cap_flags, 4408 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE); 4409 bool loopback = check_bit(ref_op->ldpc_dec.op_flags, 4410 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK); 4411 bool hc_out = check_bit(ref_op->ldpc_dec.op_flags, 4412 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE); 4413 4414 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 4415 "BURST_SIZE should be <= %u", MAX_BURST); 4416 4417 rte_bbdev_info_get(tp->dev_id, &info); 4418 4419 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 4420 "NUM_OPS cannot exceed %u for this device", 4421 info.drv.queue_size_lim); 4422 4423 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 4424 4425 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->op_params->sync, SYNC_START, 4426 rte_memory_order_relaxed); 4427 4428 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops); 4429 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops); 4430 4431 /* For throughput tests we need to disable early termination */ 4432 if (check_bit(ref_op->ldpc_dec.op_flags, RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE)) 4433 ref_op->ldpc_dec.op_flags -= RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE; 4434 4435 ref_op->ldpc_dec.iter_max = get_iter_max(); 4436 /* Since ET is disabled, the expected iter_count is iter_max */ 4437 ref_op->ldpc_dec.iter_count = ref_op->ldpc_dec.iter_max; 4438 4439 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4440 copy_reference_ldpc_dec_op(ops_enq, num_ops, 0, bufs->inputs, 4441 bufs->hard_outputs, bufs->soft_outputs, 4442 bufs->harq_inputs, bufs->harq_outputs, ref_op); 4443 4444 /* Set counter to validate the ordering */ 4445 for (j = 0; j < num_ops; ++j) 4446 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 4447 4448 for (i = 0; i < TEST_REPETITIONS; ++i) { 4449 uint32_t time_out = 0; 4450 for (j = 0; j < num_ops; ++j) { 4451 if (!loopback) 4452 mbuf_reset(ops_enq[j]->ldpc_dec.hard_output.data); 4453 if (hc_out || loopback) 4454 mbuf_reset(ops_enq[j]->ldpc_dec.harq_combined_output.data); 4455 if (ops_enq[j]->ldpc_dec.soft_output.data != NULL) 4456 mbuf_reset(ops_enq[j]->ldpc_dec.soft_output.data); 4457 } 4458 if (extDdr) 4459 preload_harq_ddr(tp->dev_id, queue_id, ops_enq, 4460 num_ops, true); 4461 start_time = rte_rdtsc_precise(); 4462 4463 for (enq = 0, deq = 0; enq < num_ops;) { 4464 num_to_enq = burst_sz; 4465 4466 if (unlikely(num_ops - enq < num_to_enq)) 4467 num_to_enq = num_ops - enq; 4468 4469 enq += rte_bbdev_enqueue_ldpc_dec_ops(tp->dev_id, 4470 queue_id, &ops_enq[enq], num_to_enq); 4471 4472 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id, 4473 queue_id, &ops_deq[deq], enq - deq); 4474 time_out++; 4475 if (time_out >= TIME_OUT_POLL) { 4476 timeout_exit(tp->dev_id); 4477 TEST_ASSERT_SUCCESS(TEST_FAILED, "Enqueue timeout!"); 4478 } 4479 } 4480 4481 /* dequeue the remaining */ 4482 time_out = 0; 4483 while (deq < enq) { 4484 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id, 4485 queue_id, &ops_deq[deq], enq - deq); 4486 time_out++; 4487 if (time_out >= TIME_OUT_POLL) { 4488 timeout_exit(tp->dev_id); 4489 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 4490 } 4491 } 4492 4493 total_time += rte_rdtsc_precise() - start_time; 4494 } 4495 4496 tp->iter_count = 0; 4497 /* get the max of iter_count for all dequeued ops */ 4498 for (i = 0; i < num_ops; ++i) { 4499 tp->iter_count = RTE_MAX(ops_enq[i]->ldpc_dec.iter_count, 4500 tp->iter_count); 4501 } 4502 if (extDdr) { 4503 /* Read loopback is not thread safe */ 4504 retrieve_harq_ddr(tp->dev_id, queue_id, ops_enq, num_ops); 4505 } 4506 4507 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 4508 ret = validate_ldpc_dec_op(ops_deq, num_ops, ref_op, 4509 tp->op_params->vector_mask); 4510 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 4511 } 4512 4513 ret = rte_bbdev_queue_stop(tp->dev_id, queue_id); 4514 if (ret != 0) 4515 printf("Failed to stop queue on dev %u q_id: %u\n", tp->dev_id, queue_id); 4516 rte_bbdev_dec_op_free_bulk(ops_enq, num_ops); 4517 4518 double tb_len_bits = calc_ldpc_dec_TB_size(ref_op); 4519 4520 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) / 4521 ((double)total_time / (double)rte_get_tsc_hz()); 4522 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) / 4523 1000000.0) / ((double)total_time / 4524 (double)rte_get_tsc_hz()); 4525 4526 return TEST_SUCCESS; 4527 } 4528 4529 static int 4530 throughput_pmd_lcore_enc(void *arg) 4531 { 4532 struct thread_params *tp = arg; 4533 uint16_t enq, deq; 4534 uint64_t total_time = 0, start_time; 4535 const uint16_t queue_id = tp->queue_id; 4536 const uint16_t burst_sz = tp->op_params->burst_sz; 4537 const uint16_t num_ops = tp->op_params->num_to_process; 4538 struct rte_bbdev_enc_op *ops_enq[num_ops]; 4539 struct rte_bbdev_enc_op *ops_deq[num_ops]; 4540 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op; 4541 struct test_buffers *bufs = NULL; 4542 int i, j, ret; 4543 struct rte_bbdev_info info; 4544 uint16_t num_to_enq; 4545 4546 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 4547 "BURST_SIZE should be <= %u", MAX_BURST); 4548 4549 rte_bbdev_info_get(tp->dev_id, &info); 4550 4551 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 4552 "NUM_OPS cannot exceed %u for this device", 4553 info.drv.queue_size_lim); 4554 4555 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 4556 4557 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->op_params->sync, SYNC_START, 4558 rte_memory_order_relaxed); 4559 4560 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops_enq, 4561 num_ops); 4562 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", 4563 num_ops); 4564 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4565 copy_reference_enc_op(ops_enq, num_ops, 0, bufs->inputs, 4566 bufs->hard_outputs, ref_op); 4567 4568 /* Set counter to validate the ordering */ 4569 for (j = 0; j < num_ops; ++j) 4570 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 4571 4572 for (i = 0; i < TEST_REPETITIONS; ++i) { 4573 uint32_t time_out = 0; 4574 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4575 for (j = 0; j < num_ops; ++j) 4576 mbuf_reset(ops_enq[j]->turbo_enc.output.data); 4577 4578 start_time = rte_rdtsc_precise(); 4579 4580 for (enq = 0, deq = 0; enq < num_ops;) { 4581 num_to_enq = burst_sz; 4582 4583 if (unlikely(num_ops - enq < num_to_enq)) 4584 num_to_enq = num_ops - enq; 4585 4586 enq += rte_bbdev_enqueue_enc_ops(tp->dev_id, 4587 queue_id, &ops_enq[enq], num_to_enq); 4588 4589 deq += rte_bbdev_dequeue_enc_ops(tp->dev_id, 4590 queue_id, &ops_deq[deq], enq - deq); 4591 time_out++; 4592 if (time_out >= TIME_OUT_POLL) { 4593 timeout_exit(tp->dev_id); 4594 TEST_ASSERT_SUCCESS(TEST_FAILED, "Enqueue timeout!"); 4595 } 4596 } 4597 4598 /* dequeue the remaining */ 4599 time_out = 0; 4600 while (deq < enq) { 4601 deq += rte_bbdev_dequeue_enc_ops(tp->dev_id, 4602 queue_id, &ops_deq[deq], enq - deq); 4603 time_out++; 4604 if (time_out >= TIME_OUT_POLL) { 4605 timeout_exit(tp->dev_id); 4606 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 4607 } 4608 } 4609 4610 total_time += rte_rdtsc_precise() - start_time; 4611 } 4612 4613 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 4614 ret = validate_enc_op(ops_deq, num_ops, ref_op); 4615 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 4616 } 4617 4618 rte_bbdev_enc_op_free_bulk(ops_enq, num_ops); 4619 4620 double tb_len_bits = calc_enc_TB_size(ref_op); 4621 4622 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) / 4623 ((double)total_time / (double)rte_get_tsc_hz()); 4624 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) 4625 / 1000000.0) / ((double)total_time / 4626 (double)rte_get_tsc_hz()); 4627 4628 return TEST_SUCCESS; 4629 } 4630 4631 static int 4632 throughput_pmd_lcore_ldpc_enc(void *arg) 4633 { 4634 struct thread_params *tp = arg; 4635 uint16_t enq, deq; 4636 uint64_t total_time = 0, start_time; 4637 const uint16_t queue_id = tp->queue_id; 4638 const uint16_t burst_sz = tp->op_params->burst_sz; 4639 const uint16_t num_ops = tp->op_params->num_to_process; 4640 struct rte_bbdev_enc_op *ops_enq[num_ops]; 4641 struct rte_bbdev_enc_op *ops_deq[num_ops]; 4642 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op; 4643 struct test_buffers *bufs = NULL; 4644 int i, j, ret; 4645 struct rte_bbdev_info info; 4646 uint16_t num_to_enq; 4647 4648 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 4649 "BURST_SIZE should be <= %u", MAX_BURST); 4650 4651 rte_bbdev_info_get(tp->dev_id, &info); 4652 4653 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 4654 "NUM_OPS cannot exceed %u for this device", 4655 info.drv.queue_size_lim); 4656 4657 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 4658 4659 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->op_params->sync, SYNC_START, 4660 rte_memory_order_relaxed); 4661 4662 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops_enq, 4663 num_ops); 4664 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", 4665 num_ops); 4666 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4667 copy_reference_ldpc_enc_op(ops_enq, num_ops, 0, bufs->inputs, 4668 bufs->hard_outputs, ref_op); 4669 4670 /* Set counter to validate the ordering */ 4671 for (j = 0; j < num_ops; ++j) 4672 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 4673 4674 for (i = 0; i < TEST_REPETITIONS; ++i) { 4675 uint32_t time_out = 0; 4676 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4677 for (j = 0; j < num_ops; ++j) 4678 mbuf_reset(ops_enq[j]->turbo_enc.output.data); 4679 4680 start_time = rte_rdtsc_precise(); 4681 4682 for (enq = 0, deq = 0; enq < num_ops;) { 4683 num_to_enq = burst_sz; 4684 4685 if (unlikely(num_ops - enq < num_to_enq)) 4686 num_to_enq = num_ops - enq; 4687 4688 enq += rte_bbdev_enqueue_ldpc_enc_ops(tp->dev_id, 4689 queue_id, &ops_enq[enq], num_to_enq); 4690 4691 deq += rte_bbdev_dequeue_ldpc_enc_ops(tp->dev_id, 4692 queue_id, &ops_deq[deq], enq - deq); 4693 time_out++; 4694 if (time_out >= TIME_OUT_POLL) { 4695 timeout_exit(tp->dev_id); 4696 TEST_ASSERT_SUCCESS(TEST_FAILED, "Enqueue timeout!"); 4697 } 4698 } 4699 4700 /* dequeue the remaining */ 4701 time_out = 0; 4702 while (deq < enq) { 4703 deq += rte_bbdev_dequeue_ldpc_enc_ops(tp->dev_id, 4704 queue_id, &ops_deq[deq], enq - deq); 4705 time_out++; 4706 if (time_out >= TIME_OUT_POLL) { 4707 timeout_exit(tp->dev_id); 4708 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 4709 } 4710 } 4711 4712 total_time += rte_rdtsc_precise() - start_time; 4713 } 4714 4715 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 4716 ret = validate_ldpc_enc_op(ops_deq, num_ops, ref_op); 4717 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 4718 } 4719 4720 rte_bbdev_enc_op_free_bulk(ops_enq, num_ops); 4721 4722 double tb_len_bits = calc_ldpc_enc_TB_size(ref_op); 4723 4724 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) / 4725 ((double)total_time / (double)rte_get_tsc_hz()); 4726 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) 4727 / 1000000.0) / ((double)total_time / 4728 (double)rte_get_tsc_hz()); 4729 4730 return TEST_SUCCESS; 4731 } 4732 4733 static int 4734 throughput_pmd_lcore_fft(void *arg) 4735 { 4736 struct thread_params *tp = arg; 4737 uint16_t enq, deq; 4738 uint64_t total_time = 0, start_time; 4739 const uint16_t queue_id = tp->queue_id; 4740 const uint16_t burst_sz = tp->op_params->burst_sz; 4741 const uint16_t num_ops = tp->op_params->num_to_process; 4742 struct rte_bbdev_fft_op *ops_enq[num_ops]; 4743 struct rte_bbdev_fft_op *ops_deq[num_ops]; 4744 struct rte_bbdev_fft_op *ref_op = tp->op_params->ref_fft_op; 4745 struct test_buffers *bufs = NULL; 4746 int i, j, ret; 4747 struct rte_bbdev_info info; 4748 uint16_t num_to_enq; 4749 4750 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 4751 "BURST_SIZE should be <= %u", MAX_BURST); 4752 4753 rte_bbdev_info_get(tp->dev_id, &info); 4754 4755 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 4756 "NUM_OPS cannot exceed %u for this device", 4757 info.drv.queue_size_lim); 4758 4759 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 4760 4761 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->op_params->sync, SYNC_START, 4762 rte_memory_order_relaxed); 4763 4764 ret = rte_bbdev_fft_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops); 4765 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops); 4766 4767 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4768 copy_reference_fft_op(ops_enq, num_ops, 0, bufs->inputs, 4769 bufs->hard_outputs, bufs->soft_outputs, bufs->harq_inputs, ref_op); 4770 4771 /* Set counter to validate the ordering */ 4772 for (j = 0; j < num_ops; ++j) 4773 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 4774 4775 for (i = 0; i < TEST_REPETITIONS; ++i) { 4776 uint32_t time_out = 0; 4777 for (j = 0; j < num_ops; ++j) 4778 mbuf_reset(ops_enq[j]->fft.base_output.data); 4779 4780 start_time = rte_rdtsc_precise(); 4781 4782 for (enq = 0, deq = 0; enq < num_ops;) { 4783 num_to_enq = burst_sz; 4784 4785 if (unlikely(num_ops - enq < num_to_enq)) 4786 num_to_enq = num_ops - enq; 4787 4788 enq += rte_bbdev_enqueue_fft_ops(tp->dev_id, 4789 queue_id, &ops_enq[enq], num_to_enq); 4790 4791 deq += rte_bbdev_dequeue_fft_ops(tp->dev_id, 4792 queue_id, &ops_deq[deq], enq - deq); 4793 time_out++; 4794 if (time_out >= TIME_OUT_POLL) { 4795 timeout_exit(tp->dev_id); 4796 TEST_ASSERT_SUCCESS(TEST_FAILED, "Enqueue timeout!"); 4797 } 4798 } 4799 4800 /* dequeue the remaining */ 4801 time_out = 0; 4802 while (deq < enq) { 4803 deq += rte_bbdev_dequeue_fft_ops(tp->dev_id, 4804 queue_id, &ops_deq[deq], enq - deq); 4805 time_out++; 4806 if (time_out >= TIME_OUT_POLL) { 4807 timeout_exit(tp->dev_id); 4808 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 4809 } 4810 } 4811 4812 total_time += rte_rdtsc_precise() - start_time; 4813 } 4814 4815 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 4816 ret = validate_fft_op(ops_deq, num_ops, ref_op); 4817 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 4818 } 4819 4820 rte_bbdev_fft_op_free_bulk(ops_enq, num_ops); 4821 4822 double tb_len_bits = calc_fft_size(ref_op); 4823 4824 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) / 4825 ((double)total_time / (double)rte_get_tsc_hz()); 4826 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) / 4827 1000000.0) / ((double)total_time / 4828 (double)rte_get_tsc_hz()); 4829 4830 return TEST_SUCCESS; 4831 } 4832 4833 static int 4834 throughput_pmd_lcore_mldts(void *arg) 4835 { 4836 struct thread_params *tp = arg; 4837 uint16_t enq, deq; 4838 uint64_t total_time = 0, start_time; 4839 const uint16_t queue_id = tp->queue_id; 4840 const uint16_t burst_sz = tp->op_params->burst_sz; 4841 const uint16_t num_ops = tp->op_params->num_to_process; 4842 struct rte_bbdev_mldts_op *ops_enq[num_ops]; 4843 struct rte_bbdev_mldts_op *ops_deq[num_ops]; 4844 struct rte_bbdev_mldts_op *ref_op = tp->op_params->ref_mldts_op; 4845 struct test_buffers *bufs = NULL; 4846 int i, j, ret; 4847 struct rte_bbdev_info info; 4848 uint16_t num_to_enq; 4849 4850 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), "BURST_SIZE should be <= %u", MAX_BURST); 4851 4852 rte_bbdev_info_get(tp->dev_id, &info); 4853 4854 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 4855 "NUM_OPS cannot exceed %u for this device", 4856 info.drv.queue_size_lim); 4857 4858 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 4859 4860 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->op_params->sync, SYNC_START, 4861 rte_memory_order_relaxed); 4862 4863 ret = rte_bbdev_mldts_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops); 4864 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops); 4865 4866 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4867 copy_reference_mldts_op(ops_enq, num_ops, 0, bufs->inputs, bufs->harq_inputs, 4868 bufs->hard_outputs, ref_op); 4869 4870 /* Set counter to validate the ordering */ 4871 for (j = 0; j < num_ops; ++j) 4872 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 4873 4874 for (i = 0; i < TEST_REPETITIONS; ++i) { 4875 uint32_t time_out = 0; 4876 for (j = 0; j < num_ops; ++j) 4877 mbuf_reset(ops_enq[j]->mldts.output.data); 4878 4879 start_time = rte_rdtsc_precise(); 4880 4881 for (enq = 0, deq = 0; enq < num_ops;) { 4882 num_to_enq = burst_sz; 4883 4884 if (unlikely(num_ops - enq < num_to_enq)) 4885 num_to_enq = num_ops - enq; 4886 4887 enq += rte_bbdev_enqueue_mldts_ops(tp->dev_id, 4888 queue_id, &ops_enq[enq], num_to_enq); 4889 4890 deq += rte_bbdev_dequeue_mldts_ops(tp->dev_id, 4891 queue_id, &ops_deq[deq], enq - deq); 4892 time_out++; 4893 if (time_out >= TIME_OUT_POLL) { 4894 timeout_exit(tp->dev_id); 4895 TEST_ASSERT_SUCCESS(TEST_FAILED, "Enqueue timeout!"); 4896 } 4897 } 4898 4899 /* dequeue the remaining */ 4900 time_out = 0; 4901 while (deq < enq) { 4902 deq += rte_bbdev_dequeue_mldts_ops(tp->dev_id, 4903 queue_id, &ops_deq[deq], enq - deq); 4904 time_out++; 4905 if (time_out >= TIME_OUT_POLL) { 4906 timeout_exit(tp->dev_id); 4907 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 4908 } 4909 } 4910 4911 total_time += rte_rdtsc_precise() - start_time; 4912 } 4913 4914 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 4915 ret = validate_mldts_op(ops_deq, num_ops, ref_op); 4916 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 4917 } 4918 4919 rte_bbdev_mldts_op_free_bulk(ops_enq, num_ops); 4920 4921 double tb_len_bits = calc_mldts_size(ref_op); 4922 4923 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) / 4924 ((double)total_time / (double)rte_get_tsc_hz()); 4925 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) / 4926 1000000.0) / ((double)total_time / 4927 (double)rte_get_tsc_hz()); 4928 4929 return TEST_SUCCESS; 4930 } 4931 4932 static void 4933 print_enc_throughput(struct thread_params *t_params, unsigned int used_cores) 4934 { 4935 unsigned int iter = 0; 4936 double total_mops = 0, total_mbps = 0; 4937 4938 for (iter = 0; iter < used_cores; iter++) { 4939 printf( 4940 "Throughput for core (%u): %.8lg Ops/s, %.8lg Mbps\n", 4941 t_params[iter].lcore_id, t_params[iter].ops_per_sec, 4942 t_params[iter].mbps); 4943 total_mops += t_params[iter].ops_per_sec; 4944 total_mbps += t_params[iter].mbps; 4945 } 4946 printf( 4947 "\nTotal throughput for %u cores: %.8lg MOPS, %.8lg Mbps\n", 4948 used_cores, total_mops, total_mbps); 4949 } 4950 4951 /* Aggregate the performance results over the number of cores used */ 4952 static void 4953 print_dec_throughput(struct thread_params *t_params, unsigned int used_cores) 4954 { 4955 unsigned int core_idx = 0; 4956 double total_mops = 0, total_mbps = 0; 4957 uint8_t iter_count = 0; 4958 4959 for (core_idx = 0; core_idx < used_cores; core_idx++) { 4960 printf( 4961 "Throughput for core (%u): %.8lg Ops/s, %.8lg Mbps @ max %u iterations\n", 4962 t_params[core_idx].lcore_id, 4963 t_params[core_idx].ops_per_sec, 4964 t_params[core_idx].mbps, 4965 t_params[core_idx].iter_count); 4966 total_mops += t_params[core_idx].ops_per_sec; 4967 total_mbps += t_params[core_idx].mbps; 4968 iter_count = RTE_MAX(iter_count, 4969 t_params[core_idx].iter_count); 4970 } 4971 printf( 4972 "\nTotal throughput for %u cores: %.8lg MOPS, %.8lg Mbps @ max %u iterations\n", 4973 used_cores, total_mops, total_mbps, iter_count); 4974 } 4975 4976 /* Aggregate the performance results over the number of cores used */ 4977 static void 4978 print_dec_bler(struct thread_params *t_params, unsigned int used_cores) 4979 { 4980 unsigned int core_idx = 0; 4981 double total_mbps = 0, total_bler = 0, total_iter = 0; 4982 double snr = get_snr(); 4983 4984 for (core_idx = 0; core_idx < used_cores; core_idx++) { 4985 printf("Core%u BLER %.1f %% - Iters %.1f - Tp %.1f Mbps %s\n", 4986 t_params[core_idx].lcore_id, 4987 t_params[core_idx].bler * 100, 4988 t_params[core_idx].iter_average, 4989 t_params[core_idx].mbps, 4990 get_vector_filename()); 4991 total_mbps += t_params[core_idx].mbps; 4992 total_bler += t_params[core_idx].bler; 4993 total_iter += t_params[core_idx].iter_average; 4994 } 4995 total_bler /= used_cores; 4996 total_iter /= used_cores; 4997 4998 printf("SNR %.2f BLER %.1f %% - Iterations %.1f %d - Tp %.3f Mbps %s\n", 4999 snr, total_bler * 100, total_iter, get_iter_max(), 5000 total_mbps, get_vector_filename()); 5001 } 5002 5003 /* 5004 * Test function that determines BLER wireless performance 5005 */ 5006 static int 5007 bler_test(struct active_device *ad, 5008 struct test_op_params *op_params) 5009 { 5010 int ret; 5011 unsigned int lcore_id, used_cores = 0; 5012 struct thread_params *t_params; 5013 struct rte_bbdev_info info; 5014 lcore_function_t *bler_function; 5015 uint16_t num_lcores; 5016 const char *op_type_str; 5017 5018 rte_bbdev_info_get(ad->dev_id, &info); 5019 5020 op_type_str = rte_bbdev_op_type_str(test_vector.op_type); 5021 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", 5022 test_vector.op_type); 5023 5024 printf("+ ------------------------------------------------------- +\n"); 5025 printf("== test: bler\ndev: %s, nb_queues: %u, burst size: %u, num ops: %u, num_lcores: %u, op type: %s, itr mode: %s, GHz: %lg\n", 5026 info.dev_name, ad->nb_queues, op_params->burst_sz, 5027 op_params->num_to_process, op_params->num_lcores, 5028 op_type_str, 5029 intr_enabled ? "Interrupt mode" : "PMD mode", 5030 (double)rte_get_tsc_hz() / 1000000000.0); 5031 5032 /* Set number of lcores */ 5033 num_lcores = (ad->nb_queues < (op_params->num_lcores)) 5034 ? ad->nb_queues 5035 : op_params->num_lcores; 5036 5037 /* Allocate memory for thread parameters structure */ 5038 t_params = rte_zmalloc(NULL, num_lcores * sizeof(struct thread_params), 5039 RTE_CACHE_LINE_SIZE); 5040 TEST_ASSERT_NOT_NULL(t_params, "Failed to alloc %zuB for t_params", 5041 RTE_ALIGN(sizeof(struct thread_params) * num_lcores, 5042 RTE_CACHE_LINE_SIZE)); 5043 5044 if ((test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) && 5045 !check_bit(test_vector.ldpc_dec.op_flags, 5046 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK) 5047 && !check_bit(test_vector.ldpc_dec.op_flags, 5048 RTE_BBDEV_LDPC_LLR_COMPRESSION)) 5049 bler_function = bler_pmd_lcore_ldpc_dec; 5050 else if ((test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) && 5051 !check_bit(test_vector.turbo_dec.op_flags, 5052 RTE_BBDEV_TURBO_SOFT_OUTPUT)) 5053 bler_function = bler_pmd_lcore_turbo_dec; 5054 else 5055 return TEST_SKIPPED; 5056 5057 rte_atomic_store_explicit(&op_params->sync, SYNC_WAIT, rte_memory_order_relaxed); 5058 5059 /* Main core is set at first entry */ 5060 t_params[0].dev_id = ad->dev_id; 5061 t_params[0].lcore_id = rte_lcore_id(); 5062 t_params[0].op_params = op_params; 5063 t_params[0].queue_id = ad->queue_ids[used_cores++]; 5064 t_params[0].iter_count = 0; 5065 5066 RTE_LCORE_FOREACH_WORKER(lcore_id) { 5067 if (used_cores >= num_lcores) 5068 break; 5069 5070 t_params[used_cores].dev_id = ad->dev_id; 5071 t_params[used_cores].lcore_id = lcore_id; 5072 t_params[used_cores].op_params = op_params; 5073 t_params[used_cores].queue_id = ad->queue_ids[used_cores]; 5074 t_params[used_cores].iter_count = 0; 5075 5076 rte_eal_remote_launch(bler_function, 5077 &t_params[used_cores++], lcore_id); 5078 } 5079 5080 rte_atomic_store_explicit(&op_params->sync, SYNC_START, rte_memory_order_relaxed); 5081 ret = bler_function(&t_params[0]); 5082 5083 /* Main core is always used */ 5084 for (used_cores = 1; used_cores < num_lcores; used_cores++) 5085 ret |= rte_eal_wait_lcore(t_params[used_cores].lcore_id); 5086 5087 print_dec_bler(t_params, num_lcores); 5088 5089 /* Return if test failed */ 5090 if (ret) { 5091 rte_free(t_params); 5092 return ret; 5093 } 5094 5095 /* Function to print something here*/ 5096 rte_free(t_params); 5097 return ret; 5098 } 5099 5100 /* 5101 * Test function that determines how long an enqueue + dequeue of a burst 5102 * takes on available lcores. 5103 */ 5104 static int 5105 throughput_test(struct active_device *ad, 5106 struct test_op_params *op_params) 5107 { 5108 int ret; 5109 unsigned int lcore_id, used_cores = 0; 5110 struct thread_params *t_params, *tp; 5111 struct rte_bbdev_info info; 5112 lcore_function_t *throughput_function; 5113 uint16_t num_lcores; 5114 const char *op_type_str; 5115 5116 rte_bbdev_info_get(ad->dev_id, &info); 5117 5118 op_type_str = rte_bbdev_op_type_str(test_vector.op_type); 5119 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", 5120 test_vector.op_type); 5121 5122 printf("+ ------------------------------------------------------- +\n"); 5123 printf("== test: throughput\ndev: %s, nb_queues: %u, burst size: %u, num ops: %u, num_lcores: %u, op type: %s, itr mode: %s, GHz: %lg\n", 5124 info.dev_name, ad->nb_queues, op_params->burst_sz, 5125 op_params->num_to_process, op_params->num_lcores, 5126 op_type_str, 5127 intr_enabled ? "Interrupt mode" : "PMD mode", 5128 (double)rte_get_tsc_hz() / 1000000000.0); 5129 5130 /* Set number of lcores */ 5131 num_lcores = (ad->nb_queues < (op_params->num_lcores)) 5132 ? ad->nb_queues 5133 : op_params->num_lcores; 5134 5135 /* Allocate memory for thread parameters structure */ 5136 t_params = rte_zmalloc(NULL, num_lcores * sizeof(struct thread_params), 5137 RTE_CACHE_LINE_SIZE); 5138 TEST_ASSERT_NOT_NULL(t_params, "Failed to alloc %zuB for t_params", 5139 RTE_ALIGN(sizeof(struct thread_params) * num_lcores, 5140 RTE_CACHE_LINE_SIZE)); 5141 5142 if (intr_enabled) { 5143 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) 5144 throughput_function = throughput_intr_lcore_dec; 5145 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 5146 throughput_function = throughput_intr_lcore_ldpc_dec; 5147 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) 5148 throughput_function = throughput_intr_lcore_enc; 5149 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) 5150 throughput_function = throughput_intr_lcore_ldpc_enc; 5151 else if (test_vector.op_type == RTE_BBDEV_OP_FFT) 5152 throughput_function = throughput_intr_lcore_fft; 5153 else if (test_vector.op_type == RTE_BBDEV_OP_MLDTS) 5154 throughput_function = throughput_intr_lcore_mldts; 5155 else 5156 throughput_function = throughput_intr_lcore_enc; 5157 5158 /* Dequeue interrupt callback registration */ 5159 ret = rte_bbdev_callback_register(ad->dev_id, 5160 RTE_BBDEV_EVENT_DEQUEUE, dequeue_event_callback, 5161 t_params); 5162 if (ret < 0) { 5163 rte_free(t_params); 5164 return ret; 5165 } 5166 } else { 5167 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) 5168 throughput_function = throughput_pmd_lcore_dec; 5169 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 5170 throughput_function = throughput_pmd_lcore_ldpc_dec; 5171 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) 5172 throughput_function = throughput_pmd_lcore_enc; 5173 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) 5174 throughput_function = throughput_pmd_lcore_ldpc_enc; 5175 else if (test_vector.op_type == RTE_BBDEV_OP_FFT) 5176 throughput_function = throughput_pmd_lcore_fft; 5177 else if (test_vector.op_type == RTE_BBDEV_OP_MLDTS) 5178 throughput_function = throughput_pmd_lcore_mldts; 5179 else 5180 throughput_function = throughput_pmd_lcore_enc; 5181 } 5182 5183 rte_atomic_store_explicit(&op_params->sync, SYNC_WAIT, rte_memory_order_relaxed); 5184 5185 /* Main core is set at first entry */ 5186 t_params[0].dev_id = ad->dev_id; 5187 t_params[0].lcore_id = rte_lcore_id(); 5188 t_params[0].op_params = op_params; 5189 t_params[0].queue_id = ad->queue_ids[used_cores++]; 5190 t_params[0].iter_count = 0; 5191 5192 RTE_LCORE_FOREACH_WORKER(lcore_id) { 5193 if (used_cores >= num_lcores) 5194 break; 5195 5196 t_params[used_cores].dev_id = ad->dev_id; 5197 t_params[used_cores].lcore_id = lcore_id; 5198 t_params[used_cores].op_params = op_params; 5199 t_params[used_cores].queue_id = ad->queue_ids[used_cores]; 5200 t_params[used_cores].iter_count = 0; 5201 5202 rte_eal_remote_launch(throughput_function, 5203 &t_params[used_cores++], lcore_id); 5204 } 5205 5206 rte_atomic_store_explicit(&op_params->sync, SYNC_START, rte_memory_order_relaxed); 5207 ret = throughput_function(&t_params[0]); 5208 5209 /* Main core is always used */ 5210 for (used_cores = 1; used_cores < num_lcores; used_cores++) 5211 ret |= rte_eal_wait_lcore(t_params[used_cores].lcore_id); 5212 5213 /* Return if test failed */ 5214 if (ret) { 5215 rte_free(t_params); 5216 return ret; 5217 } 5218 5219 /* Print throughput if interrupts are disabled and test passed */ 5220 if (!intr_enabled) { 5221 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC || 5222 test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 5223 print_dec_throughput(t_params, num_lcores); 5224 else 5225 print_enc_throughput(t_params, num_lcores); 5226 rte_free(t_params); 5227 return ret; 5228 } 5229 5230 /* In interrupt TC we need to wait for the interrupt callback to deqeue 5231 * all pending operations. Skip waiting for queues which reported an 5232 * error using processing_status variable. 5233 * Wait for main lcore operations. 5234 */ 5235 tp = &t_params[0]; 5236 while ((rte_atomic_load_explicit(&tp->nb_dequeued, rte_memory_order_relaxed) < 5237 op_params->num_to_process) && 5238 (rte_atomic_load_explicit(&tp->processing_status, rte_memory_order_relaxed) != 5239 TEST_FAILED)) 5240 rte_pause(); 5241 5242 tp->ops_per_sec /= TEST_REPETITIONS; 5243 tp->mbps /= TEST_REPETITIONS; 5244 ret |= (int)rte_atomic_load_explicit(&tp->processing_status, rte_memory_order_relaxed); 5245 5246 /* Wait for worker lcores operations */ 5247 for (used_cores = 1; used_cores < num_lcores; used_cores++) { 5248 tp = &t_params[used_cores]; 5249 5250 while ((rte_atomic_load_explicit(&tp->nb_dequeued, rte_memory_order_relaxed) < 5251 op_params->num_to_process) && 5252 (rte_atomic_load_explicit(&tp->processing_status, 5253 rte_memory_order_relaxed) != TEST_FAILED)) 5254 rte_pause(); 5255 5256 tp->ops_per_sec /= TEST_REPETITIONS; 5257 tp->mbps /= TEST_REPETITIONS; 5258 ret |= (int)rte_atomic_load_explicit(&tp->processing_status, 5259 rte_memory_order_relaxed); 5260 } 5261 5262 /* Print throughput if test passed */ 5263 if (!ret) { 5264 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC || 5265 test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 5266 print_dec_throughput(t_params, num_lcores); 5267 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC || 5268 test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) 5269 print_enc_throughput(t_params, num_lcores); 5270 } 5271 5272 rte_free(t_params); 5273 return ret; 5274 } 5275 5276 static int 5277 latency_test_dec(struct rte_mempool *mempool, 5278 struct test_buffers *bufs, struct rte_bbdev_dec_op *ref_op, 5279 uint16_t dev_id, uint16_t queue_id, 5280 const uint16_t num_to_process, uint16_t burst_sz, 5281 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time, bool disable_et) 5282 { 5283 int ret = TEST_SUCCESS; 5284 uint16_t i, j, dequeued; 5285 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 5286 uint64_t start_time = 0, last_time = 0; 5287 5288 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 5289 uint16_t enq = 0, deq = 0; 5290 uint32_t time_out = 0; 5291 bool first_time = true; 5292 last_time = 0; 5293 5294 if (unlikely(num_to_process - dequeued < burst_sz)) 5295 burst_sz = num_to_process - dequeued; 5296 5297 ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz); 5298 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_dec_op_alloc_bulk() failed"); 5299 5300 ref_op->turbo_dec.iter_max = get_iter_max(); 5301 /* For validation tests we want to enable early termination */ 5302 if (!disable_et && !check_bit(ref_op->turbo_dec.op_flags, 5303 RTE_BBDEV_TURBO_EARLY_TERMINATION)) 5304 ref_op->turbo_dec.op_flags |= RTE_BBDEV_TURBO_EARLY_TERMINATION; 5305 5306 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 5307 copy_reference_dec_op(ops_enq, burst_sz, dequeued, 5308 bufs->inputs, 5309 bufs->hard_outputs, 5310 bufs->soft_outputs, 5311 ref_op); 5312 5313 /* Set counter to validate the ordering */ 5314 for (j = 0; j < burst_sz; ++j) 5315 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 5316 5317 start_time = rte_rdtsc_precise(); 5318 5319 enq = rte_bbdev_enqueue_dec_ops(dev_id, queue_id, &ops_enq[enq], 5320 burst_sz); 5321 TEST_ASSERT(enq == burst_sz, 5322 "Error enqueueing burst, expected %u, got %u", 5323 burst_sz, enq); 5324 5325 /* Dequeue */ 5326 do { 5327 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id, 5328 &ops_deq[deq], burst_sz - deq); 5329 if (likely(first_time && (deq > 0))) { 5330 last_time = rte_rdtsc_precise() - start_time; 5331 first_time = false; 5332 } 5333 time_out++; 5334 if (time_out >= TIME_OUT_POLL) { 5335 timeout_exit(dev_id); 5336 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 5337 } 5338 } while (unlikely(burst_sz != deq)); 5339 5340 *max_time = RTE_MAX(*max_time, last_time); 5341 *min_time = RTE_MIN(*min_time, last_time); 5342 *total_time += last_time; 5343 5344 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 5345 ret = validate_dec_op(ops_deq, burst_sz, ref_op); 5346 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 5347 } 5348 5349 rte_bbdev_dec_op_free_bulk(ops_enq, deq); 5350 dequeued += deq; 5351 } 5352 5353 return i; 5354 } 5355 5356 /* Test case for latency/validation for LDPC Decoder */ 5357 static int 5358 latency_test_ldpc_dec(struct rte_mempool *mempool, 5359 struct test_buffers *bufs, struct rte_bbdev_dec_op *ref_op, 5360 int vector_mask, uint16_t dev_id, uint16_t queue_id, 5361 const uint16_t num_to_process, uint16_t burst_sz, 5362 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time, 5363 bool disable_et) 5364 { 5365 int ret = TEST_SUCCESS; 5366 uint16_t i, j, dequeued; 5367 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 5368 uint64_t start_time = 0, last_time = 0; 5369 bool extDdr = ldpc_cap_flags & 5370 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE; 5371 5372 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 5373 uint16_t enq = 0, deq = 0; 5374 uint32_t time_out = 0; 5375 bool first_time = true; 5376 last_time = 0; 5377 5378 if (unlikely(num_to_process - dequeued < burst_sz)) 5379 burst_sz = num_to_process - dequeued; 5380 5381 ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz); 5382 TEST_ASSERT_SUCCESS(ret, 5383 "rte_bbdev_dec_op_alloc_bulk() failed"); 5384 5385 /* For latency tests we need to disable early termination */ 5386 if (disable_et && check_bit(ref_op->ldpc_dec.op_flags, 5387 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE)) 5388 ref_op->ldpc_dec.op_flags -= RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE; 5389 5390 ref_op->ldpc_dec.iter_max = get_iter_max(); 5391 /* When ET is disabled, the expected iter_count is iter_max */ 5392 if (disable_et) 5393 ref_op->ldpc_dec.iter_count = ref_op->ldpc_dec.iter_max; 5394 5395 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 5396 copy_reference_ldpc_dec_op(ops_enq, burst_sz, dequeued, 5397 bufs->inputs, 5398 bufs->hard_outputs, 5399 bufs->soft_outputs, 5400 bufs->harq_inputs, 5401 bufs->harq_outputs, 5402 ref_op); 5403 5404 if (extDdr) 5405 preload_harq_ddr(dev_id, queue_id, ops_enq, 5406 burst_sz, true); 5407 5408 /* Set counter to validate the ordering */ 5409 for (j = 0; j < burst_sz; ++j) 5410 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 5411 5412 start_time = rte_rdtsc_precise(); 5413 5414 enq = rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id, 5415 &ops_enq[enq], burst_sz); 5416 TEST_ASSERT(enq == burst_sz, 5417 "Error enqueueing burst, expected %u, got %u", 5418 burst_sz, enq); 5419 5420 /* Dequeue */ 5421 do { 5422 deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id, 5423 &ops_deq[deq], burst_sz - deq); 5424 if (likely(first_time && (deq > 0))) { 5425 last_time = rte_rdtsc_precise() - start_time; 5426 first_time = false; 5427 } 5428 time_out++; 5429 if (time_out >= TIME_OUT_POLL) { 5430 timeout_exit(dev_id); 5431 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 5432 } 5433 } while (unlikely(burst_sz != deq)); 5434 5435 *max_time = RTE_MAX(*max_time, last_time); 5436 *min_time = RTE_MIN(*min_time, last_time); 5437 *total_time += last_time; 5438 5439 if (extDdr) 5440 retrieve_harq_ddr(dev_id, queue_id, ops_enq, burst_sz); 5441 5442 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 5443 ret = validate_ldpc_dec_op(ops_deq, burst_sz, ref_op, vector_mask); 5444 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 5445 } 5446 5447 rte_bbdev_dec_op_free_bulk(ops_enq, deq); 5448 dequeued += deq; 5449 } 5450 return i; 5451 } 5452 5453 static int 5454 latency_test_enc(struct rte_mempool *mempool, 5455 struct test_buffers *bufs, struct rte_bbdev_enc_op *ref_op, 5456 uint16_t dev_id, uint16_t queue_id, 5457 const uint16_t num_to_process, uint16_t burst_sz, 5458 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time) 5459 { 5460 int ret = TEST_SUCCESS; 5461 uint16_t i, j, dequeued; 5462 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 5463 uint64_t start_time = 0, last_time = 0; 5464 5465 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 5466 uint16_t enq = 0, deq = 0; 5467 uint32_t time_out = 0; 5468 bool first_time = true; 5469 last_time = 0; 5470 5471 if (unlikely(num_to_process - dequeued < burst_sz)) 5472 burst_sz = num_to_process - dequeued; 5473 5474 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz); 5475 TEST_ASSERT_SUCCESS(ret, 5476 "rte_bbdev_enc_op_alloc_bulk() failed"); 5477 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 5478 copy_reference_enc_op(ops_enq, burst_sz, dequeued, 5479 bufs->inputs, 5480 bufs->hard_outputs, 5481 ref_op); 5482 5483 /* Set counter to validate the ordering */ 5484 for (j = 0; j < burst_sz; ++j) 5485 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 5486 5487 start_time = rte_rdtsc_precise(); 5488 5489 enq = rte_bbdev_enqueue_enc_ops(dev_id, queue_id, &ops_enq[enq], 5490 burst_sz); 5491 TEST_ASSERT(enq == burst_sz, 5492 "Error enqueueing burst, expected %u, got %u", 5493 burst_sz, enq); 5494 5495 /* Dequeue */ 5496 do { 5497 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id, 5498 &ops_deq[deq], burst_sz - deq); 5499 if (likely(first_time && (deq > 0))) { 5500 last_time += rte_rdtsc_precise() - start_time; 5501 first_time = false; 5502 } 5503 time_out++; 5504 if (time_out >= TIME_OUT_POLL) { 5505 timeout_exit(dev_id); 5506 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 5507 } 5508 } while (unlikely(burst_sz != deq)); 5509 5510 *max_time = RTE_MAX(*max_time, last_time); 5511 *min_time = RTE_MIN(*min_time, last_time); 5512 *total_time += last_time; 5513 5514 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 5515 ret = validate_enc_op(ops_deq, burst_sz, ref_op); 5516 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 5517 } 5518 5519 rte_bbdev_enc_op_free_bulk(ops_enq, deq); 5520 dequeued += deq; 5521 } 5522 5523 return i; 5524 } 5525 5526 static int 5527 latency_test_ldpc_enc(struct rte_mempool *mempool, 5528 struct test_buffers *bufs, struct rte_bbdev_enc_op *ref_op, 5529 uint16_t dev_id, uint16_t queue_id, 5530 const uint16_t num_to_process, uint16_t burst_sz, 5531 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time) 5532 { 5533 int ret = TEST_SUCCESS; 5534 uint16_t i, j, dequeued; 5535 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 5536 uint64_t start_time = 0, last_time = 0; 5537 5538 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 5539 uint16_t enq = 0, deq = 0; 5540 uint32_t time_out = 0; 5541 bool first_time = true; 5542 last_time = 0; 5543 5544 if (unlikely(num_to_process - dequeued < burst_sz)) 5545 burst_sz = num_to_process - dequeued; 5546 5547 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz); 5548 TEST_ASSERT_SUCCESS(ret, 5549 "rte_bbdev_enc_op_alloc_bulk() failed"); 5550 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 5551 copy_reference_ldpc_enc_op(ops_enq, burst_sz, dequeued, 5552 bufs->inputs, 5553 bufs->hard_outputs, 5554 ref_op); 5555 5556 /* Set counter to validate the ordering */ 5557 for (j = 0; j < burst_sz; ++j) 5558 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 5559 5560 start_time = rte_rdtsc_precise(); 5561 5562 enq = rte_bbdev_enqueue_ldpc_enc_ops(dev_id, queue_id, 5563 &ops_enq[enq], burst_sz); 5564 TEST_ASSERT(enq == burst_sz, 5565 "Error enqueueing burst, expected %u, got %u", 5566 burst_sz, enq); 5567 5568 /* Dequeue */ 5569 do { 5570 deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id, 5571 &ops_deq[deq], burst_sz - deq); 5572 if (likely(first_time && (deq > 0))) { 5573 last_time += rte_rdtsc_precise() - start_time; 5574 first_time = false; 5575 } 5576 time_out++; 5577 if (time_out >= TIME_OUT_POLL) { 5578 timeout_exit(dev_id); 5579 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 5580 } 5581 } while (unlikely(burst_sz != deq)); 5582 5583 *max_time = RTE_MAX(*max_time, last_time); 5584 *min_time = RTE_MIN(*min_time, last_time); 5585 *total_time += last_time; 5586 5587 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 5588 ret = validate_enc_op(ops_deq, burst_sz, ref_op); 5589 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 5590 } 5591 5592 rte_bbdev_enc_op_free_bulk(ops_enq, deq); 5593 dequeued += deq; 5594 } 5595 5596 return i; 5597 } 5598 5599 5600 static int 5601 latency_test_fft(struct rte_mempool *mempool, 5602 struct test_buffers *bufs, struct rte_bbdev_fft_op *ref_op, 5603 uint16_t dev_id, uint16_t queue_id, 5604 const uint16_t num_to_process, uint16_t burst_sz, 5605 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time) 5606 { 5607 int ret = TEST_SUCCESS; 5608 uint16_t i, j, dequeued; 5609 struct rte_bbdev_fft_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 5610 uint64_t start_time = 0, last_time = 0; 5611 5612 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 5613 uint16_t enq = 0, deq = 0; 5614 uint32_t time_out = 0; 5615 bool first_time = true; 5616 last_time = 0; 5617 5618 if (unlikely(num_to_process - dequeued < burst_sz)) 5619 burst_sz = num_to_process - dequeued; 5620 5621 ret = rte_bbdev_fft_op_alloc_bulk(mempool, ops_enq, burst_sz); 5622 TEST_ASSERT_SUCCESS(ret, 5623 "rte_bbdev_fft_op_alloc_bulk() failed"); 5624 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 5625 copy_reference_fft_op(ops_enq, burst_sz, dequeued, 5626 bufs->inputs, 5627 bufs->hard_outputs, bufs->soft_outputs, bufs->harq_inputs, 5628 ref_op); 5629 5630 /* Set counter to validate the ordering */ 5631 for (j = 0; j < burst_sz; ++j) 5632 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 5633 5634 start_time = rte_rdtsc_precise(); 5635 5636 enq = rte_bbdev_enqueue_fft_ops(dev_id, queue_id, 5637 &ops_enq[enq], burst_sz); 5638 TEST_ASSERT(enq == burst_sz, 5639 "Error enqueueing burst, expected %u, got %u", 5640 burst_sz, enq); 5641 5642 /* Dequeue */ 5643 do { 5644 deq += rte_bbdev_dequeue_fft_ops(dev_id, queue_id, 5645 &ops_deq[deq], burst_sz - deq); 5646 if (likely(first_time && (deq > 0))) { 5647 last_time += rte_rdtsc_precise() - start_time; 5648 first_time = false; 5649 } 5650 time_out++; 5651 if (time_out >= TIME_OUT_POLL) { 5652 timeout_exit(dev_id); 5653 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 5654 } 5655 } while (unlikely(burst_sz != deq)); 5656 5657 *max_time = RTE_MAX(*max_time, last_time); 5658 *min_time = RTE_MIN(*min_time, last_time); 5659 *total_time += last_time; 5660 5661 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 5662 ret = validate_fft_op(ops_deq, burst_sz, ref_op); 5663 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 5664 } 5665 5666 rte_bbdev_fft_op_free_bulk(ops_enq, deq); 5667 dequeued += deq; 5668 } 5669 5670 return i; 5671 } 5672 5673 static int 5674 latency_test_mldts(struct rte_mempool *mempool, 5675 struct test_buffers *bufs, struct rte_bbdev_mldts_op *ref_op, 5676 uint16_t dev_id, uint16_t queue_id, 5677 const uint16_t num_to_process, uint16_t burst_sz, 5678 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time) 5679 { 5680 int ret = TEST_SUCCESS; 5681 uint16_t i, j, dequeued; 5682 struct rte_bbdev_mldts_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 5683 uint64_t start_time = 0, last_time = 0; 5684 5685 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 5686 uint16_t enq = 0, deq = 0; 5687 uint32_t time_out = 0; 5688 bool first_time = true; 5689 last_time = 0; 5690 5691 if (unlikely(num_to_process - dequeued < burst_sz)) 5692 burst_sz = num_to_process - dequeued; 5693 5694 ret = rte_bbdev_mldts_op_alloc_bulk(mempool, ops_enq, burst_sz); 5695 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_mldts_op_alloc_bulk() failed"); 5696 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 5697 copy_reference_mldts_op(ops_enq, burst_sz, dequeued, 5698 bufs->inputs, bufs->harq_inputs, 5699 bufs->hard_outputs, 5700 ref_op); 5701 5702 /* Set counter to validate the ordering */ 5703 for (j = 0; j < burst_sz; ++j) 5704 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 5705 5706 start_time = rte_rdtsc_precise(); 5707 5708 enq = rte_bbdev_enqueue_mldts_ops(dev_id, queue_id, &ops_enq[enq], burst_sz); 5709 TEST_ASSERT(enq == burst_sz, 5710 "Error enqueueing burst, expected %u, got %u", 5711 burst_sz, enq); 5712 5713 /* Dequeue */ 5714 do { 5715 deq += rte_bbdev_dequeue_mldts_ops(dev_id, queue_id, 5716 &ops_deq[deq], burst_sz - deq); 5717 if (likely(first_time && (deq > 0))) { 5718 last_time += rte_rdtsc_precise() - start_time; 5719 first_time = false; 5720 } 5721 time_out++; 5722 if (time_out >= TIME_OUT_POLL) { 5723 timeout_exit(dev_id); 5724 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 5725 } 5726 } while (unlikely(burst_sz != deq)); 5727 5728 *max_time = RTE_MAX(*max_time, last_time); 5729 *min_time = RTE_MIN(*min_time, last_time); 5730 *total_time += last_time; 5731 5732 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 5733 ret = validate_mldts_op(ops_deq, burst_sz, ref_op); 5734 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 5735 } 5736 5737 rte_bbdev_mldts_op_free_bulk(ops_enq, deq); 5738 dequeued += deq; 5739 } 5740 5741 return i; 5742 } 5743 5744 /* Common function for running validation and latency test cases */ 5745 static int 5746 validation_latency_test(struct active_device *ad, 5747 struct test_op_params *op_params, bool latency_flag) 5748 { 5749 int iter; 5750 uint16_t burst_sz = op_params->burst_sz; 5751 const uint16_t num_to_process = op_params->num_to_process; 5752 const enum rte_bbdev_op_type op_type = test_vector.op_type; 5753 const uint16_t queue_id = ad->queue_ids[0]; 5754 struct test_buffers *bufs = NULL; 5755 struct rte_bbdev_info info; 5756 uint64_t total_time, min_time, max_time; 5757 const char *op_type_str; 5758 5759 total_time = max_time = 0; 5760 min_time = UINT64_MAX; 5761 5762 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 5763 "BURST_SIZE should be <= %u", MAX_BURST); 5764 5765 rte_bbdev_info_get(ad->dev_id, &info); 5766 bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 5767 5768 op_type_str = rte_bbdev_op_type_str(op_type); 5769 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type); 5770 5771 printf("+ ------------------------------------------------------- +\n"); 5772 if (latency_flag) 5773 printf("== test: latency\ndev:"); 5774 else 5775 printf("== test: validation\ndev:"); 5776 printf("%s, burst size: %u, num ops: %u, op type: %s\n", 5777 info.dev_name, burst_sz, num_to_process, op_type_str); 5778 5779 if (op_type == RTE_BBDEV_OP_TURBO_DEC) 5780 iter = latency_test_dec(op_params->mp, bufs, 5781 op_params->ref_dec_op, ad->dev_id, queue_id, 5782 num_to_process, burst_sz, &total_time, 5783 &min_time, &max_time, latency_flag); 5784 else if (op_type == RTE_BBDEV_OP_LDPC_ENC) 5785 iter = latency_test_ldpc_enc(op_params->mp, bufs, 5786 op_params->ref_enc_op, ad->dev_id, queue_id, 5787 num_to_process, burst_sz, &total_time, 5788 &min_time, &max_time); 5789 else if (op_type == RTE_BBDEV_OP_LDPC_DEC) 5790 iter = latency_test_ldpc_dec(op_params->mp, bufs, 5791 op_params->ref_dec_op, op_params->vector_mask, 5792 ad->dev_id, queue_id, num_to_process, 5793 burst_sz, &total_time, &min_time, &max_time, 5794 latency_flag); 5795 else if (op_type == RTE_BBDEV_OP_FFT) 5796 iter = latency_test_fft(op_params->mp, bufs, 5797 op_params->ref_fft_op, 5798 ad->dev_id, queue_id, 5799 num_to_process, burst_sz, &total_time, 5800 &min_time, &max_time); 5801 else if (op_type == RTE_BBDEV_OP_MLDTS) 5802 iter = latency_test_mldts(op_params->mp, bufs, 5803 op_params->ref_mldts_op, 5804 ad->dev_id, queue_id, 5805 num_to_process, burst_sz, &total_time, 5806 &min_time, &max_time); 5807 else /* RTE_BBDEV_OP_TURBO_ENC */ 5808 iter = latency_test_enc(op_params->mp, bufs, 5809 op_params->ref_enc_op, 5810 ad->dev_id, queue_id, 5811 num_to_process, burst_sz, &total_time, 5812 &min_time, &max_time); 5813 5814 if (iter <= 0) 5815 return TEST_FAILED; 5816 5817 printf("Operation latency:\n" 5818 "\tavg: %lg cycles, %lg us\n" 5819 "\tmin: %lg cycles, %lg us\n" 5820 "\tmax: %lg cycles, %lg us\n", 5821 (double)total_time / (double)iter, 5822 (double)(total_time * 1000000) / (double)iter / 5823 (double)rte_get_tsc_hz(), (double)min_time, 5824 (double)(min_time * 1000000) / (double)rte_get_tsc_hz(), 5825 (double)max_time, (double)(max_time * 1000000) / 5826 (double)rte_get_tsc_hz()); 5827 5828 return TEST_SUCCESS; 5829 } 5830 5831 static int 5832 latency_test(struct active_device *ad, struct test_op_params *op_params) 5833 { 5834 return validation_latency_test(ad, op_params, true); 5835 } 5836 5837 static int 5838 validation_test(struct active_device *ad, struct test_op_params *op_params) 5839 { 5840 return validation_latency_test(ad, op_params, false); 5841 } 5842 5843 static int 5844 get_bbdev_queue_stats(uint16_t dev_id, uint16_t queue_id, 5845 struct rte_bbdev_stats *stats) 5846 { 5847 struct rte_bbdev *dev = &rte_bbdev_devices[dev_id]; 5848 struct rte_bbdev_stats *q_stats; 5849 5850 if (queue_id >= dev->data->num_queues) 5851 return -1; 5852 5853 q_stats = &dev->data->queues[queue_id].queue_stats; 5854 5855 stats->enqueued_count = q_stats->enqueued_count; 5856 stats->dequeued_count = q_stats->dequeued_count; 5857 stats->enqueue_err_count = q_stats->enqueue_err_count; 5858 stats->dequeue_err_count = q_stats->dequeue_err_count; 5859 stats->enqueue_warn_count = q_stats->enqueue_warn_count; 5860 stats->dequeue_warn_count = q_stats->dequeue_warn_count; 5861 stats->acc_offload_cycles = q_stats->acc_offload_cycles; 5862 stats->enqueue_depth_avail = q_stats->enqueue_depth_avail; 5863 5864 return 0; 5865 } 5866 5867 static int 5868 offload_latency_test_fft(struct rte_mempool *mempool, struct test_buffers *bufs, 5869 struct rte_bbdev_fft_op *ref_op, uint16_t dev_id, 5870 uint16_t queue_id, const uint16_t num_to_process, 5871 uint16_t burst_sz, struct test_time_stats *time_st) 5872 { 5873 int i, dequeued, ret; 5874 struct rte_bbdev_fft_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 5875 uint64_t enq_start_time, deq_start_time; 5876 uint64_t enq_sw_last_time, deq_last_time; 5877 struct rte_bbdev_stats stats; 5878 5879 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 5880 uint16_t enq = 0, deq = 0; 5881 5882 if (unlikely(num_to_process - dequeued < burst_sz)) 5883 burst_sz = num_to_process - dequeued; 5884 5885 ret = rte_bbdev_fft_op_alloc_bulk(mempool, ops_enq, burst_sz); 5886 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_fft_op_alloc_bulk() failed"); 5887 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 5888 copy_reference_fft_op(ops_enq, burst_sz, dequeued, 5889 bufs->inputs, 5890 bufs->hard_outputs, bufs->soft_outputs, bufs->harq_inputs, 5891 ref_op); 5892 5893 /* Start time meas for enqueue function offload latency */ 5894 enq_start_time = rte_rdtsc_precise(); 5895 do { 5896 enq += rte_bbdev_enqueue_fft_ops(dev_id, queue_id, 5897 &ops_enq[enq], burst_sz - enq); 5898 } while (unlikely(burst_sz != enq)); 5899 5900 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats); 5901 TEST_ASSERT_SUCCESS(ret, 5902 "Failed to get stats for queue (%u) of device (%u)", 5903 queue_id, dev_id); 5904 5905 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time - 5906 stats.acc_offload_cycles; 5907 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time, 5908 enq_sw_last_time); 5909 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time, 5910 enq_sw_last_time); 5911 time_st->enq_sw_total_time += enq_sw_last_time; 5912 5913 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time, 5914 stats.acc_offload_cycles); 5915 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time, 5916 stats.acc_offload_cycles); 5917 time_st->enq_acc_total_time += stats.acc_offload_cycles; 5918 5919 /* give time for device to process ops */ 5920 rte_delay_us(WAIT_OFFLOAD_US); 5921 5922 /* Start time meas for dequeue function offload latency */ 5923 deq_start_time = rte_rdtsc_precise(); 5924 /* Dequeue one operation */ 5925 do { 5926 deq += rte_bbdev_dequeue_fft_ops(dev_id, queue_id, 5927 &ops_deq[deq], enq); 5928 } while (unlikely(deq == 0)); 5929 5930 deq_last_time = rte_rdtsc_precise() - deq_start_time; 5931 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time, 5932 deq_last_time); 5933 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time, 5934 deq_last_time); 5935 time_st->deq_total_time += deq_last_time; 5936 5937 /* Dequeue remaining operations if needed*/ 5938 while (burst_sz != deq) 5939 deq += rte_bbdev_dequeue_fft_ops(dev_id, queue_id, 5940 &ops_deq[deq], burst_sz - deq); 5941 5942 rte_bbdev_fft_op_free_bulk(ops_enq, deq); 5943 dequeued += deq; 5944 } 5945 5946 return i; 5947 } 5948 5949 static int 5950 offload_latency_test_mldts(struct rte_mempool *mempool, struct test_buffers *bufs, 5951 struct rte_bbdev_mldts_op *ref_op, uint16_t dev_id, 5952 uint16_t queue_id, const uint16_t num_to_process, 5953 uint16_t burst_sz, struct test_time_stats *time_st) 5954 { 5955 int i, dequeued, ret; 5956 struct rte_bbdev_mldts_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 5957 uint64_t enq_start_time, deq_start_time; 5958 uint64_t enq_sw_last_time, deq_last_time; 5959 struct rte_bbdev_stats stats; 5960 5961 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 5962 uint16_t enq = 0, deq = 0; 5963 5964 if (unlikely(num_to_process - dequeued < burst_sz)) 5965 burst_sz = num_to_process - dequeued; 5966 5967 ret = rte_bbdev_mldts_op_alloc_bulk(mempool, ops_enq, burst_sz); 5968 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_mldts_op_alloc_bulk() failed"); 5969 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 5970 copy_reference_mldts_op(ops_enq, burst_sz, dequeued, 5971 bufs->inputs, bufs->harq_inputs, 5972 bufs->hard_outputs, 5973 ref_op); 5974 5975 /* Start time meas for enqueue function offload latency */ 5976 enq_start_time = rte_rdtsc_precise(); 5977 do { 5978 enq += rte_bbdev_enqueue_mldts_ops(dev_id, queue_id, 5979 &ops_enq[enq], burst_sz - enq); 5980 } while (unlikely(burst_sz != enq)); 5981 5982 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats); 5983 TEST_ASSERT_SUCCESS(ret, 5984 "Failed to get stats for queue (%u) of device (%u)", 5985 queue_id, dev_id); 5986 5987 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time - 5988 stats.acc_offload_cycles; 5989 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time, 5990 enq_sw_last_time); 5991 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time, 5992 enq_sw_last_time); 5993 time_st->enq_sw_total_time += enq_sw_last_time; 5994 5995 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time, 5996 stats.acc_offload_cycles); 5997 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time, 5998 stats.acc_offload_cycles); 5999 time_st->enq_acc_total_time += stats.acc_offload_cycles; 6000 6001 /* give time for device to process ops */ 6002 rte_delay_us(WAIT_OFFLOAD_US); 6003 6004 /* Start time meas for dequeue function offload latency */ 6005 deq_start_time = rte_rdtsc_precise(); 6006 /* Dequeue one operation */ 6007 do { 6008 deq += rte_bbdev_dequeue_mldts_ops(dev_id, queue_id, &ops_deq[deq], enq); 6009 } while (unlikely(deq == 0)); 6010 6011 deq_last_time = rte_rdtsc_precise() - deq_start_time; 6012 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time, deq_last_time); 6013 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time, deq_last_time); 6014 time_st->deq_total_time += deq_last_time; 6015 6016 /* Dequeue remaining operations if needed*/ 6017 while (burst_sz != deq) 6018 deq += rte_bbdev_dequeue_mldts_ops(dev_id, queue_id, 6019 &ops_deq[deq], burst_sz - deq); 6020 6021 rte_bbdev_mldts_op_free_bulk(ops_enq, deq); 6022 dequeued += deq; 6023 } 6024 6025 return i; 6026 } 6027 6028 static int 6029 offload_latency_test_dec(struct rte_mempool *mempool, struct test_buffers *bufs, 6030 struct rte_bbdev_dec_op *ref_op, uint16_t dev_id, 6031 uint16_t queue_id, const uint16_t num_to_process, 6032 uint16_t burst_sz, struct test_time_stats *time_st) 6033 { 6034 int i, dequeued, ret; 6035 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 6036 uint64_t enq_start_time, deq_start_time; 6037 uint64_t enq_sw_last_time, deq_last_time; 6038 struct rte_bbdev_stats stats; 6039 6040 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 6041 uint16_t enq = 0, deq = 0; 6042 6043 if (unlikely(num_to_process - dequeued < burst_sz)) 6044 burst_sz = num_to_process - dequeued; 6045 6046 ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz); 6047 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_dec_op_alloc_bulk() failed"); 6048 ref_op->turbo_dec.iter_max = get_iter_max(); 6049 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 6050 copy_reference_dec_op(ops_enq, burst_sz, dequeued, 6051 bufs->inputs, 6052 bufs->hard_outputs, 6053 bufs->soft_outputs, 6054 ref_op); 6055 6056 /* Start time meas for enqueue function offload latency */ 6057 enq_start_time = rte_rdtsc_precise(); 6058 do { 6059 enq += rte_bbdev_enqueue_dec_ops(dev_id, queue_id, 6060 &ops_enq[enq], burst_sz - enq); 6061 } while (unlikely(burst_sz != enq)); 6062 6063 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats); 6064 TEST_ASSERT_SUCCESS(ret, 6065 "Failed to get stats for queue (%u) of device (%u)", 6066 queue_id, dev_id); 6067 6068 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time - 6069 stats.acc_offload_cycles; 6070 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time, 6071 enq_sw_last_time); 6072 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time, 6073 enq_sw_last_time); 6074 time_st->enq_sw_total_time += enq_sw_last_time; 6075 6076 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time, 6077 stats.acc_offload_cycles); 6078 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time, 6079 stats.acc_offload_cycles); 6080 time_st->enq_acc_total_time += stats.acc_offload_cycles; 6081 6082 /* give time for device to process ops */ 6083 rte_delay_us(WAIT_OFFLOAD_US); 6084 6085 /* Start time meas for dequeue function offload latency */ 6086 deq_start_time = rte_rdtsc_precise(); 6087 /* Dequeue one operation */ 6088 do { 6089 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id, 6090 &ops_deq[deq], enq); 6091 } while (unlikely(deq == 0)); 6092 6093 deq_last_time = rte_rdtsc_precise() - deq_start_time; 6094 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time, 6095 deq_last_time); 6096 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time, 6097 deq_last_time); 6098 time_st->deq_total_time += deq_last_time; 6099 6100 /* Dequeue remaining operations if needed*/ 6101 while (burst_sz != deq) 6102 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id, 6103 &ops_deq[deq], burst_sz - deq); 6104 6105 rte_bbdev_dec_op_free_bulk(ops_enq, deq); 6106 dequeued += deq; 6107 } 6108 6109 return i; 6110 } 6111 6112 static int 6113 offload_latency_test_ldpc_dec(struct rte_mempool *mempool, 6114 struct test_buffers *bufs, 6115 struct rte_bbdev_dec_op *ref_op, uint16_t dev_id, 6116 uint16_t queue_id, const uint16_t num_to_process, 6117 uint16_t burst_sz, struct test_time_stats *time_st) 6118 { 6119 int i, dequeued, ret; 6120 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 6121 uint64_t enq_start_time, deq_start_time; 6122 uint64_t enq_sw_last_time, deq_last_time; 6123 struct rte_bbdev_stats stats; 6124 bool extDdr = ldpc_cap_flags & 6125 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE; 6126 6127 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 6128 uint16_t enq = 0, deq = 0; 6129 6130 if (unlikely(num_to_process - dequeued < burst_sz)) 6131 burst_sz = num_to_process - dequeued; 6132 6133 ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz); 6134 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_dec_op_alloc_bulk() failed"); 6135 ref_op->ldpc_dec.iter_max = get_iter_max(); 6136 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 6137 copy_reference_ldpc_dec_op(ops_enq, burst_sz, dequeued, 6138 bufs->inputs, 6139 bufs->hard_outputs, 6140 bufs->soft_outputs, 6141 bufs->harq_inputs, 6142 bufs->harq_outputs, 6143 ref_op); 6144 6145 if (extDdr) 6146 preload_harq_ddr(dev_id, queue_id, ops_enq, 6147 burst_sz, true); 6148 6149 /* Start time meas for enqueue function offload latency */ 6150 enq_start_time = rte_rdtsc_precise(); 6151 do { 6152 enq += rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id, 6153 &ops_enq[enq], burst_sz - enq); 6154 } while (unlikely(burst_sz != enq)); 6155 6156 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time; 6157 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats); 6158 TEST_ASSERT_SUCCESS(ret, 6159 "Failed to get stats for queue (%u) of device (%u)", 6160 queue_id, dev_id); 6161 6162 enq_sw_last_time -= stats.acc_offload_cycles; 6163 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time, 6164 enq_sw_last_time); 6165 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time, 6166 enq_sw_last_time); 6167 time_st->enq_sw_total_time += enq_sw_last_time; 6168 6169 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time, 6170 stats.acc_offload_cycles); 6171 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time, 6172 stats.acc_offload_cycles); 6173 time_st->enq_acc_total_time += stats.acc_offload_cycles; 6174 6175 /* give time for device to process ops */ 6176 rte_delay_us(WAIT_OFFLOAD_US); 6177 6178 /* Start time meas for dequeue function offload latency */ 6179 deq_start_time = rte_rdtsc_precise(); 6180 /* Dequeue one operation */ 6181 do { 6182 deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id, 6183 &ops_deq[deq], enq); 6184 } while (unlikely(deq == 0)); 6185 6186 deq_last_time = rte_rdtsc_precise() - deq_start_time; 6187 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time, 6188 deq_last_time); 6189 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time, 6190 deq_last_time); 6191 time_st->deq_total_time += deq_last_time; 6192 6193 /* Dequeue remaining operations if needed*/ 6194 while (burst_sz != deq) 6195 deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id, 6196 &ops_deq[deq], burst_sz - deq); 6197 6198 if (extDdr) { 6199 /* Read loopback is not thread safe */ 6200 retrieve_harq_ddr(dev_id, queue_id, ops_enq, burst_sz); 6201 } 6202 6203 rte_bbdev_dec_op_free_bulk(ops_enq, deq); 6204 dequeued += deq; 6205 } 6206 6207 return i; 6208 } 6209 6210 static int 6211 offload_latency_test_enc(struct rte_mempool *mempool, struct test_buffers *bufs, 6212 struct rte_bbdev_enc_op *ref_op, uint16_t dev_id, 6213 uint16_t queue_id, const uint16_t num_to_process, 6214 uint16_t burst_sz, struct test_time_stats *time_st) 6215 { 6216 int i, dequeued, ret; 6217 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 6218 uint64_t enq_start_time, deq_start_time; 6219 uint64_t enq_sw_last_time, deq_last_time; 6220 struct rte_bbdev_stats stats; 6221 6222 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 6223 uint16_t enq = 0, deq = 0; 6224 6225 if (unlikely(num_to_process - dequeued < burst_sz)) 6226 burst_sz = num_to_process - dequeued; 6227 6228 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz); 6229 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_enc_op_alloc_bulk() failed"); 6230 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 6231 copy_reference_enc_op(ops_enq, burst_sz, dequeued, 6232 bufs->inputs, 6233 bufs->hard_outputs, 6234 ref_op); 6235 6236 /* Start time meas for enqueue function offload latency */ 6237 enq_start_time = rte_rdtsc_precise(); 6238 do { 6239 enq += rte_bbdev_enqueue_enc_ops(dev_id, queue_id, 6240 &ops_enq[enq], burst_sz - enq); 6241 } while (unlikely(burst_sz != enq)); 6242 6243 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time; 6244 6245 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats); 6246 TEST_ASSERT_SUCCESS(ret, 6247 "Failed to get stats for queue (%u) of device (%u)", 6248 queue_id, dev_id); 6249 enq_sw_last_time -= stats.acc_offload_cycles; 6250 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time, 6251 enq_sw_last_time); 6252 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time, 6253 enq_sw_last_time); 6254 time_st->enq_sw_total_time += enq_sw_last_time; 6255 6256 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time, 6257 stats.acc_offload_cycles); 6258 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time, 6259 stats.acc_offload_cycles); 6260 time_st->enq_acc_total_time += stats.acc_offload_cycles; 6261 6262 /* give time for device to process ops */ 6263 rte_delay_us(WAIT_OFFLOAD_US); 6264 6265 /* Start time meas for dequeue function offload latency */ 6266 deq_start_time = rte_rdtsc_precise(); 6267 /* Dequeue one operation */ 6268 do { 6269 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id, 6270 &ops_deq[deq], enq); 6271 } while (unlikely(deq == 0)); 6272 6273 deq_last_time = rte_rdtsc_precise() - deq_start_time; 6274 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time, 6275 deq_last_time); 6276 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time, 6277 deq_last_time); 6278 time_st->deq_total_time += deq_last_time; 6279 6280 while (burst_sz != deq) 6281 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id, 6282 &ops_deq[deq], burst_sz - deq); 6283 6284 rte_bbdev_enc_op_free_bulk(ops_enq, deq); 6285 dequeued += deq; 6286 } 6287 6288 return i; 6289 } 6290 6291 static int 6292 offload_latency_test_ldpc_enc(struct rte_mempool *mempool, 6293 struct test_buffers *bufs, 6294 struct rte_bbdev_enc_op *ref_op, uint16_t dev_id, 6295 uint16_t queue_id, const uint16_t num_to_process, 6296 uint16_t burst_sz, struct test_time_stats *time_st) 6297 { 6298 int i, dequeued, ret; 6299 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 6300 uint64_t enq_start_time, deq_start_time; 6301 uint64_t enq_sw_last_time, deq_last_time; 6302 struct rte_bbdev_stats stats; 6303 6304 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 6305 uint16_t enq = 0, deq = 0; 6306 6307 if (unlikely(num_to_process - dequeued < burst_sz)) 6308 burst_sz = num_to_process - dequeued; 6309 6310 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz); 6311 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_enc_op_alloc_bulk() failed"); 6312 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 6313 copy_reference_ldpc_enc_op(ops_enq, burst_sz, dequeued, 6314 bufs->inputs, 6315 bufs->hard_outputs, 6316 ref_op); 6317 6318 /* Start time meas for enqueue function offload latency */ 6319 enq_start_time = rte_rdtsc_precise(); 6320 do { 6321 enq += rte_bbdev_enqueue_ldpc_enc_ops(dev_id, queue_id, 6322 &ops_enq[enq], burst_sz - enq); 6323 } while (unlikely(burst_sz != enq)); 6324 6325 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time; 6326 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats); 6327 TEST_ASSERT_SUCCESS(ret, 6328 "Failed to get stats for queue (%u) of device (%u)", 6329 queue_id, dev_id); 6330 6331 enq_sw_last_time -= stats.acc_offload_cycles; 6332 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time, 6333 enq_sw_last_time); 6334 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time, 6335 enq_sw_last_time); 6336 time_st->enq_sw_total_time += enq_sw_last_time; 6337 6338 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time, 6339 stats.acc_offload_cycles); 6340 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time, 6341 stats.acc_offload_cycles); 6342 time_st->enq_acc_total_time += stats.acc_offload_cycles; 6343 6344 /* give time for device to process ops */ 6345 rte_delay_us(WAIT_OFFLOAD_US); 6346 6347 /* Start time meas for dequeue function offload latency */ 6348 deq_start_time = rte_rdtsc_precise(); 6349 /* Dequeue one operation */ 6350 do { 6351 deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id, 6352 &ops_deq[deq], enq); 6353 } while (unlikely(deq == 0)); 6354 6355 deq_last_time = rte_rdtsc_precise() - deq_start_time; 6356 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time, 6357 deq_last_time); 6358 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time, 6359 deq_last_time); 6360 time_st->deq_total_time += deq_last_time; 6361 6362 while (burst_sz != deq) 6363 deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id, 6364 &ops_deq[deq], burst_sz - deq); 6365 6366 rte_bbdev_enc_op_free_bulk(ops_enq, deq); 6367 dequeued += deq; 6368 } 6369 6370 return i; 6371 } 6372 6373 static int 6374 offload_cost_test(struct active_device *ad, 6375 struct test_op_params *op_params) 6376 { 6377 int iter, ret; 6378 uint16_t burst_sz = op_params->burst_sz; 6379 const uint16_t num_to_process = op_params->num_to_process; 6380 const enum rte_bbdev_op_type op_type = test_vector.op_type; 6381 const uint16_t queue_id = ad->queue_ids[0]; 6382 struct test_buffers *bufs = NULL; 6383 struct rte_bbdev_info info; 6384 const char *op_type_str; 6385 struct test_time_stats time_st; 6386 6387 memset(&time_st, 0, sizeof(struct test_time_stats)); 6388 time_st.enq_sw_min_time = UINT64_MAX; 6389 time_st.enq_acc_min_time = UINT64_MAX; 6390 time_st.deq_min_time = UINT64_MAX; 6391 6392 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 6393 "BURST_SIZE should be <= %u", MAX_BURST); 6394 6395 rte_bbdev_info_get(ad->dev_id, &info); 6396 bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 6397 6398 op_type_str = rte_bbdev_op_type_str(op_type); 6399 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type); 6400 6401 printf("+ ------------------------------------------------------- +\n"); 6402 printf("== test: offload latency test\ndev: %s, burst size: %u, num ops: %u, op type: %s\n", 6403 info.dev_name, burst_sz, num_to_process, op_type_str); 6404 6405 if (op_type == RTE_BBDEV_OP_TURBO_DEC) 6406 iter = offload_latency_test_dec(op_params->mp, bufs, 6407 op_params->ref_dec_op, ad->dev_id, queue_id, 6408 num_to_process, burst_sz, &time_st); 6409 else if (op_type == RTE_BBDEV_OP_TURBO_ENC) 6410 iter = offload_latency_test_enc(op_params->mp, bufs, 6411 op_params->ref_enc_op, ad->dev_id, queue_id, 6412 num_to_process, burst_sz, &time_st); 6413 else if (op_type == RTE_BBDEV_OP_LDPC_ENC) 6414 iter = offload_latency_test_ldpc_enc(op_params->mp, bufs, 6415 op_params->ref_enc_op, ad->dev_id, queue_id, 6416 num_to_process, burst_sz, &time_st); 6417 else if (op_type == RTE_BBDEV_OP_LDPC_DEC) 6418 iter = offload_latency_test_ldpc_dec(op_params->mp, bufs, 6419 op_params->ref_dec_op, ad->dev_id, queue_id, 6420 num_to_process, burst_sz, &time_st); 6421 else if (op_type == RTE_BBDEV_OP_FFT) 6422 iter = offload_latency_test_fft(op_params->mp, bufs, 6423 op_params->ref_fft_op, ad->dev_id, queue_id, 6424 num_to_process, burst_sz, &time_st); 6425 else if (op_type == RTE_BBDEV_OP_MLDTS) 6426 iter = offload_latency_test_mldts(op_params->mp, bufs, 6427 op_params->ref_mldts_op, ad->dev_id, queue_id, 6428 num_to_process, burst_sz, &time_st); 6429 else 6430 iter = offload_latency_test_enc(op_params->mp, bufs, 6431 op_params->ref_enc_op, ad->dev_id, queue_id, 6432 num_to_process, burst_sz, &time_st); 6433 6434 if (iter <= 0) 6435 return TEST_FAILED; 6436 6437 printf("Enqueue driver offload cost latency:\n" 6438 "\tavg: %lg cycles, %lg us\n" 6439 "\tmin: %lg cycles, %lg us\n" 6440 "\tmax: %lg cycles, %lg us\n" 6441 "Enqueue accelerator offload cost latency:\n" 6442 "\tavg: %lg cycles, %lg us\n" 6443 "\tmin: %lg cycles, %lg us\n" 6444 "\tmax: %lg cycles, %lg us\n", 6445 (double)time_st.enq_sw_total_time / (double)iter, 6446 (double)(time_st.enq_sw_total_time * 1000000) / 6447 (double)iter / (double)rte_get_tsc_hz(), 6448 (double)time_st.enq_sw_min_time, 6449 (double)(time_st.enq_sw_min_time * 1000000) / 6450 rte_get_tsc_hz(), (double)time_st.enq_sw_max_time, 6451 (double)(time_st.enq_sw_max_time * 1000000) / 6452 rte_get_tsc_hz(), (double)time_st.enq_acc_total_time / 6453 (double)iter, 6454 (double)(time_st.enq_acc_total_time * 1000000) / 6455 (double)iter / (double)rte_get_tsc_hz(), 6456 (double)time_st.enq_acc_min_time, 6457 (double)(time_st.enq_acc_min_time * 1000000) / 6458 rte_get_tsc_hz(), (double)time_st.enq_acc_max_time, 6459 (double)(time_st.enq_acc_max_time * 1000000) / 6460 rte_get_tsc_hz()); 6461 6462 printf("Dequeue offload cost latency - one op:\n" 6463 "\tavg: %lg cycles, %lg us\n" 6464 "\tmin: %lg cycles, %lg us\n" 6465 "\tmax: %lg cycles, %lg us\n", 6466 (double)time_st.deq_total_time / (double)iter, 6467 (double)(time_st.deq_total_time * 1000000) / 6468 (double)iter / (double)rte_get_tsc_hz(), 6469 (double)time_st.deq_min_time, 6470 (double)(time_st.deq_min_time * 1000000) / 6471 rte_get_tsc_hz(), (double)time_st.deq_max_time, 6472 (double)(time_st.deq_max_time * 1000000) / 6473 rte_get_tsc_hz()); 6474 6475 struct rte_bbdev_stats stats = {0}; 6476 ret = get_bbdev_queue_stats(ad->dev_id, queue_id, &stats); 6477 TEST_ASSERT_SUCCESS(ret, 6478 "Failed to get stats for queue (%u) of device (%u)", 6479 queue_id, ad->dev_id); 6480 if (stats.enqueue_warn_count > 0) 6481 printf("Warning reported on the queue : %10"PRIu64"\n", 6482 stats.enqueue_warn_count); 6483 if (op_type != RTE_BBDEV_OP_LDPC_DEC) { 6484 TEST_ASSERT_SUCCESS(stats.enqueued_count != num_to_process, 6485 "Mismatch in enqueue count %10"PRIu64" %d", 6486 stats.enqueued_count, num_to_process); 6487 TEST_ASSERT_SUCCESS(stats.dequeued_count != num_to_process, 6488 "Mismatch in dequeue count %10"PRIu64" %d", 6489 stats.dequeued_count, num_to_process); 6490 } 6491 TEST_ASSERT_SUCCESS(stats.enqueue_err_count != 0, 6492 "Enqueue count Error %10"PRIu64"", 6493 stats.enqueue_err_count); 6494 TEST_ASSERT_SUCCESS(stats.dequeue_err_count != 0, 6495 "Dequeue count Error (%10"PRIu64"", 6496 stats.dequeue_err_count); 6497 6498 return TEST_SUCCESS; 6499 } 6500 6501 static int 6502 offload_latency_empty_q_test_dec(uint16_t dev_id, uint16_t queue_id, 6503 const uint16_t num_to_process, uint16_t burst_sz, 6504 uint64_t *deq_total_time, uint64_t *deq_min_time, 6505 uint64_t *deq_max_time, const enum rte_bbdev_op_type op_type) 6506 { 6507 int i, deq_total; 6508 struct rte_bbdev_dec_op *ops[MAX_BURST]; 6509 uint64_t deq_start_time, deq_last_time; 6510 6511 /* Test deq offload latency from an empty queue */ 6512 6513 for (i = 0, deq_total = 0; deq_total < num_to_process; 6514 ++i, deq_total += burst_sz) { 6515 deq_start_time = rte_rdtsc_precise(); 6516 6517 if (unlikely(num_to_process - deq_total < burst_sz)) 6518 burst_sz = num_to_process - deq_total; 6519 if (op_type == RTE_BBDEV_OP_LDPC_DEC) 6520 rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id, ops, 6521 burst_sz); 6522 else 6523 rte_bbdev_dequeue_dec_ops(dev_id, queue_id, ops, 6524 burst_sz); 6525 6526 deq_last_time = rte_rdtsc_precise() - deq_start_time; 6527 *deq_max_time = RTE_MAX(*deq_max_time, deq_last_time); 6528 *deq_min_time = RTE_MIN(*deq_min_time, deq_last_time); 6529 *deq_total_time += deq_last_time; 6530 } 6531 6532 return i; 6533 } 6534 6535 static int 6536 offload_latency_empty_q_test_enc(uint16_t dev_id, uint16_t queue_id, 6537 const uint16_t num_to_process, uint16_t burst_sz, 6538 uint64_t *deq_total_time, uint64_t *deq_min_time, 6539 uint64_t *deq_max_time, const enum rte_bbdev_op_type op_type) 6540 { 6541 int i, deq_total; 6542 struct rte_bbdev_enc_op *ops[MAX_BURST]; 6543 uint64_t deq_start_time, deq_last_time; 6544 6545 /* Test deq offload latency from an empty queue */ 6546 for (i = 0, deq_total = 0; deq_total < num_to_process; 6547 ++i, deq_total += burst_sz) { 6548 deq_start_time = rte_rdtsc_precise(); 6549 6550 if (unlikely(num_to_process - deq_total < burst_sz)) 6551 burst_sz = num_to_process - deq_total; 6552 if (op_type == RTE_BBDEV_OP_LDPC_ENC) 6553 rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id, ops, 6554 burst_sz); 6555 else 6556 rte_bbdev_dequeue_enc_ops(dev_id, queue_id, ops, 6557 burst_sz); 6558 6559 deq_last_time = rte_rdtsc_precise() - deq_start_time; 6560 *deq_max_time = RTE_MAX(*deq_max_time, deq_last_time); 6561 *deq_min_time = RTE_MIN(*deq_min_time, deq_last_time); 6562 *deq_total_time += deq_last_time; 6563 } 6564 6565 return i; 6566 } 6567 6568 static int 6569 offload_latency_empty_q_test(struct active_device *ad, 6570 struct test_op_params *op_params) 6571 { 6572 int iter; 6573 uint64_t deq_total_time, deq_min_time, deq_max_time; 6574 uint16_t burst_sz = op_params->burst_sz; 6575 const uint16_t num_to_process = op_params->num_to_process; 6576 const enum rte_bbdev_op_type op_type = test_vector.op_type; 6577 const uint16_t queue_id = ad->queue_ids[0]; 6578 struct rte_bbdev_info info; 6579 const char *op_type_str; 6580 6581 deq_total_time = deq_max_time = 0; 6582 deq_min_time = UINT64_MAX; 6583 6584 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 6585 "BURST_SIZE should be <= %u", MAX_BURST); 6586 6587 rte_bbdev_info_get(ad->dev_id, &info); 6588 6589 op_type_str = rte_bbdev_op_type_str(op_type); 6590 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type); 6591 6592 printf("+ ------------------------------------------------------- +\n"); 6593 printf("== test: offload latency empty dequeue\ndev: %s, burst size: %u, num ops: %u, op type: %s\n", 6594 info.dev_name, burst_sz, num_to_process, op_type_str); 6595 6596 if (op_type == RTE_BBDEV_OP_TURBO_DEC || 6597 op_type == RTE_BBDEV_OP_LDPC_DEC) 6598 iter = offload_latency_empty_q_test_dec(ad->dev_id, queue_id, 6599 num_to_process, burst_sz, &deq_total_time, 6600 &deq_min_time, &deq_max_time, op_type); 6601 else 6602 iter = offload_latency_empty_q_test_enc(ad->dev_id, queue_id, 6603 num_to_process, burst_sz, &deq_total_time, 6604 &deq_min_time, &deq_max_time, op_type); 6605 6606 if (iter <= 0) 6607 return TEST_FAILED; 6608 6609 printf("Empty dequeue offload:\n" 6610 "\tavg: %lg cycles, %lg us\n" 6611 "\tmin: %lg cycles, %lg us\n" 6612 "\tmax: %lg cycles, %lg us\n", 6613 (double)deq_total_time / (double)iter, 6614 (double)(deq_total_time * 1000000) / (double)iter / 6615 (double)rte_get_tsc_hz(), (double)deq_min_time, 6616 (double)(deq_min_time * 1000000) / rte_get_tsc_hz(), 6617 (double)deq_max_time, (double)(deq_max_time * 1000000) / 6618 rte_get_tsc_hz()); 6619 6620 return TEST_SUCCESS; 6621 } 6622 6623 static int 6624 bler_tc(void) 6625 { 6626 return run_test_case(bler_test); 6627 } 6628 6629 static int 6630 throughput_tc(void) 6631 { 6632 return run_test_case(throughput_test); 6633 } 6634 6635 static int 6636 offload_cost_tc(void) 6637 { 6638 return run_test_case(offload_cost_test); 6639 } 6640 6641 static int 6642 offload_latency_empty_q_tc(void) 6643 { 6644 return run_test_case(offload_latency_empty_q_test); 6645 } 6646 6647 static int 6648 latency_tc(void) 6649 { 6650 return run_test_case(latency_test); 6651 } 6652 6653 static int 6654 validation_tc(void) 6655 { 6656 return run_test_case(validation_test); 6657 } 6658 6659 static int 6660 interrupt_tc(void) 6661 { 6662 return run_test_case(throughput_test); 6663 } 6664 6665 static struct unit_test_suite bbdev_bler_testsuite = { 6666 .suite_name = "BBdev BLER Tests", 6667 .setup = testsuite_setup, 6668 .teardown = testsuite_teardown, 6669 .unit_test_cases = { 6670 TEST_CASE_ST(ut_setup, ut_teardown, bler_tc), 6671 TEST_CASES_END() /**< NULL terminate unit test array */ 6672 } 6673 }; 6674 6675 static struct unit_test_suite bbdev_throughput_testsuite = { 6676 .suite_name = "BBdev Throughput Tests", 6677 .setup = testsuite_setup, 6678 .teardown = testsuite_teardown, 6679 .unit_test_cases = { 6680 TEST_CASE_ST(ut_setup, ut_teardown, throughput_tc), 6681 TEST_CASES_END() /**< NULL terminate unit test array */ 6682 } 6683 }; 6684 6685 static struct unit_test_suite bbdev_validation_testsuite = { 6686 .suite_name = "BBdev Validation Tests", 6687 .setup = testsuite_setup, 6688 .teardown = testsuite_teardown, 6689 .unit_test_cases = { 6690 TEST_CASE_ST(ut_setup, ut_teardown, validation_tc), 6691 TEST_CASES_END() /**< NULL terminate unit test array */ 6692 } 6693 }; 6694 6695 static struct unit_test_suite bbdev_latency_testsuite = { 6696 .suite_name = "BBdev Latency Tests", 6697 .setup = testsuite_setup, 6698 .teardown = testsuite_teardown, 6699 .unit_test_cases = { 6700 TEST_CASE_ST(ut_setup, ut_teardown, latency_tc), 6701 TEST_CASES_END() /**< NULL terminate unit test array */ 6702 } 6703 }; 6704 6705 static struct unit_test_suite bbdev_offload_cost_testsuite = { 6706 .suite_name = "BBdev Offload Cost Tests", 6707 .setup = testsuite_setup, 6708 .teardown = testsuite_teardown, 6709 .unit_test_cases = { 6710 TEST_CASE_ST(ut_setup, ut_teardown, offload_cost_tc), 6711 TEST_CASE_ST(ut_setup, ut_teardown, offload_latency_empty_q_tc), 6712 TEST_CASES_END() /**< NULL terminate unit test array */ 6713 } 6714 }; 6715 6716 static struct unit_test_suite bbdev_interrupt_testsuite = { 6717 .suite_name = "BBdev Interrupt Tests", 6718 .setup = interrupt_testsuite_setup, 6719 .teardown = testsuite_teardown, 6720 .unit_test_cases = { 6721 TEST_CASE_ST(ut_setup, ut_teardown, interrupt_tc), 6722 TEST_CASES_END() /**< NULL terminate unit test array */ 6723 } 6724 }; 6725 6726 REGISTER_TEST_COMMAND(bler, bbdev_bler_testsuite); 6727 REGISTER_TEST_COMMAND(throughput, bbdev_throughput_testsuite); 6728 REGISTER_TEST_COMMAND(validation, bbdev_validation_testsuite); 6729 REGISTER_TEST_COMMAND(latency, bbdev_latency_testsuite); 6730 REGISTER_TEST_COMMAND(offload, bbdev_offload_cost_testsuite); 6731 REGISTER_TEST_COMMAND(interrupt, bbdev_interrupt_testsuite); 6732