1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017 Intel Corporation 3 */ 4 5 #include <stdio.h> 6 #include <stdlib.h> 7 #include <inttypes.h> 8 #include <math.h> 9 10 #include <rte_eal.h> 11 #include <rte_common.h> 12 #include <rte_dev.h> 13 #include <rte_launch.h> 14 #include <rte_bbdev.h> 15 #include <rte_cycles.h> 16 #include <rte_lcore.h> 17 #include <rte_malloc.h> 18 #include <rte_random.h> 19 #include <rte_hexdump.h> 20 #include <rte_interrupts.h> 21 22 #include "main.h" 23 #include "test_bbdev_vector.h" 24 25 #define GET_SOCKET(socket_id) (((socket_id) == SOCKET_ID_ANY) ? 0 : (socket_id)) 26 27 #define MAX_QUEUES RTE_MAX_LCORE 28 #define TEST_REPETITIONS 100 29 #define TIME_OUT_POLL 1e9 30 #define WAIT_OFFLOAD_US 1000 31 32 #ifdef RTE_BASEBAND_FPGA_LTE_FEC 33 #include <fpga_lte_fec.h> 34 #define FPGA_LTE_PF_DRIVER_NAME ("intel_fpga_lte_fec_pf") 35 #define FPGA_LTE_VF_DRIVER_NAME ("intel_fpga_lte_fec_vf") 36 #define VF_UL_4G_QUEUE_VALUE 4 37 #define VF_DL_4G_QUEUE_VALUE 4 38 #define UL_4G_BANDWIDTH 3 39 #define DL_4G_BANDWIDTH 3 40 #define UL_4G_LOAD_BALANCE 128 41 #define DL_4G_LOAD_BALANCE 128 42 #define FLR_4G_TIMEOUT 610 43 #endif 44 45 #ifdef RTE_BASEBAND_FPGA_5GNR_FEC 46 #include <rte_pmd_fpga_5gnr_fec.h> 47 #define FPGA_5GNR_PF_DRIVER_NAME ("intel_fpga_5gnr_fec_pf") 48 #define FPGA_5GNR_VF_DRIVER_NAME ("intel_fpga_5gnr_fec_vf") 49 #define VF_UL_5G_QUEUE_VALUE 4 50 #define VF_DL_5G_QUEUE_VALUE 4 51 #define UL_5G_BANDWIDTH 3 52 #define DL_5G_BANDWIDTH 3 53 #define UL_5G_LOAD_BALANCE 128 54 #define DL_5G_LOAD_BALANCE 128 55 #endif 56 57 #ifdef RTE_BASEBAND_ACC 58 #include <rte_acc_cfg.h> 59 #define ACC100PF_DRIVER_NAME ("intel_acc100_pf") 60 #define ACC100VF_DRIVER_NAME ("intel_acc100_vf") 61 #define ACC100_QMGR_NUM_AQS 16 62 #define ACC100_QMGR_NUM_QGS 2 63 #define ACC100_QMGR_AQ_DEPTH 5 64 #define ACC100_QMGR_INVALID_IDX -1 65 #define ACC100_QMGR_RR 1 66 #define ACC100_QOS_GBR 0 67 #define VRBPF_DRIVER_NAME ("intel_vran_boost_pf") 68 #define VRBVF_DRIVER_NAME ("intel_vran_boost_vf") 69 #define VRB_QMGR_NUM_AQS 16 70 #define VRB_QMGR_NUM_QGS 2 71 #define VRB_QMGR_AQ_DEPTH 5 72 #define VRB_QMGR_INVALID_IDX -1 73 #define VRB_QMGR_RR 1 74 #define VRB_QOS_GBR 0 75 #endif 76 77 #define OPS_CACHE_SIZE 256U 78 #define OPS_POOL_SIZE_MIN 511U /* 0.5K per queue */ 79 80 #define SYNC_WAIT 0 81 #define SYNC_START 1 82 83 #define INVALID_QUEUE_ID -1 84 /* Increment for next code block in external HARQ memory */ 85 #define HARQ_INCR 32768 86 /* Headroom for filler LLRs insertion in HARQ buffer */ 87 #define FILLER_HEADROOM 2048 88 /* Constants from K0 computation from 3GPP 38.212 Table 5.4.2.1-2 */ 89 #define N_ZC_1 66 /* N = 66 Zc for BG 1 */ 90 #define N_ZC_2 50 /* N = 50 Zc for BG 2 */ 91 #define K0_1_1 17 /* K0 fraction numerator for rv 1 and BG 1 */ 92 #define K0_1_2 13 /* K0 fraction numerator for rv 1 and BG 2 */ 93 #define K0_2_1 33 /* K0 fraction numerator for rv 2 and BG 1 */ 94 #define K0_2_2 25 /* K0 fraction numerator for rv 2 and BG 2 */ 95 #define K0_3_1 56 /* K0 fraction numerator for rv 3 and BG 1 */ 96 #define K0_3_2 43 /* K0 fraction numerator for rv 3 and BG 2 */ 97 #define NUM_SC_PER_RB (12) /* Number of subcarriers in a RB in 3GPP. */ 98 #define BITS_PER_LLR (8) /* Number of bits in a LLR. */ 99 100 #define HARQ_MEM_TOLERANCE 256 101 static struct test_bbdev_vector test_vector; 102 103 /* Switch between PMD and Interrupt for throughput TC */ 104 static bool intr_enabled; 105 106 /* LLR arithmetic representation for numerical conversion */ 107 static int ldpc_llr_decimals; 108 static int ldpc_llr_size; 109 /* Keep track of the LDPC decoder device capability flag */ 110 static uint32_t ldpc_cap_flags; 111 /* FFT window width predefined on device and on vector. */ 112 static int fft_window_width_dev; 113 114 /* Represents tested active devices */ 115 static struct active_device { 116 const char *driver_name; 117 uint8_t dev_id; 118 uint16_t supported_ops; 119 uint16_t queue_ids[MAX_QUEUES]; 120 uint16_t nb_queues; 121 struct rte_mempool *ops_mempool; 122 struct rte_mempool *in_mbuf_pool; 123 struct rte_mempool *hard_out_mbuf_pool; 124 struct rte_mempool *soft_out_mbuf_pool; 125 struct rte_mempool *harq_in_mbuf_pool; 126 struct rte_mempool *harq_out_mbuf_pool; 127 } active_devs[RTE_BBDEV_MAX_DEVS]; 128 129 static uint8_t nb_active_devs; 130 131 /* Data buffers used by BBDEV ops */ 132 struct test_buffers { 133 struct rte_bbdev_op_data *inputs; 134 struct rte_bbdev_op_data *hard_outputs; 135 struct rte_bbdev_op_data *soft_outputs; 136 struct rte_bbdev_op_data *harq_inputs; 137 struct rte_bbdev_op_data *harq_outputs; 138 }; 139 140 /* Operation parameters specific for given test case */ 141 struct test_op_params { 142 struct rte_mempool *mp; 143 struct rte_bbdev_dec_op *ref_dec_op; 144 struct rte_bbdev_enc_op *ref_enc_op; 145 struct rte_bbdev_fft_op *ref_fft_op; 146 struct rte_bbdev_mldts_op *ref_mldts_op; 147 uint16_t burst_sz; 148 uint16_t num_to_process; 149 uint16_t num_lcores; 150 int vector_mask; 151 RTE_ATOMIC(uint16_t) sync; 152 struct test_buffers q_bufs[RTE_MAX_NUMA_NODES][MAX_QUEUES]; 153 }; 154 155 /* Contains per lcore params */ 156 struct thread_params { 157 uint8_t dev_id; 158 uint16_t queue_id; 159 uint32_t lcore_id; 160 uint64_t start_time; 161 double ops_per_sec; 162 double mbps; 163 uint8_t iter_count; 164 double iter_average; 165 double bler; 166 RTE_ATOMIC(uint16_t) nb_dequeued; 167 RTE_ATOMIC(int16_t) processing_status; 168 RTE_ATOMIC(uint16_t) burst_sz; 169 struct test_op_params *op_params; 170 struct rte_bbdev_dec_op *dec_ops[MAX_BURST]; 171 struct rte_bbdev_enc_op *enc_ops[MAX_BURST]; 172 struct rte_bbdev_fft_op *fft_ops[MAX_BURST]; 173 struct rte_bbdev_mldts_op *mldts_ops[MAX_BURST]; 174 }; 175 176 /* Stores time statistics */ 177 struct test_time_stats { 178 /* Stores software enqueue total working time */ 179 uint64_t enq_sw_total_time; 180 /* Stores minimum value of software enqueue working time */ 181 uint64_t enq_sw_min_time; 182 /* Stores maximum value of software enqueue working time */ 183 uint64_t enq_sw_max_time; 184 /* Stores turbo enqueue total working time */ 185 uint64_t enq_acc_total_time; 186 /* Stores minimum value of accelerator enqueue working time */ 187 uint64_t enq_acc_min_time; 188 /* Stores maximum value of accelerator enqueue working time */ 189 uint64_t enq_acc_max_time; 190 /* Stores dequeue total working time */ 191 uint64_t deq_total_time; 192 /* Stores minimum value of dequeue working time */ 193 uint64_t deq_min_time; 194 /* Stores maximum value of dequeue working time */ 195 uint64_t deq_max_time; 196 }; 197 198 typedef int (test_case_function)(struct active_device *ad, 199 struct test_op_params *op_params); 200 201 /* Get device status before timeout exit */ 202 static inline void 203 timeout_exit(uint8_t dev_id) 204 { 205 struct rte_bbdev_info info; 206 rte_bbdev_info_get(dev_id, &info); 207 printf("Device Status %s\n", rte_bbdev_device_status_str(info.drv.device_status)); 208 } 209 210 static inline void 211 mbuf_reset(struct rte_mbuf *m) 212 { 213 m->pkt_len = 0; 214 215 do { 216 m->data_len = 0; 217 m = m->next; 218 } while (m != NULL); 219 } 220 221 /* Read flag value 0/1 from bitmap */ 222 static inline bool 223 check_bit(uint32_t bitmap, uint32_t bitmask) 224 { 225 return bitmap & bitmask; 226 } 227 228 static inline void 229 set_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type) 230 { 231 ad->supported_ops |= (1 << op_type); 232 } 233 234 static inline bool 235 is_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type) 236 { 237 return ad->supported_ops & (1 << op_type); 238 } 239 240 static inline bool 241 flags_match(uint32_t flags_req, uint32_t flags_present) 242 { 243 return (flags_req & flags_present) == flags_req; 244 } 245 246 static void 247 clear_soft_out_cap(uint32_t *op_flags) 248 { 249 *op_flags &= ~RTE_BBDEV_TURBO_SOFT_OUTPUT; 250 *op_flags &= ~RTE_BBDEV_TURBO_POS_LLR_1_BIT_SOFT_OUT; 251 *op_flags &= ~RTE_BBDEV_TURBO_NEG_LLR_1_BIT_SOFT_OUT; 252 } 253 254 /* This API is to convert all the test vector op data entries 255 * to big endian format. It is used when the device supports 256 * the input in the big endian format. 257 */ 258 static inline void 259 convert_op_data_to_be(void) 260 { 261 struct op_data_entries *op; 262 enum op_data_type type; 263 uint8_t nb_segs, *rem_data, temp; 264 uint32_t *data, len; 265 int complete, rem, i, j; 266 267 for (type = DATA_INPUT; type < DATA_NUM_TYPES; ++type) { 268 nb_segs = test_vector.entries[type].nb_segments; 269 op = &test_vector.entries[type]; 270 271 /* Invert byte endianness for all the segments */ 272 for (i = 0; i < nb_segs; ++i) { 273 len = op->segments[i].length; 274 data = op->segments[i].addr; 275 276 /* Swap complete u32 bytes */ 277 complete = len / 4; 278 for (j = 0; j < complete; j++) 279 data[j] = rte_bswap32(data[j]); 280 281 /* Swap any remaining bytes */ 282 rem = len % 4; 283 rem_data = (uint8_t *)&data[j]; 284 for (j = 0; j < rem/2; j++) { 285 temp = rem_data[j]; 286 rem_data[j] = rem_data[rem - j - 1]; 287 rem_data[rem - j - 1] = temp; 288 } 289 } 290 } 291 } 292 293 static int 294 check_dev_cap(const struct rte_bbdev_info *dev_info) 295 { 296 unsigned int i; 297 unsigned int nb_inputs, nb_soft_outputs, nb_hard_outputs, 298 nb_harq_inputs, nb_harq_outputs; 299 const struct rte_bbdev_op_cap *op_cap = dev_info->drv.capabilities; 300 uint8_t dev_data_endianness = dev_info->drv.data_endianness; 301 302 nb_inputs = test_vector.entries[DATA_INPUT].nb_segments; 303 nb_soft_outputs = test_vector.entries[DATA_SOFT_OUTPUT].nb_segments; 304 nb_hard_outputs = test_vector.entries[DATA_HARD_OUTPUT].nb_segments; 305 nb_harq_inputs = test_vector.entries[DATA_HARQ_INPUT].nb_segments; 306 nb_harq_outputs = test_vector.entries[DATA_HARQ_OUTPUT].nb_segments; 307 308 for (i = 0; op_cap->type != RTE_BBDEV_OP_NONE; ++i, ++op_cap) { 309 if (op_cap->type != test_vector.op_type) 310 continue; 311 312 if (dev_data_endianness == RTE_BIG_ENDIAN) 313 convert_op_data_to_be(); 314 315 if (op_cap->type == RTE_BBDEV_OP_TURBO_DEC) { 316 const struct rte_bbdev_op_cap_turbo_dec *cap = 317 &op_cap->cap.turbo_dec; 318 /* Ignore lack of soft output capability, just skip 319 * checking if soft output is valid. 320 */ 321 if ((test_vector.turbo_dec.op_flags & 322 RTE_BBDEV_TURBO_SOFT_OUTPUT) && 323 !(cap->capability_flags & 324 RTE_BBDEV_TURBO_SOFT_OUTPUT)) { 325 printf( 326 "INFO: Device \"%s\" does not support soft output - soft output flags will be ignored.\n", 327 dev_info->dev_name); 328 clear_soft_out_cap( 329 &test_vector.turbo_dec.op_flags); 330 } 331 332 if (!flags_match(test_vector.turbo_dec.op_flags, 333 cap->capability_flags)) 334 return TEST_FAILED; 335 if (nb_inputs > cap->num_buffers_src) { 336 printf("Too many inputs defined: %u, max: %u\n", 337 nb_inputs, cap->num_buffers_src); 338 return TEST_FAILED; 339 } 340 if (nb_soft_outputs > cap->num_buffers_soft_out && 341 (test_vector.turbo_dec.op_flags & 342 RTE_BBDEV_TURBO_SOFT_OUTPUT)) { 343 printf( 344 "Too many soft outputs defined: %u, max: %u\n", 345 nb_soft_outputs, 346 cap->num_buffers_soft_out); 347 return TEST_FAILED; 348 } 349 if (nb_hard_outputs > cap->num_buffers_hard_out) { 350 printf( 351 "Too many hard outputs defined: %u, max: %u\n", 352 nb_hard_outputs, 353 cap->num_buffers_hard_out); 354 return TEST_FAILED; 355 } 356 if (intr_enabled && !(cap->capability_flags & 357 RTE_BBDEV_TURBO_DEC_INTERRUPTS)) { 358 printf( 359 "Dequeue interrupts are not supported!\n"); 360 return TEST_FAILED; 361 } 362 363 return TEST_SUCCESS; 364 } else if (op_cap->type == RTE_BBDEV_OP_TURBO_ENC) { 365 const struct rte_bbdev_op_cap_turbo_enc *cap = 366 &op_cap->cap.turbo_enc; 367 368 if (!flags_match(test_vector.turbo_enc.op_flags, 369 cap->capability_flags)) 370 return TEST_FAILED; 371 if (nb_inputs > cap->num_buffers_src) { 372 printf("Too many inputs defined: %u, max: %u\n", 373 nb_inputs, cap->num_buffers_src); 374 return TEST_FAILED; 375 } 376 if (nb_hard_outputs > cap->num_buffers_dst) { 377 printf( 378 "Too many hard outputs defined: %u, max: %u\n", 379 nb_hard_outputs, cap->num_buffers_dst); 380 return TEST_FAILED; 381 } 382 if (intr_enabled && !(cap->capability_flags & 383 RTE_BBDEV_TURBO_ENC_INTERRUPTS)) { 384 printf( 385 "Dequeue interrupts are not supported!\n"); 386 return TEST_FAILED; 387 } 388 389 return TEST_SUCCESS; 390 } else if (op_cap->type == RTE_BBDEV_OP_LDPC_ENC) { 391 const struct rte_bbdev_op_cap_ldpc_enc *cap = 392 &op_cap->cap.ldpc_enc; 393 394 if (!flags_match(test_vector.ldpc_enc.op_flags, 395 cap->capability_flags)){ 396 printf("Flag Mismatch\n"); 397 return TEST_FAILED; 398 } 399 if (nb_inputs > cap->num_buffers_src) { 400 printf("Too many inputs defined: %u, max: %u\n", 401 nb_inputs, cap->num_buffers_src); 402 return TEST_FAILED; 403 } 404 if (nb_hard_outputs > cap->num_buffers_dst) { 405 printf( 406 "Too many hard outputs defined: %u, max: %u\n", 407 nb_hard_outputs, cap->num_buffers_dst); 408 return TEST_FAILED; 409 } 410 if (intr_enabled && !(cap->capability_flags & 411 RTE_BBDEV_LDPC_ENC_INTERRUPTS)) { 412 printf( 413 "Dequeue interrupts are not supported!\n"); 414 return TEST_FAILED; 415 } 416 417 return TEST_SUCCESS; 418 } else if (op_cap->type == RTE_BBDEV_OP_LDPC_DEC) { 419 const struct rte_bbdev_op_cap_ldpc_dec *cap = 420 &op_cap->cap.ldpc_dec; 421 422 if (!flags_match(test_vector.ldpc_dec.op_flags, 423 cap->capability_flags)){ 424 printf("Flag Mismatch\n"); 425 return TEST_FAILED; 426 } 427 if (nb_inputs > cap->num_buffers_src) { 428 printf("Too many inputs defined: %u, max: %u\n", 429 nb_inputs, cap->num_buffers_src); 430 return TEST_FAILED; 431 } 432 if (nb_hard_outputs > cap->num_buffers_hard_out) { 433 printf( 434 "Too many hard outputs defined: %u, max: %u\n", 435 nb_hard_outputs, 436 cap->num_buffers_hard_out); 437 return TEST_FAILED; 438 } 439 if (nb_harq_inputs > cap->num_buffers_hard_out) { 440 printf( 441 "Too many HARQ inputs defined: %u, max: %u\n", 442 nb_harq_inputs, 443 cap->num_buffers_hard_out); 444 return TEST_FAILED; 445 } 446 if (nb_harq_outputs > cap->num_buffers_hard_out) { 447 printf( 448 "Too many HARQ outputs defined: %u, max: %u\n", 449 nb_harq_outputs, 450 cap->num_buffers_hard_out); 451 return TEST_FAILED; 452 } 453 if (intr_enabled && !(cap->capability_flags & 454 RTE_BBDEV_LDPC_DEC_INTERRUPTS)) { 455 printf( 456 "Dequeue interrupts are not supported!\n"); 457 return TEST_FAILED; 458 } 459 if (intr_enabled && (test_vector.ldpc_dec.op_flags & 460 (RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE | 461 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE | 462 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK 463 ))) { 464 printf("Skip loop-back with interrupt\n"); 465 return TEST_FAILED; 466 } 467 return TEST_SUCCESS; 468 } else if (op_cap->type == RTE_BBDEV_OP_FFT) { 469 const struct rte_bbdev_op_cap_fft *cap = &op_cap->cap.fft; 470 471 if (!flags_match(test_vector.fft.op_flags, cap->capability_flags)) { 472 printf("Flag Mismatch\n"); 473 return TEST_FAILED; 474 } 475 if (nb_inputs > cap->num_buffers_src) { 476 printf("Too many inputs defined: %u, max: %u\n", 477 nb_inputs, cap->num_buffers_src); 478 return TEST_FAILED; 479 } 480 return TEST_SUCCESS; 481 } else if (op_cap->type == RTE_BBDEV_OP_MLDTS) { 482 const struct rte_bbdev_op_cap_mld *cap = &op_cap->cap.mld; 483 if (!flags_match(test_vector.mldts.op_flags, cap->capability_flags)) { 484 printf("Flag Mismatch\n"); 485 return TEST_FAILED; 486 } 487 if (nb_inputs > cap->num_buffers_src) { 488 printf("Too many inputs defined: %u, max: %u\n", 489 nb_inputs, cap->num_buffers_src); 490 return TEST_FAILED; 491 } 492 return TEST_SUCCESS; 493 } 494 } 495 496 if ((i == 0) && (test_vector.op_type == RTE_BBDEV_OP_NONE)) 497 return TEST_SUCCESS; /* Special case for NULL device */ 498 499 return TEST_FAILED; 500 } 501 502 /* calculates optimal mempool size not smaller than the val */ 503 static unsigned int 504 optimal_mempool_size(unsigned int val) 505 { 506 return rte_align32pow2(val + 1) - 1; 507 } 508 509 /* allocates mbuf mempool for inputs and outputs */ 510 static struct rte_mempool * 511 create_mbuf_pool(struct op_data_entries *entries, uint8_t dev_id, 512 int socket_id, unsigned int mbuf_pool_size, 513 const char *op_type_str) 514 { 515 unsigned int i; 516 uint32_t max_seg_sz = 0; 517 char pool_name[RTE_MEMPOOL_NAMESIZE]; 518 519 /* find max input segment size */ 520 for (i = 0; i < entries->nb_segments; ++i) 521 if (entries->segments[i].length > max_seg_sz) 522 max_seg_sz = entries->segments[i].length; 523 524 snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str, 525 dev_id); 526 return rte_pktmbuf_pool_create(pool_name, mbuf_pool_size, 0, 0, 527 RTE_MAX(max_seg_sz + RTE_PKTMBUF_HEADROOM 528 + FILLER_HEADROOM, 529 (unsigned int)RTE_MBUF_DEFAULT_BUF_SIZE), socket_id); 530 } 531 532 static int 533 create_mempools(struct active_device *ad, int socket_id, 534 enum rte_bbdev_op_type org_op_type, uint16_t num_ops) 535 { 536 struct rte_mempool *mp; 537 unsigned int ops_pool_size, mbuf_pool_size = 0; 538 char pool_name[RTE_MEMPOOL_NAMESIZE]; 539 const char *op_type_str; 540 enum rte_bbdev_op_type op_type = org_op_type; 541 542 struct op_data_entries *in = &test_vector.entries[DATA_INPUT]; 543 struct op_data_entries *hard_out = 544 &test_vector.entries[DATA_HARD_OUTPUT]; 545 struct op_data_entries *soft_out = 546 &test_vector.entries[DATA_SOFT_OUTPUT]; 547 struct op_data_entries *harq_in = 548 &test_vector.entries[DATA_HARQ_INPUT]; 549 struct op_data_entries *harq_out = 550 &test_vector.entries[DATA_HARQ_OUTPUT]; 551 552 /* allocate ops mempool */ 553 ops_pool_size = optimal_mempool_size(RTE_MAX( 554 /* Ops used plus 1 reference op */ 555 RTE_MAX((unsigned int)(ad->nb_queues * num_ops + 1), 556 /* Minimal cache size plus 1 reference op */ 557 (unsigned int)(1.5 * rte_lcore_count() * 558 OPS_CACHE_SIZE + 1)), 559 OPS_POOL_SIZE_MIN)); 560 561 if (org_op_type == RTE_BBDEV_OP_NONE) 562 op_type = RTE_BBDEV_OP_TURBO_ENC; 563 564 op_type_str = rte_bbdev_op_type_str(op_type); 565 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type); 566 567 snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str, 568 ad->dev_id); 569 mp = rte_bbdev_op_pool_create(pool_name, op_type, 570 ops_pool_size, OPS_CACHE_SIZE, socket_id); 571 TEST_ASSERT_NOT_NULL(mp, 572 "ERROR Failed to create %u items ops pool for dev %u on socket %u.", 573 ops_pool_size, 574 ad->dev_id, 575 socket_id); 576 ad->ops_mempool = mp; 577 578 /* Do not create inputs and outputs mbufs for BaseBand Null Device */ 579 if (org_op_type == RTE_BBDEV_OP_NONE) 580 return TEST_SUCCESS; 581 582 /* Inputs */ 583 if (in->nb_segments > 0) { 584 mbuf_pool_size = optimal_mempool_size(ops_pool_size * 585 in->nb_segments); 586 mp = create_mbuf_pool(in, ad->dev_id, socket_id, 587 mbuf_pool_size, "in"); 588 TEST_ASSERT_NOT_NULL(mp, 589 "ERROR Failed to create %u items input pktmbuf pool for dev %u on socket %u.", 590 mbuf_pool_size, 591 ad->dev_id, 592 socket_id); 593 ad->in_mbuf_pool = mp; 594 } 595 596 /* Hard outputs */ 597 if (hard_out->nb_segments > 0) { 598 mbuf_pool_size = optimal_mempool_size(ops_pool_size * 599 hard_out->nb_segments); 600 mp = create_mbuf_pool(hard_out, ad->dev_id, socket_id, 601 mbuf_pool_size, 602 "hard_out"); 603 TEST_ASSERT_NOT_NULL(mp, 604 "ERROR Failed to create %u items hard output pktmbuf pool for dev %u on socket %u.", 605 mbuf_pool_size, 606 ad->dev_id, 607 socket_id); 608 ad->hard_out_mbuf_pool = mp; 609 } 610 611 /* Soft outputs */ 612 if (soft_out->nb_segments > 0) { 613 mbuf_pool_size = optimal_mempool_size(ops_pool_size * 614 soft_out->nb_segments); 615 mp = create_mbuf_pool(soft_out, ad->dev_id, socket_id, 616 mbuf_pool_size, 617 "soft_out"); 618 TEST_ASSERT_NOT_NULL(mp, 619 "ERROR Failed to create %uB soft output pktmbuf pool for dev %u on socket %u.", 620 mbuf_pool_size, 621 ad->dev_id, 622 socket_id); 623 ad->soft_out_mbuf_pool = mp; 624 } 625 626 /* HARQ inputs */ 627 if (harq_in->nb_segments > 0) { 628 mbuf_pool_size = optimal_mempool_size(ops_pool_size * 629 harq_in->nb_segments); 630 mp = create_mbuf_pool(harq_in, ad->dev_id, socket_id, 631 mbuf_pool_size, 632 "harq_in"); 633 TEST_ASSERT_NOT_NULL(mp, 634 "ERROR Failed to create %uB harq input pktmbuf pool for dev %u on socket %u.", 635 mbuf_pool_size, 636 ad->dev_id, 637 socket_id); 638 ad->harq_in_mbuf_pool = mp; 639 } 640 641 /* HARQ outputs */ 642 if (harq_out->nb_segments > 0) { 643 mbuf_pool_size = optimal_mempool_size(ops_pool_size * 644 harq_out->nb_segments); 645 mp = create_mbuf_pool(harq_out, ad->dev_id, socket_id, 646 mbuf_pool_size, 647 "harq_out"); 648 TEST_ASSERT_NOT_NULL(mp, 649 "ERROR Failed to create %uB harq output pktmbuf pool for dev %u on socket %u.", 650 mbuf_pool_size, 651 ad->dev_id, 652 socket_id); 653 ad->harq_out_mbuf_pool = mp; 654 } 655 656 return TEST_SUCCESS; 657 } 658 659 static int 660 add_bbdev_dev(uint8_t dev_id, struct rte_bbdev_info *info, 661 struct test_bbdev_vector *vector) 662 { 663 int ret; 664 unsigned int queue_id; 665 struct rte_bbdev_queue_conf qconf; 666 struct active_device *ad = &active_devs[nb_active_devs]; 667 unsigned int nb_queues; 668 enum rte_bbdev_op_type op_type = vector->op_type; 669 670 /* Configure fpga lte fec with PF & VF values 671 * if '-i' flag is set and using fpga device 672 */ 673 #ifdef RTE_BASEBAND_FPGA_LTE_FEC 674 if ((get_init_device() == true) && 675 (!strcmp(info->drv.driver_name, FPGA_LTE_PF_DRIVER_NAME))) { 676 struct rte_fpga_lte_fec_conf conf; 677 unsigned int i; 678 679 printf("Configure FPGA LTE FEC Driver %s with default values\n", 680 info->drv.driver_name); 681 682 /* clear default configuration before initialization */ 683 memset(&conf, 0, sizeof(struct rte_fpga_lte_fec_conf)); 684 685 /* Set PF mode : 686 * true if PF is used for data plane 687 * false for VFs 688 */ 689 conf.pf_mode_en = true; 690 691 for (i = 0; i < FPGA_LTE_FEC_NUM_VFS; ++i) { 692 /* Number of UL queues per VF (fpga supports 8 VFs) */ 693 conf.vf_ul_queues_number[i] = VF_UL_4G_QUEUE_VALUE; 694 /* Number of DL queues per VF (fpga supports 8 VFs) */ 695 conf.vf_dl_queues_number[i] = VF_DL_4G_QUEUE_VALUE; 696 } 697 698 /* UL bandwidth. Needed for schedule algorithm */ 699 conf.ul_bandwidth = UL_4G_BANDWIDTH; 700 /* DL bandwidth */ 701 conf.dl_bandwidth = DL_4G_BANDWIDTH; 702 703 /* UL & DL load Balance Factor to 64 */ 704 conf.ul_load_balance = UL_4G_LOAD_BALANCE; 705 conf.dl_load_balance = DL_4G_LOAD_BALANCE; 706 707 /**< FLR timeout value */ 708 conf.flr_time_out = FLR_4G_TIMEOUT; 709 710 /* setup FPGA PF with configuration information */ 711 ret = rte_fpga_lte_fec_configure(info->dev_name, &conf); 712 TEST_ASSERT_SUCCESS(ret, 713 "Failed to configure 4G FPGA PF for bbdev %s", 714 info->dev_name); 715 } 716 #endif 717 #ifdef RTE_BASEBAND_FPGA_5GNR_FEC 718 if ((get_init_device() == true) && 719 (!strcmp(info->drv.driver_name, FPGA_5GNR_PF_DRIVER_NAME))) { 720 struct rte_fpga_5gnr_fec_conf conf; 721 unsigned int i; 722 723 printf("Configure FPGA 5GNR FEC Driver %s with default values\n", 724 info->drv.driver_name); 725 726 /* clear default configuration before initialization */ 727 memset(&conf, 0, sizeof(struct rte_fpga_5gnr_fec_conf)); 728 729 /* Set PF mode : 730 * true if PF is used for data plane 731 * false for VFs 732 */ 733 conf.pf_mode_en = true; 734 735 for (i = 0; i < FPGA_5GNR_FEC_NUM_VFS; ++i) { 736 /* Number of UL queues per VF (fpga supports 8 VFs) */ 737 conf.vf_ul_queues_number[i] = VF_UL_5G_QUEUE_VALUE; 738 /* Number of DL queues per VF (fpga supports 8 VFs) */ 739 conf.vf_dl_queues_number[i] = VF_DL_5G_QUEUE_VALUE; 740 } 741 742 /* UL bandwidth. Needed only for Vista Creek 5GNR schedule algorithm */ 743 conf.ul_bandwidth = UL_5G_BANDWIDTH; 744 /* DL bandwidth. Needed only for Vista Creek 5GNR schedule algorithm */ 745 conf.dl_bandwidth = DL_5G_BANDWIDTH; 746 747 /* UL & DL load Balance Factor to 64 */ 748 conf.ul_load_balance = UL_5G_LOAD_BALANCE; 749 conf.dl_load_balance = DL_5G_LOAD_BALANCE; 750 751 /* setup FPGA PF with configuration information */ 752 ret = rte_fpga_5gnr_fec_configure(info->dev_name, &conf); 753 TEST_ASSERT_SUCCESS(ret, 754 "Failed to configure 5G FPGA PF for bbdev %s", 755 info->dev_name); 756 } 757 #endif 758 #ifdef RTE_BASEBAND_ACC 759 if ((get_init_device() == true) && 760 (!strcmp(info->drv.driver_name, ACC100PF_DRIVER_NAME))) { 761 struct rte_acc_conf conf; 762 unsigned int i; 763 764 printf("Configure ACC100 FEC device %s with default values\n", 765 info->drv.driver_name); 766 767 /* clear default configuration before initialization */ 768 memset(&conf, 0, sizeof(struct rte_acc_conf)); 769 770 /* Always set in PF mode for built-in configuration */ 771 conf.pf_mode_en = true; 772 for (i = 0; i < RTE_ACC_NUM_VFS; ++i) { 773 conf.arb_dl_4g[i].gbr_threshold1 = ACC100_QOS_GBR; 774 conf.arb_dl_4g[i].gbr_threshold1 = ACC100_QOS_GBR; 775 conf.arb_dl_4g[i].round_robin_weight = ACC100_QMGR_RR; 776 conf.arb_ul_4g[i].gbr_threshold1 = ACC100_QOS_GBR; 777 conf.arb_ul_4g[i].gbr_threshold1 = ACC100_QOS_GBR; 778 conf.arb_ul_4g[i].round_robin_weight = ACC100_QMGR_RR; 779 conf.arb_dl_5g[i].gbr_threshold1 = ACC100_QOS_GBR; 780 conf.arb_dl_5g[i].gbr_threshold1 = ACC100_QOS_GBR; 781 conf.arb_dl_5g[i].round_robin_weight = ACC100_QMGR_RR; 782 conf.arb_ul_5g[i].gbr_threshold1 = ACC100_QOS_GBR; 783 conf.arb_ul_5g[i].gbr_threshold1 = ACC100_QOS_GBR; 784 conf.arb_ul_5g[i].round_robin_weight = ACC100_QMGR_RR; 785 } 786 787 conf.input_pos_llr_1_bit = true; 788 conf.output_pos_llr_1_bit = true; 789 conf.num_vf_bundles = 1; /**< Number of VF bundles to setup */ 790 791 conf.q_ul_4g.num_qgroups = ACC100_QMGR_NUM_QGS; 792 conf.q_ul_4g.first_qgroup_index = ACC100_QMGR_INVALID_IDX; 793 conf.q_ul_4g.num_aqs_per_groups = ACC100_QMGR_NUM_AQS; 794 conf.q_ul_4g.aq_depth_log2 = ACC100_QMGR_AQ_DEPTH; 795 conf.q_dl_4g.num_qgroups = ACC100_QMGR_NUM_QGS; 796 conf.q_dl_4g.first_qgroup_index = ACC100_QMGR_INVALID_IDX; 797 conf.q_dl_4g.num_aqs_per_groups = ACC100_QMGR_NUM_AQS; 798 conf.q_dl_4g.aq_depth_log2 = ACC100_QMGR_AQ_DEPTH; 799 conf.q_ul_5g.num_qgroups = ACC100_QMGR_NUM_QGS; 800 conf.q_ul_5g.first_qgroup_index = ACC100_QMGR_INVALID_IDX; 801 conf.q_ul_5g.num_aqs_per_groups = ACC100_QMGR_NUM_AQS; 802 conf.q_ul_5g.aq_depth_log2 = ACC100_QMGR_AQ_DEPTH; 803 conf.q_dl_5g.num_qgroups = ACC100_QMGR_NUM_QGS; 804 conf.q_dl_5g.first_qgroup_index = ACC100_QMGR_INVALID_IDX; 805 conf.q_dl_5g.num_aqs_per_groups = ACC100_QMGR_NUM_AQS; 806 conf.q_dl_5g.aq_depth_log2 = ACC100_QMGR_AQ_DEPTH; 807 808 /* setup PF with configuration information */ 809 ret = rte_acc_configure(info->dev_name, &conf); 810 TEST_ASSERT_SUCCESS(ret, 811 "Failed to configure ACC100 PF for bbdev %s", 812 info->dev_name); 813 } 814 if ((get_init_device() == true) && 815 (!strcmp(info->drv.driver_name, VRBPF_DRIVER_NAME))) { 816 struct rte_acc_conf conf; 817 unsigned int i; 818 819 printf("Configure Driver %s with default values\n", 820 info->drv.driver_name); 821 822 /* clear default configuration before initialization */ 823 memset(&conf, 0, sizeof(struct rte_acc_conf)); 824 825 /* Always set in PF mode for built-in configuration */ 826 conf.pf_mode_en = true; 827 for (i = 0; i < RTE_ACC_NUM_VFS; ++i) { 828 conf.arb_dl_4g[i].gbr_threshold1 = VRB_QOS_GBR; 829 conf.arb_dl_4g[i].gbr_threshold1 = VRB_QOS_GBR; 830 conf.arb_dl_4g[i].round_robin_weight = VRB_QMGR_RR; 831 conf.arb_ul_4g[i].gbr_threshold1 = VRB_QOS_GBR; 832 conf.arb_ul_4g[i].gbr_threshold1 = VRB_QOS_GBR; 833 conf.arb_ul_4g[i].round_robin_weight = VRB_QMGR_RR; 834 conf.arb_dl_5g[i].gbr_threshold1 = VRB_QOS_GBR; 835 conf.arb_dl_5g[i].gbr_threshold1 = VRB_QOS_GBR; 836 conf.arb_dl_5g[i].round_robin_weight = VRB_QMGR_RR; 837 conf.arb_ul_5g[i].gbr_threshold1 = VRB_QOS_GBR; 838 conf.arb_ul_5g[i].gbr_threshold1 = VRB_QOS_GBR; 839 conf.arb_ul_5g[i].round_robin_weight = VRB_QMGR_RR; 840 conf.arb_fft[i].gbr_threshold1 = VRB_QOS_GBR; 841 conf.arb_fft[i].gbr_threshold1 = VRB_QOS_GBR; 842 conf.arb_fft[i].round_robin_weight = VRB_QMGR_RR; 843 conf.arb_mld[i].gbr_threshold1 = VRB_QOS_GBR; 844 conf.arb_mld[i].gbr_threshold1 = VRB_QOS_GBR; 845 conf.arb_mld[i].round_robin_weight = VRB_QMGR_RR; 846 } 847 848 conf.input_pos_llr_1_bit = true; 849 conf.output_pos_llr_1_bit = true; 850 conf.num_vf_bundles = 1; /**< Number of VF bundles to setup */ 851 852 conf.q_ul_4g.num_qgroups = VRB_QMGR_NUM_QGS; 853 conf.q_ul_4g.first_qgroup_index = VRB_QMGR_INVALID_IDX; 854 conf.q_ul_4g.num_aqs_per_groups = VRB_QMGR_NUM_AQS; 855 conf.q_ul_4g.aq_depth_log2 = VRB_QMGR_AQ_DEPTH; 856 conf.q_dl_4g.num_qgroups = VRB_QMGR_NUM_QGS; 857 conf.q_dl_4g.first_qgroup_index = VRB_QMGR_INVALID_IDX; 858 conf.q_dl_4g.num_aqs_per_groups = VRB_QMGR_NUM_AQS; 859 conf.q_dl_4g.aq_depth_log2 = VRB_QMGR_AQ_DEPTH; 860 conf.q_ul_5g.num_qgroups = VRB_QMGR_NUM_QGS; 861 conf.q_ul_5g.first_qgroup_index = VRB_QMGR_INVALID_IDX; 862 conf.q_ul_5g.num_aqs_per_groups = VRB_QMGR_NUM_AQS; 863 conf.q_ul_5g.aq_depth_log2 = VRB_QMGR_AQ_DEPTH; 864 conf.q_dl_5g.num_qgroups = VRB_QMGR_NUM_QGS; 865 conf.q_dl_5g.first_qgroup_index = VRB_QMGR_INVALID_IDX; 866 conf.q_dl_5g.num_aqs_per_groups = VRB_QMGR_NUM_AQS; 867 conf.q_dl_5g.aq_depth_log2 = VRB_QMGR_AQ_DEPTH; 868 conf.q_fft.num_qgroups = VRB_QMGR_NUM_QGS; 869 conf.q_fft.first_qgroup_index = VRB_QMGR_INVALID_IDX; 870 conf.q_fft.num_aqs_per_groups = VRB_QMGR_NUM_AQS; 871 conf.q_fft.aq_depth_log2 = VRB_QMGR_AQ_DEPTH; 872 conf.q_mld.num_qgroups = VRB_QMGR_NUM_QGS; 873 conf.q_mld.first_qgroup_index = VRB_QMGR_INVALID_IDX; 874 conf.q_mld.num_aqs_per_groups = VRB_QMGR_NUM_AQS; 875 conf.q_mld.aq_depth_log2 = VRB_QMGR_AQ_DEPTH; 876 877 /* setup PF with configuration information */ 878 ret = rte_acc_configure(info->dev_name, &conf); 879 TEST_ASSERT_SUCCESS(ret, 880 "Failed to configure PF for bbdev %s", 881 info->dev_name); 882 } 883 #endif 884 /* Let's refresh this now this is configured */ 885 rte_bbdev_info_get(dev_id, info); 886 if (info->drv.device_status == RTE_BBDEV_DEV_FATAL_ERR) 887 printf("Device Status %s\n", rte_bbdev_device_status_str(info->drv.device_status)); 888 if (info->drv.fft_window_width != NULL) 889 fft_window_width_dev = info->drv.fft_window_width[0]; 890 else 891 fft_window_width_dev = 0; 892 if (fft_window_width_dev != 0) 893 printf(" FFT Window0 width %d\n", fft_window_width_dev); 894 895 nb_queues = RTE_MIN(rte_lcore_count(), info->drv.max_num_queues); 896 nb_queues = RTE_MIN(nb_queues, (unsigned int) MAX_QUEUES); 897 898 /* setup device */ 899 ret = rte_bbdev_setup_queues(dev_id, nb_queues, info->socket_id); 900 if (ret < 0) { 901 printf("rte_bbdev_setup_queues(%u, %u, %d) ret %i\n", 902 dev_id, nb_queues, info->socket_id, ret); 903 return TEST_FAILED; 904 } 905 906 /* configure interrupts if needed */ 907 if (intr_enabled) { 908 ret = rte_bbdev_intr_enable(dev_id); 909 if (ret < 0) { 910 printf("rte_bbdev_intr_enable(%u) ret %i\n", dev_id, 911 ret); 912 return TEST_FAILED; 913 } 914 } 915 916 /* setup device queues */ 917 qconf.socket = info->socket_id; 918 qconf.queue_size = info->drv.default_queue_conf.queue_size; 919 qconf.priority = 0; 920 qconf.deferred_start = 0; 921 qconf.op_type = op_type; 922 923 for (queue_id = 0; queue_id < nb_queues; ++queue_id) { 924 ret = rte_bbdev_queue_configure(dev_id, queue_id, &qconf); 925 if (ret != 0) { 926 printf( 927 "Allocated all queues (id=%u) at prio%u on dev%u\n", 928 queue_id, qconf.priority, dev_id); 929 qconf.priority++; 930 ret = rte_bbdev_queue_configure(ad->dev_id, queue_id, &qconf); 931 } 932 if (ret != 0) { 933 printf("All queues on dev %u allocated: %u\n", dev_id, queue_id); 934 break; 935 } 936 ret = rte_bbdev_queue_start(ad->dev_id, queue_id); 937 if (ret != 0) { 938 printf("Failed to start queue on dev %u q_id: %u\n", dev_id, queue_id); 939 break; 940 } 941 ad->queue_ids[queue_id] = queue_id; 942 } 943 TEST_ASSERT(queue_id != 0, 944 "ERROR Failed to configure any queues on dev %u\n" 945 "\tthe device may not support the related operation capability\n" 946 "\tor the device may not have been configured yet", dev_id); 947 ad->nb_queues = queue_id; 948 949 set_avail_op(ad, op_type); 950 951 return TEST_SUCCESS; 952 } 953 954 static int 955 add_active_device(uint8_t dev_id, struct rte_bbdev_info *info, 956 struct test_bbdev_vector *vector) 957 { 958 int ret; 959 960 active_devs[nb_active_devs].driver_name = info->drv.driver_name; 961 active_devs[nb_active_devs].dev_id = dev_id; 962 963 ret = add_bbdev_dev(dev_id, info, vector); 964 if (ret == TEST_SUCCESS) 965 ++nb_active_devs; 966 return ret; 967 } 968 969 static uint8_t 970 populate_active_devices(void) 971 { 972 int ret; 973 uint8_t dev_id; 974 uint8_t nb_devs_added = 0; 975 struct rte_bbdev_info info; 976 977 RTE_BBDEV_FOREACH(dev_id) { 978 rte_bbdev_info_get(dev_id, &info); 979 980 if (check_dev_cap(&info)) { 981 printf( 982 "Device %d (%s) does not support specified capabilities\n", 983 dev_id, info.dev_name); 984 continue; 985 } 986 987 ret = add_active_device(dev_id, &info, &test_vector); 988 if (ret != 0) { 989 printf("Adding active bbdev %s skipped\n", 990 info.dev_name); 991 continue; 992 } 993 nb_devs_added++; 994 } 995 996 return nb_devs_added; 997 } 998 999 static int 1000 read_test_vector(void) 1001 { 1002 int ret; 1003 1004 memset(&test_vector, 0, sizeof(test_vector)); 1005 printf("Test vector file = %s\n", get_vector_filename()); 1006 ret = test_bbdev_vector_read(get_vector_filename(), &test_vector); 1007 TEST_ASSERT_SUCCESS(ret, "Failed to parse file %s\n", 1008 get_vector_filename()); 1009 1010 return TEST_SUCCESS; 1011 } 1012 1013 static int 1014 testsuite_setup(void) 1015 { 1016 TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n"); 1017 1018 if (populate_active_devices() == 0) { 1019 printf("No suitable devices found!\n"); 1020 return TEST_SKIPPED; 1021 } 1022 1023 return TEST_SUCCESS; 1024 } 1025 1026 static int 1027 interrupt_testsuite_setup(void) 1028 { 1029 TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n"); 1030 1031 /* Enable interrupts */ 1032 intr_enabled = true; 1033 1034 /* Special case for NULL device (RTE_BBDEV_OP_NONE) */ 1035 if (populate_active_devices() == 0 || 1036 test_vector.op_type == RTE_BBDEV_OP_NONE) { 1037 intr_enabled = false; 1038 printf("No suitable devices found!\n"); 1039 return TEST_SKIPPED; 1040 } 1041 1042 return TEST_SUCCESS; 1043 } 1044 1045 static void 1046 testsuite_teardown(void) 1047 { 1048 uint8_t dev_id; 1049 1050 /* Unconfigure devices */ 1051 RTE_BBDEV_FOREACH(dev_id) 1052 rte_bbdev_close(dev_id); 1053 1054 /* Clear active devices structs. */ 1055 memset(active_devs, 0, sizeof(active_devs)); 1056 nb_active_devs = 0; 1057 1058 /* Disable interrupts */ 1059 intr_enabled = false; 1060 } 1061 1062 static int 1063 ut_setup(void) 1064 { 1065 uint8_t i, dev_id; 1066 1067 for (i = 0; i < nb_active_devs; i++) { 1068 dev_id = active_devs[i].dev_id; 1069 /* reset bbdev stats */ 1070 TEST_ASSERT_SUCCESS(rte_bbdev_stats_reset(dev_id), 1071 "Failed to reset stats of bbdev %u", dev_id); 1072 /* start the device */ 1073 TEST_ASSERT_SUCCESS(rte_bbdev_start(dev_id), 1074 "Failed to start bbdev %u", dev_id); 1075 } 1076 1077 return TEST_SUCCESS; 1078 } 1079 1080 static void 1081 ut_teardown(void) 1082 { 1083 uint8_t i, dev_id, ret; 1084 struct rte_bbdev_stats stats; 1085 1086 for (i = 0; i < nb_active_devs; i++) { 1087 dev_id = active_devs[i].dev_id; 1088 /* read stats and print */ 1089 ret = rte_bbdev_stats_get(dev_id, &stats); 1090 if (ret != 0) 1091 printf("Failed to get stats on bbdev %u\n", dev_id); 1092 /* Stop the device */ 1093 rte_bbdev_stop(dev_id); 1094 } 1095 } 1096 1097 static int 1098 init_op_data_objs(struct rte_bbdev_op_data *bufs, 1099 struct op_data_entries *ref_entries, 1100 struct rte_mempool *mbuf_pool, const uint16_t n, 1101 enum op_data_type op_type, uint16_t min_alignment) 1102 { 1103 int ret; 1104 unsigned int i, j; 1105 bool large_input = false; 1106 1107 for (i = 0; i < n; ++i) { 1108 char *data; 1109 struct op_data_buf *seg = &ref_entries->segments[0]; 1110 struct rte_mbuf *m_head = rte_pktmbuf_alloc(mbuf_pool); 1111 TEST_ASSERT_NOT_NULL(m_head, 1112 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)", 1113 op_type, n * ref_entries->nb_segments, 1114 mbuf_pool->size); 1115 1116 if ((seg->length + RTE_PKTMBUF_HEADROOM) > RTE_BBDEV_LDPC_E_MAX_MBUF) { 1117 /* 1118 * Special case when DPDK mbuf cannot handle 1119 * the required input size 1120 */ 1121 large_input = true; 1122 } 1123 bufs[i].data = m_head; 1124 bufs[i].offset = 0; 1125 bufs[i].length = 0; 1126 1127 if ((op_type == DATA_INPUT) || (op_type == DATA_HARQ_INPUT)) { 1128 if (large_input) { 1129 /* Allocate a fake overused mbuf */ 1130 data = rte_malloc(NULL, seg->length, 0); 1131 TEST_ASSERT_NOT_NULL(data, 1132 "rte malloc failed with %u bytes", 1133 seg->length); 1134 memcpy(data, seg->addr, seg->length); 1135 m_head->buf_addr = data; 1136 rte_mbuf_iova_set(m_head, rte_malloc_virt2iova(data)); 1137 m_head->data_off = 0; 1138 m_head->data_len = seg->length; 1139 } else { 1140 data = rte_pktmbuf_append(m_head, seg->length); 1141 TEST_ASSERT_NOT_NULL(data, 1142 "Couldn't append %u bytes to mbuf from %d data type mbuf pool", 1143 seg->length, op_type); 1144 1145 TEST_ASSERT(data == RTE_PTR_ALIGN( 1146 data, min_alignment), 1147 "Data addr in mbuf (%p) is not aligned to device min alignment (%u)", 1148 data, min_alignment); 1149 rte_memcpy(data, seg->addr, seg->length); 1150 } 1151 1152 bufs[i].length += seg->length; 1153 1154 for (j = 1; j < ref_entries->nb_segments; ++j) { 1155 struct rte_mbuf *m_tail = 1156 rte_pktmbuf_alloc(mbuf_pool); 1157 TEST_ASSERT_NOT_NULL(m_tail, 1158 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)", 1159 op_type, 1160 n * ref_entries->nb_segments, 1161 mbuf_pool->size); 1162 seg += 1; 1163 1164 data = rte_pktmbuf_append(m_tail, seg->length); 1165 TEST_ASSERT_NOT_NULL(data, 1166 "Couldn't append %u bytes to mbuf from %d data type mbuf pool", 1167 seg->length, op_type); 1168 1169 TEST_ASSERT(data == RTE_PTR_ALIGN(data, 1170 min_alignment), 1171 "Data addr in mbuf (%p) is not aligned to device min alignment (%u)", 1172 data, min_alignment); 1173 rte_memcpy(data, seg->addr, seg->length); 1174 bufs[i].length += seg->length; 1175 1176 ret = rte_pktmbuf_chain(m_head, m_tail); 1177 TEST_ASSERT_SUCCESS(ret, 1178 "Couldn't chain mbufs from %d data type mbuf pool", 1179 op_type); 1180 } 1181 } else { 1182 if (((op_type == DATA_HARD_OUTPUT) || (op_type == DATA_SOFT_OUTPUT)) 1183 && ((seg->length + RTE_PKTMBUF_HEADROOM) 1184 > RTE_BBDEV_LDPC_E_MAX_MBUF)) { 1185 /* Allocate a fake overused mbuf + margin */ 1186 data = rte_malloc(NULL, seg->length + 1024, 0); 1187 TEST_ASSERT_NOT_NULL(data, 1188 "rte malloc failed with %u bytes", 1189 seg->length + 1024); 1190 m_head->buf_addr = data; 1191 rte_mbuf_iova_set(m_head, rte_malloc_virt2iova(data)); 1192 m_head->data_off = 0; 1193 m_head->data_len = seg->length; 1194 } else { 1195 /* allocate chained-mbuf for output buffer */ 1196 for (j = 1; j < ref_entries->nb_segments; ++j) { 1197 struct rte_mbuf *m_tail = 1198 rte_pktmbuf_alloc(mbuf_pool); 1199 TEST_ASSERT_NOT_NULL(m_tail, 1200 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)", 1201 op_type, 1202 n * ref_entries->nb_segments, 1203 mbuf_pool->size); 1204 1205 ret = rte_pktmbuf_chain(m_head, m_tail); 1206 TEST_ASSERT_SUCCESS(ret, 1207 "Couldn't chain mbufs from %d data type mbuf pool", 1208 op_type); 1209 } 1210 } 1211 bufs[i].length += seg->length; 1212 } 1213 } 1214 1215 return 0; 1216 } 1217 1218 static int 1219 allocate_buffers_on_socket(struct rte_bbdev_op_data **buffers, const int len, 1220 const int socket) 1221 { 1222 int i; 1223 1224 *buffers = rte_zmalloc_socket(NULL, len, 0, socket); 1225 if (*buffers == NULL) { 1226 printf("WARNING: Failed to allocate op_data on socket %d\n", 1227 socket); 1228 /* try to allocate memory on other detected sockets */ 1229 for (i = 0; i < socket; i++) { 1230 *buffers = rte_zmalloc_socket(NULL, len, 0, i); 1231 if (*buffers != NULL) 1232 break; 1233 } 1234 } 1235 1236 return (*buffers == NULL) ? TEST_FAILED : TEST_SUCCESS; 1237 } 1238 1239 static void 1240 limit_input_llr_val_range(struct rte_bbdev_op_data *input_ops, 1241 const uint16_t n, const int8_t max_llr_modulus) 1242 { 1243 uint16_t i, byte_idx; 1244 1245 for (i = 0; i < n; ++i) { 1246 struct rte_mbuf *m = input_ops[i].data; 1247 while (m != NULL) { 1248 int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *, 1249 input_ops[i].offset); 1250 for (byte_idx = 0; byte_idx < rte_pktmbuf_data_len(m); 1251 ++byte_idx) 1252 llr[byte_idx] = round((double)max_llr_modulus * 1253 llr[byte_idx] / INT8_MAX); 1254 1255 m = m->next; 1256 } 1257 } 1258 } 1259 1260 /* 1261 * We may have to insert filler bits 1262 * when they are required by the HARQ assumption 1263 */ 1264 static void 1265 ldpc_add_filler(struct rte_bbdev_op_data *input_ops, 1266 const uint16_t n, struct test_op_params *op_params) 1267 { 1268 struct rte_bbdev_op_ldpc_dec dec = op_params->ref_dec_op->ldpc_dec; 1269 1270 if (input_ops == NULL) 1271 return; 1272 /* No need to add filler if not required by device */ 1273 if (!(ldpc_cap_flags & 1274 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_FILLERS)) 1275 return; 1276 /* No need to add filler for loopback operation */ 1277 if (dec.op_flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK) 1278 return; 1279 1280 uint16_t i, j, parity_offset; 1281 for (i = 0; i < n; ++i) { 1282 struct rte_mbuf *m = input_ops[i].data; 1283 int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *, 1284 input_ops[i].offset); 1285 parity_offset = (dec.basegraph == 1 ? 20 : 8) 1286 * dec.z_c - dec.n_filler; 1287 uint16_t new_hin_size = input_ops[i].length + dec.n_filler; 1288 m->data_len = new_hin_size; 1289 input_ops[i].length = new_hin_size; 1290 for (j = new_hin_size - 1; j >= parity_offset + dec.n_filler; 1291 j--) 1292 llr[j] = llr[j - dec.n_filler]; 1293 uint16_t llr_max_pre_scaling = (1 << (ldpc_llr_size - 1)) - 1; 1294 for (j = 0; j < dec.n_filler; j++) 1295 llr[parity_offset + j] = llr_max_pre_scaling; 1296 } 1297 } 1298 1299 static void 1300 ldpc_input_llr_scaling(struct rte_bbdev_op_data *input_ops, 1301 const uint16_t n, const int8_t llr_size, 1302 const int8_t llr_decimals) 1303 { 1304 if (input_ops == NULL) 1305 return; 1306 1307 uint16_t i, byte_idx; 1308 1309 int16_t llr_max, llr_min, llr_tmp; 1310 llr_max = (1 << (llr_size - 1)) - 1; 1311 llr_min = -llr_max; 1312 for (i = 0; i < n; ++i) { 1313 struct rte_mbuf *m = input_ops[i].data; 1314 while (m != NULL) { 1315 int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *, 1316 input_ops[i].offset); 1317 for (byte_idx = 0; byte_idx < rte_pktmbuf_data_len(m); 1318 ++byte_idx) { 1319 1320 llr_tmp = llr[byte_idx]; 1321 if (llr_decimals == 4) 1322 llr_tmp *= 8; 1323 else if (llr_decimals == 2) 1324 llr_tmp *= 2; 1325 else if (llr_decimals == 0) 1326 llr_tmp /= 2; 1327 llr_tmp = RTE_MIN(llr_max, 1328 RTE_MAX(llr_min, llr_tmp)); 1329 llr[byte_idx] = (int8_t) llr_tmp; 1330 } 1331 1332 m = m->next; 1333 } 1334 } 1335 } 1336 1337 1338 1339 static int 1340 fill_queue_buffers(struct test_op_params *op_params, 1341 struct rte_mempool *in_mp, struct rte_mempool *hard_out_mp, 1342 struct rte_mempool *soft_out_mp, 1343 struct rte_mempool *harq_in_mp, struct rte_mempool *harq_out_mp, 1344 uint16_t queue_id, 1345 const struct rte_bbdev_op_cap *capabilities, 1346 uint16_t min_alignment, const int socket_id) 1347 { 1348 int ret; 1349 enum op_data_type type; 1350 const uint16_t n = op_params->num_to_process; 1351 1352 struct rte_mempool *mbuf_pools[DATA_NUM_TYPES] = { 1353 in_mp, 1354 soft_out_mp, 1355 hard_out_mp, 1356 harq_in_mp, 1357 harq_out_mp, 1358 }; 1359 1360 struct rte_bbdev_op_data **queue_ops[DATA_NUM_TYPES] = { 1361 &op_params->q_bufs[socket_id][queue_id].inputs, 1362 &op_params->q_bufs[socket_id][queue_id].soft_outputs, 1363 &op_params->q_bufs[socket_id][queue_id].hard_outputs, 1364 &op_params->q_bufs[socket_id][queue_id].harq_inputs, 1365 &op_params->q_bufs[socket_id][queue_id].harq_outputs, 1366 }; 1367 1368 for (type = DATA_INPUT; type < DATA_NUM_TYPES; ++type) { 1369 struct op_data_entries *ref_entries = 1370 &test_vector.entries[type]; 1371 if (ref_entries->nb_segments == 0) 1372 continue; 1373 1374 ret = allocate_buffers_on_socket(queue_ops[type], 1375 n * sizeof(struct rte_bbdev_op_data), 1376 socket_id); 1377 TEST_ASSERT_SUCCESS(ret, 1378 "Couldn't allocate memory for rte_bbdev_op_data structs"); 1379 1380 ret = init_op_data_objs(*queue_ops[type], ref_entries, 1381 mbuf_pools[type], n, type, min_alignment); 1382 TEST_ASSERT_SUCCESS(ret, 1383 "Couldn't init rte_bbdev_op_data structs"); 1384 } 1385 1386 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) 1387 limit_input_llr_val_range(*queue_ops[DATA_INPUT], n, 1388 capabilities->cap.turbo_dec.max_llr_modulus); 1389 1390 if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) { 1391 bool loopback = op_params->ref_dec_op->ldpc_dec.op_flags & 1392 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK; 1393 bool llr_comp = op_params->ref_dec_op->ldpc_dec.op_flags & 1394 RTE_BBDEV_LDPC_LLR_COMPRESSION; 1395 bool harq_comp = op_params->ref_dec_op->ldpc_dec.op_flags & 1396 RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION; 1397 1398 ldpc_llr_decimals = capabilities->cap.ldpc_dec.llr_decimals; 1399 ldpc_llr_size = capabilities->cap.ldpc_dec.llr_size; 1400 ldpc_cap_flags = capabilities->cap.ldpc_dec.capability_flags; 1401 if (!loopback && !llr_comp) 1402 ldpc_input_llr_scaling(*queue_ops[DATA_INPUT], n, 1403 ldpc_llr_size, ldpc_llr_decimals); 1404 if (!loopback && !harq_comp) 1405 ldpc_input_llr_scaling(*queue_ops[DATA_HARQ_INPUT], n, 1406 ldpc_llr_size, ldpc_llr_decimals); 1407 if (!loopback) 1408 ldpc_add_filler(*queue_ops[DATA_HARQ_INPUT], n, 1409 op_params); 1410 } 1411 1412 return 0; 1413 } 1414 1415 static void 1416 free_buffers(struct active_device *ad, struct test_op_params *op_params) 1417 { 1418 unsigned int i, j; 1419 1420 rte_mempool_free(ad->ops_mempool); 1421 rte_mempool_free(ad->in_mbuf_pool); 1422 rte_mempool_free(ad->hard_out_mbuf_pool); 1423 rte_mempool_free(ad->soft_out_mbuf_pool); 1424 rte_mempool_free(ad->harq_in_mbuf_pool); 1425 rte_mempool_free(ad->harq_out_mbuf_pool); 1426 1427 for (i = 0; i < rte_lcore_count(); ++i) { 1428 for (j = 0; j < RTE_MAX_NUMA_NODES; ++j) { 1429 rte_free(op_params->q_bufs[j][i].inputs); 1430 rte_free(op_params->q_bufs[j][i].hard_outputs); 1431 rte_free(op_params->q_bufs[j][i].soft_outputs); 1432 rte_free(op_params->q_bufs[j][i].harq_inputs); 1433 rte_free(op_params->q_bufs[j][i].harq_outputs); 1434 } 1435 } 1436 } 1437 1438 static void 1439 copy_reference_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n, 1440 unsigned int start_idx, 1441 struct rte_bbdev_op_data *inputs, 1442 struct rte_bbdev_op_data *hard_outputs, 1443 struct rte_bbdev_op_data *soft_outputs, 1444 struct rte_bbdev_dec_op *ref_op) 1445 { 1446 unsigned int i; 1447 struct rte_bbdev_op_turbo_dec *turbo_dec = &ref_op->turbo_dec; 1448 1449 for (i = 0; i < n; ++i) { 1450 if (turbo_dec->code_block_mode == RTE_BBDEV_TRANSPORT_BLOCK) { 1451 ops[i]->turbo_dec.tb_params.ea = 1452 turbo_dec->tb_params.ea; 1453 ops[i]->turbo_dec.tb_params.eb = 1454 turbo_dec->tb_params.eb; 1455 ops[i]->turbo_dec.tb_params.k_pos = 1456 turbo_dec->tb_params.k_pos; 1457 ops[i]->turbo_dec.tb_params.k_neg = 1458 turbo_dec->tb_params.k_neg; 1459 ops[i]->turbo_dec.tb_params.c = 1460 turbo_dec->tb_params.c; 1461 ops[i]->turbo_dec.tb_params.c_neg = 1462 turbo_dec->tb_params.c_neg; 1463 ops[i]->turbo_dec.tb_params.cab = 1464 turbo_dec->tb_params.cab; 1465 ops[i]->turbo_dec.tb_params.r = 1466 turbo_dec->tb_params.r; 1467 } else { 1468 ops[i]->turbo_dec.cb_params.e = turbo_dec->cb_params.e; 1469 ops[i]->turbo_dec.cb_params.k = turbo_dec->cb_params.k; 1470 } 1471 1472 ops[i]->turbo_dec.ext_scale = turbo_dec->ext_scale; 1473 ops[i]->turbo_dec.iter_max = turbo_dec->iter_max; 1474 ops[i]->turbo_dec.iter_min = turbo_dec->iter_min; 1475 ops[i]->turbo_dec.op_flags = turbo_dec->op_flags; 1476 ops[i]->turbo_dec.rv_index = turbo_dec->rv_index; 1477 ops[i]->turbo_dec.num_maps = turbo_dec->num_maps; 1478 ops[i]->turbo_dec.code_block_mode = turbo_dec->code_block_mode; 1479 1480 ops[i]->turbo_dec.hard_output = hard_outputs[start_idx + i]; 1481 ops[i]->turbo_dec.input = inputs[start_idx + i]; 1482 if (soft_outputs != NULL) 1483 ops[i]->turbo_dec.soft_output = 1484 soft_outputs[start_idx + i]; 1485 } 1486 } 1487 1488 static void 1489 copy_reference_enc_op(struct rte_bbdev_enc_op **ops, unsigned int n, 1490 unsigned int start_idx, 1491 struct rte_bbdev_op_data *inputs, 1492 struct rte_bbdev_op_data *outputs, 1493 struct rte_bbdev_enc_op *ref_op) 1494 { 1495 unsigned int i; 1496 struct rte_bbdev_op_turbo_enc *turbo_enc = &ref_op->turbo_enc; 1497 for (i = 0; i < n; ++i) { 1498 if (turbo_enc->code_block_mode == RTE_BBDEV_TRANSPORT_BLOCK) { 1499 ops[i]->turbo_enc.tb_params.ea = 1500 turbo_enc->tb_params.ea; 1501 ops[i]->turbo_enc.tb_params.eb = 1502 turbo_enc->tb_params.eb; 1503 ops[i]->turbo_enc.tb_params.k_pos = 1504 turbo_enc->tb_params.k_pos; 1505 ops[i]->turbo_enc.tb_params.k_neg = 1506 turbo_enc->tb_params.k_neg; 1507 ops[i]->turbo_enc.tb_params.c = 1508 turbo_enc->tb_params.c; 1509 ops[i]->turbo_enc.tb_params.c_neg = 1510 turbo_enc->tb_params.c_neg; 1511 ops[i]->turbo_enc.tb_params.cab = 1512 turbo_enc->tb_params.cab; 1513 ops[i]->turbo_enc.tb_params.ncb_pos = 1514 turbo_enc->tb_params.ncb_pos; 1515 ops[i]->turbo_enc.tb_params.ncb_neg = 1516 turbo_enc->tb_params.ncb_neg; 1517 ops[i]->turbo_enc.tb_params.r = turbo_enc->tb_params.r; 1518 } else { 1519 ops[i]->turbo_enc.cb_params.e = turbo_enc->cb_params.e; 1520 ops[i]->turbo_enc.cb_params.k = turbo_enc->cb_params.k; 1521 ops[i]->turbo_enc.cb_params.ncb = 1522 turbo_enc->cb_params.ncb; 1523 } 1524 ops[i]->turbo_enc.rv_index = turbo_enc->rv_index; 1525 ops[i]->turbo_enc.op_flags = turbo_enc->op_flags; 1526 ops[i]->turbo_enc.code_block_mode = turbo_enc->code_block_mode; 1527 1528 ops[i]->turbo_enc.output = outputs[start_idx + i]; 1529 ops[i]->turbo_enc.input = inputs[start_idx + i]; 1530 } 1531 } 1532 1533 1534 /* Returns a random number drawn from a normal distribution 1535 * with mean of 0 and variance of 1 1536 * Marsaglia algorithm 1537 */ 1538 static double 1539 randn(int n) 1540 { 1541 double S, Z, U1, U2, u, v, fac; 1542 1543 do { 1544 U1 = (double)rand() / RAND_MAX; 1545 U2 = (double)rand() / RAND_MAX; 1546 u = 2. * U1 - 1.; 1547 v = 2. * U2 - 1.; 1548 S = u * u + v * v; 1549 } while (S >= 1 || S == 0); 1550 fac = sqrt(-2. * log(S) / S); 1551 Z = (n % 2) ? u * fac : v * fac; 1552 return Z; 1553 } 1554 1555 static inline double 1556 maxstar(double A, double B) 1557 { 1558 if (fabs(A - B) > 5) 1559 return RTE_MAX(A, B); 1560 else 1561 return RTE_MAX(A, B) + log1p(exp(-fabs(A - B))); 1562 } 1563 1564 /* 1565 * Generate Qm LLRS for Qm==8 1566 * Modulation, AWGN and LLR estimation from max log development 1567 */ 1568 static void 1569 gen_qm8_llr(int8_t *llrs, uint32_t i, double N0, double llr_max) 1570 { 1571 int qm = 8; 1572 int qam = 256; 1573 int m, k; 1574 double I, Q, p0, p1, llr_, b[qm], log_syml_prob[qam]; 1575 /* 5.1.4 of TS38.211 */ 1576 const double symbols_I[256] = { 1577 5, 5, 7, 7, 5, 5, 7, 7, 3, 3, 1, 1, 3, 3, 1, 1, 5, 1578 5, 7, 7, 5, 5, 7, 7, 3, 3, 1, 1, 3, 3, 1, 1, 11, 1579 11, 9, 9, 11, 11, 9, 9, 13, 13, 15, 15, 13, 13, 1580 15, 15, 11, 11, 9, 9, 11, 11, 9, 9, 13, 13, 15, 1581 15, 13, 13, 15, 15, 5, 5, 7, 7, 5, 5, 7, 7, 3, 3, 1582 1, 1, 3, 3, 1, 1, 5, 5, 7, 7, 5, 5, 7, 7, 3, 3, 1, 1583 1, 3, 3, 1, 1, 11, 11, 9, 9, 11, 11, 9, 9, 13, 13, 1584 15, 15, 13, 13, 15, 15, 11, 11, 9, 9, 11, 11, 9, 9, 1585 13, 13, 15, 15, 13, 13, 15, 15, -5, -5, -7, -7, -5, 1586 -5, -7, -7, -3, -3, -1, -1, -3, -3, -1, -1, -5, -5, 1587 -7, -7, -5, -5, -7, -7, -3, -3, -1, -1, -3, -3, 1588 -1, -1, -11, -11, -9, -9, -11, -11, -9, -9, -13, 1589 -13, -15, -15, -13, -13, -15, -15, -11, -11, -9, 1590 -9, -11, -11, -9, -9, -13, -13, -15, -15, -13, 1591 -13, -15, -15, -5, -5, -7, -7, -5, -5, -7, -7, -3, 1592 -3, -1, -1, -3, -3, -1, -1, -5, -5, -7, -7, -5, -5, 1593 -7, -7, -3, -3, -1, -1, -3, -3, -1, -1, -11, -11, 1594 -9, -9, -11, -11, -9, -9, -13, -13, -15, -15, -13, 1595 -13, -15, -15, -11, -11, -9, -9, -11, -11, -9, -9, 1596 -13, -13, -15, -15, -13, -13, -15, -15}; 1597 const double symbols_Q[256] = { 1598 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 11, 1599 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9, 13, 15, 13, 1600 15, 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 1601 11, 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9, 13, 1602 15, 13, 15, -5, -7, -5, -7, -3, -1, -3, -1, -5, 1603 -7, -5, -7, -3, -1, -3, -1, -11, -9, -11, -9, -13, 1604 -15, -13, -15, -11, -9, -11, -9, -13, -15, -13, 1605 -15, -5, -7, -5, -7, -3, -1, -3, -1, -5, -7, -5, 1606 -7, -3, -1, -3, -1, -11, -9, -11, -9, -13, -15, 1607 -13, -15, -11, -9, -11, -9, -13, -15, -13, -15, 5, 1608 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 11, 1609 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9, 13, 15, 1610 13, 15, 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 1611 3, 1, 11, 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9, 1612 13, 15, 13, 15, -5, -7, -5, -7, -3, -1, -3, -1, 1613 -5, -7, -5, -7, -3, -1, -3, -1, -11, -9, -11, -9, 1614 -13, -15, -13, -15, -11, -9, -11, -9, -13, -15, 1615 -13, -15, -5, -7, -5, -7, -3, -1, -3, -1, -5, -7, 1616 -5, -7, -3, -1, -3, -1, -11, -9, -11, -9, -13, -15, 1617 -13, -15, -11, -9, -11, -9, -13, -15, -13, -15}; 1618 /* Average constellation point energy */ 1619 N0 *= 170.0; 1620 for (k = 0; k < qm; k++) 1621 b[k] = llrs[qm * i + k] < 0 ? 1.0 : 0.0; 1622 /* 5.1.4 of TS38.211 */ 1623 I = (1 - 2 * b[0]) * (8 - (1 - 2 * b[2]) * 1624 (4 - (1 - 2 * b[4]) * (2 - (1 - 2 * b[6])))); 1625 Q = (1 - 2 * b[1]) * (8 - (1 - 2 * b[3]) * 1626 (4 - (1 - 2 * b[5]) * (2 - (1 - 2 * b[7])))); 1627 /* AWGN channel */ 1628 I += sqrt(N0 / 2) * randn(0); 1629 Q += sqrt(N0 / 2) * randn(1); 1630 /* 1631 * Calculate the log of the probability that each of 1632 * the constellation points was transmitted 1633 */ 1634 for (m = 0; m < qam; m++) 1635 log_syml_prob[m] = -(pow(I - symbols_I[m], 2.0) 1636 + pow(Q - symbols_Q[m], 2.0)) / N0; 1637 /* Calculate an LLR for each of the k_64QAM bits in the set */ 1638 for (k = 0; k < qm; k++) { 1639 p0 = -999999; 1640 p1 = -999999; 1641 /* For each constellation point */ 1642 for (m = 0; m < qam; m++) { 1643 if ((m >> (qm - k - 1)) & 1) 1644 p1 = maxstar(p1, log_syml_prob[m]); 1645 else 1646 p0 = maxstar(p0, log_syml_prob[m]); 1647 } 1648 /* Calculate the LLR */ 1649 llr_ = p0 - p1; 1650 llr_ *= (1 << ldpc_llr_decimals); 1651 llr_ = round(llr_); 1652 if (llr_ > llr_max) 1653 llr_ = llr_max; 1654 if (llr_ < -llr_max) 1655 llr_ = -llr_max; 1656 llrs[qm * i + k] = (int8_t) llr_; 1657 } 1658 } 1659 1660 1661 /* 1662 * Generate Qm LLRS for Qm==6 1663 * Modulation, AWGN and LLR estimation from max log development 1664 */ 1665 static void 1666 gen_qm6_llr(int8_t *llrs, uint32_t i, double N0, double llr_max) 1667 { 1668 int qm = 6; 1669 int qam = 64; 1670 int m, k; 1671 double I, Q, p0, p1, llr_, b[qm], log_syml_prob[qam]; 1672 /* 5.1.4 of TS38.211 */ 1673 const double symbols_I[64] = { 1674 3, 3, 1, 1, 3, 3, 1, 1, 5, 5, 7, 7, 5, 5, 7, 7, 1675 3, 3, 1, 1, 3, 3, 1, 1, 5, 5, 7, 7, 5, 5, 7, 7, 1676 -3, -3, -1, -1, -3, -3, -1, -1, -5, -5, -7, -7, 1677 -5, -5, -7, -7, -3, -3, -1, -1, -3, -3, -1, -1, 1678 -5, -5, -7, -7, -5, -5, -7, -7}; 1679 const double symbols_Q[64] = { 1680 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 1681 -3, -1, -3, -1, -5, -7, -5, -7, -3, -1, -3, -1, 1682 -5, -7, -5, -7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 1683 5, 7, 5, 7, -3, -1, -3, -1, -5, -7, -5, -7, 1684 -3, -1, -3, -1, -5, -7, -5, -7}; 1685 /* Average constellation point energy */ 1686 N0 *= 42.0; 1687 for (k = 0; k < qm; k++) 1688 b[k] = llrs[qm * i + k] < 0 ? 1.0 : 0.0; 1689 /* 5.1.4 of TS38.211 */ 1690 I = (1 - 2 * b[0])*(4 - (1 - 2 * b[2]) * (2 - (1 - 2 * b[4]))); 1691 Q = (1 - 2 * b[1])*(4 - (1 - 2 * b[3]) * (2 - (1 - 2 * b[5]))); 1692 /* AWGN channel */ 1693 I += sqrt(N0 / 2) * randn(0); 1694 Q += sqrt(N0 / 2) * randn(1); 1695 /* 1696 * Calculate the log of the probability that each of 1697 * the constellation points was transmitted 1698 */ 1699 for (m = 0; m < qam; m++) 1700 log_syml_prob[m] = -(pow(I - symbols_I[m], 2.0) 1701 + pow(Q - symbols_Q[m], 2.0)) / N0; 1702 /* Calculate an LLR for each of the k_64QAM bits in the set */ 1703 for (k = 0; k < qm; k++) { 1704 p0 = -999999; 1705 p1 = -999999; 1706 /* For each constellation point */ 1707 for (m = 0; m < qam; m++) { 1708 if ((m >> (qm - k - 1)) & 1) 1709 p1 = maxstar(p1, log_syml_prob[m]); 1710 else 1711 p0 = maxstar(p0, log_syml_prob[m]); 1712 } 1713 /* Calculate the LLR */ 1714 llr_ = p0 - p1; 1715 llr_ *= (1 << ldpc_llr_decimals); 1716 llr_ = round(llr_); 1717 if (llr_ > llr_max) 1718 llr_ = llr_max; 1719 if (llr_ < -llr_max) 1720 llr_ = -llr_max; 1721 llrs[qm * i + k] = (int8_t) llr_; 1722 } 1723 } 1724 1725 /* 1726 * Generate Qm LLRS for Qm==4 1727 * Modulation, AWGN and LLR estimation from max log development 1728 */ 1729 static void 1730 gen_qm4_llr(int8_t *llrs, uint32_t i, double N0, double llr_max) 1731 { 1732 int qm = 4; 1733 int qam = 16; 1734 int m, k; 1735 double I, Q, p0, p1, llr_, b[qm], log_syml_prob[qam]; 1736 /* 5.1.4 of TS38.211 */ 1737 const double symbols_I[16] = {1, 1, 3, 3, 1, 1, 3, 3, 1738 -1, -1, -3, -3, -1, -1, -3, -3}; 1739 const double symbols_Q[16] = {1, 3, 1, 3, -1, -3, -1, -3, 1740 1, 3, 1, 3, -1, -3, -1, -3}; 1741 /* Average constellation point energy */ 1742 N0 *= 10.0; 1743 for (k = 0; k < qm; k++) 1744 b[k] = llrs[qm * i + k] < 0 ? 1.0 : 0.0; 1745 /* 5.1.4 of TS38.211 */ 1746 I = (1 - 2 * b[0]) * (2 - (1 - 2 * b[2])); 1747 Q = (1 - 2 * b[1]) * (2 - (1 - 2 * b[3])); 1748 /* AWGN channel */ 1749 I += sqrt(N0 / 2) * randn(0); 1750 Q += sqrt(N0 / 2) * randn(1); 1751 /* 1752 * Calculate the log of the probability that each of 1753 * the constellation points was transmitted 1754 */ 1755 for (m = 0; m < qam; m++) 1756 log_syml_prob[m] = -(pow(I - symbols_I[m], 2.0) 1757 + pow(Q - symbols_Q[m], 2.0)) / N0; 1758 /* Calculate an LLR for each of the k_64QAM bits in the set */ 1759 for (k = 0; k < qm; k++) { 1760 p0 = -999999; 1761 p1 = -999999; 1762 /* For each constellation point */ 1763 for (m = 0; m < qam; m++) { 1764 if ((m >> (qm - k - 1)) & 1) 1765 p1 = maxstar(p1, log_syml_prob[m]); 1766 else 1767 p0 = maxstar(p0, log_syml_prob[m]); 1768 } 1769 /* Calculate the LLR */ 1770 llr_ = p0 - p1; 1771 llr_ *= (1 << ldpc_llr_decimals); 1772 llr_ = round(llr_); 1773 if (llr_ > llr_max) 1774 llr_ = llr_max; 1775 if (llr_ < -llr_max) 1776 llr_ = -llr_max; 1777 llrs[qm * i + k] = (int8_t) llr_; 1778 } 1779 } 1780 1781 static void 1782 gen_qm2_llr(int8_t *llrs, uint32_t j, double N0, double llr_max) 1783 { 1784 double b, b1, n; 1785 double coeff = 2.0 * sqrt(N0); 1786 1787 /* Ignore in vectors rare quasi null LLRs not to be saturated */ 1788 if (llrs[j] < 8 && llrs[j] > -8) 1789 return; 1790 1791 /* Note don't change sign here */ 1792 n = randn(j % 2); 1793 b1 = ((llrs[j] > 0 ? 2.0 : -2.0) 1794 + coeff * n) / N0; 1795 b = b1 * (1 << ldpc_llr_decimals); 1796 b = round(b); 1797 if (b > llr_max) 1798 b = llr_max; 1799 if (b < -llr_max) 1800 b = -llr_max; 1801 llrs[j] = (int8_t) b; 1802 } 1803 1804 /* Simple LLR generation assuming AWGN and QPSK */ 1805 static void 1806 gen_turbo_llr(int8_t *llrs, uint32_t j, double N0, double llr_max) 1807 { 1808 double b, b1, n; 1809 double coeff = 2.0 * sqrt(N0); 1810 1811 /* Ignore in vectors null LLRs not to be saturated */ 1812 if (llrs[j] == 0) 1813 return; 1814 1815 /* Note don't change sign here */ 1816 n = randn(j % 2); 1817 b1 = ((llrs[j] > 0 ? 2.0 : -2.0) 1818 + coeff * n) / N0; 1819 b = b1 * (1 << 4); 1820 b = round(b); 1821 if (b > llr_max) 1822 b = llr_max; 1823 if (b < -llr_max) 1824 b = -llr_max; 1825 llrs[j] = (int8_t) b; 1826 } 1827 1828 /* Generate LLR for a given SNR */ 1829 static void 1830 generate_llr_input(uint16_t n, struct rte_bbdev_op_data *inputs, 1831 struct rte_bbdev_dec_op *ref_op) 1832 { 1833 struct rte_mbuf *m; 1834 uint16_t qm; 1835 uint32_t i, j, e, range; 1836 double N0, llr_max; 1837 1838 e = ref_op->ldpc_dec.cb_params.e; 1839 qm = ref_op->ldpc_dec.q_m; 1840 llr_max = (1 << (ldpc_llr_size - 1)) - 1; 1841 range = e / qm; 1842 N0 = 1.0 / pow(10.0, get_snr() / 10.0); 1843 1844 for (i = 0; i < n; ++i) { 1845 m = inputs[i].data; 1846 int8_t *llrs = rte_pktmbuf_mtod_offset(m, int8_t *, 0); 1847 if (qm == 8) { 1848 for (j = 0; j < range; ++j) 1849 gen_qm8_llr(llrs, j, N0, llr_max); 1850 } else if (qm == 6) { 1851 for (j = 0; j < range; ++j) 1852 gen_qm6_llr(llrs, j, N0, llr_max); 1853 } else if (qm == 4) { 1854 for (j = 0; j < range; ++j) 1855 gen_qm4_llr(llrs, j, N0, llr_max); 1856 } else { 1857 for (j = 0; j < e; ++j) 1858 gen_qm2_llr(llrs, j, N0, llr_max); 1859 } 1860 } 1861 } 1862 1863 /* Generate LLR for turbo decoder for a given SNR */ 1864 static void 1865 generate_turbo_llr_input(uint16_t n, struct rte_bbdev_op_data *inputs, 1866 struct rte_bbdev_dec_op *ref_op) 1867 { 1868 struct rte_mbuf *m; 1869 uint32_t i, j, range; 1870 double N0, llr_max; 1871 1872 llr_max = 127; 1873 range = ref_op->turbo_dec.input.length; 1874 N0 = 1.0 / pow(10.0, get_snr() / 10.0); 1875 1876 if (range > inputs[0].data->data_len) { 1877 printf("Warning: Limiting LLR generation to first segment (%d from %d)\n", 1878 inputs[0].data->data_len, range); 1879 range = inputs[0].data->data_len; 1880 } 1881 1882 for (i = 0; i < n; ++i) { 1883 m = inputs[i].data; 1884 int8_t *llrs = rte_pktmbuf_mtod_offset(m, int8_t *, 0); 1885 for (j = 0; j < range; ++j) 1886 gen_turbo_llr(llrs, j, N0, llr_max); 1887 } 1888 } 1889 1890 static void 1891 copy_reference_ldpc_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n, 1892 unsigned int start_idx, 1893 struct rte_bbdev_op_data *inputs, 1894 struct rte_bbdev_op_data *hard_outputs, 1895 struct rte_bbdev_op_data *soft_outputs, 1896 struct rte_bbdev_op_data *harq_inputs, 1897 struct rte_bbdev_op_data *harq_outputs, 1898 struct rte_bbdev_dec_op *ref_op) 1899 { 1900 unsigned int i; 1901 struct rte_bbdev_op_ldpc_dec *ldpc_dec = &ref_op->ldpc_dec; 1902 1903 for (i = 0; i < n; ++i) { 1904 if (ldpc_dec->code_block_mode == RTE_BBDEV_TRANSPORT_BLOCK) { 1905 ops[i]->ldpc_dec.tb_params.ea = 1906 ldpc_dec->tb_params.ea; 1907 ops[i]->ldpc_dec.tb_params.eb = 1908 ldpc_dec->tb_params.eb; 1909 ops[i]->ldpc_dec.tb_params.c = 1910 ldpc_dec->tb_params.c; 1911 ops[i]->ldpc_dec.tb_params.cab = 1912 ldpc_dec->tb_params.cab; 1913 ops[i]->ldpc_dec.tb_params.r = 1914 ldpc_dec->tb_params.r; 1915 } else { 1916 ops[i]->ldpc_dec.cb_params.e = ldpc_dec->cb_params.e; 1917 } 1918 1919 ops[i]->ldpc_dec.basegraph = ldpc_dec->basegraph; 1920 ops[i]->ldpc_dec.z_c = ldpc_dec->z_c; 1921 ops[i]->ldpc_dec.q_m = ldpc_dec->q_m; 1922 ops[i]->ldpc_dec.n_filler = ldpc_dec->n_filler; 1923 ops[i]->ldpc_dec.n_cb = ldpc_dec->n_cb; 1924 ops[i]->ldpc_dec.iter_max = ldpc_dec->iter_max; 1925 ops[i]->ldpc_dec.rv_index = ldpc_dec->rv_index; 1926 ops[i]->ldpc_dec.k0 = ldpc_dec->k0; 1927 ops[i]->ldpc_dec.op_flags = ldpc_dec->op_flags; 1928 ops[i]->ldpc_dec.code_block_mode = ldpc_dec->code_block_mode; 1929 1930 if (hard_outputs != NULL) 1931 ops[i]->ldpc_dec.hard_output = 1932 hard_outputs[start_idx + i]; 1933 if (inputs != NULL) 1934 ops[i]->ldpc_dec.input = 1935 inputs[start_idx + i]; 1936 if (soft_outputs != NULL) 1937 ops[i]->ldpc_dec.soft_output = 1938 soft_outputs[start_idx + i]; 1939 if (harq_inputs != NULL) 1940 ops[i]->ldpc_dec.harq_combined_input = 1941 harq_inputs[start_idx + i]; 1942 if (harq_outputs != NULL) 1943 ops[i]->ldpc_dec.harq_combined_output = 1944 harq_outputs[start_idx + i]; 1945 } 1946 } 1947 1948 1949 static void 1950 copy_reference_ldpc_enc_op(struct rte_bbdev_enc_op **ops, unsigned int n, 1951 unsigned int start_idx, 1952 struct rte_bbdev_op_data *inputs, 1953 struct rte_bbdev_op_data *outputs, 1954 struct rte_bbdev_enc_op *ref_op) 1955 { 1956 unsigned int i; 1957 struct rte_bbdev_op_ldpc_enc *ldpc_enc = &ref_op->ldpc_enc; 1958 for (i = 0; i < n; ++i) { 1959 if (ldpc_enc->code_block_mode == RTE_BBDEV_TRANSPORT_BLOCK) { 1960 ops[i]->ldpc_enc.tb_params.ea = ldpc_enc->tb_params.ea; 1961 ops[i]->ldpc_enc.tb_params.eb = ldpc_enc->tb_params.eb; 1962 ops[i]->ldpc_enc.tb_params.cab = 1963 ldpc_enc->tb_params.cab; 1964 ops[i]->ldpc_enc.tb_params.c = ldpc_enc->tb_params.c; 1965 ops[i]->ldpc_enc.tb_params.r = ldpc_enc->tb_params.r; 1966 } else { 1967 ops[i]->ldpc_enc.cb_params.e = ldpc_enc->cb_params.e; 1968 } 1969 ops[i]->ldpc_enc.basegraph = ldpc_enc->basegraph; 1970 ops[i]->ldpc_enc.z_c = ldpc_enc->z_c; 1971 ops[i]->ldpc_enc.q_m = ldpc_enc->q_m; 1972 ops[i]->ldpc_enc.n_filler = ldpc_enc->n_filler; 1973 ops[i]->ldpc_enc.n_cb = ldpc_enc->n_cb; 1974 ops[i]->ldpc_enc.rv_index = ldpc_enc->rv_index; 1975 ops[i]->ldpc_enc.op_flags = ldpc_enc->op_flags; 1976 ops[i]->ldpc_enc.code_block_mode = ldpc_enc->code_block_mode; 1977 ops[i]->ldpc_enc.output = outputs[start_idx + i]; 1978 ops[i]->ldpc_enc.input = inputs[start_idx + i]; 1979 } 1980 } 1981 1982 static void 1983 copy_reference_fft_op(struct rte_bbdev_fft_op **ops, unsigned int n, 1984 unsigned int start_idx, struct rte_bbdev_op_data *inputs, 1985 struct rte_bbdev_op_data *outputs, struct rte_bbdev_op_data *pwrouts, 1986 struct rte_bbdev_op_data *win_inputs, struct rte_bbdev_fft_op *ref_op) 1987 { 1988 unsigned int i, j; 1989 struct rte_bbdev_op_fft *fft = &ref_op->fft; 1990 for (i = 0; i < n; i++) { 1991 ops[i]->fft.input_sequence_size = fft->input_sequence_size; 1992 ops[i]->fft.input_leading_padding = fft->input_leading_padding; 1993 ops[i]->fft.output_sequence_size = fft->output_sequence_size; 1994 ops[i]->fft.output_leading_depadding = 1995 fft->output_leading_depadding; 1996 for (j = 0; j < RTE_BBDEV_MAX_CS_2; j++) 1997 ops[i]->fft.window_index[j] = fft->window_index[j]; 1998 for (j = 0; j < RTE_BBDEV_MAX_CS; j++) { 1999 ops[i]->fft.cs_theta_0[j] = fft->cs_theta_0[j]; 2000 ops[i]->fft.cs_theta_d[j] = fft->cs_theta_d[j]; 2001 ops[i]->fft.time_offset[j] = fft->time_offset[j]; 2002 } 2003 ops[i]->fft.cs_bitmap = fft->cs_bitmap; 2004 ops[i]->fft.num_antennas_log2 = fft->num_antennas_log2; 2005 ops[i]->fft.idft_log2 = fft->idft_log2; 2006 ops[i]->fft.dft_log2 = fft->dft_log2; 2007 ops[i]->fft.cs_time_adjustment = fft->cs_time_adjustment; 2008 ops[i]->fft.idft_shift = fft->idft_shift; 2009 ops[i]->fft.dft_shift = fft->dft_shift; 2010 ops[i]->fft.ncs_reciprocal = fft->ncs_reciprocal; 2011 ops[i]->fft.power_shift = fft->power_shift; 2012 ops[i]->fft.fp16_exp_adjust = fft->fp16_exp_adjust; 2013 ops[i]->fft.output_depadded_size = fft->output_depadded_size; 2014 ops[i]->fft.freq_resample_mode = fft->freq_resample_mode; 2015 ops[i]->fft.base_output = outputs[start_idx + i]; 2016 ops[i]->fft.base_input = inputs[start_idx + i]; 2017 if (win_inputs != NULL) 2018 ops[i]->fft.dewindowing_input = win_inputs[start_idx + i]; 2019 if (pwrouts != NULL) 2020 ops[i]->fft.power_meas_output = pwrouts[start_idx + i]; 2021 ops[i]->fft.op_flags = fft->op_flags; 2022 } 2023 } 2024 2025 static void 2026 copy_reference_mldts_op(struct rte_bbdev_mldts_op **ops, unsigned int n, 2027 unsigned int start_idx, 2028 struct rte_bbdev_op_data *q_inputs, 2029 struct rte_bbdev_op_data *r_inputs, 2030 struct rte_bbdev_op_data *outputs, 2031 struct rte_bbdev_mldts_op *ref_op) 2032 { 2033 unsigned int i, j; 2034 struct rte_bbdev_op_mldts *mldts = &ref_op->mldts; 2035 for (i = 0; i < n; i++) { 2036 ops[i]->mldts.c_rep = mldts->c_rep; 2037 ops[i]->mldts.num_layers = mldts->num_layers; 2038 ops[i]->mldts.num_rbs = mldts->num_rbs; 2039 ops[i]->mldts.op_flags = mldts->op_flags; 2040 for (j = 0; j < RTE_BBDEV_MAX_MLD_LAYERS; j++) 2041 ops[i]->mldts.q_m[j] = mldts->q_m[j]; 2042 ops[i]->mldts.r_rep = mldts->r_rep; 2043 ops[i]->mldts.c_rep = mldts->c_rep; 2044 ops[i]->mldts.r_input = r_inputs[start_idx + i]; 2045 ops[i]->mldts.qhy_input = q_inputs[start_idx + i]; 2046 ops[i]->mldts.output = outputs[start_idx + i]; 2047 } 2048 } 2049 2050 static int 2051 check_dec_status_and_ordering(struct rte_bbdev_dec_op *op, 2052 unsigned int order_idx, const int expected_status) 2053 { 2054 int status = op->status; 2055 /* ignore parity mismatch false alarms for long iterations */ 2056 if (get_iter_max() >= 10) { 2057 if (!(expected_status & (1 << RTE_BBDEV_SYNDROME_ERROR)) && 2058 (status & (1 << RTE_BBDEV_SYNDROME_ERROR))) { 2059 printf("WARNING: Ignore Syndrome Check mismatch\n"); 2060 status -= (1 << RTE_BBDEV_SYNDROME_ERROR); 2061 } 2062 if ((expected_status & (1 << RTE_BBDEV_SYNDROME_ERROR)) && 2063 !(status & (1 << RTE_BBDEV_SYNDROME_ERROR))) { 2064 printf("WARNING: Ignore Syndrome Check mismatch\n"); 2065 status += (1 << RTE_BBDEV_SYNDROME_ERROR); 2066 } 2067 } 2068 2069 TEST_ASSERT(status == expected_status, 2070 "op_status (%d) != expected_status (%d)", 2071 op->status, expected_status); 2072 2073 TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data, 2074 "Ordering error, expected %p, got %p", 2075 (void *)(uintptr_t)order_idx, op->opaque_data); 2076 2077 return TEST_SUCCESS; 2078 } 2079 2080 static int 2081 check_enc_status_and_ordering(struct rte_bbdev_enc_op *op, 2082 unsigned int order_idx, const int expected_status) 2083 { 2084 TEST_ASSERT(op->status == expected_status, 2085 "op_status (%d) != expected_status (%d)", 2086 op->status, expected_status); 2087 2088 TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data, 2089 "Ordering error, expected %p, got %p", 2090 (void *)(uintptr_t)order_idx, op->opaque_data); 2091 2092 return TEST_SUCCESS; 2093 } 2094 2095 static int 2096 check_fft_status_and_ordering(struct rte_bbdev_fft_op *op, 2097 unsigned int order_idx, const int expected_status) 2098 { 2099 TEST_ASSERT(op->status == expected_status, 2100 "op_status (%d) != expected_status (%d)", 2101 op->status, expected_status); 2102 2103 TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data, 2104 "Ordering error, expected %p, got %p", 2105 (void *)(uintptr_t)order_idx, op->opaque_data); 2106 2107 return TEST_SUCCESS; 2108 } 2109 2110 static int 2111 check_mldts_status_and_ordering(struct rte_bbdev_mldts_op *op, 2112 unsigned int order_idx, const int expected_status) 2113 { 2114 TEST_ASSERT(op->status == expected_status, 2115 "op_status (%d) != expected_status (%d)", 2116 op->status, expected_status); 2117 2118 TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data, 2119 "Ordering error, expected %p, got %p", 2120 (void *)(uintptr_t)order_idx, op->opaque_data); 2121 2122 return TEST_SUCCESS; 2123 } 2124 2125 static inline int 2126 validate_op_chain(struct rte_bbdev_op_data *op, 2127 struct op_data_entries *orig_op) 2128 { 2129 uint8_t i; 2130 struct rte_mbuf *m = op->data; 2131 uint8_t nb_dst_segments = orig_op->nb_segments; 2132 uint32_t total_data_size = 0; 2133 bool ignore_mbuf = false; /* ignore mbuf limitations */ 2134 2135 TEST_ASSERT(nb_dst_segments == m->nb_segs, 2136 "Number of segments differ in original (%u) and filled (%u) op", 2137 nb_dst_segments, m->nb_segs); 2138 2139 /* Validate each mbuf segment length */ 2140 for (i = 0; i < nb_dst_segments; ++i) { 2141 /* Apply offset to the first mbuf segment */ 2142 uint16_t offset = (i == 0) ? op->offset : 0; 2143 uint16_t data_len = rte_pktmbuf_data_len(m) - offset; 2144 total_data_size += orig_op->segments[i].length; 2145 2146 if ((orig_op->segments[i].length + RTE_PKTMBUF_HEADROOM) 2147 > RTE_BBDEV_LDPC_E_MAX_MBUF) 2148 ignore_mbuf = true; 2149 if (!ignore_mbuf) 2150 TEST_ASSERT(orig_op->segments[i].length == data_len, 2151 "Length of segment differ in original (%u) and filled (%u) op", 2152 orig_op->segments[i].length, data_len); 2153 TEST_ASSERT_BUFFERS_ARE_EQUAL(orig_op->segments[i].addr, 2154 rte_pktmbuf_mtod_offset(m, uint32_t *, offset), 2155 orig_op->segments[i].length, 2156 "Output buffers (CB=%u) are not equal", i); 2157 m = m->next; 2158 } 2159 2160 /* Validate total mbuf pkt length */ 2161 uint32_t pkt_len = rte_pktmbuf_pkt_len(op->data) - op->offset; 2162 if (!ignore_mbuf) 2163 TEST_ASSERT(total_data_size == pkt_len, 2164 "Length of data differ in original (%u) and filled (%u) op", 2165 total_data_size, pkt_len); 2166 2167 return TEST_SUCCESS; 2168 } 2169 2170 /* 2171 * Compute K0 for a given configuration for HARQ output length computation 2172 * As per definition in 3GPP 38.212 Table 5.4.2.1-2 2173 */ 2174 static inline uint16_t 2175 get_k0(uint16_t n_cb, uint16_t z_c, uint8_t bg, uint8_t rv_index, uint16_t k0) 2176 { 2177 if (k0 > 0) 2178 return k0; 2179 if (rv_index == 0) 2180 return 0; 2181 uint16_t n = (bg == 1 ? N_ZC_1 : N_ZC_2) * z_c; 2182 if (n_cb == n) { 2183 if (rv_index == 1) 2184 return (bg == 1 ? K0_1_1 : K0_1_2) * z_c; 2185 else if (rv_index == 2) 2186 return (bg == 1 ? K0_2_1 : K0_2_2) * z_c; 2187 else 2188 return (bg == 1 ? K0_3_1 : K0_3_2) * z_c; 2189 } 2190 /* LBRM case - includes a division by N */ 2191 if (rv_index == 1) 2192 return (((bg == 1 ? K0_1_1 : K0_1_2) * n_cb) 2193 / n) * z_c; 2194 else if (rv_index == 2) 2195 return (((bg == 1 ? K0_2_1 : K0_2_2) * n_cb) 2196 / n) * z_c; 2197 else 2198 return (((bg == 1 ? K0_3_1 : K0_3_2) * n_cb) 2199 / n) * z_c; 2200 } 2201 2202 /* HARQ output length including the Filler bits */ 2203 static inline uint16_t 2204 compute_harq_len(struct rte_bbdev_op_ldpc_dec *ops_ld) 2205 { 2206 uint16_t k0 = 0; 2207 uint8_t max_rv = (ops_ld->rv_index == 1) ? 3 : ops_ld->rv_index; 2208 k0 = get_k0(ops_ld->n_cb, ops_ld->z_c, ops_ld->basegraph, max_rv, ops_ld->k0); 2209 /* Compute RM out size and number of rows */ 2210 uint16_t parity_offset = (ops_ld->basegraph == 1 ? 20 : 8) 2211 * ops_ld->z_c - ops_ld->n_filler; 2212 uint16_t deRmOutSize = RTE_MIN( 2213 k0 + ops_ld->cb_params.e + 2214 ((k0 > parity_offset) ? 2215 0 : ops_ld->n_filler), 2216 ops_ld->n_cb); 2217 uint16_t numRows = ((deRmOutSize + ops_ld->z_c - 1) 2218 / ops_ld->z_c); 2219 uint16_t harq_output_len = numRows * ops_ld->z_c; 2220 return harq_output_len; 2221 } 2222 2223 static inline int 2224 validate_op_harq_chain(struct rte_bbdev_op_data *op, 2225 struct op_data_entries *orig_op, 2226 struct rte_bbdev_op_ldpc_dec *ops_ld) 2227 { 2228 uint8_t i; 2229 uint32_t j, jj, k; 2230 struct rte_mbuf *m = op->data; 2231 uint8_t nb_dst_segments = orig_op->nb_segments; 2232 uint32_t total_data_size = 0; 2233 int8_t *harq_orig, *harq_out, abs_harq_origin; 2234 uint32_t byte_error = 0, cum_error = 0, error; 2235 int16_t llr_max = (1 << (ldpc_llr_size - ldpc_llr_decimals)) - 1; 2236 int16_t llr_max_pre_scaling = (1 << (ldpc_llr_size - 1)) - 1; 2237 uint16_t parity_offset; 2238 2239 TEST_ASSERT(nb_dst_segments == m->nb_segs, 2240 "Number of segments differ in original (%u) and filled (%u) op", 2241 nb_dst_segments, m->nb_segs); 2242 2243 /* Validate each mbuf segment length */ 2244 for (i = 0; i < nb_dst_segments; ++i) { 2245 /* Apply offset to the first mbuf segment */ 2246 uint16_t offset = (i == 0) ? op->offset : 0; 2247 uint16_t data_len = rte_pktmbuf_data_len(m) - offset; 2248 total_data_size += orig_op->segments[i].length; 2249 2250 TEST_ASSERT(orig_op->segments[i].length < (uint32_t)(data_len + HARQ_MEM_TOLERANCE), 2251 "Length of segment differ in original (%u) and filled (%u) op", 2252 orig_op->segments[i].length, data_len); 2253 harq_orig = (int8_t *) orig_op->segments[i].addr; 2254 harq_out = rte_pktmbuf_mtod_offset(m, int8_t *, offset); 2255 2256 /* Cannot compare HARQ output data for such cases */ 2257 if ((ldpc_llr_decimals > 1) && ((ops_ld->op_flags & RTE_BBDEV_LDPC_LLR_COMPRESSION) 2258 || (ops_ld->op_flags & RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION) 2259 || (ops_ld->op_flags & RTE_BBDEV_LDPC_HARQ_4BIT_COMPRESSION))) 2260 break; 2261 2262 if (!(ldpc_cap_flags & 2263 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_FILLERS 2264 ) || (ops_ld->op_flags & 2265 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)) { 2266 data_len -= ops_ld->z_c; 2267 parity_offset = data_len; 2268 } else { 2269 /* Compute RM out size and number of rows */ 2270 parity_offset = (ops_ld->basegraph == 1 ? 20 : 8) 2271 * ops_ld->z_c - ops_ld->n_filler; 2272 uint16_t deRmOutSize = compute_harq_len(ops_ld) - 2273 ops_ld->n_filler; 2274 if (data_len > deRmOutSize) 2275 data_len = deRmOutSize; 2276 } 2277 if (data_len > orig_op->segments[i].length) 2278 data_len = orig_op->segments[i].length; 2279 /* 2280 * HARQ output can have minor differences 2281 * due to integer representation and related scaling 2282 */ 2283 for (j = 0, jj = 0; j < data_len; j++, jj++) { 2284 if (j == parity_offset) { 2285 /* Special Handling of the filler bits */ 2286 for (k = 0; k < ops_ld->n_filler; k++) { 2287 if (harq_out[jj] != 2288 llr_max_pre_scaling) { 2289 printf("HARQ Filler issue %d: %d %d\n", 2290 jj, harq_out[jj], 2291 llr_max); 2292 byte_error++; 2293 } 2294 jj++; 2295 } 2296 } 2297 if (!(ops_ld->op_flags & 2298 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)) { 2299 if (ldpc_llr_decimals > 1) 2300 harq_out[jj] = (harq_out[jj] + 1) 2301 >> (ldpc_llr_decimals - 1); 2302 /* Saturated to S7 */ 2303 if (harq_orig[j] > llr_max) 2304 harq_orig[j] = llr_max; 2305 if (harq_orig[j] < -llr_max) 2306 harq_orig[j] = -llr_max; 2307 } 2308 if (harq_orig[j] != harq_out[jj]) { 2309 error = (harq_orig[j] > harq_out[jj]) ? 2310 harq_orig[j] - harq_out[jj] : 2311 harq_out[jj] - harq_orig[j]; 2312 abs_harq_origin = harq_orig[j] > 0 ? 2313 harq_orig[j] : 2314 -harq_orig[j]; 2315 /* Residual quantization error */ 2316 if ((error > 8 && (abs_harq_origin < 2317 (llr_max - 16))) || 2318 (error > 16)) { 2319 /* 2320 printf("HARQ mismatch %d: exp %d act %d => %d\n", 2321 j, harq_orig[j], 2322 harq_out[jj], error); 2323 */ 2324 byte_error++; 2325 cum_error += error; 2326 } 2327 } 2328 } 2329 m = m->next; 2330 } 2331 2332 if (byte_error) 2333 TEST_ASSERT(byte_error <= 1, 2334 "HARQ output mismatch (%d) %d", 2335 byte_error, cum_error); 2336 2337 /* Validate total mbuf pkt length */ 2338 uint32_t pkt_len = rte_pktmbuf_pkt_len(op->data) - op->offset; 2339 TEST_ASSERT(total_data_size < pkt_len + HARQ_MEM_TOLERANCE, 2340 "Length of data differ in original (%u) and filled (%u) op", 2341 total_data_size, pkt_len); 2342 2343 return TEST_SUCCESS; 2344 } 2345 2346 2347 static inline int 2348 validate_op_so_chain(struct rte_bbdev_op_data *op, 2349 struct op_data_entries *orig_op) 2350 { 2351 struct rte_mbuf *m = op->data; 2352 uint8_t i, nb_dst_segments = orig_op->nb_segments; 2353 uint32_t j, jj; 2354 int8_t *so_orig, *so_out; 2355 uint32_t byte_error = 0, error, margin_error = 0; 2356 2357 TEST_ASSERT(nb_dst_segments == m->nb_segs, 2358 "Number of segments differ in original (%u) and filled (%u) op", 2359 nb_dst_segments, m->nb_segs); 2360 2361 /* Validate each mbuf segment length. */ 2362 for (i = 0; i < nb_dst_segments; ++i) { 2363 /* Apply offset to the first mbuf segment. */ 2364 uint16_t offset = (i == 0) ? op->offset : 0; 2365 uint16_t data_len = rte_pktmbuf_data_len(m) - offset; 2366 2367 TEST_ASSERT(orig_op->segments[i].length == data_len, 2368 "Length of segment differ in original (%u) and filled (%u) op", 2369 orig_op->segments[i].length, data_len); 2370 so_orig = (int8_t *) orig_op->segments[i].addr; 2371 so_out = rte_pktmbuf_mtod_offset(m, int8_t *, offset); 2372 margin_error += data_len / 8; /* Allow for few % errors. */ 2373 2374 /* SO output can have minor differences due to algorithm variations. */ 2375 for (j = 0, jj = 0; j < data_len; j++, jj++) { 2376 if (so_orig[j] != so_out[jj]) { 2377 error = (so_orig[j] > so_out[jj]) ? so_orig[j] - so_out[jj] : 2378 so_out[jj] - so_orig[j]; 2379 /* Residual quantization error. */ 2380 if (error > 32) { 2381 printf("Warning: Soft mismatch %d: exp %d act %d => %d\n", 2382 j, so_orig[j], so_out[jj], error); 2383 byte_error++; 2384 } 2385 } 2386 } 2387 m = m->next; 2388 } 2389 2390 if (byte_error > margin_error) 2391 TEST_ASSERT(byte_error <= 1, "Soft output mismatch (%d) %d", 2392 byte_error, margin_error); 2393 2394 return TEST_SUCCESS; 2395 } 2396 2397 static int 2398 validate_dec_op(struct rte_bbdev_dec_op **ops, const uint16_t n, 2399 struct rte_bbdev_dec_op *ref_op) 2400 { 2401 unsigned int i; 2402 int ret; 2403 struct op_data_entries *hard_data_orig = 2404 &test_vector.entries[DATA_HARD_OUTPUT]; 2405 struct op_data_entries *soft_data_orig = 2406 &test_vector.entries[DATA_SOFT_OUTPUT]; 2407 struct rte_bbdev_op_turbo_dec *ops_td; 2408 struct rte_bbdev_op_data *hard_output; 2409 struct rte_bbdev_op_data *soft_output; 2410 2411 for (i = 0; i < n; ++i) { 2412 ops_td = &ops[i]->turbo_dec; 2413 hard_output = &ops_td->hard_output; 2414 soft_output = &ops_td->soft_output; 2415 2416 ret = check_dec_status_and_ordering(ops[i], i, ref_op->status); 2417 TEST_ASSERT_SUCCESS(ret, 2418 "Checking status and ordering for decoder failed"); 2419 2420 TEST_ASSERT_SUCCESS(validate_op_chain(hard_output, 2421 hard_data_orig), 2422 "Hard output buffers (CB=%u) are not equal", 2423 i); 2424 2425 if (ref_op->turbo_dec.op_flags & RTE_BBDEV_TURBO_SOFT_OUTPUT) 2426 TEST_ASSERT_SUCCESS(validate_op_so_chain(soft_output, 2427 soft_data_orig), 2428 "Soft output buffers (CB=%u) are not equal", 2429 i); 2430 } 2431 2432 return TEST_SUCCESS; 2433 } 2434 2435 /* Check Number of code blocks errors */ 2436 static int 2437 validate_ldpc_bler(struct rte_bbdev_dec_op **ops, const uint16_t n) 2438 { 2439 unsigned int i; 2440 struct op_data_entries *hard_data_orig = 2441 &test_vector.entries[DATA_HARD_OUTPUT]; 2442 struct rte_bbdev_op_ldpc_dec *ops_td; 2443 struct rte_bbdev_op_data *hard_output; 2444 int errors = 0; 2445 struct rte_mbuf *m; 2446 2447 for (i = 0; i < n; ++i) { 2448 ops_td = &ops[i]->ldpc_dec; 2449 hard_output = &ops_td->hard_output; 2450 m = hard_output->data; 2451 if (memcmp(rte_pktmbuf_mtod_offset(m, uint32_t *, 0), 2452 hard_data_orig->segments[0].addr, 2453 hard_data_orig->segments[0].length)) 2454 errors++; 2455 } 2456 return errors; 2457 } 2458 2459 /* Check Number of code blocks errors */ 2460 static int 2461 validate_turbo_bler(struct rte_bbdev_dec_op **ops, const uint16_t n) 2462 { 2463 unsigned int i; 2464 struct op_data_entries *hard_data_orig = &test_vector.entries[DATA_HARD_OUTPUT]; 2465 struct rte_bbdev_op_turbo_dec *ops_td; 2466 struct rte_bbdev_op_data *hard_output; 2467 int errors = 0; 2468 struct rte_mbuf *m; 2469 2470 for (i = 0; i < n; ++i) { 2471 ops_td = &ops[i]->turbo_dec; 2472 hard_output = &ops_td->hard_output; 2473 m = hard_output->data; 2474 if (memcmp(rte_pktmbuf_mtod_offset(m, uint32_t *, 0), 2475 hard_data_orig->segments[0].addr, 2476 hard_data_orig->segments[0].length)) 2477 errors++; 2478 } 2479 return errors; 2480 } 2481 2482 2483 static int 2484 validate_ldpc_dec_op(struct rte_bbdev_dec_op **ops, const uint16_t n, 2485 struct rte_bbdev_dec_op *ref_op, const int vector_mask) 2486 { 2487 unsigned int i; 2488 int ret; 2489 struct op_data_entries *hard_data_orig = 2490 &test_vector.entries[DATA_HARD_OUTPUT]; 2491 struct op_data_entries *soft_data_orig = 2492 &test_vector.entries[DATA_SOFT_OUTPUT]; 2493 struct op_data_entries *harq_data_orig = 2494 &test_vector.entries[DATA_HARQ_OUTPUT]; 2495 struct rte_bbdev_op_ldpc_dec *ops_td; 2496 struct rte_bbdev_op_data *hard_output; 2497 struct rte_bbdev_op_data *harq_output; 2498 struct rte_bbdev_op_data *soft_output; 2499 struct rte_bbdev_op_ldpc_dec *ref_td = &ref_op->ldpc_dec; 2500 2501 for (i = 0; i < n; ++i) { 2502 ops_td = &ops[i]->ldpc_dec; 2503 hard_output = &ops_td->hard_output; 2504 harq_output = &ops_td->harq_combined_output; 2505 soft_output = &ops_td->soft_output; 2506 2507 ret = check_dec_status_and_ordering(ops[i], i, ref_op->status); 2508 TEST_ASSERT_SUCCESS(ret, 2509 "Checking status and ordering for decoder failed"); 2510 if (vector_mask & TEST_BBDEV_VF_EXPECTED_ITER_COUNT) 2511 TEST_ASSERT(ops_td->iter_count <= ref_td->iter_count, 2512 "Returned iter_count (%d) > expected iter_count (%d)", 2513 ops_td->iter_count, ref_td->iter_count); 2514 /* 2515 * We can ignore output data when the decoding failed to 2516 * converge or for loop-back cases 2517 */ 2518 if (!check_bit(ops[i]->ldpc_dec.op_flags, 2519 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK 2520 ) && ( 2521 ops[i]->status & (1 << RTE_BBDEV_SYNDROME_ERROR 2522 )) == 0) 2523 TEST_ASSERT_SUCCESS(validate_op_chain(hard_output, 2524 hard_data_orig), 2525 "Hard output buffers (CB=%u) are not equal", 2526 i); 2527 2528 if (ref_op->ldpc_dec.op_flags & RTE_BBDEV_LDPC_SOFT_OUT_ENABLE) 2529 TEST_ASSERT_SUCCESS(validate_op_so_chain(soft_output, 2530 soft_data_orig), 2531 "Soft output buffers (CB=%u) are not equal", 2532 i); 2533 if (ref_op->ldpc_dec.op_flags & 2534 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE) { 2535 TEST_ASSERT_SUCCESS(validate_op_harq_chain(harq_output, 2536 harq_data_orig, ops_td), 2537 "HARQ output buffers (CB=%u) are not equal", 2538 i); 2539 } 2540 if (ref_op->ldpc_dec.op_flags & 2541 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK) 2542 TEST_ASSERT_SUCCESS(validate_op_harq_chain(harq_output, 2543 harq_data_orig, ops_td), 2544 "HARQ output buffers (CB=%u) are not equal", 2545 i); 2546 2547 } 2548 2549 return TEST_SUCCESS; 2550 } 2551 2552 2553 static int 2554 validate_enc_op(struct rte_bbdev_enc_op **ops, const uint16_t n, 2555 struct rte_bbdev_enc_op *ref_op) 2556 { 2557 unsigned int i; 2558 int ret; 2559 struct op_data_entries *hard_data_orig = 2560 &test_vector.entries[DATA_HARD_OUTPUT]; 2561 2562 for (i = 0; i < n; ++i) { 2563 ret = check_enc_status_and_ordering(ops[i], i, ref_op->status); 2564 TEST_ASSERT_SUCCESS(ret, 2565 "Checking status and ordering for encoder failed"); 2566 TEST_ASSERT_SUCCESS(validate_op_chain( 2567 &ops[i]->turbo_enc.output, 2568 hard_data_orig), 2569 "Output buffers (CB=%u) are not equal", 2570 i); 2571 } 2572 2573 return TEST_SUCCESS; 2574 } 2575 2576 static int 2577 validate_ldpc_enc_op(struct rte_bbdev_enc_op **ops, const uint16_t n, 2578 struct rte_bbdev_enc_op *ref_op) 2579 { 2580 unsigned int i; 2581 int ret; 2582 struct op_data_entries *hard_data_orig = 2583 &test_vector.entries[DATA_HARD_OUTPUT]; 2584 2585 for (i = 0; i < n; ++i) { 2586 ret = check_enc_status_and_ordering(ops[i], i, ref_op->status); 2587 TEST_ASSERT_SUCCESS(ret, 2588 "Checking status and ordering for encoder failed"); 2589 TEST_ASSERT_SUCCESS(validate_op_chain( 2590 &ops[i]->ldpc_enc.output, 2591 hard_data_orig), 2592 "Output buffers (CB=%u) are not equal", 2593 i); 2594 } 2595 2596 return TEST_SUCCESS; 2597 } 2598 2599 static inline int 2600 validate_op_fft_chain(struct rte_bbdev_op_data *op, struct op_data_entries *orig_op, 2601 bool skip_validate_output) 2602 { 2603 struct rte_mbuf *m = op->data; 2604 uint8_t i, nb_dst_segments = orig_op->nb_segments; 2605 int16_t delt, abs_delt, thres_hold = 4; 2606 uint32_t j, data_len_iq, error_num; 2607 int16_t *ref_out, *op_out; 2608 2609 TEST_ASSERT(nb_dst_segments == m->nb_segs, 2610 "Number of segments differ in original (%u) and filled (%u) op fft", 2611 nb_dst_segments, m->nb_segs); 2612 2613 /* Due to size limitation of mbuf, FFT doesn't use real mbuf. */ 2614 for (i = 0; i < nb_dst_segments; ++i) { 2615 uint16_t offset = (i == 0) ? op->offset : 0; 2616 uint32_t data_len = op->length; 2617 2618 TEST_ASSERT(orig_op->segments[i].length == data_len, 2619 "Length of segment differ in original (%u) and filled (%u) op fft", 2620 orig_op->segments[i].length, data_len); 2621 /* Divided by 2 to get the number of 16bits data. */ 2622 data_len_iq = data_len >> 1; 2623 ref_out = (int16_t *)(orig_op->segments[i].addr); 2624 op_out = rte_pktmbuf_mtod_offset(m, int16_t *, offset); 2625 error_num = 0; 2626 for (j = 0; j < data_len_iq; j++) { 2627 delt = ref_out[j] - op_out[j]; 2628 abs_delt = delt > 0 ? delt : -delt; 2629 error_num += (abs_delt > thres_hold ? 1 : 0); 2630 } 2631 if ((error_num > 0) && !skip_validate_output) { 2632 rte_memdump(stdout, "Buffer A", ref_out, data_len); 2633 rte_memdump(stdout, "Buffer B", op_out, data_len); 2634 TEST_ASSERT(error_num == 0, 2635 "FFT Output are not matched total (%u) errors (%u)", 2636 data_len_iq, error_num); 2637 } 2638 2639 m = m->next; 2640 } 2641 2642 return TEST_SUCCESS; 2643 } 2644 2645 static inline int 2646 validate_op_fft_meas_chain(struct rte_bbdev_op_data *op, struct op_data_entries *orig_op, 2647 bool skip_validate_output) 2648 { 2649 struct rte_mbuf *m = op->data; 2650 uint8_t i, nb_dst_segments = orig_op->nb_segments; 2651 double thres_hold = 1.0; 2652 uint32_t j, data_len_iq, error_num; 2653 int32_t *ref_out, *op_out; 2654 double estSNR, refSNR, delt, abs_delt; 2655 2656 TEST_ASSERT(nb_dst_segments == m->nb_segs, 2657 "Number of segments differ in original (%u) and filled (%u) op fft", 2658 nb_dst_segments, m->nb_segs); 2659 2660 /* Due to size limitation of mbuf, FFT doesn't use real mbuf. */ 2661 for (i = 0; i < nb_dst_segments; ++i) { 2662 uint16_t offset = (i == 0) ? op->offset : 0; 2663 uint32_t data_len = op->length; 2664 2665 TEST_ASSERT(orig_op->segments[i].length == data_len, 2666 "Length of segment differ in original (%u) and filled (%u) op fft", 2667 orig_op->segments[i].length, data_len); 2668 2669 /* Divided by 4 to get the number of 32 bits data. */ 2670 data_len_iq = data_len >> 2; 2671 ref_out = (int32_t *)(orig_op->segments[i].addr); 2672 op_out = rte_pktmbuf_mtod_offset(m, int32_t *, offset); 2673 error_num = 0; 2674 for (j = 0; j < data_len_iq; j++) { 2675 estSNR = 10*log10(op_out[j]); 2676 refSNR = 10*log10(ref_out[j]); 2677 delt = refSNR - estSNR; 2678 abs_delt = delt > 0 ? delt : -delt; 2679 error_num += (abs_delt > thres_hold ? 1 : 0); 2680 } 2681 if ((error_num > 0) && !skip_validate_output) { 2682 rte_memdump(stdout, "Buffer A", ref_out, data_len); 2683 rte_memdump(stdout, "Buffer B", op_out, data_len); 2684 TEST_ASSERT(error_num == 0, 2685 "FFT Output are not matched total (%u) errors (%u)", 2686 data_len_iq, error_num); 2687 } 2688 2689 m = m->next; 2690 } 2691 2692 return TEST_SUCCESS; 2693 } 2694 2695 static inline int 2696 validate_op_mldts_chain(struct rte_bbdev_op_data *op, 2697 struct op_data_entries *orig_op) 2698 { 2699 uint8_t i; 2700 struct rte_mbuf *m = op->data; 2701 uint8_t nb_dst_segments = orig_op->nb_segments; 2702 /*the result is not bit exact*/ 2703 int16_t thres_hold = 3; 2704 int16_t delt, abs_delt; 2705 uint32_t j, data_len_iq; 2706 uint32_t error_num; 2707 int8_t *ref_out; 2708 int8_t *op_out; 2709 2710 TEST_ASSERT(nb_dst_segments == m->nb_segs, 2711 "Number of segments differ in original (%u) and filled (%u) op mldts", 2712 nb_dst_segments, m->nb_segs); 2713 2714 /* Due to size limitation of mbuf, MLDTS doesn't use real mbuf. */ 2715 for (i = 0; i < nb_dst_segments; ++i) { 2716 uint16_t offset = (i == 0) ? op->offset : 0; 2717 uint32_t data_len = op->length; 2718 2719 TEST_ASSERT(orig_op->segments[i].length == data_len, 2720 "Length of segment differ in original (%u) and filled (%u) op mldts", 2721 orig_op->segments[i].length, data_len); 2722 data_len_iq = data_len; 2723 ref_out = (int8_t *)(orig_op->segments[i].addr); 2724 op_out = rte_pktmbuf_mtod_offset(m, int8_t *, offset); 2725 error_num = 0; 2726 for (j = 0; j < data_len_iq; j++) { 2727 2728 delt = ref_out[j] - op_out[j]; 2729 abs_delt = delt > 0 ? delt : -delt; 2730 error_num += (abs_delt > thres_hold ? 1 : 0); 2731 if (error_num > 0) 2732 printf("MLD Error %d: Exp %x %d Actual %x %d Diff %d\n", 2733 j, ref_out[j], ref_out[j], op_out[j], op_out[j], 2734 delt); 2735 } 2736 TEST_ASSERT(error_num == 0, 2737 "MLDTS Output are not matched total (%u) errors (%u)", 2738 data_len_iq, error_num); 2739 2740 m = m->next; 2741 } 2742 2743 return TEST_SUCCESS; 2744 } 2745 2746 static int 2747 validate_fft_op(struct rte_bbdev_fft_op **ops, const uint16_t n, 2748 struct rte_bbdev_fft_op *ref_op) 2749 { 2750 unsigned int i; 2751 int ret; 2752 struct op_data_entries *fft_data_orig = &test_vector.entries[DATA_HARD_OUTPUT]; 2753 struct op_data_entries *fft_pwr_orig = &test_vector.entries[DATA_SOFT_OUTPUT]; 2754 bool skip_validate_output = false; 2755 2756 if ((test_vector.fft_window_width_vec > 0) && 2757 (test_vector.fft_window_width_vec != fft_window_width_dev)) { 2758 printf("The vector FFT width doesn't match with device - skip %d %d\n", 2759 test_vector.fft_window_width_vec, fft_window_width_dev); 2760 skip_validate_output = true; 2761 } 2762 2763 for (i = 0; i < n; ++i) { 2764 ret = check_fft_status_and_ordering(ops[i], i, ref_op->status); 2765 TEST_ASSERT_SUCCESS(ret, "Checking status and ordering for FFT failed"); 2766 TEST_ASSERT_SUCCESS(validate_op_fft_chain( 2767 &ops[i]->fft.base_output, fft_data_orig, skip_validate_output), 2768 "FFT Output buffers (op=%u) are not matched", i); 2769 if (check_bit(ops[i]->fft.op_flags, RTE_BBDEV_FFT_POWER_MEAS)) 2770 TEST_ASSERT_SUCCESS(validate_op_fft_meas_chain( 2771 &ops[i]->fft.power_meas_output, fft_pwr_orig, skip_validate_output), 2772 "FFT Power Output buffers (op=%u) are not matched", i); 2773 } 2774 2775 return TEST_SUCCESS; 2776 } 2777 2778 static int 2779 validate_mldts_op(struct rte_bbdev_mldts_op **ops, const uint16_t n, 2780 struct rte_bbdev_mldts_op *ref_op) 2781 { 2782 unsigned int i; 2783 int ret; 2784 struct op_data_entries *mldts_data_orig = 2785 &test_vector.entries[DATA_HARD_OUTPUT]; 2786 for (i = 0; i < n; ++i) { 2787 ret = check_mldts_status_and_ordering(ops[i], i, ref_op->status); 2788 TEST_ASSERT_SUCCESS(ret, 2789 "Checking status and ordering for MLDTS failed"); 2790 TEST_ASSERT_SUCCESS(validate_op_mldts_chain( 2791 &ops[i]->mldts.output, 2792 mldts_data_orig), 2793 "MLDTS Output buffers (op=%u) are not matched", 2794 i); 2795 } 2796 2797 return TEST_SUCCESS; 2798 } 2799 2800 static void 2801 create_reference_dec_op(struct rte_bbdev_dec_op *op) 2802 { 2803 unsigned int i; 2804 struct op_data_entries *entry; 2805 2806 op->turbo_dec = test_vector.turbo_dec; 2807 entry = &test_vector.entries[DATA_INPUT]; 2808 for (i = 0; i < entry->nb_segments; ++i) 2809 op->turbo_dec.input.length += 2810 entry->segments[i].length; 2811 } 2812 2813 static void 2814 create_reference_ldpc_dec_op(struct rte_bbdev_dec_op *op) 2815 { 2816 unsigned int i; 2817 struct op_data_entries *entry; 2818 2819 op->ldpc_dec = test_vector.ldpc_dec; 2820 entry = &test_vector.entries[DATA_INPUT]; 2821 for (i = 0; i < entry->nb_segments; ++i) 2822 op->ldpc_dec.input.length += 2823 entry->segments[i].length; 2824 if (test_vector.ldpc_dec.op_flags & 2825 RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE) { 2826 entry = &test_vector.entries[DATA_HARQ_INPUT]; 2827 for (i = 0; i < entry->nb_segments; ++i) 2828 op->ldpc_dec.harq_combined_input.length += 2829 entry->segments[i].length; 2830 } 2831 } 2832 2833 static void 2834 create_reference_fft_op(struct rte_bbdev_fft_op *op) 2835 { 2836 unsigned int i; 2837 struct op_data_entries *entry; 2838 op->fft = test_vector.fft; 2839 entry = &test_vector.entries[DATA_INPUT]; 2840 for (i = 0; i < entry->nb_segments; ++i) 2841 op->fft.base_input.length += entry->segments[i].length; 2842 entry = &test_vector.entries[DATA_HARQ_INPUT]; 2843 for (i = 0; i < entry->nb_segments; ++i) 2844 op->fft.dewindowing_input.length += entry->segments[i].length; 2845 } 2846 2847 static void 2848 create_reference_mldts_op(struct rte_bbdev_mldts_op *op) 2849 { 2850 unsigned int i; 2851 struct op_data_entries *entry; 2852 op->mldts = test_vector.mldts; 2853 entry = &test_vector.entries[DATA_INPUT]; 2854 for (i = 0; i < entry->nb_segments; ++i) 2855 op->mldts.qhy_input.length += entry->segments[i].length; 2856 entry = &test_vector.entries[DATA_HARQ_INPUT]; 2857 for (i = 0; i < entry->nb_segments; ++i) 2858 op->mldts.r_input.length += entry->segments[i].length; 2859 } 2860 2861 static void 2862 create_reference_enc_op(struct rte_bbdev_enc_op *op) 2863 { 2864 unsigned int i; 2865 struct op_data_entries *entry; 2866 2867 op->turbo_enc = test_vector.turbo_enc; 2868 entry = &test_vector.entries[DATA_INPUT]; 2869 for (i = 0; i < entry->nb_segments; ++i) 2870 op->turbo_enc.input.length += 2871 entry->segments[i].length; 2872 } 2873 2874 static void 2875 create_reference_ldpc_enc_op(struct rte_bbdev_enc_op *op) 2876 { 2877 unsigned int i; 2878 struct op_data_entries *entry; 2879 2880 op->ldpc_enc = test_vector.ldpc_enc; 2881 entry = &test_vector.entries[DATA_INPUT]; 2882 for (i = 0; i < entry->nb_segments; ++i) 2883 op->ldpc_enc.input.length += 2884 entry->segments[i].length; 2885 } 2886 2887 static uint32_t 2888 calc_dec_TB_size(struct rte_bbdev_dec_op *op) 2889 { 2890 uint8_t i; 2891 uint32_t c, r, tb_size = 0; 2892 2893 if (op->turbo_dec.code_block_mode == RTE_BBDEV_CODE_BLOCK) { 2894 tb_size = op->turbo_dec.tb_params.k_neg; 2895 } else { 2896 c = op->turbo_dec.tb_params.c; 2897 r = op->turbo_dec.tb_params.r; 2898 for (i = 0; i < c-r; i++) 2899 tb_size += (r < op->turbo_dec.tb_params.c_neg) ? 2900 op->turbo_dec.tb_params.k_neg : 2901 op->turbo_dec.tb_params.k_pos; 2902 } 2903 return tb_size; 2904 } 2905 2906 static uint32_t 2907 calc_ldpc_dec_TB_size(struct rte_bbdev_dec_op *op) 2908 { 2909 uint8_t num_cbs = 0; 2910 uint32_t tb_size = 0; 2911 uint16_t sys_cols = (op->ldpc_dec.basegraph == 1) ? 22 : 10; 2912 2913 if (op->ldpc_dec.code_block_mode == RTE_BBDEV_CODE_BLOCK) 2914 num_cbs = 1; 2915 else 2916 num_cbs = op->ldpc_dec.tb_params.c - op->ldpc_dec.tb_params.r; 2917 2918 tb_size = (sys_cols * op->ldpc_dec.z_c - op->ldpc_dec.n_filler) * num_cbs; 2919 return tb_size; 2920 } 2921 2922 static uint32_t 2923 calc_enc_TB_size(struct rte_bbdev_enc_op *op) 2924 { 2925 uint8_t i; 2926 uint32_t c, r, tb_size = 0; 2927 2928 if (op->turbo_enc.code_block_mode == RTE_BBDEV_CODE_BLOCK) { 2929 tb_size = op->turbo_enc.tb_params.k_neg; 2930 } else { 2931 c = op->turbo_enc.tb_params.c; 2932 r = op->turbo_enc.tb_params.r; 2933 for (i = 0; i < c-r; i++) 2934 tb_size += (r < op->turbo_enc.tb_params.c_neg) ? 2935 op->turbo_enc.tb_params.k_neg : 2936 op->turbo_enc.tb_params.k_pos; 2937 } 2938 return tb_size; 2939 } 2940 2941 static uint32_t 2942 calc_ldpc_enc_TB_size(struct rte_bbdev_enc_op *op) 2943 { 2944 uint8_t num_cbs = 0; 2945 uint32_t tb_size = 0; 2946 uint16_t sys_cols = (op->ldpc_enc.basegraph == 1) ? 22 : 10; 2947 2948 if (op->ldpc_enc.code_block_mode == RTE_BBDEV_CODE_BLOCK) 2949 num_cbs = 1; 2950 else 2951 num_cbs = op->ldpc_enc.tb_params.c - op->ldpc_enc.tb_params.r; 2952 2953 tb_size = (sys_cols * op->ldpc_enc.z_c - op->ldpc_enc.n_filler) * num_cbs; 2954 return tb_size; 2955 } 2956 2957 static uint32_t 2958 calc_fft_size(struct rte_bbdev_fft_op *op) 2959 { 2960 uint32_t output_size; 2961 int num_cs = 0, i; 2962 for (i = 0; i < 12; i++) 2963 if (check_bit(op->fft.cs_bitmap, 1 << i)) 2964 num_cs++; 2965 output_size = (num_cs * op->fft.output_sequence_size * 4) << op->fft.num_antennas_log2; 2966 return output_size; 2967 } 2968 2969 static uint32_t 2970 calc_mldts_size(struct rte_bbdev_mldts_op *op) 2971 { 2972 uint32_t output_size = 0; 2973 uint16_t i; 2974 2975 for (i = 0; i < op->mldts.num_layers; i++) 2976 output_size += op->mldts.q_m[i]; 2977 2978 output_size *= NUM_SC_PER_RB * BITS_PER_LLR * op->mldts.num_rbs * (op->mldts.c_rep + 1); 2979 2980 return output_size; 2981 } 2982 2983 static int 2984 init_test_op_params(struct test_op_params *op_params, 2985 enum rte_bbdev_op_type op_type, const int expected_status, 2986 const int vector_mask, struct rte_mempool *ops_mp, 2987 uint16_t burst_sz, uint16_t num_to_process, uint16_t num_lcores) 2988 { 2989 int ret = 0; 2990 if (op_type == RTE_BBDEV_OP_TURBO_DEC || 2991 op_type == RTE_BBDEV_OP_LDPC_DEC) 2992 ret = rte_bbdev_dec_op_alloc_bulk(ops_mp, 2993 &op_params->ref_dec_op, 1); 2994 else if (op_type == RTE_BBDEV_OP_FFT) 2995 ret = rte_bbdev_fft_op_alloc_bulk(ops_mp, 2996 &op_params->ref_fft_op, 1); 2997 else if (op_type == RTE_BBDEV_OP_MLDTS) 2998 ret = rte_bbdev_mldts_op_alloc_bulk(ops_mp, 2999 &op_params->ref_mldts_op, 1); 3000 else 3001 ret = rte_bbdev_enc_op_alloc_bulk(ops_mp, 3002 &op_params->ref_enc_op, 1); 3003 3004 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_op_alloc_bulk() failed"); 3005 3006 op_params->mp = ops_mp; 3007 op_params->burst_sz = burst_sz; 3008 op_params->num_to_process = num_to_process; 3009 op_params->num_lcores = num_lcores; 3010 op_params->vector_mask = vector_mask; 3011 if (op_type == RTE_BBDEV_OP_TURBO_DEC || 3012 op_type == RTE_BBDEV_OP_LDPC_DEC) 3013 op_params->ref_dec_op->status = expected_status; 3014 else if (op_type == RTE_BBDEV_OP_TURBO_ENC 3015 || op_type == RTE_BBDEV_OP_LDPC_ENC) 3016 op_params->ref_enc_op->status = expected_status; 3017 else if (op_type == RTE_BBDEV_OP_FFT) 3018 op_params->ref_fft_op->status = expected_status; 3019 else if (op_type == RTE_BBDEV_OP_MLDTS) 3020 op_params->ref_mldts_op->status = expected_status; 3021 return 0; 3022 } 3023 3024 static int 3025 run_test_case_on_device(test_case_function *test_case_func, uint8_t dev_id, 3026 struct test_op_params *op_params) 3027 { 3028 int t_ret, f_ret, socket_id = SOCKET_ID_ANY; 3029 unsigned int i; 3030 struct active_device *ad; 3031 unsigned int burst_sz = get_burst_sz(); 3032 enum rte_bbdev_op_type op_type = test_vector.op_type; 3033 const struct rte_bbdev_op_cap *capabilities = NULL; 3034 3035 ad = &active_devs[dev_id]; 3036 3037 /* Check if device supports op_type */ 3038 if (!is_avail_op(ad, test_vector.op_type)) 3039 return TEST_SUCCESS; 3040 3041 struct rte_bbdev_info info; 3042 rte_bbdev_info_get(ad->dev_id, &info); 3043 socket_id = GET_SOCKET(info.socket_id); 3044 3045 f_ret = create_mempools(ad, socket_id, op_type, 3046 get_num_ops()); 3047 if (f_ret != TEST_SUCCESS) { 3048 printf("Couldn't create mempools"); 3049 goto fail; 3050 } 3051 if (op_type == RTE_BBDEV_OP_NONE) 3052 op_type = RTE_BBDEV_OP_TURBO_ENC; 3053 3054 f_ret = init_test_op_params(op_params, test_vector.op_type, 3055 test_vector.expected_status, 3056 test_vector.mask, 3057 ad->ops_mempool, 3058 burst_sz, 3059 get_num_ops(), 3060 get_num_lcores()); 3061 if (f_ret != TEST_SUCCESS) { 3062 printf("Couldn't init test op params"); 3063 goto fail; 3064 } 3065 3066 3067 /* Find capabilities */ 3068 const struct rte_bbdev_op_cap *cap = info.drv.capabilities; 3069 do { 3070 if (cap->type == test_vector.op_type) { 3071 capabilities = cap; 3072 break; 3073 } 3074 cap++; 3075 } while (cap->type != RTE_BBDEV_OP_NONE); 3076 TEST_ASSERT_NOT_NULL(capabilities, 3077 "Couldn't find capabilities"); 3078 3079 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) { 3080 create_reference_dec_op(op_params->ref_dec_op); 3081 } else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) 3082 create_reference_enc_op(op_params->ref_enc_op); 3083 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) 3084 create_reference_ldpc_enc_op(op_params->ref_enc_op); 3085 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 3086 create_reference_ldpc_dec_op(op_params->ref_dec_op); 3087 else if (test_vector.op_type == RTE_BBDEV_OP_FFT) 3088 create_reference_fft_op(op_params->ref_fft_op); 3089 else if (test_vector.op_type == RTE_BBDEV_OP_MLDTS) 3090 create_reference_mldts_op(op_params->ref_mldts_op); 3091 3092 for (i = 0; i < ad->nb_queues; ++i) { 3093 f_ret = fill_queue_buffers(op_params, 3094 ad->in_mbuf_pool, 3095 ad->hard_out_mbuf_pool, 3096 ad->soft_out_mbuf_pool, 3097 ad->harq_in_mbuf_pool, 3098 ad->harq_out_mbuf_pool, 3099 ad->queue_ids[i], 3100 capabilities, 3101 info.drv.min_alignment, 3102 socket_id); 3103 if (f_ret != TEST_SUCCESS) { 3104 printf("Couldn't init queue buffers"); 3105 goto fail; 3106 } 3107 } 3108 3109 /* Run test case function */ 3110 t_ret = test_case_func(ad, op_params); 3111 3112 /* Free active device resources and return */ 3113 free_buffers(ad, op_params); 3114 return t_ret; 3115 3116 fail: 3117 free_buffers(ad, op_params); 3118 return TEST_FAILED; 3119 } 3120 3121 /* Run given test function per active device per supported op type 3122 * per burst size. 3123 */ 3124 static int 3125 run_test_case(test_case_function *test_case_func) 3126 { 3127 int ret = 0; 3128 uint8_t dev; 3129 3130 /* Alloc op_params */ 3131 struct test_op_params *op_params = rte_zmalloc(NULL, 3132 sizeof(struct test_op_params), RTE_CACHE_LINE_SIZE); 3133 TEST_ASSERT_NOT_NULL(op_params, "Failed to alloc %zuB for op_params", 3134 RTE_ALIGN(sizeof(struct test_op_params), 3135 RTE_CACHE_LINE_SIZE)); 3136 3137 /* For each device run test case function */ 3138 for (dev = 0; dev < nb_active_devs; ++dev) 3139 ret |= run_test_case_on_device(test_case_func, dev, op_params); 3140 3141 rte_free(op_params); 3142 3143 return ret; 3144 } 3145 3146 3147 /* Push back the HARQ output from DDR to host */ 3148 static void 3149 retrieve_harq_ddr(uint16_t dev_id, uint16_t queue_id, 3150 struct rte_bbdev_dec_op **ops, 3151 const uint16_t n) 3152 { 3153 uint16_t j; 3154 int save_status, ret; 3155 uint32_t harq_offset = (uint32_t) queue_id * HARQ_INCR * MAX_OPS; 3156 struct rte_bbdev_dec_op *ops_deq[MAX_BURST]; 3157 uint32_t flags = ops[0]->ldpc_dec.op_flags; 3158 bool loopback = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK; 3159 bool mem_out = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE; 3160 bool hc_out = flags & RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE; 3161 bool h_comp = flags & RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION; 3162 for (j = 0; j < n; ++j) { 3163 if ((loopback && mem_out) || hc_out) { 3164 save_status = ops[j]->status; 3165 ops[j]->ldpc_dec.op_flags = 3166 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK + 3167 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE; 3168 if (h_comp) 3169 ops[j]->ldpc_dec.op_flags += 3170 RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION; 3171 ops[j]->ldpc_dec.harq_combined_input.offset = 3172 harq_offset; 3173 ops[j]->ldpc_dec.harq_combined_output.offset = 0; 3174 harq_offset += HARQ_INCR; 3175 if (!loopback) 3176 ops[j]->ldpc_dec.harq_combined_input.length = 3177 ops[j]->ldpc_dec.harq_combined_output.length; 3178 rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id, 3179 &ops[j], 1); 3180 ret = 0; 3181 while (ret == 0) 3182 ret = rte_bbdev_dequeue_ldpc_dec_ops( 3183 dev_id, queue_id, 3184 &ops_deq[j], 1); 3185 ops[j]->ldpc_dec.op_flags = flags; 3186 ops[j]->status = save_status; 3187 } 3188 } 3189 } 3190 3191 /* 3192 * Push back the HARQ output from HW DDR to Host 3193 * Preload HARQ memory input and adjust HARQ offset 3194 */ 3195 static void 3196 preload_harq_ddr(uint16_t dev_id, uint16_t queue_id, 3197 struct rte_bbdev_dec_op **ops, const uint16_t n, 3198 bool preload) 3199 { 3200 uint16_t j; 3201 int deq; 3202 uint32_t harq_offset = (uint32_t) queue_id * HARQ_INCR * MAX_OPS; 3203 struct rte_bbdev_op_data save_hc_in[MAX_OPS], save_hc_out[MAX_OPS]; 3204 struct rte_bbdev_dec_op *ops_deq[MAX_OPS]; 3205 uint32_t flags = ops[0]->ldpc_dec.op_flags; 3206 bool mem_in = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE; 3207 bool hc_in = flags & RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE; 3208 bool mem_out = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE; 3209 bool hc_out = flags & RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE; 3210 bool h_comp = flags & RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION; 3211 if ((mem_in || hc_in) && preload) { 3212 for (j = 0; j < n; ++j) { 3213 save_hc_in[j] = ops[j]->ldpc_dec.harq_combined_input; 3214 save_hc_out[j] = ops[j]->ldpc_dec.harq_combined_output; 3215 ops[j]->ldpc_dec.op_flags = 3216 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK + 3217 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE; 3218 if (h_comp) 3219 ops[j]->ldpc_dec.op_flags += 3220 RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION; 3221 ops[j]->ldpc_dec.harq_combined_output.offset = 3222 harq_offset; 3223 ops[j]->ldpc_dec.harq_combined_input.offset = 0; 3224 harq_offset += HARQ_INCR; 3225 } 3226 rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id, &ops[0], n); 3227 deq = 0; 3228 while (deq != n) 3229 deq += rte_bbdev_dequeue_ldpc_dec_ops( 3230 dev_id, queue_id, &ops_deq[deq], 3231 n - deq); 3232 /* Restore the operations */ 3233 for (j = 0; j < n; ++j) { 3234 ops[j]->ldpc_dec.op_flags = flags; 3235 ops[j]->ldpc_dec.harq_combined_input = save_hc_in[j]; 3236 ops[j]->ldpc_dec.harq_combined_output = save_hc_out[j]; 3237 } 3238 } 3239 harq_offset = (uint32_t) queue_id * HARQ_INCR * MAX_OPS; 3240 for (j = 0; j < n; ++j) { 3241 /* Adjust HARQ offset when we reach external DDR */ 3242 if (mem_in || hc_in) 3243 ops[j]->ldpc_dec.harq_combined_input.offset 3244 = harq_offset; 3245 if (mem_out || hc_out) 3246 ops[j]->ldpc_dec.harq_combined_output.offset 3247 = harq_offset; 3248 harq_offset += HARQ_INCR; 3249 } 3250 } 3251 3252 static void 3253 dequeue_event_callback(uint16_t dev_id, 3254 enum rte_bbdev_event_type event, void *cb_arg, 3255 void *ret_param) 3256 { 3257 int ret; 3258 uint16_t i; 3259 uint64_t total_time; 3260 uint16_t deq, burst_sz, num_ops; 3261 uint16_t queue_id = *(uint16_t *) ret_param; 3262 struct rte_bbdev_info info; 3263 double tb_len_bits; 3264 struct thread_params *tp = cb_arg; 3265 3266 /* Find matching thread params using queue_id */ 3267 for (i = 0; i < MAX_QUEUES; ++i, ++tp) 3268 if (tp->queue_id == queue_id) 3269 break; 3270 3271 if (i == MAX_QUEUES) { 3272 printf("%s: Queue_id from interrupt details was not found!\n", 3273 __func__); 3274 return; 3275 } 3276 3277 if (unlikely(event != RTE_BBDEV_EVENT_DEQUEUE)) { 3278 rte_atomic_store_explicit(&tp->processing_status, TEST_FAILED, 3279 rte_memory_order_relaxed); 3280 printf( 3281 "Dequeue interrupt handler called for incorrect event!\n"); 3282 return; 3283 } 3284 3285 burst_sz = rte_atomic_load_explicit(&tp->burst_sz, rte_memory_order_relaxed); 3286 num_ops = tp->op_params->num_to_process; 3287 3288 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) 3289 deq = rte_bbdev_dequeue_dec_ops(dev_id, queue_id, 3290 &tp->dec_ops[ 3291 rte_atomic_load_explicit(&tp->nb_dequeued, 3292 rte_memory_order_relaxed)], 3293 burst_sz); 3294 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 3295 deq = rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id, 3296 &tp->dec_ops[ 3297 rte_atomic_load_explicit(&tp->nb_dequeued, 3298 rte_memory_order_relaxed)], 3299 burst_sz); 3300 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) 3301 deq = rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id, 3302 &tp->enc_ops[ 3303 rte_atomic_load_explicit(&tp->nb_dequeued, 3304 rte_memory_order_relaxed)], 3305 burst_sz); 3306 else if (test_vector.op_type == RTE_BBDEV_OP_FFT) 3307 deq = rte_bbdev_dequeue_fft_ops(dev_id, queue_id, 3308 &tp->fft_ops[ 3309 rte_atomic_load_explicit(&tp->nb_dequeued, 3310 rte_memory_order_relaxed)], 3311 burst_sz); 3312 else if (test_vector.op_type == RTE_BBDEV_OP_MLDTS) 3313 deq = rte_bbdev_dequeue_mldts_ops(dev_id, queue_id, 3314 &tp->mldts_ops[ 3315 rte_atomic_load_explicit(&tp->nb_dequeued, 3316 rte_memory_order_relaxed)], 3317 burst_sz); 3318 else /*RTE_BBDEV_OP_TURBO_ENC*/ 3319 deq = rte_bbdev_dequeue_enc_ops(dev_id, queue_id, 3320 &tp->enc_ops[ 3321 rte_atomic_load_explicit(&tp->nb_dequeued, 3322 rte_memory_order_relaxed)], 3323 burst_sz); 3324 3325 if (deq < burst_sz) { 3326 printf( 3327 "After receiving the interrupt all operations should be dequeued. Expected: %u, got: %u\n", 3328 burst_sz, deq); 3329 rte_atomic_store_explicit(&tp->processing_status, TEST_FAILED, 3330 rte_memory_order_relaxed); 3331 return; 3332 } 3333 3334 if (rte_atomic_load_explicit(&tp->nb_dequeued, rte_memory_order_relaxed) + deq < num_ops) { 3335 rte_atomic_fetch_add_explicit(&tp->nb_dequeued, deq, rte_memory_order_relaxed); 3336 return; 3337 } 3338 3339 total_time = rte_rdtsc_precise() - tp->start_time; 3340 3341 rte_bbdev_info_get(dev_id, &info); 3342 3343 ret = TEST_SUCCESS; 3344 3345 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) { 3346 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 3347 ret = validate_dec_op(tp->dec_ops, num_ops, ref_op); 3348 /* get the max of iter_count for all dequeued ops */ 3349 for (i = 0; i < num_ops; ++i) 3350 tp->iter_count = RTE_MAX( 3351 tp->dec_ops[i]->turbo_dec.iter_count, 3352 tp->iter_count); 3353 rte_bbdev_dec_op_free_bulk(tp->dec_ops, deq); 3354 } else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) { 3355 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op; 3356 ret = validate_enc_op(tp->enc_ops, num_ops, ref_op); 3357 rte_bbdev_enc_op_free_bulk(tp->enc_ops, deq); 3358 } else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) { 3359 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op; 3360 ret = validate_ldpc_enc_op(tp->enc_ops, num_ops, ref_op); 3361 rte_bbdev_enc_op_free_bulk(tp->enc_ops, deq); 3362 } else if (test_vector.op_type == RTE_BBDEV_OP_FFT) { 3363 struct rte_bbdev_fft_op *ref_op = tp->op_params->ref_fft_op; 3364 ret = validate_fft_op(tp->fft_ops, num_ops, ref_op); 3365 rte_bbdev_fft_op_free_bulk(tp->fft_ops, deq); 3366 } else if (test_vector.op_type == RTE_BBDEV_OP_MLDTS) { 3367 struct rte_bbdev_mldts_op *ref_op = tp->op_params->ref_mldts_op; 3368 ret = validate_mldts_op(tp->mldts_ops, num_ops, ref_op); 3369 rte_bbdev_mldts_op_free_bulk(tp->mldts_ops, deq); 3370 } else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) { 3371 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 3372 ret = validate_ldpc_dec_op(tp->dec_ops, num_ops, ref_op, 3373 tp->op_params->vector_mask); 3374 rte_bbdev_dec_op_free_bulk(tp->dec_ops, deq); 3375 } 3376 3377 if (ret) { 3378 printf("Buffers validation failed\n"); 3379 rte_atomic_store_explicit(&tp->processing_status, TEST_FAILED, 3380 rte_memory_order_relaxed); 3381 } 3382 3383 switch (test_vector.op_type) { 3384 case RTE_BBDEV_OP_TURBO_DEC: 3385 tb_len_bits = calc_dec_TB_size(tp->op_params->ref_dec_op); 3386 break; 3387 case RTE_BBDEV_OP_TURBO_ENC: 3388 tb_len_bits = calc_enc_TB_size(tp->op_params->ref_enc_op); 3389 break; 3390 case RTE_BBDEV_OP_LDPC_DEC: 3391 tb_len_bits = calc_ldpc_dec_TB_size(tp->op_params->ref_dec_op); 3392 break; 3393 case RTE_BBDEV_OP_FFT: 3394 tb_len_bits = calc_fft_size(tp->op_params->ref_fft_op); 3395 break; 3396 case RTE_BBDEV_OP_MLDTS: 3397 tb_len_bits = calc_mldts_size(tp->op_params->ref_mldts_op); 3398 break; 3399 case RTE_BBDEV_OP_LDPC_ENC: 3400 tb_len_bits = calc_ldpc_enc_TB_size(tp->op_params->ref_enc_op); 3401 break; 3402 case RTE_BBDEV_OP_NONE: 3403 tb_len_bits = 0.0; 3404 break; 3405 default: 3406 printf("Unknown op type: %d\n", test_vector.op_type); 3407 rte_atomic_store_explicit(&tp->processing_status, TEST_FAILED, 3408 rte_memory_order_relaxed); 3409 return; 3410 } 3411 3412 tp->ops_per_sec += ((double)num_ops) / 3413 ((double)total_time / (double)rte_get_tsc_hz()); 3414 tp->mbps += (((double)(num_ops * tb_len_bits)) / 1000000.0) / 3415 ((double)total_time / (double)rte_get_tsc_hz()); 3416 3417 rte_atomic_fetch_add_explicit(&tp->nb_dequeued, deq, rte_memory_order_relaxed); 3418 } 3419 3420 static int 3421 throughput_intr_lcore_ldpc_dec(void *arg) 3422 { 3423 struct thread_params *tp = arg; 3424 unsigned int enqueued; 3425 const uint16_t queue_id = tp->queue_id; 3426 const uint16_t burst_sz = tp->op_params->burst_sz; 3427 const uint16_t num_to_process = tp->op_params->num_to_process; 3428 struct rte_bbdev_dec_op *ops[num_to_process]; 3429 struct test_buffers *bufs = NULL; 3430 struct rte_bbdev_info info; 3431 int ret, i, j; 3432 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 3433 uint16_t num_to_enq, enq; 3434 3435 bool loopback = check_bit(ref_op->ldpc_dec.op_flags, 3436 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK); 3437 bool hc_out = check_bit(ref_op->ldpc_dec.op_flags, 3438 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE); 3439 3440 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 3441 "BURST_SIZE should be <= %u", MAX_BURST); 3442 3443 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id), 3444 "Failed to enable interrupts for dev: %u, queue_id: %u", 3445 tp->dev_id, queue_id); 3446 3447 rte_bbdev_info_get(tp->dev_id, &info); 3448 3449 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim), 3450 "NUM_OPS cannot exceed %u for this device", 3451 info.drv.queue_size_lim); 3452 3453 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 3454 3455 rte_atomic_store_explicit(&tp->processing_status, 0, rte_memory_order_relaxed); 3456 rte_atomic_store_explicit(&tp->nb_dequeued, 0, rte_memory_order_relaxed); 3457 3458 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->op_params->sync, SYNC_START, 3459 rte_memory_order_relaxed); 3460 3461 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops, 3462 num_to_process); 3463 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", 3464 num_to_process); 3465 ref_op->ldpc_dec.iter_max = get_iter_max(); 3466 3467 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3468 copy_reference_ldpc_dec_op(ops, num_to_process, 0, bufs->inputs, 3469 bufs->hard_outputs, bufs->soft_outputs, 3470 bufs->harq_inputs, bufs->harq_outputs, ref_op); 3471 3472 /* Set counter to validate the ordering */ 3473 for (j = 0; j < num_to_process; ++j) 3474 ops[j]->opaque_data = (void *)(uintptr_t)j; 3475 3476 for (j = 0; j < TEST_REPETITIONS; ++j) { 3477 for (i = 0; i < num_to_process; ++i) { 3478 if (!loopback) 3479 mbuf_reset(ops[i]->ldpc_dec.hard_output.data); 3480 if (hc_out || loopback) 3481 mbuf_reset(ops[i]->ldpc_dec.harq_combined_output.data); 3482 if (ops[i]->ldpc_dec.soft_output.data != NULL) 3483 mbuf_reset(ops[i]->ldpc_dec.soft_output.data); 3484 } 3485 3486 tp->start_time = rte_rdtsc_precise(); 3487 for (enqueued = 0; enqueued < num_to_process;) { 3488 num_to_enq = burst_sz; 3489 3490 if (unlikely(num_to_process - enqueued < num_to_enq)) 3491 num_to_enq = num_to_process - enqueued; 3492 3493 /* Write to thread burst_sz current number of enqueued 3494 * descriptors. It ensures that proper number of 3495 * descriptors will be dequeued in callback 3496 * function - needed for last batch in case where 3497 * the number of operations is not a multiple of 3498 * burst size. 3499 */ 3500 rte_atomic_store_explicit(&tp->burst_sz, num_to_enq, 3501 rte_memory_order_relaxed); 3502 3503 enq = 0; 3504 do { 3505 enq += rte_bbdev_enqueue_ldpc_dec_ops( 3506 tp->dev_id, 3507 queue_id, &ops[enqueued], 3508 num_to_enq); 3509 } while (unlikely(num_to_enq != enq)); 3510 enqueued += enq; 3511 3512 /* Wait until processing of previous batch is 3513 * completed 3514 */ 3515 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->nb_dequeued, enqueued, 3516 rte_memory_order_relaxed); 3517 } 3518 if (j != TEST_REPETITIONS - 1) 3519 rte_atomic_store_explicit(&tp->nb_dequeued, 0, rte_memory_order_relaxed); 3520 } 3521 3522 return TEST_SUCCESS; 3523 } 3524 3525 static int 3526 throughput_intr_lcore_dec(void *arg) 3527 { 3528 struct thread_params *tp = arg; 3529 unsigned int enqueued; 3530 const uint16_t queue_id = tp->queue_id; 3531 const uint16_t burst_sz = tp->op_params->burst_sz; 3532 const uint16_t num_to_process = tp->op_params->num_to_process; 3533 struct rte_bbdev_dec_op *ops[num_to_process]; 3534 struct test_buffers *bufs = NULL; 3535 struct rte_bbdev_info info; 3536 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 3537 int ret, i, j; 3538 uint16_t num_to_enq, enq; 3539 3540 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 3541 "BURST_SIZE should be <= %u", MAX_BURST); 3542 3543 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id), 3544 "Failed to enable interrupts for dev: %u, queue_id: %u", 3545 tp->dev_id, queue_id); 3546 3547 rte_bbdev_info_get(tp->dev_id, &info); 3548 3549 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim), 3550 "NUM_OPS cannot exceed %u for this device", 3551 info.drv.queue_size_lim); 3552 3553 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 3554 3555 rte_atomic_store_explicit(&tp->processing_status, 0, rte_memory_order_relaxed); 3556 rte_atomic_store_explicit(&tp->nb_dequeued, 0, rte_memory_order_relaxed); 3557 3558 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->op_params->sync, SYNC_START, 3559 rte_memory_order_relaxed); 3560 3561 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops, 3562 num_to_process); 3563 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_to_process); 3564 ref_op->turbo_dec.iter_max = get_iter_max(); 3565 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3566 copy_reference_dec_op(ops, num_to_process, 0, bufs->inputs, 3567 bufs->hard_outputs, bufs->soft_outputs, 3568 tp->op_params->ref_dec_op); 3569 3570 /* Set counter to validate the ordering. */ 3571 for (j = 0; j < num_to_process; ++j) 3572 ops[j]->opaque_data = (void *)(uintptr_t)j; 3573 3574 for (j = 0; j < TEST_REPETITIONS; ++j) { 3575 for (i = 0; i < num_to_process; ++i) { 3576 mbuf_reset(ops[i]->turbo_dec.hard_output.data); 3577 if (ops[i]->turbo_dec.soft_output.data != NULL) 3578 mbuf_reset(ops[i]->turbo_dec.soft_output.data); 3579 } 3580 3581 tp->start_time = rte_rdtsc_precise(); 3582 for (enqueued = 0; enqueued < num_to_process;) { 3583 num_to_enq = burst_sz; 3584 3585 if (unlikely(num_to_process - enqueued < num_to_enq)) 3586 num_to_enq = num_to_process - enqueued; 3587 3588 /* Write to thread burst_sz current number of enqueued 3589 * descriptors. It ensures that proper number of 3590 * descriptors will be dequeued in callback 3591 * function - needed for last batch in case where 3592 * the number of operations is not a multiple of 3593 * burst size. 3594 */ 3595 rte_atomic_store_explicit(&tp->burst_sz, num_to_enq, 3596 rte_memory_order_relaxed); 3597 3598 enq = 0; 3599 do { 3600 enq += rte_bbdev_enqueue_dec_ops(tp->dev_id, 3601 queue_id, &ops[enqueued], 3602 num_to_enq); 3603 } while (unlikely(num_to_enq != enq)); 3604 enqueued += enq; 3605 3606 /* Wait until processing of previous batch is 3607 * completed 3608 */ 3609 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->nb_dequeued, enqueued, 3610 rte_memory_order_relaxed); 3611 } 3612 if (j != TEST_REPETITIONS - 1) 3613 rte_atomic_store_explicit(&tp->nb_dequeued, 0, rte_memory_order_relaxed); 3614 } 3615 3616 return TEST_SUCCESS; 3617 } 3618 3619 static int 3620 throughput_intr_lcore_enc(void *arg) 3621 { 3622 struct thread_params *tp = arg; 3623 unsigned int enqueued; 3624 const uint16_t queue_id = tp->queue_id; 3625 const uint16_t burst_sz = tp->op_params->burst_sz; 3626 const uint16_t num_to_process = tp->op_params->num_to_process; 3627 struct rte_bbdev_enc_op *ops[num_to_process]; 3628 struct test_buffers *bufs = NULL; 3629 struct rte_bbdev_info info; 3630 int ret, i, j; 3631 uint16_t num_to_enq, enq; 3632 3633 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 3634 "BURST_SIZE should be <= %u", MAX_BURST); 3635 3636 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id), 3637 "Failed to enable interrupts for dev: %u, queue_id: %u", 3638 tp->dev_id, queue_id); 3639 3640 rte_bbdev_info_get(tp->dev_id, &info); 3641 3642 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim), 3643 "NUM_OPS cannot exceed %u for this device", 3644 info.drv.queue_size_lim); 3645 3646 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 3647 3648 rte_atomic_store_explicit(&tp->processing_status, 0, rte_memory_order_relaxed); 3649 rte_atomic_store_explicit(&tp->nb_dequeued, 0, rte_memory_order_relaxed); 3650 3651 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->op_params->sync, SYNC_START, 3652 rte_memory_order_relaxed); 3653 3654 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops, 3655 num_to_process); 3656 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", 3657 num_to_process); 3658 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3659 copy_reference_enc_op(ops, num_to_process, 0, bufs->inputs, 3660 bufs->hard_outputs, tp->op_params->ref_enc_op); 3661 3662 /* Set counter to validate the ordering */ 3663 for (j = 0; j < num_to_process; ++j) 3664 ops[j]->opaque_data = (void *)(uintptr_t)j; 3665 3666 for (j = 0; j < TEST_REPETITIONS; ++j) { 3667 for (i = 0; i < num_to_process; ++i) 3668 mbuf_reset(ops[i]->turbo_enc.output.data); 3669 3670 tp->start_time = rte_rdtsc_precise(); 3671 for (enqueued = 0; enqueued < num_to_process;) { 3672 num_to_enq = burst_sz; 3673 3674 if (unlikely(num_to_process - enqueued < num_to_enq)) 3675 num_to_enq = num_to_process - enqueued; 3676 3677 /* Write to thread burst_sz current number of enqueued 3678 * descriptors. It ensures that proper number of 3679 * descriptors will be dequeued in callback 3680 * function - needed for last batch in case where 3681 * the number of operations is not a multiple of 3682 * burst size. 3683 */ 3684 rte_atomic_store_explicit(&tp->burst_sz, num_to_enq, 3685 rte_memory_order_relaxed); 3686 3687 enq = 0; 3688 do { 3689 enq += rte_bbdev_enqueue_enc_ops(tp->dev_id, 3690 queue_id, &ops[enqueued], 3691 num_to_enq); 3692 } while (unlikely(enq != num_to_enq)); 3693 enqueued += enq; 3694 3695 /* Wait until processing of previous batch is 3696 * completed 3697 */ 3698 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->nb_dequeued, enqueued, 3699 rte_memory_order_relaxed); 3700 } 3701 if (j != TEST_REPETITIONS - 1) 3702 rte_atomic_store_explicit(&tp->nb_dequeued, 0, rte_memory_order_relaxed); 3703 } 3704 3705 return TEST_SUCCESS; 3706 } 3707 3708 3709 static int 3710 throughput_intr_lcore_ldpc_enc(void *arg) 3711 { 3712 struct thread_params *tp = arg; 3713 unsigned int enqueued; 3714 const uint16_t queue_id = tp->queue_id; 3715 const uint16_t burst_sz = tp->op_params->burst_sz; 3716 const uint16_t num_to_process = tp->op_params->num_to_process; 3717 struct rte_bbdev_enc_op *ops[num_to_process]; 3718 struct test_buffers *bufs = NULL; 3719 struct rte_bbdev_info info; 3720 int ret, i, j; 3721 uint16_t num_to_enq, enq; 3722 3723 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 3724 "BURST_SIZE should be <= %u", MAX_BURST); 3725 3726 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id), 3727 "Failed to enable interrupts for dev: %u, queue_id: %u", 3728 tp->dev_id, queue_id); 3729 3730 rte_bbdev_info_get(tp->dev_id, &info); 3731 3732 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim), 3733 "NUM_OPS cannot exceed %u for this device", 3734 info.drv.queue_size_lim); 3735 3736 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 3737 3738 rte_atomic_store_explicit(&tp->processing_status, 0, rte_memory_order_relaxed); 3739 rte_atomic_store_explicit(&tp->nb_dequeued, 0, rte_memory_order_relaxed); 3740 3741 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->op_params->sync, SYNC_START, 3742 rte_memory_order_relaxed); 3743 3744 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops, 3745 num_to_process); 3746 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", 3747 num_to_process); 3748 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3749 copy_reference_ldpc_enc_op(ops, num_to_process, 0, 3750 bufs->inputs, bufs->hard_outputs, 3751 tp->op_params->ref_enc_op); 3752 3753 /* Set counter to validate the ordering */ 3754 for (j = 0; j < num_to_process; ++j) 3755 ops[j]->opaque_data = (void *)(uintptr_t)j; 3756 3757 for (j = 0; j < TEST_REPETITIONS; ++j) { 3758 for (i = 0; i < num_to_process; ++i) 3759 mbuf_reset(ops[i]->turbo_enc.output.data); 3760 3761 tp->start_time = rte_rdtsc_precise(); 3762 for (enqueued = 0; enqueued < num_to_process;) { 3763 num_to_enq = burst_sz; 3764 3765 if (unlikely(num_to_process - enqueued < num_to_enq)) 3766 num_to_enq = num_to_process - enqueued; 3767 3768 /* Write to thread burst_sz current number of enqueued 3769 * descriptors. It ensures that proper number of 3770 * descriptors will be dequeued in callback 3771 * function - needed for last batch in case where 3772 * the number of operations is not a multiple of 3773 * burst size. 3774 */ 3775 rte_atomic_store_explicit(&tp->burst_sz, num_to_enq, 3776 rte_memory_order_relaxed); 3777 3778 enq = 0; 3779 do { 3780 enq += rte_bbdev_enqueue_ldpc_enc_ops( 3781 tp->dev_id, 3782 queue_id, &ops[enqueued], 3783 num_to_enq); 3784 } while (unlikely(enq != num_to_enq)); 3785 enqueued += enq; 3786 3787 /* Wait until processing of previous batch is 3788 * completed 3789 */ 3790 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->nb_dequeued, enqueued, 3791 rte_memory_order_relaxed); 3792 } 3793 if (j != TEST_REPETITIONS - 1) 3794 rte_atomic_store_explicit(&tp->nb_dequeued, 0, rte_memory_order_relaxed); 3795 } 3796 3797 return TEST_SUCCESS; 3798 } 3799 3800 3801 static int 3802 throughput_intr_lcore_fft(void *arg) 3803 { 3804 struct thread_params *tp = arg; 3805 unsigned int enqueued; 3806 const uint16_t queue_id = tp->queue_id; 3807 const uint16_t burst_sz = tp->op_params->burst_sz; 3808 const uint16_t num_to_process = tp->op_params->num_to_process; 3809 struct rte_bbdev_fft_op *ops[num_to_process]; 3810 struct test_buffers *bufs = NULL; 3811 struct rte_bbdev_info info; 3812 int ret, i, j; 3813 uint16_t num_to_enq, enq; 3814 3815 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 3816 "BURST_SIZE should be <= %u", MAX_BURST); 3817 3818 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id), 3819 "Failed to enable interrupts for dev: %u, queue_id: %u", 3820 tp->dev_id, queue_id); 3821 3822 rte_bbdev_info_get(tp->dev_id, &info); 3823 3824 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim), 3825 "NUM_OPS cannot exceed %u for this device", 3826 info.drv.queue_size_lim); 3827 3828 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 3829 3830 rte_atomic_store_explicit(&tp->processing_status, 0, rte_memory_order_relaxed); 3831 rte_atomic_store_explicit(&tp->nb_dequeued, 0, rte_memory_order_relaxed); 3832 3833 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->op_params->sync, SYNC_START, 3834 rte_memory_order_relaxed); 3835 3836 ret = rte_bbdev_fft_op_alloc_bulk(tp->op_params->mp, ops, 3837 num_to_process); 3838 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", 3839 num_to_process); 3840 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3841 copy_reference_fft_op(ops, num_to_process, 0, bufs->inputs, 3842 bufs->hard_outputs, bufs->soft_outputs, bufs->harq_inputs, 3843 tp->op_params->ref_fft_op); 3844 3845 /* Set counter to validate the ordering */ 3846 for (j = 0; j < num_to_process; ++j) 3847 ops[j]->opaque_data = (void *)(uintptr_t)j; 3848 3849 for (j = 0; j < TEST_REPETITIONS; ++j) { 3850 for (i = 0; i < num_to_process; ++i) 3851 mbuf_reset(ops[i]->fft.base_output.data); 3852 3853 tp->start_time = rte_rdtsc_precise(); 3854 for (enqueued = 0; enqueued < num_to_process;) { 3855 num_to_enq = burst_sz; 3856 3857 if (unlikely(num_to_process - enqueued < num_to_enq)) 3858 num_to_enq = num_to_process - enqueued; 3859 3860 /* Write to thread burst_sz current number of enqueued 3861 * descriptors. It ensures that proper number of 3862 * descriptors will be dequeued in callback 3863 * function - needed for last batch in case where 3864 * the number of operations is not a multiple of 3865 * burst size. 3866 */ 3867 rte_atomic_store_explicit(&tp->burst_sz, num_to_enq, 3868 rte_memory_order_relaxed); 3869 3870 enq = 0; 3871 do { 3872 enq += rte_bbdev_enqueue_fft_ops(tp->dev_id, 3873 queue_id, &ops[enqueued], 3874 num_to_enq); 3875 } while (unlikely(enq != num_to_enq)); 3876 enqueued += enq; 3877 3878 /* Wait until processing of previous batch is 3879 * completed 3880 */ 3881 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->nb_dequeued, enqueued, 3882 rte_memory_order_relaxed); 3883 } 3884 if (j != TEST_REPETITIONS - 1) 3885 rte_atomic_store_explicit(&tp->nb_dequeued, 0, rte_memory_order_relaxed); 3886 } 3887 3888 return TEST_SUCCESS; 3889 } 3890 3891 static int 3892 throughput_intr_lcore_mldts(void *arg) 3893 { 3894 struct thread_params *tp = arg; 3895 unsigned int enqueued; 3896 const uint16_t queue_id = tp->queue_id; 3897 const uint16_t burst_sz = tp->op_params->burst_sz; 3898 const uint16_t num_to_process = tp->op_params->num_to_process; 3899 struct rte_bbdev_mldts_op *ops[num_to_process]; 3900 struct test_buffers *bufs = NULL; 3901 struct rte_bbdev_info info; 3902 int ret, i, j; 3903 uint16_t num_to_enq, enq; 3904 3905 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), "BURST_SIZE should be <= %u", MAX_BURST); 3906 3907 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id), 3908 "Failed to enable interrupts for dev: %u, queue_id: %u", 3909 tp->dev_id, queue_id); 3910 3911 rte_bbdev_info_get(tp->dev_id, &info); 3912 3913 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim), 3914 "NUM_OPS cannot exceed %u for this device", 3915 info.drv.queue_size_lim); 3916 3917 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 3918 3919 rte_atomic_store_explicit(&tp->processing_status, 0, rte_memory_order_relaxed); 3920 rte_atomic_store_explicit(&tp->nb_dequeued, 0, rte_memory_order_relaxed); 3921 3922 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->op_params->sync, SYNC_START, 3923 rte_memory_order_relaxed); 3924 3925 ret = rte_bbdev_mldts_op_alloc_bulk(tp->op_params->mp, ops, num_to_process); 3926 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_to_process); 3927 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3928 copy_reference_mldts_op(ops, num_to_process, 0, bufs->inputs, bufs->harq_inputs, 3929 bufs->hard_outputs, tp->op_params->ref_mldts_op); 3930 3931 /* Set counter to validate the ordering */ 3932 for (j = 0; j < num_to_process; ++j) 3933 ops[j]->opaque_data = (void *)(uintptr_t)j; 3934 3935 for (j = 0; j < TEST_REPETITIONS; ++j) { 3936 for (i = 0; i < num_to_process; ++i) 3937 mbuf_reset(ops[i]->mldts.output.data); 3938 3939 tp->start_time = rte_rdtsc_precise(); 3940 for (enqueued = 0; enqueued < num_to_process;) { 3941 num_to_enq = burst_sz; 3942 3943 if (unlikely(num_to_process - enqueued < num_to_enq)) 3944 num_to_enq = num_to_process - enqueued; 3945 3946 /* Write to thread burst_sz current number of enqueued 3947 * descriptors. It ensures that proper number of 3948 * descriptors will be dequeued in callback 3949 * function - needed for last batch in case where 3950 * the number of operations is not a multiple of 3951 * burst size. 3952 */ 3953 rte_atomic_store_explicit(&tp->burst_sz, num_to_enq, 3954 rte_memory_order_relaxed); 3955 3956 enq = 0; 3957 do { 3958 enq += rte_bbdev_enqueue_mldts_ops(tp->dev_id, 3959 queue_id, &ops[enqueued], num_to_enq); 3960 } while (unlikely(enq != num_to_enq)); 3961 enqueued += enq; 3962 3963 /* Wait until processing of previous batch is 3964 * completed 3965 */ 3966 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->nb_dequeued, enqueued, 3967 rte_memory_order_relaxed); 3968 } 3969 if (j != TEST_REPETITIONS - 1) 3970 rte_atomic_store_explicit(&tp->nb_dequeued, 0, rte_memory_order_relaxed); 3971 } 3972 3973 return TEST_SUCCESS; 3974 } 3975 3976 static int 3977 throughput_pmd_lcore_dec(void *arg) 3978 { 3979 struct thread_params *tp = arg; 3980 uint16_t enq, deq; 3981 uint64_t total_time = 0, start_time; 3982 const uint16_t queue_id = tp->queue_id; 3983 const uint16_t burst_sz = tp->op_params->burst_sz; 3984 const uint16_t num_ops = tp->op_params->num_to_process; 3985 struct rte_bbdev_dec_op *ops_enq[num_ops]; 3986 struct rte_bbdev_dec_op *ops_deq[num_ops]; 3987 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 3988 struct test_buffers *bufs = NULL; 3989 int i, j, ret; 3990 struct rte_bbdev_info info; 3991 uint16_t num_to_enq; 3992 bool so_enable; 3993 3994 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 3995 "BURST_SIZE should be <= %u", MAX_BURST); 3996 3997 rte_bbdev_info_get(tp->dev_id, &info); 3998 3999 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 4000 "NUM_OPS cannot exceed %u for this device", 4001 info.drv.queue_size_lim); 4002 4003 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 4004 4005 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->op_params->sync, SYNC_START, 4006 rte_memory_order_relaxed); 4007 4008 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops); 4009 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops); 4010 ref_op->turbo_dec.iter_max = get_iter_max(); 4011 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4012 copy_reference_dec_op(ops_enq, num_ops, 0, bufs->inputs, 4013 bufs->hard_outputs, bufs->soft_outputs, ref_op); 4014 4015 so_enable = check_bit(ops_enq[0]->turbo_dec.op_flags, RTE_BBDEV_TURBO_SOFT_OUTPUT); 4016 4017 /* Set counter to validate the ordering */ 4018 for (j = 0; j < num_ops; ++j) 4019 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 4020 4021 for (i = 0; i < TEST_REPETITIONS; ++i) { 4022 uint32_t time_out = 0; 4023 for (j = 0; j < num_ops; ++j) 4024 mbuf_reset(ops_enq[j]->turbo_dec.hard_output.data); 4025 if (so_enable) 4026 for (j = 0; j < num_ops; ++j) 4027 mbuf_reset(ops_enq[j]->turbo_dec.soft_output.data); 4028 4029 start_time = rte_rdtsc_precise(); 4030 4031 for (enq = 0, deq = 0; enq < num_ops;) { 4032 num_to_enq = burst_sz; 4033 4034 if (unlikely(num_ops - enq < num_to_enq)) 4035 num_to_enq = num_ops - enq; 4036 4037 enq += rte_bbdev_enqueue_dec_ops(tp->dev_id, 4038 queue_id, &ops_enq[enq], num_to_enq); 4039 4040 deq += rte_bbdev_dequeue_dec_ops(tp->dev_id, 4041 queue_id, &ops_deq[deq], enq - deq); 4042 time_out++; 4043 if (time_out >= TIME_OUT_POLL) { 4044 timeout_exit(tp->dev_id); 4045 TEST_ASSERT_SUCCESS(TEST_FAILED, "Enqueue timeout!"); 4046 } 4047 } 4048 4049 /* dequeue the remaining */ 4050 time_out = 0; 4051 while (deq < enq) { 4052 deq += rte_bbdev_dequeue_dec_ops(tp->dev_id, 4053 queue_id, &ops_deq[deq], enq - deq); 4054 time_out++; 4055 if (time_out >= TIME_OUT_POLL) { 4056 timeout_exit(tp->dev_id); 4057 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 4058 } 4059 } 4060 4061 total_time += rte_rdtsc_precise() - start_time; 4062 } 4063 4064 tp->iter_count = 0; 4065 /* get the max of iter_count for all dequeued ops */ 4066 for (i = 0; i < num_ops; ++i) { 4067 tp->iter_count = RTE_MAX(ops_enq[i]->turbo_dec.iter_count, 4068 tp->iter_count); 4069 } 4070 4071 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 4072 ret = validate_dec_op(ops_deq, num_ops, ref_op); 4073 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 4074 } 4075 4076 rte_bbdev_dec_op_free_bulk(ops_enq, num_ops); 4077 4078 double tb_len_bits = calc_dec_TB_size(ref_op); 4079 4080 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) / 4081 ((double)total_time / (double)rte_get_tsc_hz()); 4082 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) / 4083 1000000.0) / ((double)total_time / 4084 (double)rte_get_tsc_hz()); 4085 4086 return TEST_SUCCESS; 4087 } 4088 4089 static int 4090 bler_pmd_lcore_ldpc_dec(void *arg) 4091 { 4092 struct thread_params *tp = arg; 4093 uint16_t enq, deq; 4094 uint64_t total_time = 0, start_time; 4095 const uint16_t queue_id = tp->queue_id; 4096 const uint16_t burst_sz = tp->op_params->burst_sz; 4097 const uint16_t num_ops = tp->op_params->num_to_process; 4098 struct rte_bbdev_dec_op *ops_enq[num_ops]; 4099 struct rte_bbdev_dec_op *ops_deq[num_ops]; 4100 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 4101 struct test_buffers *bufs = NULL; 4102 int i, j, ret; 4103 float parity_bler = 0; 4104 struct rte_bbdev_info info; 4105 uint16_t num_to_enq; 4106 bool extDdr = check_bit(ldpc_cap_flags, 4107 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE); 4108 bool loopback = check_bit(ref_op->ldpc_dec.op_flags, 4109 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK); 4110 bool hc_out = check_bit(ref_op->ldpc_dec.op_flags, 4111 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE); 4112 4113 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 4114 "BURST_SIZE should be <= %u", MAX_BURST); 4115 TEST_ASSERT_SUCCESS((num_ops == 0), "NUM_OPS must be greater than 0"); 4116 4117 rte_bbdev_info_get(tp->dev_id, &info); 4118 4119 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 4120 "NUM_OPS cannot exceed %u for this device", 4121 info.drv.queue_size_lim); 4122 4123 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 4124 4125 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->op_params->sync, SYNC_START, 4126 rte_memory_order_relaxed); 4127 4128 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops); 4129 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops); 4130 4131 /* For BLER tests we need to enable early termination */ 4132 if (!check_bit(ref_op->ldpc_dec.op_flags, RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE)) 4133 ref_op->ldpc_dec.op_flags += RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE; 4134 4135 ref_op->ldpc_dec.iter_max = get_iter_max(); 4136 4137 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4138 copy_reference_ldpc_dec_op(ops_enq, num_ops, 0, bufs->inputs, 4139 bufs->hard_outputs, bufs->soft_outputs, 4140 bufs->harq_inputs, bufs->harq_outputs, ref_op); 4141 generate_llr_input(num_ops, bufs->inputs, ref_op); 4142 4143 /* Set counter to validate the ordering */ 4144 for (j = 0; j < num_ops; ++j) 4145 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 4146 4147 for (i = 0; i < 1; ++i) { /* Could add more iterations */ 4148 uint32_t time_out = 0; 4149 for (j = 0; j < num_ops; ++j) { 4150 if (!loopback) 4151 mbuf_reset(ops_enq[j]->ldpc_dec.hard_output.data); 4152 if (hc_out || loopback) 4153 mbuf_reset(ops_enq[j]->ldpc_dec.harq_combined_output.data); 4154 if (ops_enq[j]->ldpc_dec.soft_output.data != NULL) 4155 mbuf_reset(ops_enq[j]->ldpc_dec.soft_output.data); 4156 } 4157 if (extDdr) 4158 preload_harq_ddr(tp->dev_id, queue_id, ops_enq, 4159 num_ops, true); 4160 start_time = rte_rdtsc_precise(); 4161 4162 for (enq = 0, deq = 0; enq < num_ops;) { 4163 num_to_enq = burst_sz; 4164 4165 if (unlikely(num_ops - enq < num_to_enq)) 4166 num_to_enq = num_ops - enq; 4167 4168 enq += rte_bbdev_enqueue_ldpc_dec_ops(tp->dev_id, 4169 queue_id, &ops_enq[enq], num_to_enq); 4170 4171 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id, 4172 queue_id, &ops_deq[deq], enq - deq); 4173 time_out++; 4174 if (time_out >= TIME_OUT_POLL) { 4175 timeout_exit(tp->dev_id); 4176 TEST_ASSERT_SUCCESS(TEST_FAILED, "Enqueue timeout!"); 4177 } 4178 } 4179 4180 /* dequeue the remaining */ 4181 time_out = 0; 4182 while (deq < enq) { 4183 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id, 4184 queue_id, &ops_deq[deq], enq - deq); 4185 time_out++; 4186 if (time_out >= TIME_OUT_POLL) { 4187 timeout_exit(tp->dev_id); 4188 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 4189 } 4190 } 4191 4192 total_time += rte_rdtsc_precise() - start_time; 4193 } 4194 4195 tp->iter_count = 0; 4196 tp->iter_average = 0; 4197 /* get the max of iter_count for all dequeued ops */ 4198 for (i = 0; i < num_ops; ++i) { 4199 tp->iter_count = RTE_MAX(ops_enq[i]->ldpc_dec.iter_count, 4200 tp->iter_count); 4201 tp->iter_average += (double) ops_enq[i]->ldpc_dec.iter_count; 4202 if (ops_enq[i]->status & (1 << RTE_BBDEV_SYNDROME_ERROR)) 4203 parity_bler += 1.0; 4204 } 4205 4206 parity_bler /= num_ops; /* This one is based on SYND */ 4207 tp->iter_average /= num_ops; 4208 tp->bler = (double) validate_ldpc_bler(ops_deq, num_ops) / num_ops; 4209 4210 if (test_vector.op_type != RTE_BBDEV_OP_NONE 4211 && tp->bler == 0 4212 && parity_bler == 0 4213 && !hc_out) { 4214 ret = validate_ldpc_dec_op(ops_deq, num_ops, ref_op, 4215 tp->op_params->vector_mask); 4216 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 4217 } 4218 4219 rte_bbdev_dec_op_free_bulk(ops_enq, num_ops); 4220 4221 double tb_len_bits = calc_ldpc_dec_TB_size(ref_op); 4222 tp->ops_per_sec = ((double)num_ops * 1) / 4223 ((double)total_time / (double)rte_get_tsc_hz()); 4224 tp->mbps = (((double)(num_ops * 1 * tb_len_bits)) / 4225 1000000.0) / ((double)total_time / 4226 (double)rte_get_tsc_hz()); 4227 4228 return TEST_SUCCESS; 4229 } 4230 4231 4232 static int 4233 bler_pmd_lcore_turbo_dec(void *arg) 4234 { 4235 struct thread_params *tp = arg; 4236 uint16_t enq, deq; 4237 uint64_t total_time = 0, start_time; 4238 const uint16_t queue_id = tp->queue_id; 4239 const uint16_t burst_sz = tp->op_params->burst_sz; 4240 const uint16_t num_ops = tp->op_params->num_to_process; 4241 struct rte_bbdev_dec_op *ops_enq[num_ops]; 4242 struct rte_bbdev_dec_op *ops_deq[num_ops]; 4243 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 4244 struct test_buffers *bufs = NULL; 4245 int i, j, ret; 4246 struct rte_bbdev_info info; 4247 uint16_t num_to_enq; 4248 4249 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 4250 "BURST_SIZE should be <= %u", MAX_BURST); 4251 TEST_ASSERT_SUCCESS((num_ops == 0), "NUM_OPS must be greater than 0"); 4252 4253 rte_bbdev_info_get(tp->dev_id, &info); 4254 4255 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 4256 "NUM_OPS cannot exceed %u for this device", 4257 info.drv.queue_size_lim); 4258 4259 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 4260 4261 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->op_params->sync, SYNC_START, 4262 rte_memory_order_relaxed); 4263 4264 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops); 4265 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops); 4266 4267 /* For BLER tests we need to enable early termination */ 4268 if (!check_bit(ref_op->turbo_dec.op_flags, RTE_BBDEV_TURBO_EARLY_TERMINATION)) 4269 ref_op->turbo_dec.op_flags += RTE_BBDEV_TURBO_EARLY_TERMINATION; 4270 4271 ref_op->turbo_dec.iter_max = get_iter_max(); 4272 4273 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4274 copy_reference_dec_op(ops_enq, num_ops, 0, bufs->inputs, 4275 bufs->hard_outputs, bufs->soft_outputs, 4276 ref_op); 4277 generate_turbo_llr_input(num_ops, bufs->inputs, ref_op); 4278 4279 /* Set counter to validate the ordering */ 4280 for (j = 0; j < num_ops; ++j) 4281 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 4282 4283 for (i = 0; i < 1; ++i) { /* Could add more iterations */ 4284 uint32_t time_out = 0; 4285 for (j = 0; j < num_ops; ++j) { 4286 mbuf_reset( 4287 ops_enq[j]->turbo_dec.hard_output.data); 4288 } 4289 4290 start_time = rte_rdtsc_precise(); 4291 4292 for (enq = 0, deq = 0; enq < num_ops;) { 4293 num_to_enq = burst_sz; 4294 4295 if (unlikely(num_ops - enq < num_to_enq)) 4296 num_to_enq = num_ops - enq; 4297 4298 enq += rte_bbdev_enqueue_dec_ops(tp->dev_id, 4299 queue_id, &ops_enq[enq], num_to_enq); 4300 4301 deq += rte_bbdev_dequeue_dec_ops(tp->dev_id, 4302 queue_id, &ops_deq[deq], enq - deq); 4303 time_out++; 4304 if (time_out >= TIME_OUT_POLL) { 4305 timeout_exit(tp->dev_id); 4306 TEST_ASSERT_SUCCESS(TEST_FAILED, "Enqueue timeout!"); 4307 } 4308 } 4309 4310 /* dequeue the remaining */ 4311 time_out = 0; 4312 while (deq < enq) { 4313 deq += rte_bbdev_dequeue_dec_ops(tp->dev_id, 4314 queue_id, &ops_deq[deq], enq - deq); 4315 time_out++; 4316 if (time_out >= TIME_OUT_POLL) { 4317 timeout_exit(tp->dev_id); 4318 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 4319 } 4320 } 4321 4322 total_time += rte_rdtsc_precise() - start_time; 4323 } 4324 4325 tp->iter_count = 0; 4326 tp->iter_average = 0; 4327 /* get the max of iter_count for all dequeued ops */ 4328 for (i = 0; i < num_ops; ++i) { 4329 tp->iter_count = RTE_MAX(ops_enq[i]->turbo_dec.iter_count, 4330 tp->iter_count); 4331 tp->iter_average += (double) ops_enq[i]->turbo_dec.iter_count; 4332 } 4333 4334 tp->iter_average /= num_ops; 4335 tp->bler = (double) validate_turbo_bler(ops_deq, num_ops) / num_ops; 4336 4337 rte_bbdev_dec_op_free_bulk(ops_enq, num_ops); 4338 4339 double tb_len_bits = calc_dec_TB_size(ref_op); 4340 tp->ops_per_sec = ((double)num_ops * 1) / 4341 ((double)total_time / (double)rte_get_tsc_hz()); 4342 tp->mbps = (((double)(num_ops * 1 * tb_len_bits)) / 4343 1000000.0) / ((double)total_time / 4344 (double)rte_get_tsc_hz()); 4345 printf("TBS %.0f Time %.0f\n", tb_len_bits, 1000000.0 * 4346 ((double)total_time / (double)rte_get_tsc_hz())); 4347 4348 return TEST_SUCCESS; 4349 } 4350 4351 static int 4352 throughput_pmd_lcore_ldpc_dec(void *arg) 4353 { 4354 struct thread_params *tp = arg; 4355 uint16_t enq, deq; 4356 uint64_t total_time = 0, start_time; 4357 const uint16_t queue_id = tp->queue_id; 4358 const uint16_t burst_sz = tp->op_params->burst_sz; 4359 const uint16_t num_ops = tp->op_params->num_to_process; 4360 struct rte_bbdev_dec_op *ops_enq[num_ops]; 4361 struct rte_bbdev_dec_op *ops_deq[num_ops]; 4362 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 4363 struct test_buffers *bufs = NULL; 4364 int i, j, ret; 4365 struct rte_bbdev_info info; 4366 uint16_t num_to_enq; 4367 bool extDdr = check_bit(ldpc_cap_flags, 4368 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE); 4369 bool loopback = check_bit(ref_op->ldpc_dec.op_flags, 4370 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK); 4371 bool hc_out = check_bit(ref_op->ldpc_dec.op_flags, 4372 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE); 4373 4374 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 4375 "BURST_SIZE should be <= %u", MAX_BURST); 4376 4377 rte_bbdev_info_get(tp->dev_id, &info); 4378 4379 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 4380 "NUM_OPS cannot exceed %u for this device", 4381 info.drv.queue_size_lim); 4382 4383 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 4384 4385 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->op_params->sync, SYNC_START, 4386 rte_memory_order_relaxed); 4387 4388 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops); 4389 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops); 4390 4391 /* For throughput tests we need to disable early termination */ 4392 if (check_bit(ref_op->ldpc_dec.op_flags, RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE)) 4393 ref_op->ldpc_dec.op_flags -= RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE; 4394 4395 ref_op->ldpc_dec.iter_max = get_iter_max(); 4396 /* Since ET is disabled, the expected iter_count is iter_max */ 4397 ref_op->ldpc_dec.iter_count = ref_op->ldpc_dec.iter_max; 4398 4399 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4400 copy_reference_ldpc_dec_op(ops_enq, num_ops, 0, bufs->inputs, 4401 bufs->hard_outputs, bufs->soft_outputs, 4402 bufs->harq_inputs, bufs->harq_outputs, ref_op); 4403 4404 /* Set counter to validate the ordering */ 4405 for (j = 0; j < num_ops; ++j) 4406 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 4407 4408 for (i = 0; i < TEST_REPETITIONS; ++i) { 4409 uint32_t time_out = 0; 4410 for (j = 0; j < num_ops; ++j) { 4411 if (!loopback) 4412 mbuf_reset(ops_enq[j]->ldpc_dec.hard_output.data); 4413 if (hc_out || loopback) 4414 mbuf_reset(ops_enq[j]->ldpc_dec.harq_combined_output.data); 4415 if (ops_enq[j]->ldpc_dec.soft_output.data != NULL) 4416 mbuf_reset(ops_enq[j]->ldpc_dec.soft_output.data); 4417 } 4418 if (extDdr) 4419 preload_harq_ddr(tp->dev_id, queue_id, ops_enq, 4420 num_ops, true); 4421 start_time = rte_rdtsc_precise(); 4422 4423 for (enq = 0, deq = 0; enq < num_ops;) { 4424 num_to_enq = burst_sz; 4425 4426 if (unlikely(num_ops - enq < num_to_enq)) 4427 num_to_enq = num_ops - enq; 4428 4429 enq += rte_bbdev_enqueue_ldpc_dec_ops(tp->dev_id, 4430 queue_id, &ops_enq[enq], num_to_enq); 4431 4432 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id, 4433 queue_id, &ops_deq[deq], enq - deq); 4434 time_out++; 4435 if (time_out >= TIME_OUT_POLL) { 4436 timeout_exit(tp->dev_id); 4437 TEST_ASSERT_SUCCESS(TEST_FAILED, "Enqueue timeout!"); 4438 } 4439 } 4440 4441 /* dequeue the remaining */ 4442 time_out = 0; 4443 while (deq < enq) { 4444 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id, 4445 queue_id, &ops_deq[deq], enq - deq); 4446 time_out++; 4447 if (time_out >= TIME_OUT_POLL) { 4448 timeout_exit(tp->dev_id); 4449 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 4450 } 4451 } 4452 4453 total_time += rte_rdtsc_precise() - start_time; 4454 } 4455 4456 tp->iter_count = 0; 4457 /* get the max of iter_count for all dequeued ops */ 4458 for (i = 0; i < num_ops; ++i) { 4459 tp->iter_count = RTE_MAX(ops_enq[i]->ldpc_dec.iter_count, 4460 tp->iter_count); 4461 } 4462 if (extDdr) { 4463 /* Read loopback is not thread safe */ 4464 retrieve_harq_ddr(tp->dev_id, queue_id, ops_enq, num_ops); 4465 } 4466 4467 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 4468 ret = validate_ldpc_dec_op(ops_deq, num_ops, ref_op, 4469 tp->op_params->vector_mask); 4470 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 4471 } 4472 4473 ret = rte_bbdev_queue_stop(tp->dev_id, queue_id); 4474 if (ret != 0) 4475 printf("Failed to stop queue on dev %u q_id: %u\n", tp->dev_id, queue_id); 4476 rte_bbdev_dec_op_free_bulk(ops_enq, num_ops); 4477 4478 double tb_len_bits = calc_ldpc_dec_TB_size(ref_op); 4479 4480 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) / 4481 ((double)total_time / (double)rte_get_tsc_hz()); 4482 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) / 4483 1000000.0) / ((double)total_time / 4484 (double)rte_get_tsc_hz()); 4485 4486 return TEST_SUCCESS; 4487 } 4488 4489 static int 4490 throughput_pmd_lcore_enc(void *arg) 4491 { 4492 struct thread_params *tp = arg; 4493 uint16_t enq, deq; 4494 uint64_t total_time = 0, start_time; 4495 const uint16_t queue_id = tp->queue_id; 4496 const uint16_t burst_sz = tp->op_params->burst_sz; 4497 const uint16_t num_ops = tp->op_params->num_to_process; 4498 struct rte_bbdev_enc_op *ops_enq[num_ops]; 4499 struct rte_bbdev_enc_op *ops_deq[num_ops]; 4500 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op; 4501 struct test_buffers *bufs = NULL; 4502 int i, j, ret; 4503 struct rte_bbdev_info info; 4504 uint16_t num_to_enq; 4505 4506 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 4507 "BURST_SIZE should be <= %u", MAX_BURST); 4508 4509 rte_bbdev_info_get(tp->dev_id, &info); 4510 4511 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 4512 "NUM_OPS cannot exceed %u for this device", 4513 info.drv.queue_size_lim); 4514 4515 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 4516 4517 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->op_params->sync, SYNC_START, 4518 rte_memory_order_relaxed); 4519 4520 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops_enq, 4521 num_ops); 4522 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", 4523 num_ops); 4524 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4525 copy_reference_enc_op(ops_enq, num_ops, 0, bufs->inputs, 4526 bufs->hard_outputs, ref_op); 4527 4528 /* Set counter to validate the ordering */ 4529 for (j = 0; j < num_ops; ++j) 4530 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 4531 4532 for (i = 0; i < TEST_REPETITIONS; ++i) { 4533 uint32_t time_out = 0; 4534 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4535 for (j = 0; j < num_ops; ++j) 4536 mbuf_reset(ops_enq[j]->turbo_enc.output.data); 4537 4538 start_time = rte_rdtsc_precise(); 4539 4540 for (enq = 0, deq = 0; enq < num_ops;) { 4541 num_to_enq = burst_sz; 4542 4543 if (unlikely(num_ops - enq < num_to_enq)) 4544 num_to_enq = num_ops - enq; 4545 4546 enq += rte_bbdev_enqueue_enc_ops(tp->dev_id, 4547 queue_id, &ops_enq[enq], num_to_enq); 4548 4549 deq += rte_bbdev_dequeue_enc_ops(tp->dev_id, 4550 queue_id, &ops_deq[deq], enq - deq); 4551 time_out++; 4552 if (time_out >= TIME_OUT_POLL) { 4553 timeout_exit(tp->dev_id); 4554 TEST_ASSERT_SUCCESS(TEST_FAILED, "Enqueue timeout!"); 4555 } 4556 } 4557 4558 /* dequeue the remaining */ 4559 time_out = 0; 4560 while (deq < enq) { 4561 deq += rte_bbdev_dequeue_enc_ops(tp->dev_id, 4562 queue_id, &ops_deq[deq], enq - deq); 4563 time_out++; 4564 if (time_out >= TIME_OUT_POLL) { 4565 timeout_exit(tp->dev_id); 4566 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 4567 } 4568 } 4569 4570 total_time += rte_rdtsc_precise() - start_time; 4571 } 4572 4573 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 4574 ret = validate_enc_op(ops_deq, num_ops, ref_op); 4575 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 4576 } 4577 4578 rte_bbdev_enc_op_free_bulk(ops_enq, num_ops); 4579 4580 double tb_len_bits = calc_enc_TB_size(ref_op); 4581 4582 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) / 4583 ((double)total_time / (double)rte_get_tsc_hz()); 4584 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) 4585 / 1000000.0) / ((double)total_time / 4586 (double)rte_get_tsc_hz()); 4587 4588 return TEST_SUCCESS; 4589 } 4590 4591 static int 4592 throughput_pmd_lcore_ldpc_enc(void *arg) 4593 { 4594 struct thread_params *tp = arg; 4595 uint16_t enq, deq; 4596 uint64_t total_time = 0, start_time; 4597 const uint16_t queue_id = tp->queue_id; 4598 const uint16_t burst_sz = tp->op_params->burst_sz; 4599 const uint16_t num_ops = tp->op_params->num_to_process; 4600 struct rte_bbdev_enc_op *ops_enq[num_ops]; 4601 struct rte_bbdev_enc_op *ops_deq[num_ops]; 4602 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op; 4603 struct test_buffers *bufs = NULL; 4604 int i, j, ret; 4605 struct rte_bbdev_info info; 4606 uint16_t num_to_enq; 4607 4608 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 4609 "BURST_SIZE should be <= %u", MAX_BURST); 4610 4611 rte_bbdev_info_get(tp->dev_id, &info); 4612 4613 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 4614 "NUM_OPS cannot exceed %u for this device", 4615 info.drv.queue_size_lim); 4616 4617 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 4618 4619 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->op_params->sync, SYNC_START, 4620 rte_memory_order_relaxed); 4621 4622 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops_enq, 4623 num_ops); 4624 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", 4625 num_ops); 4626 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4627 copy_reference_ldpc_enc_op(ops_enq, num_ops, 0, bufs->inputs, 4628 bufs->hard_outputs, ref_op); 4629 4630 /* Set counter to validate the ordering */ 4631 for (j = 0; j < num_ops; ++j) 4632 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 4633 4634 for (i = 0; i < TEST_REPETITIONS; ++i) { 4635 uint32_t time_out = 0; 4636 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4637 for (j = 0; j < num_ops; ++j) 4638 mbuf_reset(ops_enq[j]->turbo_enc.output.data); 4639 4640 start_time = rte_rdtsc_precise(); 4641 4642 for (enq = 0, deq = 0; enq < num_ops;) { 4643 num_to_enq = burst_sz; 4644 4645 if (unlikely(num_ops - enq < num_to_enq)) 4646 num_to_enq = num_ops - enq; 4647 4648 enq += rte_bbdev_enqueue_ldpc_enc_ops(tp->dev_id, 4649 queue_id, &ops_enq[enq], num_to_enq); 4650 4651 deq += rte_bbdev_dequeue_ldpc_enc_ops(tp->dev_id, 4652 queue_id, &ops_deq[deq], enq - deq); 4653 time_out++; 4654 if (time_out >= TIME_OUT_POLL) { 4655 timeout_exit(tp->dev_id); 4656 TEST_ASSERT_SUCCESS(TEST_FAILED, "Enqueue timeout!"); 4657 } 4658 } 4659 4660 /* dequeue the remaining */ 4661 time_out = 0; 4662 while (deq < enq) { 4663 deq += rte_bbdev_dequeue_ldpc_enc_ops(tp->dev_id, 4664 queue_id, &ops_deq[deq], enq - deq); 4665 time_out++; 4666 if (time_out >= TIME_OUT_POLL) { 4667 timeout_exit(tp->dev_id); 4668 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 4669 } 4670 } 4671 4672 total_time += rte_rdtsc_precise() - start_time; 4673 } 4674 4675 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 4676 ret = validate_ldpc_enc_op(ops_deq, num_ops, ref_op); 4677 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 4678 } 4679 4680 rte_bbdev_enc_op_free_bulk(ops_enq, num_ops); 4681 4682 double tb_len_bits = calc_ldpc_enc_TB_size(ref_op); 4683 4684 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) / 4685 ((double)total_time / (double)rte_get_tsc_hz()); 4686 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) 4687 / 1000000.0) / ((double)total_time / 4688 (double)rte_get_tsc_hz()); 4689 4690 return TEST_SUCCESS; 4691 } 4692 4693 static int 4694 throughput_pmd_lcore_fft(void *arg) 4695 { 4696 struct thread_params *tp = arg; 4697 uint16_t enq, deq; 4698 uint64_t total_time = 0, start_time; 4699 const uint16_t queue_id = tp->queue_id; 4700 const uint16_t burst_sz = tp->op_params->burst_sz; 4701 const uint16_t num_ops = tp->op_params->num_to_process; 4702 struct rte_bbdev_fft_op *ops_enq[num_ops]; 4703 struct rte_bbdev_fft_op *ops_deq[num_ops]; 4704 struct rte_bbdev_fft_op *ref_op = tp->op_params->ref_fft_op; 4705 struct test_buffers *bufs = NULL; 4706 int i, j, ret; 4707 struct rte_bbdev_info info; 4708 uint16_t num_to_enq; 4709 4710 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 4711 "BURST_SIZE should be <= %u", MAX_BURST); 4712 4713 rte_bbdev_info_get(tp->dev_id, &info); 4714 4715 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 4716 "NUM_OPS cannot exceed %u for this device", 4717 info.drv.queue_size_lim); 4718 4719 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 4720 4721 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->op_params->sync, SYNC_START, 4722 rte_memory_order_relaxed); 4723 4724 ret = rte_bbdev_fft_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops); 4725 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops); 4726 4727 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4728 copy_reference_fft_op(ops_enq, num_ops, 0, bufs->inputs, 4729 bufs->hard_outputs, bufs->soft_outputs, bufs->harq_inputs, ref_op); 4730 4731 /* Set counter to validate the ordering */ 4732 for (j = 0; j < num_ops; ++j) 4733 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 4734 4735 for (i = 0; i < TEST_REPETITIONS; ++i) { 4736 uint32_t time_out = 0; 4737 for (j = 0; j < num_ops; ++j) 4738 mbuf_reset(ops_enq[j]->fft.base_output.data); 4739 4740 start_time = rte_rdtsc_precise(); 4741 4742 for (enq = 0, deq = 0; enq < num_ops;) { 4743 num_to_enq = burst_sz; 4744 4745 if (unlikely(num_ops - enq < num_to_enq)) 4746 num_to_enq = num_ops - enq; 4747 4748 enq += rte_bbdev_enqueue_fft_ops(tp->dev_id, 4749 queue_id, &ops_enq[enq], num_to_enq); 4750 4751 deq += rte_bbdev_dequeue_fft_ops(tp->dev_id, 4752 queue_id, &ops_deq[deq], enq - deq); 4753 time_out++; 4754 if (time_out >= TIME_OUT_POLL) { 4755 timeout_exit(tp->dev_id); 4756 TEST_ASSERT_SUCCESS(TEST_FAILED, "Enqueue timeout!"); 4757 } 4758 } 4759 4760 /* dequeue the remaining */ 4761 time_out = 0; 4762 while (deq < enq) { 4763 deq += rte_bbdev_dequeue_fft_ops(tp->dev_id, 4764 queue_id, &ops_deq[deq], enq - deq); 4765 time_out++; 4766 if (time_out >= TIME_OUT_POLL) { 4767 timeout_exit(tp->dev_id); 4768 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 4769 } 4770 } 4771 4772 total_time += rte_rdtsc_precise() - start_time; 4773 } 4774 4775 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 4776 ret = validate_fft_op(ops_deq, num_ops, ref_op); 4777 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 4778 } 4779 4780 rte_bbdev_fft_op_free_bulk(ops_enq, num_ops); 4781 4782 double tb_len_bits = calc_fft_size(ref_op); 4783 4784 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) / 4785 ((double)total_time / (double)rte_get_tsc_hz()); 4786 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) / 4787 1000000.0) / ((double)total_time / 4788 (double)rte_get_tsc_hz()); 4789 4790 return TEST_SUCCESS; 4791 } 4792 4793 static int 4794 throughput_pmd_lcore_mldts(void *arg) 4795 { 4796 struct thread_params *tp = arg; 4797 uint16_t enq, deq; 4798 uint64_t total_time = 0, start_time; 4799 const uint16_t queue_id = tp->queue_id; 4800 const uint16_t burst_sz = tp->op_params->burst_sz; 4801 const uint16_t num_ops = tp->op_params->num_to_process; 4802 struct rte_bbdev_mldts_op *ops_enq[num_ops]; 4803 struct rte_bbdev_mldts_op *ops_deq[num_ops]; 4804 struct rte_bbdev_mldts_op *ref_op = tp->op_params->ref_mldts_op; 4805 struct test_buffers *bufs = NULL; 4806 int i, j, ret; 4807 struct rte_bbdev_info info; 4808 uint16_t num_to_enq; 4809 4810 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), "BURST_SIZE should be <= %u", MAX_BURST); 4811 4812 rte_bbdev_info_get(tp->dev_id, &info); 4813 4814 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 4815 "NUM_OPS cannot exceed %u for this device", 4816 info.drv.queue_size_lim); 4817 4818 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 4819 4820 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->op_params->sync, SYNC_START, 4821 rte_memory_order_relaxed); 4822 4823 ret = rte_bbdev_mldts_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops); 4824 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops); 4825 4826 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4827 copy_reference_mldts_op(ops_enq, num_ops, 0, bufs->inputs, bufs->harq_inputs, 4828 bufs->hard_outputs, ref_op); 4829 4830 /* Set counter to validate the ordering */ 4831 for (j = 0; j < num_ops; ++j) 4832 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 4833 4834 for (i = 0; i < TEST_REPETITIONS; ++i) { 4835 uint32_t time_out = 0; 4836 for (j = 0; j < num_ops; ++j) 4837 mbuf_reset(ops_enq[j]->mldts.output.data); 4838 4839 start_time = rte_rdtsc_precise(); 4840 4841 for (enq = 0, deq = 0; enq < num_ops;) { 4842 num_to_enq = burst_sz; 4843 4844 if (unlikely(num_ops - enq < num_to_enq)) 4845 num_to_enq = num_ops - enq; 4846 4847 enq += rte_bbdev_enqueue_mldts_ops(tp->dev_id, 4848 queue_id, &ops_enq[enq], num_to_enq); 4849 4850 deq += rte_bbdev_dequeue_mldts_ops(tp->dev_id, 4851 queue_id, &ops_deq[deq], enq - deq); 4852 time_out++; 4853 if (time_out >= TIME_OUT_POLL) { 4854 timeout_exit(tp->dev_id); 4855 TEST_ASSERT_SUCCESS(TEST_FAILED, "Enqueue timeout!"); 4856 } 4857 } 4858 4859 /* dequeue the remaining */ 4860 time_out = 0; 4861 while (deq < enq) { 4862 deq += rte_bbdev_dequeue_mldts_ops(tp->dev_id, 4863 queue_id, &ops_deq[deq], enq - deq); 4864 time_out++; 4865 if (time_out >= TIME_OUT_POLL) { 4866 timeout_exit(tp->dev_id); 4867 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 4868 } 4869 } 4870 4871 total_time += rte_rdtsc_precise() - start_time; 4872 } 4873 4874 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 4875 ret = validate_mldts_op(ops_deq, num_ops, ref_op); 4876 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 4877 } 4878 4879 rte_bbdev_mldts_op_free_bulk(ops_enq, num_ops); 4880 4881 double tb_len_bits = calc_mldts_size(ref_op); 4882 4883 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) / 4884 ((double)total_time / (double)rte_get_tsc_hz()); 4885 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) / 4886 1000000.0) / ((double)total_time / 4887 (double)rte_get_tsc_hz()); 4888 4889 return TEST_SUCCESS; 4890 } 4891 4892 static void 4893 print_enc_throughput(struct thread_params *t_params, unsigned int used_cores) 4894 { 4895 unsigned int iter = 0; 4896 double total_mops = 0, total_mbps = 0; 4897 4898 for (iter = 0; iter < used_cores; iter++) { 4899 printf( 4900 "Throughput for core (%u): %.8lg Ops/s, %.8lg Mbps\n", 4901 t_params[iter].lcore_id, t_params[iter].ops_per_sec, 4902 t_params[iter].mbps); 4903 total_mops += t_params[iter].ops_per_sec; 4904 total_mbps += t_params[iter].mbps; 4905 } 4906 printf( 4907 "\nTotal throughput for %u cores: %.8lg MOPS, %.8lg Mbps\n", 4908 used_cores, total_mops, total_mbps); 4909 } 4910 4911 /* Aggregate the performance results over the number of cores used */ 4912 static void 4913 print_dec_throughput(struct thread_params *t_params, unsigned int used_cores) 4914 { 4915 unsigned int core_idx = 0; 4916 double total_mops = 0, total_mbps = 0; 4917 uint8_t iter_count = 0; 4918 4919 for (core_idx = 0; core_idx < used_cores; core_idx++) { 4920 printf( 4921 "Throughput for core (%u): %.8lg Ops/s, %.8lg Mbps @ max %u iterations\n", 4922 t_params[core_idx].lcore_id, 4923 t_params[core_idx].ops_per_sec, 4924 t_params[core_idx].mbps, 4925 t_params[core_idx].iter_count); 4926 total_mops += t_params[core_idx].ops_per_sec; 4927 total_mbps += t_params[core_idx].mbps; 4928 iter_count = RTE_MAX(iter_count, 4929 t_params[core_idx].iter_count); 4930 } 4931 printf( 4932 "\nTotal throughput for %u cores: %.8lg MOPS, %.8lg Mbps @ max %u iterations\n", 4933 used_cores, total_mops, total_mbps, iter_count); 4934 } 4935 4936 /* Aggregate the performance results over the number of cores used */ 4937 static void 4938 print_dec_bler(struct thread_params *t_params, unsigned int used_cores) 4939 { 4940 unsigned int core_idx = 0; 4941 double total_mbps = 0, total_bler = 0, total_iter = 0; 4942 double snr = get_snr(); 4943 4944 for (core_idx = 0; core_idx < used_cores; core_idx++) { 4945 printf("Core%u BLER %.1f %% - Iters %.1f - Tp %.1f Mbps %s\n", 4946 t_params[core_idx].lcore_id, 4947 t_params[core_idx].bler * 100, 4948 t_params[core_idx].iter_average, 4949 t_params[core_idx].mbps, 4950 get_vector_filename()); 4951 total_mbps += t_params[core_idx].mbps; 4952 total_bler += t_params[core_idx].bler; 4953 total_iter += t_params[core_idx].iter_average; 4954 } 4955 total_bler /= used_cores; 4956 total_iter /= used_cores; 4957 4958 printf("SNR %.2f BLER %.1f %% - Iterations %.1f %d - Tp %.3f Mbps %s\n", 4959 snr, total_bler * 100, total_iter, get_iter_max(), 4960 total_mbps, get_vector_filename()); 4961 } 4962 4963 /* 4964 * Test function that determines BLER wireless performance 4965 */ 4966 static int 4967 bler_test(struct active_device *ad, 4968 struct test_op_params *op_params) 4969 { 4970 int ret; 4971 unsigned int lcore_id, used_cores = 0; 4972 struct thread_params *t_params; 4973 struct rte_bbdev_info info; 4974 lcore_function_t *bler_function; 4975 uint16_t num_lcores; 4976 const char *op_type_str; 4977 4978 rte_bbdev_info_get(ad->dev_id, &info); 4979 4980 op_type_str = rte_bbdev_op_type_str(test_vector.op_type); 4981 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", 4982 test_vector.op_type); 4983 4984 printf("+ ------------------------------------------------------- +\n"); 4985 printf("== test: bler\ndev: %s, nb_queues: %u, burst size: %u, num ops: %u, num_lcores: %u, op type: %s, itr mode: %s, GHz: %lg\n", 4986 info.dev_name, ad->nb_queues, op_params->burst_sz, 4987 op_params->num_to_process, op_params->num_lcores, 4988 op_type_str, 4989 intr_enabled ? "Interrupt mode" : "PMD mode", 4990 (double)rte_get_tsc_hz() / 1000000000.0); 4991 4992 /* Set number of lcores */ 4993 num_lcores = (ad->nb_queues < (op_params->num_lcores)) 4994 ? ad->nb_queues 4995 : op_params->num_lcores; 4996 4997 /* Allocate memory for thread parameters structure */ 4998 t_params = rte_zmalloc(NULL, num_lcores * sizeof(struct thread_params), 4999 RTE_CACHE_LINE_SIZE); 5000 TEST_ASSERT_NOT_NULL(t_params, "Failed to alloc %zuB for t_params", 5001 RTE_ALIGN(sizeof(struct thread_params) * num_lcores, 5002 RTE_CACHE_LINE_SIZE)); 5003 5004 if ((test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) && 5005 !check_bit(test_vector.ldpc_dec.op_flags, 5006 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK) 5007 && !check_bit(test_vector.ldpc_dec.op_flags, 5008 RTE_BBDEV_LDPC_LLR_COMPRESSION)) 5009 bler_function = bler_pmd_lcore_ldpc_dec; 5010 else if ((test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) && 5011 !check_bit(test_vector.turbo_dec.op_flags, 5012 RTE_BBDEV_TURBO_SOFT_OUTPUT)) 5013 bler_function = bler_pmd_lcore_turbo_dec; 5014 else 5015 return TEST_SKIPPED; 5016 5017 rte_atomic_store_explicit(&op_params->sync, SYNC_WAIT, rte_memory_order_relaxed); 5018 5019 /* Main core is set at first entry */ 5020 t_params[0].dev_id = ad->dev_id; 5021 t_params[0].lcore_id = rte_lcore_id(); 5022 t_params[0].op_params = op_params; 5023 t_params[0].queue_id = ad->queue_ids[used_cores++]; 5024 t_params[0].iter_count = 0; 5025 5026 RTE_LCORE_FOREACH_WORKER(lcore_id) { 5027 if (used_cores >= num_lcores) 5028 break; 5029 5030 t_params[used_cores].dev_id = ad->dev_id; 5031 t_params[used_cores].lcore_id = lcore_id; 5032 t_params[used_cores].op_params = op_params; 5033 t_params[used_cores].queue_id = ad->queue_ids[used_cores]; 5034 t_params[used_cores].iter_count = 0; 5035 5036 rte_eal_remote_launch(bler_function, 5037 &t_params[used_cores++], lcore_id); 5038 } 5039 5040 rte_atomic_store_explicit(&op_params->sync, SYNC_START, rte_memory_order_relaxed); 5041 ret = bler_function(&t_params[0]); 5042 5043 /* Main core is always used */ 5044 for (used_cores = 1; used_cores < num_lcores; used_cores++) 5045 ret |= rte_eal_wait_lcore(t_params[used_cores].lcore_id); 5046 5047 print_dec_bler(t_params, num_lcores); 5048 5049 /* Return if test failed */ 5050 if (ret) { 5051 rte_free(t_params); 5052 return ret; 5053 } 5054 5055 /* Function to print something here*/ 5056 rte_free(t_params); 5057 return ret; 5058 } 5059 5060 /* 5061 * Test function that determines how long an enqueue + dequeue of a burst 5062 * takes on available lcores. 5063 */ 5064 static int 5065 throughput_test(struct active_device *ad, 5066 struct test_op_params *op_params) 5067 { 5068 int ret; 5069 unsigned int lcore_id, used_cores = 0; 5070 struct thread_params *t_params, *tp; 5071 struct rte_bbdev_info info; 5072 lcore_function_t *throughput_function; 5073 uint16_t num_lcores; 5074 const char *op_type_str; 5075 5076 rte_bbdev_info_get(ad->dev_id, &info); 5077 5078 op_type_str = rte_bbdev_op_type_str(test_vector.op_type); 5079 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", 5080 test_vector.op_type); 5081 5082 printf("+ ------------------------------------------------------- +\n"); 5083 printf("== test: throughput\ndev: %s, nb_queues: %u, burst size: %u, num ops: %u, num_lcores: %u, op type: %s, itr mode: %s, GHz: %lg\n", 5084 info.dev_name, ad->nb_queues, op_params->burst_sz, 5085 op_params->num_to_process, op_params->num_lcores, 5086 op_type_str, 5087 intr_enabled ? "Interrupt mode" : "PMD mode", 5088 (double)rte_get_tsc_hz() / 1000000000.0); 5089 5090 /* Set number of lcores */ 5091 num_lcores = (ad->nb_queues < (op_params->num_lcores)) 5092 ? ad->nb_queues 5093 : op_params->num_lcores; 5094 5095 /* Allocate memory for thread parameters structure */ 5096 t_params = rte_zmalloc(NULL, num_lcores * sizeof(struct thread_params), 5097 RTE_CACHE_LINE_SIZE); 5098 TEST_ASSERT_NOT_NULL(t_params, "Failed to alloc %zuB for t_params", 5099 RTE_ALIGN(sizeof(struct thread_params) * num_lcores, 5100 RTE_CACHE_LINE_SIZE)); 5101 5102 if (intr_enabled) { 5103 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) 5104 throughput_function = throughput_intr_lcore_dec; 5105 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 5106 throughput_function = throughput_intr_lcore_ldpc_dec; 5107 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) 5108 throughput_function = throughput_intr_lcore_enc; 5109 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) 5110 throughput_function = throughput_intr_lcore_ldpc_enc; 5111 else if (test_vector.op_type == RTE_BBDEV_OP_FFT) 5112 throughput_function = throughput_intr_lcore_fft; 5113 else if (test_vector.op_type == RTE_BBDEV_OP_MLDTS) 5114 throughput_function = throughput_intr_lcore_mldts; 5115 else 5116 throughput_function = throughput_intr_lcore_enc; 5117 5118 /* Dequeue interrupt callback registration */ 5119 ret = rte_bbdev_callback_register(ad->dev_id, 5120 RTE_BBDEV_EVENT_DEQUEUE, dequeue_event_callback, 5121 t_params); 5122 if (ret < 0) { 5123 rte_free(t_params); 5124 return ret; 5125 } 5126 } else { 5127 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) 5128 throughput_function = throughput_pmd_lcore_dec; 5129 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 5130 throughput_function = throughput_pmd_lcore_ldpc_dec; 5131 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) 5132 throughput_function = throughput_pmd_lcore_enc; 5133 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) 5134 throughput_function = throughput_pmd_lcore_ldpc_enc; 5135 else if (test_vector.op_type == RTE_BBDEV_OP_FFT) 5136 throughput_function = throughput_pmd_lcore_fft; 5137 else if (test_vector.op_type == RTE_BBDEV_OP_MLDTS) 5138 throughput_function = throughput_pmd_lcore_mldts; 5139 else 5140 throughput_function = throughput_pmd_lcore_enc; 5141 } 5142 5143 rte_atomic_store_explicit(&op_params->sync, SYNC_WAIT, rte_memory_order_relaxed); 5144 5145 /* Main core is set at first entry */ 5146 t_params[0].dev_id = ad->dev_id; 5147 t_params[0].lcore_id = rte_lcore_id(); 5148 t_params[0].op_params = op_params; 5149 t_params[0].queue_id = ad->queue_ids[used_cores++]; 5150 t_params[0].iter_count = 0; 5151 5152 RTE_LCORE_FOREACH_WORKER(lcore_id) { 5153 if (used_cores >= num_lcores) 5154 break; 5155 5156 t_params[used_cores].dev_id = ad->dev_id; 5157 t_params[used_cores].lcore_id = lcore_id; 5158 t_params[used_cores].op_params = op_params; 5159 t_params[used_cores].queue_id = ad->queue_ids[used_cores]; 5160 t_params[used_cores].iter_count = 0; 5161 5162 rte_eal_remote_launch(throughput_function, 5163 &t_params[used_cores++], lcore_id); 5164 } 5165 5166 rte_atomic_store_explicit(&op_params->sync, SYNC_START, rte_memory_order_relaxed); 5167 ret = throughput_function(&t_params[0]); 5168 5169 /* Main core is always used */ 5170 for (used_cores = 1; used_cores < num_lcores; used_cores++) 5171 ret |= rte_eal_wait_lcore(t_params[used_cores].lcore_id); 5172 5173 /* Return if test failed */ 5174 if (ret) { 5175 rte_free(t_params); 5176 return ret; 5177 } 5178 5179 /* Print throughput if interrupts are disabled and test passed */ 5180 if (!intr_enabled) { 5181 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC || 5182 test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 5183 print_dec_throughput(t_params, num_lcores); 5184 else 5185 print_enc_throughput(t_params, num_lcores); 5186 rte_free(t_params); 5187 return ret; 5188 } 5189 5190 /* In interrupt TC we need to wait for the interrupt callback to deqeue 5191 * all pending operations. Skip waiting for queues which reported an 5192 * error using processing_status variable. 5193 * Wait for main lcore operations. 5194 */ 5195 tp = &t_params[0]; 5196 while ((rte_atomic_load_explicit(&tp->nb_dequeued, rte_memory_order_relaxed) < 5197 op_params->num_to_process) && 5198 (rte_atomic_load_explicit(&tp->processing_status, rte_memory_order_relaxed) != 5199 TEST_FAILED)) 5200 rte_pause(); 5201 5202 tp->ops_per_sec /= TEST_REPETITIONS; 5203 tp->mbps /= TEST_REPETITIONS; 5204 ret |= (int)rte_atomic_load_explicit(&tp->processing_status, rte_memory_order_relaxed); 5205 5206 /* Wait for worker lcores operations */ 5207 for (used_cores = 1; used_cores < num_lcores; used_cores++) { 5208 tp = &t_params[used_cores]; 5209 5210 while ((rte_atomic_load_explicit(&tp->nb_dequeued, rte_memory_order_relaxed) < 5211 op_params->num_to_process) && 5212 (rte_atomic_load_explicit(&tp->processing_status, 5213 rte_memory_order_relaxed) != TEST_FAILED)) 5214 rte_pause(); 5215 5216 tp->ops_per_sec /= TEST_REPETITIONS; 5217 tp->mbps /= TEST_REPETITIONS; 5218 ret |= (int)rte_atomic_load_explicit(&tp->processing_status, 5219 rte_memory_order_relaxed); 5220 } 5221 5222 /* Print throughput if test passed */ 5223 if (!ret) { 5224 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC || 5225 test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 5226 print_dec_throughput(t_params, num_lcores); 5227 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC || 5228 test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) 5229 print_enc_throughput(t_params, num_lcores); 5230 } 5231 5232 rte_free(t_params); 5233 return ret; 5234 } 5235 5236 static int 5237 latency_test_dec(struct rte_mempool *mempool, 5238 struct test_buffers *bufs, struct rte_bbdev_dec_op *ref_op, 5239 uint16_t dev_id, uint16_t queue_id, 5240 const uint16_t num_to_process, uint16_t burst_sz, 5241 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time, bool disable_et) 5242 { 5243 int ret = TEST_SUCCESS; 5244 uint16_t i, j, dequeued; 5245 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 5246 uint64_t start_time = 0, last_time = 0; 5247 5248 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 5249 uint16_t enq = 0, deq = 0; 5250 uint32_t time_out = 0; 5251 bool first_time = true; 5252 last_time = 0; 5253 5254 if (unlikely(num_to_process - dequeued < burst_sz)) 5255 burst_sz = num_to_process - dequeued; 5256 5257 ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz); 5258 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_dec_op_alloc_bulk() failed"); 5259 5260 ref_op->turbo_dec.iter_max = get_iter_max(); 5261 /* For validation tests we want to enable early termination */ 5262 if (!disable_et && !check_bit(ref_op->turbo_dec.op_flags, 5263 RTE_BBDEV_TURBO_EARLY_TERMINATION)) 5264 ref_op->turbo_dec.op_flags |= RTE_BBDEV_TURBO_EARLY_TERMINATION; 5265 5266 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 5267 copy_reference_dec_op(ops_enq, burst_sz, dequeued, 5268 bufs->inputs, 5269 bufs->hard_outputs, 5270 bufs->soft_outputs, 5271 ref_op); 5272 5273 /* Set counter to validate the ordering */ 5274 for (j = 0; j < burst_sz; ++j) 5275 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 5276 5277 start_time = rte_rdtsc_precise(); 5278 5279 enq = rte_bbdev_enqueue_dec_ops(dev_id, queue_id, &ops_enq[enq], 5280 burst_sz); 5281 TEST_ASSERT(enq == burst_sz, 5282 "Error enqueueing burst, expected %u, got %u", 5283 burst_sz, enq); 5284 5285 /* Dequeue */ 5286 do { 5287 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id, 5288 &ops_deq[deq], burst_sz - deq); 5289 if (likely(first_time && (deq > 0))) { 5290 last_time = rte_rdtsc_precise() - start_time; 5291 first_time = false; 5292 } 5293 time_out++; 5294 if (time_out >= TIME_OUT_POLL) { 5295 timeout_exit(dev_id); 5296 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 5297 } 5298 } while (unlikely(burst_sz != deq)); 5299 5300 *max_time = RTE_MAX(*max_time, last_time); 5301 *min_time = RTE_MIN(*min_time, last_time); 5302 *total_time += last_time; 5303 5304 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 5305 ret = validate_dec_op(ops_deq, burst_sz, ref_op); 5306 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 5307 } 5308 5309 rte_bbdev_dec_op_free_bulk(ops_enq, deq); 5310 dequeued += deq; 5311 } 5312 5313 return i; 5314 } 5315 5316 /* Test case for latency/validation for LDPC Decoder */ 5317 static int 5318 latency_test_ldpc_dec(struct rte_mempool *mempool, 5319 struct test_buffers *bufs, struct rte_bbdev_dec_op *ref_op, 5320 int vector_mask, uint16_t dev_id, uint16_t queue_id, 5321 const uint16_t num_to_process, uint16_t burst_sz, 5322 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time, 5323 bool disable_et) 5324 { 5325 int ret = TEST_SUCCESS; 5326 uint16_t i, j, dequeued; 5327 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 5328 uint64_t start_time = 0, last_time = 0; 5329 bool extDdr = ldpc_cap_flags & 5330 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE; 5331 5332 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 5333 uint16_t enq = 0, deq = 0; 5334 uint32_t time_out = 0; 5335 bool first_time = true; 5336 last_time = 0; 5337 5338 if (unlikely(num_to_process - dequeued < burst_sz)) 5339 burst_sz = num_to_process - dequeued; 5340 5341 ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz); 5342 TEST_ASSERT_SUCCESS(ret, 5343 "rte_bbdev_dec_op_alloc_bulk() failed"); 5344 5345 /* For latency tests we need to disable early termination */ 5346 if (disable_et && check_bit(ref_op->ldpc_dec.op_flags, 5347 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE)) 5348 ref_op->ldpc_dec.op_flags -= RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE; 5349 5350 ref_op->ldpc_dec.iter_max = get_iter_max(); 5351 /* When ET is disabled, the expected iter_count is iter_max */ 5352 if (disable_et) 5353 ref_op->ldpc_dec.iter_count = ref_op->ldpc_dec.iter_max; 5354 5355 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 5356 copy_reference_ldpc_dec_op(ops_enq, burst_sz, dequeued, 5357 bufs->inputs, 5358 bufs->hard_outputs, 5359 bufs->soft_outputs, 5360 bufs->harq_inputs, 5361 bufs->harq_outputs, 5362 ref_op); 5363 5364 if (extDdr) 5365 preload_harq_ddr(dev_id, queue_id, ops_enq, 5366 burst_sz, true); 5367 5368 /* Set counter to validate the ordering */ 5369 for (j = 0; j < burst_sz; ++j) 5370 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 5371 5372 start_time = rte_rdtsc_precise(); 5373 5374 enq = rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id, 5375 &ops_enq[enq], burst_sz); 5376 TEST_ASSERT(enq == burst_sz, 5377 "Error enqueueing burst, expected %u, got %u", 5378 burst_sz, enq); 5379 5380 /* Dequeue */ 5381 do { 5382 deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id, 5383 &ops_deq[deq], burst_sz - deq); 5384 if (likely(first_time && (deq > 0))) { 5385 last_time = rte_rdtsc_precise() - start_time; 5386 first_time = false; 5387 } 5388 time_out++; 5389 if (time_out >= TIME_OUT_POLL) { 5390 timeout_exit(dev_id); 5391 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 5392 } 5393 } while (unlikely(burst_sz != deq)); 5394 5395 *max_time = RTE_MAX(*max_time, last_time); 5396 *min_time = RTE_MIN(*min_time, last_time); 5397 *total_time += last_time; 5398 5399 if (extDdr) 5400 retrieve_harq_ddr(dev_id, queue_id, ops_enq, burst_sz); 5401 5402 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 5403 ret = validate_ldpc_dec_op(ops_deq, burst_sz, ref_op, vector_mask); 5404 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 5405 } 5406 5407 rte_bbdev_dec_op_free_bulk(ops_enq, deq); 5408 dequeued += deq; 5409 } 5410 return i; 5411 } 5412 5413 static int 5414 latency_test_enc(struct rte_mempool *mempool, 5415 struct test_buffers *bufs, struct rte_bbdev_enc_op *ref_op, 5416 uint16_t dev_id, uint16_t queue_id, 5417 const uint16_t num_to_process, uint16_t burst_sz, 5418 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time) 5419 { 5420 int ret = TEST_SUCCESS; 5421 uint16_t i, j, dequeued; 5422 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 5423 uint64_t start_time = 0, last_time = 0; 5424 5425 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 5426 uint16_t enq = 0, deq = 0; 5427 uint32_t time_out = 0; 5428 bool first_time = true; 5429 last_time = 0; 5430 5431 if (unlikely(num_to_process - dequeued < burst_sz)) 5432 burst_sz = num_to_process - dequeued; 5433 5434 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz); 5435 TEST_ASSERT_SUCCESS(ret, 5436 "rte_bbdev_enc_op_alloc_bulk() failed"); 5437 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 5438 copy_reference_enc_op(ops_enq, burst_sz, dequeued, 5439 bufs->inputs, 5440 bufs->hard_outputs, 5441 ref_op); 5442 5443 /* Set counter to validate the ordering */ 5444 for (j = 0; j < burst_sz; ++j) 5445 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 5446 5447 start_time = rte_rdtsc_precise(); 5448 5449 enq = rte_bbdev_enqueue_enc_ops(dev_id, queue_id, &ops_enq[enq], 5450 burst_sz); 5451 TEST_ASSERT(enq == burst_sz, 5452 "Error enqueueing burst, expected %u, got %u", 5453 burst_sz, enq); 5454 5455 /* Dequeue */ 5456 do { 5457 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id, 5458 &ops_deq[deq], burst_sz - deq); 5459 if (likely(first_time && (deq > 0))) { 5460 last_time += rte_rdtsc_precise() - start_time; 5461 first_time = false; 5462 } 5463 time_out++; 5464 if (time_out >= TIME_OUT_POLL) { 5465 timeout_exit(dev_id); 5466 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 5467 } 5468 } while (unlikely(burst_sz != deq)); 5469 5470 *max_time = RTE_MAX(*max_time, last_time); 5471 *min_time = RTE_MIN(*min_time, last_time); 5472 *total_time += last_time; 5473 5474 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 5475 ret = validate_enc_op(ops_deq, burst_sz, ref_op); 5476 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 5477 } 5478 5479 rte_bbdev_enc_op_free_bulk(ops_enq, deq); 5480 dequeued += deq; 5481 } 5482 5483 return i; 5484 } 5485 5486 static int 5487 latency_test_ldpc_enc(struct rte_mempool *mempool, 5488 struct test_buffers *bufs, struct rte_bbdev_enc_op *ref_op, 5489 uint16_t dev_id, uint16_t queue_id, 5490 const uint16_t num_to_process, uint16_t burst_sz, 5491 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time) 5492 { 5493 int ret = TEST_SUCCESS; 5494 uint16_t i, j, dequeued; 5495 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 5496 uint64_t start_time = 0, last_time = 0; 5497 5498 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 5499 uint16_t enq = 0, deq = 0; 5500 uint32_t time_out = 0; 5501 bool first_time = true; 5502 last_time = 0; 5503 5504 if (unlikely(num_to_process - dequeued < burst_sz)) 5505 burst_sz = num_to_process - dequeued; 5506 5507 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz); 5508 TEST_ASSERT_SUCCESS(ret, 5509 "rte_bbdev_enc_op_alloc_bulk() failed"); 5510 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 5511 copy_reference_ldpc_enc_op(ops_enq, burst_sz, dequeued, 5512 bufs->inputs, 5513 bufs->hard_outputs, 5514 ref_op); 5515 5516 /* Set counter to validate the ordering */ 5517 for (j = 0; j < burst_sz; ++j) 5518 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 5519 5520 start_time = rte_rdtsc_precise(); 5521 5522 enq = rte_bbdev_enqueue_ldpc_enc_ops(dev_id, queue_id, 5523 &ops_enq[enq], burst_sz); 5524 TEST_ASSERT(enq == burst_sz, 5525 "Error enqueueing burst, expected %u, got %u", 5526 burst_sz, enq); 5527 5528 /* Dequeue */ 5529 do { 5530 deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id, 5531 &ops_deq[deq], burst_sz - deq); 5532 if (likely(first_time && (deq > 0))) { 5533 last_time += rte_rdtsc_precise() - start_time; 5534 first_time = false; 5535 } 5536 time_out++; 5537 if (time_out >= TIME_OUT_POLL) { 5538 timeout_exit(dev_id); 5539 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 5540 } 5541 } while (unlikely(burst_sz != deq)); 5542 5543 *max_time = RTE_MAX(*max_time, last_time); 5544 *min_time = RTE_MIN(*min_time, last_time); 5545 *total_time += last_time; 5546 5547 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 5548 ret = validate_enc_op(ops_deq, burst_sz, ref_op); 5549 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 5550 } 5551 5552 rte_bbdev_enc_op_free_bulk(ops_enq, deq); 5553 dequeued += deq; 5554 } 5555 5556 return i; 5557 } 5558 5559 5560 static int 5561 latency_test_fft(struct rte_mempool *mempool, 5562 struct test_buffers *bufs, struct rte_bbdev_fft_op *ref_op, 5563 uint16_t dev_id, uint16_t queue_id, 5564 const uint16_t num_to_process, uint16_t burst_sz, 5565 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time) 5566 { 5567 int ret = TEST_SUCCESS; 5568 uint16_t i, j, dequeued; 5569 struct rte_bbdev_fft_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 5570 uint64_t start_time = 0, last_time = 0; 5571 5572 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 5573 uint16_t enq = 0, deq = 0; 5574 uint32_t time_out = 0; 5575 bool first_time = true; 5576 last_time = 0; 5577 5578 if (unlikely(num_to_process - dequeued < burst_sz)) 5579 burst_sz = num_to_process - dequeued; 5580 5581 ret = rte_bbdev_fft_op_alloc_bulk(mempool, ops_enq, burst_sz); 5582 TEST_ASSERT_SUCCESS(ret, 5583 "rte_bbdev_fft_op_alloc_bulk() failed"); 5584 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 5585 copy_reference_fft_op(ops_enq, burst_sz, dequeued, 5586 bufs->inputs, 5587 bufs->hard_outputs, bufs->soft_outputs, bufs->harq_inputs, 5588 ref_op); 5589 5590 /* Set counter to validate the ordering */ 5591 for (j = 0; j < burst_sz; ++j) 5592 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 5593 5594 start_time = rte_rdtsc_precise(); 5595 5596 enq = rte_bbdev_enqueue_fft_ops(dev_id, queue_id, 5597 &ops_enq[enq], burst_sz); 5598 TEST_ASSERT(enq == burst_sz, 5599 "Error enqueueing burst, expected %u, got %u", 5600 burst_sz, enq); 5601 5602 /* Dequeue */ 5603 do { 5604 deq += rte_bbdev_dequeue_fft_ops(dev_id, queue_id, 5605 &ops_deq[deq], burst_sz - deq); 5606 if (likely(first_time && (deq > 0))) { 5607 last_time += rte_rdtsc_precise() - start_time; 5608 first_time = false; 5609 } 5610 time_out++; 5611 if (time_out >= TIME_OUT_POLL) { 5612 timeout_exit(dev_id); 5613 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 5614 } 5615 } while (unlikely(burst_sz != deq)); 5616 5617 *max_time = RTE_MAX(*max_time, last_time); 5618 *min_time = RTE_MIN(*min_time, last_time); 5619 *total_time += last_time; 5620 5621 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 5622 ret = validate_fft_op(ops_deq, burst_sz, ref_op); 5623 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 5624 } 5625 5626 rte_bbdev_fft_op_free_bulk(ops_enq, deq); 5627 dequeued += deq; 5628 } 5629 5630 return i; 5631 } 5632 5633 static int 5634 latency_test_mldts(struct rte_mempool *mempool, 5635 struct test_buffers *bufs, struct rte_bbdev_mldts_op *ref_op, 5636 uint16_t dev_id, uint16_t queue_id, 5637 const uint16_t num_to_process, uint16_t burst_sz, 5638 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time) 5639 { 5640 int ret = TEST_SUCCESS; 5641 uint16_t i, j, dequeued; 5642 struct rte_bbdev_mldts_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 5643 uint64_t start_time = 0, last_time = 0; 5644 5645 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 5646 uint16_t enq = 0, deq = 0; 5647 uint32_t time_out = 0; 5648 bool first_time = true; 5649 last_time = 0; 5650 5651 if (unlikely(num_to_process - dequeued < burst_sz)) 5652 burst_sz = num_to_process - dequeued; 5653 5654 ret = rte_bbdev_mldts_op_alloc_bulk(mempool, ops_enq, burst_sz); 5655 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_mldts_op_alloc_bulk() failed"); 5656 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 5657 copy_reference_mldts_op(ops_enq, burst_sz, dequeued, 5658 bufs->inputs, bufs->harq_inputs, 5659 bufs->hard_outputs, 5660 ref_op); 5661 5662 /* Set counter to validate the ordering */ 5663 for (j = 0; j < burst_sz; ++j) 5664 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 5665 5666 start_time = rte_rdtsc_precise(); 5667 5668 enq = rte_bbdev_enqueue_mldts_ops(dev_id, queue_id, &ops_enq[enq], burst_sz); 5669 TEST_ASSERT(enq == burst_sz, 5670 "Error enqueueing burst, expected %u, got %u", 5671 burst_sz, enq); 5672 5673 /* Dequeue */ 5674 do { 5675 deq += rte_bbdev_dequeue_mldts_ops(dev_id, queue_id, 5676 &ops_deq[deq], burst_sz - deq); 5677 if (likely(first_time && (deq > 0))) { 5678 last_time += rte_rdtsc_precise() - start_time; 5679 first_time = false; 5680 } 5681 time_out++; 5682 if (time_out >= TIME_OUT_POLL) { 5683 timeout_exit(dev_id); 5684 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 5685 } 5686 } while (unlikely(burst_sz != deq)); 5687 5688 *max_time = RTE_MAX(*max_time, last_time); 5689 *min_time = RTE_MIN(*min_time, last_time); 5690 *total_time += last_time; 5691 5692 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 5693 ret = validate_mldts_op(ops_deq, burst_sz, ref_op); 5694 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 5695 } 5696 5697 rte_bbdev_mldts_op_free_bulk(ops_enq, deq); 5698 dequeued += deq; 5699 } 5700 5701 return i; 5702 } 5703 5704 /* Common function for running validation and latency test cases */ 5705 static int 5706 validation_latency_test(struct active_device *ad, 5707 struct test_op_params *op_params, bool latency_flag) 5708 { 5709 int iter; 5710 uint16_t burst_sz = op_params->burst_sz; 5711 const uint16_t num_to_process = op_params->num_to_process; 5712 const enum rte_bbdev_op_type op_type = test_vector.op_type; 5713 const uint16_t queue_id = ad->queue_ids[0]; 5714 struct test_buffers *bufs = NULL; 5715 struct rte_bbdev_info info; 5716 uint64_t total_time, min_time, max_time; 5717 const char *op_type_str; 5718 5719 total_time = max_time = 0; 5720 min_time = UINT64_MAX; 5721 5722 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 5723 "BURST_SIZE should be <= %u", MAX_BURST); 5724 5725 rte_bbdev_info_get(ad->dev_id, &info); 5726 bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 5727 5728 op_type_str = rte_bbdev_op_type_str(op_type); 5729 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type); 5730 5731 printf("+ ------------------------------------------------------- +\n"); 5732 if (latency_flag) 5733 printf("== test: latency\ndev:"); 5734 else 5735 printf("== test: validation\ndev:"); 5736 printf("%s, burst size: %u, num ops: %u, op type: %s\n", 5737 info.dev_name, burst_sz, num_to_process, op_type_str); 5738 5739 if (op_type == RTE_BBDEV_OP_TURBO_DEC) 5740 iter = latency_test_dec(op_params->mp, bufs, 5741 op_params->ref_dec_op, ad->dev_id, queue_id, 5742 num_to_process, burst_sz, &total_time, 5743 &min_time, &max_time, latency_flag); 5744 else if (op_type == RTE_BBDEV_OP_LDPC_ENC) 5745 iter = latency_test_ldpc_enc(op_params->mp, bufs, 5746 op_params->ref_enc_op, ad->dev_id, queue_id, 5747 num_to_process, burst_sz, &total_time, 5748 &min_time, &max_time); 5749 else if (op_type == RTE_BBDEV_OP_LDPC_DEC) 5750 iter = latency_test_ldpc_dec(op_params->mp, bufs, 5751 op_params->ref_dec_op, op_params->vector_mask, 5752 ad->dev_id, queue_id, num_to_process, 5753 burst_sz, &total_time, &min_time, &max_time, 5754 latency_flag); 5755 else if (op_type == RTE_BBDEV_OP_FFT) 5756 iter = latency_test_fft(op_params->mp, bufs, 5757 op_params->ref_fft_op, 5758 ad->dev_id, queue_id, 5759 num_to_process, burst_sz, &total_time, 5760 &min_time, &max_time); 5761 else if (op_type == RTE_BBDEV_OP_MLDTS) 5762 iter = latency_test_mldts(op_params->mp, bufs, 5763 op_params->ref_mldts_op, 5764 ad->dev_id, queue_id, 5765 num_to_process, burst_sz, &total_time, 5766 &min_time, &max_time); 5767 else /* RTE_BBDEV_OP_TURBO_ENC */ 5768 iter = latency_test_enc(op_params->mp, bufs, 5769 op_params->ref_enc_op, 5770 ad->dev_id, queue_id, 5771 num_to_process, burst_sz, &total_time, 5772 &min_time, &max_time); 5773 5774 if (iter <= 0) 5775 return TEST_FAILED; 5776 5777 printf("Operation latency:\n" 5778 "\tavg: %lg cycles, %lg us\n" 5779 "\tmin: %lg cycles, %lg us\n" 5780 "\tmax: %lg cycles, %lg us\n", 5781 (double)total_time / (double)iter, 5782 (double)(total_time * 1000000) / (double)iter / 5783 (double)rte_get_tsc_hz(), (double)min_time, 5784 (double)(min_time * 1000000) / (double)rte_get_tsc_hz(), 5785 (double)max_time, (double)(max_time * 1000000) / 5786 (double)rte_get_tsc_hz()); 5787 5788 return TEST_SUCCESS; 5789 } 5790 5791 static int 5792 latency_test(struct active_device *ad, struct test_op_params *op_params) 5793 { 5794 return validation_latency_test(ad, op_params, true); 5795 } 5796 5797 static int 5798 validation_test(struct active_device *ad, struct test_op_params *op_params) 5799 { 5800 return validation_latency_test(ad, op_params, false); 5801 } 5802 5803 static int 5804 get_bbdev_queue_stats(uint16_t dev_id, uint16_t queue_id, 5805 struct rte_bbdev_stats *stats) 5806 { 5807 struct rte_bbdev *dev = &rte_bbdev_devices[dev_id]; 5808 struct rte_bbdev_stats *q_stats; 5809 5810 if (queue_id >= dev->data->num_queues) 5811 return -1; 5812 5813 q_stats = &dev->data->queues[queue_id].queue_stats; 5814 5815 stats->enqueued_count = q_stats->enqueued_count; 5816 stats->dequeued_count = q_stats->dequeued_count; 5817 stats->enqueue_err_count = q_stats->enqueue_err_count; 5818 stats->dequeue_err_count = q_stats->dequeue_err_count; 5819 stats->enqueue_warn_count = q_stats->enqueue_warn_count; 5820 stats->dequeue_warn_count = q_stats->dequeue_warn_count; 5821 stats->acc_offload_cycles = q_stats->acc_offload_cycles; 5822 stats->enqueue_depth_avail = q_stats->enqueue_depth_avail; 5823 5824 return 0; 5825 } 5826 5827 static int 5828 offload_latency_test_fft(struct rte_mempool *mempool, struct test_buffers *bufs, 5829 struct rte_bbdev_fft_op *ref_op, uint16_t dev_id, 5830 uint16_t queue_id, const uint16_t num_to_process, 5831 uint16_t burst_sz, struct test_time_stats *time_st) 5832 { 5833 int i, dequeued, ret; 5834 struct rte_bbdev_fft_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 5835 uint64_t enq_start_time, deq_start_time; 5836 uint64_t enq_sw_last_time, deq_last_time; 5837 struct rte_bbdev_stats stats; 5838 5839 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 5840 uint16_t enq = 0, deq = 0; 5841 5842 if (unlikely(num_to_process - dequeued < burst_sz)) 5843 burst_sz = num_to_process - dequeued; 5844 5845 ret = rte_bbdev_fft_op_alloc_bulk(mempool, ops_enq, burst_sz); 5846 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_fft_op_alloc_bulk() failed"); 5847 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 5848 copy_reference_fft_op(ops_enq, burst_sz, dequeued, 5849 bufs->inputs, 5850 bufs->hard_outputs, bufs->soft_outputs, bufs->harq_inputs, 5851 ref_op); 5852 5853 /* Start time meas for enqueue function offload latency */ 5854 enq_start_time = rte_rdtsc_precise(); 5855 do { 5856 enq += rte_bbdev_enqueue_fft_ops(dev_id, queue_id, 5857 &ops_enq[enq], burst_sz - enq); 5858 } while (unlikely(burst_sz != enq)); 5859 5860 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats); 5861 TEST_ASSERT_SUCCESS(ret, 5862 "Failed to get stats for queue (%u) of device (%u)", 5863 queue_id, dev_id); 5864 5865 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time - 5866 stats.acc_offload_cycles; 5867 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time, 5868 enq_sw_last_time); 5869 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time, 5870 enq_sw_last_time); 5871 time_st->enq_sw_total_time += enq_sw_last_time; 5872 5873 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time, 5874 stats.acc_offload_cycles); 5875 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time, 5876 stats.acc_offload_cycles); 5877 time_st->enq_acc_total_time += stats.acc_offload_cycles; 5878 5879 /* give time for device to process ops */ 5880 rte_delay_us(WAIT_OFFLOAD_US); 5881 5882 /* Start time meas for dequeue function offload latency */ 5883 deq_start_time = rte_rdtsc_precise(); 5884 /* Dequeue one operation */ 5885 do { 5886 deq += rte_bbdev_dequeue_fft_ops(dev_id, queue_id, 5887 &ops_deq[deq], enq); 5888 } while (unlikely(deq == 0)); 5889 5890 deq_last_time = rte_rdtsc_precise() - deq_start_time; 5891 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time, 5892 deq_last_time); 5893 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time, 5894 deq_last_time); 5895 time_st->deq_total_time += deq_last_time; 5896 5897 /* Dequeue remaining operations if needed*/ 5898 while (burst_sz != deq) 5899 deq += rte_bbdev_dequeue_fft_ops(dev_id, queue_id, 5900 &ops_deq[deq], burst_sz - deq); 5901 5902 rte_bbdev_fft_op_free_bulk(ops_enq, deq); 5903 dequeued += deq; 5904 } 5905 5906 return i; 5907 } 5908 5909 static int 5910 offload_latency_test_mldts(struct rte_mempool *mempool, struct test_buffers *bufs, 5911 struct rte_bbdev_mldts_op *ref_op, uint16_t dev_id, 5912 uint16_t queue_id, const uint16_t num_to_process, 5913 uint16_t burst_sz, struct test_time_stats *time_st) 5914 { 5915 int i, dequeued, ret; 5916 struct rte_bbdev_mldts_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 5917 uint64_t enq_start_time, deq_start_time; 5918 uint64_t enq_sw_last_time, deq_last_time; 5919 struct rte_bbdev_stats stats; 5920 5921 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 5922 uint16_t enq = 0, deq = 0; 5923 5924 if (unlikely(num_to_process - dequeued < burst_sz)) 5925 burst_sz = num_to_process - dequeued; 5926 5927 ret = rte_bbdev_mldts_op_alloc_bulk(mempool, ops_enq, burst_sz); 5928 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_mldts_op_alloc_bulk() failed"); 5929 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 5930 copy_reference_mldts_op(ops_enq, burst_sz, dequeued, 5931 bufs->inputs, bufs->harq_inputs, 5932 bufs->hard_outputs, 5933 ref_op); 5934 5935 /* Start time meas for enqueue function offload latency */ 5936 enq_start_time = rte_rdtsc_precise(); 5937 do { 5938 enq += rte_bbdev_enqueue_mldts_ops(dev_id, queue_id, 5939 &ops_enq[enq], burst_sz - enq); 5940 } while (unlikely(burst_sz != enq)); 5941 5942 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats); 5943 TEST_ASSERT_SUCCESS(ret, 5944 "Failed to get stats for queue (%u) of device (%u)", 5945 queue_id, dev_id); 5946 5947 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time - 5948 stats.acc_offload_cycles; 5949 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time, 5950 enq_sw_last_time); 5951 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time, 5952 enq_sw_last_time); 5953 time_st->enq_sw_total_time += enq_sw_last_time; 5954 5955 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time, 5956 stats.acc_offload_cycles); 5957 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time, 5958 stats.acc_offload_cycles); 5959 time_st->enq_acc_total_time += stats.acc_offload_cycles; 5960 5961 /* give time for device to process ops */ 5962 rte_delay_us(WAIT_OFFLOAD_US); 5963 5964 /* Start time meas for dequeue function offload latency */ 5965 deq_start_time = rte_rdtsc_precise(); 5966 /* Dequeue one operation */ 5967 do { 5968 deq += rte_bbdev_dequeue_mldts_ops(dev_id, queue_id, &ops_deq[deq], enq); 5969 } while (unlikely(deq == 0)); 5970 5971 deq_last_time = rte_rdtsc_precise() - deq_start_time; 5972 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time, deq_last_time); 5973 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time, deq_last_time); 5974 time_st->deq_total_time += deq_last_time; 5975 5976 /* Dequeue remaining operations if needed*/ 5977 while (burst_sz != deq) 5978 deq += rte_bbdev_dequeue_mldts_ops(dev_id, queue_id, 5979 &ops_deq[deq], burst_sz - deq); 5980 5981 rte_bbdev_mldts_op_free_bulk(ops_enq, deq); 5982 dequeued += deq; 5983 } 5984 5985 return i; 5986 } 5987 5988 static int 5989 offload_latency_test_dec(struct rte_mempool *mempool, struct test_buffers *bufs, 5990 struct rte_bbdev_dec_op *ref_op, uint16_t dev_id, 5991 uint16_t queue_id, const uint16_t num_to_process, 5992 uint16_t burst_sz, struct test_time_stats *time_st) 5993 { 5994 int i, dequeued, ret; 5995 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 5996 uint64_t enq_start_time, deq_start_time; 5997 uint64_t enq_sw_last_time, deq_last_time; 5998 struct rte_bbdev_stats stats; 5999 6000 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 6001 uint16_t enq = 0, deq = 0; 6002 6003 if (unlikely(num_to_process - dequeued < burst_sz)) 6004 burst_sz = num_to_process - dequeued; 6005 6006 ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz); 6007 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_dec_op_alloc_bulk() failed"); 6008 ref_op->turbo_dec.iter_max = get_iter_max(); 6009 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 6010 copy_reference_dec_op(ops_enq, burst_sz, dequeued, 6011 bufs->inputs, 6012 bufs->hard_outputs, 6013 bufs->soft_outputs, 6014 ref_op); 6015 6016 /* Start time meas for enqueue function offload latency */ 6017 enq_start_time = rte_rdtsc_precise(); 6018 do { 6019 enq += rte_bbdev_enqueue_dec_ops(dev_id, queue_id, 6020 &ops_enq[enq], burst_sz - enq); 6021 } while (unlikely(burst_sz != enq)); 6022 6023 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats); 6024 TEST_ASSERT_SUCCESS(ret, 6025 "Failed to get stats for queue (%u) of device (%u)", 6026 queue_id, dev_id); 6027 6028 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time - 6029 stats.acc_offload_cycles; 6030 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time, 6031 enq_sw_last_time); 6032 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time, 6033 enq_sw_last_time); 6034 time_st->enq_sw_total_time += enq_sw_last_time; 6035 6036 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time, 6037 stats.acc_offload_cycles); 6038 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time, 6039 stats.acc_offload_cycles); 6040 time_st->enq_acc_total_time += stats.acc_offload_cycles; 6041 6042 /* give time for device to process ops */ 6043 rte_delay_us(WAIT_OFFLOAD_US); 6044 6045 /* Start time meas for dequeue function offload latency */ 6046 deq_start_time = rte_rdtsc_precise(); 6047 /* Dequeue one operation */ 6048 do { 6049 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id, 6050 &ops_deq[deq], enq); 6051 } while (unlikely(deq == 0)); 6052 6053 deq_last_time = rte_rdtsc_precise() - deq_start_time; 6054 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time, 6055 deq_last_time); 6056 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time, 6057 deq_last_time); 6058 time_st->deq_total_time += deq_last_time; 6059 6060 /* Dequeue remaining operations if needed*/ 6061 while (burst_sz != deq) 6062 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id, 6063 &ops_deq[deq], burst_sz - deq); 6064 6065 rte_bbdev_dec_op_free_bulk(ops_enq, deq); 6066 dequeued += deq; 6067 } 6068 6069 return i; 6070 } 6071 6072 static int 6073 offload_latency_test_ldpc_dec(struct rte_mempool *mempool, 6074 struct test_buffers *bufs, 6075 struct rte_bbdev_dec_op *ref_op, uint16_t dev_id, 6076 uint16_t queue_id, const uint16_t num_to_process, 6077 uint16_t burst_sz, struct test_time_stats *time_st) 6078 { 6079 int i, dequeued, ret; 6080 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 6081 uint64_t enq_start_time, deq_start_time; 6082 uint64_t enq_sw_last_time, deq_last_time; 6083 struct rte_bbdev_stats stats; 6084 bool extDdr = ldpc_cap_flags & 6085 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE; 6086 6087 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 6088 uint16_t enq = 0, deq = 0; 6089 6090 if (unlikely(num_to_process - dequeued < burst_sz)) 6091 burst_sz = num_to_process - dequeued; 6092 6093 ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz); 6094 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_dec_op_alloc_bulk() failed"); 6095 ref_op->ldpc_dec.iter_max = get_iter_max(); 6096 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 6097 copy_reference_ldpc_dec_op(ops_enq, burst_sz, dequeued, 6098 bufs->inputs, 6099 bufs->hard_outputs, 6100 bufs->soft_outputs, 6101 bufs->harq_inputs, 6102 bufs->harq_outputs, 6103 ref_op); 6104 6105 if (extDdr) 6106 preload_harq_ddr(dev_id, queue_id, ops_enq, 6107 burst_sz, true); 6108 6109 /* Start time meas for enqueue function offload latency */ 6110 enq_start_time = rte_rdtsc_precise(); 6111 do { 6112 enq += rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id, 6113 &ops_enq[enq], burst_sz - enq); 6114 } while (unlikely(burst_sz != enq)); 6115 6116 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time; 6117 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats); 6118 TEST_ASSERT_SUCCESS(ret, 6119 "Failed to get stats for queue (%u) of device (%u)", 6120 queue_id, dev_id); 6121 6122 enq_sw_last_time -= stats.acc_offload_cycles; 6123 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time, 6124 enq_sw_last_time); 6125 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time, 6126 enq_sw_last_time); 6127 time_st->enq_sw_total_time += enq_sw_last_time; 6128 6129 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time, 6130 stats.acc_offload_cycles); 6131 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time, 6132 stats.acc_offload_cycles); 6133 time_st->enq_acc_total_time += stats.acc_offload_cycles; 6134 6135 /* give time for device to process ops */ 6136 rte_delay_us(WAIT_OFFLOAD_US); 6137 6138 /* Start time meas for dequeue function offload latency */ 6139 deq_start_time = rte_rdtsc_precise(); 6140 /* Dequeue one operation */ 6141 do { 6142 deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id, 6143 &ops_deq[deq], enq); 6144 } while (unlikely(deq == 0)); 6145 6146 deq_last_time = rte_rdtsc_precise() - deq_start_time; 6147 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time, 6148 deq_last_time); 6149 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time, 6150 deq_last_time); 6151 time_st->deq_total_time += deq_last_time; 6152 6153 /* Dequeue remaining operations if needed*/ 6154 while (burst_sz != deq) 6155 deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id, 6156 &ops_deq[deq], burst_sz - deq); 6157 6158 if (extDdr) { 6159 /* Read loopback is not thread safe */ 6160 retrieve_harq_ddr(dev_id, queue_id, ops_enq, burst_sz); 6161 } 6162 6163 rte_bbdev_dec_op_free_bulk(ops_enq, deq); 6164 dequeued += deq; 6165 } 6166 6167 return i; 6168 } 6169 6170 static int 6171 offload_latency_test_enc(struct rte_mempool *mempool, struct test_buffers *bufs, 6172 struct rte_bbdev_enc_op *ref_op, uint16_t dev_id, 6173 uint16_t queue_id, const uint16_t num_to_process, 6174 uint16_t burst_sz, struct test_time_stats *time_st) 6175 { 6176 int i, dequeued, ret; 6177 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 6178 uint64_t enq_start_time, deq_start_time; 6179 uint64_t enq_sw_last_time, deq_last_time; 6180 struct rte_bbdev_stats stats; 6181 6182 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 6183 uint16_t enq = 0, deq = 0; 6184 6185 if (unlikely(num_to_process - dequeued < burst_sz)) 6186 burst_sz = num_to_process - dequeued; 6187 6188 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz); 6189 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_enc_op_alloc_bulk() failed"); 6190 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 6191 copy_reference_enc_op(ops_enq, burst_sz, dequeued, 6192 bufs->inputs, 6193 bufs->hard_outputs, 6194 ref_op); 6195 6196 /* Start time meas for enqueue function offload latency */ 6197 enq_start_time = rte_rdtsc_precise(); 6198 do { 6199 enq += rte_bbdev_enqueue_enc_ops(dev_id, queue_id, 6200 &ops_enq[enq], burst_sz - enq); 6201 } while (unlikely(burst_sz != enq)); 6202 6203 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time; 6204 6205 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats); 6206 TEST_ASSERT_SUCCESS(ret, 6207 "Failed to get stats for queue (%u) of device (%u)", 6208 queue_id, dev_id); 6209 enq_sw_last_time -= stats.acc_offload_cycles; 6210 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time, 6211 enq_sw_last_time); 6212 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time, 6213 enq_sw_last_time); 6214 time_st->enq_sw_total_time += enq_sw_last_time; 6215 6216 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time, 6217 stats.acc_offload_cycles); 6218 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time, 6219 stats.acc_offload_cycles); 6220 time_st->enq_acc_total_time += stats.acc_offload_cycles; 6221 6222 /* give time for device to process ops */ 6223 rte_delay_us(WAIT_OFFLOAD_US); 6224 6225 /* Start time meas for dequeue function offload latency */ 6226 deq_start_time = rte_rdtsc_precise(); 6227 /* Dequeue one operation */ 6228 do { 6229 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id, 6230 &ops_deq[deq], enq); 6231 } while (unlikely(deq == 0)); 6232 6233 deq_last_time = rte_rdtsc_precise() - deq_start_time; 6234 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time, 6235 deq_last_time); 6236 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time, 6237 deq_last_time); 6238 time_st->deq_total_time += deq_last_time; 6239 6240 while (burst_sz != deq) 6241 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id, 6242 &ops_deq[deq], burst_sz - deq); 6243 6244 rte_bbdev_enc_op_free_bulk(ops_enq, deq); 6245 dequeued += deq; 6246 } 6247 6248 return i; 6249 } 6250 6251 static int 6252 offload_latency_test_ldpc_enc(struct rte_mempool *mempool, 6253 struct test_buffers *bufs, 6254 struct rte_bbdev_enc_op *ref_op, uint16_t dev_id, 6255 uint16_t queue_id, const uint16_t num_to_process, 6256 uint16_t burst_sz, struct test_time_stats *time_st) 6257 { 6258 int i, dequeued, ret; 6259 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 6260 uint64_t enq_start_time, deq_start_time; 6261 uint64_t enq_sw_last_time, deq_last_time; 6262 struct rte_bbdev_stats stats; 6263 6264 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 6265 uint16_t enq = 0, deq = 0; 6266 6267 if (unlikely(num_to_process - dequeued < burst_sz)) 6268 burst_sz = num_to_process - dequeued; 6269 6270 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz); 6271 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_enc_op_alloc_bulk() failed"); 6272 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 6273 copy_reference_ldpc_enc_op(ops_enq, burst_sz, dequeued, 6274 bufs->inputs, 6275 bufs->hard_outputs, 6276 ref_op); 6277 6278 /* Start time meas for enqueue function offload latency */ 6279 enq_start_time = rte_rdtsc_precise(); 6280 do { 6281 enq += rte_bbdev_enqueue_ldpc_enc_ops(dev_id, queue_id, 6282 &ops_enq[enq], burst_sz - enq); 6283 } while (unlikely(burst_sz != enq)); 6284 6285 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time; 6286 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats); 6287 TEST_ASSERT_SUCCESS(ret, 6288 "Failed to get stats for queue (%u) of device (%u)", 6289 queue_id, dev_id); 6290 6291 enq_sw_last_time -= stats.acc_offload_cycles; 6292 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time, 6293 enq_sw_last_time); 6294 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time, 6295 enq_sw_last_time); 6296 time_st->enq_sw_total_time += enq_sw_last_time; 6297 6298 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time, 6299 stats.acc_offload_cycles); 6300 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time, 6301 stats.acc_offload_cycles); 6302 time_st->enq_acc_total_time += stats.acc_offload_cycles; 6303 6304 /* give time for device to process ops */ 6305 rte_delay_us(WAIT_OFFLOAD_US); 6306 6307 /* Start time meas for dequeue function offload latency */ 6308 deq_start_time = rte_rdtsc_precise(); 6309 /* Dequeue one operation */ 6310 do { 6311 deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id, 6312 &ops_deq[deq], enq); 6313 } while (unlikely(deq == 0)); 6314 6315 deq_last_time = rte_rdtsc_precise() - deq_start_time; 6316 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time, 6317 deq_last_time); 6318 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time, 6319 deq_last_time); 6320 time_st->deq_total_time += deq_last_time; 6321 6322 while (burst_sz != deq) 6323 deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id, 6324 &ops_deq[deq], burst_sz - deq); 6325 6326 rte_bbdev_enc_op_free_bulk(ops_enq, deq); 6327 dequeued += deq; 6328 } 6329 6330 return i; 6331 } 6332 6333 static int 6334 offload_cost_test(struct active_device *ad, 6335 struct test_op_params *op_params) 6336 { 6337 int iter, ret; 6338 uint16_t burst_sz = op_params->burst_sz; 6339 const uint16_t num_to_process = op_params->num_to_process; 6340 const enum rte_bbdev_op_type op_type = test_vector.op_type; 6341 const uint16_t queue_id = ad->queue_ids[0]; 6342 struct test_buffers *bufs = NULL; 6343 struct rte_bbdev_info info; 6344 const char *op_type_str; 6345 struct test_time_stats time_st; 6346 6347 memset(&time_st, 0, sizeof(struct test_time_stats)); 6348 time_st.enq_sw_min_time = UINT64_MAX; 6349 time_st.enq_acc_min_time = UINT64_MAX; 6350 time_st.deq_min_time = UINT64_MAX; 6351 6352 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 6353 "BURST_SIZE should be <= %u", MAX_BURST); 6354 6355 rte_bbdev_info_get(ad->dev_id, &info); 6356 bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 6357 6358 op_type_str = rte_bbdev_op_type_str(op_type); 6359 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type); 6360 6361 printf("+ ------------------------------------------------------- +\n"); 6362 printf("== test: offload latency test\ndev: %s, burst size: %u, num ops: %u, op type: %s\n", 6363 info.dev_name, burst_sz, num_to_process, op_type_str); 6364 6365 if (op_type == RTE_BBDEV_OP_TURBO_DEC) 6366 iter = offload_latency_test_dec(op_params->mp, bufs, 6367 op_params->ref_dec_op, ad->dev_id, queue_id, 6368 num_to_process, burst_sz, &time_st); 6369 else if (op_type == RTE_BBDEV_OP_TURBO_ENC) 6370 iter = offload_latency_test_enc(op_params->mp, bufs, 6371 op_params->ref_enc_op, ad->dev_id, queue_id, 6372 num_to_process, burst_sz, &time_st); 6373 else if (op_type == RTE_BBDEV_OP_LDPC_ENC) 6374 iter = offload_latency_test_ldpc_enc(op_params->mp, bufs, 6375 op_params->ref_enc_op, ad->dev_id, queue_id, 6376 num_to_process, burst_sz, &time_st); 6377 else if (op_type == RTE_BBDEV_OP_LDPC_DEC) 6378 iter = offload_latency_test_ldpc_dec(op_params->mp, bufs, 6379 op_params->ref_dec_op, ad->dev_id, queue_id, 6380 num_to_process, burst_sz, &time_st); 6381 else if (op_type == RTE_BBDEV_OP_FFT) 6382 iter = offload_latency_test_fft(op_params->mp, bufs, 6383 op_params->ref_fft_op, ad->dev_id, queue_id, 6384 num_to_process, burst_sz, &time_st); 6385 else if (op_type == RTE_BBDEV_OP_MLDTS) 6386 iter = offload_latency_test_mldts(op_params->mp, bufs, 6387 op_params->ref_mldts_op, ad->dev_id, queue_id, 6388 num_to_process, burst_sz, &time_st); 6389 else 6390 iter = offload_latency_test_enc(op_params->mp, bufs, 6391 op_params->ref_enc_op, ad->dev_id, queue_id, 6392 num_to_process, burst_sz, &time_st); 6393 6394 if (iter <= 0) 6395 return TEST_FAILED; 6396 6397 printf("Enqueue driver offload cost latency:\n" 6398 "\tavg: %lg cycles, %lg us\n" 6399 "\tmin: %lg cycles, %lg us\n" 6400 "\tmax: %lg cycles, %lg us\n" 6401 "Enqueue accelerator offload cost latency:\n" 6402 "\tavg: %lg cycles, %lg us\n" 6403 "\tmin: %lg cycles, %lg us\n" 6404 "\tmax: %lg cycles, %lg us\n", 6405 (double)time_st.enq_sw_total_time / (double)iter, 6406 (double)(time_st.enq_sw_total_time * 1000000) / 6407 (double)iter / (double)rte_get_tsc_hz(), 6408 (double)time_st.enq_sw_min_time, 6409 (double)(time_st.enq_sw_min_time * 1000000) / 6410 rte_get_tsc_hz(), (double)time_st.enq_sw_max_time, 6411 (double)(time_st.enq_sw_max_time * 1000000) / 6412 rte_get_tsc_hz(), (double)time_st.enq_acc_total_time / 6413 (double)iter, 6414 (double)(time_st.enq_acc_total_time * 1000000) / 6415 (double)iter / (double)rte_get_tsc_hz(), 6416 (double)time_st.enq_acc_min_time, 6417 (double)(time_st.enq_acc_min_time * 1000000) / 6418 rte_get_tsc_hz(), (double)time_st.enq_acc_max_time, 6419 (double)(time_st.enq_acc_max_time * 1000000) / 6420 rte_get_tsc_hz()); 6421 6422 printf("Dequeue offload cost latency - one op:\n" 6423 "\tavg: %lg cycles, %lg us\n" 6424 "\tmin: %lg cycles, %lg us\n" 6425 "\tmax: %lg cycles, %lg us\n", 6426 (double)time_st.deq_total_time / (double)iter, 6427 (double)(time_st.deq_total_time * 1000000) / 6428 (double)iter / (double)rte_get_tsc_hz(), 6429 (double)time_st.deq_min_time, 6430 (double)(time_st.deq_min_time * 1000000) / 6431 rte_get_tsc_hz(), (double)time_st.deq_max_time, 6432 (double)(time_st.deq_max_time * 1000000) / 6433 rte_get_tsc_hz()); 6434 6435 struct rte_bbdev_stats stats = {0}; 6436 ret = get_bbdev_queue_stats(ad->dev_id, queue_id, &stats); 6437 TEST_ASSERT_SUCCESS(ret, 6438 "Failed to get stats for queue (%u) of device (%u)", 6439 queue_id, ad->dev_id); 6440 if (stats.enqueue_warn_count > 0) 6441 printf("Warning reported on the queue : %10"PRIu64"\n", 6442 stats.enqueue_warn_count); 6443 if (op_type != RTE_BBDEV_OP_LDPC_DEC) { 6444 TEST_ASSERT_SUCCESS(stats.enqueued_count != num_to_process, 6445 "Mismatch in enqueue count %10"PRIu64" %d", 6446 stats.enqueued_count, num_to_process); 6447 TEST_ASSERT_SUCCESS(stats.dequeued_count != num_to_process, 6448 "Mismatch in dequeue count %10"PRIu64" %d", 6449 stats.dequeued_count, num_to_process); 6450 } 6451 TEST_ASSERT_SUCCESS(stats.enqueue_err_count != 0, 6452 "Enqueue count Error %10"PRIu64"", 6453 stats.enqueue_err_count); 6454 TEST_ASSERT_SUCCESS(stats.dequeue_err_count != 0, 6455 "Dequeue count Error (%10"PRIu64"", 6456 stats.dequeue_err_count); 6457 6458 return TEST_SUCCESS; 6459 } 6460 6461 static int 6462 offload_latency_empty_q_test_dec(uint16_t dev_id, uint16_t queue_id, 6463 const uint16_t num_to_process, uint16_t burst_sz, 6464 uint64_t *deq_total_time, uint64_t *deq_min_time, 6465 uint64_t *deq_max_time, const enum rte_bbdev_op_type op_type) 6466 { 6467 int i, deq_total; 6468 struct rte_bbdev_dec_op *ops[MAX_BURST]; 6469 uint64_t deq_start_time, deq_last_time; 6470 6471 /* Test deq offload latency from an empty queue */ 6472 6473 for (i = 0, deq_total = 0; deq_total < num_to_process; 6474 ++i, deq_total += burst_sz) { 6475 deq_start_time = rte_rdtsc_precise(); 6476 6477 if (unlikely(num_to_process - deq_total < burst_sz)) 6478 burst_sz = num_to_process - deq_total; 6479 if (op_type == RTE_BBDEV_OP_LDPC_DEC) 6480 rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id, ops, 6481 burst_sz); 6482 else 6483 rte_bbdev_dequeue_dec_ops(dev_id, queue_id, ops, 6484 burst_sz); 6485 6486 deq_last_time = rte_rdtsc_precise() - deq_start_time; 6487 *deq_max_time = RTE_MAX(*deq_max_time, deq_last_time); 6488 *deq_min_time = RTE_MIN(*deq_min_time, deq_last_time); 6489 *deq_total_time += deq_last_time; 6490 } 6491 6492 return i; 6493 } 6494 6495 static int 6496 offload_latency_empty_q_test_enc(uint16_t dev_id, uint16_t queue_id, 6497 const uint16_t num_to_process, uint16_t burst_sz, 6498 uint64_t *deq_total_time, uint64_t *deq_min_time, 6499 uint64_t *deq_max_time, const enum rte_bbdev_op_type op_type) 6500 { 6501 int i, deq_total; 6502 struct rte_bbdev_enc_op *ops[MAX_BURST]; 6503 uint64_t deq_start_time, deq_last_time; 6504 6505 /* Test deq offload latency from an empty queue */ 6506 for (i = 0, deq_total = 0; deq_total < num_to_process; 6507 ++i, deq_total += burst_sz) { 6508 deq_start_time = rte_rdtsc_precise(); 6509 6510 if (unlikely(num_to_process - deq_total < burst_sz)) 6511 burst_sz = num_to_process - deq_total; 6512 if (op_type == RTE_BBDEV_OP_LDPC_ENC) 6513 rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id, ops, 6514 burst_sz); 6515 else 6516 rte_bbdev_dequeue_enc_ops(dev_id, queue_id, ops, 6517 burst_sz); 6518 6519 deq_last_time = rte_rdtsc_precise() - deq_start_time; 6520 *deq_max_time = RTE_MAX(*deq_max_time, deq_last_time); 6521 *deq_min_time = RTE_MIN(*deq_min_time, deq_last_time); 6522 *deq_total_time += deq_last_time; 6523 } 6524 6525 return i; 6526 } 6527 6528 static int 6529 offload_latency_empty_q_test(struct active_device *ad, 6530 struct test_op_params *op_params) 6531 { 6532 int iter; 6533 uint64_t deq_total_time, deq_min_time, deq_max_time; 6534 uint16_t burst_sz = op_params->burst_sz; 6535 const uint16_t num_to_process = op_params->num_to_process; 6536 const enum rte_bbdev_op_type op_type = test_vector.op_type; 6537 const uint16_t queue_id = ad->queue_ids[0]; 6538 struct rte_bbdev_info info; 6539 const char *op_type_str; 6540 6541 deq_total_time = deq_max_time = 0; 6542 deq_min_time = UINT64_MAX; 6543 6544 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 6545 "BURST_SIZE should be <= %u", MAX_BURST); 6546 6547 rte_bbdev_info_get(ad->dev_id, &info); 6548 6549 op_type_str = rte_bbdev_op_type_str(op_type); 6550 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type); 6551 6552 printf("+ ------------------------------------------------------- +\n"); 6553 printf("== test: offload latency empty dequeue\ndev: %s, burst size: %u, num ops: %u, op type: %s\n", 6554 info.dev_name, burst_sz, num_to_process, op_type_str); 6555 6556 if (op_type == RTE_BBDEV_OP_TURBO_DEC || 6557 op_type == RTE_BBDEV_OP_LDPC_DEC) 6558 iter = offload_latency_empty_q_test_dec(ad->dev_id, queue_id, 6559 num_to_process, burst_sz, &deq_total_time, 6560 &deq_min_time, &deq_max_time, op_type); 6561 else 6562 iter = offload_latency_empty_q_test_enc(ad->dev_id, queue_id, 6563 num_to_process, burst_sz, &deq_total_time, 6564 &deq_min_time, &deq_max_time, op_type); 6565 6566 if (iter <= 0) 6567 return TEST_FAILED; 6568 6569 printf("Empty dequeue offload:\n" 6570 "\tavg: %lg cycles, %lg us\n" 6571 "\tmin: %lg cycles, %lg us\n" 6572 "\tmax: %lg cycles, %lg us\n", 6573 (double)deq_total_time / (double)iter, 6574 (double)(deq_total_time * 1000000) / (double)iter / 6575 (double)rte_get_tsc_hz(), (double)deq_min_time, 6576 (double)(deq_min_time * 1000000) / rte_get_tsc_hz(), 6577 (double)deq_max_time, (double)(deq_max_time * 1000000) / 6578 rte_get_tsc_hz()); 6579 6580 return TEST_SUCCESS; 6581 } 6582 6583 static int 6584 bler_tc(void) 6585 { 6586 return run_test_case(bler_test); 6587 } 6588 6589 static int 6590 throughput_tc(void) 6591 { 6592 return run_test_case(throughput_test); 6593 } 6594 6595 static int 6596 offload_cost_tc(void) 6597 { 6598 return run_test_case(offload_cost_test); 6599 } 6600 6601 static int 6602 offload_latency_empty_q_tc(void) 6603 { 6604 return run_test_case(offload_latency_empty_q_test); 6605 } 6606 6607 static int 6608 latency_tc(void) 6609 { 6610 return run_test_case(latency_test); 6611 } 6612 6613 static int 6614 validation_tc(void) 6615 { 6616 return run_test_case(validation_test); 6617 } 6618 6619 static int 6620 interrupt_tc(void) 6621 { 6622 return run_test_case(throughput_test); 6623 } 6624 6625 static struct unit_test_suite bbdev_bler_testsuite = { 6626 .suite_name = "BBdev BLER Tests", 6627 .setup = testsuite_setup, 6628 .teardown = testsuite_teardown, 6629 .unit_test_cases = { 6630 TEST_CASE_ST(ut_setup, ut_teardown, bler_tc), 6631 TEST_CASES_END() /**< NULL terminate unit test array */ 6632 } 6633 }; 6634 6635 static struct unit_test_suite bbdev_throughput_testsuite = { 6636 .suite_name = "BBdev Throughput Tests", 6637 .setup = testsuite_setup, 6638 .teardown = testsuite_teardown, 6639 .unit_test_cases = { 6640 TEST_CASE_ST(ut_setup, ut_teardown, throughput_tc), 6641 TEST_CASES_END() /**< NULL terminate unit test array */ 6642 } 6643 }; 6644 6645 static struct unit_test_suite bbdev_validation_testsuite = { 6646 .suite_name = "BBdev Validation Tests", 6647 .setup = testsuite_setup, 6648 .teardown = testsuite_teardown, 6649 .unit_test_cases = { 6650 TEST_CASE_ST(ut_setup, ut_teardown, validation_tc), 6651 TEST_CASES_END() /**< NULL terminate unit test array */ 6652 } 6653 }; 6654 6655 static struct unit_test_suite bbdev_latency_testsuite = { 6656 .suite_name = "BBdev Latency Tests", 6657 .setup = testsuite_setup, 6658 .teardown = testsuite_teardown, 6659 .unit_test_cases = { 6660 TEST_CASE_ST(ut_setup, ut_teardown, latency_tc), 6661 TEST_CASES_END() /**< NULL terminate unit test array */ 6662 } 6663 }; 6664 6665 static struct unit_test_suite bbdev_offload_cost_testsuite = { 6666 .suite_name = "BBdev Offload Cost Tests", 6667 .setup = testsuite_setup, 6668 .teardown = testsuite_teardown, 6669 .unit_test_cases = { 6670 TEST_CASE_ST(ut_setup, ut_teardown, offload_cost_tc), 6671 TEST_CASE_ST(ut_setup, ut_teardown, offload_latency_empty_q_tc), 6672 TEST_CASES_END() /**< NULL terminate unit test array */ 6673 } 6674 }; 6675 6676 static struct unit_test_suite bbdev_interrupt_testsuite = { 6677 .suite_name = "BBdev Interrupt Tests", 6678 .setup = interrupt_testsuite_setup, 6679 .teardown = testsuite_teardown, 6680 .unit_test_cases = { 6681 TEST_CASE_ST(ut_setup, ut_teardown, interrupt_tc), 6682 TEST_CASES_END() /**< NULL terminate unit test array */ 6683 } 6684 }; 6685 6686 REGISTER_TEST_COMMAND(bler, bbdev_bler_testsuite); 6687 REGISTER_TEST_COMMAND(throughput, bbdev_throughput_testsuite); 6688 REGISTER_TEST_COMMAND(validation, bbdev_validation_testsuite); 6689 REGISTER_TEST_COMMAND(latency, bbdev_latency_testsuite); 6690 REGISTER_TEST_COMMAND(offload, bbdev_offload_cost_testsuite); 6691 REGISTER_TEST_COMMAND(interrupt, bbdev_interrupt_testsuite); 6692